From da7106cfd6e23bbc8ee21d6929690ccb253414d5 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Wed, 24 May 2017 09:54:00 -0300 Subject: [PATCH] extended CloudFormation models for Lambda and DynamoDB --- moto/awslambda/models.py | 79 ++++++++++++++++--- moto/cloudformation/parsing.py | 8 +- moto/cloudwatch/models.py | 21 +++++ moto/dynamodb/models.py | 22 ++++++ .../test_cloudformation_stack_crud.py | 78 ++++++++++++++++++ 5 files changed, 195 insertions(+), 13 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 1e651cb04..13d4726ac 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -4,6 +4,7 @@ import base64 import datetime import hashlib import io +import os import json import sys import zipfile @@ -16,12 +17,12 @@ except: import boto.awslambda from moto.core import BaseBackend, BaseModel from moto.s3.models import s3_backend -from moto.s3.exceptions import MissingBucket +from moto.s3.exceptions import MissingBucket, MissingKey class LambdaFunction(BaseModel): - def __init__(self, spec): + def __init__(self, spec, validate_s3=True): # required self.code = spec['Code'] self.function_name = spec['FunctionName'] @@ -58,24 +59,25 @@ class LambdaFunction(BaseModel): self.code_size = len(to_unzip_code) self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest() else: - # validate s3 bucket + # validate s3 bucket and key + key = None try: # FIXME: does not validate bucket region key = s3_backend.get_key( self.code['S3Bucket'], self.code['S3Key']) except MissingBucket: - raise ValueError( - "InvalidParameterValueException", - "Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist") - else: - # validate s3 key - if key is None: + if do_validate_s3(): + raise ValueError( + "InvalidParameterValueException", + "Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist") + except MissingKey: + if do_validate_s3(): raise ValueError( "InvalidParameterValueException", "Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.") - else: - self.code_size = key.size - self.code_sha_256 = hashlib.sha256(key.value).hexdigest() + if key: + self.code_size = key.size + self.code_sha_256 = hashlib.sha256(key.value).hexdigest() self.function_arn = 'arn:aws:lambda:123456789012:function:{0}'.format( self.function_name) @@ -209,6 +211,13 @@ class LambdaFunction(BaseModel): fn = backend.create_function(spec) return fn + def get_cfn_attribute(self, attribute_name): + from moto.cloudformation.exceptions import UnformattedGetAttTemplateException + if attribute_name == 'Arn': + region = 'us-east-1' + return 'arn:aws:lambda:{0}:123456789012:function:{1}'.format(region, self.function_name) + raise UnformattedGetAttTemplateException() + @staticmethod def _create_zipfile_from_plaintext_code(code): zip_output = io.BytesIO() @@ -219,6 +228,48 @@ class LambdaFunction(BaseModel): return zip_output.read() +class EventSourceMapping(BaseModel): + + def __init__(self, spec): + # required + self.function_name = spec['FunctionName'] + self.event_source_arn = spec['EventSourceArn'] + self.starting_position = spec['StartingPosition'] + + # optional + self.batch_size = spec.get('BatchSize', 100) + self.enabled = spec.get('Enabled', True) + self.starting_position_timestamp = spec.get('StartingPositionTimestamp', None) + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + spec = { + 'FunctionName': properties['FunctionName'], + 'EventSourceArn': properties['EventSourceArn'], + 'StartingPosition': properties['StartingPosition'] + } + optional_properties = 'BatchSize Enabled StartingPositionTimestamp'.split() + for prop in optional_properties: + if prop in properties: + spec[prop] = properties[prop] + return EventSourceMapping(spec) + + +class LambdaVersion(BaseModel): + + def __init__(self, spec): + self.version = spec['Version'] + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + spec = { + 'Version': properties.get('Version') + } + return LambdaVersion(spec) + + class LambdaBackend(BaseBackend): def __init__(self): @@ -242,6 +293,10 @@ class LambdaBackend(BaseBackend): return self._functions.values() +def do_validate_s3(): + return os.environ.get('VALIDATE_LAMBDA_S3', '') in ['', '1', 'true'] + + lambda_backends = {} for region in boto.awslambda.regions(): lambda_backends[region.name] = LambdaBackend() diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 6d38289c7..1908a2a71 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -7,7 +7,9 @@ import warnings from moto.autoscaling import models as autoscaling_models from moto.awslambda import models as lambda_models +from moto.cloudwatch import models as cloudwatch_models from moto.datapipeline import models as datapipeline_models +from moto.dynamodb import models as dynamodb_models from moto.ec2 import models as ec2_models from moto.ecs import models as ecs_models from moto.elb import models as elb_models @@ -27,7 +29,10 @@ from boto.cloudformation.stack import Output MODEL_MAP = { "AWS::AutoScaling::AutoScalingGroup": autoscaling_models.FakeAutoScalingGroup, "AWS::AutoScaling::LaunchConfiguration": autoscaling_models.FakeLaunchConfiguration, + "AWS::DynamoDB::Table": dynamodb_models.Table, + "AWS::Lambda::EventSourceMapping": lambda_models.EventSourceMapping, "AWS::Lambda::Function": lambda_models.LambdaFunction, + "AWS::Lambda::Version": lambda_models.LambdaVersion, "AWS::EC2::EIP": ec2_models.ElasticAddress, "AWS::EC2::Instance": ec2_models.Instance, "AWS::EC2::InternetGateway": ec2_models.InternetGateway, @@ -53,6 +58,7 @@ MODEL_MAP = { "AWS::IAM::InstanceProfile": iam_models.InstanceProfile, "AWS::IAM::Role": iam_models.Role, "AWS::KMS::Key": kms_models.Key, + "AWS::Logs::LogGroup": cloudwatch_models.LogGroup, "AWS::RDS::DBInstance": rds_models.Database, "AWS::RDS::DBSecurityGroup": rds_models.SecurityGroup, "AWS::RDS::DBSubnetGroup": rds_models.SubnetGroup, @@ -133,7 +139,7 @@ def clean_json(resource_json, resources_map): try: return resource.get_cfn_attribute(resource_json['Fn::GetAtt'][1]) except NotImplementedError as n: - logger.warning(n.message.format( + logger.warning(str(n).format( resource_json['Fn::GetAtt'][0])) except UnformattedGetAttTemplateException: raise ValidationError( diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index dd97ddcbb..ed0086d93 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -111,6 +111,27 @@ class CloudWatchBackend(BaseBackend): return self.metric_data +class LogGroup(BaseModel): + + def __init__(self, spec): + # required + self.name = spec['LogGroupName'] + # optional + self.tags = spec.get('Tags', []) + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + spec = { + 'LogGroupName': properties['LogGroupName'] + } + optional_properties = 'Tags'.split() + for prop in optional_properties: + if prop in properties: + spec[prop] = properties[prop] + return LogGroup(spec) + + cloudwatch_backends = {} for region in boto.ec2.cloudwatch.regions(): cloudwatch_backends[region.name] = CloudWatchBackend() diff --git a/moto/dynamodb/models.py b/moto/dynamodb/models.py index 39bf15fca..300189a0e 100644 --- a/moto/dynamodb/models.py +++ b/moto/dynamodb/models.py @@ -137,6 +137,20 @@ class Table(BaseModel): } return results + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + key_attr = [i['AttributeName'] for i in properties['KeySchema'] if i['KeyType'] == 'HASH'][0] + key_type = [i['AttributeType'] for i in properties['AttributeDefinitions'] if i['AttributeName'] == key_attr][0] + spec = { + 'name': properties['TableName'], + 'hash_key_attr': key_attr, + 'hash_key_type': key_type + } + # TODO: optional properties still missing: + # range_key_attr, range_key_type, read_capacity, write_capacity + return Table(**spec) + def __len__(self): count = 0 for key, value in self.items.items(): @@ -245,6 +259,14 @@ class Table(BaseModel): except KeyError: return None + def get_cfn_attribute(self, attribute_name): + from moto.cloudformation.exceptions import UnformattedGetAttTemplateException + if attribute_name == 'StreamArn': + region = 'us-east-1' + time = '2000-01-01T00:00:00.000' + return 'arn:aws:dynamodb:{0}:123456789012:table/{1}/stream/{2}'.format(region, self.name, time) + raise UnformattedGetAttTemplateException() + class DynamoDBBackend(BaseBackend): diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index eb3798f82..0e3634756 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import os import json import boto @@ -565,3 +566,80 @@ def test_describe_stack_events_shows_create_update_and_delete(): assert False, "Too many stack events" list(stack_events_to_look_for).should.be.empty + + +@mock_cloudformation_deprecated +@mock_route53_deprecated +def test_create_stack_lambda_and_dynamodb(): + conn = boto.connect_cloudformation() + dummy_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack Lambda Test 1", + "Parameters": { + }, + "Resources": { + "func1": { + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code": { + "S3Bucket": "bucket_123", + "S3Key": "key_123" + }, + "FunctionName": "func1", + "Handler": "handler.handler", + "Role": "role1", + "Runtime": "python2.7", + "Description": "descr", + "MemorySize": 12345, + } + }, + "func1version": { + "Type": "AWS::Lambda::LambdaVersion", + "Properties" : { + "Version": "v1.2.3" + } + }, + "tab1": { + "Type" : "AWS::DynamoDB::Table", + "Properties" : { + "TableName": "tab1", + "KeySchema": [{ + "AttributeName": "attr1", + "KeyType": "HASH" + }], + "AttributeDefinitions": [{ + "AttributeName": "attr1", + "AttributeType": "string" + }], + "ProvisionedThroughput": { + "ReadCapacityUnits": 10, + "WriteCapacityUnits": 10 + } + } + }, + "func1mapping": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties" : { + "FunctionName": "v1.2.3", + "EventSourceArn": "arn:aws:dynamodb:region:XXXXXX:table/tab1/stream/2000T00:00:00.000", + "StartingPosition": "0", + "BatchSize": 100, + "Enabled": True + } + } + }, + } + validate_s3_before = os.environ.get('VALIDATE_LAMBDA_S3', '') + try: + os.environ['VALIDATE_LAMBDA_S3'] = 'false' + conn.create_stack( + "test_stack_lambda_1", + template_body=json.dumps(dummy_template), + parameters={}.items() + ) + finally: + os.environ['VALIDATE_LAMBDA_S3'] = validate_s3_before + + stack = conn.describe_stacks()[0] + resources = stack.list_resources() + assert len(resources) == 4