From 25f9e8b58883680f8aea6acf3fd568df07103fac Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Wed, 16 Sep 2015 17:49:13 -0400 Subject: [PATCH] Change CloudFormation to support Data Pipelines --- moto/cloudformation/parsing.py | 2 + moto/datapipeline/models.py | 17 +++- moto/datapipeline/utils.py | 18 +++++ requirements-dev.txt | 1 + .../test_cloudformation_stack_integration.py | 78 +++++++++++++++++++ tests/test_datapipeline/test_datapipeline.py | 37 +++++++-- 6 files changed, 143 insertions(+), 10 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 5306ce0d9..418d736d5 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -4,6 +4,7 @@ import functools import logging from moto.autoscaling import models as autoscaling_models +from moto.datapipeline import models as datapipeline_models from moto.ec2 import models as ec2_models from moto.elb import models as elb_models from moto.iam import models as iam_models @@ -36,6 +37,7 @@ MODEL_MAP = { "AWS::EC2::VPCGatewayAttachment": ec2_models.VPCGatewayAttachment, "AWS::EC2::VPCPeeringConnection": ec2_models.VPCPeeringConnection, "AWS::ElasticLoadBalancing::LoadBalancer": elb_models.FakeLoadBalancer, + "AWS::DataPipeline::Pipeline": datapipeline_models.Pipeline, "AWS::IAM::InstanceProfile": iam_models.InstanceProfile, "AWS::IAM::Role": iam_models.Role, "AWS::RDS::DBInstance": rds_models.Database, diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 3b31c9eb2..ca2bfdfee 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import datetime import boto.datapipeline from moto.core import BaseBackend -from .utils import get_random_pipeline_id +from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys class PipelineObject(object): @@ -73,12 +73,25 @@ class Pipeline(object): def set_pipeline_objects(self, pipeline_objects): self.objects = [ PipelineObject(pipeline_object['id'], pipeline_object['name'], pipeline_object['fields']) - for pipeline_object in pipeline_objects + for pipeline_object in remove_capitalization_of_dict_keys(pipeline_objects) ] def activate(self): self.status = "SCHEDULED" + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + datapipeline_backend = datapipeline_backends[region_name] + properties = cloudformation_json["Properties"] + + cloudformation_unique_id = "cf-" + properties["Name"] + pipeline = datapipeline_backend.create_pipeline(properties["Name"], cloudformation_unique_id) + datapipeline_backend.put_pipeline_definition(pipeline.pipeline_id, properties["PipelineObjects"]) + + if properties["Activate"]: + pipeline.activate() + return pipeline + class DataPipelineBackend(BaseBackend): diff --git a/moto/datapipeline/utils.py b/moto/datapipeline/utils.py index 7de9d2732..75df4a9a5 100644 --- a/moto/datapipeline/utils.py +++ b/moto/datapipeline/utils.py @@ -1,5 +1,23 @@ +import collections +import six from moto.core.utils import get_random_hex def get_random_pipeline_id(): return "df-{0}".format(get_random_hex(length=19)) + + +def remove_capitalization_of_dict_keys(obj): + if isinstance(obj, collections.Mapping): + result = obj.__class__() + for key, value in obj.items(): + normalized_key = key[:1].lower() + key[1:] + result[normalized_key] = remove_capitalization_of_dict_keys(value) + return result + elif isinstance(obj, collections.Iterable) and not isinstance(obj, six.string_types): + result = obj.__class__() + for item in obj: + result += (remove_capitalization_of_dict_keys(item),) + return result + else: + return obj diff --git a/requirements-dev.txt b/requirements-dev.txt index 5cd0acb14..bd4b6d237 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ coverage freezegun flask boto3 +six \ No newline at end of file diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index c1305d843..a959777b8 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -3,6 +3,7 @@ import json import boto import boto.cloudformation +import boto.datapipeline import boto.ec2 import boto.ec2.autoscale import boto.ec2.elb @@ -17,6 +18,7 @@ import sure # noqa from moto import ( mock_autoscaling, mock_cloudformation, + mock_datapipeline, mock_ec2, mock_elb, mock_iam, @@ -1395,3 +1397,79 @@ def test_subnets_should_be_created_with_availability_zone(): ) subnet = vpc_conn.get_all_subnets(filters={'cidrBlock': '10.0.0.0/24'})[0] subnet.availability_zone.should.equal('us-west-1b') + + +@mock_cloudformation +@mock_datapipeline +def test_datapipeline(): + dp_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "dataPipeline": { + "Properties": { + "Activate": "true", + "Name": "testDataPipeline", + "PipelineObjects": [ + { + "Fields": [ + { + "Key": "failureAndRerunMode", + "StringValue": "CASCADE" + }, + { + "Key": "scheduleType", + "StringValue": "cron" + }, + { + "Key": "schedule", + "RefValue": "DefaultSchedule" + }, + { + "Key": "pipelineLogUri", + "StringValue": "s3://bucket/logs" + }, + { + "Key": "type", + "StringValue": "Default" + }, + ], + "Id": "Default", + "Name": "Default" + }, + { + "Fields": [ + { + "Key": "startDateTime", + "StringValue": "1970-01-01T01:00:00" + }, + { + "Key": "period", + "StringValue": "1 Day" + }, + { + "Key": "type", + "StringValue": "Schedule" + } + ], + "Id": "DefaultSchedule", + "Name": "RunOnce" + } + ], + "PipelineTags": [] + }, + "Type": "AWS::DataPipeline::Pipeline" + } + } + } + cf_conn = boto.cloudformation.connect_to_region("us-east-1") + template_json = json.dumps(dp_template) + cf_conn.create_stack( + "test_stack", + template_body=template_json, + ) + + dp_conn = boto.datapipeline.connect_to_region('us-east-1') + data_pipelines = dp_conn.list_pipelines() + + data_pipelines['PipelineIdList'].should.have.length_of(1) + data_pipelines['PipelineIdList'][0]['Name'].should.equal('testDataPipeline') \ No newline at end of file diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index e18cbec94..bc8242d0c 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -4,6 +4,7 @@ import boto.datapipeline import sure # noqa from moto import mock_datapipeline +from moto.datapipeline.utils import remove_capitalization_of_dict_keys def get_value_from_fields(key, fields): @@ -144,13 +145,33 @@ def test_listing_pipelines(): response["HasMoreResults"].should.be(False) response["Marker"].should.be.none - response["PipelineIdList"].should.equal([ + response["PipelineIdList"].should.have.length_of(2) + response["PipelineIdList"].should.contain({ + "Id": res1["pipelineId"], + "Name": "mypipeline1", + }) + response["PipelineIdList"].should.contain({ + "Id": res2["pipelineId"], + "Name": "mypipeline2" + }) + + +# testing a helper function +def test_remove_capitalization_of_dict_keys(): + result = remove_capitalization_of_dict_keys( { - "Id": res1["pipelineId"], - "Name": "mypipeline1", - }, - { - "Id": res2["pipelineId"], - "Name": "mypipeline2" + "Id": "IdValue", + "Fields": [{ + "Key": "KeyValue", + "StringValue": "StringValueValue" + }] } - ]) + ) + + result.should.equal({ + "id": "IdValue", + "fields": [{ + "key": "KeyValue", + "stringValue": "StringValueValue" + }], + })