diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..16d9d7d91 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:2 + +ADD . /moto/ +ENV PYTHONUNBUFFERED 1 + +WORKDIR /moto/ +RUN python setup.py install + +CMD ["moto_server"] + +EXPOSE 5000 diff --git a/README.md b/README.md index c458d2a6f..d80e515ed 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ It gets even better! Moto isn't just S3. Here's the status of the other AWS serv |------------------------------------------------------------------------------| | Cloudwatch | @mock_cloudwatch | basic endpoints done | |------------------------------------------------------------------------------| +| Data Pipeline | @mock_datapipeline| basic endpoints done | +|------------------------------------------------------------------------------| | DynamoDB | @mock_dynamodb | core endpoints done | | DynamoDB2 | @mock_dynamodb2 | core endpoints done - no indexes | |------------------------------------------------------------------------------| diff --git a/moto/__init__.py b/moto/__init__.py index e5a94fd80..be7cfdda3 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,11 +3,12 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.12' +__version__ = '0.4.18' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa from .cloudwatch import mock_cloudwatch # flake8: noqa +from .datapipeline import mock_datapipeline # flake8: noqa from .dynamodb import mock_dynamodb # flake8: noqa from .dynamodb2 import mock_dynamodb2 # flake8: noqa from .ec2 import mock_ec2 # flake8: noqa diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 2c8f425ac..cb95d0542 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -113,7 +113,8 @@ class FakeAutoScalingGroup(object): def __init__(self, name, availability_zones, desired_capacity, max_size, min_size, launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, - load_balancers, placement_group, termination_policies, autoscaling_backend): + load_balancers, placement_group, termination_policies, + autoscaling_backend, tags): self.autoscaling_backend = autoscaling_backend self.name = name self.availability_zones = availability_zones @@ -133,6 +134,7 @@ class FakeAutoScalingGroup(object): self.instance_states = [] self.set_desired_capacity(desired_capacity) + self.tags = tags if tags else [] @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -156,6 +158,7 @@ class FakeAutoScalingGroup(object): load_balancers=load_balancer_names, placement_group=None, termination_policies=properties.get("TerminationPolicies", []), + tags=properties.get("Tags", []), ) return group @@ -261,7 +264,7 @@ class AutoScalingBackend(BaseBackend): launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, load_balancers, - placement_group, termination_policies): + placement_group, termination_policies, tags): def make_int(value): return int(value) if value is not None else value @@ -286,6 +289,7 @@ class AutoScalingBackend(BaseBackend): placement_group=placement_group, termination_policies=termination_policies, autoscaling_backend=self, + tags=tags, ) self.autoscaling_groups[name] = group return group diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 4f5948b6d..70fda4526 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -60,6 +60,7 @@ class AutoScalingResponse(BaseResponse): load_balancers=self._get_multi_param('LoadBalancerNames.member'), placement_group=self._get_param('PlacementGroup'), termination_policies=self._get_multi_param('TerminationPolicies.member'), + tags=self._get_list_prefix('Tags.member'), ) template = self.response_template(CREATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() @@ -235,7 +236,17 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """ {% for group in groups %} - + + {% for tag in group.tags %} + + {{ tag.resource_type }} + {{ tag.resource_id }} + {{ tag.propagate_at_launch }} + {{ tag.key }} + {{ tag.value }} + + {% endfor %} + {{ group.name }} {{ group.health_check_type }} diff --git a/moto/backends.py b/moto/backends.py index b46d56c06..cb040ab93 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from moto.autoscaling import autoscaling_backend from moto.cloudwatch import cloudwatch_backend from moto.cloudformation import cloudformation_backend +from moto.datapipeline import datapipeline_backend from moto.dynamodb import dynamodb_backend from moto.dynamodb2 import dynamodb_backend2 from moto.ec2 import ec2_backend @@ -25,6 +26,7 @@ BACKENDS = { 'autoscaling': autoscaling_backend, 'cloudformation': cloudformation_backend, 'cloudwatch': cloudwatch_backend, + 'datapipeline': datapipeline_backend, 'dynamodb': dynamodb_backend, 'dynamodb2': dynamodb_backend2, 'ec2': ec2_backend, diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 5306ce0d9..418d736d5 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -4,6 +4,7 @@ import functools import logging from moto.autoscaling import models as autoscaling_models +from moto.datapipeline import models as datapipeline_models from moto.ec2 import models as ec2_models from moto.elb import models as elb_models from moto.iam import models as iam_models @@ -36,6 +37,7 @@ MODEL_MAP = { "AWS::EC2::VPCGatewayAttachment": ec2_models.VPCGatewayAttachment, "AWS::EC2::VPCPeeringConnection": ec2_models.VPCPeeringConnection, "AWS::ElasticLoadBalancing::LoadBalancer": elb_models.FakeLoadBalancer, + "AWS::DataPipeline::Pipeline": datapipeline_models.Pipeline, "AWS::IAM::InstanceProfile": iam_models.InstanceProfile, "AWS::IAM::Role": iam_models.Role, "AWS::RDS::DBInstance": rds_models.Database, diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 28db697cf..c5196b2df 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -86,9 +86,19 @@ class CloudFormationResponse(BaseResponse): def get_template(self): name_or_stack_id = self.querystring.get('StackName')[0] - stack = self.cloudformation_backend.get_stack(name_or_stack_id) - return stack.template + + response = { + "GetTemplateResponse": { + "GetTemplateResult": { + "TemplateBody": stack.template, + "ResponseMetadata": { + "RequestId": "2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE" + } + } + } + } + return json.dumps(response) def update_stack(self): stack_name = self._get_param('StackName') diff --git a/moto/core/responses.py b/moto/core/responses.py index bd54cf01c..5fda815e7 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -63,8 +63,16 @@ class DynamicDictLoader(DictLoader): class _TemplateEnvironmentMixin(object): - loader = DynamicDictLoader({}) - environment = Environment(loader=loader) + + def __init__(self): + super(_TemplateEnvironmentMixin, self).__init__() + self.loader = DynamicDictLoader({}) + self.environment = Environment(loader=self.loader, autoescape=self.should_autoescape) + + @property + def should_autoescape(self): + # Allow for subclass to overwrite + return False def contains_template(self, template_id): return self.loader.contains(template_id) @@ -73,7 +81,7 @@ class _TemplateEnvironmentMixin(object): template_id = id(source) if not self.contains_template(template_id): self.loader.update({template_id: source}) - self.environment = Environment(loader=self.loader) + self.environment = Environment(loader=self.loader, autoescape=self.should_autoescape) return self.environment.get_template(template_id) diff --git a/moto/datapipeline/__init__.py b/moto/datapipeline/__init__.py new file mode 100644 index 000000000..dcfe2f427 --- /dev/null +++ b/moto/datapipeline/__init__.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals +from .models import datapipeline_backends +from ..core.models import MockAWS + +datapipeline_backend = datapipeline_backends['us-east-1'] + + +def mock_datapipeline(func=None): + if func: + return MockAWS(datapipeline_backends)(func) + else: + return MockAWS(datapipeline_backends) diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py new file mode 100644 index 000000000..b6a70b5f1 --- /dev/null +++ b/moto/datapipeline/models.py @@ -0,0 +1,149 @@ +from __future__ import unicode_literals + +import datetime +import boto.datapipeline +from moto.core import BaseBackend +from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys + + +class PipelineObject(object): + def __init__(self, object_id, name, fields): + self.object_id = object_id + self.name = name + self.fields = fields + + def to_json(self): + return { + "fields": self.fields, + "id": self.object_id, + "name": self.name, + } + + +class Pipeline(object): + def __init__(self, name, unique_id): + self.name = name + self.unique_id = unique_id + self.description = "" + self.pipeline_id = get_random_pipeline_id() + self.creation_time = datetime.datetime.utcnow() + self.objects = [] + self.status = "PENDING" + + @property + def physical_resource_id(self): + return self.pipeline_id + + def to_meta_json(self): + return { + "id": self.pipeline_id, + "name": self.name, + } + + def to_json(self): + return { + "description": self.description, + "fields": [{ + "key": "@pipelineState", + "stringValue": self.status, + }, { + "key": "description", + "stringValue": self.description + }, { + "key": "name", + "stringValue": self.name + }, { + "key": "@creationTime", + "stringValue": datetime.datetime.strftime(self.creation_time, '%Y-%m-%dT%H-%M-%S'), + }, { + "key": "@id", + "stringValue": self.pipeline_id, + }, { + "key": "@sphere", + "stringValue": "PIPELINE" + }, { + "key": "@version", + "stringValue": "1" + }, { + "key": "@userId", + "stringValue": "924374875933" + }, { + "key": "@accountId", + "stringValue": "924374875933" + }, { + "key": "uniqueId", + "stringValue": self.unique_id + }], + "name": self.name, + "pipelineId": self.pipeline_id, + "tags": [ + ] + } + + def set_pipeline_objects(self, pipeline_objects): + self.objects = [ + PipelineObject(pipeline_object['id'], pipeline_object['name'], pipeline_object['fields']) + for pipeline_object in remove_capitalization_of_dict_keys(pipeline_objects) + ] + + def activate(self): + self.status = "SCHEDULED" + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + datapipeline_backend = datapipeline_backends[region_name] + properties = cloudformation_json["Properties"] + + cloudformation_unique_id = "cf-" + properties["Name"] + pipeline = datapipeline_backend.create_pipeline(properties["Name"], cloudformation_unique_id) + datapipeline_backend.put_pipeline_definition(pipeline.pipeline_id, properties["PipelineObjects"]) + + if properties["Activate"]: + pipeline.activate() + return pipeline + + +class DataPipelineBackend(BaseBackend): + + def __init__(self): + self.pipelines = {} + + def create_pipeline(self, name, unique_id): + pipeline = Pipeline(name, unique_id) + self.pipelines[pipeline.pipeline_id] = pipeline + return pipeline + + def list_pipelines(self): + return self.pipelines.values() + + def describe_pipelines(self, pipeline_ids): + pipelines = [pipeline for pipeline in self.pipelines.values() if pipeline.pipeline_id in pipeline_ids] + return pipelines + + def get_pipeline(self, pipeline_id): + return self.pipelines[pipeline_id] + + def put_pipeline_definition(self, pipeline_id, pipeline_objects): + pipeline = self.get_pipeline(pipeline_id) + pipeline.set_pipeline_objects(pipeline_objects) + + def get_pipeline_definition(self, pipeline_id): + pipeline = self.get_pipeline(pipeline_id) + return pipeline.objects + + def describe_objects(self, object_ids, pipeline_id): + pipeline = self.get_pipeline(pipeline_id) + pipeline_objects = [ + pipeline_object for pipeline_object in pipeline.objects + if pipeline_object.object_id in object_ids + ] + return pipeline_objects + + def activate_pipeline(self, pipeline_id): + pipeline = self.get_pipeline(pipeline_id) + pipeline.activate() + + +datapipeline_backends = {} +for region in boto.datapipeline.regions(): + datapipeline_backends[region.name] = DataPipelineBackend() diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py new file mode 100644 index 000000000..70d19d189 --- /dev/null +++ b/moto/datapipeline/responses.py @@ -0,0 +1,81 @@ +from __future__ import unicode_literals + +import json + +from moto.core.responses import BaseResponse +from .models import datapipeline_backends + + +class DataPipelineResponse(BaseResponse): + + @property + def parameters(self): + # TODO this should really be moved to core/responses.py + if self.body: + return json.loads(self.body.decode("utf-8")) + else: + return self.querystring + + @property + def datapipeline_backend(self): + return datapipeline_backends[self.region] + + def create_pipeline(self): + name = self.parameters['name'] + unique_id = self.parameters['uniqueId'] + pipeline = self.datapipeline_backend.create_pipeline(name, unique_id) + return json.dumps({ + "pipelineId": pipeline.pipeline_id, + }) + + def list_pipelines(self): + pipelines = self.datapipeline_backend.list_pipelines() + return json.dumps({ + "hasMoreResults": False, + "marker": None, + "pipelineIdList": [ + pipeline.to_meta_json() for pipeline in pipelines + ] + }) + + def describe_pipelines(self): + pipeline_ids = self.parameters["pipelineIds"] + pipelines = self.datapipeline_backend.describe_pipelines(pipeline_ids) + + return json.dumps({ + "pipelineDescriptionList": [ + pipeline.to_json() for pipeline in pipelines + ] + }) + + def put_pipeline_definition(self): + pipeline_id = self.parameters["pipelineId"] + pipeline_objects = self.parameters["pipelineObjects"] + + self.datapipeline_backend.put_pipeline_definition(pipeline_id, pipeline_objects) + return json.dumps({"errored": False}) + + def get_pipeline_definition(self): + pipeline_id = self.parameters["pipelineId"] + pipeline_definition = self.datapipeline_backend.get_pipeline_definition(pipeline_id) + return json.dumps({ + "pipelineObjects": [pipeline_object.to_json() for pipeline_object in pipeline_definition] + }) + + def describe_objects(self): + pipeline_id = self.parameters["pipelineId"] + object_ids = self.parameters["objectIds"] + + pipeline_objects = self.datapipeline_backend.describe_objects(object_ids, pipeline_id) + return json.dumps({ + "hasMoreResults": False, + "marker": None, + "pipelineObjects": [ + pipeline_object.to_json() for pipeline_object in pipeline_objects + ] + }) + + def activate_pipeline(self): + pipeline_id = self.parameters["pipelineId"] + self.datapipeline_backend.activate_pipeline(pipeline_id) + return json.dumps({}) diff --git a/moto/datapipeline/urls.py b/moto/datapipeline/urls.py new file mode 100644 index 000000000..40805874b --- /dev/null +++ b/moto/datapipeline/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import DataPipelineResponse + +url_bases = [ + "https?://datapipeline.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': DataPipelineResponse.dispatch, +} diff --git a/moto/datapipeline/utils.py b/moto/datapipeline/utils.py new file mode 100644 index 000000000..75df4a9a5 --- /dev/null +++ b/moto/datapipeline/utils.py @@ -0,0 +1,23 @@ +import collections +import six +from moto.core.utils import get_random_hex + + +def get_random_pipeline_id(): + return "df-{0}".format(get_random_hex(length=19)) + + +def remove_capitalization_of_dict_keys(obj): + if isinstance(obj, collections.Mapping): + result = obj.__class__() + for key, value in obj.items(): + normalized_key = key[:1].lower() + key[1:] + result[normalized_key] = remove_capitalization_of_dict_keys(value) + return result + elif isinstance(obj, collections.Iterable) and not isinstance(obj, six.string_types): + result = obj.__class__() + for item in obj: + result += (remove_capitalization_of_dict_keys(item),) + return result + else: + return obj diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index e7277ee6b..612a0c3d3 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -121,6 +121,17 @@ class Item(object): # TODO deal with other types self.attrs[key] = DynamoType({"S": value}) + def update_with_attribute_updates(self, attribute_updates): + for attribute_name, update_action in attribute_updates.items(): + action = update_action['Action'] + new_value = list(update_action['Value'].values())[0] + if action == 'PUT': + # TODO deal with other types + if isinstance(new_value, list) or isinstance(new_value, set): + self.attrs[attribute_name] = DynamoType({"SS": new_value}) + else: + self.attrs[attribute_name] = DynamoType({"S": new_value}) + class Table(object): @@ -411,12 +422,19 @@ class DynamoDBBackend(BaseBackend): return table.scan(scan_filters) - def update_item(self, table_name, key, update_expression): + def update_item(self, table_name, key, update_expression, attribute_updates): table = self.get_table(table_name) + if table.hash_key_attr in key: + # Sometimes the key is wrapped in a dict with the key name + key = key[table.hash_key_attr] + hash_value = DynamoType(key) item = table.get_item(hash_value) - item.update(update_expression) + if update_expression: + item.update(update_expression) + else: + item.update_with_attribute_updates(attribute_updates) return item def delete_item(self, table_name, keys): diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 57d06bbf3..2be0dda8f 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -373,8 +373,9 @@ class DynamoHandler(BaseResponse): def update_item(self): name = self.body['TableName'] key = self.body['Key'] - update_expression = self.body['UpdateExpression'] - item = dynamodb_backend2.update_item(name, key, update_expression) + update_expression = self.body.get('UpdateExpression') + attribute_updates = self.body.get('AttributeUpdates') + item = dynamodb_backend2.update_item(name, key, update_expression, attribute_updates) item_dict = item.to_json() item_dict['ConsumedCapacityUnits'] = 0.5 diff --git a/moto/ec2/models.py b/moto/ec2/models.py index efe968eb2..446561d33 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -98,7 +98,7 @@ from .utils import ( def utc_date_and_time(): - return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.000Z') def validate_resource_ids(resource_ids): @@ -710,6 +710,13 @@ class KeyPairBackend(object): return results + def import_key_pair(self, key_name, public_key_material): + if key_name in self.keypairs: + raise InvalidKeyPairDuplicateError(key_name) + self.keypairs[key_name] = keypair = random_key_pair() + keypair['name'] = key_name + return keypair + class TagBackend(object): @@ -1381,12 +1388,13 @@ class VolumeAttachment(object): class Volume(TaggedEC2Resource): - def __init__(self, ec2_backend, volume_id, size, zone): + def __init__(self, ec2_backend, volume_id, size, zone, snapshot_id=None): self.id = volume_id self.size = size self.zone = zone self.create_time = utc_date_and_time() self.attachment = None + self.snapshot_id = snapshot_id self.ec2_backend = ec2_backend @classmethod @@ -1429,10 +1437,14 @@ class EBSBackend(object): self.snapshots = {} super(EBSBackend, self).__init__() - def create_volume(self, size, zone_name): + def create_volume(self, size, zone_name, snapshot_id=None): volume_id = random_volume_id() zone = self.get_zone_by_name(zone_name) - volume = Volume(self, volume_id, size, zone) + if snapshot_id: + snapshot = self.get_snapshot(snapshot_id) + if size is None: + size = snapshot.volume.size + volume = Volume(self, volume_id, size, zone, snapshot_id) self.volumes[volume_id] = volume return volume diff --git a/moto/ec2/responses/__init__.py b/moto/ec2/responses/__init__.py index f051a7ca7..e51992e41 100644 --- a/moto/ec2/responses/__init__.py +++ b/moto/ec2/responses/__init__.py @@ -66,3 +66,7 @@ class EC2Response( def ec2_backend(self): from moto.ec2.models import ec2_backends return ec2_backends[self.region] + + @property + def should_autoescape(self): + return True diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index abb371260..5adb4c7d0 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -25,9 +25,10 @@ class ElasticBlockStore(BaseResponse): return template.render(snapshot=snapshot) def create_volume(self): - size = self.querystring.get('Size')[0] - zone = self.querystring.get('AvailabilityZone')[0] - volume = self.ec2_backend.create_volume(size, zone) + size = self._get_param('Size') + zone = self._get_param('AvailabilityZone') + snapshot_id = self._get_param('SnapshotId') + volume = self.ec2_backend.create_volume(size, zone, snapshot_id) template = self.response_template(CREATE_VOLUME_RESPONSE) return template.render(volume=volume) @@ -110,7 +111,11 @@ CREATE_VOLUME_RESPONSE = """ {% endfor %} """ + +EC2_DESCRIBE_INSTANCE_TYPES = """ + + f8b86168-d034-4e65-b48d-3b84c78e64af + + {% for instance_type in instance_types %} + + {{ instance_type.name }} + {{ instance_type.cores }} + {{ instance_type.memory }} + {{ instance_type.disk }} + {{ instance_type.storageCount }} + {{ instance_type.maxIpAddresses }} + {{ instance_type.ebsOptimizedAvailable }} + + {% endfor %} + +""" diff --git a/moto/ec2/responses/key_pairs.py b/moto/ec2/responses/key_pairs.py index 80c6442f0..128e04ba9 100644 --- a/moto/ec2/responses/key_pairs.py +++ b/moto/ec2/responses/key_pairs.py @@ -28,7 +28,11 @@ class KeyPairs(BaseResponse): return template.render(keypairs=keypairs) def import_key_pair(self): - raise NotImplementedError('KeyPairs.import_key_pair is not yet implemented') + name = self.querystring.get('KeyName')[0] + material = self.querystring.get('PublicKeyMaterial')[0] + keypair = self.ec2_backend.import_key_pair(name, material) + template = self.response_template(IMPORT_KEYPAIR_RESPONSE) + return template.render(**keypair) DESCRIBE_KEY_PAIRS_RESPONSE = """ @@ -58,3 +62,10 @@ DELETE_KEY_PAIR_RESPONSE = """ + + 471f9fdd-8fe2-4a84-86b0-bd3d3e350979 + {{ name }} + {{ fingerprint }} + """ diff --git a/moto/ec2/responses/security_groups.py b/moto/ec2/responses/security_groups.py index eec27c3aa..9a9aaafd9 100644 --- a/moto/ec2/responses/security_groups.py +++ b/moto/ec2/responses/security_groups.py @@ -9,9 +9,9 @@ def process_rules_from_querystring(querystring): except: group_name_or_id = querystring.get('GroupId')[0] - ip_protocol = querystring.get('IpPermissions.1.IpProtocol')[0] - from_port = querystring.get('IpPermissions.1.FromPort')[0] - to_port = querystring.get('IpPermissions.1.ToPort')[0] + ip_protocol = querystring.get('IpPermissions.1.IpProtocol', [None])[0] + from_port = querystring.get('IpPermissions.1.FromPort', [None])[0] + to_port = querystring.get('IpPermissions.1.ToPort', [None])[0] ip_ranges = [] for key, value in querystring.items(): if 'IpPermissions.1.IpRanges' in key: diff --git a/moto/ec2/responses/tags.py b/moto/ec2/responses/tags.py index effb4faf9..4a62261d5 100644 --- a/moto/ec2/responses/tags.py +++ b/moto/ec2/responses/tags.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from xml.sax.saxutils import escape from moto.core.responses import BaseResponse from moto.ec2.models import validate_resource_ids from moto.ec2.utils import sequence_from_querystring, tags_from_query_string, filters_from_querystring @@ -26,8 +25,6 @@ class TagResponse(BaseResponse): def describe_tags(self): filters = filters_from_querystring(querystring_dict=self.querystring) tags = self.ec2_backend.describe_tags(filters=filters) - for tag in tags: - tag['value'] = escape(tag['value']) template = self.response_template(DESCRIBE_RESPONSE) return template.render(tags=tags) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 0b2b2ff49..b16c363ea 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -453,27 +453,22 @@ def simple_aws_filter_to_re(filter_string): return tmp_filter -# not really random ( http://xkcd.com/221/ ) def random_key_pair(): + def random_hex(): + return chr(random.choice(list(range(48, 58)) + list(range(97, 102)))) + def random_fingerprint(): + return ':'.join([random_hex()+random_hex() for i in range(20)]) + def random_material(): + return ''.join([ + chr(random.choice(list(range(65, 91)) + list(range(48, 58)) + + list(range(97, 102)))) + for i in range(1000) + ]) + material = "---- BEGIN RSA PRIVATE KEY ----" + random_material() + \ + "-----END RSA PRIVATE KEY-----" return { - 'fingerprint': ('1f:51:ae:28:bf:89:e9:d8:1f:25:5d:37:2d:' - '7d:b8:ca:9f:f5:f1:6f'), - 'material': """---- BEGIN RSA PRIVATE KEY ---- -MIICiTCCAfICCQD6m7oRw0uXOjANBgkqhkiG9w0BAQUFADCBiDELMAkGA1UEBhMC -VVMxCzAJBgNVBAgTAldBMRAwDgYDVQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6 -b24xFDASBgNVBAsTC0lBTSBDb25zb2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAd -BgkqhkiG9w0BCQEWEG5vb25lQGFtYXpvbi5jb20wHhcNMTEwNDI1MjA0NTIxWhcN -MTIwNDI0MjA0NTIxWjCBiDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAwDgYD -VQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6b24xFDASBgNVBAsTC0lBTSBDb25z -b2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAdBgkqhkiG9w0BCQEWEG5vb25lQGFt -YXpvbi5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMaK0dn+a4GmWIWJ -21uUSfwfEvySWtC2XADZ4nB+BLYgVIk60CpiwsZ3G93vUEIO3IyNoH/f0wYK8m9T -rDHudUZg3qX4waLG5M43q7Wgc/MbQITxOUSQv7c7ugFFDzQGBzZswY6786m86gpE -Ibb3OhjZnzcvQAaRHhdlQWIMm2nrAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAtCu4 -nUhVVxYUntneD9+h8Mg9q6q+auNKyExzyLwaxlAoo7TJHidbtS4J5iNmZgXL0Fkb -FFBjvSfpJIlJ00zbhNYS5f6GuoEDmFJl0ZxBHjJnyp378OD8uTs7fLvjx79LjSTb -NYiytVbZPQUQ5Yaxu2jXnimvw3rrszlaEXAMPLE ------END RSA PRIVATE KEY-----""" + 'fingerprint': random_fingerprint(), + 'material': material } diff --git a/moto/glacier/responses.py b/moto/glacier/responses.py index 37cbdc4c9..eac9b94c6 100644 --- a/moto/glacier/responses.py +++ b/moto/glacier/responses.py @@ -11,6 +11,7 @@ from .utils import region_from_glacier_url, vault_from_glacier_url class GlacierResponse(_TemplateEnvironmentMixin): def __init__(self, backend): + super(GlacierResponse, self).__init__() self.backend = backend @classmethod diff --git a/moto/kinesis/exceptions.py b/moto/kinesis/exceptions.py index c6be76885..0fcb3652a 100644 --- a/moto/kinesis/exceptions.py +++ b/moto/kinesis/exceptions.py @@ -13,6 +13,15 @@ class ResourceNotFoundError(BadRequest): }) +class ResourceInUseError(BadRequest): + def __init__(self, message): + super(ResourceNotFoundError, self).__init__() + self.description = json.dumps({ + "message": message, + '__type': 'ResourceInUseException', + }) + + class StreamNotFoundError(ResourceNotFoundError): def __init__(self, stream_name): super(StreamNotFoundError, self).__init__( diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index 0b01881a4..aae4e918f 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -1,9 +1,12 @@ from __future__ import unicode_literals +import datetime +import time + import boto.kinesis from moto.compat import OrderedDict from moto.core import BaseBackend -from .exceptions import StreamNotFoundError, ShardNotFoundError +from .exceptions import StreamNotFoundError, ShardNotFoundError, ResourceInUseError from .utils import compose_shard_iterator, compose_new_shard_iterator, decompose_shard_iterator @@ -124,12 +127,82 @@ class Stream(object): } +class FirehoseRecord(object): + def __init__(self, record_data): + self.record_id = 12345678 + self.record_data = record_data + + +class DeliveryStream(object): + def __init__(self, stream_name, **stream_kwargs): + self.name = stream_name + self.redshift_username = stream_kwargs['redshift_username'] + self.redshift_password = stream_kwargs['redshift_password'] + self.redshift_jdbc_url = stream_kwargs['redshift_jdbc_url'] + self.redshift_role_arn = stream_kwargs['redshift_role_arn'] + self.redshift_copy_command = stream_kwargs['redshift_copy_command'] + + self.redshift_s3_role_arn = stream_kwargs['redshift_s3_role_arn'] + self.redshift_s3_bucket_arn = stream_kwargs['redshift_s3_bucket_arn'] + self.redshift_s3_prefix = stream_kwargs['redshift_s3_prefix'] + self.redshift_s3_compression_format = stream_kwargs.get('redshift_s3_compression_format', 'UNCOMPRESSED') + self.redshift_s3_buffering_hings = stream_kwargs['redshift_s3_buffering_hings'] + + self.records = [] + self.status = 'ACTIVE' + self.create_at = datetime.datetime.utcnow() + self.last_updated = datetime.datetime.utcnow() + + @property + def arn(self): + return 'arn:aws:firehose:us-east-1:123456789012:deliverystream/{0}'.format(self.name) + + def to_dict(self): + return { + "DeliveryStreamDescription": { + "CreateTimestamp": time.mktime(self.create_at.timetuple()), + "DeliveryStreamARN": self.arn, + "DeliveryStreamName": self.name, + "DeliveryStreamStatus": self.status, + "Destinations": [ + { + "DestinationId": "string", + "RedshiftDestinationDescription": { + "ClusterJDBCURL": self.redshift_jdbc_url, + "CopyCommand": self.redshift_copy_command, + "RoleARN": self.redshift_role_arn, + "S3DestinationDescription": { + "BucketARN": self.redshift_s3_bucket_arn, + "BufferingHints": self.redshift_s3_buffering_hings, + "CompressionFormat": self.redshift_s3_compression_format, + "Prefix": self.redshift_s3_prefix, + "RoleARN": self.redshift_s3_role_arn + }, + "Username": self.redshift_username, + }, + } + ], + "HasMoreDestinations": False, + "LastUpdateTimestamp": time.mktime(self.last_updated.timetuple()), + "VersionId": "string", + } + } + + def put_record(self, record_data): + record = FirehoseRecord(record_data) + self.records.append(record) + return record + + class KinesisBackend(BaseBackend): def __init__(self): self.streams = {} + self.delivery_streams = {} def create_stream(self, stream_name, shard_count, region): + if stream_name in self.streams: + return ResourceInUseError(stream_name) stream = Stream(stream_name, shard_count, region) self.streams[stream_name] = stream return stream @@ -180,6 +253,52 @@ class KinesisBackend(BaseBackend): return sequence_number, shard_id + def put_records(self, stream_name, records): + stream = self.describe_stream(stream_name) + + response = { + "FailedRecordCount": 0, + "Records" : [] + } + + for record in records: + partition_key = record.get("PartitionKey") + explicit_hash_key = record.get("ExplicitHashKey") + data = record.get("data") + + sequence_number, shard_id = stream.put_record( + partition_key, explicit_hash_key, None, data + ) + response['Records'].append({ + "SequenceNumber": sequence_number, + "ShardId": shard_id + }) + + return response + + ''' Firehose ''' + def create_delivery_stream(self, stream_name, **stream_kwargs): + stream = DeliveryStream(stream_name, **stream_kwargs) + self.delivery_streams[stream_name] = stream + return stream + + def get_delivery_stream(self, stream_name): + if stream_name in self.delivery_streams: + return self.delivery_streams[stream_name] + else: + raise StreamNotFoundError(stream_name) + + def list_delivery_streams(self): + return self.delivery_streams.values() + + def delete_delivery_stream(self, stream_name): + self.delivery_streams.pop(stream_name) + + def put_firehose_record(self, stream_name, record_data): + stream = self.get_delivery_stream(stream_name) + record = stream.put_record(record_data) + return record + kinesis_backends = {} for region in boto.kinesis.regions(): kinesis_backends[region.name] = KinesisBackend() diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index 4b5f13729..35500e8ac 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -16,6 +16,11 @@ class KinesisResponse(BaseResponse): def kinesis_backend(self): return kinesis_backends[self.region] + @property + def is_firehose(self): + host = self.headers.get('host') or self.headers['Host'] + return host.startswith('firehose') + def create_stream(self): stream_name = self.parameters.get('StreamName') shard_count = self.parameters.get('ShardCount') @@ -67,6 +72,8 @@ class KinesisResponse(BaseResponse): }) def put_record(self): + if self.is_firehose: + return self.firehose_put_record() stream_name = self.parameters.get("StreamName") partition_key = self.parameters.get("PartitionKey") explicit_hash_key = self.parameters.get("ExplicitHashKey") @@ -81,3 +88,83 @@ class KinesisResponse(BaseResponse): "SequenceNumber": sequence_number, "ShardId": shard_id, }) + + def put_records(self): + if self.is_firehose: + return self.firehose_put_record() + stream_name = self.parameters.get("StreamName") + records = self.parameters.get("Records") + + response = self.kinesis_backend.put_records( + stream_name, records + ) + + return json.dumps(response) + + ''' Firehose ''' + def create_delivery_stream(self): + stream_name = self.parameters['DeliveryStreamName'] + redshift_config = self.parameters.get('RedshiftDestinationConfiguration') + + if redshift_config: + redshift_s3_config = redshift_config['S3Configuration'] + stream_kwargs = { + 'redshift_username': redshift_config['Username'], + 'redshift_password': redshift_config['Password'], + 'redshift_jdbc_url': redshift_config['ClusterJDBCURL'], + 'redshift_role_arn': redshift_config['RoleARN'], + 'redshift_copy_command': redshift_config['CopyCommand'], + + 'redshift_s3_role_arn': redshift_s3_config['RoleARN'], + 'redshift_s3_bucket_arn': redshift_s3_config['BucketARN'], + 'redshift_s3_prefix': redshift_s3_config['Prefix'], + 'redshift_s3_compression_format': redshift_s3_config.get('CompressionFormat'), + 'redshift_s3_buffering_hings': redshift_s3_config['BufferingHints'], + } + stream = self.kinesis_backend.create_delivery_stream(stream_name, **stream_kwargs) + return json.dumps({ + 'DeliveryStreamARN': stream.arn + }) + + def describe_delivery_stream(self): + stream_name = self.parameters["DeliveryStreamName"] + stream = self.kinesis_backend.get_delivery_stream(stream_name) + return json.dumps(stream.to_dict()) + + def list_delivery_streams(self): + streams = self.kinesis_backend.list_delivery_streams() + return json.dumps({ + "DeliveryStreamNames": [ + stream.name for stream in streams + ], + "HasMoreDeliveryStreams": False + }) + + def delete_delivery_stream(self): + stream_name = self.parameters['DeliveryStreamName'] + self.kinesis_backend.delete_delivery_stream(stream_name) + return json.dumps({}) + + def firehose_put_record(self): + stream_name = self.parameters['DeliveryStreamName'] + record_data = self.parameters['Record']['Data'] + + record = self.kinesis_backend.put_firehose_record(stream_name, record_data) + return json.dumps({ + "RecordId": record.record_id, + }) + + def put_record_batch(self): + stream_name = self.parameters['DeliveryStreamName'] + records = self.parameters['Records'] + + request_responses = [] + for record in records: + record_response = self.kinesis_backend.put_firehose_record(stream_name, record['Data']) + request_responses.append({ + "RecordId": record_response.record_id + }) + return json.dumps({ + "FailedPutCount": 0, + "RequestResponses": request_responses, + }) diff --git a/moto/kinesis/urls.py b/moto/kinesis/urls.py index 5de870c29..a8d15eecd 100644 --- a/moto/kinesis/urls.py +++ b/moto/kinesis/urls.py @@ -3,6 +3,7 @@ from .responses import KinesisResponse url_bases = [ "https?://kinesis.(.+).amazonaws.com", + "https?://firehose.(.+).amazonaws.com", ] url_paths = { diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 5cb05ebeb..37ecbf873 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -135,7 +135,7 @@ class Database(object): "engine": properties.get("Engine"), "engine_version": properties.get("EngineVersion"), "iops": properties.get("Iops"), - "master_password": properties.get('MasterUserPassword'), + "master_user_password": properties.get('MasterUserPassword'), "master_username": properties.get('MasterUsername'), "multi_az": properties.get("MultiAZ"), "port": properties.get('Port', 3306), diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index daa068aa6..bd51f6ea0 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -27,7 +27,7 @@ class RDS2Response(BaseResponse): "engine": self._get_param("Engine"), "engine_version": self._get_param("EngineVersion"), "iops": self._get_int_param("Iops"), - "master_password": self._get_param('MasterUserPassword'), + "master_user_password": self._get_param('MasterUserPassword'), "master_username": self._get_param('MasterUsername'), "multi_az": self._get_bool_param("MultiAZ"), # OptionGroupName @@ -504,4 +504,4 @@ ADD_TAGS_TO_RESOURCE_TEMPLATE = \ REMOVE_TAGS_FROM_RESOURCE_TEMPLATE = \ """{"RemoveTagsFromResourceResponse": {"ResponseMetadata": {"RequestId": "c6499a01-a664-11e4-8069-fb454b71a80e"}}} - """ \ No newline at end of file + """ diff --git a/moto/route53/models.py b/moto/route53/models.py index 00e23c38e..0e79d617a 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -68,6 +68,7 @@ class RecordSet(object): self.records = kwargs.get('ResourceRecords', []) self.set_identifier = kwargs.get('SetIdentifier') self.weight = kwargs.get('Weight') + self.region = kwargs.get('Region') self.health_check = kwargs.get('HealthCheckId') @classmethod @@ -89,6 +90,9 @@ class RecordSet(object): {% if record_set.weight %} {{ record_set.weight }} {% endif %} + {% if record_set.region %} + {{ record_set.region }} + {% endif %} {{ record_set.ttl }} {% for record in record_set.records %} diff --git a/moto/s3/models.py b/moto/s3/models.py index 5d0bfce2d..9e023f68d 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -24,6 +24,7 @@ class FakeKey(object): self.name = name self.value = value self.last_modified = datetime.datetime.utcnow() + self.acl = get_canned_acl('private') self._storage_class = storage self._metadata = {} self._expiry = None @@ -45,6 +46,9 @@ class FakeKey(object): def set_storage_class(self, storage_class): self._storage_class = storage_class + def set_acl(self, acl): + self.acl = acl + def append_to_value(self, value): self.value += value self.last_modified = datetime.datetime.utcnow() @@ -161,6 +165,61 @@ class FakeMultipart(object): yield self.parts[part_id] +class FakeGrantee(object): + def __init__(self, id='', uri='', display_name=''): + self.id = id + self.uri = uri + self.display_name = display_name + + @property + def type(self): + return 'Group' if self.uri else 'CanonicalUser' + + +ALL_USERS_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/global/AllUsers') +AUTHENTICATED_USERS_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/global/AuthenticatedUsers') +LOG_DELIVERY_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/s3/LogDelivery') + +PERMISSION_FULL_CONTROL = 'FULL_CONTROL' +PERMISSION_WRITE = 'WRITE' +PERMISSION_READ = 'READ' +PERMISSION_WRITE_ACP = 'WRITE_ACP' +PERMISSION_READ_ACP = 'READ_ACP' + + +class FakeGrant(object): + def __init__(self, grantees, permissions): + self.grantees = grantees + self.permissions = permissions + + +class FakeAcl(object): + def __init__(self, grants=[]): + self.grants = grants + + +def get_canned_acl(acl): + owner_grantee = FakeGrantee(id='75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a') + grants = [FakeGrant([owner_grantee], [PERMISSION_FULL_CONTROL])] + if acl == 'private': + pass # no other permissions + elif acl == 'public-read': + grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ])) + elif acl == 'public-read-write': + grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ, PERMISSION_WRITE])) + elif acl == 'authenticated-read': + grants.append(FakeGrant([AUTHENTICATED_USERS_GRANTEE], [PERMISSION_READ])) + elif acl == 'bucket-owner-read': + pass # TODO: bucket owner ACL + elif acl == 'bucket-owner-full-control': + pass # TODO: bucket owner ACL + elif acl == 'log-delivery-write': + grants.append(FakeGrant([LOG_DELIVERY_GRANTEE], [PERMISSION_READ_ACP, PERMISSION_WRITE])) + else: + assert False, 'Unknown canned acl: %s' % (acl,) + return FakeAcl(grants=grants) + + class LifecycleRule(object): def __init__(self, id=None, prefix=None, status=None, expiration_days=None, expiration_date=None, transition_days=None, @@ -185,6 +244,8 @@ class FakeBucket(object): self.versioning_status = None self.rules = [] self.policy = None + self.website_configuration = None + self.acl = get_canned_acl('private') @property def location(self): @@ -213,6 +274,9 @@ class FakeBucket(object): def delete_lifecycle(self): self.rules = [] + def set_website_configuration(self, website_configuration): + self.website_configuration = website_configuration + def get_cfn_attribute(self, attribute_name): from moto.cloudformation.exceptions import UnformattedGetAttTemplateException if attribute_name == 'DomainName': @@ -221,6 +285,9 @@ class FakeBucket(object): raise NotImplementedError('"Fn::GetAtt" : [ "{0}" , "WebsiteURL" ]"') raise UnformattedGetAttTemplateException() + def set_acl(self, acl): + self.acl = acl + class S3Backend(BaseBackend): @@ -284,6 +351,14 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) bucket.set_lifecycle(rules) + def set_bucket_website_configuration(self, bucket_name, website_configuration): + bucket = self.get_bucket(bucket_name) + bucket.set_website_configuration(website_configuration) + + def get_bucket_website_configuration(self, bucket_name): + bucket = self.get_bucket(bucket_name) + return bucket.website_configuration + def set_key(self, bucket_name, key_name, value, storage=None, etag=None): key_name = clean_key_name(key_name) @@ -399,7 +474,7 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) return bucket.keys.pop(key_name) - def copy_key(self, src_bucket_name, src_key_name, dest_bucket_name, dest_key_name, storage=None): + def copy_key(self, src_bucket_name, src_key_name, dest_bucket_name, dest_key_name, storage=None, acl=None): src_key_name = clean_key_name(src_key_name) dest_key_name = clean_key_name(dest_key_name) src_bucket = self.get_bucket(src_bucket_name) @@ -409,6 +484,17 @@ class S3Backend(BaseBackend): key = key.copy(dest_key_name) dest_bucket.keys[dest_key_name] = key if storage is not None: - dest_bucket.keys[dest_key_name].set_storage_class(storage) + key.set_storage_class(storage) + if acl is not None: + key.set_acl(acl) + + def set_bucket_acl(self, bucket_name, acl): + bucket = self.get_bucket(bucket_name) + bucket.set_acl(acl) + + def get_bucket_acl(self, bucket_name): + bucket = self.get_bucket(bucket_name) + return bucket.acl + s3_backend = S3Backend() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 68d1bb318..27a6d4536 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -9,7 +9,7 @@ import xmltodict from moto.core.responses import _TemplateEnvironmentMixin from .exceptions import BucketAlreadyExists, S3ClientError, InvalidPartOrder -from .models import s3_backend +from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl from .utils import bucket_name_from_url, metadata_from_headers from xml.dom import minidom @@ -22,10 +22,18 @@ def parse_key_name(pth): class ResponseObject(_TemplateEnvironmentMixin): - def __init__(self, backend, bucket_name_from_url, parse_key_name): + def __init__(self, backend, bucket_name_from_url, parse_key_name, + is_delete_keys=None): + super(ResponseObject, self).__init__() self.backend = backend self.bucket_name_from_url = bucket_name_from_url self.parse_key_name = parse_key_name + if is_delete_keys: + self.is_delete_keys = is_delete_keys + + @staticmethod + def is_delete_keys(path, bucket_name): + return path == u'/?delete' def all_buckets(self): # No bucket specified. Listing all buckets @@ -72,7 +80,7 @@ class ResponseObject(_TemplateEnvironmentMixin): elif method == 'GET': return self._bucket_response_get(bucket_name, querystring, headers) elif method == 'PUT': - return self._bucket_response_put(body, region_name, bucket_name, querystring, headers) + return self._bucket_response_put(request, body, region_name, bucket_name, querystring, headers) elif method == 'DELETE': return self._bucket_response_delete(body, bucket_name, querystring, headers) elif method == 'POST': @@ -94,29 +102,36 @@ class ResponseObject(_TemplateEnvironmentMixin): prefix = querystring.get('prefix', [None])[0] multiparts = [upload for upload in multiparts if upload.key_name.startswith(prefix)] template = self.response_template(S3_ALL_MULTIPARTS) - return 200, headers, template.render( + return template.render( bucket_name=bucket_name, uploads=multiparts) elif 'location' in querystring: bucket = self.backend.get_bucket(bucket_name) template = self.response_template(S3_BUCKET_LOCATION) - return 200, headers, template.render(location=bucket.location) + return template.render(location=bucket.location) elif 'lifecycle' in querystring: bucket = self.backend.get_bucket(bucket_name) if not bucket.rules: return 404, headers, "NoSuchLifecycleConfiguration" template = self.response_template(S3_BUCKET_LIFECYCLE_CONFIGURATION) - return 200, headers, template.render(rules=bucket.rules) + return template.render(rules=bucket.rules) elif 'versioning' in querystring: versioning = self.backend.get_bucket_versioning(bucket_name) template = self.response_template(S3_BUCKET_GET_VERSIONING) - return 200, headers, template.render(status=versioning) + return template.render(status=versioning) elif 'policy' in querystring: policy = self.backend.get_bucket_policy(bucket_name) if not policy: template = self.response_template(S3_NO_POLICY) return 404, headers, template.render(bucket_name=bucket_name) return 200, headers, policy + elif 'website' in querystring: + website_configuration = self.backend.get_bucket_website_configuration(bucket_name) + return website_configuration + elif 'acl' in querystring: + bucket = self.backend.get_bucket(bucket_name) + template = self.response_template(S3_OBJECT_ACL_RESPONSE) + return template.render(obj=bucket) elif 'versions' in querystring: delimiter = querystring.get('delimiter', [None])[0] encoding_type = querystring.get('encoding-type', [None])[0] @@ -157,7 +172,7 @@ class ResponseObject(_TemplateEnvironmentMixin): result_folders=result_folders ) - def _bucket_response_put(self, body, region_name, bucket_name, querystring, headers): + def _bucket_response_put(self, request, body, region_name, bucket_name, querystring, headers): if 'versioning' in querystring: ver = re.search('([A-Za-z]+)', body) if ver: @@ -176,6 +191,14 @@ class ResponseObject(_TemplateEnvironmentMixin): elif 'policy' in querystring: self.backend.set_bucket_policy(bucket_name, body) return 'True' + elif 'acl' in querystring: + acl = self._acl_from_headers(request.headers) + # TODO: Support the XML-based ACL format + self.backend.set_bucket_acl(bucket_name, acl) + return "" + elif 'website' in querystring: + self.backend.set_bucket_website_configuration(bucket_name, body) + return "" else: try: new_bucket = self.backend.create_bucket(bucket_name, region_name) @@ -209,7 +232,7 @@ class ResponseObject(_TemplateEnvironmentMixin): return 409, headers, template.render(bucket=removed_bucket) def _bucket_response_post(self, request, bucket_name, headers): - if request.path == u'/?delete': + if self.is_delete_keys(request.path, bucket_name): return self._bucket_response_delete_keys(request, bucket_name, headers) # POST to bucket-url should create file from form @@ -294,7 +317,7 @@ class ResponseObject(_TemplateEnvironmentMixin): def _key_response(self, request, full_url, headers): parsed_url = urlparse(full_url) - query = parse_qs(parsed_url.query) + query = parse_qs(parsed_url.query, keep_blank_values=True) method = request.method key_name = self.parse_key_name(parsed_url.path) @@ -310,18 +333,18 @@ class ResponseObject(_TemplateEnvironmentMixin): if method == 'GET': return self._key_response_get(bucket_name, query, key_name, headers) elif method == 'PUT': - return self._key_response_put(request, parsed_url, body, bucket_name, query, key_name, headers) + return self._key_response_put(request, body, bucket_name, query, key_name, headers) elif method == 'HEAD': return self._key_response_head(bucket_name, key_name, headers) elif method == 'DELETE': return self._key_response_delete(bucket_name, query, key_name, headers) elif method == 'POST': - return self._key_response_post(request, body, parsed_url, bucket_name, query, key_name, headers) + return self._key_response_post(request, body, bucket_name, query, key_name, headers) else: raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) def _key_response_get(self, bucket_name, query, key_name, headers): - if 'uploadId' in query: + if query.get('uploadId'): upload_id = query['uploadId'][0] parts = self.backend.list_multipart(bucket_name, upload_id) template = self.response_template(S3_MULTIPART_LIST_RESPONSE) @@ -335,14 +358,18 @@ class ResponseObject(_TemplateEnvironmentMixin): version_id = query.get('versionId', [None])[0] key = self.backend.get_key( bucket_name, key_name, version_id=version_id) + if 'acl' in query: + template = self.response_template(S3_OBJECT_ACL_RESPONSE) + return 200, headers, template.render(obj=key) + if key: headers.update(key.metadata) return 200, headers, key.value else: return 404, headers, "" - def _key_response_put(self, request, parsed_url, body, bucket_name, query, key_name, headers): - if 'uploadId' in query and 'partNumber' in query: + def _key_response_put(self, request, body, bucket_name, query, key_name, headers): + if query.get('uploadId') and query.get('partNumber'): upload_id = query['uploadId'][0] part_number = int(query['partNumber'][0]) if 'x-amz-copy-source' in request.headers: @@ -361,16 +388,19 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, headers, response storage_class = request.headers.get('x-amz-storage-class', 'STANDARD') + acl = self._acl_from_headers(request.headers) - if parsed_url.query == 'acl': - # We don't implement ACL yet, so just return + if 'acl' in query: + key = self.backend.get_key(bucket_name, key_name) + # TODO: Support the XML-based ACL format + key.set_acl(acl) return 200, headers, "" if 'x-amz-copy-source' in request.headers: # Copy key src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1) self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, - storage=storage_class) + storage=storage_class, acl=acl) mdirective = request.headers.get('x-amz-metadata-directive') if mdirective is not None and mdirective == 'REPLACE': new_key = self.backend.get_key(bucket_name, key_name) @@ -393,6 +423,7 @@ class ResponseObject(_TemplateEnvironmentMixin): request.streaming = True metadata = metadata_from_headers(request.headers) new_key.set_metadata(metadata) + new_key.set_acl(acl) template = self.response_template(S3_OBJECT_RESPONSE) headers.update(new_key.response_dict) @@ -407,8 +438,40 @@ class ResponseObject(_TemplateEnvironmentMixin): else: return 404, headers, "" + def _acl_from_headers(self, headers): + canned_acl = headers.get('x-amz-acl', '') + if canned_acl: + return get_canned_acl(canned_acl) + + grants = [] + for header, value in headers.items(): + if not header.startswith('x-amz-grant-'): + continue + + permission = { + 'read': 'READ', + 'write': 'WRITE', + 'read-acp': 'READ_ACP', + 'write-acp': 'WRITE_ACP', + 'full-control': 'FULL_CONTROL', + }[header[len('x-amz-grant-'):]] + + grantees = [] + for key_and_value in value.split(","): + key, value = re.match('([^=]+)="([^"]+)"', key_and_value.strip()).groups() + if key.lower() == 'id': + grantees.append(FakeGrantee(id=value)) + else: + grantees.append(FakeGrantee(uri=value)) + grants.append(FakeGrant(grantees, [permission])) + + if grants: + return FakeAcl(grants) + else: + return None + def _key_response_delete(self, bucket_name, query, key_name, headers): - if 'uploadId' in query: + if query.get('uploadId'): upload_id = query['uploadId'][0] self.backend.cancel_multipart(bucket_name, upload_id) return 204, headers, "" @@ -428,8 +491,8 @@ class ResponseObject(_TemplateEnvironmentMixin): raise InvalidPartOrder() yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText) - def _key_response_post(self, request, body, parsed_url, bucket_name, query, key_name, headers): - if body == b'' and parsed_url.query == 'uploads': + def _key_response_post(self, request, body, bucket_name, query, key_name, headers): + if body == b'' and 'uploads' in query: metadata = metadata_from_headers(request.headers) multipart = self.backend.initiate_multipart(bucket_name, key_name, metadata) @@ -441,7 +504,7 @@ class ResponseObject(_TemplateEnvironmentMixin): ) return 200, headers, response - if 'uploadId' in query: + if query.get('uploadId'): body = self._complete_multipart_body(body) upload_id = query['uploadId'][0] key = self.backend.complete_multipart(bucket_name, upload_id, body) @@ -451,7 +514,7 @@ class ResponseObject(_TemplateEnvironmentMixin): key_name=key.name, etag=key.etag, ) - elif parsed_url.query == 'restore': + elif 'restore' in query: es = minidom.parseString(body).getElementsByTagName('Days') days = es[0].childNodes[0].wholeText key = self.backend.get_key(bucket_name, key_name) @@ -635,6 +698,37 @@ S3_OBJECT_RESPONSE = """ + + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + + {% for grant in obj.acl.grants %} + + {% for grantee in grant.grantees %} + + {% if grantee.uri %} + {{ grantee.uri }} + {% endif %} + {% if grantee.id %} + {{ grantee.id }} + {% endif %} + {% if grantee.display_name %} + {{ grantee.display_name }} + {% endif %} + + {% endfor %} + {% for permission in grant.permissions %} + {{ permission }} + {% endfor %} + + {% endfor %} + + """ + S3_OBJECT_COPY_RESPONSE = """ {{ key.etag }} @@ -710,7 +804,7 @@ S3_ALL_MULTIPARTS = """ 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a - OwnerDisplayName + webfile STANDARD 2010-11-10T20:48:33.000Z diff --git a/moto/s3bucket_path/responses.py b/moto/s3bucket_path/responses.py index 4e2f87376..6de82fbb0 100644 --- a/moto/s3bucket_path/responses.py +++ b/moto/s3bucket_path/responses.py @@ -9,8 +9,14 @@ from moto.s3.responses import ResponseObject def parse_key_name(pth): return "/".join(pth.rstrip("/").split("/")[2:]) + +def is_delete_keys(path, bucket_name): + return path == u'/' + bucket_name + u'/?delete' + + S3BucketPathResponseInstance = ResponseObject( s3bucket_path_backend, bucket_name_from_url, parse_key_name, + is_delete_keys, ) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index bd4129dc2..efb75dd9c 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -34,7 +34,7 @@ class Message(object): @property def md5(self): body_md5 = hashlib.md5() - body_md5.update(self.body.encode('utf-8')) + body_md5.update(self._body.encode('utf-8')) return body_md5.hexdigest() @property @@ -106,9 +106,10 @@ class Queue(object): 'VisibilityTimeout', 'WaitTimeSeconds'] - def __init__(self, name, visibility_timeout, wait_time_seconds): + def __init__(self, name, visibility_timeout, wait_time_seconds, region): self.name = name self.visibility_timeout = visibility_timeout or 30 + self.region = region # wait_time_seconds will be set to immediate return messages self.wait_time_seconds = wait_time_seconds or 0 @@ -179,6 +180,10 @@ class Queue(object): result[attribute] = getattr(self, camelcase_to_underscores(attribute)) return result + @property + def url(self): + return "http://sqs.{0}.amazonaws.com/123456789012/{1}".format(self.region, self.name) + @property def messages(self): return [message for message in self._messages if message.visible and not message.delayed] @@ -196,14 +201,20 @@ class Queue(object): class SQSBackend(BaseBackend): - def __init__(self): + def __init__(self, region_name): + self.region_name = region_name self.queues = {} super(SQSBackend, self).__init__() + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + def create_queue(self, name, visibility_timeout, wait_time_seconds): queue = self.queues.get(name) if queue is None: - queue = Queue(name, visibility_timeout, wait_time_seconds) + queue = Queue(name, visibility_timeout, wait_time_seconds, self.region_name) self.queues[name] = queue return queue @@ -314,4 +325,4 @@ class SQSBackend(BaseBackend): sqs_backends = {} for region in boto.sqs.regions(): - sqs_backends[region.name] = SQSBackend() + sqs_backends[region.name] = SQSBackend(region.name) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 648e939d2..d1ba5b6dd 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -11,6 +11,7 @@ from .exceptions import ( ) MAXIMUM_VISIBILTY_TIMEOUT = 43200 +MAXIMUM_MESSAGE_LENGTH = 262144 # 256 KiB DEFAULT_RECEIVED_MESSAGES = 1 SQS_REGION_REGEX = r'://(.+?)\.queue\.amazonaws\.com' @@ -106,6 +107,9 @@ class SQSResponse(BaseResponse): message = self.querystring.get("MessageBody")[0] delay_seconds = self.querystring.get('DelaySeconds') + if len(message) > MAXIMUM_MESSAGE_LENGTH: + return ERROR_TOO_LONG_RESPONSE, dict(status=400) + if delay_seconds: delay_seconds = int(delay_seconds[0]) else: @@ -232,7 +236,7 @@ class SQSResponse(BaseResponse): CREATE_QUEUE_RESPONSE = """ - http://sqs.us-east-1.amazonaws.com/123456789012/{{ queue.name }} + {{ queue.url }} {{ queue.visibility_timeout }} @@ -244,7 +248,7 @@ CREATE_QUEUE_RESPONSE = """ GET_QUEUE_URL_RESPONSE = """ - http://sqs.us-east-1.amazonaws.com/123456789012/{{ queue.name }} + {{ queue.url }} 470a6f13-2ed9-4181-ad8a-2fdea142988e @@ -254,7 +258,7 @@ GET_QUEUE_URL_RESPONSE = """ LIST_QUEUES_RESPONSE = """ {% for queue in queues %} - http://sqs.us-east-1.amazonaws.com/123456789012/{{ queue.name }} + {{ queue.url }} {% endfor %} @@ -417,3 +421,13 @@ PURGE_QUEUE_RESPONSE = """ """ + +ERROR_TOO_LONG_RESPONSE = """ + + Sender + InvalidParameterValue + One or more parameters are invalid. Reason: Message must be shorter than 262144 bytes. + + + 6fde8d1e-52cd-4581-8cd9-c512f4c64223 +""" diff --git a/requirements-dev.txt b/requirements-dev.txt index 5cd0acb14..bd4b6d237 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ coverage freezegun flask boto3 +six \ No newline at end of file diff --git a/setup.py b/setup.py index e73362fb9..a65f5e15e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.12', + version='0.4.18', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 8b8f8f320..41286442d 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import boto from boto.ec2.autoscale.launchconfig import LaunchConfiguration from boto.ec2.autoscale.group import AutoScalingGroup +from boto.ec2.autoscale import Tag import sure # noqa from moto import mock_autoscaling, mock_ec2 @@ -18,6 +19,7 @@ def test_create_autoscaling_group(): ) conn.create_launch_configuration(config) + group = AutoScalingGroup( name='tester_group', availability_zones=['us-east-1c', 'us-east-1b'], @@ -32,6 +34,13 @@ def test_create_autoscaling_group(): placement_group="test_placement", vpc_zone_identifier='subnet-1234abcd', termination_policies=["OldestInstance", "NewestInstance"], + tags=[Tag( + resource_id='tester_group', + key='test_key', + value='test_value', + propagate_at_launch=True + ) + ], ) conn.create_auto_scaling_group(group) @@ -50,6 +59,12 @@ def test_create_autoscaling_group(): list(group.load_balancers).should.equal(["test_lb"]) group.placement_group.should.equal("test_placement") list(group.termination_policies).should.equal(["OldestInstance", "NewestInstance"]) + len(list(group.tags)).should.equal(1) + tag = list(group.tags)[0] + tag.resource_id.should.equal('tester_group') + tag.key.should.equal('test_key') + tag.value.should.equal('test_value') + tag.propagate_at_launch.should.equal(True) @mock_autoscaling @@ -88,6 +103,7 @@ def test_create_autoscaling_groups_defaults(): list(group.load_balancers).should.equal([]) group.placement_group.should.equal(None) list(group.termination_policies).should.equal([]) + list(group.tags).should.equal([]) @mock_autoscaling diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index 236c33e7d..5ca20fe04 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -40,7 +40,17 @@ def test_create_stack(): stack = conn.describe_stacks()[0] stack.stack_name.should.equal('test_stack') - stack.get_template().should.equal(dummy_template) + stack.get_template().should.equal({ + 'GetTemplateResponse': { + 'GetTemplateResult': { + 'TemplateBody': dummy_template_json, + 'ResponseMetadata': { + 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' + } + } + } + + }) @mock_cloudformation @@ -83,7 +93,18 @@ def test_create_stack_from_s3_url(): stack = conn.describe_stacks()[0] stack.stack_name.should.equal('new-stack') - stack.get_template().should.equal(dummy_template) + stack.get_template().should.equal( + { + 'GetTemplateResponse': { + 'GetTemplateResult': { + 'TemplateBody': dummy_template_json, + 'ResponseMetadata': { + 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' + } + } + } + + }) @mock_cloudformation @@ -138,7 +159,17 @@ def test_get_template_by_name(): ) template = conn.get_template("test_stack") - template.should.equal(dummy_template) + template.should.equal({ + 'GetTemplateResponse': { + 'GetTemplateResult': { + 'TemplateBody': dummy_template_json, + 'ResponseMetadata': { + 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' + } + } + } + + }) @mock_cloudformation @@ -243,4 +274,13 @@ def test_stack_tags(): # conn.update_stack("test_stack", dummy_template_json2) # stack = conn.describe_stacks()[0] -# stack.get_template().should.equal(dummy_template2) +# stack.get_template().should.equal({ +# 'GetTemplateResponse': { +# 'GetTemplateResult': { +# 'TemplateBody': dummy_template_json2, +# 'ResponseMetadata': { +# 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' +# } +# } +# } +# }) diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index c1305d843..a38a1029b 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -3,6 +3,7 @@ import json import boto import boto.cloudformation +import boto.datapipeline import boto.ec2 import boto.ec2.autoscale import boto.ec2.elb @@ -17,6 +18,7 @@ import sure # noqa from moto import ( mock_autoscaling, mock_cloudformation, + mock_datapipeline, mock_ec2, mock_elb, mock_iam, @@ -287,7 +289,6 @@ def test_stack_elb_integration_with_attached_ec2_instances(): ec2_conn = boto.ec2.connect_to_region("us-west-1") reservation = ec2_conn.get_all_instances()[0] ec2_instance = reservation.instances[0] - instance_id = ec2_instance.id load_balancer.instances[0].id.should.equal(ec2_instance.id) list(load_balancer.availability_zones).should.equal(['us-east1']) @@ -1395,3 +1396,83 @@ def test_subnets_should_be_created_with_availability_zone(): ) subnet = vpc_conn.get_all_subnets(filters={'cidrBlock': '10.0.0.0/24'})[0] subnet.availability_zone.should.equal('us-west-1b') + + +@mock_cloudformation +@mock_datapipeline +def test_datapipeline(): + dp_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "dataPipeline": { + "Properties": { + "Activate": "true", + "Name": "testDataPipeline", + "PipelineObjects": [ + { + "Fields": [ + { + "Key": "failureAndRerunMode", + "StringValue": "CASCADE" + }, + { + "Key": "scheduleType", + "StringValue": "cron" + }, + { + "Key": "schedule", + "RefValue": "DefaultSchedule" + }, + { + "Key": "pipelineLogUri", + "StringValue": "s3://bucket/logs" + }, + { + "Key": "type", + "StringValue": "Default" + }, + ], + "Id": "Default", + "Name": "Default" + }, + { + "Fields": [ + { + "Key": "startDateTime", + "StringValue": "1970-01-01T01:00:00" + }, + { + "Key": "period", + "StringValue": "1 Day" + }, + { + "Key": "type", + "StringValue": "Schedule" + } + ], + "Id": "DefaultSchedule", + "Name": "RunOnce" + } + ], + "PipelineTags": [] + }, + "Type": "AWS::DataPipeline::Pipeline" + } + } + } + cf_conn = boto.cloudformation.connect_to_region("us-east-1") + template_json = json.dumps(dp_template) + stack_id = cf_conn.create_stack( + "test_stack", + template_body=template_json, + ) + + dp_conn = boto.datapipeline.connect_to_region('us-east-1') + data_pipelines = dp_conn.list_pipelines() + + data_pipelines['pipelineIdList'].should.have.length_of(1) + data_pipelines['pipelineIdList'][0]['name'].should.equal('testDataPipeline') + + stack_resources = cf_conn.list_stack_resources(stack_id) + stack_resources.should.have.length_of(1) + stack_resources[0].physical_resource_id.should.equal(data_pipelines['pipelineIdList'][0]['id']) diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py new file mode 100644 index 000000000..5a958492f --- /dev/null +++ b/tests/test_datapipeline/test_datapipeline.py @@ -0,0 +1,175 @@ +from __future__ import unicode_literals + +import boto.datapipeline +import sure # noqa + +from moto import mock_datapipeline +from moto.datapipeline.utils import remove_capitalization_of_dict_keys + + +def get_value_from_fields(key, fields): + for field in fields: + if field['key'] == key: + return field['stringValue'] + + +@mock_datapipeline +def test_create_pipeline(): + conn = boto.datapipeline.connect_to_region("us-west-2") + + res = conn.create_pipeline("mypipeline", "some-unique-id") + + pipeline_id = res["pipelineId"] + pipeline_descriptions = conn.describe_pipelines([pipeline_id])["pipelineDescriptionList"] + pipeline_descriptions.should.have.length_of(1) + + pipeline_description = pipeline_descriptions[0] + pipeline_description['name'].should.equal("mypipeline") + pipeline_description["pipelineId"].should.equal(pipeline_id) + fields = pipeline_description['fields'] + + get_value_from_fields('@pipelineState', fields).should.equal("PENDING") + get_value_from_fields('uniqueId', fields).should.equal("some-unique-id") + + +PIPELINE_OBJECTS = [ + { + "id": "Default", + "name": "Default", + "fields": [{ + "key": "workerGroup", + "stringValue": "workerGroup" + }] + }, + { + "id": "Schedule", + "name": "Schedule", + "fields": [{ + "key": "startDateTime", + "stringValue": "2012-12-12T00:00:00" + }, { + "key": "type", + "stringValue": "Schedule" + }, { + "key": "period", + "stringValue": "1 hour" + }, { + "key": "endDateTime", + "stringValue": "2012-12-21T18:00:00" + }] + }, + { + "id": "SayHello", + "name": "SayHello", + "fields": [{ + "key": "type", + "stringValue": "ShellCommandActivity" + }, { + "key": "command", + "stringValue": "echo hello" + }, { + "key": "parent", + "refValue": "Default" + }, { + "key": "schedule", + "refValue": "Schedule" + }] + } +] + + +@mock_datapipeline +def test_creating_pipeline_definition(): + conn = boto.datapipeline.connect_to_region("us-west-2") + res = conn.create_pipeline("mypipeline", "some-unique-id") + pipeline_id = res["pipelineId"] + + conn.put_pipeline_definition(PIPELINE_OBJECTS, pipeline_id) + + pipeline_definition = conn.get_pipeline_definition(pipeline_id) + pipeline_definition['pipelineObjects'].should.have.length_of(3) + default_object = pipeline_definition['pipelineObjects'][0] + default_object['name'].should.equal("Default") + default_object['id'].should.equal("Default") + default_object['fields'].should.equal([{ + "key": "workerGroup", + "stringValue": "workerGroup" + }]) + + +@mock_datapipeline +def test_describing_pipeline_objects(): + conn = boto.datapipeline.connect_to_region("us-west-2") + res = conn.create_pipeline("mypipeline", "some-unique-id") + pipeline_id = res["pipelineId"] + + conn.put_pipeline_definition(PIPELINE_OBJECTS, pipeline_id) + + objects = conn.describe_objects(["Schedule", "Default"], pipeline_id)['pipelineObjects'] + + objects.should.have.length_of(2) + default_object = [x for x in objects if x['id'] == 'Default'][0] + default_object['name'].should.equal("Default") + default_object['fields'].should.equal([{ + "key": "workerGroup", + "stringValue": "workerGroup" + }]) + + +@mock_datapipeline +def test_activate_pipeline(): + conn = boto.datapipeline.connect_to_region("us-west-2") + + res = conn.create_pipeline("mypipeline", "some-unique-id") + + pipeline_id = res["pipelineId"] + conn.activate_pipeline(pipeline_id) + + pipeline_descriptions = conn.describe_pipelines([pipeline_id])["pipelineDescriptionList"] + pipeline_descriptions.should.have.length_of(1) + pipeline_description = pipeline_descriptions[0] + fields = pipeline_description['fields'] + + get_value_from_fields('@pipelineState', fields).should.equal("SCHEDULED") + + +@mock_datapipeline +def test_listing_pipelines(): + conn = boto.datapipeline.connect_to_region("us-west-2") + res1 = conn.create_pipeline("mypipeline1", "some-unique-id1") + res2 = conn.create_pipeline("mypipeline2", "some-unique-id2") + + response = conn.list_pipelines() + + response["hasMoreResults"].should.be(False) + response["marker"].should.be.none + response["pipelineIdList"].should.have.length_of(2) + response["pipelineIdList"].should.contain({ + "id": res1["pipelineId"], + "name": "mypipeline1", + }) + response["pipelineIdList"].should.contain({ + "id": res2["pipelineId"], + "name": "mypipeline2" + }) + + +# testing a helper function +def test_remove_capitalization_of_dict_keys(): + result = remove_capitalization_of_dict_keys( + { + "Id": "IdValue", + "Fields": [{ + "Key": "KeyValue", + "StringValue": "StringValueValue" + }] + } + ) + + result.should.equal({ + "id": "IdValue", + "fields": [{ + "key": "KeyValue", + "stringValue": "StringValueValue" + }], + }) diff --git a/tests/test_datapipeline/test_server.py b/tests/test_datapipeline/test_server.py new file mode 100644 index 000000000..012c5ad55 --- /dev/null +++ b/tests/test_datapipeline/test_server.py @@ -0,0 +1,27 @@ +from __future__ import unicode_literals + +import json +import sure # noqa + +import moto.server as server +from moto import mock_datapipeline + +''' +Test the different server responses +''' + + +@mock_datapipeline +def test_list_streams(): + backend = server.create_backend_app("datapipeline") + test_client = backend.test_client() + + res = test_client.post('/', + data={"pipelineIds": ["ASdf"]}, + headers={"X-Amz-Target": "DataPipeline.DescribePipelines"}, + ) + + json_data = json.loads(res.data.decode("utf-8")) + json_data.should.equal({ + 'pipelineDescriptionList': [] + }) diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 808805b8d..6baeb8a12 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -122,6 +122,33 @@ def test_item_add_and_describe_and_update(): }) +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_item_partial_save(): + table = create_table() + + data = { + 'forum_name': 'LOLCat Forum', + 'Body': 'http://url_to_lolcat.gif', + 'SentBy': 'User A', + } + + table.put_item(data=data) + returned_item = table.get_item(forum_name="LOLCat Forum") + + returned_item['SentBy'] = 'User B' + returned_item.partial_save() + + returned_item = table.get_item( + forum_name='LOLCat Forum' + ) + dict(returned_item).should.equal({ + 'forum_name': 'LOLCat Forum', + 'Body': 'http://url_to_lolcat.gif', + 'SentBy': 'User B', + }) + + @requires_boto_gte("2.9") @mock_dynamodb2 def test_item_put_without_table(): diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index a16a6659b..06b55841e 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -33,6 +33,7 @@ def test_create_and_delete_volume(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none + @mock_ec2 def test_filter_volume_by_id(): conn = boto.connect_ec2('the_key', 'the_secret') @@ -93,7 +94,9 @@ def test_create_snapshot(): conn = boto.connect_ec2('the_key', 'the_secret') volume = conn.create_volume(80, "us-east-1a") - volume.create_snapshot('a test snapshot') + snapshot = volume.create_snapshot('a test snapshot') + snapshot.update() + snapshot.status.should.equal('completed') snapshots = conn.get_all_snapshots() snapshots.should.have.length_of(1) @@ -114,6 +117,7 @@ def test_create_snapshot(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none + @mock_ec2 def test_filter_snapshot_by_id(): conn = boto.connect_ec2('the_key', 'the_secret') @@ -134,6 +138,7 @@ def test_filter_snapshot_by_id(): s.volume_id.should.be.within([volume2.id, volume3.id]) s.region.name.should.equal(conn.region.name) + @mock_ec2 def test_snapshot_attribute(): conn = boto.connect_ec2('the_key', 'the_secret') @@ -215,6 +220,20 @@ def test_snapshot_attribute(): user_ids=['user']).should.throw(NotImplementedError) +@mock_ec2 +def test_create_volume_from_snapshot(): + conn = boto.connect_ec2('the_key', 'the_secret') + volume = conn.create_volume(80, "us-east-1a") + + snapshot = volume.create_snapshot('a test snapshot') + snapshot.update() + snapshot.status.should.equal('completed') + + new_volume = snapshot.create_volume('us-east-1a') + new_volume.size.should.equal(80) + new_volume.snapshot_id.should.equal(snapshot.id) + + @mock_ec2 def test_modify_attribute_blockDeviceMapping(): """ @@ -234,3 +253,13 @@ def test_modify_attribute_blockDeviceMapping(): instance = ec2_backends[conn.region.name].get_instance(instance.id) instance.block_device_mapping.should.have.key('/dev/sda1') instance.block_device_mapping['/dev/sda1'].delete_on_termination.should.be(True) + + +@mock_ec2 +def test_volume_tag_escaping(): + conn = boto.connect_ec2('the_key', 'the_secret') + vol = conn.create_volume(10, 'us-east-1a') + snapshot = conn.create_snapshot(vol.id, 'Desc') + snapshot.add_tags({'key': ''}) + + dict(conn.get_all_snapshots()[0].tags).should.equal({'key': ''}) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 629fc67ee..2e9b9834a 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -53,7 +53,7 @@ def test_instance_launch_and_terminate(): instances.should.have.length_of(1) instances[0].id.should.equal(instance.id) instances[0].state.should.equal('running') - instances[0].launch_time.should.equal("2014-01-01T05:00:00Z") + instances[0].launch_time.should.equal("2014-01-01T05:00:00.000Z") instances[0].vpc_id.should.equal(None) root_device_name = instances[0].root_device_name diff --git a/tests/test_ec2/test_key_pairs.py b/tests/test_ec2/test_key_pairs.py index 2390f45ce..858e064fb 100644 --- a/tests/test_ec2/test_key_pairs.py +++ b/tests/test_ec2/test_key_pairs.py @@ -85,3 +85,27 @@ def test_key_pairs_delete_exist(): r = conn.delete_key_pair('foo') r.should.be.ok assert len(conn.get_all_key_pairs()) == 0 + + +@mock_ec2 +def test_key_pairs_import(): + conn = boto.connect_ec2('the_key', 'the_secret') + kp = conn.import_key_pair('foo', b'content') + assert kp.name == 'foo' + kps = conn.get_all_key_pairs() + assert len(kps) == 1 + assert kps[0].name == 'foo' + + +@mock_ec2 +def test_key_pairs_import_exist(): + conn = boto.connect_ec2('the_key', 'the_secret') + kp = conn.import_key_pair('foo', b'content') + assert kp.name == 'foo' + assert len(conn.get_all_key_pairs()) == 1 + + with assert_raises(EC2ResponseError) as cm: + conn.create_key_pair('foo') + cm.exception.code.should.equal('InvalidKeyPair.Duplicate') + cm.exception.status.should.equal(400) + cm.exception.request_id.should_not.be.none diff --git a/tests/test_ec2/test_utils.py b/tests/test_ec2/test_utils.py new file mode 100644 index 000000000..ef540e193 --- /dev/null +++ b/tests/test_ec2/test_utils.py @@ -0,0 +1,8 @@ +from moto.ec2 import utils + + +def test_random_key_pair(): + key_pair = utils.random_key_pair() + assert len(key_pair['fingerprint']) == 59 + assert key_pair['material'].startswith('---- BEGIN RSA PRIVATE KEY ----') + assert key_pair['material'].endswith('-----END RSA PRIVATE KEY-----') diff --git a/tests/test_kinesis/test_firehose.py b/tests/test_kinesis/test_firehose.py new file mode 100644 index 000000000..37585fe5e --- /dev/null +++ b/tests/test_kinesis/test_firehose.py @@ -0,0 +1,141 @@ +from __future__ import unicode_literals + +import datetime + +from botocore.exceptions import ClientError +import boto3 +from freezegun import freeze_time +import sure # noqa + +from moto import mock_kinesis + + +def create_stream(client, stream_name): + return client.create_delivery_stream( + DeliveryStreamName=stream_name, + RedshiftDestinationConfiguration={ + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'ClusterJDBCURL': 'jdbc:redshift://host.amazonaws.com:5439/database', + 'CopyCommand': { + 'DataTableName': 'outputTable', + 'CopyOptions': "CSV DELIMITER ',' NULL '\\0'" + }, + 'Username': 'username', + 'Password': 'password', + 'S3Configuration': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'BucketARN': 'arn:aws:s3:::kinesis-test', + 'Prefix': 'myFolder/', + 'BufferingHints': { + 'SizeInMBs': 123, + 'IntervalInSeconds': 124 + }, + 'CompressionFormat': 'UNCOMPRESSED', + } + } + ) + + +@mock_kinesis +@freeze_time("2015-03-01") +def test_create_stream(): + client = boto3.client('firehose', region_name='us-east-1') + + response = create_stream(client, 'stream1') + stream_arn = response['DeliveryStreamARN'] + + response = client.describe_delivery_stream(DeliveryStreamName='stream1') + stream_description = response['DeliveryStreamDescription'] + + # Sure and Freezegun don't play nicely together + created = stream_description.pop('CreateTimestamp') + last_updated = stream_description.pop('LastUpdateTimestamp') + from dateutil.tz import tzlocal + assert created == datetime.datetime(2015, 3, 1, tzinfo=tzlocal()) + assert last_updated == datetime.datetime(2015, 3, 1, tzinfo=tzlocal()) + + stream_description.should.equal({ + 'DeliveryStreamName': 'stream1', + 'DeliveryStreamARN': stream_arn, + 'DeliveryStreamStatus': 'ACTIVE', + 'VersionId': 'string', + 'Destinations': [ + { + 'DestinationId': 'string', + 'RedshiftDestinationDescription': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'ClusterJDBCURL': 'jdbc:redshift://host.amazonaws.com:5439/database', + 'CopyCommand': { + 'DataTableName': 'outputTable', + 'CopyOptions': "CSV DELIMITER ',' NULL '\\0'" + }, + 'Username': 'username', + 'S3DestinationDescription': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'BucketARN': 'arn:aws:s3:::kinesis-test', + 'Prefix': 'myFolder/', + 'BufferingHints': { + 'SizeInMBs': 123, + 'IntervalInSeconds': 124 + }, + 'CompressionFormat': 'UNCOMPRESSED', + } + } + }, + ], + "HasMoreDestinations": False, + }) + + +@mock_kinesis +@freeze_time("2015-03-01") +def test_deescribe_non_existant_stream(): + client = boto3.client('firehose', region_name='us-east-1') + + client.describe_delivery_stream.when.called_with(DeliveryStreamName='not-a-stream').should.throw(ClientError) + + +@mock_kinesis +@freeze_time("2015-03-01") +def test_list_and_delete_stream(): + client = boto3.client('firehose', region_name='us-east-1') + + create_stream(client, 'stream1') + create_stream(client, 'stream2') + + set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal(set(['stream1', 'stream2'])) + + client.delete_delivery_stream(DeliveryStreamName='stream1') + + set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal(set(['stream2'])) + + +@mock_kinesis +def test_put_record(): + client = boto3.client('firehose', region_name='us-east-1') + + create_stream(client, 'stream1') + client.put_record( + DeliveryStreamName='stream1', + Record={ + 'Data': 'some data' + } + ) + + +@mock_kinesis +def test_put_record_batch(): + client = boto3.client('firehose', region_name='us-east-1') + + create_stream(client, 'stream1') + client.put_record_batch( + DeliveryStreamName='stream1', + Records=[ + { + 'Data': 'some data1' + }, + { + 'Data': 'some data2' + }, + ] + ) diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 669b8b2e3..5556bfc7d 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -239,3 +239,25 @@ def test_deleting_weighted_route(): cname = zone.get_cname('cname.testdns.aws.com.', all=True) # When get_cname only had one result, it returns just that result instead of a list. cname.identifier.should.equal('success-test-bar') + + +@mock_route53 +def test_deleting_latency_route(): + conn = boto.connect_route53() + + conn.create_hosted_zone("testdns.aws.com.") + zone = conn.get_zone("testdns.aws.com.") + + zone.add_cname("cname.testdns.aws.com", "example.com", identifier=('success-test-foo', 'us-west-2')) + zone.add_cname("cname.testdns.aws.com", "example.com", identifier=('success-test-bar', 'us-west-1')) + + cnames = zone.get_cname('cname.testdns.aws.com.', all=True) + cnames.should.have.length_of(2) + foo_cname = [cname for cname in cnames if cname.identifier == 'success-test-foo'][0] + foo_cname.region.should.equal('us-west-2') + + zone.delete_record(foo_cname) + cname = zone.get_cname('cname.testdns.aws.com.', all=True) + # When get_cname only had one result, it returns just that result instead of a list. + cname.identifier.should.equal('success-test-bar') + cname.region.should.equal('us-west-1') diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f5d7cba15..aff033c5f 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -726,7 +726,7 @@ def test_list_versions(): @mock_s3 -def test_acl_is_ignored_for_now(): +def test_acl_setting(): conn = boto.connect_s3() bucket = conn.create_bucket('foobar') content = b'imafile' @@ -741,6 +741,74 @@ def test_acl_is_ignored_for_now(): assert key.get_contents_as_string() == content + grants = key.get_acl().acl.grants + assert any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'READ' for g in grants), grants + + +@mock_s3 +def test_acl_setting_via_headers(): + conn = boto.connect_s3() + bucket = conn.create_bucket('foobar') + content = b'imafile' + keyname = 'test.txt' + + key = Key(bucket, name=keyname) + key.content_type = 'text/plain' + key.set_contents_from_string(content, headers={ + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/global/AllUsers"' + }) + + key = bucket.get_key(keyname) + + assert key.get_contents_as_string() == content + + grants = key.get_acl().acl.grants + assert any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'FULL_CONTROL' for g in grants), grants + + +@mock_s3 +def test_acl_switching(): + conn = boto.connect_s3() + bucket = conn.create_bucket('foobar') + content = b'imafile' + keyname = 'test.txt' + + key = Key(bucket, name=keyname) + key.content_type = 'text/plain' + key.set_contents_from_string(content, policy='public-read') + key.set_acl('private') + + grants = key.get_acl().acl.grants + assert not any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'READ' for g in grants), grants + + +@mock_s3 +def test_bucket_acl_setting(): + conn = boto.connect_s3() + bucket = conn.create_bucket('foobar') + + bucket.make_public() + + grants = bucket.get_acl().acl.grants + assert any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'READ' for g in grants), grants + + +@mock_s3 +def test_bucket_acl_switching(): + conn = boto.connect_s3() + bucket = conn.create_bucket('foobar') + bucket.make_public() + + bucket.set_acl('private') + + grants = bucket.get_acl().acl.grants + assert not any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'READ' for g in grants), grants + @mock_s3 def test_unicode_key(): @@ -902,5 +970,33 @@ def test_boto3_head_object(): s3.Object('blah', 'hello.txt').meta.client.head_object(Bucket='blah', Key='hello.txt') - with assert_raises(ClientError) as err: + with assert_raises(ClientError): s3.Object('blah', 'hello2.txt').meta.client.head_object(Bucket='blah', Key='hello_bad.txt') + + +TEST_XML = """\ + + + + index.html + + + + + test/testing + + + test.txt + + + + +""" + + +@mock_s3 +def test_website_configuration_xml(): + conn = boto.connect_s3() + bucket = conn.create_bucket('test-bucket') + bucket.set_website_configuration_xml(TEST_XML) + bucket.get_website_configuration_xml().should.equal(TEST_XML) diff --git a/tests/test_s3bucket_path/test_s3bucket_path.py b/tests/test_s3bucket_path/test_s3bucket_path.py index dfa6f2057..eff01bf55 100644 --- a/tests/test_s3bucket_path/test_s3bucket_path.py +++ b/tests/test_s3bucket_path/test_s3bucket_path.py @@ -281,3 +281,40 @@ def test_bucket_key_listing_order(): delimiter = '/' keys = [x.name for x in bucket.list(prefix + 'x', delimiter)] keys.should.equal(['toplevel/x/']) + + +@mock_s3bucket_path +def test_delete_keys(): + conn = create_connection() + bucket = conn.create_bucket('foobar') + + Key(bucket=bucket, name='file1').set_contents_from_string('abc') + Key(bucket=bucket, name='file2').set_contents_from_string('abc') + Key(bucket=bucket, name='file3').set_contents_from_string('abc') + Key(bucket=bucket, name='file4').set_contents_from_string('abc') + + result = bucket.delete_keys(['file2', 'file3']) + result.deleted.should.have.length_of(2) + result.errors.should.have.length_of(0) + keys = bucket.get_all_keys() + keys.should.have.length_of(2) + keys[0].name.should.equal('file1') + + +@mock_s3bucket_path +def test_delete_keys_with_invalid(): + conn = create_connection() + bucket = conn.create_bucket('foobar') + + Key(bucket=bucket, name='file1').set_contents_from_string('abc') + Key(bucket=bucket, name='file2').set_contents_from_string('abc') + Key(bucket=bucket, name='file3').set_contents_from_string('abc') + Key(bucket=bucket, name='file4').set_contents_from_string('abc') + + result = bucket.delete_keys(['abc', 'file3']) + + result.deleted.should.have.length_of(1) + result.errors.should.have.length_of(1) + keys = bucket.get_all_keys() + keys.should.have.length_of(3) + keys[0].name.should.equal('file1') diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index ba1e11e52..1e300fa57 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -34,6 +34,8 @@ def test_create_queues_in_multiple_region(): list(west1_conn.get_all_queues()).should.have.length_of(1) list(west2_conn.get_all_queues()).should.have.length_of(1) + west1_conn.get_all_queues()[0].url.should.equal('http://sqs.us-west-1.amazonaws.com/123456789012/test-queue') + @mock_sqs def test_get_queue(): @@ -168,6 +170,18 @@ def test_send_message_with_delay(): queue.count().should.equal(0) +@mock_sqs +def test_send_large_message_fails(): + conn = boto.connect_sqs('the_key', 'the_secret') + queue = conn.create_queue("test-queue", visibility_timeout=60) + queue.set_message_class(RawMessage) + + body_one = 'test message' * 200000 + huge_message = queue.new_message(body_one) + + queue.write.when.called_with(huge_message).should.throw(SQSError) + + @mock_sqs def test_message_becomes_inflight_when_received(): conn = boto.connect_sqs('the_key', 'the_secret')