From b59a77d5bb4c9f9f9d094fff9921ed74d035e2ac Mon Sep 17 00:00:00 2001 From: zeb Date: Fri, 4 Sep 2015 18:48:48 +0200 Subject: [PATCH 01/55] Tweak bucket.delete_keys for s3bucket_path. --- moto/s3/responses.py | 11 +++++- moto/s3bucket_path/responses.py | 6 +++ .../test_s3bucket_path/test_s3bucket_path.py | 37 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 68d1bb318..8b9fdb228 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -22,10 +22,17 @@ def parse_key_name(pth): class ResponseObject(_TemplateEnvironmentMixin): - def __init__(self, backend, bucket_name_from_url, parse_key_name): + def __init__(self, backend, bucket_name_from_url, parse_key_name, + is_delete_keys=None): self.backend = backend self.bucket_name_from_url = bucket_name_from_url self.parse_key_name = parse_key_name + if is_delete_keys: + self.is_delete_keys = is_delete_keys + + @staticmethod + def is_delete_keys(path, bucket_name): + return path == u'/?delete' def all_buckets(self): # No bucket specified. Listing all buckets @@ -209,7 +216,7 @@ class ResponseObject(_TemplateEnvironmentMixin): return 409, headers, template.render(bucket=removed_bucket) def _bucket_response_post(self, request, bucket_name, headers): - if request.path == u'/?delete': + if self.is_delete_keys(request.path, bucket_name): return self._bucket_response_delete_keys(request, bucket_name, headers) # POST to bucket-url should create file from form diff --git a/moto/s3bucket_path/responses.py b/moto/s3bucket_path/responses.py index 4e2f87376..6de82fbb0 100644 --- a/moto/s3bucket_path/responses.py +++ b/moto/s3bucket_path/responses.py @@ -9,8 +9,14 @@ from moto.s3.responses import ResponseObject def parse_key_name(pth): return "/".join(pth.rstrip("/").split("/")[2:]) + +def is_delete_keys(path, bucket_name): + return path == u'/' + bucket_name + u'/?delete' + + S3BucketPathResponseInstance = ResponseObject( s3bucket_path_backend, bucket_name_from_url, parse_key_name, + is_delete_keys, ) diff --git a/tests/test_s3bucket_path/test_s3bucket_path.py b/tests/test_s3bucket_path/test_s3bucket_path.py index dfa6f2057..eff01bf55 100644 --- a/tests/test_s3bucket_path/test_s3bucket_path.py +++ b/tests/test_s3bucket_path/test_s3bucket_path.py @@ -281,3 +281,40 @@ def test_bucket_key_listing_order(): delimiter = '/' keys = [x.name for x in bucket.list(prefix + 'x', delimiter)] keys.should.equal(['toplevel/x/']) + + +@mock_s3bucket_path +def test_delete_keys(): + conn = create_connection() + bucket = conn.create_bucket('foobar') + + Key(bucket=bucket, name='file1').set_contents_from_string('abc') + Key(bucket=bucket, name='file2').set_contents_from_string('abc') + Key(bucket=bucket, name='file3').set_contents_from_string('abc') + Key(bucket=bucket, name='file4').set_contents_from_string('abc') + + result = bucket.delete_keys(['file2', 'file3']) + result.deleted.should.have.length_of(2) + result.errors.should.have.length_of(0) + keys = bucket.get_all_keys() + keys.should.have.length_of(2) + keys[0].name.should.equal('file1') + + +@mock_s3bucket_path +def test_delete_keys_with_invalid(): + conn = create_connection() + bucket = conn.create_bucket('foobar') + + Key(bucket=bucket, name='file1').set_contents_from_string('abc') + Key(bucket=bucket, name='file2').set_contents_from_string('abc') + Key(bucket=bucket, name='file3').set_contents_from_string('abc') + Key(bucket=bucket, name='file4').set_contents_from_string('abc') + + result = bucket.delete_keys(['abc', 'file3']) + + result.deleted.should.have.length_of(1) + result.errors.should.have.length_of(1) + keys = bucket.get_all_keys() + keys.should.have.length_of(3) + keys[0].name.should.equal('file1') From 14ec3531ff5430d2927d1ee5b62231859136ee10 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 8 Sep 2015 21:36:32 +0000 Subject: [PATCH 02/55] Add support for latency based route53 records Store and marshal the region field of records Signed-off-by: Kevin Donnelly --- moto/route53/models.py | 4 ++++ tests/test_route53/test_route53.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/moto/route53/models.py b/moto/route53/models.py index 00e23c38e..0e79d617a 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -68,6 +68,7 @@ class RecordSet(object): self.records = kwargs.get('ResourceRecords', []) self.set_identifier = kwargs.get('SetIdentifier') self.weight = kwargs.get('Weight') + self.region = kwargs.get('Region') self.health_check = kwargs.get('HealthCheckId') @classmethod @@ -89,6 +90,9 @@ class RecordSet(object): {% if record_set.weight %} {{ record_set.weight }} {% endif %} + {% if record_set.region %} + {{ record_set.region }} + {% endif %} {{ record_set.ttl }} {% for record in record_set.records %} diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 669b8b2e3..5556bfc7d 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -239,3 +239,25 @@ def test_deleting_weighted_route(): cname = zone.get_cname('cname.testdns.aws.com.', all=True) # When get_cname only had one result, it returns just that result instead of a list. cname.identifier.should.equal('success-test-bar') + + +@mock_route53 +def test_deleting_latency_route(): + conn = boto.connect_route53() + + conn.create_hosted_zone("testdns.aws.com.") + zone = conn.get_zone("testdns.aws.com.") + + zone.add_cname("cname.testdns.aws.com", "example.com", identifier=('success-test-foo', 'us-west-2')) + zone.add_cname("cname.testdns.aws.com", "example.com", identifier=('success-test-bar', 'us-west-1')) + + cnames = zone.get_cname('cname.testdns.aws.com.', all=True) + cnames.should.have.length_of(2) + foo_cname = [cname for cname in cnames if cname.identifier == 'success-test-foo'][0] + foo_cname.region.should.equal('us-west-2') + + zone.delete_record(foo_cname) + cname = zone.get_cname('cname.testdns.aws.com.', all=True) + # When get_cname only had one result, it returns just that result instead of a list. + cname.identifier.should.equal('success-test-bar') + cname.region.should.equal('us-west-1') From 82eeb182a74012ef4576fa8080670a0cc6a21ce1 Mon Sep 17 00:00:00 2001 From: ZuluPro Date: Tue, 15 Sep 2015 18:25:12 -0400 Subject: [PATCH 03/55] Added DockerFile --- Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..16d9d7d91 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:2 + +ADD . /moto/ +ENV PYTHONUNBUFFERED 1 + +WORKDIR /moto/ +RUN python setup.py install + +CMD ["moto_server"] + +EXPOSE 5000 From 23c2e7835e90e81a271a9faf7b4ee9aa5dd7785f Mon Sep 17 00:00:00 2001 From: ZuluPro Date: Tue, 15 Sep 2015 19:55:26 -0400 Subject: [PATCH 04/55] Remade ec2.utils.random_key_pair for be really random --- moto/ec2/utils.py | 33 ++++++++++++++------------------- tests/test_ec2/test_utils.py | 8 ++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 tests/test_ec2/test_utils.py diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 0b2b2ff49..b16c363ea 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -453,27 +453,22 @@ def simple_aws_filter_to_re(filter_string): return tmp_filter -# not really random ( http://xkcd.com/221/ ) def random_key_pair(): + def random_hex(): + return chr(random.choice(list(range(48, 58)) + list(range(97, 102)))) + def random_fingerprint(): + return ':'.join([random_hex()+random_hex() for i in range(20)]) + def random_material(): + return ''.join([ + chr(random.choice(list(range(65, 91)) + list(range(48, 58)) + + list(range(97, 102)))) + for i in range(1000) + ]) + material = "---- BEGIN RSA PRIVATE KEY ----" + random_material() + \ + "-----END RSA PRIVATE KEY-----" return { - 'fingerprint': ('1f:51:ae:28:bf:89:e9:d8:1f:25:5d:37:2d:' - '7d:b8:ca:9f:f5:f1:6f'), - 'material': """---- BEGIN RSA PRIVATE KEY ---- -MIICiTCCAfICCQD6m7oRw0uXOjANBgkqhkiG9w0BAQUFADCBiDELMAkGA1UEBhMC -VVMxCzAJBgNVBAgTAldBMRAwDgYDVQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6 -b24xFDASBgNVBAsTC0lBTSBDb25zb2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAd -BgkqhkiG9w0BCQEWEG5vb25lQGFtYXpvbi5jb20wHhcNMTEwNDI1MjA0NTIxWhcN -MTIwNDI0MjA0NTIxWjCBiDELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAldBMRAwDgYD -VQQHEwdTZWF0dGxlMQ8wDQYDVQQKEwZBbWF6b24xFDASBgNVBAsTC0lBTSBDb25z -b2xlMRIwEAYDVQQDEwlUZXN0Q2lsYWMxHzAdBgkqhkiG9w0BCQEWEG5vb25lQGFt -YXpvbi5jb20wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMaK0dn+a4GmWIWJ -21uUSfwfEvySWtC2XADZ4nB+BLYgVIk60CpiwsZ3G93vUEIO3IyNoH/f0wYK8m9T -rDHudUZg3qX4waLG5M43q7Wgc/MbQITxOUSQv7c7ugFFDzQGBzZswY6786m86gpE -Ibb3OhjZnzcvQAaRHhdlQWIMm2nrAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAtCu4 -nUhVVxYUntneD9+h8Mg9q6q+auNKyExzyLwaxlAoo7TJHidbtS4J5iNmZgXL0Fkb -FFBjvSfpJIlJ00zbhNYS5f6GuoEDmFJl0ZxBHjJnyp378OD8uTs7fLvjx79LjSTb -NYiytVbZPQUQ5Yaxu2jXnimvw3rrszlaEXAMPLE ------END RSA PRIVATE KEY-----""" + 'fingerprint': random_fingerprint(), + 'material': material } diff --git a/tests/test_ec2/test_utils.py b/tests/test_ec2/test_utils.py new file mode 100644 index 000000000..ef540e193 --- /dev/null +++ b/tests/test_ec2/test_utils.py @@ -0,0 +1,8 @@ +from moto.ec2 import utils + + +def test_random_key_pair(): + key_pair = utils.random_key_pair() + assert len(key_pair['fingerprint']) == 59 + assert key_pair['material'].startswith('---- BEGIN RSA PRIVATE KEY ----') + assert key_pair['material'].endswith('-----END RSA PRIVATE KEY-----') From 95169c6011e4a887f86b8e5c4c6df492ace5c1a9 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 10:00:38 -0400 Subject: [PATCH 05/55] First version of datapipelines. --- moto/__init__.py | 1 + moto/backends.py | 2 + moto/datapipeline/__init__.py | 12 ++ moto/datapipeline/models.py | 122 ++++++++++++++++++ moto/datapipeline/responses.py | 68 ++++++++++ moto/datapipeline/urls.py | 10 ++ moto/datapipeline/utils.py | 5 + tests/test_datapipeline/test_datapipeline.py | 126 +++++++++++++++++++ tests/test_datapipeline/test_server.py | 25 ++++ 9 files changed, 371 insertions(+) create mode 100644 moto/datapipeline/__init__.py create mode 100644 moto/datapipeline/models.py create mode 100644 moto/datapipeline/responses.py create mode 100644 moto/datapipeline/urls.py create mode 100644 moto/datapipeline/utils.py create mode 100644 tests/test_datapipeline/test_datapipeline.py create mode 100644 tests/test_datapipeline/test_server.py diff --git a/moto/__init__.py b/moto/__init__.py index e5a94fd80..2dac1f683 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -8,6 +8,7 @@ __version__ = '0.4.12' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa from .cloudwatch import mock_cloudwatch # flake8: noqa +from .datapipeline import mock_datapipeline # flake8: noqa from .dynamodb import mock_dynamodb # flake8: noqa from .dynamodb2 import mock_dynamodb2 # flake8: noqa from .ec2 import mock_ec2 # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index b46d56c06..cb040ab93 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from moto.autoscaling import autoscaling_backend from moto.cloudwatch import cloudwatch_backend from moto.cloudformation import cloudformation_backend +from moto.datapipeline import datapipeline_backend from moto.dynamodb import dynamodb_backend from moto.dynamodb2 import dynamodb_backend2 from moto.ec2 import ec2_backend @@ -25,6 +26,7 @@ BACKENDS = { 'autoscaling': autoscaling_backend, 'cloudformation': cloudformation_backend, 'cloudwatch': cloudwatch_backend, + 'datapipeline': datapipeline_backend, 'dynamodb': dynamodb_backend, 'dynamodb2': dynamodb_backend2, 'ec2': ec2_backend, diff --git a/moto/datapipeline/__init__.py b/moto/datapipeline/__init__.py new file mode 100644 index 000000000..dcfe2f427 --- /dev/null +++ b/moto/datapipeline/__init__.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals +from .models import datapipeline_backends +from ..core.models import MockAWS + +datapipeline_backend = datapipeline_backends['us-east-1'] + + +def mock_datapipeline(func=None): + if func: + return MockAWS(datapipeline_backends)(func) + else: + return MockAWS(datapipeline_backends) diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py new file mode 100644 index 000000000..393218558 --- /dev/null +++ b/moto/datapipeline/models.py @@ -0,0 +1,122 @@ +from __future__ import unicode_literals + +import datetime +import boto.datapipeline +from moto.core import BaseBackend +from .utils import get_random_pipeline_id + + +class PipelineObject(object): + def __init__(self, object_id, name, fields): + self.object_id = object_id + self.name = name + self.fields = fields + + def to_json(self): + return { + "Fields": self.fields, + "Id": self.object_id, + "Name": self.name, + } + + +class Pipeline(object): + def __init__(self, name, unique_id): + self.name = name + self.unique_id = unique_id + self.description = "" + self.pipeline_id = get_random_pipeline_id() + self.creation_time = datetime.datetime.utcnow() + self.objects = [] + + def to_json(self): + return { + "Description": self.description, + "Fields": [{ + "key": "@pipelineState", + "stringValue": "SCHEDULED" + }, { + "key": "description", + "stringValue": self.description + }, { + "key": "name", + "stringValue": self.name + }, { + "key": "@creationTime", + "stringValue": datetime.datetime.strftime(self.creation_time, '%Y-%m-%dT%H-%M-%S'), + }, { + "key": "@id", + "stringValue": self.pipeline_id, + }, { + "key": "@sphere", + "stringValue": "PIPELINE" + }, { + "key": "@version", + "stringValue": "1" + }, { + "key": "@userId", + "stringValue": "924374875933" + }, { + "key": "@accountId", + "stringValue": "924374875933" + }, { + "key": "uniqueId", + "stringValue": self.unique_id + }], + "Name": self.name, + "PipelineId": self.pipeline_id, + "Tags": [ + ] + } + + def set_pipeline_objects(self, pipeline_objects): + self.objects = [ + PipelineObject(pipeline_object['id'], pipeline_object['name'], pipeline_object['fields']) + for pipeline_object in pipeline_objects + ] + + def activate(self): + pass + + +class DataPipelineBackend(BaseBackend): + + def __init__(self): + self.pipelines = {} + + def create_pipeline(self, name, unique_id): + pipeline = Pipeline(name, unique_id) + self.pipelines[pipeline.pipeline_id] = pipeline + return pipeline + + def describe_pipelines(self, pipeline_ids): + pipelines = [pipeline for pipeline in self.pipelines.values() if pipeline.pipeline_id in pipeline_ids] + return pipelines + + def get_pipeline(self, pipeline_id): + return self.pipelines[pipeline_id] + + def put_pipeline_definition(self, pipeline_id, pipeline_objects): + pipeline = self.get_pipeline(pipeline_id) + pipeline.set_pipeline_objects(pipeline_objects) + + def get_pipeline_definition(self, pipeline_id): + pipeline = self.get_pipeline(pipeline_id) + return pipeline.objects + + def describe_objects(self, object_ids, pipeline_id): + pipeline = self.get_pipeline(pipeline_id) + pipeline_objects = [ + pipeline_object for pipeline_object in pipeline.objects + if pipeline_object.object_id in object_ids + ] + return pipeline_objects + + def activate_pipeline(self, pipeline_id): + pipeline = self.get_pipeline(pipeline_id) + pipeline.activate() + + +datapipeline_backends = {} +for region in boto.datapipeline.regions(): + datapipeline_backends[region.name] = DataPipelineBackend() diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py new file mode 100644 index 000000000..a61ec2514 --- /dev/null +++ b/moto/datapipeline/responses.py @@ -0,0 +1,68 @@ +from __future__ import unicode_literals + +import json + +from moto.core.responses import BaseResponse +from .models import datapipeline_backends + + +class DataPipelineResponse(BaseResponse): + + @property + def parameters(self): + return json.loads(self.body.decode("utf-8")) + + @property + def datapipeline_backend(self): + return datapipeline_backends[self.region] + + def create_pipeline(self): + name = self.parameters['name'] + unique_id = self.parameters['uniqueId'] + pipeline = self.datapipeline_backend.create_pipeline(name, unique_id) + return json.dumps({ + "pipelineId": pipeline.pipeline_id, + }) + + def describe_pipelines(self): + pipeline_ids = self.parameters["pipelineIds"] + pipelines = self.datapipeline_backend.describe_pipelines(pipeline_ids) + + return json.dumps({ + "PipelineDescriptionList": [ + pipeline.to_json() for pipeline in pipelines + ] + }) + + def put_pipeline_definition(self): + pipeline_id = self.parameters["pipelineId"] + pipeline_objects = self.parameters["pipelineObjects"] + + self.datapipeline_backend.put_pipeline_definition(pipeline_id, pipeline_objects) + return json.dumps({"errored": False}) + + def get_pipeline_definition(self): + pipeline_id = self.parameters["pipelineId"] + pipeline_definition = self.datapipeline_backend.get_pipeline_definition(pipeline_id) + return json.dumps({ + "pipelineObjects": [pipeline_object.to_json() for pipeline_object in pipeline_definition] + }) + + def describe_objects(self): + pipeline_id = self.parameters["pipelineId"] + object_ids = self.parameters["objectIds"] + + pipeline_objects = self.datapipeline_backend.describe_objects(object_ids, pipeline_id) + + return json.dumps({ + "HasMoreResults": False, + "Marker": None, + "PipelineObjects": [ + pipeline_object.to_json() for pipeline_object in pipeline_objects + ] + }) + + def activate_pipeline(self): + pipeline_id = self.parameters["pipelineId"] + self.datapipeline_backend.activate_pipeline(pipeline_id) + return json.dumps({}) diff --git a/moto/datapipeline/urls.py b/moto/datapipeline/urls.py new file mode 100644 index 000000000..40805874b --- /dev/null +++ b/moto/datapipeline/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import DataPipelineResponse + +url_bases = [ + "https?://datapipeline.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': DataPipelineResponse.dispatch, +} diff --git a/moto/datapipeline/utils.py b/moto/datapipeline/utils.py new file mode 100644 index 000000000..7de9d2732 --- /dev/null +++ b/moto/datapipeline/utils.py @@ -0,0 +1,5 @@ +from moto.core.utils import get_random_hex + + +def get_random_pipeline_id(): + return "df-{0}".format(get_random_hex(length=19)) diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py new file mode 100644 index 000000000..e48606780 --- /dev/null +++ b/tests/test_datapipeline/test_datapipeline.py @@ -0,0 +1,126 @@ +from __future__ import unicode_literals + +import boto.datapipeline +import sure # noqa + +from moto import mock_datapipeline + + +@mock_datapipeline +def test_create_pipeline(): + conn = boto.datapipeline.connect_to_region("us-west-2") + + res = conn.create_pipeline("mypipeline", "some-unique-id") + + pipeline_id = res["pipelineId"] + pipeline_descriptions = conn.describe_pipelines([pipeline_id])["PipelineDescriptionList"] + pipeline_descriptions.should.have.length_of(1) + + pipeline_description = pipeline_descriptions[0] + pipeline_description['Name'].should.equal("mypipeline") + pipeline_description["PipelineId"].should.equal(pipeline_id) + fields = pipeline_description['Fields'] + + def get_value_from_fields(key, fields): + for field in fields: + if field['key'] == key: + return field['stringValue'] + + get_value_from_fields('@pipelineState', fields).should.equal("SCHEDULED") + get_value_from_fields('uniqueId', fields).should.equal("some-unique-id") + + +PIPELINE_OBJECTS = [ + { + "id": "Default", + "name": "Default", + "fields": [{ + "key": "workerGroup", + "stringValue": "workerGroup" + }] + }, + { + "id": "Schedule", + "name": "Schedule", + "fields": [{ + "key": "startDateTime", + "stringValue": "2012-12-12T00:00:00" + }, { + "key": "type", + "stringValue": "Schedule" + }, { + "key": "period", + "stringValue": "1 hour" + }, { + "key": "endDateTime", + "stringValue": "2012-12-21T18:00:00" + }] + }, + { + "id": "SayHello", + "name": "SayHello", + "fields": [{ + "key": "type", + "stringValue": "ShellCommandActivity" + }, { + "key": "command", + "stringValue": "echo hello" + }, { + "key": "parent", + "refValue": "Default" + }, { + "key": "schedule", + "refValue": "Schedule" + }] + } +] + + +@mock_datapipeline +def test_creating_pipeline_definition(): + conn = boto.datapipeline.connect_to_region("us-west-2") + res = conn.create_pipeline("mypipeline", "some-unique-id") + pipeline_id = res["pipelineId"] + + conn.put_pipeline_definition(PIPELINE_OBJECTS, pipeline_id) + + pipeline_definition = conn.get_pipeline_definition(pipeline_id) + pipeline_definition['pipelineObjects'].should.have.length_of(3) + default_object = pipeline_definition['pipelineObjects'][0] + default_object['Name'].should.equal("Default") + default_object['Id'].should.equal("Default") + default_object['Fields'].should.equal([{ + "key": "workerGroup", + "stringValue": "workerGroup" + }]) + + +@mock_datapipeline +def test_describing_pipeline_objects(): + conn = boto.datapipeline.connect_to_region("us-west-2") + res = conn.create_pipeline("mypipeline", "some-unique-id") + pipeline_id = res["pipelineId"] + + conn.put_pipeline_definition(PIPELINE_OBJECTS, pipeline_id) + + objects = conn.describe_objects(["Schedule", "Default"], pipeline_id)['PipelineObjects'] + + objects.should.have.length_of(2) + default_object = [x for x in objects if x['Id'] == 'Default'][0] + default_object['Name'].should.equal("Default") + default_object['Fields'].should.equal([{ + "key": "workerGroup", + "stringValue": "workerGroup" + }]) + + +@mock_datapipeline +def test_activate_pipeline(): + conn = boto.datapipeline.connect_to_region("us-west-2") + + res = conn.create_pipeline("mypipeline", "some-unique-id") + + pipeline_id = res["pipelineId"] + conn.activate_pipeline(pipeline_id) + + # TODO what do we need to assert here. Change in pipeline status? diff --git a/tests/test_datapipeline/test_server.py b/tests/test_datapipeline/test_server.py new file mode 100644 index 000000000..0ff03aa8d --- /dev/null +++ b/tests/test_datapipeline/test_server.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + +import json +import sure # noqa + +import moto.server as server +from moto import mock_datapipeline + +''' +Test the different server responses +''' + + +@mock_datapipeline +def test_list_streams(): + backend = server.create_backend_app("datapipeline") + test_client = backend.test_client() + + res = test_client.get('/?Action=ListStreams') + + json_data = json.loads(res.data.decode("utf-8")) + json_data.should.equal({ + "HasMoreStreams": False, + "StreamNames": [], + }) From 91a75570c61cdaf238126b9c6da399556a236561 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 10:01:13 -0400 Subject: [PATCH 06/55] Update readme. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c458d2a6f..d80e515ed 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ It gets even better! Moto isn't just S3. Here's the status of the other AWS serv |------------------------------------------------------------------------------| | Cloudwatch | @mock_cloudwatch | basic endpoints done | |------------------------------------------------------------------------------| +| Data Pipeline | @mock_datapipeline| basic endpoints done | +|------------------------------------------------------------------------------| | DynamoDB | @mock_dynamodb | core endpoints done | | DynamoDB2 | @mock_dynamodb2 | core endpoints done - no indexes | |------------------------------------------------------------------------------| From b0ea9f285943bcbea3dd14fe3c5f8aa63db937a8 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 10:11:43 -0400 Subject: [PATCH 07/55] Fix tests for server mode. --- moto/datapipeline/responses.py | 6 +++++- tests/test_datapipeline/test_server.py | 8 +++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py index a61ec2514..273543329 100644 --- a/moto/datapipeline/responses.py +++ b/moto/datapipeline/responses.py @@ -10,7 +10,11 @@ class DataPipelineResponse(BaseResponse): @property def parameters(self): - return json.loads(self.body.decode("utf-8")) + # TODO this should really be moved to core/responses.py + if self.body: + return json.loads(self.body.decode("utf-8")) + else: + return self.querystring @property def datapipeline_backend(self): diff --git a/tests/test_datapipeline/test_server.py b/tests/test_datapipeline/test_server.py index 0ff03aa8d..33a19c6f0 100644 --- a/tests/test_datapipeline/test_server.py +++ b/tests/test_datapipeline/test_server.py @@ -16,10 +16,12 @@ def test_list_streams(): backend = server.create_backend_app("datapipeline") test_client = backend.test_client() - res = test_client.get('/?Action=ListStreams') + res = test_client.post('/', + data={"pipelineIds": ["ASdf"]}, + headers={"X-Amz-Target": "DataPipeline.DescribePipelines"}, + ) json_data = json.loads(res.data.decode("utf-8")) json_data.should.equal({ - "HasMoreStreams": False, - "StreamNames": [], + 'PipelineDescriptionList': [] }) From db23b7d24cd0ddf1d2e32a94b4c3cde0399c96d9 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 11:22:52 -0400 Subject: [PATCH 08/55] Fix state to start as PENDING and only become SCHEDULED on activation. --- moto/datapipeline/models.py | 5 +++-- tests/test_datapipeline/test_datapipeline.py | 20 +++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 393218558..2eb181bb1 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -28,13 +28,14 @@ class Pipeline(object): self.pipeline_id = get_random_pipeline_id() self.creation_time = datetime.datetime.utcnow() self.objects = [] + self.status = "PENDING" def to_json(self): return { "Description": self.description, "Fields": [{ "key": "@pipelineState", - "stringValue": "SCHEDULED" + "stringValue": self.status, }, { "key": "description", "stringValue": self.description @@ -76,7 +77,7 @@ class Pipeline(object): ] def activate(self): - pass + self.status = "SCHEDULED" class DataPipelineBackend(BaseBackend): diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index e48606780..b374d9a4c 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -6,6 +6,12 @@ import sure # noqa from moto import mock_datapipeline +def get_value_from_fields(key, fields): + for field in fields: + if field['key'] == key: + return field['stringValue'] + + @mock_datapipeline def test_create_pipeline(): conn = boto.datapipeline.connect_to_region("us-west-2") @@ -21,12 +27,7 @@ def test_create_pipeline(): pipeline_description["PipelineId"].should.equal(pipeline_id) fields = pipeline_description['Fields'] - def get_value_from_fields(key, fields): - for field in fields: - if field['key'] == key: - return field['stringValue'] - - get_value_from_fields('@pipelineState', fields).should.equal("SCHEDULED") + get_value_from_fields('@pipelineState', fields).should.equal("PENDING") get_value_from_fields('uniqueId', fields).should.equal("some-unique-id") @@ -123,4 +124,9 @@ def test_activate_pipeline(): pipeline_id = res["pipelineId"] conn.activate_pipeline(pipeline_id) - # TODO what do we need to assert here. Change in pipeline status? + pipeline_descriptions = conn.describe_pipelines([pipeline_id])["PipelineDescriptionList"] + pipeline_descriptions.should.have.length_of(1) + pipeline_description = pipeline_descriptions[0] + fields = pipeline_description['Fields'] + + get_value_from_fields('@pipelineState', fields).should.equal("SCHEDULED") From 1b811e69492d554af4d1e2d9765f9b68681775ec Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 11:29:03 -0400 Subject: [PATCH 09/55] 0.4.13 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 2dac1f683..d5220e8b2 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.12' +__version__ = '0.4.13' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index e73362fb9..df18bbbf7 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.12', + version='0.4.13', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 0cef3328402b13abc8e5e8ccfcc58886b33d3430 Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Wed, 16 Sep 2015 15:49:15 -0400 Subject: [PATCH 10/55] Add support to ListPipelines --- moto/datapipeline/models.py | 3 +++ moto/datapipeline/responses.py | 10 ++++++++ tests/test_datapipeline/test_datapipeline.py | 24 ++++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 2eb181bb1..3b31c9eb2 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -90,6 +90,9 @@ class DataPipelineBackend(BaseBackend): self.pipelines[pipeline.pipeline_id] = pipeline return pipeline + def list_pipelines(self): + return self.pipelines.values() + def describe_pipelines(self, pipeline_ids): pipelines = [pipeline for pipeline in self.pipelines.values() if pipeline.pipeline_id in pipeline_ids] return pipelines diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py index 273543329..582e8504f 100644 --- a/moto/datapipeline/responses.py +++ b/moto/datapipeline/responses.py @@ -28,6 +28,16 @@ class DataPipelineResponse(BaseResponse): "pipelineId": pipeline.pipeline_id, }) + def list_pipelines(self): + pipelines = self.datapipeline_backend.list_pipelines() + return json.dumps({ + "HasMoreResults": False, + "Marker": None, + "PipelineIdList": [ + {"Id": pipeline.pipeline_id, "Name": pipeline.name} for pipeline in pipelines + ] + }) + def describe_pipelines(self): pipeline_ids = self.parameters["pipelineIds"] pipelines = self.datapipeline_backend.describe_pipelines(pipeline_ids) diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index b374d9a4c..e18cbec94 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -130,3 +130,27 @@ def test_activate_pipeline(): fields = pipeline_description['Fields'] get_value_from_fields('@pipelineState', fields).should.equal("SCHEDULED") + + +@mock_datapipeline +def test_listing_pipelines(): + conn = boto.datapipeline.connect_to_region("us-west-2") + res1 = conn.create_pipeline("mypipeline1", "some-unique-id1") + res2 = conn.create_pipeline("mypipeline2", "some-unique-id2") + pipeline_id1 = res1["pipelineId"] + pipeline_id2 = res2["pipelineId"] + + response = conn.list_pipelines() + + response["HasMoreResults"].should.be(False) + response["Marker"].should.be.none + response["PipelineIdList"].should.equal([ + { + "Id": res1["pipelineId"], + "Name": "mypipeline1", + }, + { + "Id": res2["pipelineId"], + "Name": "mypipeline2" + } + ]) From 25f9e8b58883680f8aea6acf3fd568df07103fac Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Wed, 16 Sep 2015 17:49:13 -0400 Subject: [PATCH 11/55] Change CloudFormation to support Data Pipelines --- moto/cloudformation/parsing.py | 2 + moto/datapipeline/models.py | 17 +++- moto/datapipeline/utils.py | 18 +++++ requirements-dev.txt | 1 + .../test_cloudformation_stack_integration.py | 78 +++++++++++++++++++ tests/test_datapipeline/test_datapipeline.py | 37 +++++++-- 6 files changed, 143 insertions(+), 10 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 5306ce0d9..418d736d5 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -4,6 +4,7 @@ import functools import logging from moto.autoscaling import models as autoscaling_models +from moto.datapipeline import models as datapipeline_models from moto.ec2 import models as ec2_models from moto.elb import models as elb_models from moto.iam import models as iam_models @@ -36,6 +37,7 @@ MODEL_MAP = { "AWS::EC2::VPCGatewayAttachment": ec2_models.VPCGatewayAttachment, "AWS::EC2::VPCPeeringConnection": ec2_models.VPCPeeringConnection, "AWS::ElasticLoadBalancing::LoadBalancer": elb_models.FakeLoadBalancer, + "AWS::DataPipeline::Pipeline": datapipeline_models.Pipeline, "AWS::IAM::InstanceProfile": iam_models.InstanceProfile, "AWS::IAM::Role": iam_models.Role, "AWS::RDS::DBInstance": rds_models.Database, diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 3b31c9eb2..ca2bfdfee 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import datetime import boto.datapipeline from moto.core import BaseBackend -from .utils import get_random_pipeline_id +from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys class PipelineObject(object): @@ -73,12 +73,25 @@ class Pipeline(object): def set_pipeline_objects(self, pipeline_objects): self.objects = [ PipelineObject(pipeline_object['id'], pipeline_object['name'], pipeline_object['fields']) - for pipeline_object in pipeline_objects + for pipeline_object in remove_capitalization_of_dict_keys(pipeline_objects) ] def activate(self): self.status = "SCHEDULED" + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + datapipeline_backend = datapipeline_backends[region_name] + properties = cloudformation_json["Properties"] + + cloudformation_unique_id = "cf-" + properties["Name"] + pipeline = datapipeline_backend.create_pipeline(properties["Name"], cloudformation_unique_id) + datapipeline_backend.put_pipeline_definition(pipeline.pipeline_id, properties["PipelineObjects"]) + + if properties["Activate"]: + pipeline.activate() + return pipeline + class DataPipelineBackend(BaseBackend): diff --git a/moto/datapipeline/utils.py b/moto/datapipeline/utils.py index 7de9d2732..75df4a9a5 100644 --- a/moto/datapipeline/utils.py +++ b/moto/datapipeline/utils.py @@ -1,5 +1,23 @@ +import collections +import six from moto.core.utils import get_random_hex def get_random_pipeline_id(): return "df-{0}".format(get_random_hex(length=19)) + + +def remove_capitalization_of_dict_keys(obj): + if isinstance(obj, collections.Mapping): + result = obj.__class__() + for key, value in obj.items(): + normalized_key = key[:1].lower() + key[1:] + result[normalized_key] = remove_capitalization_of_dict_keys(value) + return result + elif isinstance(obj, collections.Iterable) and not isinstance(obj, six.string_types): + result = obj.__class__() + for item in obj: + result += (remove_capitalization_of_dict_keys(item),) + return result + else: + return obj diff --git a/requirements-dev.txt b/requirements-dev.txt index 5cd0acb14..bd4b6d237 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,4 @@ coverage freezegun flask boto3 +six \ No newline at end of file diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index c1305d843..a959777b8 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -3,6 +3,7 @@ import json import boto import boto.cloudformation +import boto.datapipeline import boto.ec2 import boto.ec2.autoscale import boto.ec2.elb @@ -17,6 +18,7 @@ import sure # noqa from moto import ( mock_autoscaling, mock_cloudformation, + mock_datapipeline, mock_ec2, mock_elb, mock_iam, @@ -1395,3 +1397,79 @@ def test_subnets_should_be_created_with_availability_zone(): ) subnet = vpc_conn.get_all_subnets(filters={'cidrBlock': '10.0.0.0/24'})[0] subnet.availability_zone.should.equal('us-west-1b') + + +@mock_cloudformation +@mock_datapipeline +def test_datapipeline(): + dp_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "dataPipeline": { + "Properties": { + "Activate": "true", + "Name": "testDataPipeline", + "PipelineObjects": [ + { + "Fields": [ + { + "Key": "failureAndRerunMode", + "StringValue": "CASCADE" + }, + { + "Key": "scheduleType", + "StringValue": "cron" + }, + { + "Key": "schedule", + "RefValue": "DefaultSchedule" + }, + { + "Key": "pipelineLogUri", + "StringValue": "s3://bucket/logs" + }, + { + "Key": "type", + "StringValue": "Default" + }, + ], + "Id": "Default", + "Name": "Default" + }, + { + "Fields": [ + { + "Key": "startDateTime", + "StringValue": "1970-01-01T01:00:00" + }, + { + "Key": "period", + "StringValue": "1 Day" + }, + { + "Key": "type", + "StringValue": "Schedule" + } + ], + "Id": "DefaultSchedule", + "Name": "RunOnce" + } + ], + "PipelineTags": [] + }, + "Type": "AWS::DataPipeline::Pipeline" + } + } + } + cf_conn = boto.cloudformation.connect_to_region("us-east-1") + template_json = json.dumps(dp_template) + cf_conn.create_stack( + "test_stack", + template_body=template_json, + ) + + dp_conn = boto.datapipeline.connect_to_region('us-east-1') + data_pipelines = dp_conn.list_pipelines() + + data_pipelines['PipelineIdList'].should.have.length_of(1) + data_pipelines['PipelineIdList'][0]['Name'].should.equal('testDataPipeline') \ No newline at end of file diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index e18cbec94..bc8242d0c 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -4,6 +4,7 @@ import boto.datapipeline import sure # noqa from moto import mock_datapipeline +from moto.datapipeline.utils import remove_capitalization_of_dict_keys def get_value_from_fields(key, fields): @@ -144,13 +145,33 @@ def test_listing_pipelines(): response["HasMoreResults"].should.be(False) response["Marker"].should.be.none - response["PipelineIdList"].should.equal([ + response["PipelineIdList"].should.have.length_of(2) + response["PipelineIdList"].should.contain({ + "Id": res1["pipelineId"], + "Name": "mypipeline1", + }) + response["PipelineIdList"].should.contain({ + "Id": res2["pipelineId"], + "Name": "mypipeline2" + }) + + +# testing a helper function +def test_remove_capitalization_of_dict_keys(): + result = remove_capitalization_of_dict_keys( { - "Id": res1["pipelineId"], - "Name": "mypipeline1", - }, - { - "Id": res2["pipelineId"], - "Name": "mypipeline2" + "Id": "IdValue", + "Fields": [{ + "Key": "KeyValue", + "StringValue": "StringValueValue" + }] } - ]) + ) + + result.should.equal({ + "id": "IdValue", + "fields": [{ + "key": "KeyValue", + "stringValue": "StringValueValue" + }], + }) From 65dd7f76390889f5f77cb8135cc3ea1d3d3240d2 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 18:13:44 -0400 Subject: [PATCH 12/55] Change fields to be start with lower case based on examing of real AWS calls. --- moto/datapipeline/models.py | 22 ++++++++------ moto/datapipeline/responses.py | 2 +- tests/test_datapipeline/test_datapipeline.py | 30 +++++++++----------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index ca2bfdfee..1b97c01d1 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -14,9 +14,9 @@ class PipelineObject(object): def to_json(self): return { - "Fields": self.fields, - "Id": self.object_id, - "Name": self.name, + "fields": self.fields, + "id": self.object_id, + "name": self.name, } @@ -30,10 +30,16 @@ class Pipeline(object): self.objects = [] self.status = "PENDING" + def to_meta_json(self): + return { + "id": self.pipeline_id, + "name": self.name, + } + def to_json(self): return { - "Description": self.description, - "Fields": [{ + "description": self.description, + "fields": [{ "key": "@pipelineState", "stringValue": self.status, }, { @@ -64,9 +70,9 @@ class Pipeline(object): "key": "uniqueId", "stringValue": self.unique_id }], - "Name": self.name, - "PipelineId": self.pipeline_id, - "Tags": [ + "name": self.name, + "pipelineId": self.pipeline_id, + "tags": [ ] } diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py index 582e8504f..7ddeec061 100644 --- a/moto/datapipeline/responses.py +++ b/moto/datapipeline/responses.py @@ -34,7 +34,7 @@ class DataPipelineResponse(BaseResponse): "HasMoreResults": False, "Marker": None, "PipelineIdList": [ - {"Id": pipeline.pipeline_id, "Name": pipeline.name} for pipeline in pipelines + pipeline.to_meta_json() for pipeline in pipelines ] }) diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index bc8242d0c..27b942271 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -24,9 +24,9 @@ def test_create_pipeline(): pipeline_descriptions.should.have.length_of(1) pipeline_description = pipeline_descriptions[0] - pipeline_description['Name'].should.equal("mypipeline") - pipeline_description["PipelineId"].should.equal(pipeline_id) - fields = pipeline_description['Fields'] + pipeline_description['name'].should.equal("mypipeline") + pipeline_description["pipelineId"].should.equal(pipeline_id) + fields = pipeline_description['fields'] get_value_from_fields('@pipelineState', fields).should.equal("PENDING") get_value_from_fields('uniqueId', fields).should.equal("some-unique-id") @@ -89,9 +89,9 @@ def test_creating_pipeline_definition(): pipeline_definition = conn.get_pipeline_definition(pipeline_id) pipeline_definition['pipelineObjects'].should.have.length_of(3) default_object = pipeline_definition['pipelineObjects'][0] - default_object['Name'].should.equal("Default") - default_object['Id'].should.equal("Default") - default_object['Fields'].should.equal([{ + default_object['name'].should.equal("Default") + default_object['id'].should.equal("Default") + default_object['fields'].should.equal([{ "key": "workerGroup", "stringValue": "workerGroup" }]) @@ -108,9 +108,9 @@ def test_describing_pipeline_objects(): objects = conn.describe_objects(["Schedule", "Default"], pipeline_id)['PipelineObjects'] objects.should.have.length_of(2) - default_object = [x for x in objects if x['Id'] == 'Default'][0] - default_object['Name'].should.equal("Default") - default_object['Fields'].should.equal([{ + default_object = [x for x in objects if x['id'] == 'Default'][0] + default_object['name'].should.equal("Default") + default_object['fields'].should.equal([{ "key": "workerGroup", "stringValue": "workerGroup" }]) @@ -128,7 +128,7 @@ def test_activate_pipeline(): pipeline_descriptions = conn.describe_pipelines([pipeline_id])["PipelineDescriptionList"] pipeline_descriptions.should.have.length_of(1) pipeline_description = pipeline_descriptions[0] - fields = pipeline_description['Fields'] + fields = pipeline_description['fields'] get_value_from_fields('@pipelineState', fields).should.equal("SCHEDULED") @@ -138,8 +138,6 @@ def test_listing_pipelines(): conn = boto.datapipeline.connect_to_region("us-west-2") res1 = conn.create_pipeline("mypipeline1", "some-unique-id1") res2 = conn.create_pipeline("mypipeline2", "some-unique-id2") - pipeline_id1 = res1["pipelineId"] - pipeline_id2 = res2["pipelineId"] response = conn.list_pipelines() @@ -147,12 +145,12 @@ def test_listing_pipelines(): response["Marker"].should.be.none response["PipelineIdList"].should.have.length_of(2) response["PipelineIdList"].should.contain({ - "Id": res1["pipelineId"], - "Name": "mypipeline1", + "id": res1["pipelineId"], + "name": "mypipeline1", }) response["PipelineIdList"].should.contain({ - "Id": res2["pipelineId"], - "Name": "mypipeline2" + "id": res2["pipelineId"], + "name": "mypipeline2" }) From 66dce44214c9332ce7840bf83a8395d1891d7648 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 16 Sep 2015 18:17:20 -0400 Subject: [PATCH 13/55] Update cloudformation test for previous commit. --- .../test_cloudformation_stack_integration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index a959777b8..a47026221 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -289,7 +289,6 @@ def test_stack_elb_integration_with_attached_ec2_instances(): ec2_conn = boto.ec2.connect_to_region("us-west-1") reservation = ec2_conn.get_all_instances()[0] ec2_instance = reservation.instances[0] - instance_id = ec2_instance.id load_balancer.instances[0].id.should.equal(ec2_instance.id) list(load_balancer.availability_zones).should.equal(['us-east1']) @@ -1472,4 +1471,4 @@ def test_datapipeline(): data_pipelines = dp_conn.list_pipelines() data_pipelines['PipelineIdList'].should.have.length_of(1) - data_pipelines['PipelineIdList'][0]['Name'].should.equal('testDataPipeline') \ No newline at end of file + data_pipelines['PipelineIdList'][0]['name'].should.equal('testDataPipeline') From 8623483c0fed081c0c5a8225b4ad0d753ac251fe Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Thu, 17 Sep 2015 15:18:57 -0400 Subject: [PATCH 14/55] Change data pipeline responses to start with lowercase characters --- moto/datapipeline/responses.py | 15 +++++++-------- .../test_cloudformation_stack_integration.py | 4 ++-- tests/test_datapipeline/test_datapipeline.py | 16 ++++++++-------- tests/test_datapipeline/test_server.py | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/moto/datapipeline/responses.py b/moto/datapipeline/responses.py index 7ddeec061..70d19d189 100644 --- a/moto/datapipeline/responses.py +++ b/moto/datapipeline/responses.py @@ -31,9 +31,9 @@ class DataPipelineResponse(BaseResponse): def list_pipelines(self): pipelines = self.datapipeline_backend.list_pipelines() return json.dumps({ - "HasMoreResults": False, - "Marker": None, - "PipelineIdList": [ + "hasMoreResults": False, + "marker": None, + "pipelineIdList": [ pipeline.to_meta_json() for pipeline in pipelines ] }) @@ -43,7 +43,7 @@ class DataPipelineResponse(BaseResponse): pipelines = self.datapipeline_backend.describe_pipelines(pipeline_ids) return json.dumps({ - "PipelineDescriptionList": [ + "pipelineDescriptionList": [ pipeline.to_json() for pipeline in pipelines ] }) @@ -67,11 +67,10 @@ class DataPipelineResponse(BaseResponse): object_ids = self.parameters["objectIds"] pipeline_objects = self.datapipeline_backend.describe_objects(object_ids, pipeline_id) - return json.dumps({ - "HasMoreResults": False, - "Marker": None, - "PipelineObjects": [ + "hasMoreResults": False, + "marker": None, + "pipelineObjects": [ pipeline_object.to_json() for pipeline_object in pipeline_objects ] }) diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index a47026221..f84535425 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -1470,5 +1470,5 @@ def test_datapipeline(): dp_conn = boto.datapipeline.connect_to_region('us-east-1') data_pipelines = dp_conn.list_pipelines() - data_pipelines['PipelineIdList'].should.have.length_of(1) - data_pipelines['PipelineIdList'][0]['name'].should.equal('testDataPipeline') + data_pipelines['pipelineIdList'].should.have.length_of(1) + data_pipelines['pipelineIdList'][0]['name'].should.equal('testDataPipeline') diff --git a/tests/test_datapipeline/test_datapipeline.py b/tests/test_datapipeline/test_datapipeline.py index 27b942271..5a958492f 100644 --- a/tests/test_datapipeline/test_datapipeline.py +++ b/tests/test_datapipeline/test_datapipeline.py @@ -20,7 +20,7 @@ def test_create_pipeline(): res = conn.create_pipeline("mypipeline", "some-unique-id") pipeline_id = res["pipelineId"] - pipeline_descriptions = conn.describe_pipelines([pipeline_id])["PipelineDescriptionList"] + pipeline_descriptions = conn.describe_pipelines([pipeline_id])["pipelineDescriptionList"] pipeline_descriptions.should.have.length_of(1) pipeline_description = pipeline_descriptions[0] @@ -105,7 +105,7 @@ def test_describing_pipeline_objects(): conn.put_pipeline_definition(PIPELINE_OBJECTS, pipeline_id) - objects = conn.describe_objects(["Schedule", "Default"], pipeline_id)['PipelineObjects'] + objects = conn.describe_objects(["Schedule", "Default"], pipeline_id)['pipelineObjects'] objects.should.have.length_of(2) default_object = [x for x in objects if x['id'] == 'Default'][0] @@ -125,7 +125,7 @@ def test_activate_pipeline(): pipeline_id = res["pipelineId"] conn.activate_pipeline(pipeline_id) - pipeline_descriptions = conn.describe_pipelines([pipeline_id])["PipelineDescriptionList"] + pipeline_descriptions = conn.describe_pipelines([pipeline_id])["pipelineDescriptionList"] pipeline_descriptions.should.have.length_of(1) pipeline_description = pipeline_descriptions[0] fields = pipeline_description['fields'] @@ -141,14 +141,14 @@ def test_listing_pipelines(): response = conn.list_pipelines() - response["HasMoreResults"].should.be(False) - response["Marker"].should.be.none - response["PipelineIdList"].should.have.length_of(2) - response["PipelineIdList"].should.contain({ + response["hasMoreResults"].should.be(False) + response["marker"].should.be.none + response["pipelineIdList"].should.have.length_of(2) + response["pipelineIdList"].should.contain({ "id": res1["pipelineId"], "name": "mypipeline1", }) - response["PipelineIdList"].should.contain({ + response["pipelineIdList"].should.contain({ "id": res2["pipelineId"], "name": "mypipeline2" }) diff --git a/tests/test_datapipeline/test_server.py b/tests/test_datapipeline/test_server.py index 33a19c6f0..012c5ad55 100644 --- a/tests/test_datapipeline/test_server.py +++ b/tests/test_datapipeline/test_server.py @@ -23,5 +23,5 @@ def test_list_streams(): json_data = json.loads(res.data.decode("utf-8")) json_data.should.equal({ - 'PipelineDescriptionList': [] + 'pipelineDescriptionList': [] }) From dafddb094bb917f475ee5b079258d553fe135421 Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Thu, 17 Sep 2015 15:19:36 -0400 Subject: [PATCH 15/55] Implement CloudFormation's physical_resource_id for Data Pipeline --- moto/datapipeline/models.py | 4 ++++ .../test_cloudformation_stack_integration.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 1b97c01d1..b6a70b5f1 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -30,6 +30,10 @@ class Pipeline(object): self.objects = [] self.status = "PENDING" + @property + def physical_resource_id(self): + return self.pipeline_id + def to_meta_json(self): return { "id": self.pipeline_id, diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index f84535425..a38a1029b 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -1462,7 +1462,7 @@ def test_datapipeline(): } cf_conn = boto.cloudformation.connect_to_region("us-east-1") template_json = json.dumps(dp_template) - cf_conn.create_stack( + stack_id = cf_conn.create_stack( "test_stack", template_body=template_json, ) @@ -1472,3 +1472,7 @@ def test_datapipeline(): data_pipelines['pipelineIdList'].should.have.length_of(1) data_pipelines['pipelineIdList'][0]['name'].should.equal('testDataPipeline') + + stack_resources = cf_conn.list_stack_resources(stack_id) + stack_resources.should.have.length_of(1) + stack_resources[0].physical_resource_id.should.equal(data_pipelines['pipelineIdList'][0]['id']) From 967c778390a8d5991a475fa92856e627d3116b1b Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Thu, 17 Sep 2015 17:21:57 -0400 Subject: [PATCH 16/55] 0.4.14 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index d5220e8b2..0d7e04a70 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.13' +__version__ = '0.4.14' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index df18bbbf7..18ec4b314 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.13', + version='0.4.14', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From f72613cc47712cdf95b40fb49119c11e74d7ae56 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 19 Sep 2015 09:18:16 -0400 Subject: [PATCH 17/55] Fix queue urls for other regions. Closes #411. --- moto/sqs/models.py | 19 +++++++++++++++---- moto/sqs/responses.py | 6 +++--- tests/test_sqs/test_sqs.py | 2 ++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index bd4129dc2..bc0a5a4c6 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -106,9 +106,10 @@ class Queue(object): 'VisibilityTimeout', 'WaitTimeSeconds'] - def __init__(self, name, visibility_timeout, wait_time_seconds): + def __init__(self, name, visibility_timeout, wait_time_seconds, region): self.name = name self.visibility_timeout = visibility_timeout or 30 + self.region = region # wait_time_seconds will be set to immediate return messages self.wait_time_seconds = wait_time_seconds or 0 @@ -179,6 +180,10 @@ class Queue(object): result[attribute] = getattr(self, camelcase_to_underscores(attribute)) return result + @property + def url(self): + return "http://sqs.{0}.amazonaws.com/123456789012/{1}".format(self.region, self.name) + @property def messages(self): return [message for message in self._messages if message.visible and not message.delayed] @@ -196,14 +201,20 @@ class Queue(object): class SQSBackend(BaseBackend): - def __init__(self): + def __init__(self, region_name): + self.region_name = region_name self.queues = {} super(SQSBackend, self).__init__() + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + def create_queue(self, name, visibility_timeout, wait_time_seconds): queue = self.queues.get(name) if queue is None: - queue = Queue(name, visibility_timeout, wait_time_seconds) + queue = Queue(name, visibility_timeout, wait_time_seconds, self.region_name) self.queues[name] = queue return queue @@ -314,4 +325,4 @@ class SQSBackend(BaseBackend): sqs_backends = {} for region in boto.sqs.regions(): - sqs_backends[region.name] = SQSBackend() + sqs_backends[region.name] = SQSBackend(region.name) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 648e939d2..abae83fed 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -232,7 +232,7 @@ class SQSResponse(BaseResponse): CREATE_QUEUE_RESPONSE = """ - http://sqs.us-east-1.amazonaws.com/123456789012/{{ queue.name }} + {{ queue.url }} {{ queue.visibility_timeout }} @@ -244,7 +244,7 @@ CREATE_QUEUE_RESPONSE = """ GET_QUEUE_URL_RESPONSE = """ - http://sqs.us-east-1.amazonaws.com/123456789012/{{ queue.name }} + {{ queue.url }} 470a6f13-2ed9-4181-ad8a-2fdea142988e @@ -254,7 +254,7 @@ GET_QUEUE_URL_RESPONSE = """ LIST_QUEUES_RESPONSE = """ {% for queue in queues %} - http://sqs.us-east-1.amazonaws.com/123456789012/{{ queue.name }} + {{ queue.url }} {% endfor %} diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index ba1e11e52..a23545dcc 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -34,6 +34,8 @@ def test_create_queues_in_multiple_region(): list(west1_conn.get_all_queues()).should.have.length_of(1) list(west2_conn.get_all_queues()).should.have.length_of(1) + west1_conn.get_all_queues()[0].url.should.equal('http://sqs.us-west-1.amazonaws.com/123456789012/test-queue') + @mock_sqs def test_get_queue(): From 0999a49b5b8aa2ed3292b75ca70d1e8ca4d0634b Mon Sep 17 00:00:00 2001 From: Anthony Monthe Date: Wed, 23 Sep 2015 14:48:54 +0200 Subject: [PATCH 18/55] Added describe instance types --- moto/ec2/responses/instances.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 8689aa3a8..fdc2432c6 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +from boto.ec2.instancetype import InstanceType from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores from moto.ec2.utils import instance_ids_from_querystring, filters_from_querystring, \ @@ -78,6 +79,11 @@ class InstanceResponse(BaseResponse): template = self.response_template(EC2_INSTANCE_STATUS) return template.render(instances=instances) + def describe_instance_types(self): + instance_types = [InstanceType(name='t1.micro', cores=1, memory=644874240, disk=0)] + template = self.response_template(EC2_DESCRIBE_INSTANCE_TYPES) + return template.render(instance_types=instance_types) + def describe_instance_attribute(self): # TODO this and modify below should raise IncorrectInstanceState if # instance not in stopped state @@ -586,3 +592,21 @@ EC2_INSTANCE_STATUS = """ {% endfor %} """ + +EC2_DESCRIBE_INSTANCE_TYPES = """ + + f8b86168-d034-4e65-b48d-3b84c78e64af + + {% for instance_type in instance_types %} + + {{ instance_type.name }} + {{ instance_type.cores }} + {{ instance_type.memory }} + {{ instance_type.disk }} + {{ instance_type.storageCount }} + {{ instance_type.maxIpAddresses }} + {{ instance_type.ebsOptimizedAvailable }} + + {% endfor %} + +""" From b85b41597778195192bf5cc23503cf36893b039f Mon Sep 17 00:00:00 2001 From: ZuluPro Date: Thu, 24 Sep 2015 17:25:49 +0200 Subject: [PATCH 19/55] Implemented import key pair --- moto/ec2/models.py | 7 +++++++ moto/ec2/responses/key_pairs.py | 13 ++++++++++++- tests/test_ec2/test_key_pairs.py | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index efe968eb2..a7c0133c7 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -710,6 +710,13 @@ class KeyPairBackend(object): return results + def import_key_pair(self, key_name, public_key_material): + if key_name in self.keypairs: + raise InvalidKeyPairDuplicateError(key_name) + self.keypairs[key_name] = keypair = random_key_pair() + keypair['name'] = key_name + return keypair + class TagBackend(object): diff --git a/moto/ec2/responses/key_pairs.py b/moto/ec2/responses/key_pairs.py index 80c6442f0..128e04ba9 100644 --- a/moto/ec2/responses/key_pairs.py +++ b/moto/ec2/responses/key_pairs.py @@ -28,7 +28,11 @@ class KeyPairs(BaseResponse): return template.render(keypairs=keypairs) def import_key_pair(self): - raise NotImplementedError('KeyPairs.import_key_pair is not yet implemented') + name = self.querystring.get('KeyName')[0] + material = self.querystring.get('PublicKeyMaterial')[0] + keypair = self.ec2_backend.import_key_pair(name, material) + template = self.response_template(IMPORT_KEYPAIR_RESPONSE) + return template.render(**keypair) DESCRIBE_KEY_PAIRS_RESPONSE = """ @@ -58,3 +62,10 @@ DELETE_KEY_PAIR_RESPONSE = """ + + 471f9fdd-8fe2-4a84-86b0-bd3d3e350979 + {{ name }} + {{ fingerprint }} + """ diff --git a/tests/test_ec2/test_key_pairs.py b/tests/test_ec2/test_key_pairs.py index 2390f45ce..858e064fb 100644 --- a/tests/test_ec2/test_key_pairs.py +++ b/tests/test_ec2/test_key_pairs.py @@ -85,3 +85,27 @@ def test_key_pairs_delete_exist(): r = conn.delete_key_pair('foo') r.should.be.ok assert len(conn.get_all_key_pairs()) == 0 + + +@mock_ec2 +def test_key_pairs_import(): + conn = boto.connect_ec2('the_key', 'the_secret') + kp = conn.import_key_pair('foo', b'content') + assert kp.name == 'foo' + kps = conn.get_all_key_pairs() + assert len(kps) == 1 + assert kps[0].name == 'foo' + + +@mock_ec2 +def test_key_pairs_import_exist(): + conn = boto.connect_ec2('the_key', 'the_secret') + kp = conn.import_key_pair('foo', b'content') + assert kp.name == 'foo' + assert len(conn.get_all_key_pairs()) == 1 + + with assert_raises(EC2ResponseError) as cm: + conn.create_key_pair('foo') + cm.exception.code.should.equal('InvalidKeyPair.Duplicate') + cm.exception.status.should.equal(400) + cm.exception.request_id.should_not.be.none From 76bce7954ac58027b706d97ea4277f50a2242cd6 Mon Sep 17 00:00:00 2001 From: Andy Raines Date: Mon, 5 Oct 2015 14:14:56 +0100 Subject: [PATCH 20/55] Fixes #430: MD5 hashing should be done to the real body, not an escaped one --- moto/sqs/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index bc0a5a4c6..efb75dd9c 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -34,7 +34,7 @@ class Message(object): @property def md5(self): body_md5 = hashlib.md5() - body_md5.update(self.body.encode('utf-8')) + body_md5.update(self._body.encode('utf-8')) return body_md5.hexdigest() @property From e5675e953315a8d5b651f07d14bd2871afc99fca Mon Sep 17 00:00:00 2001 From: Hugo Lopes Tavares Date: Mon, 5 Oct 2015 15:21:30 -0400 Subject: [PATCH 21/55] Change CloudFormationResponse.get_template() to return `GetTemplateResponse/GetTemplateResult/TemplateBody` --- moto/cloudformation/responses.py | 14 +++++- .../test_cloudformation_stack_crud.py | 48 +++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 28db697cf..c5196b2df 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -86,9 +86,19 @@ class CloudFormationResponse(BaseResponse): def get_template(self): name_or_stack_id = self.querystring.get('StackName')[0] - stack = self.cloudformation_backend.get_stack(name_or_stack_id) - return stack.template + + response = { + "GetTemplateResponse": { + "GetTemplateResult": { + "TemplateBody": stack.template, + "ResponseMetadata": { + "RequestId": "2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE" + } + } + } + } + return json.dumps(response) def update_stack(self): stack_name = self._get_param('StackName') diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index 236c33e7d..5ca20fe04 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -40,7 +40,17 @@ def test_create_stack(): stack = conn.describe_stacks()[0] stack.stack_name.should.equal('test_stack') - stack.get_template().should.equal(dummy_template) + stack.get_template().should.equal({ + 'GetTemplateResponse': { + 'GetTemplateResult': { + 'TemplateBody': dummy_template_json, + 'ResponseMetadata': { + 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' + } + } + } + + }) @mock_cloudformation @@ -83,7 +93,18 @@ def test_create_stack_from_s3_url(): stack = conn.describe_stacks()[0] stack.stack_name.should.equal('new-stack') - stack.get_template().should.equal(dummy_template) + stack.get_template().should.equal( + { + 'GetTemplateResponse': { + 'GetTemplateResult': { + 'TemplateBody': dummy_template_json, + 'ResponseMetadata': { + 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' + } + } + } + + }) @mock_cloudformation @@ -138,7 +159,17 @@ def test_get_template_by_name(): ) template = conn.get_template("test_stack") - template.should.equal(dummy_template) + template.should.equal({ + 'GetTemplateResponse': { + 'GetTemplateResult': { + 'TemplateBody': dummy_template_json, + 'ResponseMetadata': { + 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' + } + } + } + + }) @mock_cloudformation @@ -243,4 +274,13 @@ def test_stack_tags(): # conn.update_stack("test_stack", dummy_template_json2) # stack = conn.describe_stacks()[0] -# stack.get_template().should.equal(dummy_template2) +# stack.get_template().should.equal({ +# 'GetTemplateResponse': { +# 'GetTemplateResult': { +# 'TemplateBody': dummy_template_json2, +# 'ResponseMetadata': { +# 'RequestId': '2d06e36c-ac1d-11e0-a958-f9382b6eb86bEXAMPLE' +# } +# } +# } +# }) From 3c38a551b2ba440fdb4d8578a9c6aef98d353224 Mon Sep 17 00:00:00 2001 From: Miles O'Connell Date: Tue, 6 Oct 2015 09:21:26 -0700 Subject: [PATCH 22/55] Adding tags to AutoScalingGroups --- moto/autoscaling/models.py | 4 +++- moto/autoscaling/responses.py | 2 ++ tests/test_autoscaling/test_autoscaling.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 2c8f425ac..db4fa95a8 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -113,7 +113,8 @@ class FakeAutoScalingGroup(object): def __init__(self, name, availability_zones, desired_capacity, max_size, min_size, launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, - load_balancers, placement_group, termination_policies, autoscaling_backend): + load_balancers, placement_group, termination_policies, + autoscaling_backend, tags): self.autoscaling_backend = autoscaling_backend self.name = name self.availability_zones = availability_zones @@ -133,6 +134,7 @@ class FakeAutoScalingGroup(object): self.instance_states = [] self.set_desired_capacity(desired_capacity) + self.tags = tags if tags else [] @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 4f5948b6d..b1fee1a38 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -60,6 +60,7 @@ class AutoScalingResponse(BaseResponse): load_balancers=self._get_multi_param('LoadBalancerNames.member'), placement_group=self._get_param('PlacementGroup'), termination_policies=self._get_multi_param('TerminationPolicies.member'), + tags=self._get_multi_param('Tags.member'), ) template = self.response_template(CREATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() @@ -85,6 +86,7 @@ class AutoScalingResponse(BaseResponse): load_balancers=self._get_multi_param('LoadBalancerNames.member'), placement_group=self._get_param('PlacementGroup'), termination_policies=self._get_multi_param('TerminationPolicies.member'), + tags=self._get_multi_param('Tags.member'), ) template = self.response_template(UPDATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 8b8f8f320..51cbfa521 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -32,6 +32,13 @@ def test_create_autoscaling_group(): placement_group="test_placement", vpc_zone_identifier='subnet-1234abcd', termination_policies=["OldestInstance", "NewestInstance"], + tags=[{ + 'ResourceId': 'tester_group', + 'ResourceType': 'auto-scaling-group', + 'Key': 'test_key', + 'Value': 'test_value', + 'PropagateAtLaunch': True, + }], ) conn.create_auto_scaling_group(group) @@ -50,6 +57,13 @@ def test_create_autoscaling_group(): list(group.load_balancers).should.equal(["test_lb"]) group.placement_group.should.equal("test_placement") list(group.termination_policies).should.equal(["OldestInstance", "NewestInstance"]) + list(group.tags).should.equal([{ + 'ResourceId': 'tester_group', + 'ResourceType': 'auto-scaling-group', + 'Key': 'test_key', + 'Value': 'test_value', + 'PropagateAtLaunch': True, + }]) @mock_autoscaling @@ -88,6 +102,7 @@ def test_create_autoscaling_groups_defaults(): list(group.load_balancers).should.equal([]) group.placement_group.should.equal(None) list(group.termination_policies).should.equal([]) + list(group.tags).should.equal([]) @mock_autoscaling From b3096af0989db4edc1b8f03051c99c48c7ac81e7 Mon Sep 17 00:00:00 2001 From: milesoc Date: Tue, 6 Oct 2015 18:02:38 +0000 Subject: [PATCH 23/55] Set tags in response, fix tests for tags --- moto/autoscaling/models.py | 6 +++-- moto/autoscaling/responses.py | 15 ++++++++--- tests/test_autoscaling/test_autoscaling.py | 29 +++++++++++----------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index db4fa95a8..cb95d0542 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -113,7 +113,7 @@ class FakeAutoScalingGroup(object): def __init__(self, name, availability_zones, desired_capacity, max_size, min_size, launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, - load_balancers, placement_group, termination_policies, + load_balancers, placement_group, termination_policies, autoscaling_backend, tags): self.autoscaling_backend = autoscaling_backend self.name = name @@ -158,6 +158,7 @@ class FakeAutoScalingGroup(object): load_balancers=load_balancer_names, placement_group=None, termination_policies=properties.get("TerminationPolicies", []), + tags=properties.get("Tags", []), ) return group @@ -263,7 +264,7 @@ class AutoScalingBackend(BaseBackend): launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, load_balancers, - placement_group, termination_policies): + placement_group, termination_policies, tags): def make_int(value): return int(value) if value is not None else value @@ -288,6 +289,7 @@ class AutoScalingBackend(BaseBackend): placement_group=placement_group, termination_policies=termination_policies, autoscaling_backend=self, + tags=tags, ) self.autoscaling_groups[name] = group return group diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index b1fee1a38..70fda4526 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -60,7 +60,7 @@ class AutoScalingResponse(BaseResponse): load_balancers=self._get_multi_param('LoadBalancerNames.member'), placement_group=self._get_param('PlacementGroup'), termination_policies=self._get_multi_param('TerminationPolicies.member'), - tags=self._get_multi_param('Tags.member'), + tags=self._get_list_prefix('Tags.member'), ) template = self.response_template(CREATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() @@ -86,7 +86,6 @@ class AutoScalingResponse(BaseResponse): load_balancers=self._get_multi_param('LoadBalancerNames.member'), placement_group=self._get_param('PlacementGroup'), termination_policies=self._get_multi_param('TerminationPolicies.member'), - tags=self._get_multi_param('Tags.member'), ) template = self.response_template(UPDATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() @@ -237,7 +236,17 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """ {% for group in groups %} - + + {% for tag in group.tags %} + + {{ tag.resource_type }} + {{ tag.resource_id }} + {{ tag.propagate_at_launch }} + {{ tag.key }} + {{ tag.value }} + + {% endfor %} + {{ group.name }} {{ group.health_check_type }} diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 51cbfa521..41286442d 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import boto from boto.ec2.autoscale.launchconfig import LaunchConfiguration from boto.ec2.autoscale.group import AutoScalingGroup +from boto.ec2.autoscale import Tag import sure # noqa from moto import mock_autoscaling, mock_ec2 @@ -18,6 +19,7 @@ def test_create_autoscaling_group(): ) conn.create_launch_configuration(config) + group = AutoScalingGroup( name='tester_group', availability_zones=['us-east-1c', 'us-east-1b'], @@ -32,13 +34,13 @@ def test_create_autoscaling_group(): placement_group="test_placement", vpc_zone_identifier='subnet-1234abcd', termination_policies=["OldestInstance", "NewestInstance"], - tags=[{ - 'ResourceId': 'tester_group', - 'ResourceType': 'auto-scaling-group', - 'Key': 'test_key', - 'Value': 'test_value', - 'PropagateAtLaunch': True, - }], + tags=[Tag( + resource_id='tester_group', + key='test_key', + value='test_value', + propagate_at_launch=True + ) + ], ) conn.create_auto_scaling_group(group) @@ -57,13 +59,12 @@ def test_create_autoscaling_group(): list(group.load_balancers).should.equal(["test_lb"]) group.placement_group.should.equal("test_placement") list(group.termination_policies).should.equal(["OldestInstance", "NewestInstance"]) - list(group.tags).should.equal([{ - 'ResourceId': 'tester_group', - 'ResourceType': 'auto-scaling-group', - 'Key': 'test_key', - 'Value': 'test_value', - 'PropagateAtLaunch': True, - }]) + len(list(group.tags)).should.equal(1) + tag = list(group.tags)[0] + tag.resource_id.should.equal('tester_group') + tag.key.should.equal('test_key') + tag.value.should.equal('test_value') + tag.propagate_at_launch.should.equal(True) @mock_autoscaling From 11cb2fba16048a272df64139121d9e6a9895de62 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 6 Oct 2015 15:20:21 -0400 Subject: [PATCH 24/55] 0.4.15 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 0d7e04a70..25031f0ed 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.14' +__version__ = '0.4.15' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index 18ec4b314..c82d53208 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.14', + version='0.4.15', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 131d44f0ad3918ab96ab01f92e0291e4ded569c4 Mon Sep 17 00:00:00 2001 From: Mike Kaplinskiy Date: Wed, 7 Oct 2015 00:04:22 -0700 Subject: [PATCH 25/55] Add S3 ACL supprt. --- moto/s3/models.py | 65 +++++++++++++++++++++++- moto/s3/responses.py | 103 +++++++++++++++++++++++++++++++++------ tests/test_s3/test_s3.py | 45 ++++++++++++++++- 3 files changed, 194 insertions(+), 19 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 5d0bfce2d..83412a3f9 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -24,6 +24,7 @@ class FakeKey(object): self.name = name self.value = value self.last_modified = datetime.datetime.utcnow() + self.acl = get_canned_acl('private') self._storage_class = storage self._metadata = {} self._expiry = None @@ -45,6 +46,9 @@ class FakeKey(object): def set_storage_class(self, storage_class): self._storage_class = storage_class + def set_acl(self, acl): + self.acl = acl + def append_to_value(self, value): self.value += value self.last_modified = datetime.datetime.utcnow() @@ -161,6 +165,61 @@ class FakeMultipart(object): yield self.parts[part_id] +class FakeGrantee(object): + def __init__(self, id='', uri='', display_name=''): + self.id = id + self.uri = uri + self.display_name = display_name + + @property + def type(self): + return 'Group' if self.uri else 'CanonicalUser' + + +ALL_USERS_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/global/AllUsers') +AUTHENTICATED_USERS_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/global/AuthenticatedUsers') +LOG_DELIVERY_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/s3/LogDelivery') + +PERMISSION_FULL_CONTROL = 'FULL_CONTROL' +PERMISSION_WRITE = 'WRITE' +PERMISSION_READ = 'READ' +PERMISSION_WRITE_ACP = 'WRITE_ACP' +PERMISSION_READ_ACP = 'READ_ACP' + + +class FakeGrant(object): + def __init__(self, grantees, permissions): + self.grantees = grantees + self.permissions = permissions + + +class FakeAcl(object): + def __init__(self, grants=[]): + self.grants = grants + + +def get_canned_acl(acl): + owner_grantee = FakeGrantee(id='75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a') + grants = [FakeGrant([owner_grantee], [PERMISSION_FULL_CONTROL])] + if acl == 'private': + pass # no other permissions + elif acl == 'public-read': + grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ])) + elif acl == 'public-read-write': + grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ, PERMISSION_WRITE])) + elif acl == 'authenticated-read': + grants.append(FakeGrant([AUTHENTICATED_USERS_GRANTEE], [PERMISSION_READ])) + elif acl == 'bucket-owner-read': + pass # TODO: bucket owner ACL + elif acl == 'bucket-owner-full-control': + pass # TODO: bucket owner ACL + elif acl == 'log-delivery-write': + grants.append(FakeGrant([LOG_DELIVERY_GRANTEE], [PERMISSION_READ_ACP, PERMISSION_WRITE])) + else: + assert False, 'Unknown canned acl: %s' % (acl,) + return FakeAcl(grants=grants) + + class LifecycleRule(object): def __init__(self, id=None, prefix=None, status=None, expiration_days=None, expiration_date=None, transition_days=None, @@ -399,7 +458,7 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) return bucket.keys.pop(key_name) - def copy_key(self, src_bucket_name, src_key_name, dest_bucket_name, dest_key_name, storage=None): + def copy_key(self, src_bucket_name, src_key_name, dest_bucket_name, dest_key_name, storage=None, acl=None): src_key_name = clean_key_name(src_key_name) dest_key_name = clean_key_name(dest_key_name) src_bucket = self.get_bucket(src_bucket_name) @@ -409,6 +468,8 @@ class S3Backend(BaseBackend): key = key.copy(dest_key_name) dest_bucket.keys[dest_key_name] = key if storage is not None: - dest_bucket.keys[dest_key_name].set_storage_class(storage) + key.set_storage_class(storage) + if acl is not None: + key.set_acl(acl) s3_backend = S3Backend() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 8b9fdb228..687b0464f 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -9,7 +9,7 @@ import xmltodict from moto.core.responses import _TemplateEnvironmentMixin from .exceptions import BucketAlreadyExists, S3ClientError, InvalidPartOrder -from .models import s3_backend +from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl from .utils import bucket_name_from_url, metadata_from_headers from xml.dom import minidom @@ -301,7 +301,7 @@ class ResponseObject(_TemplateEnvironmentMixin): def _key_response(self, request, full_url, headers): parsed_url = urlparse(full_url) - query = parse_qs(parsed_url.query) + query = parse_qs(parsed_url.query, keep_blank_values=True) method = request.method key_name = self.parse_key_name(parsed_url.path) @@ -317,18 +317,18 @@ class ResponseObject(_TemplateEnvironmentMixin): if method == 'GET': return self._key_response_get(bucket_name, query, key_name, headers) elif method == 'PUT': - return self._key_response_put(request, parsed_url, body, bucket_name, query, key_name, headers) + return self._key_response_put(request, body, bucket_name, query, key_name, headers) elif method == 'HEAD': return self._key_response_head(bucket_name, key_name, headers) elif method == 'DELETE': return self._key_response_delete(bucket_name, query, key_name, headers) elif method == 'POST': - return self._key_response_post(request, body, parsed_url, bucket_name, query, key_name, headers) + return self._key_response_post(request, body, bucket_name, query, key_name, headers) else: raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) def _key_response_get(self, bucket_name, query, key_name, headers): - if 'uploadId' in query: + if query.get('uploadId'): upload_id = query['uploadId'][0] parts = self.backend.list_multipart(bucket_name, upload_id) template = self.response_template(S3_MULTIPART_LIST_RESPONSE) @@ -342,14 +342,18 @@ class ResponseObject(_TemplateEnvironmentMixin): version_id = query.get('versionId', [None])[0] key = self.backend.get_key( bucket_name, key_name, version_id=version_id) + if 'acl' in query: + template = self.response_template(S3_OBJECT_ACL_RESPONSE) + return 200, headers, template.render(key=key) + if key: headers.update(key.metadata) return 200, headers, key.value else: return 404, headers, "" - def _key_response_put(self, request, parsed_url, body, bucket_name, query, key_name, headers): - if 'uploadId' in query and 'partNumber' in query: + def _key_response_put(self, request, body, bucket_name, query, key_name, headers): + if query.get('uploadId') and query.get('partNumber'): upload_id = query['uploadId'][0] part_number = int(query['partNumber'][0]) if 'x-amz-copy-source' in request.headers: @@ -368,16 +372,19 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, headers, response storage_class = request.headers.get('x-amz-storage-class', 'STANDARD') + acl = self._acl_from_headers(request.headers) - if parsed_url.query == 'acl': - # We don't implement ACL yet, so just return + if 'acl' in query: + key = self.backend.get_key(bucket_name, key_name) + # TODO: Support the XML-based ACL format + key.set_acl(acl) return 200, headers, "" if 'x-amz-copy-source' in request.headers: # Copy key src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1) self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, - storage=storage_class) + storage=storage_class, acl=acl) mdirective = request.headers.get('x-amz-metadata-directive') if mdirective is not None and mdirective == 'REPLACE': new_key = self.backend.get_key(bucket_name, key_name) @@ -400,6 +407,7 @@ class ResponseObject(_TemplateEnvironmentMixin): request.streaming = True metadata = metadata_from_headers(request.headers) new_key.set_metadata(metadata) + new_key.set_acl(acl) template = self.response_template(S3_OBJECT_RESPONSE) headers.update(new_key.response_dict) @@ -414,8 +422,40 @@ class ResponseObject(_TemplateEnvironmentMixin): else: return 404, headers, "" + def _acl_from_headers(self, headers): + canned_acl = headers.get('x-amz-acl', '') + if canned_acl: + return get_canned_acl(canned_acl) + + grants = [] + for header, value in headers.items(): + if not header.startswith('x-amz-grant-'): + continue + + permission = { + 'read': 'READ', + 'write': 'WRITE', + 'read-acp': 'READ_ACP', + 'write-acp': 'WRITE_ACP', + 'full-control': 'FULL_CONTROL', + }[header[len('x-amz-grant-'):]] + + grantees = [] + for key_and_value in value.split(","): + key, value = re.match('([^=]+)="([^"]+)"', key_and_value.strip()).groups() + if key.lower() == 'id': + grantees.append(FakeGrantee(id=value)) + else: + grantees.append(FakeGrantee(uri=value)) + grants.append(FakeGrant(grantees, [permission])) + + if grants: + return FakeAcl(grants) + else: + return None + def _key_response_delete(self, bucket_name, query, key_name, headers): - if 'uploadId' in query: + if query.get('uploadId'): upload_id = query['uploadId'][0] self.backend.cancel_multipart(bucket_name, upload_id) return 204, headers, "" @@ -435,8 +475,8 @@ class ResponseObject(_TemplateEnvironmentMixin): raise InvalidPartOrder() yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText) - def _key_response_post(self, request, body, parsed_url, bucket_name, query, key_name, headers): - if body == b'' and parsed_url.query == 'uploads': + def _key_response_post(self, request, body, bucket_name, query, key_name, headers): + if body == b'' and 'uploads' in query: metadata = metadata_from_headers(request.headers) multipart = self.backend.initiate_multipart(bucket_name, key_name, metadata) @@ -448,7 +488,7 @@ class ResponseObject(_TemplateEnvironmentMixin): ) return 200, headers, response - if 'uploadId' in query: + if query.get('uploadId'): body = self._complete_multipart_body(body) upload_id = query['uploadId'][0] key = self.backend.complete_multipart(bucket_name, upload_id, body) @@ -458,7 +498,7 @@ class ResponseObject(_TemplateEnvironmentMixin): key_name=key.name, etag=key.etag, ) - elif parsed_url.query == 'restore': + elif 'restore' in query: es = minidom.parseString(body).getElementsByTagName('Days') days = es[0].childNodes[0].wholeText key = self.backend.get_key(bucket_name, key_name) @@ -642,6 +682,37 @@ S3_OBJECT_RESPONSE = """ + + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + + {% for grant in key.acl.grants %} + + {% for grantee in grant.grantees %} + + {% if grantee.uri %} + {{ grantee.uri }} + {% endif %} + {% if grantee.id %} + {{ grantee.id }} + {% endif %} + {% if grantee.display_name %} + {{ grantee.display_name }} + {% endif %} + + {% endfor %} + {% for permission in grant.permissions %} + {{ permission }} + {% endfor %} + + {% endfor %} + + """ + S3_OBJECT_COPY_RESPONSE = """ {{ key.etag }} @@ -717,7 +788,7 @@ S3_ALL_MULTIPARTS = """ 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a - OwnerDisplayName + webfile STANDARD 2010-11-10T20:48:33.000Z diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f5d7cba15..0434c0d65 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -726,7 +726,7 @@ def test_list_versions(): @mock_s3 -def test_acl_is_ignored_for_now(): +def test_acl_setting(): conn = boto.connect_s3() bucket = conn.create_bucket('foobar') content = b'imafile' @@ -741,6 +741,49 @@ def test_acl_is_ignored_for_now(): assert key.get_contents_as_string() == content + grants = key.get_acl().acl.grants + assert any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'READ' for g in grants), grants + + +@mock_s3 +def test_acl_setting_via_headers(): + conn = boto.connect_s3() + bucket = conn.create_bucket('foobar') + content = b'imafile' + keyname = 'test.txt' + + key = Key(bucket, name=keyname) + key.content_type = 'text/plain' + key.set_contents_from_string(content, headers={ + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/global/AllUsers"' + }) + + key = bucket.get_key(keyname) + + assert key.get_contents_as_string() == content + + grants = key.get_acl().acl.grants + assert any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'FULL_CONTROL' for g in grants), grants + + +@mock_s3 +def test_acl_switching(): + conn = boto.connect_s3() + bucket = conn.create_bucket('foobar') + content = b'imafile' + keyname = 'test.txt' + + key = Key(bucket, name=keyname) + key.content_type = 'text/plain' + key.set_contents_from_string(content, policy='public-read') + key.set_acl('private') + + grants = key.get_acl().acl.grants + assert not any(g.uri == 'http://acs.amazonaws.com/groups/global/AllUsers' and + g.permission == 'READ' for g in grants), grants + @mock_s3 def test_unicode_key(): From 0b3ad166c05b497d026d3168698f07c303e49392 Mon Sep 17 00:00:00 2001 From: nuwan_ag Date: Wed, 14 Oct 2015 02:07:47 +1100 Subject: [PATCH 26/55] Set snapshots to be in a completed state after being created and added test case --- moto/ec2/responses/elastic_block_store.py | 4 ++-- tests/test_ec2/test_elastic_block_store.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index abb371260..876766b2f 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -198,9 +198,9 @@ DESCRIBE_SNAPSHOTS_RESPONSE = """ Date: Tue, 20 Oct 2015 09:12:59 -0700 Subject: [PATCH 27/55] Use correct kwarg when creating database --- moto/rds2/models.py | 2 +- moto/rds2/responses.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 5cb05ebeb..37ecbf873 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -135,7 +135,7 @@ class Database(object): "engine": properties.get("Engine"), "engine_version": properties.get("EngineVersion"), "iops": properties.get("Iops"), - "master_password": properties.get('MasterUserPassword'), + "master_user_password": properties.get('MasterUserPassword'), "master_username": properties.get('MasterUsername'), "multi_az": properties.get("MultiAZ"), "port": properties.get('Port', 3306), diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index daa068aa6..bd51f6ea0 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -27,7 +27,7 @@ class RDS2Response(BaseResponse): "engine": self._get_param("Engine"), "engine_version": self._get_param("EngineVersion"), "iops": self._get_int_param("Iops"), - "master_password": self._get_param('MasterUserPassword'), + "master_user_password": self._get_param('MasterUserPassword'), "master_username": self._get_param('MasterUsername'), "multi_az": self._get_bool_param("MultiAZ"), # OptionGroupName @@ -504,4 +504,4 @@ ADD_TAGS_TO_RESOURCE_TEMPLATE = \ REMOVE_TAGS_FROM_RESOURCE_TEMPLATE = \ """{"RemoveTagsFromResourceResponse": {"ResponseMetadata": {"RequestId": "c6499a01-a664-11e4-8069-fb454b71a80e"}}} - """ \ No newline at end of file + """ From e07894c6e4105300b273858e6f648ca6781a7393 Mon Sep 17 00:00:00 2001 From: Enis Afgan Date: Thu, 29 Oct 2015 07:27:35 +0100 Subject: [PATCH 28/55] When adding security group rules, allow a source group only to be specified - as per boto docs & functionality --- moto/ec2/responses/security_groups.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/ec2/responses/security_groups.py b/moto/ec2/responses/security_groups.py index eec27c3aa..9a9aaafd9 100644 --- a/moto/ec2/responses/security_groups.py +++ b/moto/ec2/responses/security_groups.py @@ -9,9 +9,9 @@ def process_rules_from_querystring(querystring): except: group_name_or_id = querystring.get('GroupId')[0] - ip_protocol = querystring.get('IpPermissions.1.IpProtocol')[0] - from_port = querystring.get('IpPermissions.1.FromPort')[0] - to_port = querystring.get('IpPermissions.1.ToPort')[0] + ip_protocol = querystring.get('IpPermissions.1.IpProtocol', [None])[0] + from_port = querystring.get('IpPermissions.1.FromPort', [None])[0] + to_port = querystring.get('IpPermissions.1.ToPort', [None])[0] ip_ranges = [] for key, value in querystring.items(): if 'IpPermissions.1.IpRanges' in key: From ac1bb336c8e645b6e61f06722bfe303936567c17 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 30 Oct 2015 09:59:57 -0400 Subject: [PATCH 29/55] firest draft of firehose support. --- moto/kinesis/models.py | 91 +++++++++++++++++++ moto/kinesis/responses.py | 74 ++++++++++++++++ moto/kinesis/urls.py | 1 + tests/test_kinesis/test_firehose.py | 132 ++++++++++++++++++++++++++++ 4 files changed, 298 insertions(+) create mode 100644 tests/test_kinesis/test_firehose.py diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index 0b01881a4..fed8214d2 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -1,5 +1,8 @@ from __future__ import unicode_literals +import datetime +import time + import boto.kinesis from moto.compat import OrderedDict from moto.core import BaseBackend @@ -124,10 +127,78 @@ class Stream(object): } +class FirehoseRecord(object): + def __init__(self, record_data): + self.record_id = 12345678 + self.record_data = record_data + + +class DeliveryStream(object): + def __init__(self, stream_name, **stream_kwargs): + self.name = stream_name + self.redshift_username = stream_kwargs['redshift_username'] + self.redshift_password = stream_kwargs['redshift_password'] + self.redshift_jdbc_url = stream_kwargs['redshift_jdbc_url'] + self.redshift_role_arn = stream_kwargs['redshift_role_arn'] + self.redshift_copy_command = stream_kwargs['redshift_copy_command'] + + self.redshift_s3_role_arn = stream_kwargs['redshift_s3_role_arn'] + self.redshift_s3_bucket_arn = stream_kwargs['redshift_s3_bucket_arn'] + self.redshift_s3_prefix = stream_kwargs['redshift_s3_prefix'] + self.redshift_s3_compression_format = stream_kwargs['redshift_s3_compression_format'] + self.redshift_s3_buffering_hings = stream_kwargs['redshift_s3_buffering_hings'] + + self.records = [] + self.status = 'ACTIVE' + self.create_at = datetime.datetime.utcnow() + self.last_updated = datetime.datetime.utcnow() + + @property + def arn(self): + return 'arn:aws:firehose:us-east-1:123456789012:deliverystream/{0}'.format(self.name) + + def to_dict(self): + return { + "DeliveryStreamDescription": { + "CreateTimestamp": time.mktime(self.create_at.timetuple()), + "DeliveryStreamARN": self.arn, + "DeliveryStreamName": self.name, + "DeliveryStreamStatus": self.status, + "Destinations": [ + { + "DestinationId": "string", + "RedshiftDestinationDescription": { + "ClusterJDBCURL": self.redshift_jdbc_url, + "CopyCommand": self.redshift_copy_command, + "RoleARN": self.redshift_role_arn, + "S3DestinationDescription": { + "BucketARN": self.redshift_s3_bucket_arn, + "BufferingHints": self.redshift_s3_buffering_hings, + "CompressionFormat": self.redshift_s3_compression_format, + "Prefix": self.redshift_s3_prefix, + "RoleARN": self.redshift_s3_role_arn + }, + "Username": self.redshift_username, + }, + } + ], + "HasMoreDestinations": False, + "LastUpdateTimestamp": time.mktime(self.last_updated.timetuple()), + "VersionId": "string", + } + } + + def put_record(self, record_data): + record = FirehoseRecord(record_data) + self.records.append(record) + return record + + class KinesisBackend(BaseBackend): def __init__(self): self.streams = {} + self.delivery_streams = {} def create_stream(self, stream_name, shard_count, region): stream = Stream(stream_name, shard_count, region) @@ -180,6 +251,26 @@ class KinesisBackend(BaseBackend): return sequence_number, shard_id + ''' Firehose ''' + def create_delivery_stream(self, stream_name, **stream_kwargs): + stream = DeliveryStream(stream_name, **stream_kwargs) + self.delivery_streams[stream_name] = stream + return stream + + def get_delivery_stream(self, stream_name): + return self.delivery_streams[stream_name] + + def list_delivery_streams(self): + return self.delivery_streams.values() + + def delete_delivery_stream(self, stream_name): + self.delivery_streams.pop(stream_name) + + def put_firehose_record(self, stream_name, record_data): + stream = self.get_delivery_stream(stream_name) + record = stream.put_record(record_data) + return record + kinesis_backends = {} for region in boto.kinesis.regions(): kinesis_backends[region.name] = KinesisBackend() diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index 4b5f13729..51ddc8d50 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -16,6 +16,10 @@ class KinesisResponse(BaseResponse): def kinesis_backend(self): return kinesis_backends[self.region] + @property + def is_firehose(self): + return self.headers['host'].startswith('firehose') + def create_stream(self): stream_name = self.parameters.get('StreamName') shard_count = self.parameters.get('ShardCount') @@ -67,6 +71,8 @@ class KinesisResponse(BaseResponse): }) def put_record(self): + if self.is_firehose: + return self.firehose_put_record() stream_name = self.parameters.get("StreamName") partition_key = self.parameters.get("PartitionKey") explicit_hash_key = self.parameters.get("ExplicitHashKey") @@ -81,3 +87,71 @@ class KinesisResponse(BaseResponse): "SequenceNumber": sequence_number, "ShardId": shard_id, }) + + ''' Firehose ''' + def create_delivery_stream(self): + stream_name = self.parameters['DeliveryStreamName'] + redshift_config = self.parameters.get('RedshiftDestinationConfiguration') + + if redshift_config: + redshift_s3_config = redshift_config['S3Configuration'] + stream_kwargs = { + 'redshift_username': redshift_config['Username'], + 'redshift_password': redshift_config['Password'], + 'redshift_jdbc_url': redshift_config['ClusterJDBCURL'], + 'redshift_role_arn': redshift_config['RoleARN'], + 'redshift_copy_command': redshift_config['CopyCommand'], + + 'redshift_s3_role_arn': redshift_s3_config['RoleARN'], + 'redshift_s3_bucket_arn': redshift_s3_config['BucketARN'], + 'redshift_s3_prefix': redshift_s3_config['Prefix'], + 'redshift_s3_compression_format': redshift_s3_config['CompressionFormat'], + 'redshift_s3_buffering_hings': redshift_s3_config['BufferingHints'], + } + stream = self.kinesis_backend.create_delivery_stream(stream_name, **stream_kwargs) + return json.dumps({ + 'DeliveryStreamARN': stream.arn + }) + + def describe_delivery_stream(self): + stream_name = self.parameters["DeliveryStreamName"] + stream = self.kinesis_backend.get_delivery_stream(stream_name) + return json.dumps(stream.to_dict()) + + def list_delivery_streams(self): + streams = self.kinesis_backend.list_delivery_streams() + return json.dumps({ + "DeliveryStreamNames": [ + stream.name for stream in streams + ], + "HasMoreDeliveryStreams": False + }) + + def delete_delivery_stream(self): + stream_name = self.parameters['DeliveryStreamName'] + self.kinesis_backend.delete_delivery_stream(stream_name) + return json.dumps({}) + + def firehose_put_record(self): + stream_name = self.parameters['DeliveryStreamName'] + record_data = self.parameters['Record']['Data'] + + record = self.kinesis_backend.put_firehose_record(stream_name, record_data) + return json.dumps({ + "RecordId": record.record_id, + }) + + def put_record_batch(self): + stream_name = self.parameters['DeliveryStreamName'] + records = self.parameters['Records'] + + request_responses = [] + for record in records: + record_response = self.kinesis_backend.put_firehose_record(stream_name, record['Data']) + request_responses.append({ + "RecordId": record_response.record_id + }) + return json.dumps({ + "FailedPutCount": 0, + "RequestResponses": request_responses, + }) diff --git a/moto/kinesis/urls.py b/moto/kinesis/urls.py index 5de870c29..a8d15eecd 100644 --- a/moto/kinesis/urls.py +++ b/moto/kinesis/urls.py @@ -3,6 +3,7 @@ from .responses import KinesisResponse url_bases = [ "https?://kinesis.(.+).amazonaws.com", + "https?://firehose.(.+).amazonaws.com", ] url_paths = { diff --git a/tests/test_kinesis/test_firehose.py b/tests/test_kinesis/test_firehose.py new file mode 100644 index 000000000..3f5e7e8dd --- /dev/null +++ b/tests/test_kinesis/test_firehose.py @@ -0,0 +1,132 @@ +from __future__ import unicode_literals + +import datetime + +import boto3 +from freezegun import freeze_time +import sure # noqa + +from moto import mock_kinesis + + +def create_stream(client, stream_name): + return client.create_delivery_stream( + DeliveryStreamName=stream_name, + RedshiftDestinationConfiguration={ + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'ClusterJDBCURL': 'jdbc:redshift://host.amazonaws.com:5439/database', + 'CopyCommand': { + 'DataTableName': 'outputTable', + 'CopyOptions': "CSV DELIMITER ',' NULL '\\0'" + }, + 'Username': 'username', + 'Password': 'password', + 'S3Configuration': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'BucketARN': 'arn:aws:s3:::kinesis-test', + 'Prefix': 'myFolder/', + 'BufferingHints': { + 'SizeInMBs': 123, + 'IntervalInSeconds': 124 + }, + 'CompressionFormat': 'UNCOMPRESSED', + } + } + ) + + +@mock_kinesis +@freeze_time("2015-03-01") +def test_create_stream(): + client = boto3.client('firehose', region_name='us-east-1') + + response = create_stream(client, 'stream1') + stream_arn = response['DeliveryStreamARN'] + + response = client.describe_delivery_stream(DeliveryStreamName='stream1') + stream_description = response['DeliveryStreamDescription'] + + # Sure and Freezegun don't play nicely together + created = stream_description.pop('CreateTimestamp') + last_updated = stream_description.pop('LastUpdateTimestamp') + from dateutil.tz import tzlocal + assert created == datetime.datetime(2015, 3, 1, tzinfo=tzlocal()) + assert last_updated == datetime.datetime(2015, 3, 1, tzinfo=tzlocal()) + + stream_description.should.equal({ + 'DeliveryStreamName': 'stream1', + 'DeliveryStreamARN': stream_arn, + 'DeliveryStreamStatus': 'ACTIVE', + 'VersionId': 'string', + 'Destinations': [ + { + 'DestinationId': 'string', + 'RedshiftDestinationDescription': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'ClusterJDBCURL': 'jdbc:redshift://host.amazonaws.com:5439/database', + 'CopyCommand': { + 'DataTableName': 'outputTable', + 'CopyOptions': "CSV DELIMITER ',' NULL '\\0'" + }, + 'Username': 'username', + 'S3DestinationDescription': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'BucketARN': 'arn:aws:s3:::kinesis-test', + 'Prefix': 'myFolder/', + 'BufferingHints': { + 'SizeInMBs': 123, + 'IntervalInSeconds': 124 + }, + 'CompressionFormat': 'UNCOMPRESSED', + } + } + }, + ], + "HasMoreDestinations": False, + }) + + +@mock_kinesis +@freeze_time("2015-03-01") +def test_list_and_delete_stream(): + client = boto3.client('firehose', region_name='us-east-1') + + create_stream(client, 'stream1') + create_stream(client, 'stream2') + + set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal({'stream1', 'stream2'}) + + client.delete_delivery_stream(DeliveryStreamName='stream1') + + set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal({'stream2'}) + + +@mock_kinesis +def test_put_record(): + client = boto3.client('firehose', region_name='us-east-1') + + create_stream(client, 'stream1') + client.put_record( + DeliveryStreamName='stream1', + Record={ + 'Data': 'some data' + } + ) + + +@mock_kinesis +def test_put_record_batch(): + client = boto3.client('firehose', region_name='us-east-1') + + create_stream(client, 'stream1') + client.put_record_batch( + DeliveryStreamName='stream1', + Records=[ + { + 'Data': 'some data1' + }, + { + 'Data': 'some data2' + }, + ] + ) From fe2126b7278568ba2026d78afbb3238ea411ea47 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 30 Oct 2015 10:04:08 -0400 Subject: [PATCH 30/55] py26 fix --- tests/test_kinesis/test_firehose.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_kinesis/test_firehose.py b/tests/test_kinesis/test_firehose.py index 3f5e7e8dd..c22562847 100644 --- a/tests/test_kinesis/test_firehose.py +++ b/tests/test_kinesis/test_firehose.py @@ -94,11 +94,11 @@ def test_list_and_delete_stream(): create_stream(client, 'stream1') create_stream(client, 'stream2') - set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal({'stream1', 'stream2'}) + set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal(set(['stream1', 'stream2'])) client.delete_delivery_stream(DeliveryStreamName='stream1') - set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal({'stream2'}) + set(client.list_delivery_streams()['DeliveryStreamNames']).should.equal(set(['stream2'])) @mock_kinesis From 015e7ea9a2d6414998250e78b510d3af0749e849 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 30 Oct 2015 10:05:53 -0400 Subject: [PATCH 31/55] py3 fix. --- moto/kinesis/responses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index 51ddc8d50..b3fa98743 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -18,7 +18,8 @@ class KinesisResponse(BaseResponse): @property def is_firehose(self): - return self.headers['host'].startswith('firehose') + host = self.headers.get('hose', self.headers['Host']) + return host.startswith('firehose') def create_stream(self): stream_name = self.parameters.get('StreamName') From 127625bdc3ebf808d678acbb8db34d4f69d5b839 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 30 Oct 2015 10:10:49 -0400 Subject: [PATCH 32/55] fix typo. --- moto/kinesis/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index b3fa98743..d238f75dd 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -18,7 +18,7 @@ class KinesisResponse(BaseResponse): @property def is_firehose(self): - host = self.headers.get('hose', self.headers['Host']) + host = self.headers.get('host', self.headers['Host']) return host.startswith('firehose') def create_stream(self): From 7fcf84b32aab8ba1245728ae5be8e84e202ca913 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 30 Oct 2015 10:13:57 -0400 Subject: [PATCH 33/55] easier fallback. --- moto/kinesis/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index d238f75dd..839bf73e1 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -18,7 +18,7 @@ class KinesisResponse(BaseResponse): @property def is_firehose(self): - host = self.headers.get('host', self.headers['Host']) + host = self.headers.get('host') or self.headers['Host'] return host.startswith('firehose') def create_stream(self): From 1b1cf40af8e1a6cd085a10d19370eade924a7e67 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 30 Oct 2015 14:18:29 -0400 Subject: [PATCH 34/55] handle optional compression format. --- moto/kinesis/models.py | 2 +- moto/kinesis/responses.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index fed8214d2..a26e45af7 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -145,7 +145,7 @@ class DeliveryStream(object): self.redshift_s3_role_arn = stream_kwargs['redshift_s3_role_arn'] self.redshift_s3_bucket_arn = stream_kwargs['redshift_s3_bucket_arn'] self.redshift_s3_prefix = stream_kwargs['redshift_s3_prefix'] - self.redshift_s3_compression_format = stream_kwargs['redshift_s3_compression_format'] + self.redshift_s3_compression_format = stream_kwargs.get('redshift_s3_compression_format', 'UNCOMPRESSED') self.redshift_s3_buffering_hings = stream_kwargs['redshift_s3_buffering_hings'] self.records = [] diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index 839bf73e1..5b8c7be06 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -106,7 +106,7 @@ class KinesisResponse(BaseResponse): 'redshift_s3_role_arn': redshift_s3_config['RoleARN'], 'redshift_s3_bucket_arn': redshift_s3_config['BucketARN'], 'redshift_s3_prefix': redshift_s3_config['Prefix'], - 'redshift_s3_compression_format': redshift_s3_config['CompressionFormat'], + 'redshift_s3_compression_format': redshift_s3_config.get('CompressionFormat'), 'redshift_s3_buffering_hings': redshift_s3_config['BufferingHints'], } stream = self.kinesis_backend.create_delivery_stream(stream_name, **stream_kwargs) From fcaa8fbce714c311d6e54c426ad49f52b983d104 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 2 Nov 2015 10:09:17 -0500 Subject: [PATCH 35/55] 0.4.16 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 25031f0ed..abfb476c1 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.15' +__version__ = '0.4.16' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index c82d53208..4bcee21ea 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.15', + version='0.4.16', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 99a4bcf98fc662fd150caab45fd299d34fc76413 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 2 Nov 2015 10:11:14 -0500 Subject: [PATCH 36/55] 0.4.17 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index abfb476c1..68755bdce 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.16' +__version__ = '0.4.17' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index 4bcee21ea..ac6509b5b 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.16', + version='0.4.17', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From d9faab3e5ed8511e7df913a735e865723a165f3c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 2 Nov 2015 13:25:31 -0500 Subject: [PATCH 37/55] Fix error for describing kinesis stream that has not been created. --- moto/kinesis/models.py | 5 ++++- tests/test_kinesis/test_firehose.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index a26e45af7..d2d0d2913 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -258,7 +258,10 @@ class KinesisBackend(BaseBackend): return stream def get_delivery_stream(self, stream_name): - return self.delivery_streams[stream_name] + if stream_name in self.delivery_streams: + return self.delivery_streams[stream_name] + else: + raise StreamNotFoundError(stream_name) def list_delivery_streams(self): return self.delivery_streams.values() diff --git a/tests/test_kinesis/test_firehose.py b/tests/test_kinesis/test_firehose.py index c22562847..37585fe5e 100644 --- a/tests/test_kinesis/test_firehose.py +++ b/tests/test_kinesis/test_firehose.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime +from botocore.exceptions import ClientError import boto3 from freezegun import freeze_time import sure # noqa @@ -86,6 +87,14 @@ def test_create_stream(): }) +@mock_kinesis +@freeze_time("2015-03-01") +def test_deescribe_non_existant_stream(): + client = boto3.client('firehose', region_name='us-east-1') + + client.describe_delivery_stream.when.called_with(DeliveryStreamName='not-a-stream').should.throw(ClientError) + + @mock_kinesis @freeze_time("2015-03-01") def test_list_and_delete_stream(): From 73452c79f7b7b6ac79459a8e5cbd9cb3994f7273 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 2 Nov 2015 23:33:39 -0500 Subject: [PATCH 38/55] Add milliseconds to EC2 launch time. Closes #445. --- moto/ec2/models.py | 2 +- tests/test_ec2/test_instances.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index a7c0133c7..0a2c3ffb7 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -98,7 +98,7 @@ from .utils import ( def utc_date_and_time(): - return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.000Z') def validate_resource_ids(resource_ids): diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 629fc67ee..2e9b9834a 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -53,7 +53,7 @@ def test_instance_launch_and_terminate(): instances.should.have.length_of(1) instances[0].id.should.equal(instance.id) instances[0].state.should.equal('running') - instances[0].launch_time.should.equal("2014-01-01T05:00:00Z") + instances[0].launch_time.should.equal("2014-01-01T05:00:00.000Z") instances[0].vpc_id.should.equal(None) root_device_name = instances[0].root_device_name From d3e4c2c4b5bf900ac1c32cbb8469ef718e14c807 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 3 Nov 2015 09:25:47 -0500 Subject: [PATCH 39/55] Add ability for specific backends to enable template escaping. Closes #441. --- moto/core/responses.py | 14 +++++++++++--- moto/ec2/responses/__init__.py | 4 ++++ moto/ec2/responses/tags.py | 3 --- moto/glacier/responses.py | 1 + moto/s3/responses.py | 1 + tests/test_ec2/test_elastic_block_store.py | 10 ++++++++++ 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index bd54cf01c..5fda815e7 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -63,8 +63,16 @@ class DynamicDictLoader(DictLoader): class _TemplateEnvironmentMixin(object): - loader = DynamicDictLoader({}) - environment = Environment(loader=loader) + + def __init__(self): + super(_TemplateEnvironmentMixin, self).__init__() + self.loader = DynamicDictLoader({}) + self.environment = Environment(loader=self.loader, autoescape=self.should_autoescape) + + @property + def should_autoescape(self): + # Allow for subclass to overwrite + return False def contains_template(self, template_id): return self.loader.contains(template_id) @@ -73,7 +81,7 @@ class _TemplateEnvironmentMixin(object): template_id = id(source) if not self.contains_template(template_id): self.loader.update({template_id: source}) - self.environment = Environment(loader=self.loader) + self.environment = Environment(loader=self.loader, autoescape=self.should_autoescape) return self.environment.get_template(template_id) diff --git a/moto/ec2/responses/__init__.py b/moto/ec2/responses/__init__.py index f051a7ca7..e51992e41 100644 --- a/moto/ec2/responses/__init__.py +++ b/moto/ec2/responses/__init__.py @@ -66,3 +66,7 @@ class EC2Response( def ec2_backend(self): from moto.ec2.models import ec2_backends return ec2_backends[self.region] + + @property + def should_autoescape(self): + return True diff --git a/moto/ec2/responses/tags.py b/moto/ec2/responses/tags.py index effb4faf9..4a62261d5 100644 --- a/moto/ec2/responses/tags.py +++ b/moto/ec2/responses/tags.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from xml.sax.saxutils import escape from moto.core.responses import BaseResponse from moto.ec2.models import validate_resource_ids from moto.ec2.utils import sequence_from_querystring, tags_from_query_string, filters_from_querystring @@ -26,8 +25,6 @@ class TagResponse(BaseResponse): def describe_tags(self): filters = filters_from_querystring(querystring_dict=self.querystring) tags = self.ec2_backend.describe_tags(filters=filters) - for tag in tags: - tag['value'] = escape(tag['value']) template = self.response_template(DESCRIBE_RESPONSE) return template.render(tags=tags) diff --git a/moto/glacier/responses.py b/moto/glacier/responses.py index 37cbdc4c9..eac9b94c6 100644 --- a/moto/glacier/responses.py +++ b/moto/glacier/responses.py @@ -11,6 +11,7 @@ from .utils import region_from_glacier_url, vault_from_glacier_url class GlacierResponse(_TemplateEnvironmentMixin): def __init__(self, backend): + super(GlacierResponse, self).__init__() self.backend = backend @classmethod diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 687b0464f..42ad92715 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,6 +24,7 @@ def parse_key_name(pth): class ResponseObject(_TemplateEnvironmentMixin): def __init__(self, backend, bucket_name_from_url, parse_key_name, is_delete_keys=None): + super(ResponseObject, self).__init__() self.backend = backend self.bucket_name_from_url = bucket_name_from_url self.parse_key_name = parse_key_name diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 0fd5d6bc2..b2308fe04 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -236,3 +236,13 @@ def test_modify_attribute_blockDeviceMapping(): instance = ec2_backends[conn.region.name].get_instance(instance.id) instance.block_device_mapping.should.have.key('/dev/sda1') instance.block_device_mapping['/dev/sda1'].delete_on_termination.should.be(True) + + +@mock_ec2 +def test_volume_tag_escaping(): + conn = boto.connect_ec2('the_key', 'the_secret') + vol = conn.create_volume(10, 'us-east-1a') + snapshot = conn.create_snapshot(vol.id, 'Desc') + snapshot.add_tags({'key': ''}) + + dict(conn.get_all_snapshots()[0].tags).should.equal({'key': ''}) From cddf139bbca5d8f8c8fb941b1b7a3e09f8ffa8f8 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 3 Nov 2015 09:37:02 -0500 Subject: [PATCH 40/55] Add ability to create EBS volumes from snapshots. Closes #447. --- moto/ec2/models.py | 11 ++++++++--- moto/ec2/responses/elastic_block_store.py | 19 ++++++++++++++----- tests/test_ec2/test_elastic_block_store.py | 17 +++++++++++++++++ 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 0a2c3ffb7..446561d33 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1388,12 +1388,13 @@ class VolumeAttachment(object): class Volume(TaggedEC2Resource): - def __init__(self, ec2_backend, volume_id, size, zone): + def __init__(self, ec2_backend, volume_id, size, zone, snapshot_id=None): self.id = volume_id self.size = size self.zone = zone self.create_time = utc_date_and_time() self.attachment = None + self.snapshot_id = snapshot_id self.ec2_backend = ec2_backend @classmethod @@ -1436,10 +1437,14 @@ class EBSBackend(object): self.snapshots = {} super(EBSBackend, self).__init__() - def create_volume(self, size, zone_name): + def create_volume(self, size, zone_name, snapshot_id=None): volume_id = random_volume_id() zone = self.get_zone_by_name(zone_name) - volume = Volume(self, volume_id, size, zone) + if snapshot_id: + snapshot = self.get_snapshot(snapshot_id) + if size is None: + size = snapshot.volume.size + volume = Volume(self, volume_id, size, zone, snapshot_id) self.volumes[volume_id] = volume return volume diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index 876766b2f..5adb4c7d0 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -25,9 +25,10 @@ class ElasticBlockStore(BaseResponse): return template.render(snapshot=snapshot) def create_volume(self): - size = self.querystring.get('Size')[0] - zone = self.querystring.get('AvailabilityZone')[0] - volume = self.ec2_backend.create_volume(size, zone) + size = self._get_param('Size') + zone = self._get_param('AvailabilityZone') + snapshot_id = self._get_param('SnapshotId') + volume = self.ec2_backend.create_volume(size, zone, snapshot_id) template = self.response_template(CREATE_VOLUME_RESPONSE) return template.render(volume=volume) @@ -110,7 +111,11 @@ CREATE_VOLUME_RESPONSE = """ Date: Tue, 3 Nov 2015 14:01:09 -0500 Subject: [PATCH 41/55] 0.4.18 --- moto/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 68755bdce..be7cfdda3 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '0.4.17' +__version__ = '0.4.18' from .autoscaling import mock_autoscaling # flake8: noqa from .cloudformation import mock_cloudformation # flake8: noqa diff --git a/setup.py b/setup.py index ac6509b5b..a65f5e15e 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ extras_require = { setup( name='moto', - version='0.4.17', + version='0.4.18', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 18d63a6cfe162ada40b6fe37929f7f1630c17f18 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 4 Nov 2015 18:55:41 -0500 Subject: [PATCH 42/55] Add basics of S# website configuration. Closes #442. --- moto/s3/models.py | 12 ++++++++++++ moto/s3/responses.py | 14 ++++++++++---- tests/test_s3/test_s3.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 83412a3f9..0375775cf 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -244,6 +244,7 @@ class FakeBucket(object): self.versioning_status = None self.rules = [] self.policy = None + self.website_configuration = None @property def location(self): @@ -272,6 +273,9 @@ class FakeBucket(object): def delete_lifecycle(self): self.rules = [] + def set_website_configuration(self, website_configuration): + self.website_configuration = website_configuration + def get_cfn_attribute(self, attribute_name): from moto.cloudformation.exceptions import UnformattedGetAttTemplateException if attribute_name == 'DomainName': @@ -343,6 +347,14 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) bucket.set_lifecycle(rules) + def set_bucket_website_configuration(self, bucket_name, website_configuration): + bucket = self.get_bucket(bucket_name) + bucket.set_website_configuration(website_configuration) + + def get_bucket_website_configuration(self, bucket_name): + bucket = self.get_bucket(bucket_name) + return bucket.website_configuration + def set_key(self, bucket_name, key_name, value, storage=None, etag=None): key_name = clean_key_name(key_name) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 42ad92715..ff174854f 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -102,29 +102,32 @@ class ResponseObject(_TemplateEnvironmentMixin): prefix = querystring.get('prefix', [None])[0] multiparts = [upload for upload in multiparts if upload.key_name.startswith(prefix)] template = self.response_template(S3_ALL_MULTIPARTS) - return 200, headers, template.render( + return template.render( bucket_name=bucket_name, uploads=multiparts) elif 'location' in querystring: bucket = self.backend.get_bucket(bucket_name) template = self.response_template(S3_BUCKET_LOCATION) - return 200, headers, template.render(location=bucket.location) + return template.render(location=bucket.location) elif 'lifecycle' in querystring: bucket = self.backend.get_bucket(bucket_name) if not bucket.rules: return 404, headers, "NoSuchLifecycleConfiguration" template = self.response_template(S3_BUCKET_LIFECYCLE_CONFIGURATION) - return 200, headers, template.render(rules=bucket.rules) + return template.render(rules=bucket.rules) elif 'versioning' in querystring: versioning = self.backend.get_bucket_versioning(bucket_name) template = self.response_template(S3_BUCKET_GET_VERSIONING) - return 200, headers, template.render(status=versioning) + return template.render(status=versioning) elif 'policy' in querystring: policy = self.backend.get_bucket_policy(bucket_name) if not policy: template = self.response_template(S3_NO_POLICY) return 404, headers, template.render(bucket_name=bucket_name) return 200, headers, policy + elif 'website' in querystring: + website_configuration = self.backend.get_bucket_website_configuration(bucket_name) + return website_configuration elif 'versions' in querystring: delimiter = querystring.get('delimiter', [None])[0] encoding_type = querystring.get('encoding-type', [None])[0] @@ -184,6 +187,9 @@ class ResponseObject(_TemplateEnvironmentMixin): elif 'policy' in querystring: self.backend.set_bucket_policy(bucket_name, body) return 'True' + elif 'website' in querystring: + self.backend.set_bucket_website_configuration(bucket_name, body) + return "" else: try: new_bucket = self.backend.create_bucket(bucket_name, region_name) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 0434c0d65..b4bd0e880 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -945,5 +945,33 @@ def test_boto3_head_object(): s3.Object('blah', 'hello.txt').meta.client.head_object(Bucket='blah', Key='hello.txt') - with assert_raises(ClientError) as err: + with assert_raises(ClientError): s3.Object('blah', 'hello2.txt').meta.client.head_object(Bucket='blah', Key='hello_bad.txt') + + +TEST_XML = """\ + + + + index.html + + + + + test/testing + + + test.txt + + + + +""" + + +@mock_s3 +def test_website_configuration_xml(): + conn = boto.connect_s3() + bucket = conn.create_bucket('test-bucket') + bucket.set_website_configuration_xml(TEST_XML) + bucket.get_website_configuration_xml().should.equal(TEST_XML) From 8d41d0019b7d60f9eb9681c7cc963aee45d4301f Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 7 Nov 2015 16:45:24 -0500 Subject: [PATCH 43/55] Add basic support for AttributeUpdates in Dynamo update_item. Closes #449. --- moto/dynamodb2/models.py | 19 +++++++++++-- moto/dynamodb2/responses.py | 5 ++-- .../test_dynamodb_table_without_range_key.py | 27 +++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index e7277ee6b..4832d7944 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -121,6 +121,14 @@ class Item(object): # TODO deal with other types self.attrs[key] = DynamoType({"S": value}) + def update_with_attribute_updates(self, attribute_updates): + for attribute_name, update_action in attribute_updates.items(): + action = update_action['Action'] + new_value = update_action['Value'].values()[0] + if action == 'PUT': + # TODO deal with other types + self.attrs[attribute_name] = DynamoType({"S": new_value}) + class Table(object): @@ -411,12 +419,19 @@ class DynamoDBBackend(BaseBackend): return table.scan(scan_filters) - def update_item(self, table_name, key, update_expression): + def update_item(self, table_name, key, update_expression, attribute_updates): table = self.get_table(table_name) + if table.hash_key_attr in key: + # Sometimes the key is wrapped in a dict with the key name + key = key[table.hash_key_attr] + hash_value = DynamoType(key) item = table.get_item(hash_value) - item.update(update_expression) + if update_expression: + item.update(update_expression) + else: + item.update_with_attribute_updates(attribute_updates) return item def delete_item(self, table_name, keys): diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 57d06bbf3..2be0dda8f 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -373,8 +373,9 @@ class DynamoHandler(BaseResponse): def update_item(self): name = self.body['TableName'] key = self.body['Key'] - update_expression = self.body['UpdateExpression'] - item = dynamodb_backend2.update_item(name, key, update_expression) + update_expression = self.body.get('UpdateExpression') + attribute_updates = self.body.get('AttributeUpdates') + item = dynamodb_backend2.update_item(name, key, update_expression, attribute_updates) item_dict = item.to_json() item_dict['ConsumedCapacityUnits'] = 0.5 diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 808805b8d..6baeb8a12 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -122,6 +122,33 @@ def test_item_add_and_describe_and_update(): }) +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_item_partial_save(): + table = create_table() + + data = { + 'forum_name': 'LOLCat Forum', + 'Body': 'http://url_to_lolcat.gif', + 'SentBy': 'User A', + } + + table.put_item(data=data) + returned_item = table.get_item(forum_name="LOLCat Forum") + + returned_item['SentBy'] = 'User B' + returned_item.partial_save() + + returned_item = table.get_item( + forum_name='LOLCat Forum' + ) + dict(returned_item).should.equal({ + 'forum_name': 'LOLCat Forum', + 'Body': 'http://url_to_lolcat.gif', + 'SentBy': 'User B', + }) + + @requires_boto_gte("2.9") @mock_dynamodb2 def test_item_put_without_table(): From ab3682a55c1c4f2983819e73a8cc59c0159a33f3 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 7 Nov 2015 16:58:39 -0500 Subject: [PATCH 44/55] py3 fix. --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 4832d7944..8266d5588 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -124,7 +124,7 @@ class Item(object): def update_with_attribute_updates(self, attribute_updates): for attribute_name, update_action in attribute_updates.items(): action = update_action['Action'] - new_value = update_action['Value'].values()[0] + new_value = list(update_action['Value'].values())[0] if action == 'PUT': # TODO deal with other types self.attrs[attribute_name] = DynamoType({"S": new_value}) From 5b2a7242198c138f67d5342bb4ffa7c08e55d8c8 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Mon, 9 Nov 2015 19:29:08 -0500 Subject: [PATCH 45/55] Check SQS message size --- moto/sqs/exceptions.py | 5 +++++ moto/sqs/models.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py index d72cfdffc..c9d0d73c5 100644 --- a/moto/sqs/exceptions.py +++ b/moto/sqs/exceptions.py @@ -11,6 +11,11 @@ class ReceiptHandleIsInvalid(Exception): status_code = 400 +class InvalidParameterValue(Exception): + description = "One or more parameters are invalid. Reason: Message must be shorter than 262144 bytes." + status_code = 400 + + class MessageAttributesInvalid(Exception): status_code = 400 diff --git a/moto/sqs/models.py b/moto/sqs/models.py index efb75dd9c..725465498 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -12,7 +12,8 @@ from moto.core.utils import camelcase_to_underscores, get_random_message_id from .utils import generate_receipt_handle, unix_time_millis from .exceptions import ( ReceiptHandleIsInvalid, - MessageNotInflight + MessageNotInflight, + InvalidParameterValue, ) DEFAULT_ACCOUNT_ID = 123456789012 @@ -251,6 +252,9 @@ class SQSBackend(BaseBackend): else: delay_seconds = queue.delay_seconds + if len(message_body) > self.maximum_message_size: + raise InvalidParameterValue + message_id = get_random_message_id() message = Message(message_id, message_body) From 540ee79ad9c024015936efe9fae26663ba764c7f Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Mon, 9 Nov 2015 19:51:17 -0500 Subject: [PATCH 46/55] Put the size check in the queue --- moto/sqs/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 725465498..17c33e103 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -190,6 +190,9 @@ class Queue(object): return [message for message in self._messages if message.visible and not message.delayed] def add_message(self, message): + if len(message) > self.maximum_message_size: + raise InvalidParameterValue() + self._messages.append(message) def get_cfn_attribute(self, attribute_name): @@ -252,9 +255,6 @@ class SQSBackend(BaseBackend): else: delay_seconds = queue.delay_seconds - if len(message_body) > self.maximum_message_size: - raise InvalidParameterValue - message_id = get_random_message_id() message = Message(message_id, message_body) From b732e116a4416c008211f214018ba520672ae8c1 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Mon, 9 Nov 2015 19:53:14 -0500 Subject: [PATCH 47/55] Try the length of the body --- moto/sqs/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 17c33e103..b5a64874d 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -190,7 +190,7 @@ class Queue(object): return [message for message in self._messages if message.visible and not message.delayed] def add_message(self, message): - if len(message) > self.maximum_message_size: + if len(message.body) > self.maximum_message_size: raise InvalidParameterValue() self._messages.append(message) From f8cbcfc098b760ac58c6cb87516235d058ffa548 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Mon, 9 Nov 2015 20:03:52 -0500 Subject: [PATCH 48/55] How about in the SQSResponse object --- moto/sqs/models.py | 4 ---- moto/sqs/responses.py | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index b5a64874d..653c5316a 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -121,7 +121,6 @@ class Queue(object): self.created_timestamp = now self.delay_seconds = 0 self.last_modified_timestamp = now - self.maximum_message_size = 64 << 10 self.message_retention_period = 86400 * 4 # four days self.queue_arn = 'arn:aws:sqs:sqs.us-east-1:123456789012:%s' % self.name self.receive_message_wait_time_seconds = 0 @@ -190,9 +189,6 @@ class Queue(object): return [message for message in self._messages if message.visible and not message.delayed] def add_message(self, message): - if len(message.body) > self.maximum_message_size: - raise InvalidParameterValue() - self._messages.append(message) def get_cfn_attribute(self, attribute_name): diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index abae83fed..82d24cb6a 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -11,6 +11,7 @@ from .exceptions import ( ) MAXIMUM_VISIBILTY_TIMEOUT = 43200 +MAXIMUM_MESSAGE_LENGTH = 262144 # 256 KiB DEFAULT_RECEIVED_MESSAGES = 1 SQS_REGION_REGEX = r'://(.+?)\.queue\.amazonaws\.com' @@ -106,6 +107,9 @@ class SQSResponse(BaseResponse): message = self.querystring.get("MessageBody")[0] delay_seconds = self.querystring.get('DelaySeconds') + if len(message) > MAXIMUM_MESSAGE_LENGTH: + return "One or more parameters are invalid. Reason: Message must be shorter than 262144 bytes.", dict(status=400) + if delay_seconds: delay_seconds = int(delay_seconds[0]) else: From 97b7781c13944344c93c5deca99f918709d25bf1 Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Mon, 9 Nov 2015 20:09:08 -0500 Subject: [PATCH 49/55] Include XML response template --- moto/sqs/responses.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 82d24cb6a..d1ba5b6dd 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -108,7 +108,7 @@ class SQSResponse(BaseResponse): delay_seconds = self.querystring.get('DelaySeconds') if len(message) > MAXIMUM_MESSAGE_LENGTH: - return "One or more parameters are invalid. Reason: Message must be shorter than 262144 bytes.", dict(status=400) + return ERROR_TOO_LONG_RESPONSE, dict(status=400) if delay_seconds: delay_seconds = int(delay_seconds[0]) @@ -421,3 +421,13 @@ PURGE_QUEUE_RESPONSE = """ """ + +ERROR_TOO_LONG_RESPONSE = """ + + Sender + InvalidParameterValue + One or more parameters are invalid. Reason: Message must be shorter than 262144 bytes. + + + 6fde8d1e-52cd-4581-8cd9-c512f4c64223 +""" From 27e7767883153ce3aa17a9842da4fcf6c85b9f1d Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Mon, 9 Nov 2015 20:19:51 -0500 Subject: [PATCH 50/55] Remove stuff that doesn't need to change --- moto/sqs/exceptions.py | 5 ----- moto/sqs/models.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py index c9d0d73c5..d72cfdffc 100644 --- a/moto/sqs/exceptions.py +++ b/moto/sqs/exceptions.py @@ -11,11 +11,6 @@ class ReceiptHandleIsInvalid(Exception): status_code = 400 -class InvalidParameterValue(Exception): - description = "One or more parameters are invalid. Reason: Message must be shorter than 262144 bytes." - status_code = 400 - - class MessageAttributesInvalid(Exception): status_code = 400 diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 653c5316a..efb75dd9c 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -12,8 +12,7 @@ from moto.core.utils import camelcase_to_underscores, get_random_message_id from .utils import generate_receipt_handle, unix_time_millis from .exceptions import ( ReceiptHandleIsInvalid, - MessageNotInflight, - InvalidParameterValue, + MessageNotInflight ) DEFAULT_ACCOUNT_ID = 123456789012 @@ -121,6 +120,7 @@ class Queue(object): self.created_timestamp = now self.delay_seconds = 0 self.last_modified_timestamp = now + self.maximum_message_size = 64 << 10 self.message_retention_period = 86400 * 4 # four days self.queue_arn = 'arn:aws:sqs:sqs.us-east-1:123456789012:%s' % self.name self.receive_message_wait_time_seconds = 0 From a4e86494e13960fe604fa18bc92fbf17ec4390eb Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Tue, 10 Nov 2015 11:24:55 -0500 Subject: [PATCH 51/55] Add a test for a message that is too long --- tests/test_sqs/test_sqs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index a23545dcc..43ab21d5c 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -170,6 +170,18 @@ def test_send_message_with_delay(): queue.count().should.equal(0) +@mock_sqs +def test_send_large_message_fails(): + conn = boto.connect_sqs('the_key', 'the_secret') + queue = conn.create_queue("test-queue", visibility_timeout=60) + queue.set_message_class(RawMessage) + + body_one = 'test message' * 20000 + huge_message = queue.new_message(body_one) + + queue.write.when.called_with(huge_message).should.throw(SQSError) + + @mock_sqs def test_message_becomes_inflight_when_received(): conn = boto.connect_sqs('the_key', 'the_secret') From 5115e50bd063d4c4f2d9381ebcd39c7d711b8b2c Mon Sep 17 00:00:00 2001 From: Ian Dees Date: Tue, 10 Nov 2015 11:39:00 -0500 Subject: [PATCH 52/55] The message has to be bigger --- tests/test_sqs/test_sqs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 43ab21d5c..1e300fa57 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -176,7 +176,7 @@ def test_send_large_message_fails(): queue = conn.create_queue("test-queue", visibility_timeout=60) queue.set_message_class(RawMessage) - body_one = 'test message' * 20000 + body_one = 'test message' * 200000 huge_message = queue.new_message(body_one) queue.write.when.called_with(huge_message).should.throw(SQSError) From c38731ecbbfbbfc66fe41c0cf14d6d86ee17ebd2 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 11 Nov 2015 20:26:29 -0500 Subject: [PATCH 53/55] Add ACL support for S3 buckets. --- moto/s3/models.py | 13 +++++++++++++ moto/s3/responses.py | 17 +++++++++++++---- tests/test_s3/test_s3.py | 25 +++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 0375775cf..9e023f68d 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -245,6 +245,7 @@ class FakeBucket(object): self.rules = [] self.policy = None self.website_configuration = None + self.acl = get_canned_acl('private') @property def location(self): @@ -284,6 +285,9 @@ class FakeBucket(object): raise NotImplementedError('"Fn::GetAtt" : [ "{0}" , "WebsiteURL" ]"') raise UnformattedGetAttTemplateException() + def set_acl(self, acl): + self.acl = acl + class S3Backend(BaseBackend): @@ -484,4 +488,13 @@ class S3Backend(BaseBackend): if acl is not None: key.set_acl(acl) + def set_bucket_acl(self, bucket_name, acl): + bucket = self.get_bucket(bucket_name) + bucket.set_acl(acl) + + def get_bucket_acl(self, bucket_name): + bucket = self.get_bucket(bucket_name) + return bucket.acl + + s3_backend = S3Backend() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ff174854f..27a6d4536 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -80,7 +80,7 @@ class ResponseObject(_TemplateEnvironmentMixin): elif method == 'GET': return self._bucket_response_get(bucket_name, querystring, headers) elif method == 'PUT': - return self._bucket_response_put(body, region_name, bucket_name, querystring, headers) + return self._bucket_response_put(request, body, region_name, bucket_name, querystring, headers) elif method == 'DELETE': return self._bucket_response_delete(body, bucket_name, querystring, headers) elif method == 'POST': @@ -128,6 +128,10 @@ class ResponseObject(_TemplateEnvironmentMixin): elif 'website' in querystring: website_configuration = self.backend.get_bucket_website_configuration(bucket_name) return website_configuration + elif 'acl' in querystring: + bucket = self.backend.get_bucket(bucket_name) + template = self.response_template(S3_OBJECT_ACL_RESPONSE) + return template.render(obj=bucket) elif 'versions' in querystring: delimiter = querystring.get('delimiter', [None])[0] encoding_type = querystring.get('encoding-type', [None])[0] @@ -168,7 +172,7 @@ class ResponseObject(_TemplateEnvironmentMixin): result_folders=result_folders ) - def _bucket_response_put(self, body, region_name, bucket_name, querystring, headers): + def _bucket_response_put(self, request, body, region_name, bucket_name, querystring, headers): if 'versioning' in querystring: ver = re.search('([A-Za-z]+)', body) if ver: @@ -187,6 +191,11 @@ class ResponseObject(_TemplateEnvironmentMixin): elif 'policy' in querystring: self.backend.set_bucket_policy(bucket_name, body) return 'True' + elif 'acl' in querystring: + acl = self._acl_from_headers(request.headers) + # TODO: Support the XML-based ACL format + self.backend.set_bucket_acl(bucket_name, acl) + return "" elif 'website' in querystring: self.backend.set_bucket_website_configuration(bucket_name, body) return "" @@ -351,7 +360,7 @@ class ResponseObject(_TemplateEnvironmentMixin): bucket_name, key_name, version_id=version_id) if 'acl' in query: template = self.response_template(S3_OBJECT_ACL_RESPONSE) - return 200, headers, template.render(key=key) + return 200, headers, template.render(obj=key) if key: headers.update(key.metadata) @@ -696,7 +705,7 @@ S3_OBJECT_ACL_RESPONSE = """ webfile - {% for grant in key.acl.grants %} + {% for grant in obj.acl.grants %} {% for grantee in grant.grantees %} Date: Wed, 11 Nov 2015 21:59:55 -0500 Subject: [PATCH 54/55] Add support for partial updates and SS in dynamodb. --- moto/dynamodb2/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 8266d5588..612a0c3d3 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -127,7 +127,10 @@ class Item(object): new_value = list(update_action['Value'].values())[0] if action == 'PUT': # TODO deal with other types - self.attrs[attribute_name] = DynamoType({"S": new_value}) + if isinstance(new_value, list) or isinstance(new_value, set): + self.attrs[attribute_name] = DynamoType({"SS": new_value}) + else: + self.attrs[attribute_name] = DynamoType({"S": new_value}) class Table(object): From f93b9a86e9faf7846888acf37f705d07e2bfa647 Mon Sep 17 00:00:00 2001 From: mfranke Date: Thu, 12 Nov 2015 10:05:02 +0100 Subject: [PATCH 55/55] add put_records API fix create_stream API to get right response in case of stream already exists --- moto/kinesis/exceptions.py | 9 +++++++++ moto/kinesis/models.py | 27 ++++++++++++++++++++++++++- moto/kinesis/responses.py | 12 ++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/moto/kinesis/exceptions.py b/moto/kinesis/exceptions.py index c6be76885..0fcb3652a 100644 --- a/moto/kinesis/exceptions.py +++ b/moto/kinesis/exceptions.py @@ -13,6 +13,15 @@ class ResourceNotFoundError(BadRequest): }) +class ResourceInUseError(BadRequest): + def __init__(self, message): + super(ResourceNotFoundError, self).__init__() + self.description = json.dumps({ + "message": message, + '__type': 'ResourceInUseException', + }) + + class StreamNotFoundError(ResourceNotFoundError): def __init__(self, stream_name): super(StreamNotFoundError, self).__init__( diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index d2d0d2913..aae4e918f 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -6,7 +6,7 @@ import time import boto.kinesis from moto.compat import OrderedDict from moto.core import BaseBackend -from .exceptions import StreamNotFoundError, ShardNotFoundError +from .exceptions import StreamNotFoundError, ShardNotFoundError, ResourceInUseError from .utils import compose_shard_iterator, compose_new_shard_iterator, decompose_shard_iterator @@ -201,6 +201,8 @@ class KinesisBackend(BaseBackend): self.delivery_streams = {} def create_stream(self, stream_name, shard_count, region): + if stream_name in self.streams: + return ResourceInUseError(stream_name) stream = Stream(stream_name, shard_count, region) self.streams[stream_name] = stream return stream @@ -251,6 +253,29 @@ class KinesisBackend(BaseBackend): return sequence_number, shard_id + def put_records(self, stream_name, records): + stream = self.describe_stream(stream_name) + + response = { + "FailedRecordCount": 0, + "Records" : [] + } + + for record in records: + partition_key = record.get("PartitionKey") + explicit_hash_key = record.get("ExplicitHashKey") + data = record.get("data") + + sequence_number, shard_id = stream.put_record( + partition_key, explicit_hash_key, None, data + ) + response['Records'].append({ + "SequenceNumber": sequence_number, + "ShardId": shard_id + }) + + return response + ''' Firehose ''' def create_delivery_stream(self, stream_name, **stream_kwargs): stream = DeliveryStream(stream_name, **stream_kwargs) diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index 5b8c7be06..35500e8ac 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -89,6 +89,18 @@ class KinesisResponse(BaseResponse): "ShardId": shard_id, }) + def put_records(self): + if self.is_firehose: + return self.firehose_put_record() + stream_name = self.parameters.get("StreamName") + records = self.parameters.get("Records") + + response = self.kinesis_backend.put_records( + stream_name, records + ) + + return json.dumps(response) + ''' Firehose ''' def create_delivery_stream(self): stream_name = self.parameters['DeliveryStreamName']