From 6560063ceba44514263af69811950b6a580b85fd Mon Sep 17 00:00:00 2001 From: Alfred Moreno Date: Thu, 22 Sep 2016 18:38:47 -0700 Subject: [PATCH 01/18] Fix tagging code. Unit tests were passing but the underlying implementation wasn't producing the expected result --- moto/route53/models.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index c15862770..6b293a1ca 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -238,18 +238,19 @@ class Route53Backend(BaseBackend): def change_tags_for_resource(self, resource_id, tags): if 'Tag' in tags: - for key, tag in tags.items(): - for t in tag: - self.resource_tags[resource_id][t['Key']] = t['Value'] - + if isinstance(tags['Tag'], list): + for tag in tags['Tag']: + self.resource_tags[resource_id][tag['Key']] = tag['Value'] + else: + key, value = (tags['Tag']['Key'], tags['Tag']['Value']) + self.resource_tags[resource_id][key] = value else: - for _, keys in tags.items(): - if isinstance(keys, list): - for key in keys: + if 'Key' in tags: + if isinstance(tags['Key'], list): + for key in tags['Key']: del(self.resource_tags[resource_id][key]) else: - del(self.resource_tags[resource_id][keys]) - + del(self.resource_tags[resource_id][tags['Key']]) def list_tags_for_resource(self, resource_id): if resource_id in self.resource_tags: From 40ad92b4bc3c90e459d6097ffc6324ad9d57c428 Mon Sep 17 00:00:00 2001 From: Alfred Moreno Date: Thu, 22 Sep 2016 18:44:07 -0700 Subject: [PATCH 02/18] Add a unit test to make sure multiple tags were actually associated with a resourceId --- tests/test_route53/test_route53.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index b3bed8272..c6699bdac 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -354,6 +354,8 @@ def test_list_or_change_tags_for_resource_request(): response['ResourceTagSet']['Tags'].should.contain(tag1) response['ResourceTagSet']['Tags'].should.contain(tag2) + len(response['ResourceTagSet']['Tags']).should.equal(2) + # Try to remove the tags conn.change_tags_for_resource( ResourceType='healthcheck', From 6b74487b317fe5f2ab6090775da57e6e912b92c1 Mon Sep 17 00:00:00 2001 From: Georges Chaudy Date: Fri, 23 Sep 2016 16:14:17 +0100 Subject: [PATCH 03/18] fix: dynamodb2 conditions --- moto/dynamodb2/models.py | 9 +++-- .../test_dynamodb_table_without_range_key.py | 33 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 0a994f783..a70d6347d 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -275,9 +275,14 @@ class Table(object): raise ValueError("The conditional request failed") elif key not in current_attr: raise ValueError("The conditional request failed") - elif DynamoType(val['Value']).value != current_attr[key].value: + elif 'Value' in val and DynamoType(val['Value']).value != current_attr[key].value: raise ValueError("The conditional request failed") - + elif 'ComparisonOperator' in val: + comparison_func = get_comparison_func(val['ComparisonOperator']) + dynamo_types = [DynamoType(ele) for ele in val["AttributeValueList"]] + for t in dynamo_types: + if not comparison_func(current_attr[key].value, t.value): + raise ValueError('The conditional request failed') if range_value: self.items[hash_value][range_value] = item else: diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py index 5a32c6b20..691e14818 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py @@ -8,6 +8,7 @@ from freezegun import freeze_time from boto.exception import JSONResponseError from moto import mock_dynamodb2 from tests.helpers import requires_boto_gte +import botocore try: from boto.dynamodb2.fields import HashKey from boto.dynamodb2.table import Table @@ -469,6 +470,7 @@ def test_update_item_set(): }) + @mock_dynamodb2 def test_failed_overwrite(): table = Table.create('messages', schema=[ @@ -585,6 +587,37 @@ def test_boto3_conditions(): response['Items'][0].should.equal({"username": "johndoe"}) +@mock_dynamodb2 +def test_boto3_put_item_conditions_fails(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.put_item.when.called_with( + Item={'username': 'johndoe', 'foo': 'baz'}, + Expected={ + 'foo': { + 'ComparisonOperator': 'NE', + 'AttributeValueList': ['bar'] + } + }).should.throw(botocore.client.ClientError) + + +@mock_dynamodb2 +def test_boto3_put_item_conditions_pass(): + table = _create_user_table() + table.put_item(Item={'username': 'johndoe', 'foo': 'bar'}) + table.put_item( + Item={'username': 'johndoe', 'foo': 'baz'}, + Expected={ + 'foo': { + 'ComparisonOperator': 'EQ', + 'AttributeValueList': ['bar'] + } + }) + returned_item = table.get_item(Key={'username': 'johndoe'}) + assert dict(returned_item)['Item']['foo'].should.equal("baz") + + + @mock_dynamodb2 def test_scan_pagination(): table = _create_user_table() From bd3dd23c6b3877b397d431402b7aa89bf0bcbac7 Mon Sep 17 00:00:00 2001 From: Georges Chaudy Date: Fri, 23 Sep 2016 17:07:49 +0100 Subject: [PATCH 04/18] fix: change dynamodb2 batch write response --- moto/dynamodb2/responses.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 76bba9e49..91d2f45ee 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -179,14 +179,14 @@ class DynamoHandler(BaseResponse): item = dynamodb_backend2.delete_item(table_name, keys) response = { - "Responses": { - "Thread": { - "ConsumedCapacityUnits": 1.0 - }, - "Reply": { - "ConsumedCapacityUnits": 1.0 - } - }, + "ConsumedCapacity": [ + { + 'TableName': table_name, + 'CapacityUnits': 1.0, + 'Table': {'CapacityUnits': 1.0} + } for table_name, table_requests in table_batches.items() + ], + "ItemCollectionMetrics": {}, "UnprocessedItems": {} } From e6bf5550ceb92a8a9ef0598523ff1c0fdf81d104 Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Mon, 26 Sep 2016 13:44:05 +0100 Subject: [PATCH 05/18] setup.py - use extras_require for flask Adding `moto` to the requirements file for my Django project pulls in `flask`, even though I don't need the server component. By making the `server` 'extra' then there is no hard requirement on flask. --- README.md | 14 +++++++++++--- setup.py | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 16e1ccc95..a4f500dfa 100644 --- a/README.md +++ b/README.md @@ -200,16 +200,24 @@ In general, Moto doesn't rely on anything specific to Boto. It only mocks AWS en ## Stand-alone Server Mode -Moto also comes with a stand-alone server mode. This allows you to utilize the backend structure of Moto even if you don't use Python. +Moto also has a stand-alone server mode. This allows you to utilize +the backend structure of Moto even if you don't use Python. -To run a service: +It uses flask, which isn't a default dependency. You can install the +server 'extra' package with: + +```python +pip install moto[server] +``` + +You can then start it running a service: ```console $ moto_server ec2 * Running on http://127.0.0.1:5000/ ``` -You can also pass the port as the second argument: +You can also pass the port: ```console $ moto_server ec2 -p3000 diff --git a/setup.py b/setup.py index 928f28c75..e4da5a4f6 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ from setuptools import setup, find_packages install_requires = [ "Jinja2", "boto>=2.36.0", - "flask", "httpretty==0.8.10", "requests", "xmltodict", @@ -18,6 +17,8 @@ install_requires = [ extras_require = { # No builtin OrderedDict before 2.7 ':python_version=="2.6"': ['ordereddict'], + + 'server': ['flask'], } setup( From 8720601eaa16364cc48a6d4659c26ae98f12c462 Mon Sep 17 00:00:00 2001 From: Andrew Gross Date: Thu, 29 Sep 2016 16:37:34 -0400 Subject: [PATCH 06/18] Remove sure and freezegun from setup.py, they still exist in requirements-dev.txt --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index 928f28c75..68ad6a9da 100644 --- a/setup.py +++ b/setup.py @@ -11,8 +11,6 @@ install_requires = [ "xmltodict", "six", "werkzeug", - "sure", - "freezegun" ] extras_require = { From 3a7fec4032c048e5881a2778ec9c391b53da44fd Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Mon, 3 Oct 2016 01:33:59 +1100 Subject: [PATCH 07/18] Add ARN and Alias functionality to KMS --- moto/kms/models.py | 27 +++++++--- moto/kms/responses.py | 14 ++--- tests/test_kms/test_kms.py | 101 +++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 14 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 7ca6e24b5..167303ff1 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -77,11 +77,19 @@ class KmsBackend(BaseBackend): return self.keys.pop(key_id) def describe_key(self, key_id): - return self.keys[key_id] + # allow the different methods (alias, ARN :key/, keyId, ARN alias) to describe key not just KeyId + key_id = self.get_key_id(key_id) + if r'alias/' in str(key_id).lower(): + key_id = self.get_key_id_from_alias(key_id.split('alias/')[1]) + return self.keys[self.get_key_id(key_id)] def list_keys(self): return self.keys.values() + def get_key_id(self, key_id): + # Allow use of ARN as well as pure KeyId + return str(key_id).split(r':key/')[1] if r':key/' in str(key_id).lower() else key_id + def alias_exists(self, alias_name): for aliases in self.key_to_aliases.values(): if alias_name in aliases: @@ -99,21 +107,26 @@ class KmsBackend(BaseBackend): def get_all_aliases(self): return self.key_to_aliases + def get_key_id_from_alias(self, alias_name): + for key_id, aliases in dict(self.key_to_aliases).iteritems(): + if alias_name in ",".join(aliases): + return key_id + return None + def enable_key_rotation(self, key_id): - self.keys[key_id].key_rotation_status = True + self.keys[self.get_key_id(key_id)].key_rotation_status = True def disable_key_rotation(self, key_id): - self.keys[key_id].key_rotation_status = False + self.keys[self.get_key_id(key_id)].key_rotation_status = False def get_key_rotation_status(self, key_id): - return self.keys[key_id].key_rotation_status + return self.keys[self.get_key_id(key_id)].key_rotation_status def put_key_policy(self, key_id, policy): - self.keys[key_id].policy = policy + self.keys[self.get_key_id(key_id)].policy = policy def get_key_policy(self, key_id): - return self.keys[key_id].policy - + return self.keys[self.get_key_id(key_id)].policy kms_backends = {} for region in boto.kms.regions(): diff --git a/moto/kms/responses.py b/moto/kms/responses.py index fe5af07ae..69a75c473 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -37,7 +37,7 @@ class KmsResponse(BaseResponse): def describe_key(self): key_id = self.parameters.get('KeyId') try: - key = self.kms_backend.describe_key(key_id) + key = self.kms_backend.describe_key(self.kms_backend.get_key_id(key_id)) except KeyError: headers = dict(self.headers) headers['status'] = 404 @@ -140,7 +140,7 @@ class KmsResponse(BaseResponse): def enable_key_rotation(self): key_id = self.parameters.get('KeyId') - _assert_valid_key_id(key_id) + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: self.kms_backend.enable_key_rotation(key_id) except KeyError: @@ -152,7 +152,7 @@ class KmsResponse(BaseResponse): def disable_key_rotation(self): key_id = self.parameters.get('KeyId') - _assert_valid_key_id(key_id) + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: self.kms_backend.disable_key_rotation(key_id) except KeyError: @@ -163,7 +163,7 @@ class KmsResponse(BaseResponse): def get_key_rotation_status(self): key_id = self.parameters.get('KeyId') - _assert_valid_key_id(key_id) + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: rotation_enabled = self.kms_backend.get_key_rotation_status(key_id) except KeyError: @@ -176,7 +176,7 @@ class KmsResponse(BaseResponse): key_id = self.parameters.get('KeyId') policy_name = self.parameters.get('PolicyName') policy = self.parameters.get('Policy') - _assert_valid_key_id(key_id) + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) _assert_default_policy(policy_name) try: @@ -191,7 +191,7 @@ class KmsResponse(BaseResponse): def get_key_policy(self): key_id = self.parameters.get('KeyId') policy_name = self.parameters.get('PolicyName') - _assert_valid_key_id(key_id) + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) _assert_default_policy(policy_name) try: @@ -203,7 +203,7 @@ class KmsResponse(BaseResponse): def list_key_policies(self): key_id = self.parameters.get('KeyId') - _assert_valid_key_id(key_id) + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: self.kms_backend.describe_key(key_id) except KeyError: diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index fe1394434..f228f4661 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -30,6 +30,39 @@ def test_describe_key(): key['KeyMetadata']['KeyUsage'].should.equal("ENCRYPT_DECRYPT") +@mock_kms +def test_describe_key_via_alias(): + conn = boto.kms.connect_to_region("us-west-2") + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + conn.create_alias(alias_name='alias/my-key-alias', target_key_id=key['KeyMetadata']['KeyId']) + + alias_key = conn.describe_key('alias/my-key-alias') + alias_key['KeyMetadata']['Description'].should.equal("my key") + alias_key['KeyMetadata']['KeyUsage'].should.equal("ENCRYPT_DECRYPT") + alias_key['KeyMetadata']['Arn'].should.equal(key['KeyMetadata']['Arn']) + + +@mock_kms +def test_describe_key_via_alias_not_found(): + conn = boto.kms.connect_to_region("us-west-2") + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + conn.create_alias(alias_name='alias/my-key-alias', target_key_id=key['KeyMetadata']['KeyId']) + + conn.describe_key.when.called_with('alias/not-found-alias').should.throw(JSONResponseError) + + +@mock_kms +def test_describe_key_via_arn(): + conn = boto.kms.connect_to_region("us-west-2") + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + arn = key['KeyMetadata']['Arn'] + + the_key = conn.describe_key(arn) + the_key['KeyMetadata']['Description'].should.equal("my key") + the_key['KeyMetadata']['KeyUsage'].should.equal("ENCRYPT_DECRYPT") + the_key['KeyMetadata']['KeyId'].should.equal(key['KeyMetadata']['KeyId']) + + @mock_kms def test_describe_missing_key(): conn = boto.kms.connect_to_region("us-west-2") @@ -58,6 +91,18 @@ def test_enable_key_rotation(): conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(True) +@mock_kms +def test_enable_key_rotation_via_arn(): + conn = boto.kms.connect_to_region("us-west-2") + + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + key_id = key['KeyMetadata']['Arn'] + + conn.enable_key_rotation(key_id) + + conn.get_key_rotation_status(key_id)['KeyRotationEnabled'].should.equal(True) + + @mock_kms def test_enable_key_rotation_with_missing_key(): @@ -65,6 +110,18 @@ def test_enable_key_rotation_with_missing_key(): conn.enable_key_rotation.when.called_with("not-a-key").should.throw(JSONResponseError) +@mock_kms +def test_enable_key_rotation_with_alias_name_should_fail(): + conn = boto.kms.connect_to_region("us-west-2") + key = conn.create_key(policy="my policy", description="my key", key_usage='ENCRYPT_DECRYPT') + conn.create_alias(alias_name='alias/my-key-alias', target_key_id=key['KeyMetadata']['KeyId']) + + alias_key = conn.describe_key('alias/my-key-alias') + alias_key['KeyMetadata']['Arn'].should.equal(key['KeyMetadata']['Arn']) + + conn.enable_key_rotation.when.called_with('alias/my-alias').should.throw(JSONResponseError) + + @mock_kms def test_disable_key_rotation(): conn = boto.kms.connect_to_region("us-west-2") @@ -121,6 +178,14 @@ def test_get_key_policy(): policy = conn.get_key_policy(key_id, 'default') policy['Policy'].should.equal('my policy') +@mock_kms +def test_get_key_policy_via_arn(): + conn = boto.kms.connect_to_region('us-west-2') + + key = conn.create_key(policy='my policy', description='my key1', key_usage='ENCRYPT_DECRYPT') + policy = conn.get_key_policy(key['KeyMetadata']['Arn'], 'default') + + policy['Policy'].should.equal('my policy') @mock_kms def test_put_key_policy(): @@ -134,6 +199,42 @@ def test_put_key_policy(): policy['Policy'].should.equal('new policy') +@mock_kms +def test_put_key_policy_via_arn(): + conn = boto.kms.connect_to_region('us-west-2') + + key = conn.create_key(policy='my policy', description='my key1', key_usage='ENCRYPT_DECRYPT') + key_id = key['KeyMetadata']['Arn'] + + conn.put_key_policy(key_id, 'default', 'new policy') + policy = conn.get_key_policy(key_id, 'default') + policy['Policy'].should.equal('new policy') + + +@mock_kms +def test_put_key_policy_via_alias_should_not_update(): + conn = boto.kms.connect_to_region('us-west-2') + + key = conn.create_key(policy='my policy', description='my key1', key_usage='ENCRYPT_DECRYPT') + conn.create_alias(alias_name='alias/my-key-alias', target_key_id=key['KeyMetadata']['KeyId']) + + conn.put_key_policy.when.called_with('alias/my-key-alias', 'default', 'new policy').should.throw(JSONResponseError) + + policy = conn.get_key_policy(key['KeyMetadata']['KeyId'], 'default') + policy['Policy'].should.equal('my policy') + + +@mock_kms +def test_put_key_policy(): + conn = boto.kms.connect_to_region('us-west-2') + + key = conn.create_key(policy='my policy', description='my key1', key_usage='ENCRYPT_DECRYPT') + conn.put_key_policy(key['KeyMetadata']['Arn'], 'default', 'new policy') + + policy = conn.get_key_policy(key['KeyMetadata']['KeyId'], 'default') + policy['Policy'].should.equal('new policy') + + @mock_kms def test_list_key_policies(): conn = boto.kms.connect_to_region('us-west-2') From 0062c8da1a48d2e41f67e7a160679e4ec42ea143 Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Mon, 3 Oct 2016 01:51:59 +1100 Subject: [PATCH 08/18] Fix up models to work with Python 3 --- moto/kms/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 167303ff1..0bfe5791f 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -108,7 +108,7 @@ class KmsBackend(BaseBackend): return self.key_to_aliases def get_key_id_from_alias(self, alias_name): - for key_id, aliases in dict(self.key_to_aliases).iteritems(): + for key_id, aliases in dict(self.key_to_aliases).items(): if alias_name in ",".join(aliases): return key_id return None From 6c577091da23efe7ccf0b24fdd81a64fbff2dd1f Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Thu, 6 Oct 2016 19:52:23 +1000 Subject: [PATCH 09/18] lambderize the moto lambda --- moto/awslambda/models.py | 56 +++++++++-- moto/awslambda/responses.py | 2 +- tests/test_awslambda/test_lambda.py | 97 +++++++++++++------ .../test_cloudformation_stack_integration.py | 39 +++++--- 4 files changed, 147 insertions(+), 47 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index d5be3b3be..ff21805d3 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -3,7 +3,11 @@ from __future__ import unicode_literals import base64 import datetime import hashlib +import io import json +import StringIO +import sys +import zipfile import boto.awslambda from moto.core import BaseBackend @@ -34,9 +38,13 @@ class LambdaFunction(object): self.version = '$LATEST' self.last_modified = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') if 'ZipFile' in self.code: - code = base64.b64decode(self.code['ZipFile']) - self.code_size = len(code) - self.code_sha_256 = hashlib.sha256(code).hexdigest() + to_unzip_code = base64.b64decode(self.code['ZipFile']) + zbuffer = io.BytesIO() + zbuffer.write(to_unzip_code) + zip_file = zipfile.ZipFile(zbuffer, 'r', zipfile.ZIP_DEFLATED) + self.code = zip_file.read("".join(zip_file.namelist())) + self.code_size = len(to_unzip_code) + self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest() else: # validate s3 bucket try: @@ -93,15 +101,47 @@ class LambdaFunction(object): "Configuration": self.get_configuration(), } + def _invoke_lambda(self, code, event={}, context={}): + # TO DO: context not yet implemented + try: + codeOut = StringIO.StringIO() + codeErr = StringIO.StringIO() + mycode = "\n".join([self.code, 'print lambda_handler(%s, %s)' % (event, context)]) + #print "moto_lambda_debug: ", mycode + sys.stdout = codeOut + sys.stderr = codeErr + exec(mycode, {'event': event, 'context': context}) + exec_err = codeErr.getvalue() + exec_out = codeOut.getvalue() + result = "\n".join([exec_out, exec_err]) + except Exception as ex: + result = 'Exception %s, %s' % (ex, ex.message) + finally: + codeErr.close() + codeOut.close() + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + return result + + def is_json(self, test_str): + try: + response = json.loads(test_str) + except: + response = test_str + return response + def invoke(self, request, headers): payload = dict() # Get the invocation type: + invoke_type = request.headers.get("x-amz-invocation-type") + response = self._invoke_lambda(code=self.code, event=self.is_json(request.body)) if request.headers.get("x-amz-invocation-type") == "RequestResponse": - encoded = base64.b64encode("Some log file output...".encode('utf-8')) + encoded = base64.b64encode(response.encode('utf-8')) + payload['result'] = encoded headers["x-amz-log-result"] = encoded.decode('utf-8') - - payload["result"] = "Good" + elif request.headers.get("x-amz-invocation-type") == "Event": + payload['result'] = 'good' # nothing should be sent back possibly headers etc. return json.dumps(payload, indent=4) @@ -154,3 +194,7 @@ class LambdaBackend(BaseBackend): lambda_backends = {} for region in boto.awslambda.regions(): lambda_backends[region.name] = LambdaBackend() + +# Handle us forgotten regions, unless Lambda truly only runs out of US and EU????? +for region in ['ap-southeast-2']: + lambda_backends[region] = LambdaBackend() diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 98458cc2c..468a95766 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -43,7 +43,7 @@ class LambdaResponse(BaseResponse): if lambda_backend.has_function(function_name): fn = lambda_backend.get_function(function_name) payload = fn.invoke(request, headers) - return 200, headers, payload + return 202, headers, payload else: return 404, headers, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index e706a013f..ee17312a6 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import base64 import botocore.client import boto3 import hashlib @@ -8,44 +9,57 @@ import zipfile import sure # noqa from freezegun import freeze_time -from moto import mock_lambda, mock_s3 +from moto import mock_lambda, mock_s3, mock_ec2 -def get_test_zip_file(): +def _process_lamda(pfunc): zip_output = io.BytesIO() - zip_file = zipfile.ZipFile(zip_output, 'w') - zip_file.writestr('lambda_function.py', b'''\ -def handler(event, context): - return "hello world" -''') + zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) + zip_file.writestr('lambda_function.zip', pfunc) zip_file.close() zip_output.seek(0) return zip_output.read() -@mock_lambda -def test_list_functions(): - conn = boto3.client('lambda', 'us-west-2') +def get_test_zip_file1(): + pfunc = b""" +def lambda_handler(event, context): + return (event, context) +""" + return _process_lamda(pfunc) - result = conn.list_functions() - result['Functions'].should.have.length_of(0) +def get_test_zip_file2(): + pfunc = b""" +def lambda_handler(event, context): + volume_id = event.get('volume_id') + print 'get volume details for %s' % volume_id + import boto3 + ec2 = boto3.resource('ec2', region_name='us-west-2') + vol = ec2.Volume(volume_id) + print 'Volume - %s state=%s, size=%s' % (volume_id, vol.state, vol.size) +""" + return _process_lamda(pfunc) @mock_lambda @mock_s3 +def test_list_functions(): + conn = boto3.client('lambda', 'us-west-2') + result = conn.list_functions() + result['Functions'].should.have.length_of(0) + +@mock_lambda @freeze_time('2015-01-01 00:00:00') def test_invoke_function(): conn = boto3.client('lambda', 'us-west-2') - - zip_content = get_test_zip_file() conn.create_function( FunctionName='testFunction', Runtime='python2.7', Role='test-iam-role', Handler='lambda_function.handler', Code={ - 'ZipFile': zip_content, + 'ZipFile': get_test_zip_file1(), }, Description='test lambda function', Timeout=3, @@ -53,8 +67,8 @@ def test_invoke_function(): Publish=True, ) - success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload='{}') - success_result["StatusCode"].should.equal(200) + success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload="Mostly Harmless") + success_result["StatusCode"].should.equal(202) conn.invoke.when.called_with( FunctionName='notAFunction', @@ -62,11 +76,42 @@ def test_invoke_function(): Payload='{}' ).should.throw(botocore.client.ClientError) - success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload='{}') - success_result["StatusCode"].should.equal(200) + success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload='{"msg": "So long and thanks for all the fish"}') + success_result["StatusCode"].should.equal(202) import base64 - base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal("Some log file output...") + base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal("({u'msg': u'So long and thanks for all the fish'}, {})\n\n") + +@mock_ec2 +@mock_lambda +@freeze_time('2015-01-01 00:00:00') +def test_invoke_function_get_ec2_volume(): + conn = boto3.resource("ec2", "us-west-2") + vol = conn.create_volume(Size=99, AvailabilityZone='us-west-2') + vol = conn.Volume(vol.id) + + conn = boto3.client('lambda', 'us-west-2') + conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.handler', + Code={ + 'ZipFile': get_test_zip_file2(), + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + import json + success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload=json.dumps({'volume_id': vol.id})) + success_result["StatusCode"].should.equal(202) + + import base64 + msg = 'get volume details for %s\nVolume - %s state=%s, size=%s\nNone\n\n' % (vol.id, vol.id, vol.state, vol.size) + base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal(msg) @mock_lambda @@ -101,7 +146,7 @@ def test_create_function_from_aws_bucket(): s3_conn = boto3.client('s3', 'us-west-2') s3_conn.create_bucket(Bucket='test-bucket') - zip_content = get_test_zip_file() + zip_content = get_test_zip_file2() s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) conn = boto3.client('lambda', 'us-west-2') @@ -151,8 +196,7 @@ def test_create_function_from_aws_bucket(): @freeze_time('2015-01-01 00:00:00') def test_create_function_from_zipfile(): conn = boto3.client('lambda', 'us-west-2') - - zip_content = get_test_zip_file() + zip_content = get_test_zip_file1() result = conn.create_function( FunctionName='testFunction', Runtime='python2.7', @@ -196,7 +240,7 @@ def test_get_function(): s3_conn = boto3.client('s3', 'us-west-2') s3_conn.create_bucket(Bucket='test-bucket') - zip_content = get_test_zip_file() + zip_content = get_test_zip_file1() s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) conn = boto3.client('lambda', 'us-west-2') @@ -245,14 +289,13 @@ def test_get_function(): }) - @mock_lambda @mock_s3 def test_delete_function(): s3_conn = boto3.client('s3', 'us-west-2') s3_conn.create_bucket(Bucket='test-bucket') - zip_content = get_test_zip_file() + zip_content = get_test_zip_file2() s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) conn = boto3.client('lambda', 'us-west-2') @@ -289,7 +332,7 @@ def test_list_create_list_get_delete_list(): s3_conn = boto3.client('s3', 'us-west-2') s3_conn.create_bucket(Bucket='test-bucket') - zip_content = get_test_zip_file() + zip_content = get_test_zip_file2() s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) conn = boto3.client('lambda', 'us-west-2') diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index cbdb3d221..bd8b32fab 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import json +import base64 import boto import boto.cloudformation import boto.datapipeline @@ -1724,10 +1725,29 @@ def test_datapipeline(): stack_resources.should.have.length_of(1) stack_resources[0].physical_resource_id.should.equal(data_pipelines['pipelineIdList'][0]['id']) +def _process_lamda(pfunc): + import io + import zipfile + zip_output = io.BytesIO() + zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED) + zip_file.writestr('lambda_function.zip', pfunc) + zip_file.close() + zip_output.seek(0) + return zip_output.read() + + +def get_test_zip_file1(): + pfunc = b""" +def lambda_handler(event, context): + return (event, context) +""" + return _process_lamda(pfunc) + @mock_cloudformation @mock_lambda def test_lambda_function(): + # switch this to python as backend lambda only supports python execution. conn = boto3.client('lambda', 'us-east-1') template = { "AWSTemplateFormatVersion": "2010-09-09", @@ -1736,22 +1756,15 @@ def test_lambda_function(): "Type": "AWS::Lambda::Function", "Properties": { "Code": { - "ZipFile": {"Fn::Join": [ - "\n", - """ - exports.handler = function(event, context) { - context.succeed(); - } - """.splitlines() - ]} + "ZipFile": base64.b64encode(get_test_zip_file1()) }, - "Handler": "index.handler", + "Handler": "lambda_function.handler", "Description": "Test function", "MemorySize": 128, "Role": "test-role", - "Runtime": "nodejs", + "Runtime": "python2.7" } - }, + } } } @@ -1765,10 +1778,10 @@ def test_lambda_function(): result = conn.list_functions() result['Functions'].should.have.length_of(1) result['Functions'][0]['Description'].should.equal('Test function') - result['Functions'][0]['Handler'].should.equal('index.handler') + result['Functions'][0]['Handler'].should.equal('lambda_function.handler') result['Functions'][0]['MemorySize'].should.equal(128) result['Functions'][0]['Role'].should.equal('test-role') - result['Functions'][0]['Runtime'].should.equal('nodejs') + result['Functions'][0]['Runtime'].should.equal('python2.7') @mock_cloudformation From 7c3005e58258a0bbc80379c46212e0cc2a5445f6 Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Fri, 7 Oct 2016 00:14:47 +1000 Subject: [PATCH 10/18] attmpt 2 to resolve StringIO not being within Python 3 anymore --- moto/awslambda/models.py | 17 +++++++++++------ tests/test_awslambda/test_lambda.py | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index ff21805d3..824bc13a2 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -1,14 +1,19 @@ from __future__ import unicode_literals +import StringIO import base64 import datetime import hashlib import io import json -import StringIO import sys import zipfile +try: + from StringIO import StringIO +except: + from io import StringIO + import boto.awslambda from moto.core import BaseBackend from moto.s3.models import s3_backend @@ -104,18 +109,18 @@ class LambdaFunction(object): def _invoke_lambda(self, code, event={}, context={}): # TO DO: context not yet implemented try: - codeOut = StringIO.StringIO() - codeErr = StringIO.StringIO() - mycode = "\n".join([self.code, 'print lambda_handler(%s, %s)' % (event, context)]) + codeOut = StringIO() + codeErr = StringIO() + mycode = "\n".join([self.code, 'print lambda_handler(%s, %s)' % (event, context)]) #print "moto_lambda_debug: ", mycode sys.stdout = codeOut sys.stderr = codeErr - exec(mycode, {'event': event, 'context': context}) + exec mycode exec_err = codeErr.getvalue() exec_out = codeOut.getvalue() result = "\n".join([exec_out, exec_err]) except Exception as ex: - result = 'Exception %s, %s' % (ex, ex.message) + result = '%s\n\n\nException %s, %s' % (mycode, ex, ex.message) finally: codeErr.close() codeOut.close() diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index ee17312a6..a82100bc4 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -5,6 +5,7 @@ import botocore.client import boto3 import hashlib import io +import json import zipfile import sure # noqa @@ -67,7 +68,7 @@ def test_invoke_function(): Publish=True, ) - success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload="Mostly Harmless") + success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload='Mostly Harmless') success_result["StatusCode"].should.equal(202) conn.invoke.when.called_with( @@ -76,10 +77,9 @@ def test_invoke_function(): Payload='{}' ).should.throw(botocore.client.ClientError) - success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload='{"msg": "So long and thanks for all the fish"}') + success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', + Payload=json.dumps({'msg': 'So long and thanks for all the fish'})) success_result["StatusCode"].should.equal(202) - - import base64 base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal("({u'msg': u'So long and thanks for all the fish'}, {})\n\n") @mock_ec2 From 5500cc3e6fd0f4768c51c626f125572e082cdddc Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Fri, 7 Oct 2016 00:18:39 +1000 Subject: [PATCH 11/18] attmpt 3 not liking Python 3 very much at the moment --- moto/awslambda/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 824bc13a2..32718714e 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -115,7 +115,7 @@ class LambdaFunction(object): #print "moto_lambda_debug: ", mycode sys.stdout = codeOut sys.stderr = codeErr - exec mycode + exec(mycode) exec_err = codeErr.getvalue() exec_out = codeOut.getvalue() result = "\n".join([exec_out, exec_err]) From 76266b616382700082f7bab09552b154f9068386 Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Fri, 7 Oct 2016 00:26:00 +1000 Subject: [PATCH 12/18] attmpt 4 thought i got rid of import StringIO - rockstar status not looking so good now --- moto/awslambda/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 32718714e..0e24330c4 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import StringIO import base64 import datetime import hashlib From de9d31bc0dc68b71404d8b9f262360f354dbb92b Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Fri, 7 Oct 2016 00:41:31 +1000 Subject: [PATCH 13/18] attmpt 5 --- moto/awslambda/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 0e24330c4..684f1a248 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -107,11 +107,12 @@ class LambdaFunction(object): def _invoke_lambda(self, code, event={}, context={}): # TO DO: context not yet implemented + mycode = "\n".join([self.code, 'print lambda_handler(%s, %s)' % (event, context)]) + #print "moto_lambda_debug: ", mycode + try: codeOut = StringIO() codeErr = StringIO() - mycode = "\n".join([self.code, 'print lambda_handler(%s, %s)' % (event, context)]) - #print "moto_lambda_debug: ", mycode sys.stdout = codeOut sys.stderr = codeErr exec(mycode) From 95c90ae15f919a346ab5d8b5d60860e0c125e824 Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Fri, 7 Oct 2016 00:50:33 +1000 Subject: [PATCH 14/18] attmpt 6 is it the print statements or unicode? --- moto/awslambda/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 684f1a248..e857bd5c1 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -107,8 +107,8 @@ class LambdaFunction(object): def _invoke_lambda(self, code, event={}, context={}): # TO DO: context not yet implemented - mycode = "\n".join([self.code, 'print lambda_handler(%s, %s)' % (event, context)]) - #print "moto_lambda_debug: ", mycode + mycode = "\n".join([self.code, 'print(lambda_handler(%s, %s))' % (event, context)]) + print("moto_lambda_debug: ", mycode) try: codeOut = StringIO() From dc98cf6f64b00ec53eda8a139cd58c36a85bc95e Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Mon, 10 Oct 2016 01:13:52 +1000 Subject: [PATCH 15/18] argh 2 days trying to work python3 into working python2 :( --- moto/awslambda/models.py | 51 ++++++++++------ tests/test_awslambda/test_lambda.py | 59 ++++++++++++++----- .../test_cloudformation_stack_integration.py | 4 +- 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index e857bd5c1..563e7f43b 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -42,7 +42,12 @@ class LambdaFunction(object): self.version = '$LATEST' self.last_modified = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') if 'ZipFile' in self.code: - to_unzip_code = base64.b64decode(self.code['ZipFile']) + # more hackery to handle unicode/bytes/str in python3 and python2 - argh! + try: + to_unzip_code = base64.b64decode(bytes(self.code['ZipFile'], 'utf-8')) + except Exception: + to_unzip_code = base64.b64decode(self.code['ZipFile']) + zbuffer = io.BytesIO() zbuffer.write(to_unzip_code) zip_file = zipfile.ZipFile(zbuffer, 'r', zipfile.ZIP_DEFLATED) @@ -105,10 +110,28 @@ class LambdaFunction(object): "Configuration": self.get_configuration(), } + def convert(self, s): + try: + return str(s, encoding='utf8') + except: + return s + + def is_json(self, test_str): + try: + response = json.loads(test_str) + except: + response = test_str + return response + def _invoke_lambda(self, code, event={}, context={}): # TO DO: context not yet implemented - mycode = "\n".join([self.code, 'print(lambda_handler(%s, %s))' % (event, context)]) - print("moto_lambda_debug: ", mycode) + try: + mycode = "\n".join(['import json', + self.convert(self.code), + self.convert('print(lambda_handler(%s, %s))' % (self.is_json(self.convert(event)), context))]) + print("moto_lambda_debug: ", mycode) + except Exception as ex: + print('fuck ', ex) try: codeOut = StringIO() @@ -118,35 +141,25 @@ class LambdaFunction(object): exec(mycode) exec_err = codeErr.getvalue() exec_out = codeOut.getvalue() - result = "\n".join([exec_out, exec_err]) + result = "\n".join([exec_out, self.convert(exec_err)]) except Exception as ex: - result = '%s\n\n\nException %s, %s' % (mycode, ex, ex.message) + result = '%s\n\n\nException %s' % (mycode, ex) finally: codeErr.close() codeOut.close() sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ - return result - - def is_json(self, test_str): - try: - response = json.loads(test_str) - except: - response = test_str - return response + return self.convert(result) def invoke(self, request, headers): payload = dict() # Get the invocation type: - invoke_type = request.headers.get("x-amz-invocation-type") - response = self._invoke_lambda(code=self.code, event=self.is_json(request.body)) + r = self._invoke_lambda(code=self.code, event=request.body) if request.headers.get("x-amz-invocation-type") == "RequestResponse": - encoded = base64.b64encode(response.encode('utf-8')) - payload['result'] = encoded + encoded = base64.b64encode(r.encode('utf-8')) headers["x-amz-log-result"] = encoded.decode('utf-8') - elif request.headers.get("x-amz-invocation-type") == "Event": - payload['result'] = 'good' # nothing should be sent back possibly headers etc. + payload['result'] = headers["x-amz-log-result"] return json.dumps(payload, indent=4) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index a82100bc4..6b0655d98 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -23,7 +23,7 @@ def _process_lamda(pfunc): def get_test_zip_file1(): - pfunc = b""" + pfunc = """ def lambda_handler(event, context): return (event, context) """ @@ -31,14 +31,14 @@ def lambda_handler(event, context): def get_test_zip_file2(): - pfunc = b""" + pfunc = """ def lambda_handler(event, context): volume_id = event.get('volume_id') - print 'get volume details for %s' % volume_id + print('get volume details for %s' % volume_id) import boto3 ec2 = boto3.resource('ec2', region_name='us-west-2') vol = ec2.Volume(volume_id) - print 'Volume - %s state=%s, size=%s' % (volume_id, vol.state, vol.size) + print('Volume - %s state=%s, size=%s' % (volume_id, vol.state, vol.size)) """ return _process_lamda(pfunc) @@ -52,7 +52,7 @@ def test_list_functions(): @mock_lambda @freeze_time('2015-01-01 00:00:00') -def test_invoke_function(): +def test_invoke_event_function(): conn = boto3.client('lambda', 'us-west-2') conn.create_function( FunctionName='testFunction', @@ -68,7 +68,7 @@ def test_invoke_function(): Publish=True, ) - success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload='Mostly Harmless') + success_result = conn.invoke(FunctionName='testFunction', InvocationType='Event', Payload=json.dumps({'msg': 'Mostly Harmless'})) success_result["StatusCode"].should.equal(202) conn.invoke.when.called_with( @@ -77,10 +77,31 @@ def test_invoke_function(): Payload='{}' ).should.throw(botocore.client.ClientError) + +@mock_lambda +@freeze_time('2015-01-01 00:00:00') +def test_invoke_requestresponse_function(): + conn = boto3.client('lambda', 'us-west-2') + conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.handler', + Code={ + 'ZipFile': get_test_zip_file1(), + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + success_result = conn.invoke(FunctionName='testFunction', InvocationType='RequestResponse', Payload=json.dumps({'msg': 'So long and thanks for all the fish'})) success_result["StatusCode"].should.equal(202) - base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal("({u'msg': u'So long and thanks for all the fish'}, {})\n\n") + + #nasty hack - hope someone has better solution dealing with unicode tests working for Py2 and Py3. + base64.b64decode(success_result["LogResult"]).decode('utf-8').replace("u'", "'").should.equal("({'msg': 'So long and thanks for all the fish'}, {})\n\n") @mock_ec2 @mock_lambda @@ -111,7 +132,8 @@ def test_invoke_function_get_ec2_volume(): import base64 msg = 'get volume details for %s\nVolume - %s state=%s, size=%s\nNone\n\n' % (vol.id, vol.id, vol.state, vol.size) - base64.b64decode(success_result["LogResult"]).decode('utf-8').should.equal(msg) + # yet again hacky solution to allow code to run tests for python2 and python3 - pls someone fix :( + base64.b64decode(success_result["LogResult"]).decode('utf-8').replace("u'", "'").should.equal(msg) @mock_lambda @@ -145,8 +167,8 @@ def test_create_based_on_s3_with_missing_bucket(): def test_create_function_from_aws_bucket(): s3_conn = boto3.client('s3', 'us-west-2') s3_conn.create_bucket(Bucket='test-bucket') - zip_content = get_test_zip_file2() + s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) conn = boto3.client('lambda', 'us-west-2') @@ -168,7 +190,8 @@ def test_create_function_from_aws_bucket(): "SubnetIds": ["subnet-123abc"], }, ) - result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + result['ResponseMetadata'].pop('RetryAttempts', None) # Botocore inserts retry attempts not seen in Python27 result.should.equal({ 'FunctionName': 'testFunction', 'FunctionArn': 'arn:aws:lambda:123456789012:function:testFunction', @@ -187,7 +210,6 @@ def test_create_function_from_aws_bucket(): "SubnetIds": ["subnet-123abc"], "VpcId": "vpc-123abc" }, - 'ResponseMetadata': {'HTTPStatusCode': 201}, }) @@ -210,7 +232,9 @@ def test_create_function_from_zipfile(): MemorySize=128, Publish=True, ) - result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + result['ResponseMetadata'].pop('RetryAttempts', None) # Botocore inserts retry attempts not seen in Python27 + result.should.equal({ 'FunctionName': 'testFunction', 'FunctionArn': 'arn:aws:lambda:123456789012:function:testFunction', @@ -260,7 +284,8 @@ def test_get_function(): ) result = conn.get_function(FunctionName='testFunction') - result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + result['ResponseMetadata'].pop('RetryAttempts', None) # Botocore inserts retry attempts not seen in Python27 result.should.equal({ "Code": { @@ -315,7 +340,9 @@ def test_delete_function(): ) success_result = conn.delete_function(FunctionName='testFunction') - success_result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + success_result['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + success_result['ResponseMetadata'].pop('RetryAttempts', None) # Botocore inserts retry attempts not seen in Python27 + success_result.should.equal({'ResponseMetadata': {'HTTPStatusCode': 204}}) conn.delete_function.when.called_with(FunctionName='testFunctionThatDoesntExist').should.throw(botocore.client.ClientError) @@ -380,7 +407,9 @@ def test_list_create_list_get_delete_list(): conn.list_functions()['Functions'].should.equal([expected_function_result['Configuration']]) func = conn.get_function(FunctionName='testFunction') - func['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + func['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + func['ResponseMetadata'].pop('RetryAttempts', None) # Botocore inserts retry attempts not seen in Python27 + func.should.equal(expected_function_result) conn.delete_function(FunctionName='testFunction') diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index bd8b32fab..d3928dc48 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -1737,7 +1737,7 @@ def _process_lamda(pfunc): def get_test_zip_file1(): - pfunc = b""" + pfunc = """ def lambda_handler(event, context): return (event, context) """ @@ -1756,7 +1756,7 @@ def test_lambda_function(): "Type": "AWS::Lambda::Function", "Properties": { "Code": { - "ZipFile": base64.b64encode(get_test_zip_file1()) + "ZipFile": base64.b64encode(get_test_zip_file1()).decode('utf-8') }, "Handler": "lambda_function.handler", "Description": "Test function", From c4ea3cf751e8f5899bcd4389315b8dd627b885da Mon Sep 17 00:00:00 2001 From: rocky4570fft Date: Mon, 10 Oct 2016 01:22:16 +1000 Subject: [PATCH 16/18] correct expletive and remove debug line --- moto/awslambda/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 563e7f43b..260d2d338 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -129,9 +129,9 @@ class LambdaFunction(object): mycode = "\n".join(['import json', self.convert(self.code), self.convert('print(lambda_handler(%s, %s))' % (self.is_json(self.convert(event)), context))]) - print("moto_lambda_debug: ", mycode) + #print("moto_lambda_debug: ", mycode) except Exception as ex: - print('fuck ', ex) + print("Exception %s", ex) try: codeOut = StringIO() From ddf2f5a754455fe2b1b434089cade9083d6b7d33 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 9 Oct 2016 20:23:56 -0400 Subject: [PATCH 17/18] Fix firehose to work without Redshift config. --- moto/kinesis/models.py | 75 ++++++++++++++++++----------- moto/kinesis/responses.py | 10 ++++ tests/test_kinesis/test_firehose.py | 54 +++++++++++++++++++++ 3 files changed, 112 insertions(+), 27 deletions(-) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index 4de10a554..e0e20da3f 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -176,17 +176,23 @@ class FirehoseRecord(object): class DeliveryStream(object): def __init__(self, stream_name, **stream_kwargs): self.name = stream_name - self.redshift_username = stream_kwargs['redshift_username'] - self.redshift_password = stream_kwargs['redshift_password'] - self.redshift_jdbc_url = stream_kwargs['redshift_jdbc_url'] - self.redshift_role_arn = stream_kwargs['redshift_role_arn'] - self.redshift_copy_command = stream_kwargs['redshift_copy_command'] + self.redshift_username = stream_kwargs.get('redshift_username') + self.redshift_password = stream_kwargs.get('redshift_password') + self.redshift_jdbc_url = stream_kwargs.get('redshift_jdbc_url') + self.redshift_role_arn = stream_kwargs.get('redshift_role_arn') + self.redshift_copy_command = stream_kwargs.get('redshift_copy_command') - self.redshift_s3_role_arn = stream_kwargs['redshift_s3_role_arn'] - self.redshift_s3_bucket_arn = stream_kwargs['redshift_s3_bucket_arn'] - self.redshift_s3_prefix = stream_kwargs['redshift_s3_prefix'] + self.s3_role_arn = stream_kwargs.get('s3_role_arn') + self.s3_bucket_arn = stream_kwargs.get('s3_bucket_arn') + self.s3_prefix = stream_kwargs.get('s3_prefix') + self.s3_compression_format = stream_kwargs.get('s3_compression_format', 'UNCOMPRESSED') + self.s3_buffering_hings = stream_kwargs.get('s3_buffering_hings') + + self.redshift_s3_role_arn = stream_kwargs.get('redshift_s3_role_arn') + self.redshift_s3_bucket_arn = stream_kwargs.get('redshift_s3_bucket_arn') + self.redshift_s3_prefix = stream_kwargs.get('redshift_s3_prefix') self.redshift_s3_compression_format = stream_kwargs.get('redshift_s3_compression_format', 'UNCOMPRESSED') - self.redshift_s3_buffering_hings = stream_kwargs['redshift_s3_buffering_hings'] + self.redshift_s3_buffering_hings = stream_kwargs.get('redshift_s3_buffering_hings') self.records = [] self.status = 'ACTIVE' @@ -197,6 +203,38 @@ class DeliveryStream(object): def arn(self): return 'arn:aws:firehose:us-east-1:123456789012:deliverystream/{0}'.format(self.name) + def destinations_to_dict(self): + if self.s3_role_arn: + return [{ + 'DestinationId': 'string', + 'S3DestinationDescription': { + 'RoleARN': self.s3_role_arn, + 'BucketARN': self.s3_bucket_arn, + 'Prefix': self.s3_prefix, + 'BufferingHints': self.s3_buffering_hings, + 'CompressionFormat': self.s3_compression_format, + } + }] + else: + return [{ + "DestinationId": "string", + "RedshiftDestinationDescription": { + "ClusterJDBCURL": self.redshift_jdbc_url, + "CopyCommand": self.redshift_copy_command, + "RoleARN": self.redshift_role_arn, + "S3DestinationDescription": { + "BucketARN": self.redshift_s3_bucket_arn, + "BufferingHints": self.redshift_s3_buffering_hings, + "CompressionFormat": self.redshift_s3_compression_format, + "Prefix": self.redshift_s3_prefix, + "RoleARN": self.redshift_s3_role_arn + }, + "Username": self.redshift_username, + }, + } + ] + + def to_dict(self): return { "DeliveryStreamDescription": { @@ -204,24 +242,7 @@ class DeliveryStream(object): "DeliveryStreamARN": self.arn, "DeliveryStreamName": self.name, "DeliveryStreamStatus": self.status, - "Destinations": [ - { - "DestinationId": "string", - "RedshiftDestinationDescription": { - "ClusterJDBCURL": self.redshift_jdbc_url, - "CopyCommand": self.redshift_copy_command, - "RoleARN": self.redshift_role_arn, - "S3DestinationDescription": { - "BucketARN": self.redshift_s3_bucket_arn, - "BufferingHints": self.redshift_s3_buffering_hings, - "CompressionFormat": self.redshift_s3_compression_format, - "Prefix": self.redshift_s3_prefix, - "RoleARN": self.redshift_s3_role_arn - }, - "Username": self.redshift_username, - }, - } - ], + "Destinations": self.destinations_to_dict(), "HasMoreDestinations": False, "LastUpdateTimestamp": time.mktime(self.last_updated.timetuple()), "VersionId": "string", diff --git a/moto/kinesis/responses.py b/moto/kinesis/responses.py index b52bdedf0..264f53a2c 100644 --- a/moto/kinesis/responses.py +++ b/moto/kinesis/responses.py @@ -139,6 +139,16 @@ class KinesisResponse(BaseResponse): 'redshift_s3_compression_format': redshift_s3_config.get('CompressionFormat'), 'redshift_s3_buffering_hings': redshift_s3_config['BufferingHints'], } + else: + # S3 Config + s3_config = self.parameters['S3DestinationConfiguration'] + stream_kwargs = { + 's3_role_arn': s3_config['RoleARN'], + 's3_bucket_arn': s3_config['BucketARN'], + 's3_prefix': s3_config['Prefix'], + 's3_compression_format': s3_config.get('CompressionFormat'), + 's3_buffering_hings': s3_config['BufferingHints'], + } stream = self.kinesis_backend.create_delivery_stream(stream_name, **stream_kwargs) return json.dumps({ 'DeliveryStreamARN': stream.arn diff --git a/tests/test_kinesis/test_firehose.py b/tests/test_kinesis/test_firehose.py index 37585fe5e..14ee1916b 100644 --- a/tests/test_kinesis/test_firehose.py +++ b/tests/test_kinesis/test_firehose.py @@ -87,6 +87,60 @@ def test_create_stream(): }) +@mock_kinesis +@freeze_time("2015-03-01") +def test_create_stream_without_redshift(): + client = boto3.client('firehose', region_name='us-east-1') + + response = client.create_delivery_stream( + DeliveryStreamName="stream1", + S3DestinationConfiguration={ + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'BucketARN': 'arn:aws:s3:::kinesis-test', + 'Prefix': 'myFolder/', + 'BufferingHints': { + 'SizeInMBs': 123, + 'IntervalInSeconds': 124 + }, + 'CompressionFormat': 'UNCOMPRESSED', + } + ) + stream_arn = response['DeliveryStreamARN'] + + response = client.describe_delivery_stream(DeliveryStreamName='stream1') + stream_description = response['DeliveryStreamDescription'] + + # Sure and Freezegun don't play nicely together + created = stream_description.pop('CreateTimestamp') + last_updated = stream_description.pop('LastUpdateTimestamp') + from dateutil.tz import tzlocal + assert created == datetime.datetime(2015, 3, 1, tzinfo=tzlocal()) + assert last_updated == datetime.datetime(2015, 3, 1, tzinfo=tzlocal()) + + stream_description.should.equal({ + 'DeliveryStreamName': 'stream1', + 'DeliveryStreamARN': stream_arn, + 'DeliveryStreamStatus': 'ACTIVE', + 'VersionId': 'string', + 'Destinations': [ + { + 'DestinationId': 'string', + 'S3DestinationDescription': { + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'RoleARN': 'arn:aws:iam::123456789012:role/firehose_delivery_role', + 'BucketARN': 'arn:aws:s3:::kinesis-test', + 'Prefix': 'myFolder/', + 'BufferingHints': { + 'SizeInMBs': 123, + 'IntervalInSeconds': 124 + }, + 'CompressionFormat': 'UNCOMPRESSED', + } + }, + ], + "HasMoreDestinations": False, + }) + @mock_kinesis @freeze_time("2015-03-01") def test_deescribe_non_existant_stream(): From fef3437db2d377b6058eec1188c6808c588b20ad Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 9 Oct 2016 21:20:53 -0400 Subject: [PATCH 18/18] Throw error on terminating empty instance list. Closes #697. --- moto/ec2/models.py | 2 ++ tests/test_ec2/test_instances.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index f90ceae9f..e3342c8c8 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -632,6 +632,8 @@ class InstanceBackend(object): def terminate_instances(self, instance_ids): terminated_instances = [] + if not instance_ids: + raise EC2ClientError("InvalidParameterCombination", "No instances specified") for instance in self.get_multi_instances_by_id(instance_ids): instance.terminate() terminated_instances.append(instance) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 58b1f693a..e3959d8e0 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -73,6 +73,13 @@ def test_instance_launch_and_terminate(): instance = reservations[0].instances[0] instance.state.should.equal('terminated') + +@mock_ec2 +def test_terminate_empty_instances(): + conn = boto.connect_ec2('the_key', 'the_secret') + conn.terminate_instances.when.called_with([]).should.throw(EC2ResponseError) + + @freeze_time("2014-01-01 05:00:00") @mock_ec2 def test_instance_attach_volume(): @@ -330,6 +337,7 @@ def test_get_instances_filtering_by_tag(): reservations[0].instances[0].id.should.equal(instance1.id) reservations[0].instances[1].id.should.equal(instance3.id) + @mock_ec2 def test_get_instances_filtering_by_tag_value(): conn = boto.connect_ec2()