From ab595279ad67254a50da6713c30f02ba1876215c Mon Sep 17 00:00:00 2001 From: Brian Pandola Date: Sat, 30 Sep 2017 21:51:05 -0700 Subject: [PATCH 01/20] Fix boto/boto3 multi-param discrepancies boto uses the param.member.N syntax, but boto3 replaces the generic .member with a more specific identifier. Example: boto: ClusterSecurityGroups.member.N boto3: ClusterSecurityGroups.ClusterSecurityGroupName.N This commit addresses this issue for the ClusterSecurityGroups, SubnetIds, and VpcSecurityGroupIds parameters. --- moto/redshift/responses.py | 44 +++++++++++++---------- tests/test_redshift/test_redshift.py | 52 ++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index 0dbf35cb2..52ca908e8 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -66,6 +66,24 @@ class RedshiftResponse(BaseResponse): count += 1 return unpacked_list + def _get_cluster_security_groups(self): + cluster_security_groups = self._get_multi_param('ClusterSecurityGroups.member') + if not cluster_security_groups: + cluster_security_groups = self._get_multi_param('ClusterSecurityGroups.ClusterSecurityGroupName') + return cluster_security_groups + + def _get_vpc_security_group_ids(self): + vpc_security_group_ids = self._get_multi_param('VpcSecurityGroupIds.member') + if not vpc_security_group_ids: + vpc_security_group_ids = self._get_multi_param('VpcSecurityGroupIds.VpcSecurityGroupId') + return vpc_security_group_ids + + def _get_subnet_ids(self): + subnet_ids = self._get_multi_param('SubnetIds.member') + if not subnet_ids: + subnet_ids = self._get_multi_param('SubnetIds.SubnetIdentifier') + return subnet_ids + def create_cluster(self): cluster_kwargs = { "cluster_identifier": self._get_param('ClusterIdentifier'), @@ -74,8 +92,8 @@ class RedshiftResponse(BaseResponse): "master_user_password": self._get_param('MasterUserPassword'), "db_name": self._get_param('DBName'), "cluster_type": self._get_param('ClusterType'), - "cluster_security_groups": self._get_multi_param('ClusterSecurityGroups.member'), - "vpc_security_group_ids": self._get_multi_param('VpcSecurityGroupIds.member'), + "cluster_security_groups": self._get_cluster_security_groups(), + "vpc_security_group_ids": self._get_vpc_security_group_ids(), "cluster_subnet_group_name": self._get_param('ClusterSubnetGroupName'), "availability_zone": self._get_param('AvailabilityZone'), "preferred_maintenance_window": self._get_param('PreferredMaintenanceWindow'), @@ -116,10 +134,8 @@ class RedshiftResponse(BaseResponse): "publicly_accessible": self._get_param("PubliclyAccessible"), "cluster_parameter_group_name": self._get_param( 'ClusterParameterGroupName'), - "cluster_security_groups": self._get_multi_param( - 'ClusterSecurityGroups.member'), - "vpc_security_group_ids": self._get_multi_param( - 'VpcSecurityGroupIds.member'), + "cluster_security_groups": self._get_cluster_security_groups(), + "vpc_security_group_ids": self._get_vpc_security_group_ids(), "preferred_maintenance_window": self._get_param( 'PreferredMaintenanceWindow'), "automated_snapshot_retention_period": self._get_int_param( @@ -161,8 +177,8 @@ class RedshiftResponse(BaseResponse): "node_type": self._get_param('NodeType'), "master_user_password": self._get_param('MasterUserPassword'), "cluster_type": self._get_param('ClusterType'), - "cluster_security_groups": self._get_multi_param('ClusterSecurityGroups.member'), - "vpc_security_group_ids": self._get_multi_param('VpcSecurityGroupIds.member'), + "cluster_security_groups": self._get_cluster_security_groups(), + "vpc_security_group_ids": self._get_vpc_security_group_ids(), "cluster_subnet_group_name": self._get_param('ClusterSubnetGroupName'), "preferred_maintenance_window": self._get_param('PreferredMaintenanceWindow'), "cluster_parameter_group_name": self._get_param('ClusterParameterGroupName'), @@ -173,12 +189,6 @@ class RedshiftResponse(BaseResponse): "publicly_accessible": self._get_param("PubliclyAccessible"), "encrypted": self._get_param("Encrypted"), } - # There's a bug in boto3 where the security group ids are not passed - # according to the AWS documentation - if not request_kwargs['vpc_security_group_ids']: - request_kwargs['vpc_security_group_ids'] = self._get_multi_param( - 'VpcSecurityGroupIds.VpcSecurityGroupId') - cluster_kwargs = {} # We only want parameters that were actually passed in, otherwise # we'll stomp all over our cluster metadata with None values. @@ -217,11 +227,7 @@ class RedshiftResponse(BaseResponse): def create_cluster_subnet_group(self): cluster_subnet_group_name = self._get_param('ClusterSubnetGroupName') description = self._get_param('Description') - subnet_ids = self._get_multi_param('SubnetIds.member') - # There's a bug in boto3 where the subnet ids are not passed - # according to the AWS documentation - if not subnet_ids: - subnet_ids = self._get_multi_param('SubnetIds.SubnetIdentifier') + subnet_ids = self._get_subnet_ids() tags = self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value')) subnet_group = self.redshift_backend.create_cluster_subnet_group( diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index dca475374..cebaa3ec7 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -216,6 +216,33 @@ def test_create_cluster_with_security_group(): set(group_names).should.equal(set(["security_group1", "security_group2"])) +@mock_redshift +def test_create_cluster_with_security_group_boto3(): + client = boto3.client('redshift', region_name='us-east-1') + client.create_cluster_security_group( + ClusterSecurityGroupName="security_group1", + Description="This is my security group", + ) + client.create_cluster_security_group( + ClusterSecurityGroupName="security_group2", + Description="This is my security group", + ) + + cluster_identifier = 'my_cluster' + client.create_cluster( + ClusterIdentifier=cluster_identifier, + NodeType="dw.hs1.xlarge", + MasterUsername="username", + MasterUserPassword="password", + ClusterSecurityGroups=["security_group1", "security_group2"] + ) + response = client.describe_clusters(ClusterIdentifier=cluster_identifier) + cluster = response['Clusters'][0] + group_names = [group['ClusterSecurityGroupName'] + for group in cluster['ClusterSecurityGroups']] + set(group_names).should.equal({"security_group1", "security_group2"}) + + @mock_redshift_deprecated @mock_ec2_deprecated def test_create_cluster_with_vpc_security_groups(): @@ -242,6 +269,31 @@ def test_create_cluster_with_vpc_security_groups(): list(group_ids).should.equal([security_group.id]) +@mock_redshift +@mock_ec2 +def test_create_cluster_with_vpc_security_groups_boto3(): + ec2 = boto3.resource('ec2', region_name='us-east-1') + vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16') + client = boto3.client('redshift', region_name='us-east-1') + cluster_id = 'my_cluster' + security_group = ec2.create_security_group( + Description="vpc_security_group", + GroupName="a group", + VpcId=vpc.id) + client.create_cluster( + ClusterIdentifier=cluster_id, + NodeType="dw.hs1.xlarge", + MasterUsername="username", + MasterUserPassword="password", + VpcSecurityGroupIds=[security_group.id], + ) + response = client.describe_clusters(ClusterIdentifier=cluster_id) + cluster = response['Clusters'][0] + group_ids = [group['VpcSecurityGroupId'] + for group in cluster['VpcSecurityGroups']] + list(group_ids).should.equal([security_group.id]) + + @mock_redshift_deprecated def test_create_cluster_with_parameter_group(): conn = boto.connect_redshift() From 37ae61871c06412157b5b178546e7e63d34bd914 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 15:01:33 -0700 Subject: [PATCH 02/20] add model methods for iam attached group policies --- moto/iam/models.py | 68 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index a7e584284..0f0f4c058 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -82,6 +82,10 @@ class ManagedPolicy(Policy): self.attachment_count -= 1 del role.managed_policies[self.name] + def attach_to_group(self, group): + self.attachment_count += 1 + group.managed_policies[self.name] = self + def attach_to_user(self, user): self.attachment_count += 1 user.managed_policies[self.name] = self @@ -249,6 +253,7 @@ class Group(BaseModel): ) self.users = [] + self.managed_policies = {} self.policies = {} def get_cfn_attribute(self, attribute_name): @@ -433,14 +438,36 @@ class IAMBackend(BaseBackend): except KeyError: raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) + def attach_group_policy(self, policy_arn, group_name): + arns = dict((p.arn, p) for p in self.managed_policies.values()) + try: + policy = arns[policy_arn] + except KeyError: + raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) + policy.attach_to_group(self.get_group(group_name)) + + def detach_group_policy(self, policy_arn, group_name): + arns = dict((p.arn, p) for p in self.managed_policies.values()) + try: + policy = arns[policy_arn] + except KeyError: + raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) + policy.detach_from_group(self.get_group(group_name)) + def attach_user_policy(self, policy_arn, user_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) - policy = arns[policy_arn] + try: + policy = arns[policy_arn] + except KeyError: + raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) policy.attach_to_user(self.get_user(user_name)) def detach_user_policy(self, policy_arn, user_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) - policy = arns[policy_arn] + try: + policy = arns[policy_arn] + except KeyError: + raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) policy.detach_from_user(self.get_user(user_name)) def create_policy(self, description, path, policy_document, policy_name): @@ -458,39 +485,15 @@ class IAMBackend(BaseBackend): def list_attached_role_policies(self, role_name, marker=None, max_items=100, path_prefix='/'): policies = self.get_role(role_name).managed_policies.values() + return self._filter_attached_policies(policies, marker, max_items, path_prefix) - if path_prefix: - policies = [p for p in policies if p.path.startswith(path_prefix)] - - policies = sorted(policies, key=lambda policy: policy.name) - start_idx = int(marker) if marker else 0 - - policies = policies[start_idx:start_idx + max_items] - - if len(policies) < max_items: - marker = None - else: - marker = str(start_idx + max_items) - - return policies, marker + def list_attached_group_policies(self, group_name, marker=None, max_items=100, path_prefix='/'): + policies = self.get_group(group_name).managed_policies.values() + return self._filter_attached_policies(policies, marker, max_items, path_prefix) def list_attached_user_policies(self, user_name, marker=None, max_items=100, path_prefix='/'): policies = self.get_user(user_name).managed_policies.values() - - if path_prefix: - policies = [p for p in policies if p.path.startswith(path_prefix)] - - policies = sorted(policies, key=lambda policy: policy.name) - start_idx = int(marker) if marker else 0 - - policies = policies[start_idx:start_idx + max_items] - - if len(policies) < max_items: - marker = None - else: - marker = str(start_idx + max_items) - - return policies, marker + return self._filter_attached_policies(policies, marker, max_items, path_prefix) def list_policies(self, marker, max_items, only_attached, path_prefix, scope): policies = self.managed_policies.values() @@ -504,6 +507,9 @@ class IAMBackend(BaseBackend): policies = [p for p in policies if not isinstance( p, AWSManagedPolicy)] + return self._filter_attached_policies(policies, marker, max_items, path_prefix) + + def _filter_attached_policies(self, policies, marker, max_items, path_prefix): if path_prefix: policies = [p for p in policies if p.path.startswith(path_prefix)] From 3788e42f3518ab96ae00945a3e26a9379804fa79 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 15:01:52 -0700 Subject: [PATCH 03/20] implement handlers for iam attached group policies --- moto/iam/responses.py | 59 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 13688869e..6ca49b830 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -20,6 +20,20 @@ class IamResponse(BaseResponse): template = self.response_template(GENERIC_EMPTY_TEMPLATE) return template.render(name="DetachRolePolicyResponse") + def attach_group_policy(self): + policy_arn = self._get_param('PolicyArn') + group_name = self._get_param('GroupName') + iam_backend.attach_group_policy(policy_arn, group_name) + template = self.response_template(ATTACH_GROUP_POLICY_TEMPLATE) + return template.render() + + def detach_group_policy(self): + policy_arn = self._get_param('PolicyArn') + group_name = self._get_param('GroupName') + iam_backend.detach_group_policy(policy_arn, group_name) + template = self.response_template(DETACH_GROUP_POLICY_TEMPLATE) + return template.render() + def attach_user_policy(self): policy_arn = self._get_param('PolicyArn') user_name = self._get_param('UserName') @@ -54,6 +68,17 @@ class IamResponse(BaseResponse): template = self.response_template(LIST_ATTACHED_ROLE_POLICIES_TEMPLATE) return template.render(policies=policies, marker=marker) + def list_attached_group_policies(self): + marker = self._get_param('Marker') + max_items = self._get_int_param('MaxItems', 100) + path_prefix = self._get_param('PathPrefix', '/') + group_name = self._get_param('GroupName') + policies, marker = iam_backend.list_attached_group_policies( + group_name, marker=marker, max_items=max_items, + path_prefix=path_prefix) + template = self.response_template(LIST_ATTACHED_GROUP_POLICIES_TEMPLATE) + return template.render(policies=policies, marker=marker) + def list_attached_user_policies(self): marker = self._get_param('Marker') max_items = self._get_int_param('MaxItems', 100) @@ -520,6 +545,18 @@ DETACH_USER_POLICY_TEMPLATE = """ """ +ATTACH_GROUP_POLICY_TEMPLATE = """ + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + +DETACH_GROUP_POLICY_TEMPLATE = """ + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + CREATE_POLICY_TEMPLATE = """ @@ -560,6 +597,28 @@ LIST_ATTACHED_ROLE_POLICIES_TEMPLATE = """ """ +LIST_ATTACHED_GROUP_POLICIES_TEMPLATE = """ + + {% if marker is none %} + false + {% else %} + true + {{ marker }} + {% endif %} + + {% for policy in policies %} + + {{ policy.name }} + {{ policy.arn }} + + {% endfor %} + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + LIST_ATTACHED_USER_POLICIES_TEMPLATE = """ {% if marker is none %} From 9f02a84d8da4127f581ac920fdc6e6b0beb9666e Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 15:02:00 -0700 Subject: [PATCH 04/20] test attaching group policies --- tests/test_iam/test_iam_groups.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_iam/test_iam_groups.py b/tests/test_iam/test_iam_groups.py index 9d5095884..5270fb96e 100644 --- a/tests/test_iam/test_iam_groups.py +++ b/tests/test_iam/test_iam_groups.py @@ -82,6 +82,23 @@ def test_put_group_policy(): conn.put_group_policy('my-group', 'my-policy', '{"some": "json"}') +@mock_iam +def test_attach_group_policies(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_group(GroupName='my-group') + conn.list_attached_group_policies(GroupName='my-group')['AttachedPolicies'].should.be.empty + policy_arn = 'arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role' + conn.list_attached_group_policies(GroupName='my-group')['AttachedPolicies'].should.be.empty + conn.attach_group_policy(GroupName='my-group', PolicyArn=policy_arn) + conn.list_attached_group_policies(GroupName='my-group')['AttachedPolicies'].should.equal( + [ + { + 'PolicyName': 'AmazonElasticMapReduceforEC2Role', + 'PolicyArn': policy_arn, + } + ]) + + @mock_iam_deprecated() def test_get_group_policy(): conn = boto.connect_iam() From cdb1ebf6669ab95838112a9105d12753dca9a12e Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 15:02:06 -0700 Subject: [PATCH 05/20] pep8 fixes --- tests/test_iam/test_iam_groups.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_iam/test_iam_groups.py b/tests/test_iam/test_iam_groups.py index 5270fb96e..2b308f70a 100644 --- a/tests/test_iam/test_iam_groups.py +++ b/tests/test_iam/test_iam_groups.py @@ -107,7 +107,8 @@ def test_get_group_policy(): conn.get_group_policy('my-group', 'my-policy') conn.put_group_policy('my-group', 'my-policy', '{"some": "json"}') - policy = conn.get_group_policy('my-group', 'my-policy') + conn.get_group_policy('my-group', 'my-policy') + @mock_iam_deprecated() def test_get_all_group_policies(): @@ -124,6 +125,6 @@ def test_get_all_group_policies(): def test_list_group_policies(): conn = boto3.client('iam', region_name='us-east-1') conn.create_group(GroupName='my-group') - policies = conn.list_group_policies(GroupName='my-group')['PolicyNames'].should.be.empty + conn.list_group_policies(GroupName='my-group')['PolicyNames'].should.be.empty conn.put_group_policy(GroupName='my-group', PolicyName='my-policy', PolicyDocument='{"some": "json"}') - policies = conn.list_group_policies(GroupName='my-group')['PolicyNames'].should.equal(['my-policy']) + conn.list_group_policies(GroupName='my-group')['PolicyNames'].should.equal(['my-policy']) From 353f8387a2926c0f3776992bdd657f084b9e3ae0 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 15:04:59 -0700 Subject: [PATCH 06/20] implementing detach_group_policy --- moto/iam/models.py | 32 ++++++++++--------------------- tests/test_iam/test_iam_groups.py | 3 +++ 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 0f0f4c058..18ed513b4 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -74,25 +74,13 @@ class ManagedPolicy(Policy): is_attachable = True - def attach_to_role(self, role): + def attach_to(self, obj): self.attachment_count += 1 - role.managed_policies[self.name] = self + obj.managed_policies[self.name] = self - def detach_from_role(self, role): + def detach_from(self, obj): self.attachment_count -= 1 - del role.managed_policies[self.name] - - def attach_to_group(self, group): - self.attachment_count += 1 - group.managed_policies[self.name] = self - - def attach_to_user(self, user): - self.attachment_count += 1 - user.managed_policies[self.name] = self - - def detach_from_user(self, user): - self.attachment_count -= 1 - del user.managed_policies[self.name] + del obj.managed_policies[self.name] class AWSManagedPolicy(ManagedPolicy): @@ -428,13 +416,13 @@ class IAMBackend(BaseBackend): def attach_role_policy(self, policy_arn, role_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) policy = arns[policy_arn] - policy.attach_to_role(self.get_role(role_name)) + policy.attach_to(self.get_role(role_name)) def detach_role_policy(self, policy_arn, role_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) try: policy = arns[policy_arn] - policy.detach_from_role(self.get_role(role_name)) + policy.detach_from(self.get_role(role_name)) except KeyError: raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) @@ -444,7 +432,7 @@ class IAMBackend(BaseBackend): policy = arns[policy_arn] except KeyError: raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) - policy.attach_to_group(self.get_group(group_name)) + policy.attach_to(self.get_group(group_name)) def detach_group_policy(self, policy_arn, group_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) @@ -452,7 +440,7 @@ class IAMBackend(BaseBackend): policy = arns[policy_arn] except KeyError: raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) - policy.detach_from_group(self.get_group(group_name)) + policy.detach_from(self.get_group(group_name)) def attach_user_policy(self, policy_arn, user_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) @@ -460,7 +448,7 @@ class IAMBackend(BaseBackend): policy = arns[policy_arn] except KeyError: raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) - policy.attach_to_user(self.get_user(user_name)) + policy.attach_to(self.get_user(user_name)) def detach_user_policy(self, policy_arn, user_name): arns = dict((p.arn, p) for p in self.managed_policies.values()) @@ -468,7 +456,7 @@ class IAMBackend(BaseBackend): policy = arns[policy_arn] except KeyError: raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn)) - policy.detach_from_user(self.get_user(user_name)) + policy.detach_from(self.get_user(user_name)) def create_policy(self, description, path, policy_document, policy_name): policy = ManagedPolicy( diff --git a/tests/test_iam/test_iam_groups.py b/tests/test_iam/test_iam_groups.py index 2b308f70a..49c7987f6 100644 --- a/tests/test_iam/test_iam_groups.py +++ b/tests/test_iam/test_iam_groups.py @@ -98,6 +98,9 @@ def test_attach_group_policies(): } ]) + conn.detach_group_policy(GroupName='my-group', PolicyArn=policy_arn) + conn.list_attached_group_policies(GroupName='my-group')['AttachedPolicies'].should.be.empty + @mock_iam_deprecated() def test_get_group_policy(): From ea26466e6d498f2ed77b024b32c141ad8b306182 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Mon, 2 Oct 2017 07:17:02 +0900 Subject: [PATCH 07/20] Add more files and lines when scaffolding (#1222) * auto-generatr urls.py * add mocks to moto/__init__.py and moto/backends.py * add uri to urls.py * change output of scaffold.py --- scripts/scaffold.py | 114 +++++++++++++++++++++++++++++++- scripts/template/lib/urls.py.j2 | 9 +++ 2 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 scripts/template/lib/urls.py.j2 diff --git a/scripts/scaffold.py b/scripts/scaffold.py index 5373be40d..b1c9f3a0f 100755 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -107,6 +107,62 @@ def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None f.write(rendered) +def append_mock_to_init_py(service): + path = os.path.join(os.path.dirname(__file__), '..', 'moto', '__init__.py') + with open(path) as f: + lines = [_.replace('\n', '') for _ in f.readlines()] + + if any(_ for _ in lines if re.match('^from.*mock_{}.*$'.format(service), _)): + return + filtered_lines = [_ for _ in lines if re.match('^from.*mock.*$', _)] + last_import_line_index = lines.index(filtered_lines[-1]) + + new_line = 'from .{} import mock_{} # flake8: noqa'.format(service, service) + lines.insert(last_import_line_index + 1, new_line) + + body = '\n'.join(lines) + '\n' + with open(path, 'w') as f: + f.write(body) + + +def append_mock_import_to_backends_py(service): + path = os.path.join(os.path.dirname(__file__), '..', 'moto', 'backends.py') + with open(path) as f: + lines = [_.replace('\n', '') for _ in f.readlines()] + + if any(_ for _ in lines if re.match('^from moto\.{}.*{}_backends.*$'.format(service, service), _)): + return + filtered_lines = [_ for _ in lines if re.match('^from.*backends.*$', _)] + last_import_line_index = lines.index(filtered_lines[-1]) + + new_line = 'from moto.{} import {}_backends'.format(service, service) + lines.insert(last_import_line_index + 1, new_line) + + body = '\n'.join(lines) + '\n' + with open(path, 'w') as f: + f.write(body) + +def append_mock_dict_to_backends_py(service): + path = os.path.join(os.path.dirname(__file__), '..', 'moto', 'backends.py') + with open(path) as f: + lines = [_.replace('\n', '') for _ in f.readlines()] + + # 'xray': xray_backends + if any(_ for _ in lines if re.match(".*'{}': {}_backends.*".format(service, service), _)): + return + filtered_lines = [_ for _ in lines if re.match(".*'.*':.*_backends.*", _)] + last_elem_line_index = lines.index(filtered_lines[-1]) + + new_line = " '{}': {}_backends,".format(service, service) + prev_line = lines[last_elem_line_index] + if not prev_line.endswith('{') and not prev_line.endswith(','): + lines[last_elem_line_index] += ',' + lines.insert(last_elem_line_index + 1, new_line) + + body = '\n'.join(lines) + '\n' + with open(path, 'w') as f: + f.write(body) + def initialize_service(service, operation, api_protocol): """create lib and test dirs if not exist """ @@ -115,11 +171,14 @@ def initialize_service(service, operation, api_protocol): print_progress('Initializing service', service, 'green') - service_class = boto3.client(service).__class__.__name__ + client = boto3.client(service) + service_class = client.__class__.__name__ + endpoint_prefix = client._service_model.endpoint_prefix tmpl_context = { 'service': service, - 'service_class': service_class + 'service_class': service_class, + 'endpoint_prefix': endpoint_prefix } # initialize service directory @@ -148,6 +207,11 @@ def initialize_service(service, operation, api_protocol): tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename ) + # append mock to init files + append_mock_to_init_py(service) + append_mock_import_to_backends_py(service) + append_mock_dict_to_backends_py(service) + def to_upper_camel_case(s): return ''.join([_.title() for _ in s.split('_')]) @@ -324,6 +388,41 @@ def insert_code_to_class(path, base_class, new_code): f.write(body) +def insert_url(service, operation): + client = boto3.client(service) + service_class = client.__class__.__name__ + aws_operation_name = to_upper_camel_case(operation) + uri = client._service_model.operation_model(aws_operation_name).http['requestUri'] + + path = os.path.join(os.path.dirname(__file__), '..', 'moto', service, 'urls.py') + with open(path) as f: + lines = [_.replace('\n', '') for _ in f.readlines()] + + if any(_ for _ in lines if re.match(uri, _)): + return + + url_paths_found = False + last_elem_line_index = -1 + for i, line in enumerate(lines): + if line.startswith('url_paths'): + url_paths_found = True + if url_paths_found and line.startswith('}'): + last_elem_line_index = i - 1 + + prev_line = lines[last_elem_line_index] + if not prev_line.endswith('{') and not prev_line.endswith(','): + lines[last_elem_line_index] += ',' + + new_line = " '{0}%s$': %sResponse.dispatch," % ( + uri, service_class + ) + lines.insert(last_elem_line_index + 1, new_line) + + body = '\n'.join(lines) + '\n' + with open(path, 'w') as f: + f.write(body) + + def insert_query_codes(service, operation): func_in_responses = get_function_in_responses(service, operation, 'query') func_in_models = get_function_in_models(service, operation) @@ -346,6 +445,9 @@ def insert_query_codes(service, operation): print_progress('inserting code', models_path, 'green') insert_code_to_class(models_path, BaseBackend, func_in_models) + # edit urls.py + insert_url(service, operation) + def insert_json_codes(service, operation): func_in_responses = get_function_in_responses(service, operation, 'json') func_in_models = get_function_in_models(service, operation) @@ -360,6 +462,9 @@ def insert_json_codes(service, operation): print_progress('inserting code', models_path, 'green') insert_code_to_class(models_path, BaseBackend, func_in_models) + # edit urls.py + insert_url(service, operation) + def insert_restjson_codes(service, operation): func_in_models = get_function_in_models(service, operation) @@ -369,6 +474,9 @@ def insert_restjson_codes(service, operation): print_progress('inserting code', models_path, 'green') insert_code_to_class(models_path, BaseBackend, func_in_models) + # edit urls.py + insert_url(service, operation) + @click.command() def main(): service, operation = select_service_and_operation() @@ -383,7 +491,7 @@ def main(): else: print_progress('skip inserting code', 'api protocol "{}" is not supported'.format(api_protocol), 'yellow') - click.echo('You will still need to make "{0}/urls.py", add the backend into "backends.py" and add the mock into "__init__.py"'.format(service)) + click.echo('You will still need to add the mock into "__init__.py"'.format(service)) if __name__ == '__main__': main() diff --git a/scripts/template/lib/urls.py.j2 b/scripts/template/lib/urls.py.j2 new file mode 100644 index 000000000..53cc03c0e --- /dev/null +++ b/scripts/template/lib/urls.py.j2 @@ -0,0 +1,9 @@ +from __future__ import unicode_literals +from .responses import {{ service_class }}Response + +url_bases = [ + "https?://{{ endpoint_prefix }}.(.+).amazonaws.com", +] + +url_paths = { +} From c5b6f5ea07321e5ae72b8fd85b1d456dae229750 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 17:00:48 -0700 Subject: [PATCH 08/20] bumping to version 1.1.20 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d4ce3d5f1..378119925 100755 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ else: setup( name='moto', - version='1.1.19', + version='1.1.20', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 6abd929c4964f03333f40d44435d3d8c08afa11a Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 17:02:52 -0700 Subject: [PATCH 09/20] Update changelog for 1.1.20 --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40edb4204..084540e62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,16 @@ Moto Changelog Latest ------ -1.1.16 +1.1.20 +----- + + * Improved `make scaffold` + * Implemented IAM attached group policies + * Redshift: fixed multi-params + * Redshift: implement taggable resources + * Lambda + SNS: Major enhancements + +1.1.19 ----- * Fixing regression from 1.1.15 From e8c868f1b700b5ef3e58dd027fbd6303e434c568 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Sun, 1 Oct 2017 17:06:24 -0700 Subject: [PATCH 10/20] adding logs to release notes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 084540e62..109cdcf31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Latest * Improved `make scaffold` * Implemented IAM attached group policies + * Implemented skeleton of Cloudwatch Logs * Redshift: fixed multi-params * Redshift: implement taggable resources * Lambda + SNS: Major enhancements From 04542dccc033f6931acebe82f2382e01b4b46981 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 12:35:52 -0700 Subject: [PATCH 11/20] implement elbv2 ResourceInUseError --- moto/elbv2/exceptions.py | 7 +++++++ moto/elbv2/models.py | 21 +++++++++++++++++++-- tests/test_elbv2/test_elbv2.py | 7 +++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/moto/elbv2/exceptions.py b/moto/elbv2/exceptions.py index 0947535eb..0bf9649d7 100644 --- a/moto/elbv2/exceptions.py +++ b/moto/elbv2/exceptions.py @@ -152,6 +152,13 @@ class InvalidDescribeRulesRequest(ELBClientError): ) +class ResourceInUseError(ELBClientError): + + def __init__(self, msg="A specified resource is in use"): + super(ResourceInUseError, self).__init__( + "ResourceInUse", msg) + + class RuleNotFoundError(ELBClientError): def __init__(self): diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 3c6afe7f5..8aa9ee9f0 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -21,6 +21,7 @@ from .exceptions import ( InvalidActionTypeError, ActionTargetGroupNotFoundError, InvalidDescribeRulesRequest, + ResourceInUseError, RuleNotFoundError, DuplicatePriorityError, InvalidTargetGroupNameError, @@ -426,10 +427,17 @@ class ELBv2Backend(BaseBackend): # however, boto3 does't raise error even if rule is not found def delete_target_group(self, target_group_arn): - target_group = self.target_groups.pop(target_group_arn, None) + if target_group_arn not in self.target_groups: + raise TargetGroupNotFoundError() + + target_group = self.target_groups[target_group_arn] if target_group: + if self._any_listener_using(target_group_arn): + raise ResourceInUseError( + "The target group '{}' is currently in use by a listener or a rule".format( + target_group_arn)) + del self.target_groups[target_group_arn] return target_group - raise TargetGroupNotFoundError() def delete_listener(self, listener_arn): for load_balancer in self.load_balancers.values(): @@ -539,6 +547,15 @@ class ELBv2Backend(BaseBackend): modified_rules.append(given_rule) return modified_rules + def _any_listener_using(self, target_group_arn): + for load_balancer in self.load_balancers.values(): + for listener in load_balancer.listeners.values(): + for rule in listener.rules: + for action in rule.actions: + if action.get('target_group_arn') == target_group_arn: + return True + return False + elbv2_backends = {} for region in ec2_backends.keys(): diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 21799ddcf..1a8494bd3 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -306,6 +306,13 @@ def test_create_target_group_and_listeners(): response = conn.describe_listeners(ListenerArns=[http_listener_arn, https_listener_arn]) response.get('Listeners').should.have.length_of(2) + # Try to delete the target group and it fails because there's a + # listener referencing it + with assert_raises(ClientError) as e: + conn.delete_target_group(TargetGroupArn=target_group.get('TargetGroupArn')) + e.exception.operation_name.should.equal('DeleteTargetGroup') + e.exception.args.should.equal(("An error occurred (ResourceInUse) when calling the DeleteTargetGroup operation: The target group 'arn:aws:elasticloadbalancing:us-east-1:1:targetgroup/a-target/50dc6c495c0c9188' is currently in use by a listener or a rule", )) # NOQA + # Delete one listener response = conn.describe_listeners(LoadBalancerArn=load_balancer_arn) response.get('Listeners').should.have.length_of(2) From f6166f841ad402353a6e482a59003e468846e447 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 12:36:47 -0700 Subject: [PATCH 12/20] running autopep8 against test_elbv2.py --- tests/test_elbv2/test_elbv2.py | 284 +++++++++++++++++++++++---------- 1 file changed, 201 insertions(+), 83 deletions(-) diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 1a8494bd3..8224a348d 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -14,10 +14,17 @@ def test_create_load_balancer(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') response = conn.create_load_balancer( Name='my-lb', @@ -29,7 +36,8 @@ def test_create_load_balancer(): lb = response.get('LoadBalancers')[0] lb.get('DNSName').should.equal("my-lb-1.us-east-1.elb.amazonaws.com") - lb.get('LoadBalancerArn').should.equal('arn:aws:elasticloadbalancing:us-east-1:1:loadbalancer/my-lb/50dc6c495c0c9188') + lb.get('LoadBalancerArn').should.equal( + 'arn:aws:elasticloadbalancing:us-east-1:1:loadbalancer/my-lb/50dc6c495c0c9188') lb.get('SecurityGroups').should.equal([security_group.id]) lb.get('AvailabilityZones').should.equal([ {'SubnetId': subnet1.id, 'ZoneName': 'us-east-1a'}, @@ -37,7 +45,8 @@ def test_create_load_balancer(): # Ensure the tags persisted response = conn.describe_tags(ResourceArns=[lb.get('LoadBalancerArn')]) - tags = {d['Key']: d['Value'] for d in response['TagDescriptions'][0]['Tags']} + tags = {d['Key']: d['Value'] + for d in response['TagDescriptions'][0]['Tags']} tags.should.equal({'key_name': 'a_value'}) @@ -47,10 +56,17 @@ def test_describe_load_balancers(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') conn.create_load_balancer( Name='my-lb', @@ -65,11 +81,14 @@ def test_describe_load_balancers(): lb = response.get('LoadBalancers')[0] lb.get('LoadBalancerName').should.equal('my-lb') - response = conn.describe_load_balancers(LoadBalancerArns=[lb.get('LoadBalancerArn')]) - response.get('LoadBalancers')[0].get('LoadBalancerName').should.equal('my-lb') + response = conn.describe_load_balancers( + LoadBalancerArns=[lb.get('LoadBalancerArn')]) + response.get('LoadBalancers')[0].get( + 'LoadBalancerName').should.equal('my-lb') response = conn.describe_load_balancers(Names=['my-lb']) - response.get('LoadBalancers')[0].get('LoadBalancerName').should.equal('my-lb') + response.get('LoadBalancers')[0].get( + 'LoadBalancerName').should.equal('my-lb') with assert_raises(ClientError): conn.describe_load_balancers(LoadBalancerArns=['not-a/real/arn']) @@ -84,10 +103,17 @@ def test_add_remove_tags(): ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') conn.create_load_balancer( Name='my-lb', @@ -197,10 +223,19 @@ def test_create_elb_in_multiple_region(): conn = boto3.client('elbv2', region_name=region) ec2 = boto3.resource('ec2', region_name=region) - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') - vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone=region + 'a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone=region + 'b') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') + vpc = ec2.create_vpc( + CidrBlock='172.28.7.0/24', + InstanceTenancy='default') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone=region + 'a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone=region + 'b') conn.create_load_balancer( Name='my-lb', @@ -210,10 +245,14 @@ def test_create_elb_in_multiple_region(): Tags=[{'Key': 'key_name', 'Value': 'a_value'}]) list( - boto3.client('elbv2', region_name='us-west-1').describe_load_balancers().get('LoadBalancers') + boto3.client( + 'elbv2', + region_name='us-west-1').describe_load_balancers().get('LoadBalancers') ).should.have.length_of(1) list( - boto3.client('elbv2', region_name='us-west-2').describe_load_balancers().get('LoadBalancers') + boto3.client( + 'elbv2', + region_name='us-west-2').describe_load_balancers().get('LoadBalancers') ).should.have.length_of(1) @@ -223,10 +262,17 @@ def test_create_target_group_and_listeners(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') response = conn.create_load_balancer( Name='my-lb', @@ -254,7 +300,8 @@ def test_create_target_group_and_listeners(): target_group_arn = target_group['TargetGroupArn'] # Add tags to the target group - conn.add_tags(ResourceArns=[target_group_arn], Tags=[{'Key': 'target', 'Value': 'group'}]) + conn.add_tags(ResourceArns=[target_group_arn], Tags=[ + {'Key': 'target', 'Value': 'group'}]) conn.describe_tags(ResourceArns=[target_group_arn])['TagDescriptions'][0]['Tags'].should.equal( [{'Key': 'target', 'Value': 'group'}]) @@ -281,7 +328,8 @@ def test_create_target_group_and_listeners(): LoadBalancerArn=load_balancer_arn, Protocol='HTTPS', Port=443, - Certificates=[{'CertificateArn': 'arn:aws:iam:123456789012:server-certificate/test-cert'}], + Certificates=[ + {'CertificateArn': 'arn:aws:iam:123456789012:server-certificate/test-cert'}], DefaultActions=[{'Type': 'forward', 'TargetGroupArn': target_group.get('TargetGroupArn')}]) listener = response.get('Listeners')[0] listener.get('Port').should.equal(443) @@ -303,13 +351,17 @@ def test_create_target_group_and_listeners(): listener.get('Port').should.equal(443) listener.get('Protocol').should.equal('HTTPS') - response = conn.describe_listeners(ListenerArns=[http_listener_arn, https_listener_arn]) + response = conn.describe_listeners( + ListenerArns=[ + http_listener_arn, + https_listener_arn]) response.get('Listeners').should.have.length_of(2) # Try to delete the target group and it fails because there's a # listener referencing it with assert_raises(ClientError) as e: - conn.delete_target_group(TargetGroupArn=target_group.get('TargetGroupArn')) + conn.delete_target_group( + TargetGroupArn=target_group.get('TargetGroupArn')) e.exception.operation_name.should.equal('DeleteTargetGroup') e.exception.args.should.equal(("An error occurred (ResourceInUse) when calling the DeleteTargetGroup operation: The target group 'arn:aws:elasticloadbalancing:us-east-1:1:targetgroup/a-target/50dc6c495c0c9188' is currently in use by a listener or a rule", )) # NOQA @@ -328,7 +380,10 @@ def test_create_target_group_and_listeners(): response.get('LoadBalancers').should.have.length_of(0) # And it deleted the remaining listener - response = conn.describe_listeners(ListenerArns=[http_listener_arn, https_listener_arn]) + response = conn.describe_listeners( + ListenerArns=[ + http_listener_arn, + https_listener_arn]) response.get('Listeners').should.have.length_of(0) # But not the target groups @@ -366,7 +421,13 @@ def test_create_invalid_target_group(): UnhealthyThresholdCount=2, Matcher={'HttpCode': '200'}) - invalid_names = ['-name', 'name-', '-name-', 'example.com', 'test@test', 'Na--me'] + invalid_names = [ + '-name', + 'name-', + '-name-', + 'example.com', + 'test@test', + 'Na--me'] for name in invalid_names: with assert_raises(ClientError): conn.create_target_group( @@ -406,10 +467,17 @@ def test_describe_paginated_balancers(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') for i in range(51): conn.create_load_balancer( @@ -421,7 +489,8 @@ def test_describe_paginated_balancers(): resp = conn.describe_load_balancers() resp['LoadBalancers'].should.have.length_of(50) - resp['NextMarker'].should.equal(resp['LoadBalancers'][-1]['LoadBalancerName']) + resp['NextMarker'].should.equal( + resp['LoadBalancers'][-1]['LoadBalancerName']) resp2 = conn.describe_load_balancers(Marker=resp['NextMarker']) resp2['LoadBalancers'].should.have.length_of(1) assert 'NextToken' not in resp2.keys() @@ -433,10 +502,17 @@ def test_delete_load_balancer(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') response = conn.create_load_balancer( Name='my-lb', @@ -459,10 +535,17 @@ def test_register_targets(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') conn.create_load_balancer( Name='my-lb', @@ -487,7 +570,8 @@ def test_register_targets(): target_group = response.get('TargetGroups')[0] # No targets registered yet - response = conn.describe_target_health(TargetGroupArn=target_group.get('TargetGroupArn')) + response = conn.describe_target_health( + TargetGroupArn=target_group.get('TargetGroupArn')) response.get('TargetHealthDescriptions').should.have.length_of(0) response = ec2.create_instances( @@ -508,14 +592,16 @@ def test_register_targets(): }, ]) - response = conn.describe_target_health(TargetGroupArn=target_group.get('TargetGroupArn')) + response = conn.describe_target_health( + TargetGroupArn=target_group.get('TargetGroupArn')) response.get('TargetHealthDescriptions').should.have.length_of(2) response = conn.deregister_targets( TargetGroupArn=target_group.get('TargetGroupArn'), Targets=[{'Id': instance_id2}]) - response = conn.describe_target_health(TargetGroupArn=target_group.get('TargetGroupArn')) + response = conn.describe_target_health( + TargetGroupArn=target_group.get('TargetGroupArn')) response.get('TargetHealthDescriptions').should.have.length_of(1) @@ -525,10 +611,17 @@ def test_target_group_attributes(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') response = conn.create_load_balancer( Name='my-lb', @@ -564,9 +657,11 @@ def test_target_group_attributes(): target_group_arn = target_group['TargetGroupArn'] # The attributes should start with the two defaults - response = conn.describe_target_group_attributes(TargetGroupArn=target_group_arn) + response = conn.describe_target_group_attributes( + TargetGroupArn=target_group_arn) response['Attributes'].should.have.length_of(2) - attributes = {attr['Key']: attr['Value'] for attr in response['Attributes']} + attributes = {attr['Key']: attr['Value'] + for attr in response['Attributes']} attributes['deregistration_delay.timeout_seconds'].should.equal('300') attributes['stickiness.enabled'].should.equal('false') @@ -586,14 +681,17 @@ def test_target_group_attributes(): # The response should have only the keys updated response['Attributes'].should.have.length_of(2) - attributes = {attr['Key']: attr['Value'] for attr in response['Attributes']} + attributes = {attr['Key']: attr['Value'] + for attr in response['Attributes']} attributes['stickiness.type'].should.equal('lb_cookie') attributes['stickiness.enabled'].should.equal('true') # These new values should be in the full attribute list - response = conn.describe_target_group_attributes(TargetGroupArn=target_group_arn) + response = conn.describe_target_group_attributes( + TargetGroupArn=target_group_arn) response['Attributes'].should.have.length_of(3) - attributes = {attr['Key']: attr['Value'] for attr in response['Attributes']} + attributes = {attr['Key']: attr['Value'] + for attr in response['Attributes']} attributes['stickiness.type'].should.equal('lb_cookie') attributes['stickiness.enabled'].should.equal('true') @@ -604,10 +702,17 @@ def test_handle_listener_rules(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') response = conn.create_load_balancer( Name='my-lb', @@ -656,11 +761,11 @@ def test_handle_listener_rules(): Priority=priority, Conditions=[{ 'Field': 'host-header', - 'Values': [ host ] + 'Values': [host] }, - { + { 'Field': 'path-pattern', - 'Values': [ path_pattern ] + 'Values': [path_pattern] }], Actions=[{ 'TargetGroupArn': target_group.get('TargetGroupArn'), @@ -678,11 +783,11 @@ def test_handle_listener_rules(): Priority=priority, Conditions=[{ 'Field': 'host-header', - 'Values': [ host ] + 'Values': [host] }, - { + { 'Field': 'path-pattern', - 'Values': [ path_pattern ] + 'Values': [path_pattern] }], Actions=[{ 'TargetGroupArn': target_group.get('TargetGroupArn'), @@ -698,11 +803,11 @@ def test_handle_listener_rules(): Priority=priority, Conditions=[{ 'Field': 'host-header', - 'Values': [ host ] + 'Values': [host] }, - { + { 'Field': 'path-pattern', - 'Values': [ path_pattern ] + 'Values': [path_pattern] }], Actions=[{ 'TargetGroupArn': target_group.get('TargetGroupArn'), @@ -710,7 +815,6 @@ def test_handle_listener_rules(): }] ) - # test for describe listeners obtained_rules = conn.describe_rules(ListenerArn=http_listener_arn) len(obtained_rules['Rules']).should.equal(3) @@ -723,15 +827,20 @@ def test_handle_listener_rules(): obtained_rules['Rules'].should.equal([first_rule]) # test for pagination - obtained_rules = conn.describe_rules(ListenerArn=http_listener_arn, PageSize=1) + obtained_rules = conn.describe_rules( + ListenerArn=http_listener_arn, PageSize=1) len(obtained_rules['Rules']).should.equal(1) obtained_rules.should.have.key('NextMarker') next_marker = obtained_rules['NextMarker'] - following_rules = conn.describe_rules(ListenerArn=http_listener_arn, PageSize=1, Marker=next_marker) + following_rules = conn.describe_rules( + ListenerArn=http_listener_arn, + PageSize=1, + Marker=next_marker) len(following_rules['Rules']).should.equal(1) following_rules.should.have.key('NextMarker') - following_rules['Rules'][0]['RuleArn'].should_not.equal(obtained_rules['Rules'][0]['RuleArn']) + following_rules['Rules'][0]['RuleArn'].should_not.equal( + obtained_rules['Rules'][0]['RuleArn']) # test for invalid describe rule request with assert_raises(ClientError): @@ -750,13 +859,13 @@ def test_handle_listener_rules(): modified_rule = conn.modify_rule( RuleArn=first_rule['RuleArn'], Conditions=[{ - 'Field': 'host-header', - 'Values': [ new_host ] - }, + 'Field': 'host-header', + 'Values': [new_host] + }, { 'Field': 'path-pattern', - 'Values': [ new_path_pattern ] - }] + 'Values': [new_path_pattern] + }] )['Rules'][0] rules = conn.describe_rules(ListenerArn=http_listener_arn) @@ -764,12 +873,14 @@ def test_handle_listener_rules(): modified_rule.should.equal(obtained_rule) obtained_rule['Conditions'][0]['Values'][0].should.equal(new_host) obtained_rule['Conditions'][1]['Values'][0].should.equal(new_path_pattern) - obtained_rule['Actions'][0]['TargetGroupArn'].should.equal(target_group.get('TargetGroupArn')) + obtained_rule['Actions'][0]['TargetGroupArn'].should.equal( + target_group.get('TargetGroupArn')) # modify priority conn.set_rule_priorities( RulePriorities=[ - {'RuleArn': first_rule['RuleArn'], 'Priority': int(first_rule['Priority']) - 1} + {'RuleArn': first_rule['RuleArn'], + 'Priority': int(first_rule['Priority']) - 1} ] ) with assert_raises(ClientError): @@ -794,11 +905,11 @@ def test_handle_listener_rules(): Priority=safe_priority, Conditions=[{ 'Field': 'host-header', - 'Values': [ host ] + 'Values': [host] }, - { + { 'Field': 'path-pattern', - 'Values': [ path_pattern ] + 'Values': [path_pattern] }], Actions=[{ 'TargetGroupArn': target_group.get('TargetGroupArn'), @@ -815,11 +926,11 @@ def test_handle_listener_rules(): Priority=safe_priority, Conditions=[{ 'Field': 'host-header', - 'Values': [ host ] + 'Values': [host] }, - { + { 'Field': 'path-pattern', - 'Values': [ path_pattern ] + 'Values': [path_pattern] }], Actions=[{ 'TargetGroupArn': invalid_target_group_arn, @@ -835,7 +946,7 @@ def test_handle_listener_rules(): Priority=safe_priority, Conditions=[{ 'Field': 'xxxxxxx', - 'Values': [ host ] + 'Values': [host] }], Actions=[{ 'TargetGroupArn': target_group.get('TargetGroupArn'), @@ -882,10 +993,17 @@ def test_describe_invalid_target_group(): conn = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') - security_group = ec2.create_security_group(GroupName='a-security-group', Description='First One') + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') - subnet1 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1a') - subnet2 = ec2.create_subnet(VpcId=vpc.id, CidrBlock='172.28.7.192/26', AvailabilityZone='us-east-1b') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1b') response = conn.create_load_balancer( Name='my-lb', From 068c0617daacb609e6c6c3fffeafbc126e17878e Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 12:38:36 -0700 Subject: [PATCH 13/20] fixing last pep8 violations on elbv2 test --- tests/test_elbv2/test_elbv2.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 8224a348d..98634c677 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -796,9 +796,8 @@ def test_handle_listener_rules(): ) # test for PriorityInUse - host2 = 'yyy.example.com' with assert_raises(ClientError): - r = conn.create_rule( + conn.create_rule( ListenerArn=http_listener_arn, Priority=priority, Conditions=[{ @@ -900,7 +899,7 @@ def test_handle_listener_rules(): # test for invalid action type safe_priority = 2 with assert_raises(ClientError): - r = conn.create_rule( + conn.create_rule( ListenerArn=http_listener_arn, Priority=safe_priority, Conditions=[{ @@ -921,7 +920,7 @@ def test_handle_listener_rules(): safe_priority = 2 invalid_target_group_arn = target_group.get('TargetGroupArn') + 'x' with assert_raises(ClientError): - r = conn.create_rule( + conn.create_rule( ListenerArn=http_listener_arn, Priority=safe_priority, Conditions=[{ @@ -941,7 +940,7 @@ def test_handle_listener_rules(): # test for invalid condition field_name safe_priority = 2 with assert_raises(ClientError): - r = conn.create_rule( + conn.create_rule( ListenerArn=http_listener_arn, Priority=safe_priority, Conditions=[{ @@ -957,7 +956,7 @@ def test_handle_listener_rules(): # test for emptry condition value safe_priority = 2 with assert_raises(ClientError): - r = conn.create_rule( + conn.create_rule( ListenerArn=http_listener_arn, Priority=safe_priority, Conditions=[{ @@ -973,7 +972,7 @@ def test_handle_listener_rules(): # test for multiple condition value safe_priority = 2 with assert_raises(ClientError): - r = conn.create_rule( + conn.create_rule( ListenerArn=http_listener_arn, Priority=safe_priority, Conditions=[{ @@ -1012,7 +1011,7 @@ def test_describe_invalid_target_group(): Scheme='internal', Tags=[{'Key': 'key_name', 'Value': 'a_value'}]) - load_balancer_arn = response.get('LoadBalancers')[0].get('LoadBalancerArn') + response.get('LoadBalancers')[0].get('LoadBalancerArn') response = conn.create_target_group( Name='a-target', @@ -1027,7 +1026,6 @@ def test_describe_invalid_target_group(): HealthyThresholdCount=5, UnhealthyThresholdCount=2, Matcher={'HttpCode': '200'}) - target_group = response.get('TargetGroups')[0] # Check error raises correctly with assert_raises(ClientError): From 867fc3b7f7c6a53cba4249b1c1485c6c092e6b3f Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 13:35:53 -0700 Subject: [PATCH 14/20] Removing dicttoxml dependency --- moto/redshift/responses.py | 23 +++++++++++++++++++++-- setup.py | 1 - 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index 52ca908e8..58983310f 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals import json -import dicttoxml +import xmltodict + from jinja2 import Template from six import iteritems @@ -26,6 +27,24 @@ def convert_json_error_to_xml(json_error): return template.render(code=code, message=message) +def itemize(data): + """ + The xmltodict.unparse requires we modify the shape of the input dictionary slightly. Instead of a dict of the form: + {'key': ['value1', 'value2']} + We must provide: + {'key': {'item': ['value1', 'value2']}} + """ + if isinstance(data, dict): + ret = {} + for key in data: + ret[key] = itemize(data[key]) + return ret + elif isinstance(data, list): + return {'item': [itemize(value) for value in data]} + else: + return data + + class RedshiftResponse(BaseResponse): @property @@ -36,7 +55,7 @@ class RedshiftResponse(BaseResponse): if self.request_json: return json.dumps(response) else: - xml = dicttoxml.dicttoxml(response, attr_type=False, root=False) + xml = xmltodict.unparse(itemize(response), full_document=False) return xml.decode("utf-8") def call_action(self): diff --git a/setup.py b/setup.py index 378119925..0770c098f 100755 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ install_requires = [ "cryptography>=2.0.0", "requests>=2.5", "xmltodict", - "dicttoxml", "six>1.9", "werkzeug", "pyaml", From 95a4bd5a7b4d73de8efcc2834b5543a3baef26cc Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 15:25:02 -0700 Subject: [PATCH 15/20] supporting python 3 --- moto/redshift/responses.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index 58983310f..a320f9cae 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -56,7 +56,9 @@ class RedshiftResponse(BaseResponse): return json.dumps(response) else: xml = xmltodict.unparse(itemize(response), full_document=False) - return xml.decode("utf-8") + if hasattr(xml, 'decode'): + xml = xml.decode('utf-8') + return xml def call_action(self): status, headers, body = super(RedshiftResponse, self).call_action() From 6e889daaa89dd24ad8888c59898a417437a1c6f1 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 15:35:36 -0700 Subject: [PATCH 16/20] bumping to version 1.1.21 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0770c098f..3f6804ce0 100755 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ else: setup( name='moto', - version='1.1.20', + version='1.1.21', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 74cbd08816f3c26e4ada5b3613756300363ca383 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 2 Oct 2017 15:36:31 -0700 Subject: [PATCH 17/20] changelog for 1.1.21 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 109cdcf31..bbce6c343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Moto Changelog Latest ------ +1.1.21 +----- + + * ELBv2 bugfixes + * Removing GPL'd dependency + 1.1.20 ----- From fc9c2509228c0bd04300347d50d46d0ff1c7087f Mon Sep 17 00:00:00 2001 From: Chris Keogh Date: Tue, 3 Oct 2017 13:33:50 +1300 Subject: [PATCH 18/20] add basic awslambda get_policy --- moto/awslambda/responses.py | 14 ++++++++++++++ moto/awslambda/urls.py | 3 ++- tests/test_awslambda/test_lambda.py | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index cf92e66f4..972cd7a4e 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -57,6 +57,20 @@ class LambdaResponse(BaseResponse): else: raise ValueError("Cannot handle {0} request".format(request.method)) + def policy(self, request, full_url, headers): + if request.method == 'GET': + return self._get_policy(request, full_url, headers) + + def _get_policy(self, request, full_url, headers): + lambda_backend = self.get_lambda_backend(full_url) + + path = request.path if hasattr(request, 'path') else request.path_url + function_name = path.split('/')[-2] + if lambda_backend.has_function(function_name): + return 200, {}, json.dumps(dict(Policy='test_policy')) + else: + return 404, {}, "{}" + def _invoke(self, request, full_url): response_headers = {} lambda_backend = self.get_lambda_backend(full_url) diff --git a/moto/awslambda/urls.py b/moto/awslambda/urls.py index 0fec24bab..005785f19 100644 --- a/moto/awslambda/urls.py +++ b/moto/awslambda/urls.py @@ -12,5 +12,6 @@ url_paths = { r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/?$': response.function, r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invocations/?$': response.invoke, r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invoke-async/?$': response.invoke_async, - r'{0}/(?P[^/]+)/tags/(?P.+)': response.tag + r'{0}/(?P[^/]+)/tags/(?P.+)': response.tag, + r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/policy/?$': response.policy } diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 6b67ce0f0..c7fee2745 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -645,3 +645,26 @@ def test_get_function_created_with_zipfile(): } }, ) + +@mock_lambda +def get_function_policy(): + conn = boto3.client('lambda', 'us-west-2') + zip_content = get_test_zip_file1() + result = conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.handler', + Code={ + 'ZipFile': zip_content, + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + response = conn.get_policy( + FunctionName='testFunction' + ) + assert response['Policy'] == 'test_policy' \ No newline at end of file From b994cf5291888858c46609449ff0d5d940ea5a59 Mon Sep 17 00:00:00 2001 From: Chris Keogh Date: Tue, 3 Oct 2017 13:54:37 +1300 Subject: [PATCH 19/20] add more realistic policy --- moto/awslambda/responses.py | 5 ++++- tests/test_awslambda/test_lambda.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 972cd7a4e..94c381f5e 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -67,7 +67,10 @@ class LambdaResponse(BaseResponse): path = request.path if hasattr(request, 'path') else request.path_url function_name = path.split('/')[-2] if lambda_backend.has_function(function_name): - return 200, {}, json.dumps(dict(Policy='test_policy')) + policy = ("{\"Statement\":[{\"Action\":[\"lambda:InvokeFunction\"]," + "\"Resource\":\"arn:aws:lambda:us-west-2:account-id:function:helloworld\"," + "\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"account-id\"},\"Sid\":\"3\"}]}") + return 200, {}, json.dumps(dict(Policy=policy)) else: return 404, {}, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index c7fee2745..163fa306f 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -667,4 +667,8 @@ def get_function_policy(): response = conn.get_policy( FunctionName='testFunction' ) - assert response['Policy'] == 'test_policy' \ No newline at end of file + + assert 'Policy' in response + assert isinstance(response['Policy'], str) + res = json.loads(response['Policy']) + assert res['Statement'][0]['Action'] == ['lambda:InvokeFunction'] \ No newline at end of file From 9bb07e6b6e42811fa768849618a7c32d9f55d4d1 Mon Sep 17 00:00:00 2001 From: Chris Keogh Date: Tue, 3 Oct 2017 15:23:00 +1300 Subject: [PATCH 20/20] add awslambda.add_permission --- moto/awslambda/models.py | 4 +++ moto/awslambda/responses.py | 20 ++++++++++--- tests/test_awslambda/test_lambda.py | 46 ++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index d22d1a7f4..935abbcd6 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -132,6 +132,7 @@ class LambdaFunction(BaseModel): self.logs_backend = logs_backends[self.region] self.environment_vars = spec.get('Environment', {}).get('Variables', {}) self.docker_client = docker.from_env() + self.policy = "" # Unfortunately mocking replaces this method w/o fallback enabled, so we # need to replace it if we detect it's been mocked @@ -527,6 +528,9 @@ class LambdaBackend(BaseBackend): pass # Don't care + def add_policy(self, function_name, policy): + self.get_function(function_name).policy = policy + def do_validate_s3(): return os.environ.get('VALIDATE_LAMBDA_S3', '') in ['', '1', 'true'] diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 94c381f5e..5215f63c5 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -60,6 +60,20 @@ class LambdaResponse(BaseResponse): def policy(self, request, full_url, headers): if request.method == 'GET': return self._get_policy(request, full_url, headers) + if request.method == 'POST': + return self._add_policy(request, full_url, headers) + + def _add_policy(self, request, full_url, headers): + lambda_backend = self.get_lambda_backend(full_url) + + path = request.path if hasattr(request, 'path') else request.path_url + function_name = path.split('/')[-2] + if lambda_backend.has_function(function_name): + policy = request.body.decode('utf8') + lambda_backend.add_policy(function_name, policy) + return 200, {}, json.dumps(dict(Statement=policy)) + else: + return 404, {}, "{}" def _get_policy(self, request, full_url, headers): lambda_backend = self.get_lambda_backend(full_url) @@ -67,10 +81,8 @@ class LambdaResponse(BaseResponse): path = request.path if hasattr(request, 'path') else request.path_url function_name = path.split('/')[-2] if lambda_backend.has_function(function_name): - policy = ("{\"Statement\":[{\"Action\":[\"lambda:InvokeFunction\"]," - "\"Resource\":\"arn:aws:lambda:us-west-2:account-id:function:helloworld\"," - "\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"account-id\"},\"Sid\":\"3\"}]}") - return 200, {}, json.dumps(dict(Policy=policy)) + function = lambda_backend.get_function(function_name) + return 200, {}, json.dumps(dict(Policy="{\"Statement\":[" + function.policy + "]}")) else: return 404, {}, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 163fa306f..317e9f4a2 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -646,6 +646,39 @@ def test_get_function_created_with_zipfile(): }, ) +@mock_lambda +def add_function_permission(): + conn = boto3.client('lambda', 'us-west-2') + zip_content = get_test_zip_file1() + result = conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.handler', + Code={ + 'ZipFile': zip_content, + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + response = conn.add_permission( + FunctionName='testFunction', + StatementId='1', + Action="lambda:InvokeFunction", + Principal='432143214321', + SourceArn="arn:aws:lambda:us-west-2:account-id:function:helloworld", + SourceAccount='123412341234', + EventSourceToken='blah', + Qualifier='2' + ) + assert 'Statement' in response + res = json.loads(response['Statement']) + assert res['Action'] == "lambda:InvokeFunction" + + @mock_lambda def get_function_policy(): conn = boto3.client('lambda', 'us-west-2') @@ -664,6 +697,17 @@ def get_function_policy(): Publish=True, ) + response = conn.add_permission( + FunctionName='testFunction', + StatementId='1', + Action="lambda:InvokeFunction", + Principal='432143214321', + SourceArn="arn:aws:lambda:us-west-2:account-id:function:helloworld", + SourceAccount='123412341234', + EventSourceToken='blah', + Qualifier='2' + ) + response = conn.get_policy( FunctionName='testFunction' ) @@ -671,4 +715,4 @@ def get_function_policy(): assert 'Policy' in response assert isinstance(response['Policy'], str) res = json.loads(response['Policy']) - assert res['Statement'][0]['Action'] == ['lambda:InvokeFunction'] \ No newline at end of file + assert res['Statement'][0]['Action'] == 'lambda:InvokeFunction'