From 4c605ba403c6f7a4f9b5e61fe7bbd85da8196964 Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Fri, 19 Oct 2018 17:17:26 -0400 Subject: [PATCH 01/12] Add Python 3.7 support, remove Python 3.3 support As of #1733, Python 3.7 is supported, so reflect that in the Trove classifiers. As of 2017-09-29, Python 3.3 is end-of-life and no longer receives updates of any kind (including security fixes), so remove it from the list of supported versions. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 74683836e..5bdaef8ef 100755 --- a/setup.py +++ b/setup.py @@ -78,10 +78,10 @@ setup( "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "License :: OSI Approved :: Apache Software License", "Topic :: Software Development :: Testing", ], From 7e863b0260669646c78ffe8daf8a08ba268dc24f Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Thu, 21 Feb 2019 22:08:46 +0100 Subject: [PATCH 02/12] Add attributes parameter support for sns create_topic API --- moto/sns/models.py | 7 +++++-- moto/sns/responses.py | 3 ++- tests/test_sns/test_topics_boto3.py | 12 ++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 41e83aba4..ac3bf14ba 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -12,7 +12,7 @@ from boto3 import Session from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel -from moto.core.utils import iso_8601_datetime_with_milliseconds +from moto.core.utils import iso_8601_datetime_with_milliseconds, camelcase_to_underscores from moto.sqs import sqs_backends from moto.awslambda import lambda_backends @@ -243,11 +243,14 @@ class SNSBackend(BaseBackend): def update_sms_attributes(self, attrs): self.sms_attributes.update(attrs) - def create_topic(self, name): + def create_topic(self, name, attributes=None): fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{1,256}$', name) if fails_constraints: raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.") candidate_topic = Topic(name, self) + if attributes: + for attribute in attributes: + setattr(candidate_topic, camelcase_to_underscores(attribute), attributes[attribute]) if candidate_topic.arn in self.topics: return self.topics[candidate_topic.arn] else: diff --git a/moto/sns/responses.py b/moto/sns/responses.py index 8c1bb885e..440115429 100644 --- a/moto/sns/responses.py +++ b/moto/sns/responses.py @@ -75,7 +75,8 @@ class SNSResponse(BaseResponse): def create_topic(self): name = self._get_param('Name') - topic = self.backend.create_topic(name) + attributes = self._get_attributes() + topic = self.backend.create_topic(name, attributes) if self.request_json: return json.dumps({ diff --git a/tests/test_sns/test_topics_boto3.py b/tests/test_sns/test_topics_boto3.py index 7d9a27b18..870fa6f6e 100644 --- a/tests/test_sns/test_topics_boto3.py +++ b/tests/test_sns/test_topics_boto3.py @@ -32,6 +32,18 @@ def test_create_and_delete_topic(): topics = topics_json["Topics"] topics.should.have.length_of(0) + +@mock_sns +def test_create_topic_with_attributes(): + conn = boto3.client("sns", region_name="us-east-1") + conn.create_topic(Name='some-topic-with-attribute', Attributes={'DisplayName': 'test-topic'}) + topics_json = conn.list_topics() + topic_arn = topics_json["Topics"][0]['TopicArn'] + + attributes = conn.get_topic_attributes(TopicArn=topic_arn)['Attributes'] + attributes['DisplayName'].should.equal('test-topic') + + @mock_sns def test_create_topic_should_be_indempodent(): conn = boto3.client("sns", region_name="us-east-1") From bb44af2ccfe58874e3558a33b571ad3ebee12842 Mon Sep 17 00:00:00 2001 From: Hunter Jarrell Date: Fri, 14 Jun 2019 15:50:37 -0400 Subject: [PATCH 03/12] Add CreateDate to iam list_groups_for_user. Add the CreateDate field to the list_groups_for_user to match the correct AWS response. Fixes #2242 --- moto/iam/responses.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 05624101a..b923a5250 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1349,6 +1349,7 @@ LIST_GROUPS_FOR_USER_TEMPLATE = """ {{ group.name }} {{ group.id }} {{ group.arn }} + {{ group.created_iso_8601 }} {% endfor %} @@ -1651,6 +1652,7 @@ LIST_GROUPS_FOR_USER_TEMPLATE = """ {{ group.name }} {{ group.id }} {{ group.arn }} + {{ group.created_iso_8601 }} {% endfor %} From 6e97881896cc40fb8baeeb9cbffb6db9c489437b Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Mon, 17 Jun 2019 15:53:32 +0200 Subject: [PATCH 04/12] Route53 Delete: respect the given Type --- moto/route53/models.py | 8 ++++++-- moto/route53/responses.py | 2 +- tests/test_route53/test_route53.py | 20 +++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 3760d3817..ac19c04d9 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -190,9 +190,13 @@ class FakeZone(BaseModel): self.rrsets.append(new_rrset) return new_rrset - def delete_rrset_by_name(self, name): + def delete_rrset(self, rrset): self.rrsets = [ - record_set for record_set in self.rrsets if record_set.name != name] + record_set + for record_set in self.rrsets + if record_set.name != rrset['Name'] or + record_set.type_ != rrset['Type'] + ] def delete_rrset_by_id(self, set_identifier): self.rrsets = [ diff --git a/moto/route53/responses.py b/moto/route53/responses.py index 98ffa4c47..bf705c87f 100644 --- a/moto/route53/responses.py +++ b/moto/route53/responses.py @@ -147,7 +147,7 @@ class Route53(BaseResponse): the_zone.delete_rrset_by_id( record_set["SetIdentifier"]) else: - the_zone.delete_rrset_by_name(record_set["Name"]) + the_zone.delete_rrset(record_set) return 200, headers, CHANGE_RRSET_RESPONSE diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index d730f8dcf..a1c2f4913 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -110,6 +110,7 @@ def test_rrset(): changes = ResourceRecordSets(conn, zoneid) changes.add_change("DELETE", "foo.bar.testdns.aws.com", "A") + changes.add_change("DELETE", "foo.bar.testdns.aws.com", "TXT") changes.commit() changes = ResourceRecordSets(conn, zoneid) @@ -582,7 +583,7 @@ def test_change_resource_record_sets_crud_valid(): cname_record_detail['TTL'].should.equal(60) cname_record_detail['ResourceRecords'].should.equal([{'Value': '192.168.1.1'}]) - # Delete record. + # Delete record with wrong type. delete_payload = { 'Comment': 'delete prod.redis.db', 'Changes': [ @@ -597,6 +598,23 @@ def test_change_resource_record_sets_crud_valid(): } conn.change_resource_record_sets(HostedZoneId=hosted_zone_id, ChangeBatch=delete_payload) response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) + len(response['ResourceRecordSets']).should.equal(1) + + # Delete record. + delete_payload = { + 'Comment': 'delete prod.redis.db', + 'Changes': [ + { + 'Action': 'DELETE', + 'ResourceRecordSet': { + 'Name': 'prod.redis.db', + 'Type': 'A', + } + } + ] + } + conn.change_resource_record_sets(HostedZoneId=hosted_zone_id, ChangeBatch=delete_payload) + response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) len(response['ResourceRecordSets']).should.equal(0) From a5d1b22534b4b16e39be5004012bd95ad895a645 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Tue, 18 Jun 2019 11:03:28 +0200 Subject: [PATCH 05/12] Fix CloudFormation usage --- moto/route53/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index ac19c04d9..9961f4157 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -119,7 +119,7 @@ class RecordSet(BaseModel): properties["HostedZoneId"]) try: - hosted_zone.delete_rrset_by_name(resource_name) + hosted_zone.delete_rrset({'Name': resource_name}) except KeyError: pass @@ -162,7 +162,7 @@ class RecordSet(BaseModel): self.hosted_zone_name) if not hosted_zone: hosted_zone = route53_backend.get_hosted_zone(self.hosted_zone_id) - hosted_zone.delete_rrset_by_name(self.name) + hosted_zone.delete_rrset({'Name': self.name, 'Type': self.type_}) class FakeZone(BaseModel): @@ -195,7 +195,7 @@ class FakeZone(BaseModel): record_set for record_set in self.rrsets if record_set.name != rrset['Name'] or - record_set.type_ != rrset['Type'] + (rrset.get('Type') is not None and record_set.type_ != rrset['Type']) ] def delete_rrset_by_id(self, set_identifier): From 5f46aa8c504b2d65bc22c4df652ce8fed59017fa Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Mon, 17 Jun 2019 16:59:07 +0200 Subject: [PATCH 06/12] Reduced readability to please flake8 --- moto/route53/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 9961f4157..da298144f 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -195,7 +195,7 @@ class FakeZone(BaseModel): record_set for record_set in self.rrsets if record_set.name != rrset['Name'] or - (rrset.get('Type') is not None and record_set.type_ != rrset['Type']) + (rrset.get('Type') is not None and record_set.type_ != rrset['Type']) ] def delete_rrset_by_id(self, set_identifier): From 6ac315b90329d86f0ddeb4623f7985cdf14c4092 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Tue, 2 Jul 2019 12:24:19 +0200 Subject: [PATCH 07/12] Fixed broken tests due to policy validation. --- tests/test_iam/test_iam.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 5be83e417..290197c6d 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -361,15 +361,15 @@ def test_create_many_policy_versions(): conn = boto3.client('iam', region_name='us-east-1') conn.create_policy( PolicyName="TestCreateManyPolicyVersions", - PolicyDocument='{"some":"policy"}') + PolicyDocument=MOCK_POLICY) for _ in range(0, 4): conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestCreateManyPolicyVersions", - PolicyDocument='{"some":"policy"}') + PolicyDocument=MOCK_POLICY) with assert_raises(ClientError): conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestCreateManyPolicyVersions", - PolicyDocument='{"some":"policy"}') + PolicyDocument=MOCK_POLICY) @mock_iam @@ -377,22 +377,22 @@ def test_set_default_policy_version(): conn = boto3.client('iam', region_name='us-east-1') conn.create_policy( PolicyName="TestSetDefaultPolicyVersion", - PolicyDocument='{"first":"policy"}') + PolicyDocument=MOCK_POLICY) conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestSetDefaultPolicyVersion", - PolicyDocument='{"second":"policy"}', + PolicyDocument=MOCK_POLICY_2, SetAsDefault=True) conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestSetDefaultPolicyVersion", - PolicyDocument='{"third":"policy"}', + PolicyDocument=MOCK_POLICY_3, SetAsDefault=True) versions = conn.list_policy_versions( PolicyArn="arn:aws:iam::123456789012:policy/TestSetDefaultPolicyVersion") - versions.get('Versions')[0].get('Document').should.equal({'first': 'policy'}) + versions.get('Versions')[0].get('Document').should.equal(json.loads(MOCK_POLICY)) versions.get('Versions')[0].get('IsDefaultVersion').shouldnt.be.ok - versions.get('Versions')[1].get('Document').should.equal({'second': 'policy'}) + versions.get('Versions')[1].get('Document').should.equal(json.loads(MOCK_POLICY_2)) versions.get('Versions')[1].get('IsDefaultVersion').shouldnt.be.ok - versions.get('Versions')[2].get('Document').should.equal({'third': 'policy'}) + versions.get('Versions')[2].get('Document').should.equal(json.loads(MOCK_POLICY_3)) versions.get('Versions')[2].get('IsDefaultVersion').should.be.ok @@ -434,7 +434,6 @@ def test_get_policy_version(): retrieved = conn.get_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicyVersion", VersionId=version.get('PolicyVersion').get('VersionId')) - retrieved.get('PolicyVersion').get('Document').should.equal({'some': 'policy'}) retrieved.get('PolicyVersion').get('Document').should.equal(json.loads(MOCK_POLICY)) retrieved.get('PolicyVersion').get('IsDefaultVersion').shouldnt.be.ok @@ -525,10 +524,10 @@ def test_delete_default_policy_version(): conn = boto3.client('iam', region_name='us-east-1') conn.create_policy( PolicyName="TestDeletePolicyVersion", - PolicyDocument='{"first":"policy"}') + PolicyDocument=MOCK_POLICY) conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", - PolicyDocument='{"second":"policy"}') + PolicyDocument=MOCK_POLICY_2) with assert_raises(ClientError): conn.delete_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", From f11a5dcf6b783ccbbf4bb00531d726152d54c71f Mon Sep 17 00:00:00 2001 From: Cory Dolphin Date: Tue, 2 Jul 2019 18:21:19 -0700 Subject: [PATCH 08/12] Fix socket.fakesock compatibility --- moto/packages/httpretty/core.py | 24 ++++++++++++++--- tests/test_core/test_socket.py | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 tests/test_core/test_socket.py diff --git a/moto/packages/httpretty/core.py b/moto/packages/httpretty/core.py index 4eb92108f..f94723017 100644 --- a/moto/packages/httpretty/core.py +++ b/moto/packages/httpretty/core.py @@ -268,10 +268,26 @@ class fakesock(object): _sent_data = [] def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, - protocol=0): - self.truesock = (old_socket(family, type, protocol) - if httpretty.allow_net_connect - else None) + proto=0, fileno=None, _sock=None): + """ + Matches both the Python 2 API: + def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None): + https://github.com/python/cpython/blob/2.7/Lib/socket.py + + and the Python 3 API: + def __init__(self, family=-1, type=-1, proto=-1, fileno=None): + https://github.com/python/cpython/blob/3.5/Lib/socket.py + """ + if httpretty.allow_net_connect: + if PY3: + self.truesock = old_socket(family, type, proto, fileno) + else: + # If Python 2, if parameters are passed as arguments, instead of kwargs, + # the 4th argument `_sock` will be interpreted as the `fileno`. + # Check if _sock is none, and if so, pass fileno. + self.truesock = old_socket(family, type, proto, fileno or _sock) + else: + self.truesock = None self._closed = True self.fd = FakeSockFile() self.fd.socket = self diff --git a/tests/test_core/test_socket.py b/tests/test_core/test_socket.py new file mode 100644 index 000000000..2e73d7b5f --- /dev/null +++ b/tests/test_core/test_socket.py @@ -0,0 +1,48 @@ +import unittest +from moto import mock_dynamodb2_deprecated, mock_dynamodb2 +import socket + +from six import PY3 + + +class TestSocketPair(unittest.TestCase): + + @mock_dynamodb2_deprecated + def test_asyncio_deprecated(self): + if PY3: + self.assertIn( + 'moto.packages.httpretty.core.fakesock.socket', + str(socket.socket), + 'Our mock should be present' + ) + import asyncio + self.assertIsNotNone(asyncio.get_event_loop()) + + @mock_dynamodb2_deprecated + def test_socket_pair_deprecated(self): + + # In Python2, the fakesocket is not set, for some reason. + if PY3: + self.assertIn( + 'moto.packages.httpretty.core.fakesock.socket', + str(socket.socket), + 'Our mock should be present' + ) + a, b = socket.socketpair() + self.assertIsNotNone(a) + self.assertIsNotNone(b) + if a: + a.close() + if b: + b.close() + + + @mock_dynamodb2 + def test_socket_pair(self): + a, b = socket.socketpair() + self.assertIsNotNone(a) + self.assertIsNotNone(b) + if a: + a.close() + if b: + b.close() From d3495e408e42edb5ab8c3200d75825ed2a14640c Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 3 Jul 2019 14:48:07 -0700 Subject: [PATCH 09/12] bump PyYAML minimum version to address https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-18342 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 593d248e9..7286507e8 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ install_requires = [ "xmltodict", "six>1.9", "werkzeug", - "PyYAML", + "PyYAML>=5.1", "pytz", "python-dateutil<3.0.0,>=2.1", "python-jose<4.0.0", From af0205b6a33ab00f6b2ad2833cf9cc0998386407 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 6 Jul 2019 17:40:36 -0500 Subject: [PATCH 10/12] Fix wrong tag for cloudwatch metrics response. Closes #2267. --- moto/cloudwatch/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 8118f35ba..bf176e1be 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -275,7 +275,7 @@ GET_METRIC_STATISTICS_TEMPLATE = """ s" - $url").mkString(System.lineSeparator)) + println() +} From 67326ace4fb6a01f522ee390a6e225c4ce11ccac Mon Sep 17 00:00:00 2001 From: wndhydrnt Date: Sun, 7 Jul 2019 21:45:51 +0200 Subject: [PATCH 12/12] Raise exception if a role policy is not found --- moto/iam/models.py | 1 + tests/test_iam/test_iam.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/moto/iam/models.py b/moto/iam/models.py index 457535148..f92568df4 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -685,6 +685,7 @@ class IAMBackend(BaseBackend): for p, d in role.policies.items(): if p == policy_name: return p, d + raise IAMNotFoundException("Policy Document {0} not attached to role {1}".format(policy_name, role_name)) def list_role_policies(self, role_name): role = self.get_role(role_name) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 3dfc05267..e7507e2e5 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -311,6 +311,15 @@ def test_put_role_policy(): policy.should.equal("test policy") +@mock_iam +def test_get_role_policy(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_role( + RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="my-path") + with assert_raises(conn.exceptions.NoSuchEntityException): + conn.get_role_policy(RoleName="my-role", PolicyName="does-not-exist") + + @mock_iam_deprecated() def test_update_assume_role_policy(): conn = boto.connect_iam()