Merged upstream master

moto/cloudwatch/responses.py

@@ -275,7 +275,7 @@ GET_METRIC_STATISTICS_TEMPLATE = """<GetMetricStatisticsResponse xmlns="http://m
       <Label>{{ label }}</Label>
       <Datapoints>
         {% for datapoint in datapoints %}
-            <Datapoint>
+            <member>
               {% if datapoint.sum is not none %}
               <Sum>{{ datapoint.sum }}</Sum>
               {% endif %}
@@ -302,7 +302,7 @@ GET_METRIC_STATISTICS_TEMPLATE = """<GetMetricStatisticsResponse xmlns="http://m
 
               <Timestamp>{{ datapoint.timestamp }}</Timestamp>
               <Unit>{{ datapoint.unit }}</Unit>
-            </Datapoint>
+            </member>
         {% endfor %}
       </Datapoints>
     </GetMetricStatisticsResult>

moto/iam/models.py

@@ -685,6 +685,7 @@ class IAMBackend(BaseBackend):
         for p, d in role.policies.items():
             if p == policy_name:
                 return p, d
+        raise IAMNotFoundException("Policy Document {0} not attached to role {1}".format(policy_name, role_name))
 
     def list_role_policies(self, role_name):
         role = self.get_role(role_name)
@@ -777,7 +778,6 @@ class IAMBackend(BaseBackend):
         policy = self.get_policy(policy_arn)
         if not policy:
             raise IAMNotFoundException("Policy not found")
-            
         if len(policy.versions) >= 5:
             raise IAMLimitExceededException("A managed policy can have up to 5 versions. Before you create a new version, you must delete an existing version.")
         set_as_default = (set_as_default == "true")  # convert it to python bool

moto/iam/responses.py

@@ -1349,6 +1349,7 @@ LIST_GROUPS_FOR_USER_TEMPLATE = """<ListGroupsForUserResponse>
             <GroupName>{{ group.name }}</GroupName>
             <GroupId>{{ group.id }}</GroupId>
             <Arn>{{ group.arn }}</Arn>
+            <CreateDate>{{ group.created_iso_8601 }}</CreateDate>
         </member>
         {% endfor %}
     </Groups>
@@ -1652,6 +1653,7 @@ LIST_GROUPS_FOR_USER_TEMPLATE = """<ListGroupsForUserResponse>
             <GroupName>{{ group.name }}</GroupName>
             <GroupId>{{ group.id }}</GroupId>
             <Arn>{{ group.arn }}</Arn>
+            <CreateDate>{{ group.created_iso_8601 }}</CreateDate>
         </member>
         {% endfor %}
     </Groups>

moto/packages/httpretty/core.py

@@ -268,10 +268,26 @@ class fakesock(object):
         _sent_data = []
 
         def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM,
-                     protocol=0):
-            self.truesock = (old_socket(family, type, protocol)
-                             if httpretty.allow_net_connect
-                             else None)
+                     proto=0, fileno=None, _sock=None):
+            """
+                Matches both the Python 2 API:
+                    def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
+                    https://github.com/python/cpython/blob/2.7/Lib/socket.py
+
+                and the Python 3 API:
+                    def __init__(self, family=-1, type=-1, proto=-1, fileno=None):
+                    https://github.com/python/cpython/blob/3.5/Lib/socket.py
+            """
+            if httpretty.allow_net_connect:
+                if PY3:
+                    self.truesock = old_socket(family, type, proto, fileno)
+                else:
+                    # If Python 2, if parameters are passed as arguments, instead of kwargs,
+                    # the 4th argument `_sock` will be interpreted as the `fileno`.
+                    # Check if _sock is none, and if so, pass fileno.
+                    self.truesock = old_socket(family, type, proto, fileno or _sock)
+            else:
+                self.truesock = None
             self._closed = True
             self.fd = FakeSockFile()
             self.fd.socket = self

moto/route53/models.py

@@ -119,7 +119,7 @@ class RecordSet(BaseModel):
                 properties["HostedZoneId"])
 
         try:
-            hosted_zone.delete_rrset_by_name(resource_name)
+            hosted_zone.delete_rrset({'Name': resource_name})
         except KeyError:
             pass
 
@@ -162,7 +162,7 @@ class RecordSet(BaseModel):
             self.hosted_zone_name)
         if not hosted_zone:
             hosted_zone = route53_backend.get_hosted_zone(self.hosted_zone_id)
-        hosted_zone.delete_rrset_by_name(self.name)
+        hosted_zone.delete_rrset({'Name': self.name, 'Type': self.type_})
 
 
 def reverse_domain_name(domain_name):
@@ -196,9 +196,13 @@ class FakeZone(BaseModel):
             self.rrsets.append(new_rrset)
         return new_rrset
 
-    def delete_rrset_by_name(self, name):
+    def delete_rrset(self, rrset):
         self.rrsets = [
-            record_set for record_set in self.rrsets if record_set.name != name]
+            record_set
+            for record_set in self.rrsets
+            if record_set.name != rrset['Name'] or
+            (rrset.get('Type') is not None and record_set.type_ != rrset['Type'])
+        ]
 
     def delete_rrset_by_id(self, set_identifier):
         self.rrsets = [

moto/route53/responses.py

@@ -147,7 +147,7 @@ class Route53(BaseResponse):
                         the_zone.delete_rrset_by_id(
                             record_set["SetIdentifier"])
                     else:
-                        the_zone.delete_rrset_by_name(record_set["Name"])
+                        the_zone.delete_rrset(record_set)
 
             return 200, headers, CHANGE_RRSET_RESPONSE
 

moto/sns/models.py

@@ -12,7 +12,7 @@ from boto3 import Session
 
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel
-from moto.core.utils import iso_8601_datetime_with_milliseconds
+from moto.core.utils import iso_8601_datetime_with_milliseconds, camelcase_to_underscores
 from moto.sqs import sqs_backends
 from moto.awslambda import lambda_backends
 
@@ -243,11 +243,14 @@ class SNSBackend(BaseBackend):
     def update_sms_attributes(self, attrs):
         self.sms_attributes.update(attrs)
 
-    def create_topic(self, name):
+    def create_topic(self, name, attributes=None):
         fails_constraints = not re.match(r'^[a-zA-Z0-9_-]{1,256}$', name)
         if fails_constraints:
             raise InvalidParameterValue("Topic names must be made up of only uppercase and lowercase ASCII letters, numbers, underscores, and hyphens, and must be between 1 and 256 characters long.")
         candidate_topic = Topic(name, self)
+        if attributes:
+            for attribute in attributes:
+                setattr(candidate_topic, camelcase_to_underscores(attribute), attributes[attribute])
         if candidate_topic.arn in self.topics:
             return self.topics[candidate_topic.arn]
         else:

moto/sns/responses.py

@@ -75,7 +75,8 @@ class SNSResponse(BaseResponse):
 
     def create_topic(self):
         name = self._get_param('Name')
-        topic = self.backend.create_topic(name)
+        attributes = self._get_attributes()
+        topic = self.backend.create_topic(name, attributes)
 
         if self.request_json:
             return json.dumps({

other_langs/sqsSample.scala

@@ -0,0 +1,25 @@
+package com.amazonaws.examples
+
+import com.amazonaws.client.builder.AwsClientBuilder
+import com.amazonaws.regions.{Region, Regions}
+import com.amazonaws.services.sqs.AmazonSQSClientBuilder
+
+import scala.jdk.CollectionConverters._
+
+object QueueTest extends App {
+  val region = Region.getRegion(Regions.US_WEST_2).getName
+  val serviceEndpoint = "http://localhost:5000"
+
+  val amazonSqs =  AmazonSQSClientBuilder.standard()
+    .withEndpointConfiguration(
+      new AwsClientBuilder.EndpointConfiguration(serviceEndpoint, region))
+    .build
+
+  val queueName = "my-first-queue"
+  amazonSqs.createQueue(queueName)
+
+  val urls = amazonSqs.listQueues().getQueueUrls.asScala
+  println("Listing queues")
+  println(urls.map(url => s" - $url").mkString(System.lineSeparator))
+  println()
+}

setup.py

@@ -38,7 +38,7 @@ install_requires = [
     "xmltodict",
     "six>1.9",
     "werkzeug",
-    "PyYAML",
+    "PyYAML>=5.1",
     "pytz",
     "python-dateutil<3.0.0,>=2.1",
     "python-jose<4.0.0",
@@ -89,10 +89,10 @@ setup(
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "License :: OSI Approved :: Apache Software License",
         "Topic :: Software Development :: Testing",
     ],

tests/test_core/test_socket.py

@@ -0,0 +1,48 @@
+import unittest
+from moto import mock_dynamodb2_deprecated, mock_dynamodb2
+import socket
+
+from six import PY3
+
+
+class TestSocketPair(unittest.TestCase):
+
+    @mock_dynamodb2_deprecated
+    def test_asyncio_deprecated(self):
+        if PY3:
+            self.assertIn(
+                'moto.packages.httpretty.core.fakesock.socket',
+                str(socket.socket),
+                'Our mock should be present'
+            )
+            import asyncio
+            self.assertIsNotNone(asyncio.get_event_loop())
+
+    @mock_dynamodb2_deprecated
+    def test_socket_pair_deprecated(self):
+
+        # In Python2, the fakesocket is not set, for some reason.
+        if PY3:
+            self.assertIn(
+                'moto.packages.httpretty.core.fakesock.socket',
+                str(socket.socket),
+                'Our mock should be present'
+            )
+        a, b = socket.socketpair()
+        self.assertIsNotNone(a)
+        self.assertIsNotNone(b)
+        if a:
+            a.close()
+        if b:
+            b.close()
+
+
+    @mock_dynamodb2
+    def test_socket_pair(self):
+        a, b = socket.socketpair()
+        self.assertIsNotNone(a)
+        self.assertIsNotNone(b)
+        if a:
+            a.close()
+        if b:
+            b.close()

tests/test_iam/test_iam.py

@@ -311,6 +311,15 @@ def test_put_role_policy():
     policy.should.equal("test policy")
 
 
+@mock_iam
+def test_get_role_policy():
+    conn = boto3.client('iam', region_name='us-east-1')
+    conn.create_role(
+        RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="my-path")
+    with assert_raises(conn.exceptions.NoSuchEntityException):
+        conn.get_role_policy(RoleName="my-role", PolicyName="does-not-exist")
+
+
 @mock_iam_deprecated()
 def test_update_assume_role_policy():
     conn = boto.connect_iam()
@@ -361,15 +370,15 @@ def test_create_many_policy_versions():
     conn = boto3.client('iam', region_name='us-east-1')
     conn.create_policy(
         PolicyName="TestCreateManyPolicyVersions",
-        PolicyDocument='{"some":"policy"}')
+        PolicyDocument=MOCK_POLICY)
     for _ in range(0, 4):
         conn.create_policy_version(
             PolicyArn="arn:aws:iam::123456789012:policy/TestCreateManyPolicyVersions",
-            PolicyDocument='{"some":"policy"}')
+            PolicyDocument=MOCK_POLICY)
     with assert_raises(ClientError):
         conn.create_policy_version(
             PolicyArn="arn:aws:iam::123456789012:policy/TestCreateManyPolicyVersions",
-            PolicyDocument='{"some":"policy"}')
+            PolicyDocument=MOCK_POLICY)
 
 
 @mock_iam
@@ -377,22 +386,22 @@ def test_set_default_policy_version():
     conn = boto3.client('iam', region_name='us-east-1')
     conn.create_policy(
         PolicyName="TestSetDefaultPolicyVersion",
-        PolicyDocument='{"first":"policy"}')
+        PolicyDocument=MOCK_POLICY)
     conn.create_policy_version(
         PolicyArn="arn:aws:iam::123456789012:policy/TestSetDefaultPolicyVersion",
-        PolicyDocument='{"second":"policy"}',
+        PolicyDocument=MOCK_POLICY_2,
         SetAsDefault=True)
     conn.create_policy_version(
         PolicyArn="arn:aws:iam::123456789012:policy/TestSetDefaultPolicyVersion",
-        PolicyDocument='{"third":"policy"}',
+        PolicyDocument=MOCK_POLICY_3,
         SetAsDefault=True)
     versions = conn.list_policy_versions(
         PolicyArn="arn:aws:iam::123456789012:policy/TestSetDefaultPolicyVersion")
-    versions.get('Versions')[0].get('Document').should.equal({'first': 'policy'})
+    versions.get('Versions')[0].get('Document').should.equal(json.loads(MOCK_POLICY))
     versions.get('Versions')[0].get('IsDefaultVersion').shouldnt.be.ok
-    versions.get('Versions')[1].get('Document').should.equal({'second': 'policy'})
+    versions.get('Versions')[1].get('Document').should.equal(json.loads(MOCK_POLICY_2))
     versions.get('Versions')[1].get('IsDefaultVersion').shouldnt.be.ok
-    versions.get('Versions')[2].get('Document').should.equal({'third': 'policy'})
+    versions.get('Versions')[2].get('Document').should.equal(json.loads(MOCK_POLICY_3))
     versions.get('Versions')[2].get('IsDefaultVersion').should.be.ok
 
 
@@ -524,10 +533,10 @@ def test_delete_default_policy_version():
     conn = boto3.client('iam', region_name='us-east-1')
     conn.create_policy(
         PolicyName="TestDeletePolicyVersion",
-        PolicyDocument='{"first":"policy"}')
+        PolicyDocument=MOCK_POLICY)
     conn.create_policy_version(
         PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion",
-        PolicyDocument='{"second":"policy"}')
+        PolicyDocument=MOCK_POLICY_2)
     with assert_raises(ClientError):
         conn.delete_policy_version(
             PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion",

tests/test_route53/test_route53.py

@@ -110,6 +110,7 @@ def test_rrset():
 
     changes = ResourceRecordSets(conn, zoneid)
     changes.add_change("DELETE", "foo.bar.testdns.aws.com", "A")
+    changes.add_change("DELETE", "foo.bar.testdns.aws.com", "TXT")
     changes.commit()
 
     changes = ResourceRecordSets(conn, zoneid)
@@ -582,7 +583,7 @@ def test_change_resource_record_sets_crud_valid():
     cname_record_detail['TTL'].should.equal(60)
     cname_record_detail['ResourceRecords'].should.equal([{'Value': '192.168.1.1'}])
 
-    # Delete record.
+    # Delete record with wrong type.
     delete_payload = {
         'Comment': 'delete prod.redis.db',
         'Changes': [
@@ -597,6 +598,23 @@ def test_change_resource_record_sets_crud_valid():
     }
     conn.change_resource_record_sets(HostedZoneId=hosted_zone_id, ChangeBatch=delete_payload)
     response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id)
+    len(response['ResourceRecordSets']).should.equal(1)
+
+    # Delete record.
+    delete_payload = {
+        'Comment': 'delete prod.redis.db',
+        'Changes': [
+            {
+                'Action': 'DELETE',
+                'ResourceRecordSet': {
+                    'Name': 'prod.redis.db',
+                    'Type': 'A',
+                }
+            }
+        ]
+    }
+    conn.change_resource_record_sets(HostedZoneId=hosted_zone_id, ChangeBatch=delete_payload)
+    response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id)
     len(response['ResourceRecordSets']).should.equal(0)
 
 

tests/test_sns/test_topics_boto3.py

@@ -32,6 +32,18 @@ def test_create_and_delete_topic():
         topics = topics_json["Topics"]
         topics.should.have.length_of(0)
 
+
+@mock_sns
+def test_create_topic_with_attributes():
+    conn = boto3.client("sns", region_name="us-east-1")
+    conn.create_topic(Name='some-topic-with-attribute', Attributes={'DisplayName': 'test-topic'})
+    topics_json = conn.list_topics()
+    topic_arn = topics_json["Topics"][0]['TopicArn']
+
+    attributes = conn.get_topic_attributes(TopicArn=topic_arn)['Attributes']
+    attributes['DisplayName'].should.equal('test-topic')
+
+
 @mock_sns
 def test_create_topic_should_be_indempodent():
     conn = boto3.client("sns", region_name="us-east-1")