From 199ff05e4ef54f3988e6c6d6f1d20dee15959825 Mon Sep 17 00:00:00 2001 From: John Corrales Date: Sat, 14 Dec 2019 19:21:41 -0800 Subject: [PATCH 01/48] parent 0f67a74d254127d4a64570145966679ab68d9f3d author John Corrales 1576380101 -0800 committer John Corrales 1576633072 -0800 added send_ssh_public_key --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/__init__.py | 1 + moto/backends.py | 2 ++ moto/ec2_instance_connect/__init__.py | 4 ++++ moto/ec2_instance_connect/models.py | 11 +++++++++ moto/ec2_instance_connect/responses.py | 9 ++++++++ moto/ec2_instance_connect/urls.py | 6 +++++ .../test_ec2_instance_connect_boto3.py | 23 +++++++++++++++++++ 8 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 moto/ec2_instance_connect/__init__.py create mode 100644 moto/ec2_instance_connect/models.py create mode 100644 moto/ec2_instance_connect/responses.py create mode 100644 moto/ec2_instance_connect/urls.py create mode 100644 tests/test_ec2_instance_connect/test_ec2_instance_connect_boto3.py diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 5d9f18ebf..e243c16aa 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2425,7 +2425,7 @@ ## ec2-instance-connect 0% implemented -- [ ] send_ssh_public_key +- [x] send_ssh_public_key ## ecr 27% implemented diff --git a/moto/__init__.py b/moto/__init__.py index 767c0ee27..7006ee588 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -20,6 +20,7 @@ from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # noqa from .dynamodbstreams import mock_dynamodbstreams # noqa from .ec2 import mock_ec2, mock_ec2_deprecated # noqa +from .ec2_instance_connect import mock_ec2_instance_connect # noqa from .ecr import mock_ecr, mock_ecr_deprecated # noqa from .ecs import mock_ecs, mock_ecs_deprecated # noqa from .elb import mock_elb, mock_elb_deprecated # noqa diff --git a/moto/backends.py b/moto/backends.py index 53a5cafc3..9a741d6f3 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -18,6 +18,7 @@ from moto.dynamodb import dynamodb_backends from moto.dynamodb2 import dynamodb_backends2 from moto.dynamodbstreams import dynamodbstreams_backends from moto.ec2 import ec2_backends +from moto.ec2_instance_connect import ec2_instance_connect_backends from moto.ecr import ecr_backends from moto.ecs import ecs_backends from moto.elb import elb_backends @@ -69,6 +70,7 @@ BACKENDS = { "dynamodb2": dynamodb_backends2, "dynamodbstreams": dynamodbstreams_backends, "ec2": ec2_backends, + "ec2_instance_connect": ec2_instance_connect_backends, "ecr": ecr_backends, "ecs": ecs_backends, "elb": elb_backends, diff --git a/moto/ec2_instance_connect/__init__.py b/moto/ec2_instance_connect/__init__.py new file mode 100644 index 000000000..c20d59cfa --- /dev/null +++ b/moto/ec2_instance_connect/__init__.py @@ -0,0 +1,4 @@ +from ..core.models import base_decorator +from .models import ec2_instance_connect_backends + +mock_ec2_instance_connect = base_decorator(ec2_instance_connect_backends) diff --git a/moto/ec2_instance_connect/models.py b/moto/ec2_instance_connect/models.py new file mode 100644 index 000000000..cc8cc3f33 --- /dev/null +++ b/moto/ec2_instance_connect/models.py @@ -0,0 +1,11 @@ +import boto +from moto.core import BaseBackend + + +class Ec2InstanceConnectBackend(BaseBackend): + pass + + +ec2_instance_connect_backends = {} +for region in boto.ec2.regions(): + ec2_instance_connect_backends[region.name] = Ec2InstanceConnectBackend() diff --git a/moto/ec2_instance_connect/responses.py b/moto/ec2_instance_connect/responses.py new file mode 100644 index 000000000..462f1fddc --- /dev/null +++ b/moto/ec2_instance_connect/responses.py @@ -0,0 +1,9 @@ +import json +from moto.core.responses import BaseResponse + + +class Ec2InstanceConnectResponse(BaseResponse): + def send_ssh_public_key(self): + return json.dumps( + {"RequestId": "example-2a47-4c91-9700-e37e85162cb6", "Success": True} + ) diff --git a/moto/ec2_instance_connect/urls.py b/moto/ec2_instance_connect/urls.py new file mode 100644 index 000000000..829a2145b --- /dev/null +++ b/moto/ec2_instance_connect/urls.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .responses import Ec2InstanceConnectResponse + +url_bases = ["https?://ec2-instance-connect.(.+).amazonaws.com"] + +url_paths = {"{0}/$": Ec2InstanceConnectResponse.dispatch} diff --git a/tests/test_ec2_instance_connect/test_ec2_instance_connect_boto3.py b/tests/test_ec2_instance_connect/test_ec2_instance_connect_boto3.py new file mode 100644 index 000000000..eb685d80a --- /dev/null +++ b/tests/test_ec2_instance_connect/test_ec2_instance_connect_boto3.py @@ -0,0 +1,23 @@ +import boto3 + +from moto import mock_ec2_instance_connect + +pubkey = """ssh-rsa +AAAAB3NzaC1yc2EAAAADAQABAAABAQDV5+voluw2zmzqpqCAqtsyoP01TQ8Ydx1eS1yD6wUsHcPqMIqpo57YxiC8XPwrdeKQ6GG6MC3bHsgXoPypGP0LyixbiuLTU31DnnqorcHt4bWs6rQa7dK2pCCflz2fhYRt5ZjqSNsAKivIbqkH66JozN0SySIka3kEV79GdB0BicioKeEJlCwM9vvxafyzjWf/z8E0lh4ni3vkLpIVJ0t5l+Qd9QMJrT6Is0SCQPVagTYZoi8+fWDoGsBa8vyRwDjEzBl28ZplKh9tSyDkRIYszWTpmK8qHiqjLYZBfAxXjGJbEYL1iig4ZxvbYzKEiKSBi1ZMW9iWjHfZDZuxXAmB +example +""" + + +@mock_ec2_instance_connect +def test_send_ssh_public_key(): + client = boto3.client("ec2-instance-connect", region_name="us-east-1") + fake_request_id = "example-2a47-4c91-9700-e37e85162cb6" + + response = client.send_ssh_public_key( + InstanceId="i-abcdefg12345", + InstanceOSUser="ec2-user", + SSHPublicKey=pubkey, + AvailabilityZone="us-east-1a", + ) + + assert response["RequestId"] == fake_request_id From d508bd72ce06f5d179569860ae90facfe958a9d4 Mon Sep 17 00:00:00 2001 From: John Corrales Date: Thu, 19 Dec 2019 17:47:17 -0800 Subject: [PATCH 02/48] escape the dots --- moto/ec2/urls.py | 2 +- moto/ec2_instance_connect/urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/ec2/urls.py b/moto/ec2/urls.py index b83a9e950..4d85b2f56 100644 --- a/moto/ec2/urls.py +++ b/moto/ec2/urls.py @@ -2,6 +2,6 @@ from __future__ import unicode_literals from .responses import EC2Response -url_bases = ["https?://ec2.(.+).amazonaws.com(|.cn)"] +url_bases = ["https?://ec2\.(.+)\.amazonaws\.com(|\.cn)"] url_paths = {"{0}/": EC2Response.dispatch} diff --git a/moto/ec2_instance_connect/urls.py b/moto/ec2_instance_connect/urls.py index 829a2145b..e7078264f 100644 --- a/moto/ec2_instance_connect/urls.py +++ b/moto/ec2_instance_connect/urls.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals from .responses import Ec2InstanceConnectResponse -url_bases = ["https?://ec2-instance-connect.(.+).amazonaws.com"] +url_bases = ["https?://ec2-instance-connect\.(.+)\.amazonaws\.com"] url_paths = {"{0}/$": Ec2InstanceConnectResponse.dispatch} From 1415a9359675c28a4a2e0138a5f011342ee3016c Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Thu, 19 Dec 2019 18:30:43 -0800 Subject: [PATCH 03/48] Implement List user tags --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/iam/models.py | 11 ++++++-- moto/iam/responses.py | 27 ++++++++++++++++-- tests/test_iam/test_iam.py | 57 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 88 insertions(+), 9 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 5d9f18ebf..6a3b80cbd 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3447,7 +3447,7 @@ - [X] list_signing_certificates - [ ] list_ssh_public_keys - [X] list_user_policies -- [ ] list_user_tags +- [X] list_user_tags - [X] list_users - [X] list_virtual_mfa_devices - [X] put_group_policy diff --git a/moto/iam/models.py b/moto/iam/models.py index 5bbd9235d..18b3a7a6f 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -543,7 +543,7 @@ class Group(BaseModel): class User(BaseModel): - def __init__(self, name, path=None): + def __init__(self, name, path=None, tags=None): self.name = name self.id = random_resource_id() self.path = path if path else "/" @@ -556,6 +556,7 @@ class User(BaseModel): self.password = None self.password_reset_required = False self.signing_certificates = {} + self.tags = tags @property def arn(self): @@ -1421,13 +1422,13 @@ class IAMBackend(BaseBackend): "The group with name {0} cannot be found.".format(group_name) ) - def create_user(self, user_name, path="/"): + def create_user(self, user_name, path="/", tags=None): if user_name in self.users: raise IAMConflictException( "EntityAlreadyExists", "User {0} already exists".format(user_name) ) - user = User(user_name, path) + user = User(user_name, path, tags) self.users[user_name] = user return user @@ -1583,6 +1584,10 @@ class IAMBackend(BaseBackend): user = self.get_user(user_name) return user.policies.keys() + def list_user_tags(self, user_name): + user = self.get_user(user_name) + return user.tags + def put_user_policy(self, user_name, policy_name, policy_json): user = self.get_user(user_name) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 45bd28c36..06561d4c4 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -440,8 +440,8 @@ class IamResponse(BaseResponse): def create_user(self): user_name = self._get_param("UserName") path = self._get_param("Path") - - user = iam_backend.create_user(user_name, path) + tags = self._get_multi_param("Tags.member") + user = iam_backend.create_user(user_name, path, tags) template = self.response_template(USER_TEMPLATE) return template.render(action="Create", user=user) @@ -538,6 +538,12 @@ class IamResponse(BaseResponse): template = self.response_template(LIST_USER_POLICIES_TEMPLATE) return template.render(policies=policies) + def list_user_tags(self): + user_name = self._get_param("UserName") + tags = iam_backend.list_user_tags(user_name) + template = self.response_template(LIST_USER_TAGS_TEMPLATE) + return template.render(user_tags=tags or []) + def put_user_policy(self): user_name = self._get_param("UserName") policy_name = self._get_param("PolicyName") @@ -1699,6 +1705,23 @@ LIST_USER_POLICIES_TEMPLATE = """ """ +LIST_USER_TAGS_TEMPLATE = """ + + + {% for tag in user_tags %} + + {{ tag.Key }} + {{ tag.Value }} + + {% endfor %} + + false + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + CREATE_ACCESS_KEY_TEMPLATE = """ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 6311dce9c..cabb6d037 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1737,9 +1737,7 @@ def test_delete_saml_provider(): def test_create_role_defaults(): """Tests default values""" conn = boto3.client("iam", region_name="us-east-1") - conn.create_role( - RoleName="my-role", AssumeRolePolicyDocument="{}", - ) + conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="{}") # Get role: role = conn.get_role(RoleName="my-role")["Role"] @@ -2672,3 +2670,56 @@ def test_get_account_summary(): "GroupsQuota": 300, } ) + + +@mock_iam() +def test_list_user_tags(): + """Tests both setting a tags on a user in create_user and list_user_tags""" + conn = boto3.client("iam", region_name="us-east-1") + conn.create_user(UserName="kenny-bania") + conn.create_user( + UserName="jackie-chiles", Tags=[{"Key": "Sue-Allen", "Value": "Oh-Henry"}] + ) + conn.create_user( + UserName="cosmo", + Tags=[ + {"Key": "Stan", "Value": "The Caddy"}, + {"Key": "like-a", "Value": "glove"}, + ], + ) + + assert conn.list_user_tags(UserName="kenny-bania") == { + "Tags": [], + "IsTruncated": False, + "ResponseMetadata": { + "RequestId": "7a62c49f-347e-4fc4-9331-6e8eEXAMPLE", + "HTTPStatusCode": 200, + "HTTPHeaders": {"server": "amazon.com"}, + "RetryAttempts": 0, + }, + } + + assert conn.list_user_tags(UserName="jackie-chiles") == { + "Tags": [{"Key": "Sue-Allen", "Value": "Oh-Henry"}], + "IsTruncated": False, + "ResponseMetadata": { + "RequestId": "7a62c49f-347e-4fc4-9331-6e8eEXAMPLE", + "HTTPStatusCode": 200, + "HTTPHeaders": {"server": "amazon.com"}, + "RetryAttempts": 0, + }, + } + + assert conn.list_user_tags(UserName="cosmo") == { + "Tags": [ + {"Key": "Stan", "Value": "The Caddy"}, + {"Key": "like-a", "Value": "glove"}, + ], + "IsTruncated": False, + "ResponseMetadata": { + "RequestId": "7a62c49f-347e-4fc4-9331-6e8eEXAMPLE", + "HTTPStatusCode": 200, + "HTTPHeaders": {"server": "amazon.com"}, + "RetryAttempts": 0, + }, + } From c2e444a210740ee10c751c3c03b9db38e20ee671 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Fri, 20 Dec 2019 10:54:33 -0800 Subject: [PATCH 04/48] Update tests. --- tests/test_iam/test_iam.py | 45 ++++++++++---------------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index cabb6d037..9a2c1f0dd 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -2687,39 +2687,16 @@ def test_list_user_tags(): {"Key": "like-a", "Value": "glove"}, ], ) + response = conn.list_user_tags(UserName="kenny-bania") + response["Tags"].should.equal([]) + response["IsTruncated"].should_not.be.ok - assert conn.list_user_tags(UserName="kenny-bania") == { - "Tags": [], - "IsTruncated": False, - "ResponseMetadata": { - "RequestId": "7a62c49f-347e-4fc4-9331-6e8eEXAMPLE", - "HTTPStatusCode": 200, - "HTTPHeaders": {"server": "amazon.com"}, - "RetryAttempts": 0, - }, - } + response = conn.list_user_tags(UserName="jackie-chiles") + response["Tags"].should.equal([{"Key": "Sue-Allen", "Value": "Oh-Henry"}]) + response["IsTruncated"].should_not.be.ok - assert conn.list_user_tags(UserName="jackie-chiles") == { - "Tags": [{"Key": "Sue-Allen", "Value": "Oh-Henry"}], - "IsTruncated": False, - "ResponseMetadata": { - "RequestId": "7a62c49f-347e-4fc4-9331-6e8eEXAMPLE", - "HTTPStatusCode": 200, - "HTTPHeaders": {"server": "amazon.com"}, - "RetryAttempts": 0, - }, - } - - assert conn.list_user_tags(UserName="cosmo") == { - "Tags": [ - {"Key": "Stan", "Value": "The Caddy"}, - {"Key": "like-a", "Value": "glove"}, - ], - "IsTruncated": False, - "ResponseMetadata": { - "RequestId": "7a62c49f-347e-4fc4-9331-6e8eEXAMPLE", - "HTTPStatusCode": 200, - "HTTPHeaders": {"server": "amazon.com"}, - "RetryAttempts": 0, - }, - } + response = conn.list_user_tags(UserName="cosmo") + response["Tags"].should.equal( + [{"Key": "Stan", "Value": "The Caddy"}, {"Key": "like-a", "Value": "glove"}] + ) + response["IsTruncated"].should_not.be.ok From a6f14eee3fc1570f8bb26929001aa1cea4973816 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Fri, 20 Dec 2019 11:16:17 -0800 Subject: [PATCH 05/48] Add test to repo stream issue --- tests/test_dynamodb2/test_dynamodb.py | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 1a8a70615..01768e8e4 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -348,6 +348,35 @@ def test_put_item_with_special_chars(): '"': {"S": "foo"}, }, ) +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_put_item_with_streams(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + StreamSpecification={'StreamEnabled': True, 'StreamViewType': 'NEW_AND_OLD_IMAGES'}, + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}} + }, + ) @requires_boto_gte("2.9") From 0a7d299da3594ca90082c2389c39dd3e750a57a8 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Fri, 20 Dec 2019 11:18:04 -0800 Subject: [PATCH 06/48] Fix json encoding issue. --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 121f564a4..784dfc5d7 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -586,7 +586,7 @@ class StreamRecord(BaseModel): self.record["dynamodb"]["OldImage"] = old_a # This is a substantial overestimate but it's the easiest to do now - self.record["dynamodb"]["SizeBytes"] = len(json.dumps(self.record["dynamodb"])) + self.record["dynamodb"]["SizeBytes"] = len(dynamo_json_dump(self.record["dynamodb"])) def to_json(self): return self.record From 5bd3827b26109199e03ed609c1020540dc101e69 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Fri, 20 Dec 2019 11:30:36 -0800 Subject: [PATCH 07/48] run black --- moto/dynamodb2/models.py | 4 +++- tests/test_dynamodb2/test_dynamodb.py | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 784dfc5d7..0a1b2679f 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -586,7 +586,9 @@ class StreamRecord(BaseModel): self.record["dynamodb"]["OldImage"] = old_a # This is a substantial overestimate but it's the easiest to do now - self.record["dynamodb"]["SizeBytes"] = len(dynamo_json_dump(self.record["dynamodb"])) + self.record["dynamodb"]["SizeBytes"] = len( + dynamo_json_dump(self.record["dynamodb"]) + ) def to_json(self): return self.record diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 01768e8e4..a64ba4a8a 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -348,6 +348,8 @@ def test_put_item_with_special_chars(): '"': {"S": "foo"}, }, ) + + @requires_boto_gte("2.9") @mock_dynamodb2 def test_put_item_with_streams(): @@ -363,7 +365,10 @@ def test_put_item_with_streams(): TableName=name, KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - StreamSpecification={'StreamEnabled': True, 'StreamViewType': 'NEW_AND_OLD_IMAGES'}, + StreamSpecification={ + "StreamEnabled": True, + "StreamViewType": "NEW_AND_OLD_IMAGES", + }, ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, ) @@ -374,7 +379,7 @@ def test_put_item_with_streams(): "subject": {"S": "Check this out!"}, "Body": {"S": "http://url_to_lolcat.gif"}, "SentBy": {"S": "test"}, - "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}} + "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, }, ) From bcc4a7486912235d8c1306e3fc266ab48348444a Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 21 Dec 2019 12:08:13 +0100 Subject: [PATCH 08/48] avoid rendering "None" as S3 Prefix value --- moto/s3/responses.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 3fa793f25..71f21c8e1 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1491,7 +1491,9 @@ S3_ALL_BUCKETS = """ {{ bucket.name }} + {% if prefix != None %} {{ prefix }} + {% endif %} {{ max_keys }} {{ delimiter }} {{ is_truncated }} @@ -1523,7 +1525,9 @@ S3_BUCKET_GET_RESPONSE = """ S3_BUCKET_GET_RESPONSE_V2 = """ {{ bucket.name }} +{% if prefix != None %} {{ prefix }} +{% endif %} {{ max_keys }} {{ key_count }} {% if delimiter %} @@ -1684,7 +1688,9 @@ S3_BUCKET_GET_VERSIONING = """ S3_BUCKET_GET_VERSIONS = """ {{ bucket.name }} + {% if prefix != None %} {{ prefix }} + {% endif %} {{ key_marker }} {{ max_keys }} {{ is_truncated }} From 4ad111830b79ed6bbfd8462cf0a5d9c3a4ccc2d7 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 21 Dec 2019 13:27:49 +0100 Subject: [PATCH 09/48] fix tests --- tests/test_s3/test_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 3cf3bc6f1..682213d13 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1261,7 +1261,7 @@ def test_boto3_list_objects_truncated_response(): assert listed_object["Key"] == "one" assert resp["MaxKeys"] == 1 assert resp["IsTruncated"] == True - assert resp["Prefix"] == "None" + assert resp.get("Prefix") is None assert resp["Delimiter"] == "None" assert "NextMarker" in resp @@ -1274,7 +1274,7 @@ def test_boto3_list_objects_truncated_response(): assert listed_object["Key"] == "three" assert resp["MaxKeys"] == 1 assert resp["IsTruncated"] == True - assert resp["Prefix"] == "None" + assert resp.get("Prefix") is None assert resp["Delimiter"] == "None" assert "NextMarker" in resp @@ -1287,7 +1287,7 @@ def test_boto3_list_objects_truncated_response(): assert listed_object["Key"] == "two" assert resp["MaxKeys"] == 1 assert resp["IsTruncated"] == False - assert resp["Prefix"] == "None" + assert resp.get("Prefix") is None assert resp["Delimiter"] == "None" assert "NextMarker" not in resp From df951facc53ff1c30332ec56846a14c7db5dac3e Mon Sep 17 00:00:00 2001 From: gruebel Date: Sun, 22 Dec 2019 11:42:15 +0100 Subject: [PATCH 10/48] Add codepipeline.list_tags_for_resource --- moto/codepipeline/models.py | 15 + moto/codepipeline/responses.py | 7 + tests/test_codepipeline/test_codepipeline.py | 469 ++++--------------- 3 files changed, 119 insertions(+), 372 deletions(-) diff --git a/moto/codepipeline/models.py b/moto/codepipeline/models.py index b3e76f838..888491296 100644 --- a/moto/codepipeline/models.py +++ b/moto/codepipeline/models.py @@ -145,6 +145,21 @@ class CodePipelineBackend(BaseBackend): def delete_pipeline(self, name): self.pipelines.pop(name, None) + def list_tags_for_resource(self, arn): + name = arn.split(":")[-1] + pipeline = self.pipelines.get(name) + + if not pipeline: + raise ResourceNotFoundException( + "The account with id '{0}' does not include a pipeline with the name '{1}'".format( + ACCOUNT_ID, name + ) + ) + + tags = [{"key": key, "value": value} for key, value in pipeline.tags.items()] + + return tags + codepipeline_backends = {} for region in Session().get_available_regions("codepipeline"): diff --git a/moto/codepipeline/responses.py b/moto/codepipeline/responses.py index f2eee4d4e..75a2ce800 100644 --- a/moto/codepipeline/responses.py +++ b/moto/codepipeline/responses.py @@ -39,3 +39,10 @@ class CodePipelineResponse(BaseResponse): self.codepipeline_backend.delete_pipeline(self._get_param("name")) return "" + + def list_tags_for_resource(self): + tags = self.codepipeline_backend.list_tags_for_resource( + self._get_param("resourceArn") + ) + + return json.dumps({"tags": tags}) diff --git a/tests/test_codepipeline/test_codepipeline.py b/tests/test_codepipeline/test_codepipeline.py index 926d7f873..b93a15fc7 100644 --- a/tests/test_codepipeline/test_codepipeline.py +++ b/tests/test_codepipeline/test_codepipeline.py @@ -13,52 +13,7 @@ from moto import mock_codepipeline, mock_iam def test_create_pipeline(): client = boto3.client("codepipeline", region_name="us-east-1") - response = client.create_pipeline( - pipeline={ - "name": "test-pipeline", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - }, - tags=[{"key": "key", "value": "value"}], - ) + response = create_basic_codepipeline(client, "test-pipeline") response["pipeline"].should.equal( { @@ -120,98 +75,10 @@ def test_create_pipeline(): def test_create_pipeline_errors(): client = boto3.client("codepipeline", region_name="us-east-1") client_iam = boto3.client("iam", region_name="us-east-1") - client.create_pipeline( - pipeline={ - "name": "test-pipeline", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - } - ) + create_basic_codepipeline(client, "test-pipeline") with assert_raises(ClientError) as e: - client.create_pipeline( - pipeline={ - "name": "test-pipeline", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - } - ) + create_basic_codepipeline(client, "test-pipeline") ex = e.exception ex.operation_name.should.equal("CreatePipeline") ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) @@ -348,52 +215,7 @@ def test_create_pipeline_errors(): @mock_codepipeline def test_get_pipeline(): client = boto3.client("codepipeline", region_name="us-east-1") - client.create_pipeline( - pipeline={ - "name": "test-pipeline", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - }, - tags=[{"key": "key", "value": "value"}], - ) + create_basic_codepipeline(client, "test-pipeline") response = client.get_pipeline(name="test-pipeline") @@ -474,53 +296,7 @@ def test_get_pipeline_errors(): @mock_codepipeline def test_update_pipeline(): client = boto3.client("codepipeline", region_name="us-east-1") - role_arn = get_role_arn() - client.create_pipeline( - pipeline={ - "name": "test-pipeline", - "roleArn": role_arn, - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - }, - tags=[{"key": "key", "value": "value"}], - ) + create_basic_codepipeline(client, "test-pipeline") response = client.get_pipeline(name="test-pipeline") created_time = response["metadata"]["created"] @@ -529,7 +305,7 @@ def test_update_pipeline(): response = client.update_pipeline( pipeline={ "name": "test-pipeline", - "roleArn": role_arn, + "roleArn": get_role_arn(), "artifactStore": { "type": "S3", "location": "codepipeline-us-east-1-123456789012", @@ -692,105 +468,19 @@ def test_update_pipeline_errors(): @mock_codepipeline def test_list_pipelines(): client = boto3.client("codepipeline", region_name="us-east-1") - client.create_pipeline( - pipeline={ - "name": "test-pipeline-1", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - }, - ) - client.create_pipeline( - pipeline={ - "name": "test-pipeline-2", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - }, - ) + name_1 = "test-pipeline-1" + create_basic_codepipeline(client, name_1) + name_2 = "test-pipeline-2" + create_basic_codepipeline(client, name_2) response = client.list_pipelines() response["pipelines"].should.have.length_of(2) - response["pipelines"][0]["name"].should.equal("test-pipeline-1") + response["pipelines"][0]["name"].should.equal(name_1) response["pipelines"][0]["version"].should.equal(1) response["pipelines"][0]["created"].should.be.a(datetime) response["pipelines"][0]["updated"].should.be.a(datetime) - response["pipelines"][1]["name"].should.equal("test-pipeline-2") + response["pipelines"][1]["name"].should.equal(name_2) response["pipelines"][1]["version"].should.equal(1) response["pipelines"][1]["created"].should.be.a(datetime) response["pipelines"][1]["updated"].should.be.a(datetime) @@ -799,68 +489,54 @@ def test_list_pipelines(): @mock_codepipeline def test_delete_pipeline(): client = boto3.client("codepipeline", region_name="us-east-1") - client.create_pipeline( - pipeline={ - "name": "test-pipeline", - "roleArn": get_role_arn(), - "artifactStore": { - "type": "S3", - "location": "codepipeline-us-east-1-123456789012", - }, - "stages": [ - { - "name": "Stage-1", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Source", - "owner": "AWS", - "provider": "S3", - "version": "1", - }, - "configuration": { - "S3Bucket": "test-bucket", - "S3ObjectKey": "test-object", - }, - "outputArtifacts": [{"name": "artifact"},], - }, - ], - }, - { - "name": "Stage-2", - "actions": [ - { - "name": "Action-1", - "actionTypeId": { - "category": "Approval", - "owner": "AWS", - "provider": "Manual", - "version": "1", - }, - }, - ], - }, - ], - }, - ) + name = "test-pipeline" + create_basic_codepipeline(client, name) client.list_pipelines()["pipelines"].should.have.length_of(1) - client.delete_pipeline(name="test-pipeline") + client.delete_pipeline(name=name) client.list_pipelines()["pipelines"].should.have.length_of(0) # deleting a not existing pipeline, should raise no exception - client.delete_pipeline(name="test-pipeline") + client.delete_pipeline(name=name) + + +@mock_codepipeline +def test_list_tags_for_resource(): + client = boto3.client("codepipeline", region_name="us-east-1") + name = "test-pipeline" + create_basic_codepipeline(client, name) + + response = client.list_tags_for_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name) + ) + response["tags"].should.equal([{"key": "key", "value": "value"}]) + + +@mock_codepipeline +def test_list_tags_for_resource_errors(): + client = boto3.client("codepipeline", region_name="us-east-1") + + with assert_raises(ClientError) as e: + client.list_tags_for_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:not-existing" + ) + ex = e.exception + ex.operation_name.should.equal("ListTagsForResource") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("ResourceNotFoundException") + ex.response["Error"]["Message"].should.equal( + "The account with id '123456789012' does not include a pipeline with the name 'not-existing'" + ) @mock_iam def get_role_arn(): - iam = boto3.client("iam", region_name="us-east-1") + client = boto3.client("iam", region_name="us-east-1") try: - return iam.get_role(RoleName="test-role")["Role"]["Arn"] + return client.get_role(RoleName="test-role")["Role"]["Arn"] except ClientError: - return iam.create_role( + return client.create_role( RoleName="test-role", AssumeRolePolicyDocument=json.dumps( { @@ -875,3 +551,52 @@ def get_role_arn(): } ), )["Role"]["Arn"] + + +def create_basic_codepipeline(client, name): + return client.create_pipeline( + pipeline={ + "name": name, + "roleArn": get_role_arn(), + "artifactStore": { + "type": "S3", + "location": "codepipeline-us-east-1-123456789012", + }, + "stages": [ + { + "name": "Stage-1", + "actions": [ + { + "name": "Action-1", + "actionTypeId": { + "category": "Source", + "owner": "AWS", + "provider": "S3", + "version": "1", + }, + "configuration": { + "S3Bucket": "test-bucket", + "S3ObjectKey": "test-object", + }, + "outputArtifacts": [{"name": "artifact"},], + }, + ], + }, + { + "name": "Stage-2", + "actions": [ + { + "name": "Action-1", + "actionTypeId": { + "category": "Approval", + "owner": "AWS", + "provider": "Manual", + "version": "1", + }, + }, + ], + }, + ], + }, + tags=[{"key": "key", "value": "value"}], + ) From b96a46b98fb717ae9fe660999a69e426a029a4b0 Mon Sep 17 00:00:00 2001 From: gruebel Date: Mon, 23 Dec 2019 19:33:37 +0100 Subject: [PATCH 11/48] Add codepipeline.tag_resource --- moto/codepipeline/exceptions.py | 16 +++++ moto/codepipeline/models.py | 32 +++++++++ moto/codepipeline/responses.py | 7 ++ tests/test_codepipeline/test_codepipeline.py | 72 ++++++++++++++++++++ 4 files changed, 127 insertions(+) diff --git a/moto/codepipeline/exceptions.py b/moto/codepipeline/exceptions.py index e455298cd..a4db9aab1 100644 --- a/moto/codepipeline/exceptions.py +++ b/moto/codepipeline/exceptions.py @@ -26,3 +26,19 @@ class ResourceNotFoundException(JsonRESTError): super(ResourceNotFoundException, self).__init__( "ResourceNotFoundException", message ) + + +class InvalidTagsException(JsonRESTError): + code = 400 + + def __init__(self, message): + super(InvalidTagsException, self).__init__("InvalidTagsException", message) + + +class TooManyTagsException(JsonRESTError): + code = 400 + + def __init__(self, arn): + super(TooManyTagsException, self).__init__( + "TooManyTagsException", "Tag limit exceeded for resource [{}].".format(arn) + ) diff --git a/moto/codepipeline/models.py b/moto/codepipeline/models.py index 888491296..4a8b89617 100644 --- a/moto/codepipeline/models.py +++ b/moto/codepipeline/models.py @@ -12,6 +12,8 @@ from moto.codepipeline.exceptions import ( InvalidStructureException, PipelineNotFoundException, ResourceNotFoundException, + InvalidTagsException, + TooManyTagsException, ) from moto.core import BaseBackend, BaseModel @@ -54,6 +56,18 @@ class CodePipeline(BaseModel): return pipeline + def validate_tags(self, tags): + for tag in tags: + if tag["key"].startswith("aws:"): + raise InvalidTagsException( + "Not allowed to modify system tags. " + "System tags start with 'aws:'. " + "msg=[Caller is an end user and not allowed to mutate system tags]" + ) + + if (len(self.tags) + len(tags)) > 50: + raise TooManyTagsException(self._arn) + class CodePipelineBackend(BaseBackend): def __init__(self): @@ -93,6 +107,8 @@ class CodePipelineBackend(BaseBackend): self.pipelines[pipeline["name"]] = CodePipeline(region, pipeline) if tags: + self.pipelines[pipeline["name"]].validate_tags(tags) + new_tags = {tag["key"]: tag["value"] for tag in tags} self.pipelines[pipeline["name"]].tags.update(new_tags) @@ -160,6 +176,22 @@ class CodePipelineBackend(BaseBackend): return tags + def tag_resource(self, arn, tags): + name = arn.split(":")[-1] + pipeline = self.pipelines.get(name) + + if not pipeline: + raise ResourceNotFoundException( + "The account with id '{0}' does not include a pipeline with the name '{1}'".format( + ACCOUNT_ID, name + ) + ) + + pipeline.validate_tags(tags) + + for tag in tags: + pipeline.tags.update({tag["key"]: tag["value"]}) + codepipeline_backends = {} for region in Session().get_available_regions("codepipeline"): diff --git a/moto/codepipeline/responses.py b/moto/codepipeline/responses.py index 75a2ce800..df1bf220f 100644 --- a/moto/codepipeline/responses.py +++ b/moto/codepipeline/responses.py @@ -46,3 +46,10 @@ class CodePipelineResponse(BaseResponse): ) return json.dumps({"tags": tags}) + + def tag_resource(self): + self.codepipeline_backend.tag_resource( + self._get_param("resourceArn"), self._get_param("tags") + ) + + return "" diff --git a/tests/test_codepipeline/test_codepipeline.py b/tests/test_codepipeline/test_codepipeline.py index b93a15fc7..e71e24f76 100644 --- a/tests/test_codepipeline/test_codepipeline.py +++ b/tests/test_codepipeline/test_codepipeline.py @@ -530,6 +530,78 @@ def test_list_tags_for_resource_errors(): ) +@mock_codepipeline +def test_tag_resource(): + client = boto3.client("codepipeline", region_name="us-east-1") + name = "test-pipeline" + create_basic_codepipeline(client, name) + + client.tag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name), + tags=[{"key": "key-2", "value": "value-2"}], + ) + + response = client.list_tags_for_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name) + ) + response["tags"].should.equal( + [{"key": "key", "value": "value"}, {"key": "key-2", "value": "value-2"}] + ) + + +@mock_codepipeline +def test_tag_resource_errors(): + client = boto3.client("codepipeline", region_name="us-east-1") + name = "test-pipeline" + create_basic_codepipeline(client, name) + + with assert_raises(ClientError) as e: + client.tag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:not-existing", + tags=[{"key": "key-2", "value": "value-2"}], + ) + ex = e.exception + ex.operation_name.should.equal("TagResource") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("ResourceNotFoundException") + ex.response["Error"]["Message"].should.equal( + "The account with id '123456789012' does not include a pipeline with the name 'not-existing'" + ) + + with assert_raises(ClientError) as e: + client.tag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name), + tags=[{"key": "aws:key", "value": "value"}], + ) + ex = e.exception + ex.operation_name.should.equal("TagResource") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidTagsException") + ex.response["Error"]["Message"].should.equal( + "Not allowed to modify system tags. " + "System tags start with 'aws:'. " + "msg=[Caller is an end user and not allowed to mutate system tags]" + ) + + with assert_raises(ClientError) as e: + client.tag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name), + tags=[ + {"key": "key-{}".format(i), "value": "value-{}".format(i)} + for i in range(50) + ], + ) + ex = e.exception + ex.operation_name.should.equal("TagResource") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("TooManyTagsException") + ex.response["Error"]["Message"].should.equal( + "Tag limit exceeded for resource [arn:aws:codepipeline:us-east-1:123456789012:{}].".format( + name + ) + ) + + @mock_iam def get_role_arn(): client = boto3.client("iam", region_name="us-east-1") From 8331d480bab8b0556718d9af0c73e19eb0a05104 Mon Sep 17 00:00:00 2001 From: gruebel Date: Mon, 23 Dec 2019 19:50:16 +0100 Subject: [PATCH 12/48] Add codepipeline.untag_resource --- IMPLEMENTATION_COVERAGE.md | 8 ++-- moto/codepipeline/models.py | 14 ++++++ moto/codepipeline/responses.py | 7 +++ tests/test_codepipeline/test_codepipeline.py | 46 ++++++++++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 5e6ef1c9e..a1b0ffb5e 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -1374,7 +1374,7 @@ - [ ] update_profiling_group ## codepipeline -13% implemented +22% implemented - [ ] acknowledge_job - [ ] acknowledge_third_party_job - [ ] create_custom_action_type @@ -1394,7 +1394,7 @@ - [ ] list_action_types - [ ] list_pipeline_executions - [X] list_pipelines -- [ ] list_tags_for_resource +- [X] list_tags_for_resource - [ ] list_webhooks - [ ] poll_for_jobs - [ ] poll_for_third_party_jobs @@ -1408,8 +1408,8 @@ - [ ] register_webhook_with_third_party - [ ] retry_stage_execution - [ ] start_pipeline_execution -- [ ] tag_resource -- [ ] untag_resource +- [X] tag_resource +- [X] untag_resource - [X] update_pipeline ## codestar diff --git a/moto/codepipeline/models.py b/moto/codepipeline/models.py index 4a8b89617..556682c10 100644 --- a/moto/codepipeline/models.py +++ b/moto/codepipeline/models.py @@ -192,6 +192,20 @@ class CodePipelineBackend(BaseBackend): for tag in tags: pipeline.tags.update({tag["key"]: tag["value"]}) + def untag_resource(self, arn, tag_keys): + name = arn.split(":")[-1] + pipeline = self.pipelines.get(name) + + if not pipeline: + raise ResourceNotFoundException( + "The account with id '{0}' does not include a pipeline with the name '{1}'".format( + ACCOUNT_ID, name + ) + ) + + for key in tag_keys: + pipeline.tags.pop(key, None) + codepipeline_backends = {} for region in Session().get_available_regions("codepipeline"): diff --git a/moto/codepipeline/responses.py b/moto/codepipeline/responses.py index df1bf220f..0223dfae6 100644 --- a/moto/codepipeline/responses.py +++ b/moto/codepipeline/responses.py @@ -53,3 +53,10 @@ class CodePipelineResponse(BaseResponse): ) return "" + + def untag_resource(self): + self.codepipeline_backend.untag_resource( + self._get_param("resourceArn"), self._get_param("tagKeys") + ) + + return "" diff --git a/tests/test_codepipeline/test_codepipeline.py b/tests/test_codepipeline/test_codepipeline.py index e71e24f76..a40efa05c 100644 --- a/tests/test_codepipeline/test_codepipeline.py +++ b/tests/test_codepipeline/test_codepipeline.py @@ -602,6 +602,52 @@ def test_tag_resource_errors(): ) +@mock_codepipeline +def test_untag_resource(): + client = boto3.client("codepipeline", region_name="us-east-1") + name = "test-pipeline" + create_basic_codepipeline(client, name) + + response = client.list_tags_for_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name) + ) + response["tags"].should.equal([{"key": "key", "value": "value"}]) + + client.untag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name), + tagKeys=["key"], + ) + + response = client.list_tags_for_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name) + ) + response["tags"].should.have.length_of(0) + + # removing a not existing tag should raise no exception + client.untag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:{}".format(name), + tagKeys=["key"], + ) + + +@mock_codepipeline +def test_untag_resource_errors(): + client = boto3.client("codepipeline", region_name="us-east-1") + + with assert_raises(ClientError) as e: + client.untag_resource( + resourceArn="arn:aws:codepipeline:us-east-1:123456789012:not-existing", + tagKeys=["key"], + ) + ex = e.exception + ex.operation_name.should.equal("UntagResource") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("ResourceNotFoundException") + ex.response["Error"]["Message"].should.equal( + "The account with id '123456789012' does not include a pipeline with the name 'not-existing'" + ) + + @mock_iam def get_role_arn(): client = boto3.client("iam", region_name="us-east-1") From 9455ab0e53eb18dcd25bbd37b8fe84d58027eb49 Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 24 Dec 2019 13:52:33 +0100 Subject: [PATCH 13/48] Fix Python 2.7 tests --- moto/codepipeline/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/codepipeline/models.py b/moto/codepipeline/models.py index 556682c10..e14c0c2d8 100644 --- a/moto/codepipeline/models.py +++ b/moto/codepipeline/models.py @@ -112,7 +112,7 @@ class CodePipelineBackend(BaseBackend): new_tags = {tag["key"]: tag["value"] for tag in tags} self.pipelines[pipeline["name"]].tags.update(new_tags) - return pipeline, tags + return pipeline, sorted(tags, key=lambda i: i["key"]) def get_pipeline(self, name): codepipeline = self.pipelines.get(name) @@ -174,7 +174,7 @@ class CodePipelineBackend(BaseBackend): tags = [{"key": key, "value": value} for key, value in pipeline.tags.items()] - return tags + return sorted(tags, key=lambda i: i["key"]) def tag_resource(self, arn, tags): name = arn.split(":")[-1] From 299e7851d6bfbdbf0c10e18d1c3ae991a15428e1 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Tue, 24 Dec 2019 10:23:46 -0800 Subject: [PATCH 14/48] Add assertions. --- tests/test_dynamodb2/test_dynamodb.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index a64ba4a8a..773ff61e1 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -9,7 +9,7 @@ from boto3.dynamodb.conditions import Attr, Key import sure # noqa import requests from moto import mock_dynamodb2, mock_dynamodb2_deprecated -from moto.dynamodb2 import dynamodb_backend2 +from moto.dynamodb2 import dynamodb_backend2, dynamodb_backends2 from boto.exception import JSONResponseError from botocore.exceptions import ClientError, ParamValidationError from tests.helpers import requires_boto_gte @@ -383,6 +383,23 @@ def test_put_item_with_streams(): }, ) + result = conn.get_item(TableName=name, Key={"forum_name": {"S": "LOLCat Forum"}}) + + result["Item"].should.be.equal( + { + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, + } + ) + stream_shard = dynamodb_backends2["us-west-2"].get_table(name).stream_shard + len(stream_shard.items).should.be.equal(1) + stream_record = stream_shard.items[0].record + stream_record["eventName"].should.be.equal("INSERT") + stream_record["dynamodb"]["SizeBytes"].should.be.equal(447) + @requires_boto_gte("2.9") @mock_dynamodb2 From 7d4c15d53e41b19d8b341d38bba8c6caa9847100 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Tue, 24 Dec 2019 11:01:54 -0800 Subject: [PATCH 15/48] skip part of the test in server mode. --- tests/test_dynamodb2/test_dynamodb.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 773ff61e1..831538054 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -394,9 +394,12 @@ def test_put_item_with_streams(): "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, } ) - stream_shard = dynamodb_backends2["us-west-2"].get_table(name).stream_shard - len(stream_shard.items).should.be.equal(1) - stream_record = stream_shard.items[0].record + table = dynamodb_backends2["us-west-2"].get_table(name) + if not table: + # There is no way to access stream data over the API, so this part can't run in server-tests mode. + return + len(table.stream_shard.items).should.be.equal(1) + stream_record = table.stream_shard.items[0].record stream_record["eventName"].should.be.equal("INSERT") stream_record["dynamodb"]["SizeBytes"].should.be.equal(447) From a820aada42e21a880d88debc81604a7cd6a56695 Mon Sep 17 00:00:00 2001 From: Jovan Zivanov Date: Thu, 26 Dec 2019 14:23:53 +0100 Subject: [PATCH 16/48] add codecommit create, get and delete repository --- IMPLEMENTATION_COVERAGE.md | 6 +- moto/__init__.py | 1 + moto/codecommit/__init__.py | 4 + moto/codecommit/exceptions.py | 32 +++++ moto/codecommit/models.py | 63 +++++++++ moto/codecommit/responses.py | 46 ++++++ moto/codecommit/urls.py | 6 + tests/test_codecommit/test_codecommit.py | 170 +++++++++++++++++++++++ 8 files changed, 325 insertions(+), 3 deletions(-) create mode 100644 moto/codecommit/__init__.py create mode 100644 moto/codecommit/exceptions.py create mode 100644 moto/codecommit/models.py create mode 100644 moto/codecommit/responses.py create mode 100644 moto/codecommit/urls.py create mode 100644 tests/test_codecommit/test_codecommit.py diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 5e6ef1c9e..245cf4705 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -1240,14 +1240,14 @@ - [ ] create_commit - [ ] create_pull_request - [ ] create_pull_request_approval_rule -- [ ] create_repository +- [X] create_repository - [ ] create_unreferenced_merge_commit - [ ] delete_approval_rule_template - [ ] delete_branch - [ ] delete_comment_content - [ ] delete_file - [ ] delete_pull_request_approval_rule -- [ ] delete_repository +- [X] delete_repository - [ ] describe_merge_conflicts - [ ] describe_pull_request_events - [ ] disassociate_approval_rule_template_from_repository @@ -1268,7 +1268,7 @@ - [ ] get_pull_request - [ ] get_pull_request_approval_states - [ ] get_pull_request_override_state -- [ ] get_repository +- [X] get_repository - [ ] get_repository_triggers - [ ] list_approval_rule_templates - [ ] list_associated_approval_rule_templates_for_repository diff --git a/moto/__init__.py b/moto/__init__.py index a9f1bb8ba..db79c59f6 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -9,6 +9,7 @@ from .batch import mock_batch # noqa from .cloudformation import mock_cloudformation # noqa from .cloudformation import mock_cloudformation_deprecated # noqa from .cloudwatch import mock_cloudwatch, mock_cloudwatch_deprecated # noqa +from .codecommit import mock_codecommit from .codepipeline import mock_codepipeline # noqa from .cognitoidentity import mock_cognitoidentity # noqa from .cognitoidentity import mock_cognitoidentity_deprecated # noqa diff --git a/moto/codecommit/__init__.py b/moto/codecommit/__init__.py new file mode 100644 index 000000000..c1da043ee --- /dev/null +++ b/moto/codecommit/__init__.py @@ -0,0 +1,4 @@ +from .models import codecommit_backends +from ..core.models import base_decorator + +mock_codecommit = base_decorator(codecommit_backends) \ No newline at end of file diff --git a/moto/codecommit/exceptions.py b/moto/codecommit/exceptions.py new file mode 100644 index 000000000..2cb3f7fae --- /dev/null +++ b/moto/codecommit/exceptions.py @@ -0,0 +1,32 @@ +from moto.core.exceptions import JsonRESTError + + +class RepositoryNameExistsException(JsonRESTError): + code = 400 + + def __init__(self, repository_name): + super(RepositoryNameExistsException, self).__init__( + "RepositoryNameExistsException", "Repository named {0} already exists".format(repository_name) + ) + + +class RepositoryDoesNotExistException(JsonRESTError): + code = 400 + + def __init__(self, repository_name): + super(RepositoryDoesNotExistException, self).__init__( + "RepositoryDoesNotExistException", "{0} does not exist".format(repository_name) + ) + + +class InvalidRepositoryNameException(JsonRESTError): + code = 400 + + def __init__(self): + super(InvalidRepositoryNameException, self).__init__( + "InvalidRepositoryNameException", "The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. " + ) diff --git a/moto/codecommit/models.py b/moto/codecommit/models.py new file mode 100644 index 000000000..e691507b6 --- /dev/null +++ b/moto/codecommit/models.py @@ -0,0 +1,63 @@ +from boto3 import Session +from moto.core import BaseBackend, BaseModel +from moto.core.utils import iso_8601_datetime_with_milliseconds +from datetime import datetime +from moto.iam.models import ACCOUNT_ID +from .exceptions import RepositoryDoesNotExistException, RepositoryNameExistsException +import uuid + + +class CodeCommit(BaseModel): + def __init__(self, region, repository_description, repository_name): + current_date = iso_8601_datetime_with_milliseconds(datetime.utcnow()) + self.repository_metadata = dict() + self.repository_metadata["repositoryName"] = repository_name + self.repository_metadata["cloneUrlSsh"] = "ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + region, repository_name + ) + self.repository_metadata["cloneUrlHttp"] = "https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + region, repository_name + ) + self.repository_metadata["creationDate"] = current_date + self.repository_metadata["lastModifiedDate"] = current_date + self.repository_metadata["repositoryDescription"] = repository_description + self.repository_metadata["repositoryId"] = str(uuid.uuid4()) + self.repository_metadata["Arn"] = "arn:aws:codecommit:{0}:{1}:{2}".format( + region, ACCOUNT_ID, repository_name + ) + self.repository_metadata["accountId"] = ACCOUNT_ID + + +class CodeCommitBackend(BaseBackend): + def __init__(self): + self.repositories = {} + + def create_repository(self, region, repository_name, repository_description): + repository = self.repositories.get(repository_name) + if repository: + raise RepositoryNameExistsException(repository_name) + + self.repositories[repository_name] = CodeCommit(region, repository_description, repository_name) + + return self.repositories[repository_name].repository_metadata + + def get_repository(self, repository_name): + repository = self.repositories.get(repository_name) + if not repository: + raise RepositoryDoesNotExistException(repository_name) + + return repository.repository_metadata + + def delete_repository(self, repository_name): + repository = self.repositories.get(repository_name) + + if repository: + self.repositories.pop(repository_name) + return repository.repository_metadata.get("repositoryId") + + return None + + +codecommit_backends = {} +for region in Session().get_available_regions("codecommit"): + codecommit_backends[region] = CodeCommitBackend() diff --git a/moto/codecommit/responses.py b/moto/codecommit/responses.py new file mode 100644 index 000000000..1ee177f37 --- /dev/null +++ b/moto/codecommit/responses.py @@ -0,0 +1,46 @@ +import json +import re + +from moto.core.responses import BaseResponse +from .models import codecommit_backends +from .exceptions import InvalidRepositoryNameException + + +class CodeCommitResponse(BaseResponse): + @property + def codecommit_backend(self): + return codecommit_backends[self.region] + + def create_repository(self): + if not self._is_repository_name_valid(self._get_param("repositoryName")): + raise InvalidRepositoryNameException() + + repository_metadata = self.codecommit_backend.create_repository( + self.region, self._get_param("repositoryName"), self._get_param("repositoryDescription") + ) + + return json.dumps({"repositoryMetadata": repository_metadata}) + + def get_repository(self): + if not self._is_repository_name_valid(self._get_param("repositoryName")): + raise InvalidRepositoryNameException() + + repository_metadata = self.codecommit_backend.get_repository(self._get_param("repositoryName")) + + return json.dumps({"repositoryMetadata": repository_metadata}) + + def delete_repository(self): + if not self._is_repository_name_valid(self._get_param("repositoryName")): + raise InvalidRepositoryNameException() + + repository_id = self.codecommit_backend.delete_repository(self._get_param("repositoryName")) + + if repository_id: + return json.dumps({"repositoryId": repository_id}) + + return json.dumps({}) + + def _is_repository_name_valid(self, repository_name): + name_regex = re.compile(r"[\w\.-]+") + result = name_regex.fullmatch(repository_name) + return result diff --git a/moto/codecommit/urls.py b/moto/codecommit/urls.py new file mode 100644 index 000000000..1e3cdb1b4 --- /dev/null +++ b/moto/codecommit/urls.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .responses import CodeCommitResponse + +url_bases = ["https?://codecommit.(.+).amazonaws.com"] + +url_paths = {"{0}/$": CodeCommitResponse.dispatch} diff --git a/tests/test_codecommit/test_codecommit.py b/tests/test_codecommit/test_codecommit.py new file mode 100644 index 000000000..c62c0127f --- /dev/null +++ b/tests/test_codecommit/test_codecommit.py @@ -0,0 +1,170 @@ +import boto3 + +import sure # noqa +from moto import mock_codecommit +from moto.iam.models import ACCOUNT_ID +from botocore.exceptions import ClientError +from nose.tools import assert_raises + + +@mock_codecommit +def test_create_repository(): + client = boto3.client("codecommit", region_name="eu-central-1") + response = client.create_repository( + repositoryName='repository_one', + repositoryDescription='description repo one' + ) + + response.should_not.be.none + response["repositoryMetadata"].should_not.be.none + response["repositoryMetadata"]["creationDate"].should_not.be.none + response["repositoryMetadata"]["lastModifiedDate"].should_not.be.none + response["repositoryMetadata"]["repositoryId"].should_not.be.empty + response["repositoryMetadata"]["repositoryName"].should.equal("repository_one") + response["repositoryMetadata"]["repositoryDescription"].should.equal('description repo one') + response["repositoryMetadata"]["cloneUrlSsh"].should.equal("ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}". + format("eu-central-1", 'repository_one')) + response["repositoryMetadata"]["cloneUrlHttp"].should.equal("https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}". + format("eu-central-1", 'repository_one')) + response["repositoryMetadata"]["Arn"].should.equal("arn:aws:codecommit:{0}:{1}:{2}".format( + "eu-central-1", ACCOUNT_ID, 'repository_one' + )) + response["repositoryMetadata"]["accountId"].should.equal(ACCOUNT_ID) + + response = client.create_repository( + repositoryName='repository_two' + ) + + response.should_not.be.none + response.get("repositoryMetadata").should_not.be.none + response.get("repositoryMetadata").get("repositoryName").should.equal("repository_two") + response.get("repositoryMetadata").get("repositoryDescription").should.be.none + + with assert_raises(ClientError) as e: + client.create_repository( + repositoryName='repository_two' + ) + ex = e.exception + ex.operation_name.should.equal("CreateRepository") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("RepositoryNameExistsException") + ex.response["Error"]["Message"].should.equal("Repository named {0} already exists".format("repository_two")) + + +@mock_codecommit +def test_get_repository(): + client = boto3.client("codecommit", region_name="eu-central-1") + + repository_name = 'repository_one' + + client.create_repository( + repositoryName=repository_name, + repositoryDescription='description repo one' + ) + + response = client.get_repository( + repositoryName=repository_name + ) + + response.should_not.be.none + response.get("repositoryMetadata").should_not.be.none + response.get("repositoryMetadata").get("creationDate").should_not.be.none + response.get("repositoryMetadata").get("lastModifiedDate").should_not.be.none + response.get("repositoryMetadata").get("repositoryId").should_not.be.empty + response.get("repositoryMetadata").get("repositoryName").should.equal(repository_name) + response.get("repositoryMetadata").get("repositoryDescription").should.equal('description repo one') + response.get("repositoryMetadata").get("cloneUrlSsh") \ + .should.equal("ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format("eu-central-1", 'repository_one')) + response.get("repositoryMetadata").get("cloneUrlHttp") \ + .should.equal("https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format("eu-central-1", 'repository_one')) + response.get("repositoryMetadata").get("Arn") \ + .should.equal("arn:aws:codecommit:{0}:{1}:{2}".format("eu-central-1", ACCOUNT_ID, 'repository_one' + )) + response.get("repositoryMetadata").get("accountId").should.equal(ACCOUNT_ID) + + client = boto3.client("codecommit", region_name="us-east-1") + + with assert_raises(ClientError) as e: + client.get_repository( + repositoryName=repository_name + ) + ex = e.exception + ex.operation_name.should.equal("GetRepository") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("RepositoryDoesNotExistException") + ex.response["Error"]["Message"].should.equal("{0} does not exist".format(repository_name)) + + +@mock_codecommit +def test_invalid_repository_name(): + client = boto3.client("codecommit", region_name="eu-central-1") + + with assert_raises(ClientError) as e: + client.create_repository( + repositoryName='repository_one-@#@' + ) + ex = e.exception + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidRepositoryNameException") + ex.response["Error"]["Message"].should.equal("The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. ") + with assert_raises(ClientError) as e: + client.create_repository( + repositoryName='!_repository_one' + ) + + with assert_raises(ClientError) as e: + client.create_repository( + repositoryName='_rep@ository_one' + ) + + with assert_raises(ClientError) as e: + client.get_repository( + repositoryName='_rep@ository_one' + ) + + +@mock_codecommit +def test_delete_repository(): + client = boto3.client("codecommit", region_name="us-east-1") + + response = client.create_repository( + repositoryName='repository_one' + ) + + repository_id_create = response.get("repositoryMetadata").get("repositoryId") + + response = client.delete_repository( + repositoryName='repository_one' + ) + + response.get('repositoryId').should_not.be.none + repository_id_create.should.equal(response.get("repositoryId")) + + response = client.delete_repository( + repositoryName='unknown_repository' + ) + + response.get('repositoryId').should.be.none + + +@mock_codecommit +def test_delete_repository_invalid_repository_name(): + client = boto3.client("codecommit", region_name="us-east-1") + + with assert_raises(ClientError) as e: + client.delete_repository( + repositoryName='_rep@ository_one' + ) + ex = e.exception + ex.operation_name.should.equal("DeleteRepository") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidRepositoryNameException") + ex.response["Error"]["Message"].should.equal("The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. ") \ No newline at end of file From e20deb4acdacc0b590ce7f67507a932483904f99 Mon Sep 17 00:00:00 2001 From: Jovan Zivanov Date: Thu, 26 Dec 2019 15:02:24 +0100 Subject: [PATCH 17/48] fix linter exceptions --- moto/__init__.py | 2 +- moto/codecommit/__init__.py | 2 +- moto/codecommit/exceptions.py | 17 +-- moto/codecommit/models.py | 12 +- moto/codecommit/responses.py | 12 +- tests/test_codecommit/test_codecommit.py | 152 ++++++++++++----------- 6 files changed, 109 insertions(+), 88 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index db79c59f6..835e37e2e 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -9,7 +9,7 @@ from .batch import mock_batch # noqa from .cloudformation import mock_cloudformation # noqa from .cloudformation import mock_cloudformation_deprecated # noqa from .cloudwatch import mock_cloudwatch, mock_cloudwatch_deprecated # noqa -from .codecommit import mock_codecommit +from .codecommit import mock_codecommit # noqa from .codepipeline import mock_codepipeline # noqa from .cognitoidentity import mock_cognitoidentity # noqa from .cognitoidentity import mock_cognitoidentity_deprecated # noqa diff --git a/moto/codecommit/__init__.py b/moto/codecommit/__init__.py index c1da043ee..6c5a8f5ad 100644 --- a/moto/codecommit/__init__.py +++ b/moto/codecommit/__init__.py @@ -1,4 +1,4 @@ from .models import codecommit_backends from ..core.models import base_decorator -mock_codecommit = base_decorator(codecommit_backends) \ No newline at end of file +mock_codecommit = base_decorator(codecommit_backends) diff --git a/moto/codecommit/exceptions.py b/moto/codecommit/exceptions.py index 2cb3f7fae..136af50f1 100644 --- a/moto/codecommit/exceptions.py +++ b/moto/codecommit/exceptions.py @@ -6,7 +6,8 @@ class RepositoryNameExistsException(JsonRESTError): def __init__(self, repository_name): super(RepositoryNameExistsException, self).__init__( - "RepositoryNameExistsException", "Repository named {0} already exists".format(repository_name) + "RepositoryNameExistsException", + "Repository named {0} already exists".format(repository_name), ) @@ -15,7 +16,8 @@ class RepositoryDoesNotExistException(JsonRESTError): def __init__(self, repository_name): super(RepositoryDoesNotExistException, self).__init__( - "RepositoryDoesNotExistException", "{0} does not exist".format(repository_name) + "RepositoryDoesNotExistException", + "{0} does not exist".format(repository_name), ) @@ -24,9 +26,10 @@ class InvalidRepositoryNameException(JsonRESTError): def __init__(self): super(InvalidRepositoryNameException, self).__init__( - "InvalidRepositoryNameException", "The repository name is not valid. Repository names can be any valid " - "combination of letters, numbers, " - "periods, underscores, and dashes between 1 and 100 characters in " - "length. Names are case sensitive. " - "For more information, see Limits in the AWS CodeCommit User Guide. " + "InvalidRepositoryNameException", + "The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. ", ) diff --git a/moto/codecommit/models.py b/moto/codecommit/models.py index e691507b6..6a4e82ad2 100644 --- a/moto/codecommit/models.py +++ b/moto/codecommit/models.py @@ -12,10 +12,14 @@ class CodeCommit(BaseModel): current_date = iso_8601_datetime_with_milliseconds(datetime.utcnow()) self.repository_metadata = dict() self.repository_metadata["repositoryName"] = repository_name - self.repository_metadata["cloneUrlSsh"] = "ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + self.repository_metadata[ + "cloneUrlSsh" + ] = "ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( region, repository_name ) - self.repository_metadata["cloneUrlHttp"] = "https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + self.repository_metadata[ + "cloneUrlHttp" + ] = "https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( region, repository_name ) self.repository_metadata["creationDate"] = current_date @@ -37,7 +41,9 @@ class CodeCommitBackend(BaseBackend): if repository: raise RepositoryNameExistsException(repository_name) - self.repositories[repository_name] = CodeCommit(region, repository_description, repository_name) + self.repositories[repository_name] = CodeCommit( + region, repository_description, repository_name + ) return self.repositories[repository_name].repository_metadata diff --git a/moto/codecommit/responses.py b/moto/codecommit/responses.py index 1ee177f37..3e2cf2539 100644 --- a/moto/codecommit/responses.py +++ b/moto/codecommit/responses.py @@ -16,7 +16,9 @@ class CodeCommitResponse(BaseResponse): raise InvalidRepositoryNameException() repository_metadata = self.codecommit_backend.create_repository( - self.region, self._get_param("repositoryName"), self._get_param("repositoryDescription") + self.region, + self._get_param("repositoryName"), + self._get_param("repositoryDescription"), ) return json.dumps({"repositoryMetadata": repository_metadata}) @@ -25,7 +27,9 @@ class CodeCommitResponse(BaseResponse): if not self._is_repository_name_valid(self._get_param("repositoryName")): raise InvalidRepositoryNameException() - repository_metadata = self.codecommit_backend.get_repository(self._get_param("repositoryName")) + repository_metadata = self.codecommit_backend.get_repository( + self._get_param("repositoryName") + ) return json.dumps({"repositoryMetadata": repository_metadata}) @@ -33,7 +37,9 @@ class CodeCommitResponse(BaseResponse): if not self._is_repository_name_valid(self._get_param("repositoryName")): raise InvalidRepositoryNameException() - repository_id = self.codecommit_backend.delete_repository(self._get_param("repositoryName")) + repository_id = self.codecommit_backend.delete_repository( + self._get_param("repositoryName") + ) if repository_id: return json.dumps({"repositoryId": repository_id}) diff --git a/tests/test_codecommit/test_codecommit.py b/tests/test_codecommit/test_codecommit.py index c62c0127f..8be10033a 100644 --- a/tests/test_codecommit/test_codecommit.py +++ b/tests/test_codecommit/test_codecommit.py @@ -11,8 +11,7 @@ from nose.tools import assert_raises def test_create_repository(): client = boto3.client("codecommit", region_name="eu-central-1") response = client.create_repository( - repositoryName='repository_one', - repositoryDescription='description repo one' + repositoryName="repository_one", repositoryDescription="description repo one" ) response.should_not.be.none @@ -21,78 +20,97 @@ def test_create_repository(): response["repositoryMetadata"]["lastModifiedDate"].should_not.be.none response["repositoryMetadata"]["repositoryId"].should_not.be.empty response["repositoryMetadata"]["repositoryName"].should.equal("repository_one") - response["repositoryMetadata"]["repositoryDescription"].should.equal('description repo one') - response["repositoryMetadata"]["cloneUrlSsh"].should.equal("ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}". - format("eu-central-1", 'repository_one')) - response["repositoryMetadata"]["cloneUrlHttp"].should.equal("https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}". - format("eu-central-1", 'repository_one')) - response["repositoryMetadata"]["Arn"].should.equal("arn:aws:codecommit:{0}:{1}:{2}".format( - "eu-central-1", ACCOUNT_ID, 'repository_one' - )) + response["repositoryMetadata"]["repositoryDescription"].should.equal( + "description repo one" + ) + response["repositoryMetadata"]["cloneUrlSsh"].should.equal( + "ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + "eu-central-1", "repository_one" + ) + ) + response["repositoryMetadata"]["cloneUrlHttp"].should.equal( + "https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + "eu-central-1", "repository_one" + ) + ) + response["repositoryMetadata"]["Arn"].should.equal( + "arn:aws:codecommit:{0}:{1}:{2}".format( + "eu-central-1", ACCOUNT_ID, "repository_one" + ) + ) response["repositoryMetadata"]["accountId"].should.equal(ACCOUNT_ID) - response = client.create_repository( - repositoryName='repository_two' - ) + response = client.create_repository(repositoryName="repository_two") response.should_not.be.none response.get("repositoryMetadata").should_not.be.none - response.get("repositoryMetadata").get("repositoryName").should.equal("repository_two") + response.get("repositoryMetadata").get("repositoryName").should.equal( + "repository_two" + ) response.get("repositoryMetadata").get("repositoryDescription").should.be.none with assert_raises(ClientError) as e: - client.create_repository( - repositoryName='repository_two' - ) + client.create_repository(repositoryName="repository_two") ex = e.exception ex.operation_name.should.equal("CreateRepository") ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.response["Error"]["Code"].should.contain("RepositoryNameExistsException") - ex.response["Error"]["Message"].should.equal("Repository named {0} already exists".format("repository_two")) + ex.response["Error"]["Message"].should.equal( + "Repository named {0} already exists".format("repository_two") + ) @mock_codecommit def test_get_repository(): client = boto3.client("codecommit", region_name="eu-central-1") - repository_name = 'repository_one' + repository_name = "repository_one" client.create_repository( - repositoryName=repository_name, - repositoryDescription='description repo one' + repositoryName=repository_name, repositoryDescription="description repo one" ) - response = client.get_repository( - repositoryName=repository_name - ) + response = client.get_repository(repositoryName=repository_name) response.should_not.be.none response.get("repositoryMetadata").should_not.be.none response.get("repositoryMetadata").get("creationDate").should_not.be.none response.get("repositoryMetadata").get("lastModifiedDate").should_not.be.none response.get("repositoryMetadata").get("repositoryId").should_not.be.empty - response.get("repositoryMetadata").get("repositoryName").should.equal(repository_name) - response.get("repositoryMetadata").get("repositoryDescription").should.equal('description repo one') - response.get("repositoryMetadata").get("cloneUrlSsh") \ - .should.equal("ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format("eu-central-1", 'repository_one')) - response.get("repositoryMetadata").get("cloneUrlHttp") \ - .should.equal("https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format("eu-central-1", 'repository_one')) - response.get("repositoryMetadata").get("Arn") \ - .should.equal("arn:aws:codecommit:{0}:{1}:{2}".format("eu-central-1", ACCOUNT_ID, 'repository_one' - )) + response.get("repositoryMetadata").get("repositoryName").should.equal( + repository_name + ) + response.get("repositoryMetadata").get("repositoryDescription").should.equal( + "description repo one" + ) + response.get("repositoryMetadata").get("cloneUrlSsh").should.equal( + "ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + "eu-central-1", "repository_one" + ) + ) + response.get("repositoryMetadata").get("cloneUrlHttp").should.equal( + "https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + "eu-central-1", "repository_one" + ) + ) + response.get("repositoryMetadata").get("Arn").should.equal( + "arn:aws:codecommit:{0}:{1}:{2}".format( + "eu-central-1", ACCOUNT_ID, "repository_one" + ) + ) response.get("repositoryMetadata").get("accountId").should.equal(ACCOUNT_ID) client = boto3.client("codecommit", region_name="us-east-1") with assert_raises(ClientError) as e: - client.get_repository( - repositoryName=repository_name - ) + client.get_repository(repositoryName=repository_name) ex = e.exception ex.operation_name.should.equal("GetRepository") ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.response["Error"]["Code"].should.contain("RepositoryDoesNotExistException") - ex.response["Error"]["Message"].should.equal("{0} does not exist".format(repository_name)) + ex.response["Error"]["Message"].should.equal( + "{0} does not exist".format(repository_name) + ) @mock_codecommit @@ -100,55 +118,43 @@ def test_invalid_repository_name(): client = boto3.client("codecommit", region_name="eu-central-1") with assert_raises(ClientError) as e: - client.create_repository( - repositoryName='repository_one-@#@' - ) + client.create_repository(repositoryName="repository_one-@#@") ex = e.exception ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.response["Error"]["Code"].should.contain("InvalidRepositoryNameException") - ex.response["Error"]["Message"].should.equal("The repository name is not valid. Repository names can be any valid " - "combination of letters, numbers, " - "periods, underscores, and dashes between 1 and 100 characters in " - "length. Names are case sensitive. " - "For more information, see Limits in the AWS CodeCommit User Guide. ") + ex.response["Error"]["Message"].should.equal( + "The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. " + ) with assert_raises(ClientError) as e: - client.create_repository( - repositoryName='!_repository_one' - ) + client.create_repository(repositoryName="!_repository_one") with assert_raises(ClientError) as e: - client.create_repository( - repositoryName='_rep@ository_one' - ) + client.create_repository(repositoryName="_rep@ository_one") with assert_raises(ClientError) as e: - client.get_repository( - repositoryName='_rep@ository_one' - ) + client.get_repository(repositoryName="_rep@ository_one") @mock_codecommit def test_delete_repository(): client = boto3.client("codecommit", region_name="us-east-1") - response = client.create_repository( - repositoryName='repository_one' - ) + response = client.create_repository(repositoryName="repository_one") repository_id_create = response.get("repositoryMetadata").get("repositoryId") - response = client.delete_repository( - repositoryName='repository_one' - ) + response = client.delete_repository(repositoryName="repository_one") - response.get('repositoryId').should_not.be.none + response.get("repositoryId").should_not.be.none repository_id_create.should.equal(response.get("repositoryId")) - response = client.delete_repository( - repositoryName='unknown_repository' - ) + response = client.delete_repository(repositoryName="unknown_repository") - response.get('repositoryId').should.be.none + response.get("repositoryId").should.be.none @mock_codecommit @@ -156,15 +162,15 @@ def test_delete_repository_invalid_repository_name(): client = boto3.client("codecommit", region_name="us-east-1") with assert_raises(ClientError) as e: - client.delete_repository( - repositoryName='_rep@ository_one' - ) + client.delete_repository(repositoryName="_rep@ository_one") ex = e.exception ex.operation_name.should.equal("DeleteRepository") ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.response["Error"]["Code"].should.contain("InvalidRepositoryNameException") - ex.response["Error"]["Message"].should.equal("The repository name is not valid. Repository names can be any valid " - "combination of letters, numbers, " - "periods, underscores, and dashes between 1 and 100 characters in " - "length. Names are case sensitive. " - "For more information, see Limits in the AWS CodeCommit User Guide. ") \ No newline at end of file + ex.response["Error"]["Message"].should.equal( + "The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. " + ) From cb1eb79b894f25c47c19da841599583da5beca8e Mon Sep 17 00:00:00 2001 From: Jovan Zivanov Date: Thu, 26 Dec 2019 16:06:53 +0100 Subject: [PATCH 18/48] add tests for codecommit --- tests/test_codecommit/test_codecommit.py | 68 ++++++++++++++++++++---- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/tests/test_codecommit/test_codecommit.py b/tests/test_codecommit/test_codecommit.py index 8be10033a..6e916f20a 100644 --- a/tests/test_codecommit/test_codecommit.py +++ b/tests/test_codecommit/test_codecommit.py @@ -40,6 +40,11 @@ def test_create_repository(): ) response["repositoryMetadata"]["accountId"].should.equal(ACCOUNT_ID) + +@mock_codecommit +def test_create_repository_without_description(): + client = boto3.client("codecommit", region_name="eu-central-1") + response = client.create_repository(repositoryName="repository_two") response.should_not.be.none @@ -48,9 +53,39 @@ def test_create_repository(): "repository_two" ) response.get("repositoryMetadata").get("repositoryDescription").should.be.none + response["repositoryMetadata"].should_not.be.none + response["repositoryMetadata"]["creationDate"].should_not.be.none + response["repositoryMetadata"]["lastModifiedDate"].should_not.be.none + response["repositoryMetadata"]["repositoryId"].should_not.be.empty + response["repositoryMetadata"]["cloneUrlSsh"].should.equal( + "ssh://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + "eu-central-1", "repository_two" + ) + ) + response["repositoryMetadata"]["cloneUrlHttp"].should.equal( + "https://git-codecommit.{0}.amazonaws.com/v1/repos/{1}".format( + "eu-central-1", "repository_two" + ) + ) + response["repositoryMetadata"]["Arn"].should.equal( + "arn:aws:codecommit:{0}:{1}:{2}".format( + "eu-central-1", ACCOUNT_ID, "repository_two" + ) + ) + response["repositoryMetadata"]["accountId"].should.equal(ACCOUNT_ID) + + +@mock_codecommit +def test_create_repository_repository_name_exists(): + client = boto3.client("codecommit", region_name="eu-central-1") + + client.create_repository(repositoryName="repository_two") with assert_raises(ClientError) as e: - client.create_repository(repositoryName="repository_two") + client.create_repository( + repositoryName="repository_two", + repositoryDescription="description repo two", + ) ex = e.exception ex.operation_name.should.equal("CreateRepository") ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) @@ -60,6 +95,25 @@ def test_create_repository(): ) +@mock_codecommit +def test_create_repository_invalid_repository_name(): + client = boto3.client("codecommit", region_name="eu-central-1") + + with assert_raises(ClientError) as e: + client.create_repository(repositoryName="in_123_valid_@#$_characters") + ex = e.exception + ex.operation_name.should.equal("CreateRepository") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidRepositoryNameException") + ex.response["Error"]["Message"].should.equal( + "The repository name is not valid. Repository names can be any valid " + "combination of letters, numbers, " + "periods, underscores, and dashes between 1 and 100 characters in " + "length. Names are case sensitive. " + "For more information, see Limits in the AWS CodeCommit User Guide. " + ) + + @mock_codecommit def test_get_repository(): client = boto3.client("codecommit", region_name="eu-central-1") @@ -114,11 +168,11 @@ def test_get_repository(): @mock_codecommit -def test_invalid_repository_name(): +def test_get_repository_invalid_repository_name(): client = boto3.client("codecommit", region_name="eu-central-1") with assert_raises(ClientError) as e: - client.create_repository(repositoryName="repository_one-@#@") + client.get_repository(repositoryName="repository_one-@#@") ex = e.exception ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.response["Error"]["Code"].should.contain("InvalidRepositoryNameException") @@ -129,14 +183,6 @@ def test_invalid_repository_name(): "length. Names are case sensitive. " "For more information, see Limits in the AWS CodeCommit User Guide. " ) - with assert_raises(ClientError) as e: - client.create_repository(repositoryName="!_repository_one") - - with assert_raises(ClientError) as e: - client.create_repository(repositoryName="_rep@ository_one") - - with assert_raises(ClientError) as e: - client.get_repository(repositoryName="_rep@ository_one") @mock_codecommit From cba1b2e18026d1f45ba5e6536d2313c5be5c4201 Mon Sep 17 00:00:00 2001 From: Jovan Zivanov Date: Thu, 26 Dec 2019 16:30:06 +0100 Subject: [PATCH 19/48] [codecommit] fix repository name check --- moto/codecommit/responses.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/moto/codecommit/responses.py b/moto/codecommit/responses.py index 3e2cf2539..3c6fdc5ea 100644 --- a/moto/codecommit/responses.py +++ b/moto/codecommit/responses.py @@ -6,13 +6,23 @@ from .models import codecommit_backends from .exceptions import InvalidRepositoryNameException +def _is_repository_name_valid(repository_name): + name_regex = re.compile(r"[\w\.-]+") + result = name_regex.split(repository_name) + if len(result) > 0: + for match in result: + if len(match) > 0: + return False + return True + + class CodeCommitResponse(BaseResponse): @property def codecommit_backend(self): return codecommit_backends[self.region] def create_repository(self): - if not self._is_repository_name_valid(self._get_param("repositoryName")): + if not _is_repository_name_valid(self._get_param("repositoryName")): raise InvalidRepositoryNameException() repository_metadata = self.codecommit_backend.create_repository( @@ -24,7 +34,7 @@ class CodeCommitResponse(BaseResponse): return json.dumps({"repositoryMetadata": repository_metadata}) def get_repository(self): - if not self._is_repository_name_valid(self._get_param("repositoryName")): + if not _is_repository_name_valid(self._get_param("repositoryName")): raise InvalidRepositoryNameException() repository_metadata = self.codecommit_backend.get_repository( @@ -34,7 +44,7 @@ class CodeCommitResponse(BaseResponse): return json.dumps({"repositoryMetadata": repository_metadata}) def delete_repository(self): - if not self._is_repository_name_valid(self._get_param("repositoryName")): + if not _is_repository_name_valid(self._get_param("repositoryName")): raise InvalidRepositoryNameException() repository_id = self.codecommit_backend.delete_repository( @@ -45,8 +55,3 @@ class CodeCommitResponse(BaseResponse): return json.dumps({"repositoryId": repository_id}) return json.dumps({}) - - def _is_repository_name_valid(self, repository_name): - name_regex = re.compile(r"[\w\.-]+") - result = name_regex.fullmatch(repository_name) - return result From d7ba355a65f17664c56a7fedd7423a16605f2469 Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 26 Dec 2019 17:12:22 +0100 Subject: [PATCH 20/48] Add missing regions to all services --- moto/apigateway/models.py | 8 ++++ moto/athena/models.py | 9 +++- moto/awslambda/models.py | 16 ++++--- moto/batch/models.py | 14 ++++-- moto/cloudformation/models.py | 14 +++++- moto/cloudwatch/models.py | 13 ++++- moto/codepipeline/models.py | 6 +++ moto/cognitoidentity/models.py | 13 ++++- moto/cognitoidp/models.py | 11 ++++- moto/config/models.py | 7 ++- moto/datapipeline/models.py | 12 ++++- moto/datasync/models.py | 11 +++-- moto/dynamodb2/models.py | 13 +++-- moto/dynamodbstreams/models.py | 18 +++++-- moto/ec2/models.py | 48 ++++++++++++++++++- moto/ecs/models.py | 14 ++++-- moto/emr/models.py | 9 +++- moto/events/models.py | 11 +++-- moto/glacier/models.py | 10 +++- moto/iot/models.py | 11 +++-- moto/iotdata/models.py | 12 +++-- moto/kinesis/models.py | 10 +++- moto/kms/models.py | 9 +++- moto/logs/models.py | 12 +++-- moto/polly/models.py | 13 +++-- moto/rds2/models.py | 11 +++-- moto/redshift/models.py | 11 +++-- moto/resourcegroups/models.py | 18 +++++-- moto/resourcegroupstaggingapi/models.py | 20 +++++--- moto/secretsmanager/models.py | 16 +++++-- moto/sns/models.py | 4 ++ moto/sqs/models.py | 9 +++- moto/stepfunctions/models.py | 16 +++++-- moto/swf/models/__init__.py | 9 +++- scripts/template/lib/models.py.j2 | 11 +++-- .../test_availability_zones_and_regions.py | 4 +- tests/test_ec2/test_regions.py | 10 +++- tests/test_glacier/test_glacier_jobs.py | 2 +- 38 files changed, 358 insertions(+), 107 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 8b5fb787f..fd2fb7064 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -847,3 +847,11 @@ class APIGatewayBackend(BaseBackend): apigateway_backends = {} for region_name in Session().get_available_regions("apigateway"): apigateway_backends[region_name] = APIGatewayBackend(region_name) +for region_name in Session().get_available_regions( + "apigateway", partition_name="aws-us-gov" +): + apigateway_backends[region_name] = APIGatewayBackend(region_name) +for region_name in Session().get_available_regions( + "apigateway", partition_name="aws-cn" +): + apigateway_backends[region_name] = APIGatewayBackend(region_name) diff --git a/moto/athena/models.py b/moto/athena/models.py index 2f41046a9..6aeca0ffa 100644 --- a/moto/athena/models.py +++ b/moto/athena/models.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals import time -import boto3 +from boto3 import Session + from moto.core import BaseBackend, BaseModel from moto.core import ACCOUNT_ID @@ -77,5 +78,9 @@ class AthenaBackend(BaseBackend): athena_backends = {} -for region in boto3.Session().get_available_regions("athena"): +for region in Session().get_available_regions("athena"): + athena_backends[region] = AthenaBackend(region) +for region in Session().get_available_regions("athena", partition_name="aws-us-gov"): + athena_backends[region] = AthenaBackend(region) +for region in Session().get_available_regions("athena", partition_name="aws-cn"): athena_backends[region] = AthenaBackend(region) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index b1b8f57a8..5795ff5df 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -24,6 +24,8 @@ import weakref import requests.adapters import boto.awslambda +from boto3 import Session + from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError from moto.iam.models import iam_backend @@ -1043,10 +1045,10 @@ def do_validate_s3(): return os.environ.get("VALIDATE_LAMBDA_S3", "") in ["", "1", "true"] -# Handle us forgotten regions, unless Lambda truly only runs out of US and -lambda_backends = { - _region.name: LambdaBackend(_region.name) for _region in boto.awslambda.regions() -} - -lambda_backends["ap-southeast-2"] = LambdaBackend("ap-southeast-2") -lambda_backends["us-gov-west-1"] = LambdaBackend("us-gov-west-1") +lambda_backends = {} +for region in Session().get_available_regions("lambda"): + lambda_backends[region] = LambdaBackend(region) +for region in Session().get_available_regions("lambda", partition_name="aws-us-gov"): + lambda_backends[region] = LambdaBackend(region) +for region in Session().get_available_regions("lambda", partition_name="aws-cn"): + lambda_backends[region] = LambdaBackend(region) diff --git a/moto/batch/models.py b/moto/batch/models.py index e12cc8f84..fc35f2997 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -1,5 +1,4 @@ from __future__ import unicode_literals -import boto3 import re import requests.adapters from itertools import cycle @@ -12,6 +11,8 @@ import docker import functools import threading import dateutil.parser +from boto3 import Session + from moto.core import BaseBackend, BaseModel from moto.iam import iam_backends from moto.ec2 import ec2_backends @@ -1317,7 +1318,10 @@ class BatchBackend(BaseBackend): job.terminate(reason) -available_regions = boto3.session.Session().get_available_regions("batch") -batch_backends = { - region: BatchBackend(region_name=region) for region in available_regions -} +batch_backends = {} +for region in Session().get_available_regions("batch"): + batch_backends[region] = BatchBackend(region) +for region in Session().get_available_regions("batch", partition_name="aws-us-gov"): + batch_backends[region] = BatchBackend(region) +for region in Session().get_available_regions("batch", partition_name="aws-cn"): + batch_backends[region] = BatchBackend(region) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 71ceaf168..073d84ce8 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -5,6 +5,8 @@ import yaml import uuid import boto.cloudformation +from boto3 import Session + from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel @@ -717,5 +719,13 @@ class CloudFormationBackend(BaseBackend): cloudformation_backends = {} -for region in boto.cloudformation.regions(): - cloudformation_backends[region.name] = CloudFormationBackend() +for region in Session().get_available_regions("cloudformation"): + cloudformation_backends[region] = CloudFormationBackend() +for region in Session().get_available_regions( + "cloudformation", partition_name="aws-us-gov" +): + cloudformation_backends[region] = CloudFormationBackend() +for region in Session().get_available_regions( + "cloudformation", partition_name="aws-cn" +): + cloudformation_backends[region] = CloudFormationBackend() diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 662005237..18f5965a8 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -1,4 +1,7 @@ import json + +from boto3 import Session + from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError @@ -431,5 +434,11 @@ class LogGroup(BaseModel): cloudwatch_backends = {} -for region in boto.ec2.cloudwatch.regions(): - cloudwatch_backends[region.name] = CloudWatchBackend() +for region in Session().get_available_regions("cloudwatch"): + cloudwatch_backends[region] = CloudWatchBackend() +for region in Session().get_available_regions( + "cloudwatch", partition_name="aws-us-gov" +): + cloudwatch_backends[region] = CloudWatchBackend() +for region in Session().get_available_regions("cloudwatch", partition_name="aws-cn"): + cloudwatch_backends[region] = CloudWatchBackend() diff --git a/moto/codepipeline/models.py b/moto/codepipeline/models.py index b3e76f838..b67d48989 100644 --- a/moto/codepipeline/models.py +++ b/moto/codepipeline/models.py @@ -149,3 +149,9 @@ class CodePipelineBackend(BaseBackend): codepipeline_backends = {} for region in Session().get_available_regions("codepipeline"): codepipeline_backends[region] = CodePipelineBackend() +for region in Session().get_available_regions( + "codepipeline", partition_name="aws-us-gov" +): + codepipeline_backends[region] = CodePipelineBackend() +for region in Session().get_available_regions("codepipeline", partition_name="aws-cn"): + codepipeline_backends[region] = CodePipelineBackend() diff --git a/moto/cognitoidentity/models.py b/moto/cognitoidentity/models.py index 2a4f5d4bc..20fd1241e 100644 --- a/moto/cognitoidentity/models.py +++ b/moto/cognitoidentity/models.py @@ -4,6 +4,7 @@ import datetime import json import boto.cognito.identity +from boto3 import Session from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel @@ -136,5 +137,13 @@ class CognitoIdentityBackend(BaseBackend): cognitoidentity_backends = {} -for region in boto.cognito.identity.regions(): - cognitoidentity_backends[region.name] = CognitoIdentityBackend(region.name) +for region in Session().get_available_regions("cognito-identity"): + cognitoidentity_backends[region] = CognitoIdentityBackend(region) +for region in Session().get_available_regions( + "cognito-identity", partition_name="aws-us-gov" +): + cognitoidentity_backends[region] = CognitoIdentityBackend(region) +for region in Session().get_available_regions( + "cognito-identity", partition_name="aws-cn" +): + cognitoidentity_backends[region] = CognitoIdentityBackend(region) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 6700920ce..70db9b7fa 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -10,6 +10,7 @@ import time import uuid import boto.cognito.identity +from boto3 import Session from jose import jws from moto.compat import OrderedDict @@ -749,8 +750,14 @@ class CognitoIdpBackend(BaseBackend): cognitoidp_backends = {} -for region in boto.cognito.identity.regions(): - cognitoidp_backends[region.name] = CognitoIdpBackend(region.name) +for region in Session().get_available_regions("cognito-idp"): + cognitoidp_backends[region] = CognitoIdpBackend(region) +for region in Session().get_available_regions( + "cognito-idp", partition_name="aws-us-gov" +): + cognitoidp_backends[region] = CognitoIdpBackend(region) +for region in Session().get_available_regions("cognito-idp", partition_name="aws-cn"): + cognitoidp_backends[region] = CognitoIdpBackend(region) # Hack to help moto-server process requests on localhost, where the region isn't diff --git a/moto/config/models.py b/moto/config/models.py index 9015762fe..45dccd1ba 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -1084,6 +1084,9 @@ class ConfigBackend(BaseBackend): config_backends = {} -boto3_session = Session() -for region in boto3_session.get_available_regions("config"): +for region in Session().get_available_regions("config"): + config_backends[region] = ConfigBackend() +for region in Session().get_available_regions("config", partition_name="aws-us-gov"): + config_backends[region] = ConfigBackend() +for region in Session().get_available_regions("config", partition_name="aws-cn"): config_backends[region] = ConfigBackend() diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index cc1fe777e..1964db008 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import datetime import boto.datapipeline +from boto3 import Session + from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys @@ -142,5 +144,11 @@ class DataPipelineBackend(BaseBackend): datapipeline_backends = {} -for region in boto.datapipeline.regions(): - datapipeline_backends[region.name] = DataPipelineBackend() +for region in Session().get_available_regions("datapipeline"): + datapipeline_backends[region] = DataPipelineBackend() +for region in Session().get_available_regions( + "datapipeline", partition_name="aws-us-gov" +): + datapipeline_backends[region] = DataPipelineBackend() +for region in Session().get_available_regions("datapipeline", partition_name="aws-cn"): + datapipeline_backends[region] = DataPipelineBackend(region) diff --git a/moto/datasync/models.py b/moto/datasync/models.py index 17a2659fb..702cace5b 100644 --- a/moto/datasync/models.py +++ b/moto/datasync/models.py @@ -1,4 +1,5 @@ -import boto3 +from boto3 import Session + from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel @@ -226,5 +227,9 @@ class DataSyncBackend(BaseBackend): datasync_backends = {} -for region in boto3.Session().get_available_regions("datasync"): - datasync_backends[region] = DataSyncBackend(region_name=region) +for region in Session().get_available_regions("datasync"): + datasync_backends[region] = DataSyncBackend(region) +for region in Session().get_available_regions("datasync", partition_name="aws-us-gov"): + datasync_backends[region] = DataSyncBackend(region) +for region in Session().get_available_regions("datasync", partition_name="aws-cn"): + datasync_backends[region] = DataSyncBackend(region) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 121f564a4..1fcde527c 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -8,7 +8,7 @@ import re import uuid import six -import boto3 +from boto3 import Session from botocore.exceptions import ParamValidationError from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel @@ -1484,7 +1484,10 @@ class DynamoDBBackend(BaseBackend): return table.ttl -available_regions = boto3.session.Session().get_available_regions("dynamodb") -dynamodb_backends = { - region: DynamoDBBackend(region_name=region) for region in available_regions -} +dynamodb_backends = {} +for region in Session().get_available_regions("dynamodb"): + dynamodb_backends[region] = DynamoDBBackend(region) +for region in Session().get_available_regions("dynamodb", partition_name="aws-us-gov"): + dynamodb_backends[region] = DynamoDBBackend(region) +for region in Session().get_available_regions("dynamodb", partition_name="aws-cn"): + dynamodb_backends[region] = DynamoDBBackend(region) diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index 6e99d8ef6..dc6f0e0d3 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -2,9 +2,10 @@ from __future__ import unicode_literals import os import json -import boto3 import base64 +from boto3 import Session + from moto.core import BaseBackend, BaseModel from moto.dynamodb2.models import dynamodb_backends @@ -139,7 +140,14 @@ class DynamoDBStreamsBackend(BaseBackend): return json.dumps(shard_iterator.get(limit)) -available_regions = boto3.session.Session().get_available_regions("dynamodbstreams") -dynamodbstreams_backends = { - region: DynamoDBStreamsBackend(region=region) for region in available_regions -} +dynamodbstreams_backends = {} +for region in Session().get_available_regions("dynamodbstreams"): + dynamodbstreams_backends[region] = DynamoDBStreamsBackend(region) +for region in Session().get_available_regions( + "dynamodbstreams", partition_name="aws-us-gov" +): + dynamodbstreams_backends[region] = DynamoDBStreamsBackend(region) +for region in Session().get_available_regions( + "dynamodbstreams", partition_name="aws-cn" +): + dynamodbstreams_backends[region] = DynamoDBStreamsBackend(region) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 374494faa..e492ae7c6 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -8,6 +8,8 @@ import os import re import six import warnings + +from boto3 import Session from pkg_resources import resource_filename import boto.ec2 @@ -1473,7 +1475,13 @@ class Zone(object): class RegionsAndZonesBackend(object): - regions = [Region(ri.name, ri.endpoint) for ri in boto.ec2.regions()] + regions = [] + for region in Session().get_available_regions("ec2"): + regions.append(Region(region, "ec2.{}.amazonaws.com".format(region))) + for region in Session().get_available_regions("ec2", partition_name="aws-us-gov"): + regions.append(Region(region, "ec2.{}.amazonaws.com".format(region))) + for region in Session().get_available_regions("ec2", partition_name="aws-cn"): + regions.append(Region(region, "ec2.{}.amazonaws.com.cn".format(region))) zones = { "ap-south-1": [ @@ -1536,6 +1544,11 @@ class RegionsAndZonesBackend(object): zone_id="apne1-az2", ), ], + "ap-east-1": [ + Zone(region_name="ap-east-1", name="ap-east-1a", zone_id="ape1-az1"), + Zone(region_name="ap-east-1", name="ap-east-1b", zone_id="ape1-az2"), + Zone(region_name="ap-east-1", name="ap-east-1c", zone_id="ape1-az3"), + ], "sa-east-1": [ Zone(region_name="sa-east-1", name="sa-east-1a", zone_id="sae1-az1"), Zone(region_name="sa-east-1", name="sa-east-1c", zone_id="sae1-az3"), @@ -1605,10 +1618,32 @@ class RegionsAndZonesBackend(object): Zone(region_name="us-west-2", name="us-west-2b", zone_id="usw2-az1"), Zone(region_name="us-west-2", name="us-west-2c", zone_id="usw2-az3"), ], + "me-south-1": [ + Zone(region_name="me-south-1", name="me-south-1a", zone_id="mes1-az1"), + Zone(region_name="me-south-1", name="me-south-1b", zone_id="mes1-az2"), + Zone(region_name="me-south-1", name="me-south-1c", zone_id="mes1-az3"), + ], "cn-north-1": [ Zone(region_name="cn-north-1", name="cn-north-1a", zone_id="cnn1-az1"), Zone(region_name="cn-north-1", name="cn-north-1b", zone_id="cnn1-az2"), ], + "cn-northwest-1": [ + Zone( + region_name="cn-northwest-1", + name="cn-northwest-1a", + zone_id="cnnw1-az1", + ), + Zone( + region_name="cn-northwest-1", + name="cn-northwest-1b", + zone_id="cnnw1-az2", + ), + Zone( + region_name="cn-northwest-1", + name="cn-northwest-1c", + zone_id="cnnw1-az3", + ), + ], "us-gov-west-1": [ Zone( region_name="us-gov-west-1", name="us-gov-west-1a", zone_id="usgw1-az1" @@ -1620,6 +1655,17 @@ class RegionsAndZonesBackend(object): region_name="us-gov-west-1", name="us-gov-west-1c", zone_id="usgw1-az3" ), ], + "us-gov-east-1": [ + Zone( + region_name="us-gov-east-1", name="us-gov-east-1a", zone_id="usge1-az1" + ), + Zone( + region_name="us-gov-east-1", name="us-gov-east-1b", zone_id="usge1-az2" + ), + Zone( + region_name="us-gov-east-1", name="us-gov-east-1c", zone_id="usge1-az3" + ), + ], } def describe_regions(self, region_names=[]): diff --git a/moto/ecs/models.py b/moto/ecs/models.py index c9dc998ee..845bdf650 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -3,9 +3,10 @@ import re import uuid from datetime import datetime from random import random, randint -import boto3 import pytz +from boto3 import Session + from moto.core.exceptions import JsonRESTError from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -1302,7 +1303,10 @@ class EC2ContainerServiceBackend(BaseBackend): raise NotImplementedError() -available_regions = boto3.session.Session().get_available_regions("ecs") -ecs_backends = { - region: EC2ContainerServiceBackend(region) for region in available_regions -} +ecs_backends = {} +for region in Session().get_available_regions("ecs"): + ecs_backends[region] = EC2ContainerServiceBackend(region) +for region in Session().get_available_regions("ecs", partition_name="aws-us-gov"): + ecs_backends[region] = EC2ContainerServiceBackend(region) +for region in Session().get_available_regions("ecs", partition_name="aws-cn"): + ecs_backends[region] = EC2ContainerServiceBackend(region) diff --git a/moto/emr/models.py b/moto/emr/models.py index b62ce7932..3e10eff1a 100644 --- a/moto/emr/models.py +++ b/moto/emr/models.py @@ -4,6 +4,7 @@ from datetime import timedelta import boto.emr import pytz +from boto3 import Session from dateutil.parser import parse as dtparse from moto.core import BaseBackend, BaseModel from moto.emr.exceptions import EmrError @@ -460,5 +461,9 @@ class ElasticMapReduceBackend(BaseBackend): emr_backends = {} -for region in boto.emr.regions(): - emr_backends[region.name] = ElasticMapReduceBackend(region.name) +for region in Session().get_available_regions("emr"): + emr_backends[region] = ElasticMapReduceBackend(region) +for region in Session().get_available_regions("emr", partition_name="aws-us-gov"): + emr_backends[region] = ElasticMapReduceBackend(region) +for region in Session().get_available_regions("emr", partition_name="aws-cn"): + emr_backends[region] = ElasticMapReduceBackend(region) diff --git a/moto/events/models.py b/moto/events/models.py index 0298c7c69..548d41393 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -1,7 +1,7 @@ import os import re import json -import boto3 +from boto3 import Session from moto.core.exceptions import JsonRESTError from moto.core import BaseBackend, BaseModel @@ -362,5 +362,10 @@ class EventsBackend(BaseBackend): self.event_buses.pop(name, None) -available_regions = boto3.session.Session().get_available_regions("events") -events_backends = {region: EventsBackend(region) for region in available_regions} +events_backends = {} +for region in Session().get_available_regions("events"): + events_backends[region] = EventsBackend(region) +for region in Session().get_available_regions("events", partition_name="aws-us-gov"): + events_backends[region] = EventsBackend(region) +for region in Session().get_available_regions("events", partition_name="aws-cn"): + events_backends[region] = EventsBackend(region) diff --git a/moto/glacier/models.py b/moto/glacier/models.py index 6a3fc074d..ff87a34a6 100644 --- a/moto/glacier/models.py +++ b/moto/glacier/models.py @@ -6,6 +6,8 @@ import datetime import boto.glacier +from boto3 import Session + from moto.core import BaseBackend, BaseModel from .utils import get_job_id @@ -221,5 +223,9 @@ class GlacierBackend(BaseBackend): glacier_backends = {} -for region in boto.glacier.regions(): - glacier_backends[region.name] = GlacierBackend(region) +for region in Session().get_available_regions("glacier"): + glacier_backends[region] = GlacierBackend(region) +for region in Session().get_available_regions("glacier", partition_name="aws-us-gov"): + glacier_backends[region] = GlacierBackend(region) +for region in Session().get_available_regions("glacier", partition_name="aws-cn"): + glacier_backends[region] = GlacierBackend(region) diff --git a/moto/iot/models.py b/moto/iot/models.py index 74a3e992c..d59d7533c 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -9,7 +9,7 @@ import uuid from collections import OrderedDict from datetime import datetime -import boto3 +from boto3 import Session from moto.core import BaseBackend, BaseModel from .exceptions import ( @@ -825,5 +825,10 @@ class IoTBackend(BaseBackend): return self.jobs[job_id] -available_regions = boto3.session.Session().get_available_regions("iot") -iot_backends = {region: IoTBackend(region) for region in available_regions} +iot_backends = {} +for region in Session().get_available_regions("iot"): + iot_backends[region] = IoTBackend(region) +for region in Session().get_available_regions("iot", partition_name="aws-us-gov"): + iot_backends[region] = IoTBackend(region) +for region in Session().get_available_regions("iot", partition_name="aws-cn"): + iot_backends[region] = IoTBackend(region) diff --git a/moto/iotdata/models.py b/moto/iotdata/models.py index e534e1d1f..41b69bc7f 100644 --- a/moto/iotdata/models.py +++ b/moto/iotdata/models.py @@ -1,8 +1,9 @@ from __future__ import unicode_literals import json import time -import boto3 import jsondiff +from boto3 import Session + from moto.core import BaseBackend, BaseModel from moto.iot import iot_backends from .exceptions import ( @@ -205,5 +206,10 @@ class IoTDataPlaneBackend(BaseBackend): return None -available_regions = boto3.session.Session().get_available_regions("iot-data") -iotdata_backends = {region: IoTDataPlaneBackend(region) for region in available_regions} +iotdata_backends = {} +for region in Session().get_available_regions("iot-data"): + iotdata_backends[region] = IoTDataPlaneBackend(region) +for region in Session().get_available_regions("iot-data", partition_name="aws-us-gov"): + iotdata_backends[region] = IoTDataPlaneBackend(region) +for region in Session().get_available_regions("iot-data", partition_name="aws-cn"): + iotdata_backends[region] = IoTDataPlaneBackend(region) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index 48642f197..cdb81a565 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -10,6 +10,8 @@ import itertools from operator import attrgetter from hashlib import md5 +from boto3 import Session + from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -530,5 +532,9 @@ class KinesisBackend(BaseBackend): kinesis_backends = {} -for region in boto.kinesis.regions(): - kinesis_backends[region.name] = KinesisBackend() +for region in Session().get_available_regions("kinesis"): + kinesis_backends[region] = KinesisBackend() +for region in Session().get_available_regions("kinesis", partition_name="aws-us-gov"): + kinesis_backends[region] = KinesisBackend() +for region in Session().get_available_regions("kinesis", partition_name="aws-cn"): + kinesis_backends[region] = KinesisBackend() diff --git a/moto/kms/models.py b/moto/kms/models.py index 9d7739779..8fa18346a 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -5,6 +5,7 @@ from collections import defaultdict from datetime import datetime, timedelta import boto.kms +from boto3 import Session from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_without_milliseconds @@ -284,5 +285,9 @@ class KmsBackend(BaseBackend): kms_backends = {} -for region in boto.kms.regions(): - kms_backends[region.name] = KmsBackend() +for region in Session().get_available_regions("kms"): + kms_backends[region] = KmsBackend() +for region in Session().get_available_regions("kms", partition_name="aws-us-gov"): + kms_backends[region] = KmsBackend() +for region in Session().get_available_regions("kms", partition_name="aws-cn"): + kms_backends[region] = KmsBackend() diff --git a/moto/logs/models.py b/moto/logs/models.py index d0639524e..a3f87fefe 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -1,3 +1,5 @@ +from boto3 import Session + from moto.core import BaseBackend import boto.logs from moto.core.utils import unix_time_millis @@ -558,6 +560,10 @@ class LogsBackend(BaseBackend): log_group.untag(tags) -logs_backends = { - region.name: LogsBackend(region.name) for region in boto.logs.regions() -} +logs_backends = {} +for region in Session().get_available_regions("logs"): + logs_backends[region] = LogsBackend(region) +for region in Session().get_available_regions("logs", partition_name="aws-us-gov"): + logs_backends[region] = LogsBackend(region) +for region in Session().get_available_regions("logs", partition_name="aws-cn"): + logs_backends[region] = LogsBackend(region) diff --git a/moto/polly/models.py b/moto/polly/models.py index f91c80c64..f76bf4a88 100644 --- a/moto/polly/models.py +++ b/moto/polly/models.py @@ -3,6 +3,8 @@ from xml.etree import ElementTree as ET import datetime import boto3 +from boto3 import Session + from moto.core import BaseBackend, BaseModel from .resources import VOICE_DATA @@ -113,7 +115,10 @@ class PollyBackend(BaseBackend): self._lexicons[name] = lexicon -available_regions = boto3.session.Session().get_available_regions("polly") -polly_backends = { - region: PollyBackend(region_name=region) for region in available_regions -} +polly_backends = {} +for region in Session().get_available_regions("polly"): + polly_backends[region] = PollyBackend(region) +for region in Session().get_available_regions("polly", partition_name="aws-us-gov"): + polly_backends[region] = PollyBackend(region) +for region in Session().get_available_regions("polly", partition_name="aws-cn"): + polly_backends[region] = PollyBackend(region) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 686d22ccf..df7b9f8c3 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -6,6 +6,7 @@ import os from collections import defaultdict import boto.rds2 +from boto3 import Session from jinja2 import Template from re import compile as re_compile from moto.cloudformation.exceptions import UnformattedGetAttTemplateException @@ -1501,6 +1502,10 @@ class DBParameterGroup(object): return db_parameter_group -rds2_backends = dict( - (region.name, RDS2Backend(region.name)) for region in boto.rds2.regions() -) +rds2_backends = {} +for region in Session().get_available_regions("rds"): + rds2_backends[region] = RDS2Backend(region) +for region in Session().get_available_regions("rds", partition_name="aws-us-gov"): + rds2_backends[region] = RDS2Backend(region) +for region in Session().get_available_regions("rds", partition_name="aws-cn"): + rds2_backends[region] = RDS2Backend(region) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 2c57c0f06..faaa03652 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -4,6 +4,7 @@ import copy import datetime import boto.redshift +from boto3 import Session from botocore.exceptions import ClientError from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel @@ -897,7 +898,9 @@ class RedshiftBackend(BaseBackend): redshift_backends = {} -for region in boto.redshift.regions(): - redshift_backends[region.name] = RedshiftBackend( - ec2_backends[region.name], region.name - ) +for region in Session().get_available_regions("redshift"): + redshift_backends[region] = RedshiftBackend(ec2_backends[region], region) +for region in Session().get_available_regions("redshift", partition_name="aws-us-gov"): + redshift_backends[region] = RedshiftBackend(ec2_backends[region], region) +for region in Session().get_available_regions("redshift", partition_name="aws-cn"): + redshift_backends[region] = RedshiftBackend(ec2_backends[region], region) diff --git a/moto/resourcegroups/models.py b/moto/resourcegroups/models.py index 7d4d88230..4dd96408a 100644 --- a/moto/resourcegroups/models.py +++ b/moto/resourcegroups/models.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals from builtins import str -import boto3 import json import re +from boto3 import Session + from moto.core import BaseBackend, BaseModel from moto.core import ACCOUNT_ID from .exceptions import BadRequestException @@ -350,7 +351,14 @@ class ResourceGroupsBackend(BaseBackend): return self.groups.by_name[group_name] -available_regions = boto3.session.Session().get_available_regions("resource-groups") -resourcegroups_backends = { - region: ResourceGroupsBackend(region_name=region) for region in available_regions -} +resourcegroups_backends = {} +for region in Session().get_available_regions("resource-groups"): + resourcegroups_backends[region] = ResourceGroupsBackend(region) +for region in Session().get_available_regions( + "resource-groups", partition_name="aws-us-gov" +): + resourcegroups_backends[region] = ResourceGroupsBackend(region) +for region in Session().get_available_regions( + "resource-groups", partition_name="aws-cn" +): + resourcegroups_backends[region] = ResourceGroupsBackend(region) diff --git a/moto/resourcegroupstaggingapi/models.py b/moto/resourcegroupstaggingapi/models.py index 7b0c03a88..850ab5c04 100644 --- a/moto/resourcegroupstaggingapi/models.py +++ b/moto/resourcegroupstaggingapi/models.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals import uuid -import boto3 import six +from boto3 import Session + from moto.core import BaseBackend from moto.core.exceptions import RESTError @@ -636,9 +637,14 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # return failed_resources_map -available_regions = boto3.session.Session().get_available_regions( - "resourcegroupstaggingapi" -) -resourcegroupstaggingapi_backends = { - region: ResourceGroupsTaggingAPIBackend(region) for region in available_regions -} +resourcegroupstaggingapi_backends = {} +for region in Session().get_available_regions("resourcegroupstaggingapi"): + resourcegroupstaggingapi_backends[region] = ResourceGroupsTaggingAPIBackend(region) +for region in Session().get_available_regions( + "resourcegroupstaggingapi", partition_name="aws-us-gov" +): + resourcegroupstaggingapi_backends[region] = ResourceGroupsTaggingAPIBackend(region) +for region in Session().get_available_regions( + "resourcegroupstaggingapi", partition_name="aws-cn" +): + resourcegroupstaggingapi_backends[region] = ResourceGroupsTaggingAPIBackend(region) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 2a1a336d9..2ed29057a 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -7,6 +7,7 @@ import uuid import datetime import boto3 +from boto3 import Session from moto.core import BaseBackend, BaseModel from .exceptions import ( @@ -491,7 +492,14 @@ class SecretsManagerBackend(BaseBackend): ) -available_regions = boto3.session.Session().get_available_regions("secretsmanager") -secretsmanager_backends = { - region: SecretsManagerBackend(region_name=region) for region in available_regions -} +secretsmanager_backends = {} +for region in Session().get_available_regions("secretsmanager"): + secretsmanager_backends[region] = SecretsManagerBackend(region_name=region) +for region in Session().get_available_regions( + "secretsmanager", partition_name="aws-us-gov" +): + secretsmanager_backends[region] = SecretsManagerBackend(region_name=region) +for region in Session().get_available_regions( + "secretsmanager", partition_name="aws-cn" +): + secretsmanager_backends[region] = SecretsManagerBackend(region_name=region) diff --git a/moto/sns/models.py b/moto/sns/models.py index cdc50f640..695639084 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -707,6 +707,10 @@ class SNSBackend(BaseBackend): sns_backends = {} for region in Session().get_available_regions("sns"): sns_backends[region] = SNSBackend(region) +for region in Session().get_available_regions("sns", partition_name="aws-us-gov"): + sns_backends[region] = SNSBackend(region) +for region in Session().get_available_regions("sns", partition_name="aws-cn"): + sns_backends[region] = SNSBackend(region) DEFAULT_EFFECTIVE_DELIVERY_POLICY = { diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 4e6282f56..78ddc38b4 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -9,6 +9,7 @@ import struct from xml.sax.saxutils import escape import boto.sqs +from boto3 import Session from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel @@ -857,5 +858,9 @@ class SQSBackend(BaseBackend): sqs_backends = {} -for region in boto.sqs.regions(): - sqs_backends[region.name] = SQSBackend(region.name) +for region in Session().get_available_regions("sqs"): + sqs_backends[region] = SQSBackend(region) +for region in Session().get_available_regions("sqs", partition_name="aws-us-gov"): + sqs_backends[region] = SQSBackend(region) +for region in Session().get_available_regions("sqs", partition_name="aws-cn"): + sqs_backends[region] = SQSBackend(region) diff --git a/moto/stepfunctions/models.py b/moto/stepfunctions/models.py index 665f3b777..6bcd6c48b 100644 --- a/moto/stepfunctions/models.py +++ b/moto/stepfunctions/models.py @@ -1,6 +1,9 @@ import boto import re from datetime import datetime + +from boto3 import Session + from moto.core import BaseBackend from moto.core.utils import iso_8601_datetime_without_milliseconds from moto.sts.models import ACCOUNT_ID @@ -280,7 +283,12 @@ class StepFunctionBackend(BaseBackend): return ACCOUNT_ID -stepfunction_backends = { - _region.name: StepFunctionBackend(_region.name) - for _region in boto.awslambda.regions() -} +stepfunction_backends = {} +for region in Session().get_available_regions("stepfunctions"): + stepfunction_backends[region] = StepFunctionBackend(region) +for region in Session().get_available_regions( + "stepfunctions", partition_name="aws-us-gov" +): + stepfunction_backends[region] = StepFunctionBackend(region) +for region in Session().get_available_regions("stepfunctions", partition_name="aws-cn"): + stepfunction_backends[region] = StepFunctionBackend(region) diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 50cc29bb3..5637b8410 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import boto.swf +from boto3 import Session from moto.core import BaseBackend @@ -418,5 +419,9 @@ class SWFBackend(BaseBackend): swf_backends = {} -for region in boto.swf.regions(): - swf_backends[region.name] = SWFBackend(region.name) +for region in Session().get_available_regions("swf"): + swf_backends[region] = SWFBackend(region) +for region in Session().get_available_regions("swf", partition_name="aws-us-gov"): + swf_backends[region] = SWFBackend(region) +for region in Session().get_available_regions("swf", partition_name="aws-cn"): + swf_backends[region] = SWFBackend(region) diff --git a/scripts/template/lib/models.py.j2 b/scripts/template/lib/models.py.j2 index 28fa4a4e1..84f8dad71 100644 --- a/scripts/template/lib/models.py.j2 +++ b/scripts/template/lib/models.py.j2 @@ -1,5 +1,5 @@ from __future__ import unicode_literals -import boto3 +from boto3 import Session from moto.core import BaseBackend, BaseModel @@ -16,5 +16,10 @@ class {{ service_class }}Backend(BaseBackend): # add methods from here -available_regions = boto3.session.Session().get_available_regions("{{ service }}") -{{ escaped_service }}_backends = {region: {{ service_class }}Backend(region) for region in available_regions} +{{ escaped_service }}_backends = {} +for region in Session().get_available_regions("{{ service }}"): + {{ escaped_service }}_backends[region] = {{ service_class }}Backend() +for region in Session().get_available_regions("{{ service }}", partition_name="aws-us-gov"): + {{ escaped_service }}_backends[region] = {{ service_class }}Backend() +for region in Session().get_available_regions("{{ service }}", partition_name="aws-cn"): + {{ escaped_service }}_backends[region] = {{ service_class }}Backend() diff --git a/tests/test_ec2/test_availability_zones_and_regions.py b/tests/test_ec2/test_availability_zones_and_regions.py index 349be7936..4adc87efe 100644 --- a/tests/test_ec2/test_availability_zones_and_regions.py +++ b/tests/test_ec2/test_availability_zones_and_regions.py @@ -11,7 +11,7 @@ from moto import mock_ec2, mock_ec2_deprecated def test_describe_regions(): conn = boto.connect_ec2("the_key", "the_secret") regions = conn.get_all_regions() - regions.should.have.length_of(16) + regions.should.have.length_of(22) for region in regions: region.endpoint.should.contain(region.name) @@ -32,7 +32,7 @@ def test_availability_zones(): def test_boto3_describe_regions(): ec2 = boto3.client("ec2", "us-east-1") resp = ec2.describe_regions() - resp["Regions"].should.have.length_of(16) + resp["Regions"].should.have.length_of(22) for rec in resp["Regions"]: rec["Endpoint"].should.contain(rec["RegionName"]) diff --git a/tests/test_ec2/test_regions.py b/tests/test_ec2/test_regions.py index 551b739f2..3504a2b5a 100644 --- a/tests/test_ec2/test_regions.py +++ b/tests/test_ec2/test_regions.py @@ -3,13 +3,21 @@ import boto.ec2 import boto.ec2.autoscale import boto.ec2.elb import sure +from boto3 import Session + from moto import mock_ec2_deprecated, mock_autoscaling_deprecated, mock_elb_deprecated from moto.ec2 import ec2_backends def test_use_boto_regions(): - boto_regions = {r.name for r in boto.ec2.regions()} + boto_regions = set() + for region in Session().get_available_regions("ec2"): + boto_regions.add(region) + for region in Session().get_available_regions("ec2", partition_name="aws-us-gov"): + boto_regions.add(region) + for region in Session().get_available_regions("ec2", partition_name="aws-cn"): + boto_regions.add(region) moto_regions = set(ec2_backends) moto_regions.should.equal(boto_regions) diff --git a/tests/test_glacier/test_glacier_jobs.py b/tests/test_glacier/test_glacier_jobs.py index 11077d7f2..cba2c1a27 100644 --- a/tests/test_glacier/test_glacier_jobs.py +++ b/tests/test_glacier/test_glacier_jobs.py @@ -44,7 +44,7 @@ def test_describe_job(): joboutput.should.have.key("Tier").which.should.equal("Standard") joboutput.should.have.key("StatusCode").which.should.equal("InProgress") joboutput.should.have.key("VaultARN").which.should.equal( - "arn:aws:glacier:RegionInfo:us-west-2:012345678901:vaults/my_vault" + "arn:aws:glacier:us-west-2:012345678901:vaults/my_vault" ) From 45922fd4efc4497b2fadac85469acd352b7f0096 Mon Sep 17 00:00:00 2001 From: Jovan Zivanov Date: Thu, 26 Dec 2019 17:21:37 +0100 Subject: [PATCH 21/48] [codecommit] add support for server mode --- moto/backends.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/backends.py b/moto/backends.py index 9295bc758..bfc2b398e 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -8,6 +8,7 @@ from moto.awslambda import lambda_backends from moto.batch import batch_backends from moto.cloudformation import cloudformation_backends from moto.cloudwatch import cloudwatch_backends +from moto.codecommit import codecommit_backends from moto.codepipeline import codepipeline_backends from moto.cognitoidentity import cognitoidentity_backends from moto.cognitoidp import cognitoidp_backends @@ -61,6 +62,7 @@ BACKENDS = { "batch": batch_backends, "cloudformation": cloudformation_backends, "cloudwatch": cloudwatch_backends, + "codecommit": codecommit_backends, "codepipeline": codepipeline_backends, "cognito-identity": cognitoidentity_backends, "cognito-idp": cognitoidp_backends, From 9d9b6208786498b158941b53715668ebfd79fef3 Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 26 Dec 2019 21:03:49 +0100 Subject: [PATCH 22/48] Fixed linter errors --- moto/awslambda/models.py | 1 - moto/cloudformation/models.py | 1 - moto/cloudwatch/models.py | 1 - moto/cognitoidentity/models.py | 1 - moto/cognitoidp/models.py | 1 - moto/datapipeline/models.py | 1 - moto/ec2/models.py | 2 -- moto/emr/models.py | 1 - moto/glacier/models.py | 2 -- moto/kinesis/models.py | 1 - moto/kms/models.py | 1 - moto/logs/models.py | 1 - moto/polly/models.py | 1 - moto/rds2/models.py | 1 - moto/redshift/models.py | 1 - moto/secretsmanager/models.py | 1 - moto/sqs/models.py | 1 - moto/stepfunctions/models.py | 1 - moto/swf/models/__init__.py | 1 - 19 files changed, 21 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 5795ff5df..95a5c4ad5 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -23,7 +23,6 @@ import traceback import weakref import requests.adapters -import boto.awslambda from boto3 import Session from moto.core import BaseBackend, BaseModel diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 073d84ce8..0ae5d1ae4 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -4,7 +4,6 @@ import json import yaml import uuid -import boto.cloudformation from boto3 import Session from moto.compat import OrderedDict diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 18f5965a8..13b31ddfe 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -5,7 +5,6 @@ from boto3 import Session from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError -import boto.ec2.cloudwatch from datetime import datetime, timedelta from dateutil.tz import tzutc from uuid import uuid4 diff --git a/moto/cognitoidentity/models.py b/moto/cognitoidentity/models.py index 20fd1241e..ae9f308c2 100644 --- a/moto/cognitoidentity/models.py +++ b/moto/cognitoidentity/models.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import datetime import json -import boto.cognito.identity from boto3 import Session from moto.compat import OrderedDict diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 70db9b7fa..478ceffb2 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -9,7 +9,6 @@ import os import time import uuid -import boto.cognito.identity from boto3 import Session from jose import jws diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index 1964db008..d93deea61 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import datetime -import boto.datapipeline from boto3 import Session from moto.compat import OrderedDict diff --git a/moto/ec2/models.py b/moto/ec2/models.py index e492ae7c6..93a350914 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -12,8 +12,6 @@ import warnings from boto3 import Session from pkg_resources import resource_filename -import boto.ec2 - from collections import defaultdict import weakref from datetime import datetime diff --git a/moto/emr/models.py b/moto/emr/models.py index 3e10eff1a..713b15b9f 100644 --- a/moto/emr/models.py +++ b/moto/emr/models.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from datetime import datetime from datetime import timedelta -import boto.emr import pytz from boto3 import Session from dateutil.parser import parse as dtparse diff --git a/moto/glacier/models.py b/moto/glacier/models.py index ff87a34a6..9e91ea3a5 100644 --- a/moto/glacier/models.py +++ b/moto/glacier/models.py @@ -4,8 +4,6 @@ import hashlib import datetime - -import boto.glacier from boto3 import Session from moto.core import BaseBackend, BaseModel diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index cdb81a565..ec9655bfa 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import datetime import time -import boto.kinesis import re import six import itertools diff --git a/moto/kms/models.py b/moto/kms/models.py index 8fa18346a..22f0039b2 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -4,7 +4,6 @@ import os from collections import defaultdict from datetime import datetime, timedelta -import boto.kms from boto3 import Session from moto.core import BaseBackend, BaseModel diff --git a/moto/logs/models.py b/moto/logs/models.py index a3f87fefe..7448319db 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -1,7 +1,6 @@ from boto3 import Session from moto.core import BaseBackend -import boto.logs from moto.core.utils import unix_time_millis from .exceptions import ( ResourceNotFoundException, diff --git a/moto/polly/models.py b/moto/polly/models.py index f76bf4a88..cf4c8ab03 100644 --- a/moto/polly/models.py +++ b/moto/polly/models.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from xml.etree import ElementTree as ET import datetime -import boto3 from boto3 import Session from moto.core import BaseBackend, BaseModel diff --git a/moto/rds2/models.py b/moto/rds2/models.py index df7b9f8c3..e648765b7 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -5,7 +5,6 @@ import datetime import os from collections import defaultdict -import boto.rds2 from boto3 import Session from jinja2 import Template from re import compile as re_compile diff --git a/moto/redshift/models.py b/moto/redshift/models.py index faaa03652..17840fb86 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import copy import datetime -import boto.redshift from boto3 import Session from botocore.exceptions import ClientError from moto.compat import OrderedDict diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 2ed29057a..294a6401e 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -6,7 +6,6 @@ import json import uuid import datetime -import boto3 from boto3 import Session from moto.core import BaseBackend, BaseModel diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 78ddc38b4..40dd6ba97 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -8,7 +8,6 @@ import six import struct from xml.sax.saxutils import escape -import boto.sqs from boto3 import Session from moto.core.exceptions import RESTError diff --git a/moto/stepfunctions/models.py b/moto/stepfunctions/models.py index 6bcd6c48b..de530b863 100644 --- a/moto/stepfunctions/models.py +++ b/moto/stepfunctions/models.py @@ -1,4 +1,3 @@ -import boto import re from datetime import datetime diff --git a/moto/swf/models/__init__.py b/moto/swf/models/__init__.py index 5637b8410..e5b285f5b 100644 --- a/moto/swf/models/__init__.py +++ b/moto/swf/models/__init__.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import boto.swf from boto3 import Session from moto.core import BaseBackend From 2fb5004dc2147925b512fa9706061cbb39486a62 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 27 Dec 2019 16:04:12 +0100 Subject: [PATCH 23/48] Fix sns unsubscribe & delete_topic behavior --- moto/sns/models.py | 6 +--- tests/test_sns/test_subscriptions.py | 18 +++++++++-- tests/test_sns/test_subscriptions_boto3.py | 36 +++++++++++++++------- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index cdc50f640..680176bd3 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -397,10 +397,6 @@ class SNSBackend(BaseBackend): return self._get_values_nexttoken(self.topics, next_token) def delete_topic(self, arn): - topic = self.get_topic(arn) - subscriptions = self._get_topic_subscriptions(topic) - for sub in subscriptions: - self.unsubscribe(sub.arn) self.topics.pop(arn) def get_topic(self, arn): @@ -466,7 +462,7 @@ class SNSBackend(BaseBackend): return None def unsubscribe(self, subscription_arn): - self.subscriptions.pop(subscription_arn) + self.subscriptions.pop(subscription_arn, None) def list_subscriptions(self, topic_arn=None, next_token=None): if topic_arn: diff --git a/tests/test_sns/test_subscriptions.py b/tests/test_sns/test_subscriptions.py index fbd4274f4..f773438d7 100644 --- a/tests/test_sns/test_subscriptions.py +++ b/tests/test_sns/test_subscriptions.py @@ -54,9 +54,10 @@ def test_deleting_subscriptions_by_deleting_topic(): ]["Subscriptions"] subscriptions.should.have.length_of(1) subscription = subscriptions[0] + subscription_arn = subscription["SubscriptionArn"] subscription["TopicArn"].should.equal(topic_arn) subscription["Protocol"].should.equal("http") - subscription["SubscriptionArn"].should.contain(topic_arn) + subscription_arn.should.contain(topic_arn) subscription["Endpoint"].should.equal("http://example.com/") # Now delete the topic @@ -67,12 +68,25 @@ def test_deleting_subscriptions_by_deleting_topic(): topics = topics_json["ListTopicsResponse"]["ListTopicsResult"]["Topics"] topics.should.have.length_of(0) - # And there should be zero subscriptions left + # And the subscription should still be left + subscriptions = conn.get_all_subscriptions()["ListSubscriptionsResponse"][ + "ListSubscriptionsResult" + ]["Subscriptions"] + subscriptions.should.have.length_of(1) + subscription = subscriptions[0] + subscription["SubscriptionArn"].should.equal(subscription_arn) + + # Now delete hanging subscription + conn.unsubscribe(subscription_arn) + subscriptions = conn.get_all_subscriptions()["ListSubscriptionsResponse"][ "ListSubscriptionsResult" ]["Subscriptions"] subscriptions.should.have.length_of(0) + # Deleting it again should not result in any error + conn.unsubscribe(subscription_arn) + @mock_sns_deprecated def test_getting_subscriptions_by_topic(): diff --git a/tests/test_sns/test_subscriptions_boto3.py b/tests/test_sns/test_subscriptions_boto3.py index faf3ae4a5..d91b3566b 100644 --- a/tests/test_sns/test_subscriptions_boto3.py +++ b/tests/test_sns/test_subscriptions_boto3.py @@ -97,34 +97,48 @@ def test_creating_subscription(): @mock_sns -def test_deleting_subscriptions_by_deleting_topic(): - conn = boto3.client("sns", region_name="us-east-1") - conn.create_topic(Name="some-topic") - response = conn.list_topics() +def test_unsubscribe_from_deleted_topic(): + client = boto3.client("sns", region_name="us-east-1") + client.create_topic(Name="some-topic") + response = client.list_topics() topic_arn = response["Topics"][0]["TopicArn"] - conn.subscribe(TopicArn=topic_arn, Protocol="http", Endpoint="http://example.com/") + client.subscribe( + TopicArn=topic_arn, Protocol="http", Endpoint="http://example.com/" + ) - subscriptions = conn.list_subscriptions()["Subscriptions"] + subscriptions = client.list_subscriptions()["Subscriptions"] subscriptions.should.have.length_of(1) subscription = subscriptions[0] + subscription_arn = subscription["SubscriptionArn"] subscription["TopicArn"].should.equal(topic_arn) subscription["Protocol"].should.equal("http") - subscription["SubscriptionArn"].should.contain(topic_arn) + subscription_arn.should.contain(topic_arn) subscription["Endpoint"].should.equal("http://example.com/") # Now delete the topic - conn.delete_topic(TopicArn=topic_arn) + client.delete_topic(TopicArn=topic_arn) # And there should now be 0 topics - topics_json = conn.list_topics() + topics_json = client.list_topics() topics = topics_json["Topics"] topics.should.have.length_of(0) - # And there should be zero subscriptions left - subscriptions = conn.list_subscriptions()["Subscriptions"] + # And the subscription should still be left + subscriptions = client.list_subscriptions()["Subscriptions"] + subscriptions.should.have.length_of(1) + subscription = subscriptions[0] + subscription["SubscriptionArn"].should.equal(subscription_arn) + + # Now delete hanging subscription + client.unsubscribe(SubscriptionArn=subscription_arn) + + subscriptions = client.list_subscriptions()["Subscriptions"] subscriptions.should.have.length_of(0) + # Deleting it again should not result in any error + client.unsubscribe(SubscriptionArn=subscription_arn) + @mock_sns def test_getting_subscriptions_by_topic(): From 877f3b056aebf619ccb2fafa91ef334dc43bfaae Mon Sep 17 00:00:00 2001 From: Dejan Levec Date: Fri, 27 Dec 2019 18:53:14 +0100 Subject: [PATCH 24/48] Add IsTruncated to Route53.list_resource_record_sets --- moto/route53/responses.py | 1 + tests/test_route53/test_route53.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/moto/route53/responses.py b/moto/route53/responses.py index 3e688b65d..077c89a2c 100644 --- a/moto/route53/responses.py +++ b/moto/route53/responses.py @@ -271,6 +271,7 @@ LIST_RRSET_RESPONSE = """ diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 0e9a1e2c0..746c78719 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -862,6 +862,8 @@ def test_list_resource_record_sets_name_type_filters(): StartRecordName=all_records[start_with][1], ) + response["IsTruncated"].should.equal(False) + returned_records = [ (record["Type"], record["Name"]) for record in response["ResourceRecordSets"] ] From cd9b7072a2a5b8842e203d18f1ab5af8bdd1ac1d Mon Sep 17 00:00:00 2001 From: gruebel Date: Sat, 28 Dec 2019 21:22:16 +0100 Subject: [PATCH 25/48] Adjust ec2 regions tests --- tests/test_ec2/test_availability_zones_and_regions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ec2/test_availability_zones_and_regions.py b/tests/test_ec2/test_availability_zones_and_regions.py index 4adc87efe..d5355f3b1 100644 --- a/tests/test_ec2/test_availability_zones_and_regions.py +++ b/tests/test_ec2/test_availability_zones_and_regions.py @@ -11,7 +11,7 @@ from moto import mock_ec2, mock_ec2_deprecated def test_describe_regions(): conn = boto.connect_ec2("the_key", "the_secret") regions = conn.get_all_regions() - regions.should.have.length_of(22) + len(regions).should.be.greater_than(1) for region in regions: region.endpoint.should.contain(region.name) @@ -32,7 +32,7 @@ def test_availability_zones(): def test_boto3_describe_regions(): ec2 = boto3.client("ec2", "us-east-1") resp = ec2.describe_regions() - resp["Regions"].should.have.length_of(22) + len(resp["Regions"]).should.be.greater_than(1) for rec in resp["Regions"]: rec["Endpoint"].should.contain(rec["RegionName"]) From 000cb968a44277d5dcd696f7745775468e3d45e6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 5 Jan 2020 11:36:51 +0000 Subject: [PATCH 26/48] #2623 - Only return response from lambda, skip log output --- moto/awslambda/models.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 95a5c4ad5..38ff81fb2 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -53,9 +53,6 @@ try: except ImportError: from backports.tempfile import TemporaryDirectory -# The lambci container is returning a special escape character for the "RequestID" fields. Unicode 033: -# _stderr_regex = re.compile(r"START|END|REPORT RequestId: .*") -_stderr_regex = re.compile(r"\033\[\d+.*") _orig_adapter_send = requests.adapters.HTTPAdapter.send docker_3 = docker.__version__[0] >= "3" @@ -385,7 +382,7 @@ class LambdaFunction(BaseModel): try: # TODO: I believe we can keep the container running and feed events as needed # also need to hook it up to the other services so it can make kws/s3 etc calls - # Should get invoke_id /RequestId from invovation + # Should get invoke_id /RequestId from invocation env_vars = { "AWS_LAMBDA_FUNCTION_TIMEOUT": self.timeout, "AWS_LAMBDA_FUNCTION_NAME": self.function_name, @@ -453,14 +450,9 @@ class LambdaFunction(BaseModel): if exit_code != 0: raise Exception("lambda invoke failed output: {}".format(output)) - # strip out RequestId lines (TODO: This will return an additional '\n' in the response) - output = os.linesep.join( - [ - line - for line in self.convert(output).splitlines() - if not _stderr_regex.match(line) - ] - ) + # We only care about the response from the lambda + # Which is the last line of the output, according to https://github.com/lambci/docker-lambda/issues/25 + output = output.splitlines()[-1] return output, False except BaseException as e: traceback.print_exc() From eab9e15bf08bf89ec8552c614a39457899b8427f Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 5 Jan 2020 15:01:31 +0000 Subject: [PATCH 27/48] #2623 - Fix and simplify test in ServerMode --- tests/test_awslambda/test_lambda.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 6fd97e325..e378f6ee2 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -58,8 +58,7 @@ def lambda_handler(event, context): volume_id = event.get('volume_id') vol = ec2.Volume(volume_id) - print('get volume details for %s\\nVolume - %s state=%s, size=%s' % (volume_id, volume_id, vol.state, vol.size)) - return event + return {'id': vol.id, 'state': vol.state, 'size': vol.size} """.format( base_url="motoserver:5000" if settings.TEST_SERVER_MODE @@ -181,27 +180,11 @@ if settings.TEST_SERVER_MODE: Payload=json.dumps(in_data), ) result["StatusCode"].should.equal(202) - msg = "get volume details for %s\nVolume - %s state=%s, size=%s\n%s" % ( - vol.id, - vol.id, - vol.state, - vol.size, - json.dumps(in_data).replace( - " ", "" - ), # Makes the tests pass as the result is missing the whitespace + actual_payload = result["Payload"].read().decode("utf-8") + expected_payload = json.dumps( + {"id": vol.id, "state": vol.state, "size": vol.size} ) - - log_result = base64.b64decode(result["LogResult"]).decode("utf-8") - - # The Docker lambda invocation will return an additional '\n', so need to replace it: - log_result = log_result.replace("\n\n", "\n") - log_result.should.equal(msg) - - payload = result["Payload"].read().decode("utf-8") - - # The Docker lambda invocation will return an additional '\n', so need to replace it: - payload = payload.replace("\n\n", "\n") - payload.should.equal(msg) + actual_payload.should.equal(expected_payload) @mock_logs From 68d882e6c0408b029ab0be5a8641d19c7652a154 Mon Sep 17 00:00:00 2001 From: Franz See Date: Sun, 5 Jan 2020 23:55:04 +0800 Subject: [PATCH 28/48] moto/issues/2672 | Modified 'token_use' to return 'id' for an id token, and 'access' for an access token --- moto/cognitoidp/models.py | 8 ++++---- tests/test_cognitoidp/test_cognitoidp.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 6700920ce..082fa5189 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -108,7 +108,7 @@ class CognitoIdpUserPool(BaseModel): return user_pool_json - def create_jwt(self, client_id, username, expires_in=60 * 60, extra_data={}): + def create_jwt(self, client_id, username, token_use, expires_in=60 * 60, extra_data={}): now = int(time.time()) payload = { "iss": "https://cognito-idp.{}.amazonaws.com/{}".format( @@ -116,7 +116,7 @@ class CognitoIdpUserPool(BaseModel): ), "sub": self.users[username].id, "aud": client_id, - "token_use": "id", + "token_use": token_use, "auth_time": now, "exp": now + expires_in, } @@ -125,7 +125,7 @@ class CognitoIdpUserPool(BaseModel): return jws.sign(payload, self.json_web_key, algorithm="RS256"), expires_in def create_id_token(self, client_id, username): - id_token, expires_in = self.create_jwt(client_id, username) + id_token, expires_in = self.create_jwt(client_id, username, "id") self.id_tokens[id_token] = (client_id, username) return id_token, expires_in @@ -137,7 +137,7 @@ class CognitoIdpUserPool(BaseModel): def create_access_token(self, client_id, username): extra_data = self.get_user_extra_data_by_client_id(client_id, username) access_token, expires_in = self.create_jwt( - client_id, username, extra_data=extra_data + client_id, username, "access", extra_data=extra_data ) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 7ac1038b0..71a6e3191 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -1142,12 +1142,13 @@ def test_token_legitimacy(): id_claims = json.loads(jws.verify(id_token, json_web_key, "RS256")) id_claims["iss"].should.equal(issuer) id_claims["aud"].should.equal(client_id) + id_claims["token_use"].should.equal("id") access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) for k, v in outputs["additional_fields"].items(): access_claims[k].should.equal(v) - + access_claims["token_use"].should.equal("access") @mock_cognitoidp def test_change_password(): From a8e1a3bf08312581bf4fae1908cc1bcb76aef7d6 Mon Sep 17 00:00:00 2001 From: Franz See Date: Mon, 6 Jan 2020 13:29:23 +0800 Subject: [PATCH 29/48] moto/issues/2672 | Formatted using black --- moto/cognitoidp/models.py | 4 +++- tests/test_cognitoidp/test_cognitoidp.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 082fa5189..b67239e93 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -108,7 +108,9 @@ class CognitoIdpUserPool(BaseModel): return user_pool_json - def create_jwt(self, client_id, username, token_use, expires_in=60 * 60, extra_data={}): + def create_jwt( + self, client_id, username, token_use, expires_in=60 * 60, extra_data={} + ): now = int(time.time()) payload = { "iss": "https://cognito-idp.{}.amazonaws.com/{}".format( diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 71a6e3191..79e6dbbb8 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -1150,6 +1150,7 @@ def test_token_legitimacy(): access_claims[k].should.equal(v) access_claims["token_use"].should.equal("access") + @mock_cognitoidp def test_change_password(): conn = boto3.client("cognito-idp", "us-west-2") From 5f59cb7fb0551eb39659228b311f894fa55fef96 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 6 Jan 2020 08:16:09 +0000 Subject: [PATCH 30/48] #2674 - ListAppend should also work when adding maps to a list --- moto/dynamodb2/models.py | 2 +- tests/test_dynamodb2/test_dynamodb.py | 52 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index d4907cba5..2313a6e41 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -457,7 +457,7 @@ class Item(BaseModel): ) if not old_list.is_list(): raise ParamValidationError - old_list.value.extend(new_value["L"]) + old_list.value.extend([DynamoType(v) for v in new_value["L"]]) value = old_list return value diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 831538054..2d961b406 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3489,6 +3489,58 @@ def test_update_supports_nested_list_append_onto_another_list(): ) +@mock_dynamodb2 +def test_update_supports_list_append_maps(): + client = boto3.client("dynamodb", region_name="us-west-1") + client.create_table( + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "rid", "AttributeType": "S"}, + ], + TableName="TestTable", + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "rid", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="TestTable", + Item={ + "id": {"S": "nested_list_append"}, + "rid": {"S": "range_key"}, + "a": {"L": [{"M": {"b": {"S": "bar1"}}}]}, + }, + ) + + # Update item using list_append expression + client.update_item( + TableName="TestTable", + Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}}, + UpdateExpression="SET a = list_append(a, :i)", + ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}}, + ) + + # Verify item is appended to the existing list + result = client.query( + TableName="TestTable", + KeyConditionExpression="id = :i AND begins_with(rid, :r)", + ExpressionAttributeValues={ + ":i": {"S": "nested_list_append"}, + ":r": {"S": "range_key"}, + }, + )["Items"] + result.should.equal( + [ + { + "a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}, + "rid": {"S": "range_key"}, + "id": {"S": "nested_list_append"}, + } + ] + ) + + @mock_dynamodb2 def test_update_catches_invalid_list_append_operation(): client = boto3.client("dynamodb", region_name="us-east-1") From d06a5d3a2b2947e029f544ebbd84eca43e1f6eb5 Mon Sep 17 00:00:00 2001 From: Patrick Delaney Date: Tue, 7 Jan 2020 10:12:50 -0500 Subject: [PATCH 31/48] fix: small fixes to get scripts/scaffold.py working --- scripts/scaffold.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/scaffold.py b/scripts/scaffold.py index be154f103..43a648b48 100755 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -20,8 +20,8 @@ import jinja2 from prompt_toolkit import ( prompt ) -from prompt_toolkit.contrib.completers import WordCompleter -from prompt_toolkit.shortcuts import print_tokens +from prompt_toolkit.completion import WordCompleter +from prompt_toolkit.shortcuts import print_formatted_text from botocore import xform_name from botocore.session import Session @@ -149,12 +149,12 @@ def append_mock_dict_to_backends_py(service): with open(path) as f: lines = [_.replace('\n', '') for _ in f.readlines()] - if any(_ for _ in lines if re.match(".*'{}': {}_backends.*".format(service, service), _)): + if any(_ for _ in lines if re.match(".*\"{}\": {}_backends.*".format(service, service), _)): return - filtered_lines = [_ for _ in lines if re.match(".*'.*':.*_backends.*", _)] + filtered_lines = [_ for _ in lines if re.match(".*\".*\":.*_backends.*", _)] last_elem_line_index = lines.index(filtered_lines[-1]) - new_line = " '{}': {}_backends,".format(service, get_escaped_service(service)) + new_line = " \"{}\": {}_backends,".format(service, get_escaped_service(service)) prev_line = lines[last_elem_line_index] if not prev_line.endswith('{') and not prev_line.endswith(','): lines[last_elem_line_index] += ',' From cba3cfc3843de2bd3fb6d1ad6867761af877843d Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 9 Jan 2020 09:10:16 +0000 Subject: [PATCH 32/48] Escape curly braces in formatting string --- tests/test_awslambda/test_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index e378f6ee2..2d9a6bd5d 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -58,7 +58,7 @@ def lambda_handler(event, context): volume_id = event.get('volume_id') vol = ec2.Volume(volume_id) - return {'id': vol.id, 'state': vol.state, 'size': vol.size} + return {{'id': vol.id, 'state': vol.state, 'size': vol.size}} """.format( base_url="motoserver:5000" if settings.TEST_SERVER_MODE From 58844830199ed269b44e17ef8e839b2be1cfd2b0 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 9 Jan 2020 10:08:35 +0000 Subject: [PATCH 33/48] Compare map, instead of string repr --- tests/test_awslambda/test_lambda.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 2d9a6bd5d..2835729f8 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -180,10 +180,8 @@ if settings.TEST_SERVER_MODE: Payload=json.dumps(in_data), ) result["StatusCode"].should.equal(202) - actual_payload = result["Payload"].read().decode("utf-8") - expected_payload = json.dumps( - {"id": vol.id, "state": vol.state, "size": vol.size} - ) + actual_payload = json.loads(result["Payload"].read().decode("utf-8")) + expected_payload = {"id": vol.id, "state": vol.state, "size": vol.size} actual_payload.should.equal(expected_payload) From 2cb3f327de85268165b63f8576e4e135eb59333c Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Thu, 9 Jan 2020 22:50:55 -0600 Subject: [PATCH 34/48] Store 'networkMode' in ECS Task Definitions instead of just throwing it away --- moto/ecs/models.py | 9 +++++++-- moto/ecs/responses.py | 3 ++- tests/test_ecs/test_ecs_boto3.py | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 845bdf650..30075f7f0 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -118,6 +118,7 @@ class TaskDefinition(BaseObject): revision, container_definitions, region_name, + network_mode=None, volumes=None, tags=None, ): @@ -132,6 +133,10 @@ class TaskDefinition(BaseObject): self.volumes = [] else: self.volumes = volumes + if network_mode is None: + self.network_mode = 'bridge' + else: + self.network_mode = network_mode @property def response_object(self): @@ -553,7 +558,7 @@ class EC2ContainerServiceBackend(BaseBackend): raise Exception("{0} is not a cluster".format(cluster_name)) def register_task_definition( - self, family, container_definitions, volumes, tags=None + self, family, container_definitions, volumes=None, network_mode=None, tags=None ): if family in self.task_definitions: last_id = self._get_last_task_definition_revision_id(family) @@ -562,7 +567,7 @@ class EC2ContainerServiceBackend(BaseBackend): self.task_definitions[family] = {} revision = 1 task_definition = TaskDefinition( - family, revision, container_definitions, self.region_name, volumes, tags + family, revision, container_definitions, self.region_name, volumes=volumes, network_mode=network_mode, tags=tags ) self.task_definitions[family][revision] = task_definition diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index d08bded2c..ebbfeb84b 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -62,8 +62,9 @@ class EC2ContainerServiceResponse(BaseResponse): container_definitions = self._get_param("containerDefinitions") volumes = self._get_param("volumes") tags = self._get_param("tags") + network_mode = self._get_param('networkMode') task_definition = self.ecs_backend.register_task_definition( - family, container_definitions, volumes, tags + family, container_definitions, volumes=volumes, network_mode=network_mode, tags=tags, ) return json.dumps({"taskDefinition": task_definition.response_object}) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 973c95b81..75598f6e5 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -94,6 +94,7 @@ def test_register_task_definition(): "logConfiguration": {"logDriver": "json-file"}, } ], + networkMode='bridge', tags=[ {"key": "createdBy", "value": "moto-unittest"}, {"key": "foo", "value": "bar"}, @@ -124,6 +125,7 @@ def test_register_task_definition(): response["taskDefinition"]["containerDefinitions"][0]["logConfiguration"][ "logDriver" ].should.equal("json-file") + response['taskDefinition']['networkMode'].should.equal('bridge') @mock_ecs From fd1fdde1bf8c0b3b179057e24fbce1c2b1f0a3bf Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Thu, 9 Jan 2020 23:45:14 -0600 Subject: [PATCH 35/48] Allow black to reformat correctly --- moto/ecs/models.py | 10 ++++++++-- moto/ecs/responses.py | 8 ++++++-- tests/test_ecs/test_ecs_boto3.py | 4 ++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 30075f7f0..30e4687c4 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -134,7 +134,7 @@ class TaskDefinition(BaseObject): else: self.volumes = volumes if network_mode is None: - self.network_mode = 'bridge' + self.network_mode = "bridge" else: self.network_mode = network_mode @@ -567,7 +567,13 @@ class EC2ContainerServiceBackend(BaseBackend): self.task_definitions[family] = {} revision = 1 task_definition = TaskDefinition( - family, revision, container_definitions, self.region_name, volumes=volumes, network_mode=network_mode, tags=tags + family, + revision, + container_definitions, + self.region_name, + volumes=volumes, + network_mode=network_mode, + tags=tags, ) self.task_definitions[family][revision] = task_definition diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index ebbfeb84b..49bf022b4 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -62,9 +62,13 @@ class EC2ContainerServiceResponse(BaseResponse): container_definitions = self._get_param("containerDefinitions") volumes = self._get_param("volumes") tags = self._get_param("tags") - network_mode = self._get_param('networkMode') + network_mode = self._get_param("networkMode") task_definition = self.ecs_backend.register_task_definition( - family, container_definitions, volumes=volumes, network_mode=network_mode, tags=tags, + family, + container_definitions, + volumes=volumes, + network_mode=network_mode, + tags=tags, ) return json.dumps({"taskDefinition": task_definition.response_object}) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 75598f6e5..f1f1e04ae 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -94,7 +94,7 @@ def test_register_task_definition(): "logConfiguration": {"logDriver": "json-file"}, } ], - networkMode='bridge', + networkMode="bridge", tags=[ {"key": "createdBy", "value": "moto-unittest"}, {"key": "foo", "value": "bar"}, @@ -125,7 +125,7 @@ def test_register_task_definition(): response["taskDefinition"]["containerDefinitions"][0]["logConfiguration"][ "logDriver" ].should.equal("json-file") - response['taskDefinition']['networkMode'].should.equal('bridge') + response["taskDefinition"]["networkMode"].should.equal("bridge") @mock_ecs From 6dac06ed7c8994719a5ac485d471d2c22045a05d Mon Sep 17 00:00:00 2001 From: Sebastian P Date: Fri, 10 Jan 2020 16:08:34 +0100 Subject: [PATCH 36/48] setup.py: Unlock use with jsondiff >1.1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 97a6341ff..d09f8fc7b 100755 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "python-jose<4.0.0", "mock", "docker>=2.5.1", - "jsondiff==1.1.2", + "jsondiff>=1.1.2", "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", "idna<2.9,>=2.5", From 9ce1ee49d763dc73d6630065c2b7adbacb279ad1 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 12 Jan 2020 12:05:08 +0000 Subject: [PATCH 37/48] #2626 - DynamoDB - FilterExpression should ignore items with non-existent attribute --- moto/dynamodb2/comparisons.py | 8 -------- tests/test_dynamodb2/test_dynamodb.py | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 69d7f74e0..372f612c3 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -979,8 +979,6 @@ class OpLessThan(Op): # In python3 None is not a valid comparator when using < or > so must be handled specially if lhs and rhs: return lhs < rhs - elif lhs is None and rhs: - return True else: return False @@ -994,8 +992,6 @@ class OpGreaterThan(Op): # In python3 None is not a valid comparator when using < or > so must be handled specially if lhs and rhs: return lhs > rhs - elif lhs and rhs is None: - return True else: return False @@ -1027,8 +1023,6 @@ class OpLessThanOrEqual(Op): # In python3 None is not a valid comparator when using < or > so must be handled specially if lhs and rhs: return lhs <= rhs - elif lhs is None and rhs or lhs is None and rhs is None: - return True else: return False @@ -1042,8 +1036,6 @@ class OpGreaterThanOrEqual(Op): # In python3 None is not a valid comparator when using < or > so must be handled specially if lhs and rhs: return lhs >= rhs - elif lhs and rhs is None or lhs is None and rhs is None: - return True else: return False diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 831538054..b63a7c19e 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1719,6 +1719,32 @@ def test_scan_filter4(): assert response["Count"] == 0 +@mock_dynamodb2 +def test_scan_filter_should_not_return_non_existing_attributes(): + table_name = "my-table" + item = {"partitionKey": "pk-2", "my-attr": 42} + # Create table + res = boto3.resource("dynamodb") + res.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "partitionKey", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "partitionKey", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + table = res.Table(table_name) + # Insert items + table.put_item(Item={"partitionKey": "pk-1"}) + table.put_item(Item=item) + # Verify a few operations + # Assert we only find the item that has this attribute + table.scan(FilterExpression=Attr("my-attr").lt(43))["Items"].should.equal([item]) + table.scan(FilterExpression=Attr("my-attr").lte(42))["Items"].should.equal([item]) + table.scan(FilterExpression=Attr("my-attr").gte(42))["Items"].should.equal([item]) + table.scan(FilterExpression=Attr("my-attr").gt(41))["Items"].should.equal([item]) + # Sanity check that we can't find the item if the FE is wrong + table.scan(FilterExpression=Attr("my-attr").gt(43))["Items"].should.equal([]) + + @mock_dynamodb2 def test_bad_scan_filter(): client = boto3.client("dynamodb", region_name="us-east-1") From 8c920cce109552d4841bf1ff72fda53f69b3bd45 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 12 Jan 2020 12:20:55 +0000 Subject: [PATCH 38/48] Specify region in tests --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index b63a7c19e..333eba135 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1724,7 +1724,7 @@ def test_scan_filter_should_not_return_non_existing_attributes(): table_name = "my-table" item = {"partitionKey": "pk-2", "my-attr": 42} # Create table - res = boto3.resource("dynamodb") + res = boto3.resource("dynamodb", region_name="us-east-1") res.create_table( TableName=table_name, KeySchema=[{"AttributeName": "partitionKey", "KeyType": "HASH"}], From fba84ec34b2e0dbf9e8ade3ef0965de882e76bab Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 14 Jan 2020 12:28:48 +0530 Subject: [PATCH 39/48] Fixed a typo in README.md - related to https://github.com/spulec/moto/issues/2691 --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4024328a9..f5c45a6b6 100644 --- a/README.md +++ b/README.md @@ -283,14 +283,14 @@ def test_describe_instances_allowed(): ] } access_key = ... - # create access key for an IAM user/assumed role that has the policy above. + # create access key for an IAM user/assumed role that has the policy above. # this part should call __exactly__ 4 AWS actions, so that authentication and authorization starts exactly after this - + client = boto3.client('ec2', region_name='us-east-1', aws_access_key_id=access_key['AccessKeyId'], aws_secret_access_key=access_key['SecretAccessKey']) - - # if the IAM principal whose access key is used, does not have the permission to describe instances, this will fail + + # if the IAM principal whose access key is used, does not have the permission to describe instances, this will fail instances = client.describe_instances()['Reservations'][0]['Instances'] assert len(instances) == 0 ``` @@ -310,16 +310,16 @@ You need to ensure that the mocks are actually in place. Changes made to recent have altered some of the mock behavior. In short, you need to ensure that you _always_ do the following: 1. Ensure that your tests have dummy environment variables set up: - + export AWS_ACCESS_KEY_ID='testing' export AWS_SECRET_ACCESS_KEY='testing' export AWS_SECURITY_TOKEN='testing' export AWS_SESSION_TOKEN='testing' - -1. __VERY IMPORTANT__: ensure that you have your mocks set up __BEFORE__ your `boto3` client is established. + +1. __VERY IMPORTANT__: ensure that you have your mocks set up __BEFORE__ your `boto3` client is established. This can typically happen if you import a module that has a `boto3` client instantiated outside of a function. See the pesky imports section below on how to work around this. - + ### Example on usage? If you are a user of [pytest](https://pytest.org/en/latest/), you can leverage [pytest fixtures](https://pytest.org/en/latest/fixture.html#fixture) to help set up your mocks and other AWS resources that you would need. @@ -354,7 +354,7 @@ def cloudwatch(aws_credentials): ... etc. ``` -In the code sample above, all of the AWS/mocked fixtures take in a parameter of `aws_credentials`, +In the code sample above, all of the AWS/mocked fixtures take in a parameter of `aws_credentials`, which sets the proper fake environment variables. The fake environment variables are used so that `botocore` doesn't try to locate real credentials on your system. @@ -364,7 +364,7 @@ def test_create_bucket(s3): # s3 is a fixture defined above that yields a boto3 s3 client. # Feel free to instantiate another boto3 S3 client -- Keep note of the region though. s3.create_bucket(Bucket="somebucket") - + result = s3.list_buckets() assert len(result['Buckets']) == 1 assert result['Buckets'][0]['Name'] == 'somebucket' @@ -373,7 +373,7 @@ def test_create_bucket(s3): ### What about those pesky imports? Recall earlier, it was mentioned that mocks should be established __BEFORE__ the clients are set up. One way to avoid import issues is to make use of local Python imports -- i.e. import the module inside of the unit -test you want to run vs. importing at the top of the file. +test you want to run vs. importing at the top of the file. Example: ```python @@ -381,12 +381,12 @@ def test_something(s3): from some.package.that.does.something.with.s3 import some_func # <-- Local import for unit test # ^^ Importing here ensures that the mock has been established. - sume_func() # The mock has been established from the "s3" pytest fixture, so this function that uses + some_func() # The mock has been established from the "s3" pytest fixture, so this function that uses # a package-level S3 client will properly use the mock and not reach out to AWS. ``` ### Other caveats -For Tox, Travis CI, and other build systems, you might need to also perform a `touch ~/.aws/credentials` +For Tox, Travis CI, and other build systems, you might need to also perform a `touch ~/.aws/credentials` command before running the tests. As long as that file is present (empty preferably) and the environment variables above are set, you should be good to go. From db559e7e06bcadad70be048aeb4f1b1119671375 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Tue, 14 Jan 2020 09:55:32 -0800 Subject: [PATCH 40/48] Fix some typos --- moto/ec2/responses/security_groups.py | 8 ++++---- tests/test_autoscaling/test_autoscaling.py | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/moto/ec2/responses/security_groups.py b/moto/ec2/responses/security_groups.py index 6f2926f61..f0002d5bd 100644 --- a/moto/ec2/responses/security_groups.py +++ b/moto/ec2/responses/security_groups.py @@ -104,7 +104,7 @@ class SecurityGroups(BaseResponse): if self.is_not_dryrun("GrantSecurityGroupIngress"): for args in self._process_rules_from_querystring(): self.ec2_backend.authorize_security_group_ingress(*args) - return AUTHORIZE_SECURITY_GROUP_INGRESS_REPONSE + return AUTHORIZE_SECURITY_GROUP_INGRESS_RESPONSE def create_security_group(self): name = self._get_param("GroupName") @@ -158,7 +158,7 @@ class SecurityGroups(BaseResponse): if self.is_not_dryrun("RevokeSecurityGroupIngress"): for args in self._process_rules_from_querystring(): self.ec2_backend.revoke_security_group_ingress(*args) - return REVOKE_SECURITY_GROUP_INGRESS_REPONSE + return REVOKE_SECURITY_GROUP_INGRESS_RESPONSE CREATE_SECURITY_GROUP_RESPONSE = """ @@ -265,12 +265,12 @@ DESCRIBE_SECURITY_GROUPS_RESPONSE = ( """ ) -AUTHORIZE_SECURITY_GROUP_INGRESS_REPONSE = """ +AUTHORIZE_SECURITY_GROUP_INGRESS_RESPONSE = """ 59dbff89-35bd-4eac-99ed-be587EXAMPLE true """ -REVOKE_SECURITY_GROUP_INGRESS_REPONSE = """ +REVOKE_SECURITY_GROUP_INGRESS_RESPONSE = """ 59dbff89-35bd-4eac-99ed-be587EXAMPLE true """ diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index c46bc7219..2e7255381 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -706,14 +706,14 @@ def test_create_autoscaling_group_boto3(): "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "propogated-tag-key", - "Value": "propogate-tag-value", + "Value": "propagate-tag-value", "PropagateAtLaunch": True, }, { "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "not-propogated-tag-key", - "Value": "not-propogate-tag-value", + "Value": "not-propagate-tag-value", "PropagateAtLaunch": False, }, ], @@ -744,14 +744,14 @@ def test_create_autoscaling_group_from_instance(): "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "propogated-tag-key", - "Value": "propogate-tag-value", + "Value": "propagate-tag-value", "PropagateAtLaunch": True, }, { "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "not-propogated-tag-key", - "Value": "not-propogate-tag-value", + "Value": "not-propagate-tag-value", "PropagateAtLaunch": False, }, ], @@ -1062,7 +1062,7 @@ def test_detach_one_instance_decrement(): "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "propogated-tag-key", - "Value": "propogate-tag-value", + "Value": "propagate-tag-value", "PropagateAtLaunch": True, } ], @@ -1116,7 +1116,7 @@ def test_detach_one_instance(): "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "propogated-tag-key", - "Value": "propogate-tag-value", + "Value": "propagate-tag-value", "PropagateAtLaunch": True, } ], @@ -1169,7 +1169,7 @@ def test_attach_one_instance(): "ResourceId": "test_asg", "ResourceType": "auto-scaling-group", "Key": "propogated-tag-key", - "Value": "propogate-tag-value", + "Value": "propagate-tag-value", "PropagateAtLaunch": True, } ], From db75c9e25ca49da7c1bb7e330579db695fbeff3b Mon Sep 17 00:00:00 2001 From: Franz See Date: Sun, 5 Jan 2020 23:13:36 +0800 Subject: [PATCH 41/48] moto/issues/2670 | Moved population of user attributes from accessToken to idToken --- moto/cognitoidp/models.py | 6 +++--- tests/test_cognitoidp/test_cognitoidp.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 78025627a..9f39d7a5f 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -127,7 +127,8 @@ class CognitoIdpUserPool(BaseModel): return jws.sign(payload, self.json_web_key, algorithm="RS256"), expires_in def create_id_token(self, client_id, username): - id_token, expires_in = self.create_jwt(client_id, username, "id") + extra_data = self.get_user_extra_data_by_client_id(client_id, username) + id_token, expires_in = self.create_jwt(client_id, username, "id", extra_data=extra_data) self.id_tokens[id_token] = (client_id, username) return id_token, expires_in @@ -137,9 +138,8 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - extra_data = self.get_user_extra_data_by_client_id(client_id, username) access_token, expires_in = self.create_jwt( - client_id, username, "access", extra_data=extra_data + client_id, username, "access" ) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 79e6dbbb8..6a13683f0 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -1143,11 +1143,11 @@ def test_token_legitimacy(): id_claims["iss"].should.equal(issuer) id_claims["aud"].should.equal(client_id) id_claims["token_use"].should.equal("id") + for k, v in outputs["additional_fields"].items(): + id_claims[k].should.equal(v) access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) - for k, v in outputs["additional_fields"].items(): - access_claims[k].should.equal(v) access_claims["token_use"].should.equal("access") From 44e92f58ec44250c0701209549104d4545304ae8 Mon Sep 17 00:00:00 2001 From: Franz See Date: Wed, 15 Jan 2020 23:33:26 +0800 Subject: [PATCH 42/48] moto/issues/2670 | Used black to format the code --- moto/cognitoidp/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 9f39d7a5f..96b23a404 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -128,7 +128,9 @@ class CognitoIdpUserPool(BaseModel): def create_id_token(self, client_id, username): extra_data = self.get_user_extra_data_by_client_id(client_id, username) - id_token, expires_in = self.create_jwt(client_id, username, "id", extra_data=extra_data) + id_token, expires_in = self.create_jwt( + client_id, username, "id", extra_data=extra_data + ) self.id_tokens[id_token] = (client_id, username) return id_token, expires_in @@ -138,9 +140,7 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt( - client_id, username, "access" - ) + access_token, expires_in = self.create_jwt(client_id, username, "access") self.access_tokens[access_token] = (client_id, username) return access_token, expires_in From 33661d267e699a83828abc43cd2250b1e033fe9a Mon Sep 17 00:00:00 2001 From: Charles Park Date: Thu, 16 Jan 2020 16:33:59 -0500 Subject: [PATCH 43/48] Fix spelling typo --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 6311597fe..22ac97228 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -76,7 +76,7 @@ Currently implemented Services: +---------------------------+-----------------------+------------------------------------+ | Logs | @mock_logs | basic endpoints done | +---------------------------+-----------------------+------------------------------------+ -| Organizations | @mock_organizations | some core edpoints done | +| Organizations | @mock_organizations | some core endpoints done | +---------------------------+-----------------------+------------------------------------+ | Polly | @mock_polly | all endpoints done | +---------------------------+-----------------------+------------------------------------+ From 6f02782624e0a60d388b2ba56cf7be9cf359582b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 22 Jan 2020 11:30:17 +0000 Subject: [PATCH 44/48] #2627 - Change comparison to differentiate between 0 and None --- moto/dynamodb2/comparisons.py | 8 ++--- tests/test_dynamodb2/test_dynamodb.py | 42 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 372f612c3..29951d92d 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -977,7 +977,7 @@ class OpLessThan(Op): lhs = self.lhs.expr(item) rhs = self.rhs.expr(item) # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs and rhs: + if lhs is not None and rhs is not None: return lhs < rhs else: return False @@ -990,7 +990,7 @@ class OpGreaterThan(Op): lhs = self.lhs.expr(item) rhs = self.rhs.expr(item) # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs and rhs: + if lhs is not None and rhs is not None: return lhs > rhs else: return False @@ -1021,7 +1021,7 @@ class OpLessThanOrEqual(Op): lhs = self.lhs.expr(item) rhs = self.rhs.expr(item) # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs and rhs: + if lhs is not None and rhs is not None: return lhs <= rhs else: return False @@ -1034,7 +1034,7 @@ class OpGreaterThanOrEqual(Op): lhs = self.lhs.expr(item) rhs = self.rhs.expr(item) # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs and rhs: + if lhs is not None and rhs is not None: return lhs >= rhs else: return False diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 333eba135..1a0865ba8 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2531,6 +2531,48 @@ def test_condition_expressions(): ) +@mock_dynamodb2 +def test_condition_expression_numerical_attribute(): + dynamodb = boto3.resource("dynamodb") + dynamodb.create_table( + TableName="my-table", + KeySchema=[{"AttributeName": "partitionKey", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "partitionKey", "AttributeType": "S"}], + ) + table = dynamodb.Table("my-table") + table.put_item(Item={"partitionKey": "pk-pos", "myAttr": 5}) + table.put_item(Item={"partitionKey": "pk-neg", "myAttr": -5}) + + # try to update the item we put in the table using numerical condition expression + # Specifically, verify that we can compare with a zero-value + # First verify that > and >= work on positive numbers + update_numerical_con_expr( + key="pk-pos", con_expr="myAttr > :zero", res="6", table=table + ) + update_numerical_con_expr( + key="pk-pos", con_expr="myAttr >= :zero", res="7", table=table + ) + # Second verify that < and <= work on negative numbers + update_numerical_con_expr( + key="pk-neg", con_expr="myAttr < :zero", res="-4", table=table + ) + update_numerical_con_expr( + key="pk-neg", con_expr="myAttr <= :zero", res="-3", table=table + ) + + +def update_numerical_con_expr(key, con_expr, res, table): + table.update_item( + Key={"partitionKey": key}, + UpdateExpression="ADD myAttr :one", + ExpressionAttributeValues={":zero": 0, ":one": 1}, + ConditionExpression=con_expr, + ) + table.get_item(Key={"partitionKey": key})["Item"]["myAttr"].should.equal( + Decimal(res) + ) + + @mock_dynamodb2 def test_condition_expression__attr_doesnt_exist(): client = boto3.client("dynamodb", region_name="us-east-1") From 7ff7ee4e8ebf621ad24cb4101c9b7069e43867c6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 22 Jan 2020 11:42:06 +0000 Subject: [PATCH 45/48] Test fix - Region must be specified --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 1a0865ba8..5a978edc0 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2533,7 +2533,7 @@ def test_condition_expressions(): @mock_dynamodb2 def test_condition_expression_numerical_attribute(): - dynamodb = boto3.resource("dynamodb") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") dynamodb.create_table( TableName="my-table", KeySchema=[{"AttributeName": "partitionKey", "KeyType": "HASH"}], From a32b3c4b597c24844e2ee3af24b48cce746bf0d0 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 22 Jan 2020 19:38:07 -0600 Subject: [PATCH 46/48] Fix SQS get_queue_attributes to allow RedrivePolicy. Closes #2682. --- moto/sqs/models.py | 1 + tests/test_sqs/test_sqs.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 40dd6ba97..8b8263e3c 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -183,6 +183,7 @@ class Queue(BaseModel): "MaximumMessageSize", "MessageRetentionPeriod", "QueueArn", + "RedrivePolicy", "ReceiveMessageWaitTimeSeconds", "VisibilityTimeout", ] diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 639d6e51c..c74c3822a 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -331,7 +331,20 @@ def test_delete_queue(): @mock_sqs def test_get_queue_attributes(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + + dlq_resp = client.create_queue(QueueName="test-dlr-queue") + dlq_arn1 = client.get_queue_attributes(QueueUrl=dlq_resp["QueueUrl"])["Attributes"][ + "QueueArn" + ] + + response = client.create_queue( + QueueName="test-queue", + Attributes={ + "RedrivePolicy": json.dumps( + {"deadLetterTargetArn": dlq_arn1, "maxReceiveCount": 2} + ), + }, + ) queue_url = response["QueueUrl"] response = client.get_queue_attributes(QueueUrl=queue_url) @@ -356,6 +369,7 @@ def test_get_queue_attributes(): "ApproximateNumberOfMessages", "MaximumMessageSize", "QueueArn", + "RedrivePolicy", "VisibilityTimeout", ], ) @@ -366,6 +380,9 @@ def test_get_queue_attributes(): "MaximumMessageSize": "65536", "QueueArn": "arn:aws:sqs:us-east-1:{}:test-queue".format(ACCOUNT_ID), "VisibilityTimeout": "30", + "RedrivePolicy": json.dumps( + {"deadLetterTargetArn": dlq_arn1, "maxReceiveCount": 2} + ), } ) From d73a548bb0b72c8ccbb70e316ae5112f2f9264c5 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 22 Jan 2020 19:45:09 -0600 Subject: [PATCH 47/48] Remove duplicate StorageClass in S3_MULTIPART_LIST_RESPONSE. --- moto/s3/responses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 71f21c8e1..c8f4e082b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1869,7 +1869,6 @@ S3_MULTIPART_LIST_RESPONSE = """ 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a webfile - STANDARD 1 {{ count }} {{ count }} From 19bf8bf76207a01a168ec842c2e4e917adc466dd Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 22 Jan 2020 20:43:34 -0600 Subject: [PATCH 48/48] Change S3 S3_ALL_BUCKETS response to return bucket creation_date in iso format. --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index c8f4e082b..a04427172 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1482,7 +1482,7 @@ S3_ALL_BUCKETS = """