From ed81e36faffda8b1ab4a5d3176fc65307fe8a34d Mon Sep 17 00:00:00 2001 From: Andrey Kislyuk Date: Wed, 22 Jan 2020 16:08:42 -0800 Subject: [PATCH 01/38] awslambda: explicitly specify json-file log driver This is analogous to #2635. --- moto/awslambda/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 38ff81fb2..e43f8e5d0 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -409,6 +409,7 @@ class LambdaFunction(BaseModel): volumes=["{}:/var/task".format(data_vol.name)], environment=env_vars, detach=True, + log_config=docker.types.LogConfig(type=docker.types.LogConfig.types.JSON), **run_kwargs ) finally: From e3906043d7a77ccb3c656088bff5e8fce2386ec0 Mon Sep 17 00:00:00 2001 From: Andrey Kislyuk Date: Wed, 22 Jan 2020 16:58:25 -0800 Subject: [PATCH 02/38] Fix linter error --- moto/awslambda/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index e43f8e5d0..4a223821b 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -394,6 +394,7 @@ class LambdaFunction(BaseModel): env_vars.update(self.environment_vars) container = output = exit_code = None + log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON) with _DockerDataVolumeContext(self) as data_vol: try: run_kwargs = ( @@ -409,7 +410,7 @@ class LambdaFunction(BaseModel): volumes=["{}:/var/task".format(data_vol.name)], environment=env_vars, detach=True, - log_config=docker.types.LogConfig(type=docker.types.LogConfig.types.JSON), + log_config=log_config, **run_kwargs ) finally: From 55a1c2fb590b333e43353a7bd01790990da07a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Matt=C3=A9?= Date: Tue, 28 Jan 2020 20:45:19 -0300 Subject: [PATCH 03/38] Support greedy resource path --- moto/apigateway/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index fd2fb7064..748a09e0f 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -556,7 +556,7 @@ class APIGatewayBackend(BaseBackend): return resource def create_resource(self, function_id, parent_resource_id, path_part): - if not re.match("^\\{?[a-zA-Z0-9._-]+\\}?$", path_part): + if not re.match("^\\{?[a-zA-Z0-9._-]+\\+?\\}?$", path_part): raise InvalidResourcePathException() api = self.get_rest_api(function_id) child = api.add_child(path=path_part, parent_id=parent_resource_id) From be8eab18e91f3b10e6b741c195e2b66cc4407ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Matt=C3=A9?= Date: Tue, 28 Jan 2020 20:56:13 -0300 Subject: [PATCH 04/38] Update InvalidResourcePathException message --- moto/apigateway/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/apigateway/exceptions.py b/moto/apigateway/exceptions.py index 434ebc467..2a306ab99 100644 --- a/moto/apigateway/exceptions.py +++ b/moto/apigateway/exceptions.py @@ -39,7 +39,7 @@ class InvalidResourcePathException(BadRequestException): def __init__(self): super(InvalidResourcePathException, self).__init__( "BadRequestException", - "Resource's path part only allow a-zA-Z0-9._- and curly braces at the beginning and the end.", + "Resource's path part only allow a-zA-Z0-9._- and curly braces at the beginning and the end and an optional plus sign before the closing brace.", ) From cf65cfc6ec3a5ae50498ea9938ba8dbe826bb8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADcio=20Matt=C3=A9?= Date: Wed, 29 Jan 2020 16:28:37 -0300 Subject: [PATCH 05/38] Update API Gateway resource name test --- tests/test_apigateway/test_apigateway.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 59c0c07f6..601aa2952 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -58,15 +58,15 @@ def test_create_resource__validate_name(): 0 ]["id"] - invalid_names = ["/users", "users/", "users/{user_id}", "us{er"] - valid_names = ["users", "{user_id}", "user_09", "good-dog"] + invalid_names = ["/users", "users/", "users/{user_id}", "us{er", "us+er"] + valid_names = ["users", "{user_id}", "{proxy+}", "user_09", "good-dog"] # All invalid names should throw an exception for name in invalid_names: with assert_raises(ClientError) as ex: client.create_resource(restApiId=api_id, parentId=root_id, pathPart=name) ex.exception.response["Error"]["Code"].should.equal("BadRequestException") ex.exception.response["Error"]["Message"].should.equal( - "Resource's path part only allow a-zA-Z0-9._- and curly braces at the beginning and the end." + "Resource's path part only allow a-zA-Z0-9._- and curly braces at the beginning and the end and an optional plus sign before the closing brace." ) # All valid names should go through for name in valid_names: From c877266f8648664045b7be9fa2519269c898b825 Mon Sep 17 00:00:00 2001 From: Brandon Bradley Date: Wed, 29 Jan 2020 16:27:56 -0600 Subject: [PATCH 06/38] fix 500 error on non-existing stack name --- moto/cloudformation/models.py | 2 ++ moto/cloudformation/responses.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 0ae5d1ae4..c05783fb4 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -677,6 +677,8 @@ class CloudFormationBackend(BaseBackend): def list_stack_resources(self, stack_name_or_id): stack = self.get_stack(stack_name_or_id) + if stack is None: + return [] return stack.stack_resources def delete_stack(self, name_or_stack_id): diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index bf68a6325..7effb03fa 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -229,6 +229,9 @@ class CloudFormationResponse(BaseResponse): stack_name_or_id = self._get_param("StackName") resources = self.cloudformation_backend.list_stack_resources(stack_name_or_id) + if not resources: + raise ValidationError(stack_name_or_id) + template = self.response_template(LIST_STACKS_RESOURCES_RESPONSE) return template.render(resources=resources) From 44024ab74b8a3a4bb300a85d0c5f679d6465d3dd Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 30 Jan 2020 22:42:27 +0100 Subject: [PATCH 07/38] Fix sqs permission handling & add more error handling --- moto/sqs/exceptions.py | 25 +++++ moto/sqs/models.py | 101 ++++++++++++++++--- tests/test_sqs/test_sqs.py | 198 +++++++++++++++++++++++++++++++++++-- 3 files changed, 303 insertions(+), 21 deletions(-) diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py index 01123d777..77d7b9fb2 100644 --- a/moto/sqs/exceptions.py +++ b/moto/sqs/exceptions.py @@ -99,3 +99,28 @@ class InvalidAttributeName(RESTError): super(InvalidAttributeName, self).__init__( "InvalidAttributeName", "Unknown Attribute {}.".format(attribute_name) ) + + +class InvalidParameterValue(RESTError): + code = 400 + + def __init__(self, message): + super(InvalidParameterValue, self).__init__("InvalidParameterValue", message) + + +class MissingParameter(RESTError): + code = 400 + + def __init__(self): + super(MissingParameter, self).__init__( + "MissingParameter", "The request must contain the parameter Actions." + ) + + +class OverLimit(RESTError): + code = 403 + + def __init__(self, count): + super(OverLimit, self).__init__( + "OverLimit", "{} Actions were found, maximum allowed is 7.".format(count) + ) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 8b8263e3c..8fbe90108 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -30,6 +30,9 @@ from .exceptions import ( BatchEntryIdsNotDistinct, TooManyEntriesInBatchRequest, InvalidAttributeName, + InvalidParameterValue, + MissingParameter, + OverLimit, ) from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID @@ -183,6 +186,7 @@ class Queue(BaseModel): "MaximumMessageSize", "MessageRetentionPeriod", "QueueArn", + "Policy", "RedrivePolicy", "ReceiveMessageWaitTimeSeconds", "VisibilityTimeout", @@ -195,6 +199,8 @@ class Queue(BaseModel): "DeleteMessage", "GetQueueAttributes", "GetQueueUrl", + "ListDeadLetterSourceQueues", + "PurgeQueue", "ReceiveMessage", "SendMessage", ) @@ -273,7 +279,7 @@ class Queue(BaseModel): if key in bool_fields: value = value == "true" - if key == "RedrivePolicy" and value is not None: + if key in ["Policy", "RedrivePolicy"] and value is not None: continue setattr(self, camelcase_to_underscores(key), value) @@ -281,6 +287,9 @@ class Queue(BaseModel): if attributes.get("RedrivePolicy", None): self._setup_dlq(attributes["RedrivePolicy"]) + if attributes.get("Policy"): + self.policy = attributes["Policy"] + self.last_modified_timestamp = now def _setup_dlq(self, policy): @@ -472,6 +481,24 @@ class Queue(BaseModel): return self.name raise UnformattedGetAttTemplateException() + @property + def policy(self): + if self._policy_json.get("Statement"): + return json.dumps(self._policy_json) + else: + return None + + @policy.setter + def policy(self, policy): + if policy: + self._policy_json = json.loads(policy) + else: + self._policy_json = { + "Version": "2012-10-17", + "Id": "{}/SQSDefaultPolicy".format(self.queue_arn), + "Statement": [], + } + class SQSBackend(BaseBackend): def __init__(self, region_name): @@ -802,25 +829,75 @@ class SQSBackend(BaseBackend): def add_permission(self, queue_name, actions, account_ids, label): queue = self.get_queue(queue_name) - if actions is None or len(actions) == 0: - raise RESTError("InvalidParameterValue", "Need at least one Action") - if account_ids is None or len(account_ids) == 0: - raise RESTError("InvalidParameterValue", "Need at least one Account ID") + if not actions: + raise MissingParameter() - if not all([item in Queue.ALLOWED_PERMISSIONS for item in actions]): - raise RESTError("InvalidParameterValue", "Invalid permissions") + if not account_ids: + raise InvalidParameterValue( + "Value [] for parameter PrincipalId is invalid. Reason: Unable to verify." + ) - queue.permissions[label] = (account_ids, actions) + count = len(actions) + if count > 7: + raise OverLimit(count) + + invalid_action = next( + (action for action in actions if action not in Queue.ALLOWED_PERMISSIONS), + None, + ) + if invalid_action: + raise InvalidParameterValue( + "Value SQS:{} for parameter ActionName is invalid. " + "Reason: Only the queue owner is allowed to invoke this action.".format( + invalid_action + ) + ) + + policy = queue._policy_json + statement = next( + ( + statement + for statement in policy["Statement"] + if statement["Sid"] == label + ), + None, + ) + if statement: + raise InvalidParameterValue( + "Value {} for parameter Label is invalid. " + "Reason: Already exists.".format(label) + ) + + principals = [ + "arn:aws:iam::{}:root".format(account_id) for account_id in account_ids + ] + actions = ["SQS:{}".format(action) for action in actions] + + statement = { + "Sid": label, + "Effect": "Allow", + "Principal": {"AWS": principals[0] if len(principals) == 1 else principals}, + "Action": actions[0] if len(actions) == 1 else actions, + "Resource": queue.queue_arn, + } + + queue._policy_json["Statement"].append(statement) def remove_permission(self, queue_name, label): queue = self.get_queue(queue_name) - if label not in queue.permissions: - raise RESTError( - "InvalidParameterValue", "Permission doesnt exist for the given label" + statements = queue._policy_json["Statement"] + statements_new = [ + statement for statement in statements if statement["Sid"] != label + ] + + if len(statements) == len(statements_new): + raise InvalidParameterValue( + "Value {} for parameter Label is invalid. " + "Reason: can't find label on existing policy.".format(label) ) - del queue.permissions[label] + queue._policy_json["Statement"] = statements_new def tag_queue(self, queue_name, tags): queue = self.get_queue(queue_name) diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 1eb511db0..93d388117 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -132,6 +132,35 @@ def test_create_queue_with_tags(): ) +@mock_sqs +def test_create_queue_with_policy(): + client = boto3.client("sqs", region_name="us-east-1") + response = client.create_queue( + QueueName="test-queue", + Attributes={ + "Policy": json.dumps( + { + "Version": "2012-10-17", + "Id": "test", + "Statement": [{"Effect": "Allow", "Principal": "*", "Action": "*"}], + } + ) + }, + ) + queue_url = response["QueueUrl"] + + response = client.get_queue_attributes( + QueueUrl=queue_url, AttributeNames=["Policy"] + ) + json.loads(response["Attributes"]["Policy"]).should.equal( + { + "Version": "2012-10-17", + "Id": "test", + "Statement": [{"Effect": "Allow", "Principal": "*", "Action": "*"}], + } + ) + + @mock_sqs def test_get_queue_url(): client = boto3.client("sqs", region_name="us-east-1") @@ -1186,18 +1215,169 @@ def test_permissions(): Actions=["SendMessage"], ) - with assert_raises(ClientError): - client.add_permission( - QueueUrl=queue_url, - Label="account2", - AWSAccountIds=["222211111111"], - Actions=["SomeRubbish"], - ) + response = client.get_queue_attributes( + QueueUrl=queue_url, AttributeNames=["Policy"] + ) + policy = json.loads(response["Attributes"]["Policy"]) + policy["Version"].should.equal("2012-10-17") + policy["Id"].should.equal( + "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo/SQSDefaultPolicy" + ) + sorted(policy["Statement"], key=lambda x: x["Sid"]).should.equal( + [ + { + "Sid": "account1", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::111111111111:root"}, + "Action": "SQS:*", + "Resource": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo", + }, + { + "Sid": "account2", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::222211111111:root"}, + "Action": "SQS:SendMessage", + "Resource": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo", + }, + ] + ) client.remove_permission(QueueUrl=queue_url, Label="account2") - with assert_raises(ClientError): - client.remove_permission(QueueUrl=queue_url, Label="non_existent") + response = client.get_queue_attributes( + QueueUrl=queue_url, AttributeNames=["Policy"] + ) + json.loads(response["Attributes"]["Policy"]).should.equal( + { + "Version": "2012-10-17", + "Id": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo/SQSDefaultPolicy", + "Statement": [ + { + "Sid": "account1", + "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::111111111111:root"}, + "Action": "SQS:*", + "Resource": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo", + }, + ], + } + ) + + +@mock_sqs +def test_add_permission_errors(): + client = boto3.client("sqs", region_name="us-east-1") + response = client.create_queue(QueueName="test-queue") + queue_url = response["QueueUrl"] + client.add_permission( + QueueUrl=queue_url, + Label="test", + AWSAccountIds=["111111111111"], + Actions=["ReceiveMessage"], + ) + + with assert_raises(ClientError) as e: + client.add_permission( + QueueUrl=queue_url, + Label="test", + AWSAccountIds=["111111111111"], + Actions=["ReceiveMessage", "SendMessage"], + ) + ex = e.exception + ex.operation_name.should.equal("AddPermission") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidParameterValue") + ex.response["Error"]["Message"].should.equal( + "Value test for parameter Label is invalid. " "Reason: Already exists." + ) + + with assert_raises(ClientError) as e: + client.add_permission( + QueueUrl=queue_url, + Label="test-2", + AWSAccountIds=["111111111111"], + Actions=["RemovePermission"], + ) + ex = e.exception + ex.operation_name.should.equal("AddPermission") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidParameterValue") + ex.response["Error"]["Message"].should.equal( + "Value SQS:RemovePermission for parameter ActionName is invalid. " + "Reason: Only the queue owner is allowed to invoke this action." + ) + + with assert_raises(ClientError) as e: + client.add_permission( + QueueUrl=queue_url, + Label="test-2", + AWSAccountIds=["111111111111"], + Actions=[], + ) + ex = e.exception + ex.operation_name.should.equal("AddPermission") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("MissingParameter") + ex.response["Error"]["Message"].should.equal( + "The request must contain the parameter Actions." + ) + + with assert_raises(ClientError) as e: + client.add_permission( + QueueUrl=queue_url, + Label="test-2", + AWSAccountIds=[], + Actions=["ReceiveMessage"], + ) + ex = e.exception + ex.operation_name.should.equal("AddPermission") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidParameterValue") + ex.response["Error"]["Message"].should.equal( + "Value [] for parameter PrincipalId is invalid. Reason: Unable to verify." + ) + + with assert_raises(ClientError) as e: + client.add_permission( + QueueUrl=queue_url, + Label="test-2", + AWSAccountIds=["111111111111"], + Actions=[ + "ChangeMessageVisibility", + "DeleteMessage", + "GetQueueAttributes", + "GetQueueUrl", + "ListDeadLetterSourceQueues", + "PurgeQueue", + "ReceiveMessage", + "SendMessage", + ], + ) + ex = e.exception + ex.operation_name.should.equal("AddPermission") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(403) + ex.response["Error"]["Code"].should.contain("OverLimit") + ex.response["Error"]["Message"].should.equal( + "8 Actions were found, maximum allowed is 7." + ) + + +@mock_sqs +def test_remove_permission_errors(): + client = boto3.client("sqs", region_name="us-east-1") + response = client.create_queue(QueueName="test-queue") + queue_url = response["QueueUrl"] + + with assert_raises(ClientError) as e: + client.remove_permission(QueueUrl=queue_url, Label="test") + ex = e.exception + ex.operation_name.should.equal("RemovePermission") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidParameterValue") + ex.response["Error"]["Message"].should.equal( + "Value test for parameter Label is invalid. " + "Reason: can't find label on existing policy." + ) @mock_sqs From b7795b7111158c3974040cf24ad1b56cfc1042c8 Mon Sep 17 00:00:00 2001 From: Brandon Bradley Date: Thu, 30 Jan 2020 16:35:19 -0600 Subject: [PATCH 08/38] test for ListStackResources --- .../test_cloudformation_stack_crud_boto3.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 40fb2d669..a3e5097d7 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -522,6 +522,16 @@ def test_boto3_list_stack_set_operations(): list_operation["Summaries"][-1]["Action"].should.equal("UPDATE") +@mock_cloudformation +def test_boto3_bad_list_stack_resources(): + cf_conn = boto3.client("cloudformation", region_name="us-east-1") + cf_conn.create_stack_set( + StackSetName="test_stack_set", TemplateBody=dummy_template_json + ) + with assert_raises(ClientError): + cf_conn.list_stack_resources(StackName="test_stack_set") + + @mock_cloudformation def test_boto3_delete_stack_set(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") From 8b3c2b66544d82b91472fe3ad792ce5c18b773d3 Mon Sep 17 00:00:00 2001 From: Brandon Bradley Date: Thu, 30 Jan 2020 17:50:21 -0600 Subject: [PATCH 09/38] fix test --- .../test_cloudformation_stack_crud_boto3.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index a3e5097d7..b7e86a1d5 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -525,9 +525,6 @@ def test_boto3_list_stack_set_operations(): @mock_cloudformation def test_boto3_bad_list_stack_resources(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") - cf_conn.create_stack_set( - StackSetName="test_stack_set", TemplateBody=dummy_template_json - ) with assert_raises(ClientError): cf_conn.list_stack_resources(StackName="test_stack_set") From 40bd4f16039d908a0f25735f6b35e8c347ef2f95 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 31 Jan 2020 17:16:42 +0100 Subject: [PATCH 10/38] Fix kms.create_key default output --- moto/kms/models.py | 14 +++++++++++++- tests/test_kms/test_kms.py | 17 +++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 22f0039b2..cceb96342 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -9,6 +9,8 @@ from boto3 import Session from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_without_milliseconds +from moto.iam.models import ACCOUNT_ID + from .utils import decrypt, encrypt, generate_key_id, generate_master_key @@ -21,11 +23,16 @@ class Key(BaseModel): self.description = description self.enabled = True self.region = region - self.account_id = "012345678912" + self.account_id = ACCOUNT_ID self.key_rotation_status = False self.deletion_date = None self.tags = tags or {} self.key_material = generate_master_key() + self.origin = "AWS_KMS" + self.key_manager = "CUSTOMER" + self.customer_master_key_spec = "SYMMETRIC_DEFAULT" + self.encryption_algorithms = ["SYMMETRIC_DEFAULT"] + self.signing_algorithms = None @property def physical_resource_id(self): @@ -43,11 +50,16 @@ class Key(BaseModel): "AWSAccountId": self.account_id, "Arn": self.arn, "CreationDate": iso_8601_datetime_without_milliseconds(datetime.now()), + "CustomerMasterKeySpec": self.customer_master_key_spec, "Description": self.description, "Enabled": self.enabled, + "EncryptionAlgorithms": self.encryption_algorithms, "KeyId": self.id, + "KeyManager": self.key_manager, "KeyUsage": self.key_usage, "KeyState": self.key_state, + "Origin": self.origin, + "SigningAlgorithms": self.signing_algorithms, } } if self.key_state == "PendingDeletion": diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 70fa68787..8c2843ee4 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -46,10 +46,23 @@ def test_create_key(): Tags=[{"TagKey": "project", "TagValue": "moto"}], ) + key["KeyMetadata"]["Arn"].should.equal( + "arn:aws:kms:us-east-1:123456789012:key/{}".format( + key["KeyMetadata"]["KeyId"] + ) + ) + key["KeyMetadata"]["AWSAccountId"].should.equal("123456789012") + key["KeyMetadata"]["CreationDate"].should.be.a(datetime) + key["KeyMetadata"]["CustomerMasterKeySpec"].should.equal("SYMMETRIC_DEFAULT") key["KeyMetadata"]["Description"].should.equal("my key") + key["KeyMetadata"]["Enabled"].should.be.ok + key["KeyMetadata"]["EncryptionAlgorithms"].should.equal(["SYMMETRIC_DEFAULT"]) + key["KeyMetadata"]["KeyId"].should_not.be.empty + key["KeyMetadata"]["KeyManager"].should.equal("CUSTOMER") + key["KeyMetadata"]["KeyState"].should.equal("Enabled") key["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") - key["KeyMetadata"]["Enabled"].should.equal(True) - key["KeyMetadata"]["CreationDate"].should.be.a(date) + key["KeyMetadata"]["Origin"].should.equal("AWS_KMS") + key["KeyMetadata"].should_not.have.key("SigningAlgorithms") @mock_kms_deprecated From 27ce0b7ab14f89ff87774bf04b4b44a44a61fca2 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Fri, 31 Jan 2020 12:49:10 -0800 Subject: [PATCH 11/38] Botocore no longer needs an older version of python-dateutil. https://github.com/boto/botocore/pull/1910 https://github.com/boto/botocore/issues/1872 https://github.com/spulec/moto/pull/2570 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d09f8fc7b..1dde71ac7 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ install_requires = [ "werkzeug", "PyYAML>=5.1", "pytz", - "python-dateutil<2.8.1,>=2.1", + "python-dateutil<3.0.0,>=2.1", "python-jose<4.0.0", "mock", "docker>=2.5.1", From 800e5ab7d2e25970515b9612f6e998d9434e76d4 Mon Sep 17 00:00:00 2001 From: Brandon Bradley Date: Sat, 1 Feb 2020 14:52:48 -0600 Subject: [PATCH 12/38] requested changes from review --- moto/cloudformation/models.py | 2 +- moto/cloudformation/responses.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index c05783fb4..b32d63b32 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -678,7 +678,7 @@ class CloudFormationBackend(BaseBackend): def list_stack_resources(self, stack_name_or_id): stack = self.get_stack(stack_name_or_id) if stack is None: - return [] + return None return stack.stack_resources def delete_stack(self, name_or_stack_id): diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 7effb03fa..77a3051fd 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -229,7 +229,7 @@ class CloudFormationResponse(BaseResponse): stack_name_or_id = self._get_param("StackName") resources = self.cloudformation_backend.list_stack_resources(stack_name_or_id) - if not resources: + if resources is None: raise ValidationError(stack_name_or_id) template = self.response_template(LIST_STACKS_RESOURCES_RESPONSE) From c36371e235a92a45dbd49aefca3cbd8d24c76fdd Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 12:47:54 -0300 Subject: [PATCH 13/38] Add failing test for database creation with iam --- tests/test_rds2/test_rds2.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 9a5a73678..6dc42fbc1 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -1689,3 +1689,18 @@ def test_create_parameter_group_with_tags(): ResourceName="arn:aws:rds:us-west-2:1234567890:pg:test" ) result["TagList"].should.equal([{"Value": "bar", "Key": "foo"}]) + + +@mock_rds2 +def test_create_database_with_iam_authentication(): + conn = boto3.client("rds", region_name="us-west-2") + + database = conn.create_db_instance( + DBInstanceIdentifier="rds", + DBInstanceClass="db.t1.micro", + Engine="postgres", + EnableIAMDatabaseAuthentication=True, + ) + + db_instance = database["DBInstance"] + db_instance["IAMDatabaseAuthenticationEnabled"].should.equal(True) From ec66670315750fd1c60df369c8e12a05f0b4498e Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 12:50:46 -0300 Subject: [PATCH 14/38] Add enable_iam_database_authentication parameter in RDS2Response --- moto/rds2/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index 7c815b2d5..cdffdd40e 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -27,6 +27,7 @@ class RDS2Response(BaseResponse): "db_subnet_group_name": self._get_param("DBSubnetGroupName"), "engine": self._get_param("Engine"), "engine_version": self._get_param("EngineVersion"), + "enable_iam_database_authentication": self._get_bool_param("EnableIAMDatabaseAuthentication"), "license_model": self._get_param("LicenseModel"), "iops": self._get_int_param("Iops"), "kms_key_id": self._get_param("KmsKeyId"), From dfd21187e143e58f412d03a48c05529600f99cbe Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 12:55:05 -0300 Subject: [PATCH 15/38] Change iam_database_authentication_enabled to enabled_iam_database_authentication in accordance with aws docs --- moto/rds2/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index d2aa24a20..68acef0a0 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -130,7 +130,9 @@ class Database(BaseModel): if not self.option_group_name and self.engine in self.default_option_groups: self.option_group_name = self.default_option_groups[self.engine] self.character_set_name = kwargs.get("character_set_name", None) - self.iam_database_authentication_enabled = False + self.enable_iam_database_authentication = kwargs.get( + "enable_iam_database_authentication", False + ) self.dbi_resource_id = "db-M5ENSHXFPU6XHZ4G4ZEI5QIO2U" self.tags = kwargs.get("tags", []) From 51e787fba6b24af7284394126ae5f1ced96a4f35 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 12:57:16 -0300 Subject: [PATCH 16/38] Add enable_iam_database_authentication in 'to_xml' method --- moto/rds2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 68acef0a0..aae708cdd 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -216,7 +216,7 @@ class Database(BaseModel): {{ database.source_db_identifier }} {% endif %} {{ database.engine }} - {{database.iam_database_authentication_enabled }} + {{database.enable_iam_database_authentication|lower }} {{ database.license_model }} {{ database.engine_version }} From eb0687eeaa3fb544c139c0dec43b2b6b828dbaa8 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 13:08:13 -0300 Subject: [PATCH 17/38] Add failing test for EnableIAMDatabaseAuthentication snapshot --- tests/test_rds2/test_rds2.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 6dc42fbc1..7e6670e9b 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -1704,3 +1704,21 @@ def test_create_database_with_iam_authentication(): db_instance = database["DBInstance"] db_instance["IAMDatabaseAuthenticationEnabled"].should.equal(True) + + +@mock_rds2 +def test_create_db_snapshot_with_iam_authentication(): + conn = boto3.client("rds", region_name="us-west-2") + + conn.create_db_instance( + DBInstanceIdentifier="rds", + DBInstanceClass="db.t1.micro", + Engine="postgres", + EnableIAMDatabaseAuthentication=True, + ) + + snapshot = conn.create_db_snapshot( + DBInstanceIdentifier="rds", DBSnapshotIdentifier="snapshot" + ).get("DBSnapshot") + + snapshot.get("IAMDatabaseAuthenticationEnabled").should.equal(True) From 06e4cafd20aa5140d89a952a97b35c8833999194 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 13:10:04 -0300 Subject: [PATCH 18/38] Add enable_iam_database_authentication variable into snapshot 'to_xml' --- moto/rds2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index aae708cdd..963af1c63 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -544,7 +544,7 @@ class Snapshot(BaseModel): {{ database.kms_key_id }} {{ snapshot.snapshot_arn }} - false + {{ database.enable_iam_database_authentication|lower }} """ ) return template.render(snapshot=self, database=self.database) From 9f8388e4021f082b2199d698d56da941c551dd53 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 13:19:50 -0300 Subject: [PATCH 19/38] Change test name in favor of abbreviation --- tests/test_rds2/test_rds2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 7e6670e9b..e93ff43e9 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -1692,7 +1692,7 @@ def test_create_parameter_group_with_tags(): @mock_rds2 -def test_create_database_with_iam_authentication(): +def test_create_db_with_iam_authentication(): conn = boto3.client("rds", region_name="us-west-2") database = conn.create_db_instance( From f0509276d892258aeee1f869b4ac1f08120feaa5 Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sun, 2 Feb 2020 13:46:01 -0300 Subject: [PATCH 20/38] Apply black in responses.py --- moto/rds2/responses.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index cdffdd40e..b63e9f8b8 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -27,7 +27,9 @@ class RDS2Response(BaseResponse): "db_subnet_group_name": self._get_param("DBSubnetGroupName"), "engine": self._get_param("Engine"), "engine_version": self._get_param("EngineVersion"), - "enable_iam_database_authentication": self._get_bool_param("EnableIAMDatabaseAuthentication"), + "enable_iam_database_authentication": self._get_bool_param( + "EnableIAMDatabaseAuthentication" + ), "license_model": self._get_param("LicenseModel"), "iops": self._get_int_param("Iops"), "kms_key_id": self._get_param("KmsKeyId"), From c9995412b525769303c24171c7d9c54dd6b5098a Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 3 Feb 2020 10:21:22 -0600 Subject: [PATCH 21/38] add support for apigateway fields with default values including apiKeySource, endpointConfiguration, and tags --- moto/apigateway/models.py | 29 +++++- moto/apigateway/responses.py | 43 ++++++++- tests/test_apigateway/test_apigateway.py | 117 ++++++++++++++++++++++- 3 files changed, 184 insertions(+), 5 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index fd2fb7064..234d6636c 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -394,12 +394,17 @@ class UsagePlanKey(BaseModel, dict): class RestAPI(BaseModel): - def __init__(self, id, region_name, name, description): + def __init__(self, id, region_name, name, description, **kwargs): self.id = id self.region_name = region_name self.name = name self.description = description self.create_date = int(time.time()) + self.api_key_source = kwargs.get("api_key_source") or "HEADER" + self.endpoint_configuration = kwargs.get("endpoint_configuration") or { + "types": ["EDGE"] + } + self.tags = kwargs.get("tags") or {} self.deployments = {} self.stages = {} @@ -416,6 +421,9 @@ class RestAPI(BaseModel): "name": self.name, "description": self.description, "createdDate": int(time.time()), + "apiKeySource": self.api_key_source, + "endpointConfiguration": self.endpoint_configuration, + "tags": self.tags, } def add_child(self, path, parent_id=None): @@ -529,9 +537,24 @@ class APIGatewayBackend(BaseBackend): self.__dict__ = {} self.__init__(region_name) - def create_rest_api(self, name, description): + def create_rest_api( + self, + name, + description, + api_key_source=None, + endpoint_configuration=None, + tags=None, + ): api_id = create_id() - rest_api = RestAPI(api_id, self.region_name, name, description) + rest_api = RestAPI( + api_id, + self.region_name, + name, + description, + api_key_source=api_key_source, + endpoint_configuration=endpoint_configuration, + tags=tags, + ) self.apis[api_id] = rest_api return rest_api diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index c4c7b403e..e10d670c5 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -12,6 +12,9 @@ from .exceptions import ( ApiKeyAlreadyExists, ) +API_KEY_SOURCES = ["AUTHORIZER", "HEADER"] +ENDPOINT_CONFIGURATION_TYPES = ["PRIVATE", "EDGE", "REGIONAL"] + class APIGatewayResponse(BaseResponse): def error(self, type_, message, status=400): @@ -45,7 +48,45 @@ class APIGatewayResponse(BaseResponse): elif self.method == "POST": name = self._get_param("name") description = self._get_param("description") - rest_api = self.backend.create_rest_api(name, description) + api_key_source = self._get_param("apiKeySource") + endpoint_configuration = self._get_param("endpointConfiguration") + tags = self._get_param("tags") + + # Param validation + if api_key_source and api_key_source not in API_KEY_SOURCES: + return self.error( + "ValidationException", + ( + "1 validation error detected: " + "Value '{api_key_source}' at 'createRestApiInput.apiKeySource' failed " + "to satisfy constraint: Member must satisfy enum value set: " + "[AUTHORIZER, HEADER]" + ).format(api_key_source=api_key_source), + ) + + if endpoint_configuration and "types" in endpoint_configuration: + invalid_types = list( + set(endpoint_configuration["types"]) + - set(ENDPOINT_CONFIGURATION_TYPES) + ) + if invalid_types: + return self.error( + "ValidationException", + ( + "1 validation error detected: Value '{endpoint_type}' " + "at 'createRestApiInput.endpointConfiguration.types' failed " + "to satisfy constraint: Member must satisfy enum value set: " + "[PRIVATE, EDGE, REGIONAL]" + ).format(endpoint_type=invalid_types[0]), + ) + + rest_api = self.backend.create_rest_api( + name, + description, + api_key_source=api_key_source, + endpoint_configuration=endpoint_configuration, + tags=tags, + ) return 200, {}, json.dumps(rest_api.to_dict()) def restapis_individual(self, request, full_url, headers): diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 59c0c07f6..37bcc97f7 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -26,7 +26,14 @@ def test_create_and_get_rest_api(): response.pop("ResponseMetadata") response.pop("createdDate") response.should.equal( - {"id": api_id, "name": "my_api", "description": "this is my api"} + { + "id": api_id, + "name": "my_api", + "description": "this is my api", + "apiKeySource": "HEADER", + "endpointConfiguration": {"types": ["EDGE"]}, + "tags": {}, + } ) @@ -47,6 +54,114 @@ def test_list_and_delete_apis(): len(response["items"]).should.equal(1) +@mock_apigateway +def test_create_rest_api_with_tags(): + client = boto3.client("apigateway", region_name="us-west-2") + + response = client.create_rest_api( + name="my_api", description="this is my api", tags={"MY_TAG1": "MY_VALUE1"} + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + + assert "tags" in response + response["tags"].should.equal({"MY_TAG1": "MY_VALUE1"}) + + +@mock_apigateway +def test_create_rest_api_invalid_apikeysource(): + client = boto3.client("apigateway", region_name="us-west-2") + + with assert_raises(ClientError) as ex: + client.create_rest_api( + name="my_api", + description="this is my api", + apiKeySource="not a valid api key source", + ) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + + +@mock_apigateway +def test_create_rest_api_valid_apikeysources(): + client = boto3.client("apigateway", region_name="us-west-2") + + # 1. test creating rest api with HEADER apiKeySource + response = client.create_rest_api( + name="my_api", description="this is my api", apiKeySource="HEADER", + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + response["apiKeySource"].should.equal("HEADER") + + # 2. test creating rest api with AUTHORIZER apiKeySource + response = client.create_rest_api( + name="my_api2", description="this is my api", apiKeySource="AUTHORIZER", + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + response["apiKeySource"].should.equal("AUTHORIZER") + + +@mock_apigateway +def test_create_rest_api_invalid_endpointconfiguration(): + client = boto3.client("apigateway", region_name="us-west-2") + + with assert_raises(ClientError) as ex: + client.create_rest_api( + name="my_api", + description="this is my api", + endpointConfiguration={"types": ["INVALID"]}, + ) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + + +@mock_apigateway +def test_create_rest_api_valid_endpointconfigurations(): + client = boto3.client("apigateway", region_name="us-west-2") + + # 1. test creating rest api with PRIVATE endpointConfiguration + response = client.create_rest_api( + name="my_api", + description="this is my api", + endpointConfiguration={"types": ["PRIVATE"]}, + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + response["endpointConfiguration"].should.equal( + {"types": ["PRIVATE"],} + ) + + # 2. test creating rest api with REGIONAL endpointConfiguration + response = client.create_rest_api( + name="my_api2", + description="this is my api", + endpointConfiguration={"types": ["REGIONAL"]}, + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + response["endpointConfiguration"].should.equal( + {"types": ["REGIONAL"],} + ) + + # 3. test creating rest api with EDGE endpointConfiguration + response = client.create_rest_api( + name="my_api3", + description="this is my api", + endpointConfiguration={"types": ["EDGE"]}, + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + response["endpointConfiguration"].should.equal( + {"types": ["EDGE"],} + ) + + @mock_apigateway def test_create_resource__validate_name(): client = boto3.client("apigateway", region_name="us-west-2") From 6d64b12b4117b4b85af92f7aeca8a78e49fa9bc8 Mon Sep 17 00:00:00 2001 From: rossjones Date: Tue, 4 Feb 2020 10:02:43 +0000 Subject: [PATCH 22/38] Remove ResourceWarnings when loading AMIS and INSTANCE_TYPES When loading AMIS and INSTANCE_TYPES in moto.ec2.models a file handle is potentially leaked when loading the JSON. This results in a ResourceWarning which is a bit of unnecessary noise. Rather than pass a call to open() to json.load() this instead uses a context-manager in a small private helper function. This fixes https://github.com/spulec/moto/issues/2620 --- moto/ec2/models.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 93a350914..a0c886087 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -139,17 +139,22 @@ from .utils import ( rsa_public_key_fingerprint, ) -INSTANCE_TYPES = json.load( - open(resource_filename(__name__, "resources/instance_types.json"), "r") + +def _load_resource(filename): + with open(filename, "r") as f: + return json.load(f) + + +INSTANCE_TYPES = _load_resource( + resource_filename(__name__, "resources/instance_types.json") ) -AMIS = json.load( - open( - os.environ.get("MOTO_AMIS_PATH") - or resource_filename(__name__, "resources/amis.json"), - "r", - ) + +AMIS = _load_resource( + os.environ.get("MOTO_AMIS_PATH") + or resource_filename(__name__, "resources/amis.json"), ) + OWNER_ID = "111122223333" From 4f0c06ca5322274e9d7c1744f86115b354755e84 Mon Sep 17 00:00:00 2001 From: Jay Udey Date: Tue, 4 Feb 2020 14:04:45 -0600 Subject: [PATCH 23/38] handle map or list parameters --- moto/ses/models.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/moto/ses/models.py b/moto/ses/models.py index eacdd8458..4b6ce52c8 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -148,11 +148,15 @@ class SESBackend(BaseBackend): def __type_of_message__(self, destinations): """Checks the destination for any special address that could indicate delivery, complaint or bounce like in SES simulator""" - alladdress = ( - destinations.get("ToAddresses", []) - + destinations.get("CcAddresses", []) - + destinations.get("BccAddresses", []) - ) + if isinstance(destinations, list): + alladdress = destinations + else: + alladdress = ( + destinations.get("ToAddresses", []) + + destinations.get("CcAddresses", []) + + destinations.get("BccAddresses", []) + ) + for addr in alladdress: if SESFeedback.SUCCESS_ADDR in addr: return SESFeedback.DELIVERY From bb64258a8f2fd994d40405f76440c47f12012ef9 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Tue, 4 Feb 2020 17:06:55 -0800 Subject: [PATCH 24/38] Fixed issue with Lambda invoke via ARN - Fixed an issue where Lambda invokes via an ARN was hitting real AWS. --- moto/awslambda/responses.py | 3 ++- moto/awslambda/urls.py | 1 + tests/test_awslambda/test_lambda.py | 37 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index e1713ce52..bac670b8e 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -176,7 +176,8 @@ class LambdaResponse(BaseResponse): def _invoke(self, request, full_url): response_headers = {} - function_name = self.path.rsplit("/", 2)[-2] + # URL Decode in case it's a ARN: + function_name = unquote(self.path.rsplit("/", 2)[-2]) qualifier = self._get_param("qualifier") response_header, payload = self.lambda_backend.invoke( diff --git a/moto/awslambda/urls.py b/moto/awslambda/urls.py index 6c9b736a6..c25e58dba 100644 --- a/moto/awslambda/urls.py +++ b/moto/awslambda/urls.py @@ -12,6 +12,7 @@ url_paths = { r"{0}/(?P[^/]+)/event-source-mappings/?$": response.event_source_mappings, r"{0}/(?P[^/]+)/event-source-mappings/(?P[\w_-]+)/?$": response.event_source_mapping, r"{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invocations/?$": response.invoke, + r"{0}/(?P[^/]+)/functions/(?P.+)/invocations/?$": response.invoke, r"{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invoke-async/?$": response.invoke_async, r"{0}/(?P[^/]+)/tags/(?P.+)": response.tag, r"{0}/(?P[^/]+)/functions/(?P[\w_-]+)/policy/(?P[\w_-]+)$": response.policy, diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index dfd6431e7..4db13d220 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -124,6 +124,43 @@ def test_invoke_requestresponse_function(): json.loads(payload).should.equal(in_data) +@mock_lambda +def test_invoke_requestresponse_function_with_arn(): + from moto.awslambda.models import ACCOUNT_ID + + conn = boto3.client("lambda", "us-west-2") + conn.create_function( + FunctionName="testFunction", + Runtime="python2.7", + Role=get_role_name(), + Handler="lambda_function.lambda_handler", + Code={"ZipFile": get_test_zip_file1()}, + Description="test lambda function", + Timeout=3, + MemorySize=128, + Publish=True, + ) + + in_data = {"msg": "So long and thanks for all the fish"} + success_result = conn.invoke( + FunctionName="arn:aws:lambda:us-west-2:{}:function:testFunction".format( + ACCOUNT_ID + ), + InvocationType="RequestResponse", + Payload=json.dumps(in_data), + ) + + success_result["StatusCode"].should.equal(202) + result_obj = json.loads( + base64.b64decode(success_result["LogResult"]).decode("utf-8") + ) + + result_obj.should.equal(in_data) + + payload = success_result["Payload"].read().decode("utf-8") + json.loads(payload).should.equal(in_data) + + @mock_lambda def test_invoke_event_function(): conn = boto3.client("lambda", "us-west-2") From 4bae0339c2f09b84639c64a7f7776bbc03aa87e5 Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Wed, 5 Feb 2020 12:03:24 +0100 Subject: [PATCH 25/38] Add Filter parameter for cognitoidp list_users() --- moto/cognitoidp/responses.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index 80247b076..a170b7541 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -279,9 +279,13 @@ class CognitoIdpResponse(BaseResponse): user_pool_id = self._get_param("UserPoolId") limit = self._get_param("Limit") token = self._get_param("PaginationToken") + filt = self._get_param("Filter") users, token = cognitoidp_backends[self.region].list_users( user_pool_id, limit=limit, pagination_token=token ) + if filt: + name, value = filt.replace('"', '').split('=') + users = [user for user in users for attribute in user.attributes if attribute['Name'] == name and attribute['Value'] == value] response = {"Users": [user.to_json(extended=True) for user in users]} if token: response["PaginationToken"] = str(token) From 8115dd2d1b0b6b89a8ceba50d8a7edc309f49e52 Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Wed, 5 Feb 2020 12:03:33 +0100 Subject: [PATCH 26/38] Add test --- tests/test_cognitoidp/test_cognitoidp.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 6a13683f0..27a6841f4 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -958,6 +958,15 @@ def test_list_users(): result["Users"].should.have.length_of(1) result["Users"][0]["Username"].should.equal(username) + username_bis = str(uuid.uuid4()) + conn.admin_create_user( + UserPoolId=user_pool_id, Username=username_bis, + UserAttributes=[{'Name': 'phone_number', 'Value': '+33666666666'}] + ) + result = conn.list_users(UserPoolId=user_pool_id, Filter='phone_number="+33666666666') + result["Users"].should.have.length_of(1) + result["Users"][0]["Username"].should.equal(username_bis) + @mock_cognitoidp def test_list_users_returns_limit_items(): From d8d057711dcf88a526b7bb454fa21e6050f122b4 Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Wed, 5 Feb 2020 14:19:08 +0100 Subject: [PATCH 27/38] Change from black linter --- moto/cognitoidp/responses.py | 9 +++++++-- tests/test_cognitoidp/test_cognitoidp.py | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index a170b7541..fa3b7b0b5 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -284,8 +284,13 @@ class CognitoIdpResponse(BaseResponse): user_pool_id, limit=limit, pagination_token=token ) if filt: - name, value = filt.replace('"', '').split('=') - users = [user for user in users for attribute in user.attributes if attribute['Name'] == name and attribute['Value'] == value] + name, value = filt.replace('"', "").split("=") + users = [ + user + for user in users + for attribute in user.attributes + if attribute["Name"] == name and attribute["Value"] == value + ] response = {"Users": [user.to_json(extended=True) for user in users]} if token: response["PaginationToken"] = str(token) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 27a6841f4..2f7ed11e5 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -960,10 +960,13 @@ def test_list_users(): username_bis = str(uuid.uuid4()) conn.admin_create_user( - UserPoolId=user_pool_id, Username=username_bis, - UserAttributes=[{'Name': 'phone_number', 'Value': '+33666666666'}] + UserPoolId=user_pool_id, + Username=username_bis, + UserAttributes=[{"Name": "phone_number", "Value": "+33666666666"}], + ) + result = conn.list_users( + UserPoolId=user_pool_id, Filter='phone_number="+33666666666' ) - result = conn.list_users(UserPoolId=user_pool_id, Filter='phone_number="+33666666666') result["Users"].should.have.length_of(1) result["Users"][0]["Username"].should.equal(username_bis) From 1321943d60eaab533396fa32d30a831b34b4474e Mon Sep 17 00:00:00 2001 From: Jay Udey Date: Wed, 5 Feb 2020 09:03:45 -0600 Subject: [PATCH 28/38] add test verifying solution --- tests/test_ses/test_ses_sns_boto3.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/test_ses/test_ses_sns_boto3.py b/tests/test_ses/test_ses_sns_boto3.py index fc58d88aa..43d4000bf 100644 --- a/tests/test_ses/test_ses_sns_boto3.py +++ b/tests/test_ses/test_ses_sns_boto3.py @@ -40,6 +40,8 @@ def __setup_feedback_env__( ) # Verify SES domain ses_conn.verify_domain_identity(Domain=domain) + # Specify email address to allow for raw e-mails to be processed + ses_conn.verify_email_identity(EmailAddress="test@example.com") # Setup SES notification topic if expected_msg is not None: ses_conn.set_identity_notification_topic( @@ -47,7 +49,7 @@ def __setup_feedback_env__( ) -def __test_sns_feedback__(addr, expected_msg): +def __test_sns_feedback__(addr, expected_msg, raw_email=False): region_name = "us-east-1" ses_conn = boto3.client("ses", region_name=region_name) sns_conn = boto3.client("sns", region_name=region_name) @@ -73,7 +75,18 @@ def __test_sns_feedback__(addr, expected_msg): "Body": {"Text": {"Data": "test body"}}, }, ) - ses_conn.send_email(**kwargs) + if raw_email: + kwargs.pop("Message") + kwargs.pop("Destination") + kwargs.update( + { + "Destinations": [addr + "@" + domain], + "RawMessage": {"Data": bytearray("raw_email", "utf-8")}, + } + ) + ses_conn.send_raw_email(**kwargs) + else: + ses_conn.send_email(**kwargs) # Wait for messages in the queues queue = sqs_conn.get_queue_by_name(QueueName=queue) @@ -112,3 +125,12 @@ def test_sns_feedback_complaint(): @mock_ses def test_sns_feedback_delivery(): __test_sns_feedback__(SESFeedback.SUCCESS_ADDR, SESFeedback.DELIVERY) + + +@mock_sqs +@mock_sns +@mock_ses +def test_sns_feedback_delivery_raw_email(): + __test_sns_feedback__( + SESFeedback.SUCCESS_ADDR, SESFeedback.DELIVERY, raw_email=True + ) From 14ebf29a61119649b00469ab6400948d603cb0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Tomak?= Date: Thu, 6 Feb 2020 11:49:41 +0100 Subject: [PATCH 29/38] Add UpdateOrganizationalUnit endpoint to Organizations API --- moto/organizations/models.py | 5 +++++ moto/organizations/responses.py | 5 +++++ .../test_organizations_boto3.py | 17 +++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 42e4dd00a..9be129fa7 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -222,6 +222,11 @@ class OrganizationsBackend(BaseBackend): self.attach_policy(PolicyId=utils.DEFAULT_POLICY_ID, TargetId=new_ou.id) return new_ou.describe() + def update_organizational_unit(self, **kwargs): + ou = self.get_organizational_unit_by_id(kwargs["OrganizationalUnitId"]) + ou.name = kwargs["Name"] + return ou.describe() + def get_organizational_unit_by_id(self, ou_id): ou = next((ou for ou in self.ou if ou.id == ou_id), None) if ou is None: diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 7c42eb4ec..ba7dd4453 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -36,6 +36,11 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.create_organizational_unit(**self.request_params) ) + def update_organizational_unit(self): + return json.dumps( + self.organizations_backend.update_organizational_unit(**self.request_params) + ) + def describe_organizational_unit(self): return json.dumps( self.organizations_backend.describe_organizational_unit( diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index dd79ae787..ab3ddf671 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -713,3 +713,20 @@ def test_untag_resource_errors(): ex.response["Error"]["Message"].should.equal( "You provided a value that does not match the required pattern." ) + + +@mock_organizations +def test_update_organizational_unit(): + client = boto3.client("organizations", region_name="us-east-1") + org = client.create_organization(FeatureSet="ALL")["Organization"] + root_id = client.list_roots()["Roots"][0]["Id"] + ou_name = "ou01" + response = client.create_organizational_unit(ParentId=root_id, Name=ou_name) + validate_organizational_unit(org, response) + response["OrganizationalUnit"]["Name"].should.equal(ou_name) + new_ou_name = "ou02" + response = client.update_organizational_unit( + OrganizationalUnitId=response["OrganizationalUnit"]["Id"], Name=new_ou_name + ) + validate_organizational_unit(org, response) + response["OrganizationalUnit"]["Name"].should.equal(new_ou_name) From fc9eab25919eea0759e1b3146ad111532d1ddfa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Tomak?= Date: Thu, 6 Feb 2020 12:38:37 +0100 Subject: [PATCH 30/38] Raise DuplicateOrganizationalUnitException Calling UpdateOrganizationalUnit with name that already exists should raise proper error. --- moto/organizations/exceptions.py | 10 +++++++++ moto/organizations/models.py | 8 ++++++- .../test_organizations_boto3.py | 21 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/moto/organizations/exceptions.py b/moto/organizations/exceptions.py index 01b98da7e..b40908862 100644 --- a/moto/organizations/exceptions.py +++ b/moto/organizations/exceptions.py @@ -10,3 +10,13 @@ class InvalidInputException(JsonRESTError): "InvalidInputException", "You provided a value that does not match the required pattern.", ) + + +class DuplicateOrganizationalUnitException(JsonRESTError): + code = 400 + + def __init__(self): + super(DuplicateOrganizationalUnitException, self).__init__( + "DuplicateOrganizationalUnitException", + "An OU with the same name already exists.", + ) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 9be129fa7..0db069f9a 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -8,7 +8,10 @@ from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError from moto.core.utils import unix_time from moto.organizations import utils -from moto.organizations.exceptions import InvalidInputException +from moto.organizations.exceptions import ( + InvalidInputException, + DuplicateOrganizationalUnitException, +) class FakeOrganization(BaseModel): @@ -223,6 +226,9 @@ class OrganizationsBackend(BaseBackend): return new_ou.describe() def update_organizational_unit(self, **kwargs): + for ou in self.ou: + if ou.name == kwargs["Name"]: + raise DuplicateOrganizationalUnitException ou = self.get_organizational_unit_by_id(kwargs["OrganizationalUnitId"]) ou.name = kwargs["Name"] return ou.describe() diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index ab3ddf671..876e83712 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -730,3 +730,24 @@ def test_update_organizational_unit(): ) validate_organizational_unit(org, response) response["OrganizationalUnit"]["Name"].should.equal(new_ou_name) + + +@mock_organizations +def test_update_organizational_unit_duplicate_error(): + client = boto3.client("organizations", region_name="us-east-1") + org = client.create_organization(FeatureSet="ALL")["Organization"] + root_id = client.list_roots()["Roots"][0]["Id"] + ou_name = "ou01" + response = client.create_organizational_unit(ParentId=root_id, Name=ou_name) + validate_organizational_unit(org, response) + response["OrganizationalUnit"]["Name"].should.equal(ou_name) + with assert_raises(ClientError) as e: + client.update_organizational_unit( + OrganizationalUnitId=response["OrganizationalUnit"]["Id"], Name=ou_name + ) + exc = e.exception + exc.operation_name.should.equal("UpdateOrganizationalUnit") + exc.response["Error"]["Code"].should.contain("DuplicateOrganizationalUnitException") + exc.response["Error"]["Message"].should.equal( + "An OU with the same name already exists." + ) From 5d050444915ab0b8798e9895f99e8f6bbdbccd6c Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 6 Feb 2020 17:57:00 +0100 Subject: [PATCH 31/38] Add CustomerMasterKeySpec parameter handling --- moto/kms/models.py | 43 ++++++++++++++++++++++++++++----- moto/kms/responses.py | 3 ++- tests/test_kms/test_kms.py | 47 ++++++++++++++++++++++++++++++++++++ tests/test_kms/test_utils.py | 8 +++--- 4 files changed, 90 insertions(+), 11 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index cceb96342..1015aa72a 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -15,7 +15,7 @@ from .utils import decrypt, encrypt, generate_key_id, generate_master_key class Key(BaseModel): - def __init__(self, policy, key_usage, description, tags, region): + def __init__(self, policy, key_usage, customer_master_key_spec, description, tags, region): self.id = generate_key_id() self.policy = policy self.key_usage = key_usage @@ -30,9 +30,7 @@ class Key(BaseModel): self.key_material = generate_master_key() self.origin = "AWS_KMS" self.key_manager = "CUSTOMER" - self.customer_master_key_spec = "SYMMETRIC_DEFAULT" - self.encryption_algorithms = ["SYMMETRIC_DEFAULT"] - self.signing_algorithms = None + self.customer_master_key_spec = customer_master_key_spec or "SYMMETRIC_DEFAULT" @property def physical_resource_id(self): @@ -44,6 +42,38 @@ class Key(BaseModel): self.region, self.account_id, self.id ) + @property + def encryption_algorithms(self): + if self.key_usage == "SIGN_VERIFY": + return None + elif self.customer_master_key_spec == "SYMMETRIC_DEFAULT": + return ["SYMMETRIC_DEFAULT"] + else: + return [ + "RSAES_OAEP_SHA_1", + "RSAES_OAEP_SHA_256" + ] + + @property + def signing_algorithms(self): + if self.key_usage == "ENCRYPT_DECRYPT": + return None + elif self.customer_master_key_spec in ["ECC_NIST_P256", "ECC_SECG_P256K1"]: + return ["ECDSA_SHA_256"] + elif self.customer_master_key_spec == "ECC_NIST_P384": + return ["ECDSA_SHA_384"] + elif self.customer_master_key_spec == "ECC_NIST_P521": + return ["ECDSA_SHA_512"] + else: + return [ + "RSASSA_PKCS1_V1_5_SHA_256", + "RSASSA_PKCS1_V1_5_SHA_384", + "RSASSA_PKCS1_V1_5_SHA_512", + "RSASSA_PSS_SHA_256", + "RSASSA_PSS_SHA_384", + "RSASSA_PSS_SHA_512" + ] + def to_dict(self): key_dict = { "KeyMetadata": { @@ -81,6 +111,7 @@ class Key(BaseModel): key = kms_backend.create_key( policy=properties["KeyPolicy"], key_usage="ENCRYPT_DECRYPT", + customer_master_key_spec="SYMMETRIC_DEFAULT", description=properties["Description"], tags=properties.get("Tags"), region=region_name, @@ -102,8 +133,8 @@ class KmsBackend(BaseBackend): self.keys = {} self.key_to_aliases = defaultdict(set) - def create_key(self, policy, key_usage, description, tags, region): - key = Key(policy, key_usage, description, tags, region) + def create_key(self, policy, key_usage, customer_master_key_spec, description, tags, region): + key = Key(policy, key_usage, customer_master_key_spec, description, tags, region) self.keys[key.id] = key return key diff --git a/moto/kms/responses.py b/moto/kms/responses.py index d3a9726e1..15b990bbb 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -118,11 +118,12 @@ class KmsResponse(BaseResponse): """https://docs.aws.amazon.com/kms/latest/APIReference/API_CreateKey.html""" policy = self.parameters.get("Policy") key_usage = self.parameters.get("KeyUsage") + customer_master_key_spec = self.parameters.get("CustomerMasterKeySpec") description = self.parameters.get("Description") tags = self.parameters.get("Tags") key = self.kms_backend.create_key( - policy, key_usage, description, tags, self.region + policy, key_usage, customer_master_key_spec, description, tags, self.region ) return json.dumps(key.to_dict()) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 8c2843ee4..c5a49b974 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -64,6 +64,53 @@ def test_create_key(): key["KeyMetadata"]["Origin"].should.equal("AWS_KMS") key["KeyMetadata"].should_not.have.key("SigningAlgorithms") + key = conn.create_key( + KeyUsage = "ENCRYPT_DECRYPT", + CustomerMasterKeySpec = 'RSA_2048', + ) + + sorted(key["KeyMetadata"]["EncryptionAlgorithms"]).should.equal(["RSAES_OAEP_SHA_1", "RSAES_OAEP_SHA_256"]) + key["KeyMetadata"].should_not.have.key("SigningAlgorithms") + + key = conn.create_key( + KeyUsage = "SIGN_VERIFY", + CustomerMasterKeySpec = 'RSA_2048', + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + sorted(key["KeyMetadata"]["SigningAlgorithms"]).should.equal([ + "RSASSA_PKCS1_V1_5_SHA_256", + "RSASSA_PKCS1_V1_5_SHA_384", + "RSASSA_PKCS1_V1_5_SHA_512", + "RSASSA_PSS_SHA_256", + "RSASSA_PSS_SHA_384", + "RSASSA_PSS_SHA_512" + ]) + + key = conn.create_key( + KeyUsage = "SIGN_VERIFY", + CustomerMasterKeySpec = 'ECC_SECG_P256K1', + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_256"]) + + key = conn.create_key( + KeyUsage = "SIGN_VERIFY", + CustomerMasterKeySpec = 'ECC_NIST_P384', + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_384"]) + + key = conn.create_key( + KeyUsage = "SIGN_VERIFY", + CustomerMasterKeySpec = 'ECC_NIST_P521', + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_512"]) + @mock_kms_deprecated def test_describe_key(): diff --git a/tests/test_kms/test_utils.py b/tests/test_kms/test_utils.py index f5478e0ef..4c84ed127 100644 --- a/tests/test_kms/test_utils.py +++ b/tests/test_kms/test_utils.py @@ -102,7 +102,7 @@ def test_deserialize_ciphertext_blob(raw, serialized): @parameterized(((ec[0],) for ec in ENCRYPTION_CONTEXT_VECTORS)) def test_encrypt_decrypt_cycle(encryption_context): plaintext = b"some secret plaintext" - master_key = Key("nop", "nop", "nop", [], "nop") + master_key = Key("nop", "nop", "nop", "nop", [], "nop") master_key_map = {master_key.id: master_key} ciphertext_blob = encrypt( @@ -133,7 +133,7 @@ def test_encrypt_unknown_key_id(): def test_decrypt_invalid_ciphertext_format(): - master_key = Key("nop", "nop", "nop", [], "nop") + master_key = Key("nop", "nop", "nop", "nop", [], "nop") master_key_map = {master_key.id: master_key} with assert_raises(InvalidCiphertextException): @@ -153,7 +153,7 @@ def test_decrypt_unknwown_key_id(): def test_decrypt_invalid_ciphertext(): - master_key = Key("nop", "nop", "nop", [], "nop") + master_key = Key("nop", "nop", "nop", "nop", [], "nop") master_key_map = {master_key.id: master_key} ciphertext_blob = ( master_key.id.encode("utf-8") + b"123456789012" @@ -171,7 +171,7 @@ def test_decrypt_invalid_ciphertext(): def test_decrypt_invalid_encryption_context(): plaintext = b"some secret plaintext" - master_key = Key("nop", "nop", "nop", [], "nop") + master_key = Key("nop", "nop", "nop", "nop", [], "nop") master_key_map = {master_key.id: master_key} ciphertext_blob = encrypt( From b4c9b76ca958223f54c6b8cc22b85bc100f48c18 Mon Sep 17 00:00:00 2001 From: Terry Griffin <“griffint61@users.noreply.github.com”> Date: Thu, 6 Feb 2020 15:26:20 -0800 Subject: [PATCH 32/38] Added 'x-amzn-ErrorType' in return header from lambda:get_function for missing function --- moto/awslambda/responses.py | 2 +- tests/test_awslambda/test_lambda.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index bac670b8e..3152ea6f6 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -295,7 +295,7 @@ class LambdaResponse(BaseResponse): code["Configuration"]["FunctionArn"] += ":$LATEST" return 200, {}, json.dumps(code) else: - return 404, {}, "{}" + return 404, {"x-amzn-ErrorType": "ResourceNotFoundException"}, "{}" def _get_aws_region(self, full_url): region = self.region_regex.search(full_url) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 4db13d220..f1265ce71 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -78,7 +78,7 @@ def lambda_handler(event, context): def get_test_zip_file4(): pfunc = """ -def lambda_handler(event, context): +def lambda_handler(event, context): raise Exception('I failed!') """ return _process_lambda(pfunc) @@ -455,7 +455,7 @@ def test_get_function(): ) # Test get function when can't find function name - with assert_raises(ClientError): + with assert_raises(conn.exceptions.ResourceNotFoundException): conn.get_function(FunctionName="junk", Qualifier="$LATEST") From 4833419499c1310ebd2a0012b4b7ba842146ae41 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 7 Feb 2020 15:38:37 +0100 Subject: [PATCH 33/38] Fix CreationDate handling --- moto/kms/models.py | 32 ++++++++++---------- tests/test_kms/test_kms.py | 61 +++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 1015aa72a..ff5d0a356 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta from boto3 import Session from moto.core import BaseBackend, BaseModel -from moto.core.utils import iso_8601_datetime_without_milliseconds +from moto.core.utils import unix_time from moto.iam.models import ACCOUNT_ID @@ -15,8 +15,11 @@ from .utils import decrypt, encrypt, generate_key_id, generate_master_key class Key(BaseModel): - def __init__(self, policy, key_usage, customer_master_key_spec, description, tags, region): + def __init__( + self, policy, key_usage, customer_master_key_spec, description, tags, region + ): self.id = generate_key_id() + self.creation_date = unix_time() self.policy = policy self.key_usage = key_usage self.key_state = "Enabled" @@ -49,10 +52,7 @@ class Key(BaseModel): elif self.customer_master_key_spec == "SYMMETRIC_DEFAULT": return ["SYMMETRIC_DEFAULT"] else: - return [ - "RSAES_OAEP_SHA_1", - "RSAES_OAEP_SHA_256" - ] + return ["RSAES_OAEP_SHA_1", "RSAES_OAEP_SHA_256"] @property def signing_algorithms(self): @@ -71,7 +71,7 @@ class Key(BaseModel): "RSASSA_PKCS1_V1_5_SHA_512", "RSASSA_PSS_SHA_256", "RSASSA_PSS_SHA_384", - "RSASSA_PSS_SHA_512" + "RSASSA_PSS_SHA_512", ] def to_dict(self): @@ -79,7 +79,7 @@ class Key(BaseModel): "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, - "CreationDate": iso_8601_datetime_without_milliseconds(datetime.now()), + "CreationDate": self.creation_date, "CustomerMasterKeySpec": self.customer_master_key_spec, "Description": self.description, "Enabled": self.enabled, @@ -93,9 +93,7 @@ class Key(BaseModel): } } if self.key_state == "PendingDeletion": - key_dict["KeyMetadata"][ - "DeletionDate" - ] = iso_8601_datetime_without_milliseconds(self.deletion_date) + key_dict["KeyMetadata"]["DeletionDate"] = unix_time(self.deletion_date) return key_dict def delete(self, region_name): @@ -133,8 +131,12 @@ class KmsBackend(BaseBackend): self.keys = {} self.key_to_aliases = defaultdict(set) - def create_key(self, policy, key_usage, customer_master_key_spec, description, tags, region): - key = Key(policy, key_usage, customer_master_key_spec, description, tags, region) + def create_key( + self, policy, key_usage, customer_master_key_spec, description, tags, region + ): + key = Key( + policy, key_usage, customer_master_key_spec, description, tags, region + ) self.keys[key.id] = key return key @@ -258,9 +260,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].deletion_date = datetime.now() + timedelta( days=pending_window_in_days ) - return iso_8601_datetime_without_milliseconds( - self.keys[key_id].deletion_date - ) + return unix_time(self.keys[key_id].deletion_date) def encrypt(self, key_id, plaintext, encryption_context): key_id = self.any_id_to_key_id(key_id) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index c5a49b974..c924af76d 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -65,47 +65,44 @@ def test_create_key(): key["KeyMetadata"].should_not.have.key("SigningAlgorithms") key = conn.create_key( - KeyUsage = "ENCRYPT_DECRYPT", - CustomerMasterKeySpec = 'RSA_2048', + KeyUsage="ENCRYPT_DECRYPT", CustomerMasterKeySpec="RSA_2048", ) - sorted(key["KeyMetadata"]["EncryptionAlgorithms"]).should.equal(["RSAES_OAEP_SHA_1", "RSAES_OAEP_SHA_256"]) + sorted(key["KeyMetadata"]["EncryptionAlgorithms"]).should.equal( + ["RSAES_OAEP_SHA_1", "RSAES_OAEP_SHA_256"] + ) key["KeyMetadata"].should_not.have.key("SigningAlgorithms") - key = conn.create_key( - KeyUsage = "SIGN_VERIFY", - CustomerMasterKeySpec = 'RSA_2048', - ) + key = conn.create_key(KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="RSA_2048",) key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") - sorted(key["KeyMetadata"]["SigningAlgorithms"]).should.equal([ - "RSASSA_PKCS1_V1_5_SHA_256", - "RSASSA_PKCS1_V1_5_SHA_384", - "RSASSA_PKCS1_V1_5_SHA_512", - "RSASSA_PSS_SHA_256", - "RSASSA_PSS_SHA_384", - "RSASSA_PSS_SHA_512" - ]) + sorted(key["KeyMetadata"]["SigningAlgorithms"]).should.equal( + [ + "RSASSA_PKCS1_V1_5_SHA_256", + "RSASSA_PKCS1_V1_5_SHA_384", + "RSASSA_PKCS1_V1_5_SHA_512", + "RSASSA_PSS_SHA_256", + "RSASSA_PSS_SHA_384", + "RSASSA_PSS_SHA_512", + ] + ) key = conn.create_key( - KeyUsage = "SIGN_VERIFY", - CustomerMasterKeySpec = 'ECC_SECG_P256K1', + KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_SECG_P256K1", ) key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_256"]) key = conn.create_key( - KeyUsage = "SIGN_VERIFY", - CustomerMasterKeySpec = 'ECC_NIST_P384', + KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_NIST_P384", ) key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_384"]) key = conn.create_key( - KeyUsage = "SIGN_VERIFY", - CustomerMasterKeySpec = 'ECC_NIST_P521', + KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_NIST_P521", ) key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") @@ -125,6 +122,28 @@ def test_describe_key(): key["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") +@mock_kms +def test_boto3_describe_key(): + client = boto3.client("kms", region_name="us-east-1") + response = client.create_key(Description="my key", KeyUsage="ENCRYPT_DECRYPT",) + key_id = response["KeyMetadata"]["KeyId"] + + response = client.describe_key(KeyId=key_id) + + response["KeyMetadata"]["AWSAccountId"].should.equal("123456789012") + response["KeyMetadata"]["CreationDate"].should.be.a(datetime) + response["KeyMetadata"]["CustomerMasterKeySpec"].should.equal("SYMMETRIC_DEFAULT") + response["KeyMetadata"]["Description"].should.equal("my key") + response["KeyMetadata"]["Enabled"].should.be.ok + response["KeyMetadata"]["EncryptionAlgorithms"].should.equal(["SYMMETRIC_DEFAULT"]) + response["KeyMetadata"]["KeyId"].should_not.be.empty + response["KeyMetadata"]["KeyManager"].should.equal("CUSTOMER") + response["KeyMetadata"]["KeyState"].should.equal("Enabled") + response["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") + response["KeyMetadata"]["Origin"].should.equal("AWS_KMS") + response["KeyMetadata"].should_not.have.key("SigningAlgorithms") + + @mock_kms_deprecated def test_describe_key_via_alias(): conn = boto.kms.connect_to_region("us-west-2") From ec56351416d080ed07506153c1929a1f182c6d96 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 7 Feb 2020 16:28:23 +0100 Subject: [PATCH 34/38] Move boto3 tests to separate file --- tests/test_kms/test_kms.py | 623 +----------------------------- tests/test_kms/test_kms_boto3.py | 638 +++++++++++++++++++++++++++++++ 2 files changed, 639 insertions(+), 622 deletions(-) create mode 100644 tests/test_kms/test_kms_boto3.py diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index c924af76d..9ce324373 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -1,25 +1,18 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from datetime import date -from datetime import datetime -from dateutil.tz import tzutc import base64 -import os import re -import boto3 import boto.kms -import botocore.exceptions import six import sure # noqa from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException -from freezegun import freeze_time from nose.tools import assert_raises from parameterized import parameterized from moto.kms.exceptions import NotFoundException as MotoNotFoundException -from moto import mock_kms, mock_kms_deprecated +from moto import mock_kms_deprecated PLAINTEXT_VECTORS = ( (b"some encodeable plaintext",), @@ -35,80 +28,6 @@ def _get_encoded_value(plaintext): return plaintext.encode("utf-8") -@mock_kms -def test_create_key(): - conn = boto3.client("kms", region_name="us-east-1") - with freeze_time("2015-01-01 00:00:00"): - key = conn.create_key( - Policy="my policy", - Description="my key", - KeyUsage="ENCRYPT_DECRYPT", - Tags=[{"TagKey": "project", "TagValue": "moto"}], - ) - - key["KeyMetadata"]["Arn"].should.equal( - "arn:aws:kms:us-east-1:123456789012:key/{}".format( - key["KeyMetadata"]["KeyId"] - ) - ) - key["KeyMetadata"]["AWSAccountId"].should.equal("123456789012") - key["KeyMetadata"]["CreationDate"].should.be.a(datetime) - key["KeyMetadata"]["CustomerMasterKeySpec"].should.equal("SYMMETRIC_DEFAULT") - key["KeyMetadata"]["Description"].should.equal("my key") - key["KeyMetadata"]["Enabled"].should.be.ok - key["KeyMetadata"]["EncryptionAlgorithms"].should.equal(["SYMMETRIC_DEFAULT"]) - key["KeyMetadata"]["KeyId"].should_not.be.empty - key["KeyMetadata"]["KeyManager"].should.equal("CUSTOMER") - key["KeyMetadata"]["KeyState"].should.equal("Enabled") - key["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") - key["KeyMetadata"]["Origin"].should.equal("AWS_KMS") - key["KeyMetadata"].should_not.have.key("SigningAlgorithms") - - key = conn.create_key( - KeyUsage="ENCRYPT_DECRYPT", CustomerMasterKeySpec="RSA_2048", - ) - - sorted(key["KeyMetadata"]["EncryptionAlgorithms"]).should.equal( - ["RSAES_OAEP_SHA_1", "RSAES_OAEP_SHA_256"] - ) - key["KeyMetadata"].should_not.have.key("SigningAlgorithms") - - key = conn.create_key(KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="RSA_2048",) - - key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") - sorted(key["KeyMetadata"]["SigningAlgorithms"]).should.equal( - [ - "RSASSA_PKCS1_V1_5_SHA_256", - "RSASSA_PKCS1_V1_5_SHA_384", - "RSASSA_PKCS1_V1_5_SHA_512", - "RSASSA_PSS_SHA_256", - "RSASSA_PSS_SHA_384", - "RSASSA_PSS_SHA_512", - ] - ) - - key = conn.create_key( - KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_SECG_P256K1", - ) - - key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") - key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_256"]) - - key = conn.create_key( - KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_NIST_P384", - ) - - key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") - key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_384"]) - - key = conn.create_key( - KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_NIST_P521", - ) - - key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") - key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_512"]) - - @mock_kms_deprecated def test_describe_key(): conn = boto.kms.connect_to_region("us-west-2") @@ -122,28 +41,6 @@ def test_describe_key(): key["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") -@mock_kms -def test_boto3_describe_key(): - client = boto3.client("kms", region_name="us-east-1") - response = client.create_key(Description="my key", KeyUsage="ENCRYPT_DECRYPT",) - key_id = response["KeyMetadata"]["KeyId"] - - response = client.describe_key(KeyId=key_id) - - response["KeyMetadata"]["AWSAccountId"].should.equal("123456789012") - response["KeyMetadata"]["CreationDate"].should.be.a(datetime) - response["KeyMetadata"]["CustomerMasterKeySpec"].should.equal("SYMMETRIC_DEFAULT") - response["KeyMetadata"]["Description"].should.equal("my key") - response["KeyMetadata"]["Enabled"].should.be.ok - response["KeyMetadata"]["EncryptionAlgorithms"].should.equal(["SYMMETRIC_DEFAULT"]) - response["KeyMetadata"]["KeyId"].should_not.be.empty - response["KeyMetadata"]["KeyManager"].should.equal("CUSTOMER") - response["KeyMetadata"]["KeyState"].should.equal("Enabled") - response["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") - response["KeyMetadata"]["Origin"].should.equal("AWS_KMS") - response["KeyMetadata"].should_not.have.key("SigningAlgorithms") - - @mock_kms_deprecated def test_describe_key_via_alias(): conn = boto.kms.connect_to_region("us-west-2") @@ -175,22 +72,6 @@ def test_describe_key_via_alias_not_found(): ) -@parameterized( - ( - ("alias/does-not-exist",), - ("arn:aws:kms:us-east-1:012345678912:alias/does-not-exist",), - ("invalid",), - ) -) -@mock_kms -def test_describe_key_via_alias_invalid_alias(key_id): - client = boto3.client("kms", region_name="us-east-1") - client.create_key(Description="key") - - with assert_raises(client.exceptions.NotFoundException): - client.describe_key(KeyId=key_id) - - @mock_kms_deprecated def test_describe_key_via_arn(): conn = boto.kms.connect_to_region("us-west-2") @@ -318,71 +199,6 @@ def test_generate_data_key(): response["KeyId"].should.equal(key_arn) -@mock_kms -def test_boto3_generate_data_key(): - kms = boto3.client("kms", region_name="us-west-2") - - key = kms.create_key() - key_id = key["KeyMetadata"]["KeyId"] - key_arn = key["KeyMetadata"]["Arn"] - - response = kms.generate_data_key(KeyId=key_id, NumberOfBytes=32) - - # CiphertextBlob must NOT be base64-encoded - with assert_raises(Exception): - base64.b64decode(response["CiphertextBlob"], validate=True) - # Plaintext must NOT be base64-encoded - with assert_raises(Exception): - base64.b64decode(response["Plaintext"], validate=True) - - response["KeyId"].should.equal(key_arn) - - -@parameterized(PLAINTEXT_VECTORS) -@mock_kms -def test_encrypt(plaintext): - client = boto3.client("kms", region_name="us-west-2") - - key = client.create_key(Description="key") - key_id = key["KeyMetadata"]["KeyId"] - key_arn = key["KeyMetadata"]["Arn"] - - response = client.encrypt(KeyId=key_id, Plaintext=plaintext) - response["CiphertextBlob"].should_not.equal(plaintext) - - # CiphertextBlob must NOT be base64-encoded - with assert_raises(Exception): - base64.b64decode(response["CiphertextBlob"], validate=True) - - response["KeyId"].should.equal(key_arn) - - -@parameterized(PLAINTEXT_VECTORS) -@mock_kms -def test_decrypt(plaintext): - client = boto3.client("kms", region_name="us-west-2") - - key = client.create_key(Description="key") - key_id = key["KeyMetadata"]["KeyId"] - key_arn = key["KeyMetadata"]["Arn"] - - encrypt_response = client.encrypt(KeyId=key_id, Plaintext=plaintext) - - client.create_key(Description="key") - # CiphertextBlob must NOT be base64-encoded - with assert_raises(Exception): - base64.b64decode(encrypt_response["CiphertextBlob"], validate=True) - - decrypt_response = client.decrypt(CiphertextBlob=encrypt_response["CiphertextBlob"]) - - # Plaintext must NOT be base64-encoded - with assert_raises(Exception): - base64.b64decode(decrypt_response["Plaintext"], validate=True) - - decrypt_response["Plaintext"].should.equal(_get_encoded_value(plaintext)) - decrypt_response["KeyId"].should.equal(key_arn) - - @mock_kms_deprecated def test_disable_key_rotation_with_missing_key(): conn = boto.kms.connect_to_region("us-west-2") @@ -853,25 +669,6 @@ def test__list_aliases(): len(aliases).should.equal(7) -@parameterized( - ( - ("not-a-uuid",), - ("alias/DoesNotExist",), - ("arn:aws:kms:us-east-1:012345678912:alias/DoesNotExist",), - ("d25652e4-d2d2-49f7-929a-671ccda580c6",), - ( - "arn:aws:kms:us-east-1:012345678912:key/d25652e4-d2d2-49f7-929a-671ccda580c6", - ), - ) -) -@mock_kms -def test_invalid_key_ids(key_id): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.generate_data_key(KeyId=key_id, NumberOfBytes=5) - - @mock_kms_deprecated def test__assert_default_policy(): from moto.kms.responses import _assert_default_policy @@ -882,421 +679,3 @@ def test__assert_default_policy(): _assert_default_policy.when.called_with("default").should_not.throw( MotoNotFoundException ) - - -@parameterized(PLAINTEXT_VECTORS) -@mock_kms -def test_kms_encrypt_boto3(plaintext): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="key") - response = client.encrypt(KeyId=key["KeyMetadata"]["KeyId"], Plaintext=plaintext) - - response = client.decrypt(CiphertextBlob=response["CiphertextBlob"]) - response["Plaintext"].should.equal(_get_encoded_value(plaintext)) - - -@mock_kms -def test_disable_key(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="disable-key") - client.disable_key(KeyId=key["KeyMetadata"]["KeyId"]) - - result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) - assert result["KeyMetadata"]["Enabled"] == False - assert result["KeyMetadata"]["KeyState"] == "Disabled" - - -@mock_kms -def test_enable_key(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="enable-key") - client.disable_key(KeyId=key["KeyMetadata"]["KeyId"]) - client.enable_key(KeyId=key["KeyMetadata"]["KeyId"]) - - result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) - assert result["KeyMetadata"]["Enabled"] == True - assert result["KeyMetadata"]["KeyState"] == "Enabled" - - -@mock_kms -def test_schedule_key_deletion(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="schedule-key-deletion") - if os.environ.get("TEST_SERVER_MODE", "false").lower() == "false": - with freeze_time("2015-01-01 12:00:00"): - response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) - assert response["KeyId"] == key["KeyMetadata"]["KeyId"] - assert response["DeletionDate"] == datetime( - 2015, 1, 31, 12, 0, tzinfo=tzutc() - ) - else: - # Can't manipulate time in server mode - response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) - assert response["KeyId"] == key["KeyMetadata"]["KeyId"] - - result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) - assert result["KeyMetadata"]["Enabled"] == False - assert result["KeyMetadata"]["KeyState"] == "PendingDeletion" - assert "DeletionDate" in result["KeyMetadata"] - - -@mock_kms -def test_schedule_key_deletion_custom(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="schedule-key-deletion") - if os.environ.get("TEST_SERVER_MODE", "false").lower() == "false": - with freeze_time("2015-01-01 12:00:00"): - response = client.schedule_key_deletion( - KeyId=key["KeyMetadata"]["KeyId"], PendingWindowInDays=7 - ) - assert response["KeyId"] == key["KeyMetadata"]["KeyId"] - assert response["DeletionDate"] == datetime( - 2015, 1, 8, 12, 0, tzinfo=tzutc() - ) - else: - # Can't manipulate time in server mode - response = client.schedule_key_deletion( - KeyId=key["KeyMetadata"]["KeyId"], PendingWindowInDays=7 - ) - assert response["KeyId"] == key["KeyMetadata"]["KeyId"] - - result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) - assert result["KeyMetadata"]["Enabled"] == False - assert result["KeyMetadata"]["KeyState"] == "PendingDeletion" - assert "DeletionDate" in result["KeyMetadata"] - - -@mock_kms -def test_cancel_key_deletion(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="cancel-key-deletion") - client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) - response = client.cancel_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) - assert response["KeyId"] == key["KeyMetadata"]["KeyId"] - - result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) - assert result["KeyMetadata"]["Enabled"] == False - assert result["KeyMetadata"]["KeyState"] == "Disabled" - assert "DeletionDate" not in result["KeyMetadata"] - - -@mock_kms -def test_update_key_description(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="old_description") - key_id = key["KeyMetadata"]["KeyId"] - - result = client.update_key_description(KeyId=key_id, Description="new_description") - assert "ResponseMetadata" in result - - -@mock_kms -def test_tag_resource(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="cancel-key-deletion") - response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) - - keyid = response["KeyId"] - response = client.tag_resource( - KeyId=keyid, Tags=[{"TagKey": "string", "TagValue": "string"}] - ) - - # Shouldn't have any data, just header - assert len(response.keys()) == 1 - - -@mock_kms -def test_list_resource_tags(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="cancel-key-deletion") - response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) - - keyid = response["KeyId"] - response = client.tag_resource( - KeyId=keyid, Tags=[{"TagKey": "string", "TagValue": "string"}] - ) - - response = client.list_resource_tags(KeyId=keyid) - assert response["Tags"][0]["TagKey"] == "string" - assert response["Tags"][0]["TagValue"] == "string" - - -@parameterized( - ( - (dict(KeySpec="AES_256"), 32), - (dict(KeySpec="AES_128"), 16), - (dict(NumberOfBytes=64), 64), - (dict(NumberOfBytes=1), 1), - (dict(NumberOfBytes=1024), 1024), - ) -) -@mock_kms -def test_generate_data_key_sizes(kwargs, expected_key_length): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="generate-data-key-size") - - response = client.generate_data_key(KeyId=key["KeyMetadata"]["KeyId"], **kwargs) - - assert len(response["Plaintext"]) == expected_key_length - - -@mock_kms -def test_generate_data_key_decrypt(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="generate-data-key-decrypt") - - resp1 = client.generate_data_key( - KeyId=key["KeyMetadata"]["KeyId"], KeySpec="AES_256" - ) - resp2 = client.decrypt(CiphertextBlob=resp1["CiphertextBlob"]) - - assert resp1["Plaintext"] == resp2["Plaintext"] - - -@parameterized( - ( - (dict(KeySpec="AES_257"),), - (dict(KeySpec="AES_128", NumberOfBytes=16),), - (dict(NumberOfBytes=2048),), - (dict(NumberOfBytes=0),), - (dict(),), - ) -) -@mock_kms -def test_generate_data_key_invalid_size_params(kwargs): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="generate-data-key-size") - - with assert_raises( - (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError) - ) as err: - client.generate_data_key(KeyId=key["KeyMetadata"]["KeyId"], **kwargs) - - -@parameterized( - ( - ("alias/DoesNotExist",), - ("arn:aws:kms:us-east-1:012345678912:alias/DoesNotExist",), - ("d25652e4-d2d2-49f7-929a-671ccda580c6",), - ( - "arn:aws:kms:us-east-1:012345678912:key/d25652e4-d2d2-49f7-929a-671ccda580c6", - ), - ) -) -@mock_kms -def test_generate_data_key_invalid_key(key_id): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.generate_data_key(KeyId=key_id, KeySpec="AES_256") - - -@parameterized( - ( - ("alias/DoesExist", False), - ("arn:aws:kms:us-east-1:012345678912:alias/DoesExist", False), - ("", True), - ("arn:aws:kms:us-east-1:012345678912:key/", True), - ) -) -@mock_kms -def test_generate_data_key_all_valid_key_ids(prefix, append_key_id): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key() - key_id = key["KeyMetadata"]["KeyId"] - client.create_alias(AliasName="alias/DoesExist", TargetKeyId=key_id) - - target_id = prefix - if append_key_id: - target_id += key_id - - client.generate_data_key(KeyId=key_id, NumberOfBytes=32) - - -@mock_kms -def test_generate_data_key_without_plaintext_decrypt(): - client = boto3.client("kms", region_name="us-east-1") - key = client.create_key(Description="generate-data-key-decrypt") - - resp1 = client.generate_data_key_without_plaintext( - KeyId=key["KeyMetadata"]["KeyId"], KeySpec="AES_256" - ) - - assert "Plaintext" not in resp1 - - -@parameterized(PLAINTEXT_VECTORS) -@mock_kms -def test_re_encrypt_decrypt(plaintext): - client = boto3.client("kms", region_name="us-west-2") - - key_1 = client.create_key(Description="key 1") - key_1_id = key_1["KeyMetadata"]["KeyId"] - key_1_arn = key_1["KeyMetadata"]["Arn"] - key_2 = client.create_key(Description="key 2") - key_2_id = key_2["KeyMetadata"]["KeyId"] - key_2_arn = key_2["KeyMetadata"]["Arn"] - - encrypt_response = client.encrypt( - KeyId=key_1_id, Plaintext=plaintext, EncryptionContext={"encryption": "context"} - ) - - re_encrypt_response = client.re_encrypt( - CiphertextBlob=encrypt_response["CiphertextBlob"], - SourceEncryptionContext={"encryption": "context"}, - DestinationKeyId=key_2_id, - DestinationEncryptionContext={"another": "context"}, - ) - - # CiphertextBlob must NOT be base64-encoded - with assert_raises(Exception): - base64.b64decode(re_encrypt_response["CiphertextBlob"], validate=True) - - re_encrypt_response["SourceKeyId"].should.equal(key_1_arn) - re_encrypt_response["KeyId"].should.equal(key_2_arn) - - decrypt_response_1 = client.decrypt( - CiphertextBlob=encrypt_response["CiphertextBlob"], - EncryptionContext={"encryption": "context"}, - ) - decrypt_response_1["Plaintext"].should.equal(_get_encoded_value(plaintext)) - decrypt_response_1["KeyId"].should.equal(key_1_arn) - - decrypt_response_2 = client.decrypt( - CiphertextBlob=re_encrypt_response["CiphertextBlob"], - EncryptionContext={"another": "context"}, - ) - decrypt_response_2["Plaintext"].should.equal(_get_encoded_value(plaintext)) - decrypt_response_2["KeyId"].should.equal(key_2_arn) - - decrypt_response_1["Plaintext"].should.equal(decrypt_response_2["Plaintext"]) - - -@mock_kms -def test_re_encrypt_to_invalid_destination(): - client = boto3.client("kms", region_name="us-west-2") - - key = client.create_key(Description="key 1") - key_id = key["KeyMetadata"]["KeyId"] - - encrypt_response = client.encrypt(KeyId=key_id, Plaintext=b"some plaintext") - - with assert_raises(client.exceptions.NotFoundException): - client.re_encrypt( - CiphertextBlob=encrypt_response["CiphertextBlob"], - DestinationKeyId="alias/DoesNotExist", - ) - - -@parameterized(((12,), (44,), (91,), (1,), (1024,))) -@mock_kms -def test_generate_random(number_of_bytes): - client = boto3.client("kms", region_name="us-west-2") - - response = client.generate_random(NumberOfBytes=number_of_bytes) - - response["Plaintext"].should.be.a(bytes) - len(response["Plaintext"]).should.equal(number_of_bytes) - - -@parameterized( - ( - (2048, botocore.exceptions.ClientError), - (1025, botocore.exceptions.ClientError), - (0, botocore.exceptions.ParamValidationError), - (-1, botocore.exceptions.ParamValidationError), - (-1024, botocore.exceptions.ParamValidationError), - ) -) -@mock_kms -def test_generate_random_invalid_number_of_bytes(number_of_bytes, error_type): - client = boto3.client("kms", region_name="us-west-2") - - with assert_raises(error_type): - client.generate_random(NumberOfBytes=number_of_bytes) - - -@mock_kms -def test_enable_key_rotation_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.enable_key_rotation(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_disable_key_rotation_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.disable_key_rotation(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_enable_key_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.enable_key(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_disable_key_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.disable_key(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_cancel_key_deletion_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.cancel_key_deletion(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_schedule_key_deletion_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.schedule_key_deletion(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_get_key_rotation_status_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.get_key_rotation_status(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_get_key_policy_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.get_key_policy( - KeyId="12366f9b-1230-123d-123e-123e6ae60c02", PolicyName="default" - ) - - -@mock_kms -def test_list_key_policies_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.list_key_policies(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") - - -@mock_kms -def test_put_key_policy_key_not_found(): - client = boto3.client("kms", region_name="us-east-1") - - with assert_raises(client.exceptions.NotFoundException): - client.put_key_policy( - KeyId="00000000-0000-0000-0000-000000000000", - PolicyName="default", - Policy="new policy", - ) diff --git a/tests/test_kms/test_kms_boto3.py b/tests/test_kms/test_kms_boto3.py new file mode 100644 index 000000000..c125c0557 --- /dev/null +++ b/tests/test_kms/test_kms_boto3.py @@ -0,0 +1,638 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from datetime import datetime +from dateutil.tz import tzutc +import base64 +import os + +import boto3 +import botocore.exceptions +import six +import sure # noqa +from freezegun import freeze_time +from nose.tools import assert_raises +from parameterized import parameterized + +from moto import mock_kms + +PLAINTEXT_VECTORS = ( + (b"some encodeable plaintext",), + (b"some unencodeable plaintext \xec\x8a\xcf\xb6r\xe9\xb5\xeb\xff\xa23\x16",), + ("some unicode characters ø˚∆øˆˆ∆ßçøˆˆçßøˆ¨¥",), +) + + +def _get_encoded_value(plaintext): + if isinstance(plaintext, six.binary_type): + return plaintext + + return plaintext.encode("utf-8") + + +@mock_kms +def test_create_key(): + conn = boto3.client("kms", region_name="us-east-1") + key = conn.create_key( + Policy="my policy", + Description="my key", + KeyUsage="ENCRYPT_DECRYPT", + Tags=[{"TagKey": "project", "TagValue": "moto"}], + ) + + key["KeyMetadata"]["Arn"].should.equal( + "arn:aws:kms:us-east-1:123456789012:key/{}".format(key["KeyMetadata"]["KeyId"]) + ) + key["KeyMetadata"]["AWSAccountId"].should.equal("123456789012") + key["KeyMetadata"]["CreationDate"].should.be.a(datetime) + key["KeyMetadata"]["CustomerMasterKeySpec"].should.equal("SYMMETRIC_DEFAULT") + key["KeyMetadata"]["Description"].should.equal("my key") + key["KeyMetadata"]["Enabled"].should.be.ok + key["KeyMetadata"]["EncryptionAlgorithms"].should.equal(["SYMMETRIC_DEFAULT"]) + key["KeyMetadata"]["KeyId"].should_not.be.empty + key["KeyMetadata"]["KeyManager"].should.equal("CUSTOMER") + key["KeyMetadata"]["KeyState"].should.equal("Enabled") + key["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") + key["KeyMetadata"]["Origin"].should.equal("AWS_KMS") + key["KeyMetadata"].should_not.have.key("SigningAlgorithms") + + key = conn.create_key(KeyUsage="ENCRYPT_DECRYPT", CustomerMasterKeySpec="RSA_2048",) + + sorted(key["KeyMetadata"]["EncryptionAlgorithms"]).should.equal( + ["RSAES_OAEP_SHA_1", "RSAES_OAEP_SHA_256"] + ) + key["KeyMetadata"].should_not.have.key("SigningAlgorithms") + + key = conn.create_key(KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="RSA_2048",) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + sorted(key["KeyMetadata"]["SigningAlgorithms"]).should.equal( + [ + "RSASSA_PKCS1_V1_5_SHA_256", + "RSASSA_PKCS1_V1_5_SHA_384", + "RSASSA_PKCS1_V1_5_SHA_512", + "RSASSA_PSS_SHA_256", + "RSASSA_PSS_SHA_384", + "RSASSA_PSS_SHA_512", + ] + ) + + key = conn.create_key( + KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_SECG_P256K1", + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_256"]) + + key = conn.create_key( + KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_NIST_P384", + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_384"]) + + key = conn.create_key( + KeyUsage="SIGN_VERIFY", CustomerMasterKeySpec="ECC_NIST_P521", + ) + + key["KeyMetadata"].should_not.have.key("EncryptionAlgorithms") + key["KeyMetadata"]["SigningAlgorithms"].should.equal(["ECDSA_SHA_512"]) + + +@mock_kms +def test_describe_key(): + client = boto3.client("kms", region_name="us-east-1") + response = client.create_key(Description="my key", KeyUsage="ENCRYPT_DECRYPT",) + key_id = response["KeyMetadata"]["KeyId"] + + response = client.describe_key(KeyId=key_id) + + response["KeyMetadata"]["AWSAccountId"].should.equal("123456789012") + response["KeyMetadata"]["CreationDate"].should.be.a(datetime) + response["KeyMetadata"]["CustomerMasterKeySpec"].should.equal("SYMMETRIC_DEFAULT") + response["KeyMetadata"]["Description"].should.equal("my key") + response["KeyMetadata"]["Enabled"].should.be.ok + response["KeyMetadata"]["EncryptionAlgorithms"].should.equal(["SYMMETRIC_DEFAULT"]) + response["KeyMetadata"]["KeyId"].should_not.be.empty + response["KeyMetadata"]["KeyManager"].should.equal("CUSTOMER") + response["KeyMetadata"]["KeyState"].should.equal("Enabled") + response["KeyMetadata"]["KeyUsage"].should.equal("ENCRYPT_DECRYPT") + response["KeyMetadata"]["Origin"].should.equal("AWS_KMS") + response["KeyMetadata"].should_not.have.key("SigningAlgorithms") + + +@parameterized( + ( + ("alias/does-not-exist",), + ("arn:aws:kms:us-east-1:012345678912:alias/does-not-exist",), + ("invalid",), + ) +) +@mock_kms +def test_describe_key_via_alias_invalid_alias(key_id): + client = boto3.client("kms", region_name="us-east-1") + client.create_key(Description="key") + + with assert_raises(client.exceptions.NotFoundException): + client.describe_key(KeyId=key_id) + + +@mock_kms +def test_generate_data_key(): + kms = boto3.client("kms", region_name="us-west-2") + + key = kms.create_key() + key_id = key["KeyMetadata"]["KeyId"] + key_arn = key["KeyMetadata"]["Arn"] + + response = kms.generate_data_key(KeyId=key_id, NumberOfBytes=32) + + # CiphertextBlob must NOT be base64-encoded + with assert_raises(Exception): + base64.b64decode(response["CiphertextBlob"], validate=True) + # Plaintext must NOT be base64-encoded + with assert_raises(Exception): + base64.b64decode(response["Plaintext"], validate=True) + + response["KeyId"].should.equal(key_arn) + + +@parameterized(PLAINTEXT_VECTORS) +@mock_kms +def test_encrypt(plaintext): + client = boto3.client("kms", region_name="us-west-2") + + key = client.create_key(Description="key") + key_id = key["KeyMetadata"]["KeyId"] + key_arn = key["KeyMetadata"]["Arn"] + + response = client.encrypt(KeyId=key_id, Plaintext=plaintext) + response["CiphertextBlob"].should_not.equal(plaintext) + + # CiphertextBlob must NOT be base64-encoded + with assert_raises(Exception): + base64.b64decode(response["CiphertextBlob"], validate=True) + + response["KeyId"].should.equal(key_arn) + + +@parameterized(PLAINTEXT_VECTORS) +@mock_kms +def test_decrypt(plaintext): + client = boto3.client("kms", region_name="us-west-2") + + key = client.create_key(Description="key") + key_id = key["KeyMetadata"]["KeyId"] + key_arn = key["KeyMetadata"]["Arn"] + + encrypt_response = client.encrypt(KeyId=key_id, Plaintext=plaintext) + + client.create_key(Description="key") + # CiphertextBlob must NOT be base64-encoded + with assert_raises(Exception): + base64.b64decode(encrypt_response["CiphertextBlob"], validate=True) + + decrypt_response = client.decrypt(CiphertextBlob=encrypt_response["CiphertextBlob"]) + + # Plaintext must NOT be base64-encoded + with assert_raises(Exception): + base64.b64decode(decrypt_response["Plaintext"], validate=True) + + decrypt_response["Plaintext"].should.equal(_get_encoded_value(plaintext)) + decrypt_response["KeyId"].should.equal(key_arn) + + +@parameterized( + ( + ("not-a-uuid",), + ("alias/DoesNotExist",), + ("arn:aws:kms:us-east-1:012345678912:alias/DoesNotExist",), + ("d25652e4-d2d2-49f7-929a-671ccda580c6",), + ( + "arn:aws:kms:us-east-1:012345678912:key/d25652e4-d2d2-49f7-929a-671ccda580c6", + ), + ) +) +@mock_kms +def test_invalid_key_ids(key_id): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.generate_data_key(KeyId=key_id, NumberOfBytes=5) + + +@parameterized(PLAINTEXT_VECTORS) +@mock_kms +def test_kms_encrypt(plaintext): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="key") + response = client.encrypt(KeyId=key["KeyMetadata"]["KeyId"], Plaintext=plaintext) + + response = client.decrypt(CiphertextBlob=response["CiphertextBlob"]) + response["Plaintext"].should.equal(_get_encoded_value(plaintext)) + + +@mock_kms +def test_disable_key(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="disable-key") + client.disable_key(KeyId=key["KeyMetadata"]["KeyId"]) + + result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == "Disabled" + + +@mock_kms +def test_enable_key(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="enable-key") + client.disable_key(KeyId=key["KeyMetadata"]["KeyId"]) + client.enable_key(KeyId=key["KeyMetadata"]["KeyId"]) + + result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) + assert result["KeyMetadata"]["Enabled"] == True + assert result["KeyMetadata"]["KeyState"] == "Enabled" + + +@mock_kms +def test_schedule_key_deletion(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="schedule-key-deletion") + if os.environ.get("TEST_SERVER_MODE", "false").lower() == "false": + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) + assert response["KeyId"] == key["KeyMetadata"]["KeyId"] + assert response["DeletionDate"] == datetime( + 2015, 1, 31, 12, 0, tzinfo=tzutc() + ) + else: + # Can't manipulate time in server mode + response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) + assert response["KeyId"] == key["KeyMetadata"]["KeyId"] + + result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == "PendingDeletion" + assert "DeletionDate" in result["KeyMetadata"] + + +@mock_kms +def test_schedule_key_deletion_custom(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="schedule-key-deletion") + if os.environ.get("TEST_SERVER_MODE", "false").lower() == "false": + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key["KeyMetadata"]["KeyId"], PendingWindowInDays=7 + ) + assert response["KeyId"] == key["KeyMetadata"]["KeyId"] + assert response["DeletionDate"] == datetime( + 2015, 1, 8, 12, 0, tzinfo=tzutc() + ) + else: + # Can't manipulate time in server mode + response = client.schedule_key_deletion( + KeyId=key["KeyMetadata"]["KeyId"], PendingWindowInDays=7 + ) + assert response["KeyId"] == key["KeyMetadata"]["KeyId"] + + result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == "PendingDeletion" + assert "DeletionDate" in result["KeyMetadata"] + + +@mock_kms +def test_cancel_key_deletion(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="cancel-key-deletion") + client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) + response = client.cancel_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) + assert response["KeyId"] == key["KeyMetadata"]["KeyId"] + + result = client.describe_key(KeyId=key["KeyMetadata"]["KeyId"]) + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == "Disabled" + assert "DeletionDate" not in result["KeyMetadata"] + + +@mock_kms +def test_update_key_description(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="old_description") + key_id = key["KeyMetadata"]["KeyId"] + + result = client.update_key_description(KeyId=key_id, Description="new_description") + assert "ResponseMetadata" in result + + +@mock_kms +def test_tag_resource(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="cancel-key-deletion") + response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) + + keyid = response["KeyId"] + response = client.tag_resource( + KeyId=keyid, Tags=[{"TagKey": "string", "TagValue": "string"}] + ) + + # Shouldn't have any data, just header + assert len(response.keys()) == 1 + + +@mock_kms +def test_list_resource_tags(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="cancel-key-deletion") + response = client.schedule_key_deletion(KeyId=key["KeyMetadata"]["KeyId"]) + + keyid = response["KeyId"] + response = client.tag_resource( + KeyId=keyid, Tags=[{"TagKey": "string", "TagValue": "string"}] + ) + + response = client.list_resource_tags(KeyId=keyid) + assert response["Tags"][0]["TagKey"] == "string" + assert response["Tags"][0]["TagValue"] == "string" + + +@parameterized( + ( + (dict(KeySpec="AES_256"), 32), + (dict(KeySpec="AES_128"), 16), + (dict(NumberOfBytes=64), 64), + (dict(NumberOfBytes=1), 1), + (dict(NumberOfBytes=1024), 1024), + ) +) +@mock_kms +def test_generate_data_key_sizes(kwargs, expected_key_length): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="generate-data-key-size") + + response = client.generate_data_key(KeyId=key["KeyMetadata"]["KeyId"], **kwargs) + + assert len(response["Plaintext"]) == expected_key_length + + +@mock_kms +def test_generate_data_key_decrypt(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="generate-data-key-decrypt") + + resp1 = client.generate_data_key( + KeyId=key["KeyMetadata"]["KeyId"], KeySpec="AES_256" + ) + resp2 = client.decrypt(CiphertextBlob=resp1["CiphertextBlob"]) + + assert resp1["Plaintext"] == resp2["Plaintext"] + + +@parameterized( + ( + (dict(KeySpec="AES_257"),), + (dict(KeySpec="AES_128", NumberOfBytes=16),), + (dict(NumberOfBytes=2048),), + (dict(NumberOfBytes=0),), + (dict(),), + ) +) +@mock_kms +def test_generate_data_key_invalid_size_params(kwargs): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="generate-data-key-size") + + with assert_raises( + (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError) + ) as err: + client.generate_data_key(KeyId=key["KeyMetadata"]["KeyId"], **kwargs) + + +@parameterized( + ( + ("alias/DoesNotExist",), + ("arn:aws:kms:us-east-1:012345678912:alias/DoesNotExist",), + ("d25652e4-d2d2-49f7-929a-671ccda580c6",), + ( + "arn:aws:kms:us-east-1:012345678912:key/d25652e4-d2d2-49f7-929a-671ccda580c6", + ), + ) +) +@mock_kms +def test_generate_data_key_invalid_key(key_id): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.generate_data_key(KeyId=key_id, KeySpec="AES_256") + + +@parameterized( + ( + ("alias/DoesExist", False), + ("arn:aws:kms:us-east-1:012345678912:alias/DoesExist", False), + ("", True), + ("arn:aws:kms:us-east-1:012345678912:key/", True), + ) +) +@mock_kms +def test_generate_data_key_all_valid_key_ids(prefix, append_key_id): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key() + key_id = key["KeyMetadata"]["KeyId"] + client.create_alias(AliasName="alias/DoesExist", TargetKeyId=key_id) + + target_id = prefix + if append_key_id: + target_id += key_id + + client.generate_data_key(KeyId=key_id, NumberOfBytes=32) + + +@mock_kms +def test_generate_data_key_without_plaintext_decrypt(): + client = boto3.client("kms", region_name="us-east-1") + key = client.create_key(Description="generate-data-key-decrypt") + + resp1 = client.generate_data_key_without_plaintext( + KeyId=key["KeyMetadata"]["KeyId"], KeySpec="AES_256" + ) + + assert "Plaintext" not in resp1 + + +@parameterized(PLAINTEXT_VECTORS) +@mock_kms +def test_re_encrypt_decrypt(plaintext): + client = boto3.client("kms", region_name="us-west-2") + + key_1 = client.create_key(Description="key 1") + key_1_id = key_1["KeyMetadata"]["KeyId"] + key_1_arn = key_1["KeyMetadata"]["Arn"] + key_2 = client.create_key(Description="key 2") + key_2_id = key_2["KeyMetadata"]["KeyId"] + key_2_arn = key_2["KeyMetadata"]["Arn"] + + encrypt_response = client.encrypt( + KeyId=key_1_id, Plaintext=plaintext, EncryptionContext={"encryption": "context"} + ) + + re_encrypt_response = client.re_encrypt( + CiphertextBlob=encrypt_response["CiphertextBlob"], + SourceEncryptionContext={"encryption": "context"}, + DestinationKeyId=key_2_id, + DestinationEncryptionContext={"another": "context"}, + ) + + # CiphertextBlob must NOT be base64-encoded + with assert_raises(Exception): + base64.b64decode(re_encrypt_response["CiphertextBlob"], validate=True) + + re_encrypt_response["SourceKeyId"].should.equal(key_1_arn) + re_encrypt_response["KeyId"].should.equal(key_2_arn) + + decrypt_response_1 = client.decrypt( + CiphertextBlob=encrypt_response["CiphertextBlob"], + EncryptionContext={"encryption": "context"}, + ) + decrypt_response_1["Plaintext"].should.equal(_get_encoded_value(plaintext)) + decrypt_response_1["KeyId"].should.equal(key_1_arn) + + decrypt_response_2 = client.decrypt( + CiphertextBlob=re_encrypt_response["CiphertextBlob"], + EncryptionContext={"another": "context"}, + ) + decrypt_response_2["Plaintext"].should.equal(_get_encoded_value(plaintext)) + decrypt_response_2["KeyId"].should.equal(key_2_arn) + + decrypt_response_1["Plaintext"].should.equal(decrypt_response_2["Plaintext"]) + + +@mock_kms +def test_re_encrypt_to_invalid_destination(): + client = boto3.client("kms", region_name="us-west-2") + + key = client.create_key(Description="key 1") + key_id = key["KeyMetadata"]["KeyId"] + + encrypt_response = client.encrypt(KeyId=key_id, Plaintext=b"some plaintext") + + with assert_raises(client.exceptions.NotFoundException): + client.re_encrypt( + CiphertextBlob=encrypt_response["CiphertextBlob"], + DestinationKeyId="alias/DoesNotExist", + ) + + +@parameterized(((12,), (44,), (91,), (1,), (1024,))) +@mock_kms +def test_generate_random(number_of_bytes): + client = boto3.client("kms", region_name="us-west-2") + + response = client.generate_random(NumberOfBytes=number_of_bytes) + + response["Plaintext"].should.be.a(bytes) + len(response["Plaintext"]).should.equal(number_of_bytes) + + +@parameterized( + ( + (2048, botocore.exceptions.ClientError), + (1025, botocore.exceptions.ClientError), + (0, botocore.exceptions.ParamValidationError), + (-1, botocore.exceptions.ParamValidationError), + (-1024, botocore.exceptions.ParamValidationError), + ) +) +@mock_kms +def test_generate_random_invalid_number_of_bytes(number_of_bytes, error_type): + client = boto3.client("kms", region_name="us-west-2") + + with assert_raises(error_type): + client.generate_random(NumberOfBytes=number_of_bytes) + + +@mock_kms +def test_enable_key_rotation_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.enable_key_rotation(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_disable_key_rotation_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.disable_key_rotation(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_enable_key_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.enable_key(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_disable_key_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.disable_key(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_cancel_key_deletion_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.cancel_key_deletion(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_schedule_key_deletion_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.schedule_key_deletion(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_get_key_rotation_status_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.get_key_rotation_status(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_get_key_policy_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.get_key_policy( + KeyId="12366f9b-1230-123d-123e-123e6ae60c02", PolicyName="default" + ) + + +@mock_kms +def test_list_key_policies_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.list_key_policies(KeyId="12366f9b-1230-123d-123e-123e6ae60c02") + + +@mock_kms +def test_put_key_policy_key_not_found(): + client = boto3.client("kms", region_name="us-east-1") + + with assert_raises(client.exceptions.NotFoundException): + client.put_key_policy( + KeyId="00000000-0000-0000-0000-000000000000", + PolicyName="default", + Policy="new policy", + ) From 0b7e990bbfed5e5ba8fd80570519dcd81d0a7e04 Mon Sep 17 00:00:00 2001 From: jmsanders <10291790+jmsanders@users.noreply.github.com> Date: Fri, 7 Feb 2020 15:50:08 -0600 Subject: [PATCH 35/38] Limit SQS list_queues response to 1000 queues The maximum number of queues that the ListQueues API can return is 1000: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ListQueues.html --- moto/sqs/models.py | 2 +- tests/test_sqs/test_sqs.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 8fbe90108..a54d91c43 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -567,7 +567,7 @@ class SQSBackend(BaseBackend): for name, q in self.queues.items(): if prefix_re.search(name): qs.append(q) - return qs + return qs[:1000] def get_queue(self, queue_name): queue = self.queues.get(queue_name) diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 93d388117..f2ab8c37c 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1759,3 +1759,23 @@ def test_receive_message_for_queue_with_receive_message_wait_time_seconds_set(): ) queue.receive_messages() + + +@mock_sqs +def test_list_queues_limits_to_1000_queues(): + client = boto3.client("sqs", region_name="us-east-1") + + for i in range(1001): + client.create_queue(QueueName="test-queue-{0}".format(i)) + + client.list_queues()["QueueUrls"].should.have.length_of(1000) + client.list_queues(QueueNamePrefix="test-queue")["QueueUrls"].should.have.length_of( + 1000 + ) + + resource = boto3.resource("sqs", region_name="us-east-1") + + list(resource.queues.all()).should.have.length_of(1000) + list(resource.queues.filter(QueueNamePrefix="test-queue")).should.have.length_of( + 1000 + ) From d4caf14b61ef1ba1a0bf3998b4773acd67eafb7b Mon Sep 17 00:00:00 2001 From: Nikita Antonenkov Date: Sat, 1 Feb 2020 22:00:15 +0100 Subject: [PATCH 36/38] Fixed UnboundLocalError in dynamodb2.query when no filters are passed --- moto/dynamodb2/responses.py | 7 +++++++ tests/test_dynamodb2/test_dynamodb.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index c9f3529a9..d3767c3fd 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -508,6 +508,13 @@ class DynamoHandler(BaseResponse): # 'KeyConditions': {u'forum_name': {u'ComparisonOperator': u'EQ', u'AttributeValueList': [{u'S': u'the-key'}]}} key_conditions = self.body.get("KeyConditions") query_filters = self.body.get("QueryFilter") + + if not (key_conditions or query_filters): + return self.error( + "com.amazonaws.dynamodb.v20111205#ValidationException", + "Either KeyConditions or QueryFilter should be present", + ) + if key_conditions: ( hash_key_name, diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 2e3f9fdbb..ec01889ae 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3721,3 +3721,24 @@ def test_allow_update_to_item_with_different_type(): table.get_item(Key={"job_id": "b"})["Item"]["job_details"][ "job_name" ].should.be.equal({"nested": "yes"}) + + +@mock_dynamodb2 +def test_query_catches_when_no_filters(): + dynamo = boto3.resource("dynamodb", region_name="eu-central-1") + dynamo.create_table( + AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], + TableName="origin-rbu-dev", + KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamo.Table("origin-rbu-dev") + + with assert_raises(ClientError) as ex: + table.query(TableName="original-rbu-dev") + + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "Either KeyConditions or QueryFilter should be present" + ) From 0ac92969362f47c215e3a3309586d3ce950de28e Mon Sep 17 00:00:00 2001 From: Nikita Antonenkov Date: Sat, 1 Feb 2020 22:05:05 +0100 Subject: [PATCH 37/38] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0282e3caf..fb9bd51de 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ env/ .vscode/ tests/file.tmp .eggs/ +.mypy_cache/ +*.tmp From f70cd0182e413bca58be077e7f1f90e50ec83f62 Mon Sep 17 00:00:00 2001 From: Terry Griffin <“griffint61@users.noreply.github.com”> Date: Mon, 10 Feb 2020 09:18:25 -0800 Subject: [PATCH 38/38] Fixed test_lambda_can_be_deleted_by_cloudformation for new (correct) error code. --- tests/test_awslambda/test_lambda_cloudformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_awslambda/test_lambda_cloudformation.py b/tests/test_awslambda/test_lambda_cloudformation.py index a5d4d23fd..f57354d69 100644 --- a/tests/test_awslambda/test_lambda_cloudformation.py +++ b/tests/test_awslambda/test_lambda_cloudformation.py @@ -94,7 +94,7 @@ def test_lambda_can_be_deleted_by_cloudformation(): # Verify function was deleted with assert_raises(ClientError) as e: lmbda.get_function(FunctionName=created_fn_name) - e.exception.response["Error"]["Code"].should.equal("404") + e.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") def create_stack(cf, s3):