From 24e942b50e4ea9b4913f48777041551316f0fe22 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 23 Jul 2018 14:38:52 +1000 Subject: [PATCH 01/64] Update list of implemented endpoints for cognito-idp --- IMPLEMENTATION_COVERAGE.md | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 17b864dc3..b8c8dd1b7 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -831,16 +831,16 @@ - [ ] add_custom_attributes - [ ] admin_add_user_to_group - [ ] admin_confirm_sign_up -- [ ] admin_create_user -- [ ] admin_delete_user +- [X] admin_create_user +- [X] admin_delete_user - [ ] admin_delete_user_attributes - [ ] admin_disable_provider_for_user - [ ] admin_disable_user - [ ] admin_enable_user - [ ] admin_forget_device - [ ] admin_get_device -- [ ] admin_get_user -- [ ] admin_initiate_auth +- [X] admin_get_user +- [X] admin_initiate_auth - [ ] admin_link_provider_for_user - [ ] admin_list_devices - [ ] admin_list_groups_for_user @@ -855,32 +855,32 @@ - [ ] admin_update_user_attributes - [ ] admin_user_global_sign_out - [ ] associate_software_token -- [ ] change_password +- [X] change_password - [ ] confirm_device -- [ ] confirm_forgot_password +- [X] confirm_forgot_password - [ ] confirm_sign_up - [ ] create_group -- [ ] create_identity_provider +- [X] create_identity_provider - [ ] create_resource_server - [ ] create_user_import_job -- [ ] create_user_pool -- [ ] create_user_pool_client -- [ ] create_user_pool_domain +- [X] create_user_pool +- [X] create_user_pool_client +- [X] create_user_pool_domain - [ ] delete_group -- [ ] delete_identity_provider +- [X] delete_identity_provider - [ ] delete_resource_server - [ ] delete_user - [ ] delete_user_attributes -- [ ] delete_user_pool -- [ ] delete_user_pool_client -- [ ] delete_user_pool_domain -- [ ] describe_identity_provider +- [X] delete_user_pool +- [X] delete_user_pool_client +- [X] delete_user_pool_domain +- [X] describe_identity_provider - [ ] describe_resource_server - [ ] describe_risk_configuration - [ ] describe_user_import_job -- [ ] describe_user_pool -- [ ] describe_user_pool_client -- [ ] describe_user_pool_domain +- [X] describe_user_pool +- [X] describe_user_pool_client +- [X] describe_user_pool_domain - [ ] forget_device - [ ] forgot_password - [ ] get_csv_header @@ -896,15 +896,15 @@ - [ ] initiate_auth - [ ] list_devices - [ ] list_groups -- [ ] list_identity_providers +- [X] list_identity_providers - [ ] list_resource_servers - [ ] list_user_import_jobs -- [ ] list_user_pool_clients -- [ ] list_user_pools -- [ ] list_users +- [X] list_user_pool_clients +- [X] list_user_pools +- [X] list_users - [ ] list_users_in_group - [ ] resend_confirmation_code -- [ ] respond_to_auth_challenge +- [X] respond_to_auth_challenge - [ ] set_risk_configuration - [ ] set_ui_customization - [ ] set_user_mfa_preference @@ -920,7 +920,7 @@ - [ ] update_resource_server - [ ] update_user_attributes - [ ] update_user_pool -- [ ] update_user_pool_client +- [X] update_user_pool_client - [ ] verify_software_token - [ ] verify_user_attribute From 04fdd5617a2131dc68d2f7461718b70e62215e93 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 23 Jul 2018 16:26:54 +1000 Subject: [PATCH 02/64] Implement group management for cognito-idp --- IMPLEMENTATION_COVERAGE.md | 8 +- moto/cognitoidp/exceptions.py | 10 +++ moto/cognitoidp/models.py | 67 ++++++++++++++++- moto/cognitoidp/responses.py | 41 ++++++++++ tests/test_cognitoidp/test_cognitoidp.py | 96 +++++++++++++++++++++++- 5 files changed, 214 insertions(+), 8 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index b8c8dd1b7..cceaa3c8a 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -859,14 +859,14 @@ - [ ] confirm_device - [X] confirm_forgot_password - [ ] confirm_sign_up -- [ ] create_group +- [X] create_group - [X] create_identity_provider - [ ] create_resource_server - [ ] create_user_import_job - [X] create_user_pool - [X] create_user_pool_client - [X] create_user_pool_domain -- [ ] delete_group +- [X] delete_group - [X] delete_identity_provider - [ ] delete_resource_server - [ ] delete_user @@ -885,7 +885,7 @@ - [ ] forgot_password - [ ] get_csv_header - [ ] get_device -- [ ] get_group +- [X] get_group - [ ] get_identity_provider_by_identifier - [ ] get_signing_certificate - [ ] get_ui_customization @@ -895,7 +895,7 @@ - [ ] global_sign_out - [ ] initiate_auth - [ ] list_devices -- [ ] list_groups +- [X] list_groups - [X] list_identity_providers - [ ] list_resource_servers - [ ] list_user_import_jobs diff --git a/moto/cognitoidp/exceptions.py b/moto/cognitoidp/exceptions.py index 1f1ec2309..452670213 100644 --- a/moto/cognitoidp/exceptions.py +++ b/moto/cognitoidp/exceptions.py @@ -24,6 +24,16 @@ class UserNotFoundError(BadRequest): }) +class GroupExistsException(BadRequest): + + def __init__(self, message): + super(GroupExistsException, self).__init__() + self.description = json.dumps({ + "message": message, + '__type': 'GroupExistsException', + }) + + class NotAuthorizedError(BadRequest): def __init__(self, message): diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 10da0c6ff..a11eb435c 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -11,8 +11,7 @@ from jose import jws from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel -from .exceptions import NotAuthorizedError, ResourceNotFoundError, UserNotFoundError - +from .exceptions import GroupExistsException, NotAuthorizedError, ResourceNotFoundError, UserNotFoundError UserStatus = { "FORCE_CHANGE_PASSWORD": "FORCE_CHANGE_PASSWORD", @@ -33,6 +32,7 @@ class CognitoIdpUserPool(BaseModel): self.clients = OrderedDict() self.identity_providers = OrderedDict() + self.groups = OrderedDict() self.users = OrderedDict() self.refresh_tokens = {} self.access_tokens = {} @@ -185,6 +185,29 @@ class CognitoIdpIdentityProvider(BaseModel): return identity_provider_json +class CognitoIdpGroup(BaseModel): + + def __init__(self, user_pool_id, group_name, description, role_arn, precedence): + self.user_pool_id = user_pool_id + self.group_name = group_name + self.description = description or "" + self.role_arn = role_arn + self.precedence = precedence + self.last_modified_date = datetime.datetime.now() + self.creation_date = self.last_modified_date + + def to_json(self): + return { + "GroupName": self.group_name, + "UserPoolId": self.user_pool_id, + "Description": self.description, + "RoleArn": self.role_arn, + "Precedence": self.precedence, + "LastModifiedDate": time.mktime(self.last_modified_date.timetuple()), + "CreationDate": time.mktime(self.creation_date.timetuple()), + } + + class CognitoIdpUser(BaseModel): def __init__(self, user_pool_id, username, password, status, attributes): @@ -367,6 +390,46 @@ class CognitoIdpBackend(BaseBackend): del user_pool.identity_providers[name] + # Group + def create_group(self, user_pool_id, group_name, description, role_arn, precedence): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + group = CognitoIdpGroup(user_pool_id, group_name, description, role_arn, precedence) + if group.group_name in user_pool.groups: + raise GroupExistsException("A group with the name already exists") + user_pool.groups[group.group_name] = group + + return group + + def get_group(self, user_pool_id, group_name): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + if group_name not in user_pool.groups: + raise ResourceNotFoundError(group_name) + + return user_pool.groups[group_name] + + def list_groups(self, user_pool_id): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + return user_pool.groups.values() + + def delete_group(self, user_pool_id, group_name): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + if group_name not in user_pool.groups: + raise ResourceNotFoundError(group_name) + + del user_pool.groups[group_name] + # User def admin_create_user(self, user_pool_id, username, temporary_password, attributes): user_pool = self.user_pools.get(user_pool_id) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index e6f20367e..a2f67ba84 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -129,6 +129,47 @@ class CognitoIdpResponse(BaseResponse): cognitoidp_backends[self.region].delete_identity_provider(user_pool_id, name) return "" + # Group + def create_group(self): + group_name = self._get_param("GroupName") + user_pool_id = self._get_param("UserPoolId") + description = self._get_param("Description") + role_arn = self._get_param("RoleArn") + precedence = self._get_param("Precedence") + + group = cognitoidp_backends[self.region].create_group( + user_pool_id, + group_name, + description, + role_arn, + precedence, + ) + + return json.dumps({ + "Group": group.to_json(), + }) + + def get_group(self): + group_name = self._get_param("GroupName") + user_pool_id = self._get_param("UserPoolId") + group = cognitoidp_backends[self.region].get_group(user_pool_id, group_name) + return json.dumps({ + "Group": group.to_json(), + }) + + def list_groups(self): + user_pool_id = self._get_param("UserPoolId") + groups = cognitoidp_backends[self.region].list_groups(user_pool_id) + return json.dumps({ + "Groups": [group.to_json() for group in groups], + }) + + def delete_group(self): + group_name = self._get_param("GroupName") + user_pool_id = self._get_param("UserPoolId") + cognitoidp_backends[self.region].delete_group(user_pool_id, group_name) + return "" + # User def admin_create_user(self): user_pool_id = self._get_param("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 56d7c08a8..c226f91b7 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -1,14 +1,18 @@ from __future__ import unicode_literals -import boto3 import json import os +import random import uuid +import boto3 +# noinspection PyUnresolvedReferences +import sure # noqa +from botocore.exceptions import ClientError from jose import jws +from nose.tools import assert_raises from moto import mock_cognitoidp -import sure # noqa @mock_cognitoidp @@ -323,6 +327,94 @@ def test_delete_identity_providers(): caught.should.be.true +@mock_cognitoidp +def test_create_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + description = str(uuid.uuid4()) + role_arn = "arn:aws:iam:::role/my-iam-role" + precedence = random.randint(0, 100000) + + result = conn.create_group( + GroupName=group_name, + UserPoolId=user_pool_id, + Description=description, + RoleArn=role_arn, + Precedence=precedence, + ) + + result["Group"]["GroupName"].should.equal(group_name) + result["Group"]["UserPoolId"].should.equal(user_pool_id) + result["Group"]["Description"].should.equal(description) + result["Group"]["RoleArn"].should.equal(role_arn) + result["Group"]["Precedence"].should.equal(precedence) + result["Group"]["LastModifiedDate"].should.be.a("datetime.datetime") + result["Group"]["CreationDate"].should.be.a("datetime.datetime") + + +@mock_cognitoidp +def test_create_group_with_duplicate_name_raises_error(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + with assert_raises(ClientError) as cm: + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + cm.exception.operation_name.should.equal('CreateGroup') + cm.exception.response['Error']['Code'].should.equal('GroupExistsException') + cm.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + + +@mock_cognitoidp +def test_get_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + result = conn.get_group(GroupName=group_name, UserPoolId=user_pool_id) + + result["Group"]["GroupName"].should.equal(group_name) + result["Group"]["UserPoolId"].should.equal(user_pool_id) + result["Group"]["LastModifiedDate"].should.be.a("datetime.datetime") + result["Group"]["CreationDate"].should.be.a("datetime.datetime") + + +@mock_cognitoidp +def test_list_groups(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + result = conn.list_groups(UserPoolId=user_pool_id) + + result["Groups"].should.have.length_of(1) + result["Groups"][0]["GroupName"].should.equal(group_name) + + +@mock_cognitoidp +def test_delete_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + + with assert_raises(ClientError) as cm: + conn.get_group(GroupName=group_name, UserPoolId=user_pool_id) + cm.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') + + @mock_cognitoidp def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") From ffa7560d026b9d0a3380b3c7cb3006d9e89af76a Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 23 Jul 2018 20:54:51 +1000 Subject: [PATCH 03/64] Implement user-group relationships for cognito-idp --- IMPLEMENTATION_COVERAGE.md | 10 +- moto/cognitoidp/models.py | 38 ++++++ moto/cognitoidp/responses.py | 42 +++++++ tests/test_cognitoidp/test_cognitoidp.py | 153 ++++++++++++++++++++++- 4 files changed, 237 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index cceaa3c8a..47420e8a4 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -827,9 +827,9 @@ - [ ] unlink_identity - [ ] update_identity_pool -## cognito-idp - 0% implemented +## cognito-idp - 34% implemented - [ ] add_custom_attributes -- [ ] admin_add_user_to_group +- [X] admin_add_user_to_group - [ ] admin_confirm_sign_up - [X] admin_create_user - [X] admin_delete_user @@ -843,9 +843,9 @@ - [X] admin_initiate_auth - [ ] admin_link_provider_for_user - [ ] admin_list_devices -- [ ] admin_list_groups_for_user +- [X] admin_list_groups_for_user - [ ] admin_list_user_auth_events -- [ ] admin_remove_user_from_group +- [X] admin_remove_user_from_group - [ ] admin_reset_user_password - [ ] admin_respond_to_auth_challenge - [ ] admin_set_user_mfa_preference @@ -902,7 +902,7 @@ - [X] list_user_pool_clients - [X] list_user_pools - [X] list_users -- [ ] list_users_in_group +- [X] list_users_in_group - [ ] resend_confirmation_code - [X] respond_to_auth_challenge - [ ] set_risk_configuration diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index a11eb435c..349d1e319 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -196,6 +196,10 @@ class CognitoIdpGroup(BaseModel): self.last_modified_date = datetime.datetime.now() self.creation_date = self.last_modified_date + # Users who are members of this group. + # Note that these links are bidirectional. + self.users = set() + def to_json(self): return { "GroupName": self.group_name, @@ -221,6 +225,10 @@ class CognitoIdpUser(BaseModel): self.create_date = datetime.datetime.utcnow() self.last_modified_date = datetime.datetime.utcnow() + # Groups this user is a member of. + # Note that these links are bidirectional. + self.groups = set() + def _base_json(self): return { "UserPoolId": self.user_pool_id, @@ -428,8 +436,34 @@ class CognitoIdpBackend(BaseBackend): if group_name not in user_pool.groups: raise ResourceNotFoundError(group_name) + group = user_pool.groups[group_name] + for user in group.users: + user.groups.remove(group) + del user_pool.groups[group_name] + def admin_add_user_to_group(self, user_pool_id, group_name, username): + group = self.get_group(user_pool_id, group_name) + user = self.admin_get_user(user_pool_id, username) + + group.users.add(user) + user.groups.add(group) + + def list_users_in_group(self, user_pool_id, group_name): + group = self.get_group(user_pool_id, group_name) + return list(group.users) + + def admin_list_groups_for_user(self, user_pool_id, username): + user = self.admin_get_user(user_pool_id, username) + return list(user.groups) + + def admin_remove_user_from_group(self, user_pool_id, group_name, username): + group = self.get_group(user_pool_id, group_name) + user = self.admin_get_user(user_pool_id, username) + + group.users.discard(user) + user.groups.discard(group) + # User def admin_create_user(self, user_pool_id, username, temporary_password, attributes): user_pool = self.user_pools.get(user_pool_id) @@ -465,6 +499,10 @@ class CognitoIdpBackend(BaseBackend): if username not in user_pool.users: raise ResourceNotFoundError(username) + user = user_pool.users[username] + for group in user.groups: + group.users.remove(user) + del user_pool.users[username] def _log_user_in(self, user_pool, client, username): diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index a2f67ba84..7d19fa3e8 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -170,6 +170,48 @@ class CognitoIdpResponse(BaseResponse): cognitoidp_backends[self.region].delete_group(user_pool_id, group_name) return "" + def admin_add_user_to_group(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + group_name = self._get_param("GroupName") + + cognitoidp_backends[self.region].admin_add_user_to_group( + user_pool_id, + group_name, + username, + ) + + return "" + + def list_users_in_group(self): + user_pool_id = self._get_param("UserPoolId") + group_name = self._get_param("GroupName") + users = cognitoidp_backends[self.region].list_users_in_group(user_pool_id, group_name) + return json.dumps({ + "Users": [user.to_json(extended=True) for user in users], + }) + + def admin_list_groups_for_user(self): + username = self._get_param("Username") + user_pool_id = self._get_param("UserPoolId") + groups = cognitoidp_backends[self.region].admin_list_groups_for_user(user_pool_id, username) + return json.dumps({ + "Groups": [group.to_json() for group in groups], + }) + + def admin_remove_user_from_group(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + group_name = self._get_param("GroupName") + + cognitoidp_backends[self.region].admin_remove_user_from_group( + user_pool_id, + group_name, + username, + ) + + return "" + # User def admin_create_user(self): user_pool_id = self._get_param("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index c226f91b7..07edc990c 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -408,13 +408,164 @@ def test_delete_group(): group_name = str(uuid.uuid4()) conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) - conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + result = conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected with assert_raises(ClientError) as cm: conn.get_group(GroupName=group_name, UserPoolId=user_pool_id) cm.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') +@mock_cognitoidp +def test_admin_add_user_to_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + result = conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + + +@mock_cognitoidp +def test_admin_add_user_to_group_again_is_noop(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + +@mock_cognitoidp +def test_list_users_in_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + result = conn.list_users_in_group(UserPoolId=user_pool_id, GroupName=group_name) + + result["Users"].should.have.length_of(1) + result["Users"][0]["Username"].should.equal(username) + + +@mock_cognitoidp +def test_list_users_in_group_ignores_deleted_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + username2 = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username2) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username2, GroupName=group_name) + conn.admin_delete_user(UserPoolId=user_pool_id, Username=username) + + result = conn.list_users_in_group(UserPoolId=user_pool_id, GroupName=group_name) + + result["Users"].should.have.length_of(1) + result["Users"][0]["Username"].should.equal(username2) + + +@mock_cognitoidp +def test_admin_list_groups_for_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + result = conn.admin_list_groups_for_user(Username=username, UserPoolId=user_pool_id) + + result["Groups"].should.have.length_of(1) + result["Groups"][0]["GroupName"].should.equal(group_name) + + +@mock_cognitoidp +def test_admin_list_groups_for_user_ignores_deleted_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + group_name2 = str(uuid.uuid4()) + conn.create_group(GroupName=group_name2, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name2) + conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + + result = conn.admin_list_groups_for_user(Username=username, UserPoolId=user_pool_id) + + result["Groups"].should.have.length_of(1) + result["Groups"][0]["GroupName"].should.equal(group_name2) + + +@mock_cognitoidp +def test_admin_remove_user_from_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + result = conn.admin_remove_user_from_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + conn.list_users_in_group(UserPoolId=user_pool_id, GroupName=group_name) \ + ["Users"].should.have.length_of(0) + conn.admin_list_groups_for_user(Username=username, UserPoolId=user_pool_id) \ + ["Groups"].should.have.length_of(0) + + +@mock_cognitoidp +def test_admin_remove_user_from_group_again_is_noop(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + @mock_cognitoidp def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") From b4c44a820f48d22090e131e27ae8f58738649ecb Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Fri, 19 Oct 2018 17:09:19 -0400 Subject: [PATCH 04/64] Add a long description to setup Include the readme as the `long_description` argument to `setup` and set its content type appropriately. This allows PyPI to render the content correctly. --- setup.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup.py b/setup.py index 98780dd5a..17e585912 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,23 @@ #!/usr/bin/env python from __future__ import unicode_literals +import codecs +import os +import re import setuptools from setuptools import setup, find_packages import sys +# Borrowed from pip at https://github.com/pypa/pip/blob/62c27dee45625e1b63d1e023b0656310f276e050/setup.py#L11-L15 +here = os.path.abspath(os.path.dirname(__file__)) + +def read(*parts): + # intentionally *not* adding an encoding option to open, See: + # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 + with codecs.open(os.path.join(here, *parts), 'r') as fp: + return fp.read() + + install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", @@ -43,6 +56,8 @@ setup( version='1.3.6', description='A library that allows your python tests to easily' ' mock out the boto library', + long_description=read('README.md'), + long_description_content_type='text/markdown', author='Steve Pulec', author_email='spulec@gmail.com', url='https://github.com/spulec/moto', From ede02e2c2a319fbbe5921800af5aae680b01ca15 Mon Sep 17 00:00:00 2001 From: Adam Davis Date: Mon, 22 Oct 2018 13:37:52 -0700 Subject: [PATCH 05/64] Without double quotes, pip would not install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 791226d6b..d6e9f30a1 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ It uses flask, which isn't a default dependency. You can install the server 'extra' package with: ```python -pip install moto[server] +pip install "moto[server]" ``` You can then start it running a service: From 544050ab270e45af6a87d43809f3330592cb057c Mon Sep 17 00:00:00 2001 From: Drew Pearce Date: Tue, 23 Oct 2018 15:37:28 -0400 Subject: [PATCH 06/64] added tests for dynamodb not equals --- tests/test_dynamodb2/test_dynamodb.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index afc919dd7..ad73de41b 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -815,6 +815,16 @@ def test_scan_filter(): ) assert response['Count'] == 1 + response = table.scan( + FilterExpression=Attr('app').ne('app2') + ) + assert response['Count'] == 1 + + response = table.scan( + FilterExpression=Attr('app').ne('app1') + ) + assert response['Count'] == 0 + @mock_dynamodb2 def test_scan_filter2(): @@ -872,6 +882,26 @@ def test_scan_filter3(): ) assert response['Count'] == 1 + response = table.scan( + FilterExpression=Attr('active').ne(True) + ) + assert response['Count'] == 0 + + response = table.scan( + FilterExpression=Attr('active').ne(False) + ) + assert response['Count'] == 1 + + response = table.scan( + FilterExpression=Attr('app').ne(1) + ) + assert response['Count'] == 0 + + response = table.scan( + FilterExpression=Attr('app').ne(2) + ) + assert response['Count'] == 1 + @mock_dynamodb2 def test_scan_filter4(): From ecc7c244675c1054e89f7b2efef2e5b0c0fcc45e Mon Sep 17 00:00:00 2001 From: Drew Pearce Date: Tue, 23 Oct 2018 15:54:56 -0400 Subject: [PATCH 07/64] simple fix for not equals in dynamodb filter expressions. i suspect this was just a typo --- moto/dynamodb2/comparisons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 53226c557..6d37345fe 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -383,7 +383,7 @@ class OpNotEqual(Op): def expr(self, item): lhs = self._lhs(item) rhs = self._rhs(item) - return lhs == rhs + return lhs != rhs class OpLessThanOrEqual(Op): From aa4be6fcad087139f2087c9794def59de1c118d3 Mon Sep 17 00:00:00 2001 From: George Alton Date: Wed, 24 Oct 2018 15:10:28 +0100 Subject: [PATCH 08/64] Adds limiting/pagination to cognitoidp list_* functions --- moto/cognitoidp/models.py | 50 ++++- moto/cognitoidp/responses.py | 53 +++-- tests/test_cognitoidp/test_cognitoidp.py | 244 +++++++++++++++++++++++ 3 files changed, 329 insertions(+), 18 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 476d470b9..521701de7 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import datetime +import functools import json import os import time @@ -20,6 +21,43 @@ UserStatus = { } +def paginate(limit, start_arg="next_token", limit_arg="max_results"): + """Returns a limited result list, and an offset into list of remaining items + + Takes the next_token, and max_results kwargs given to a function and handles + the slicing of the results. The kwarg `next_token` is the offset into the + list to begin slicing from. `max_results` is the size of the result required + + If the max_results is not supplied then the `limit` parameter is used as a + default + + :param limit_arg: the name of argument in the decorated function that + controls amount of items returned + :param start_arg: the name of the argument in the decorated that provides + the starting offset + :param limit: A default maximum items to return + :return: a tuple containing a list of items, and the offset into the list + """ + default_start = 0 + + def outer_wrapper(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + # setup + start = int(default_start if kwargs.get(start_arg) is None else kwargs[start_arg]) + stop = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg]) + end = start + stop + # call + result = func(*args, **kwargs) + # modify + results = list(result) + limited_results = results[start: end] + next_token = end if end < len(results) else None + return limited_results, next_token + return wrapper + return outer_wrapper + + class CognitoIdpUserPool(BaseModel): def __init__(self, region, name, extended_config): @@ -242,7 +280,8 @@ class CognitoIdpBackend(BaseBackend): self.user_pools[user_pool.id] = user_pool return user_pool - def list_user_pools(self): + @paginate(60) + def list_user_pools(self, max_results=None, next_token=None): return self.user_pools.values() def describe_user_pool(self, user_pool_id): @@ -289,7 +328,8 @@ class CognitoIdpBackend(BaseBackend): user_pool.clients[user_pool_client.id] = user_pool_client return user_pool_client - def list_user_pool_clients(self, user_pool_id): + @paginate(60) + def list_user_pool_clients(self, user_pool_id, max_results=None, next_token=None): user_pool = self.user_pools.get(user_pool_id) if not user_pool: raise ResourceNotFoundError(user_pool_id) @@ -339,7 +379,8 @@ class CognitoIdpBackend(BaseBackend): user_pool.identity_providers[name] = identity_provider return identity_provider - def list_identity_providers(self, user_pool_id): + @paginate(60) + def list_identity_providers(self, user_pool_id, max_results=None, next_token=None): user_pool = self.user_pools.get(user_pool_id) if not user_pool: raise ResourceNotFoundError(user_pool_id) @@ -387,7 +428,8 @@ class CognitoIdpBackend(BaseBackend): return user_pool.users[username] - def list_users(self, user_pool_id): + @paginate(60, "pagination_token", "limit") + def list_users(self, user_pool_id, pagination_token=None, limit=None): user_pool = self.user_pools.get(user_pool_id) if not user_pool: raise ResourceNotFoundError(user_pool_id) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index 50939786b..cb5374a6b 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -22,10 +22,17 @@ class CognitoIdpResponse(BaseResponse): }) def list_user_pools(self): - user_pools = cognitoidp_backends[self.region].list_user_pools() - return json.dumps({ - "UserPools": [user_pool.to_json() for user_pool in user_pools] - }) + max_results = self._get_param("MaxResults") + next_token = self._get_param("NextToken", "0") + user_pools, next_token = cognitoidp_backends[self.region].list_user_pools( + max_results=max_results, next_token=next_token + ) + response = { + "UserPools": [user_pool.to_json() for user_pool in user_pools], + } + if next_token: + response["NextToken"] = str(next_token) + return json.dumps(response) def describe_user_pool(self): user_pool_id = self._get_param("UserPoolId") @@ -72,10 +79,16 @@ class CognitoIdpResponse(BaseResponse): def list_user_pool_clients(self): user_pool_id = self._get_param("UserPoolId") - user_pool_clients = cognitoidp_backends[self.region].list_user_pool_clients(user_pool_id) - return json.dumps({ + max_results = self._get_param("MaxResults") + next_token = self._get_param("NextToken", "0") + user_pool_clients, next_token = cognitoidp_backends[self.region].list_user_pool_clients(user_pool_id, + max_results=max_results, next_token=next_token) + response = { "UserPoolClients": [user_pool_client.to_json() for user_pool_client in user_pool_clients] - }) + } + if next_token: + response["NextToken"] = str(next_token) + return json.dumps(response) def describe_user_pool_client(self): user_pool_id = self._get_param("UserPoolId") @@ -110,10 +123,17 @@ class CognitoIdpResponse(BaseResponse): def list_identity_providers(self): user_pool_id = self._get_param("UserPoolId") - identity_providers = cognitoidp_backends[self.region].list_identity_providers(user_pool_id) - return json.dumps({ + max_results = self._get_param("MaxResults") + next_token = self._get_param("NextToken", "0") + identity_providers, next_token = cognitoidp_backends[self.region].list_identity_providers( + user_pool_id, max_results=max_results, next_token=next_token + ) + response = { "Providers": [identity_provider.to_json() for identity_provider in identity_providers] - }) + } + if next_token: + response["NextToken"] = str(next_token) + return json.dumps(response) def describe_identity_provider(self): user_pool_id = self._get_param("UserPoolId") @@ -155,10 +175,15 @@ class CognitoIdpResponse(BaseResponse): def list_users(self): user_pool_id = self._get_param("UserPoolId") - users = cognitoidp_backends[self.region].list_users(user_pool_id) - return json.dumps({ - "Users": [user.to_json(extended=True) for user in users] - }) + limit = self._get_param("Limit") + token = self._get_param("PaginationToken") + users, token = cognitoidp_backends[self.region].list_users(user_pool_id, + limit=limit, + pagination_token=token) + response = {"Users": [user.to_json(extended=True) for user in users]} + if token: + response["PaginationToken"] = str(token) + return json.dumps(response) def admin_disable_user(self): user_pool_id = self._get_param("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index f72a44762..362740ce3 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -41,6 +41,56 @@ def test_list_user_pools(): result["UserPools"][0]["Name"].should.equal(name) +@mock_cognitoidp +def test_list_user_pools_returns_max_items(): + conn = boto3.client("cognito-idp", "us-west-2") + + # Given 10 user pools + pool_count = 10 + for i in range(pool_count): + conn.create_user_pool(PoolName=str(uuid.uuid4())) + + max_results = 5 + result = conn.list_user_pools(MaxResults=max_results) + result["UserPools"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pools_returns_next_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + + # Given 10 user pool clients + pool_count = 10 + for i in range(pool_count): + conn.create_user_pool(PoolName=str(uuid.uuid4())) + + max_results = 5 + result = conn.list_user_pools(MaxResults=max_results) + result["UserPools"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + next_token = result["NextToken"] + result_2 = conn.list_user_pools(MaxResults=max_results, NextToken=next_token) + result_2["UserPools"].should.have.length_of(max_results) + result_2.shouldnt.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pools_when_max_items_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + + # Given 10 user pool clients + pool_count = 10 + for i in range(pool_count): + conn.create_user_pool(PoolName=str(uuid.uuid4())) + + max_results = pool_count + 5 + result = conn.list_user_pools(MaxResults=max_results) + result["UserPools"].should.have.length_of(pool_count) + result.shouldnt.have.key("NextToken") + + @mock_cognitoidp def test_describe_user_pool(): conn = boto3.client("cognito-idp", "us-west-2") @@ -140,6 +190,67 @@ def test_list_user_pool_clients(): result["UserPoolClients"][0]["ClientName"].should.equal(client_name) +@mock_cognitoidp +def test_list_user_pool_clients_returns_max_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 user pool clients + client_count = 10 + for i in range(client_count): + client_name = str(uuid.uuid4()) + conn.create_user_pool_client(UserPoolId=user_pool_id, + ClientName=client_name) + max_results = 5 + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results) + result["UserPoolClients"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pool_clients_returns_next_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 user pool clients + client_count = 10 + for i in range(client_count): + client_name = str(uuid.uuid4()) + conn.create_user_pool_client(UserPoolId=user_pool_id, + ClientName=client_name) + max_results = 5 + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results) + result["UserPoolClients"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + next_token = result["NextToken"] + result_2 = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results, + NextToken=next_token) + result_2["UserPoolClients"].should.have.length_of(max_results) + result_2.shouldnt.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pool_clients_when_max_items_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 user pool clients + client_count = 10 + for i in range(client_count): + client_name = str(uuid.uuid4()) + conn.create_user_pool_client(UserPoolId=user_pool_id, + ClientName=client_name) + max_results = client_count + 5 + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results) + result["UserPoolClients"].should.have.length_of(client_count) + result.shouldnt.have.key("NextToken") + + @mock_cognitoidp def test_describe_user_pool_client(): conn = boto3.client("cognito-idp", "us-west-2") @@ -264,6 +375,83 @@ def test_list_identity_providers(): result["Providers"][0]["ProviderType"].should.equal(provider_type) +@mock_cognitoidp +def test_list_identity_providers_returns_max_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 identity providers linked to a user pool + identity_provider_count = 10 + for i in range(identity_provider_count): + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + max_results = 5 + result = conn.list_identity_providers(UserPoolId=user_pool_id, + MaxResults=max_results) + result["Providers"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + +@mock_cognitoidp +def test_list_identity_providers_returns_next_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 identity providers linked to a user pool + identity_provider_count = 10 + for i in range(identity_provider_count): + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + max_results = 5 + result = conn.list_identity_providers(UserPoolId=user_pool_id, MaxResults=max_results) + result["Providers"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + next_token = result["NextToken"] + result_2 = conn.list_identity_providers(UserPoolId=user_pool_id, + MaxResults=max_results, + NextToken=next_token) + result_2["Providers"].should.have.length_of(max_results) + result_2.shouldnt.have.key("NextToken") + + +@mock_cognitoidp +def test_list_identity_providers_when_max_items_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 identity providers linked to a user pool + identity_provider_count = 10 + for i in range(identity_provider_count): + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + max_results = identity_provider_count + 5 + result = conn.list_identity_providers(UserPoolId=user_pool_id, MaxResults=max_results) + result["Providers"].should.have.length_of(identity_provider_count) + result.shouldnt.have.key("NextToken") + + @mock_cognitoidp def test_describe_identity_providers(): conn = boto3.client("cognito-idp", "us-west-2") @@ -396,6 +584,62 @@ def test_list_users(): result["Users"][0]["Username"].should.equal(username) +@mock_cognitoidp +def test_list_users_returns_limit_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 users + user_count = 10 + for i in range(user_count): + conn.admin_create_user(UserPoolId=user_pool_id, + Username=str(uuid.uuid4())) + max_results = 5 + result = conn.list_users(UserPoolId=user_pool_id, Limit=max_results) + result["Users"].should.have.length_of(max_results) + result.should.have.key("PaginationToken") + + +@mock_cognitoidp +def test_list_users_returns_pagination_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 users + user_count = 10 + for i in range(user_count): + conn.admin_create_user(UserPoolId=user_pool_id, + Username=str(uuid.uuid4())) + + max_results = 5 + result = conn.list_users(UserPoolId=user_pool_id, Limit=max_results) + result["Users"].should.have.length_of(max_results) + result.should.have.key("PaginationToken") + + next_token = result["PaginationToken"] + result_2 = conn.list_users(UserPoolId=user_pool_id, + Limit=max_results, PaginationToken=next_token) + result_2["Users"].should.have.length_of(max_results) + result_2.shouldnt.have.key("PaginationToken") + + +@mock_cognitoidp +def test_list_users_when_limit_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 users + user_count = 10 + for i in range(user_count): + conn.admin_create_user(UserPoolId=user_pool_id, + Username=str(uuid.uuid4())) + + max_results = user_count + 5 + result = conn.list_users(UserPoolId=user_pool_id, Limit=max_results) + result["Users"].should.have.length_of(user_count) + result.shouldnt.have.key("PaginationToken") + + @mock_cognitoidp def test_admin_disable_user(): conn = boto3.client("cognito-idp", "us-west-2") From 80f860727fbacece3717d258d9079faae8fc5334 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Thu, 25 Oct 2018 20:34:53 +0900 Subject: [PATCH 09/64] Add support for IoT attach_policy --- IMPLEMENTATION_COVERAGE.md | 4 ++-- moto/iot/models.py | 8 ++++++++ moto/iot/responses.py | 9 +++++++++ tests/test_iot/test_iot.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7c68c0e31..7a1e2e7aa 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2376,11 +2376,11 @@ - [ ] unsubscribe_from_event - [ ] update_assessment_target -## iot - 30% implemented +## iot - 31% implemented - [ ] accept_certificate_transfer - [X] add_thing_to_thing_group - [ ] associate_targets_with_job -- [ ] attach_policy +- [X] attach_policy - [X] attach_principal_policy - [X] attach_thing_principal - [ ] cancel_certificate_transfer diff --git a/moto/iot/models.py b/moto/iot/models.py index c36bb985f..db9ad3817 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -429,6 +429,14 @@ class IoTBackend(BaseBackend): pass raise ResourceNotFoundException() + def attach_policy(self, policy_name, target): + principal = self._get_principal(target) + policy = self.get_policy(policy_name) + k = (target, policy_name) + if k in self.principal_policies: + return + self.principal_policies[k] = (principal, policy) + def attach_principal_policy(self, policy_name, principal_arn): principal = self._get_principal(principal_arn) policy = self.get_policy(policy_name) diff --git a/moto/iot/responses.py b/moto/iot/responses.py index 006c4c4cc..042e5a314 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -224,6 +224,15 @@ class IoTResponse(BaseResponse): ) return json.dumps(dict()) + def attach_policy(self): + policy_name = self._get_param("policyName") + target = self._get_param('target') + self.iot_backend.attach_policy( + policy_name=policy_name, + target=target, + ) + return json.dumps(dict()) + def attach_principal_policy(self): policy_name = self._get_param("policyName") principal = self.headers.get('x-amzn-iot-principal') diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 5c6effd7a..9082203d9 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -309,6 +309,35 @@ def test_policy(): @mock_iot def test_principal_policy(): + client = boto3.client('iot', region_name='ap-northeast-1') + policy_name = 'my-policy' + doc = '{}' + client.create_policy(policyName=policy_name, policyDocument=doc) + cert = client.create_keys_and_certificate(setAsActive=True) + cert_arn = cert['certificateArn'] + + client.attach_policy(policyName=policy_name, target=cert_arn) + + res = client.list_principal_policies(principal=cert_arn) + res.should.have.key('policies').which.should.have.length_of(1) + for policy in res['policies']: + policy.should.have.key('policyName').which.should_not.be.none + policy.should.have.key('policyArn').which.should_not.be.none + + res = client.list_policy_principals(policyName=policy_name) + res.should.have.key('principals').which.should.have.length_of(1) + for principal in res['principals']: + principal.should_not.be.none + + client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + res = client.list_principal_policies(principal=cert_arn) + res.should.have.key('policies').which.should.have.length_of(0) + res = client.list_policy_principals(policyName=policy_name) + res.should.have.key('principals').which.should.have.length_of(0) + + +@mock_iot +def test_principal_policy_deprecated(): client = boto3.client('iot', region_name='ap-northeast-1') policy_name = 'my-policy' doc = '{}' From b485122ec6e3bae5f80dc39742b323f4eaf5ac06 Mon Sep 17 00:00:00 2001 From: zane Date: Wed, 24 Oct 2018 14:06:23 -0700 Subject: [PATCH 10/64] refactor to store multiple scrects, use uuid --- .gitignore | 2 +- moto/secretsmanager/models.py | 79 ++++++++++--------- moto/secretsmanager/utils.py | 5 +- .../test_secretsmanager.py | 18 +++-- tests/test_secretsmanager/test_server.py | 41 ++++++++-- 5 files changed, 91 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index 7f57e98e9..cfda51440 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ python_env .ropeproject/ .pytest_cache/ venv/ - +.vscode/ diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 1404a0ec8..7f89e2eb6 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import time import json +import uuid import boto3 @@ -18,10 +19,6 @@ class SecretsManager(BaseModel): def __init__(self, region_name, **kwargs): self.region = region_name - self.secret_id = kwargs.get('secret_id', '') - self.version_id = kwargs.get('version_id', '') - self.version_stage = kwargs.get('version_stage', '') - self.secret_string = '' class SecretsManagerBackend(BaseBackend): @@ -29,14 +26,7 @@ class SecretsManagerBackend(BaseBackend): def __init__(self, region_name=None, **kwargs): super(SecretsManagerBackend, self).__init__() self.region = region_name - self.secret_id = kwargs.get('secret_id', '') - self.name = kwargs.get('name', '') - self.createdate = int(time.time()) - self.secret_string = '' - self.rotation_enabled = False - self.rotation_lambda_arn = '' - self.auto_rotate_after_days = 0 - self.version_id = '' + self.secrets = {} def reset(self): region_name = self.region @@ -44,36 +34,49 @@ class SecretsManagerBackend(BaseBackend): self.__init__(region_name) def _is_valid_identifier(self, identifier): - return identifier in (self.name, self.secret_id) + return identifier in self.secrets def get_secret_value(self, secret_id, version_id, version_stage): if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException() + secret = self.secrets[secret_id] + response = json.dumps({ - "ARN": secret_arn(self.region, self.secret_id), - "Name": self.name, - "VersionId": "A435958A-D821-4193-B719-B7769357AER4", - "SecretString": self.secret_string, + "ARN": secret_arn(self.region, secret['secret_id']), + "Name": secret['name'], + "VersionId": secret['version_id'], + "SecretString": secret['secret_string'], "VersionStages": [ "AWSCURRENT", ], - "CreatedDate": "2018-05-23 13:16:57.198000" + "CreatedDate": secret['createdate'] }) return response def create_secret(self, name, secret_string, **kwargs): - self.secret_string = secret_string - self.secret_id = name - self.name = name + generated_version_id = str(uuid.uuid4()) + + secret = { + 'secret_string': secret_string, + 'secret_id': name, + 'name': name, + 'createdate': int(time.time()), + 'rotation_enabled': False, + 'rotation_lambda_arn': '', + 'auto_rotate_after_days': 0, + 'version_id': generated_version_id + } + + self.secrets[name] = secret response = json.dumps({ "ARN": secret_arn(self.region, name), - "Name": self.name, - "VersionId": "A435958A-D821-4193-B719-B7769357AER4", + "Name": name, + "VersionId": generated_version_id, }) return response @@ -82,15 +85,17 @@ class SecretsManagerBackend(BaseBackend): if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException + secret = self.secrets[secret_id] + response = json.dumps({ - "ARN": secret_arn(self.region, self.secret_id), - "Name": self.name, + "ARN": secret_arn(self.region, secret['secret_id']), + "Name": secret['name'], "Description": "", "KmsKeyId": "", - "RotationEnabled": self.rotation_enabled, - "RotationLambdaARN": self.rotation_lambda_arn, + "RotationEnabled": secret['rotation_enabled'], + "RotationLambdaARN": secret['rotation_lambda_arn'], "RotationRules": { - "AutomaticallyAfterDays": self.auto_rotate_after_days + "AutomaticallyAfterDays": secret['auto_rotate_after_days'] }, "LastRotatedDate": None, "LastChangedDate": None, @@ -141,17 +146,19 @@ class SecretsManagerBackend(BaseBackend): ) raise InvalidParameterException(msg) - self.version_id = client_request_token or '' - self.rotation_lambda_arn = rotation_lambda_arn or '' + secret = self.secrets[secret_id] + + secret['version_id'] = client_request_token or '' + secret['rotation_lambda_arn'] = rotation_lambda_arn or '' if rotation_rules: - self.auto_rotate_after_days = rotation_rules.get(rotation_days, 0) - if self.auto_rotate_after_days > 0: - self.rotation_enabled = True + secret['auto_rotate_after_days'] = rotation_rules.get(rotation_days, 0) + if secret['auto_rotate_after_days'] > 0: + secret['rotation_enabled'] = True response = json.dumps({ - "ARN": secret_arn(self.region, self.secret_id), - "Name": self.name, - "VersionId": self.version_id + "ARN": secret_arn(self.region, secret['secret_id']), + "Name": secret['name'], + "VersionId": secret['version_id'] }) return response diff --git a/moto/secretsmanager/utils.py b/moto/secretsmanager/utils.py index 2cb92020a..231fea296 100644 --- a/moto/secretsmanager/utils.py +++ b/moto/secretsmanager/utils.py @@ -52,8 +52,9 @@ def random_password(password_length, exclude_characters, exclude_numbers, def secret_arn(region, secret_id): - return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-rIjad".format( - region, secret_id) + id_string = ''.join(random.choice(string.ascii_letters) for _ in range(5)) + return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-{2}".format( + region, secret_id, id_string) def _exclude_characters(password, exclude_characters): diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index ec384a660..0e0b98b1e 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -39,8 +39,7 @@ def test_create_secret(): conn = boto3.client('secretsmanager', region_name='us-east-1') result = conn.create_secret(Name='test-secret', SecretString="foosecret") - assert result['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') + assert result['ARN'] assert result['Name'] == 'test-secret' secret = conn.get_secret_value(SecretId='test-secret') assert secret['SecretString'] == 'foosecret' @@ -159,10 +158,17 @@ def test_describe_secret(): conn.create_secret(Name='test-secret', SecretString='foosecret') + conn.create_secret(Name='test-secret-2', + SecretString='barsecret') + secret_description = conn.describe_secret(SecretId='test-secret') + secret_description_2 = conn.describe_secret(SecretId='test-secret-2') + assert secret_description # Returned dict is not empty - assert secret_description['ARN'] == ( - 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad') + assert secret_description['Name'] == ('test-secret') + assert secret_description['ARN'] != '' # Test arn not empty + assert secret_description_2['Name'] == ('test-secret-2') + assert secret_description_2['ARN'] != '' # Test arn not empty @mock_secretsmanager def test_describe_secret_that_does_not_exist(): @@ -190,9 +196,7 @@ def test_rotate_secret(): rotated_secret = conn.rotate_secret(SecretId=secret_name) assert rotated_secret - assert rotated_secret['ARN'] == ( - 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad' - ) + assert rotated_secret['ARN'] != '' # Test arn not empty assert rotated_secret['Name'] == secret_name assert rotated_secret['VersionId'] != '' diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index e573f9b67..d0f495f57 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -82,11 +82,20 @@ def test_create_secret(): headers={ "X-Amz-Target": "secretsmanager.CreateSecret"}, ) + res_2 = test_client.post('/', + data={"Name": "test-secret-2", + "SecretString": "bar-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) json_data = json.loads(res.data.decode("utf-8")) - assert json_data['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') + assert json_data['ARN'] != '' assert json_data['Name'] == 'test-secret' + + json_data_2 = json.loads(res_2.data.decode("utf-8")) + assert json_data_2['ARN'] != '' + assert json_data_2['Name'] == 'test-secret-2' @mock_secretsmanager def test_describe_secret(): @@ -107,12 +116,30 @@ def test_describe_secret(): "X-Amz-Target": "secretsmanager.DescribeSecret" }, ) + + create_secret_2 = test_client.post('/', + data={"Name": "test-secret-2", + "SecretString": "barsecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + describe_secret_2 = test_client.post('/', + data={"SecretId": "test-secret-2"}, + headers={ + "X-Amz-Target": "secretsmanager.DescribeSecret" + }, + ) json_data = json.loads(describe_secret.data.decode("utf-8")) assert json_data # Returned dict is not empty - assert json_data['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' - ) + assert json_data['ARN'] != '' + assert json_data['Name'] == 'test-secret' + + json_data_2 = json.loads(describe_secret_2.data.decode("utf-8")) + assert json_data_2 # Returned dict is not empty + assert json_data_2['ARN'] != '' + assert json_data_2['Name'] == 'test-secret-2' @mock_secretsmanager def test_describe_secret_that_does_not_exist(): @@ -179,9 +206,7 @@ def test_rotate_secret(): json_data = json.loads(rotate_secret.data.decode("utf-8")) assert json_data # Returned dict is not empty - assert json_data['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' - ) + assert json_data['ARN'] != '' assert json_data['Name'] == 'test-secret' assert json_data['VersionId'] == client_request_token From 5ca68fbf06adecb0c0478f102a9cc7d331aa36c4 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Fri, 26 Oct 2018 22:12:26 +0300 Subject: [PATCH 11/64] Update jsondiff to 1.1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 98780dd5a..34002b79d 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ install_requires = [ "python-jose<3.0.0", "mock", "docker>=2.5.1", - "jsondiff==1.1.1", + "jsondiff==1.1.2", "aws-xray-sdk<0.96,>=0.93", "responses>=0.9.0", ] From 090cad8c88169e3f6bd049d023d4e4d2fe309025 Mon Sep 17 00:00:00 2001 From: Jamie Starke Date: Fri, 26 Oct 2018 21:54:01 -0700 Subject: [PATCH 12/64] [1009] Converts ECS Instance full arn to instance_id for storage --- moto/ecs/models.py | 2 ++ tests/test_ecs/test_ecs_boto3.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index d00853843..4a6737ceb 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -769,6 +769,8 @@ class EC2ContainerServiceBackend(BaseBackend): Container instances status should be one of [ACTIVE,DRAINING]") failures = [] container_instance_objects = [] + list_container_instance_ids = [x.split('/')[-1] + for x in list_container_instance_ids] for container_instance_id in list_container_instance_ids: container_instance = self.container_instances[cluster_name].get(container_instance_id, None) if container_instance is not None: diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 70c1463ee..7274e62c1 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -910,6 +910,65 @@ def test_update_container_instances_state(): status='test_status').should.throw(Exception) +@mock_ec2 +@mock_ecs +def test_update_container_instances_state_by_arn(): + ecs_client = boto3.client('ecs', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + test_cluster_name = 'test_ecs_cluster' + _ = ecs_client.create_cluster( + clusterName=test_cluster_name + ) + + instance_to_create = 3 + test_instance_arns = [] + for i in range(0, instance_to_create): + test_instance = ec2.create_instances( + ImageId="ami-1234abcd", + MinCount=1, + MaxCount=1, + )[0] + + instance_id_document = json.dumps( + ec2_utils.generate_instance_identity_document(test_instance) + ) + + response = ecs_client.register_container_instance( + cluster=test_cluster_name, + instanceIdentityDocument=instance_id_document) + + test_instance_arns.append(response['containerInstance']['containerInstanceArn']) + + response = ecs_client.update_container_instances_state(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='DRAINING') + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_statuses = [ci['status'] for ci in response['containerInstances']] + for status in response_statuses: + status.should.equal('DRAINING') + response = ecs_client.update_container_instances_state(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='DRAINING') + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_statuses = [ci['status'] for ci in response['containerInstances']] + for status in response_statuses: + status.should.equal('DRAINING') + response = ecs_client.update_container_instances_state(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='ACTIVE') + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_statuses = [ci['status'] for ci in response['containerInstances']] + for status in response_statuses: + status.should.equal('ACTIVE') + ecs_client.update_container_instances_state.when.called_with(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='test_status').should.throw(Exception) + + @mock_ec2 @mock_ecs def test_run_task(): From 249dd7059e9c91712e459355126b0d4b57b25583 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Sun, 28 Oct 2018 17:13:17 +0900 Subject: [PATCH 13/64] add test case for IoT attach_policy do nothing if policy have already attached to certificate --- tests/test_iot/test_iot.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 9082203d9..47ea9d59b 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -318,6 +318,15 @@ def test_principal_policy(): client.attach_policy(policyName=policy_name, target=cert_arn) + res = client.list_principal_policies(principal=cert_arn) + res.should.have.key('policies').which.should.have.length_of(1) + for policy in res['policies']: + policy.should.have.key('policyName').which.should_not.be.none + policy.should.have.key('policyArn').which.should_not.be.none + + # do nothing if policy have already attached to certificate + client.attach_policy(policyName=policy_name, target=cert_arn) + res = client.list_principal_policies(principal=cert_arn) res.should.have.key('policies').which.should.have.length_of(1) for policy in res['policies']: From 9ba28a05b8b5aa709bde7188c6886f81155fd231 Mon Sep 17 00:00:00 2001 From: George Alton Date: Sun, 28 Oct 2018 11:00:21 +0000 Subject: [PATCH 14/64] avoids copying entire result into a new list --- moto/cognitoidp/models.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 521701de7..edeb7b128 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime import functools +import itertools import json import os import time @@ -43,16 +44,12 @@ def paginate(limit, start_arg="next_token", limit_arg="max_results"): def outer_wrapper(func): @functools.wraps(func) def wrapper(*args, **kwargs): - # setup start = int(default_start if kwargs.get(start_arg) is None else kwargs[start_arg]) - stop = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg]) - end = start + stop - # call + lim = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg]) + stop = start + lim result = func(*args, **kwargs) - # modify - results = list(result) - limited_results = results[start: end] - next_token = end if end < len(results) else None + limited_results = list(itertools.islice(result, start, stop)) + next_token = stop if stop < len(result) else None return limited_results, next_token return wrapper return outer_wrapper From a0708a70fcc1b265a25fccb01ec1ca67e409a5ce Mon Sep 17 00:00:00 2001 From: Mark Challoner Date: Mon, 29 Oct 2018 13:29:53 +0000 Subject: [PATCH 15/64] Fix Tags parameter on CloudFormation create_change_set method. --- moto/cloudformation/responses.py | 3 ++- .../test_cloudformation_stack_crud_boto3.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index a1295a20d..fe436939f 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -87,7 +87,8 @@ class CloudFormationResponse(BaseResponse): role_arn = self._get_param('RoleARN') update_or_create = self._get_param('ChangeSetType', 'CREATE') parameters_list = self._get_list_prefix("Parameters.member") - tags = {tag[0]: tag[1] for tag in self._get_list_prefix("Tags.member")} + tags = dict((item['key'], item['value']) + for item in self._get_list_prefix("Tags.member")) parameters = {param['parameter_key']: param['parameter_value'] for param in parameters_list} if template_url: diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 9bfae6174..064e0fb33 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -391,6 +391,9 @@ def test_create_change_set_from_s3_url(): TemplateURL=key_url, ChangeSetName='NewChangeSet', ChangeSetType='CREATE', + Tags=[ + {'Key': 'tag-key', 'Value': 'tag-value'} + ], ) assert 'arn:aws:cloudformation:us-west-1:123456789:changeSet/NewChangeSet/' in response['Id'] assert 'arn:aws:cloudformation:us-east-1:123456789:stack/NewStack' in response['StackId'] From 3d71a67794be7d4d3e2542d513f9e376ddb7635c Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Tue, 30 Oct 2018 14:38:59 +0900 Subject: [PATCH 16/64] Add some validations for IoT delete operations fix #1908 --- moto/iot/exceptions.py | 17 +++++++ moto/iot/models.py | 30 ++++++++++++- tests/test_iot/test_iot.py | 92 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/moto/iot/exceptions.py b/moto/iot/exceptions.py index 7bbdb706d..3af3751d9 100644 --- a/moto/iot/exceptions.py +++ b/moto/iot/exceptions.py @@ -31,3 +31,20 @@ class VersionConflictException(IoTClientError): 'VersionConflictException', 'The version for thing %s does not match the expected version.' % name ) + + +class CertificateStateException(IoTClientError): + def __init__(self, msg, cert_id): + self.code = 406 + super(CertificateStateException, self).__init__( + 'CertificateStateException', + '%s Id: %s' % (msg, cert_id) + ) + + +class DeleteConflictException(IoTClientError): + def __init__(self, msg): + self.code = 409 + super(DeleteConflictException, self).__init__( + 'DeleteConflictException', msg + ) diff --git a/moto/iot/models.py b/moto/iot/models.py index c36bb985f..4b0b260e1 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -13,6 +13,8 @@ import boto3 from moto.core import BaseBackend, BaseModel from .exceptions import ( + CertificateStateException, + DeleteConflictException, ResourceNotFoundException, InvalidRequestException, VersionConflictException @@ -378,7 +380,25 @@ class IoTBackend(BaseBackend): return certificate, key_pair def delete_certificate(self, certificate_id): - self.describe_certificate(certificate_id) + cert = self.describe_certificate(certificate_id) + if cert.status == 'ACTIVE': + raise CertificateStateException( + 'Certificate must be deactivated (not ACTIVE) before deletion.', certificate_id) + + certs = [k[0] for k, v in self.principal_things.items() + if self._get_principal(k[0]).certificate_id == certificate_id] + if len(certs) > 0: + raise DeleteConflictException( + 'Things must be detached before deletion (arn: %s)' % certs[0] + ) + + certs = [k[0] for k, v in self.principal_policies.items() + if self._get_principal(k[0]).certificate_id == certificate_id] + if len(certs) > 0: + raise DeleteConflictException( + 'Certificate policies must be detached before deletion (arn: %s)' % certs[0] + ) + del self.certificates[certificate_id] def describe_certificate(self, certificate_id): @@ -411,6 +431,14 @@ class IoTBackend(BaseBackend): return policies[0] def delete_policy(self, policy_name): + + policies = [k[1] for k, v in self.principal_policies.items() if k[1] == policy_name] + if len(policies) > 0: + raise DeleteConflictException( + 'The policy cannot be deleted as the policy is attached to one or more principals (name=%s)' + % policy_name + ) + policy = self.get_policy(policy_name) del self.policies[policy.name] diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 5c6effd7a..21b670d46 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -5,6 +5,8 @@ import sure # noqa import boto3 from moto import mock_iot +from botocore.exceptions import ClientError +from nose.tools import assert_raises @mock_iot @@ -261,6 +263,96 @@ def test_certs(): res.should.have.key('certificates').which.should.have.length_of(0) +@mock_iot +def test_delete_policy_validation(): + doc = """{ + "Version": "2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":[ + "iot: *" + ], + "Resource":"*" + } + ] + } + """ + client = boto3.client('iot', region_name='ap-northeast-1') + cert = client.create_keys_and_certificate(setAsActive=True) + cert_arn = cert['certificateArn'] + policy_name = 'my-policy' + client.create_policy(policyName=policy_name, policyDocument=doc) + client.attach_principal_policy(policyName=policy_name, principal=cert_arn) + + with assert_raises(ClientError) as e: + client.delete_policy(policyName=policy_name) + e.exception.response['Error']['Message'].should.contain( + 'The policy cannot be deleted as the policy is attached to one or more principals (name=%s)' % policy_name) + res = client.list_policies() + res.should.have.key('policies').which.should.have.length_of(1) + + client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + client.delete_policy(policyName=policy_name) + res = client.list_policies() + res.should.have.key('policies').which.should.have.length_of(0) + + +@mock_iot +def test_delete_certificate_validation(): + doc = """{ + "Version": "2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":[ + "iot: *" + ], + "Resource":"*" + } + ] + } + """ + client = boto3.client('iot', region_name='ap-northeast-1') + cert = client.create_keys_and_certificate(setAsActive=True) + cert_id = cert['certificateId'] + cert_arn = cert['certificateArn'] + policy_name = 'my-policy' + thing_name = 'thing-1' + client.create_policy(policyName=policy_name, policyDocument=doc) + client.attach_principal_policy(policyName=policy_name, principal=cert_arn) + client.create_thing(thingName=thing_name) + client.attach_thing_principal(thingName=thing_name, principal=cert_arn) + + with assert_raises(ClientError) as e: + client.delete_certificate(certificateId=cert_id) + e.exception.response['Error']['Message'].should.contain( + 'Certificate must be deactivated (not ACTIVE) before deletion.') + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(1) + + client.update_certificate(certificateId=cert_id, newStatus='REVOKED') + with assert_raises(ClientError) as e: + client.delete_certificate(certificateId=cert_id) + e.exception.response['Error']['Message'].should.contain( + 'Things must be detached before deletion (arn: %s)' % cert_arn) + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(1) + + client.detach_thing_principal(thingName=thing_name, principal=cert_arn) + with assert_raises(ClientError) as e: + client.delete_certificate(certificateId=cert_id) + e.exception.response['Error']['Message'].should.contain( + 'Certificate policies must be detached before deletion (arn: %s)' % cert_arn) + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(1) + + client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + client.delete_certificate(certificateId=cert_id) + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(0) + + @mock_iot def test_certs_create_inactive(): client = boto3.client('iot', region_name='ap-northeast-1') From e38eea751f55d2e139f3271a73f69d35ff64129b Mon Sep 17 00:00:00 2001 From: jamesandres Date: Wed, 31 Oct 2018 11:39:49 +0000 Subject: [PATCH 17/64] Go easier on the CPU when moto sqs is idle For our local development setup we have found that moto is using around 25% CPU constantly. Digging in with gdb it turned out that it was burning that CPU in the sleeping loop. Here i'm increasing the sleep by 10x which brings the idle CPU usage down by 10x (to ~2%). I'm not familiar enough with the moto/sqs codebase to know if lengthening this sleep will have an adverse effect; however, in other Python dev I've noticed that (in Python 2.7 anyway..) Python threading won't context switch a thread until a sleep of at least 0.01 seconds is performed (shockingly long!). So based on this guesswork I suspect sleeping for 0.01 seconds won't cause any grief. --- moto/sqs/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index f3262a988..1404ded75 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -534,7 +534,7 @@ class SQSBackend(BaseBackend): break import time - time.sleep(0.001) + time.sleep(0.01) continue previous_result_count = len(result) From 67cb2e25bbc365484806daccc97d56cc95d6afe1 Mon Sep 17 00:00:00 2001 From: Pall Valmundsson Date: Thu, 1 Nov 2018 19:51:17 +0000 Subject: [PATCH 18/64] Support IAM Credential Report in boto3 Lowercase XML element names in API responses seem to cause issues for boto3. --- moto/iam/responses.py | 8 ++++---- tests/test_iam/test_iam.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 22558f3f6..91ba09543 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1243,8 +1243,8 @@ LIST_ACCESS_KEYS_TEMPLATE = """ CREDENTIAL_REPORT_GENERATING = """ - STARTED - No report exists. Starting a new report generation task + STARTED + No report exists. Starting a new report generation task fa788a82-aa8a-11e4-a278-1786c418872b" @@ -1253,7 +1253,7 @@ CREDENTIAL_REPORT_GENERATING = """ CREDENTIAL_REPORT_GENERATED = """ - COMPLETE + COMPLETE fa788a82-aa8a-11e4-a278-1786c418872b" @@ -1262,7 +1262,7 @@ CREDENTIAL_REPORT_GENERATED = """ CREDENTIAL_REPORT = """ - {{ report }} + {{ report }} 2015-02-02T20:02:02Z text/csv diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index bc23ff712..34e198886 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -536,6 +536,14 @@ def test_generate_credential_report(): result['generate_credential_report_response'][ 'generate_credential_report_result']['state'].should.equal('COMPLETE') +@mock_iam +def test_generate_credential_report_boto3(): + conn = boto3.client('iam', region_name='us-east-1') + result = conn.generate_credential_report() + result['State'].should.equal('STARTED') + result = conn.generate_credential_report() + result['State'].should.equal('COMPLETE') + @mock_iam_deprecated() def test_get_credential_report(): @@ -551,6 +559,19 @@ def test_get_credential_report(): 'get_credential_report_result']['content'].encode('ascii')).decode('ascii') report.should.match(r'.*my-user.*') +@mock_iam +def test_get_credential_report(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_user(UserName='my-user') + with assert_raises(ClientError): + conn.get_credential_report() + result = conn.generate_credential_report() + while result['State'] != 'COMPLETE': + result = conn.generate_credential_report() + result = conn.get_credential_report() + report = result['Content'].decode('utf-8') + report.should.match(r'.*my-user.*') + @requires_boto_gte('2.39') @mock_iam_deprecated() From e9f8bec91a46c939fc8c23d3b6cac1de30383585 Mon Sep 17 00:00:00 2001 From: Pall Valmundsson Date: Thu, 1 Nov 2018 19:58:09 +0000 Subject: [PATCH 19/64] Fix boto3 IAM test function names --- tests/test_iam/test_iam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 34e198886..c33bd7fab 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -537,7 +537,7 @@ def test_generate_credential_report(): 'generate_credential_report_result']['state'].should.equal('COMPLETE') @mock_iam -def test_generate_credential_report_boto3(): +def test_boto3_generate_credential_report(): conn = boto3.client('iam', region_name='us-east-1') result = conn.generate_credential_report() result['State'].should.equal('STARTED') @@ -560,7 +560,7 @@ def test_get_credential_report(): report.should.match(r'.*my-user.*') @mock_iam -def test_get_credential_report(): +def test_boto3_get_credential_report(): conn = boto3.client('iam', region_name='us-east-1') conn.create_user(UserName='my-user') with assert_raises(ClientError): From b4b0ae50773f0cd9192f0c908e54010d924ed8ef Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Fri, 19 Oct 2018 16:40:58 -0700 Subject: [PATCH 20/64] Some IAM fixes. - Fixed InstanceProfiles having `Path` set to `None`. - Added in some dynamic `CreateDate`s. - Fixed missing Instance Profile ID's being sent over --- moto/iam/models.py | 2 + moto/iam/responses.py | 104 ++++++++++++++++++------------------- tests/test_iam/test_iam.py | 67 +++++++++++++----------- 3 files changed, 90 insertions(+), 83 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4a5240a08..accdf0334 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -117,6 +117,7 @@ class Role(BaseModel): self.path = path self.policies = {} self.managed_policies = {} + self.create_date = datetime.now(pytz.utc) @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -168,6 +169,7 @@ class InstanceProfile(BaseModel): self.name = name self.path = path self.roles = roles if roles else [] + self.create_date = datetime.now(pytz.utc) @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 22558f3f6..43b438c96 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -201,7 +201,7 @@ class IamResponse(BaseResponse): def create_instance_profile(self): profile_name = self._get_param('InstanceProfileName') - path = self._get_param('Path') + path = self._get_param('Path', '/') profile = iam_backend.create_instance_profile( profile_name, path, role_ids=[]) @@ -734,7 +734,7 @@ CREATE_INSTANCE_PROFILE_TEMPLATE = """ {% for profile in instance_profiles %} - {{ profile.id }} - - {% for role in profile.roles %} - - {{ role.path }} - {{ role.arn }} - {{ role.name }} - {{ role.assume_policy_document }} - 2012-05-09T15:45:35Z - {{ role.id }} - - {% endfor %} - - {{ profile.name }} - {{ profile.path }} - {{ profile.arn }} - 2012-05-09T16:27:11Z + {{ profile.id }} + + {% for role in profile.roles %} + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_policy_document }} + {{ role.create_date }} + {{ role.id }} + + {% endfor %} + + {{ profile.name }} + {{ profile.path }} + {{ profile.arn }} + {{ profile.create_date }} {% endfor %} @@ -1382,7 +1382,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ user.path }} {{ user.name }} {{ user.arn }} - 2012-05-09T15:45:35Z + {{ user.created_iso_8601 }} {% endfor %} @@ -1401,7 +1401,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ group.name }} {{ group.path }} {{ group.arn }} - 2012-05-09T16:27:11Z + {{ group.create_date }} {% endfor %} @@ -1421,23 +1421,23 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {% for profile in instance_profiles %} - {{ profile.id }} - - {% for role in profile.roles %} - - {{ role.path }} - {{ role.arn }} - {{ role.name }} - {{ role.assume_role_policy_document }} - 2012-05-09T15:45:35Z - {{ role.id }} - - {% endfor %} - - {{ profile.name }} - {{ profile.path }} - {{ profile.arn }} - 2012-05-09T16:27:11Z + {{ profile.id }} + + {% for role in profile.roles %} + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_role_policy_document }} + {{ role.create_date }} + {{ role.id }} + + {% endfor %} + + {{ profile.name }} + {{ profile.path }} + {{ profile.arn }} + {{ profile.create_date }} {% endfor %} @@ -1445,7 +1445,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ role.arn }} {{ role.name }} {{ role.assume_role_policy_document }} - 2014-07-30T17:09:20Z + {{ role.create_date }} {{ role.id }} {% endfor %} @@ -1474,9 +1474,9 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {{ policy.arn }} 1 - 2012-05-09T16:27:11Z + {{ policy.create_datetime }} true - 2012-05-09T16:27:11Z + {{ policy.update_datetime }} {% endfor %} diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index bc23ff712..72b9205dd 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -108,6 +108,10 @@ def test_create_role_and_instance_profile(): conn.list_roles().roles[0].role_name.should.equal('my-role') + # Test with an empty path: + profile = conn.create_instance_profile('my-other-profile') + profile.path.should.equal('/') + @mock_iam_deprecated() def test_remove_role_from_instance_profile(): @@ -700,10 +704,10 @@ def test_get_account_authorization_details(): import json conn = boto3.client('iam', region_name='us-east-1') conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") - conn.create_user(Path='/', UserName='testCloudAuxUser') - conn.create_group(Path='/', GroupName='testCloudAuxGroup') + conn.create_user(Path='/', UserName='testUser') + conn.create_group(Path='/', GroupName='testGroup') conn.create_policy( - PolicyName='testCloudAuxPolicy', + PolicyName='testPolicy', Path='/', PolicyDocument=json.dumps({ "Version": "2012-10-17", @@ -715,46 +719,47 @@ def test_get_account_authorization_details(): } ] }), - Description='Test CloudAux Policy' + Description='Test Policy' ) + conn.create_instance_profile(InstanceProfileName='ipn') + conn.add_role_to_instance_profile(InstanceProfileName='ipn', RoleName='my-role') + result = conn.get_account_authorization_details(Filter=['Role']) - len(result['RoleDetailList']) == 1 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 0 - len(result['Policies']) == 0 + assert len(result['RoleDetailList']) == 1 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) == 0 + assert len(result['RoleDetailList'][0]['InstanceProfileList']) == 1 result = conn.get_account_authorization_details(Filter=['User']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 1 - len(result['GroupDetailList']) == 0 - len(result['Policies']) == 0 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 1 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) == 0 result = conn.get_account_authorization_details(Filter=['Group']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 1 - len(result['Policies']) == 0 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 1 + assert len(result['Policies']) == 0 result = conn.get_account_authorization_details(Filter=['LocalManagedPolicy']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 0 - len(result['Policies']) == 1 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) == 1 # Check for greater than 1 since this should always be greater than one but might change. # See iam/aws_managed_policies.py result = conn.get_account_authorization_details(Filter=['AWSManagedPolicy']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 0 - len(result['Policies']) > 1 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) > 1 result = conn.get_account_authorization_details() - len(result['RoleDetailList']) == 1 - len(result['UserDetailList']) == 1 - len(result['GroupDetailList']) == 1 - len(result['Policies']) > 1 - - - + assert len(result['RoleDetailList']) == 1 + assert len(result['UserDetailList']) == 1 + assert len(result['GroupDetailList']) == 1 + assert len(result['Policies']) > 1 From 6e17ba51c6ee91b1a6109da5887706c07b39679e Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Sun, 21 Oct 2018 17:24:45 -0700 Subject: [PATCH 21/64] Fixed a truncation bug for `list_user_policies`. --- moto/iam/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 43b438c96..3d54b9331 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1199,8 +1199,8 @@ LIST_USER_POLICIES_TEMPLATE = """ {{ policy }} {% endfor %} + false - false 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE From 94b5438d76271c342b98e81655c1e258a68538ad Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Wed, 24 Oct 2018 18:00:52 -0700 Subject: [PATCH 22/64] Added IAM User signing certificate support --- moto/iam/exceptions.py | 8 ++++ moto/iam/models.py | 59 ++++++++++++++++++++++++++- moto/iam/responses.py | 82 ++++++++++++++++++++++++++++++++++++++ moto/iam/utils.py | 3 +- tests/test_iam/test_iam.py | 76 +++++++++++++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 3 deletions(-) diff --git a/moto/iam/exceptions.py b/moto/iam/exceptions.py index 84f15f51f..61922ea18 100644 --- a/moto/iam/exceptions.py +++ b/moto/iam/exceptions.py @@ -24,3 +24,11 @@ class IAMReportNotPresentException(RESTError): def __init__(self, message): super(IAMReportNotPresentException, self).__init__( "ReportNotPresent", message) + + +class MalformedCertificate(RESTError): + code = 400 + + def __init__(self, cert): + super(MalformedCertificate, self).__init__( + 'MalformedCertificate', 'Certificate {cert} is malformed'.format(cert=cert)) diff --git a/moto/iam/models.py b/moto/iam/models.py index accdf0334..20804f4d3 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1,14 +1,18 @@ from __future__ import unicode_literals import base64 +import sys from datetime import datetime import json +from cryptography import x509 +from cryptography.hazmat.backends import default_backend + import pytz from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_without_milliseconds from .aws_managed_policies import aws_managed_policies_data -from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException +from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, MalformedCertificate from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id ACCOUNT_ID = 123456789012 @@ -215,6 +219,16 @@ class Certificate(BaseModel): return "arn:aws:iam::{0}:server-certificate{1}{2}".format(ACCOUNT_ID, self.path, self.cert_name) +class SigningCertificate(BaseModel): + + def __init__(self, id, user_name, body): + self.id = id + self.user_name = user_name + self.body = body + self.upload_date = datetime.strftime(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S") + self.status = 'Active' + + class AccessKey(BaseModel): def __init__(self, user_name): @@ -299,6 +313,7 @@ class User(BaseModel): self.access_keys = [] self.password = None self.password_reset_required = False + self.signing_certificates = {} @property def arn(self): @@ -767,6 +782,48 @@ class IAMBackend(BaseBackend): return users + def upload_signing_certificate(self, user_name, body): + user = self.get_user(user_name) + cert_id = random_resource_id(size=32) + + # Validate the signing cert: + try: + if sys.version_info < (3, 0): + data = bytes(body) + else: + data = bytes(body, 'utf8') + + x509.load_pem_x509_certificate(data, default_backend()) + + except Exception: + raise MalformedCertificate(body) + + user.signing_certificates[cert_id] = SigningCertificate(cert_id, user_name, body) + + return user.signing_certificates[cert_id] + + def delete_signing_certificate(self, user_name, cert_id): + user = self.get_user(user_name) + + try: + del user.signing_certificates[cert_id] + except KeyError: + raise IAMNotFoundException("The Certificate with id {id} cannot be found.".format(id=cert_id)) + + def list_signing_certificates(self, user_name): + user = self.get_user(user_name) + + return list(user.signing_certificates.values()) + + def update_signing_certificate(self, user_name, cert_id, status): + user = self.get_user(user_name) + + try: + user.signing_certificates[cert_id].status = status + + except KeyError: + raise IAMNotFoundException("The Certificate with id {id} cannot be found.".format(id=cert_id)) + def create_login_profile(self, user_name, password): # This does not currently deal with PasswordPolicyViolation. user = self.get_user(user_name) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 3d54b9331..fa33cefea 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -552,6 +552,38 @@ class IamResponse(BaseResponse): roles=account_details['roles'] ) + def upload_signing_certificate(self): + user_name = self._get_param('UserName') + cert_body = self._get_param('CertificateBody') + + cert = iam_backend.upload_signing_certificate(user_name, cert_body) + template = self.response_template(UPLOAD_SIGNING_CERTIFICATE_TEMPLATE) + return template.render(cert=cert) + + def update_signing_certificate(self): + user_name = self._get_param('UserName') + cert_id = self._get_param('CertificateId') + status = self._get_param('Status') + + iam_backend.update_signing_certificate(user_name, cert_id, status) + template = self.response_template(UPDATE_SIGNING_CERTIFICATE_TEMPLATE) + return template.render() + + def delete_signing_certificate(self): + user_name = self._get_param('UserName') + cert_id = self._get_param('CertificateId') + + iam_backend.delete_signing_certificate(user_name, cert_id) + template = self.response_template(DELETE_SIGNING_CERTIFICATE_TEMPLATE) + return template.render() + + def list_signing_certificates(self): + user_name = self._get_param('UserName') + + certs = iam_backend.list_signing_certificates(user_name) + template = self.response_template(LIST_SIGNING_CERTIFICATES_TEMPLATE) + return template.render(user_name=user_name, certificates=certs) + ATTACH_ROLE_POLICY_TEMPLATE = """ @@ -1485,3 +1517,53 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """92e79ae7-7399-11e4-8c85-4b53eEXAMPLE """ + + +UPLOAD_SIGNING_CERTIFICATE_TEMPLATE = """ + + + {{ cert.user_name }} + {{ cert.id }} + {{ cert.body }} + {{ cert.status }} + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + + +UPDATE_SIGNING_CERTIFICATE_TEMPLATE = """ + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + +""" + + +DELETE_SIGNING_CERTIFICATE_TEMPLATE = """ + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + + +LIST_SIGNING_CERTIFICATES_TEMPLATE = """ + + {{ user_name }} + + {% for cert in certificates %} + + {{ user_name }} + {{ cert.id }} + {{ cert.body }} + {{ cert.status }} + + {% endfor %} + + false + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" diff --git a/moto/iam/utils.py b/moto/iam/utils.py index 1fae85a6c..f59bdfffe 100644 --- a/moto/iam/utils.py +++ b/moto/iam/utils.py @@ -12,8 +12,7 @@ def random_alphanumeric(length): ) -def random_resource_id(): - size = 20 +def random_resource_id(size=20): chars = list(range(10)) + list(string.ascii_lowercase) return ''.join(six.text_type(random.choice(chars)) for x in range(size)) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 72b9205dd..7bf38f48e 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -14,6 +14,19 @@ from nose.tools import raises from tests.helpers import requires_boto_gte +MOCK_CERT = """-----BEGIN CERTIFICATE----- +MIIBpzCCARACCQCY5yOdxCTrGjANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQKDAxt +b3RvIHRlc3RpbmcwIBcNMTgxMTA1MTkwNTIwWhgPMjI5MjA4MTkxOTA1MjBaMBcx +FTATBgNVBAoMDG1vdG8gdGVzdGluZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC +gYEA1Jn3g2h7LD3FLqdpcYNbFXCS4V4eDpuTCje9vKFcC3pi/01147X3zdfPy8Mt +ZhKxcREOwm4NXykh23P9KW7fBovpNwnbYsbPqj8Hf1ZaClrgku1arTVhEnKjx8zO +vaR/bVLCss4uE0E0VM1tJn/QGQsfthFsjuHtwx8uIWz35tUCAwEAATANBgkqhkiG +9w0BAQsFAAOBgQBWdOQ7bDc2nWkUhFjZoNIZrqjyNdjlMUndpwREVD7FQ/DuxJMj +FyDHrtlrS80dPUQWNYHw++oACDpWO01LGLPPrGmuO/7cOdojPEd852q5gd+7W9xt +8vUH+pBa6IBLbvBp+szli51V3TLSWcoyy4ceJNQU2vCkTLoFdS0RLd/7tQ== +-----END CERTIFICATE-----""" + + @mock_iam_deprecated() def test_get_all_server_certs(): conn = boto.connect_iam() @@ -763,3 +776,66 @@ def test_get_account_authorization_details(): assert len(result['UserDetailList']) == 1 assert len(result['GroupDetailList']) == 1 assert len(result['Policies']) > 1 + + +@mock_iam +def test_signing_certs(): + client = boto3.client('iam', region_name='us-east-1') + + # Create the IAM user first: + client.create_user(UserName='testing') + + # Upload the cert: + resp = client.upload_signing_certificate(UserName='testing', CertificateBody=MOCK_CERT)['Certificate'] + cert_id = resp['CertificateId'] + + assert resp['UserName'] == 'testing' + assert resp['Status'] == 'Active' + assert resp['CertificateBody'] == MOCK_CERT + assert resp['CertificateId'] + + # Upload a the cert with an invalid body: + with assert_raises(ClientError) as ce: + client.upload_signing_certificate(UserName='testing', CertificateBody='notacert') + assert ce.exception.response['Error']['Code'] == 'MalformedCertificate' + + # Upload with an invalid user: + with assert_raises(ClientError): + client.upload_signing_certificate(UserName='notauser', CertificateBody=MOCK_CERT) + + # Update: + client.update_signing_certificate(UserName='testing', CertificateId=cert_id, Status='Inactive') + + with assert_raises(ClientError): + client.update_signing_certificate(UserName='notauser', CertificateId=cert_id, Status='Inactive') + + with assert_raises(ClientError) as ce: + client.update_signing_certificate(UserName='testing', CertificateId='x' * 32, Status='Inactive') + + assert ce.exception.response['Error']['Message'] == 'The Certificate with id {id} cannot be found.'.format( + id='x' * 32) + + # List the certs: + resp = client.list_signing_certificates(UserName='testing')['Certificates'] + assert len(resp) == 1 + assert resp[0]['CertificateBody'] == MOCK_CERT + assert resp[0]['Status'] == 'Inactive' # Changed with the update call above. + + with assert_raises(ClientError): + client.list_signing_certificates(UserName='notauser') + + # Delete: + client.delete_signing_certificate(UserName='testing', CertificateId=cert_id) + + with assert_raises(ClientError): + client.delete_signing_certificate(UserName='notauser', CertificateId=cert_id) + + with assert_raises(ClientError) as ce: + client.delete_signing_certificate(UserName='testing', CertificateId=cert_id) + + assert ce.exception.response['Error']['Message'] == 'The Certificate with id {id} cannot be found.'.format( + id=cert_id) + + # Verify that it's not in the list: + resp = client.list_signing_certificates(UserName='testing') + assert not resp['Certificates'] From b66965e6e879a1647ca031448d8536f2e129d2c4 Mon Sep 17 00:00:00 2001 From: adam davis Date: Fri, 2 Nov 2018 16:04:17 -0700 Subject: [PATCH 23/64] Adding cloudformation-validate. Cfn-lint does the heavy lifting. --- moto/batch/responses.py | 2 +- moto/cloudformation/models.py | 4 + moto/cloudformation/responses.py | 27 +++++ moto/cloudformation/utils.py | 33 ++++++ setup.py | 1 + tests/test_cloudformation/test_validate.py | 115 +++++++++++++++++++++ 6 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 tests/test_cloudformation/test_validate.py diff --git a/moto/batch/responses.py b/moto/batch/responses.py index e626b7d4c..7fb606184 100644 --- a/moto/batch/responses.py +++ b/moto/batch/responses.py @@ -27,7 +27,7 @@ class BatchResponse(BaseResponse): elif not hasattr(self, '_json'): try: self._json = json.loads(self.body) - except json.JSONDecodeError: + except ValueError: print() return self._json diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index e5ab7255d..6ec821b42 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -13,6 +13,7 @@ from .utils import ( generate_changeset_id, generate_stack_id, yaml_tag_constructor, + validate_template_cfn_lint, ) from .exceptions import ValidationError @@ -270,6 +271,9 @@ class CloudFormationBackend(BaseBackend): next_token = str(token + 100) if len(all_exports) > token + 100 else None return exports, next_token + def validate_template(self, template): + return validate_template_cfn_lint(template) + def _validate_export_uniqueness(self, stack): new_stack_export_names = [x.name for x in stack.exports] export_names = self.exports.keys() diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index a1295a20d..0964287a3 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import json +import yaml from six.moves.urllib.parse import urlparse from moto.core.responses import BaseResponse @@ -294,6 +295,32 @@ class CloudFormationResponse(BaseResponse): template = self.response_template(LIST_EXPORTS_RESPONSE) return template.render(exports=exports, next_token=next_token) + def validate_template(self): + cfn_lint = self.cloudformation_backend.validate_template(self._get_param('TemplateBody')) + if cfn_lint: + raise ValidationError(cfn_lint[0].message) + description = "" + try: + description = json.loads(self._get_param('TemplateBody'))['Description'] + except (ValueError, KeyError): + pass + try: + description = yaml.load(self._get_param('TemplateBody'))['Description'] + except (yaml.ParserError, KeyError): + pass + template = self.response_template(VALIDATE_STACK_RESPONSE_TEMPLATE) + return template.render(description=description) + + +VALIDATE_STACK_RESPONSE_TEMPLATE = """ + + + + +{{ description }} + + +""" CREATE_STACK_RESPONSE_TEMPLATE = """ diff --git a/moto/cloudformation/utils.py b/moto/cloudformation/utils.py index f3b8874ed..f963ce7c8 100644 --- a/moto/cloudformation/utils.py +++ b/moto/cloudformation/utils.py @@ -3,6 +3,9 @@ import uuid import six import random import yaml +import os + +from cfnlint import decode, core def generate_stack_id(stack_name): @@ -38,3 +41,33 @@ def yaml_tag_constructor(loader, tag, node): key = 'Fn::{}'.format(tag[1:]) return {key: _f(loader, tag, node)} + + +def validate_template_cfn_lint(template): + + # Save the template to a temporary file -- cfn-lint requires a file + filename = "file.tmp" + with open(filename, "w") as file: + file.write(template) + abs_filename = os.path.abspath(filename) + + # decode handles both yaml and json + template, matches = decode.decode(abs_filename, False) + + # Set cfn-lint to info + core.configure_logging(None) + + # Initialize the ruleset to be applied (no overrules, no excludes) + rules = core.get_rules([], [], []) + + # Use us-east-1 region (spec file) for validation + regions = ['us-east-1'] + + # Process all the rules and gather the errors + matches = core.run_checks( + abs_filename, + template, + rules, + regions) + + return matches diff --git a/setup.py b/setup.py index 98780dd5a..873ca16b2 100755 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ install_requires = [ "jsondiff==1.1.1", "aws-xray-sdk<0.96,>=0.93", "responses>=0.9.0", + "cfn-lint" ] extras_require = { diff --git a/tests/test_cloudformation/test_validate.py b/tests/test_cloudformation/test_validate.py new file mode 100644 index 000000000..e2c3af05d --- /dev/null +++ b/tests/test_cloudformation/test_validate.py @@ -0,0 +1,115 @@ +from collections import OrderedDict +import json +import yaml +import os +import boto3 +from nose.tools import raises +import botocore + + +from moto.cloudformation.exceptions import ValidationError +from moto.cloudformation.models import FakeStack +from moto.cloudformation.parsing import resource_class_from_type, parse_condition, Export +from moto.sqs.models import Queue +from moto.s3.models import FakeBucket +from moto.cloudformation.utils import yaml_tag_constructor +from boto.cloudformation.stack import Output +from moto import mock_cloudformation, mock_s3, mock_sqs, mock_ec2 + +json_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack 1", + "Resources": { + "EC2Instance1": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-d3adb33f", + "KeyName": "dummy", + "InstanceType": "t2.micro", + "Tags": [ + { + "Key": "Description", + "Value": "Test tag" + }, + { + "Key": "Name", + "Value": "Name tag for tests" + } + ] + } + } + } +} + +# One resource is required +json_bad_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack 1" +} + +dummy_template_json = json.dumps(json_template) +dummy_bad_template_json = json.dumps(json_bad_template) + + +@mock_cloudformation +def test_boto3_json_validate_successful(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + response = cf_conn.validate_template( + TemplateBody=dummy_template_json, + ) + assert response['Description'] == "Stack 1" + assert response['Parameters'] == [] + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + +@mock_cloudformation +def test_boto3_json_invalid_missing_resource(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + try: + cf_conn.validate_template( + TemplateBody=dummy_bad_template_json, + ) + assert False + except botocore.exceptions.ClientError as e: + assert str(e) == 'An error occurred (ValidationError) when calling the ValidateTemplate operation: Stack' \ + ' with id Missing top level item Resources to file module does not exist' + assert True + + +yaml_template = """ + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template + Resources: + S3Bucket: + Type: AWS::S3::Bucket + Properties: + AccessControl: PublicRead + BucketName: cf-test-bucket-1 +""" + +yaml_bad_template = """ + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template +""" + +@mock_cloudformation +def test_boto3_yaml_validate_successful(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + response = cf_conn.validate_template( + TemplateBody=yaml_template, + ) + assert response['Description'] == "Simple CloudFormation Test Template" + assert response['Parameters'] == [] + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + +@mock_cloudformation +def test_boto3_yaml_invalid_missing_resource(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + try: + cf_conn.validate_template( + TemplateBody=yaml_bad_template, + ) + assert False + except botocore.exceptions.ClientError as e: + assert str(e) == 'An error occurred (ValidationError) when calling the ValidateTemplate operation: Stack' \ + ' with id Missing top level item Resources to file module does not exist' + assert True From 4af9407ef4ac22ba2a13a442d44ced30ef74cef8 Mon Sep 17 00:00:00 2001 From: Justin Dray Date: Tue, 6 Nov 2018 00:03:09 +0000 Subject: [PATCH 24/64] Update to jose 3.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 046ecf6e0..811be3855 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ install_requires = [ "pyaml", "pytz", "python-dateutil<3.0.0,>=2.1", - "python-jose<3.0.0", + "python-jose<4.0.0", "mock", "docker>=2.5.1", "jsondiff==1.1.1", From e52158f811818b84f86736727de8868b76180d64 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Tue, 6 Nov 2018 17:12:13 +0900 Subject: [PATCH 25/64] Add support for IoT detach_policy --- IMPLEMENTATION_COVERAGE.md | 4 ++-- moto/iot/models.py | 9 +++++++++ moto/iot/responses.py | 9 +++++++++ tests/test_iot/test_iot.py | 7 ++++++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7a1e2e7aa..0a00dec94 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2376,7 +2376,7 @@ - [ ] unsubscribe_from_event - [ ] update_assessment_target -## iot - 31% implemented +## iot - 32% implemented - [ ] accept_certificate_transfer - [X] add_thing_to_thing_group - [ ] associate_targets_with_job @@ -2429,7 +2429,7 @@ - [X] describe_thing_group - [ ] describe_thing_registration_task - [X] describe_thing_type -- [ ] detach_policy +- [X] detach_policy - [X] detach_principal_policy - [X] detach_thing_principal - [ ] disable_topic_rule diff --git a/moto/iot/models.py b/moto/iot/models.py index db9ad3817..5c33aecda 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -445,6 +445,15 @@ class IoTBackend(BaseBackend): return self.principal_policies[k] = (principal, policy) + def detach_policy(self, policy_name, target): + # this may raises ResourceNotFoundException + self._get_principal(target) + self.get_policy(policy_name) + k = (target, policy_name) + if k not in self.principal_policies: + raise ResourceNotFoundException() + del self.principal_policies[k] + def detach_principal_policy(self, policy_name, principal_arn): # this may raises ResourceNotFoundException self._get_principal(principal_arn) diff --git a/moto/iot/responses.py b/moto/iot/responses.py index 042e5a314..214576f52 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -242,6 +242,15 @@ class IoTResponse(BaseResponse): ) return json.dumps(dict()) + def detach_policy(self): + policy_name = self._get_param("policyName") + target = self._get_param('target') + self.iot_backend.detach_policy( + policy_name=policy_name, + target=target, + ) + return json.dumps(dict()) + def detach_principal_policy(self): policy_name = self._get_param("policyName") principal = self.headers.get('x-amzn-iot-principal') diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 47ea9d59b..161c5832a 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -5,6 +5,8 @@ import sure # noqa import boto3 from moto import mock_iot +from botocore.exceptions import ClientError +from nose.tools import assert_raises @mock_iot @@ -338,11 +340,14 @@ def test_principal_policy(): for principal in res['principals']: principal.should_not.be.none - client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + client.detach_policy(policyName=policy_name, target=cert_arn) res = client.list_principal_policies(principal=cert_arn) res.should.have.key('policies').which.should.have.length_of(0) res = client.list_policy_principals(policyName=policy_name) res.should.have.key('principals').which.should.have.length_of(0) + with assert_raises(ClientError) as e: + client.detach_policy(policyName=policy_name, target=cert_arn) + e.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') @mock_iot From 0b57ffe26a894ecc7d39a9c1155dadedc0c5f7ba Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Wed, 7 Nov 2018 15:03:25 -0500 Subject: [PATCH 26/64] Add StreamSpecification to dynamodb2 package --- moto/dynamodb2/models.py | 22 +++++++++- moto/dynamodb2/responses.py | 9 +++- tests/test_dynamodb2/test_dynamodb.py | 59 +++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a54c4f7d0..16e97ea2f 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -294,7 +294,7 @@ class Item(BaseModel): class Table(BaseModel): - def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None): + def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None, streams=None): self.name = table_name self.attr = attr self.schema = schema @@ -325,10 +325,18 @@ class Table(BaseModel): 'TimeToLiveStatus': 'DISABLED' # One of 'ENABLING'|'DISABLING'|'ENABLED'|'DISABLED', # 'AttributeName': 'string' # Can contain this } + self.set_stream_specification(streams) def _generate_arn(self, name): return 'arn:aws:dynamodb:us-east-1:123456789011:table/' + name + def set_stream_specification(self, streams): + self.stream_specification = streams + if streams and streams.get('StreamEnabled'): + self.latest_stream_label = datetime.datetime.utcnow().isoformat() + else: + self.latest_stream_label = None + def describe(self, base_key='TableDescription'): results = { base_key: { @@ -345,6 +353,11 @@ class Table(BaseModel): 'LocalSecondaryIndexes': [index for index in self.indexes], } } + if self.stream_specification: + results[base_key]['StreamSpecification'] = self.stream_specification + if self.latest_stream_label: + results[base_key]['LatestStreamLabel'] = self.latest_stream_label + results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label return results def __len__(self): @@ -680,6 +693,13 @@ class DynamoDBBackend(BaseBackend): table.throughput = throughput return table + def update_table_streams(self, name, stream_specification): + table = self.tables[name] + if stream_specification['StreamEnabled'] and table.latest_stream_label: + raise ValueError('Table already has stream enabled') + table.set_stream_specification(stream_specification) + return table + def update_table_global_indexes(self, name, global_index_updates): table = self.tables[name] gsis_by_name = dict((i['IndexName'], i) for i in table.global_indexes) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index e2f1ef1cc..73bd3ae38 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -104,13 +104,16 @@ class DynamoHandler(BaseResponse): # getting the indexes global_indexes = body.get("GlobalSecondaryIndexes", []) local_secondary_indexes = body.get("LocalSecondaryIndexes", []) + # get the stream specification + streams = body.get("StreamSpecification") table = self.dynamodb_backend.create_table(table_name, schema=key_schema, throughput=throughput, attr=attr, global_indexes=global_indexes, - indexes=local_secondary_indexes) + indexes=local_secondary_indexes, + streams=streams) if table is not None: return dynamo_json_dump(table.describe()) else: @@ -163,12 +166,16 @@ class DynamoHandler(BaseResponse): def update_table(self): name = self.body['TableName'] + table = self.dynamodb_backend.get_table(name) if 'GlobalSecondaryIndexUpdates' in self.body: table = self.dynamodb_backend.update_table_global_indexes( name, self.body['GlobalSecondaryIndexUpdates']) if 'ProvisionedThroughput' in self.body: throughput = self.body["ProvisionedThroughput"] table = self.dynamodb_backend.update_table_throughput(name, throughput) + if 'StreamSpecification' in self.body: + table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) + return dynamo_json_dump(table.describe()) def describe_table(self): diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index afc919dd7..7f30bbccf 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1336,3 +1336,62 @@ def test_query_global_secondary_index_when_created_via_update_table_resource(): assert len(forum_and_subject_items) == 1 assert forum_and_subject_items[0] == {'user_id': Decimal('1'), 'forum_name': 'cats', 'subject': 'my pet is the cutest'} + + +@mock_dynamodb2 +def test_dynamodb_streams_1(): + conn = boto3.client('dynamodb', region_name='us-east-1') + + resp = conn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1}, + StreamSpecification={ + 'StreamEnabled': True, + 'StreamViewType': 'NEW_AND_OLD_IMAGES' + } + ) + + assert 'StreamSpecification' in resp['TableDescription'] + assert resp['TableDescription']['StreamSpecification'] == { + 'StreamEnabled': True, + 'StreamViewType': 'NEW_AND_OLD_IMAGES' + } + assert 'LatestStreamLabel' in resp['TableDescription'] + assert 'LatestStreamArn' in resp['TableDescription'] + + resp = conn.delete_table(TableName='test-streams') + + assert 'StreamSpecification' in resp['TableDescription'] + + +@mock_dynamodb2 +def test_dynamodb_streams_2(): + conn = boto3.client('dynamodb', region_name='us-east-1') + + resp = conn.create_table( + TableName='test-stream-update', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1}, + ) + + assert 'StreamSpecification' not in resp['TableDescription'] + + resp = conn.update_table( + TableName='test-stream-update', + StreamSpecification={ + 'StreamEnabled': True, + 'StreamViewType': 'NEW_IMAGE' + } + ) + + assert 'StreamSpecification' in resp['TableDescription'] + assert resp['TableDescription']['StreamSpecification'] == { + 'StreamEnabled': True, + 'StreamViewType': 'NEW_IMAGE' + } + assert 'LatestStreamLabel' in resp['TableDescription'] + assert 'LatestStreamArn' in resp['TableDescription'] + From 519899f74f6d50c33ffe82d8d6fee2c86afd3a8e Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Wed, 7 Nov 2018 17:10:00 -0500 Subject: [PATCH 27/64] Much of the way towards complete DynamoDB Streams implementation --- moto/__init__.py | 1 + moto/backends.py | 1 + moto/dynamodb2/models.py | 37 ++++++- moto/dynamodbstreams/__init__.py | 6 ++ moto/dynamodbstreams/models.py | 99 +++++++++++++++++++ moto/dynamodbstreams/responses.py | 29 ++++++ moto/dynamodbstreams/urls.py | 10 ++ setup.cfg | 2 +- .../test_dynamodbstreams.py | 96 ++++++++++++++++++ 9 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 moto/dynamodbstreams/__init__.py create mode 100644 moto/dynamodbstreams/models.py create mode 100644 moto/dynamodbstreams/responses.py create mode 100644 moto/dynamodbstreams/urls.py create mode 100644 tests/test_dynamodbstreams/test_dynamodbstreams.py diff --git a/moto/__init__.py b/moto/__init__.py index dd3593d5d..e86c499a7 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -16,6 +16,7 @@ from .cognitoidp import mock_cognitoidp, mock_cognitoidp_deprecated # flake8: n from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated # flake8: noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # flake8: noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # flake8: noqa +from .dynamodbstreams import mock_dynamodbstreams # flake8: noqa from .ec2 import mock_ec2, mock_ec2_deprecated # flake8: noqa from .ecr import mock_ecr, mock_ecr_deprecated # flake8: noqa from .ecs import mock_ecs, mock_ecs_deprecated # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index d95424385..7df167a06 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -12,6 +12,7 @@ from moto.core import moto_api_backends from moto.datapipeline import datapipeline_backends from moto.dynamodb import dynamodb_backends from moto.dynamodb2 import dynamodb_backends2 +from moto.dynamodbstreams import dynamodbstreams_backends from moto.ec2 import ec2_backends from moto.ecr import ecr_backends from moto.ecs import ecs_backends diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 16e97ea2f..d58241cf6 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -292,6 +292,34 @@ class Item(BaseModel): 'ADD not supported for %s' % ', '.join(update_action['Value'].keys())) +class StreamShard(BaseModel): + def __init__(self, table): + self.table = table + self.id = 'shardId-00000001541626099285-f35f62ef' + self.starting_sequence_number = 1100000000017454423009 + self.items = [] + self.created_on = datetime.datetime.utcnow() + + def to_json(self): + return { + 'ShardId': self.id, + 'SequenceNumberRange': { + 'StartingSequenceNumber': str(self.starting_sequence_number) + } + } + + def add(self, old, new): + t = self.table.stream_specification['StreamViewType'] + if t == 'KEYS_ONLY': + self.items.append(new.key) + elif t == 'NEW_IMAGE': + self.items.append(new) + elif t == 'OLD_IMAGE': + self.items.append(old) + elif t == 'NEW_AND_OLD_IMAGES': + self.items.append((old, new)) + + class Table(BaseModel): def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None, streams=None): @@ -334,8 +362,10 @@ class Table(BaseModel): self.stream_specification = streams if streams and streams.get('StreamEnabled'): self.latest_stream_label = datetime.datetime.utcnow().isoformat() + self.stream_shard = StreamShard(self) else: self.latest_stream_label = None + self.stream_shard = None def describe(self, base_key='TableDescription'): results = { @@ -398,6 +428,7 @@ class Table(BaseModel): else: range_value = None + current = self.get_item(hash_value, lookup_range_value) item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs) @@ -413,8 +444,6 @@ class Table(BaseModel): else: lookup_range_value = DynamoType(expected_range_value) - current = self.get_item(hash_value, lookup_range_value) - if current is None: current_attr = {} elif hasattr(current, 'attrs'): @@ -445,6 +474,10 @@ class Table(BaseModel): self.items[hash_value][range_value] = item else: self.items[hash_value] = item + + if self.stream_shard is not None: + self.stream_shard.add(current, item) + return item def __nonzero__(self): diff --git a/moto/dynamodbstreams/__init__.py b/moto/dynamodbstreams/__init__.py new file mode 100644 index 000000000..b35879eba --- /dev/null +++ b/moto/dynamodbstreams/__init__.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .models import dynamodbstreams_backends +from ..core.models import base_decorator + +dynamodbstreams_backend = dynamodbstreams_backends['us-east-1'] +mock_dynamodbstreams = base_decorator(dynamodbstreams_backends) diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py new file mode 100644 index 000000000..35704de63 --- /dev/null +++ b/moto/dynamodbstreams/models.py @@ -0,0 +1,99 @@ +from __future__ import unicode_literals + +import os +import json +import boto3 +import base64 +import datetime + +from moto.core import BaseBackend, BaseModel +from moto.dynamodb2.models import dynamodb_backends + + +class ShardIterator(BaseModel): + def __init__(self, stream_shard, shard_iterator_type, sequence_number=None): + self.id = base64.b64encode(os.urandom(472)).decode('utf-8') + self.stream_shard = stream_shard + self.shard_iterator_type = shard_iterator_type + if shard_iterator_type == 'TRIM_HORIZON': + self.sequence_number = stream_shard.starting_sequence_number + elif shard_iterator_type == 'LATEST': + self.sequence_number = stream_shard.starting_sequence_number + len(stream_shard.items) + elif shard_iterator_type == 'AT_SEQUENCE_NUMBER': + self.sequence_number = sequence_number + elif shard_iterator_type == 'AFTER_SEQUENCE_NUMBER': + self.sequence_number = sequence_number + 1 + + def to_json(self): + return { + 'ShardIterator': '{}/stream/{}|1|{}'.format( + self.stream_shard.table.table_arn, + self.stream_shard.table.latest_stream_label, + self.id) + } + + +class DynamoDBStreamsBackend(BaseBackend): + def __init__(self, region): + self.region = region + + def reset(self): + region = self.region + self.__dict__ = {} + self.__init__(region) + + @property + def dynamodb(self): + return dynamodb_backends[self.region] + + def _get_table_from_arn(self, arn): + table_name = arn.split(':', 6)[5].split('/')[1] + return self.dynamodb.get_table(table_name) + + def describe_stream(self, arn): + table = self._get_table_from_arn(arn) + resp = {'StreamDescription': { + 'StreamArn': arn, + 'StreamLabel': table.latest_stream_label, + 'StreamStatus': ('ENABLED' if table.latest_stream_label + else 'DISABLED'), + 'StreamViewType': table.stream_specification['StreamViewType'], + 'CreationRequestDateTime': table.stream_shard.created_on.isoformat(), + 'TableName': table.name, + 'KeySchema': table.schema, + 'Shards': ([table.stream_shard.to_json()] if table.stream_shard + else []) + }} + + return json.dumps(resp) + + def list_streams(self, table_name=None): + streams = [] + for table in self.dynamodb.tables.values(): + if table_name is not None and table.name != table_name: + continue + if table.latest_stream_label: + d = table.describe(base_key='Table') + streams.append({ + 'StreamArn': d['Table']['LatestStreamArn'], + 'TableName': d['Table']['TableName'], + 'StreamLabel': d['Table']['LatestStreamLabel'] + }) + + return json.dumps({'Streams': streams}) + + def get_shard_iterator(self, arn, shard_id, shard_iterator_type, sequence_number=None): + table = self._get_table_from_arn(arn) + assert table.stream_shard.id == shard_id + + shard_iterator = ShardIterator(table.stream_shard, shard_iterator_type, + sequence_number) + + return json.dumps(shard_iterator.to_json()) + + + +available_regions = boto3.session.Session().get_available_regions( + 'dynamodbstreams') +dynamodbstreams_backends = {region: DynamoDBStreamsBackend(region=region) + for region in available_regions} diff --git a/moto/dynamodbstreams/responses.py b/moto/dynamodbstreams/responses.py new file mode 100644 index 000000000..ef05c1718 --- /dev/null +++ b/moto/dynamodbstreams/responses.py @@ -0,0 +1,29 @@ +from __future__ import unicode_literals + +import json + +from moto.core.responses import BaseResponse + +from .models import dynamodbstreams_backends + + +class DynamoDBStreamsHandler(BaseResponse): + + @property + def backend(self): + return dynamodbstreams_backends[self.region] + + def describe_stream(self): + arn = self._get_param('StreamArn') + return self.backend.describe_stream(arn) + + def list_streams(self): + table_name = self._get_param('TableName') + return self.backend.list_streams(table_name) + + def get_shard_iterator(self): + arn = self._get_param('StreamArn') + shard_id = self._get_param('ShardId') + shard_iterator_type = self._get_param('ShardIteratorType') + return self.backend.get_shard_iterator(arn, shard_id, + shard_iterator_type) diff --git a/moto/dynamodbstreams/urls.py b/moto/dynamodbstreams/urls.py new file mode 100644 index 000000000..1d0f94c35 --- /dev/null +++ b/moto/dynamodbstreams/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import DynamoDBStreamsHandler + +url_bases = [ + "https?://streams.dynamodb.(.+).amazonaws.com" +] + +url_paths = { + "{0}/$": DynamoDBStreamsHandler.dispatch, +} diff --git a/setup.cfg b/setup.cfg index fb04c16a8..9dbd988db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [nosetests] verbosity=1 detailed-errors=1 -with-coverage=1 +#with-coverage=1 cover-package=moto [bdist_wheel] diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py new file mode 100644 index 000000000..a10445aac --- /dev/null +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -0,0 +1,96 @@ +from __future__ import unicode_literals, print_function + +import boto3 +from moto import mock_dynamodb2, mock_dynamodbstreams + + +class TestClass(): + stream_arn = None + mocks = [] + + def setup(self): + self.mocks = [mock_dynamodb2(), mock_dynamodbstreams()] + for m in self.mocks: + m.start() + + # create a table with a stream + conn = boto3.client('dynamodb', region_name='us-east-1') + + resp = conn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', + 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1}, + StreamSpecification={ + 'StreamEnabled': True, + 'StreamViewType': 'NEW_AND_OLD_IMAGES' + } + ) + self.stream_arn = resp['TableDescription']['LatestStreamArn'] + + def teardown(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + conn.delete_table(TableName='test-streams') + self.stream_arn = None + + for m in self.mocks: + m.stop() + + + def test_verify_stream(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + resp = conn.describe_table(TableName='test-streams') + assert 'LatestStreamArn' in resp['Table'] + + def test_describe_stream(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + assert 'StreamDescription' in resp + desc = resp['StreamDescription'] + assert desc['StreamArn'] == self.stream_arn + assert desc['TableName'] == 'test-streams' + + def test_list_streams(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.list_streams() + assert resp['Streams'][0]['StreamArn'] == self.stream_arn + + resp = conn.list_streams(TableName='no-stream') + assert not resp['Streams'] + + def test_get_shard_iterator(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = conn.get_shard_iterator( + StreamArn=self.stream_arn, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON' + ) + assert 'ShardIterator' in resp + + def test_get_records(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = conn.get_shard_iterator( + StreamArn=self.stream_arn, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON' + ) + iterator_id = resp['ShardIterator'] + + resp = conn.get_records(ShardIterator=iterator_id) + assert 'Records' in resp + + # TODO: Add tests for inserting records into the stream, and + # the various stream types + From 0f6086f708112c024eb7f3079339b1698a507012 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 10:54:54 -0500 Subject: [PATCH 28/64] Finalize implementation of DynamoDB Streams --- moto/dynamodb2/models.py | 97 ++++++++++++++----- moto/dynamodbstreams/models.py | 44 +++++++-- moto/dynamodbstreams/responses.py | 7 ++ setup.cfg | 2 +- .../test_dynamodbstreams.py | 49 +++++++++- 5 files changed, 166 insertions(+), 33 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index d58241cf6..4283c038b 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals -from collections import defaultdict +from collections import defaultdict, namedtuple import copy import datetime import decimal import json import re +import uuid import boto3 from moto.compat import OrderedDict @@ -292,6 +293,44 @@ class Item(BaseModel): 'ADD not supported for %s' % ', '.join(update_action['Value'].keys())) +class StreamRecord(BaseModel): + def __init__(self, table, stream_type, event_name, old, new, seq): + old_a = old.to_json()['Attributes'] if old is not None else {} + new_a = new.to_json()['Attributes'] if new is not None else {} + + rec = old if old is not None else new + keys = {table.hash_key_attr: rec.hash_key.to_json()} + if table.range_key_attr is not None: + keys[table.range_key_attr] = rec.range_key.to_json() + + self.record = { + 'eventID': uuid.uuid4().hex, + 'eventName': event_name, + 'eventSource': 'aws:dynamodb', + 'eventVersion': '1.0', + 'awsRegion': 'us-east-1', + 'dynamodb': { + 'StreamViewType': stream_type, + 'ApproximateCreationDateTime': datetime.datetime.utcnow().isoformat(), + 'SequenceNumber': seq, + 'SizeBytes': 1, + 'Keys': keys + } + } + + if stream_type in ('NEW_IMAGE', 'NEW_AND_OLD_IMAGES'): + self.record['dynamodb']['NewImage'] = new_a + if stream_type in ('OLD_IMAGE', 'NEW_AND_OLD_IMAGES'): + self.record['dynamodb']['OldImage'] = old_a + + # This is a substantial overestimate but it's the easiest to do now + self.record['dynamodb']['SizeBytes'] = len( + json.dumps(self.record['dynamodb'])) + + def to_json(self): + return self.record + + class StreamShard(BaseModel): def __init__(self, table): self.table = table @@ -310,15 +349,22 @@ class StreamShard(BaseModel): def add(self, old, new): t = self.table.stream_specification['StreamViewType'] - if t == 'KEYS_ONLY': - self.items.append(new.key) - elif t == 'NEW_IMAGE': - self.items.append(new) - elif t == 'OLD_IMAGE': - self.items.append(old) - elif t == 'NEW_AND_OLD_IMAGES': - self.items.append((old, new)) - + if old is None: + event_name = 'INSERT' + elif new is None: + event_name = 'DELETE' + else: + event_name = 'MODIFY' + seq = len(self.items) + self.starting_sequence_number + self.items.append( + StreamRecord(self.table, t, event_name, old, new, seq)) + + def get(self, start, quantity): + start -= self.starting_sequence_number + assert start >= 0 + end = start + quantity + return [i.to_json() for i in self.items[start:end]] + class Table(BaseModel): @@ -428,22 +474,22 @@ class Table(BaseModel): else: range_value = None + if expected is None: + expected = {} + lookup_range_value = range_value + else: + expected_range_value = expected.get( + self.range_key_attr, {}).get("Value") + if(expected_range_value is None): + lookup_range_value = range_value + else: + lookup_range_value = DynamoType(expected_range_value) current = self.get_item(hash_value, lookup_range_value) + item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs) if not overwrite: - if expected is None: - expected = {} - lookup_range_value = range_value - else: - expected_range_value = expected.get( - self.range_key_attr, {}).get("Value") - if(expected_range_value is None): - lookup_range_value = range_value - else: - lookup_range_value = DynamoType(expected_range_value) - if current is None: current_attr = {} elif hasattr(current, 'attrs'): @@ -508,9 +554,14 @@ class Table(BaseModel): def delete_item(self, hash_key, range_key): try: if range_key: - return self.items[hash_key].pop(range_key) + item = self.items[hash_key].pop(range_key) else: - return self.items.pop(hash_key) + item = self.items.pop(hash_key) + + if self.stream_shard is not None: + self.stream_shard.add(item, None) + + return item except KeyError: return None diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index 35704de63..7b43eb744 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -11,8 +11,9 @@ from moto.dynamodb2.models import dynamodb_backends class ShardIterator(BaseModel): - def __init__(self, stream_shard, shard_iterator_type, sequence_number=None): + def __init__(self, streams_backend, stream_shard, shard_iterator_type, sequence_number=None): self.id = base64.b64encode(os.urandom(472)).decode('utf-8') + self.streams_backend = streams_backend self.stream_shard = stream_shard self.shard_iterator_type = shard_iterator_type if shard_iterator_type == 'TRIM_HORIZON': @@ -24,18 +25,43 @@ class ShardIterator(BaseModel): elif shard_iterator_type == 'AFTER_SEQUENCE_NUMBER': self.sequence_number = sequence_number + 1 + @property + def arn(self): + return '{}/stream/{}|1|{}'.format( + self.stream_shard.table.table_arn, + self.stream_shard.table.latest_stream_label, + self.id) + def to_json(self): return { - 'ShardIterator': '{}/stream/{}|1|{}'.format( - self.stream_shard.table.table_arn, - self.stream_shard.table.latest_stream_label, - self.id) + 'ShardIterator': self.arn + } + + def get(self, limit=1000): + items = self.stream_shard.get(self.sequence_number, limit) + try: + last_sequence_number = max(i['dynamodb']['SequenceNumber'] for i in items) + new_shard_iterator = ShardIterator(self.streams_backend, + self.stream_shard, + 'AFTER_SEQUENCE_NUMBER', + last_sequence_number) + except ValueError: + new_shard_iterator = ShardIterator(self.streams_backend, + self.stream_shard, + 'AT_SEQUENCE_NUMBER', + self.sequence_number) + + self.streams_backend.shard_iterators[new_shard_iterator.arn] = new_shard_iterator + return { + 'NextShardIterator': new_shard_iterator.arn, + 'Records': items } class DynamoDBStreamsBackend(BaseBackend): def __init__(self, region): self.region = region + self.shard_iterators = {} def reset(self): region = self.region @@ -86,11 +112,17 @@ class DynamoDBStreamsBackend(BaseBackend): table = self._get_table_from_arn(arn) assert table.stream_shard.id == shard_id - shard_iterator = ShardIterator(table.stream_shard, shard_iterator_type, + shard_iterator = ShardIterator(self, table.stream_shard, + shard_iterator_type, sequence_number) + self.shard_iterators[shard_iterator.arn] = shard_iterator return json.dumps(shard_iterator.to_json()) + def get_records(self, iterator_arn, limit): + shard_iterator = self.shard_iterators[iterator_arn] + return json.dumps(shard_iterator.get(limit)) + available_regions = boto3.session.Session().get_available_regions( diff --git a/moto/dynamodbstreams/responses.py b/moto/dynamodbstreams/responses.py index ef05c1718..c07377d38 100644 --- a/moto/dynamodbstreams/responses.py +++ b/moto/dynamodbstreams/responses.py @@ -27,3 +27,10 @@ class DynamoDBStreamsHandler(BaseResponse): shard_iterator_type = self._get_param('ShardIteratorType') return self.backend.get_shard_iterator(arn, shard_id, shard_iterator_type) + + def get_records(self): + arn = self._get_param('ShardIterator') + limit = self._get_param('Limit') + if limit is None: + limit = 1000 + return self.backend.get_records(arn, limit) diff --git a/setup.cfg b/setup.cfg index 9dbd988db..fb04c16a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [nosetests] verbosity=1 detailed-errors=1 -#with-coverage=1 +with-coverage=1 cover-package=moto [bdist_wheel] diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index a10445aac..94c0c51b2 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -75,7 +75,7 @@ class TestClass(): ) assert 'ShardIterator' in resp - def test_get_records(self): + def test_get_records_empty(self): conn = boto3.client('dynamodbstreams', region_name='us-east-1') resp = conn.describe_stream(StreamArn=self.stream_arn) @@ -90,7 +90,50 @@ class TestClass(): resp = conn.get_records(ShardIterator=iterator_id) assert 'Records' in resp + assert len(resp['Records']) == 0 - # TODO: Add tests for inserting records into the stream, and - # the various stream types + def test_get_records_seq(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + + conn.put_item( + TableName='test-streams', + Item={ + 'id': {'S': 'entry1'}, + 'first_col': {'S': 'foo'} + } + ) + conn.put_item( + TableName='test-streams', + Item={ + 'id': {'S': 'entry1'}, + 'first_col': {'S': 'bar'}, + 'second_col': {'S': 'baz'} + } + ) + conn.delete_item( + TableName='test-streams', + Key={'id': {'S': 'entry1'}} + ) + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = conn.get_shard_iterator( + StreamArn=self.stream_arn, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON' + ) + iterator_id = resp['ShardIterator'] + + resp = conn.get_records(ShardIterator=iterator_id) + assert len(resp['Records']) == 3 + assert resp['Records'][0]['eventName'] == 'INSERT' + assert resp['Records'][1]['eventName'] == 'MODIFY' + assert resp['Records'][2]['eventName'] == 'DELETE' + + # now try fetching from the next shard iterator, it should be + # empty + resp = conn.get_records(ShardIterator=resp['NextShardIterator']) + assert len(resp['Records']) == 0 From ff6a57f44391e2e8523c1ca49022ec39ed0facd6 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 11:08:24 -0500 Subject: [PATCH 29/64] Fix flake8 failures --- moto/backends.py | 1 + moto/dynamodb2/models.py | 16 ++++++++-------- moto/dynamodb2/responses.py | 2 +- moto/dynamodbstreams/models.py | 14 ++++++-------- moto/dynamodbstreams/responses.py | 4 +--- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/moto/backends.py b/moto/backends.py index 7df167a06..1a333415e 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -60,6 +60,7 @@ BACKENDS = { 'datapipeline': datapipeline_backends, 'dynamodb': dynamodb_backends, 'dynamodb2': dynamodb_backends2, + 'dynamodbstreams': dynamodbstreams_backends, 'ec2': ec2_backends, 'ecr': ecr_backends, 'ecs': ecs_backends, diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 4283c038b..e2882f1e4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -from collections import defaultdict, namedtuple +from collections import defaultdict import copy import datetime import decimal @@ -302,7 +302,7 @@ class StreamRecord(BaseModel): keys = {table.hash_key_attr: rec.hash_key.to_json()} if table.range_key_attr is not None: keys[table.range_key_attr] = rec.range_key.to_json() - + self.record = { 'eventID': uuid.uuid4().hex, 'eventName': event_name, @@ -317,7 +317,7 @@ class StreamRecord(BaseModel): 'Keys': keys } } - + if stream_type in ('NEW_IMAGE', 'NEW_AND_OLD_IMAGES'): self.record['dynamodb']['NewImage'] = new_a if stream_type in ('OLD_IMAGE', 'NEW_AND_OLD_IMAGES'): @@ -364,7 +364,7 @@ class StreamShard(BaseModel): assert start >= 0 end = start + quantity return [i.to_json() for i in self.items[start:end]] - + class Table(BaseModel): @@ -485,7 +485,7 @@ class Table(BaseModel): else: lookup_range_value = DynamoType(expected_range_value) current = self.get_item(hash_value, lookup_range_value) - + item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs) @@ -520,10 +520,10 @@ class Table(BaseModel): self.items[hash_value][range_value] = item else: self.items[hash_value] = item - + if self.stream_shard is not None: self.stream_shard.add(current, item) - + return item def __nonzero__(self): @@ -560,7 +560,7 @@ class Table(BaseModel): if self.stream_shard is not None: self.stream_shard.add(item, None) - + return item except KeyError: return None diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 73bd3ae38..d2bf99bfe 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -175,7 +175,7 @@ class DynamoHandler(BaseResponse): table = self.dynamodb_backend.update_table_throughput(name, throughput) if 'StreamSpecification' in self.body: table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) - + return dynamo_json_dump(table.describe()) def describe_table(self): diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index 7b43eb744..41cc6e280 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -4,7 +4,6 @@ import os import json import boto3 import base64 -import datetime from moto.core import BaseBackend, BaseModel from moto.dynamodb2.models import dynamodb_backends @@ -31,7 +30,7 @@ class ShardIterator(BaseModel): self.stream_shard.table.table_arn, self.stream_shard.table.latest_stream_label, self.id) - + def to_json(self): return { 'ShardIterator': self.arn @@ -67,7 +66,7 @@ class DynamoDBStreamsBackend(BaseBackend): region = self.region self.__dict__ = {} self.__init__(region) - + @property def dynamodb(self): return dynamodb_backends[self.region] @@ -75,7 +74,7 @@ class DynamoDBStreamsBackend(BaseBackend): def _get_table_from_arn(self, arn): table_name = arn.split(':', 6)[5].split('/')[1] return self.dynamodb.get_table(table_name) - + def describe_stream(self, arn): table = self._get_table_from_arn(arn) resp = {'StreamDescription': { @@ -90,7 +89,7 @@ class DynamoDBStreamsBackend(BaseBackend): 'Shards': ([table.stream_shard.to_json()] if table.stream_shard else []) }} - + return json.dumps(resp) def list_streams(self, table_name=None): @@ -105,7 +104,7 @@ class DynamoDBStreamsBackend(BaseBackend): 'TableName': d['Table']['TableName'], 'StreamLabel': d['Table']['LatestStreamLabel'] }) - + return json.dumps({'Streams': streams}) def get_shard_iterator(self, arn, shard_id, shard_iterator_type, sequence_number=None): @@ -116,14 +115,13 @@ class DynamoDBStreamsBackend(BaseBackend): shard_iterator_type, sequence_number) self.shard_iterators[shard_iterator.arn] = shard_iterator - + return json.dumps(shard_iterator.to_json()) def get_records(self, iterator_arn, limit): shard_iterator = self.shard_iterators[iterator_arn] return json.dumps(shard_iterator.get(limit)) - available_regions = boto3.session.Session().get_available_regions( 'dynamodbstreams') diff --git a/moto/dynamodbstreams/responses.py b/moto/dynamodbstreams/responses.py index c07377d38..c9c113615 100644 --- a/moto/dynamodbstreams/responses.py +++ b/moto/dynamodbstreams/responses.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import json - from moto.core.responses import BaseResponse from .models import dynamodbstreams_backends @@ -12,7 +10,7 @@ class DynamoDBStreamsHandler(BaseResponse): @property def backend(self): return dynamodbstreams_backends[self.region] - + def describe_stream(self): arn = self._get_param('StreamArn') return self.backend.describe_stream(arn) From 9d190aa04e4c66831449a1a5cd3673c35e995bf9 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 13:22:24 -0500 Subject: [PATCH 30/64] Tweak functionality and add tests --- moto/dynamodb2/models.py | 17 ++-- .../test_dynamodbstreams.py | 99 ++++++++++++++++++- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index e2882f1e4..d15b9fce5 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -406,10 +406,13 @@ class Table(BaseModel): def set_stream_specification(self, streams): self.stream_specification = streams - if streams and streams.get('StreamEnabled'): + if streams and (streams.get('StreamEnabled') + or streams.get('StreamViewType')): + self.stream_specification['StreamEnabled'] = True self.latest_stream_label = datetime.datetime.utcnow().isoformat() self.stream_shard = StreamShard(self) else: + self.stream_specification = {'StreamEnabled': False} self.latest_stream_label = None self.stream_shard = None @@ -429,11 +432,11 @@ class Table(BaseModel): 'LocalSecondaryIndexes': [index for index in self.indexes], } } - if self.stream_specification: + if self.stream_specification and self.stream_specification['StreamEnabled']: results[base_key]['StreamSpecification'] = self.stream_specification - if self.latest_stream_label: - results[base_key]['LatestStreamLabel'] = self.latest_stream_label - results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label + if self.latest_stream_label: + results[base_key]['LatestStreamLabel'] = self.latest_stream_label + results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label return results def __len__(self): @@ -779,7 +782,9 @@ class DynamoDBBackend(BaseBackend): def update_table_streams(self, name, stream_specification): table = self.tables[name] - if stream_specification['StreamEnabled'] and table.latest_stream_label: + if ((stream_specification.get('StreamEnabled') + or stream_specification.get('StreamViewType')) + and table.latest_stream_label): raise ValueError('Table already has stream enabled') table.set_stream_specification(stream_specification) return table diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index 94c0c51b2..7e4025626 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -1,10 +1,12 @@ from __future__ import unicode_literals, print_function +from nose.tools import assert_raises + import boto3 from moto import mock_dynamodb2, mock_dynamodbstreams -class TestClass(): +class TestCore(): stream_arn = None mocks = [] @@ -84,7 +86,7 @@ class TestClass(): resp = conn.get_shard_iterator( StreamArn=self.stream_arn, ShardId=shard_id, - ShardIteratorType='TRIM_HORIZON' + ShardIteratorType='LATEST' ) iterator_id = resp['ShardIterator'] @@ -137,3 +139,96 @@ class TestClass(): # empty resp = conn.get_records(ShardIterator=resp['NextShardIterator']) assert len(resp['Records']) == 0 + + +class TestEdges(): + mocks = [] + + def setup(self): + self.mocks = [mock_dynamodb2(), mock_dynamodbstreams()] + for m in self.mocks: + m.start() + + def teardown(self): + for m in self.mocks: + m.stop() + + + def test_enable_stream_on_table(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + resp = conn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', + 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1} + ) + assert 'StreamSpecification' not in resp['TableDescription'] + + resp = conn.update_table( + TableName='test-streams', + StreamSpecification={ + 'StreamViewType': 'KEYS_ONLY' + } + ) + assert 'StreamSpecification' in resp['TableDescription'] + assert resp['TableDescription']['StreamSpecification'] == { + 'StreamEnabled': True, + 'StreamViewType': 'KEYS_ONLY' + } + assert 'LatestStreamLabel' in resp['TableDescription'] + + # now try to enable it again + with assert_raises(ValueError): + resp = conn.update_table( + TableName='test-streams', + StreamSpecification={ + 'StreamViewType': 'OLD_IMAGES' + } + ) + + def test_stream_with_range_key(self): + dyn = boto3.client('dynamodb', region_name='us-east-1') + + resp = dyn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}, + {'AttributeName': 'color', 'KeyType': 'RANGE'}], + AttributeDefinitions=[{'AttributeName': 'id', + 'AttributeType': 'S'}, + {'AttributeName': 'color', + 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1}, + StreamSpecification={ + 'StreamViewType': 'NEW_IMAGES' + } + ) + stream_arn = resp['TableDescription']['LatestStreamArn'] + + streams = boto3.client('dynamodbstreams', region_name='us-east-1') + resp = streams.describe_stream(StreamArn=stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = streams.get_shard_iterator( + StreamArn=stream_arn, + ShardId=shard_id, + ShardIteratorType='LATEST' + ) + iterator_id = resp['ShardIterator'] + + dyn.put_item( + TableName='test-streams', + Item={'id': {'S': 'row1'}, 'color': {'S': 'blue'}} + ) + dyn.put_item( + TableName='test-streams', + Item={'id': {'S': 'row2'}, 'color': {'S': 'green'}} + ) + + resp = streams.get_records(ShardIterator=iterator_id) + assert len(resp['Records']) == 2 + assert resp['Records'][0]['eventName'] == 'INSERT' + assert resp['Records'][1]['eventName'] == 'INSERT' + From e7b01292e96172a1d716fc80c7df628e00c2b83e Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 13:57:44 -0500 Subject: [PATCH 31/64] Fix test failures in server mode --- moto/dynamodb2/responses.py | 6 +++++- moto/server.py | 11 +++++++---- tests/test_dynamodbstreams/test_dynamodbstreams.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d2bf99bfe..79dd749e2 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -174,7 +174,11 @@ class DynamoHandler(BaseResponse): throughput = self.body["ProvisionedThroughput"] table = self.dynamodb_backend.update_table_throughput(name, throughput) if 'StreamSpecification' in self.body: - table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) + try: + table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) + except ValueError: + er = 'com.amazonaws.dynamodb.v20111205#ResourceInUseException' + return self.error(er, 'Cannot enable stream') return dynamo_json_dump(table.describe()) diff --git a/moto/server.py b/moto/server.py index ba2470478..5ad02d383 100644 --- a/moto/server.py +++ b/moto/server.py @@ -80,10 +80,13 @@ class DomainDispatcherApplication(object): region = 'us-east-1' if service == 'dynamodb': - dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] - # If Newer API version, use dynamodb2 - if dynamo_api_version > "20111205": - host = "dynamodb2" + if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): + host = 'dynamodbstreams' + else: + dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] + # If Newer API version, use dynamodb2 + if dynamo_api_version > "20111205": + host = "dynamodb2" else: host = "{service}.{region}.amazonaws.com".format( service=service, region=region) diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index 7e4025626..b60c21053 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -180,7 +180,7 @@ class TestEdges(): assert 'LatestStreamLabel' in resp['TableDescription'] # now try to enable it again - with assert_raises(ValueError): + with assert_raises(conn.exceptions.ResourceInUseException): resp = conn.update_table( TableName='test-streams', StreamSpecification={ From 770ad1db565e345b1f11edfcba0225307d52be5d Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 16:21:06 -0500 Subject: [PATCH 32/64] Correct behavior of ReturnValues parameter to put_item and update_item --- moto/dynamodb2/responses.py | 38 +++++++++++++++- tests/test_dynamodb2/test_dynamodb.py | 63 +++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index e2f1ef1cc..cb716d563 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -183,6 +183,7 @@ class DynamoHandler(BaseResponse): def put_item(self): name = self.body['TableName'] item = self.body['Item'] + return_values = self.body.get('ReturnValues', 'NONE') if has_empty_keys_or_values(item): return get_empty_str_error() @@ -193,6 +194,13 @@ class DynamoHandler(BaseResponse): else: expected = None + if return_values == 'ALL_OLD': + existing_item = self.dynamodb_backend.get_item(name, item) + if existing_item: + existing_attributes = existing_item.to_json()['Attributes'] + else: + existing_attributes = {} + # Attempt to parse simple ConditionExpressions into an Expected # expression if not expected: @@ -228,6 +236,10 @@ class DynamoHandler(BaseResponse): 'TableName': name, 'CapacityUnits': 1 } + if return_values == 'ALL_OLD': + item_dict['Attributes'] = existing_attributes + else: + item_dict.pop('Attributes', None) return dynamo_json_dump(item_dict) else: er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' @@ -527,9 +539,9 @@ class DynamoHandler(BaseResponse): return dynamo_json_dump(item_dict) def update_item(self): - name = self.body['TableName'] key = self.body['Key'] + return_values = self.body.get('ReturnValues', 'NONE') update_expression = self.body.get('UpdateExpression') attribute_updates = self.body.get('AttributeUpdates') expression_attribute_names = self.body.get( @@ -537,6 +549,10 @@ class DynamoHandler(BaseResponse): expression_attribute_values = self.body.get( 'ExpressionAttributeValues', {}) existing_item = self.dynamodb_backend.get_item(name, key) + if existing_item: + existing_attributes = existing_item.to_json()['Attributes'] + else: + existing_attributes = {} if has_empty_keys_or_values(expression_attribute_values): return get_empty_str_error() @@ -591,8 +607,26 @@ class DynamoHandler(BaseResponse): 'TableName': name, 'CapacityUnits': 0.5 } - if not existing_item: + unchanged_attributes = { + k for k in existing_attributes.keys() + if existing_attributes[k] == item_dict['Attributes'].get(k) + } + changed_attributes = set(existing_attributes.keys()).union(item_dict['Attributes'].keys()).difference(unchanged_attributes) + + if return_values == 'NONE': item_dict['Attributes'] = {} + elif return_values == 'ALL_OLD': + item_dict['Attributes'] = existing_attributes + elif return_values == 'UPDATED_OLD': + item_dict['Attributes'] = { + k: v for k, v in existing_attributes.items() + if k in changed_attributes + } + elif return_values == 'UPDATED_NEW': + item_dict['Attributes'] = { + k: v for k, v in item_dict['Attributes'].items() + if k in changed_attributes + } return dynamo_json_dump(item_dict) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index afc919dd7..5bfe8f9bd 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1246,6 +1246,69 @@ def test_update_if_not_exists(): assert resp['Items'][0]['created_at'] == 123 +# https://github.com/spulec/moto/issues/1937 +@mock_dynamodb2 +def test_update_return_attributes(): + dynamodb = boto3.client('dynamodb', region_name='us-east-1') + + dynamodb.create_table( + TableName='moto-test', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} + ) + + def update(col, to, rv): + return dynamodb.update_item( + TableName='moto-test', + Key={'id': {'S': 'foo'}}, + AttributeUpdates={col: {'Value': {'S': to}, 'Action': 'PUT'}}, + ReturnValues=rv + ) + + r = update('col1', 'val1', 'ALL_NEW') + assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} + + r = update('col1', 'val2', 'ALL_OLD') + assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} + + r = update('col2', 'val3', 'UPDATED_NEW') + assert r['Attributes'] == {'col2': {'S': 'val3'}} + + r = update('col2', 'val4', 'UPDATED_OLD') + assert r['Attributes'] == {'col2': {'S': 'val3'}} + + r = update('col1', 'val5', 'NONE') + assert r['Attributes'] == {} + + +@mock_dynamodb2 +def test_put_return_attributes(): + dynamodb = boto3.client('dynamodb', region_name='us-east-1') + + dynamodb.create_table( + TableName='moto-test', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} + ) + + r = dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'col1': {'S': 'val1'}}, + ReturnValues='NONE' + ) + assert 'Attributes' not in r + + r = dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'col1': {'S': 'val2'}}, + ReturnValues='ALL_OLD' + ) + assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} + + + @mock_dynamodb2 def test_query_global_secondary_index_when_created_via_update_table_resource(): dynamodb = boto3.resource('dynamodb', region_name='us-east-1') From 46c0f8915f913b92190986ba74ea2fb266dd28c9 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Fri, 9 Nov 2018 09:32:19 -0500 Subject: [PATCH 33/64] Fix flake8 failures (boo) --- moto/dynamodb2/models.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index d15b9fce5..8187ceaf9 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -406,8 +406,7 @@ class Table(BaseModel): def set_stream_specification(self, streams): self.stream_specification = streams - if streams and (streams.get('StreamEnabled') - or streams.get('StreamViewType')): + if streams and (streams.get('StreamEnabled') or streams.get('StreamViewType')): self.stream_specification['StreamEnabled'] = True self.latest_stream_label = datetime.datetime.utcnow().isoformat() self.stream_shard = StreamShard(self) @@ -782,9 +781,7 @@ class DynamoDBBackend(BaseBackend): def update_table_streams(self, name, stream_specification): table = self.tables[name] - if ((stream_specification.get('StreamEnabled') - or stream_specification.get('StreamViewType')) - and table.latest_stream_label): + if (stream_specification.get('StreamEnabled') or stream_specification.get('StreamViewType')) and table.latest_stream_label: raise ValueError('Table already has stream enabled') table.set_stream_specification(stream_specification) return table From 2ec32c80f797039da8d7d7dd7096241f960ab33f Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Fri, 9 Nov 2018 13:21:38 -0500 Subject: [PATCH 34/64] Merge in functionality from #1899 --- moto/dynamodb2/responses.py | 15 ++++++++++++++- tests/test_dynamodb2/test_dynamodb.py | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index cb716d563..03abf73c3 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -185,6 +185,10 @@ class DynamoHandler(BaseResponse): item = self.body['Item'] return_values = self.body.get('ReturnValues', 'NONE') + if return_values not in ('ALL_OLD', 'NONE'): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Return values set to invalid value') + if has_empty_keys_or_values(item): return get_empty_str_error() @@ -524,7 +528,11 @@ class DynamoHandler(BaseResponse): def delete_item(self): name = self.body['TableName'] keys = self.body['Key'] - return_values = self.body.get('ReturnValues', '') + return_values = self.body.get('ReturnValues', 'NONE') + if return_values not in ('ALL_OLD', 'NONE'): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Return values set to invalid value') + table = self.dynamodb_backend.get_table(name) if not table: er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException' @@ -554,6 +562,11 @@ class DynamoHandler(BaseResponse): else: existing_attributes = {} + if return_values not in ('NONE', 'ALL_OLD', 'ALL_NEW', 'UPDATED_OLD', + 'UPDATED_NEW'): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Return values set to invalid value') + if has_empty_keys_or_values(expression_attribute_values): return get_empty_str_error() diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 5bfe8f9bd..47a714b1d 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1000,6 +1000,11 @@ def test_delete_item(): response = table.scan() assert response['Count'] == 2 + # Test ReturnValues validation + with assert_raises(ClientError) as ex: + table.delete_item(Key={'client': 'client1', 'app': 'app1'}, + ReturnValues='ALL_NEW') + # Test deletion and returning old value response = table.delete_item(Key={'client': 'client1', 'app': 'app1'}, ReturnValues='ALL_OLD') response['Attributes'].should.contain('client') @@ -1281,6 +1286,9 @@ def test_update_return_attributes(): r = update('col1', 'val5', 'NONE') assert r['Attributes'] == {} + with assert_raises(ClientError) as ex: + r = update('col1', 'val6', 'WRONG') + @mock_dynamodb2 def test_put_return_attributes(): @@ -1306,7 +1314,16 @@ def test_put_return_attributes(): ReturnValues='ALL_OLD' ) assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} - + + with assert_raises(ClientError) as ex: + dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'col1': {'S': 'val3'}}, + ReturnValues='ALL_NEW' + ) + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal('Return values set to invalid value') @mock_dynamodb2 From 6e6f23a1b0b7f4c9bce4060b5d32433c776c2dba Mon Sep 17 00:00:00 2001 From: Mark Challoner Date: Wed, 14 Nov 2018 12:03:42 +0000 Subject: [PATCH 35/64] Set deleted status on vpc peer deletion. --- moto/ec2/models.py | 9 ++++++--- tests/test_ec2/test_vpc_peering.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index b94cac479..f7d1eb044 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2230,6 +2230,10 @@ class VPCPeeringConnectionStatus(object): self.code = code self.message = message + def deleted(self): + self.code = 'deleted' + self.message = 'Deleted by {deleter ID}' + def initiating(self): self.code = 'initiating-request' self.message = 'Initiating Request to {accepter ID}' @@ -2292,9 +2296,8 @@ class VPCPeeringConnectionBackend(object): return self.vpc_pcxs.get(vpc_pcx_id) def delete_vpc_peering_connection(self, vpc_pcx_id): - deleted = self.vpc_pcxs.pop(vpc_pcx_id, None) - if not deleted: - raise InvalidVPCPeeringConnectionIdError(vpc_pcx_id) + deleted = self.get_vpc_peering_connection(vpc_pcx_id) + deleted._status.deleted() return deleted def accept_vpc_peering_connection(self, vpc_pcx_id): diff --git a/tests/test_ec2/test_vpc_peering.py b/tests/test_ec2/test_vpc_peering.py index 1f98791b3..082499a72 100644 --- a/tests/test_ec2/test_vpc_peering.py +++ b/tests/test_ec2/test_vpc_peering.py @@ -89,7 +89,8 @@ def test_vpc_peering_connections_delete(): verdict.should.equal(True) all_vpc_pcxs = conn.get_all_vpc_peering_connections() - all_vpc_pcxs.should.have.length_of(0) + all_vpc_pcxs.should.have.length_of(1) + all_vpc_pcxs[0]._status.code.should.equal('deleted') with assert_raises(EC2ResponseError) as cm: conn.delete_vpc_peering_connection("pcx-1234abcd") From b5eb724773d23f34fc9093bb06b6cdb5813c668b Mon Sep 17 00:00:00 2001 From: Jon Miller Date: Thu, 15 Nov 2018 15:24:45 -0800 Subject: [PATCH 36/64] Add default path to Role & InstanceProfile --- moto/iam/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4a5240a08..522915a24 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -114,7 +114,7 @@ class Role(BaseModel): self.id = role_id self.name = name self.assume_role_policy_document = assume_role_policy_document - self.path = path + self.path = path or '/' self.policies = {} self.managed_policies = {} @@ -166,7 +166,7 @@ class InstanceProfile(BaseModel): def __init__(self, instance_profile_id, name, path, roles): self.id = instance_profile_id self.name = name - self.path = path + self.path = path or '/' self.roles = roles if roles else [] @classmethod From cf5bd7665cff971b8f0e470e96773bcbe1e5ad36 Mon Sep 17 00:00:00 2001 From: Lorenz Hufnagel Date: Fri, 16 Nov 2018 12:23:39 +0100 Subject: [PATCH 37/64] Mock AWS credentials https://github.com/spulec/moto/issues/1924 --- .travis.yml | 2 -- moto/core/models.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a5de0fa2..d386102fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,6 @@ matrix: sudo: true before_install: - export BOTO_CONFIG=/dev/null - - export AWS_SECRET_ACCESS_KEY=foobar_secret - - export AWS_ACCESS_KEY_ID=foobar_key install: # We build moto first so the docker container doesn't try to compile it as well, also note we don't use # -d for docker run so the logs show up in travis diff --git a/moto/core/models.py b/moto/core/models.py index 19267ca08..9fe1e96bd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import functools import inspect +import os import re import six from io import BytesIO @@ -21,6 +22,11 @@ from .utils import ( ) +# "Mock" the AWS credentials as they can't be mocked in Botocore currently +os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") + + class BaseMockAWS(object): nested_count = 0 From 57fa11136b3c04d53195653dfca36e55d1485693 Mon Sep 17 00:00:00 2001 From: Joe Engel Date: Mon, 19 Nov 2018 15:47:21 -0800 Subject: [PATCH 38/64] Add functionalities for SAML Providers --- IMPLEMENTATION_COVERAGE.md | 10 ++--- moto/iam/models.py | 39 +++++++++++++++++ moto/iam/responses.py | 88 ++++++++++++++++++++++++++++++++++++++ tests/test_iam/test_iam.py | 46 ++++++++++++++++++++ 4 files changed, 178 insertions(+), 5 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7c68c0e31..d8c226e52 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2225,7 +2225,7 @@ - [X] create_policy - [X] create_policy_version - [X] create_role -- [ ] create_saml_provider +- [X] create_saml_provider - [ ] create_service_linked_role - [ ] create_service_specific_credential - [X] create_user @@ -2243,7 +2243,7 @@ - [X] delete_policy_version - [X] delete_role - [X] delete_role_policy -- [ ] delete_saml_provider +- [X] delete_saml_provider - [X] delete_server_certificate - [ ] delete_service_linked_role - [ ] delete_service_specific_credential @@ -2273,7 +2273,7 @@ - [X] get_policy_version - [X] get_role - [X] get_role_policy -- [ ] get_saml_provider +- [X] get_saml_provider - [X] get_server_certificate - [ ] get_service_linked_role_deletion_status - [ ] get_ssh_public_key @@ -2296,7 +2296,7 @@ - [X] list_policy_versions - [X] list_role_policies - [ ] list_roles -- [ ] list_saml_providers +- [X] list_saml_providers - [ ] list_server_certificates - [ ] list_service_specific_credentials - [ ] list_signing_certificates @@ -2323,7 +2323,7 @@ - [ ] update_open_id_connect_provider_thumbprint - [ ] update_role - [ ] update_role_description -- [ ] update_saml_provider +- [X] update_saml_provider - [ ] update_server_certificate - [ ] update_service_specific_credential - [ ] update_signing_certificate diff --git a/moto/iam/models.py b/moto/iam/models.py index 4a5240a08..c6f9e8113 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -50,6 +50,16 @@ class Policy(BaseModel): self.update_datetime = datetime.now(pytz.utc) +class SAMLProvider(BaseModel): + def __init__(self, name, saml_metadata_document=None): + self.name = name + self.saml_metadata_document = saml_metadata_document + + @property + def arn(self): + return "arn:aws:iam::{0}:saml-provider/{1}".format(ACCOUNT_ID, self.name) + + class PolicyVersion(object): def __init__(self, @@ -427,6 +437,7 @@ class IAMBackend(BaseBackend): self.credential_report = None self.managed_policies = self._init_managed_policies() self.account_aliases = [] + self.saml_providers = {} super(IAMBackend, self).__init__() def _init_managed_policies(self): @@ -937,5 +948,33 @@ class IAMBackend(BaseBackend): 'managed_policies': returned_policies } + def create_saml_provider(self, name, saml_metadata_document): + saml_provider = SAMLProvider(name, saml_metadata_document) + self.saml_providers[name] = saml_provider + return saml_provider + + def update_saml_provider(self, saml_provider_arn, saml_metadata_document): + saml_provider = self.get_saml_provider(saml_provider_arn) + saml_provider.saml_metadata_document = saml_metadata_document + return saml_provider + + def delete_saml_provider(self, saml_provider_arn): + try: + for saml_provider in list(self.list_saml_providers()): + if saml_provider.arn == saml_provider_arn: + del self.saml_providers[saml_provider.name] + except KeyError: + raise IAMNotFoundException( + "SAMLProvider {0} not found".format(saml_provider_arn)) + + def list_saml_providers(self): + return self.saml_providers.values() + + def get_saml_provider(self, saml_provider_arn): + for saml_provider in self.list_saml_providers(): + if saml_provider.arn == saml_provider_arn: + return saml_provider + raise IAMNotFoundException("SamlProvider {0} not found".format(saml_provider_arn)) + iam_backend = IAMBackend() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 22558f3f6..e05e3b3e7 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -552,6 +552,42 @@ class IamResponse(BaseResponse): roles=account_details['roles'] ) + def create_saml_provider(self): + saml_provider_name = self._get_param('Name') + saml_metadata_document = self._get_param('SAMLMetadataDocument') + saml_provider = iam_backend.create_saml_provider(saml_provider_name, saml_metadata_document) + + template = self.response_template(CREATE_SAML_PROVIDER_TEMPLATE) + return template.render(saml_provider=saml_provider) + + def update_saml_provider(self): + saml_provider_arn = self._get_param('SAMLProviderArn') + saml_metadata_document = self._get_param('SAMLMetadataDocument') + saml_provider = iam_backend.update_saml_provider(saml_provider_arn, saml_metadata_document) + + template = self.response_template(UPDATE_SAML_PROVIDER_TEMPLATE) + return template.render(saml_provider=saml_provider) + + def delete_saml_provider(self): + saml_provider_arn = self._get_param('SAMLProviderArn') + iam_backend.delete_saml_provider(saml_provider_arn) + + template = self.response_template(DELETE_SAML_PROVIDER_TEMPLATE) + return template.render() + + def list_saml_providers(self): + saml_providers = iam_backend.list_saml_providers() + + template = self.response_template(LIST_SAML_PROVIDERS_TEMPLATE) + return template.render(saml_providers=saml_providers) + + def get_saml_provider(self): + saml_provider_arn = self._get_param('SAMLProviderArn') + saml_provider = iam_backend.get_saml_provider(saml_provider_arn) + + template = self.response_template(GET_SAML_PROVIDER_TEMPLATE) + return template.render(saml_provider=saml_provider) + ATTACH_ROLE_POLICY_TEMPLATE = """ @@ -1485,3 +1521,55 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """92e79ae7-7399-11e4-8c85-4b53eEXAMPLE """ + +CREATE_SAML_PROVIDER_TEMPLATE = """ + + {{ saml_provider.arn }} + + + 29f47818-99f5-11e1-a4c3-27EXAMPLE804 + +""" + +LIST_SAML_PROVIDERS_TEMPLATE = """ + + + {% for saml_provider in saml_providers %} + + {{ saml_provider.arn }} + 2032-05-09T16:27:11Z + 2012-05-09T16:27:03Z + + {% endfor %} + + + + fd74fa8d-99f3-11e1-a4c3-27EXAMPLE804 + +""" + +GET_SAML_PROVIDER_TEMPLATE = """ + + 2012-05-09T16:27:11Z + 2015-12-31T21:59:59Z + {{ saml_provider.saml_metadata_document }} + + + 29f47818-99f5-11e1-a4c3-27EXAMPLE804 + +""" + +DELETE_SAML_PROVIDER_TEMPLATE = """ + + c749ee7f-99ef-11e1-a4c3-27EXAMPLE804 + +""" + +UPDATE_SAML_PROVIDER_TEMPLATE = """ + + {{ saml_provider.arn }} + + + 29f47818-99f5-11e1-a4c3-27EXAMPLE804 + +""" diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index bc23ff712..a2c6b596e 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -3,7 +3,9 @@ import base64 import boto import boto3 +import os import sure # noqa +import sys from boto.exception import BotoServerError from botocore.exceptions import ClientError from moto import mock_iam, mock_iam_deprecated @@ -756,5 +758,49 @@ def test_get_account_authorization_details(): len(result['GroupDetailList']) == 1 len(result['Policies']) > 1 +@mock_iam() +def test_create_saml_provider(): + conn = boto3.client('iam', region_name='us-east-1') + response = conn.create_saml_provider( + Name="TestSAMLProvider", + SAMLMetadataDocument='a' * 1024 + ) + response['SAMLProviderArn'].should.equal("arn:aws:iam::123456789012:saml-provider/TestSAMLProvider") +@mock_iam() +def test_get_saml_provider(): + conn = boto3.client('iam', region_name='us-east-1') + saml_provider_create = conn.create_saml_provider( + Name="TestSAMLProvider", + SAMLMetadataDocument='a' * 1024 + ) + response = conn.get_saml_provider( + SAMLProviderArn=saml_provider_create['SAMLProviderArn'] + ) + response['SAMLMetadataDocument'].should.equal('a' * 1024) +@mock_iam() +def test_list_saml_providers(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_saml_provider( + Name="TestSAMLProvider", + SAMLMetadataDocument='a' * 1024 + ) + response = conn.list_saml_providers() + response['SAMLProviderList'][0]['Arn'].should.equal("arn:aws:iam::123456789012:saml-provider/TestSAMLProvider") + +@mock_iam() +def test_delete_saml_provider(): + conn = boto3.client('iam', region_name='us-east-1') + saml_provider_create = conn.create_saml_provider( + Name="TestSAMLProvider", + SAMLMetadataDocument='a' * 1024 + ) + response = conn.list_saml_providers() + print(response) + len(response['SAMLProviderList']).should.equal(1) + conn.delete_saml_provider( + SAMLProviderArn=saml_provider_create['SAMLProviderArn'] + ) + response = conn.list_saml_providers() + len(response['SAMLProviderList']).should.equal(0) From 293b25a8f934fbdf2127fd2888c7323f7681d521 Mon Sep 17 00:00:00 2001 From: Tatsuya Hoshino Date: Tue, 20 Nov 2018 21:43:59 +0900 Subject: [PATCH 39/64] Add a missing trailing dot to the Name of ResourceRecordSet AWS Route53 treats www.example.com (without a trailing dot) and www.example.com. (with a trailing dot) as identical. Hence, after creating a `www.example.com` record, `www.example.com.` name is saved in Route53. But moto treated `www.example.com` and `www.example.com.` as different. This commit fixes the moto behavior. --- moto/route53/responses.py | 3 +++ tests/test_route53/test_route53.py | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/moto/route53/responses.py b/moto/route53/responses.py index 6679e7945..98ffa4c47 100644 --- a/moto/route53/responses.py +++ b/moto/route53/responses.py @@ -123,6 +123,9 @@ class Route53(BaseResponse): """ % (record_set['Name'], the_zone.name) return 400, headers, error_msg + if not record_set['Name'].endswith('.'): + record_set['Name'] += '.' + if action in ('CREATE', 'UPSERT'): if 'ResourceRecords' in record_set: resource_records = list( diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 76217b9d9..1ced9d937 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -164,8 +164,8 @@ def test_alias_rrset(): rrsets = conn.get_all_rrsets(zoneid, type="A") rrset_records = [(rr_set.name, rr) for rr_set in rrsets for rr in rr_set.resource_records] rrset_records.should.have.length_of(2) - rrset_records.should.contain(('foo.alias.testdns.aws.com', 'foo.testdns.aws.com')) - rrset_records.should.contain(('bar.alias.testdns.aws.com', 'bar.testdns.aws.com')) + rrset_records.should.contain(('foo.alias.testdns.aws.com.', 'foo.testdns.aws.com')) + rrset_records.should.contain(('bar.alias.testdns.aws.com.', 'bar.testdns.aws.com')) rrsets[0].resource_records[0].should.equal('foo.testdns.aws.com') rrsets = conn.get_all_rrsets(zoneid, type="CNAME") rrsets.should.have.length_of(1) @@ -525,7 +525,7 @@ def test_change_resource_record_sets_crud_valid(): { 'Action': 'CREATE', 'ResourceRecordSet': { - 'Name': 'prod.redis.db', + 'Name': 'prod.redis.db.', 'Type': 'A', 'TTL': 10, 'ResourceRecords': [{ @@ -540,7 +540,7 @@ def test_change_resource_record_sets_crud_valid(): response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) len(response['ResourceRecordSets']).should.equal(1) a_record_detail = response['ResourceRecordSets'][0] - a_record_detail['Name'].should.equal('prod.redis.db') + a_record_detail['Name'].should.equal('prod.redis.db.') a_record_detail['Type'].should.equal('A') a_record_detail['TTL'].should.equal(10) a_record_detail['ResourceRecords'].should.equal([{'Value': '127.0.0.1'}]) @@ -552,7 +552,7 @@ def test_change_resource_record_sets_crud_valid(): { 'Action': 'UPSERT', 'ResourceRecordSet': { - 'Name': 'prod.redis.db', + 'Name': 'prod.redis.db.', 'Type': 'CNAME', 'TTL': 60, 'ResourceRecords': [{ @@ -567,7 +567,7 @@ def test_change_resource_record_sets_crud_valid(): response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) len(response['ResourceRecordSets']).should.equal(1) cname_record_detail = response['ResourceRecordSets'][0] - cname_record_detail['Name'].should.equal('prod.redis.db') + cname_record_detail['Name'].should.equal('prod.redis.db.') cname_record_detail['Type'].should.equal('CNAME') cname_record_detail['TTL'].should.equal(60) cname_record_detail['ResourceRecords'].should.equal([{'Value': '192.168.1.1'}]) @@ -688,12 +688,12 @@ def test_list_resource_record_sets_name_type_filters(): # record_type, record_name all_records = [ - ('A', 'a.a.db'), - ('A', 'a.b.db'), - ('A', 'b.b.db'), - ('CNAME', 'b.b.db'), - ('CNAME', 'b.c.db'), - ('CNAME', 'c.c.db') + ('A', 'a.a.db.'), + ('A', 'a.b.db.'), + ('A', 'b.b.db.'), + ('CNAME', 'b.b.db.'), + ('CNAME', 'b.c.db.'), + ('CNAME', 'c.c.db.') ] for record_type, record_name in all_records: create_resource_record_set(record_type, record_name) From 96f9896885063bec511e195885088a074fa4b5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Bataille?= Date: Tue, 20 Nov 2018 18:17:21 +0100 Subject: [PATCH 40/64] test(#1959): define dummy aws credentials for test --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 0f3f1466a..1830c2e1b 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,10 @@ deps = commands = {envpython} setup.py test nosetests {posargs} +setenv = + AWS_ACCESS_KEY_ID=dummy_key + AWS_SECRET_ACCESS_KEY=dummy_secret + AWS_DEFAULT_REGION=us-east-1 [flake8] ignore = E128,E501 From 9291ff533a467a08eb1fe50924036f4b05a0ce07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Bataille?= Date: Tue, 20 Nov 2018 18:39:31 +0100 Subject: [PATCH 41/64] test(#1959): LocationConstraint us-east-1 should be refused --- tests/test_s3/test_s3.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 6e339abb6..18fb768ef 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1300,6 +1300,16 @@ def test_bucket_create_duplicate(): exc.exception.response['Error']['Code'].should.equal('BucketAlreadyExists') +@mock_s3 +def test_bucket_create_force_us_east_1(): + s3 = boto3.resource('s3', region_name='us-east-1') + with assert_raises(ClientError) as exc: + s3.create_bucket(Bucket="blah", CreateBucketConfiguration={ + 'LocationConstraint': 'us-east-1', + }) + exc.exception.response['Error']['Code'].should.equal('InvalidLocationConstraint') + + @mock_s3 def test_boto3_bucket_create_eu_central(): s3 = boto3.resource('s3', region_name='eu-central-1') From 437eb892e23dbae7a159a67f16073d46f23977e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Bataille?= Date: Tue, 20 Nov 2018 18:52:50 +0100 Subject: [PATCH 42/64] feat(#1959): LocationConstraint us-east-1 is not accepted by the CreateBucket operation --- moto/s3/responses.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 13e5f87d9..d27351e14 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -432,8 +432,19 @@ class ResponseObject(_TemplateEnvironmentMixin): else: if body: + # us-east-1, the default AWS region behaves a bit differently + # - you should not use it as a location constraint --> it fails + # - querying the location constraint returns None try: - region_name = xmltodict.parse(body)['CreateBucketConfiguration']['LocationConstraint'] + forced_region = xmltodict.parse(body)['CreateBucketConfiguration']['LocationConstraint'] + + if forced_region == DEFAULT_REGION_NAME: + raise S3ClientError( + 'InvalidLocationConstraint', + 'The specified location-constraint is not valid' + ) + else: + region_name = forced_region except KeyError: pass From b0eb7b263e9a9f0aa1d33ef8cee54cba64a7976d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Bataille?= Date: Tue, 20 Nov 2018 19:42:51 +0100 Subject: [PATCH 43/64] test(#1959): us-east-1 located bucket should return a None LocationConstraint --- tests/test_s3/test_s3.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 18fb768ef..6541e77b8 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -977,6 +977,15 @@ def test_bucket_location(): bucket.get_location().should.equal("us-west-2") +@mock_s3_deprecated +def test_bucket_location_us_east_1(): + cli = boto3.client('s3') + bucket_name = 'mybucket' + # No LocationConstraint ==> us-east-1 + cli.create_bucket(Bucket=bucket_name) + cli.get_bucket_location(Bucket=bucket_name)['LocationConstraint'].should.equal(None) + + @mock_s3_deprecated def test_ranged_get(): conn = boto.connect_s3() From 34ac5c72b992be546648bea960d56cbd28e0b993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Bataille?= Date: Tue, 20 Nov 2018 19:50:42 +0100 Subject: [PATCH 44/64] feat(#1959): bucket in us-east-1 return None as LocationConstraint --- moto/s3/responses.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index d27351e14..1eb010842 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -193,7 +193,13 @@ class ResponseObject(_TemplateEnvironmentMixin): elif 'location' in querystring: bucket = self.backend.get_bucket(bucket_name) template = self.response_template(S3_BUCKET_LOCATION) - return template.render(location=bucket.location) + + location = bucket.location + # us-east-1 is different - returns a None location + if location == DEFAULT_REGION_NAME: + location = None + + return template.render(location=location) elif 'lifecycle' in querystring: bucket = self.backend.get_bucket(bucket_name) if not bucket.rules: @@ -1187,7 +1193,7 @@ S3_DELETE_BUCKET_WITH_ITEMS_ERROR = """ """ S3_BUCKET_LOCATION = """ -{{ location }}""" +{% if location != None %}{{ location }}{% endif %}""" S3_BUCKET_LIFECYCLE_CONFIGURATION = """ From fb7e52beccd5cc15e10823630de3d99e1a998ba4 Mon Sep 17 00:00:00 2001 From: Jon Michaelchuck Date: Thu, 22 Nov 2018 06:08:03 -0600 Subject: [PATCH 45/64] Check bucket name length at CreateBucket Check that s3 bucket names follow the documented length restriction: 'Bucket names must be at least 3 and no more than 63 characters long.' See https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html --- moto/s3/exceptions.py | 10 ++++++++++ moto/s3/models.py | 9 +++++++-- tests/test_s3/test_s3.py | 14 ++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 26515dfd2..d6815ce72 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -178,3 +178,13 @@ class InvalidStorageClass(S3ClientError): "InvalidStorageClass", "The storage class you specified is not valid", *args, **kwargs) + + +class InvalidBucketName(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(InvalidBucketName, self).__init__( + "InvalidBucketName", + "The specified bucket is not valid.", + *args, **kwargs) diff --git a/moto/s3/models.py b/moto/s3/models.py index bb4d7848c..70b71645c 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -14,10 +14,13 @@ import six from bisect import insort from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime -from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \ - InvalidNotificationDestination, MalformedXML, InvalidStorageClass +from .exceptions import BucketAlreadyExists, MissingBucket, InvalidBucketName, InvalidPart, \ + EntityTooSmall, MissingKey, InvalidNotificationDestination, MalformedXML, \ + InvalidStorageClass from .utils import clean_key_name, _VersionedKeyStore +MAX_BUCKET_NAME_LENGTH = 63 +MIN_BUCKET_NAME_LENGTH = 3 UPLOAD_ID_BYTES = 43 UPLOAD_PART_MIN_SIZE = 5242880 STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"] @@ -634,6 +637,8 @@ class S3Backend(BaseBackend): def create_bucket(self, bucket_name, region_name): if bucket_name in self.buckets: raise BucketAlreadyExists(bucket=bucket_name) + if not MIN_BUCKET_NAME_LENGTH <= len(bucket_name) <= MAX_BUCKET_NAME_LENGTH: + raise InvalidBucketName() new_bucket = FakeBucket(name=bucket_name, region_name=region_name) self.buckets[bucket_name] = new_bucket return new_bucket diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 6e339abb6..35f79d3e2 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2581,3 +2581,17 @@ TEST_XML = """\ """ + +@mock_s3 +def test_boto3_bucket_name_too_long(): + s3 = boto3.client('s3', region_name='us-east-1') + with assert_raises(ClientError) as exc: + s3.create_bucket(Bucket='x'*64) + exc.exception.response['Error']['Code'].should.equal('InvalidBucketName') + +@mock_s3 +def test_boto3_bucket_name_too_short(): + s3 = boto3.client('s3', region_name='us-east-1') + with assert_raises(ClientError) as exc: + s3.create_bucket(Bucket='x'*2) + exc.exception.response['Error']['Code'].should.equal('InvalidBucketName') From 69e093fcea48e76eb434e84965343f66d4a8babe Mon Sep 17 00:00:00 2001 From: Jon Michaelchuck Date: Thu, 22 Nov 2018 06:16:37 -0600 Subject: [PATCH 46/64] flake8 indentation fix --- moto/s3/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 70b71645c..401f7b3df 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -15,8 +15,7 @@ from bisect import insort from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime from .exceptions import BucketAlreadyExists, MissingBucket, InvalidBucketName, InvalidPart, \ - EntityTooSmall, MissingKey, InvalidNotificationDestination, MalformedXML, \ - InvalidStorageClass + EntityTooSmall, MissingKey, InvalidNotificationDestination, MalformedXML, InvalidStorageClass from .utils import clean_key_name, _VersionedKeyStore MAX_BUCKET_NAME_LENGTH = 63 From 4092657472cab070dd9531bd82afd4b45fe95bb3 Mon Sep 17 00:00:00 2001 From: Andrew McCall Date: Thu, 22 Nov 2018 17:10:33 +0000 Subject: [PATCH 47/64] Added double toto_str/from_str, returned by emr create_cluster --- moto/core/responses.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/moto/core/responses.py b/moto/core/responses.py index 0f133e72c..8fb247f75 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -718,6 +718,8 @@ def to_str(value, spec): return str(value) elif vtype == 'float': return str(value) + elif vtype == 'double': + return str(value) elif vtype == 'timestamp': return datetime.datetime.utcfromtimestamp( value).replace(tzinfo=pytz.utc).isoformat() @@ -737,6 +739,8 @@ def from_str(value, spec): return int(value) elif vtype == 'float': return float(value) + elif vtype == 'double': + return float(value) elif vtype == 'timestamp': return value elif vtype == 'string': From df2120f38ca4fed270255955aefffed072071e02 Mon Sep 17 00:00:00 2001 From: Ka Wai Wan Date: Sat, 24 Nov 2018 02:32:39 -0800 Subject: [PATCH 48/64] Add instance protection support in autoscaling, with tests --- moto/autoscaling/models.py | 61 ++++++-- moto/autoscaling/responses.py | 23 +++ tests/test_autoscaling/test_autoscaling.py | 166 ++++++++++++++++++++- 3 files changed, 228 insertions(+), 22 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 0ebc4c465..27e81a87c 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -17,10 +17,12 @@ ASG_NAME_TAG = "aws:autoscaling:groupName" class InstanceState(object): - def __init__(self, instance, lifecycle_state="InService", health_status="Healthy"): + def __init__(self, instance, lifecycle_state="InService", + health_status="Healthy", protected_from_scale_in=False): self.instance = instance self.lifecycle_state = lifecycle_state self.health_status = health_status + self.protected_from_scale_in = protected_from_scale_in class FakeScalingPolicy(BaseModel): @@ -152,7 +154,8 @@ class FakeAutoScalingGroup(BaseModel): min_size, launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, load_balancers, target_group_arns, placement_group, termination_policies, - autoscaling_backend, tags): + autoscaling_backend, tags, + new_instances_protected_from_scale_in=False): self.autoscaling_backend = autoscaling_backend self.name = name @@ -178,6 +181,7 @@ class FakeAutoScalingGroup(BaseModel): self.target_group_arns = target_group_arns self.placement_group = placement_group self.termination_policies = termination_policies + self.new_instances_protected_from_scale_in = new_instances_protected_from_scale_in self.suspended_processes = [] self.instance_states = [] @@ -210,6 +214,8 @@ class FakeAutoScalingGroup(BaseModel): placement_group=None, termination_policies=properties.get("TerminationPolicies", []), tags=properties.get("Tags", []), + new_instances_protected_from_scale_in=properties.get( + "NewInstancesProtectedFromScaleIn", False) ) return group @@ -238,7 +244,8 @@ class FakeAutoScalingGroup(BaseModel): def update(self, availability_zones, desired_capacity, max_size, min_size, launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, - placement_group, termination_policies): + placement_group, termination_policies, + new_instances_protected_from_scale_in=None): if availability_zones: self.availability_zones = availability_zones if max_size is not None: @@ -256,6 +263,8 @@ class FakeAutoScalingGroup(BaseModel): self.health_check_period = health_check_period if health_check_type is not None: self.health_check_type = health_check_type + if new_instances_protected_from_scale_in is not None: + self.new_instances_protected_from_scale_in = new_instances_protected_from_scale_in if desired_capacity is not None: self.set_desired_capacity(desired_capacity) @@ -280,12 +289,16 @@ class FakeAutoScalingGroup(BaseModel): else: # Need to remove some instances count_to_remove = curr_instance_count - self.desired_capacity - instances_to_remove = self.instance_states[:count_to_remove] - instance_ids_to_remove = [ - instance.instance.id for instance in instances_to_remove] - self.autoscaling_backend.ec2_backend.terminate_instances( - instance_ids_to_remove) - self.instance_states = self.instance_states[count_to_remove:] + instances_to_remove = [ # only remove unprotected + state for state in self.instance_states + if not state.protected_from_scale_in + ][:count_to_remove] + if instances_to_remove: # just in case not instances to remove + instance_ids_to_remove = [ + instance.instance.id for instance in instances_to_remove] + self.autoscaling_backend.ec2_backend.terminate_instances( + instance_ids_to_remove) + self.instance_states = list(set(self.instance_states) - set(instances_to_remove)) def get_propagated_tags(self): propagated_tags = {} @@ -310,7 +323,10 @@ class FakeAutoScalingGroup(BaseModel): ) for instance in reservation.instances: instance.autoscaling_group = self - self.instance_states.append(InstanceState(instance)) + self.instance_states.append(InstanceState( + instance, + protected_from_scale_in=self.new_instances_protected_from_scale_in, + )) def append_target_groups(self, target_group_arns): append = [x for x in target_group_arns if x not in self.target_group_arns] @@ -372,7 +388,8 @@ class AutoScalingBackend(BaseBackend): default_cooldown, health_check_period, health_check_type, load_balancers, target_group_arns, placement_group, - termination_policies, tags): + termination_policies, tags, + new_instances_protected_from_scale_in=False): def make_int(value): return int(value) if value is not None else value @@ -403,6 +420,7 @@ class AutoScalingBackend(BaseBackend): termination_policies=termination_policies, autoscaling_backend=self, tags=tags, + new_instances_protected_from_scale_in=new_instances_protected_from_scale_in, ) self.autoscaling_groups[name] = group @@ -415,12 +433,14 @@ class AutoScalingBackend(BaseBackend): launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, placement_group, - termination_policies): + termination_policies, + new_instances_protected_from_scale_in=None): group = self.autoscaling_groups[name] group.update(availability_zones, desired_capacity, max_size, min_size, launch_config_name, vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, - placement_group, termination_policies) + placement_group, termination_policies, + new_instances_protected_from_scale_in=new_instances_protected_from_scale_in) return group def describe_auto_scaling_groups(self, names): @@ -448,7 +468,13 @@ class AutoScalingBackend(BaseBackend): raise ResourceContentionError else: group.desired_capacity = original_size + len(instance_ids) - new_instances = [InstanceState(self.ec2_backend.get_instance(x)) for x in instance_ids] + new_instances = [ + InstanceState( + self.ec2_backend.get_instance(x), + protected_from_scale_in=group.new_instances_protected_from_scale_in, + ) + for x in instance_ids + ] for instance in new_instances: self.ec2_backend.create_tags([instance.instance.id], {ASG_NAME_TAG: group.name}) group.instance_states.extend(new_instances) @@ -626,6 +652,13 @@ class AutoScalingBackend(BaseBackend): group = self.autoscaling_groups[group_name] group.suspended_processes = scaling_processes or [] + def set_instance_protection(self, group_name, instance_ids, protected_from_scale_in): + group = self.autoscaling_groups[group_name] + protected_instances = [ + x for x in group.instance_states if x.instance.id in instance_ids] + for instance in protected_instances: + instance.protected_from_scale_in = protected_from_scale_in + autoscaling_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 5586c51dd..065f32ebe 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -85,6 +85,8 @@ class AutoScalingResponse(BaseResponse): termination_policies=self._get_multi_param( 'TerminationPolicies.member'), tags=self._get_list_prefix('Tags.member'), + new_instances_protected_from_scale_in=self._get_bool_param( + 'NewInstancesProtectedFromScaleIn', False) ) template = self.response_template(CREATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() @@ -192,6 +194,8 @@ class AutoScalingResponse(BaseResponse): placement_group=self._get_param('PlacementGroup'), termination_policies=self._get_multi_param( 'TerminationPolicies.member'), + new_instances_protected_from_scale_in=self._get_bool_param( + 'NewInstancesProtectedFromScaleIn', None) ) template = self.response_template(UPDATE_AUTOSCALING_GROUP_TEMPLATE) return template.render() @@ -290,6 +294,15 @@ class AutoScalingResponse(BaseResponse): template = self.response_template(SUSPEND_PROCESSES_TEMPLATE) return template.render() + def set_instance_protection(self): + group_name = self._get_param('AutoScalingGroupName') + instance_ids = self._get_multi_param('InstanceIds.member') + protected_from_scale_in = self._get_bool_param('ProtectedFromScaleIn') + self.autoscaling_backend.set_instance_protection( + group_name, instance_ids, protected_from_scale_in) + template = self.response_template(SET_INSTANCE_PROTECTION_TEMPLATE) + return template.render() + CREATE_LAUNCH_CONFIGURATION_TEMPLATE = """ @@ -490,6 +503,7 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """{{ instance_state.instance.id }} {{ group.launch_config_name }} {{ instance_state.lifecycle_state }} + {{ instance_state.protected_from_scale_in|string|lower }} {% endfor %} @@ -530,6 +544,7 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """{{ group.placement_group }} {% endif %} + {{ group.new_instances_protected_from_scale_in|string|lower }} {% endfor %} @@ -565,6 +580,7 @@ DESCRIBE_AUTOSCALING_INSTANCES_TEMPLATE = """{{ instance_state.instance.id }} {{ instance_state.instance.autoscaling_group.launch_config_name }} {{ instance_state.lifecycle_state }} + {{ instance_state.protected_from_scale_in|string|lower }} {% endfor %} @@ -668,3 +684,10 @@ SET_INSTANCE_HEALTH_TEMPLATE = """ + + +{{ requestid }} + +""" diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index f86ca2b81..b1a65fb7e 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -710,6 +710,7 @@ def test_create_autoscaling_group_boto3(): 'PropagateAtLaunch': False }], VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=False, ) response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) @@ -728,13 +729,48 @@ def test_describe_autoscaling_groups_boto3(): MaxSize=20, DesiredCapacity=5, VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=True, ) + response = client.describe_auto_scaling_groups( AutoScalingGroupNames=["test_asg"] ) response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) - response['AutoScalingGroups'][0][ - 'AutoScalingGroupName'].should.equal('test_asg') + group = response['AutoScalingGroups'][0] + group['AutoScalingGroupName'].should.equal('test_asg') + group['NewInstancesProtectedFromScaleIn'].should.equal(True) + group['Instances'][0]['ProtectedFromScaleIn'].should.equal(True) + + +@mock_autoscaling +def test_describe_autoscaling_instances_boto3(): + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + _ = client.create_launch_configuration( + LaunchConfigurationName='test_launch_configuration' + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + LaunchConfigurationName='test_launch_configuration', + MinSize=0, + MaxSize=20, + DesiredCapacity=5, + VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=True, + ) + + response = client.describe_auto_scaling_groups( + AutoScalingGroupNames=["test_asg"] + ) + instance_ids = [ + instance['InstanceId'] + for instance in response['AutoScalingGroups'][0]['Instances'] + ] + + response = client.describe_auto_scaling_instances(InstanceIds=instance_ids) + for instance in response['AutoScalingInstances']: + instance['AutoScalingGroupName'].should.equal('test_asg') + instance['ProtectedFromScaleIn'].should.equal(True) @mock_autoscaling @@ -751,17 +787,21 @@ def test_update_autoscaling_group_boto3(): MaxSize=20, DesiredCapacity=5, VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=True, ) - response = client.update_auto_scaling_group( + _ = client.update_auto_scaling_group( AutoScalingGroupName='test_asg', MinSize=1, + NewInstancesProtectedFromScaleIn=False, ) response = client.describe_auto_scaling_groups( AutoScalingGroupNames=["test_asg"] ) - response['AutoScalingGroups'][0]['MinSize'].should.equal(1) + group = response['AutoScalingGroups'][0] + group['MinSize'].should.equal(1) + group['NewInstancesProtectedFromScaleIn'].should.equal(False) @mock_autoscaling @@ -992,9 +1032,7 @@ def test_attach_one_instance(): 'PropagateAtLaunch': True }], VPCZoneIdentifier=mocked_networking['subnet1'], - ) - response = client.describe_auto_scaling_groups( - AutoScalingGroupNames=['test_asg'] + NewInstancesProtectedFromScaleIn=True, ) ec2 = boto3.resource('ec2', 'us-east-1') @@ -1009,7 +1047,11 @@ def test_attach_one_instance(): response = client.describe_auto_scaling_groups( AutoScalingGroupNames=['test_asg'] ) - response['AutoScalingGroups'][0]['Instances'].should.have.length_of(3) + instances = response['AutoScalingGroups'][0]['Instances'] + instances.should.have.length_of(3) + for instance in instances: + instance['ProtectedFromScaleIn'].should.equal(True) + @mock_autoscaling @mock_ec2 @@ -1100,3 +1142,111 @@ def test_suspend_processes(): launch_suspended = True assert launch_suspended is True + +@mock_autoscaling +def test_set_instance_protection(): + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + _ = client.create_launch_configuration( + LaunchConfigurationName='test_launch_configuration' + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + LaunchConfigurationName='test_launch_configuration', + MinSize=0, + MaxSize=20, + DesiredCapacity=5, + VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=False, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=['test_asg']) + instance_ids = [ + instance['InstanceId'] + for instance in response['AutoScalingGroups'][0]['Instances'] + ] + protected = instance_ids[:3] + + _ = client.set_instance_protection( + AutoScalingGroupName='test_asg', + InstanceIds=protected, + ProtectedFromScaleIn=True, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=['test_asg']) + for instance in response['AutoScalingGroups'][0]['Instances']: + instance['ProtectedFromScaleIn'].should.equal( + instance['InstanceId'] in protected + ) + + +@mock_autoscaling +def test_set_desired_capacity_up_boto3(): + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + _ = client.create_launch_configuration( + LaunchConfigurationName='test_launch_configuration' + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + LaunchConfigurationName='test_launch_configuration', + MinSize=0, + MaxSize=20, + DesiredCapacity=5, + VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=True, + ) + + _ = client.set_desired_capacity( + AutoScalingGroupName='test_asg', + DesiredCapacity=10, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=['test_asg']) + instances = response['AutoScalingGroups'][0]['Instances'] + instances.should.have.length_of(10) + for instance in instances: + instance['ProtectedFromScaleIn'].should.equal(True) + + +@mock_autoscaling +def test_set_desired_capacity_down_boto3(): + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + _ = client.create_launch_configuration( + LaunchConfigurationName='test_launch_configuration' + ) + _ = client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + LaunchConfigurationName='test_launch_configuration', + MinSize=0, + MaxSize=20, + DesiredCapacity=5, + VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=True, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=['test_asg']) + instance_ids = [ + instance['InstanceId'] + for instance in response['AutoScalingGroups'][0]['Instances'] + ] + unprotected, protected = instance_ids[:2], instance_ids[2:] + + _ = client.set_instance_protection( + AutoScalingGroupName='test_asg', + InstanceIds=unprotected, + ProtectedFromScaleIn=False, + ) + + _ = client.set_desired_capacity( + AutoScalingGroupName='test_asg', + DesiredCapacity=1, + ) + + response = client.describe_auto_scaling_groups(AutoScalingGroupNames=['test_asg']) + group = response['AutoScalingGroups'][0] + group['DesiredCapacity'].should.equal(1) + instance_ids = {instance['InstanceId'] for instance in group['Instances']} + set(protected).should.equal(instance_ids) + set(unprotected).should_not.be.within(instance_ids) # only unprotected killed From 009b02bcd521626996d38d4983d02a4cf3d890de Mon Sep 17 00:00:00 2001 From: Jon Michaelchuck Date: Mon, 26 Nov 2018 15:56:46 -0800 Subject: [PATCH 49/64] Raise a client error if PutBucketTags request contains duplicate keys A PutBucketTags request with duplicate keys will raise a ClientError with code InvalidTag and message 'Cannot provide multiple Tags with the same key'. --- moto/s3/exceptions.py | 9 +++++++++ moto/s3/models.py | 5 ++++- tests/test_s3/test_s3.py | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 26515dfd2..f78e24943 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -178,3 +178,12 @@ class InvalidStorageClass(S3ClientError): "InvalidStorageClass", "The storage class you specified is not valid", *args, **kwargs) + +class DuplicateTagKeys(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(DuplicateTagKeys, self).__init__( + "InvalidTag", + "Cannot provide multiple Tags with the same key", + *args, **kwargs) diff --git a/moto/s3/models.py b/moto/s3/models.py index bb4d7848c..fd53417fa 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -15,7 +15,7 @@ from bisect import insort from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \ - InvalidNotificationDestination, MalformedXML, InvalidStorageClass + InvalidNotificationDestination, MalformedXML, InvalidStorageClass, DuplicateTagKeys from .utils import clean_key_name, _VersionedKeyStore UPLOAD_ID_BYTES = 43 @@ -773,6 +773,9 @@ class S3Backend(BaseBackend): return key def put_bucket_tagging(self, bucket_name, tagging): + tag_keys = [tag.key for tag in tagging.tag_set.tags] + if len(tag_keys) != len(set(tag_keys)): + raise DuplicateTagKeys() bucket = self.get_bucket(bucket_name) bucket.set_tags(tagging) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 6e339abb6..ffafc0dfd 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1553,6 +1553,24 @@ def test_boto3_put_bucket_tagging(): }) resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + # With duplicate tag keys: + with assert_raises(ClientError) as err: + resp = s3.put_bucket_tagging(Bucket=bucket_name, + Tagging={ + "TagSet": [ + { + "Key": "TagOne", + "Value": "ValueOne" + }, + { + "Key": "TagOne", + "Value": "ValueOneAgain" + } + ] + }) + e = err.exception + e.response["Error"]["Code"].should.equal("InvalidTag") + e.response["Error"]["Message"].should.equal("Cannot provide multiple Tags with the same key") @mock_s3 def test_boto3_get_bucket_tagging(): From 2d554cd098bbaff1db4c5901d5b535e68a727899 Mon Sep 17 00:00:00 2001 From: grahamlyons Date: Mon, 26 Nov 2018 23:58:41 +0000 Subject: [PATCH 50/64] Return the deleted stacks in the list stacks call This matches the behaviour of the AWS API. --- moto/cloudformation/models.py | 6 +++++- .../test_cloudformation_stack_crud_boto3.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index e5ab7255d..536931c25 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -223,7 +223,11 @@ class CloudFormationBackend(BaseBackend): return list(stacks) def list_stacks(self): - return self.stacks.values() + return [ + v for v in self.stacks.values() + ] + [ + v for v in self.deleted_stacks.values() + ] def get_stack(self, name_or_stack_id): all_stacks = dict(self.deleted_stacks, **self.stacks) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 9bfae6174..19d025753 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -532,6 +532,21 @@ def test_delete_stack_by_name(): cf_conn.describe_stacks()['Stacks'].should.have.length_of(0) +@mock_cloudformation +def test_delete_stack(): + cf = boto3.client('cloudformation', region_name='us-east-1') + cf.create_stack( + StackName="test_stack", + TemplateBody=dummy_template_json, + ) + + cf.delete_stack( + StackName="test_stack", + ) + stacks = cf.list_stacks() + assert stacks['StackSummaries'][0]['StackStatus'] == 'DELETE_COMPLETE' + + @mock_cloudformation def test_describe_deleted_stack(): cf_conn = boto3.client('cloudformation', region_name='us-east-1') From 4de92accab606e5a21409c36f682939c0c5f1027 Mon Sep 17 00:00:00 2001 From: grahamlyons Date: Tue, 27 Nov 2018 11:11:13 +0000 Subject: [PATCH 51/64] Fix existing tests which use `list_stacks` The tests are expecting that deleted stacks are not listed when in fact they should be. --- .../test_cloudformation_stack_crud.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py index 801faf8a1..b7906632b 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py @@ -266,9 +266,9 @@ def test_delete_stack_by_name(): template_body=dummy_template_json, ) - conn.list_stacks().should.have.length_of(1) + conn.describe_stacks().should.have.length_of(1) conn.delete_stack("test_stack") - conn.list_stacks().should.have.length_of(0) + conn.describe_stacks().should.have.length_of(0) @mock_cloudformation_deprecated @@ -279,9 +279,9 @@ def test_delete_stack_by_id(): template_body=dummy_template_json, ) - conn.list_stacks().should.have.length_of(1) + conn.describe_stacks().should.have.length_of(1) conn.delete_stack(stack_id) - conn.list_stacks().should.have.length_of(0) + conn.describe_stacks().should.have.length_of(0) with assert_raises(BotoServerError): conn.describe_stacks("test_stack") @@ -296,9 +296,9 @@ def test_delete_stack_with_resource_missing_delete_attr(): template_body=dummy_template_json3, ) - conn.list_stacks().should.have.length_of(1) + conn.describe_stacks().should.have.length_of(1) conn.delete_stack("test_stack") - conn.list_stacks().should.have.length_of(0) + conn.describe_stacks().should.have.length_of(0) @mock_cloudformation_deprecated From 7189d019dfa326978ea4d8006355d1f5ba302c4f Mon Sep 17 00:00:00 2001 From: martynaspaulikas Date: Tue, 27 Nov 2018 11:28:09 +0000 Subject: [PATCH 52/64] Implemented get_access_key_last_used Written test that still does not work due to: ParamValidationError: Parameter validation failed: Unknown parameter in input: "UserName", must be one of: AccessKeyId Refactored update_access_key and delete_access_key functions --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/iam/models.py | 32 ++++++++++++++++++++++---------- moto/iam/responses.py | 7 +++++++ tests/test_iam/test_iam.py | 13 +++++++++++++ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7c68c0e31..e3bac1918 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2257,7 +2257,7 @@ - [X] detach_user_policy - [X] enable_mfa_device - [ ] generate_credential_report -- [ ] get_access_key_last_used +- [X] get_access_key_last_used - [X] get_account_authorization_details - [ ] get_account_password_policy - [ ] get_account_summary diff --git a/moto/iam/models.py b/moto/iam/models.py index 4a5240a08..32e1ea5ee 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -224,6 +224,10 @@ class AccessKey(BaseModel): datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ" ) + self.last_used = datetime.strftime( + datetime.utcnow(), + "%Y-%m-%dT%H:%M:%SZ" + ) def get_cfn_attribute(self, attribute_name): from moto.cloudformation.exceptions import UnformattedGetAttTemplateException @@ -351,22 +355,25 @@ class User(BaseModel): def get_all_access_keys(self): return self.access_keys + def get_access_key_last_used(self, access_key_id): + key = self.get_access_key_by_id(access_key_id) + return key.last_used + def delete_access_key(self, access_key_id): - for key in self.access_keys: - if key.access_key_id == access_key_id: - self.access_keys.remove(key) - break - else: - raise IAMNotFoundException( - "Key {0} not found".format(access_key_id)) + key = self.get_access_key_by_id(access_key_id) + self.access_keys.remove(key) def update_access_key(self, access_key_id, status): + key = self.get_access_key_by_id(access_key_id) + key.status = status + + def get_access_key_by_id(self, access_key_id): for key in self.access_keys: if key.access_key_id == access_key_id: - key.status = status - break + return key else: - raise IAMNotFoundException("The Access Key with id {0} cannot be found".format(access_key_id)) + raise IAMNotFoundException( + "The Access Key with id {0} cannot be found".format(access_key_id)) def get_cfn_attribute(self, attribute_name): from moto.cloudformation.exceptions import UnformattedGetAttTemplateException @@ -838,6 +845,11 @@ class IAMBackend(BaseBackend): user = self.get_user(user_name) user.update_access_key(access_key_id, status) + def get_access_key_last_used(self, user_name, access_key_id): + user = self.get_user(user_name) + last_used = user.get_access_key_last_used(access_key_id) + return last_used + def get_all_access_keys(self, user_name, marker=None, max_items=None): user = self.get_user(user_name) keys = user.get_all_access_keys() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 22558f3f6..c3c6f8f8a 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -454,6 +454,13 @@ class IamResponse(BaseResponse): template = self.response_template(GENERIC_EMPTY_TEMPLATE) return template.render(name='UpdateAccessKey') + def get_access_key_last_used(self): + user_name = self._get_param('UserName') + access_key_id = self._get_param('AccessKeyId') + iam_backend.get_access_key_last_used(user_name, access_key_id) + template = self.response_template(GENERIC_EMPTY_TEMPLATE) + return template.render(name='GetAccessKeyLastUsed') + def list_access_keys(self): user_name = self._get_param('UserName') diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index bc23ff712..343d42b3a 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -695,6 +695,19 @@ def test_update_access_key(): resp['AccessKeyMetadata'][0]['Status'].should.equal('Inactive') +@mock_iam +def test_get_access_key_last_used(): + iam = boto3.resource('iam', region_name='us-east-1') + client = iam.meta.client + username = 'test-user' + iam.create_user(UserName=username) + with assert_raises(ClientError): + client.get_access_key_last_used(UserName=username, AccessKeyId='non-existent-key') + key = client.create_access_key(UserName=username)['AccessKey'] + resp = client.get_access_key_last_used(UserName=username, AccessKeyId=key['AccessKeyId']) + resp.should.equal(key.last_used) + + @mock_iam def test_get_account_authorization_details(): import json From 81dea64a9e072bc6fe2ae76317645e59c5b97b7d Mon Sep 17 00:00:00 2001 From: Craig Anderson Date: Tue, 27 Nov 2018 15:02:46 +0000 Subject: [PATCH 53/64] Ignore pyenv file. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7f57e98e9..f0118e85e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ python_env .ropeproject/ .pytest_cache/ venv/ - +.python-version From 96ed66c08d384fd3543804adf43253a2b61a5a79 Mon Sep 17 00:00:00 2001 From: Craig Anderson Date: Tue, 27 Nov 2018 15:31:56 +0000 Subject: [PATCH 54/64] Add AllowedPattern to SSM describe_parameters response (#1955) --- moto/ssm/models.py | 15 +++++++++++---- moto/ssm/responses.py | 3 ++- tests/test_ssm/test_ssm_boto3.py | 4 +++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index f16a7d981..2f316a3ac 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -14,10 +14,12 @@ import itertools class Parameter(BaseModel): - def __init__(self, name, value, type, description, keyid, last_modified_date, version): + def __init__(self, name, value, type, description, allowed_pattern, keyid, + last_modified_date, version): self.name = name self.type = type self.description = description + self.allowed_pattern = allowed_pattern self.keyid = keyid self.last_modified_date = last_modified_date self.version = version @@ -58,6 +60,10 @@ class Parameter(BaseModel): if self.keyid: r['KeyId'] = self.keyid + + if self.allowed_pattern: + r['AllowedPattern'] = self.allowed_pattern + return r @@ -291,7 +297,8 @@ class SimpleSystemManagerBackend(BaseBackend): return self._parameters[name] return None - def put_parameter(self, name, description, value, type, keyid, overwrite): + def put_parameter(self, name, description, value, type, allowed_pattern, + keyid, overwrite): previous_parameter = self._parameters.get(name) version = 1 @@ -302,8 +309,8 @@ class SimpleSystemManagerBackend(BaseBackend): return last_modified_date = time.time() - self._parameters[name] = Parameter( - name, value, type, description, keyid, last_modified_date, version) + self._parameters[name] = Parameter(name, value, type, description, + allowed_pattern, keyid, last_modified_date, version) return version def add_tags_to_resource(self, resource_type, resource_id, tags): diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index eb05e51b6..c47d4127a 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -160,11 +160,12 @@ class SimpleSystemManagerResponse(BaseResponse): description = self._get_param('Description') value = self._get_param('Value') type_ = self._get_param('Type') + allowed_pattern = self._get_param('AllowedPattern') keyid = self._get_param('KeyId') overwrite = self._get_param('Overwrite', False) result = self.ssm_backend.put_parameter( - name, description, value, type_, keyid, overwrite) + name, description, value, type_, allowed_pattern, keyid, overwrite) if result is None: error = { diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index f8ef3a237..94682429f 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -319,13 +319,15 @@ def test_describe_parameters(): Name='test', Description='A test parameter', Value='value', - Type='String') + Type='String', + AllowedPattern=r'.*') response = client.describe_parameters() len(response['Parameters']).should.equal(1) response['Parameters'][0]['Name'].should.equal('test') response['Parameters'][0]['Type'].should.equal('String') + response['Parameters'][0]['AllowedPattern'].should.equal(r'.*') @mock_ssm From 9418a6916d561ca4202f92d94192827eae371ef6 Mon Sep 17 00:00:00 2001 From: martynaspaulikas Date: Tue, 27 Nov 2018 16:12:41 +0000 Subject: [PATCH 55/64] Fix tests and functionality of get_access_key_last_used() --- Makefile | 1 + moto/iam/models.py | 25 +++++++++++++++++-------- moto/iam/responses.py | 20 +++++++++++++++----- tests/test_iam/test_iam.py | 14 ++++++++++---- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index f224d7091..de08c6f74 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ test: lint rm -f .coverage rm -rf cover @nosetests -sv --with-coverage --cover-html ./tests/ $(TEST_EXCLUDE) + test_server: @TEST_SERVER_MODE=true nosetests -sv --with-coverage --cover-html ./tests/ diff --git a/moto/iam/models.py b/moto/iam/models.py index 32e1ea5ee..f3d640940 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -355,10 +355,6 @@ class User(BaseModel): def get_all_access_keys(self): return self.access_keys - def get_access_key_last_used(self, access_key_id): - key = self.get_access_key_by_id(access_key_id) - return key.last_used - def delete_access_key(self, access_key_id): key = self.get_access_key_by_id(access_key_id) self.access_keys.remove(key) @@ -845,10 +841,23 @@ class IAMBackend(BaseBackend): user = self.get_user(user_name) user.update_access_key(access_key_id, status) - def get_access_key_last_used(self, user_name, access_key_id): - user = self.get_user(user_name) - last_used = user.get_access_key_last_used(access_key_id) - return last_used + def get_access_key_last_used(self, access_key_id): + access_keys_list = self.get_all_access_keys_for_all_users() + for key in access_keys_list: + if key.access_key_id == access_key_id: + return { + 'user_name': key.user_name, + 'last_used': key.last_used + } + else: + raise IAMNotFoundException( + "The Access Key with id {0} cannot be found".format(access_key_id)) + + def get_all_access_keys_for_all_users(self): + access_keys_list = [] + for user_name in self.users: + access_keys_list += self.get_all_access_keys(user_name) + return access_keys_list def get_all_access_keys(self, user_name, marker=None, max_items=None): user = self.get_user(user_name) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index c3c6f8f8a..1674b1b53 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -455,15 +455,13 @@ class IamResponse(BaseResponse): return template.render(name='UpdateAccessKey') def get_access_key_last_used(self): - user_name = self._get_param('UserName') access_key_id = self._get_param('AccessKeyId') - iam_backend.get_access_key_last_used(user_name, access_key_id) - template = self.response_template(GENERIC_EMPTY_TEMPLATE) - return template.render(name='GetAccessKeyLastUsed') + last_used_response = iam_backend.get_access_key_last_used(access_key_id) + template = self.response_template(GET_ACCESS_KEY_LAST_USED_TEMPLATE) + return template.render(user_name=last_used_response["user_name"], last_used=last_used_response["last_used"]) def list_access_keys(self): user_name = self._get_param('UserName') - keys = iam_backend.get_all_access_keys(user_name) template = self.response_template(LIST_ACCESS_KEYS_TEMPLATE) return template.render(user_name=user_name, keys=keys) @@ -1247,6 +1245,18 @@ LIST_ACCESS_KEYS_TEMPLATE = """ """ + +GET_ACCESS_KEY_LAST_USED_TEMPLATE = """ + + + {{ user_name }} + + {{ last_used }} + + + +""" + CREDENTIAL_REPORT_GENERATING = """ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 343d42b3a..b6c836b17 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -11,6 +11,7 @@ from moto.iam.models import aws_managed_policies from nose.tools import assert_raises, assert_equals from nose.tools import raises +from datetime import datetime from tests.helpers import requires_boto_gte @@ -702,10 +703,15 @@ def test_get_access_key_last_used(): username = 'test-user' iam.create_user(UserName=username) with assert_raises(ClientError): - client.get_access_key_last_used(UserName=username, AccessKeyId='non-existent-key') - key = client.create_access_key(UserName=username)['AccessKey'] - resp = client.get_access_key_last_used(UserName=username, AccessKeyId=key['AccessKeyId']) - resp.should.equal(key.last_used) + client.get_access_key_last_used(AccessKeyId='non-existent-key-id') + create_key_response = client.create_access_key(UserName=username)['AccessKey'] + resp = client.get_access_key_last_used(AccessKeyId=create_key_response['AccessKeyId']) + + datetime.strftime(resp["AccessKeyLastUsed"]["LastUsedDate"], "%Y-%m-%d").should.equal(datetime.strftime( + datetime.utcnow(), + "%Y-%m-%d" + )) + resp["UserName"].should.equal(create_key_response["UserName"]) @mock_iam From d29869bf9b678c5573f0ff343dd204c2459142a3 Mon Sep 17 00:00:00 2001 From: Jon Michaelchuck Date: Tue, 27 Nov 2018 08:32:30 -0800 Subject: [PATCH 56/64] flake8 fix --- moto/s3/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index f78e24943..c7d82ddfd 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -179,6 +179,7 @@ class InvalidStorageClass(S3ClientError): "The storage class you specified is not valid", *args, **kwargs) + class DuplicateTagKeys(S3ClientError): code = 400 From 5a3b5cab29aec9219e1d01c3143de41ee472955d Mon Sep 17 00:00:00 2001 From: vadym-serdiuk Date: Wed, 28 Nov 2018 17:33:22 +0200 Subject: [PATCH 57/64] Strip parenthesis in the KeyConditionExpression The "bloop" package uses parenthesis in the KeyConditionExpression, so query method returns nothing due to the wrong parsing of the parameters. --- moto/dynamodb2/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index e2f1ef1cc..aeddb76bf 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -311,7 +311,7 @@ class DynamoHandler(BaseResponse): def query(self): name = self.body['TableName'] # {u'KeyConditionExpression': u'#n0 = :v0', u'ExpressionAttributeValues': {u':v0': {u'S': u'johndoe'}}, u'ExpressionAttributeNames': {u'#n0': u'username'}} - key_condition_expression = self.body.get('KeyConditionExpression') + key_condition_expression = self.body.get('KeyConditionExpression').strip('()') projection_expression = self.body.get('ProjectionExpression') expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) filter_expression = self.body.get('FilterExpression') From 5db35ef168ff0ae9a1ebd397885cdaa326221f5c Mon Sep 17 00:00:00 2001 From: amitchakote7 Date: Thu, 29 Nov 2018 15:39:39 +1100 Subject: [PATCH 58/64] Added TargetGroupARNs to DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE --- moto/autoscaling/responses.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 5586c51dd..845db0136 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -508,6 +508,15 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """ {% endif %} + {% if group.target_group_arns %} + + {% for target_group_arn in group.target_group_arns %} + {{ target_group_arn }} + {% endfor %} + + {% else %} + + {% endif %} {{ group.min_size }} {% if group.vpc_zone_identifier %} {{ group.vpc_zone_identifier }} From 7d472896e1177d3e1144eb4d36ad3bbaf8ff771a Mon Sep 17 00:00:00 2001 From: vadym-serdiuk Date: Tue, 4 Dec 2018 12:28:17 +0200 Subject: [PATCH 59/64] Move parenthesis stripping down --- moto/dynamodb2/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index aeddb76bf..12530716c 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -311,7 +311,7 @@ class DynamoHandler(BaseResponse): def query(self): name = self.body['TableName'] # {u'KeyConditionExpression': u'#n0 = :v0', u'ExpressionAttributeValues': {u':v0': {u'S': u'johndoe'}}, u'ExpressionAttributeNames': {u'#n0': u'username'}} - key_condition_expression = self.body.get('KeyConditionExpression').strip('()') + key_condition_expression = self.body.get('KeyConditionExpression') projection_expression = self.body.get('ProjectionExpression') expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) filter_expression = self.body.get('FilterExpression') @@ -385,7 +385,7 @@ class DynamoHandler(BaseResponse): range_values = [value_alias_map[ range_key_expression_components[2]]] else: - hash_key_expression = key_condition_expression + hash_key_expression = key_condition_expression.strip('()') range_comparison = None range_values = [] From 2038fa92be7ce144260a574d9945c9c390f792a1 Mon Sep 17 00:00:00 2001 From: Taro Sato Date: Wed, 5 Dec 2018 16:17:28 -0800 Subject: [PATCH 60/64] Activate proper pagination for S3 common prefixes --- moto/s3/responses.py | 15 +++++++++++---- tests/test_s3/test_s3.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 13e5f87d9..1d8cf8c53 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -338,9 +338,15 @@ class ResponseObject(_TemplateEnvironmentMixin): if continuation_token or start_after: limit = continuation_token or start_after - result_keys = self._get_results_from_token(result_keys, limit) + if not delimiter: + result_keys = self._get_results_from_token(result_keys, limit) + else: + result_folders = self._get_results_from_token(result_folders, limit) - result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys) + if not delimiter: + result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys) + else: + result_folders, is_truncated, next_continuation_token = self._truncate_result(result_folders, max_keys) return template.render( bucket=bucket, @@ -358,7 +364,7 @@ class ResponseObject(_TemplateEnvironmentMixin): def _get_results_from_token(self, result_keys, token): continuation_index = 0 for key in result_keys: - if key.name > token: + if (key.name if isinstance(key, FakeKey) else key) > token: break continuation_index += 1 return result_keys[continuation_index:] @@ -367,7 +373,8 @@ class ResponseObject(_TemplateEnvironmentMixin): if len(result_keys) > max_keys: is_truncated = 'true' result_keys = result_keys[:max_keys] - next_continuation_token = result_keys[-1].name + item = result_keys[-1] + next_continuation_token = (item.name if isinstance(item, FakeKey) else item) else: is_truncated = 'false' next_continuation_token = None diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 6e339abb6..f5575c803 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1163,6 +1163,30 @@ def test_boto3_list_keys_xml_escaped(): assert 'Owner' not in resp['Contents'][0] +@mock_s3 +def test_boto3_list_objects_v2_common_prefix_pagination(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket='mybucket') + + max_keys = 1 + keys = ['test/{i}/{i}'.format(i=i) for i in range(3)] + for key in keys: + s3.put_object(Bucket='mybucket', Key=key, Body=b'v') + + prefixes = [] + args = {"Bucket": 'mybucket', "Delimiter": "/", "Prefix": "test/", "MaxKeys": max_keys} + resp = {"IsTruncated": True} + while resp.get("IsTruncated", False): + if "NextContinuationToken" in resp: + args["ContinuationToken"] = resp["NextContinuationToken"] + resp = s3.list_objects_v2(**args) + if "CommonPrefixes" in resp: + assert len(resp["CommonPrefixes"]) == max_keys + prefixes.extend(i["Prefix"] for i in resp["CommonPrefixes"]) + + assert prefixes == [k[:k.rindex('/') + 1] for k in keys] + + @mock_s3 def test_boto3_list_objects_v2_truncated_response(): s3 = boto3.client('s3', region_name='us-east-1') From a744adbcc5f3ca88aed521bd6b4e5c02e95687ad Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Thu, 6 Dec 2018 12:18:59 -0600 Subject: [PATCH 61/64] AWS is case-sensitive when using is-public to filter for AMIs and expects lower-case values --- moto/ec2/models.py | 2 +- tests/test_ec2/test_amis.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index b94cac479..31720073b 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1115,7 +1115,7 @@ class Ami(TaggedEC2Resource): elif filter_name == 'image-id': return self.id elif filter_name == 'is-public': - return str(self.is_public) + return self.is_public_string elif filter_name == 'state': return self.state elif filter_name == 'name': diff --git a/tests/test_ec2/test_amis.py b/tests/test_ec2/test_amis.py index a8d4d1b67..fd7234511 100644 --- a/tests/test_ec2/test_amis.py +++ b/tests/test_ec2/test_amis.py @@ -258,11 +258,11 @@ def test_ami_filters(): amis_by_name = conn.get_all_images(filters={'name': imageA.name}) set([ami.id for ami in amis_by_name]).should.equal(set([imageA.id])) - amis_by_public = conn.get_all_images(filters={'is-public': True}) + amis_by_public = conn.get_all_images(filters={'is-public': 'true'}) set([ami.id for ami in amis_by_public]).should.contain(imageB.id) len(amis_by_public).should.equal(35) - amis_by_nonpublic = conn.get_all_images(filters={'is-public': False}) + amis_by_nonpublic = conn.get_all_images(filters={'is-public': 'false'}) set([ami.id for ami in amis_by_nonpublic]).should.contain(imageA.id) len(amis_by_nonpublic).should.equal(1) From 99afb38524ab4211bdd2ea9320535aed4d321adb Mon Sep 17 00:00:00 2001 From: Trevor Edwards Date: Fri, 7 Dec 2018 13:22:38 -0800 Subject: [PATCH 62/64] Replace pyaml with PyYAML dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a1b8c5dae..0598d7a10 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ install_requires = [ "xmltodict", "six>1.9", "werkzeug", - "pyaml", + "PyYAML", "pytz", "python-dateutil<3.0.0,>=2.1", "python-jose<3.0.0", From 08d17a7a13f6ea9d655a3b336f1cc595ee9c4ff4 Mon Sep 17 00:00:00 2001 From: zane Date: Fri, 21 Dec 2018 14:04:52 -0800 Subject: [PATCH 63/64] adding Tags support --- moto/secretsmanager/models.py | 12 ++++-------- moto/secretsmanager/responses.py | 4 +++- .../test_secretsmanager/test_secretsmanager.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 7f89e2eb6..1350ab469 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -56,7 +56,7 @@ class SecretsManagerBackend(BaseBackend): return response - def create_secret(self, name, secret_string, **kwargs): + def create_secret(self, name, secret_string, tags, **kwargs): generated_version_id = str(uuid.uuid4()) @@ -68,7 +68,8 @@ class SecretsManagerBackend(BaseBackend): 'rotation_enabled': False, 'rotation_lambda_arn': '', 'auto_rotate_after_days': 0, - 'version_id': generated_version_id + 'version_id': generated_version_id, + 'tags': tags } self.secrets[name] = secret @@ -101,12 +102,7 @@ class SecretsManagerBackend(BaseBackend): "LastChangedDate": None, "LastAccessedDate": None, "DeletedDate": None, - "Tags": [ - { - "Key": "", - "Value": "" - }, - ] + "Tags": secret['tags'] }) return response diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index b8b6872a8..932e7bfd7 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -19,9 +19,11 @@ class SecretsManagerResponse(BaseResponse): def create_secret(self): name = self._get_param('Name') secret_string = self._get_param('SecretString') + tags = self._get_param('Tags', if_none=[]) return secretsmanager_backends[self.region].create_secret( name=name, - secret_string=secret_string + secret_string=secret_string, + tags=tags ) def get_random_password(self): diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 0e0b98b1e..169282421 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -44,6 +44,23 @@ def test_create_secret(): secret = conn.get_secret_value(SecretId='test-secret') assert secret['SecretString'] == 'foosecret' +@mock_secretsmanager +def test_create_secret_with_tags(): + conn = boto3.client('secretsmanager', region_name='us-east-1') + secret_name = 'test-secret-with-tags' + + result = conn.create_secret( + Name=secret_name, + SecretString="foosecret", + Tags=[{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}] + ) + assert result['ARN'] + assert result['Name'] == secret_name + secret_value = conn.get_secret_value(SecretId=secret_name) + assert secret_value['SecretString'] == 'foosecret' + secret_details = conn.describe_secret(SecretId=secret_name) + assert secret_details['Tags'] == [{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}] + @mock_secretsmanager def test_get_random_password_default_length(): conn = boto3.client('secretsmanager', region_name='us-west-2') From f4767c805ed7c81513dea2bb8be3432b739675a0 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 28 Dec 2018 19:38:09 -0500 Subject: [PATCH 64/64] Uncomment EMR FailureDetails. Closes #1891. --- moto/emr/responses.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 49e37ab9a..933e0177b 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -613,13 +613,11 @@ DESCRIBE_STEP_TEMPLATE = """