From 1267e201c53039146955a3df0085d520f5f1d063 Mon Sep 17 00:00:00 2001 From: Robert Rose Date: Thu, 28 Jun 2018 12:31:08 -0700 Subject: [PATCH 01/72] Updated index.rst to fix overflow The EC2 endpoint status was overflowing and creating a scroll bar on my screen. It was bugging me so I fixed it via the GitHub web interface. Will test to ensure it builds correctly when I get home from work. --- docs/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 321342401..66e12e4bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,11 +34,11 @@ Currently implemented Services: | - DynamoDB2 | - @mock_dynamodb2 | - core endpoints + partial indexes| +-----------------------+---------------------+-----------------------------------+ | EC2 | @mock_ec2 | core endpoints done | -| - AMI | | core endpoints done | -| - EBS | | core endpoints done | -| - Instances | | all endpoints done | -| - Security Groups | | core endpoints done | -| - Tags | | all endpoints done | +| - AMI | | - core endpoints done | +| - EBS | | - core endpoints done | +| - Instances | | - all endpoints done | +| - Security Groups | | - core endpoints done | +| - Tags | | - all endpoints done | +-----------------------+---------------------+-----------------------------------+ | ECS | @mock_ecs | basic endpoints done | +-----------------------+---------------------+-----------------------------------+ From f23288d9b92d8519239a2beeeb0dc27b97676f17 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:13 +0200 Subject: [PATCH 02/72] Changed the 'create_access_token' function in order to add the extra data into 'create_jwt' function --- moto/cognitoidp/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89f..1119edcbe 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -84,7 +84,11 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt(client_id, username) + extra_data = self.get_user_extra_data_by_client_id( + client_id, username + ) + access_token, expires_in = self.create_jwt(client_id, username, + extra_data=extra_data) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in @@ -97,6 +101,21 @@ class CognitoIdpUserPool(BaseModel): id_token, _ = self.create_id_token(client_id, username) return access_token, id_token, expires_in + def get_user_extra_data_by_client_id(self, client_id, username): + extra_data = {} + current_client = self.clients.get(client_id, None) + if current_client: + for readable_field in current_client.get_readable_fields(): + attribute = list(filter( + lambda f: f['Name'] == readable_field, + self.users.get(username).attributes + )) + if len(attribute) > 0: + extra_data.update({ + attribute[0]['Name']: attribute[0]['Value'] + }) + return extra_data + class CognitoIdpUserPoolDomain(BaseModel): @@ -138,6 +157,9 @@ class CognitoIdpUserPoolClient(BaseModel): return user_pool_client_json + def get_readable_fields(self): + return self.extended_config.get('ReadAttributes', []) + class CognitoIdpIdentityProvider(BaseModel): From 4776228b6824e31a43060082ef4ef15b490de3af Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:32 +0200 Subject: [PATCH 03/72] Testing new feature --- tests/test_cognitoidp/test_cognitoidp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469ce..fda24146d 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -325,6 +325,7 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") + username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] @@ -399,15 +400,22 @@ def authentication_flow(conn): username = str(uuid.uuid4()) temporary_password = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + user_attribute_name = str(uuid.uuid4()) + user_attribute_value = str(uuid.uuid4()) client_id = conn.create_user_pool_client( UserPoolId=user_pool_id, ClientName=str(uuid.uuid4()), + ReadAttributes=[user_attribute_name] )["UserPoolClient"]["ClientId"] conn.admin_create_user( UserPoolId=user_pool_id, Username=username, TemporaryPassword=temporary_password, + UserAttributes=[{ + 'Name': user_attribute_name, + 'Value': user_attribute_value + }] ) result = conn.admin_initiate_auth( @@ -446,6 +454,9 @@ def authentication_flow(conn): "access_token": result["AuthenticationResult"]["AccessToken"], "username": username, "password": new_password, + "additional_fields": { + user_attribute_name: user_attribute_value + } } @@ -475,6 +486,8 @@ def test_token_legitimacy(): access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) + for k, v in outputs["additional_fields"].items(): + access_claims[k].should.equal(v) @mock_cognitoidp From df28ec03d2cc8c44387fa0ac41f57a3431dd6a45 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:07:27 +0200 Subject: [PATCH 04/72] Added extra test --- tests/test_cognitoidp/test_cognitoidp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index fda24146d..d1d57f307 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,8 +4,10 @@ import boto3 import json import os import uuid +import jwt from jose import jws + from moto import mock_cognitoidp import sure # noqa @@ -446,6 +448,9 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none + jwt.decode( + result["AuthenticationResult"]["AccessToken"], verify=False + ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From aae3ab6b900b879841e75474a9fcc716f238009f Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:26:09 +0200 Subject: [PATCH 05/72] Added package dependency --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 655be0616..11f59e069 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ click==6.7 inflection==0.3.1 lxml==4.0.0 beautifulsoup4==4.6.0 +PyJWT==1.6.4 From f9762a5ecb98599109172d0af5295b6f13d1c57f Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:52:48 +0200 Subject: [PATCH 06/72] Removing failing test in order to check coverage and Travis test --- requirements-dev.txt | 1 - tests/test_cognitoidp/test_cognitoidp.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 11f59e069..655be0616 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,3 @@ click==6.7 inflection==0.3.1 lxml==4.0.0 beautifulsoup4==4.6.0 -PyJWT==1.6.4 diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index d1d57f307..98153feef 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,7 +4,6 @@ import boto3 import json import os import uuid -import jwt from jose import jws @@ -448,9 +447,6 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none - jwt.decode( - result["AuthenticationResult"]["AccessToken"], verify=False - ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From 2253bbf36123cf542f95f33b0037984686d199d6 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:13 +0200 Subject: [PATCH 07/72] Changed the 'create_access_token' function in order to add the extra data into 'create_jwt' function --- moto/cognitoidp/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89f..1119edcbe 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -84,7 +84,11 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt(client_id, username) + extra_data = self.get_user_extra_data_by_client_id( + client_id, username + ) + access_token, expires_in = self.create_jwt(client_id, username, + extra_data=extra_data) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in @@ -97,6 +101,21 @@ class CognitoIdpUserPool(BaseModel): id_token, _ = self.create_id_token(client_id, username) return access_token, id_token, expires_in + def get_user_extra_data_by_client_id(self, client_id, username): + extra_data = {} + current_client = self.clients.get(client_id, None) + if current_client: + for readable_field in current_client.get_readable_fields(): + attribute = list(filter( + lambda f: f['Name'] == readable_field, + self.users.get(username).attributes + )) + if len(attribute) > 0: + extra_data.update({ + attribute[0]['Name']: attribute[0]['Value'] + }) + return extra_data + class CognitoIdpUserPoolDomain(BaseModel): @@ -138,6 +157,9 @@ class CognitoIdpUserPoolClient(BaseModel): return user_pool_client_json + def get_readable_fields(self): + return self.extended_config.get('ReadAttributes', []) + class CognitoIdpIdentityProvider(BaseModel): From ab8f4dd159fd66ba8f9dedb9e8445a3ca33fb308 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:32 +0200 Subject: [PATCH 08/72] Testing new feature --- tests/test_cognitoidp/test_cognitoidp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469ce..fda24146d 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -325,6 +325,7 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") + username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] @@ -399,15 +400,22 @@ def authentication_flow(conn): username = str(uuid.uuid4()) temporary_password = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + user_attribute_name = str(uuid.uuid4()) + user_attribute_value = str(uuid.uuid4()) client_id = conn.create_user_pool_client( UserPoolId=user_pool_id, ClientName=str(uuid.uuid4()), + ReadAttributes=[user_attribute_name] )["UserPoolClient"]["ClientId"] conn.admin_create_user( UserPoolId=user_pool_id, Username=username, TemporaryPassword=temporary_password, + UserAttributes=[{ + 'Name': user_attribute_name, + 'Value': user_attribute_value + }] ) result = conn.admin_initiate_auth( @@ -446,6 +454,9 @@ def authentication_flow(conn): "access_token": result["AuthenticationResult"]["AccessToken"], "username": username, "password": new_password, + "additional_fields": { + user_attribute_name: user_attribute_value + } } @@ -475,6 +486,8 @@ def test_token_legitimacy(): access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) + for k, v in outputs["additional_fields"].items(): + access_claims[k].should.equal(v) @mock_cognitoidp From 9adb7c818df7bbaf842c49b82cba8579fb7bd8a9 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:07:27 +0200 Subject: [PATCH 09/72] Added extra test --- tests/test_cognitoidp/test_cognitoidp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index fda24146d..d1d57f307 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,8 +4,10 @@ import boto3 import json import os import uuid +import jwt from jose import jws + from moto import mock_cognitoidp import sure # noqa @@ -446,6 +448,9 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none + jwt.decode( + result["AuthenticationResult"]["AccessToken"], verify=False + ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From c2b8a7bbb0d1b53b20e7b019a1c8b65b9359abab Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:26:09 +0200 Subject: [PATCH 10/72] Added package dependency --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 111cd5f3f..54802ad89 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ click==6.7 inflection==0.3.1 lxml==4.2.3 beautifulsoup4==4.6.0 +PyJWT==1.6.4 From e69d2834e8905f42eb02ae0a451acfe891e64a50 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:52:48 +0200 Subject: [PATCH 11/72] Removing failing test in order to check coverage and Travis test --- requirements-dev.txt | 1 - tests/test_cognitoidp/test_cognitoidp.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 54802ad89..111cd5f3f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,3 @@ click==6.7 inflection==0.3.1 lxml==4.2.3 beautifulsoup4==4.6.0 -PyJWT==1.6.4 diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index d1d57f307..98153feef 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,7 +4,6 @@ import boto3 import json import os import uuid -import jwt from jose import jws @@ -448,9 +447,6 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none - jwt.decode( - result["AuthenticationResult"]["AccessToken"], verify=False - ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From da76ea633bfc6fef1ac1619dd311c379b253b833 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 28 Sep 2018 16:31:57 +0200 Subject: [PATCH 12/72] Removed redundand line --- tests/test_cognitoidp/test_cognitoidp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 98153feef..b879cc42a 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -326,7 +326,6 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") - username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] From edbc57e00d74bdfc39a604e692c5607ca57ac705 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 11:35:37 -0700 Subject: [PATCH 13/72] add support for AWS Organizations endpoints covers so far: - create_organization - describe_organization - create_account - describe_account - list_accounts all tests passing. could use some advise from maintaners. --- moto/__init__.py | 1 + moto/backends.py | 2 + moto/organizations/__init__.py | 6 + moto/organizations/models.py | 131 ++++++++++++ moto/organizations/responses.py | 48 +++++ moto/organizations/urls.py | 10 + moto/organizations/utils.py | 34 +++ tests/test_organizations/__init__.py | 0 tests/test_organizations/object_syntax.py | 24 +++ .../test_organizations_boto3.py | 193 ++++++++++++++++++ .../test_organizations_utils.py | 26 +++ 11 files changed, 475 insertions(+) create mode 100644 moto/organizations/__init__.py create mode 100644 moto/organizations/models.py create mode 100644 moto/organizations/responses.py create mode 100644 moto/organizations/urls.py create mode 100644 moto/organizations/utils.py create mode 100644 tests/test_organizations/__init__.py create mode 100644 tests/test_organizations/object_syntax.py create mode 100644 tests/test_organizations/test_organizations_boto3.py create mode 100644 tests/test_organizations/test_organizations_utils.py diff --git a/moto/__init__.py b/moto/__init__.py index 0ce5e54d1..2301b7ca1 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -27,6 +27,7 @@ from .glacier import mock_glacier, mock_glacier_deprecated # flake8: noqa from .iam import mock_iam, mock_iam_deprecated # flake8: noqa from .kinesis import mock_kinesis, mock_kinesis_deprecated # flake8: noqa from .kms import mock_kms, mock_kms_deprecated # flake8: noqa +from .organizations import mock_organizations # flake8: noqa from .opsworks import mock_opsworks, mock_opsworks_deprecated # flake8: noqa from .polly import mock_polly # flake8: noqa from .rds import mock_rds, mock_rds_deprecated # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index cd8fe174f..25fcec09a 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -26,6 +26,7 @@ from moto.kinesis import kinesis_backends from moto.kms import kms_backends from moto.logs import logs_backends from moto.opsworks import opsworks_backends +from moto.organizations import organizations_backends from moto.polly import polly_backends from moto.rds2 import rds2_backends from moto.redshift import redshift_backends @@ -72,6 +73,7 @@ BACKENDS = { 'kinesis': kinesis_backends, 'kms': kms_backends, 'opsworks': opsworks_backends, + 'organizations': organizations_backends, 'polly': polly_backends, 'redshift': redshift_backends, 'rds': rds2_backends, diff --git a/moto/organizations/__init__.py b/moto/organizations/__init__.py new file mode 100644 index 000000000..372782dd3 --- /dev/null +++ b/moto/organizations/__init__.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .models import organizations_backend +from ..core.models import base_decorator + +organizations_backends = {"global": organizations_backend} +mock_organizations = base_decorator(organizations_backends) diff --git a/moto/organizations/models.py b/moto/organizations/models.py new file mode 100644 index 000000000..50212af29 --- /dev/null +++ b/moto/organizations/models.py @@ -0,0 +1,131 @@ +from __future__ import unicode_literals + +import datetime +import time + +from moto.core import BaseBackend, BaseModel +from moto.core.utils import unix_time +from moto.organizations import utils + +MASTER_ACCOUNT_ID = '123456789012' +MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' +MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' +ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' + + +class FakeOrganization(BaseModel): + + def __init__(self, feature_set): + self.id = utils.make_random_org_id() + self.feature_set = feature_set + self.master_account_id = MASTER_ACCOUNT_ID + self.master_account_email = MASTER_ACCOUNT_EMAIL + self.available_policy_types = [{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }] + + @property + def arn(self): + return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) + + @property + def master_account_arn(self): + return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) + + def _describe(self): + return { + 'Organization': { + 'Id': self.id, + 'Arn': self.arn, + 'FeatureSet': self.feature_set, + 'MasterAccountArn': self.master_account_arn, + 'MasterAccountId': self.master_account_id, + 'MasterAccountEmail': self.master_account_email, + 'AvailablePolicyTypes': self.available_policy_types, + } + } + + +class FakeAccount(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.create_account_status_id = utils.make_random_create_account_status_id() + self.account_id = utils.make_random_account_id() + self.account_name = kwargs['AccountName'] + self.email = kwargs['Email'] + self.create_time = datetime.datetime.utcnow() + self.status = 'ACTIVE' + self.joined_method = 'CREATED' + + @property + def arn(self): + return ACCOUNT_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.account_id + ) + + @property + def create_account_status(self): + return { + 'CreateAccountStatus': { + 'Id': self.create_account_status_id, + 'AccountName': self.account_name, + 'State': 'SUCCEEDED', + 'RequestedTimestamp': unix_time(self.create_time), + 'CompletedTimestamp': unix_time(self.create_time), + 'AccountId': self.account_id, + } + } + + def describe(self): + return { + 'Account': { + 'Id': self.account_id, + 'Arn': self.arn, + 'Email': self.email, + 'Name': self.account_name, + 'Status': self.status, + 'JoinedMethod': self.joined_method, + 'JoinedTimestamp': unix_time(self.create_time), + } + } + + +class OrganizationsBackend(BaseBackend): + + def __init__(self): + self.org = None + self.accounts = [] + + def create_organization(self, **kwargs): + self.org = FakeOrganization(kwargs['FeatureSet']) + return self.org._describe() + + def describe_organization(self): + return self.org._describe() + + def create_account(self, **kwargs): + new_account = FakeAccount(self.org, **kwargs) + self.accounts.append(new_account) + return new_account.create_account_status + + def describe_account(self, **kwargs): + account = [account for account in self.accounts + if account.account_id == kwargs['AccountId']][0] + return account.describe() + + def list_accounts(self): + return dict( + Accounts=[account.describe()['Account'] for account in self.accounts] + ) + + +organizations_backend = OrganizationsBackend() + + + diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py new file mode 100644 index 000000000..10288dedb --- /dev/null +++ b/moto/organizations/responses.py @@ -0,0 +1,48 @@ +from __future__ import unicode_literals +import json + +from moto.core.responses import BaseResponse +from .models import organizations_backend + + +class OrganizationsResponse(BaseResponse): + + @property + def organizations_backend(self): + return organizations_backend + + @property + def request_params(self): + try: + return json.loads(self.body) + except ValueError: + return {} + + def _get_param(self, param, default=None): + return self.request_params.get(param, default) + + def create_organization(self): + return json.dumps( + self.organizations_backend.create_organization(**self.request_params) + ) + + def describe_organization(self): + return json.dumps( + self.organizations_backend.describe_organization() + ) + + def create_account(self): + return json.dumps( + self.organizations_backend.create_account(**self.request_params) + ) + + def describe_account(self): + return json.dumps( + self.organizations_backend.describe_account(**self.request_params) + ) + + def list_accounts(self): + return json.dumps( + self.organizations_backend.list_accounts() + ) + diff --git a/moto/organizations/urls.py b/moto/organizations/urls.py new file mode 100644 index 000000000..7911f5b53 --- /dev/null +++ b/moto/organizations/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import OrganizationsResponse + +url_bases = [ + "https?://organizations.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': OrganizationsResponse.dispatch, +} diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py new file mode 100644 index 000000000..5916dc5ea --- /dev/null +++ b/moto/organizations/utils.py @@ -0,0 +1,34 @@ +from __future__ import unicode_literals + +import random +import string + +CHARSET=string.ascii_lowercase + string.digits +ORG_ID_SIZE = 10 +ROOT_ID_SIZE = 4 +ACCOUNT_ID_SIZE = 12 +CREATE_ACCOUNT_STATUS_ID_SIZE = 8 + + +def make_random_org_id(): + # The regex pattern for an organization ID string requires "o-" + # followed by from 10 to 32 lower-case letters or digits. + # e.g. 'o-vipjnq5z86' + return 'o-' + ''.join(random.choice(CHARSET) for x in range(ORG_ID_SIZE)) + +def make_random_root_id(): + # The regex pattern for a root ID string requires "r-" followed by + # from 4 to 32 lower-case letters or digits. + # e.g. 'r-3zwx' + return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) + +def make_random_account_id(): + # The regex pattern for an account ID string requires exactly 12 digits. + # e.g. '488633172133' + return ''.join([random.choice(string.digits) for n in range(ACCOUNT_ID_SIZE)]) + +def make_random_create_account_status_id(): + # The regex pattern for an create account request ID string requires + # "car-" followed by from 8 to 32 lower-case letters or digits. + # e.g. 'car-35gxzwrp' + return 'car-' + ''.join(random.choice(CHARSET) for x in range(CREATE_ACCOUNT_STATUS_ID_SIZE)) diff --git a/tests/test_organizations/__init__.py b/tests/test_organizations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py new file mode 100644 index 000000000..35437be62 --- /dev/null +++ b/tests/test_organizations/object_syntax.py @@ -0,0 +1,24 @@ +""" +Temporary functions for checking object structures while specing out +models. This module will go away. +""" + +import yaml +import moto +from moto import organizations as orgs + + +# utils +print(orgs.utils.make_random_org_id()) +print(orgs.utils.make_random_root_id()) +print(orgs.utils.make_random_account_id()) +print(orgs.utils.make_random_create_account_id()) + +# models +my_org = orgs.models.FakeOrganization(feature_set = 'ALL') +print(yaml.dump(my_org._describe())) +#assert False + +my_account = orgs.models.FakeAccount(my_org, AccountName='blee01', Email='blee01@moto-example.org') +print(yaml.dump(my_account)) +#assert False diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py new file mode 100644 index 000000000..fa70f3306 --- /dev/null +++ b/tests/test_organizations/test_organizations_boto3.py @@ -0,0 +1,193 @@ +from __future__ import unicode_literals + +import boto3 +import botocore.exceptions +import sure # noqa +import yaml +import re +import datetime + +import moto +from moto import mock_organizations +from moto.organizations.models import ( + MASTER_ACCOUNT_ID, + MASTER_ACCOUNT_EMAIL, + ORGANIZATION_ARN_FORMAT, + MASTER_ACCOUNT_ARN_FORMAT, + ACCOUNT_ARN_FORMAT, +) +from .test_organizations_utils import ( + ORG_ID_REGEX, + ROOT_ID_REGEX, + ACCOUNT_ID_REGEX, + CREATE_ACCOUNT_STATUS_ID_REGEX, +) + +EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" + + +def validate_organization(response): + org = response['Organization'] + sorted(org.keys()).should.equal([ + 'Arn', + 'AvailablePolicyTypes', + 'FeatureSet', + 'Id', + 'MasterAccountArn', + 'MasterAccountEmail', + 'MasterAccountId', + ]) + org['Id'].should.match(ORG_ID_REGEX) + org['MasterAccountId'].should.equal(MASTER_ACCOUNT_ID) + org['MasterAccountArn'].should.equal(MASTER_ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['Arn'].should.equal(ORGANIZATION_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['MasterAccountEmail'].should.equal(MASTER_ACCOUNT_EMAIL) + org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) + org['AvailablePolicyTypes'].should.equal([{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }]) + # + #'Organization': { + # 'Id': 'string', + # 'Arn': 'string', + # 'FeatureSet': 'ALL'|'CONSOLIDATED_BILLING', + # 'MasterAccountArn': 'string', + # 'MasterAccountId': 'string', + # 'MasterAccountEmail': 'string', + # 'AvailablePolicyTypes': [ + # { + # 'Type': 'SERVICE_CONTROL_POLICY', + # 'Status': 'ENABLED'|'PENDING_ENABLE'|'PENDING_DISABLE' + # }, + # ] + #} + +def validate_account(org, account): + sorted(account.keys()).should.equal([ + 'Arn', + 'Email', + 'Id', + 'JoinedMethod', + 'JoinedTimestamp', + 'Name', + 'Status', + ]) + account['Id'].should.match(ACCOUNT_ID_REGEX) + account['Arn'].should.equal(ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + account['Id'], + )) + account['Email'].should.match(EMAIL_REGEX) + account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) + account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) + account['Name'].should.be.a(str) + account['JoinedTimestamp'].should.be.a(datetime.datetime) + #'Account': { + # 'Id': 'string', + # 'Arn': 'string', + # 'Email': 'string', + # 'Name': 'string', + # 'Status': 'ACTIVE'|'SUSPENDED', + # 'JoinedMethod': 'INVITED'|'CREATED', + # 'JoinedTimestamp': datetime(2015, 1, 1) + #} + +def validate_create_account_status(create_status): + sorted(create_status.keys()).should.equal([ + 'AccountId', + 'AccountName', + 'CompletedTimestamp', + 'Id', + 'RequestedTimestamp', + 'State', + ]) + create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) + create_status['AccountName'].should.be.a(str) + create_status['State'].should.equal('SUCCEEDED') + create_status['RequestedTimestamp'].should.be.a(datetime.datetime) + create_status['CompletedTimestamp'].should.be.a(datetime.datetime) + #'CreateAccountStatus': { + # 'Id': 'string', + # 'AccountName': 'string', + # 'State': 'IN_PROGRESS'|'SUCCEEDED'|'FAILED', + # 'RequestedTimestamp': datetime(2015, 1, 1), + # 'CompletedTimestamp': datetime(2015, 1, 1), + # 'AccountId': 'string', + # 'FailureReason': 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' + #} + +@mock_organizations +def test_create_organization(): + client = boto3.client('organizations', region_name='us-east-1') + response = client.create_organization(FeatureSet='ALL') + #print(yaml.dump(response)) + validate_organization(response) + response['Organization']['FeatureSet'].should.equal('ALL') + #assert False + +@mock_organizations +def test_describe_organization(): + client = boto3.client('organizations', region_name='us-east-1') + client.create_organization(FeatureSet='ALL') + response = client.describe_organization() + #print(yaml.dump(response)) + validate_organization(response) + #assert False + + +mockname = 'mock-account' +mockdomain = 'moto-example.org' +mockemail = '@'.join([mockname, mockdomain]) + +@mock_organizations +def test_create_account(): + client = boto3.client('organizations', region_name='us-east-1') + client.create_organization(FeatureSet='ALL') + create_status = client.create_account( + AccountName=mockname, Email=mockemail)['CreateAccountStatus'] + #print(yaml.dump(create_status, default_flow_style=False)) + validate_create_account_status(create_status) + create_status['AccountName'].should.equal(mockname) + #assert False + +@mock_organizations +def test_describe_account(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + account_id = client.create_account( + AccountName=mockname, Email=mockemail)['CreateAccountStatus']['AccountId'] + response = client.describe_account(AccountId=account_id) + #print(yaml.dump(response, default_flow_style=False)) + validate_account(org, response['Account']) + response['Account']['Name'].should.equal(mockname) + response['Account']['Email'].should.equal(mockemail) + #assert False + +@mock_organizations +def test_list_accounts(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + for i in range(5): + name = mockname + str(i) + email = name + '@' + mockdomain + client.create_account(AccountName=name, Email=email) + response = client.list_accounts() + #print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('Accounts') + accounts = response['Accounts'] + len(accounts).should.equal(5) + for account in accounts: + validate_account(org, account) + accounts[3]['Name'].should.equal(mockname + '3') + accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) + #assert False + diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py new file mode 100644 index 000000000..8144c327f --- /dev/null +++ b/tests/test_organizations/test_organizations_utils.py @@ -0,0 +1,26 @@ +from __future__ import unicode_literals + +import sure # noqa +import moto +from moto.organizations import utils + +ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE +ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE +CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + +def test_make_random_org_id(): + org_id = utils.make_random_org_id() + org_id.should.match(ORG_ID_REGEX) + +def test_make_random_root_id(): + org_id = utils.make_random_root_id() + org_id.should.match(ROOT_ID_REGEX) + +def test_make_random_account_id(): + account_id = utils.make_random_account_id() + account_id.should.match(ACCOUNT_ID_REGEX) + +def test_make_random_create_account_status_id(): + create_account_status_id = utils.make_random_create_account_status_id() + create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From f20898da0ef6c9a8c46866fb596e713054afc349 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 12:28:19 -0700 Subject: [PATCH 14/72] add info on organizations support to docs --- IMPLEMENTATION_COVERAGE.md | 12 ++++++------ README.md | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 411f55a8b..837850ac6 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,22 +3147,22 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 0% implemented +## organizations - 8% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake -- [ ] create_account -- [ ] create_organization +- [X] create_account +- [X] create_organization - [ ] create_organizational_unit - [ ] create_policy - [ ] decline_handshake - [ ] delete_organization - [ ] delete_organizational_unit - [ ] delete_policy -- [ ] describe_account +- [X] describe_account - [ ] describe_create_account_status - [ ] describe_handshake -- [ ] describe_organization +- [X] describe_organization - [ ] describe_organizational_unit - [ ] describe_policy - [ ] detach_policy @@ -3173,7 +3173,7 @@ - [ ] enable_policy_type - [ ] invite_account_to_organization - [ ] leave_organization -- [ ] list_accounts +- [X] list_accounts - [ ] list_accounts_for_parent - [ ] list_aws_service_access_for_organization - [ ] list_children diff --git a/README.md b/README.md index a6926a58f..e4fcb6508 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,8 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | KMS | @mock_kms | basic endpoints done | |------------------------------------------------------------------------------| +| Organizations | @mock_organizations | some endpoints done | +|------------------------------------------------------------------------------| | Polly | @mock_polly | all endpoints done | |------------------------------------------------------------------------------| | RDS | @mock_rds | core endpoints done | From c40d2be646884e333098bdb5fb1941f8f327cba4 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 13:23:15 -0700 Subject: [PATCH 15/72] organizations: clean up for flake8 --- moto/organizations/models.py | 8 +-- moto/organizations/responses.py | 1 - moto/organizations/utils.py | 11 ++-- tests/test_organizations/object_syntax.py | 3 +- .../test_organizations_boto3.py | 53 +++++-------------- .../test_organizations_utils.py | 5 +- 6 files changed, 26 insertions(+), 55 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 50212af29..e1d59c1da 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import datetime -import time from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -25,7 +24,7 @@ class FakeOrganization(BaseModel): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - + @property def arn(self): return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) @@ -115,7 +114,7 @@ class OrganizationsBackend(BaseBackend): return new_account.create_account_status def describe_account(self, **kwargs): - account = [account for account in self.accounts + account = [account for account in self.accounts if account.account_id == kwargs['AccountId']][0] return account.describe() @@ -126,6 +125,3 @@ class OrganizationsBackend(BaseBackend): organizations_backend = OrganizationsBackend() - - - diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 10288dedb..7c8d45014 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -45,4 +45,3 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.list_accounts() ) - diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index 5916dc5ea..1b111c1a7 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import random import string -CHARSET=string.ascii_lowercase + string.digits +CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 ACCOUNT_ID_SIZE = 12 @@ -11,24 +11,27 @@ CREATE_ACCOUNT_STATUS_ID_SIZE = 8 def make_random_org_id(): - # The regex pattern for an organization ID string requires "o-" + # The regex pattern for an organization ID string requires "o-" # followed by from 10 to 32 lower-case letters or digits. # e.g. 'o-vipjnq5z86' return 'o-' + ''.join(random.choice(CHARSET) for x in range(ORG_ID_SIZE)) + def make_random_root_id(): - # The regex pattern for a root ID string requires "r-" followed by + # The regex pattern for a root ID string requires "r-" followed by # from 4 to 32 lower-case letters or digits. # e.g. 'r-3zwx' return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) + def make_random_account_id(): # The regex pattern for an account ID string requires exactly 12 digits. # e.g. '488633172133' return ''.join([random.choice(string.digits) for n in range(ACCOUNT_ID_SIZE)]) + def make_random_create_account_status_id(): - # The regex pattern for an create account request ID string requires + # The regex pattern for an create account request ID string requires # "car-" followed by from 8 to 32 lower-case letters or digits. # e.g. 'car-35gxzwrp' return 'car-' + ''.join(random.choice(CHARSET) for x in range(CREATE_ACCOUNT_STATUS_ID_SIZE)) diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py index 35437be62..2779d1d07 100644 --- a/tests/test_organizations/object_syntax.py +++ b/tests/test_organizations/object_syntax.py @@ -4,7 +4,6 @@ models. This module will go away. """ import yaml -import moto from moto import organizations as orgs @@ -15,7 +14,7 @@ print(orgs.utils.make_random_account_id()) print(orgs.utils.make_random_create_account_id()) # models -my_org = orgs.models.FakeOrganization(feature_set = 'ALL') +my_org = orgs.models.FakeOrganization(feature_set='ALL') print(yaml.dump(my_org._describe())) #assert False diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index fa70f3306..7ae8d9577 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals import boto3 -import botocore.exceptions import sure # noqa -import yaml -import re import datetime -import moto from moto import mock_organizations from moto.organizations.models import ( MASTER_ACCOUNT_ID, @@ -53,21 +49,7 @@ def validate_organization(response): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }]) - # - #'Organization': { - # 'Id': 'string', - # 'Arn': 'string', - # 'FeatureSet': 'ALL'|'CONSOLIDATED_BILLING', - # 'MasterAccountArn': 'string', - # 'MasterAccountId': 'string', - # 'MasterAccountEmail': 'string', - # 'AvailablePolicyTypes': [ - # { - # 'Type': 'SERVICE_CONTROL_POLICY', - # 'Status': 'ENABLED'|'PENDING_ENABLE'|'PENDING_DISABLE' - # }, - # ] - #} + def validate_account(org, account): sorted(account.keys()).should.equal([ @@ -84,21 +66,13 @@ def validate_account(org, account): org['MasterAccountId'], org['Id'], account['Id'], - )) + )) account['Email'].should.match(EMAIL_REGEX) account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) account['Name'].should.be.a(str) account['JoinedTimestamp'].should.be.a(datetime.datetime) - #'Account': { - # 'Id': 'string', - # 'Arn': 'string', - # 'Email': 'string', - # 'Name': 'string', - # 'Status': 'ACTIVE'|'SUSPENDED', - # 'JoinedMethod': 'INVITED'|'CREATED', - # 'JoinedTimestamp': datetime(2015, 1, 1) - #} + def validate_create_account_status(create_status): sorted(create_status.keys()).should.equal([ @@ -115,15 +89,7 @@ def validate_create_account_status(create_status): create_status['State'].should.equal('SUCCEEDED') create_status['RequestedTimestamp'].should.be.a(datetime.datetime) create_status['CompletedTimestamp'].should.be.a(datetime.datetime) - #'CreateAccountStatus': { - # 'Id': 'string', - # 'AccountName': 'string', - # 'State': 'IN_PROGRESS'|'SUCCEEDED'|'FAILED', - # 'RequestedTimestamp': datetime(2015, 1, 1), - # 'CompletedTimestamp': datetime(2015, 1, 1), - # 'AccountId': 'string', - # 'FailureReason': 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' - #} + @mock_organizations def test_create_organization(): @@ -134,6 +100,7 @@ def test_create_organization(): response['Organization']['FeatureSet'].should.equal('ALL') #assert False + @mock_organizations def test_describe_organization(): client = boto3.client('organizations', region_name='us-east-1') @@ -148,23 +115,27 @@ mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) + @mock_organizations def test_create_account(): client = boto3.client('organizations', region_name='us-east-1') client.create_organization(FeatureSet='ALL') create_status = client.create_account( - AccountName=mockname, Email=mockemail)['CreateAccountStatus'] + AccountName=mockname, Email=mockemail + )['CreateAccountStatus'] #print(yaml.dump(create_status, default_flow_style=False)) validate_create_account_status(create_status) create_status['AccountName'].should.equal(mockname) #assert False + @mock_organizations def test_describe_account(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] account_id = client.create_account( - AccountName=mockname, Email=mockemail)['CreateAccountStatus']['AccountId'] + AccountName=mockname, Email=mockemail + )['CreateAccountStatus']['AccountId'] response = client.describe_account(AccountId=account_id) #print(yaml.dump(response, default_flow_style=False)) validate_account(org, response['Account']) @@ -172,6 +143,7 @@ def test_describe_account(): response['Account']['Email'].should.equal(mockemail) #assert False + @mock_organizations def test_list_accounts(): client = boto3.client('organizations', region_name='us-east-1') @@ -190,4 +162,3 @@ def test_list_accounts(): accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) #assert False - diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py index 8144c327f..3e29d5cb0 100644 --- a/tests/test_organizations/test_organizations_utils.py +++ b/tests/test_organizations/test_organizations_utils.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import sure # noqa -import moto from moto.organizations import utils ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE @@ -9,18 +8,22 @@ ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + def test_make_random_org_id(): org_id = utils.make_random_org_id() org_id.should.match(ORG_ID_REGEX) + def test_make_random_root_id(): org_id = utils.make_random_root_id() org_id.should.match(ROOT_ID_REGEX) + def test_make_random_account_id(): account_id = utils.make_random_account_id() account_id.should.match(ACCOUNT_ID_REGEX) + def test_make_random_create_account_status_id(): create_account_status_id = utils.make_random_create_account_status_id() create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From 6c0c6148f15b7a9133a0c9cce190e6870f787d6e Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 10:31:16 -0700 Subject: [PATCH 16/72] organizations: add endpoint list_roots --- moto/organizations/models.py | 70 +++++++++++++++++-- moto/organizations/responses.py | 5 ++ moto/organizations/utils.py | 14 ++++ tests/test_organizations/object_syntax.py | 6 +- .../test_organizations_boto3.py | 24 +++++++ .../test_organizations_utils.py | 11 ++- 6 files changed, 122 insertions(+), 8 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index e1d59c1da..265654030 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -11,7 +11,7 @@ MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' - +ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' class FakeOrganization(BaseModel): @@ -33,7 +33,7 @@ class FakeOrganization(BaseModel): def master_account_arn(self): return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) - def _describe(self): + def describe(self): return { 'Organization': { 'Id': self.id, @@ -95,18 +95,80 @@ class FakeAccount(BaseModel): } +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id() + self.name = kwargs['Name'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + + +class FakeRoot(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_root_id() + self.name = 'Root' + self.policy_types = [{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }] + + @property + def arn(self): + return ROOT_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + 'PolicyTypes': self.policy_types + } + + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] + self.roots = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - return self.org._describe() + self.roots.append(FakeRoot(self.org)) + return self.org.describe() def describe_organization(self): - return self.org._describe() + return self.org.describe() + + def list_roots(self): + return dict( + Roots=[root.describe() for root in self.roots] + ) def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 7c8d45014..1804a3fcf 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -31,6 +31,11 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.describe_organization() ) + def list_roots(self): + return json.dumps( + self.organizations_backend.list_roots() + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index 1b111c1a7..c7e5c71cd 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -7,6 +7,7 @@ CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 ACCOUNT_ID_SIZE = 12 +OU_ID_SUFFIX_SIZE = 8 CREATE_ACCOUNT_STATUS_ID_SIZE = 8 @@ -24,6 +25,19 @@ def make_random_root_id(): return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) +def make_random_ou_id(root_id): + # The regex pattern for an organizational unit ID string requires "ou-" + # followed by from 4 to 32 lower-case letters or digits (the ID of the root + # that contains the OU) followed by a second "-" dash and from 8 to 32 + # additional lower-case letters or digits. + # e.g. ou-g8sd-5oe3bjaw + return '-'.join([ + 'ou', + root_id.partition('-')[2], + ''.join(random.choice(CHARSET) for x in range(OU_ID_SUFFIX_SIZE)), + ]) + + def make_random_account_id(): # The regex pattern for an account ID string requires exactly 12 digits. # e.g. '488633172133' diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py index 2779d1d07..3fb86b9d5 100644 --- a/tests/test_organizations/object_syntax.py +++ b/tests/test_organizations/object_syntax.py @@ -9,9 +9,11 @@ from moto import organizations as orgs # utils print(orgs.utils.make_random_org_id()) -print(orgs.utils.make_random_root_id()) +root_id = orgs.utils.make_random_root_id() +print(root_id) +print(orgs.utils.make_random_ou_id(root_id)) print(orgs.utils.make_random_account_id()) -print(orgs.utils.make_random_create_account_id()) +print(orgs.utils.make_random_create_account_status_id()) # models my_org = orgs.models.FakeOrganization(feature_set='ALL') diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 7ae8d9577..f4dc3b445 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import boto3 import sure # noqa import datetime +import yaml from moto import mock_organizations from moto.organizations.models import ( @@ -11,6 +12,7 @@ from moto.organizations.models import ( ORGANIZATION_ARN_FORMAT, MASTER_ACCOUNT_ARN_FORMAT, ACCOUNT_ARN_FORMAT, + ROOT_ARN_FORMAT, ) from .test_organizations_utils import ( ORG_ID_REGEX, @@ -111,6 +113,28 @@ def test_describe_organization(): #assert False +@mock_organizations +def test_list_roots(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + response = client.list_roots() + #print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('Roots').should.be.a(list) + response['Roots'].should_not.be.empty + root = response['Roots'][0] + root.should.have.key('Id').should.match(ROOT_ID_REGEX) + root.should.have.key('Arn').should.equal(ROOT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + root['Id'], + )) + root.should.have.key('Name').should.be.a(str) + root.should.have.key('PolicyTypes').should.be.a(list) + root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') + root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') + #assert False + + mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py index 3e29d5cb0..d27201446 100644 --- a/tests/test_organizations/test_organizations_utils.py +++ b/tests/test_organizations/test_organizations_utils.py @@ -5,6 +5,7 @@ from moto.organizations import utils ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE @@ -15,8 +16,14 @@ def test_make_random_org_id(): def test_make_random_root_id(): - org_id = utils.make_random_root_id() - org_id.should.match(ROOT_ID_REGEX) + root_id = utils.make_random_root_id() + root_id.should.match(ROOT_ID_REGEX) + + +def test_make_random_ou_id(): + root_id = utils.make_random_root_id() + ou_id = utils.make_random_ou_id(root_id) + ou_id.should.match(OU_ID_REGEX) def test_make_random_account_id(): From beebb9abc871f826597e12ba29bcdaa2cd78f4af Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 11:49:26 -0700 Subject: [PATCH 17/72] organizations: add 2 more endpoints create_organizational_unit describe_organizational_unit --- IMPLEMENTATION_COVERAGE.md | 8 +-- moto/organizations/models.py | 72 +++++++++++-------- moto/organizations/responses.py | 10 +++ .../test_organizations_boto3.py | 49 +++++++++++++ 4 files changed, 107 insertions(+), 32 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 837850ac6..7b4bc5e4d 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,13 +3147,13 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 8% implemented +## organizations - 19% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake - [X] create_account - [X] create_organization -- [ ] create_organizational_unit +- [X] create_organizational_unit - [ ] create_policy - [ ] decline_handshake - [ ] delete_organization @@ -3163,7 +3163,7 @@ - [ ] describe_create_account_status - [ ] describe_handshake - [X] describe_organization -- [ ] describe_organizational_unit +- [X] describe_organizational_unit - [ ] describe_policy - [ ] detach_policy - [ ] disable_aws_service_access @@ -3184,7 +3184,7 @@ - [ ] list_parents - [ ] list_policies - [ ] list_policies_for_target -- [ ] list_roots +- [X] list_roots - [ ] list_targets_for_policy - [ ] move_account - [ ] remove_account_from_organization diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 265654030..461072ce3 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -12,6 +12,7 @@ ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' +OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' class FakeOrganization(BaseModel): @@ -95,32 +96,6 @@ class FakeAccount(BaseModel): } -class FakeOrganizationalUnit(BaseModel): - - def __init__(self, organization, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id() - self.name = kwargs['Name'] - - @property - def arn(self): - return OU_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) - - def describe(self): - return { - 'OrganizationalUnit': { - 'Id': self.id, - 'Arn': self.arn, - 'Name': self.name, - } - } - - class FakeRoot(BaseModel): def __init__(self, organization, **kwargs): @@ -150,12 +125,40 @@ class FakeRoot(BaseModel): } + +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, root_id, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id(root_id) + self.name = kwargs['Name'] + self.parent_id = kwargs['ParentId'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] self.roots = [] + self.ou = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) @@ -170,14 +173,27 @@ class OrganizationsBackend(BaseBackend): Roots=[root.describe() for root in self.roots] ) + def create_organizational_unit(self, **kwargs): + new_ou = FakeOrganizationalUnit(self.org, self.roots[0].id, **kwargs) + self.ou.append(new_ou) + return new_ou.describe() + + def describe_organizational_unit(self, **kwargs): + ou = [ + ou for ou in self.ou if ou.id == kwargs['OrganizationalUnitId'] + ].pop(0) + return ou.describe() + def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) return new_account.create_account_status def describe_account(self, **kwargs): - account = [account for account in self.accounts - if account.account_id == kwargs['AccountId']][0] + account = [ + account for account in self.accounts + if account.account_id == kwargs['AccountId'] + ].pop(0) return account.describe() def list_accounts(self): diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 1804a3fcf..4f0643cf6 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -36,6 +36,16 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.list_roots() ) + def create_organizational_unit(self): + return json.dumps( + self.organizations_backend.create_organizational_unit(**self.request_params) + ) + + def describe_organizational_unit(self): + return json.dumps( + self.organizations_backend.describe_organizational_unit(**self.request_params) + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index f4dc3b445..0e398bd43 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -13,10 +13,12 @@ from moto.organizations.models import ( MASTER_ACCOUNT_ARN_FORMAT, ACCOUNT_ARN_FORMAT, ROOT_ARN_FORMAT, + OU_ARN_FORMAT, ) from .test_organizations_utils import ( ORG_ID_REGEX, ROOT_ID_REGEX, + OU_ID_REGEX, ACCOUNT_ID_REGEX, CREATE_ACCOUNT_STATUS_ID_REGEX, ) @@ -53,6 +55,18 @@ def validate_organization(response): }]) +def validate_organizationa_unit(org, response): + response.should.have.key('OrganizationalUnit').should.be.a(dict) + ou = response['OrganizationalUnit'] + ou.should.have.key('Id').should.match(OU_ID_REGEX) + ou.should.have.key('Arn').should.equal(OU_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + ou['Id'], + )) + ou.should.have.key('Name').should.equal(ou_name) + + def validate_account(org, account): sorted(account.keys()).should.equal([ 'Arn', @@ -113,6 +127,9 @@ def test_describe_organization(): #assert False +# Organizational Units +ou_name = 'ou01' + @mock_organizations def test_list_roots(): client = boto3.client('organizations', region_name='us-east-1') @@ -135,6 +152,38 @@ def test_list_roots(): #assert False +@mock_organizations +def test_create_organizational_unit(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + response = client.create_organizational_unit( + ParentId=root_id, + Name=ou_name, + ) + #print(yaml.dump(response, default_flow_style=False)) + validate_organizationa_unit(org, response) + #assert False + + +@mock_organizations +def test_describe_organizational_unit(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou_id = client.create_organizational_unit( + ParentId=root_id, + Name=ou_name, + )['OrganizationalUnit']['Id'] + response = client.describe_organizational_unit( + OrganizationalUnitId=ou_id, + ) + print(yaml.dump(response, default_flow_style=False)) + validate_organizationa_unit(org, response) + #assert False + + +# Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) From fc2447c6a49139ce9fd06bbc48aa23f3fd1bcae7 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 13:58:27 -0700 Subject: [PATCH 18/72] organiziaions: 2 new endpoints: list_organizational_units_for_parents list_parents --- IMPLEMENTATION_COVERAGE.md | 6 +- moto/organizations/models.py | 32 ++++++++++- moto/organizations/responses.py | 10 ++++ .../test_organizations_boto3.py | 57 ++++++++++++++++--- 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7b4bc5e4d..84c574bc3 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 19% implemented +## organizations - 20% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3180,8 +3180,8 @@ - [ ] list_create_account_status - [ ] list_handshakes_for_account - [ ] list_handshakes_for_organization -- [ ] list_organizational_units_for_parent -- [ ] list_parents +- [X] list_organizational_units_for_parent +- [X] list_parents - [ ] list_policies - [ ] list_policies_for_target - [X] list_roots diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 461072ce3..b46c62798 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -14,6 +14,7 @@ ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' + class FakeOrganization(BaseModel): def __init__(self, feature_set): @@ -125,7 +126,6 @@ class FakeRoot(BaseModel): } - class FakeOrganizationalUnit(BaseModel): def __init__(self, organization, root_id, **kwargs): @@ -152,6 +152,7 @@ class FakeOrganizationalUnit(BaseModel): } } + class OrganizationsBackend(BaseBackend): def __init__(self): @@ -184,6 +185,35 @@ class OrganizationsBackend(BaseBackend): ].pop(0) return ou.describe() + def list_organizational_units_for_parent(self, **kwargs): + return dict( + OrganizationalUnits=[ + { + 'Id': ou.id, + 'Arn': ou.arn, + 'Name': ou.name, + } + for ou in self.ou + if ou.parent_id == kwargs['ParentId'] + ] + ) + + def list_parents(self, **kwargs): + parent_id = [ + ou.parent_id for ou in self.ou if ou.id == kwargs['ChildId'] + ].pop(0) + root_parents = [ + dict(Id=root.id, Type='ROOT') + for root in self.roots + if root.id == parent_id + ] + ou_parents = [ + dict(Id=ou.id, Type='ORGANIZATIONAL_UNIT') + for ou in self.ou + if ou.id == parent_id + ] + return dict(Parents=root_parents + ou_parents) + def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 4f0643cf6..9fdb73317 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -46,6 +46,16 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.describe_organizational_unit(**self.request_params) ) + def list_organizational_units_for_parent(self): + return json.dumps( + self.organizations_backend.list_organizational_units_for_parent(**self.request_params) + ) + + def list_parents(self): + return json.dumps( + self.organizations_backend.list_parents(**self.request_params) + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 0e398bd43..8375ffbe3 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -55,7 +55,7 @@ def validate_organization(response): }]) -def validate_organizationa_unit(org, response): +def validate_organizational_unit(org, response): response.should.have.key('OrganizationalUnit').should.be.a(dict) ou = response['OrganizationalUnit'] ou.should.have.key('Id').should.match(OU_ID_REGEX) @@ -64,7 +64,7 @@ def validate_organizationa_unit(org, response): org['Id'], ou['Id'], )) - ou.should.have.key('Name').should.equal(ou_name) + ou.should.have.key('Name').should.be.a(str) def validate_account(org, account): @@ -128,7 +128,6 @@ def test_describe_organization(): # Organizational Units -ou_name = 'ou01' @mock_organizations def test_list_roots(): @@ -157,12 +156,14 @@ def test_create_organizational_unit(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] root_id = client.list_roots()['Roots'][0]['Id'] + ou_name = 'ou01' response = client.create_organizational_unit( ParentId=root_id, Name=ou_name, ) #print(yaml.dump(response, default_flow_style=False)) - validate_organizationa_unit(org, response) + validate_organizational_unit(org, response) + response['OrganizationalUnit']['Name'].should.equal(ou_name) #assert False @@ -173,13 +174,51 @@ def test_describe_organizational_unit(): root_id = client.list_roots()['Roots'][0]['Id'] ou_id = client.create_organizational_unit( ParentId=root_id, - Name=ou_name, + Name='ou01', )['OrganizationalUnit']['Id'] - response = client.describe_organizational_unit( - OrganizationalUnitId=ou_id, - ) + response = client.describe_organizational_unit(OrganizationalUnitId=ou_id) print(yaml.dump(response, default_flow_style=False)) - validate_organizationa_unit(org, response) + validate_organizational_unit(org, response) + #assert False + + +@mock_organizations +def test_list_organizational_units_for_parent(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + client.create_organizational_unit(ParentId=root_id, Name='ou01') + client.create_organizational_unit(ParentId=root_id, Name='ou02') + client.create_organizational_unit(ParentId=root_id, Name='ou03') + response = client.list_organizational_units_for_parent(ParentId=root_id) + print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('OrganizationalUnits').should.be.a(list) + for ou in response['OrganizationalUnits']: + validate_organizational_unit(org, dict(OrganizationalUnit=ou)) + #assert False + + +@mock_organizations +def test_list_parents(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + response01 = client.list_parents(ChildId=ou01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + response02 = client.list_parents(ChildId=ou02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') #assert False From 009dcdb21a10399f7b4fb6f9465267de0e7324bb Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 15:25:34 -0700 Subject: [PATCH 19/72] organizations: and another 2 endpoints: list_accounts_for_parent move_account --- IMPLEMENTATION_COVERAGE.md | 6 ++-- moto/organizations/models.py | 25 +++++++++++-- moto/organizations/responses.py | 10 ++++++ .../test_organizations_boto3.py | 36 +++++++++++++++++++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 84c574bc3..e1943aa9a 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 20% implemented +## organizations - 28% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3174,7 +3174,7 @@ - [ ] invite_account_to_organization - [ ] leave_organization - [X] list_accounts -- [ ] list_accounts_for_parent +- [X] list_accounts_for_parent - [ ] list_aws_service_access_for_organization - [ ] list_children - [ ] list_create_account_status @@ -3186,7 +3186,7 @@ - [ ] list_policies_for_target - [X] list_roots - [ ] list_targets_for_policy -- [ ] move_account +- [X] move_account - [ ] remove_account_from_organization - [ ] update_organizational_unit - [ ] update_policy diff --git a/moto/organizations/models.py b/moto/organizations/models.py index b46c62798..bbcc1479b 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -51,7 +51,7 @@ class FakeOrganization(BaseModel): class FakeAccount(BaseModel): - def __init__(self, organization, **kwargs): + def __init__(self, organization, root_id, **kwargs): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() @@ -61,6 +61,7 @@ class FakeAccount(BaseModel): self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' self.joined_method = 'CREATED' + self.parent_id = root_id @property def arn(self): @@ -215,7 +216,7 @@ class OrganizationsBackend(BaseBackend): return dict(Parents=root_parents + ou_parents) def create_account(self, **kwargs): - new_account = FakeAccount(self.org, **kwargs) + new_account = FakeAccount(self.org, self.roots[0].id, **kwargs) self.accounts.append(new_account) return new_account.create_account_status @@ -231,5 +232,25 @@ class OrganizationsBackend(BaseBackend): Accounts=[account.describe()['Account'] for account in self.accounts] ) + def list_accounts_for_parent(self, **kwargs): + return dict( + Accounts=[ + account.describe()['Account'] + for account in self.accounts + if account.parent_id == kwargs['ParentId'] + ] + ) + + def move_account(self, **kwargs): + new_parent_id = kwargs['DestinationParentId'] + all_parent_id = [parent.id for parent in self.roots + self.ou] + account = [ + account for account in self.accounts if account.account_id == kwargs['AccountId'] + ].pop(0) + assert new_parent_id in all_parent_id + assert account.parent_id == kwargs['SourceParentId'] + index = self.accounts.index(account) + self.accounts[index].parent_id = new_parent_id + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 9fdb73317..6684ae685 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -70,3 +70,13 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.list_accounts() ) + + def list_accounts_for_parent(self): + return json.dumps( + self.organizations_backend.list_accounts_for_parent(**self.request_params) + ) + + def move_account(self): + return json.dumps( + self.organizations_backend.move_account(**self.request_params) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 8375ffbe3..5355e2716 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -274,3 +274,39 @@ def test_list_accounts(): accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) #assert False + + +@mock_organizations +def test_list_accounts_for_parent(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + account_id = client.create_account( + AccountName=mockname, + Email=mockemail, + )['CreateAccountStatus']['AccountId'] + response = client.list_accounts_for_parent(ParentId=root_id) + #print(yaml.dump(response, default_flow_style=False)) + account_id.should.be.within([account['Id'] for account in response['Accounts']]) + #assert False + + +@mock_organizations +def test_move_account(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + account_id = client.create_account( + AccountName=mockname, Email=mockemail + )['CreateAccountStatus']['AccountId'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + client.move_account( + AccountId=account_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response = client.list_accounts_for_parent(ParentId=ou01_id) + #print(yaml.dump(response, default_flow_style=False)) + account_id.should.be.within([account['Id'] for account in response['Accounts']]) + #assert False From 9b5c6c4f0f04392ab40df7404020fd70340e7347 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 18:48:56 -0700 Subject: [PATCH 20/72] organizations.model.FakeAccount: rename attributes: account_id -> id account_name -> name --- moto/organizations/models.py | 71 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index bbcc1479b..8a00918d1 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -55,8 +55,8 @@ class FakeAccount(BaseModel): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() - self.account_id = utils.make_random_account_id() - self.account_name = kwargs['AccountName'] + self.id = utils.make_random_account_id() + self.name = kwargs['AccountName'] self.email = kwargs['Email'] self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' @@ -68,7 +68,7 @@ class FakeAccount(BaseModel): return ACCOUNT_ARN_FORMAT.format( self.master_account_id, self.organization_id, - self.account_id + self.id ) @property @@ -76,21 +76,21 @@ class FakeAccount(BaseModel): return { 'CreateAccountStatus': { 'Id': self.create_account_status_id, - 'AccountName': self.account_name, + 'AccountName': self.name, 'State': 'SUCCEEDED', 'RequestedTimestamp': unix_time(self.create_time), 'CompletedTimestamp': unix_time(self.create_time), - 'AccountId': self.account_id, + 'AccountId': self.id, } } def describe(self): return { 'Account': { - 'Id': self.account_id, + 'Id': self.id, 'Arn': self.arn, 'Email': self.email, - 'Name': self.account_name, + 'Name': self.name, 'Status': self.status, 'JoinedMethod': self.joined_method, 'JoinedTimestamp': unix_time(self.create_time), @@ -98,6 +98,33 @@ class FakeAccount(BaseModel): } +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, root_id, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id(root_id) + self.name = kwargs['Name'] + self.parent_id = kwargs['ParentId'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + + class FakeRoot(BaseModel): def __init__(self, organization, **kwargs): @@ -127,32 +154,6 @@ class FakeRoot(BaseModel): } -class FakeOrganizationalUnit(BaseModel): - - def __init__(self, organization, root_id, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id(root_id) - self.name = kwargs['Name'] - self.parent_id = kwargs['ParentId'] - - @property - def arn(self): - return OU_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) - - def describe(self): - return { - 'OrganizationalUnit': { - 'Id': self.id, - 'Arn': self.arn, - 'Name': self.name, - } - } - class OrganizationsBackend(BaseBackend): @@ -223,7 +224,7 @@ class OrganizationsBackend(BaseBackend): def describe_account(self, **kwargs): account = [ account for account in self.accounts - if account.account_id == kwargs['AccountId'] + if account.id == kwargs['AccountId'] ].pop(0) return account.describe() @@ -245,7 +246,7 @@ class OrganizationsBackend(BaseBackend): new_parent_id = kwargs['DestinationParentId'] all_parent_id = [parent.id for parent in self.roots + self.ou] account = [ - account for account in self.accounts if account.account_id == kwargs['AccountId'] + account for account in self.accounts if account.id == kwargs['AccountId'] ].pop(0) assert new_parent_id in all_parent_id assert account.parent_id == kwargs['SourceParentId'] From 30a9aa33e55e35070eb2468c0d377eb79e7a14e3 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 20:39:13 -0700 Subject: [PATCH 21/72] organizations: endpoint list_parents now support account_id param refactered classes: FakeRoot inherits from FakeOrganizationsUnit add root_id attribute to class FakeOrganization dropped 'roots' attribute from class OrganizationaBackend --- moto/organizations/models.py | 84 ++++++++++--------- .../test_organizations_boto3.py | 79 +++++++++++------ 2 files changed, 98 insertions(+), 65 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 8a00918d1..609ce3831 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import datetime +import re from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -19,6 +20,7 @@ class FakeOrganization(BaseModel): def __init__(self, feature_set): self.id = utils.make_random_org_id() + self.root_id = utils.make_random_root_id() self.feature_set = feature_set self.master_account_id = MASTER_ACCOUNT_ID self.master_account_email = MASTER_ACCOUNT_EMAIL @@ -51,7 +53,7 @@ class FakeOrganization(BaseModel): class FakeAccount(BaseModel): - def __init__(self, organization, root_id, **kwargs): + def __init__(self, organization, **kwargs): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() @@ -61,7 +63,7 @@ class FakeAccount(BaseModel): self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' self.joined_method = 'CREATED' - self.parent_id = root_id + self.parent_id = organization.root_id @property def arn(self): @@ -100,16 +102,18 @@ class FakeAccount(BaseModel): class FakeOrganizationalUnit(BaseModel): - def __init__(self, organization, root_id, **kwargs): + def __init__(self, organization, **kwargs): + self.type = 'ORGANIZATIONAL_UNIT' self.organization_id = organization.id self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id(root_id) - self.name = kwargs['Name'] - self.parent_id = kwargs['ParentId'] + self.id = utils.make_random_ou_id(organization.root_id) + self.name = kwargs.get('Name') + self.parent_id = kwargs.get('ParentId') + self._arn_format = OU_ARN_FORMAT @property def arn(self): - return OU_ARN_FORMAT.format( + return self._arn_format.format( self.master_account_id, self.organization_id, self.id @@ -125,25 +129,18 @@ class FakeOrganizationalUnit(BaseModel): } -class FakeRoot(BaseModel): +class FakeRoot(FakeOrganizationalUnit): def __init__(self, organization, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_root_id() + super().__init__(organization, **kwargs) + self.type = 'ROOT' + self.id = organization.root_id self.name = 'Root' self.policy_types = [{ 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - - @property - def arn(self): - return ROOT_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) + self._arn_format = ROOT_ARN_FORMAT def describe(self): return { @@ -154,18 +151,16 @@ class FakeRoot(BaseModel): } - class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] - self.roots = [] self.ou = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - self.roots.append(FakeRoot(self.org)) + self.ou.append(FakeRoot(self.org)) return self.org.describe() def describe_organization(self): @@ -173,11 +168,11 @@ class OrganizationsBackend(BaseBackend): def list_roots(self): return dict( - Roots=[root.describe() for root in self.roots] + Roots=[ou.describe() for ou in self.ou if isinstance(ou, FakeRoot)] ) def create_organizational_unit(self, **kwargs): - new_ou = FakeOrganizationalUnit(self.org, self.roots[0].id, **kwargs) + new_ou = FakeOrganizationalUnit(self.org, **kwargs) self.ou.append(new_ou) return new_ou.describe() @@ -201,23 +196,29 @@ class OrganizationsBackend(BaseBackend): ) def list_parents(self, **kwargs): - parent_id = [ - ou.parent_id for ou in self.ou if ou.id == kwargs['ChildId'] - ].pop(0) - root_parents = [ - dict(Id=root.id, Type='ROOT') - for root in self.roots - if root.id == parent_id - ] - ou_parents = [ - dict(Id=ou.id, Type='ORGANIZATIONAL_UNIT') - for ou in self.ou - if ou.id == parent_id - ] - return dict(Parents=root_parents + ou_parents) + if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): + parent_id = [ + account.parent_id for account in self.accounts + if account.id == kwargs['ChildId'] + ].pop(0) + else: + parent_id = [ + ou.parent_id for ou in self.ou + if ou.id == kwargs['ChildId'] + ].pop(0) + return dict( + Parents=[ + { + 'Id': ou.id, + 'Type': ou.type, + } + for ou in self.ou + if ou.id == parent_id + ] + ) def create_account(self, **kwargs): - new_account = FakeAccount(self.org, self.roots[0].id, **kwargs) + new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) return new_account.create_account_status @@ -244,9 +245,10 @@ class OrganizationsBackend(BaseBackend): def move_account(self, **kwargs): new_parent_id = kwargs['DestinationParentId'] - all_parent_id = [parent.id for parent in self.roots + self.ou] + all_parent_id = [parent.id for parent in self.ou] account = [ - account for account in self.accounts if account.id == kwargs['AccountId'] + account for account in self.accounts + if account.id == kwargs['AccountId'] ].pop(0) assert new_parent_id in all_parent_id assert account.parent_id == kwargs['SourceParentId'] diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 5355e2716..2b692489f 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -198,30 +198,6 @@ def test_list_organizational_units_for_parent(): #assert False -@mock_organizations -def test_list_parents(): - client = boto3.client('organizations', region_name='us-east-1') - org = client.create_organization(FeatureSet='ALL')['Organization'] - root_id = client.list_roots()['Roots'][0]['Id'] - - ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') - ou01_id = ou01['OrganizationalUnit']['Id'] - response01 = client.list_parents(ChildId=ou01_id) - #print(yaml.dump(response01, default_flow_style=False)) - response01.should.have.key('Parents').should.be.a(list) - response01['Parents'][0].should.have.key('Id').should.equal(root_id) - response01['Parents'][0].should.have.key('Type').should.equal('ROOT') - - ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') - ou02_id = ou02['OrganizationalUnit']['Id'] - response02 = client.list_parents(ChildId=ou02_id) - #print(yaml.dump(response02, default_flow_style=False)) - response02.should.have.key('Parents').should.be.a(list) - response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) - response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False - - # Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' @@ -310,3 +286,58 @@ def test_move_account(): #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) #assert False + + +@mock_organizations +def test_list_parents_for_ou(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + response01 = client.list_parents(ChildId=ou01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + response02 = client.list_parents(ChildId=ou02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') + #assert False + + +@mock_organizations +def test_list_parents_for_accounts(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + account01_id = client.create_account( + AccountName='account01', + Email='account01@moto-example.org' + )['CreateAccountStatus']['AccountId'] + account02_id = client.create_account( + AccountName='account02', + Email='account02@moto-example.org' + )['CreateAccountStatus']['AccountId'] + client.move_account( + AccountId=account02_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response01 = client.list_parents(ChildId=account01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + response02 = client.list_parents(ChildId=account02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') + #assert False From 8f400b7110b7804ab7445c5aab3e909376833e79 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 22:19:42 -0700 Subject: [PATCH 22/72] organizations: add endpoint list_chilren --- moto/organizations/models.py | 60 ++++++++++++------- moto/organizations/responses.py | 5 ++ .../test_organizations_boto3.py | 41 +++++++++++++ 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 609ce3831..2e0d6b40d 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -195,28 +195,6 @@ class OrganizationsBackend(BaseBackend): ] ) - def list_parents(self, **kwargs): - if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): - parent_id = [ - account.parent_id for account in self.accounts - if account.id == kwargs['ChildId'] - ].pop(0) - else: - parent_id = [ - ou.parent_id for ou in self.ou - if ou.id == kwargs['ChildId'] - ].pop(0) - return dict( - Parents=[ - { - 'Id': ou.id, - 'Type': ou.type, - } - for ou in self.ou - if ou.id == parent_id - ] - ) - def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) @@ -255,5 +233,43 @@ class OrganizationsBackend(BaseBackend): index = self.accounts.index(account) self.accounts[index].parent_id = new_parent_id + def list_parents(self, **kwargs): + if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): + obj_list = self.accounts + else: + obj_list = self.ou + parent_id = [ + obj.parent_id for obj in obj_list + if obj.id == kwargs['ChildId'] + ].pop(0) + return dict( + Parents=[ + { + 'Id': ou.id, + 'Type': ou.type, + } + for ou in self.ou + if ou.id == parent_id + ] + ) + + def list_children(self, **kwargs): + if kwargs['ChildType'] == 'ACCOUNT': + obj_list = self.accounts + elif kwargs['ChildType'] == 'ORGANIZATIONAL_UNIT': + obj_list = self.ou + else: + raise ValueError + return dict( + Children=[ + { + 'Id': obj.id, + 'Type': kwargs['ChildType'], + } + for obj in obj_list + if obj.parent_id == kwargs['ParentId'] + ] + ) + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 6684ae685..966c3fbf3 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -80,3 +80,8 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.move_account(**self.request_params) ) + + def list_children(self): + return json.dumps( + self.organizations_backend.list_children(**self.request_params) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 2b692489f..496bcd74d 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -341,3 +341,44 @@ def test_list_parents_for_accounts(): response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') #assert False + + +@mock_organizations +def test_list_chidlren(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + account01_id = client.create_account( + AccountName='account01', + Email='account01@moto-example.org' + )['CreateAccountStatus']['AccountId'] + account02_id = client.create_account( + AccountName='account02', + Email='account02@moto-example.org' + )['CreateAccountStatus']['AccountId'] + client.move_account( + AccountId=account02_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response01 = client.list_children(ParentId=root_id, ChildType='ACCOUNT') + response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') + response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') + response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') + #print(yaml.dump(response01, default_flow_style=False)) + #print(yaml.dump(response02, default_flow_style=False)) + #print(yaml.dump(response03, default_flow_style=False)) + #print(yaml.dump(response04, default_flow_style=False)) + response01['Children'][0]['Id'].should.equal(account01_id) + response01['Children'][0]['Type'].should.equal('ACCOUNT') + response02['Children'][0]['Id'].should.equal(ou01_id) + response02['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + response03['Children'][0]['Id'].should.equal(account02_id) + response03['Children'][0]['Type'].should.equal('ACCOUNT') + response04['Children'][0]['Id'].should.equal(ou02_id) + response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + #assert False From 01912bdca7e2603035c4905c0274f835045d2a2a Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Mon, 16 Jul 2018 07:23:06 -0700 Subject: [PATCH 23/72] organizations: fix python 2.7 test errors --- moto/organizations/models.py | 2 +- tests/test_organizations/test_organizations_boto3.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 2e0d6b40d..a165395f7 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -132,7 +132,7 @@ class FakeOrganizationalUnit(BaseModel): class FakeRoot(FakeOrganizationalUnit): def __init__(self, organization, **kwargs): - super().__init__(organization, **kwargs) + super(FakeRoot, self).__init__(organization, **kwargs) self.type = 'ROOT' self.id = organization.root_id self.name = 'Root' diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 496bcd74d..0ac2b61bf 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -4,6 +4,7 @@ import boto3 import sure # noqa import datetime import yaml +import six from moto import mock_organizations from moto.organizations.models import ( @@ -64,7 +65,7 @@ def validate_organizational_unit(org, response): org['Id'], ou['Id'], )) - ou.should.have.key('Name').should.be.a(str) + ou.should.have.key('Name').should.be.a(six.string_types) def validate_account(org, account): @@ -86,7 +87,7 @@ def validate_account(org, account): account['Email'].should.match(EMAIL_REGEX) account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) - account['Name'].should.be.a(str) + account['Name'].should.be.a(six.string_types) account['JoinedTimestamp'].should.be.a(datetime.datetime) @@ -101,7 +102,7 @@ def validate_create_account_status(create_status): ]) create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) - create_status['AccountName'].should.be.a(str) + create_status['AccountName'].should.be.a(six.string_types) create_status['State'].should.equal('SUCCEEDED') create_status['RequestedTimestamp'].should.be.a(datetime.datetime) create_status['CompletedTimestamp'].should.be.a(datetime.datetime) @@ -144,7 +145,7 @@ def test_list_roots(): org['Id'], root['Id'], )) - root.should.have.key('Name').should.be.a(str) + root.should.have.key('Name').should.be.a(six.string_types) root.should.have.key('PolicyTypes').should.be.a(list) root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') From 40e422b74d56e545f4a1a903213a56bebff5c53f Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Thu, 19 Jul 2018 11:50:24 -0700 Subject: [PATCH 24/72] [issue #1720] Add support for AWS Organizations ready for pull request did a little cleanup refactoring local tests pass --- moto/organizations/models.py | 22 +-- moto/organizations/utils.py | 8 + tests/test_organizations/object_syntax.py | 25 --- .../organizations_test_utils.py | 136 ++++++++++++++++ .../test_organizations_boto3.py | 153 +----------------- .../test_organizations_utils.py | 36 ----- 6 files changed, 158 insertions(+), 222 deletions(-) delete mode 100644 tests/test_organizations/object_syntax.py create mode 100644 tests/test_organizations/organizations_test_utils.py delete mode 100644 tests/test_organizations/test_organizations_utils.py diff --git a/moto/organizations/models.py b/moto/organizations/models.py index a165395f7..0f3c67400 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -7,14 +7,6 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time from moto.organizations import utils -MASTER_ACCOUNT_ID = '123456789012' -MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' -ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' -MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' -ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' -ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' -OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' - class FakeOrganization(BaseModel): @@ -22,8 +14,8 @@ class FakeOrganization(BaseModel): self.id = utils.make_random_org_id() self.root_id = utils.make_random_root_id() self.feature_set = feature_set - self.master_account_id = MASTER_ACCOUNT_ID - self.master_account_email = MASTER_ACCOUNT_EMAIL + self.master_account_id = utils.MASTER_ACCOUNT_ID + self.master_account_email = utils.MASTER_ACCOUNT_EMAIL self.available_policy_types = [{ 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' @@ -31,11 +23,11 @@ class FakeOrganization(BaseModel): @property def arn(self): - return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) + return utils.ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) @property def master_account_arn(self): - return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) + return utils.MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) def describe(self): return { @@ -67,7 +59,7 @@ class FakeAccount(BaseModel): @property def arn(self): - return ACCOUNT_ARN_FORMAT.format( + return utils.ACCOUNT_ARN_FORMAT.format( self.master_account_id, self.organization_id, self.id @@ -109,7 +101,7 @@ class FakeOrganizationalUnit(BaseModel): self.id = utils.make_random_ou_id(organization.root_id) self.name = kwargs.get('Name') self.parent_id = kwargs.get('ParentId') - self._arn_format = OU_ARN_FORMAT + self._arn_format = utils.OU_ARN_FORMAT @property def arn(self): @@ -140,7 +132,7 @@ class FakeRoot(FakeOrganizationalUnit): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - self._arn_format = ROOT_ARN_FORMAT + self._arn_format = utils.ROOT_ARN_FORMAT def describe(self): return { diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index c7e5c71cd..007afa6ed 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -3,6 +3,14 @@ from __future__ import unicode_literals import random import string +MASTER_ACCOUNT_ID = '123456789012' +MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' +MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' +ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' +ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' +OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' + CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py deleted file mode 100644 index 3fb86b9d5..000000000 --- a/tests/test_organizations/object_syntax.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Temporary functions for checking object structures while specing out -models. This module will go away. -""" - -import yaml -from moto import organizations as orgs - - -# utils -print(orgs.utils.make_random_org_id()) -root_id = orgs.utils.make_random_root_id() -print(root_id) -print(orgs.utils.make_random_ou_id(root_id)) -print(orgs.utils.make_random_account_id()) -print(orgs.utils.make_random_create_account_status_id()) - -# models -my_org = orgs.models.FakeOrganization(feature_set='ALL') -print(yaml.dump(my_org._describe())) -#assert False - -my_account = orgs.models.FakeAccount(my_org, AccountName='blee01', Email='blee01@moto-example.org') -print(yaml.dump(my_account)) -#assert False diff --git a/tests/test_organizations/organizations_test_utils.py b/tests/test_organizations/organizations_test_utils.py new file mode 100644 index 000000000..6548b1830 --- /dev/null +++ b/tests/test_organizations/organizations_test_utils.py @@ -0,0 +1,136 @@ +from __future__ import unicode_literals + +import six +import sure # noqa +import datetime +from moto.organizations import utils + +EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" +ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE +ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) +ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE +CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + + +def test_make_random_org_id(): + org_id = utils.make_random_org_id() + org_id.should.match(ORG_ID_REGEX) + + +def test_make_random_root_id(): + root_id = utils.make_random_root_id() + root_id.should.match(ROOT_ID_REGEX) + + +def test_make_random_ou_id(): + root_id = utils.make_random_root_id() + ou_id = utils.make_random_ou_id(root_id) + ou_id.should.match(OU_ID_REGEX) + + +def test_make_random_account_id(): + account_id = utils.make_random_account_id() + account_id.should.match(ACCOUNT_ID_REGEX) + + +def test_make_random_create_account_status_id(): + create_account_status_id = utils.make_random_create_account_status_id() + create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + + +def validate_organization(response): + org = response['Organization'] + sorted(org.keys()).should.equal([ + 'Arn', + 'AvailablePolicyTypes', + 'FeatureSet', + 'Id', + 'MasterAccountArn', + 'MasterAccountEmail', + 'MasterAccountId', + ]) + org['Id'].should.match(ORG_ID_REGEX) + org['MasterAccountId'].should.equal(utils.MASTER_ACCOUNT_ID) + org['MasterAccountArn'].should.equal(utils.MASTER_ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['Arn'].should.equal(utils.ORGANIZATION_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['MasterAccountEmail'].should.equal(utils.MASTER_ACCOUNT_EMAIL) + org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) + org['AvailablePolicyTypes'].should.equal([{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }]) + + +def validate_roots(org, response): + response.should.have.key('Roots').should.be.a(list) + response['Roots'].should_not.be.empty + root = response['Roots'][0] + root.should.have.key('Id').should.match(ROOT_ID_REGEX) + root.should.have.key('Arn').should.equal(utils.ROOT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + root['Id'], + )) + root.should.have.key('Name').should.be.a(six.string_types) + root.should.have.key('PolicyTypes').should.be.a(list) + root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') + root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') + + +def validate_organizational_unit(org, response): + response.should.have.key('OrganizationalUnit').should.be.a(dict) + ou = response['OrganizationalUnit'] + ou.should.have.key('Id').should.match(OU_ID_REGEX) + ou.should.have.key('Arn').should.equal(utils.OU_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + ou['Id'], + )) + ou.should.have.key('Name').should.be.a(six.string_types) + + +def validate_account(org, account): + sorted(account.keys()).should.equal([ + 'Arn', + 'Email', + 'Id', + 'JoinedMethod', + 'JoinedTimestamp', + 'Name', + 'Status', + ]) + account['Id'].should.match(ACCOUNT_ID_REGEX) + account['Arn'].should.equal(utils.ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + account['Id'], + )) + account['Email'].should.match(EMAIL_REGEX) + account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) + account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) + account['Name'].should.be.a(six.string_types) + account['JoinedTimestamp'].should.be.a(datetime.datetime) + + +def validate_create_account_status(create_status): + sorted(create_status.keys()).should.equal([ + 'AccountId', + 'AccountName', + 'CompletedTimestamp', + 'Id', + 'RequestedTimestamp', + 'State', + ]) + create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) + create_status['AccountName'].should.be.a(six.string_types) + create_status['State'].should.equal('SUCCEEDED') + create_status['RequestedTimestamp'].should.be.a(datetime.datetime) + create_status['CompletedTimestamp'].should.be.a(datetime.datetime) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 0ac2b61bf..c2f06774a 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -2,120 +2,24 @@ from __future__ import unicode_literals import boto3 import sure # noqa -import datetime import yaml -import six from moto import mock_organizations -from moto.organizations.models import ( - MASTER_ACCOUNT_ID, - MASTER_ACCOUNT_EMAIL, - ORGANIZATION_ARN_FORMAT, - MASTER_ACCOUNT_ARN_FORMAT, - ACCOUNT_ARN_FORMAT, - ROOT_ARN_FORMAT, - OU_ARN_FORMAT, +from .organizations_test_utils import ( + validate_organization, + validate_roots, + validate_organizational_unit, + validate_account, + validate_create_account_status, ) -from .test_organizations_utils import ( - ORG_ID_REGEX, - ROOT_ID_REGEX, - OU_ID_REGEX, - ACCOUNT_ID_REGEX, - CREATE_ACCOUNT_STATUS_ID_REGEX, -) - -EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" - - -def validate_organization(response): - org = response['Organization'] - sorted(org.keys()).should.equal([ - 'Arn', - 'AvailablePolicyTypes', - 'FeatureSet', - 'Id', - 'MasterAccountArn', - 'MasterAccountEmail', - 'MasterAccountId', - ]) - org['Id'].should.match(ORG_ID_REGEX) - org['MasterAccountId'].should.equal(MASTER_ACCOUNT_ID) - org['MasterAccountArn'].should.equal(MASTER_ACCOUNT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - )) - org['Arn'].should.equal(ORGANIZATION_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - )) - org['MasterAccountEmail'].should.equal(MASTER_ACCOUNT_EMAIL) - org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) - org['AvailablePolicyTypes'].should.equal([{ - 'Type': 'SERVICE_CONTROL_POLICY', - 'Status': 'ENABLED' - }]) - - -def validate_organizational_unit(org, response): - response.should.have.key('OrganizationalUnit').should.be.a(dict) - ou = response['OrganizationalUnit'] - ou.should.have.key('Id').should.match(OU_ID_REGEX) - ou.should.have.key('Arn').should.equal(OU_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - ou['Id'], - )) - ou.should.have.key('Name').should.be.a(six.string_types) - - -def validate_account(org, account): - sorted(account.keys()).should.equal([ - 'Arn', - 'Email', - 'Id', - 'JoinedMethod', - 'JoinedTimestamp', - 'Name', - 'Status', - ]) - account['Id'].should.match(ACCOUNT_ID_REGEX) - account['Arn'].should.equal(ACCOUNT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - account['Id'], - )) - account['Email'].should.match(EMAIL_REGEX) - account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) - account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) - account['Name'].should.be.a(six.string_types) - account['JoinedTimestamp'].should.be.a(datetime.datetime) - - -def validate_create_account_status(create_status): - sorted(create_status.keys()).should.equal([ - 'AccountId', - 'AccountName', - 'CompletedTimestamp', - 'Id', - 'RequestedTimestamp', - 'State', - ]) - create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) - create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) - create_status['AccountName'].should.be.a(six.string_types) - create_status['State'].should.equal('SUCCEEDED') - create_status['RequestedTimestamp'].should.be.a(datetime.datetime) - create_status['CompletedTimestamp'].should.be.a(datetime.datetime) @mock_organizations def test_create_organization(): client = boto3.client('organizations', region_name='us-east-1') response = client.create_organization(FeatureSet='ALL') - #print(yaml.dump(response)) validate_organization(response) response['Organization']['FeatureSet'].should.equal('ALL') - #assert False @mock_organizations @@ -123,9 +27,7 @@ def test_describe_organization(): client = boto3.client('organizations', region_name='us-east-1') client.create_organization(FeatureSet='ALL') response = client.describe_organization() - #print(yaml.dump(response)) validate_organization(response) - #assert False # Organizational Units @@ -135,21 +37,7 @@ def test_list_roots(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] response = client.list_roots() - #print(yaml.dump(response, default_flow_style=False)) - response.should.have.key('Roots').should.be.a(list) - response['Roots'].should_not.be.empty - root = response['Roots'][0] - root.should.have.key('Id').should.match(ROOT_ID_REGEX) - root.should.have.key('Arn').should.equal(ROOT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - root['Id'], - )) - root.should.have.key('Name').should.be.a(six.string_types) - root.should.have.key('PolicyTypes').should.be.a(list) - root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') - root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') - #assert False + validate_roots(org, response) @mock_organizations @@ -162,10 +50,8 @@ def test_create_organizational_unit(): ParentId=root_id, Name=ou_name, ) - #print(yaml.dump(response, default_flow_style=False)) validate_organizational_unit(org, response) response['OrganizationalUnit']['Name'].should.equal(ou_name) - #assert False @mock_organizations @@ -178,9 +64,7 @@ def test_describe_organizational_unit(): Name='ou01', )['OrganizationalUnit']['Id'] response = client.describe_organizational_unit(OrganizationalUnitId=ou_id) - print(yaml.dump(response, default_flow_style=False)) validate_organizational_unit(org, response) - #assert False @mock_organizations @@ -192,11 +76,9 @@ def test_list_organizational_units_for_parent(): client.create_organizational_unit(ParentId=root_id, Name='ou02') client.create_organizational_unit(ParentId=root_id, Name='ou03') response = client.list_organizational_units_for_parent(ParentId=root_id) - print(yaml.dump(response, default_flow_style=False)) response.should.have.key('OrganizationalUnits').should.be.a(list) for ou in response['OrganizationalUnits']: validate_organizational_unit(org, dict(OrganizationalUnit=ou)) - #assert False # Accounts @@ -212,10 +94,8 @@ def test_create_account(): create_status = client.create_account( AccountName=mockname, Email=mockemail )['CreateAccountStatus'] - #print(yaml.dump(create_status, default_flow_style=False)) validate_create_account_status(create_status) create_status['AccountName'].should.equal(mockname) - #assert False @mock_organizations @@ -226,11 +106,9 @@ def test_describe_account(): AccountName=mockname, Email=mockemail )['CreateAccountStatus']['AccountId'] response = client.describe_account(AccountId=account_id) - #print(yaml.dump(response, default_flow_style=False)) validate_account(org, response['Account']) response['Account']['Name'].should.equal(mockname) response['Account']['Email'].should.equal(mockemail) - #assert False @mock_organizations @@ -242,7 +120,6 @@ def test_list_accounts(): email = name + '@' + mockdomain client.create_account(AccountName=name, Email=email) response = client.list_accounts() - #print(yaml.dump(response, default_flow_style=False)) response.should.have.key('Accounts') accounts = response['Accounts'] len(accounts).should.equal(5) @@ -250,7 +127,6 @@ def test_list_accounts(): validate_account(org, account) accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) - #assert False @mock_organizations @@ -263,9 +139,7 @@ def test_list_accounts_for_parent(): Email=mockemail, )['CreateAccountStatus']['AccountId'] response = client.list_accounts_for_parent(ParentId=root_id) - #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) - #assert False @mock_organizations @@ -284,9 +158,7 @@ def test_move_account(): DestinationParentId=ou01_id, ) response = client.list_accounts_for_parent(ParentId=ou01_id) - #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) - #assert False @mock_organizations @@ -297,18 +169,15 @@ def test_list_parents_for_ou(): ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') ou01_id = ou01['OrganizationalUnit']['Id'] response01 = client.list_parents(ChildId=ou01_id) - #print(yaml.dump(response01, default_flow_style=False)) response01.should.have.key('Parents').should.be.a(list) response01['Parents'][0].should.have.key('Id').should.equal(root_id) response01['Parents'][0].should.have.key('Type').should.equal('ROOT') ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') ou02_id = ou02['OrganizationalUnit']['Id'] response02 = client.list_parents(ChildId=ou02_id) - #print(yaml.dump(response02, default_flow_style=False)) response02.should.have.key('Parents').should.be.a(list) response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False @mock_organizations @@ -332,16 +201,13 @@ def test_list_parents_for_accounts(): DestinationParentId=ou01_id, ) response01 = client.list_parents(ChildId=account01_id) - #print(yaml.dump(response01, default_flow_style=False)) response01.should.have.key('Parents').should.be.a(list) response01['Parents'][0].should.have.key('Id').should.equal(root_id) response01['Parents'][0].should.have.key('Type').should.equal('ROOT') response02 = client.list_parents(ChildId=account02_id) - #print(yaml.dump(response02, default_flow_style=False)) response02.should.have.key('Parents').should.be.a(list) response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False @mock_organizations @@ -370,10 +236,6 @@ def test_list_chidlren(): response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') - #print(yaml.dump(response01, default_flow_style=False)) - #print(yaml.dump(response02, default_flow_style=False)) - #print(yaml.dump(response03, default_flow_style=False)) - #print(yaml.dump(response04, default_flow_style=False)) response01['Children'][0]['Id'].should.equal(account01_id) response01['Children'][0]['Type'].should.equal('ACCOUNT') response02['Children'][0]['Id'].should.equal(ou01_id) @@ -382,4 +244,3 @@ def test_list_chidlren(): response03['Children'][0]['Type'].should.equal('ACCOUNT') response04['Children'][0]['Id'].should.equal(ou02_id) response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') - #assert False diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py deleted file mode 100644 index d27201446..000000000 --- a/tests/test_organizations/test_organizations_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import unicode_literals - -import sure # noqa -from moto.organizations import utils - -ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE -ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE -OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) -ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE -CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE - - -def test_make_random_org_id(): - org_id = utils.make_random_org_id() - org_id.should.match(ORG_ID_REGEX) - - -def test_make_random_root_id(): - root_id = utils.make_random_root_id() - root_id.should.match(ROOT_ID_REGEX) - - -def test_make_random_ou_id(): - root_id = utils.make_random_root_id() - ou_id = utils.make_random_ou_id(root_id) - ou_id.should.match(OU_ID_REGEX) - - -def test_make_random_account_id(): - account_id = utils.make_random_account_id() - account_id.should.match(ACCOUNT_ID_REGEX) - - -def test_make_random_create_account_status_id(): - create_account_status_id = utils.make_random_create_account_status_id() - create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From 05928b1497831e38a642baadc438b8238d65b4bf Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Fri, 20 Jul 2018 13:54:53 -0700 Subject: [PATCH 25/72] [issue #1720] Add support for AWS Organizations added exception handling in class OrganizationsBackend --- IMPLEMENTATION_COVERAGE.md | 4 +- README.md | 2 +- moto/organizations/models.py | 76 ++++++++++++------- .../test_organizations_boto3.py | 69 ++++++++++++++++- 4 files changed, 120 insertions(+), 31 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index e1943aa9a..d19d2473e 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 28% implemented +## organizations - 30% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3176,7 +3176,7 @@ - [X] list_accounts - [X] list_accounts_for_parent - [ ] list_aws_service_access_for_organization -- [ ] list_children +- [X] list_children - [ ] list_create_account_status - [ ] list_handshakes_for_account - [ ] list_handshakes_for_organization diff --git a/README.md b/README.md index e4fcb6508..189bf2c4f 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | KMS | @mock_kms | basic endpoints done | |------------------------------------------------------------------------------| -| Organizations | @mock_organizations | some endpoints done | +| Organizations | @mock_organizations | some core endpoints done | |------------------------------------------------------------------------------| | Polly | @mock_polly | all endpoints done | |------------------------------------------------------------------------------| diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 0f3c67400..91896f537 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -4,6 +4,7 @@ import datetime import re from moto.core import BaseBackend, BaseModel +from moto.core.exceptions import RESTError from moto.core.utils import unix_time from moto.organizations import utils @@ -168,13 +169,31 @@ class OrganizationsBackend(BaseBackend): self.ou.append(new_ou) return new_ou.describe() + def get_organizational_unit_by_id(self, ou_id): + ou = next((ou for ou in self.ou if ou.id == ou_id), None) + if ou is None: + raise RESTError( + 'OrganizationalUnitNotFoundException', + "You specified an organizational unit that doesn't exist." + ) + return ou + + def validate_parent_id(self, parent_id): + try: + self.get_organizational_unit_by_id(parent_id) + except RESTError as e: + raise RESTError( + 'ParentNotFoundException', + "You specified parent that doesn't exist." + ) + return parent_id + def describe_organizational_unit(self, **kwargs): - ou = [ - ou for ou in self.ou if ou.id == kwargs['OrganizationalUnitId'] - ].pop(0) + ou = self.get_organizational_unit_by_id(kwargs['OrganizationalUnitId']) return ou.describe() def list_organizational_units_for_parent(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) return dict( OrganizationalUnits=[ { @@ -183,7 +202,7 @@ class OrganizationsBackend(BaseBackend): 'Name': ou.name, } for ou in self.ou - if ou.parent_id == kwargs['ParentId'] + if ou.parent_id == parent_id ] ) @@ -192,11 +211,20 @@ class OrganizationsBackend(BaseBackend): self.accounts.append(new_account) return new_account.create_account_status - def describe_account(self, **kwargs): - account = [ + def get_account_by_id(self, account_id): + account = next(( account for account in self.accounts - if account.id == kwargs['AccountId'] - ].pop(0) + if account.id == account_id + ), None) + if account is None: + raise RESTError( + 'AccountNotFoundException', + "You specified an account that doesn't exist." + ) + return account + + def describe_account(self, **kwargs): + account = self.get_account_by_id(kwargs['AccountId']) return account.describe() def list_accounts(self): @@ -205,35 +233,27 @@ class OrganizationsBackend(BaseBackend): ) def list_accounts_for_parent(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) return dict( Accounts=[ account.describe()['Account'] for account in self.accounts - if account.parent_id == kwargs['ParentId'] + if account.parent_id == parent_id ] ) def move_account(self, **kwargs): - new_parent_id = kwargs['DestinationParentId'] - all_parent_id = [parent.id for parent in self.ou] - account = [ - account for account in self.accounts - if account.id == kwargs['AccountId'] - ].pop(0) - assert new_parent_id in all_parent_id - assert account.parent_id == kwargs['SourceParentId'] + new_parent_id = self.validate_parent_id(kwargs['DestinationParentId']) + self.validate_parent_id(kwargs['SourceParentId']) + account = self.get_account_by_id(kwargs['AccountId']) index = self.accounts.index(account) self.accounts[index].parent_id = new_parent_id def list_parents(self, **kwargs): if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): - obj_list = self.accounts + child_object = self.get_account_by_id(kwargs['ChildId']) else: - obj_list = self.ou - parent_id = [ - obj.parent_id for obj in obj_list - if obj.id == kwargs['ChildId'] - ].pop(0) + child_object = self.get_organizational_unit_by_id(kwargs['ChildId']) return dict( Parents=[ { @@ -241,17 +261,21 @@ class OrganizationsBackend(BaseBackend): 'Type': ou.type, } for ou in self.ou - if ou.id == parent_id + if ou.id == child_object.parent_id ] ) def list_children(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) if kwargs['ChildType'] == 'ACCOUNT': obj_list = self.accounts elif kwargs['ChildType'] == 'ORGANIZATIONAL_UNIT': obj_list = self.ou else: - raise ValueError + raise RESTError( + 'InvalidInputException', + 'You specified an invalid value.' + ) return dict( Children=[ { @@ -259,7 +283,7 @@ class OrganizationsBackend(BaseBackend): 'Type': kwargs['ChildType'], } for obj in obj_list - if obj.parent_id == kwargs['ParentId'] + if obj.parent_id == parent_id ] ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index c2f06774a..ae9bacd82 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -2,9 +2,11 @@ from __future__ import unicode_literals import boto3 import sure # noqa -import yaml +from botocore.exceptions import ClientError +from nose.tools import assert_raises from moto import mock_organizations +from moto.organizations import utils from .organizations_test_utils import ( validate_organization, validate_roots, @@ -67,6 +69,20 @@ def test_describe_organizational_unit(): validate_organizational_unit(org, response) +@mock_organizations +def test_describe_organizational_unit_exception(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + with assert_raises(ClientError) as e: + response = client.describe_organizational_unit( + OrganizationalUnitId=utils.make_random_root_id() + ) + ex = e.exception + ex.operation_name.should.equal('DescribeOrganizationalUnit') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('OrganizationalUnitNotFoundException') + + @mock_organizations def test_list_organizational_units_for_parent(): client = boto3.client('organizations', region_name='us-east-1') @@ -81,6 +97,19 @@ def test_list_organizational_units_for_parent(): validate_organizational_unit(org, dict(OrganizationalUnit=ou)) +@mock_organizations +def test_list_organizational_units_for_parent_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.list_organizational_units_for_parent( + ParentId=utils.make_random_root_id() + ) + ex = e.exception + ex.operation_name.should.equal('ListOrganizationalUnitsForParent') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('ParentNotFoundException') + + # Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' @@ -111,6 +140,17 @@ def test_describe_account(): response['Account']['Email'].should.equal(mockemail) +@mock_organizations +def test_describe_account_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.describe_account(AccountId=utils.make_random_account_id()) + ex = e.exception + ex.operation_name.should.equal('DescribeAccount') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('AccountNotFoundException') + + @mock_organizations def test_list_accounts(): client = boto3.client('organizations', region_name='us-east-1') @@ -211,7 +251,7 @@ def test_list_parents_for_accounts(): @mock_organizations -def test_list_chidlren(): +def test_list_children(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] root_id = client.list_roots()['Roots'][0]['Id'] @@ -244,3 +284,28 @@ def test_list_chidlren(): response03['Children'][0]['Type'].should.equal('ACCOUNT') response04['Children'][0]['Id'].should.equal(ou02_id) response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + + +@mock_organizations +def test_list_children_exception(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + with assert_raises(ClientError) as e: + response = client.list_children( + ParentId=utils.make_random_root_id(), + ChildType='ACCOUNT' + ) + ex = e.exception + ex.operation_name.should.equal('ListChildren') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('ParentNotFoundException') + with assert_raises(ClientError) as e: + response = client.list_children( + ParentId=root_id, + ChildType='BLEE' + ) + ex = e.exception + ex.operation_name.should.equal('ListChildren') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('InvalidInputException') From 4356e951e10795fcb9b73ac6b67540fbfeda2001 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Fri, 20 Jul 2018 14:07:04 -0700 Subject: [PATCH 26/72] [issue #1720] Add support for AWS Organizations fix travis build error --- moto/organizations/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 91896f537..c02b1a492 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -181,7 +181,7 @@ class OrganizationsBackend(BaseBackend): def validate_parent_id(self, parent_id): try: self.get_organizational_unit_by_id(parent_id) - except RESTError as e: + except RESTError: raise RESTError( 'ParentNotFoundException', "You specified parent that doesn't exist." From b8be517be08c5d7b42315ac1af7da23d978d5f9f Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Thu, 26 Jul 2018 12:20:43 -0700 Subject: [PATCH 27/72] organizations support: add exception handling for describe_organizations --- moto/organizations/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index c02b1a492..9d5fe3886 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -157,6 +157,11 @@ class OrganizationsBackend(BaseBackend): return self.org.describe() def describe_organization(self): + if not self.org: + raise RESTError( + 'AWSOrganizationsNotInUseException', + "Your account is not a member of an organization." + ) return self.org.describe() def list_roots(self): From 3afb2862c0cdb8e9d229b7c7e4141e1abfa57d00 Mon Sep 17 00:00:00 2001 From: William Richard Date: Mon, 1 Oct 2018 16:30:23 -0400 Subject: [PATCH 28/72] Filter event log ids should be strings Based on the boto docs, eventId should be returned as a string. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs.html#CloudWatchLogs.Client.filter_log_events --- moto/logs/models.py | 2 +- tests/test_logs/test_logs.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index a4ff9db46..ca1fdc4ad 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -19,7 +19,7 @@ class LogEvent: def to_filter_dict(self): return { - "eventId": self.eventId, + "eventId": str(self.eventId), "ingestionTime": self.ingestionTime, # "logStreamName": "message": self.message, diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 05bd3c823..e3d46fd87 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -121,4 +121,8 @@ def test_filter_logs_interleaved(): interleaved=True, ) events = res['events'] - events.should.have.length_of(2) + for original_message, resulting_event in zip(messages, events): + resulting_event['eventId'].should.equal(str(resulting_event['eventId'])) + resulting_event['timestamp'].should.equal(original_message['timestamp']) + resulting_event['message'].should.equal(original_message['message']) + From ea4fcaa82a969ffdc2482fbd18c41cf117e22cc9 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 00:40:28 -0500 Subject: [PATCH 29/72] add support for NoncurrentVersionTransition, NoncurrentVersionExpiration, and AbortIncompleteMultipartUpload actions to s3 lifecycle rules --- moto/s3/models.py | 24 ++++-- moto/s3/responses.py | 16 ++++ tests/test_s3/test_s3_lifecycle.py | 129 +++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 6 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index f3994b5d8..4b1a343d1 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -341,8 +341,9 @@ class LifecycleAndFilter(BaseModel): class LifecycleRule(BaseModel): def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None, - expiration_date=None, transition_days=None, expired_object_delete_marker=None, - transition_date=None, storage_class=None): + expiration_date=None, transition_days=None, transition_date=None, storage_class=None, + expired_object_delete_marker=None, nve_noncurrent_days=None, nvt_noncurrent_days=None, + nvt_storage_class=None, aimu_days=None): self.id = id self.prefix = prefix self.filter = lc_filter @@ -351,8 +352,12 @@ class LifecycleRule(BaseModel): self.expiration_date = expiration_date self.transition_days = transition_days self.transition_date = transition_date - self.expired_object_delete_marker = expired_object_delete_marker self.storage_class = storage_class + self.expired_object_delete_marker = expired_object_delete_marker + self.nve_noncurrent_days = nve_noncurrent_days + self.nvt_noncurrent_days = nvt_noncurrent_days + self.nvt_storage_class = nvt_storage_class + self.aimu_days = aimu_days class CorsRule(BaseModel): @@ -414,8 +419,12 @@ class FakeBucket(BaseModel): def set_lifecycle(self, rules): self.rules = [] for rule in rules: + # Extract actions from Lifecycle rule expiration = rule.get('Expiration') transition = rule.get('Transition') + nve = rule.get('NoncurrentVersionExpiration') + nvt = rule.get('NoncurrentVersionTransition') + aimu = rule.get('AbortIncompleteMultipartUpload') eodm = None if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: @@ -459,11 +468,14 @@ class FakeBucket(BaseModel): status=rule['Status'], expiration_days=expiration.get('Days') if expiration else None, expiration_date=expiration.get('Date') if expiration else None, - expired_object_delete_marker=eodm, transition_days=transition.get('Days') if transition else None, transition_date=transition.get('Date') if transition else None, - storage_class=transition[ - 'StorageClass'] if transition else None, + storage_class=transition.get('StorageClass') if transition else None, + expired_object_delete_marker=eodm, + nve_noncurrent_days = nve.get('NoncurrentDays') if nve else None, + nvt_noncurrent_days = nvt.get('NoncurrentDays') if nvt else None, + nvt_storage_class = nvt.get('StorageClass') if nvt else None, + aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, )) def delete_lifecycle(self): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f8dc7e42b..de101a193 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1228,6 +1228,22 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% endif %} {% endif %} + {% if rule.nvt_noncurrent_days and rule.nvt_storage_class %} + + {{ rule.nvt_noncurrent_days }} + {{ rule.nvt_storage_class }} + + {% endif %} + {% if rule.nve_noncurrent_days %} + + {{ rule.nve_noncurrent_days }} + + {% endif %} + {% if rule.aimu_days %} + + {{ rule.aimu_days }} + + {% endif %} {% endfor %} diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index d176e95c6..9c4bd49bf 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -191,6 +191,135 @@ def test_lifecycle_with_eodm(): assert err.exception.response["Error"]["Code"] == "MalformedXML" +@mock_s3 +def test_lifecycle_with_nve(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "NoncurrentVersionExpiration": { + "NoncurrentDays": 30 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 30 + + # Change NoncurrentDays: + lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] = 10 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10 + + # With failures for missing children: + del lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + +@mock_s3 +def test_lifecycle_with_nvt(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "NoncurrentVersionTransition": { + "NoncurrentDays": 30, + "StorageClass": "ONEZONE_IA" + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 30 + assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "ONEZONE_IA" + + # Change NoncurrentDays: + lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 10 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 10 + + # Change StorageClass: + lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] = "GLACIER" + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "GLACIER" + + # With failures for missing children: + del lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 30 + + del lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + +@mock_s3 +def test_lifecycle_with_aimu(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 7 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 7 + + # Change DaysAfterInitiation: + lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] = 30 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30 + + # With failures for missing children: + del lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + @mock_s3_deprecated def test_lifecycle_with_glacier_transition(): conn = boto.s3.connect_to_region("us-west-1") From 691a8722a898edaa824d941a79a11a7e6f6627dd Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 00:45:47 -0500 Subject: [PATCH 30/72] formatting fix for flake8 due to extra spaces --- moto/s3/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 4b1a343d1..d889b8cab 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -472,9 +472,9 @@ class FakeBucket(BaseModel): transition_date=transition.get('Date') if transition else None, storage_class=transition.get('StorageClass') if transition else None, expired_object_delete_marker=eodm, - nve_noncurrent_days = nve.get('NoncurrentDays') if nve else None, - nvt_noncurrent_days = nvt.get('NoncurrentDays') if nvt else None, - nvt_storage_class = nvt.get('StorageClass') if nvt else None, + nve_noncurrent_days=nve.get('NoncurrentDays') if nve else None, + nvt_noncurrent_days=nvt.get('NoncurrentDays') if nvt else None, + nvt_storage_class=nvt.get('StorageClass') if nvt else None, aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, )) From 9b5f983cb5b4f02a9d7d0842f4f73e01bcab671b Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 01:11:11 -0500 Subject: [PATCH 31/72] add action validation to set_lifecycle() --- moto/s3/models.py | 35 +++++++++++++++++++++++------- tests/test_s3/test_s3_lifecycle.py | 22 +++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index d889b8cab..fc977bb99 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -419,12 +419,31 @@ class FakeBucket(BaseModel): def set_lifecycle(self, rules): self.rules = [] for rule in rules: - # Extract actions from Lifecycle rule + # Extract and validate actions from Lifecycle rule expiration = rule.get('Expiration') transition = rule.get('Transition') - nve = rule.get('NoncurrentVersionExpiration') - nvt = rule.get('NoncurrentVersionTransition') - aimu = rule.get('AbortIncompleteMultipartUpload') + + nve_noncurrent_days = None + if rule.get('NoncurrentVersionExpiration'): + if not rule["NoncurrentVersionExpiration"].get('NoncurrentDays'): + raise MalformedXML() + nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] + + nvt_noncurrent_days = None + nvt_storage_class = None + if rule.get('NoncurrentVersionTransition'): + if not rule["NoncurrentVersionTransition"].get('NoncurrentDays'): + raise MalformedXML() + if not rule["NoncurrentVersionTransition"].get('StorageClass'): + raise MalformedXML() + nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"] + nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] + + aimu_days = None + if rule.get('AbortIncompleteMultipartUpload'): + if not rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation'): + raise MalformedXML() + aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] eodm = None if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: @@ -472,10 +491,10 @@ class FakeBucket(BaseModel): transition_date=transition.get('Date') if transition else None, storage_class=transition.get('StorageClass') if transition else None, expired_object_delete_marker=eodm, - nve_noncurrent_days=nve.get('NoncurrentDays') if nve else None, - nvt_noncurrent_days=nvt.get('NoncurrentDays') if nvt else None, - nvt_storage_class=nvt.get('StorageClass') if nvt else None, - aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, + nve_noncurrent_days=nve_noncurrent_days, + nvt_noncurrent_days=nvt_noncurrent_days, + nvt_storage_class=nvt_storage_class, + aimu_days=aimu_days, )) def delete_lifecycle(self): diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index 9c4bd49bf..ff89dc113 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -237,10 +237,10 @@ def test_lifecycle_with_nvt(): lfc = { "Rules": [ { - "NoncurrentVersionTransition": { + "NoncurrentVersionTransitions": [{ "NoncurrentDays": 30, "StorageClass": "ONEZONE_IA" - }, + }], "ID": "wholebucket", "Filter": { "Prefix": "" @@ -252,31 +252,31 @@ def test_lifecycle_with_nvt(): client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 30 - assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "ONEZONE_IA" + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 30 + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "ONEZONE_IA" # Change NoncurrentDays: - lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 10 + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 10 client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 10 + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 10 # Change StorageClass: - lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] = "GLACIER" + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] = "GLACIER" client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "GLACIER" + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "GLACIER" # With failures for missing children: - del lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] + del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] with assert_raises(ClientError) as err: client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) assert err.exception.response["Error"]["Code"] == "MalformedXML" - lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 30 + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 30 - del lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] + del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] with assert_raises(ClientError) as err: client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) assert err.exception.response["Error"]["Code"] == "MalformedXML" From a1a8ac7286ab30c2888b879bc16ecd2bd91dd1d7 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 01:26:09 -0500 Subject: [PATCH 32/72] check for None in lifecycle actions --- moto/s3/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index fc977bb99..39f36982d 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -425,23 +425,23 @@ class FakeBucket(BaseModel): nve_noncurrent_days = None if rule.get('NoncurrentVersionExpiration'): - if not rule["NoncurrentVersionExpiration"].get('NoncurrentDays'): + if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None: raise MalformedXML() nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] nvt_noncurrent_days = None nvt_storage_class = None if rule.get('NoncurrentVersionTransition'): - if not rule["NoncurrentVersionTransition"].get('NoncurrentDays'): + if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None: raise MalformedXML() - if not rule["NoncurrentVersionTransition"].get('StorageClass'): + if rule["NoncurrentVersionTransition"].get('StorageClass') is None: raise MalformedXML() nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"] nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] aimu_days = None if rule.get('AbortIncompleteMultipartUpload'): - if not rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation'): + if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None: raise MalformedXML() aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] From 5783d662064e248084675c9358bc7e50c4666c51 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Tue, 2 Oct 2018 17:25:14 +0100 Subject: [PATCH 33/72] Mock more of the Glue Data Catalog APIs This adds some of the missing Get/Update/Create APIs relating to the Glue data catalog -- but not yet all of them, and none of the Batch* API calls. --- moto/glue/exceptions.py | 59 +++- moto/glue/models.py | 104 ++++++- moto/glue/responses.py | 103 +++++-- tests/test_glue/fixtures/datacatalog.py | 25 ++ tests/test_glue/helpers.py | 81 +++++- tests/test_glue/test_datacatalog.py | 362 ++++++++++++++++++++++-- 6 files changed, 673 insertions(+), 61 deletions(-) diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py index 62ea1525c..8972adb35 100644 --- a/moto/glue/exceptions.py +++ b/moto/glue/exceptions.py @@ -6,19 +6,56 @@ class GlueClientError(JsonRESTError): code = 400 -class DatabaseAlreadyExistsException(GlueClientError): - def __init__(self): - self.code = 400 - super(DatabaseAlreadyExistsException, self).__init__( - 'DatabaseAlreadyExistsException', - 'Database already exists.' +class AlreadyExistsException(GlueClientError): + def __init__(self, typ): + super(GlueClientError, self).__init__( + 'AlreadyExistsException', + '%s already exists.' % (typ), ) -class TableAlreadyExistsException(GlueClientError): +class DatabaseAlreadyExistsException(AlreadyExistsException): def __init__(self): - self.code = 400 - super(TableAlreadyExistsException, self).__init__( - 'TableAlreadyExistsException', - 'Table already exists.' + super(DatabaseAlreadyExistsException, self).__init__('Database') + + +class TableAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super(TableAlreadyExistsException, self).__init__('Table') + + +class PartitionAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super(PartitionAlreadyExistsException, self).__init__('Partition') + + +class EntityNotFoundException(GlueClientError): + def __init__(self, msg): + super(GlueClientError, self).__init__( + 'EntityNotFoundException', + msg, ) + + +class DatabaseNotFoundException(EntityNotFoundException): + def __init__(self, db): + super(DatabaseNotFoundException, self).__init__( + 'Database %s not found.' % db, + ) + + +class TableNotFoundException(EntityNotFoundException): + def __init__(self, tbl): + super(TableNotFoundException, self).__init__( + 'Table %s not found.' % tbl, + ) + + +class PartitionNotFoundException(EntityNotFoundException): + def __init__(self): + super(PartitionNotFoundException, self).__init__("Cannot find partition.") + + +class VersionNotFoundException(EntityNotFoundException): + def __init__(self): + super(VersionNotFoundException, self).__init__("Version not found.") diff --git a/moto/glue/models.py b/moto/glue/models.py index 09b7d60ed..bcf2ec4bf 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -1,8 +1,19 @@ from __future__ import unicode_literals +import time + from moto.core import BaseBackend, BaseModel from moto.compat import OrderedDict -from.exceptions import DatabaseAlreadyExistsException, TableAlreadyExistsException +from.exceptions import ( + JsonRESTError, + DatabaseAlreadyExistsException, + DatabaseNotFoundException, + TableAlreadyExistsException, + TableNotFoundException, + PartitionAlreadyExistsException, + PartitionNotFoundException, + VersionNotFoundException, +) class GlueBackend(BaseBackend): @@ -19,7 +30,10 @@ class GlueBackend(BaseBackend): return database def get_database(self, database_name): - return self.databases[database_name] + try: + return self.databases[database_name] + except KeyError: + raise DatabaseNotFoundException(database_name) def create_table(self, database_name, table_name, table_input): database = self.get_database(database_name) @@ -33,7 +47,10 @@ class GlueBackend(BaseBackend): def get_table(self, database_name, table_name): database = self.get_database(database_name) - return database.tables[table_name] + try: + return database.tables[table_name] + except KeyError: + raise TableNotFoundException(table_name) def get_tables(self, database_name): database = self.get_database(database_name) @@ -52,9 +69,84 @@ class FakeTable(BaseModel): def __init__(self, database_name, table_name, table_input): self.database_name = database_name self.name = table_name - self.table_input = table_input - self.storage_descriptor = self.table_input.get('StorageDescriptor', {}) - self.partition_keys = self.table_input.get('PartitionKeys', []) + self.partitions = OrderedDict() + self.versions = [] + self.update(table_input) + + def update(self, table_input): + self.versions.append(table_input) + + def get_version(self, ver): + try: + if not isinstance(ver, int): + # "1" goes to [0] + ver = int(ver) - 1 + except ValueError as e: + raise JsonRESTError("InvalidInputException", str(e)) + + try: + return self.versions[ver] + except IndexError: + raise VersionNotFoundException() + + def as_dict(self, version=-1): + obj = { + 'DatabaseName': self.database_name, + 'Name': self.name, + } + obj.update(self.get_version(version)) + return obj + + def create_partition(self, partiton_input): + partition = FakePartition(self.database_name, self.name, partiton_input) + key = str(partition.values) + if key in self.partitions: + raise PartitionAlreadyExistsException() + self.partitions[str(partition.values)] = partition + + def get_partitions(self): + return [p for str_part_values, p in self.partitions.items()] + + def get_partition(self, values): + try: + return self.partitions[str(values)] + except KeyError: + raise PartitionNotFoundException() + + def update_partition(self, old_values, partiton_input): + partition = FakePartition(self.database_name, self.name, partiton_input) + key = str(partition.values) + if old_values == partiton_input['Values']: + # Altering a partition in place. Don't remove it so the order of + # returned partitions doesn't change + if key not in self.partitions: + raise PartitionNotFoundException() + else: + removed = self.partitions.pop(str(old_values), None) + if removed is None: + raise PartitionNotFoundException() + if key in self.partitions: + # Trying to update to overwrite a partition that exists + raise PartitionAlreadyExistsException() + self.partitions[key] = partition + + +class FakePartition(BaseModel): + def __init__(self, database_name, table_name, partiton_input): + self.creation_time = time.time() + self.database_name = database_name + self.table_name = table_name + self.partition_input = partiton_input + self.values = self.partition_input.get('Values', []) + + def as_dict(self): + obj = { + 'DatabaseName': self.database_name, + 'TableName': self.table_name, + 'CreationTime': self.creation_time, + } + obj.update(self.partition_input) + return obj glue_backend = GlueBackend() diff --git a/moto/glue/responses.py b/moto/glue/responses.py index bb64c40d4..84cc6f901 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -37,27 +37,94 @@ class GlueResponse(BaseResponse): database_name = self.parameters.get('DatabaseName') table_name = self.parameters.get('Name') table = self.glue_backend.get_table(database_name, table_name) + + return json.dumps({'Table': table.as_dict()}) + + def update_table(self): + database_name = self.parameters.get('DatabaseName') + table_input = self.parameters.get('TableInput') + table_name = table_input.get('Name') + table = self.glue_backend.get_table(database_name, table_name) + table.update(table_input) + return "" + + def get_table_versions(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + return json.dumps({ - 'Table': { - 'DatabaseName': table.database_name, - 'Name': table.name, - 'PartitionKeys': table.partition_keys, - 'StorageDescriptor': table.storage_descriptor - } + "TableVersions": [ + { + "Table": table.as_dict(version=n), + "VersionId": str(n + 1), + } for n in range(len(table.versions)) + ], + }) + + def get_table_version(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + ver_id = self.parameters.get('VersionId') + + return json.dumps({ + "TableVersion": { + "Table": table.as_dict(version=ver_id), + "VersionId": ver_id, + }, }) def get_tables(self): database_name = self.parameters.get('DatabaseName') tables = self.glue_backend.get_tables(database_name) - return json.dumps( - { - 'TableList': [ - { - 'DatabaseName': table.database_name, - 'Name': table.name, - 'PartitionKeys': table.partition_keys, - 'StorageDescriptor': table.storage_descriptor - } for table in tables - ] - } - ) + return json.dumps({ + 'TableList': [ + table.as_dict() for table in tables + ] + }) + + def get_partitions(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + if 'Expression' in self.parameters: + raise NotImplementedError("Expression filtering in get_partitions is not implemented in moto") + table = self.glue_backend.get_table(database_name, table_name) + + return json.dumps({ + 'Partitions': [ + p.as_dict() for p in table.get_partitions() + ] + }) + + def get_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + values = self.parameters.get('PartitionValues') + + table = self.glue_backend.get_table(database_name, table_name) + + p = table.get_partition(values) + + return json.dumps({'Partition': p.as_dict()}) + + def create_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_input = self.parameters.get('PartitionInput') + + table = self.glue_backend.get_table(database_name, table_name) + table.create_partition(part_input) + + return "" + + def update_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_input = self.parameters.get('PartitionInput') + part_to_update = self.parameters.get('PartitionValueList') + + table = self.glue_backend.get_table(database_name, table_name) + table.update_partition(part_to_update, part_input) + + return "" diff --git a/tests/test_glue/fixtures/datacatalog.py b/tests/test_glue/fixtures/datacatalog.py index b2efe4154..edad2f0f4 100644 --- a/tests/test_glue/fixtures/datacatalog.py +++ b/tests/test_glue/fixtures/datacatalog.py @@ -29,3 +29,28 @@ TABLE_INPUT = { }, 'TableType': 'EXTERNAL_TABLE', } + + +PARTITION_INPUT = { + # 'DatabaseName': 'dbname', + 'StorageDescriptor': { + 'BucketColumns': [], + 'Columns': [], + 'Compressed': False, + 'InputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', + 'Location': 's3://.../partition=value', + 'NumberOfBuckets': -1, + 'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', + 'Parameters': {}, + 'SerdeInfo': { + 'Parameters': {'path': 's3://...', 'serialization.format': '1'}, + 'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'}, + 'SkewedInfo': {'SkewedColumnNames': [], + 'SkewedColumnValueLocationMaps': {}, + 'SkewedColumnValues': []}, + 'SortColumns': [], + 'StoredAsSubDirectories': False, + }, + # 'TableName': 'source_table', + # 'Values': ['2018-06-26'], +} diff --git a/tests/test_glue/helpers.py b/tests/test_glue/helpers.py index 4a51f9117..331b99867 100644 --- a/tests/test_glue/helpers.py +++ b/tests/test_glue/helpers.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import copy -from .fixtures.datacatalog import TABLE_INPUT +from .fixtures.datacatalog import TABLE_INPUT, PARTITION_INPUT def create_database(client, database_name): @@ -17,22 +17,38 @@ def get_database(client, database_name): return client.get_database(Name=database_name) -def create_table_input(table_name, s3_location, columns=[], partition_keys=[]): +def create_table_input(database_name, table_name, columns=[], partition_keys=[]): table_input = copy.deepcopy(TABLE_INPUT) table_input['Name'] = table_name table_input['PartitionKeys'] = partition_keys table_input['StorageDescriptor']['Columns'] = columns - table_input['StorageDescriptor']['Location'] = s3_location + table_input['StorageDescriptor']['Location'] = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) return table_input -def create_table(client, database_name, table_name, table_input): +def create_table(client, database_name, table_name, table_input=None, **kwargs): + if table_input is None: + table_input = create_table_input(database_name, table_name, **kwargs) + return client.create_table( DatabaseName=database_name, TableInput=table_input ) +def update_table(client, database_name, table_name, table_input=None, **kwargs): + if table_input is None: + table_input = create_table_input(database_name, table_name, **kwargs) + + return client.update_table( + DatabaseName=database_name, + TableInput=table_input, + ) + + def get_table(client, database_name, table_name): return client.get_table( DatabaseName=database_name, @@ -44,3 +60,60 @@ def get_tables(client, database_name): return client.get_tables( DatabaseName=database_name ) + + +def get_table_versions(client, database_name, table_name): + return client.get_table_versions( + DatabaseName=database_name, + TableName=table_name + ) + + +def get_table_version(client, database_name, table_name, version_id): + return client.get_table_version( + DatabaseName=database_name, + TableName=table_name, + VersionId=version_id, + ) + + +def create_partition_input(database_name, table_name, values=[], columns=[]): + root_path = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) + + part_input = copy.deepcopy(PARTITION_INPUT) + part_input['Values'] = values + part_input['StorageDescriptor']['Columns'] = columns + part_input['StorageDescriptor']['SerdeInfo']['Parameters']['path'] = root_path + return part_input + + +def create_partition(client, database_name, table_name, partiton_input=None, **kwargs): + if partiton_input is None: + partiton_input = create_partition_input(database_name, table_name, **kwargs) + return client.create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInput=partiton_input + ) + + +def update_partition(client, database_name, table_name, old_values=[], partiton_input=None, **kwargs): + if partiton_input is None: + partiton_input = create_partition_input(database_name, table_name, **kwargs) + return client.update_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInput=partiton_input, + PartitionValueList=old_values, + ) + + +def get_partition(client, database_name, table_name, values): + return client.get_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 7dabeb1f3..a457d5127 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -1,10 +1,15 @@ from __future__ import unicode_literals import sure # noqa +import re from nose.tools import assert_raises import boto3 from botocore.client import ClientError + +from datetime import datetime +import pytz + from moto import mock_glue from . import helpers @@ -30,7 +35,19 @@ def test_create_database_already_exists(): with assert_raises(ClientError) as exc: helpers.create_database(client, database_name) - exc.exception.response['Error']['Code'].should.equal('DatabaseAlreadyExistsException') + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_get_database_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'nosuchdatabase' + + with assert_raises(ClientError) as exc: + helpers.get_database(client, database_name) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found') @mock_glue @@ -40,12 +57,7 @@ def test_create_table(): helpers.create_database(client, database_name) table_name = 'myspecialtable' - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - - table_input = helpers.create_table_input(table_name, s3_location) + table_input = helpers.create_table_input(database_name, table_name) helpers.create_table(client, database_name, table_name, table_input) response = helpers.get_table(client, database_name, table_name) @@ -63,18 +75,12 @@ def test_create_table_already_exists(): helpers.create_database(client, database_name) table_name = 'cantcreatethistabletwice' - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - - table_input = helpers.create_table_input(table_name, s3_location) - helpers.create_table(client, database_name, table_name, table_input) + helpers.create_table(client, database_name, table_name) with assert_raises(ClientError) as exc: - helpers.create_table(client, database_name, table_name, table_input) + helpers.create_table(client, database_name, table_name) - exc.exception.response['Error']['Code'].should.equal('TableAlreadyExistsException') + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') @mock_glue @@ -87,11 +93,7 @@ def test_get_tables(): table_inputs = {} for table_name in table_names: - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - table_input = helpers.create_table_input(table_name, s3_location) + table_input = helpers.create_table_input(database_name, table_name) table_inputs[table_name] = table_input helpers.create_table(client, database_name, table_name, table_input) @@ -99,10 +101,326 @@ def test_get_tables(): tables = response['TableList'] - assert len(tables) == 3 + tables.should.have.length_of(3) for table in tables: table_name = table['Name'] table_name.should.equal(table_inputs[table_name]['Name']) table['StorageDescriptor'].should.equal(table_inputs[table_name]['StorageDescriptor']) table['PartitionKeys'].should.equal(table_inputs[table_name]['PartitionKeys']) + + +@mock_glue +def test_get_table_versions(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + table_name = 'myfirsttable' + version_inputs = {} + + table_input = helpers.create_table_input(database_name, table_name) + helpers.create_table(client, database_name, table_name, table_input) + version_inputs["1"] = table_input + + columns = [{'Name': 'country', 'Type': 'string'}] + table_input = helpers.create_table_input(database_name, table_name, columns=columns) + helpers.update_table(client, database_name, table_name, table_input) + version_inputs["2"] = table_input + + # Updateing with an indentical input should still create a new version + helpers.update_table(client, database_name, table_name, table_input) + version_inputs["3"] = table_input + + response = helpers.get_table_versions(client, database_name, table_name) + + vers = response['TableVersions'] + + vers.should.have.length_of(3) + vers[0]['Table']['StorageDescriptor']['Columns'].should.equal([]) + vers[-1]['Table']['StorageDescriptor']['Columns'].should.equal(columns) + + for n, ver in enumerate(vers): + n = str(n + 1) + ver['VersionId'].should.equal(n) + ver['Table']['Name'].should.equal(table_name) + ver['Table']['StorageDescriptor'].should.equal(version_inputs[n]['StorageDescriptor']) + ver['Table']['PartitionKeys'].should.equal(version_inputs[n]['PartitionKeys']) + + response = helpers.get_table_version(client, database_name, table_name, "3") + ver = response['TableVersion'] + + ver['VersionId'].should.equal("3") + ver['Table']['Name'].should.equal(table_name) + ver['Table']['StorageDescriptor']['Columns'].should.equal(columns) + + +@mock_glue +def test_get_table_version_not_found(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_table_version(client, database_name, 'myfirsttable', "20") + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('version', re.I) + + +@mock_glue +def test_get_table_version_invalid_input(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_table_version(client, database_name, 'myfirsttable', "10not-an-int") + + exc.exception.response['Error']['Code'].should.equal('InvalidInputException') + + +@mock_glue +def test_get_table_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + with assert_raises(ClientError) as exc: + helpers.get_table(client, database_name, 'myfirsttable') + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Table myfirsttable not found') + + +@mock_glue +def test_get_table_when_database_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'nosuchdatabase' + + with assert_raises(ClientError) as exc: + helpers.get_table(client, database_name, 'myfirsttable') + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found') + + +@mock_glue +def test_get_partitions_empty(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + + response['Partitions'].should.have.length_of(0) + + +@mock_glue +def test_create_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + before = datetime.now(pytz.utc) + + part_input = helpers.create_partition_input(database_name, table_name, values=values) + helpers.create_partition(client, database_name, table_name, part_input) + + after = datetime.now(pytz.utc) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + + partitions = response['Partitions'] + + partitions.should.have.length_of(1) + + partition = partitions[0] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor'].should.equal(part_input['StorageDescriptor']) + partition['Values'].should.equal(values) + partition['CreationTime'].should.be.greater_than(before) + partition['CreationTime'].should.be.lower_than(after) + + +@mock_glue +def test_create_partition_already_exist(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + helpers.create_partition(client, database_name, table_name, values=values) + + with assert_raises(ClientError) as exc: + helpers.create_partition(client, database_name, table_name, values=values) + + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_get_partition_not_found(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_partition(client, database_name, table_name, values) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_get_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values[1]) + + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['Values'].should.equal(values[1]) + + +@mock_glue +def test_update_partition_not_found_moving(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=['0000-00-00'], values=['2018-10-02']) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_update_partition_not_found_change_in_place(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=values, values=values) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_update_partition_cannot_overwrite(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=values[0], values=values[1]) + + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_update_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + helpers.create_partition(client, database_name, table_name, values=values) + + response = helpers.update_partition( + client, + database_name, + table_name, + old_values=values, + values=values, + columns=[{'Name': 'country', 'Type': 'string'}], + ) + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values) + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) + + +@mock_glue +def test_update_partition_move(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + new_values = ['2018-09-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + helpers.create_partition(client, database_name, table_name, values=values) + + response = helpers.update_partition( + client, + database_name, + table_name, + old_values=values, + values=new_values, + columns=[{'Name': 'country', 'Type': 'string'}], + ) + + with assert_raises(ClientError) as exc: + helpers.get_partition(client, database_name, table_name, values) + + # Old partition shouldn't exist anymore + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=new_values) + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) From 5b3b52752de5b4edca796c33d9dfd46b99c24a82 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Thu, 4 Oct 2018 10:25:16 -0500 Subject: [PATCH 34/72] explicitly check that lifecycle actions are not None when setting lifecycle --- moto/s3/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 39f36982d..bb4d7848c 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -424,14 +424,14 @@ class FakeBucket(BaseModel): transition = rule.get('Transition') nve_noncurrent_days = None - if rule.get('NoncurrentVersionExpiration'): + if rule.get('NoncurrentVersionExpiration') is not None: if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None: raise MalformedXML() nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] nvt_noncurrent_days = None nvt_storage_class = None - if rule.get('NoncurrentVersionTransition'): + if rule.get('NoncurrentVersionTransition') is not None: if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None: raise MalformedXML() if rule["NoncurrentVersionTransition"].get('StorageClass') is None: @@ -440,7 +440,7 @@ class FakeBucket(BaseModel): nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] aimu_days = None - if rule.get('AbortIncompleteMultipartUpload'): + if rule.get('AbortIncompleteMultipartUpload') is not None: if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None: raise MalformedXML() aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] From 12e0a38b56794888ce9eb572ab9c3dadaef96dba Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 11:04:55 -0500 Subject: [PATCH 35/72] add TODO for testing exceptions with aimu and nve --- tests/test_s3/test_s3_lifecycle.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index ff89dc113..3d533a641 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -222,11 +222,7 @@ def test_lifecycle_with_nve(): assert len(result["Rules"]) == 1 assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10 - # With failures for missing children: - del lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] - with assert_raises(ClientError) as err: - client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) - assert err.exception.response["Error"]["Code"] == "MalformedXML" + # TODO: Add test for failures due to missing children @mock_s3 @@ -313,11 +309,7 @@ def test_lifecycle_with_aimu(): assert len(result["Rules"]) == 1 assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30 - # With failures for missing children: - del lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] - with assert_raises(ClientError) as err: - client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) - assert err.exception.response["Error"]["Code"] == "MalformedXML" + # TODO: Add test for failures due to missing children @mock_s3_deprecated From 60ec840eef44c395e83342469ab81c0ca5a46daf Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 15:55:47 -0500 Subject: [PATCH 36/72] add disable_key, enable_key, cancel_key_deletion, and schedule_key_deletion actions to KMS endpoint --- moto/kms/models.py | 30 +++++++++++++- moto/kms/responses.py | 46 ++++++++++++++++++++++ tests/test_kms/test_kms.py | 80 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 89ebf0082..30a01d366 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -12,11 +12,13 @@ class Key(BaseModel): self.id = generate_key_id() self.policy = policy self.key_usage = key_usage + self.key_state = "Enabled" self.description = description self.enabled = True self.region = region self.account_id = "0123456789012" self.key_rotation_status = False + self.deletion_date = None @property def physical_resource_id(self): @@ -27,7 +29,7 @@ class Key(BaseModel): return "arn:aws:kms:{0}:{1}:key/{2}".format(self.region, self.account_id, self.id) def to_dict(self): - return { + key_dict = { "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, @@ -36,8 +38,11 @@ class Key(BaseModel): "Enabled": self.enabled, "KeyId": self.id, "KeyUsage": self.key_usage, + "KeyState": self.key_state, } } + key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date if self.key_state == 'PendingDeletion' + return key_dict def delete(self, region_name): kms_backends[region_name].delete_key(self.id) @@ -138,6 +143,29 @@ class KmsBackend(BaseBackend): def get_key_policy(self, key_id): return self.keys[self.get_key_id(key_id)].policy + def disable_key(self, key_id): + if key_id in self.keys: + self.keys[key_id].enabled = False + self.keys[key_id].key_state = 'Disabled' + + def enable_key(self, key_id): + if key_id in self.keys: + self.keys[key_id].enabled = True + self.keys[key_id].key_state = 'Enabled' + + def cancel_key_deletion(self, key_id): + if key_id in self.keys: + self.keys[key_id].key_state = 'Disabled' + self.keys[key_id].deletion_date = None + + def schedule_key_deletion(self, key_id, pending_window_in_days=30): + if key_id in self.keys: + if 7 <= pending_window_in_days <= 30: + self.keys[key_id].enabled = False + self.keys[key_id].key_state = 'PendingDeletion' + self.keys[key_id].deletion_date = datetime.now() + timedelta(days=pending_window_in_days) + return self.keys[key_id].deletion_date + kms_backends = {} for region in boto.kms.regions(): diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 0f544e954..e782d862e 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -233,6 +233,52 @@ class KmsResponse(BaseResponse): value = self.parameters.get("CiphertextBlob") return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8")}) + def disable_key(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.disable_key(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def enable_key(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.enable_key(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def cancel_key_deletion(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.cancel_key_deletion(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps({'KeyId': key_id}) + + def schedule_key_deletion(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + return json.dumps({ + 'KeyId': key_id, + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + }) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + def _assert_valid_key_id(key_id): if not re.match(r'^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$', key_id, re.IGNORECASE): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 96715de71..9779f02a6 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -617,3 +617,83 @@ def test_kms_encrypt_boto3(): response = client.decrypt(CiphertextBlob=response['CiphertextBlob']) response['Plaintext'].should.equal(b'bar') + + +@mock_kms +def test_disable_key(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='disable-key') + client.disable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + + result = client.describe_key(KeyId='disable-key') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'Disabled' + + +@mock_kms +def test_enable_key(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='enable-key') + client.disable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + client.enable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + + result = client.describe_key(KeyId='enable-key') + assert result["KeyMetadata"]["Enabled"] == True + assert result["KeyMetadata"]["KeyState"] == 'Enabled' + + +@mock_kms +def test_schedule_key_deletion(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='schedule-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + + result = client.describe_key(KeyId='schedule-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' + assert 'DeletionDate' in result["KeyMetadata"] + + +@mock_kms +def test_schedule_key_deletion_custom(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='schedule-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + + result = client.describe_key(KeyId='schedule-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' + assert 'DeletionDate' in result["KeyMetadata"] + + +@mock_kms +def test_cancel_key_deletion(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='cancel-key-deletion') + client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + response = client.cancel_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'cancel-key-deletion' + + result = client.describe_key(KeyId='cancel-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'Disabled' + assert 'DeletionDate' not in result["KeyMetadata"] From 15c24e49f0633b7cbe09378dfe22dc94e9190e1f Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 16:00:20 -0500 Subject: [PATCH 37/72] fix formatting for including DeletionDate in response --- moto/kms/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 30a01d366..4ac6ee845 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -41,7 +41,8 @@ class Key(BaseModel): "KeyState": self.key_state, } } - key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date if self.key_state == 'PendingDeletion' + if self.key_state == 'PendingDeletion': + key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date return key_dict def delete(self, region_name): From 7e96203020ea4cd873e0a585884b156896f49265 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 16:21:16 -0500 Subject: [PATCH 38/72] add freezegun and test DeletionDate for chedule_key_deletion --- moto/kms/models.py | 1 + tests/test_kms/test_kms.py | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 4ac6ee845..113bd1733 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -4,6 +4,7 @@ import boto.kms from moto.core import BaseBackend, BaseModel from .utils import generate_key_id from collections import defaultdict +from datetime import datetime, timedelta class Key(BaseModel): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 9779f02a6..287ef9158 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -8,6 +8,8 @@ from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises +from freezegun import freeze_time +from datetime import datetime, timedelta @mock_kms_deprecated @@ -652,11 +654,12 @@ def test_enable_key(): def test_schedule_key_deletion(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(description='schedule-key-deletion') - response = client.schedule_key_deletion( - KeyId=key['KeyMetadata']['KeyId'] - ) - assert response['KeyId'] == 'schedule-key-deletion' - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) result = client.describe_key(KeyId='schedule-key-deletion') assert result["KeyMetadata"]["Enabled"] == False @@ -668,12 +671,13 @@ def test_schedule_key_deletion(): def test_schedule_key_deletion_custom(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(description='schedule-key-deletion') - response = client.schedule_key_deletion( - KeyId=key['KeyMetadata']['KeyId'], - PendingWindowInDays=7 - ) - assert response['KeyId'] == 'schedule-key-deletion' - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) result = client.describe_key(KeyId='schedule-key-deletion') assert result["KeyMetadata"]["Enabled"] == False From 695b4349ba62865cd3e9987af65c39a761f2c35b Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 21:43:12 -0500 Subject: [PATCH 39/72] indentation fix --- moto/kms/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index e782d862e..dc37c8b59 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -271,8 +271,8 @@ class KmsResponse(BaseResponse): _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ - 'KeyId': key_id, - 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + 'KeyId': key_id, + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From a29daf411baf056f3fc924abae7e7bcbe224f5cc Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 21:56:32 -0500 Subject: [PATCH 40/72] fix invalid variables used in kms testing --- tests/test_kms/test_kms.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 287ef9158..ab7513b4d 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -623,8 +623,8 @@ def test_kms_encrypt_boto3(): @mock_kms def test_disable_key(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='disable-key') + client = boto3.client('kms') + key = client.create_key(Description='disable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] ) @@ -636,8 +636,8 @@ def test_disable_key(): @mock_kms def test_enable_key(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='enable-key') + client = boto3.client('kms') + key = client.create_key(Description='enable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] ) @@ -652,8 +652,8 @@ def test_enable_key(): @mock_kms def test_schedule_key_deletion(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='schedule-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] @@ -669,8 +669,8 @@ def test_schedule_key_deletion(): @mock_kms def test_schedule_key_deletion_custom(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='schedule-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'], @@ -687,8 +687,8 @@ def test_schedule_key_deletion_custom(): @mock_kms def test_cancel_key_deletion(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='cancel-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='cancel-key-deletion') client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) From 786b9ca519f830424bdddbcb69e0322f40c0d9b0 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 22:17:48 -0500 Subject: [PATCH 41/72] need region for kms client --- tests/test_kms/test_kms.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index ab7513b4d..0dc93b252 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -623,7 +623,7 @@ def test_kms_encrypt_boto3(): @mock_kms def test_disable_key(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='disable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] @@ -636,7 +636,7 @@ def test_disable_key(): @mock_kms def test_enable_key(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='enable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] @@ -652,7 +652,7 @@ def test_enable_key(): @mock_kms def test_schedule_key_deletion(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( @@ -669,7 +669,7 @@ def test_schedule_key_deletion(): @mock_kms def test_schedule_key_deletion_custom(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( @@ -687,7 +687,7 @@ def test_schedule_key_deletion_custom(): @mock_kms def test_cancel_key_deletion(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='cancel-key-deletion') client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] From 372f749831344f979d349238e15585f68dabc5ca Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 22:46:19 -0500 Subject: [PATCH 42/72] format DeletionDate properly for JSON serialization --- moto/kms/responses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index dc37c8b59..fe7d28523 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -9,6 +9,7 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException from moto.core.responses import BaseResponse +from moto.core.utils import iso_8601_datetime_without_milliseconds from .models import kms_backends reserved_aliases = [ @@ -272,7 +273,7 @@ class KmsResponse(BaseResponse): try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id)) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From f596069dabd1852f67f3a8a63e0e01029089cb43 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 23:35:34 -0500 Subject: [PATCH 43/72] use initial KeyMetadata for identifying keys in KMS tests --- tests/test_kms/test_kms.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 0dc93b252..69cd508ba 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -658,10 +658,10 @@ def test_schedule_key_deletion(): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) - assert response['KeyId'] == 'schedule-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] assert response['DeletionDate'] == datetime.now() + timedelta(days=30) - result = client.describe_key(KeyId='schedule-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' assert 'DeletionDate' in result["KeyMetadata"] @@ -676,10 +676,10 @@ def test_schedule_key_deletion_custom(): KeyId=key['KeyMetadata']['KeyId'], PendingWindowInDays=7 ) - assert response['KeyId'] == 'schedule-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] assert response['DeletionDate'] == datetime.now() + timedelta(days=7) - result = client.describe_key(KeyId='schedule-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' assert 'DeletionDate' in result["KeyMetadata"] @@ -695,9 +695,9 @@ def test_cancel_key_deletion(): response = client.cancel_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) - assert response['KeyId'] == 'cancel-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] - result = client.describe_key(KeyId='cancel-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' assert 'DeletionDate' not in result["KeyMetadata"] From 6277983e3f01daa19247e0f217941fd2d3ad4b1d Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 23:48:19 -0500 Subject: [PATCH 44/72] missed some KeyMetadata and need to transform datetime for testing --- tests/test_kms/test_kms.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 69cd508ba..a06a07324 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -7,6 +7,7 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated +from moto.core.utils import iso_8601_datetime_without_milliseconds from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta @@ -629,7 +630,7 @@ def test_disable_key(): KeyId=key['KeyMetadata']['KeyId'] ) - result = client.describe_key(KeyId='disable-key') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' @@ -645,7 +646,7 @@ def test_enable_key(): KeyId=key['KeyMetadata']['KeyId'] ) - result = client.describe_key(KeyId='enable-key') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == True assert result["KeyMetadata"]["KeyState"] == 'Enabled' @@ -659,7 +660,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=30)) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -677,7 +678,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=7)) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 21c8914efe65465ef90351dfa13441bf74e67427 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 00:13:47 -0500 Subject: [PATCH 45/72] include pending days input for schedule key deletion and update tests since boto client returns DeletionDate as datetime --- moto/kms/responses.py | 3 ++- tests/test_kms/test_kms.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index fe7d28523..831045e3c 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -269,11 +269,12 @@ class KmsResponse(BaseResponse): def schedule_key_deletion(self): key_id = self.parameters.get('KeyId') + pending_window_in_days = self.parameters.get('PendingWindowInDays') _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id)) + 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days)) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index a06a07324..218df1d8f 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -7,7 +7,6 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated -from moto.core.utils import iso_8601_datetime_without_milliseconds from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta @@ -660,7 +659,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=30)) + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -678,7 +677,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=7)) + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 59c233f43158d453fd9f2392a8db4d451e34a46d Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 00:33:23 -0500 Subject: [PATCH 46/72] avoid needing to import datetime and dealing with timezone vs naive datetimes in tests --- tests/test_kms/test_kms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 218df1d8f..b09459e1d 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -9,7 +9,6 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time -from datetime import datetime, timedelta @mock_kms_deprecated @@ -659,7 +658,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + assert response['DeletionDate'] == '2015-01-31T12:00:00.000Z' result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -677,7 +676,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + assert response['DeletionDate'] == '2015-01-08T12:00:00.000Z' result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 9b25d56a3585fcc5a90a8e606b63cacf170d4351 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:18:26 -0500 Subject: [PATCH 47/72] need datetime for tests since thats what boto3 returns and add default for PendingWindowInDays --- moto/kms/models.py | 2 +- moto/kms/responses.py | 5 ++++- tests/test_kms/test_kms.py | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 113bd1733..01b3d9711 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -160,7 +160,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].key_state = 'Disabled' self.keys[key_id].deletion_date = None - def schedule_key_deletion(self, key_id, pending_window_in_days=30): + def schedule_key_deletion(self, key_id, pending_window_in_days): if key_id in self.keys: if 7 <= pending_window_in_days <= 30: self.keys[key_id].enabled = False diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 831045e3c..7caeafcaa 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -269,7 +269,10 @@ class KmsResponse(BaseResponse): def schedule_key_deletion(self): key_id = self.parameters.get('KeyId') - pending_window_in_days = self.parameters.get('PendingWindowInDays') + if self.parameters.get('PendingWindowInDays') is None: + pending_window_in_days = 30 + else: + pending_window_in_days = self.parameters.get('PendingWindowInDays') _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index b09459e1d..e16bd2cea 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -9,6 +9,7 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time +from datetime import datetime, timedelta @mock_kms_deprecated @@ -658,7 +659,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == '2015-01-31T12:00:00.000Z' + assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -676,7 +677,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == '2015-01-08T12:00:00.000Z' + assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 76baab74ad6a1c4ebe8d3037bf202ca473187875 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:33:02 -0500 Subject: [PATCH 48/72] missing tzlocal --- tests/test_kms/test_kms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index e16bd2cea..159f45af4 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -10,6 +10,7 @@ from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta +from dateutil.tz import tzlocal @mock_kms_deprecated From 398dcd8230d8a2e235527bb72a0766b02d3cd6fb Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:47:22 -0500 Subject: [PATCH 49/72] transform DeletionDate in model instead to accomodate Key.to_dict --- moto/kms/models.py | 5 +++-- moto/kms/responses.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 01b3d9711..bb39d1b24 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import boto.kms from moto.core import BaseBackend, BaseModel +from moto.core.utils import iso_8601_datetime_without_milliseconds from .utils import generate_key_id from collections import defaultdict from datetime import datetime, timedelta @@ -43,7 +44,7 @@ class Key(BaseModel): } } if self.key_state == 'PendingDeletion': - key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date + key_dict['KeyMetadata']['DeletionDate'] = iso_8601_datetime_without_milliseconds(self.deletion_date) return key_dict def delete(self, region_name): @@ -166,7 +167,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].enabled = False self.keys[key_id].key_state = 'PendingDeletion' self.keys[key_id].deletion_date = datetime.now() + timedelta(days=pending_window_in_days) - return self.keys[key_id].deletion_date + return iso_8601_datetime_without_milliseconds(self.keys[key_id].deletion_date) kms_backends = {} diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 7caeafcaa..5883f51ec 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -9,7 +9,6 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException from moto.core.responses import BaseResponse -from moto.core.utils import iso_8601_datetime_without_milliseconds from .models import kms_backends reserved_aliases = [ @@ -277,7 +276,7 @@ class KmsResponse(BaseResponse): try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days)) + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From c2595b2eef9c147ccc03d2ce7ce2b59b6cec4d35 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 08:29:21 -0500 Subject: [PATCH 50/72] cant manipulate time in server mode tests --- tests/test_kms/test_kms.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 159f45af4..4f8503ceb 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -655,12 +655,19 @@ def test_enable_key(): def test_schedule_key_deletion(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') - with freeze_time("2015-01-01 12:00:00"): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'false': + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == key['KeyMetadata']['KeyId'] + assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) + else: + # Can't manipulate time in server mode response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -672,13 +679,21 @@ def test_schedule_key_deletion(): def test_schedule_key_deletion_custom(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') - with freeze_time("2015-01-01 12:00:00"): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'false': + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == key['KeyMetadata']['KeyId'] + assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) + else: + # Can't manipulate time in server mode response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'], PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 181e9690b84db61f180d31f30ea3e92776cb9a49 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 08:38:49 -0500 Subject: [PATCH 51/72] need os for checking server mode env variable --- tests/test_kms/test_kms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 4f8503ceb..8bccae27a 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -import re +import os, re import boto3 import boto.kms From c1ebec1b352cf29b9ba666019954b6667757d738 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 10:17:51 -0500 Subject: [PATCH 52/72] remove start_time from attrib comparison in test_copy_snapshot() --- tests/test_ec2/test_elastic_block_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 8930838c6..442e41dde 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -615,8 +615,8 @@ def test_copy_snapshot(): dest = dest_ec2.Snapshot(copy_snapshot_response['SnapshotId']) attribs = ['data_encryption_key_id', 'encrypted', - 'kms_key_id', 'owner_alias', 'owner_id', 'progress', - 'start_time', 'state', 'state_message', + 'kms_key_id', 'owner_alias', 'owner_id', + 'progress', 'state', 'state_message', 'tags', 'volume_id', 'volume_size'] for attrib in attribs: From d9577f9d3d8c7e4aa32bdb346a7d8a5c4a55d2f4 Mon Sep 17 00:00:00 2001 From: George Alton Date: Mon, 8 Oct 2018 19:04:47 +0100 Subject: [PATCH 53/72] Ensures a UserPool Id starts like {region}_ --- moto/cognitoidp/models.py | 2 +- tests/test_cognitoidp/test_cognitoidp.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89f..db54ded7c 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -24,7 +24,7 @@ class CognitoIdpUserPool(BaseModel): def __init__(self, region, name, extended_config): self.region = region - self.id = str(uuid.uuid4()) + self.id = "{}_{}".format(self.region, str(uuid.uuid4().hex)) self.name = name self.status = None self.extended_config = extended_config or {} diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469ce..76e86a691 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -24,6 +24,7 @@ def test_create_user_pool(): ) result["UserPool"]["Id"].should_not.be.none + result["UserPool"]["Id"].should.match(r'[\w-]+_[0-9a-zA-Z]+') result["UserPool"]["Name"].should.equal(name) result["UserPool"]["LambdaConfig"]["PreSignUp"].should.equal(value) From 7a88e634eb0e7780a07645faad5545ce2689531d Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Mon, 8 Oct 2018 15:27:19 -0700 Subject: [PATCH 54/72] organizations: add exception test for describe_organization endpoint --- tests/test_organizations/test_organizations_boto3.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index ae9bacd82..dfac5feeb 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -32,6 +32,17 @@ def test_describe_organization(): validate_organization(response) +@mock_organizations +def test_describe_organization_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.describe_organization() + ex = e.exception + ex.operation_name.should.equal('DescribeOrganization') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('AWSOrganizationsNotInUseException') + + # Organizational Units @mock_organizations From 9081a160d3df9d36c27af5efc87d81457bd74e4f Mon Sep 17 00:00:00 2001 From: Danny Cosson Date: Tue, 9 Oct 2018 10:28:15 -0700 Subject: [PATCH 55/72] fixes for cognito identity library --- moto/cognitoidentity/responses.py | 5 ++++- moto/cognitoidentity/utils.py | 2 +- tests/test_cognitoidentity/test_cognitoidentity.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/cognitoidentity/responses.py b/moto/cognitoidentity/responses.py index ea54b2cff..e7b428329 100644 --- a/moto/cognitoidentity/responses.py +++ b/moto/cognitoidentity/responses.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse from .models import cognitoidentity_backends +from .utils import get_random_identity_id class CognitoIdentityResponse(BaseResponse): @@ -31,4 +32,6 @@ class CognitoIdentityResponse(BaseResponse): return cognitoidentity_backends[self.region].get_credentials_for_identity(self._get_param('IdentityId')) def get_open_id_token_for_developer_identity(self): - return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity(self._get_param('IdentityId')) + return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity( + self._get_param('IdentityId') or get_random_identity_id(self.region) + ) diff --git a/moto/cognitoidentity/utils.py b/moto/cognitoidentity/utils.py index 359631763..6143d5121 100644 --- a/moto/cognitoidentity/utils.py +++ b/moto/cognitoidentity/utils.py @@ -2,4 +2,4 @@ from moto.core.utils import get_random_hex def get_random_identity_id(region): - return "{0}:{0}".format(region, get_random_hex(length=19)) + return "{0}:{1}".format(region, get_random_hex(length=19)) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index a38107b99..ac79fa223 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -31,6 +31,7 @@ def test_create_identity_pool(): # testing a helper function def test_get_random_identity_id(): assert len(get_random_identity_id('us-west-2')) > 0 + assert len(get_random_identity_id('us-west-2').split(':')[1]) == 19 @mock_cognitoidentity @@ -69,3 +70,16 @@ def test_get_open_id_token_for_developer_identity(): ) assert len(result['Token']) assert result['IdentityId'] == '12345' + +@mock_cognitoidentity +def test_get_open_id_token_for_developer_identity_when_no_explicit_identity_id(): + conn = boto3.client('cognito-identity', 'us-west-2') + result = conn.get_open_id_token_for_developer_identity( + IdentityPoolId='us-west-2:12345', + Logins={ + 'someurl': '12345' + }, + TokenDuration=123 + ) + assert len(result['Token']) > 0 + assert len(result['IdentityId']) > 0 From 2c15d71c2c872d2d452e16b61d5faf613073338b Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Thu, 11 Oct 2018 18:21:53 +0900 Subject: [PATCH 56/72] Allow spaces to if_not_exists --- moto/dynamodb2/models.py | 2 +- tests/test_dynamodb2/test_dynamodb.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b327c7a4b..0fa96f3b6 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -154,7 +154,7 @@ class Item(BaseModel): # If not exists, changes value to a default if needed, else its the same as it was if value.startswith('if_not_exists'): # Function signature - match = re.match(r'.*if_not_exists\((?P.+),\s*(?P.+)\).*', value) + match = re.match(r'.*if_not_exists\s*\((?P.+),\s*(?P.+)\).*', value) if not match: raise TypeError diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 243de2701..70feaf7f6 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1220,7 +1220,8 @@ def test_update_if_not_exists(): 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET created_at = if_not_exists(created_at, :created_at)', + # if_not_exists without space + UpdateExpression='SET created_at=if_not_exists(created_at,:created_at)', ExpressionAttributeValues={ ':created_at': 123 } @@ -1233,7 +1234,8 @@ def test_update_if_not_exists(): 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET created_at = if_not_exists(created_at, :created_at)', + # if_not_exists with space + UpdateExpression='SET created_at = if_not_exists (created_at, :created_at)', ExpressionAttributeValues={ ':created_at': 456 } From cf157287e707117447ccb637d8ffbf642eaad240 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Fri, 12 Oct 2018 16:08:05 +0900 Subject: [PATCH 57/72] Fix wrong type if exists --- moto/dynamodb2/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b327c7a4b..02b2933f2 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -162,12 +162,13 @@ class Item(BaseModel): # If it already exists, get its value so we dont overwrite it if path in self.attrs: - value = self.attrs[path].cast_value + value = self.attrs[path] - if value in expression_attribute_values: - value = DynamoType(expression_attribute_values[value]) - else: - value = DynamoType({"S": value}) + if type(value) != DynamoType: + if value in expression_attribute_values: + value = DynamoType(expression_attribute_values[value]) + else: + value = DynamoType({"S": value}) if '.' not in key: self.attrs[key] = value From 13c2e699328eebca5ae8d92797a9fc031b461752 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Fri, 12 Oct 2018 16:59:52 +0900 Subject: [PATCH 58/72] Allow extra spaces to attribute_exists and attribute_not_exists too --- moto/dynamodb2/responses.py | 8 ++++---- tests/test_dynamodb2/test_dynamodb.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 493e17833..e2f1ef1cc 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -204,9 +204,9 @@ class DynamoHandler(BaseResponse): if cond_items: expected = {} overwrite = False - exists_re = re.compile('^attribute_exists\((.*)\)$') + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( - '^attribute_not_exists\((.*)\)$') + '^attribute_not_exists\s*\((.*)\)$') for cond in cond_items: exists_m = exists_re.match(cond) @@ -556,9 +556,9 @@ class DynamoHandler(BaseResponse): if cond_items: expected = {} - exists_re = re.compile('^attribute_exists\((.*)\)$') + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( - '^attribute_not_exists\((.*)\)$') + '^attribute_not_exists\s*\((.*)\)$') for cond in cond_items: exists_m = exists_re.match(cond) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 70feaf7f6..afc919dd7 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -700,8 +700,8 @@ def test_filter_expression(): filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id IN :v0', {}, {':v0': {'NS': [7, 8, 9]}}) filter_expr.expr(row1).should.be(True) - # attribute function tests - filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists(User)', {}, {}) + # attribute function tests (with extra spaces) + filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists (User)', {}, {}) filter_expr.expr(row1).should.be(True) filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_type(Id, N)', {}, {}) From 27ca96519b5517fa51b954972ee95c46f77d8782 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 15 Oct 2018 01:37:25 -0400 Subject: [PATCH 59/72] Fix extra whitespace in s3. Closes #1844. --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index de101a193..962025cb1 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1449,7 +1449,7 @@ S3_MULTIPART_LIST_RESPONSE = """ STANDARD 1 - {{ count }} + {{ count }} {{ count }} false {% for part in parts %} From 81f96c4ceb1934f317901bea98012bb8926839c6 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Wed, 17 Oct 2018 11:08:44 +1100 Subject: [PATCH 60/72] Don't compare a dict_keys object to a list, since it is always False --- moto/dynamodb2/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 63ad20df6..a54c4f7d0 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -265,9 +265,9 @@ class Item(BaseModel): self.attrs[attribute_name] = DynamoType({"SS": new_value}) elif isinstance(new_value, dict): self.attrs[attribute_name] = DynamoType({"M": new_value}) - elif update_action['Value'].keys() == ['N']: + elif set(update_action['Value'].keys()) == set(['N']): self.attrs[attribute_name] = DynamoType({"N": new_value}) - elif update_action['Value'].keys() == ['NULL']: + elif set(update_action['Value'].keys()) == set(['NULL']): if attribute_name in self.attrs: del self.attrs[attribute_name] else: From 8ae1a2b357ff9a3d4b8c14b22229e5d6138ec9e5 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Tue, 16 Oct 2018 17:14:23 -0700 Subject: [PATCH 61/72] Fixes for IAM Groups --- moto/iam/models.py | 10 +++++++++- moto/iam/responses.py | 4 +++- tests/test_iam/test_iam_groups.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4d884fa2f..4a5240a08 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -255,7 +255,15 @@ class Group(BaseModel): @property def arn(self): - return "arn:aws:iam::{0}:group/{1}".format(ACCOUNT_ID, self.path) + if self.path == '/': + return "arn:aws:iam::{0}:group/{1}".format(ACCOUNT_ID, self.name) + + else: + return "arn:aws:iam::{0}:group/{1}/{2}".format(ACCOUNT_ID, self.path, self.name) + + @property + def create_date(self): + return self.created def get_policy(self, policy_name): try: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 9e8d21396..f7b373db7 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -285,7 +285,7 @@ class IamResponse(BaseResponse): def create_group(self): group_name = self._get_param('GroupName') - path = self._get_param('Path') + path = self._get_param('Path', '/') group = iam_backend.create_group(group_name, path) template = self.response_template(CREATE_GROUP_TEMPLATE) @@ -1007,6 +1007,7 @@ CREATE_GROUP_TEMPLATE = """ {{ group.name }} {{ group.id }} {{ group.arn }} + {{ group.create_date }} @@ -1021,6 +1022,7 @@ GET_GROUP_TEMPLATE = """ {{ group.name }} {{ group.id }} {{ group.arn }} + {{ group.create_date }} {% for user in group.users %} diff --git a/tests/test_iam/test_iam_groups.py b/tests/test_iam/test_iam_groups.py index 49c7987f6..0d4756f75 100644 --- a/tests/test_iam/test_iam_groups.py +++ b/tests/test_iam/test_iam_groups.py @@ -1,4 +1,7 @@ from __future__ import unicode_literals + +from datetime import datetime + import boto import boto3 import sure # noqa @@ -25,6 +28,25 @@ def test_get_group(): conn.get_group('not-group') +@mock_iam() +def test_get_group_current(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_group(GroupName='my-group') + result = conn.get_group(GroupName='my-group') + + assert result['Group']['Path'] == '/' + assert result['Group']['GroupName'] == 'my-group' + assert isinstance(result['Group']['CreateDate'], datetime) + assert result['Group']['GroupId'] + assert result['Group']['Arn'] == 'arn:aws:iam::123456789012:group/my-group' + assert not result['Users'] + + # Make a group with a different path: + other_group = conn.create_group(GroupName='my-other-group', Path='some/location') + assert other_group['Group']['Path'] == 'some/location' + assert other_group['Group']['Arn'] == 'arn:aws:iam::123456789012:group/some/location/my-other-group' + + @mock_iam_deprecated() def test_get_all_groups(): conn = boto.connect_iam() From 1b42c7bf7a53cd2cfd755216e6a9a3a4e591167d Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Fri, 27 Jul 2018 16:10:26 +1000 Subject: [PATCH 62/72] Be able to change `enabled` status for cognito-idp users --- IMPLEMENTATION_COVERAGE.md | 4 +-- moto/cognitoidp/models.py | 8 ++++++ moto/cognitoidp/responses.py | 12 +++++++++ tests/test_cognitoidp/test_cognitoidp.py | 32 ++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 17b864dc3..7c68c0e31 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -835,8 +835,8 @@ - [ ] admin_delete_user - [ ] admin_delete_user_attributes - [ ] admin_disable_provider_for_user -- [ ] admin_disable_user -- [ ] admin_enable_user +- [X] admin_disable_user +- [X] admin_enable_user - [ ] admin_forget_device - [ ] admin_get_device - [ ] admin_get_user diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 10da0c6ff..4c0132d31 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -394,6 +394,14 @@ class CognitoIdpBackend(BaseBackend): return user_pool.users.values() + def admin_disable_user(self, user_pool_id, username): + user = self.admin_get_user(user_pool_id, username) + user.enabled = False + + def admin_enable_user(self, user_pool_id, username): + user = self.admin_get_user(user_pool_id, username) + user.enabled = True + def admin_delete_user(self, user_pool_id, username): user_pool = self.user_pools.get(user_pool_id) if not user_pool: diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index e6f20367e..50939786b 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -160,6 +160,18 @@ class CognitoIdpResponse(BaseResponse): "Users": [user.to_json(extended=True) for user in users] }) + def admin_disable_user(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + cognitoidp_backends[self.region].admin_disable_user(user_pool_id, username) + return "" + + def admin_enable_user(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + cognitoidp_backends[self.region].admin_enable_user(user_pool_id, username) + return "" + def admin_delete_user(self): user_pool_id = self._get_param("UserPoolId") username = self._get_param("Username") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 56d7c08a8..b001d8f0c 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -343,6 +343,7 @@ def test_admin_create_user(): result["User"]["Attributes"].should.have.length_of(1) result["User"]["Attributes"][0]["Name"].should.equal("thing") result["User"]["Attributes"][0]["Value"].should.equal(value) + result["User"]["Enabled"].should.equal(True) @mock_cognitoidp @@ -379,6 +380,37 @@ def test_list_users(): result["Users"][0]["Username"].should.equal(username) +@mock_cognitoidp +def test_admin_disable_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + result = conn.admin_disable_user(UserPoolId=user_pool_id, Username=username) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) \ + ["Enabled"].should.equal(False) + + +@mock_cognitoidp +def test_admin_enable_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + conn.admin_disable_user(UserPoolId=user_pool_id, Username=username) + + result = conn.admin_enable_user(UserPoolId=user_pool_id, Username=username) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) \ + ["Enabled"].should.equal(True) + + @mock_cognitoidp def test_admin_delete_user(): conn = boto3.client("cognito-idp", "us-west-2") From d9190245108576103db7eb391d38d44dfe39b6dc Mon Sep 17 00:00:00 2001 From: George Alton Date: Wed, 17 Oct 2018 13:44:00 +0100 Subject: [PATCH 63/72] Adds keyId support to apigateway get_usage_plans apigateway is able to filter the result set, returning only usage plans with the given keyId. This commit supports filtering the usage plans returned to the user by filtering the list of usage plans by checking for usage plan keys --- moto/apigateway/models.py | 11 ++++++-- moto/apigateway/responses.py | 3 ++- tests/test_apigateway/test_apigateway.py | 33 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 4094c7a69..db4746a0e 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -606,8 +606,15 @@ class APIGatewayBackend(BaseBackend): self.usage_plans[plan['id']] = plan return plan - def get_usage_plans(self): - return list(self.usage_plans.values()) + def get_usage_plans(self, api_key_id=None): + plans = list(self.usage_plans.values()) + if api_key_id is not None: + plans = [ + plan + for plan in plans + if self.usage_plan_keys.get(plan['id'], {}).get(api_key_id, False) + ] + return plans def get_usage_plan(self, usage_plan_id): return self.usage_plans[usage_plan_id] diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 7364ae2cb..bc4d262cd 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -255,7 +255,8 @@ class APIGatewayResponse(BaseResponse): if self.method == 'POST': usage_plan_response = self.backend.create_usage_plan(json.loads(self.body)) elif self.method == 'GET': - usage_plans_response = self.backend.get_usage_plans() + api_key_id = self.querystring.get("keyId", [None])[0] + usage_plans_response = self.backend.get_usage_plans(api_key_id=api_key_id) return 200, {}, json.dumps({"item": usage_plans_response}) return 200, {}, json.dumps(usage_plan_response) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 8a2c4370d..5954de8ca 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1084,3 +1084,36 @@ def test_create_usage_plan_key_non_existent_api_key(): # Attempt to create a usage plan key for a API key that doesn't exists payload = {'usagePlanId': usage_plan_id, 'keyId': 'non-existent', 'keyType': 'API_KEY' } client.create_usage_plan_key.when.called_with(**payload).should.throw(ClientError) + + +@mock_apigateway +def test_get_usage_plans_using_key_id(): + region_name = 'us-west-2' + client = boto3.client('apigateway', region_name=region_name) + + # Create 2 Usage Plans + # one will be attached to an API Key, the other will remain unattached + attached_plan = client.create_usage_plan(name='Attached') + unattached_plan = client.create_usage_plan(name='Unattached') + + # Create an API key + # to attach to the usage plan + key_name = 'test-api-key' + response = client.create_api_key(name=key_name) + key_id = response["id"] + + # Create a Usage Plan Key + # Attached the Usage Plan and API Key + key_type = 'API_KEY' + payload = {'usagePlanId': attached_plan['id'], 'keyId': key_id, 'keyType': key_type} + response = client.create_usage_plan_key(**payload) + + # All usage plans should be returned when keyId is not included + all_plans = client.get_usage_plans() + len(all_plans['items']).should.equal(2) + + # Only the usage plan attached to the given api key are included + only_plans_with_key = client.get_usage_plans(keyId=key_id) + len(only_plans_with_key['items']).should.equal(1) + only_plans_with_key['items'][0]['name'].should.equal(attached_plan['name']) + only_plans_with_key['items'][0]['id'].should.equal(attached_plan['id']) From 2d2708cfd7c5a9f1457c94e482d82d7ee5757ffc Mon Sep 17 00:00:00 2001 From: George Alton Date: Wed, 17 Oct 2018 18:39:52 +0100 Subject: [PATCH 64/72] Missing users now raise a UserNotFoundException A missing user in a cognito user pool has raises a UserNotFoundException, not a ResourceNotFoundException. This commit corrects the behaviour so that the correct exception is raised --- moto/cognitoidp/models.py | 4 ++-- tests/test_cognitoidp/test_cognitoidp.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 4c0132d31..476d470b9 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -383,7 +383,7 @@ class CognitoIdpBackend(BaseBackend): raise ResourceNotFoundError(user_pool_id) if username not in user_pool.users: - raise ResourceNotFoundError(username) + raise UserNotFoundError(username) return user_pool.users[username] @@ -408,7 +408,7 @@ class CognitoIdpBackend(BaseBackend): raise ResourceNotFoundError(user_pool_id) if username not in user_pool.users: - raise ResourceNotFoundError(username) + raise UserNotFoundError(username) del user_pool.users[username] diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b001d8f0c..f72a44762 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -368,6 +368,22 @@ def test_admin_get_user(): result["UserAttributes"][0]["Value"].should.equal(value) +@mock_cognitoidp +def test_admin_get_missing_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + caught = False + try: + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) + except conn.exceptions.UserNotFoundException: + caught = True + + caught.should.be.true + + @mock_cognitoidp def test_list_users(): conn = boto3.client("cognito-idp", "us-west-2") @@ -423,7 +439,7 @@ def test_admin_delete_user(): caught = False try: conn.admin_get_user(UserPoolId=user_pool_id, Username=username) - except conn.exceptions.ResourceNotFoundException: + except conn.exceptions.UserNotFoundException: caught = True caught.should.be.true From 4a7ed0d43e6dff07c26e434b3df2af55521a7632 Mon Sep 17 00:00:00 2001 From: Will Bengtson Date: Wed, 17 Oct 2018 15:48:13 -0700 Subject: [PATCH 65/72] remove the marker since this is truncated --- moto/iam/responses.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index f7b373db7..22558f3f6 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1386,10 +1386,6 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {% endfor %} - - EXAMPLEkakv9BCuUNFDtxWSyfzetYwEx2ADc8dnzfvERF5S6YMvXKx41t6gCl/eeaCX3Jo94/ - bKqezEAg8TEVS99EKFLxm3jtbpl25FDWEXAMPLE - {% for group in groups %} From 8e909f580a13bc87e6281cea8d44c8ccf32cc2ac Mon Sep 17 00:00:00 2001 From: Jordan Guymon Date: Thu, 6 Sep 2018 15:15:27 -0700 Subject: [PATCH 66/72] MockAWS implementation using botocore event hooks --- moto/apigateway/models.py | 4 +- moto/awslambda/responses.py | 6 +-- moto/core/models.py | 86 ++++++++++++++++++++++++++++++++++++- moto/core/utils.py | 11 +++++ moto/s3/responses.py | 8 +++- requirements-dev.txt | 2 +- setup.py | 4 +- 7 files changed, 110 insertions(+), 11 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index db4746a0e..41a49e361 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -10,6 +10,7 @@ from boto3.session import Session import responses from moto.core import BaseBackend, BaseModel from .utils import create_id +from moto.core.utils import path_url from .exceptions import StageNotFoundException, ApiKeyNotFoundException STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -372,7 +373,8 @@ class RestAPI(BaseModel): # TODO deal with no matching resource def resource_callback(self, request): - path_after_stage_name = '/'.join(request.path_url.split("/")[2:]) + path = path_url(request.url) + path_after_stage_name = '/'.join(path.split("/")[2:]) if not path_after_stage_name: path_after_stage_name = '/' diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 2c8a54523..1a9a4df83 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -7,7 +7,7 @@ try: except ImportError: from urllib.parse import unquote -from moto.core.utils import amz_crc32, amzn_request_id +from moto.core.utils import amz_crc32, amzn_request_id, path_url from moto.core.responses import BaseResponse from .models import lambda_backends @@ -94,7 +94,7 @@ class LambdaResponse(BaseResponse): return self._add_policy(request, full_url, headers) def _add_policy(self, request, full_url, headers): - path = request.path if hasattr(request, 'path') else request.path_url + path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] if self.lambda_backend.get_function(function_name): policy = request.body.decode('utf8') @@ -104,7 +104,7 @@ class LambdaResponse(BaseResponse): return 404, {}, "{}" def _get_policy(self, request, full_url, headers): - path = request.path if hasattr(request, 'path') else request.path_url + path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] if self.lambda_backend.get_function(function_name): lambda_function = self.lambda_backend.get_function(function_name) diff --git a/moto/core/models.py b/moto/core/models.py index adc06a9c0..b12374fdd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -2,11 +2,14 @@ from __future__ import unicode_literals from __future__ import absolute_import -from collections import defaultdict import functools import inspect import re import six +from io import BytesIO +from collections import defaultdict +from botocore.handlers import BUILTIN_HANDLERS +from botocore.awsrequest import AWSResponse from moto import settings import responses @@ -233,7 +236,86 @@ class ResponsesMockAWS(BaseMockAWS): pass -MockAWS = ResponsesMockAWS +BOTOCORE_HTTP_METHODS = [ + 'GET', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH', 'POST', 'PUT' +] + + +class MockRawResponse(BytesIO): + def __init__(self, input): + if isinstance(input, six.text_type): + input = input.encode('utf-8') + super(MockRawResponse, self).__init__(input) + + def stream(self, **kwargs): + contents = self.read() + while contents: + yield contents + contents = self.read() + + +class BotocoreStubber(object): + def __init__(self): + self.enabled = False + self.methods = defaultdict(list) + + def reset(self): + self.methods.clear() + + def register_response(self, method, pattern, response): + matchers = self.methods[method] + matchers.append((pattern, response)) + + def __call__(self, event_name, request, **kwargs): + if not self.enabled: + return None + + response = None + response_callback = None + found_index = None + matchers = self.methods.get(request.method) + + base_url = request.url.split('?', 1)[0] + for i, (pattern, callback) in enumerate(matchers): + if pattern.match(base_url): + if found_index is None: + found_index = i + response_callback = callback + else: + matchers.pop(found_index) + break + + if response_callback is not None: + for header, value in request.headers.items(): + if isinstance(value, six.binary_type): + request.headers[header] = value.decode('utf-8') + status, headers, body = response_callback(request, request.url, request.headers) + body = MockRawResponse(body) + response = AWSResponse(request.url, status, headers, body) + + return response + +botocore_stubber = BotocoreStubber() +BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) + +class BotocoreEventMockAWS(BaseMockAWS): + def reset(self): + botocore_stubber.reset() + + def enable_patching(self): + botocore_stubber.enabled = True + for method in BOTOCORE_HTTP_METHODS: + for backend in self.backends_for_urls.values(): + for key, value in backend.urls.items(): + pattern = re.compile(key) + botocore_stubber.register_response(method, pattern, value) + + def disable_patching(self): + botocore_stubber.enabled = False + self.reset() + + +MockAWS = BotocoreEventMockAWS class ServerModeMockAWS(BaseMockAWS): diff --git a/moto/core/utils.py b/moto/core/utils.py index 86e7632b0..777a03752 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -8,6 +8,7 @@ import random import re import six import string +from six.moves.urllib.parse import urlparse REQUEST_ID_LONG = string.digits + string.ascii_uppercase @@ -286,3 +287,13 @@ def amzn_request_id(f): return status, headers, body return _wrapper + + +def path_url(url): + parsed_url = urlparse(url) + path = parsed_url.path + if not path: + path = '/' + if parsed_url.query: + path = path + '?' + parsed_url.query + return path diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 962025cb1..13e5f87d9 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -10,6 +10,7 @@ import xmltodict from moto.packages.httpretty.core import HTTPrettyRequest from moto.core.responses import _TemplateEnvironmentMixin +from moto.core.utils import path_url from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, \ parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys @@ -487,7 +488,7 @@ class ResponseObject(_TemplateEnvironmentMixin): if isinstance(request, HTTPrettyRequest): path = request.path else: - path = request.full_path if hasattr(request, 'full_path') else request.path_url + path = request.full_path if hasattr(request, 'full_path') else path_url(request.url) if self.is_delete_keys(request, path, bucket_name): return self._bucket_response_delete_keys(request, body, bucket_name, headers) @@ -708,7 +709,10 @@ class ResponseObject(_TemplateEnvironmentMixin): # Copy key # you can have a quoted ?version=abc with a version Id, so work on # we need to parse the unquoted string first - src_key_parsed = urlparse(request.headers.get("x-amz-copy-source")) + src_key = request.headers.get("x-amz-copy-source") + if isinstance(src_key, six.binary_type): + src_key = src_key.decode('utf-8') + src_key_parsed = urlparse(src_key) src_bucket, src_key = unquote(src_key_parsed.path).\ lstrip("/").split("/", 1) src_version_id = parse_qs(src_key_parsed.query).get( diff --git a/requirements-dev.txt b/requirements-dev.txt index 111cd5f3f..f87ab3db6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ freezegun flask boto>=2.45.0 boto3>=1.4.4 -botocore>=1.8.36 +botocore>=1.12.13 six>=1.9 prompt-toolkit==1.0.14 click==6.7 diff --git a/setup.py b/setup.py index 98780dd5a..66dba0f2f 100755 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ import sys install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.6.16,<1.8", - "botocore>=1.9.16,<1.11", + "boto3>=1.6.16", + "botocore>=1.12.13", "cryptography>=2.3.0", "requests>=2.5", "xmltodict", From fd4e5248559545a9c559cc42b263a9b61970c9c1 Mon Sep 17 00:00:00 2001 From: Jordan Guymon Date: Mon, 1 Oct 2018 09:45:12 -0700 Subject: [PATCH 67/72] Use env credentials for all tests --- .travis.yml | 4 ++-- moto/core/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de22818b8..3a5de0fa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: sudo: true before_install: - export BOTO_CONFIG=/dev/null + - export AWS_SECRET_ACCESS_KEY=foobar_secret + - export AWS_ACCESS_KEY_ID=foobar_key install: # We build moto first so the docker container doesn't try to compile it as well, also note we don't use # -d for docker run so the logs show up in travis @@ -32,8 +34,6 @@ install: if [ "$TEST_SERVER_MODE" = "true" ]; then docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh & - export AWS_SECRET_ACCESS_KEY=foobar_secret - export AWS_ACCESS_KEY_ID=foobar_key fi travis_retry pip install boto==2.45.0 travis_retry pip install boto3 diff --git a/moto/core/models.py b/moto/core/models.py index b12374fdd..d5f7b8427 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -295,9 +295,11 @@ class BotocoreStubber(object): return response + botocore_stubber = BotocoreStubber() BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) + class BotocoreEventMockAWS(BaseMockAWS): def reset(self): botocore_stubber.reset() From b20e190995ed469244da3a7e556d76aeaf0cdfa5 Mon Sep 17 00:00:00 2001 From: Lorenz Hufnagel Date: Sun, 14 Oct 2018 19:58:56 +0200 Subject: [PATCH 68/72] Try to get tests running --- moto/core/models.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/moto/core/models.py b/moto/core/models.py index d5f7b8427..19267ca08 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -303,6 +303,7 @@ BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) class BotocoreEventMockAWS(BaseMockAWS): def reset(self): botocore_stubber.reset() + responses_mock.reset() def enable_patching(self): botocore_stubber.enabled = True @@ -312,10 +313,32 @@ class BotocoreEventMockAWS(BaseMockAWS): pattern = re.compile(key) botocore_stubber.register_response(method, pattern, value) + if not hasattr(responses_mock, '_patcher') or not hasattr(responses_mock._patcher, 'target'): + responses_mock.start() + + for method in RESPONSES_METHODS: + # for backend in default_backends.values(): + for backend in self.backends_for_urls.values(): + for key, value in backend.urls.items(): + responses_mock.add( + CallbackResponse( + method=method, + url=re.compile(key), + callback=convert_flask_to_responses_response(value), + stream=True, + match_querystring=False, + ) + ) + def disable_patching(self): botocore_stubber.enabled = False self.reset() + try: + responses_mock.stop() + except RuntimeError: + pass + MockAWS = BotocoreEventMockAWS From 75f2c56a3643b053fbddf458e6e48e5a8e5e6214 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 30 Oct 2018 22:03:09 -0400 Subject: [PATCH 69/72] Fix ecs error response to be json. --- moto/ecs/exceptions.py | 4 +++- tests/test_ecs/test_ecs_boto3.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index c23d6fd1d..a780dc7c2 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -8,4 +8,6 @@ class ServiceNotFoundException(RESTError): def __init__(self, service_name): super(ServiceNotFoundException, self).__init__( error_type="ServiceNotFoundException", - message="The service {0} does not exist".format(service_name)) + message="The service {0} does not exist".format(service_name), + template='error_json', + ) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 70c1463ee..a0e8318da 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -631,7 +631,22 @@ def test_delete_service(): response['service']['schedulingStrategy'].should.equal('REPLICA') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') - + + +@mock_ecs +def test_update_non_existant_service(): + client = boto3.client('ecs', region_name='us-east-1') + try: + client.update_service( + cluster="my-clustet", + service="my-service", + desiredCount=0, + ) + except ClientError as exc: + error_code = exc.response['Error']['Code'] + error_code.should.equal('ServiceNotFoundException') + else: + raise Exception("Didn't raise ClientError") @mock_ec2 From a8bc7a608e7b7de47b1f8ad1f9dec8ceecf2dd1c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 30 Oct 2018 22:09:47 -0400 Subject: [PATCH 70/72] Lint. --- moto/ecs/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index a780dc7c2..bb7e685c8 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -10,4 +10,4 @@ class ServiceNotFoundException(RESTError): error_type="ServiceNotFoundException", message="The service {0} does not exist".format(service_name), template='error_json', - ) + ) From 90a62b56400251620684a094d85e458bc3abb17e Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 4 Nov 2018 17:30:44 -0500 Subject: [PATCH 71/72] 1.3.7 --- CHANGELOG.md | 5 +++++ moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7ee4448..f42619b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Moto Changelog =================== +1.3.7 +----- + + * Switch from mocking requests to using before-send for AWS calls + 1.3.6 ----- diff --git a/moto/__init__.py b/moto/__init__.py index 6992c535e..dd3593d5d 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.6' +__version__ = '1.3.7' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 66dba0f2f..046ecf6e0 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ else: setup( name='moto', - version='1.3.6', + version='1.3.7', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From ed861ecae1039a048a6350a4ff832ef094cdf2c2 Mon Sep 17 00:00:00 2001 From: Niko Eckerskorn Date: Thu, 15 Nov 2018 18:29:05 +1100 Subject: [PATCH 72/72] Loosen aws-xray-sdk requirements (#1948) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 046ecf6e0..a1b8c5dae 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_requires = [ "mock", "docker>=2.5.1", "jsondiff==1.1.1", - "aws-xray-sdk<0.96,>=0.93", + "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", ]