From c95d472bf5ef9746521cf54c83bb61333c3eafcd Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Tue, 3 Sep 2019 14:54:46 +0200 Subject: [PATCH 01/51] Add (failing) test for ElasticBeanstalk --- tests/test_eb/test_eb.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_eb/test_eb.py diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py new file mode 100644 index 000000000..924ed3adc --- /dev/null +++ b/tests/test_eb/test_eb.py @@ -0,0 +1,15 @@ +import boto3 +from moto import mock_eb + + +@mock_eb +def test_application(): + # Create Elastic Beanstalk Application + eb_client = boto3.client('elasticbeanstalk', region_name='us-east-1') + + eb_client.create_application( + ApplicationName="myapp", + ) + + eb_apps = eb_client.describe_applications() + eb_apps['Applications'][0]['ApplicationName'].should.equal("myapp") From 336f50349af0eddbb5d82776258abd9f847fa989 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Tue, 3 Sep 2019 16:10:32 +0200 Subject: [PATCH 02/51] Add sub-minimal mocking of elasticbeanstalk:create_application() --- moto/__init__.py | 1 + moto/eb/__init__.py | 4 ++ moto/eb/exceptions.py | 7 +++ moto/eb/models.py | 37 ++++++++++++++++ moto/eb/responses.py | 92 ++++++++++++++++++++++++++++++++++++++++ moto/eb/urls.py | 11 +++++ tests/test_eb/test_eb.py | 34 ++++++++++++--- 7 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 moto/eb/__init__.py create mode 100644 moto/eb/exceptions.py create mode 100644 moto/eb/models.py create mode 100644 moto/eb/responses.py create mode 100644 moto/eb/urls.py diff --git a/moto/__init__.py b/moto/__init__.py index 8594cedd2..7cb6d0e39 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -18,6 +18,7 @@ from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated # fla from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # flake8: noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # flake8: noqa from .dynamodbstreams import mock_dynamodbstreams # flake8: noqa +from .eb import mock_eb # flake8: noqa from .ec2 import mock_ec2, mock_ec2_deprecated # flake8: noqa from .ecr import mock_ecr, mock_ecr_deprecated # flake8: noqa from .ecs import mock_ecs, mock_ecs_deprecated # flake8: noqa diff --git a/moto/eb/__init__.py b/moto/eb/__init__.py new file mode 100644 index 000000000..3e06e9595 --- /dev/null +++ b/moto/eb/__init__.py @@ -0,0 +1,4 @@ +from .models import eb_backends +from moto.core.models import base_decorator + +mock_eb = base_decorator(eb_backends) diff --git a/moto/eb/exceptions.py b/moto/eb/exceptions.py new file mode 100644 index 000000000..c470d5317 --- /dev/null +++ b/moto/eb/exceptions.py @@ -0,0 +1,7 @@ +from moto.core.exceptions import RESTError + + +class InvalidParameterValueError(RESTError): + def __init__(self, message): + super(InvalidParameterValueError, self).__init__( + "InvalidParameterValue", message) diff --git a/moto/eb/models.py b/moto/eb/models.py new file mode 100644 index 000000000..246d33cde --- /dev/null +++ b/moto/eb/models.py @@ -0,0 +1,37 @@ +import boto.beanstalk + +from moto.core import BaseBackend, BaseModel +from .exceptions import InvalidParameterValueError + + +class FakeApplication(BaseModel): + def __init__(self, application_name): + self.application_name = application_name + + +class EBBackend(BaseBackend): + def __init__(self, region): + self.region = region + self.applications = dict() + + def reset(self): + # preserve region + region = self.region + self._reset_model_refs() + self.__dict__ = {} + self.__init__(region) + + def create_application(self, application_name): + if application_name in self.applications: + raise InvalidParameterValueError( + "Application {} already exists.".format(application_name) + ) + new_app = FakeApplication( + application_name=application_name, + ) + self.applications[application_name] = new_app + return new_app + + +eb_backends = dict((region.name, EBBackend(region.name)) + for region in boto.beanstalk.regions()) diff --git a/moto/eb/responses.py b/moto/eb/responses.py new file mode 100644 index 000000000..9cf8b2e47 --- /dev/null +++ b/moto/eb/responses.py @@ -0,0 +1,92 @@ +from moto.core.responses import BaseResponse +from .models import eb_backends + +EB_CREATE_APPLICATION = """ + + + + + 2019-09-03T13:08:29.049Z + + + + false + 180 + false + + + false + 200 + false + + + + arn:aws:elasticbeanstalk:{{ region_name }}:111122223333:application/{{ application_name }} + {{ application.application_name }} + 2019-09-03T13:08:29.049Z + + + + 1b6173c8-13aa-4b0a-99e9-eb36a1fb2778 + + +""" + + +EB_DESCRIBE_APPLICATIONS = """ + + + + {% for application in applications %} + + + 2019-09-03T13:08:29.049Z + + + + 180 + false + false + + + false + 200 + false + + + + arn:aws:elasticbeanstalk:{{ region_name }}:387323646340:application/{{ application.name }} + {{ application.application_name }} + 2019-09-03T13:08:29.049Z + + {% endfor %} + + + + 015a05eb-282e-4b76-bd18-663fdfaf42e4 + + +""" + + +class EBResponse(BaseResponse): + @property + def backend(self): + return eb_backends[self.region] + + def create_application(self): + app = self.backend.create_application( + application_name=self._get_param('ApplicationName'), + ) + + template = self.response_template(EB_CREATE_APPLICATION) + return template.render( + region_name=self.backend.region, + application=app, + ) + + def describe_applications(self): + template = self.response_template(EB_DESCRIBE_APPLICATIONS) + return template.render( + applications=self.backend.applications.values(), + ) diff --git a/moto/eb/urls.py b/moto/eb/urls.py new file mode 100644 index 000000000..4cd4add13 --- /dev/null +++ b/moto/eb/urls.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from .responses import EBResponse + +url_bases = [ + r"https?://elasticbeanstalk.(?P[a-zA-Z0-9\-_]+).amazonaws.com", +] + +url_paths = { + '{0}/$': EBResponse.dispatch, +} diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 924ed3adc..9e863e7f5 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -1,15 +1,39 @@ import boto3 +import sure # noqa +from botocore.exceptions import ClientError + from moto import mock_eb @mock_eb -def test_application(): +def test_create_application(): # Create Elastic Beanstalk Application - eb_client = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + app = conn.create_application( + ApplicationName="myapp", + ) + app['Application']['ApplicationName'].should.equal("myapp") - eb_client.create_application( + +@mock_eb +def test_create_application_dup(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + conn.create_application.when.called_with( + ApplicationName="myapp", + ).should.throw(ClientError) + + +@mock_eb +def test_describe_applications(): + # Create Elastic Beanstalk Application + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( ApplicationName="myapp", ) - eb_apps = eb_client.describe_applications() - eb_apps['Applications'][0]['ApplicationName'].should.equal("myapp") + apps = conn.describe_applications() + len(apps['Applications']).should.equal(1) + apps['Applications'][0]['ApplicationName'].should.equal('myapp') From 6f23a39fc26c3cf51b4f0e2b49277be85024d666 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 15:33:15 +0200 Subject: [PATCH 03/51] Add minimal mocking of elasticbeanstalk:create_environment, describe_environments and list_available_solution_stacks --- moto/eb/models.py | 25 + moto/eb/responses.py | 1297 +++++++++++++++++++++++++++++++++++++- tests/test_eb/test_eb.py | 41 ++ 3 files changed, 1343 insertions(+), 20 deletions(-) diff --git a/moto/eb/models.py b/moto/eb/models.py index 246d33cde..5b4655175 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -1,12 +1,37 @@ +import weakref + import boto.beanstalk from moto.core import BaseBackend, BaseModel from .exceptions import InvalidParameterValueError +class FakeEnvironment(BaseModel): + def __init__(self, application, environment_name): + self.environment_name = environment_name + self.application = weakref.proxy(application) # weakref to break circular dependencies + + @property + def application_name(self): + return self.application.application_name + + class FakeApplication(BaseModel): def __init__(self, application_name): self.application_name = application_name + self.environments = dict() + + def create_environment(self, environment_name): + if environment_name in self.environments: + raise InvalidParameterValueError + + env = FakeEnvironment( + application=self, + environment_name=environment_name, + ) + self.environments[environment_name] = env + + return env class EBBackend(BaseBackend): diff --git a/moto/eb/responses.py b/moto/eb/responses.py index 9cf8b2e47..fecdb8c21 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -1,5 +1,67 @@ from moto.core.responses import BaseResponse -from .models import eb_backends +from .models import eb_backends, EBBackend +from .exceptions import InvalidParameterValueError + + +class EBResponse(BaseResponse): + @property + def backend(self): + """ + :rtype: EBBackend + """ + return eb_backends[self.region] + + def create_application(self): + app = self.backend.create_application( + application_name=self._get_param('ApplicationName'), + ) + + template = self.response_template(EB_CREATE_APPLICATION) + return template.render( + region_name=self.backend.region, + application=app, + ) + + def describe_applications(self): + template = self.response_template(EB_DESCRIBE_APPLICATIONS) + return template.render( + applications=self.backend.applications.values(), + ) + + def create_environment(self): + application_name = self._get_param('ApplicationName') + environment_name = self._get_param('EnvironmentName') + try: + app = self.backend.applications[application_name] + except KeyError: + raise InvalidParameterValueError( + "No Application named \'{}\' found.".format(application_name) + ) + + env = app.create_environment(environment_name=environment_name) + + template = self.response_template(EB_CREATE_ENVIRONMENT) + return template.render( + environment=env, + region=self.backend.region, + ) + + def describe_environments(self): + envs = [] + + for app in self.backend.applications.values(): + for env in app.environments.values(): + envs.append(env) + + template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) + return template.render( + environments=envs, + ) + + @staticmethod + def list_available_solution_stacks(): + return EB_LIST_AVAILABLE_SOLUTION_STACKS + EB_CREATE_APPLICATION = """ @@ -55,7 +117,7 @@ EB_DESCRIBE_APPLICATIONS = """ - arn:aws:elasticbeanstalk:{{ region_name }}:387323646340:application/{{ application.name }} + arn:aws:elasticbeanstalk:{{ region_name }}:111122223333:application/{{ application.name }} {{ application.application_name }} 2019-09-03T13:08:29.049Z @@ -69,24 +131,1219 @@ EB_DESCRIBE_APPLICATIONS = """ """ -class EBResponse(BaseResponse): - @property - def backend(self): - return eb_backends[self.region] +EB_CREATE_ENVIRONMENT = """ + + + {{ environment.solution_stack_name }} + Grey + arn:aws:elasticbeanstalk:{{ region }}:111122223333:environment/{{ environment.application_name }}/{{ environment.environment_name }} + 2019-09-04T09:41:24.222Z + 2019-09-04T09:41:24.222Z + {{ environment_id }} + arn:aws:elasticbeanstalk:{{ region }}::platform/{{ environment.platform_arn }} + + WebServer + Standard + 1.0 + + {{ environment.environment_name }} + {{ environment.application_name }} + Launching + + + 18dc8158-f5d7-4d5a-82ef-07fcaadf81c6 + + +""" - def create_application(self): - app = self.backend.create_application( - application_name=self._get_param('ApplicationName'), - ) - template = self.response_template(EB_CREATE_APPLICATION) - return template.render( - region_name=self.backend.region, - application=app, - ) +EB_DESCRIBE_ENVIRONMENTS = """ + + + + {% for env in environments %} + + {{ env.solution_stack_name }} + Grey + arn:aws:elasticbeanstalk:{{ region }}:123456789012:environment/{{ env.application_name }}/{{ env.environment_name }} + false + 2019-08-30T09:35:10.913Z + false + + 2019-08-22T07:02:47.332Z + {{ env.environment_id }} + 1 + arn:aws:elasticbeanstalk:{{ region }}::platform/{{ env.platform_arn }} + + WebServer + Standard + 1.0 + + No Data + {{ env.environment_name }} + + + + {{ env.application_name }} + Ready + + {% endfor %} + + + + dd56b215-01a0-40b2-bd1e-57589c39424f + + +""" - def describe_applications(self): - template = self.response_template(EB_DESCRIBE_APPLICATIONS) - return template.render( - applications=self.backend.applications.values(), - ) + +# Current list as of 2019-09-04 +EB_LIST_AVAILABLE_SOLUTION_STACKS = """ + + + + 64bit Amazon Linux 2018.03 v4.10.1 running Node.js + 64bit Amazon Linux 2018.03 v4.9.2 running Node.js + 64bit Amazon Linux 2018.03 v4.8.0 running Node.js + 64bit Amazon Linux 2018.03 v4.6.0 running Node.js + 64bit Amazon Linux 2018.03 v4.5.3 running Node.js + 64bit Amazon Linux 2018.03 v4.5.1 running Node.js + 64bit Amazon Linux 2018.03 v4.5.0 running Node.js + 64bit Amazon Linux 2017.09 v4.4.6 running Node.js + 64bit Amazon Linux 2017.09 v4.4.5 running Node.js + 64bit Amazon Linux 2017.09 v4.4.4 running Node.js + 64bit Amazon Linux 2017.09 v4.4.2 running Node.js + 64bit Amazon Linux 2017.09 v4.4.0 running Node.js + 64bit Amazon Linux 2017.03 v4.3.0 running Node.js + 64bit Amazon Linux 2017.03 v4.2.2 running Node.js + 64bit Amazon Linux 2017.03 v4.2.1 running Node.js + 64bit Amazon Linux 2017.03 v4.2.0 running Node.js + 64bit Amazon Linux 2017.03 v4.1.1 running Node.js + 64bit Amazon Linux 2017.03 v4.1.0 running Node.js + 64bit Amazon Linux 2016.09 v4.0.1 running Node.js + 64bit Amazon Linux 2016.09 v4.0.0 running Node.js + 64bit Amazon Linux 2016.09 v3.3.1 running Node.js + 64bit Amazon Linux 2016.09 v3.1.0 running Node.js + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.4 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.5 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.6 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.12 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.7 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.5 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.4 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.3 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.2 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.1 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.0 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 5.6 + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.1 + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.5 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.5 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.1 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.5 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.1 + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.1 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.5 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.1 + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.1 + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.5 + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.6 + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.3 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.4 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.5 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.6 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.1 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.0 running PHP 7.0 + 64bit Amazon Linux 2016.09 v2.3.2 running PHP 7.0 + 64bit Amazon Linux 2016.09 v2.3.1 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.6 + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.4 + 64bit Amazon Linux 2018.03 v2.9.1 running Python + 64bit Amazon Linux 2018.03 v2.9.1 running Python 2.7 + 64bit Amazon Linux 2018.03 v2.7.5 running Python 3.6 + 64bit Amazon Linux 2018.03 v2.7.1 running Python 3.6 + 64bit Amazon Linux 2018.03 v2.7.0 running Python 3.6 + 64bit Amazon Linux 2017.09 v2.6.4 running Python 3.6 + 64bit Amazon Linux 2017.09 v2.6.1 running Python 3.6 + 64bit Amazon Linux 2017.03 v2.4.0 running Python 3.4 + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 1.9.3 + 64bit Amazon Linux 2018.03 v2.8.0 running Ruby 2.5 (Passenger Standalone) + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Puma) + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8.5 Java 8 + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8 Java 8 + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 7 + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 6 + 64bit Amazon Linux 2018.03 v3.1.1 running Tomcat 8.5 Java 8 + 64bit Amazon Linux 2017.03 v2.6.5 running Tomcat 8 Java 8 + 64bit Amazon Linux 2017.03 v2.6.2 running Tomcat 8 Java 8 + 64bit Amazon Linux 2017.03 v2.6.1 running Tomcat 8 Java 8 + 64bit Amazon Linux 2017.03 v2.6.0 running Tomcat 8 Java 8 + 64bit Amazon Linux 2016.09 v2.5.4 running Tomcat 8 Java 8 + 64bit Amazon Linux 2016.03 v2.1.0 running Tomcat 8 Java 8 + 64bit Windows Server Core 2016 v2.2.1 running IIS 10.0 + 64bit Windows Server 2016 v2.2.1 running IIS 10.0 + 64bit Windows Server Core 2012 R2 v2.2.1 running IIS 8.5 + 64bit Windows Server 2012 R2 v2.2.1 running IIS 8.5 + 64bit Windows Server Core 2016 v1.2.0 running IIS 10.0 + 64bit Windows Server 2016 v1.2.0 running IIS 10.0 + 64bit Windows Server Core 2012 R2 v1.2.0 running IIS 8.5 + 64bit Windows Server 2012 R2 v1.2.0 running IIS 8.5 + 64bit Windows Server 2012 v1.2.0 running IIS 8 + 64bit Windows Server 2008 R2 v1.2.0 running IIS 7.5 + 64bit Windows Server Core 2012 R2 running IIS 8.5 + 64bit Windows Server 2012 R2 running IIS 8.5 + 64bit Windows Server 2012 running IIS 8 + 64bit Windows Server 2008 R2 running IIS 7.5 + 64bit Amazon Linux 2018.03 v2.12.16 running Docker 18.06.1-ce + 64bit Amazon Linux 2016.09 v2.5.2 running Docker 1.12.6 + 64bit Amazon Linux 2018.03 v2.15.2 running Multi-container Docker 18.06.1-ce (Generic) + 64bit Debian jessie v2.12.16 running Go 1.4 (Preconfigured - Docker) + 64bit Debian jessie v2.12.16 running Go 1.3 (Preconfigured - Docker) + 64bit Debian jessie v2.12.16 running Python 3.4 (Preconfigured - Docker) + 64bit Debian jessie v2.10.0 running Python 3.4 (Preconfigured - Docker) + 64bit Amazon Linux 2018.03 v2.9.1 running Java 8 + 64bit Amazon Linux 2018.03 v2.9.1 running Java 7 + 64bit Amazon Linux 2018.03 v2.8.0 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.6 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.5 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.4 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.2 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.1 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.8 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.5 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.4 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.3 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.0 running Java 8 + 64bit Amazon Linux 2017.03 v2.5.4 running Java 8 + 64bit Amazon Linux 2017.03 v2.5.3 running Java 8 + 64bit Amazon Linux 2017.03 v2.5.2 running Java 8 + 64bit Amazon Linux 2016.09 v2.4.4 running Java 8 + 64bit Amazon Linux 2018.03 v2.12.1 running Go 1.12.7 + 64bit Amazon Linux 2018.03 v2.6.14 running Packer 1.0.3 + 64bit Amazon Linux 2018.03 v2.12.16 running GlassFish 5.0 Java 8 (Preconfigured - Docker) + + + + 64bit Amazon Linux 2018.03 v4.10.1 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.9.2 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.8.0 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.6.0 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.5.3 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.5.1 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.5.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.6 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.5 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.4 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.2 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.3.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.2.2 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.2.1 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.2.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.1.1 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.1.0 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v4.0.1 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v4.0.0 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v3.3.1 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v3.1.0 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.12 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.7 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.5 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.4 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.3 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.2 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.1 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.5 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.3 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2016.09 v2.3.2 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2016.09 v2.3.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.4 + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python 2.7 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.5 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.0 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.1 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.0 running Python 3.4 + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 1.9.3 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.0 running Ruby 2.5 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Puma) + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8.5 Java 8 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 7 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 6 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.1.1 running Tomcat 8.5 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.5 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.2 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.1 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.0 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2016.09 v2.5.4 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2016.03 v2.1.0 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Windows Server Core 2016 v2.2.1 running IIS 10.0 + + zip + + + + 64bit Windows Server 2016 v2.2.1 running IIS 10.0 + + zip + + + + 64bit Windows Server Core 2012 R2 v2.2.1 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 R2 v2.2.1 running IIS 8.5 + + zip + + + + 64bit Windows Server Core 2016 v1.2.0 running IIS 10.0 + + zip + + + + 64bit Windows Server 2016 v1.2.0 running IIS 10.0 + + zip + + + + 64bit Windows Server Core 2012 R2 v1.2.0 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 R2 v1.2.0 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 v1.2.0 running IIS 8 + + zip + + + + 64bit Windows Server 2008 R2 v1.2.0 running IIS 7.5 + + zip + + + + 64bit Windows Server Core 2012 R2 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 R2 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 running IIS 8 + + zip + + + + 64bit Windows Server 2008 R2 running IIS 7.5 + + zip + + + + 64bit Amazon Linux 2018.03 v2.12.16 running Docker 18.06.1-ce + + + + 64bit Amazon Linux 2016.09 v2.5.2 running Docker 1.12.6 + + + + 64bit Amazon Linux 2018.03 v2.15.2 running Multi-container Docker 18.06.1-ce (Generic) + + zip + json + + + + 64bit Debian jessie v2.12.16 running Go 1.4 (Preconfigured - Docker) + + zip + + + + 64bit Debian jessie v2.12.16 running Go 1.3 (Preconfigured - Docker) + + zip + + + + 64bit Debian jessie v2.12.16 running Python 3.4 (Preconfigured - Docker) + + zip + + + + 64bit Debian jessie v2.10.0 running Python 3.4 (Preconfigured - Docker) + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Java 7 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.8.0 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.6 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.5 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.2 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.8 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.5 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.03 v2.5.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.03 v2.5.3 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.03 v2.5.2 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2016.09 v2.4.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.12.1 running Go 1.12.7 + + zip + + + + 64bit Amazon Linux 2018.03 v2.6.14 running Packer 1.0.3 + + + + 64bit Amazon Linux 2018.03 v2.12.16 running GlassFish 5.0 Java 8 (Preconfigured - Docker) + + zip + + + + + + bd6bd2b2-9983-4845-b53b-fe53e8a5e1e7 + + +""" diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 9e863e7f5..aafe524fd 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -37,3 +37,44 @@ def test_describe_applications(): apps = conn.describe_applications() len(apps['Applications']).should.equal(1) apps['Applications'][0]['ApplicationName'].should.equal('myapp') + + +@mock_eb +def test_create_environment(): + # Create Elastic Beanstalk Environment + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + app = conn.create_application( + ApplicationName="myapp", + ) + env = conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + ) + env['EnvironmentName'].should.equal("myenv") + + +@mock_eb +def test_describe_environments(): + # List Elastic Beanstalk Envs + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + ) + + envs = conn.describe_environments() + envs = envs['Environments'] + len(envs).should.equal(1) + envs[0]['ApplicationName'].should.equal('myapp') + envs[0]['EnvironmentName'].should.equal('myenv') + + +@mock_eb +def test_list_available_solution_stacks(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + stacks = conn.list_available_solution_stacks() + len(stacks['SolutionStacks']).should.be.greater_than(0) + len(stacks['SolutionStacks']).should.be.equal(len(stacks['SolutionStackDetails'])) From 91fb40810242213349e0f436e01464425fdd0928 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 16:25:43 +0200 Subject: [PATCH 04/51] Move tags_from_query_string to core.utils --- moto/core/utils.py | 17 +++++++++++++++++ moto/ec2/responses/tags.py | 3 ++- moto/ec2/utils.py | 17 ----------------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index ca670e871..acf76bb48 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -297,3 +297,20 @@ def path_url(url): if parsed_url.query: path = path + '?' + parsed_url.query return path + + +def tags_from_query_string(querystring_dict): + prefix = 'Tag' + suffix = 'Key' + response_values = {} + for key, value in querystring_dict.items(): + if key.startswith(prefix) and key.endswith(suffix): + tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") + tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] + tag_value_key = "Tag.{0}.Value".format(tag_index) + if tag_value_key in querystring_dict: + response_values[tag_key] = querystring_dict.get(tag_value_key)[ + 0] + else: + response_values[tag_key] = None + return response_values diff --git a/moto/ec2/responses/tags.py b/moto/ec2/responses/tags.py index 65d3da255..37f2c3bea 100644 --- a/moto/ec2/responses/tags.py +++ b/moto/ec2/responses/tags.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse from moto.ec2.models import validate_resource_ids -from moto.ec2.utils import tags_from_query_string, filters_from_querystring +from moto.ec2.utils import filters_from_querystring +from moto.core.utils import tags_from_query_string class TagResponse(BaseResponse): diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index e67cb39f4..f0d58d5fc 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -198,23 +198,6 @@ def split_route_id(route_id): return values[0], values[1] -def tags_from_query_string(querystring_dict): - prefix = 'Tag' - suffix = 'Key' - response_values = {} - for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] - tag_value_key = "Tag.{0}.Value".format(tag_index) - if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[ - 0] - else: - response_values[tag_key] = None - return response_values - - def dhcp_configuration_from_querystring(querystring, option=u'DhcpConfiguration'): """ turn: From 9bfbd8e0088d93ccf7c0e4d81526f45db8f9bf50 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 16:55:34 +0200 Subject: [PATCH 05/51] Make tags_from_query_string() more flexible --- moto/core/utils.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index acf76bb48..6f75619d4 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -299,15 +299,27 @@ def path_url(url): return path -def tags_from_query_string(querystring_dict): - prefix = 'Tag' - suffix = 'Key' +def tags_from_query_string( + querystring_dict, + prefix="Tag", + key_suffix="Key", + value_suffix="Value" +): response_values = {} for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] - tag_value_key = "Tag.{0}.Value".format(tag_index) + if key.startswith(prefix) and key.endswith(key_suffix): + tag_index = key.replace(prefix + ".", "").replace("." + key_suffix, "") + tag_key = querystring_dict.get( + "{prefix}.{index}.{key_suffix}".format( + prefix=prefix, + index=tag_index, + key_suffix=key_suffix, + ))[0] + tag_value_key = "{prefix}.{index}.{value_suffix}".format( + prefix=prefix, + index=tag_index, + value_suffix=value_suffix, + ) if tag_value_key in querystring_dict: response_values[tag_key] = querystring_dict.get(tag_value_key)[ 0] From 7f387b0bb9842d3561f59ed7fa70b92e17791909 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 16:56:06 +0200 Subject: [PATCH 06/51] Add elasticbeanstalk Tags handling --- moto/eb/exceptions.py | 6 +++ moto/eb/models.py | 47 +++++++++++++++++++-- moto/eb/responses.py | 88 +++++++++++++++++++++++++++++++++++++--- tests/test_eb/test_eb.py | 72 ++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 10 deletions(-) diff --git a/moto/eb/exceptions.py b/moto/eb/exceptions.py index c470d5317..bf3a89618 100644 --- a/moto/eb/exceptions.py +++ b/moto/eb/exceptions.py @@ -5,3 +5,9 @@ class InvalidParameterValueError(RESTError): def __init__(self, message): super(InvalidParameterValueError, self).__init__( "InvalidParameterValue", message) + + +class ResourceNotFoundException(RESTError): + def __init__(self, message): + super(ResourceNotFoundException, self).__init__( + "ResourceNotFoundException", message) diff --git a/moto/eb/models.py b/moto/eb/models.py index 5b4655175..c3c2aa20c 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -7,32 +7,70 @@ from .exceptions import InvalidParameterValueError class FakeEnvironment(BaseModel): - def __init__(self, application, environment_name): - self.environment_name = environment_name + def __init__( + self, + application, + environment_name, + tags, + ): self.application = weakref.proxy(application) # weakref to break circular dependencies + self.environment_name = environment_name + self.tags = tags @property def application_name(self): return self.application.application_name + @property + def environment_arn(self): + return 'arn:aws:elasticbeanstalk:{region}:{account_id}:' \ + 'environment/{application_name}/{environment_name}'.format( + region=self.region, + account_id='123456789012', + application_name=self.application_name, + environment_name=self.environment_name, + ) + + @property + def platform_arn(self): + return 'TODO' # TODO + + @property + def solution_stack_name(self): + return 'TODO' # TODO + + @property + def region(self): + return self.application.region + class FakeApplication(BaseModel): - def __init__(self, application_name): + def __init__(self, backend, application_name): + self.backend = weakref.proxy(backend) # weakref to break cycles self.application_name = application_name self.environments = dict() - def create_environment(self, environment_name): + def create_environment( + self, + environment_name, + tags, + ): if environment_name in self.environments: raise InvalidParameterValueError env = FakeEnvironment( application=self, environment_name=environment_name, + tags=tags, ) self.environments[environment_name] = env return env + @property + def region(self): + return self.backend.region + class EBBackend(BaseBackend): def __init__(self, region): @@ -52,6 +90,7 @@ class EBBackend(BaseBackend): "Application {} already exists.".format(application_name) ) new_app = FakeApplication( + backend=self, application_name=application_name, ) self.applications[application_name] = new_app diff --git a/moto/eb/responses.py b/moto/eb/responses.py index fecdb8c21..fbace1938 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -1,6 +1,7 @@ from moto.core.responses import BaseResponse +from moto.core.utils import tags_from_query_string from .models import eb_backends, EBBackend -from .exceptions import InvalidParameterValueError +from .exceptions import InvalidParameterValueError, ResourceNotFoundException class EBResponse(BaseResponse): @@ -38,7 +39,11 @@ class EBResponse(BaseResponse): "No Application named \'{}\' found.".format(application_name) ) - env = app.create_environment(environment_name=environment_name) + tags = tags_from_query_string(self.querystring, prefix="Tags.member") + env = app.create_environment( + environment_name=environment_name, + tags=tags, + ) template = self.response_template(EB_CREATE_ENVIRONMENT) return template.render( @@ -62,6 +67,48 @@ class EBResponse(BaseResponse): def list_available_solution_stacks(): return EB_LIST_AVAILABLE_SOLUTION_STACKS + def _find_environment_by_arn(self, arn): + for app in self.backend.applications.keys(): + for env in self.backend.applications[app].environments.values(): + if env.environment_arn == arn: + return env + raise KeyError() + + def update_tags_for_resource(self): + resource_arn = self._get_param('ResourceArn') + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN \'{}\'.".format(resource_arn) + ) + + tags_to_add = tags_from_query_string(self.querystring, prefix="TagsToAdd.member") + for key, value in tags_to_add.items(): + res.tags[key] = value + + tags_to_remove = self._get_multi_param('TagsToRemove.member') + for key in tags_to_remove: + del res.tags[key] + + return EB_UPDATE_TAGS_FOR_RESOURCE + + def list_tags_for_resource(self): + resource_arn = self._get_param('ResourceArn') + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN \'{}\'.".format(resource_arn) + ) + tags = res.tags + + template = self.response_template(EB_LIST_TAGS_FOR_RESOURCE) + return template.render( + tags=tags, + arn=resource_arn, + ) + EB_CREATE_APPLICATION = """ @@ -136,11 +183,11 @@ EB_CREATE_ENVIRONMENT = """ {{ environment.solution_stack_name }} Grey - arn:aws:elasticbeanstalk:{{ region }}:111122223333:environment/{{ environment.application_name }}/{{ environment.environment_name }} + {{ environment.environment_arn }} 2019-09-04T09:41:24.222Z 2019-09-04T09:41:24.222Z {{ environment_id }} - arn:aws:elasticbeanstalk:{{ region }}::platform/{{ environment.platform_arn }} + {{ environment.platform_arn }} WebServer Standard @@ -165,7 +212,7 @@ EB_DESCRIBE_ENVIRONMENTS = """ {{ env.solution_stack_name }} Grey - arn:aws:elasticbeanstalk:{{ region }}:123456789012:environment/{{ env.application_name }}/{{ env.environment_name }} + {{ env.environment_arn }} false 2019-08-30T09:35:10.913Z false @@ -173,7 +220,7 @@ EB_DESCRIBE_ENVIRONMENTS = """ 2019-08-22T07:02:47.332Z {{ env.environment_id }} 1 - arn:aws:elasticbeanstalk:{{ region }}::platform/{{ env.platform_arn }} + {{ env.platform_arn }} WebServer Standard @@ -1347,3 +1394,32 @@ EB_LIST_AVAILABLE_SOLUTION_STACKS = """ """ + + +EB_UPDATE_TAGS_FOR_RESOURCE = """ + + + f355d788-e67e-440f-b915-99e35254ffee + + +""" + + +EB_LIST_TAGS_FOR_RESOURCE = """ + + + + {% for key, value in tags.items() %} + + {{ key }} + {{ value }} + + {% endfor %} + + {{ arn }} + + + 178e410f-3b57-456f-a64c-a3b6a16da9ab + + +""" diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index aafe524fd..2b5be4490 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -72,6 +72,78 @@ def test_describe_environments(): envs[0]['EnvironmentName'].should.equal('myenv') +def tags_dict_to_list(tag_dict): + tag_list = [] + for key, value in tag_dict.items(): + tag_list.append({'Key': key, 'Value': value}) + return tag_list + + +def tags_list_to_dict(tag_list): + tag_dict = {} + for tag in tag_list: + tag_dict[tag['Key']] = tag['Value'] + return tag_dict + + +@mock_eb +def test_create_environment_tags(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + env_tags = {'initial key': 'initial value'} + env = conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + Tags=tags_dict_to_list(env_tags), + ) + + tags = conn.list_tags_for_resource( + ResourceArn=env['EnvironmentArn'], + ) + tags['ResourceArn'].should.equal(env['EnvironmentArn']) + tags_list_to_dict(tags['ResourceTags']).should.equal(env_tags) + + +@mock_eb +def test_update_tags(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + env_tags = { + 'initial key': 'initial value', + 'to remove': 'delete me', + 'to update': 'original', + } + env = conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + Tags=tags_dict_to_list(env_tags), + ) + + extra_env_tags = { + 'to update': 'new', + 'extra key': 'extra value', + } + conn.update_tags_for_resource( + ResourceArn=env['EnvironmentArn'], + TagsToAdd=tags_dict_to_list(extra_env_tags), + TagsToRemove=['to remove'], + ) + + total_env_tags = env_tags.copy() + total_env_tags.update(extra_env_tags) + del total_env_tags['to remove'] + + tags = conn.list_tags_for_resource( + ResourceArn=env['EnvironmentArn'], + ) + tags['ResourceArn'].should.equal(env['EnvironmentArn']) + tags_list_to_dict(tags['ResourceTags']).should.equal(total_env_tags) + + @mock_eb def test_list_available_solution_stacks(): conn = boto3.client('elasticbeanstalk', region_name='us-east-1') From 8f51bd6116b7194d2ef553064a66ff4bb3af734a Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Thu, 5 Sep 2019 11:38:19 +0200 Subject: [PATCH 07/51] EB: pass through SolutionStackName --- moto/eb/models.py | 8 ++++---- moto/eb/responses.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/eb/models.py b/moto/eb/models.py index c3c2aa20c..fa7345f0d 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -11,10 +11,12 @@ class FakeEnvironment(BaseModel): self, application, environment_name, + solution_stack_name, tags, ): self.application = weakref.proxy(application) # weakref to break circular dependencies self.environment_name = environment_name + self.solution_stack_name = solution_stack_name self.tags = tags @property @@ -35,10 +37,6 @@ class FakeEnvironment(BaseModel): def platform_arn(self): return 'TODO' # TODO - @property - def solution_stack_name(self): - return 'TODO' # TODO - @property def region(self): return self.application.region @@ -53,6 +51,7 @@ class FakeApplication(BaseModel): def create_environment( self, environment_name, + solution_stack_name, tags, ): if environment_name in self.environments: @@ -61,6 +60,7 @@ class FakeApplication(BaseModel): env = FakeEnvironment( application=self, environment_name=environment_name, + solution_stack_name=solution_stack_name, tags=tags, ) self.environments[environment_name] = env diff --git a/moto/eb/responses.py b/moto/eb/responses.py index fbace1938..c93efc3a1 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -31,7 +31,6 @@ class EBResponse(BaseResponse): def create_environment(self): application_name = self._get_param('ApplicationName') - environment_name = self._get_param('EnvironmentName') try: app = self.backend.applications[application_name] except KeyError: @@ -41,7 +40,8 @@ class EBResponse(BaseResponse): tags = tags_from_query_string(self.querystring, prefix="Tags.member") env = app.create_environment( - environment_name=environment_name, + environment_name=self._get_param('EnvironmentName'), + solution_stack_name=self._get_param('SolutionStackName'), tags=tags, ) From 7fae0d52ad6220998ef07dca7cf7de79680c2c80 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Thu, 5 Sep 2019 14:17:55 +0200 Subject: [PATCH 08/51] Fix linting --- moto/eb/models.py | 12 ++++++------ moto/eb/responses.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/moto/eb/models.py b/moto/eb/models.py index fa7345f0d..4490bbd0c 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -26,12 +26,12 @@ class FakeEnvironment(BaseModel): @property def environment_arn(self): return 'arn:aws:elasticbeanstalk:{region}:{account_id}:' \ - 'environment/{application_name}/{environment_name}'.format( - region=self.region, - account_id='123456789012', - application_name=self.application_name, - environment_name=self.environment_name, - ) + 'environment/{application_name}/{environment_name}'.format( + region=self.region, + account_id='123456789012', + application_name=self.application_name, + environment_name=self.environment_name, + ) @property def platform_arn(self): diff --git a/moto/eb/responses.py b/moto/eb/responses.py index c93efc3a1..905780c44 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -1,6 +1,6 @@ from moto.core.responses import BaseResponse from moto.core.utils import tags_from_query_string -from .models import eb_backends, EBBackend +from .models import eb_backends from .exceptions import InvalidParameterValueError, ResourceNotFoundException From b1da99aedaee0d7db7c8d68dfd477440705648e7 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 12:29:04 +0000 Subject: [PATCH 09/51] #2797 - DynamoDB - Allow case insensitive AND in KeyConditionExpression --- moto/dynamodb2/responses.py | 6 ++++-- tests/test_dynamodb2/test_dynamodb.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index c72ded2c3..c13078a72 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -459,8 +459,10 @@ class DynamoHandler(BaseResponse): for k, v in six.iteritems(self.body.get("ExpressionAttributeNames", {})) ) - if " AND " in key_condition_expression: - expressions = key_condition_expression.split(" AND ", 1) + if " and " in key_condition_expression.lower(): + expressions = re.split( + " AND ", key_condition_expression, maxsplit=1, flags=re.IGNORECASE + ) index_hash_key = [key for key in index if key["KeyType"] == "HASH"][0] hash_key_var = reverse_attribute_lookup.get( diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..191f19c36 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1408,6 +1408,13 @@ def test_filter_expression(): filter_expr.expr(row1).should.be(True) filter_expr.expr(row2).should.be(False) + # lowercase AND test + filter_expr = moto.dynamodb2.comparisons.get_filter_expression( + "Id > :v0 and Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + # OR test filter_expr = moto.dynamodb2.comparisons.get_filter_expression( "Id = :v0 OR Id=:v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "8"}} @@ -2719,7 +2726,7 @@ def test_query_gsi_with_range_key(): res = dynamodb.query( TableName="test", IndexName="test_gsi", - KeyConditionExpression="gsi_hash_key = :gsi_hash_key AND gsi_range_key = :gsi_range_key", + KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key", ExpressionAttributeValues={ ":gsi_hash_key": {"S": "key1"}, ":gsi_range_key": {"S": "range1"}, From 5b596c8a78ffc0c5a6ee1b9b28629c3f04bd5396 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 15:17:55 +0000 Subject: [PATCH 10/51] #2699 - EC2 - Add Volumes using CloudFormation --- moto/ec2/models.py | 14 ++++++++++- tests/test_ec2/test_instances.py | 40 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index be39bab28..1b363a193 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -556,6 +556,10 @@ class Instance(TaggedEC2Resource, BotoInstance): # worst case we'll get IP address exaustion... rarely pass + def add_block_device(self, size, device_path): + volume = self.ec2_backend.create_volume(size, self.region_name) + self.ec2_backend.attach_volume(volume.id, self.id, device_path) + def setup_defaults(self): # Default have an instance with root volume should you not wish to # override with attach volume cmd. @@ -620,6 +624,7 @@ class Instance(TaggedEC2Resource, BotoInstance): subnet_id=properties.get("SubnetId"), key_name=properties.get("KeyName"), private_ip=properties.get("PrivateIpAddress"), + block_device_mappings=properties.get("BlockDeviceMappings", {}), ) instance = reservation.instances[0] for tag in properties.get("Tags", []): @@ -872,7 +877,14 @@ class InstanceBackend(object): ) new_reservation.instances.append(new_instance) new_instance.add_tags(instance_tags) - new_instance.setup_defaults() + if "block_device_mappings" in kwargs: + for block_device in kwargs["block_device_mappings"]: + new_instance.add_block_device( + block_device["Ebs"]["VolumeSize"], block_device["DeviceName"] + ) + else: + new_instance.setup_defaults() + return new_reservation def start_instances(self, instance_ids): diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..4d1cbb28d 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -9,6 +9,7 @@ from nose.tools import assert_raises import base64 import datetime import ipaddress +import json import six import boto @@ -18,7 +19,7 @@ from boto.exception import EC2ResponseError, EC2ResponseError from freezegun import freeze_time import sure # noqa -from moto import mock_ec2_deprecated, mock_ec2 +from moto import mock_ec2_deprecated, mock_ec2, mock_cloudformation from tests.helpers import requires_boto_gte @@ -1399,3 +1400,40 @@ def test_describe_instance_attribute(): invalid_instance_attribute=invalid_instance_attribute ) ex.exception.response["Error"]["Message"].should.equal(message) + + +@mock_ec2 +@mock_cloudformation +def test_volume_size_through_cloudformation(): + ec2 = boto3.client("ec2", region_name="us-east-1") + cf = boto3.client("cloudformation", region_name="us-east-1") + + volume_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "testInstance": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-d3adb33f", + "KeyName": "dummy", + "InstanceType": "t2.micro", + "BlockDeviceMappings": [ + {"DeviceName": "/dev/sda2", "Ebs": {"VolumeSize": "50"}} + ], + "Tags": [ + {"Key": "foo", "Value": "bar"}, + {"Key": "blah", "Value": "baz"}, + ], + }, + } + }, + } + template_json = json.dumps(volume_template) + cf.create_stack(StackName="test_stack", TemplateBody=template_json) + instances = ec2.describe_instances() + volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][ + "Ebs" + ] + + volumes = ec2.describe_volumes(VolumeIds=[volume["VolumeId"]]) + volumes["Volumes"][0]["Size"].should.equal(50) From da1a2118bb12ca3279952d88154ed221f9f0fd1e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 16:17:21 +0000 Subject: [PATCH 11/51] EC2 - Verify default block exists before tearing down --- moto/ec2/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 1b363a193..c391c88f3 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -567,9 +567,10 @@ class Instance(TaggedEC2Resource, BotoInstance): self.ec2_backend.attach_volume(volume.id, self.id, "/dev/sda1") def teardown_defaults(self): - volume_id = self.block_device_mapping["/dev/sda1"].volume_id - self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") - self.ec2_backend.delete_volume(volume_id) + if "/dev/sda1" in self.block_device_mapping: + volume_id = self.block_device_mapping["/dev/sda1"].volume_id + self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") + self.ec2_backend.delete_volume(volume_id) @property def get_block_device_mapping(self): From e82e1e3f397cd610d3ed0316c37325cdfe55926b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 21 Mar 2020 12:20:09 +0000 Subject: [PATCH 12/51] DynamoDB - Add 1MB item size check --- moto/dynamodb2/models.py | 11 ++++++++ tests/test_dynamodb2/test_dynamodb.py | 38 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 54dccd56d..a35eded61 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -285,6 +285,9 @@ class Item(BaseModel): def __repr__(self): return "Item: {0}".format(self.to_json()) + def size(self): + return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + def to_json(self): attributes = {} for attribute_key, attribute in self.attrs.items(): @@ -1123,6 +1126,14 @@ class Table(BaseModel): break last_evaluated_key = None + size_limit = 1000000 # DynamoDB has a 1MB size limit + item_size = sum([res.size() for res in results]) + if item_size > size_limit: + item_size = idx = 0 + while item_size + results[idx].size() < size_limit: + item_size += results[idx].size() + idx += 1 + limit = min(limit, idx) if limit else idx if limit and len(results) > limit: results = results[:limit] last_evaluated_key = {self.hash_key_attr: results[-1].hash_key} diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..daae79232 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4132,3 +4132,41 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_dynamodb_max_1mb_limit(): + ddb = boto3.resource("dynamodb", region_name="eu-west-1") + + table_name = "populated-mock-table" + table = ddb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "partition_key", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "SORT"}, + ], + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + # Populate the table + items = [ + { + "partition_key": "partition_key_val", # size=30 + "sort_key": "sort_key_value____" + str(i), # size=30 + } + for i in range(10000, 29999) + ] + with table.batch_writer() as batch: + for item in items: + batch.put_item(Item=item) + + response = table.query( + KeyConditionExpression=Key("partition_key").eq("partition_key_val") + ) + # We shouldn't get everything back - the total result set is well over 1MB + assert response["Count"] < len(items) + response["LastEvaluatedKey"].shouldnt.be(None) From c3865532f9ca6237261591277bde1afbe910099e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 23 Mar 2020 15:53:39 +0000 Subject: [PATCH 13/51] #2711 - Register default S3 metrics in CloudWatch --- moto/cloudwatch/models.py | 50 ++++++++++++++++--- moto/cloudwatch/responses.py | 5 +- moto/s3/models.py | 33 ++++++++++++ moto/s3/utils.py | 6 +++ tests/test_cloudwatch/test_cloudwatch.py | 40 +++++++++++++-- .../test_cloudwatch/test_cloudwatch_boto3.py | 16 ++++-- 6 files changed, 133 insertions(+), 17 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..523eb10f3 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -22,6 +22,14 @@ class Dimension(object): self.name = name self.value = value + def __eq__(self, item): + if isinstance(item, Dimension): + return self.name == item.name and self.value == item.value + return False + + def __ne__(self, item): # Only needed on Py2; Py3 defines it implicitly + return self != item + def daterange(start, stop, step=timedelta(days=1), inclusive=False): """ @@ -124,6 +132,17 @@ class MetricDatum(BaseModel): Dimension(dimension["Name"], dimension["Value"]) for dimension in dimensions ] + def filter(self, namespace, name, dimensions): + if namespace and namespace != self.namespace: + return False + if name and name != self.name: + return False + if dimensions and any( + Dimension(d["Name"], d["Value"]) not in self.dimensions for d in dimensions + ): + return False + return True + class Dashboard(BaseModel): def __init__(self, name, body): @@ -202,6 +221,15 @@ class CloudWatchBackend(BaseBackend): self.metric_data = [] self.paged_metric_data = {} + @property + # Retrieve a list of all OOTB metrics that are provided by metrics providers + # Computed on the fly + def aws_metric_data(self): + md = [] + for name, service in metric_providers.items(): + md.extend(service.get_cloudwatch_metrics()) + return md + def put_metric_alarm( self, name, @@ -334,7 +362,7 @@ class CloudWatchBackend(BaseBackend): return data def get_all_metrics(self): - return self.metric_data + return self.metric_data + self.aws_metric_data def put_dashboard(self, name, body): self.dashboards[name] = Dashboard(name, body) @@ -386,7 +414,7 @@ class CloudWatchBackend(BaseBackend): self.alarms[alarm_name].update_state(reason, reason_data, state_value) - def list_metrics(self, next_token, namespace, metric_name): + def list_metrics(self, next_token, namespace, metric_name, dimensions): if next_token: if next_token not in self.paged_metric_data: raise RESTError( @@ -397,15 +425,16 @@ class CloudWatchBackend(BaseBackend): del self.paged_metric_data[next_token] # Cant reuse same token twice return self._get_paginated(metrics) else: - metrics = self.get_filtered_metrics(metric_name, namespace) + metrics = self.get_filtered_metrics(metric_name, namespace, dimensions) return self._get_paginated(metrics) - def get_filtered_metrics(self, metric_name, namespace): + def get_filtered_metrics(self, metric_name, namespace, dimensions): metrics = self.get_all_metrics() - if namespace: - metrics = [md for md in metrics if md.namespace == namespace] - if metric_name: - metrics = [md for md in metrics if md.name == metric_name] + metrics = [ + md + for md in metrics + if md.filter(namespace=namespace, name=metric_name, dimensions=dimensions) + ] return metrics def _get_paginated(self, metrics): @@ -443,3 +472,8 @@ for region in Session().get_available_regions( cloudwatch_backends[region] = CloudWatchBackend() for region in Session().get_available_regions("cloudwatch", partition_name="aws-cn"): cloudwatch_backends[region] = CloudWatchBackend() + +# List of services that provide OOTB CW metrics +# See the S3Backend constructor for an example +# TODO: We might have to separate this out per region for non-global services +metric_providers = {} diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7993c9f06..dccc30216 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -124,9 +124,10 @@ class CloudWatchResponse(BaseResponse): def list_metrics(self): namespace = self._get_param("Namespace") metric_name = self._get_param("MetricName") + dimensions = self._get_multi_param("Dimensions.member") next_token = self._get_param("NextToken") next_token, metrics = self.cloudwatch_backend.list_metrics( - next_token, namespace, metric_name + next_token, namespace, metric_name, dimensions ) template = self.response_template(LIST_METRICS_TEMPLATE) return template.render(metrics=metrics, next_token=next_token) @@ -342,7 +343,7 @@ LIST_METRICS_TEMPLATE = """ Date: Tue, 24 Mar 2020 09:24:38 +0000 Subject: [PATCH 14/51] #2810 - EC2 - Explicitly set ebs_optimized to False if not specified --- moto/ec2/responses/instances.py | 2 +- tests/test_ec2/test_instances.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 29c346f82..ba15be8d0 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -52,7 +52,7 @@ class InstanceResponse(BaseResponse): private_ip = self._get_param("PrivateIpAddress") associate_public_ip = self._get_param("AssociatePublicIpAddress") key_name = self._get_param("KeyName") - ebs_optimized = self._get_param("EbsOptimized") + ebs_optimized = self._get_param("EbsOptimized") or False instance_initiated_shutdown_behavior = self._get_param( "InstanceInitiatedShutdownBehavior" ) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..d40aca000 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1319,6 +1319,12 @@ def test_create_instance_ebs_optimized(): instance.load() instance.ebs_optimized.should.be(False) + instance = ec2_resource.create_instances( + ImageId="ami-12345678", MaxCount=1, MinCount=1, + )[0] + instance.load() + instance.ebs_optimized.should.be(False) + @mock_ec2 def test_run_multiple_instances_in_same_command(): From 04f488da62462a926d4ee61ad303583bdaf836a8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 24 Mar 2020 10:22:08 +0000 Subject: [PATCH 15/51] #2388 - CloudFormation - CreateChangeSet does not create resources, as per spec --- moto/cloudformation/models.py | 14 +++++++++++--- moto/cloudformation/parsing.py | 4 +++- .../test_cloudformation_stack_crud_boto3.py | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 8136e353d..281ab5e19 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -239,8 +239,11 @@ class FakeStack(BaseModel): self.cross_stack_resources = cross_stack_resources or {} self.resource_map = self._create_resource_map() self.output_map = self._create_output_map() - self._add_stack_event("CREATE_COMPLETE") - self.status = "CREATE_COMPLETE" + if create_change_set: + self.status = "REVIEW_IN_PROGRESS" + else: + self.create_resources() + self._add_stack_event("CREATE_COMPLETE") self.creation_time = datetime.utcnow() def _create_resource_map(self): @@ -253,7 +256,7 @@ class FakeStack(BaseModel): self.template_dict, self.cross_stack_resources, ) - resource_map.create() + resource_map.load() return resource_map def _create_output_map(self): @@ -326,6 +329,10 @@ class FakeStack(BaseModel): def exports(self): return self.output_map.exports + def create_resources(self): + self.resource_map.create() + self.status = "CREATE_COMPLETE" + def update(self, template, role_arn=None, parameters=None, tags=None): self._add_stack_event( "UPDATE_IN_PROGRESS", resource_status_reason="User Initiated" @@ -640,6 +647,7 @@ class CloudFormationBackend(BaseBackend): else: stack._add_stack_event("UPDATE_IN_PROGRESS") stack._add_stack_event("UPDATE_COMPLETE") + stack.create_resources() return True def describe_stacks(self, name_or_stack_id): diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 79276c8fc..6789c0007 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -529,14 +529,16 @@ class ResourceMap(collections_abc.Mapping): for condition_name in self.lazy_condition_map: self.lazy_condition_map[condition_name] - def create(self): + def load(self): self.load_mapping() self.transform_mapping() self.load_parameters() self.load_conditions() + def create(self): # Since this is a lazy map, to create every object we just need to # iterate through self. + # Assumes that self.load() has been called before self.tags.update( { "aws:cloudformation:stack-name": self.get("AWS::StackName"), diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 5444c2278..4df1ff5d2 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -835,8 +835,10 @@ def test_describe_change_set(): ) stack = cf_conn.describe_change_set(ChangeSetName="NewChangeSet") + stack["ChangeSetName"].should.equal("NewChangeSet") stack["StackName"].should.equal("NewStack") + stack["Status"].should.equal("REVIEW_IN_PROGRESS") cf_conn.create_change_set( StackName="NewStack", @@ -851,15 +853,30 @@ def test_describe_change_set(): @mock_cloudformation +@mock_ec2 def test_execute_change_set_w_arn(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") + ec2 = boto3.client("ec2", region_name="us-east-1") + # Verify no instances exist at the moment + ec2.describe_instances()["Reservations"].should.have.length_of(0) + # Create a Change set, and verify no resources have been created yet change_set = cf_conn.create_change_set( StackName="NewStack", TemplateBody=dummy_template_json, ChangeSetName="NewChangeSet", ChangeSetType="CREATE", ) + ec2.describe_instances()["Reservations"].should.have.length_of(0) + cf_conn.describe_change_set(ChangeSetName="NewChangeSet")["Status"].should.equal( + "REVIEW_IN_PROGRESS" + ) + # Execute change set cf_conn.execute_change_set(ChangeSetName=change_set["Id"]) + # Verify that the status has changed, and the appropriate resources have been created + cf_conn.describe_change_set(ChangeSetName="NewChangeSet")["Status"].should.equal( + "CREATE_COMPLETE" + ) + ec2.describe_instances()["Reservations"].should.have.length_of(1) @mock_cloudformation From fb0de99e81dc0af7644faa66113c1aec60b589ea Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 28 Mar 2020 13:41:17 +0000 Subject: [PATCH 16/51] #2239 - Initial implementation of CW.get_metric_data --- moto/cloudwatch/models.py | 37 +++ moto/cloudwatch/responses.py | 41 +++ .../test_cloudwatch/test_cloudwatch_boto3.py | 259 ++++++++++++++++++ 3 files changed, 337 insertions(+) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..bddb94a12 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -295,6 +295,43 @@ class CloudWatchBackend(BaseBackend): ) ) + def get_metric_data(self, queries, start_time, end_time): + period_data = [ + md for md in self.metric_data if start_time <= md.timestamp <= end_time + ] + results = [] + for query in queries: + query_ns = query["metric_stat._metric._namespace"] + query_name = query["metric_stat._metric._metric_name"] + query_data = [ + md + for md in period_data + if md.namespace == query_ns and md.name == query_name + ] + metric_values = [m.value for m in query_data] + result_vals = [] + stat = query["metric_stat._stat"] + if len(metric_values) > 0: + if stat == "Average": + result_vals.append(sum(metric_values) / len(metric_values)) + elif stat == "Minimum": + result_vals.append(min(metric_values)) + elif stat == "Maximum": + result_vals.append(max(metric_values)) + elif stat == "Sum": + result_vals.append(sum(metric_values)) + + label = query["metric_stat._metric._metric_name"] + " " + stat + results.append( + { + "id": query["id"], + "label": label, + "vals": result_vals, + "timestamps": [datetime.now() for _ in result_vals], + } + ) + return results + def get_metric_statistics( self, namespace, metric_name, start_time, end_time, period, stats ): diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7993c9f06..7e75a38f0 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -92,6 +92,18 @@ class CloudWatchResponse(BaseResponse): template = self.response_template(PUT_METRIC_DATA_TEMPLATE) return template.render() + @amzn_request_id + def get_metric_data(self): + start = dtparse(self._get_param("StartTime")) + end = dtparse(self._get_param("EndTime")) + queries = self._get_list_prefix("MetricDataQueries.member") + results = self.cloudwatch_backend.get_metric_data( + start_time=start, end_time=end, queries=queries + ) + + template = self.response_template(GET_METRIC_DATA_TEMPLATE) + return template.render(results=results) + @amzn_request_id def get_metric_statistics(self): namespace = self._get_param("Namespace") @@ -285,6 +297,35 @@ PUT_METRIC_DATA_TEMPLATE = """ + + + {{ request_id }} + + + + + {% for result in results %} + + {{ result.id }} + + Complete + + {% for val in result.timestamps %} + {{ val }} + {% endfor %} + + + {% for val in result.vals %} + {{ val }} + {% endfor %} + + + {% endfor %} + + +""" + GET_METRIC_STATISTICS_TEMPLATE = """ diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 7fe144052..2b1caff02 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -3,6 +3,7 @@ import boto3 from botocore.exceptions import ClientError from datetime import datetime, timedelta +from freezegun import freeze_time from nose.tools import assert_raises from uuid import uuid4 import pytz @@ -211,6 +212,35 @@ def test_get_metric_statistics(): datapoint["Sum"].should.equal(1.5) +@mock_cloudwatch +@freeze_time("2020-02-10 18:44:05") +def test_custom_timestamp(): + utc_now = datetime.now(tz=pytz.utc) + time = "2020-02-10T18:44:09Z" + cw = boto3.client("cloudwatch", "eu-west-1") + + cw.put_metric_data( + Namespace="tester", + MetricData=[dict(MetricName="metric1", Value=1.5, Timestamp=time)], + ) + + cw.put_metric_data( + Namespace="tester", + MetricData=[ + dict(MetricName="metric2", Value=1.5, Timestamp=datetime(2020, 2, 10)) + ], + ) + + stats = cw.get_metric_statistics( + Namespace="tester", + MetricName="metric", + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + Period=60, + Statistics=["SampleCount", "Sum"], + ) + + @mock_cloudwatch def test_list_metrics(): cloudwatch = boto3.client("cloudwatch", "eu-west-1") @@ -292,3 +322,232 @@ def create_metrics(cloudwatch, namespace, metrics=5, data_points=5): Namespace=namespace, MetricData=[{"MetricName": metric_name, "Value": j, "Unit": "Seconds"}], ) + + +@mock_cloudwatch +def test_get_metric_data_within_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + values = [0, 2, 4, 3.5, 7, 100] + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + {"MetricName": "metric1", "Value": val, "Unit": "Seconds"} for val in values + ], + ) + # get_metric_data + stats = ["Average", "Sum", "Minimum", "Maximum"] + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result_" + stat, + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": stat, + }, + } + for stat in stats + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Average/Min/Max/Sum is returned as expected + avg = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Average" + ][0] + avg["Label"].should.equal("metric1 Average") + avg["StatusCode"].should.equal("Complete") + [int(val) for val in avg["Values"]].should.equal([19]) + + sum_ = [res for res in response["MetricDataResults"] if res["Id"] == "result_Sum"][ + 0 + ] + sum_["Label"].should.equal("metric1 Sum") + sum_["StatusCode"].should.equal("Complete") + [val for val in sum_["Values"]].should.equal([sum(values)]) + + min_ = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Minimum" + ][0] + min_["Label"].should.equal("metric1 Minimum") + min_["StatusCode"].should.equal("Complete") + [int(val) for val in min_["Values"]].should.equal([0]) + + max_ = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Maximum" + ][0] + max_["Label"].should.equal("metric1 Maximum") + max_["StatusCode"].should.equal("Complete") + [int(val) for val in max_["Values"]].should.equal([100]) + + +@mock_cloudwatch +def test_get_metric_data_partially_within_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + yesterday = utc_now - timedelta(days=1) + last_week = utc_now - timedelta(days=7) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + values = [0, 2, 4, 3.5, 7, 100] + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 10, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 20, + "Unit": "Seconds", + "Timestamp": yesterday, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": last_week, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result", + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + } + ], + StartTime=yesterday - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Last week's data is not returned + len(response["MetricDataResults"]).should.equal(1) + sum_ = response["MetricDataResults"][0] + sum_["Label"].should.equal("metric1 Sum") + sum_["StatusCode"].should.equal("Complete") + sum_["Values"].should.equal([30.0]) + + +@mock_cloudwatch +def test_get_metric_data_outside_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + last_week = utc_now - timedelta(days=7) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": last_week, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result", + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + } + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Last week's data is not returned + len(response["MetricDataResults"]).should.equal(1) + response["MetricDataResults"][0]["Id"].should.equal("result") + response["MetricDataResults"][0]["StatusCode"].should.equal("Complete") + response["MetricDataResults"][0]["Values"].should.equal([]) + + +@mock_cloudwatch +def test_get_metric_data_for_multiple_metrics(): + utc_now = datetime.now(tz=pytz.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace = "my_namespace/" + # put metric data + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric2", + "Value": 25, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result1", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + }, + { + "Id": "result2", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric2"}, + "Period": 60, + "Stat": "Sum", + }, + }, + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + len(response["MetricDataResults"]).should.equal(2) + + res1 = [res for res in response["MetricDataResults"] if res["Id"] == "result1"][0] + res1["Values"].should.equal([50.0]) + + res2 = [res for res in response["MetricDataResults"] if res["Id"] == "result2"][0] + res2["Values"].should.equal([25.0]) From 888e0c31a0fd94c8854f93ec88e3b45ebfeeb98b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 13:42:00 +0100 Subject: [PATCH 17/51] Linting --- moto/__init__.py | 15 +++--- moto/core/utils.py | 19 +++---- moto/eb/exceptions.py | 6 ++- moto/eb/models.py | 35 ++++++------ moto/eb/responses.py | 47 +++++++--------- moto/eb/urls.py | 2 +- tests/test_eb/test_eb.py | 112 ++++++++++++++++----------------------- 7 files changed, 96 insertions(+), 140 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index c2caa8df0..9b59f18eb 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -1,11 +1,4 @@ from __future__ import unicode_literals -import logging -# logging.getLogger('boto').setLevel(logging.CRITICAL) - - -__title__ = "moto" -__version__ = "1.3.15.dev" - from .acm import mock_acm # noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # noqa @@ -28,7 +21,7 @@ from .datasync import mock_datasync # noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # noqa from .dynamodbstreams import mock_dynamodbstreams # noqa -from .eb import mock_eb # flake8: noqa +from .eb import mock_eb # noqa from .ec2 import mock_ec2, mock_ec2_deprecated # noqa from .ec2_instance_connect import mock_ec2_instance_connect # noqa from .ecr import mock_ecr, mock_ecr_deprecated # noqa @@ -65,6 +58,12 @@ from .sts import mock_sts, mock_sts_deprecated # noqa from .swf import mock_swf, mock_swf_deprecated # noqa from .xray import XRaySegment, mock_xray, mock_xray_client # noqa +# import logging +# logging.getLogger('boto').setLevel(logging.CRITICAL) + +__title__ = "moto" +__version__ = "1.3.15.dev" + try: # Need to monkey-patch botocore requests back to underlying urllib3 classes diff --git a/moto/core/utils.py b/moto/core/utils.py index 59079784a..dce9f675c 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -331,10 +331,7 @@ def py2_strip_unicode_keys(blob): def tags_from_query_string( - querystring_dict, - prefix="Tag", - key_suffix="Key", - value_suffix="Value" + querystring_dict, prefix="Tag", key_suffix="Key", value_suffix="Value" ): response_values = {} for key, value in querystring_dict.items(): @@ -342,18 +339,14 @@ def tags_from_query_string( tag_index = key.replace(prefix + ".", "").replace("." + key_suffix, "") tag_key = querystring_dict.get( "{prefix}.{index}.{key_suffix}".format( - prefix=prefix, - index=tag_index, - key_suffix=key_suffix, - ))[0] + prefix=prefix, index=tag_index, key_suffix=key_suffix, + ) + )[0] tag_value_key = "{prefix}.{index}.{value_suffix}".format( - prefix=prefix, - index=tag_index, - value_suffix=value_suffix, + prefix=prefix, index=tag_index, value_suffix=value_suffix, ) if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[ - 0] + response_values[tag_key] = querystring_dict.get(tag_value_key)[0] else: response_values[tag_key] = None return response_values diff --git a/moto/eb/exceptions.py b/moto/eb/exceptions.py index bf3a89618..f1e27c564 100644 --- a/moto/eb/exceptions.py +++ b/moto/eb/exceptions.py @@ -4,10 +4,12 @@ from moto.core.exceptions import RESTError class InvalidParameterValueError(RESTError): def __init__(self, message): super(InvalidParameterValueError, self).__init__( - "InvalidParameterValue", message) + "InvalidParameterValue", message + ) class ResourceNotFoundException(RESTError): def __init__(self, message): super(ResourceNotFoundException, self).__init__( - "ResourceNotFoundException", message) + "ResourceNotFoundException", message + ) diff --git a/moto/eb/models.py b/moto/eb/models.py index 4490bbd0c..71873f30c 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -8,13 +8,11 @@ from .exceptions import InvalidParameterValueError class FakeEnvironment(BaseModel): def __init__( - self, - application, - environment_name, - solution_stack_name, - tags, + self, application, environment_name, solution_stack_name, tags, ): - self.application = weakref.proxy(application) # weakref to break circular dependencies + self.application = weakref.proxy( + application + ) # weakref to break circular dependencies self.environment_name = environment_name self.solution_stack_name = solution_stack_name self.tags = tags @@ -25,17 +23,19 @@ class FakeEnvironment(BaseModel): @property def environment_arn(self): - return 'arn:aws:elasticbeanstalk:{region}:{account_id}:' \ - 'environment/{application_name}/{environment_name}'.format( + return ( + "arn:aws:elasticbeanstalk:{region}:{account_id}:" + "environment/{application_name}/{environment_name}".format( region=self.region, - account_id='123456789012', + account_id="123456789012", application_name=self.application_name, environment_name=self.environment_name, ) + ) @property def platform_arn(self): - return 'TODO' # TODO + return "TODO" # TODO @property def region(self): @@ -49,10 +49,7 @@ class FakeApplication(BaseModel): self.environments = dict() def create_environment( - self, - environment_name, - solution_stack_name, - tags, + self, environment_name, solution_stack_name, tags, ): if environment_name in self.environments: raise InvalidParameterValueError @@ -89,13 +86,11 @@ class EBBackend(BaseBackend): raise InvalidParameterValueError( "Application {} already exists.".format(application_name) ) - new_app = FakeApplication( - backend=self, - application_name=application_name, - ) + new_app = FakeApplication(backend=self, application_name=application_name,) self.applications[application_name] = new_app return new_app -eb_backends = dict((region.name, EBBackend(region.name)) - for region in boto.beanstalk.regions()) +eb_backends = dict( + (region.name, EBBackend(region.name)) for region in boto.beanstalk.regions() +) diff --git a/moto/eb/responses.py b/moto/eb/responses.py index 905780c44..6178e4a7f 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -14,42 +14,34 @@ class EBResponse(BaseResponse): def create_application(self): app = self.backend.create_application( - application_name=self._get_param('ApplicationName'), + application_name=self._get_param("ApplicationName"), ) template = self.response_template(EB_CREATE_APPLICATION) - return template.render( - region_name=self.backend.region, - application=app, - ) + return template.render(region_name=self.backend.region, application=app,) def describe_applications(self): template = self.response_template(EB_DESCRIBE_APPLICATIONS) - return template.render( - applications=self.backend.applications.values(), - ) + return template.render(applications=self.backend.applications.values(),) def create_environment(self): - application_name = self._get_param('ApplicationName') + application_name = self._get_param("ApplicationName") try: app = self.backend.applications[application_name] except KeyError: raise InvalidParameterValueError( - "No Application named \'{}\' found.".format(application_name) + "No Application named '{}' found.".format(application_name) ) tags = tags_from_query_string(self.querystring, prefix="Tags.member") env = app.create_environment( - environment_name=self._get_param('EnvironmentName'), - solution_stack_name=self._get_param('SolutionStackName'), + environment_name=self._get_param("EnvironmentName"), + solution_stack_name=self._get_param("SolutionStackName"), tags=tags, ) template = self.response_template(EB_CREATE_ENVIRONMENT) - return template.render( - environment=env, - region=self.backend.region, - ) + return template.render(environment=env, region=self.backend.region,) def describe_environments(self): envs = [] @@ -59,9 +51,7 @@ class EBResponse(BaseResponse): envs.append(env) template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) - return template.render( - environments=envs, - ) + return template.render(environments=envs,) @staticmethod def list_available_solution_stacks(): @@ -75,39 +65,38 @@ class EBResponse(BaseResponse): raise KeyError() def update_tags_for_resource(self): - resource_arn = self._get_param('ResourceArn') + resource_arn = self._get_param("ResourceArn") try: res = self._find_environment_by_arn(resource_arn) except KeyError: raise ResourceNotFoundException( - "Resource not found for ARN \'{}\'.".format(resource_arn) + "Resource not found for ARN '{}'.".format(resource_arn) ) - tags_to_add = tags_from_query_string(self.querystring, prefix="TagsToAdd.member") + tags_to_add = tags_from_query_string( + self.querystring, prefix="TagsToAdd.member" + ) for key, value in tags_to_add.items(): res.tags[key] = value - tags_to_remove = self._get_multi_param('TagsToRemove.member') + tags_to_remove = self._get_multi_param("TagsToRemove.member") for key in tags_to_remove: del res.tags[key] return EB_UPDATE_TAGS_FOR_RESOURCE def list_tags_for_resource(self): - resource_arn = self._get_param('ResourceArn') + resource_arn = self._get_param("ResourceArn") try: res = self._find_environment_by_arn(resource_arn) except KeyError: raise ResourceNotFoundException( - "Resource not found for ARN \'{}\'.".format(resource_arn) + "Resource not found for ARN '{}'.".format(resource_arn) ) tags = res.tags template = self.response_template(EB_LIST_TAGS_FOR_RESOURCE) - return template.render( - tags=tags, - arn=resource_arn, - ) + return template.render(tags=tags, arn=resource_arn,) EB_CREATE_APPLICATION = """ diff --git a/moto/eb/urls.py b/moto/eb/urls.py index 4cd4add13..2d57f7f9d 100644 --- a/moto/eb/urls.py +++ b/moto/eb/urls.py @@ -7,5 +7,5 @@ url_bases = [ ] url_paths = { - '{0}/$': EBResponse.dispatch, + "{0}/$": EBResponse.dispatch, } diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 2b5be4490..1064bf31a 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -8,114 +8,94 @@ from moto import mock_eb @mock_eb def test_create_application(): # Create Elastic Beanstalk Application - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - app = conn.create_application( - ApplicationName="myapp", - ) - app['Application']['ApplicationName'].should.equal("myapp") + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + app = conn.create_application(ApplicationName="myapp",) + app["Application"]["ApplicationName"].should.equal("myapp") @mock_eb def test_create_application_dup(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) + conn.create_application.when.called_with(ApplicationName="myapp",).should.throw( + ClientError ) - conn.create_application.when.called_with( - ApplicationName="myapp", - ).should.throw(ClientError) @mock_eb def test_describe_applications(): # Create Elastic Beanstalk Application - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) apps = conn.describe_applications() - len(apps['Applications']).should.equal(1) - apps['Applications'][0]['ApplicationName'].should.equal('myapp') + len(apps["Applications"]).should.equal(1) + apps["Applications"][0]["ApplicationName"].should.equal("myapp") @mock_eb def test_create_environment(): # Create Elastic Beanstalk Environment - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - app = conn.create_application( - ApplicationName="myapp", - ) - env = conn.create_environment( - ApplicationName="myapp", - EnvironmentName="myenv", - ) - env['EnvironmentName'].should.equal("myenv") + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + app = conn.create_application(ApplicationName="myapp",) + env = conn.create_environment(ApplicationName="myapp", EnvironmentName="myenv",) + env["EnvironmentName"].should.equal("myenv") @mock_eb def test_describe_environments(): # List Elastic Beanstalk Envs - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) conn.create_environment( - ApplicationName="myapp", - EnvironmentName="myenv", + ApplicationName="myapp", EnvironmentName="myenv", ) envs = conn.describe_environments() - envs = envs['Environments'] + envs = envs["Environments"] len(envs).should.equal(1) - envs[0]['ApplicationName'].should.equal('myapp') - envs[0]['EnvironmentName'].should.equal('myenv') + envs[0]["ApplicationName"].should.equal("myapp") + envs[0]["EnvironmentName"].should.equal("myenv") def tags_dict_to_list(tag_dict): tag_list = [] for key, value in tag_dict.items(): - tag_list.append({'Key': key, 'Value': value}) + tag_list.append({"Key": key, "Value": value}) return tag_list def tags_list_to_dict(tag_list): tag_dict = {} for tag in tag_list: - tag_dict[tag['Key']] = tag['Value'] + tag_dict[tag["Key"]] = tag["Value"] return tag_dict @mock_eb def test_create_environment_tags(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) - env_tags = {'initial key': 'initial value'} + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) + env_tags = {"initial key": "initial value"} env = conn.create_environment( ApplicationName="myapp", EnvironmentName="myenv", Tags=tags_dict_to_list(env_tags), ) - tags = conn.list_tags_for_resource( - ResourceArn=env['EnvironmentArn'], - ) - tags['ResourceArn'].should.equal(env['EnvironmentArn']) - tags_list_to_dict(tags['ResourceTags']).should.equal(env_tags) + tags = conn.list_tags_for_resource(ResourceArn=env["EnvironmentArn"],) + tags["ResourceArn"].should.equal(env["EnvironmentArn"]) + tags_list_to_dict(tags["ResourceTags"]).should.equal(env_tags) @mock_eb def test_update_tags(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) env_tags = { - 'initial key': 'initial value', - 'to remove': 'delete me', - 'to update': 'original', + "initial key": "initial value", + "to remove": "delete me", + "to update": "original", } env = conn.create_environment( ApplicationName="myapp", @@ -124,29 +104,27 @@ def test_update_tags(): ) extra_env_tags = { - 'to update': 'new', - 'extra key': 'extra value', + "to update": "new", + "extra key": "extra value", } conn.update_tags_for_resource( - ResourceArn=env['EnvironmentArn'], + ResourceArn=env["EnvironmentArn"], TagsToAdd=tags_dict_to_list(extra_env_tags), - TagsToRemove=['to remove'], + TagsToRemove=["to remove"], ) total_env_tags = env_tags.copy() total_env_tags.update(extra_env_tags) - del total_env_tags['to remove'] + del total_env_tags["to remove"] - tags = conn.list_tags_for_resource( - ResourceArn=env['EnvironmentArn'], - ) - tags['ResourceArn'].should.equal(env['EnvironmentArn']) - tags_list_to_dict(tags['ResourceTags']).should.equal(total_env_tags) + tags = conn.list_tags_for_resource(ResourceArn=env["EnvironmentArn"],) + tags["ResourceArn"].should.equal(env["EnvironmentArn"]) + tags_list_to_dict(tags["ResourceTags"]).should.equal(total_env_tags) @mock_eb def test_list_available_solution_stacks(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") stacks = conn.list_available_solution_stacks() - len(stacks['SolutionStacks']).should.be.greater_than(0) - len(stacks['SolutionStacks']).should.be.equal(len(stacks['SolutionStackDetails'])) + len(stacks["SolutionStacks"]).should.be.greater_than(0) + len(stacks["SolutionStacks"]).should.be.equal(len(stacks["SolutionStackDetails"])) From c32c17a13ed94b3867336e2a9e16c5890ad75973 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 13:49:19 +0100 Subject: [PATCH 18/51] Remove duplicated method --- moto/ec2/utils.py | 16 ---------------- moto/emr/responses.py | 5 +++-- moto/emr/utils.py | 16 ---------------- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 74fe3d27b..61d22d8b2 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -196,22 +196,6 @@ def split_route_id(route_id): return values[0], values[1] -def tags_from_query_string(querystring_dict): - prefix = "Tag" - suffix = "Key" - response_values = {} - for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] - tag_value_key = "Tag.{0}.Value".format(tag_index) - if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[0] - else: - response_values[tag_key] = None - return response_values - - def dhcp_configuration_from_querystring(querystring, option="DhcpConfiguration"): """ turn: diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 3708db0ed..d2b234ced 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -10,9 +10,10 @@ from six.moves.urllib.parse import urlparse from moto.core.responses import AWSServiceSpec from moto.core.responses import BaseResponse from moto.core.responses import xml_to_json_response +from moto.core.utils import tags_from_query_string from .exceptions import EmrError from .models import emr_backends -from .utils import steps_from_query_string, tags_from_query_string +from .utils import steps_from_query_string def generate_boto3_response(operation): @@ -91,7 +92,7 @@ class ElasticMapReduceResponse(BaseResponse): @generate_boto3_response("AddTags") def add_tags(self): cluster_id = self._get_param("ResourceId") - tags = tags_from_query_string(self.querystring) + tags = tags_from_query_string(self.querystring, prefix="Tags") self.backend.add_tags(cluster_id, tags) template = self.response_template(ADD_TAGS_TEMPLATE) return template.render() diff --git a/moto/emr/utils.py b/moto/emr/utils.py index 0f75995b8..fb33214c8 100644 --- a/moto/emr/utils.py +++ b/moto/emr/utils.py @@ -22,22 +22,6 @@ def random_instance_group_id(size=13): return "i-{0}".format(random_id()) -def tags_from_query_string(querystring_dict): - prefix = "Tags" - suffix = "Key" - response_values = {} - for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tags.{0}.Key".format(tag_index))[0] - tag_value_key = "Tags.{0}.Value".format(tag_index) - if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[0] - else: - response_values[tag_key] = None - return response_values - - def steps_from_query_string(querystring_dict): steps = [] for step in querystring_dict: From 7d524eaec9bb49f8d3e8e55a7f84c1876cd4e3d1 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 14:08:22 +0100 Subject: [PATCH 19/51] Elastic Beanstalk - Rename and Add Implementation Coverage --- IMPLEMENTATION_COVERAGE.md | 14 +++--- moto/__init__.py | 2 +- moto/{eb => elasticbeanstalk}/__init__.py | 2 +- moto/{eb => elasticbeanstalk}/exceptions.py | 0 moto/{eb => elasticbeanstalk}/models.py | 50 ++++++++++++++++++++- moto/{eb => elasticbeanstalk}/responses.py | 41 +++-------------- moto/{eb => elasticbeanstalk}/urls.py | 0 tests/test_eb/test_eb.py | 18 ++++---- 8 files changed, 74 insertions(+), 53 deletions(-) rename moto/{eb => elasticbeanstalk}/__init__.py (59%) rename moto/{eb => elasticbeanstalk}/exceptions.py (100%) rename moto/{eb => elasticbeanstalk}/models.py (61%) rename moto/{eb => elasticbeanstalk}/responses.py (97%) rename moto/{eb => elasticbeanstalk}/urls.py (100%) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 705618524..bd9e9a4cd 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2878,15 +2878,15 @@ - [ ] test_failover ## elasticbeanstalk -0% implemented +13% implemented - [ ] abort_environment_update - [ ] apply_environment_managed_action - [ ] check_dns_availability - [ ] compose_environments -- [ ] create_application +- [X] create_application - [ ] create_application_version - [ ] create_configuration_template -- [ ] create_environment +- [X] create_environment - [ ] create_platform_version - [ ] create_storage_location - [ ] delete_application @@ -2903,13 +2903,13 @@ - [ ] describe_environment_managed_action_history - [ ] describe_environment_managed_actions - [ ] describe_environment_resources -- [ ] describe_environments +- [X] describe_environments - [ ] describe_events - [ ] describe_instances_health - [ ] describe_platform_version -- [ ] list_available_solution_stacks +- [X] list_available_solution_stacks - [ ] list_platform_versions -- [ ] list_tags_for_resource +- [X] list_tags_for_resource - [ ] rebuild_environment - [ ] request_environment_info - [ ] restart_app_server @@ -2921,7 +2921,7 @@ - [ ] update_application_version - [ ] update_configuration_template - [ ] update_environment -- [ ] update_tags_for_resource +- [X] update_tags_for_resource - [ ] validate_configuration_settings ## elastictranscoder diff --git a/moto/__init__.py b/moto/__init__.py index 9b59f18eb..4c9d4753c 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -21,7 +21,7 @@ from .datasync import mock_datasync # noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # noqa from .dynamodbstreams import mock_dynamodbstreams # noqa -from .eb import mock_eb # noqa +from .elasticbeanstalk import mock_elasticbeanstalk # noqa from .ec2 import mock_ec2, mock_ec2_deprecated # noqa from .ec2_instance_connect import mock_ec2_instance_connect # noqa from .ecr import mock_ecr, mock_ecr_deprecated # noqa diff --git a/moto/eb/__init__.py b/moto/elasticbeanstalk/__init__.py similarity index 59% rename from moto/eb/__init__.py rename to moto/elasticbeanstalk/__init__.py index 3e06e9595..851fa445b 100644 --- a/moto/eb/__init__.py +++ b/moto/elasticbeanstalk/__init__.py @@ -1,4 +1,4 @@ from .models import eb_backends from moto.core.models import base_decorator -mock_eb = base_decorator(eb_backends) +mock_elasticbeanstalk = base_decorator(eb_backends) diff --git a/moto/eb/exceptions.py b/moto/elasticbeanstalk/exceptions.py similarity index 100% rename from moto/eb/exceptions.py rename to moto/elasticbeanstalk/exceptions.py diff --git a/moto/eb/models.py b/moto/elasticbeanstalk/models.py similarity index 61% rename from moto/eb/models.py rename to moto/elasticbeanstalk/models.py index 71873f30c..83ad65ab0 100644 --- a/moto/eb/models.py +++ b/moto/elasticbeanstalk/models.py @@ -3,7 +3,7 @@ import weakref import boto.beanstalk from moto.core import BaseBackend, BaseModel -from .exceptions import InvalidParameterValueError +from .exceptions import InvalidParameterValueError, ResourceNotFoundException class FakeEnvironment(BaseModel): @@ -90,6 +90,54 @@ class EBBackend(BaseBackend): self.applications[application_name] = new_app return new_app + def create_environment(self, app, environment_name, stack_name, tags): + return app.create_environment( + environment_name=environment_name, + solution_stack_name=stack_name, + tags=tags, + ) + + def describe_environments(self): + envs = [] + for app in self.applications.values(): + for env in app.environments.values(): + envs.append(env) + return envs + + def list_available_solution_stacks(self): + # Implemented in response.py + pass + + def update_tags_for_resource(self, resource_arn, tags_to_add, tags_to_remove): + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN '{}'.".format(resource_arn) + ) + + for key, value in tags_to_add.items(): + res.tags[key] = value + + for key in tags_to_remove: + del res.tags[key] + + def list_tags_for_resource(self, resource_arn): + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN '{}'.".format(resource_arn) + ) + return res.tags + + def _find_environment_by_arn(self, arn): + for app in self.applications.keys(): + for env in self.applications[app].environments.values(): + if env.environment_arn == arn: + return env + raise KeyError() + eb_backends = dict( (region.name, EBBackend(region.name)) for region in boto.beanstalk.regions() diff --git a/moto/eb/responses.py b/moto/elasticbeanstalk/responses.py similarity index 97% rename from moto/eb/responses.py rename to moto/elasticbeanstalk/responses.py index 6178e4a7f..0416121b2 100644 --- a/moto/eb/responses.py +++ b/moto/elasticbeanstalk/responses.py @@ -1,7 +1,7 @@ from moto.core.responses import BaseResponse from moto.core.utils import tags_from_query_string from .models import eb_backends -from .exceptions import InvalidParameterValueError, ResourceNotFoundException +from .exceptions import InvalidParameterValueError class EBResponse(BaseResponse): @@ -34,9 +34,10 @@ class EBResponse(BaseResponse): ) tags = tags_from_query_string(self.querystring, prefix="Tags.member") - env = app.create_environment( + env = self.backend.create_environment( + app, environment_name=self._get_param("EnvironmentName"), - solution_stack_name=self._get_param("SolutionStackName"), + stack_name=self._get_param("SolutionStackName"), tags=tags, ) @@ -44,11 +45,7 @@ class EBResponse(BaseResponse): return template.render(environment=env, region=self.backend.region,) def describe_environments(self): - envs = [] - - for app in self.backend.applications.values(): - for env in app.environments.values(): - envs.append(env) + envs = self.backend.describe_environments() template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) return template.render(environments=envs,) @@ -57,43 +54,19 @@ class EBResponse(BaseResponse): def list_available_solution_stacks(): return EB_LIST_AVAILABLE_SOLUTION_STACKS - def _find_environment_by_arn(self, arn): - for app in self.backend.applications.keys(): - for env in self.backend.applications[app].environments.values(): - if env.environment_arn == arn: - return env - raise KeyError() - def update_tags_for_resource(self): resource_arn = self._get_param("ResourceArn") - try: - res = self._find_environment_by_arn(resource_arn) - except KeyError: - raise ResourceNotFoundException( - "Resource not found for ARN '{}'.".format(resource_arn) - ) - tags_to_add = tags_from_query_string( self.querystring, prefix="TagsToAdd.member" ) - for key, value in tags_to_add.items(): - res.tags[key] = value - tags_to_remove = self._get_multi_param("TagsToRemove.member") - for key in tags_to_remove: - del res.tags[key] + self.backend.update_tags_for_resource(resource_arn, tags_to_add, tags_to_remove) return EB_UPDATE_TAGS_FOR_RESOURCE def list_tags_for_resource(self): resource_arn = self._get_param("ResourceArn") - try: - res = self._find_environment_by_arn(resource_arn) - except KeyError: - raise ResourceNotFoundException( - "Resource not found for ARN '{}'.".format(resource_arn) - ) - tags = res.tags + tags = self.backend.list_tags_for_resource(resource_arn) template = self.response_template(EB_LIST_TAGS_FOR_RESOURCE) return template.render(tags=tags, arn=resource_arn,) diff --git a/moto/eb/urls.py b/moto/elasticbeanstalk/urls.py similarity index 100% rename from moto/eb/urls.py rename to moto/elasticbeanstalk/urls.py diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 1064bf31a..42eb09be3 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -2,10 +2,10 @@ import boto3 import sure # noqa from botocore.exceptions import ClientError -from moto import mock_eb +from moto import mock_elasticbeanstalk -@mock_eb +@mock_elasticbeanstalk def test_create_application(): # Create Elastic Beanstalk Application conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -13,7 +13,7 @@ def test_create_application(): app["Application"]["ApplicationName"].should.equal("myapp") -@mock_eb +@mock_elasticbeanstalk def test_create_application_dup(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") conn.create_application(ApplicationName="myapp",) @@ -22,7 +22,7 @@ def test_create_application_dup(): ) -@mock_eb +@mock_elasticbeanstalk def test_describe_applications(): # Create Elastic Beanstalk Application conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -33,7 +33,7 @@ def test_describe_applications(): apps["Applications"][0]["ApplicationName"].should.equal("myapp") -@mock_eb +@mock_elasticbeanstalk def test_create_environment(): # Create Elastic Beanstalk Environment conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -42,7 +42,7 @@ def test_create_environment(): env["EnvironmentName"].should.equal("myenv") -@mock_eb +@mock_elasticbeanstalk def test_describe_environments(): # List Elastic Beanstalk Envs conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -72,7 +72,7 @@ def tags_list_to_dict(tag_list): return tag_dict -@mock_eb +@mock_elasticbeanstalk def test_create_environment_tags(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") conn.create_application(ApplicationName="myapp",) @@ -88,7 +88,7 @@ def test_create_environment_tags(): tags_list_to_dict(tags["ResourceTags"]).should.equal(env_tags) -@mock_eb +@mock_elasticbeanstalk def test_update_tags(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") conn.create_application(ApplicationName="myapp",) @@ -122,7 +122,7 @@ def test_update_tags(): tags_list_to_dict(tags["ResourceTags"]).should.equal(total_env_tags) -@mock_eb +@mock_elasticbeanstalk def test_list_available_solution_stacks(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") stacks = conn.list_available_solution_stacks() From 551dc024595cc602091ecf640311a3db4a52d6ca Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 16:28:36 +0100 Subject: [PATCH 20/51] ElasticBeanstalk - Fix tests in Python2 and ServerMode --- moto/backends.py | 2 ++ moto/elasticbeanstalk/models.py | 16 ++++++++++++---- moto/elasticbeanstalk/responses.py | 3 +-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/moto/backends.py b/moto/backends.py index a358b8fd2..a48df74a4 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -23,6 +23,7 @@ from moto.ec2 import ec2_backends from moto.ec2_instance_connect import ec2_instance_connect_backends from moto.ecr import ecr_backends from moto.ecs import ecs_backends +from moto.elasticbeanstalk import eb_backends from moto.elb import elb_backends from moto.elbv2 import elbv2_backends from moto.emr import emr_backends @@ -77,6 +78,7 @@ BACKENDS = { "ec2_instance_connect": ec2_instance_connect_backends, "ecr": ecr_backends, "ecs": ecs_backends, + "elasticbeanstalk": eb_backends, "elb": elb_backends, "elbv2": elbv2_backends, "events": events_backends, diff --git a/moto/elasticbeanstalk/models.py b/moto/elasticbeanstalk/models.py index 83ad65ab0..3767846c1 100644 --- a/moto/elasticbeanstalk/models.py +++ b/moto/elasticbeanstalk/models.py @@ -1,6 +1,6 @@ import weakref -import boto.beanstalk +from boto3 import Session from moto.core import BaseBackend, BaseModel from .exceptions import InvalidParameterValueError, ResourceNotFoundException @@ -139,6 +139,14 @@ class EBBackend(BaseBackend): raise KeyError() -eb_backends = dict( - (region.name, EBBackend(region.name)) for region in boto.beanstalk.regions() -) +eb_backends = {} +for region in Session().get_available_regions("elasticbeanstalk"): + eb_backends[region] = EBBackend(region) +for region in Session().get_available_regions( + "elasticbeanstalk", partition_name="aws-us-gov" +): + eb_backends[region] = EBBackend(region) +for region in Session().get_available_regions( + "elasticbeanstalk", partition_name="aws-cn" +): + eb_backends[region] = EBBackend(region) diff --git a/moto/elasticbeanstalk/responses.py b/moto/elasticbeanstalk/responses.py index 0416121b2..387cbb3ea 100644 --- a/moto/elasticbeanstalk/responses.py +++ b/moto/elasticbeanstalk/responses.py @@ -50,8 +50,7 @@ class EBResponse(BaseResponse): template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) return template.render(environments=envs,) - @staticmethod - def list_available_solution_stacks(): + def list_available_solution_stacks(self): return EB_LIST_AVAILABLE_SOLUTION_STACKS def update_tags_for_resource(self): From 6dd6686afcc5c9dc40a9ba90b5b853b2a9f60e48 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 31 Mar 2020 11:10:38 +0100 Subject: [PATCH 21/51] Use TaggingService for S3 Buckets --- moto/s3/models.py | 23 ++++++++++++++++++----- moto/s3/responses.py | 8 ++++---- moto/utilities/tagging_service.py | 12 ++++++++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 8c2a86f41..aede52d26 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -22,6 +22,7 @@ import six from bisect import insort from moto.core import ACCOUNT_ID, BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime +from moto.utilities.tagging_service import TaggingService from .exceptions import ( BucketAlreadyExists, MissingBucket, @@ -787,7 +788,6 @@ class FakeBucket(BaseModel): self.policy = None self.website_configuration = None self.acl = get_canned_acl("private") - self.tags = FakeTagging() self.cors = [] self.logging = {} self.notification_configuration = None @@ -1085,6 +1085,10 @@ class FakeBucket(BaseModel): def set_acl(self, acl): self.acl = acl + @property + def arn(self): + return "arn:aws:s3:::{}".format(self.name) + @property def physical_resource_id(self): return self.name @@ -1110,7 +1114,7 @@ class FakeBucket(BaseModel): int(time.mktime(self.creation_date.timetuple())) ), # PY2 and 3 compatible "configurationItemMD5Hash": "", - "arn": "arn:aws:s3:::{}".format(self.name), + "arn": self.arn, "resourceType": "AWS::S3::Bucket", "resourceId": self.name, "resourceName": self.name, @@ -1119,7 +1123,7 @@ class FakeBucket(BaseModel): "resourceCreationTime": str(self.creation_date), "relatedEvents": [], "relationships": [], - "tags": {tag.key: tag.value for tag in self.tagging.tag_set.tags}, + "tags": s3_backend.tagger.get_tag_dict_for_resource(self.arn), "configuration": { "name": self.name, "owner": {"id": OWNER}, @@ -1181,6 +1185,7 @@ class S3Backend(BaseBackend): def __init__(self): self.buckets = {} self.account_public_access_block = None + self.tagger = TaggingService() def create_bucket(self, bucket_name, region_name): if bucket_name in self.buckets: @@ -1357,16 +1362,24 @@ class S3Backend(BaseBackend): key.set_tagging(tagging) return key + def get_bucket_tags(self, bucket_name): + bucket = self.get_bucket(bucket_name) + return self.tagger.list_tags_for_resource(bucket.arn) + def put_bucket_tagging(self, bucket_name, tagging): tag_keys = [tag.key for tag in tagging.tag_set.tags] if len(tag_keys) != len(set(tag_keys)): raise DuplicateTagKeys() bucket = self.get_bucket(bucket_name) - bucket.set_tags(tagging) + self.tagger.delete_all_tags_for_resource(bucket.arn) + self.tagger.tag_resource( + bucket.arn, + [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + ) def delete_bucket_tagging(self, bucket_name): bucket = self.get_bucket(bucket_name) - bucket.delete_tags() + self.tagger.delete_all_tags_for_resource(bucket.arn) def put_bucket_cors(self, bucket_name, cors_rules): bucket = self.get_bucket(bucket_name) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 197cd9080..f3a5eeaac 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -378,13 +378,13 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_OBJECT_ACL_RESPONSE) return template.render(obj=bucket) elif "tagging" in querystring: - bucket = self.backend.get_bucket(bucket_name) + tags = self.backend.get_bucket_tags(bucket_name)["Tags"] # "Special Error" if no tags: - if len(bucket.tagging.tag_set.tags) == 0: + if len(tags) == 0: template = self.response_template(S3_NO_BUCKET_TAGGING) return 404, {}, template.render(bucket_name=bucket_name) template = self.response_template(S3_BUCKET_TAGGING_RESPONSE) - return template.render(bucket=bucket) + return template.render(tags=tags) elif "logging" in querystring: bucket = self.backend.get_bucket(bucket_name) if not bucket.logging: @@ -1929,7 +1929,7 @@ S3_OBJECT_TAGGING_RESPONSE = """\ S3_BUCKET_TAGGING_RESPONSE = """ - {% for tag in bucket.tagging.tag_set.tags %} + {% for tag in tags %} {{ tag.key }} {{ tag.value }} diff --git a/moto/utilities/tagging_service.py b/moto/utilities/tagging_service.py index 89b857277..8c3228552 100644 --- a/moto/utilities/tagging_service.py +++ b/moto/utilities/tagging_service.py @@ -5,15 +5,23 @@ class TaggingService: self.valueName = valueName self.tags = {} + def get_tag_dict_for_resource(self, arn): + result = {} + if self.has_tags(arn): + for k, v in self.tags[arn].items(): + result[k] = v + return result + def list_tags_for_resource(self, arn): result = [] - if arn in self.tags: + if self.has_tags(arn): for k, v in self.tags[arn].items(): result.append({self.keyName: k, self.valueName: v}) return {self.tagName: result} def delete_all_tags_for_resource(self, arn): - del self.tags[arn] + if self.has_tags(arn): + del self.tags[arn] def has_tags(self, arn): return arn in self.tags From f7ad4cbc09164205d9f216355cec6c921170d3f9 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 31 Mar 2020 12:04:04 +0100 Subject: [PATCH 22/51] Use TaggingService for S3 Objects --- moto/s3/models.py | 30 ++++++++++++------- moto/s3/responses.py | 28 ++++++------------ moto/utilities/tagging_service.py | 6 ++++ tests/test_s3/test_s3.py | 9 ++++-- tests/test_utilities/test_tagging_service.py | 31 ++++++++++++++++++++ 5 files changed, 71 insertions(+), 33 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index aede52d26..b5224b64a 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -95,6 +95,7 @@ class FakeKey(BaseModel): version_id=0, max_buffer_size=DEFAULT_KEY_BUFFER_SIZE, multipart=None, + bucket_name=None, ): self.name = name self.last_modified = datetime.datetime.utcnow() @@ -106,8 +107,8 @@ class FakeKey(BaseModel): self._etag = etag self._version_id = version_id self._is_versioned = is_versioned - self._tagging = FakeTagging() self.multipart = multipart + self.bucket_name = bucket_name self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size) self._max_buffer_size = max_buffer_size @@ -127,6 +128,13 @@ class FakeKey(BaseModel): self.lock.release() return r + @property + def arn(self): + # S3 Objects don't have an ARN, but we do need something unique when creating tags against this resource + return "arn:aws:s3:::{}/{}/{}".format( + self.bucket_name, self.name, self.version_id + ) + @value.setter def value(self, new_value): self._value_buffer.seek(0) @@ -153,9 +161,6 @@ class FakeKey(BaseModel): self._metadata = {} self._metadata.update(metadata) - def set_tagging(self, tagging): - self._tagging = tagging - def set_storage_class(self, storage): if storage is not None and storage not in STORAGE_CLASS: raise InvalidStorageClass(storage=storage) @@ -211,10 +216,6 @@ class FakeKey(BaseModel): def metadata(self): return self._metadata - @property - def tagging(self): - return self._tagging - @property def response_dict(self): res = { @@ -1355,11 +1356,17 @@ class S3Backend(BaseBackend): else: return None - def set_key_tagging(self, bucket_name, key_name, tagging, version_id=None): - key = self.get_key(bucket_name, key_name, version_id) + def get_key_tags(self, key): + return self.tagger.list_tags_for_resource(key.arn) + + def set_key_tags(self, key, tagging, key_name=None): if key is None: raise MissingKey(key_name) - key.set_tagging(tagging) + self.tagger.delete_all_tags_for_resource(key.arn) + self.tagger.tag_resource( + key.arn, + [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + ) return key def get_bucket_tags(self, bucket_name): @@ -1587,6 +1594,7 @@ class S3Backend(BaseBackend): key = self.get_key(src_bucket_name, src_key_name, version_id=src_version_id) new_key = key.copy(dest_key_name, dest_bucket.is_versioned) + self.tagger.copy_tags(key.arn, new_key.arn) if storage is not None: new_key.set_storage_class(storage) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f3a5eeaac..4e3b9a67b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -383,7 +383,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if len(tags) == 0: template = self.response_template(S3_NO_BUCKET_TAGGING) return 404, {}, template.render(bucket_name=bucket_name) - template = self.response_template(S3_BUCKET_TAGGING_RESPONSE) + template = self.response_template(S3_OBJECT_TAGGING_RESPONSE) return template.render(tags=tags) elif "logging" in querystring: bucket = self.backend.get_bucket(bucket_name) @@ -1091,8 +1091,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_OBJECT_ACL_RESPONSE) return 200, response_headers, template.render(obj=key) if "tagging" in query: + tags = self.backend.get_key_tags(key)["Tags"] template = self.response_template(S3_OBJECT_TAGGING_RESPONSE) - return 200, response_headers, template.render(obj=key) + return 200, response_headers, template.render(tags=tags) response_headers.update(key.metadata) response_headers.update(key.response_dict) @@ -1164,8 +1165,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): version_id = query["versionId"][0] else: version_id = None + key = self.backend.get_key(bucket_name, key_name, version_id=version_id) tagging = self._tagging_from_xml(body) - self.backend.set_key_tagging(bucket_name, key_name, tagging, version_id) + self.backend.set_key_tags(key, tagging, key_name) return 200, response_headers, "" if "x-amz-copy-source" in request.headers: @@ -1206,7 +1208,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): tdirective = request.headers.get("x-amz-tagging-directive") if tdirective == "REPLACE": tagging = self._tagging_from_headers(request.headers) - new_key.set_tagging(tagging) + self.backend.set_key_tags(new_key, tagging) template = self.response_template(S3_OBJECT_COPY_RESPONSE) response_headers.update(new_key.response_dict) return 200, response_headers, template.render(key=new_key) @@ -1230,7 +1232,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): new_key.website_redirect_location = request.headers.get( "x-amz-website-redirect-location" ) - new_key.set_tagging(tagging) + self.backend.set_key_tags(new_key, tagging) template = self.response_template(S3_OBJECT_RESPONSE) response_headers.update(new_key.response_dict) @@ -1916,23 +1918,11 @@ S3_OBJECT_ACL_RESPONSE = """ S3_OBJECT_TAGGING_RESPONSE = """\ - - {% for tag in obj.tagging.tag_set.tags %} - - {{ tag.key }} - {{ tag.value }} - - {% endfor %} - -""" - -S3_BUCKET_TAGGING_RESPONSE = """ - {% for tag in tags %} - {{ tag.key }} - {{ tag.value }} + {{ tag.Key }} + {{ tag.Value }} {% endfor %} diff --git a/moto/utilities/tagging_service.py b/moto/utilities/tagging_service.py index 8c3228552..2d6ac99c9 100644 --- a/moto/utilities/tagging_service.py +++ b/moto/utilities/tagging_service.py @@ -35,6 +35,12 @@ class TaggingService: else: self.tags[arn][t[self.keyName]] = None + def copy_tags(self, from_arn, to_arn): + if self.has_tags(from_arn): + self.tag_resource( + to_arn, self.list_tags_for_resource(from_arn)[self.tagName] + ) + def untag_resource_using_names(self, arn, tag_names): for name in tag_names: if name in self.tags.get(arn, {}): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 303ed523d..4ddc160a8 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -3255,7 +3255,8 @@ def test_boto3_put_object_tagging_on_earliest_version(): # Older version has tags while the most recent does not resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) resp["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - resp["TagSet"].should.equal( + sorted_tagset = sorted(resp["TagSet"], key=lambda t: t["Key"]) + sorted_tagset.should.equal( [{"Key": "item1", "Value": "foo"}, {"Key": "item2", "Value": "bar"}] ) @@ -3333,7 +3334,8 @@ def test_boto3_put_object_tagging_on_both_version(): resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) resp["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - resp["TagSet"].should.equal( + sorted_tagset = sorted(resp["TagSet"], key=lambda t: t["Key"]) + sorted_tagset.should.equal( [{"Key": "item1", "Value": "foo"}, {"Key": "item2", "Value": "bar"}] ) @@ -3341,7 +3343,8 @@ def test_boto3_put_object_tagging_on_both_version(): Bucket=bucket_name, Key=key, VersionId=second_object.id ) resp["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - resp["TagSet"].should.equal( + sorted_tagset = sorted(resp["TagSet"], key=lambda t: t["Key"]) + sorted_tagset.should.equal( [{"Key": "item1", "Value": "baz"}, {"Key": "item2", "Value": "bin"}] ) diff --git a/tests/test_utilities/test_tagging_service.py b/tests/test_utilities/test_tagging_service.py index 249e903fe..1eac276a1 100644 --- a/tests/test_utilities/test_tagging_service.py +++ b/tests/test_utilities/test_tagging_service.py @@ -77,3 +77,34 @@ def test_extract_tag_names(): expected = ["key1", "key2"] expected.should.be.equal(actual) + + +def test_copy_non_existing_arn(): + svc = TaggingService() + tags = [{"Key": "key1", "Value": "value1"}, {"Key": "key2", "Value": "value2"}] + svc.tag_resource("new_arn", tags) + # + svc.copy_tags("non_existing_arn", "new_arn") + # Copying from a non-existing ARN should a NOOP + # Assert the old tags still exist + actual = sorted( + svc.list_tags_for_resource("new_arn")["Tags"], key=lambda t: t["Key"] + ) + actual.should.equal(tags) + + +def test_copy_existing_arn(): + svc = TaggingService() + tags_old_arn = [{"Key": "key1", "Value": "value1"}] + tags_new_arn = [{"Key": "key2", "Value": "value2"}] + svc.tag_resource("old_arn", tags_old_arn) + svc.tag_resource("new_arn", tags_new_arn) + # + svc.copy_tags("old_arn", "new_arn") + # Assert the old tags still exist + actual = sorted( + svc.list_tags_for_resource("new_arn")["Tags"], key=lambda t: t["Key"] + ) + actual.should.equal( + [{"Key": "key1", "Value": "value1"}, {"Key": "key2", "Value": "value2"}] + ) From 8dbfd43c5c7556af546764d16b2f49d57a3127c4 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 1 Apr 2020 15:35:25 +0100 Subject: [PATCH 23/51] Use TaggingService for S3 - Cleanup --- moto/s3/models.py | 59 +++++++------------------------ moto/s3/responses.py | 60 +++++++++++++------------------- tests/test_config/test_config.py | 2 ++ tests/test_s3/test_s3.py | 11 ++---- 4 files changed, 40 insertions(+), 92 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index b5224b64a..44a94e7a3 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -35,7 +35,6 @@ from .exceptions import ( MalformedXML, InvalidStorageClass, InvalidTargetBucketForLogging, - DuplicateTagKeys, CrossLocationLoggingProhibitted, NoSuchPublicAccessBlockConfiguration, InvalidPublicAccessBlockConfiguration, @@ -473,26 +472,10 @@ def get_canned_acl(acl): return FakeAcl(grants=grants) -class FakeTagging(BaseModel): - def __init__(self, tag_set=None): - self.tag_set = tag_set or FakeTagSet() - - -class FakeTagSet(BaseModel): - def __init__(self, tags=None): - self.tags = tags or [] - - -class FakeTag(BaseModel): - def __init__(self, key, value=None): - self.key = key - self.value = value - - class LifecycleFilter(BaseModel): def __init__(self, prefix=None, tag=None, and_filter=None): self.prefix = prefix - self.tag = tag + (self.tag_key, self.tag_value) = tag if tag else (None, None) self.and_filter = and_filter def to_config_dict(self): @@ -501,11 +484,11 @@ class LifecycleFilter(BaseModel): "predicate": {"type": "LifecyclePrefixPredicate", "prefix": self.prefix} } - elif self.tag: + elif self.tag_key: return { "predicate": { "type": "LifecycleTagPredicate", - "tag": {"key": self.tag.key, "value": self.tag.value}, + "tag": {"key": self.tag_key, "value": self.tag_value}, } } @@ -529,12 +512,9 @@ class LifecycleAndFilter(BaseModel): if self.prefix is not None: data.append({"type": "LifecyclePrefixPredicate", "prefix": self.prefix}) - for tag in self.tags: + for key, value in self.tags.items(): data.append( - { - "type": "LifecycleTagPredicate", - "tag": {"key": tag.key, "value": tag.value}, - } + {"type": "LifecycleTagPredicate", "tag": {"key": key, "value": value},} ) return data @@ -880,7 +860,7 @@ class FakeBucket(BaseModel): and_filter = None if rule["Filter"].get("And"): filters += 1 - and_tags = [] + and_tags = {} if rule["Filter"]["And"].get("Tag"): if not isinstance(rule["Filter"]["And"]["Tag"], list): rule["Filter"]["And"]["Tag"] = [ @@ -888,7 +868,7 @@ class FakeBucket(BaseModel): ] for t in rule["Filter"]["And"]["Tag"]: - and_tags.append(FakeTag(t["Key"], t.get("Value", ""))) + and_tags[t["Key"]] = t.get("Value", "") try: and_prefix = ( @@ -902,7 +882,7 @@ class FakeBucket(BaseModel): filter_tag = None if rule["Filter"].get("Tag"): filters += 1 - filter_tag = FakeTag( + filter_tag = ( rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", ""), ) @@ -989,16 +969,6 @@ class FakeBucket(BaseModel): def delete_cors(self): self.cors = [] - def set_tags(self, tagging): - self.tags = tagging - - def delete_tags(self): - self.tags = FakeTagging() - - @property - def tagging(self): - return self.tags - def set_logging(self, logging_config, bucket_backend): if not logging_config: self.logging = {} @@ -1359,13 +1329,12 @@ class S3Backend(BaseBackend): def get_key_tags(self, key): return self.tagger.list_tags_for_resource(key.arn) - def set_key_tags(self, key, tagging, key_name=None): + def set_key_tags(self, key, tags, key_name=None): if key is None: raise MissingKey(key_name) self.tagger.delete_all_tags_for_resource(key.arn) self.tagger.tag_resource( - key.arn, - [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + key.arn, [{"Key": key, "Value": value} for key, value in tags.items()], ) return key @@ -1373,15 +1342,11 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) return self.tagger.list_tags_for_resource(bucket.arn) - def put_bucket_tagging(self, bucket_name, tagging): - tag_keys = [tag.key for tag in tagging.tag_set.tags] - if len(tag_keys) != len(set(tag_keys)): - raise DuplicateTagKeys() + def put_bucket_tags(self, bucket_name, tags): bucket = self.get_bucket(bucket_name) self.tagger.delete_all_tags_for_resource(bucket.arn) self.tagger.tag_resource( - bucket.arn, - [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + bucket.arn, [{"Key": key, "Value": value} for key, value in tags.items()], ) def delete_bucket_tagging(self, bucket_name): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 4e3b9a67b..913b20861 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,6 +24,7 @@ from moto.s3bucket_path.utils import ( from .exceptions import ( BucketAlreadyExists, + DuplicateTagKeys, S3ClientError, MissingBucket, MissingKey, @@ -43,9 +44,6 @@ from .models import ( FakeGrant, FakeAcl, FakeKey, - FakeTagging, - FakeTagSet, - FakeTag, ) from .utils import ( bucket_name_from_url, @@ -652,7 +650,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return "" elif "tagging" in querystring: tagging = self._bucket_tagging_from_xml(body) - self.backend.put_bucket_tagging(bucket_name, tagging) + self.backend.put_bucket_tags(bucket_name, tagging) return "" elif "website" in querystring: self.backend.set_bucket_website_configuration(bucket_name, body) @@ -1361,55 +1359,45 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return None def _tagging_from_headers(self, headers): + tags = {} if headers.get("x-amz-tagging"): parsed_header = parse_qs(headers["x-amz-tagging"], keep_blank_values=True) - tags = [] for tag in parsed_header.items(): - tags.append(FakeTag(tag[0], tag[1][0])) - - tag_set = FakeTagSet(tags) - tagging = FakeTagging(tag_set) - return tagging - else: - return FakeTagging() + tags[tag[0]] = tag[1][0] + return tags def _tagging_from_xml(self, xml): parsed_xml = xmltodict.parse(xml, force_list={"Tag": True}) - tags = [] + tags = {} for tag in parsed_xml["Tagging"]["TagSet"]["Tag"]: - tags.append(FakeTag(tag["Key"], tag["Value"])) + tags[tag["Key"]] = tag["Value"] - tag_set = FakeTagSet(tags) - tagging = FakeTagging(tag_set) - return tagging + return tags def _bucket_tagging_from_xml(self, xml): parsed_xml = xmltodict.parse(xml) - tags = [] + tags = {} # Optional if no tags are being sent: if parsed_xml["Tagging"].get("TagSet"): # If there is only 1 tag, then it's not a list: if not isinstance(parsed_xml["Tagging"]["TagSet"]["Tag"], list): - tags.append( - FakeTag( - parsed_xml["Tagging"]["TagSet"]["Tag"]["Key"], - parsed_xml["Tagging"]["TagSet"]["Tag"]["Value"], - ) - ) + tags[parsed_xml["Tagging"]["TagSet"]["Tag"]["Key"]] = parsed_xml[ + "Tagging" + ]["TagSet"]["Tag"]["Value"] else: for tag in parsed_xml["Tagging"]["TagSet"]["Tag"]: - tags.append(FakeTag(tag["Key"], tag["Value"])) + if tag["Key"] in tags: + raise DuplicateTagKeys() + tags[tag["Key"]] = tag["Value"] # Verify that "aws:" is not in the tags. If so, then this is a problem: - for tag in tags: - if tag.key.startswith("aws:"): + for key, _ in tags.items(): + if key.startswith("aws:"): raise NoSystemTags() - tag_set = FakeTagSet(tags) - tagging = FakeTagging(tag_set) - return tagging + return tags def _cors_from_xml(self, xml): parsed_xml = xmltodict.parse(xml) @@ -1730,10 +1718,10 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% if rule.filter.prefix != None %} {{ rule.filter.prefix }} {% endif %} - {% if rule.filter.tag %} + {% if rule.filter.tag_key %} - {{ rule.filter.tag.key }} - {{ rule.filter.tag.value }} + {{ rule.filter.tag_key }} + {{ rule.filter.tag_value }} {% endif %} {% if rule.filter.and_filter %} @@ -1741,10 +1729,10 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% if rule.filter.and_filter.prefix != None %} {{ rule.filter.and_filter.prefix }} {% endif %} - {% for tag in rule.filter.and_filter.tags %} + {% for key, value in rule.filter.and_filter.tags.items() %} - {{ tag.key }} - {{ tag.value }} + {{ key }} + {{ value }} {% endfor %} diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 1ffd52a2c..1bf39428e 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -11,6 +11,8 @@ from moto import mock_s3 from moto.config import mock_config from moto.core import ACCOUNT_ID +import sure # noqa + @mock_config def test_put_configuration_recorder(): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 4ddc160a8..e2acf32f2 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4295,24 +4295,17 @@ def test_s3_config_dict(): FakeAcl, FakeGrant, FakeGrantee, - FakeTag, - FakeTagging, - FakeTagSet, OWNER, ) # Without any buckets: assert not s3_config_query.get_config_resource("some_bucket") - tags = FakeTagging( - FakeTagSet( - [FakeTag("someTag", "someValue"), FakeTag("someOtherTag", "someOtherValue")] - ) - ) + tags = {"someTag": "someValue", "someOtherTag": "someOtherValue"} # With 1 bucket in us-west-2: s3_config_query.backends["global"].create_bucket("bucket1", "us-west-2") - s3_config_query.backends["global"].put_bucket_tagging("bucket1", tags) + s3_config_query.backends["global"].put_bucket_tags("bucket1", tags) # With a log bucket: s3_config_query.backends["global"].create_bucket("logbucket", "us-west-2") From dff1ab580b20a22a0b12f8731e86d9468f42cf4b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 1 Apr 2020 16:15:03 +0100 Subject: [PATCH 24/51] Extend new S3 tag structure to ResourceGroupStaging API --- moto/resourcegroupstaggingapi/models.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/moto/resourcegroupstaggingapi/models.py b/moto/resourcegroupstaggingapi/models.py index d05a53f81..b6e35d586 100644 --- a/moto/resourcegroupstaggingapi/models.py +++ b/moto/resourcegroupstaggingapi/models.py @@ -145,10 +145,7 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Do S3, resource type s3 if not resource_type_filters or "s3" in resource_type_filters: for bucket in self.s3_backend.buckets.values(): - tags = [] - for tag in bucket.tags.tag_set.tags: - tags.append({"Key": tag.key, "Value": tag.value}) - + tags = self.s3_backend.tagger.list_tags_for_resource(bucket.arn)["Tags"] if not tags or not tag_filter( tags ): # Skip if no tags, or invalid filter @@ -362,8 +359,9 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Do S3, resource type s3 for bucket in self.s3_backend.buckets.values(): - for tag in bucket.tags.tag_set.tags: - yield tag.key + tags = self.s3_backend.tagger.get_tag_dict_for_resource(bucket.arn) + for key, _ in tags.items(): + yield key # EC2 tags def get_ec2_keys(res_id): @@ -414,9 +412,10 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Do S3, resource type s3 for bucket in self.s3_backend.buckets.values(): - for tag in bucket.tags.tag_set.tags: - if tag.key == tag_key: - yield tag.value + tags = self.s3_backend.tagger.get_tag_dict_for_resource(bucket.arn) + for key, value in tags.items(): + if key == tag_key: + yield value # EC2 tags def get_ec2_values(res_id): From 9ab02e17d528d461b5a05b7aafa623addae65966 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 3 Apr 2020 10:30:05 +0100 Subject: [PATCH 25/51] #883 - Lambda - Add test to verify remove_permission functinonality --- moto/awslambda/models.py | 4 ++-- moto/awslambda/responses.py | 6 ++--- tests/test_awslambda/test_lambda.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 9cdf2397c..589a790ae 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -1006,11 +1006,11 @@ class LambdaBackend(BaseBackend): return True return False - def add_policy_statement(self, function_name, raw): + def add_permission(self, function_name, raw): fn = self.get_function(function_name) fn.policy.add_statement(raw) - def del_policy_statement(self, function_name, sid, revision=""): + def remove_permission(self, function_name, sid, revision=""): fn = self.get_function(function_name) fn.policy.del_statement(sid, revision) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index ce6c93f16..4213840f6 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -146,7 +146,7 @@ class LambdaResponse(BaseResponse): function_name = path.split("/")[-2] if self.lambda_backend.get_function(function_name): statement = self.body - self.lambda_backend.add_policy_statement(function_name, statement) + self.lambda_backend.add_permission(function_name, statement) return 200, {}, json.dumps({"Statement": statement}) else: return 404, {}, "{}" @@ -166,9 +166,7 @@ class LambdaResponse(BaseResponse): statement_id = path.split("/")[-1].split("?")[0] revision = querystring.get("RevisionId", "") if self.lambda_backend.get_function(function_name): - self.lambda_backend.del_policy_statement( - function_name, statement_id, revision - ) + self.lambda_backend.remove_permission(function_name, statement_id, revision) return 204, {}, "{}" else: return 404, {}, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index eb8453e43..e67576518 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1677,6 +1677,42 @@ def test_create_function_with_unknown_arn(): ) +@mock_lambda +def test_remove_function_permission(): + conn = boto3.client("lambda", _lambda_region) + zip_content = get_test_zip_file1() + conn.create_function( + FunctionName="testFunction", + Runtime="python2.7", + Role=(get_role_name()), + Handler="lambda_function.handler", + Code={"ZipFile": zip_content}, + Description="test lambda function", + Timeout=3, + MemorySize=128, + Publish=True, + ) + + conn.add_permission( + FunctionName="testFunction", + StatementId="1", + Action="lambda:InvokeFunction", + Principal="432143214321", + SourceArn="arn:aws:lambda:us-west-2:account-id:function:helloworld", + SourceAccount="123412341234", + EventSourceToken="blah", + Qualifier="2", + ) + + remove = conn.remove_permission( + FunctionName="testFunction", StatementId="1", Qualifier="2", + ) + remove["ResponseMetadata"]["HTTPStatusCode"].should.equal(204) + policy = conn.get_policy(FunctionName="testFunction", Qualifier="2")["Policy"] + policy = json.loads(policy) + policy["Statement"].should.equal([]) + + def create_invalid_lambda(role): conn = boto3.client("lambda", _lambda_region) zip_content = get_test_zip_file1() From 280db9df6c43f606721d07b51e104eda8e065313 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 4 Apr 2020 14:09:38 +0100 Subject: [PATCH 26/51] #2800 - CognitoIdentity - Fix format of Identity ID --- moto/cognitoidentity/utils.py | 4 ++-- tests/test_cognitoidentity/test_cognitoidentity.py | 8 +++++--- tests/test_cognitoidentity/test_server.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/moto/cognitoidentity/utils.py b/moto/cognitoidentity/utils.py index 6143d5121..54016ad17 100644 --- a/moto/cognitoidentity/utils.py +++ b/moto/cognitoidentity/utils.py @@ -1,5 +1,5 @@ -from moto.core.utils import get_random_hex +from uuid import uuid4 def get_random_identity_id(region): - return "{0}:{1}".format(region, get_random_hex(length=19)) + return "{0}:{1}".format(region, uuid4()) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index 8eae183c6..0ec7acfb0 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -7,6 +7,7 @@ from nose.tools import assert_raises from moto import mock_cognitoidentity from moto.cognitoidentity.utils import get_random_identity_id from moto.core import ACCOUNT_ID +from uuid import UUID @mock_cognitoidentity @@ -83,8 +84,10 @@ def test_describe_identity_pool_with_invalid_id_raises_error(): # testing a helper function def test_get_random_identity_id(): - assert len(get_random_identity_id("us-west-2")) > 0 - assert len(get_random_identity_id("us-west-2").split(":")[1]) == 19 + identity_id = get_random_identity_id("us-west-2") + region, id = identity_id.split(":") + region.should.equal("us-west-2") + UUID(id, version=4) # Will throw an error if it's not a valid UUID @mock_cognitoidentity @@ -96,7 +99,6 @@ def test_get_id(): IdentityPoolId="us-west-2:12345", Logins={"someurl": "12345"}, ) - print(result) assert ( result.get("IdentityId", "").startswith("us-west-2") or result.get("ResponseMetadata").get("HTTPStatusCode") == 200 diff --git a/tests/test_cognitoidentity/test_server.py b/tests/test_cognitoidentity/test_server.py index 903dae290..8c4229f06 100644 --- a/tests/test_cognitoidentity/test_server.py +++ b/tests/test_cognitoidentity/test_server.py @@ -48,6 +48,5 @@ def test_get_id(): }, ) - print(res.data) json_data = json.loads(res.data.decode("utf-8")) assert ":" in json_data["IdentityId"] From 54f51fc7c159d2f6993d3cce1561ccaba2bf3e9a Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 8 Apr 2020 10:49:58 +0100 Subject: [PATCH 27/51] DynamoDB - TransactWriteItems implementation --- moto/dynamodb2/models.py | 90 ++++++- moto/dynamodb2/responses.py | 24 +- tests/test_dynamodb2/test_dynamodb.py | 352 ++++++++++++++++++++++++++ 3 files changed, 459 insertions(+), 7 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 152e719c4..c0e55bd5b 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1406,9 +1406,9 @@ class DynamoDBBackend(BaseBackend): table_name, key, update_expression, - attribute_updates, expression_attribute_names, expression_attribute_values, + attribute_updates=None, expected=None, condition_expression=None, ): @@ -1516,6 +1516,94 @@ class DynamoDBBackend(BaseBackend): return table.ttl + def transact_write_items(self, transact_items): + # Create a backup in case any of the transactions fail + original_table_state = copy.deepcopy(self.tables) + try: + for item in transact_items: + if "ConditionCheck" in item: + item = item["ConditionCheck"] + key = item["Key"] + table_name = item["TableName"] + condition_expression = item.get("ConditionExpression", None) + expression_attribute_names = item.get( + "ExpressionAttributeNames", None + ) + expression_attribute_values = item.get( + "ExpressionAttributeValues", None + ) + current = self.get_item(table_name, key) + + condition_op = get_filter_expression( + condition_expression, + expression_attribute_names, + expression_attribute_values, + ) + if not condition_op.expr(current): + raise ValueError("The conditional request failed") + elif "Put" in item: + item = item["Put"] + attrs = item["Item"] + table_name = item["TableName"] + condition_expression = item.get("ConditionExpression", None) + expression_attribute_names = item.get( + "ExpressionAttributeNames", None + ) + expression_attribute_values = item.get( + "ExpressionAttributeValues", None + ) + self.put_item( + table_name, + attrs, + condition_expression=condition_expression, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + ) + elif "Delete" in item: + item = item["Delete"] + key = item["Key"] + table_name = item["TableName"] + condition_expression = item.get("ConditionExpression", None) + expression_attribute_names = item.get( + "ExpressionAttributeNames", None + ) + expression_attribute_values = item.get( + "ExpressionAttributeValues", None + ) + self.delete_item( + table_name, + key, + condition_expression=condition_expression, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + ) + elif "Update" in item: + item = item["Update"] + key = item["Key"] + table_name = item["TableName"] + update_expression = item["UpdateExpression"] + condition_expression = item.get("ConditionExpression", None) + expression_attribute_names = item.get( + "ExpressionAttributeNames", None + ) + expression_attribute_values = item.get( + "ExpressionAttributeValues", None + ) + self.update_item( + table_name, + key, + update_expression=update_expression, + condition_expression=condition_expression, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + ) + else: + raise ValueError + except: # noqa: E722 Do not use bare except + # Rollback to the original state, and reraise the error + self.tables = original_table_state + raise + dynamodb_backends = {} for region in Session().get_available_regions("dynamodb"): diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 78126f7f1..9b13f20a6 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -757,12 +757,12 @@ class DynamoHandler(BaseResponse): item = self.dynamodb_backend.update_item( name, key, - update_expression, - attribute_updates, - expression_attribute_names, - expression_attribute_values, - expected, - condition_expression, + update_expression=update_expression, + attribute_updates=attribute_updates, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + expected=expected, + condition_expression=condition_expression, ) except InvalidUpdateExpression: er = "com.amazonaws.dynamodb.v20111205#ValidationException" @@ -925,3 +925,15 @@ class DynamoHandler(BaseResponse): result.update({"ConsumedCapacity": [v for v in consumed_capacity.values()]}) return dynamo_json_dump(result) + + def transact_write_items(self): + transact_items = self.body["TransactItems"] + try: + self.dynamodb_backend.transact_write_items(transact_items) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" + return self.error( + er, "A condition specified in the operation could not be evaluated." + ) + response = {"ConsumedCapacity": [], "ItemCollectionMetrics": {}} + return dynamo_json_dump(response) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index bec24c966..90deab6be 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4177,3 +4177,355 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_transact_write_items_put(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Put multiple items + dynamodb.transact_write_items( + TransactItems=[ + { + "Put": { + "Item": {"id": {"S": "foo{}".format(str(i))}, "foo": {"S": "bar"},}, + "TableName": "test-table", + } + } + for i in range(0, 5) + ] + ) + # Assert all are present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(5) + + +@mock_dynamodb2 +def test_transact_write_items_put_conditional_expressions(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo2"},}, + ) + # Put multiple items + with assert_raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Put": { + "Item": { + "id": {"S": "foo{}".format(str(i))}, + "foo": {"S": "bar"}, + }, + "TableName": "test-table", + "ConditionExpression": "#i <> :i", + "ExpressionAttributeNames": {"#i": "id"}, + "ExpressionAttributeValues": { + ":i": { + "S": "foo2" + } # This item already exist, so the ConditionExpression should fail + }, + } + } + for i in range(0, 5) + ] + ) + # Assert the exception is correct + ex.exception.response["Error"]["Code"].should.equal( + "ConditionalCheckFailedException" + ) + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "A condition specified in the operation could not be evaluated." + ) + # Assert all are present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"id": {"S": "foo2"}}) + + +@mock_dynamodb2 +def test_transact_write_items_conditioncheck_passes(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item without email address + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo"},}, + ) + # Put an email address, after verifying it doesn't exist yet + dynamodb.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + }, + { + "Put": { + "Item": { + "id": {"S": "foo"}, + "email_address": {"S": "test@moto.com"}, + }, + "TableName": "test-table", + } + }, + ] + ) + # Assert all are present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb2 +def test_transact_write_items_conditioncheck_fails(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item with email address + dynamodb.put_item( + TableName="test-table", + Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, + ) + # Try to put an email address, but verify whether it exists + # ConditionCheck should fail + with assert_raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + }, + { + "Put": { + "Item": { + "id": {"S": "foo"}, + "email_address": {"S": "update@moto.com"}, + }, + "TableName": "test-table", + } + }, + ] + ) + # Assert the exception is correct + ex.exception.response["Error"]["Code"].should.equal( + "ConditionalCheckFailedException" + ) + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "A condition specified in the operation could not be evaluated." + ) + + # Assert the original email address is still present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb2 +def test_transact_write_items_delete(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo"},}, + ) + # Delete the item + dynamodb.transact_write_items( + TransactItems=[ + {"Delete": {"Key": {"id": {"S": "foo"}}, "TableName": "test-table",}} + ] + ) + # Assert the item is deleted + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(0) + + +@mock_dynamodb2 +def test_transact_write_items_delete_with_successful_condition_expression(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item without email address + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo"},}, + ) + # ConditionExpression will pass - no email address has been specified yet + dynamodb.transact_write_items( + TransactItems=[ + { + "Delete": { + "Key": {"id": {"S": "foo"},}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + } + ] + ) + # Assert the item is deleted + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(0) + + +@mock_dynamodb2 +def test_transact_write_items_delete_with_failed_condition_expression(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item with email address + dynamodb.put_item( + TableName="test-table", + Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, + ) + # Try to delete an item that does not have an email address + # ConditionCheck should fail + with assert_raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Delete": { + "Key": {"id": {"S": "foo"},}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + } + ] + ) + # Assert the exception is correct + ex.exception.response["Error"]["Code"].should.equal( + "ConditionalCheckFailedException" + ) + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "A condition specified in the operation could not be evaluated." + ) + # Assert the original item is still present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb2 +def test_transact_write_items_update(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item + dynamodb.put_item(TableName="test-table", Item={"id": {"S": "foo"}}) + # Update the item + dynamodb.transact_write_items( + TransactItems=[ + { + "Update": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "UpdateExpression": "SET #e = :v", + "ExpressionAttributeNames": {"#e": "email_address"}, + "ExpressionAttributeValues": {":v": {"S": "test@moto.com"}}, + } + } + ] + ) + # Assert the item is updated + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}) + + +@mock_dynamodb2 +def test_transact_write_items_update_with_failed_condition_expression(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item with email address + dynamodb.put_item( + TableName="test-table", + Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, + ) + # Try to update an item that does not have an email address + # ConditionCheck should fail + with assert_raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Update": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "UpdateExpression": "SET #e = :v", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + "ExpressionAttributeValues": {":v": {"S": "update@moto.com"}}, + } + } + ] + ) + # Assert the exception is correct + ex.exception.response["Error"]["Code"].should.equal( + "ConditionalCheckFailedException" + ) + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "A condition specified in the operation could not be evaluated." + ) + # Assert the original item is still present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) From 1e1fe3ee4bffb5470795d8aab16fa3de1145f5a6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:13 +0100 Subject: [PATCH 28/51] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a35eded61..de2a06fd4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -286,7 +286,7 @@ class Item(BaseModel): return "Item: {0}".format(self.to_json()) def size(self): - return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + return sum(bytesize(key) + value.size() for key, value in self.attrs.items()) def to_json(self): attributes = {} From 8122a40be064f28dd0aa2ea8567cc2fb2ce4dea8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:20 +0100 Subject: [PATCH 29/51] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index de2a06fd4..62c60efb0 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1127,7 +1127,7 @@ class Table(BaseModel): last_evaluated_key = None size_limit = 1000000 # DynamoDB has a 1MB size limit - item_size = sum([res.size() for res in results]) + item_size = sum(res.size() for res in results) if item_size > size_limit: item_size = idx = 0 while item_size + results[idx].size() < size_limit: From c2b4c397f272f80684c395150159028ed265fd20 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:53:15 +0100 Subject: [PATCH 30/51] DDB test - Fix KeySchema, and set BillingMode for easier online testing --- tests/test_dynamodb2/test_dynamodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index daae79232..e00a45e1d 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4143,13 +4143,13 @@ def test_dynamodb_max_1mb_limit(): TableName=table_name, KeySchema=[ {"AttributeName": "partition_key", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "SORT"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, ], AttributeDefinitions=[ {"AttributeName": "partition_key", "AttributeType": "S"}, {"AttributeName": "sort_key", "AttributeType": "S"}, ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + BillingMode="PAY_PER_REQUEST", ) # Populate the table @@ -4170,3 +4170,4 @@ def test_dynamodb_max_1mb_limit(): # We shouldn't get everything back - the total result set is well over 1MB assert response["Count"] < len(items) response["LastEvaluatedKey"].shouldnt.be(None) + From a6902e87137da229d63d138b898a63ffd12fe326 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 15 Apr 2020 07:26:09 +0100 Subject: [PATCH 31/51] Update tests/test_dynamodb2/test_dynamodb.py Co-Authored-By: Guilherme Martins Crocetti --- tests/test_dynamodb2/test_dynamodb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e00a45e1d..2b4c0969c 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4168,6 +4168,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - assert response["Count"] < len(items) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) - From 870b34ba7693e88df38d2f2765b972cfda955cee Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 16 Apr 2020 07:09:50 +0100 Subject: [PATCH 32/51] Spacing --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 8a8c69a8c..a0a5e6406 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4213,5 +4213,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - len(items).should.be.greater_than(response["Count"]) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) From b7f4ae21d17cc16580295bb5d6741bffb243e6ed Mon Sep 17 00:00:00 2001 From: Erik Hovland Date: Wed, 15 Apr 2020 20:08:44 -0700 Subject: [PATCH 33/51] Add assume_role_with_saml to STSBackend. Add the assume_role_with_saml method to the STSBackend class. --- moto/sts/models.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/moto/sts/models.py b/moto/sts/models.py index 12824b2ed..b274b1acd 100644 --- a/moto/sts/models.py +++ b/moto/sts/models.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +from base64 import b64decode import datetime +import xmltodict from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.core import ACCOUNT_ID @@ -79,5 +81,24 @@ class STSBackend(BaseBackend): def assume_role_with_web_identity(self, **kwargs): return self.assume_role(**kwargs) + def assume_role_with_saml(self, **kwargs): + del kwargs["principal_arn"] + saml_assertion_encoded = kwargs.pop("saml_assertion") + saml_assertion_decoded = b64decode(saml_assertion_encoded) + saml_assertion = xmltodict.parse(saml_assertion_decoded.decode("utf-8")) + kwargs["duration"] = int( + saml_assertion["samlp:Response"]["Assertion"]["AttributeStatement"][ + "Attribute" + ][2]["AttributeValue"] + ) + kwargs["role_session_name"] = saml_assertion["samlp:Response"]["Assertion"][ + "AttributeStatement" + ]["Attribute"][0]["AttributeValue"] + kwargs["external_id"] = None + kwargs["policy"] = None + role = AssumedRole(**kwargs) + self.assumed_roles.append(role) + return role + sts_backend = STSBackend() From b10718eea7fde315003c2e8ee83bd92a2a5d03fe Mon Sep 17 00:00:00 2001 From: Erik Hovland Date: Wed, 15 Apr 2020 20:10:22 -0700 Subject: [PATCH 34/51] Add AssumeRoleWithSAML response to responses.py. Add the AssumeRoleWithSAML response to the available STS responses. --- moto/sts/responses.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/moto/sts/responses.py b/moto/sts/responses.py index f36799b03..9af2c3e12 100644 --- a/moto/sts/responses.py +++ b/moto/sts/responses.py @@ -71,6 +71,19 @@ class TokenResponse(BaseResponse): template = self.response_template(ASSUME_ROLE_WITH_WEB_IDENTITY_RESPONSE) return template.render(role=role) + def assume_role_with_saml(self): + role_arn = self.querystring.get("RoleArn")[0] + principal_arn = self.querystring.get("PrincipalArn")[0] + saml_assertion = self.querystring.get("SAMLAssertion")[0] + + role = sts_backend.assume_role_with_saml( + role_arn=role_arn, + principal_arn=principal_arn, + saml_assertion=saml_assertion, + ) + template = self.response_template(ASSUME_ROLE_WITH_SAML_RESPONSE) + return template.render(role=role) + def get_caller_identity(self): template = self.response_template(GET_CALLER_IDENTITY_RESPONSE) @@ -168,6 +181,30 @@ ASSUME_ROLE_WITH_WEB_IDENTITY_RESPONSE = """""" +ASSUME_ROLE_WITH_SAML_RESPONSE = """ + + https://signin.aws.amazon.com/saml + + {{ role.user_id }} + {{ role.arn }} + + + {{ role.access_key_id }} + {{ role.secret_access_key }} + {{ role.session_token }} + {{ role.expiration_ISO8601 }} + + {{ role.user_id }} + B64EncodedStringOfHashOfIssuerAccountIdAndUserId= + persistent + http://localhost:3000/ + + + c6104cbe-af31-11e0-8154-cbc7ccf896c7 + +""" + + GET_CALLER_IDENTITY_RESPONSE = """ {{ arn }} From 88494c58f9a45a3d100837d74ad9b4bbc9e9d24e Mon Sep 17 00:00:00 2001 From: Erik Hovland Date: Wed, 15 Apr 2020 20:11:33 -0700 Subject: [PATCH 35/51] Add a test for assume_role_with_saml. Add a test with SAML assertion to test the assume_role_with_saml method in the STSBackend. --- tests/test_sts/test_sts.py | 123 +++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tests/test_sts/test_sts.py b/tests/test_sts/test_sts.py index 4dee9184f..efc04beb4 100644 --- a/tests/test_sts/test_sts.py +++ b/tests/test_sts/test_sts.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +from base64 import b64encode import json import boto @@ -103,6 +104,128 @@ def test_assume_role(): ) +@freeze_time("2012-01-01 12:00:00") +@mock_sts +def test_assume_role_with_saml(): + client = boto3.client("sts", region_name="us-east-1") + + session_name = "session-name" + policy = json.dumps( + { + "Statement": [ + { + "Sid": "Stmt13690092345534", + "Action": ["S3:ListBucket"], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::foobar-tester"], + } + ] + } + ) + role_name = "test-role" + provider_name = "TestProvFed" + user_name = "testuser" + role_input = "arn:aws:iam::{account_id}:role/{role_name}".format( + account_id=ACCOUNT_ID, role_name=role_name + ) + principal_role = "arn:aws:iam:{account_id}:saml-provider/{provider_name}".format( + account_id=ACCOUNT_ID, provider_name=provider_name + ) + saml_assertion = """ + + + http://localhost/ + + + + + http://localhost:3000/ + + + + + + + + + + + NTIyMzk0ZGI4MjI0ZjI5ZGNhYjkyOGQyZGQ1NTZjODViZjk5YTY4ODFjOWRjNjkyYzZmODY2ZDQ4NjlkZjY3YSAgLQo= + + + NTIyMzk0ZGI4MjI0ZjI5ZGNhYjkyOGQyZGQ1NTZjODViZjk5YTY4ODFjOWRjNjkyYzZmODY2ZDQ4NjlkZjY3YSAgLQo= + + + NTIyMzk0ZGI4MjI0ZjI5ZGNhYjkyOGQyZGQ1NTZjODViZjk5YTY4ODFjOWRjNjkyYzZmODY2ZDQ4NjlkZjY3YSAgLQo= + + + + + {username} + + + + + + + urn:amazon:webservices + + + + + {username}@localhost + + + arn:aws:iam::{account_id}:saml-provider/{provider_name},arn:aws:iam::{account_id}:role/{role_name} + + + 900 + + + + + urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport + + + +""".format( + account_id=ACCOUNT_ID, + role_name=role_name, + provider_name=provider_name, + username=user_name, + ).replace( + "\n", "" + ) + + assume_role_response = client.assume_role_with_saml( + RoleArn=role_input, + PrincipalArn=principal_role, + SAMLAssertion=b64encode(saml_assertion.encode("utf-8")).decode("utf-8"), + ) + + credentials = assume_role_response["Credentials"] + if not settings.TEST_SERVER_MODE: + credentials["Expiration"].isoformat().should.equal("2012-01-01T12:15:00+00:00") + credentials["SessionToken"].should.have.length_of(356) + assert credentials["SessionToken"].startswith("FQoGZXIvYXdzE") + credentials["AccessKeyId"].should.have.length_of(20) + assert credentials["AccessKeyId"].startswith("ASIA") + credentials["SecretAccessKey"].should.have.length_of(40) + + assume_role_response["AssumedRoleUser"]["Arn"].should.equal( + "arn:aws:sts::{account_id}:assumed-role/{role_name}/{fed_name}@localhost".format( + account_id=ACCOUNT_ID, role_name=role_name, fed_name=user_name + ) + ) + assert assume_role_response["AssumedRoleUser"]["AssumedRoleId"].startswith("AROA") + assert assume_role_response["AssumedRoleUser"]["AssumedRoleId"].endswith( + ":{fed_name}@localhost".format(fed_name=user_name) + ) + assume_role_response["AssumedRoleUser"]["AssumedRoleId"].should.have.length_of( + 21 + 1 + len("{fed_name}@localhost".format(fed_name=user_name)) + ) + + @freeze_time("2012-01-01 12:00:00") @mock_sts_deprecated def test_assume_role_with_web_identity(): From 50111929cc16ea270b6c7d266c934777c15c9ad5 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 22 Apr 2020 12:18:27 +0100 Subject: [PATCH 36/51] STS - Handle AssumeRoleWithSAML as an unsigned request --- moto/server.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/moto/server.py b/moto/server.py index 92fe6f229..7987a629d 100644 --- a/moto/server.py +++ b/moto/server.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import argparse +import io import json import re import sys @@ -29,6 +30,7 @@ UNSIGNED_REQUESTS = { "AWSCognitoIdentityService": ("cognito-identity", "us-east-1"), "AWSCognitoIdentityProviderService": ("cognito-idp", "us-east-1"), } +UNSIGNED_ACTIONS = {"AssumeRoleWithSAML": ("sts", "us-east-1")} class DomainDispatcherApplication(object): @@ -77,9 +79,13 @@ class DomainDispatcherApplication(object): else: # Unsigned request target = environ.get("HTTP_X_AMZ_TARGET") + action = self.get_action_from_body(environ) if target: service, _ = target.split(".", 1) service, region = UNSIGNED_REQUESTS.get(service, DEFAULT_SERVICE_REGION) + elif action and action in UNSIGNED_ACTIONS: + # See if we can match the Action to a known service + service, region = UNSIGNED_ACTIONS.get(action) else: # S3 is the last resort when the target is also unknown service, region = DEFAULT_SERVICE_REGION @@ -130,6 +136,22 @@ class DomainDispatcherApplication(object): self.app_instances[backend] = app return app + def get_action_from_body(self, environ): + body = None + try: + request_body_size = int(environ.get("CONTENT_LENGTH", 0)) + if "wsgi.input" in environ: + body = environ["wsgi.input"].read(request_body_size).decode("utf-8") + body_dict = dict(x.split("=") for x in str(body).split("&")) + return body_dict["Action"] + except ValueError: + pass + finally: + if body: + # We've consumed the body = need to reset it + environ["wsgi.input"] = io.StringIO(body) + return None + def __call__(self, environ, start_response): backend_app = self.get_application(environ) return backend_app(environ, start_response) From 25d1e1059e6ad28050147dc2257e6a12846396a9 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 22 Apr 2020 14:07:19 +0100 Subject: [PATCH 37/51] STS - Only check request-body of eligible requests for Actions --- moto/server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/server.py b/moto/server.py index 7987a629d..498f6c504 100644 --- a/moto/server.py +++ b/moto/server.py @@ -139,12 +139,16 @@ class DomainDispatcherApplication(object): def get_action_from_body(self, environ): body = None try: - request_body_size = int(environ.get("CONTENT_LENGTH", 0)) - if "wsgi.input" in environ: + # AWS requests use querystrings as the body (Action=x&Data=y&...) + simple_form = environ["CONTENT_TYPE"].startswith( + "application/x-www-form-urlencoded" + ) + request_body_size = int(environ["CONTENT_LENGTH"]) + if simple_form and request_body_size: body = environ["wsgi.input"].read(request_body_size).decode("utf-8") - body_dict = dict(x.split("=") for x in str(body).split("&")) + body_dict = dict(x.split("=") for x in body.split("&")) return body_dict["Action"] - except ValueError: + except (KeyError, ValueError): pass finally: if body: From 9ed6e52d0ab86a0bd7b00caeb4af2c2cfb54bf42 Mon Sep 17 00:00:00 2001 From: Antoine Wendlinger Date: Wed, 22 Apr 2020 19:31:43 +0200 Subject: [PATCH 38/51] Handle VersionId in S3:delete_objects VersionId is not read in delete_objects requests, and the behavior differs from its singular counterpart delete_object. This fixes the issue. --- moto/s3/responses.py | 26 +++++++++++++++++--------- tests/test_s3/test_s3.py | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 442489a8a..ec6015f7a 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -840,26 +840,33 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _bucket_response_delete_keys(self, request, body, bucket_name): template = self.response_template(S3_DELETE_KEYS_RESPONSE) - keys = minidom.parseString(body).getElementsByTagName("Key") - deleted_names = [] + objects = minidom.parseString(body).getElementsByTagName("Object") + + deleted_objects = [] error_names = [] - if len(keys) == 0: + if len(objects) == 0: raise MalformedXML() - for k in keys: - key_name = k.firstChild.nodeValue + for object_ in objects: + key_name = object_.getElementsByTagName("Key")[0].firstChild.nodeValue + version_id_node = object_.getElementsByTagName("VersionId") + if version_id_node: + version_id = version_id_node[0].firstChild.nodeValue + else: + version_id = None + success = self.backend.delete_key( - bucket_name, undo_clean_key_name(key_name) + bucket_name, undo_clean_key_name(key_name), version_id=version_id ) if success: - deleted_names.append(key_name) + deleted_objects.append((key_name, version_id)) else: error_names.append(key_name) return ( 200, {}, - template.render(deleted=deleted_names, delete_errors=error_names), + template.render(deleted=deleted_objects, delete_errors=error_names), ) def _handle_range_header(self, request, headers, response_content): @@ -1861,9 +1868,10 @@ S3_BUCKET_GET_VERSIONS = """ S3_DELETE_KEYS_RESPONSE = """ -{% for k in deleted %} +{% for k, v in deleted %} {{k}} +{% if v %}{{v}}{% endif %} {% endfor %} {% for k in delete_errors %} diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index ffbd73966..4a94c9c38 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2218,6 +2218,29 @@ def test_boto3_deleted_versionings_list(): assert len(listed["Contents"]) == 1 +@mock_s3 +def test_boto3_delete_objects_for_specific_version_id(): + client = boto3.client("s3", region_name=DEFAULT_REGION_NAME) + client.create_bucket(Bucket="blah") + client.put_bucket_versioning( + Bucket="blah", VersioningConfiguration={"Status": "Enabled"} + ) + + client.put_object(Bucket="blah", Key="test1", Body=b"test1a") + client.put_object(Bucket="blah", Key="test1", Body=b"test1b") + + response = client.list_object_versions(Bucket="blah", Prefix="test1") + id_to_delete = [v["VersionId"] for v in response["Versions"] if v["IsLatest"]][0] + + response = client.delete_objects( + Bucket="blah", Delete={"Objects": [{"Key": "test1", "VersionId": id_to_delete}]} + ) + assert response["Deleted"] == [{"Key": "test1", "VersionId": id_to_delete}] + + listed = client.list_objects_v2(Bucket="blah") + assert len(listed["Contents"]) == 1 + + @mock_s3 def test_boto3_delete_versioned_bucket(): client = boto3.client("s3", region_name=DEFAULT_REGION_NAME) From 3e145ef8df0433141a8c17bd33505991a25bf4be Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Apr 2020 16:12:55 +0200 Subject: [PATCH 39/51] Do not remove tags after secret update, handle description --- moto/secretsmanager/models.py | 21 ++++++++++++++++----- moto/secretsmanager/responses.py | 2 ++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 11a024be6..3a13d1119 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -121,8 +121,12 @@ class SecretsManagerBackend(BaseBackend): "You can't perform this operation on the secret because it was marked for deletion." ) + secret = self.secrets[secret_id] + tags = secret["tags"] + description = secret["description"] + version_id = self._add_secret( - secret_id, secret_string=secret_string, secret_binary=secret_binary + secret_id, secret_string=secret_string, secret_binary=secret_binary, description=description, tags=tags ) response = json.dumps( @@ -136,7 +140,7 @@ class SecretsManagerBackend(BaseBackend): return response def create_secret( - self, name, secret_string=None, secret_binary=None, tags=[], **kwargs + self, name, secret_string=None, secret_binary=None, description=None, tags=[], **kwargs ): # error if secret exists @@ -146,7 +150,7 @@ class SecretsManagerBackend(BaseBackend): ) version_id = self._add_secret( - name, secret_string=secret_string, secret_binary=secret_binary, tags=tags + name, secret_string=secret_string, secret_binary=secret_binary, description=description, tags=tags ) response = json.dumps( @@ -164,6 +168,7 @@ class SecretsManagerBackend(BaseBackend): secret_id, secret_string=None, secret_binary=None, + description=None, tags=[], version_id=None, version_stages=None, @@ -216,13 +221,18 @@ class SecretsManagerBackend(BaseBackend): secret["rotation_lambda_arn"] = "" secret["auto_rotate_after_days"] = 0 secret["tags"] = tags + secret["description"] = description return version_id def put_secret_value(self, secret_id, secret_string, secret_binary, version_stages): + secret = self.secrets[secret_id] + tags = secret["tags"] + description = secret["description"] + version_id = self._add_secret( - secret_id, secret_string, secret_binary, version_stages=version_stages + secret_id, secret_string, secret_binary, description=description, tags=tags, version_stages=version_stages ) response = json.dumps( @@ -310,6 +320,7 @@ class SecretsManagerBackend(BaseBackend): self._add_secret( secret_id, old_secret_version["secret_string"], + secret["description"], secret["tags"], version_id=new_version_id, version_stages=["AWSCURRENT"], @@ -416,7 +427,7 @@ class SecretsManagerBackend(BaseBackend): { "ARN": secret_arn(self.region, secret["secret_id"]), "DeletedDate": secret.get("deleted_date", None), - "Description": "", + "Description": secret.get["description"], "KmsKeyId": "", "LastAccessedDate": None, "LastChangedDate": None, diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index 757b888a3..9a899c90d 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -21,11 +21,13 @@ class SecretsManagerResponse(BaseResponse): name = self._get_param("Name") secret_string = self._get_param("SecretString") secret_binary = self._get_param("SecretBinary") + description = self._get_param("Description", if_none="") tags = self._get_param("Tags", if_none=[]) return secretsmanager_backends[self.region].create_secret( name=name, secret_string=secret_string, secret_binary=secret_binary, + description=description, tags=tags, ) From 6483e3be806f25f02632f0f53f8810c8ae212468 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Apr 2020 18:17:03 +0200 Subject: [PATCH 40/51] do not require secret to exist on PutSecretValue operation --- moto/secretsmanager/models.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 3a13d1119..07a112fbc 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -227,9 +227,13 @@ class SecretsManagerBackend(BaseBackend): def put_secret_value(self, secret_id, secret_string, secret_binary, version_stages): - secret = self.secrets[secret_id] - tags = secret["tags"] - description = secret["description"] + if secret_id in self.secrets.keys(): + secret = self.secrets[secret_id] + tags = secret["tags"] + description = secret["description"] + else: + tags = [] + description = "" version_id = self._add_secret( secret_id, secret_string, secret_binary, description=description, tags=tags, version_stages=version_stages @@ -427,7 +431,7 @@ class SecretsManagerBackend(BaseBackend): { "ARN": secret_arn(self.region, secret["secret_id"]), "DeletedDate": secret.get("deleted_date", None), - "Description": secret.get["description"], + "Description": secret.get("description", ""), "KmsKeyId": "", "LastAccessedDate": None, "LastChangedDate": None, From ef67aee1a38e7b722d395f424aff206bc63af0dd Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Apr 2020 18:53:24 +0200 Subject: [PATCH 41/51] apply black formatting --- moto/secretsmanager/models.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 07a112fbc..7762d41bc 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -126,7 +126,11 @@ class SecretsManagerBackend(BaseBackend): description = secret["description"] version_id = self._add_secret( - secret_id, secret_string=secret_string, secret_binary=secret_binary, description=description, tags=tags + secret_id, + secret_string=secret_string, + secret_binary=secret_binary, + description=description, + tags=tags, ) response = json.dumps( @@ -140,7 +144,13 @@ class SecretsManagerBackend(BaseBackend): return response def create_secret( - self, name, secret_string=None, secret_binary=None, description=None, tags=[], **kwargs + self, + name, + secret_string=None, + secret_binary=None, + description=None, + tags=[], + **kwargs ): # error if secret exists @@ -150,7 +160,11 @@ class SecretsManagerBackend(BaseBackend): ) version_id = self._add_secret( - name, secret_string=secret_string, secret_binary=secret_binary, description=description, tags=tags + name, + secret_string=secret_string, + secret_binary=secret_binary, + description=description, + tags=tags, ) response = json.dumps( @@ -236,7 +250,12 @@ class SecretsManagerBackend(BaseBackend): description = "" version_id = self._add_secret( - secret_id, secret_string, secret_binary, description=description, tags=tags, version_stages=version_stages + secret_id, + secret_string, + secret_binary, + description=description, + tags=tags, + version_stages=version_stages, ) response = json.dumps( From b63110be9e7fc249eda1528a9161fa1870f0484a Mon Sep 17 00:00:00 2001 From: = Date: Fri, 24 Apr 2020 21:47:11 +0200 Subject: [PATCH 42/51] handle description in describe secret operation, add tests --- moto/secretsmanager/models.py | 2 +- .../test_secretsmanager.py | 101 ++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 7762d41bc..29bd6c96e 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -279,7 +279,7 @@ class SecretsManagerBackend(BaseBackend): { "ARN": secret_arn(self.region, secret["secret_id"]), "Name": secret["name"], - "Description": "", + "Description": secret.get("description", ""), "KmsKeyId": "", "RotationEnabled": secret["rotation_enabled"], "RotationLambdaARN": secret["rotation_lambda_arn"], diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 49d1dc925..6ec53460a 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -137,6 +137,45 @@ def test_create_secret_with_tags(): ] +@mock_secretsmanager +def test_create_secret_with_description(): + conn = boto3.client("secretsmanager", region_name="us-east-1") + secret_name = "test-secret-with-tags" + + result = conn.create_secret( + Name=secret_name, SecretString="foosecret", Description="desc" + ) + assert result["ARN"] + assert result["Name"] == secret_name + secret_value = conn.get_secret_value(SecretId=secret_name) + assert secret_value["SecretString"] == "foosecret" + secret_details = conn.describe_secret(SecretId=secret_name) + assert secret_details["Description"] == "desc" + + +@mock_secretsmanager +def test_create_secret_with_tags_and_description(): + conn = boto3.client("secretsmanager", region_name="us-east-1") + secret_name = "test-secret-with-tags" + + result = conn.create_secret( + Name=secret_name, + SecretString="foosecret", + Description="desc", + Tags=[{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}], + ) + assert result["ARN"] + assert result["Name"] == secret_name + secret_value = conn.get_secret_value(SecretId=secret_name) + assert secret_value["SecretString"] == "foosecret" + secret_details = conn.describe_secret(SecretId=secret_name) + assert secret_details["Tags"] == [ + {"Key": "Foo", "Value": "Bar"}, + {"Key": "Mykey", "Value": "Myvalue"}, + ] + assert secret_details["Description"] == "desc" + + @mock_secretsmanager def test_delete_secret(): conn = boto3.client("secretsmanager", region_name="us-west-2") @@ -690,6 +729,31 @@ def test_put_secret_value_versions_differ_if_same_secret_put_twice(): assert first_version_id != second_version_id +@mock_secretsmanager +def test_put_secret_value_maintains_description_and_tags(): + conn = boto3.client("secretsmanager", region_name="us-west-2") + + conn.create_secret( + Name=DEFAULT_SECRET_NAME, + SecretString="foosecret", + Description="desc", + Tags=[{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}], + ) + + conn = boto3.client("secretsmanager", region_name="us-west-2") + conn.put_secret_value( + SecretId=DEFAULT_SECRET_NAME, + SecretString="dupe_secret", + VersionStages=["AWSCURRENT"], + ) + secret_details = conn.describe_secret(SecretId=DEFAULT_SECRET_NAME) + assert secret_details["Tags"] == [ + {"Key": "Foo", "Value": "Bar"}, + {"Key": "Mykey", "Value": "Myvalue"}, + ] + assert secret_details["Description"] == "desc" + + @mock_secretsmanager def test_can_list_secret_version_ids(): conn = boto3.client("secretsmanager", region_name="us-west-2") @@ -739,6 +803,43 @@ def test_update_secret(): assert created_secret["VersionId"] != updated_secret["VersionId"] +@mock_secretsmanager +def test_update_secret_with_tags_and_description(): + conn = boto3.client("secretsmanager", region_name="us-west-2") + + created_secret = conn.create_secret( + Name="test-secret", + SecretString="foosecret", + Description="desc", + Tags=[{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}], + ) + + assert created_secret["ARN"] + assert created_secret["Name"] == "test-secret" + assert created_secret["VersionId"] != "" + + secret = conn.get_secret_value(SecretId="test-secret") + assert secret["SecretString"] == "foosecret" + + updated_secret = conn.update_secret( + SecretId="test-secret", SecretString="barsecret" + ) + + assert updated_secret["ARN"] + assert updated_secret["Name"] == "test-secret" + assert updated_secret["VersionId"] != "" + + secret = conn.get_secret_value(SecretId="test-secret") + assert secret["SecretString"] == "barsecret" + assert created_secret["VersionId"] != updated_secret["VersionId"] + secret_details = conn.describe_secret(SecretId="test-secret") + assert secret_details["Tags"] == [ + {"Key": "Foo", "Value": "Bar"}, + {"Key": "Mykey", "Value": "Myvalue"}, + ] + assert secret_details["Description"] == "desc" + + @mock_secretsmanager def test_update_secret_which_does_not_exit(): conn = boto3.client("secretsmanager", region_name="us-west-2") From a658900d69ca4ae36a4b265161809a529aabb211 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 03:13:36 -0500 Subject: [PATCH 43/51] Add policy to apigateway --- moto/apigateway/models.py | 4 ++++ moto/apigateway/responses.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index e5e5e3bfd..e011af601 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -461,6 +461,7 @@ class RestAPI(BaseModel): self.description = description self.create_date = int(time.time()) self.api_key_source = kwargs.get("api_key_source") or "HEADER" + self.policy = kwargs.get("policy") or None self.endpoint_configuration = kwargs.get("endpoint_configuration") or { "types": ["EDGE"] } @@ -485,6 +486,7 @@ class RestAPI(BaseModel): "apiKeySource": self.api_key_source, "endpointConfiguration": self.endpoint_configuration, "tags": self.tags, + "policy": self.policy, } def add_child(self, path, parent_id=None): @@ -713,6 +715,7 @@ class APIGatewayBackend(BaseBackend): api_key_source=None, endpoint_configuration=None, tags=None, + policy=None, ): api_id = create_id() rest_api = RestAPI( @@ -723,6 +726,7 @@ class APIGatewayBackend(BaseBackend): api_key_source=api_key_source, endpoint_configuration=endpoint_configuration, tags=tags, + policy=policy, ) self.apis[api_id] = rest_api return rest_api diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 822d4c0ce..a3c41a6d4 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -59,6 +59,7 @@ class APIGatewayResponse(BaseResponse): api_key_source = self._get_param("apiKeySource") endpoint_configuration = self._get_param("endpointConfiguration") tags = self._get_param("tags") + policy = self._get_param("policy") # Param validation if api_key_source and api_key_source not in API_KEY_SOURCES: @@ -94,6 +95,7 @@ class APIGatewayResponse(BaseResponse): api_key_source=api_key_source, endpoint_configuration=endpoint_configuration, tags=tags, + policy=policy, ) return 200, {}, json.dumps(rest_api.to_dict()) From 0828c5af9dfff7430537cfb26cc62a8523d9cef3 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 03:27:59 -0500 Subject: [PATCH 44/51] Add unit test for add apigateway with policy --- tests/test_apigateway/test_apigateway.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 596ed2dd4..107dc5d05 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -69,6 +69,24 @@ def test_create_rest_api_with_tags(): response["tags"].should.equal({"MY_TAG1": "MY_VALUE1"}) +@mock_apigateway +def test_create_rest_api_with_policy(): + client = boto3.client("apigateway", region_name="us-west-2") + + policy = "{\"Version\": \"2012-10-17\",\"Statement\": []}" + response = client.create_rest_api( + name="my_api", + description="this is my api", + policy=policy + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + + assert "policy" in response + response["policy"].should.equal(policy) + + @mock_apigateway def test_create_rest_api_invalid_apikeysource(): client = boto3.client("apigateway", region_name="us-west-2") From 637e0188a2ab81bf3a72b7ddae2677f235c50973 Mon Sep 17 00:00:00 2001 From: Olivier Parent Colombel Date: Mon, 20 Apr 2020 20:54:31 +0200 Subject: [PATCH 45/51] Allow S3 keys to start with leading slashes. --- moto/s3/responses.py | 3 ++- moto/s3/urls.py | 2 +- tests/test_s3/test_s3.py | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 442489a8a..ce1e6128d 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -134,7 +134,8 @@ ACTION_MAP = { def parse_key_name(pth): - return pth.lstrip("/") + # strip the first '/' left by urlparse + return pth[1:] if pth.startswith('/') else pth def is_delete_keys(request, path, bucket_name): diff --git a/moto/s3/urls.py b/moto/s3/urls.py index 752762184..4c4e9ea76 100644 --- a/moto/s3/urls.py +++ b/moto/s3/urls.py @@ -15,5 +15,5 @@ url_paths = { # path-based bucket + key "{0}/(?P[^/]+)/(?P.+)": S3ResponseInstance.key_or_control_response, # subdomain bucket + key with empty first part of path - "{0}//(?P.*)$": S3ResponseInstance.key_or_control_response, + "{0}/(?P/.*)$": S3ResponseInstance.key_or_control_response, } diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index ffbd73966..3048f6507 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -3744,6 +3744,28 @@ def test_root_dir_with_empty_name_works(): store_and_read_back_a_key("/") +@parameterized(['mybucket', 'my.bucket']) +@mock_s3 +def test_leading_slashes_not_removed(bucket_name): + """Make sure that leading slashes are not removed internally.""" + s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME) + s3.create_bucket(Bucket=bucket_name) + + uploaded_key = '/key' + invalid_key_1 = 'key' + invalid_key_2 = '//key' + + s3.put_object(Bucket=bucket_name, Key=uploaded_key, Body=b'Some body') + + with assert_raises(ClientError) as e: + s3.get_object(Bucket=bucket_name, Key=invalid_key_1) + e.exception.response["Error"]["Code"].should.equal("NoSuchKey") + + with assert_raises(ClientError) as e: + s3.get_object(Bucket=bucket_name, Key=invalid_key_2) + e.exception.response["Error"]["Code"].should.equal("NoSuchKey") + + @parameterized( [("foo/bar/baz",), ("foo",), ("foo/run_dt%3D2019-01-01%252012%253A30%253A00",)] ) From d852f7dd063ae17cc1fa7f97bc3510e3daef55e9 Mon Sep 17 00:00:00 2001 From: Olivier Parent Colombel Date: Sat, 25 Apr 2020 15:10:23 +0200 Subject: [PATCH 46/51] Fixing lint errors. --- moto/s3/responses.py | 2 +- tests/test_s3/test_s3.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ce1e6128d..71c424244 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -135,7 +135,7 @@ ACTION_MAP = { def parse_key_name(pth): # strip the first '/' left by urlparse - return pth[1:] if pth.startswith('/') else pth + return pth[1:] if pth.startswith("/") else pth def is_delete_keys(request, path, bucket_name): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 3048f6507..fea76b9e3 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -3744,18 +3744,18 @@ def test_root_dir_with_empty_name_works(): store_and_read_back_a_key("/") -@parameterized(['mybucket', 'my.bucket']) +@parameterized(["mybucket", "my.bucket"]) @mock_s3 def test_leading_slashes_not_removed(bucket_name): """Make sure that leading slashes are not removed internally.""" s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME) s3.create_bucket(Bucket=bucket_name) - uploaded_key = '/key' - invalid_key_1 = 'key' - invalid_key_2 = '//key' + uploaded_key = "/key" + invalid_key_1 = "key" + invalid_key_2 = "//key" - s3.put_object(Bucket=bucket_name, Key=uploaded_key, Body=b'Some body') + s3.put_object(Bucket=bucket_name, Key=uploaded_key, Body=b"Some body") with assert_raises(ClientError) as e: s3.get_object(Bucket=bucket_name, Key=invalid_key_1) From 4a800d8f2c8677b098ea0a2c41deface8236c267 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 11:24:54 -0500 Subject: [PATCH 47/51] Updated for black --- tests/test_apigateway/test_apigateway.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 107dc5d05..b04328a03 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -73,11 +73,9 @@ def test_create_rest_api_with_tags(): def test_create_rest_api_with_policy(): client = boto3.client("apigateway", region_name="us-west-2") - policy = "{\"Version\": \"2012-10-17\",\"Statement\": []}" + policy = '{"Version": "2012-10-17","Statement": []}' response = client.create_rest_api( - name="my_api", - description="this is my api", - policy=policy + name="my_api", description="this is my api", policy=policy ) api_id = response["id"] From ec731ac901563d256d8b24779e35050f06a9bfba Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sun, 26 Apr 2020 15:12:33 +0100 Subject: [PATCH 48/51] Improve DDB expressions support4: Execution using AST Part of structured approach for UpdateExpressions: 1) Expression gets parsed into a tokenlist (tokenized) 2) Tokenlist get transformed to expression tree (AST) 3) The AST gets validated (full semantic correctness) 4) AST gets processed to perform the update -> this commit This commit uses the AST to execute the UpdateExpression. All the existing tests pass. The only tests that have been updated are in test_dynamodb_table_with_range_key.py because they wrongly allow adding a set to a path that doesn't exist. This has been alligend to correspond to the behavior of AWS DynamoDB. This commit will resolve https://github.com/spulec/moto/issues/2806 Multiple tests have been implemented that verify this. --- moto/dynamodb2/exceptions.py | 18 + moto/dynamodb2/models/__init__.py | 214 +-------- moto/dynamodb2/models/dynamo_type.py | 106 ++++- moto/dynamodb2/parsing/executors.py | 262 ++++++++++ moto/dynamodb2/parsing/validators.py | 127 +++-- tests/test_dynamodb2/test_dynamodb.py | 271 ++++++++++- .../test_dynamodb2/test_dynamodb_executor.py | 446 ++++++++++++++++++ .../test_dynamodb_table_with_range_key.py | 25 +- 8 files changed, 1200 insertions(+), 269 deletions(-) create mode 100644 moto/dynamodb2/parsing/executors.py create mode 100644 tests/test_dynamodb2/test_dynamodb_executor.py diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py index 5dd87ef6b..18e498a90 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb2/exceptions.py @@ -39,6 +39,17 @@ class AttributeDoesNotExist(MockValidationException): super(AttributeDoesNotExist, self).__init__(self.attr_does_not_exist_msg) +class ProvidedKeyDoesNotExist(MockValidationException): + provided_key_does_not_exist_msg = ( + "The provided key element does not match the schema" + ) + + def __init__(self): + super(ProvidedKeyDoesNotExist, self).__init__( + self.provided_key_does_not_exist_msg + ) + + class ExpressionAttributeNameNotDefined(InvalidUpdateExpression): name_not_defined_msg = "An expression attribute name used in the document path is not defined; attribute name: {n}" @@ -131,3 +142,10 @@ class IncorrectOperandType(InvalidUpdateExpression): super(IncorrectOperandType, self).__init__( self.inv_operand_msg.format(f=operator_or_function, t=operand_type) ) + + +class IncorrectDataType(MockValidationException): + inc_data_type_msg = "An operand in the update expression has an incorrect data type" + + def __init__(self): + super(IncorrectDataType, self).__init__(self.inc_data_type_msg) diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 3ddbcbc54..33ee1747d 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -8,7 +8,6 @@ import re import uuid from boto3 import Session -from botocore.exceptions import ParamValidationError from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -20,8 +19,9 @@ from moto.dynamodb2.exceptions import ( ItemSizeTooLarge, ItemSizeToUpdateTooLarge, ) -from moto.dynamodb2.models.utilities import bytesize, attribute_is_list +from moto.dynamodb2.models.utilities import bytesize from moto.dynamodb2.models.dynamo_type import DynamoType +from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor from moto.dynamodb2.parsing.expressions import UpdateExpressionParser from moto.dynamodb2.parsing.validators import UpdateExpressionValidator @@ -71,6 +71,17 @@ class Item(BaseModel): for key, value in attrs.items(): self.attrs[key] = DynamoType(value) + def __eq__(self, other): + return all( + [ + self.hash_key == other.hash_key, + self.hash_key_type == other.hash_key_type, + self.range_key == other.range_key, + self.range_key_type == other.range_key_type, + self.attrs == other.attrs, + ] + ) + def __repr__(self): return "Item: {0}".format(self.to_json()) @@ -94,192 +105,6 @@ class Item(BaseModel): included = self.attrs return {"Item": included} - def update( - self, update_expression, expression_attribute_names, expression_attribute_values - ): - # Update subexpressions are identifiable by the operator keyword, so split on that and - # get rid of the empty leading string. - parts = [ - p - for p in re.split( - r"\b(SET|REMOVE|ADD|DELETE)\b", update_expression, flags=re.I - ) - if p - ] - # make sure that we correctly found only operator/value pairs - assert ( - len(parts) % 2 == 0 - ), "Mismatched operators and values in update expression: '{}'".format( - update_expression - ) - for action, valstr in zip(parts[:-1:2], parts[1::2]): - action = action.upper() - - # "Should" retain arguments in side (...) - values = re.split(r",(?![^(]*\))", valstr) - for value in values: - # A Real value - value = value.lstrip(":").rstrip(",").strip() - for k, v in expression_attribute_names.items(): - value = re.sub(r"{0}\b".format(k), v, value) - - if action == "REMOVE": - key = value - attr, list_index = attribute_is_list(key.split(".")[0]) - if "." not in key: - if list_index: - new_list = DynamoType(self.attrs[attr]) - new_list.delete(None, list_index) - self.attrs[attr] = new_list - else: - self.attrs.pop(value, None) - else: - # Handle nested dict updates - self.attrs[attr].delete(".".join(key.split(".")[1:])) - elif action == "SET": - key, value = value.split("=", 1) - key = key.strip() - value = value.strip() - - # check whether key is a list - attr, list_index = attribute_is_list(key.split(".")[0]) - # If value not exists, changes value to a default if needed, else its the same as it was - value = self._get_default(value) - # If operation == list_append, get the original value and append it - value = self._get_appended_list(value, expression_attribute_values) - - if type(value) != DynamoType: - if value in expression_attribute_values: - dyn_value = DynamoType(expression_attribute_values[value]) - else: - dyn_value = DynamoType({"S": value}) - else: - dyn_value = value - - if "." in key and attr not in self.attrs: - raise ValueError # Setting nested attr not allowed if first attr does not exist yet - elif attr not in self.attrs: - try: - self.attrs[attr] = dyn_value # set new top-level attribute - except ItemSizeTooLarge: - raise ItemSizeToUpdateTooLarge() - else: - self.attrs[attr].set( - ".".join(key.split(".")[1:]), dyn_value, list_index - ) # set value recursively - - elif action == "ADD": - key, value = value.split(" ", 1) - key = key.strip() - value_str = value.strip() - if value_str in expression_attribute_values: - dyn_value = DynamoType(expression_attribute_values[value]) - else: - raise TypeError - - # Handle adding numbers - value gets added to existing value, - # or added to 0 if it doesn't exist yet - if dyn_value.is_number(): - existing = self.attrs.get(key, DynamoType({"N": "0"})) - if not existing.same_type(dyn_value): - raise TypeError() - self.attrs[key] = DynamoType( - { - "N": str( - decimal.Decimal(existing.value) - + decimal.Decimal(dyn_value.value) - ) - } - ) - - # Handle adding sets - value is added to the set, or set is - # created with only this value if it doesn't exist yet - # New value must be of same set type as previous value - elif dyn_value.is_set(): - key_head = key.split(".")[0] - key_tail = ".".join(key.split(".")[1:]) - if key_head not in self.attrs: - self.attrs[key_head] = DynamoType({dyn_value.type: {}}) - existing = self.attrs.get(key_head) - existing = existing.get(key_tail) - if existing.value and not existing.same_type(dyn_value): - raise TypeError() - new_set = set(existing.value or []).union(dyn_value.value) - existing.set( - key=None, - new_value=DynamoType({dyn_value.type: list(new_set)}), - ) - else: # Number and Sets are the only supported types for ADD - raise TypeError - - elif action == "DELETE": - key, value = value.split(" ", 1) - key = key.strip() - value_str = value.strip() - if value_str in expression_attribute_values: - dyn_value = DynamoType(expression_attribute_values[value]) - else: - raise TypeError - - if not dyn_value.is_set(): - raise TypeError - key_head = key.split(".")[0] - key_tail = ".".join(key.split(".")[1:]) - existing = self.attrs.get(key_head) - existing = existing.get(key_tail) - if existing: - if not existing.same_type(dyn_value): - raise TypeError - new_set = set(existing.value).difference(dyn_value.value) - existing.set( - key=None, - new_value=DynamoType({existing.type: list(new_set)}), - ) - else: - raise NotImplementedError( - "{} update action not yet supported".format(action) - ) - - def _get_appended_list(self, value, expression_attribute_values): - if type(value) != DynamoType: - list_append_re = re.match("list_append\\((.+),(.+)\\)", value) - if list_append_re: - new_value = expression_attribute_values[list_append_re.group(2).strip()] - old_list_key = list_append_re.group(1) - # old_key could be a function itself (if_not_exists) - if old_list_key.startswith("if_not_exists"): - old_list = self._get_default(old_list_key) - if not isinstance(old_list, DynamoType): - old_list = DynamoType(expression_attribute_values[old_list]) - else: - old_list = self.attrs[old_list_key.split(".")[0]] - if "." in old_list_key: - # Value is nested inside a map - find the appropriate child attr - old_list = old_list.child_attr( - ".".join(old_list_key.split(".")[1:]) - ) - if not old_list.is_list(): - raise ParamValidationError - old_list.value.extend([DynamoType(v) for v in new_value["L"]]) - value = old_list - return value - - def _get_default(self, value): - if value.startswith("if_not_exists"): - # Function signature - match = re.match( - r".*if_not_exists\s*\((?P.+),\s*(?P.+)\).*", value - ) - if not match: - raise TypeError - - path, value = match.groups() - - # If it already exists, get its value so we dont overwrite it - if path in self.attrs: - value = self.attrs[path] - return value - def update_with_attribute_updates(self, attribute_updates): for attribute_name, update_action in attribute_updates.items(): action = update_action["Action"] @@ -1266,17 +1091,18 @@ class DynamoDBBackend(BaseBackend): item = table.get_item(hash_value, range_value) if update_expression: - UpdateExpressionValidator( + validated_ast = UpdateExpressionValidator( update_expression_ast, expression_attribute_names=expression_attribute_names, expression_attribute_values=expression_attribute_values, item=item, ).validate() - item.update( - update_expression, - expression_attribute_names, - expression_attribute_values, - ) + try: + UpdateExpressionExecutor( + validated_ast, item, expression_attribute_names + ).execute() + except ItemSizeTooLarge: + raise ItemSizeToUpdateTooLarge() else: item.update_with_attribute_updates(attribute_updates) if table.stream_shard is not None: diff --git a/moto/dynamodb2/models/dynamo_type.py b/moto/dynamodb2/models/dynamo_type.py index a3199dcaa..1fc1bcef3 100644 --- a/moto/dynamodb2/models/dynamo_type.py +++ b/moto/dynamodb2/models/dynamo_type.py @@ -1,10 +1,53 @@ import six from moto.dynamodb2.comparisons import get_comparison_func -from moto.dynamodb2.exceptions import InvalidUpdateExpression +from moto.dynamodb2.exceptions import InvalidUpdateExpression, IncorrectDataType from moto.dynamodb2.models.utilities import attribute_is_list, bytesize +class DDBType(object): + """ + Official documentation at https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_AttributeValue.html + """ + + BINARY_SET = "BS" + NUMBER_SET = "NS" + STRING_SET = "SS" + STRING = "S" + NUMBER = "N" + MAP = "M" + LIST = "L" + BOOLEAN = "BOOL" + BINARY = "B" + NULL = "NULL" + + +class DDBTypeConversion(object): + _human_type_mapping = { + val: key.replace("_", " ") + for key, val in DDBType.__dict__.items() + if key.upper() == key + } + + @classmethod + def get_human_type(cls, abbreviated_type): + """ + Args: + abbreviated_type(str): An attribute of DDBType + + Returns: + str: The human readable form of the DDBType. + """ + try: + human_type_str = cls._human_type_mapping[abbreviated_type] + except KeyError: + raise ValueError( + "Invalid abbreviated_type {at}".format(at=abbreviated_type) + ) + + return human_type_str + + class DynamoType(object): """ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html#DataModelDataTypes @@ -50,13 +93,22 @@ class DynamoType(object): self.value = new_value.value else: if attr not in self.value: # nonexistingattribute - type_of_new_attr = "M" if "." in key else new_value.type + type_of_new_attr = DDBType.MAP if "." in key else new_value.type self.value[attr] = DynamoType({type_of_new_attr: {}}) # {'M': {'foo': DynamoType}} ==> DynamoType.set(new_value) self.value[attr].set( ".".join(key.split(".")[1:]), new_value, list_index ) + def __contains__(self, item): + if self.type == DDBType.STRING: + return False + try: + self.__getitem__(item) + return True + except KeyError: + return False + def delete(self, key, index=None): if index: if not key: @@ -126,27 +178,35 @@ class DynamoType(object): def __add__(self, other): if self.type != other.type: raise TypeError("Different types of operandi is not allowed.") - if self.type == "N": - return DynamoType({"N": "{v}".format(v=int(self.value) + int(other.value))}) + if self.is_number(): + self_value = float(self.value) if "." in self.value else int(self.value) + other_value = float(other.value) if "." in other.value else int(other.value) + return DynamoType( + {DDBType.NUMBER: "{v}".format(v=self_value + other_value)} + ) else: - raise TypeError("Sum only supported for Numbers.") + raise IncorrectDataType() def __sub__(self, other): if self.type != other.type: raise TypeError("Different types of operandi is not allowed.") - if self.type == "N": - return DynamoType({"N": "{v}".format(v=int(self.value) - int(other.value))}) + if self.type == DDBType.NUMBER: + self_value = float(self.value) if "." in self.value else int(self.value) + other_value = float(other.value) if "." in other.value else int(other.value) + return DynamoType( + {DDBType.NUMBER: "{v}".format(v=self_value - other_value)} + ) else: raise TypeError("Sum only supported for Numbers.") def __getitem__(self, item): if isinstance(item, six.string_types): # If our DynamoType is a map it should be subscriptable with a key - if self.type == "M": + if self.type == DDBType.MAP: return self.value[item] elif isinstance(item, int): # If our DynamoType is a list is should be subscriptable with an index - if self.type == "L": + if self.type == DDBType.LIST: return self.value[item] raise TypeError( "This DynamoType {dt} is not subscriptable by a {it}".format( @@ -154,6 +214,20 @@ class DynamoType(object): ) ) + def __setitem__(self, key, value): + if isinstance(key, int): + if self.is_list(): + if key >= len(self.value): + # DynamoDB doesn't care you are out of box just add it to the end. + self.value.append(value) + else: + self.value[key] = value + elif isinstance(key, six.string_types): + if self.is_map(): + self.value[key] = value + else: + raise NotImplementedError("No set_item for {t}".format(t=type(key))) + @property def cast_value(self): if self.is_number(): @@ -222,16 +296,22 @@ class DynamoType(object): return comparison_func(self.cast_value, *range_values) def is_number(self): - return self.type == "N" + return self.type == DDBType.NUMBER def is_set(self): - return self.type == "SS" or self.type == "NS" or self.type == "BS" + return self.type in (DDBType.STRING_SET, DDBType.NUMBER_SET, DDBType.BINARY_SET) def is_list(self): - return self.type == "L" + return self.type == DDBType.LIST def is_map(self): - return self.type == "M" + return self.type == DDBType.MAP def same_type(self, other): return self.type == other.type + + def pop(self, key, *args, **kwargs): + if self.is_map() or self.is_list(): + self.value.pop(key, *args, **kwargs) + else: + raise TypeError("pop not supported for DynamoType {t}".format(t=self.type)) diff --git a/moto/dynamodb2/parsing/executors.py b/moto/dynamodb2/parsing/executors.py new file mode 100644 index 000000000..8c51c9cec --- /dev/null +++ b/moto/dynamodb2/parsing/executors.py @@ -0,0 +1,262 @@ +from abc import abstractmethod + +from moto.dynamodb2.exceptions import IncorrectOperandType, IncorrectDataType +from moto.dynamodb2.models import DynamoType +from moto.dynamodb2.models.dynamo_type import DDBTypeConversion, DDBType +from moto.dynamodb2.parsing.ast_nodes import ( + UpdateExpressionSetAction, + UpdateExpressionDeleteAction, + UpdateExpressionRemoveAction, + UpdateExpressionAddAction, + UpdateExpressionPath, + DDBTypedValue, + ExpressionAttribute, + ExpressionSelector, + ExpressionAttributeName, +) +from moto.dynamodb2.parsing.validators import ExpressionPathResolver + + +class NodeExecutor(object): + def __init__(self, ast_node, expression_attribute_names): + self.node = ast_node + self.expression_attribute_names = expression_attribute_names + + @abstractmethod + def execute(self, item): + pass + + def get_item_part_for_path_nodes(self, item, path_nodes): + """ + For a list of path nodes travers the item by following the path_nodes + Args: + item(Item): + path_nodes(list): + + Returns: + + """ + if len(path_nodes) == 0: + return item.attrs + else: + return ExpressionPathResolver( + self.expression_attribute_names + ).resolve_expression_path_nodes_to_dynamo_type(item, path_nodes) + + def get_item_before_end_of_path(self, item): + """ + Get the part ot the item where the item will perform the action. For most actions this should be the parent. As + that element will need to be modified by the action. + Args: + item(Item): + + Returns: + DynamoType or dict: The path to be set + """ + return self.get_item_part_for_path_nodes( + item, self.get_path_expression_nodes()[:-1] + ) + + def get_item_at_end_of_path(self, item): + """ + For a DELETE the path points at the stringset so we need to evaluate the full path. + Args: + item(Item): + + Returns: + DynamoType or dict: The path to be set + """ + return self.get_item_part_for_path_nodes(item, self.get_path_expression_nodes()) + + # Get the part ot the item where the item will perform the action. For most actions this should be the parent. As + # that element will need to be modified by the action. + get_item_part_in_which_to_perform_action = get_item_before_end_of_path + + def get_path_expression_nodes(self): + update_expression_path = self.node.children[0] + assert isinstance(update_expression_path, UpdateExpressionPath) + return update_expression_path.children + + def get_element_to_action(self): + return self.get_path_expression_nodes()[-1] + + def get_action_value(self): + """ + + Returns: + DynamoType: The value to be set + """ + ddb_typed_value = self.node.children[1] + assert isinstance(ddb_typed_value, DDBTypedValue) + dynamo_type_value = ddb_typed_value.children[0] + assert isinstance(dynamo_type_value, DynamoType) + return dynamo_type_value + + +class SetExecutor(NodeExecutor): + def execute(self, item): + self.set( + item_part_to_modify_with_set=self.get_item_part_in_which_to_perform_action( + item + ), + element_to_set=self.get_element_to_action(), + value_to_set=self.get_action_value(), + expression_attribute_names=self.expression_attribute_names, + ) + + @classmethod + def set( + cls, + item_part_to_modify_with_set, + element_to_set, + value_to_set, + expression_attribute_names, + ): + if isinstance(element_to_set, ExpressionAttribute): + attribute_name = element_to_set.get_attribute_name() + item_part_to_modify_with_set[attribute_name] = value_to_set + elif isinstance(element_to_set, ExpressionSelector): + index = element_to_set.get_index() + item_part_to_modify_with_set[index] = value_to_set + elif isinstance(element_to_set, ExpressionAttributeName): + attribute_name = expression_attribute_names[ + element_to_set.get_attribute_name_placeholder() + ] + item_part_to_modify_with_set[attribute_name] = value_to_set + else: + raise NotImplementedError( + "Moto does not support setting {t} yet".format(t=type(element_to_set)) + ) + + +class DeleteExecutor(NodeExecutor): + operator = "operator: DELETE" + + def execute(self, item): + string_set_to_remove = self.get_action_value() + assert isinstance(string_set_to_remove, DynamoType) + if not string_set_to_remove.is_set(): + raise IncorrectOperandType( + self.operator, + DDBTypeConversion.get_human_type(string_set_to_remove.type), + ) + + string_set = self.get_item_at_end_of_path(item) + assert isinstance(string_set, DynamoType) + if string_set.type != string_set_to_remove.type: + raise IncorrectDataType() + # String set is currently implemented as a list + string_set_list = string_set.value + + stringset_to_remove_list = string_set_to_remove.value + + for value in stringset_to_remove_list: + try: + string_set_list.remove(value) + except (KeyError, ValueError): + # DynamoDB does not mind if value is not present + pass + + +class RemoveExecutor(NodeExecutor): + def execute(self, item): + element_to_remove = self.get_element_to_action() + if isinstance(element_to_remove, ExpressionAttribute): + attribute_name = element_to_remove.get_attribute_name() + self.get_item_part_in_which_to_perform_action(item).pop( + attribute_name, None + ) + elif isinstance(element_to_remove, ExpressionAttributeName): + attribute_name = self.expression_attribute_names[ + element_to_remove.get_attribute_name_placeholder() + ] + self.get_item_part_in_which_to_perform_action(item).pop( + attribute_name, None + ) + elif isinstance(element_to_remove, ExpressionSelector): + index = element_to_remove.get_index() + try: + self.get_item_part_in_which_to_perform_action(item).pop(index) + except IndexError: + # DynamoDB does not care that index is out of bounds, it will just do nothing. + pass + else: + raise NotImplementedError( + "Moto does not support setting {t} yet".format( + t=type(element_to_remove) + ) + ) + + +class AddExecutor(NodeExecutor): + def execute(self, item): + value_to_add = self.get_action_value() + if isinstance(value_to_add, DynamoType): + if value_to_add.is_set(): + current_string_set = self.get_item_at_end_of_path(item) + assert isinstance(current_string_set, DynamoType) + if not current_string_set.type == value_to_add.type: + raise IncorrectDataType() + # Sets are implemented as list + for value in value_to_add.value: + if value in current_string_set.value: + continue + else: + current_string_set.value.append(value) + elif value_to_add.type == DDBType.NUMBER: + existing_value = self.get_item_at_end_of_path(item) + assert isinstance(existing_value, DynamoType) + if not existing_value.type == DDBType.NUMBER: + raise IncorrectDataType() + new_value = existing_value + value_to_add + SetExecutor.set( + item_part_to_modify_with_set=self.get_item_before_end_of_path(item), + element_to_set=self.get_element_to_action(), + value_to_set=new_value, + expression_attribute_names=self.expression_attribute_names, + ) + else: + raise IncorrectDataType() + + +class UpdateExpressionExecutor(object): + execution_map = { + UpdateExpressionSetAction: SetExecutor, + UpdateExpressionAddAction: AddExecutor, + UpdateExpressionRemoveAction: RemoveExecutor, + UpdateExpressionDeleteAction: DeleteExecutor, + } + + def __init__(self, update_ast, item, expression_attribute_names): + self.update_ast = update_ast + self.item = item + self.expression_attribute_names = expression_attribute_names + + def execute(self, node=None): + """ + As explained in moto.dynamodb2.parsing.expressions.NestableExpressionParserMixin._create_node the order of nodes + in the AST can be translated of the order of statements in the expression. As such we can start at the root node + and process the nodes 1-by-1. If no specific execution for the node type is defined we can execute the children + in order since it will be a container node that is expandable and left child will be first in the statement. + + Args: + node(Node): + + Returns: + None + """ + if node is None: + node = self.update_ast + + node_executor = self.get_specific_execution(node) + if node_executor is None: + for node in node.children: + self.execute(node) + else: + node_executor(node, self.expression_attribute_names).execute(self.item) + + def get_specific_execution(self, node): + for node_class in self.execution_map: + if isinstance(node, node_class): + return self.execution_map[node_class] + return None diff --git a/moto/dynamodb2/parsing/validators.py b/moto/dynamodb2/parsing/validators.py index 180c7a874..f924a713c 100644 --- a/moto/dynamodb2/parsing/validators.py +++ b/moto/dynamodb2/parsing/validators.py @@ -11,6 +11,7 @@ from moto.dynamodb2.exceptions import ( ExpressionAttributeNameNotDefined, IncorrectOperandType, InvalidUpdateExpressionInvalidDocumentPath, + ProvidedKeyDoesNotExist, ) from moto.dynamodb2.models import DynamoType from moto.dynamodb2.parsing.ast_nodes import ( @@ -56,6 +57,76 @@ class ExpressionAttributeValueProcessor(DepthFirstTraverser): return DDBTypedValue(DynamoType(target)) +class ExpressionPathResolver(object): + def __init__(self, expression_attribute_names): + self.expression_attribute_names = expression_attribute_names + + @classmethod + def raise_exception_if_keyword(cls, attribute): + if attribute.upper() in ReservedKeywords.get_reserved_keywords(): + raise AttributeIsReservedKeyword(attribute) + + def resolve_expression_path(self, item, update_expression_path): + assert isinstance(update_expression_path, UpdateExpressionPath) + return self.resolve_expression_path_nodes(item, update_expression_path.children) + + def resolve_expression_path_nodes(self, item, update_expression_path_nodes): + target = item.attrs + + for child in update_expression_path_nodes: + # First replace placeholder with attribute_name + attr_name = None + if isinstance(child, ExpressionAttributeName): + attr_placeholder = child.get_attribute_name_placeholder() + try: + attr_name = self.expression_attribute_names[attr_placeholder] + except KeyError: + raise ExpressionAttributeNameNotDefined(attr_placeholder) + elif isinstance(child, ExpressionAttribute): + attr_name = child.get_attribute_name() + self.raise_exception_if_keyword(attr_name) + if attr_name is not None: + # Resolv attribute_name + try: + target = target[attr_name] + except (KeyError, TypeError): + if child == update_expression_path_nodes[-1]: + return NoneExistingPath(creatable=True) + return NoneExistingPath() + else: + if isinstance(child, ExpressionPathDescender): + continue + elif isinstance(child, ExpressionSelector): + index = child.get_index() + if target.is_list(): + try: + target = target[index] + except IndexError: + # When a list goes out of bounds when assigning that is no problem when at the assignment + # side. It will just append to the list. + if child == update_expression_path_nodes[-1]: + return NoneExistingPath(creatable=True) + return NoneExistingPath() + else: + raise InvalidUpdateExpressionInvalidDocumentPath + else: + raise NotImplementedError( + "Path resolution for {t}".format(t=type(child)) + ) + if not isinstance(target, DynamoType): + print(target) + return DDBTypedValue(target) + + def resolve_expression_path_nodes_to_dynamo_type( + self, item, update_expression_path_nodes + ): + node = self.resolve_expression_path_nodes(item, update_expression_path_nodes) + if isinstance(node, NoneExistingPath): + raise ProvidedKeyDoesNotExist() + assert isinstance(node, DDBTypedValue) + return node.get_value() + + class ExpressionAttributeResolvingProcessor(DepthFirstTraverser): def _processing_map(self): return { @@ -107,55 +178,9 @@ class ExpressionAttributeResolvingProcessor(DepthFirstTraverser): return node def resolve_expression_path(self, node): - assert isinstance(node, UpdateExpressionPath) - - target = deepcopy(self.item.attrs) - for child in node.children: - # First replace placeholder with attribute_name - attr_name = None - if isinstance(child, ExpressionAttributeName): - attr_placeholder = child.get_attribute_name_placeholder() - try: - attr_name = self.expression_attribute_names[attr_placeholder] - except KeyError: - raise ExpressionAttributeNameNotDefined(attr_placeholder) - elif isinstance(child, ExpressionAttribute): - attr_name = child.get_attribute_name() - self.raise_exception_if_keyword(attr_name) - if attr_name is not None: - # Resolv attribute_name - try: - target = target[attr_name] - except (KeyError, TypeError): - if child == node.children[-1]: - return NoneExistingPath(creatable=True) - return NoneExistingPath() - else: - if isinstance(child, ExpressionPathDescender): - continue - elif isinstance(child, ExpressionSelector): - index = child.get_index() - if target.is_list(): - try: - target = target[index] - except IndexError: - # When a list goes out of bounds when assigning that is no problem when at the assignment - # side. It will just append to the list. - if child == node.children[-1]: - return NoneExistingPath(creatable=True) - return NoneExistingPath() - else: - raise InvalidUpdateExpressionInvalidDocumentPath - else: - raise NotImplementedError( - "Path resolution for {t}".format(t=type(child)) - ) - return DDBTypedValue(DynamoType(target)) - - @classmethod - def raise_exception_if_keyword(cls, attribute): - if attribute.upper() in ReservedKeywords.get_reserved_keywords(): - raise AttributeIsReservedKeyword(attribute) + return ExpressionPathResolver( + self.expression_attribute_names + ).resolve_expression_path(self.item, node) class UpdateExpressionFunctionEvaluator(DepthFirstTraverser): @@ -183,7 +208,9 @@ class UpdateExpressionFunctionEvaluator(DepthFirstTraverser): assert isinstance(result, (DDBTypedValue, NoneExistingPath)) return result elif function_name == "list_append": - first_arg = self.get_list_from_ddb_typed_value(first_arg, function_name) + first_arg = deepcopy( + self.get_list_from_ddb_typed_value(first_arg, function_name) + ) second_arg = self.get_list_from_ddb_typed_value(second_arg, function_name) for list_element in second_arg.value: first_arg.value.append(list_element) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 089782e77..b1bf18f0a 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1,21 +1,17 @@ from __future__ import unicode_literals, print_function -import re from decimal import Decimal -import six import boto import boto3 from boto3.dynamodb.conditions import Attr, Key import re -import requests import sure # noqa from moto import mock_dynamodb2, mock_dynamodb2_deprecated from moto.dynamodb2 import dynamodb_backend2, dynamodb_backends2 from boto.exception import JSONResponseError from botocore.exceptions import ClientError, ParamValidationError from tests.helpers import requires_boto_gte -import tests.backport_assert_raises import moto.dynamodb2.comparisons import moto.dynamodb2.models @@ -3221,6 +3217,25 @@ def test_remove_top_level_attribute(): result.should.equal({"id": {"S": "foo"}}) +@mock_dynamodb2 +def test_remove_top_level_attribute_non_existent(): + """ + Remove statements do not require attribute to exist they silently pass + """ + table_name = "test_remove" + client = create_table_with_list(table_name) + ddb_item = {"id": {"S": "foo"}, "item": {"S": "bar"}} + client.put_item(TableName=table_name, Item=ddb_item) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="REMOVE non_existent_attribute", + ExpressionAttributeNames={"#i": "item"}, + ) + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + result.should.equal(ddb_item) + + @mock_dynamodb2 def test_remove_list_index__remove_existing_index(): table_name = "test_list_index_access" @@ -4331,3 +4346,251 @@ def test_list_tables_exclusive_start_table_name_empty(): resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever") len(resp["TableNames"]).should.equal(0) + + +def assert_correct_client_error( + client_error, code, message_template, message_values=None, braces=None +): + """ + Assert whether a client_error is as expected. Allow for a list of values to be passed into the message + + Args: + client_error(ClientError): The ClientError exception that was raised + code(str): The code for the error (e.g. ValidationException) + message_template(str): Error message template. if message_values is not None then this template has a {values} + as placeholder. For example: + 'Value provided in ExpressionAttributeValues unused in expressions: keys: {values}' + message_values(list of str|None): The values that are passed in the error message + braces(list of str|None): List of length 2 with opening and closing brace for the values. By default it will be + surrounded by curly brackets + """ + braces = braces or ["{", "}"] + assert client_error.response["Error"]["Code"] == code + if message_values is not None: + values_string = "{open_brace}(?P.*){close_brace}".format( + open_brace=braces[0], close_brace=braces[1] + ) + re_msg = re.compile(message_template.format(values=values_string)) + match_result = re_msg.match(client_error.response["Error"]["Message"]) + assert match_result is not None + values_string = match_result.groupdict()["values"] + values = [key for key in values_string.split(", ")] + assert len(message_values) == len(values) + for value in message_values: + assert value in values + else: + assert client_error.response["Error"]["Message"] == message_template + + +def create_simple_table_and_return_client(): + dynamodb = boto3.client("dynamodb", region_name="eu-west-1") + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"},], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "MyStr": {"S": "1"},}, + ) + return dynamodb + + +# https://github.com/spulec/moto/issues/2806 +# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html +# #DDB-UpdateItem-request-UpdateExpression +@mock_dynamodb2 +def test_update_item_with_attribute_in_right_hand_side_and_operation(): + dynamodb = create_simple_table_and_return_client() + + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myNum = myNum+:val", + ExpressionAttributeValues={":val": {"N": "3"}}, + ) + + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) + assert result["Item"]["myNum"]["N"] == "4" + + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myNum = myNum - :val", + ExpressionAttributeValues={":val": {"N": "1"}}, + ) + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) + assert result["Item"]["myNum"]["N"] == "3" + + +@mock_dynamodb2 +def test_non_existing_attribute_should_raise_exception(): + """ + Does error message get correctly raised if attribute is referenced but it does not exist for the item. + """ + dynamodb = create_simple_table_and_return_client() + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = no_attr + MyStr", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "The provided expression refers to an attribute that does not exist in the item", + ) + + +@mock_dynamodb2 +def test_update_expression_with_plus_in_attribute_name(): + """ + Does error message get correctly raised if attribute contains a plus and is passed in without an AttributeName. And + lhs & rhs are not attribute IDs by themselve. + """ + dynamodb = create_simple_table_and_return_client() + + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "my+Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, + ) + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = my+Num", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "The provided expression refers to an attribute that does not exist in the item", + ) + + +@mock_dynamodb2 +def test_update_expression_with_minus_in_attribute_name(): + """ + Does error message get correctly raised if attribute contains a minus and is passed in without an AttributeName. And + lhs & rhs are not attribute IDs by themselve. + """ + dynamodb = create_simple_table_and_return_client() + + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "my-Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, + ) + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = my-Num", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "The provided expression refers to an attribute that does not exist in the item", + ) + + +@mock_dynamodb2 +def test_update_expression_with_space_in_attribute_name(): + """ + Does error message get correctly raised if attribute contains a space and is passed in without an AttributeName. And + lhs & rhs are not attribute IDs by themselves. + """ + dynamodb = create_simple_table_and_return_client() + + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "my Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, + ) + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = my Num", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_raise_syntax_error(e, "Num", "my Num") + + +@mock_dynamodb2 +def test_summing_up_2_strings_raises_exception(): + """ + Update set supports different DynamoDB types but some operations are not supported. For example summing up 2 strings + raises an exception. It results in ClientError with code ValidationException: + Saying An operand in the update expression has an incorrect data type + """ + dynamodb = create_simple_table_and_return_client() + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = MyStr + MyStr", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "An operand in the update expression has an incorrect data type", + ) + + +# https://github.com/spulec/moto/issues/2806 +@mock_dynamodb2 +def test_update_item_with_attribute_in_right_hand_side(): + """ + After tokenization and building expression make sure referenced attributes are replaced with their current value + """ + dynamodb = create_simple_table_and_return_client() + + # Make sure there are 2 values + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "myVal1": {"S": "Value1"}, "myVal2": {"S": "Value2"}}, + ) + + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myVal1 = myVal2", + ) + + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) + assert result["Item"]["myVal1"]["S"] == result["Item"]["myVal2"]["S"] == "Value2" + + +@mock_dynamodb2 +def test_multiple_updates(): + dynamodb = create_simple_table_and_return_client() + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "path": {"N": "6"}}, + ) + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myNum = #p + :val, newAttr = myNum", + ExpressionAttributeValues={":val": {"N": "1"}}, + ExpressionAttributeNames={"#p": "path"}, + ) + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})["Item"] + expected_result = { + "myNum": {"N": "7"}, + "newAttr": {"N": "1"}, + "path": {"N": "6"}, + "id": {"S": "1"}, + } + assert result == expected_result diff --git a/tests/test_dynamodb2/test_dynamodb_executor.py b/tests/test_dynamodb2/test_dynamodb_executor.py new file mode 100644 index 000000000..4ef0bb423 --- /dev/null +++ b/tests/test_dynamodb2/test_dynamodb_executor.py @@ -0,0 +1,446 @@ +from moto.dynamodb2.exceptions import IncorrectOperandType, IncorrectDataType +from moto.dynamodb2.models import Item, DynamoType +from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor +from moto.dynamodb2.parsing.expressions import UpdateExpressionParser +from moto.dynamodb2.parsing.validators import UpdateExpressionValidator +from parameterized import parameterized + + +def test_execution_of_if_not_exists_not_existing_value(): + update_expression = "SET a = if_not_exists(b, a)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "A"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "A"}}, + ) + assert expected_item == item + + +def test_execution_of_if_not_exists_with_existing_attribute_should_return_attribute(): + update_expression = "SET a = if_not_exists(b, a)" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "A"}, "b": {"S": "B"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"S": "B"}, "b": {"S": "B"}}, + ) + assert expected_item == item + + +def test_execution_of_if_not_exists_with_existing_attribute_should_return_value(): + update_expression = "SET a = if_not_exists(b, :val)" + update_expression_values = {":val": {"N": "4"}} + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "b": {"N": "3"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=update_expression_values, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "b": {"N": "3"}, "a": {"N": "3"}}, + ) + assert expected_item == item + + +def test_execution_of_if_not_exists_with_non_existing_attribute_should_return_value(): + update_expression = "SET a = if_not_exists(b, :val)" + update_expression_values = {":val": {"N": "4"}} + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=update_expression_values, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"N": "4"}}, + ) + assert expected_item == item + + +def test_execution_of_sum_operation(): + update_expression = "SET a = a + b" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"N": "7"}, "b": {"N": "4"}}, + ) + assert expected_item == item + + +def test_execution_of_remove(): + update_expression = "Remove a" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "1"}, "b": {"N": "4"}}, + ) + assert expected_item == item + + +def test_execution_of_remove_in_map(): + update_expression = "Remove itemmap.itemlist[1].foo11" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={ + "id": {"S": "foo2"}, + "itemmap": { + "M": { + "itemlist": { + "L": [ + {"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}}, + {"M": {"foo10": {"S": "bar1"}, "foo11": {"S": "bar2"}}}, + ] + } + } + }, + }, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={ + "id": {"S": "foo2"}, + "itemmap": { + "M": { + "itemlist": { + "L": [ + {"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}}, + {"M": {"foo10": {"S": "bar1"},}}, + ] + } + } + }, + }, + ) + assert expected_item == item + + +def test_execution_of_remove_in_list(): + update_expression = "Remove itemmap.itemlist[1]" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={ + "id": {"S": "foo2"}, + "itemmap": { + "M": { + "itemlist": { + "L": [ + {"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}}, + {"M": {"foo10": {"S": "bar1"}, "foo11": {"S": "bar2"}}}, + ] + } + } + }, + }, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=None, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={ + "id": {"S": "foo2"}, + "itemmap": { + "M": { + "itemlist": { + "L": [{"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}},] + } + } + }, + }, + ) + assert expected_item == item + + +def test_execution_of_delete_element_from_set(): + update_expression = "delete s :value" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value2", "value3"]},}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":value": {"SS": ["value2", "value5"]}}, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value3"]},}, + ) + assert expected_item == item + + +def test_execution_of_add_number(): + update_expression = "add s :value" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"N": "5"},}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":value": {"N": "10"}}, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"N": "15"}}, + ) + assert expected_item == item + + +def test_execution_of_add_set_to_a_number(): + update_expression = "add s :value" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"N": "5"},}, + ) + try: + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":value": {"SS": ["s1"]}}, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"N": "15"}}, + ) + assert expected_item == item + assert False + except IncorrectDataType: + assert True + + +def test_execution_of_add_to_a_set(): + update_expression = "ADD s :value" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value2", "value3"]},}, + ) + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":value": {"SS": ["value2", "value5"]}}, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + expected_item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={ + "id": {"S": "foo2"}, + "s": {"SS": ["value1", "value2", "value3", "value5"]}, + }, + ) + assert expected_item == item + + +@parameterized( + [ + ({":value": {"S": "10"}}, "STRING",), + ({":value": {"N": "10"}}, "NUMBER",), + ({":value": {"B": "10"}}, "BINARY",), + ({":value": {"BOOL": True}}, "BOOLEAN",), + ({":value": {"NULL": True}}, "NULL",), + ({":value": {"M": {"el0": {"S": "10"}}}}, "MAP",), + ({":value": {"L": []}}, "LIST",), + ] +) +def test_execution_of__delete_element_from_set_invalid_value( + expression_attribute_values, unexpected_data_type +): + """A delete statement must use a value of type SS in order to delete elements from a set.""" + update_expression = "delete s :value" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value2", "value3"]},}, + ) + try: + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values=expression_attribute_values, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + assert False, "Must raise exception" + except IncorrectOperandType as e: + assert e.operator_or_function == "operator: DELETE" + assert e.operand_type == unexpected_data_type + + +def test_execution_of_delete_element_from_a_string_attribute(): + """A delete statement must use a value of type SS in order to delete elements from a set.""" + update_expression = "delete s :value" + update_expression_ast = UpdateExpressionParser.make(update_expression) + item = Item( + hash_key=DynamoType({"S": "id"}), + hash_key_type="TYPE", + range_key=None, + range_key_type=None, + attrs={"id": {"S": "foo2"}, "s": {"S": "5"},}, + ) + try: + validated_ast = UpdateExpressionValidator( + update_expression_ast, + expression_attribute_names=None, + expression_attribute_values={":value": {"SS": ["value2"]}}, + item=item, + ).validate() + UpdateExpressionExecutor(validated_ast, item, None).execute() + assert False, "Must raise exception" + except IncorrectDataType: + assert True diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index 1aa2175c1..6fba713ec 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -8,6 +8,8 @@ from boto3.dynamodb.conditions import Key from botocore.exceptions import ClientError import sure # noqa from freezegun import freeze_time +from nose.tools import assert_raises + from moto import mock_dynamodb2, mock_dynamodb2_deprecated from boto.exception import JSONResponseError from tests.helpers import requires_boto_gte @@ -1273,6 +1275,15 @@ def test_update_item_with_expression(): ) +def assert_failure_due_to_key_not_in_schema(func, **kwargs): + with assert_raises(ClientError) as ex: + func(**kwargs) + ex.exception.response["Error"]["Code"].should.equal("ValidationException") + ex.exception.response["Error"]["Message"].should.equal( + "The provided key element does not match the schema" + ) + + @mock_dynamodb2 def test_update_item_add_with_expression(): table = _create_table_with_range_key() @@ -1299,14 +1310,13 @@ def test_update_item_add_with_expression(): dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item) # Update item to add a string value to a non-existing set - # Should just create the set in the background - table.update_item( + # Should throw: 'The provided key element does not match the schema' + assert_failure_due_to_key_not_in_schema( + table.update_item, Key=item_key, UpdateExpression="ADD non_existing_str_set :v", ExpressionAttributeValues={":v": {"item4"}}, ) - current_item["non_existing_str_set"] = {"item4"} - dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item) # Update item to add a num value to a num set table.update_item( @@ -1381,15 +1391,14 @@ def test_update_item_add_with_nested_sets(): dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item) # Update item to add a string value to a non-existing set - # Should just create the set in the background - table.update_item( + # Should raise + assert_failure_due_to_key_not_in_schema( + table.update_item, Key=item_key, UpdateExpression="ADD #ns.#ne :v", ExpressionAttributeNames={"#ns": "nested", "#ne": "non_existing_str_set"}, ExpressionAttributeValues={":v": {"new_item"}}, ) - current_item["nested"]["non_existing_str_set"] = {"new_item"} - dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item) @mock_dynamodb2 From 0bd586eb67323b501752023050cbc69854de9805 Mon Sep 17 00:00:00 2001 From: pvbouwel Date: Sun, 26 Apr 2020 16:19:24 +0100 Subject: [PATCH 49/51] Place reserved_keywords.txt not in root. Do not use data_files in setup.py but MANIFEST.in Otherwise some enviroments throw errors when trying to create the data file. This was raised in: https://github.com/spulec/moto/pull/2885#discussion_r415150276 --- MANIFEST.in | 1 + setup.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index bd7eb968a..51d1b223c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,6 @@ include requirements.txt requirements-dev.txt tox.ini include moto/ec2/resources/instance_types.json include moto/ec2/resources/amis.json include moto/cognitoidp/resources/*.json +include moto/dynamodb2/parsing/reserved_keywords.txt recursive-include moto/templates * recursive-include tests * diff --git a/setup.py b/setup.py index adc5e4bb9..684c0dcea 100755 --- a/setup.py +++ b/setup.py @@ -101,5 +101,4 @@ setup( project_urls={ "Documentation": "http://docs.getmoto.org/en/latest/", }, - data_files=[('', ['moto/dynamodb2/parsing/reserved_keywords.txt'])], ) From 41abd4344bdcb2dfd5dd4fcebf9bc6be325c052e Mon Sep 17 00:00:00 2001 From: Antoine Wendlinger Date: Mon, 27 Apr 2020 11:42:27 +0200 Subject: [PATCH 50/51] Use xmltodict for parsing --- moto/s3/responses.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ec6015f7a..fa6f8e568 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -839,21 +839,22 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _bucket_response_delete_keys(self, request, body, bucket_name): template = self.response_template(S3_DELETE_KEYS_RESPONSE) + body_dict = xmltodict.parse(body) - objects = minidom.parseString(body).getElementsByTagName("Object") - - deleted_objects = [] - error_names = [] + objects = body_dict["Delete"].get("Object", []) + if not isinstance(objects, list): + # We expect a list of objects, but when there is a single node xmltodict does not + # return a list. + objects = [objects] if len(objects) == 0: raise MalformedXML() + deleted_objects = [] + error_names = [] + for object_ in objects: - key_name = object_.getElementsByTagName("Key")[0].firstChild.nodeValue - version_id_node = object_.getElementsByTagName("VersionId") - if version_id_node: - version_id = version_id_node[0].firstChild.nodeValue - else: - version_id = None + key_name = object_["Key"] + version_id = object_.get("VersionId", None) success = self.backend.delete_key( bucket_name, undo_clean_key_name(key_name), version_id=version_id From dd22e7855a2d28b969a955ec940c30bf5d141804 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 27 Apr 2020 12:48:23 -0700 Subject: [PATCH 51/51] Fixed a regression with CloudWatch --- moto/cloudwatch/responses.py | 2 +- moto/s3/models.py | 6 +- tests/test_cloudwatch/test_cloudwatch.py | 63 ++++++++++--------- .../test_cloudwatch/test_cloudwatch_boto3.py | 8 +-- 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 93abb8b95..56ba68bb9 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -384,7 +384,7 @@ LIST_METRICS_TEMPLATE = """