From c95d472bf5ef9746521cf54c83bb61333c3eafcd Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Tue, 3 Sep 2019 14:54:46 +0200 Subject: [PATCH 01/46] Add (failing) test for ElasticBeanstalk --- tests/test_eb/test_eb.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_eb/test_eb.py diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py new file mode 100644 index 000000000..924ed3adc --- /dev/null +++ b/tests/test_eb/test_eb.py @@ -0,0 +1,15 @@ +import boto3 +from moto import mock_eb + + +@mock_eb +def test_application(): + # Create Elastic Beanstalk Application + eb_client = boto3.client('elasticbeanstalk', region_name='us-east-1') + + eb_client.create_application( + ApplicationName="myapp", + ) + + eb_apps = eb_client.describe_applications() + eb_apps['Applications'][0]['ApplicationName'].should.equal("myapp") From 336f50349af0eddbb5d82776258abd9f847fa989 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Tue, 3 Sep 2019 16:10:32 +0200 Subject: [PATCH 02/46] Add sub-minimal mocking of elasticbeanstalk:create_application() --- moto/__init__.py | 1 + moto/eb/__init__.py | 4 ++ moto/eb/exceptions.py | 7 +++ moto/eb/models.py | 37 ++++++++++++++++ moto/eb/responses.py | 92 ++++++++++++++++++++++++++++++++++++++++ moto/eb/urls.py | 11 +++++ tests/test_eb/test_eb.py | 34 ++++++++++++--- 7 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 moto/eb/__init__.py create mode 100644 moto/eb/exceptions.py create mode 100644 moto/eb/models.py create mode 100644 moto/eb/responses.py create mode 100644 moto/eb/urls.py diff --git a/moto/__init__.py b/moto/__init__.py index 8594cedd2..7cb6d0e39 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -18,6 +18,7 @@ from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated # fla from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # flake8: noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # flake8: noqa from .dynamodbstreams import mock_dynamodbstreams # flake8: noqa +from .eb import mock_eb # flake8: noqa from .ec2 import mock_ec2, mock_ec2_deprecated # flake8: noqa from .ecr import mock_ecr, mock_ecr_deprecated # flake8: noqa from .ecs import mock_ecs, mock_ecs_deprecated # flake8: noqa diff --git a/moto/eb/__init__.py b/moto/eb/__init__.py new file mode 100644 index 000000000..3e06e9595 --- /dev/null +++ b/moto/eb/__init__.py @@ -0,0 +1,4 @@ +from .models import eb_backends +from moto.core.models import base_decorator + +mock_eb = base_decorator(eb_backends) diff --git a/moto/eb/exceptions.py b/moto/eb/exceptions.py new file mode 100644 index 000000000..c470d5317 --- /dev/null +++ b/moto/eb/exceptions.py @@ -0,0 +1,7 @@ +from moto.core.exceptions import RESTError + + +class InvalidParameterValueError(RESTError): + def __init__(self, message): + super(InvalidParameterValueError, self).__init__( + "InvalidParameterValue", message) diff --git a/moto/eb/models.py b/moto/eb/models.py new file mode 100644 index 000000000..246d33cde --- /dev/null +++ b/moto/eb/models.py @@ -0,0 +1,37 @@ +import boto.beanstalk + +from moto.core import BaseBackend, BaseModel +from .exceptions import InvalidParameterValueError + + +class FakeApplication(BaseModel): + def __init__(self, application_name): + self.application_name = application_name + + +class EBBackend(BaseBackend): + def __init__(self, region): + self.region = region + self.applications = dict() + + def reset(self): + # preserve region + region = self.region + self._reset_model_refs() + self.__dict__ = {} + self.__init__(region) + + def create_application(self, application_name): + if application_name in self.applications: + raise InvalidParameterValueError( + "Application {} already exists.".format(application_name) + ) + new_app = FakeApplication( + application_name=application_name, + ) + self.applications[application_name] = new_app + return new_app + + +eb_backends = dict((region.name, EBBackend(region.name)) + for region in boto.beanstalk.regions()) diff --git a/moto/eb/responses.py b/moto/eb/responses.py new file mode 100644 index 000000000..9cf8b2e47 --- /dev/null +++ b/moto/eb/responses.py @@ -0,0 +1,92 @@ +from moto.core.responses import BaseResponse +from .models import eb_backends + +EB_CREATE_APPLICATION = """ + + + + + 2019-09-03T13:08:29.049Z + + + + false + 180 + false + + + false + 200 + false + + + + arn:aws:elasticbeanstalk:{{ region_name }}:111122223333:application/{{ application_name }} + {{ application.application_name }} + 2019-09-03T13:08:29.049Z + + + + 1b6173c8-13aa-4b0a-99e9-eb36a1fb2778 + + +""" + + +EB_DESCRIBE_APPLICATIONS = """ + + + + {% for application in applications %} + + + 2019-09-03T13:08:29.049Z + + + + 180 + false + false + + + false + 200 + false + + + + arn:aws:elasticbeanstalk:{{ region_name }}:387323646340:application/{{ application.name }} + {{ application.application_name }} + 2019-09-03T13:08:29.049Z + + {% endfor %} + + + + 015a05eb-282e-4b76-bd18-663fdfaf42e4 + + +""" + + +class EBResponse(BaseResponse): + @property + def backend(self): + return eb_backends[self.region] + + def create_application(self): + app = self.backend.create_application( + application_name=self._get_param('ApplicationName'), + ) + + template = self.response_template(EB_CREATE_APPLICATION) + return template.render( + region_name=self.backend.region, + application=app, + ) + + def describe_applications(self): + template = self.response_template(EB_DESCRIBE_APPLICATIONS) + return template.render( + applications=self.backend.applications.values(), + ) diff --git a/moto/eb/urls.py b/moto/eb/urls.py new file mode 100644 index 000000000..4cd4add13 --- /dev/null +++ b/moto/eb/urls.py @@ -0,0 +1,11 @@ +from __future__ import unicode_literals + +from .responses import EBResponse + +url_bases = [ + r"https?://elasticbeanstalk.(?P[a-zA-Z0-9\-_]+).amazonaws.com", +] + +url_paths = { + '{0}/$': EBResponse.dispatch, +} diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 924ed3adc..9e863e7f5 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -1,15 +1,39 @@ import boto3 +import sure # noqa +from botocore.exceptions import ClientError + from moto import mock_eb @mock_eb -def test_application(): +def test_create_application(): # Create Elastic Beanstalk Application - eb_client = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + app = conn.create_application( + ApplicationName="myapp", + ) + app['Application']['ApplicationName'].should.equal("myapp") - eb_client.create_application( + +@mock_eb +def test_create_application_dup(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + conn.create_application.when.called_with( + ApplicationName="myapp", + ).should.throw(ClientError) + + +@mock_eb +def test_describe_applications(): + # Create Elastic Beanstalk Application + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( ApplicationName="myapp", ) - eb_apps = eb_client.describe_applications() - eb_apps['Applications'][0]['ApplicationName'].should.equal("myapp") + apps = conn.describe_applications() + len(apps['Applications']).should.equal(1) + apps['Applications'][0]['ApplicationName'].should.equal('myapp') From 6f23a39fc26c3cf51b4f0e2b49277be85024d666 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 15:33:15 +0200 Subject: [PATCH 03/46] Add minimal mocking of elasticbeanstalk:create_environment, describe_environments and list_available_solution_stacks --- moto/eb/models.py | 25 + moto/eb/responses.py | 1297 +++++++++++++++++++++++++++++++++++++- tests/test_eb/test_eb.py | 41 ++ 3 files changed, 1343 insertions(+), 20 deletions(-) diff --git a/moto/eb/models.py b/moto/eb/models.py index 246d33cde..5b4655175 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -1,12 +1,37 @@ +import weakref + import boto.beanstalk from moto.core import BaseBackend, BaseModel from .exceptions import InvalidParameterValueError +class FakeEnvironment(BaseModel): + def __init__(self, application, environment_name): + self.environment_name = environment_name + self.application = weakref.proxy(application) # weakref to break circular dependencies + + @property + def application_name(self): + return self.application.application_name + + class FakeApplication(BaseModel): def __init__(self, application_name): self.application_name = application_name + self.environments = dict() + + def create_environment(self, environment_name): + if environment_name in self.environments: + raise InvalidParameterValueError + + env = FakeEnvironment( + application=self, + environment_name=environment_name, + ) + self.environments[environment_name] = env + + return env class EBBackend(BaseBackend): diff --git a/moto/eb/responses.py b/moto/eb/responses.py index 9cf8b2e47..fecdb8c21 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -1,5 +1,67 @@ from moto.core.responses import BaseResponse -from .models import eb_backends +from .models import eb_backends, EBBackend +from .exceptions import InvalidParameterValueError + + +class EBResponse(BaseResponse): + @property + def backend(self): + """ + :rtype: EBBackend + """ + return eb_backends[self.region] + + def create_application(self): + app = self.backend.create_application( + application_name=self._get_param('ApplicationName'), + ) + + template = self.response_template(EB_CREATE_APPLICATION) + return template.render( + region_name=self.backend.region, + application=app, + ) + + def describe_applications(self): + template = self.response_template(EB_DESCRIBE_APPLICATIONS) + return template.render( + applications=self.backend.applications.values(), + ) + + def create_environment(self): + application_name = self._get_param('ApplicationName') + environment_name = self._get_param('EnvironmentName') + try: + app = self.backend.applications[application_name] + except KeyError: + raise InvalidParameterValueError( + "No Application named \'{}\' found.".format(application_name) + ) + + env = app.create_environment(environment_name=environment_name) + + template = self.response_template(EB_CREATE_ENVIRONMENT) + return template.render( + environment=env, + region=self.backend.region, + ) + + def describe_environments(self): + envs = [] + + for app in self.backend.applications.values(): + for env in app.environments.values(): + envs.append(env) + + template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) + return template.render( + environments=envs, + ) + + @staticmethod + def list_available_solution_stacks(): + return EB_LIST_AVAILABLE_SOLUTION_STACKS + EB_CREATE_APPLICATION = """ @@ -55,7 +117,7 @@ EB_DESCRIBE_APPLICATIONS = """ - arn:aws:elasticbeanstalk:{{ region_name }}:387323646340:application/{{ application.name }} + arn:aws:elasticbeanstalk:{{ region_name }}:111122223333:application/{{ application.name }} {{ application.application_name }} 2019-09-03T13:08:29.049Z @@ -69,24 +131,1219 @@ EB_DESCRIBE_APPLICATIONS = """ """ -class EBResponse(BaseResponse): - @property - def backend(self): - return eb_backends[self.region] +EB_CREATE_ENVIRONMENT = """ + + + {{ environment.solution_stack_name }} + Grey + arn:aws:elasticbeanstalk:{{ region }}:111122223333:environment/{{ environment.application_name }}/{{ environment.environment_name }} + 2019-09-04T09:41:24.222Z + 2019-09-04T09:41:24.222Z + {{ environment_id }} + arn:aws:elasticbeanstalk:{{ region }}::platform/{{ environment.platform_arn }} + + WebServer + Standard + 1.0 + + {{ environment.environment_name }} + {{ environment.application_name }} + Launching + + + 18dc8158-f5d7-4d5a-82ef-07fcaadf81c6 + + +""" - def create_application(self): - app = self.backend.create_application( - application_name=self._get_param('ApplicationName'), - ) - template = self.response_template(EB_CREATE_APPLICATION) - return template.render( - region_name=self.backend.region, - application=app, - ) +EB_DESCRIBE_ENVIRONMENTS = """ + + + + {% for env in environments %} + + {{ env.solution_stack_name }} + Grey + arn:aws:elasticbeanstalk:{{ region }}:123456789012:environment/{{ env.application_name }}/{{ env.environment_name }} + false + 2019-08-30T09:35:10.913Z + false + + 2019-08-22T07:02:47.332Z + {{ env.environment_id }} + 1 + arn:aws:elasticbeanstalk:{{ region }}::platform/{{ env.platform_arn }} + + WebServer + Standard + 1.0 + + No Data + {{ env.environment_name }} + + + + {{ env.application_name }} + Ready + + {% endfor %} + + + + dd56b215-01a0-40b2-bd1e-57589c39424f + + +""" - def describe_applications(self): - template = self.response_template(EB_DESCRIBE_APPLICATIONS) - return template.render( - applications=self.backend.applications.values(), - ) + +# Current list as of 2019-09-04 +EB_LIST_AVAILABLE_SOLUTION_STACKS = """ + + + + 64bit Amazon Linux 2018.03 v4.10.1 running Node.js + 64bit Amazon Linux 2018.03 v4.9.2 running Node.js + 64bit Amazon Linux 2018.03 v4.8.0 running Node.js + 64bit Amazon Linux 2018.03 v4.6.0 running Node.js + 64bit Amazon Linux 2018.03 v4.5.3 running Node.js + 64bit Amazon Linux 2018.03 v4.5.1 running Node.js + 64bit Amazon Linux 2018.03 v4.5.0 running Node.js + 64bit Amazon Linux 2017.09 v4.4.6 running Node.js + 64bit Amazon Linux 2017.09 v4.4.5 running Node.js + 64bit Amazon Linux 2017.09 v4.4.4 running Node.js + 64bit Amazon Linux 2017.09 v4.4.2 running Node.js + 64bit Amazon Linux 2017.09 v4.4.0 running Node.js + 64bit Amazon Linux 2017.03 v4.3.0 running Node.js + 64bit Amazon Linux 2017.03 v4.2.2 running Node.js + 64bit Amazon Linux 2017.03 v4.2.1 running Node.js + 64bit Amazon Linux 2017.03 v4.2.0 running Node.js + 64bit Amazon Linux 2017.03 v4.1.1 running Node.js + 64bit Amazon Linux 2017.03 v4.1.0 running Node.js + 64bit Amazon Linux 2016.09 v4.0.1 running Node.js + 64bit Amazon Linux 2016.09 v4.0.0 running Node.js + 64bit Amazon Linux 2016.09 v3.3.1 running Node.js + 64bit Amazon Linux 2016.09 v3.1.0 running Node.js + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.4 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.5 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.6 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.12 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.7 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.5 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.4 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.3 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.2 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.1 running PHP 7.2 + 64bit Amazon Linux 2018.03 v2.8.0 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 5.6 + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.1 + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.1 + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.5 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.5 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.1 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.5 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.1 + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.1 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.4 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.5 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.6 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.0 + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.1 + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.1 + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.5 + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.6 + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.3 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.4 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.5 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.6 + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.1 running PHP 7.0 + 64bit Amazon Linux 2017.03 v2.4.0 running PHP 7.0 + 64bit Amazon Linux 2016.09 v2.3.2 running PHP 7.0 + 64bit Amazon Linux 2016.09 v2.3.1 running PHP 7.0 + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.6 + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.4 + 64bit Amazon Linux 2018.03 v2.9.1 running Python + 64bit Amazon Linux 2018.03 v2.9.1 running Python 2.7 + 64bit Amazon Linux 2018.03 v2.7.5 running Python 3.6 + 64bit Amazon Linux 2018.03 v2.7.1 running Python 3.6 + 64bit Amazon Linux 2018.03 v2.7.0 running Python 3.6 + 64bit Amazon Linux 2017.09 v2.6.4 running Python 3.6 + 64bit Amazon Linux 2017.09 v2.6.1 running Python 3.6 + 64bit Amazon Linux 2017.03 v2.4.0 running Python 3.4 + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Puma) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 1.9.3 + 64bit Amazon Linux 2018.03 v2.8.0 running Ruby 2.5 (Passenger Standalone) + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Puma) + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Passenger Standalone) + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8.5 Java 8 + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8 Java 8 + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 7 + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 6 + 64bit Amazon Linux 2018.03 v3.1.1 running Tomcat 8.5 Java 8 + 64bit Amazon Linux 2017.03 v2.6.5 running Tomcat 8 Java 8 + 64bit Amazon Linux 2017.03 v2.6.2 running Tomcat 8 Java 8 + 64bit Amazon Linux 2017.03 v2.6.1 running Tomcat 8 Java 8 + 64bit Amazon Linux 2017.03 v2.6.0 running Tomcat 8 Java 8 + 64bit Amazon Linux 2016.09 v2.5.4 running Tomcat 8 Java 8 + 64bit Amazon Linux 2016.03 v2.1.0 running Tomcat 8 Java 8 + 64bit Windows Server Core 2016 v2.2.1 running IIS 10.0 + 64bit Windows Server 2016 v2.2.1 running IIS 10.0 + 64bit Windows Server Core 2012 R2 v2.2.1 running IIS 8.5 + 64bit Windows Server 2012 R2 v2.2.1 running IIS 8.5 + 64bit Windows Server Core 2016 v1.2.0 running IIS 10.0 + 64bit Windows Server 2016 v1.2.0 running IIS 10.0 + 64bit Windows Server Core 2012 R2 v1.2.0 running IIS 8.5 + 64bit Windows Server 2012 R2 v1.2.0 running IIS 8.5 + 64bit Windows Server 2012 v1.2.0 running IIS 8 + 64bit Windows Server 2008 R2 v1.2.0 running IIS 7.5 + 64bit Windows Server Core 2012 R2 running IIS 8.5 + 64bit Windows Server 2012 R2 running IIS 8.5 + 64bit Windows Server 2012 running IIS 8 + 64bit Windows Server 2008 R2 running IIS 7.5 + 64bit Amazon Linux 2018.03 v2.12.16 running Docker 18.06.1-ce + 64bit Amazon Linux 2016.09 v2.5.2 running Docker 1.12.6 + 64bit Amazon Linux 2018.03 v2.15.2 running Multi-container Docker 18.06.1-ce (Generic) + 64bit Debian jessie v2.12.16 running Go 1.4 (Preconfigured - Docker) + 64bit Debian jessie v2.12.16 running Go 1.3 (Preconfigured - Docker) + 64bit Debian jessie v2.12.16 running Python 3.4 (Preconfigured - Docker) + 64bit Debian jessie v2.10.0 running Python 3.4 (Preconfigured - Docker) + 64bit Amazon Linux 2018.03 v2.9.1 running Java 8 + 64bit Amazon Linux 2018.03 v2.9.1 running Java 7 + 64bit Amazon Linux 2018.03 v2.8.0 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.6 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.5 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.4 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.2 running Java 8 + 64bit Amazon Linux 2018.03 v2.7.1 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.8 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.5 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.4 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.3 running Java 8 + 64bit Amazon Linux 2017.09 v2.6.0 running Java 8 + 64bit Amazon Linux 2017.03 v2.5.4 running Java 8 + 64bit Amazon Linux 2017.03 v2.5.3 running Java 8 + 64bit Amazon Linux 2017.03 v2.5.2 running Java 8 + 64bit Amazon Linux 2016.09 v2.4.4 running Java 8 + 64bit Amazon Linux 2018.03 v2.12.1 running Go 1.12.7 + 64bit Amazon Linux 2018.03 v2.6.14 running Packer 1.0.3 + 64bit Amazon Linux 2018.03 v2.12.16 running GlassFish 5.0 Java 8 (Preconfigured - Docker) + + + + 64bit Amazon Linux 2018.03 v4.10.1 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.9.2 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.8.0 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.6.0 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.5.3 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.5.1 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v4.5.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.6 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.5 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.4 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.2 running Node.js + + zip + + + + 64bit Amazon Linux 2017.09 v4.4.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.3.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.2.2 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.2.1 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.2.0 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.1.1 running Node.js + + zip + + + + 64bit Amazon Linux 2017.03 v4.1.0 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v4.0.1 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v4.0.0 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v3.3.1 running Node.js + + zip + + + + 64bit Amazon Linux 2016.09 v3.1.0 running Node.js + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.14 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.12 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.7 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.6 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.5 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.4 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.3 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.2 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.1 running PHP 7.2 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.6 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.5 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.2 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.5.0 running PHP 7.1 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.3 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.4 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.5 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 5.6 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.2 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.0 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2016.09 v2.3.2 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2016.09 v2.3.1 running PHP 7.0 + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python 3.4 + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Python 2.7 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.5 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2018.03 v2.7.0 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2017.09 v2.6.1 running Python 3.6 + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.0 running Python 3.4 + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Puma) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.6 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.5 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.4 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.3 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.2 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.1 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 2.0 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v2.10.1 running Ruby 1.9.3 + + zip + + + + 64bit Amazon Linux 2018.03 v2.8.0 running Ruby 2.5 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Puma) + + zip + + + + 64bit Amazon Linux 2017.03 v2.4.4 running Ruby 2.3 (Passenger Standalone) + + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8.5 Java 8 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 7 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.2.1 running Tomcat 7 Java 6 + + war + zip + + + + 64bit Amazon Linux 2018.03 v3.1.1 running Tomcat 8.5 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.5 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.2 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.1 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2017.03 v2.6.0 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2016.09 v2.5.4 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Amazon Linux 2016.03 v2.1.0 running Tomcat 8 Java 8 + + war + zip + + + + 64bit Windows Server Core 2016 v2.2.1 running IIS 10.0 + + zip + + + + 64bit Windows Server 2016 v2.2.1 running IIS 10.0 + + zip + + + + 64bit Windows Server Core 2012 R2 v2.2.1 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 R2 v2.2.1 running IIS 8.5 + + zip + + + + 64bit Windows Server Core 2016 v1.2.0 running IIS 10.0 + + zip + + + + 64bit Windows Server 2016 v1.2.0 running IIS 10.0 + + zip + + + + 64bit Windows Server Core 2012 R2 v1.2.0 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 R2 v1.2.0 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 v1.2.0 running IIS 8 + + zip + + + + 64bit Windows Server 2008 R2 v1.2.0 running IIS 7.5 + + zip + + + + 64bit Windows Server Core 2012 R2 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 R2 running IIS 8.5 + + zip + + + + 64bit Windows Server 2012 running IIS 8 + + zip + + + + 64bit Windows Server 2008 R2 running IIS 7.5 + + zip + + + + 64bit Amazon Linux 2018.03 v2.12.16 running Docker 18.06.1-ce + + + + 64bit Amazon Linux 2016.09 v2.5.2 running Docker 1.12.6 + + + + 64bit Amazon Linux 2018.03 v2.15.2 running Multi-container Docker 18.06.1-ce (Generic) + + zip + json + + + + 64bit Debian jessie v2.12.16 running Go 1.4 (Preconfigured - Docker) + + zip + + + + 64bit Debian jessie v2.12.16 running Go 1.3 (Preconfigured - Docker) + + zip + + + + 64bit Debian jessie v2.12.16 running Python 3.4 (Preconfigured - Docker) + + zip + + + + 64bit Debian jessie v2.10.0 running Python 3.4 (Preconfigured - Docker) + + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.9.1 running Java 7 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.8.0 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.6 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.5 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.2 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.7.1 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.8 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.5 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.3 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.09 v2.6.0 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.03 v2.5.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.03 v2.5.3 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2017.03 v2.5.2 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2016.09 v2.4.4 running Java 8 + + jar + zip + + + + 64bit Amazon Linux 2018.03 v2.12.1 running Go 1.12.7 + + zip + + + + 64bit Amazon Linux 2018.03 v2.6.14 running Packer 1.0.3 + + + + 64bit Amazon Linux 2018.03 v2.12.16 running GlassFish 5.0 Java 8 (Preconfigured - Docker) + + zip + + + + + + bd6bd2b2-9983-4845-b53b-fe53e8a5e1e7 + + +""" diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 9e863e7f5..aafe524fd 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -37,3 +37,44 @@ def test_describe_applications(): apps = conn.describe_applications() len(apps['Applications']).should.equal(1) apps['Applications'][0]['ApplicationName'].should.equal('myapp') + + +@mock_eb +def test_create_environment(): + # Create Elastic Beanstalk Environment + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + app = conn.create_application( + ApplicationName="myapp", + ) + env = conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + ) + env['EnvironmentName'].should.equal("myenv") + + +@mock_eb +def test_describe_environments(): + # List Elastic Beanstalk Envs + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + ) + + envs = conn.describe_environments() + envs = envs['Environments'] + len(envs).should.equal(1) + envs[0]['ApplicationName'].should.equal('myapp') + envs[0]['EnvironmentName'].should.equal('myenv') + + +@mock_eb +def test_list_available_solution_stacks(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + stacks = conn.list_available_solution_stacks() + len(stacks['SolutionStacks']).should.be.greater_than(0) + len(stacks['SolutionStacks']).should.be.equal(len(stacks['SolutionStackDetails'])) From 91fb40810242213349e0f436e01464425fdd0928 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 16:25:43 +0200 Subject: [PATCH 04/46] Move tags_from_query_string to core.utils --- moto/core/utils.py | 17 +++++++++++++++++ moto/ec2/responses/tags.py | 3 ++- moto/ec2/utils.py | 17 ----------------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index ca670e871..acf76bb48 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -297,3 +297,20 @@ def path_url(url): if parsed_url.query: path = path + '?' + parsed_url.query return path + + +def tags_from_query_string(querystring_dict): + prefix = 'Tag' + suffix = 'Key' + response_values = {} + for key, value in querystring_dict.items(): + if key.startswith(prefix) and key.endswith(suffix): + tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") + tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] + tag_value_key = "Tag.{0}.Value".format(tag_index) + if tag_value_key in querystring_dict: + response_values[tag_key] = querystring_dict.get(tag_value_key)[ + 0] + else: + response_values[tag_key] = None + return response_values diff --git a/moto/ec2/responses/tags.py b/moto/ec2/responses/tags.py index 65d3da255..37f2c3bea 100644 --- a/moto/ec2/responses/tags.py +++ b/moto/ec2/responses/tags.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse from moto.ec2.models import validate_resource_ids -from moto.ec2.utils import tags_from_query_string, filters_from_querystring +from moto.ec2.utils import filters_from_querystring +from moto.core.utils import tags_from_query_string class TagResponse(BaseResponse): diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index e67cb39f4..f0d58d5fc 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -198,23 +198,6 @@ def split_route_id(route_id): return values[0], values[1] -def tags_from_query_string(querystring_dict): - prefix = 'Tag' - suffix = 'Key' - response_values = {} - for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] - tag_value_key = "Tag.{0}.Value".format(tag_index) - if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[ - 0] - else: - response_values[tag_key] = None - return response_values - - def dhcp_configuration_from_querystring(querystring, option=u'DhcpConfiguration'): """ turn: From 9bfbd8e0088d93ccf7c0e4d81526f45db8f9bf50 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 16:55:34 +0200 Subject: [PATCH 05/46] Make tags_from_query_string() more flexible --- moto/core/utils.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index acf76bb48..6f75619d4 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -299,15 +299,27 @@ def path_url(url): return path -def tags_from_query_string(querystring_dict): - prefix = 'Tag' - suffix = 'Key' +def tags_from_query_string( + querystring_dict, + prefix="Tag", + key_suffix="Key", + value_suffix="Value" +): response_values = {} for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] - tag_value_key = "Tag.{0}.Value".format(tag_index) + if key.startswith(prefix) and key.endswith(key_suffix): + tag_index = key.replace(prefix + ".", "").replace("." + key_suffix, "") + tag_key = querystring_dict.get( + "{prefix}.{index}.{key_suffix}".format( + prefix=prefix, + index=tag_index, + key_suffix=key_suffix, + ))[0] + tag_value_key = "{prefix}.{index}.{value_suffix}".format( + prefix=prefix, + index=tag_index, + value_suffix=value_suffix, + ) if tag_value_key in querystring_dict: response_values[tag_key] = querystring_dict.get(tag_value_key)[ 0] From 7f387b0bb9842d3561f59ed7fa70b92e17791909 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Wed, 4 Sep 2019 16:56:06 +0200 Subject: [PATCH 06/46] Add elasticbeanstalk Tags handling --- moto/eb/exceptions.py | 6 +++ moto/eb/models.py | 47 +++++++++++++++++++-- moto/eb/responses.py | 88 +++++++++++++++++++++++++++++++++++++--- tests/test_eb/test_eb.py | 72 ++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 10 deletions(-) diff --git a/moto/eb/exceptions.py b/moto/eb/exceptions.py index c470d5317..bf3a89618 100644 --- a/moto/eb/exceptions.py +++ b/moto/eb/exceptions.py @@ -5,3 +5,9 @@ class InvalidParameterValueError(RESTError): def __init__(self, message): super(InvalidParameterValueError, self).__init__( "InvalidParameterValue", message) + + +class ResourceNotFoundException(RESTError): + def __init__(self, message): + super(ResourceNotFoundException, self).__init__( + "ResourceNotFoundException", message) diff --git a/moto/eb/models.py b/moto/eb/models.py index 5b4655175..c3c2aa20c 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -7,32 +7,70 @@ from .exceptions import InvalidParameterValueError class FakeEnvironment(BaseModel): - def __init__(self, application, environment_name): - self.environment_name = environment_name + def __init__( + self, + application, + environment_name, + tags, + ): self.application = weakref.proxy(application) # weakref to break circular dependencies + self.environment_name = environment_name + self.tags = tags @property def application_name(self): return self.application.application_name + @property + def environment_arn(self): + return 'arn:aws:elasticbeanstalk:{region}:{account_id}:' \ + 'environment/{application_name}/{environment_name}'.format( + region=self.region, + account_id='123456789012', + application_name=self.application_name, + environment_name=self.environment_name, + ) + + @property + def platform_arn(self): + return 'TODO' # TODO + + @property + def solution_stack_name(self): + return 'TODO' # TODO + + @property + def region(self): + return self.application.region + class FakeApplication(BaseModel): - def __init__(self, application_name): + def __init__(self, backend, application_name): + self.backend = weakref.proxy(backend) # weakref to break cycles self.application_name = application_name self.environments = dict() - def create_environment(self, environment_name): + def create_environment( + self, + environment_name, + tags, + ): if environment_name in self.environments: raise InvalidParameterValueError env = FakeEnvironment( application=self, environment_name=environment_name, + tags=tags, ) self.environments[environment_name] = env return env + @property + def region(self): + return self.backend.region + class EBBackend(BaseBackend): def __init__(self, region): @@ -52,6 +90,7 @@ class EBBackend(BaseBackend): "Application {} already exists.".format(application_name) ) new_app = FakeApplication( + backend=self, application_name=application_name, ) self.applications[application_name] = new_app diff --git a/moto/eb/responses.py b/moto/eb/responses.py index fecdb8c21..fbace1938 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -1,6 +1,7 @@ from moto.core.responses import BaseResponse +from moto.core.utils import tags_from_query_string from .models import eb_backends, EBBackend -from .exceptions import InvalidParameterValueError +from .exceptions import InvalidParameterValueError, ResourceNotFoundException class EBResponse(BaseResponse): @@ -38,7 +39,11 @@ class EBResponse(BaseResponse): "No Application named \'{}\' found.".format(application_name) ) - env = app.create_environment(environment_name=environment_name) + tags = tags_from_query_string(self.querystring, prefix="Tags.member") + env = app.create_environment( + environment_name=environment_name, + tags=tags, + ) template = self.response_template(EB_CREATE_ENVIRONMENT) return template.render( @@ -62,6 +67,48 @@ class EBResponse(BaseResponse): def list_available_solution_stacks(): return EB_LIST_AVAILABLE_SOLUTION_STACKS + def _find_environment_by_arn(self, arn): + for app in self.backend.applications.keys(): + for env in self.backend.applications[app].environments.values(): + if env.environment_arn == arn: + return env + raise KeyError() + + def update_tags_for_resource(self): + resource_arn = self._get_param('ResourceArn') + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN \'{}\'.".format(resource_arn) + ) + + tags_to_add = tags_from_query_string(self.querystring, prefix="TagsToAdd.member") + for key, value in tags_to_add.items(): + res.tags[key] = value + + tags_to_remove = self._get_multi_param('TagsToRemove.member') + for key in tags_to_remove: + del res.tags[key] + + return EB_UPDATE_TAGS_FOR_RESOURCE + + def list_tags_for_resource(self): + resource_arn = self._get_param('ResourceArn') + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN \'{}\'.".format(resource_arn) + ) + tags = res.tags + + template = self.response_template(EB_LIST_TAGS_FOR_RESOURCE) + return template.render( + tags=tags, + arn=resource_arn, + ) + EB_CREATE_APPLICATION = """ @@ -136,11 +183,11 @@ EB_CREATE_ENVIRONMENT = """ {{ environment.solution_stack_name }} Grey - arn:aws:elasticbeanstalk:{{ region }}:111122223333:environment/{{ environment.application_name }}/{{ environment.environment_name }} + {{ environment.environment_arn }} 2019-09-04T09:41:24.222Z 2019-09-04T09:41:24.222Z {{ environment_id }} - arn:aws:elasticbeanstalk:{{ region }}::platform/{{ environment.platform_arn }} + {{ environment.platform_arn }} WebServer Standard @@ -165,7 +212,7 @@ EB_DESCRIBE_ENVIRONMENTS = """ {{ env.solution_stack_name }} Grey - arn:aws:elasticbeanstalk:{{ region }}:123456789012:environment/{{ env.application_name }}/{{ env.environment_name }} + {{ env.environment_arn }} false 2019-08-30T09:35:10.913Z false @@ -173,7 +220,7 @@ EB_DESCRIBE_ENVIRONMENTS = """ 2019-08-22T07:02:47.332Z {{ env.environment_id }} 1 - arn:aws:elasticbeanstalk:{{ region }}::platform/{{ env.platform_arn }} + {{ env.platform_arn }} WebServer Standard @@ -1347,3 +1394,32 @@ EB_LIST_AVAILABLE_SOLUTION_STACKS = """ """ + + +EB_UPDATE_TAGS_FOR_RESOURCE = """ + + + f355d788-e67e-440f-b915-99e35254ffee + + +""" + + +EB_LIST_TAGS_FOR_RESOURCE = """ + + + + {% for key, value in tags.items() %} + + {{ key }} + {{ value }} + + {% endfor %} + + {{ arn }} + + + 178e410f-3b57-456f-a64c-a3b6a16da9ab + + +""" diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index aafe524fd..2b5be4490 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -72,6 +72,78 @@ def test_describe_environments(): envs[0]['EnvironmentName'].should.equal('myenv') +def tags_dict_to_list(tag_dict): + tag_list = [] + for key, value in tag_dict.items(): + tag_list.append({'Key': key, 'Value': value}) + return tag_list + + +def tags_list_to_dict(tag_list): + tag_dict = {} + for tag in tag_list: + tag_dict[tag['Key']] = tag['Value'] + return tag_dict + + +@mock_eb +def test_create_environment_tags(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + env_tags = {'initial key': 'initial value'} + env = conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + Tags=tags_dict_to_list(env_tags), + ) + + tags = conn.list_tags_for_resource( + ResourceArn=env['EnvironmentArn'], + ) + tags['ResourceArn'].should.equal(env['EnvironmentArn']) + tags_list_to_dict(tags['ResourceTags']).should.equal(env_tags) + + +@mock_eb +def test_update_tags(): + conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn.create_application( + ApplicationName="myapp", + ) + env_tags = { + 'initial key': 'initial value', + 'to remove': 'delete me', + 'to update': 'original', + } + env = conn.create_environment( + ApplicationName="myapp", + EnvironmentName="myenv", + Tags=tags_dict_to_list(env_tags), + ) + + extra_env_tags = { + 'to update': 'new', + 'extra key': 'extra value', + } + conn.update_tags_for_resource( + ResourceArn=env['EnvironmentArn'], + TagsToAdd=tags_dict_to_list(extra_env_tags), + TagsToRemove=['to remove'], + ) + + total_env_tags = env_tags.copy() + total_env_tags.update(extra_env_tags) + del total_env_tags['to remove'] + + tags = conn.list_tags_for_resource( + ResourceArn=env['EnvironmentArn'], + ) + tags['ResourceArn'].should.equal(env['EnvironmentArn']) + tags_list_to_dict(tags['ResourceTags']).should.equal(total_env_tags) + + @mock_eb def test_list_available_solution_stacks(): conn = boto3.client('elasticbeanstalk', region_name='us-east-1') From 8f51bd6116b7194d2ef553064a66ff4bb3af734a Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Thu, 5 Sep 2019 11:38:19 +0200 Subject: [PATCH 07/46] EB: pass through SolutionStackName --- moto/eb/models.py | 8 ++++---- moto/eb/responses.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/eb/models.py b/moto/eb/models.py index c3c2aa20c..fa7345f0d 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -11,10 +11,12 @@ class FakeEnvironment(BaseModel): self, application, environment_name, + solution_stack_name, tags, ): self.application = weakref.proxy(application) # weakref to break circular dependencies self.environment_name = environment_name + self.solution_stack_name = solution_stack_name self.tags = tags @property @@ -35,10 +37,6 @@ class FakeEnvironment(BaseModel): def platform_arn(self): return 'TODO' # TODO - @property - def solution_stack_name(self): - return 'TODO' # TODO - @property def region(self): return self.application.region @@ -53,6 +51,7 @@ class FakeApplication(BaseModel): def create_environment( self, environment_name, + solution_stack_name, tags, ): if environment_name in self.environments: @@ -61,6 +60,7 @@ class FakeApplication(BaseModel): env = FakeEnvironment( application=self, environment_name=environment_name, + solution_stack_name=solution_stack_name, tags=tags, ) self.environments[environment_name] = env diff --git a/moto/eb/responses.py b/moto/eb/responses.py index fbace1938..c93efc3a1 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -31,7 +31,6 @@ class EBResponse(BaseResponse): def create_environment(self): application_name = self._get_param('ApplicationName') - environment_name = self._get_param('EnvironmentName') try: app = self.backend.applications[application_name] except KeyError: @@ -41,7 +40,8 @@ class EBResponse(BaseResponse): tags = tags_from_query_string(self.querystring, prefix="Tags.member") env = app.create_environment( - environment_name=environment_name, + environment_name=self._get_param('EnvironmentName'), + solution_stack_name=self._get_param('SolutionStackName'), tags=tags, ) From 7fae0d52ad6220998ef07dca7cf7de79680c2c80 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Thu, 5 Sep 2019 14:17:55 +0200 Subject: [PATCH 08/46] Fix linting --- moto/eb/models.py | 12 ++++++------ moto/eb/responses.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/moto/eb/models.py b/moto/eb/models.py index fa7345f0d..4490bbd0c 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -26,12 +26,12 @@ class FakeEnvironment(BaseModel): @property def environment_arn(self): return 'arn:aws:elasticbeanstalk:{region}:{account_id}:' \ - 'environment/{application_name}/{environment_name}'.format( - region=self.region, - account_id='123456789012', - application_name=self.application_name, - environment_name=self.environment_name, - ) + 'environment/{application_name}/{environment_name}'.format( + region=self.region, + account_id='123456789012', + application_name=self.application_name, + environment_name=self.environment_name, + ) @property def platform_arn(self): diff --git a/moto/eb/responses.py b/moto/eb/responses.py index c93efc3a1..905780c44 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -1,6 +1,6 @@ from moto.core.responses import BaseResponse from moto.core.utils import tags_from_query_string -from .models import eb_backends, EBBackend +from .models import eb_backends from .exceptions import InvalidParameterValueError, ResourceNotFoundException From b1da99aedaee0d7db7c8d68dfd477440705648e7 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 12:29:04 +0000 Subject: [PATCH 09/46] #2797 - DynamoDB - Allow case insensitive AND in KeyConditionExpression --- moto/dynamodb2/responses.py | 6 ++++-- tests/test_dynamodb2/test_dynamodb.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index c72ded2c3..c13078a72 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -459,8 +459,10 @@ class DynamoHandler(BaseResponse): for k, v in six.iteritems(self.body.get("ExpressionAttributeNames", {})) ) - if " AND " in key_condition_expression: - expressions = key_condition_expression.split(" AND ", 1) + if " and " in key_condition_expression.lower(): + expressions = re.split( + " AND ", key_condition_expression, maxsplit=1, flags=re.IGNORECASE + ) index_hash_key = [key for key in index if key["KeyType"] == "HASH"][0] hash_key_var = reverse_attribute_lookup.get( diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..191f19c36 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1408,6 +1408,13 @@ def test_filter_expression(): filter_expr.expr(row1).should.be(True) filter_expr.expr(row2).should.be(False) + # lowercase AND test + filter_expr = moto.dynamodb2.comparisons.get_filter_expression( + "Id > :v0 and Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + # OR test filter_expr = moto.dynamodb2.comparisons.get_filter_expression( "Id = :v0 OR Id=:v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "8"}} @@ -2719,7 +2726,7 @@ def test_query_gsi_with_range_key(): res = dynamodb.query( TableName="test", IndexName="test_gsi", - KeyConditionExpression="gsi_hash_key = :gsi_hash_key AND gsi_range_key = :gsi_range_key", + KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key", ExpressionAttributeValues={ ":gsi_hash_key": {"S": "key1"}, ":gsi_range_key": {"S": "range1"}, From 5b596c8a78ffc0c5a6ee1b9b28629c3f04bd5396 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 15:17:55 +0000 Subject: [PATCH 10/46] #2699 - EC2 - Add Volumes using CloudFormation --- moto/ec2/models.py | 14 ++++++++++- tests/test_ec2/test_instances.py | 40 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index be39bab28..1b363a193 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -556,6 +556,10 @@ class Instance(TaggedEC2Resource, BotoInstance): # worst case we'll get IP address exaustion... rarely pass + def add_block_device(self, size, device_path): + volume = self.ec2_backend.create_volume(size, self.region_name) + self.ec2_backend.attach_volume(volume.id, self.id, device_path) + def setup_defaults(self): # Default have an instance with root volume should you not wish to # override with attach volume cmd. @@ -620,6 +624,7 @@ class Instance(TaggedEC2Resource, BotoInstance): subnet_id=properties.get("SubnetId"), key_name=properties.get("KeyName"), private_ip=properties.get("PrivateIpAddress"), + block_device_mappings=properties.get("BlockDeviceMappings", {}), ) instance = reservation.instances[0] for tag in properties.get("Tags", []): @@ -872,7 +877,14 @@ class InstanceBackend(object): ) new_reservation.instances.append(new_instance) new_instance.add_tags(instance_tags) - new_instance.setup_defaults() + if "block_device_mappings" in kwargs: + for block_device in kwargs["block_device_mappings"]: + new_instance.add_block_device( + block_device["Ebs"]["VolumeSize"], block_device["DeviceName"] + ) + else: + new_instance.setup_defaults() + return new_reservation def start_instances(self, instance_ids): diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..4d1cbb28d 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -9,6 +9,7 @@ from nose.tools import assert_raises import base64 import datetime import ipaddress +import json import six import boto @@ -18,7 +19,7 @@ from boto.exception import EC2ResponseError, EC2ResponseError from freezegun import freeze_time import sure # noqa -from moto import mock_ec2_deprecated, mock_ec2 +from moto import mock_ec2_deprecated, mock_ec2, mock_cloudformation from tests.helpers import requires_boto_gte @@ -1399,3 +1400,40 @@ def test_describe_instance_attribute(): invalid_instance_attribute=invalid_instance_attribute ) ex.exception.response["Error"]["Message"].should.equal(message) + + +@mock_ec2 +@mock_cloudformation +def test_volume_size_through_cloudformation(): + ec2 = boto3.client("ec2", region_name="us-east-1") + cf = boto3.client("cloudformation", region_name="us-east-1") + + volume_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "testInstance": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-d3adb33f", + "KeyName": "dummy", + "InstanceType": "t2.micro", + "BlockDeviceMappings": [ + {"DeviceName": "/dev/sda2", "Ebs": {"VolumeSize": "50"}} + ], + "Tags": [ + {"Key": "foo", "Value": "bar"}, + {"Key": "blah", "Value": "baz"}, + ], + }, + } + }, + } + template_json = json.dumps(volume_template) + cf.create_stack(StackName="test_stack", TemplateBody=template_json) + instances = ec2.describe_instances() + volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][ + "Ebs" + ] + + volumes = ec2.describe_volumes(VolumeIds=[volume["VolumeId"]]) + volumes["Volumes"][0]["Size"].should.equal(50) From da1a2118bb12ca3279952d88154ed221f9f0fd1e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 20 Mar 2020 16:17:21 +0000 Subject: [PATCH 11/46] EC2 - Verify default block exists before tearing down --- moto/ec2/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 1b363a193..c391c88f3 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -567,9 +567,10 @@ class Instance(TaggedEC2Resource, BotoInstance): self.ec2_backend.attach_volume(volume.id, self.id, "/dev/sda1") def teardown_defaults(self): - volume_id = self.block_device_mapping["/dev/sda1"].volume_id - self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") - self.ec2_backend.delete_volume(volume_id) + if "/dev/sda1" in self.block_device_mapping: + volume_id = self.block_device_mapping["/dev/sda1"].volume_id + self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") + self.ec2_backend.delete_volume(volume_id) @property def get_block_device_mapping(self): From e82e1e3f397cd610d3ed0316c37325cdfe55926b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 21 Mar 2020 12:20:09 +0000 Subject: [PATCH 12/46] DynamoDB - Add 1MB item size check --- moto/dynamodb2/models.py | 11 ++++++++ tests/test_dynamodb2/test_dynamodb.py | 38 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 54dccd56d..a35eded61 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -285,6 +285,9 @@ class Item(BaseModel): def __repr__(self): return "Item: {0}".format(self.to_json()) + def size(self): + return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + def to_json(self): attributes = {} for attribute_key, attribute in self.attrs.items(): @@ -1123,6 +1126,14 @@ class Table(BaseModel): break last_evaluated_key = None + size_limit = 1000000 # DynamoDB has a 1MB size limit + item_size = sum([res.size() for res in results]) + if item_size > size_limit: + item_size = idx = 0 + while item_size + results[idx].size() < size_limit: + item_size += results[idx].size() + idx += 1 + limit = min(limit, idx) if limit else idx if limit and len(results) > limit: results = results[:limit] last_evaluated_key = {self.hash_key_attr: results[-1].hash_key} diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..daae79232 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4132,3 +4132,41 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_dynamodb_max_1mb_limit(): + ddb = boto3.resource("dynamodb", region_name="eu-west-1") + + table_name = "populated-mock-table" + table = ddb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "partition_key", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "SORT"}, + ], + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + # Populate the table + items = [ + { + "partition_key": "partition_key_val", # size=30 + "sort_key": "sort_key_value____" + str(i), # size=30 + } + for i in range(10000, 29999) + ] + with table.batch_writer() as batch: + for item in items: + batch.put_item(Item=item) + + response = table.query( + KeyConditionExpression=Key("partition_key").eq("partition_key_val") + ) + # We shouldn't get everything back - the total result set is well over 1MB + assert response["Count"] < len(items) + response["LastEvaluatedKey"].shouldnt.be(None) From c3865532f9ca6237261591277bde1afbe910099e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 23 Mar 2020 15:53:39 +0000 Subject: [PATCH 13/46] #2711 - Register default S3 metrics in CloudWatch --- moto/cloudwatch/models.py | 50 ++++++++++++++++--- moto/cloudwatch/responses.py | 5 +- moto/s3/models.py | 33 ++++++++++++ moto/s3/utils.py | 6 +++ tests/test_cloudwatch/test_cloudwatch.py | 40 +++++++++++++-- .../test_cloudwatch/test_cloudwatch_boto3.py | 16 ++++-- 6 files changed, 133 insertions(+), 17 deletions(-) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..523eb10f3 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -22,6 +22,14 @@ class Dimension(object): self.name = name self.value = value + def __eq__(self, item): + if isinstance(item, Dimension): + return self.name == item.name and self.value == item.value + return False + + def __ne__(self, item): # Only needed on Py2; Py3 defines it implicitly + return self != item + def daterange(start, stop, step=timedelta(days=1), inclusive=False): """ @@ -124,6 +132,17 @@ class MetricDatum(BaseModel): Dimension(dimension["Name"], dimension["Value"]) for dimension in dimensions ] + def filter(self, namespace, name, dimensions): + if namespace and namespace != self.namespace: + return False + if name and name != self.name: + return False + if dimensions and any( + Dimension(d["Name"], d["Value"]) not in self.dimensions for d in dimensions + ): + return False + return True + class Dashboard(BaseModel): def __init__(self, name, body): @@ -202,6 +221,15 @@ class CloudWatchBackend(BaseBackend): self.metric_data = [] self.paged_metric_data = {} + @property + # Retrieve a list of all OOTB metrics that are provided by metrics providers + # Computed on the fly + def aws_metric_data(self): + md = [] + for name, service in metric_providers.items(): + md.extend(service.get_cloudwatch_metrics()) + return md + def put_metric_alarm( self, name, @@ -334,7 +362,7 @@ class CloudWatchBackend(BaseBackend): return data def get_all_metrics(self): - return self.metric_data + return self.metric_data + self.aws_metric_data def put_dashboard(self, name, body): self.dashboards[name] = Dashboard(name, body) @@ -386,7 +414,7 @@ class CloudWatchBackend(BaseBackend): self.alarms[alarm_name].update_state(reason, reason_data, state_value) - def list_metrics(self, next_token, namespace, metric_name): + def list_metrics(self, next_token, namespace, metric_name, dimensions): if next_token: if next_token not in self.paged_metric_data: raise RESTError( @@ -397,15 +425,16 @@ class CloudWatchBackend(BaseBackend): del self.paged_metric_data[next_token] # Cant reuse same token twice return self._get_paginated(metrics) else: - metrics = self.get_filtered_metrics(metric_name, namespace) + metrics = self.get_filtered_metrics(metric_name, namespace, dimensions) return self._get_paginated(metrics) - def get_filtered_metrics(self, metric_name, namespace): + def get_filtered_metrics(self, metric_name, namespace, dimensions): metrics = self.get_all_metrics() - if namespace: - metrics = [md for md in metrics if md.namespace == namespace] - if metric_name: - metrics = [md for md in metrics if md.name == metric_name] + metrics = [ + md + for md in metrics + if md.filter(namespace=namespace, name=metric_name, dimensions=dimensions) + ] return metrics def _get_paginated(self, metrics): @@ -443,3 +472,8 @@ for region in Session().get_available_regions( cloudwatch_backends[region] = CloudWatchBackend() for region in Session().get_available_regions("cloudwatch", partition_name="aws-cn"): cloudwatch_backends[region] = CloudWatchBackend() + +# List of services that provide OOTB CW metrics +# See the S3Backend constructor for an example +# TODO: We might have to separate this out per region for non-global services +metric_providers = {} diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7993c9f06..dccc30216 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -124,9 +124,10 @@ class CloudWatchResponse(BaseResponse): def list_metrics(self): namespace = self._get_param("Namespace") metric_name = self._get_param("MetricName") + dimensions = self._get_multi_param("Dimensions.member") next_token = self._get_param("NextToken") next_token, metrics = self.cloudwatch_backend.list_metrics( - next_token, namespace, metric_name + next_token, namespace, metric_name, dimensions ) template = self.response_template(LIST_METRICS_TEMPLATE) return template.render(metrics=metrics, next_token=next_token) @@ -342,7 +343,7 @@ LIST_METRICS_TEMPLATE = """ Date: Tue, 24 Mar 2020 09:24:38 +0000 Subject: [PATCH 14/46] #2810 - EC2 - Explicitly set ebs_optimized to False if not specified --- moto/ec2/responses/instances.py | 2 +- tests/test_ec2/test_instances.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index 29c346f82..ba15be8d0 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -52,7 +52,7 @@ class InstanceResponse(BaseResponse): private_ip = self._get_param("PrivateIpAddress") associate_public_ip = self._get_param("AssociatePublicIpAddress") key_name = self._get_param("KeyName") - ebs_optimized = self._get_param("EbsOptimized") + ebs_optimized = self._get_param("EbsOptimized") or False instance_initiated_shutdown_behavior = self._get_param( "InstanceInitiatedShutdownBehavior" ) diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index 85ba0fe01..d40aca000 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1319,6 +1319,12 @@ def test_create_instance_ebs_optimized(): instance.load() instance.ebs_optimized.should.be(False) + instance = ec2_resource.create_instances( + ImageId="ami-12345678", MaxCount=1, MinCount=1, + )[0] + instance.load() + instance.ebs_optimized.should.be(False) + @mock_ec2 def test_run_multiple_instances_in_same_command(): From 04f488da62462a926d4ee61ad303583bdaf836a8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 24 Mar 2020 10:22:08 +0000 Subject: [PATCH 15/46] #2388 - CloudFormation - CreateChangeSet does not create resources, as per spec --- moto/cloudformation/models.py | 14 +++++++++++--- moto/cloudformation/parsing.py | 4 +++- .../test_cloudformation_stack_crud_boto3.py | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 8136e353d..281ab5e19 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -239,8 +239,11 @@ class FakeStack(BaseModel): self.cross_stack_resources = cross_stack_resources or {} self.resource_map = self._create_resource_map() self.output_map = self._create_output_map() - self._add_stack_event("CREATE_COMPLETE") - self.status = "CREATE_COMPLETE" + if create_change_set: + self.status = "REVIEW_IN_PROGRESS" + else: + self.create_resources() + self._add_stack_event("CREATE_COMPLETE") self.creation_time = datetime.utcnow() def _create_resource_map(self): @@ -253,7 +256,7 @@ class FakeStack(BaseModel): self.template_dict, self.cross_stack_resources, ) - resource_map.create() + resource_map.load() return resource_map def _create_output_map(self): @@ -326,6 +329,10 @@ class FakeStack(BaseModel): def exports(self): return self.output_map.exports + def create_resources(self): + self.resource_map.create() + self.status = "CREATE_COMPLETE" + def update(self, template, role_arn=None, parameters=None, tags=None): self._add_stack_event( "UPDATE_IN_PROGRESS", resource_status_reason="User Initiated" @@ -640,6 +647,7 @@ class CloudFormationBackend(BaseBackend): else: stack._add_stack_event("UPDATE_IN_PROGRESS") stack._add_stack_event("UPDATE_COMPLETE") + stack.create_resources() return True def describe_stacks(self, name_or_stack_id): diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 79276c8fc..6789c0007 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -529,14 +529,16 @@ class ResourceMap(collections_abc.Mapping): for condition_name in self.lazy_condition_map: self.lazy_condition_map[condition_name] - def create(self): + def load(self): self.load_mapping() self.transform_mapping() self.load_parameters() self.load_conditions() + def create(self): # Since this is a lazy map, to create every object we just need to # iterate through self. + # Assumes that self.load() has been called before self.tags.update( { "aws:cloudformation:stack-name": self.get("AWS::StackName"), diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 5444c2278..4df1ff5d2 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -835,8 +835,10 @@ def test_describe_change_set(): ) stack = cf_conn.describe_change_set(ChangeSetName="NewChangeSet") + stack["ChangeSetName"].should.equal("NewChangeSet") stack["StackName"].should.equal("NewStack") + stack["Status"].should.equal("REVIEW_IN_PROGRESS") cf_conn.create_change_set( StackName="NewStack", @@ -851,15 +853,30 @@ def test_describe_change_set(): @mock_cloudformation +@mock_ec2 def test_execute_change_set_w_arn(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") + ec2 = boto3.client("ec2", region_name="us-east-1") + # Verify no instances exist at the moment + ec2.describe_instances()["Reservations"].should.have.length_of(0) + # Create a Change set, and verify no resources have been created yet change_set = cf_conn.create_change_set( StackName="NewStack", TemplateBody=dummy_template_json, ChangeSetName="NewChangeSet", ChangeSetType="CREATE", ) + ec2.describe_instances()["Reservations"].should.have.length_of(0) + cf_conn.describe_change_set(ChangeSetName="NewChangeSet")["Status"].should.equal( + "REVIEW_IN_PROGRESS" + ) + # Execute change set cf_conn.execute_change_set(ChangeSetName=change_set["Id"]) + # Verify that the status has changed, and the appropriate resources have been created + cf_conn.describe_change_set(ChangeSetName="NewChangeSet")["Status"].should.equal( + "CREATE_COMPLETE" + ) + ec2.describe_instances()["Reservations"].should.have.length_of(1) @mock_cloudformation From fb0de99e81dc0af7644faa66113c1aec60b589ea Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 28 Mar 2020 13:41:17 +0000 Subject: [PATCH 16/46] #2239 - Initial implementation of CW.get_metric_data --- moto/cloudwatch/models.py | 37 +++ moto/cloudwatch/responses.py | 41 +++ .../test_cloudwatch/test_cloudwatch_boto3.py | 259 ++++++++++++++++++ 3 files changed, 337 insertions(+) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index a8a1b1d19..bddb94a12 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -295,6 +295,43 @@ class CloudWatchBackend(BaseBackend): ) ) + def get_metric_data(self, queries, start_time, end_time): + period_data = [ + md for md in self.metric_data if start_time <= md.timestamp <= end_time + ] + results = [] + for query in queries: + query_ns = query["metric_stat._metric._namespace"] + query_name = query["metric_stat._metric._metric_name"] + query_data = [ + md + for md in period_data + if md.namespace == query_ns and md.name == query_name + ] + metric_values = [m.value for m in query_data] + result_vals = [] + stat = query["metric_stat._stat"] + if len(metric_values) > 0: + if stat == "Average": + result_vals.append(sum(metric_values) / len(metric_values)) + elif stat == "Minimum": + result_vals.append(min(metric_values)) + elif stat == "Maximum": + result_vals.append(max(metric_values)) + elif stat == "Sum": + result_vals.append(sum(metric_values)) + + label = query["metric_stat._metric._metric_name"] + " " + stat + results.append( + { + "id": query["id"], + "label": label, + "vals": result_vals, + "timestamps": [datetime.now() for _ in result_vals], + } + ) + return results + def get_metric_statistics( self, namespace, metric_name, start_time, end_time, period, stats ): diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7993c9f06..7e75a38f0 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -92,6 +92,18 @@ class CloudWatchResponse(BaseResponse): template = self.response_template(PUT_METRIC_DATA_TEMPLATE) return template.render() + @amzn_request_id + def get_metric_data(self): + start = dtparse(self._get_param("StartTime")) + end = dtparse(self._get_param("EndTime")) + queries = self._get_list_prefix("MetricDataQueries.member") + results = self.cloudwatch_backend.get_metric_data( + start_time=start, end_time=end, queries=queries + ) + + template = self.response_template(GET_METRIC_DATA_TEMPLATE) + return template.render(results=results) + @amzn_request_id def get_metric_statistics(self): namespace = self._get_param("Namespace") @@ -285,6 +297,35 @@ PUT_METRIC_DATA_TEMPLATE = """ + + + {{ request_id }} + + + + + {% for result in results %} + + {{ result.id }} + + Complete + + {% for val in result.timestamps %} + {{ val }} + {% endfor %} + + + {% for val in result.vals %} + {{ val }} + {% endfor %} + + + {% endfor %} + + +""" + GET_METRIC_STATISTICS_TEMPLATE = """ diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 7fe144052..2b1caff02 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -3,6 +3,7 @@ import boto3 from botocore.exceptions import ClientError from datetime import datetime, timedelta +from freezegun import freeze_time from nose.tools import assert_raises from uuid import uuid4 import pytz @@ -211,6 +212,35 @@ def test_get_metric_statistics(): datapoint["Sum"].should.equal(1.5) +@mock_cloudwatch +@freeze_time("2020-02-10 18:44:05") +def test_custom_timestamp(): + utc_now = datetime.now(tz=pytz.utc) + time = "2020-02-10T18:44:09Z" + cw = boto3.client("cloudwatch", "eu-west-1") + + cw.put_metric_data( + Namespace="tester", + MetricData=[dict(MetricName="metric1", Value=1.5, Timestamp=time)], + ) + + cw.put_metric_data( + Namespace="tester", + MetricData=[ + dict(MetricName="metric2", Value=1.5, Timestamp=datetime(2020, 2, 10)) + ], + ) + + stats = cw.get_metric_statistics( + Namespace="tester", + MetricName="metric", + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + Period=60, + Statistics=["SampleCount", "Sum"], + ) + + @mock_cloudwatch def test_list_metrics(): cloudwatch = boto3.client("cloudwatch", "eu-west-1") @@ -292,3 +322,232 @@ def create_metrics(cloudwatch, namespace, metrics=5, data_points=5): Namespace=namespace, MetricData=[{"MetricName": metric_name, "Value": j, "Unit": "Seconds"}], ) + + +@mock_cloudwatch +def test_get_metric_data_within_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + values = [0, 2, 4, 3.5, 7, 100] + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + {"MetricName": "metric1", "Value": val, "Unit": "Seconds"} for val in values + ], + ) + # get_metric_data + stats = ["Average", "Sum", "Minimum", "Maximum"] + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result_" + stat, + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": stat, + }, + } + for stat in stats + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Average/Min/Max/Sum is returned as expected + avg = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Average" + ][0] + avg["Label"].should.equal("metric1 Average") + avg["StatusCode"].should.equal("Complete") + [int(val) for val in avg["Values"]].should.equal([19]) + + sum_ = [res for res in response["MetricDataResults"] if res["Id"] == "result_Sum"][ + 0 + ] + sum_["Label"].should.equal("metric1 Sum") + sum_["StatusCode"].should.equal("Complete") + [val for val in sum_["Values"]].should.equal([sum(values)]) + + min_ = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Minimum" + ][0] + min_["Label"].should.equal("metric1 Minimum") + min_["StatusCode"].should.equal("Complete") + [int(val) for val in min_["Values"]].should.equal([0]) + + max_ = [ + res for res in response["MetricDataResults"] if res["Id"] == "result_Maximum" + ][0] + max_["Label"].should.equal("metric1 Maximum") + max_["StatusCode"].should.equal("Complete") + [int(val) for val in max_["Values"]].should.equal([100]) + + +@mock_cloudwatch +def test_get_metric_data_partially_within_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + yesterday = utc_now - timedelta(days=1) + last_week = utc_now - timedelta(days=7) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + values = [0, 2, 4, 3.5, 7, 100] + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 10, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 20, + "Unit": "Seconds", + "Timestamp": yesterday, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": last_week, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result", + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + } + ], + StartTime=yesterday - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Last week's data is not returned + len(response["MetricDataResults"]).should.equal(1) + sum_ = response["MetricDataResults"][0] + sum_["Label"].should.equal("metric1 Sum") + sum_["StatusCode"].should.equal("Complete") + sum_["Values"].should.equal([30.0]) + + +@mock_cloudwatch +def test_get_metric_data_outside_timeframe(): + utc_now = datetime.now(tz=pytz.utc) + last_week = utc_now - timedelta(days=7) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace1 = "my_namespace/" + # put metric data + cloudwatch.put_metric_data( + Namespace=namespace1, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": last_week, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result", + "MetricStat": { + "Metric": {"Namespace": namespace1, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + } + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + # Assert Last week's data is not returned + len(response["MetricDataResults"]).should.equal(1) + response["MetricDataResults"][0]["Id"].should.equal("result") + response["MetricDataResults"][0]["StatusCode"].should.equal("Complete") + response["MetricDataResults"][0]["Values"].should.equal([]) + + +@mock_cloudwatch +def test_get_metric_data_for_multiple_metrics(): + utc_now = datetime.now(tz=pytz.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace = "my_namespace/" + # put metric data + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric1", + "Value": 50, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric2", + "Value": 25, + "Unit": "Seconds", + "Timestamp": utc_now, + } + ], + ) + # get_metric_data + response = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result1", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric1"}, + "Period": 60, + "Stat": "Sum", + }, + }, + { + "Id": "result2", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric2"}, + "Period": 60, + "Stat": "Sum", + }, + }, + ], + StartTime=utc_now - timedelta(seconds=60), + EndTime=utc_now + timedelta(seconds=60), + ) + # + len(response["MetricDataResults"]).should.equal(2) + + res1 = [res for res in response["MetricDataResults"] if res["Id"] == "result1"][0] + res1["Values"].should.equal([50.0]) + + res2 = [res for res in response["MetricDataResults"] if res["Id"] == "result2"][0] + res2["Values"].should.equal([25.0]) From 888e0c31a0fd94c8854f93ec88e3b45ebfeeb98b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 13:42:00 +0100 Subject: [PATCH 17/46] Linting --- moto/__init__.py | 15 +++--- moto/core/utils.py | 19 +++---- moto/eb/exceptions.py | 6 ++- moto/eb/models.py | 35 ++++++------ moto/eb/responses.py | 47 +++++++--------- moto/eb/urls.py | 2 +- tests/test_eb/test_eb.py | 112 ++++++++++++++++----------------------- 7 files changed, 96 insertions(+), 140 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index c2caa8df0..9b59f18eb 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -1,11 +1,4 @@ from __future__ import unicode_literals -import logging -# logging.getLogger('boto').setLevel(logging.CRITICAL) - - -__title__ = "moto" -__version__ = "1.3.15.dev" - from .acm import mock_acm # noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # noqa @@ -28,7 +21,7 @@ from .datasync import mock_datasync # noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # noqa from .dynamodbstreams import mock_dynamodbstreams # noqa -from .eb import mock_eb # flake8: noqa +from .eb import mock_eb # noqa from .ec2 import mock_ec2, mock_ec2_deprecated # noqa from .ec2_instance_connect import mock_ec2_instance_connect # noqa from .ecr import mock_ecr, mock_ecr_deprecated # noqa @@ -65,6 +58,12 @@ from .sts import mock_sts, mock_sts_deprecated # noqa from .swf import mock_swf, mock_swf_deprecated # noqa from .xray import XRaySegment, mock_xray, mock_xray_client # noqa +# import logging +# logging.getLogger('boto').setLevel(logging.CRITICAL) + +__title__ = "moto" +__version__ = "1.3.15.dev" + try: # Need to monkey-patch botocore requests back to underlying urllib3 classes diff --git a/moto/core/utils.py b/moto/core/utils.py index 59079784a..dce9f675c 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -331,10 +331,7 @@ def py2_strip_unicode_keys(blob): def tags_from_query_string( - querystring_dict, - prefix="Tag", - key_suffix="Key", - value_suffix="Value" + querystring_dict, prefix="Tag", key_suffix="Key", value_suffix="Value" ): response_values = {} for key, value in querystring_dict.items(): @@ -342,18 +339,14 @@ def tags_from_query_string( tag_index = key.replace(prefix + ".", "").replace("." + key_suffix, "") tag_key = querystring_dict.get( "{prefix}.{index}.{key_suffix}".format( - prefix=prefix, - index=tag_index, - key_suffix=key_suffix, - ))[0] + prefix=prefix, index=tag_index, key_suffix=key_suffix, + ) + )[0] tag_value_key = "{prefix}.{index}.{value_suffix}".format( - prefix=prefix, - index=tag_index, - value_suffix=value_suffix, + prefix=prefix, index=tag_index, value_suffix=value_suffix, ) if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[ - 0] + response_values[tag_key] = querystring_dict.get(tag_value_key)[0] else: response_values[tag_key] = None return response_values diff --git a/moto/eb/exceptions.py b/moto/eb/exceptions.py index bf3a89618..f1e27c564 100644 --- a/moto/eb/exceptions.py +++ b/moto/eb/exceptions.py @@ -4,10 +4,12 @@ from moto.core.exceptions import RESTError class InvalidParameterValueError(RESTError): def __init__(self, message): super(InvalidParameterValueError, self).__init__( - "InvalidParameterValue", message) + "InvalidParameterValue", message + ) class ResourceNotFoundException(RESTError): def __init__(self, message): super(ResourceNotFoundException, self).__init__( - "ResourceNotFoundException", message) + "ResourceNotFoundException", message + ) diff --git a/moto/eb/models.py b/moto/eb/models.py index 4490bbd0c..71873f30c 100644 --- a/moto/eb/models.py +++ b/moto/eb/models.py @@ -8,13 +8,11 @@ from .exceptions import InvalidParameterValueError class FakeEnvironment(BaseModel): def __init__( - self, - application, - environment_name, - solution_stack_name, - tags, + self, application, environment_name, solution_stack_name, tags, ): - self.application = weakref.proxy(application) # weakref to break circular dependencies + self.application = weakref.proxy( + application + ) # weakref to break circular dependencies self.environment_name = environment_name self.solution_stack_name = solution_stack_name self.tags = tags @@ -25,17 +23,19 @@ class FakeEnvironment(BaseModel): @property def environment_arn(self): - return 'arn:aws:elasticbeanstalk:{region}:{account_id}:' \ - 'environment/{application_name}/{environment_name}'.format( + return ( + "arn:aws:elasticbeanstalk:{region}:{account_id}:" + "environment/{application_name}/{environment_name}".format( region=self.region, - account_id='123456789012', + account_id="123456789012", application_name=self.application_name, environment_name=self.environment_name, ) + ) @property def platform_arn(self): - return 'TODO' # TODO + return "TODO" # TODO @property def region(self): @@ -49,10 +49,7 @@ class FakeApplication(BaseModel): self.environments = dict() def create_environment( - self, - environment_name, - solution_stack_name, - tags, + self, environment_name, solution_stack_name, tags, ): if environment_name in self.environments: raise InvalidParameterValueError @@ -89,13 +86,11 @@ class EBBackend(BaseBackend): raise InvalidParameterValueError( "Application {} already exists.".format(application_name) ) - new_app = FakeApplication( - backend=self, - application_name=application_name, - ) + new_app = FakeApplication(backend=self, application_name=application_name,) self.applications[application_name] = new_app return new_app -eb_backends = dict((region.name, EBBackend(region.name)) - for region in boto.beanstalk.regions()) +eb_backends = dict( + (region.name, EBBackend(region.name)) for region in boto.beanstalk.regions() +) diff --git a/moto/eb/responses.py b/moto/eb/responses.py index 905780c44..6178e4a7f 100644 --- a/moto/eb/responses.py +++ b/moto/eb/responses.py @@ -14,42 +14,34 @@ class EBResponse(BaseResponse): def create_application(self): app = self.backend.create_application( - application_name=self._get_param('ApplicationName'), + application_name=self._get_param("ApplicationName"), ) template = self.response_template(EB_CREATE_APPLICATION) - return template.render( - region_name=self.backend.region, - application=app, - ) + return template.render(region_name=self.backend.region, application=app,) def describe_applications(self): template = self.response_template(EB_DESCRIBE_APPLICATIONS) - return template.render( - applications=self.backend.applications.values(), - ) + return template.render(applications=self.backend.applications.values(),) def create_environment(self): - application_name = self._get_param('ApplicationName') + application_name = self._get_param("ApplicationName") try: app = self.backend.applications[application_name] except KeyError: raise InvalidParameterValueError( - "No Application named \'{}\' found.".format(application_name) + "No Application named '{}' found.".format(application_name) ) tags = tags_from_query_string(self.querystring, prefix="Tags.member") env = app.create_environment( - environment_name=self._get_param('EnvironmentName'), - solution_stack_name=self._get_param('SolutionStackName'), + environment_name=self._get_param("EnvironmentName"), + solution_stack_name=self._get_param("SolutionStackName"), tags=tags, ) template = self.response_template(EB_CREATE_ENVIRONMENT) - return template.render( - environment=env, - region=self.backend.region, - ) + return template.render(environment=env, region=self.backend.region,) def describe_environments(self): envs = [] @@ -59,9 +51,7 @@ class EBResponse(BaseResponse): envs.append(env) template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) - return template.render( - environments=envs, - ) + return template.render(environments=envs,) @staticmethod def list_available_solution_stacks(): @@ -75,39 +65,38 @@ class EBResponse(BaseResponse): raise KeyError() def update_tags_for_resource(self): - resource_arn = self._get_param('ResourceArn') + resource_arn = self._get_param("ResourceArn") try: res = self._find_environment_by_arn(resource_arn) except KeyError: raise ResourceNotFoundException( - "Resource not found for ARN \'{}\'.".format(resource_arn) + "Resource not found for ARN '{}'.".format(resource_arn) ) - tags_to_add = tags_from_query_string(self.querystring, prefix="TagsToAdd.member") + tags_to_add = tags_from_query_string( + self.querystring, prefix="TagsToAdd.member" + ) for key, value in tags_to_add.items(): res.tags[key] = value - tags_to_remove = self._get_multi_param('TagsToRemove.member') + tags_to_remove = self._get_multi_param("TagsToRemove.member") for key in tags_to_remove: del res.tags[key] return EB_UPDATE_TAGS_FOR_RESOURCE def list_tags_for_resource(self): - resource_arn = self._get_param('ResourceArn') + resource_arn = self._get_param("ResourceArn") try: res = self._find_environment_by_arn(resource_arn) except KeyError: raise ResourceNotFoundException( - "Resource not found for ARN \'{}\'.".format(resource_arn) + "Resource not found for ARN '{}'.".format(resource_arn) ) tags = res.tags template = self.response_template(EB_LIST_TAGS_FOR_RESOURCE) - return template.render( - tags=tags, - arn=resource_arn, - ) + return template.render(tags=tags, arn=resource_arn,) EB_CREATE_APPLICATION = """ diff --git a/moto/eb/urls.py b/moto/eb/urls.py index 4cd4add13..2d57f7f9d 100644 --- a/moto/eb/urls.py +++ b/moto/eb/urls.py @@ -7,5 +7,5 @@ url_bases = [ ] url_paths = { - '{0}/$': EBResponse.dispatch, + "{0}/$": EBResponse.dispatch, } diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 2b5be4490..1064bf31a 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -8,114 +8,94 @@ from moto import mock_eb @mock_eb def test_create_application(): # Create Elastic Beanstalk Application - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - app = conn.create_application( - ApplicationName="myapp", - ) - app['Application']['ApplicationName'].should.equal("myapp") + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + app = conn.create_application(ApplicationName="myapp",) + app["Application"]["ApplicationName"].should.equal("myapp") @mock_eb def test_create_application_dup(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) + conn.create_application.when.called_with(ApplicationName="myapp",).should.throw( + ClientError ) - conn.create_application.when.called_with( - ApplicationName="myapp", - ).should.throw(ClientError) @mock_eb def test_describe_applications(): # Create Elastic Beanstalk Application - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) apps = conn.describe_applications() - len(apps['Applications']).should.equal(1) - apps['Applications'][0]['ApplicationName'].should.equal('myapp') + len(apps["Applications"]).should.equal(1) + apps["Applications"][0]["ApplicationName"].should.equal("myapp") @mock_eb def test_create_environment(): # Create Elastic Beanstalk Environment - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - app = conn.create_application( - ApplicationName="myapp", - ) - env = conn.create_environment( - ApplicationName="myapp", - EnvironmentName="myenv", - ) - env['EnvironmentName'].should.equal("myenv") + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + app = conn.create_application(ApplicationName="myapp",) + env = conn.create_environment(ApplicationName="myapp", EnvironmentName="myenv",) + env["EnvironmentName"].should.equal("myenv") @mock_eb def test_describe_environments(): # List Elastic Beanstalk Envs - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) conn.create_environment( - ApplicationName="myapp", - EnvironmentName="myenv", + ApplicationName="myapp", EnvironmentName="myenv", ) envs = conn.describe_environments() - envs = envs['Environments'] + envs = envs["Environments"] len(envs).should.equal(1) - envs[0]['ApplicationName'].should.equal('myapp') - envs[0]['EnvironmentName'].should.equal('myenv') + envs[0]["ApplicationName"].should.equal("myapp") + envs[0]["EnvironmentName"].should.equal("myenv") def tags_dict_to_list(tag_dict): tag_list = [] for key, value in tag_dict.items(): - tag_list.append({'Key': key, 'Value': value}) + tag_list.append({"Key": key, "Value": value}) return tag_list def tags_list_to_dict(tag_list): tag_dict = {} for tag in tag_list: - tag_dict[tag['Key']] = tag['Value'] + tag_dict[tag["Key"]] = tag["Value"] return tag_dict @mock_eb def test_create_environment_tags(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) - env_tags = {'initial key': 'initial value'} + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) + env_tags = {"initial key": "initial value"} env = conn.create_environment( ApplicationName="myapp", EnvironmentName="myenv", Tags=tags_dict_to_list(env_tags), ) - tags = conn.list_tags_for_resource( - ResourceArn=env['EnvironmentArn'], - ) - tags['ResourceArn'].should.equal(env['EnvironmentArn']) - tags_list_to_dict(tags['ResourceTags']).should.equal(env_tags) + tags = conn.list_tags_for_resource(ResourceArn=env["EnvironmentArn"],) + tags["ResourceArn"].should.equal(env["EnvironmentArn"]) + tags_list_to_dict(tags["ResourceTags"]).should.equal(env_tags) @mock_eb def test_update_tags(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') - conn.create_application( - ApplicationName="myapp", - ) + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") + conn.create_application(ApplicationName="myapp",) env_tags = { - 'initial key': 'initial value', - 'to remove': 'delete me', - 'to update': 'original', + "initial key": "initial value", + "to remove": "delete me", + "to update": "original", } env = conn.create_environment( ApplicationName="myapp", @@ -124,29 +104,27 @@ def test_update_tags(): ) extra_env_tags = { - 'to update': 'new', - 'extra key': 'extra value', + "to update": "new", + "extra key": "extra value", } conn.update_tags_for_resource( - ResourceArn=env['EnvironmentArn'], + ResourceArn=env["EnvironmentArn"], TagsToAdd=tags_dict_to_list(extra_env_tags), - TagsToRemove=['to remove'], + TagsToRemove=["to remove"], ) total_env_tags = env_tags.copy() total_env_tags.update(extra_env_tags) - del total_env_tags['to remove'] + del total_env_tags["to remove"] - tags = conn.list_tags_for_resource( - ResourceArn=env['EnvironmentArn'], - ) - tags['ResourceArn'].should.equal(env['EnvironmentArn']) - tags_list_to_dict(tags['ResourceTags']).should.equal(total_env_tags) + tags = conn.list_tags_for_resource(ResourceArn=env["EnvironmentArn"],) + tags["ResourceArn"].should.equal(env["EnvironmentArn"]) + tags_list_to_dict(tags["ResourceTags"]).should.equal(total_env_tags) @mock_eb def test_list_available_solution_stacks(): - conn = boto3.client('elasticbeanstalk', region_name='us-east-1') + conn = boto3.client("elasticbeanstalk", region_name="us-east-1") stacks = conn.list_available_solution_stacks() - len(stacks['SolutionStacks']).should.be.greater_than(0) - len(stacks['SolutionStacks']).should.be.equal(len(stacks['SolutionStackDetails'])) + len(stacks["SolutionStacks"]).should.be.greater_than(0) + len(stacks["SolutionStacks"]).should.be.equal(len(stacks["SolutionStackDetails"])) From c32c17a13ed94b3867336e2a9e16c5890ad75973 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 13:49:19 +0100 Subject: [PATCH 18/46] Remove duplicated method --- moto/ec2/utils.py | 16 ---------------- moto/emr/responses.py | 5 +++-- moto/emr/utils.py | 16 ---------------- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 74fe3d27b..61d22d8b2 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -196,22 +196,6 @@ def split_route_id(route_id): return values[0], values[1] -def tags_from_query_string(querystring_dict): - prefix = "Tag" - suffix = "Key" - response_values = {} - for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tag.{0}.Key".format(tag_index))[0] - tag_value_key = "Tag.{0}.Value".format(tag_index) - if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[0] - else: - response_values[tag_key] = None - return response_values - - def dhcp_configuration_from_querystring(querystring, option="DhcpConfiguration"): """ turn: diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 3708db0ed..d2b234ced 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -10,9 +10,10 @@ from six.moves.urllib.parse import urlparse from moto.core.responses import AWSServiceSpec from moto.core.responses import BaseResponse from moto.core.responses import xml_to_json_response +from moto.core.utils import tags_from_query_string from .exceptions import EmrError from .models import emr_backends -from .utils import steps_from_query_string, tags_from_query_string +from .utils import steps_from_query_string def generate_boto3_response(operation): @@ -91,7 +92,7 @@ class ElasticMapReduceResponse(BaseResponse): @generate_boto3_response("AddTags") def add_tags(self): cluster_id = self._get_param("ResourceId") - tags = tags_from_query_string(self.querystring) + tags = tags_from_query_string(self.querystring, prefix="Tags") self.backend.add_tags(cluster_id, tags) template = self.response_template(ADD_TAGS_TEMPLATE) return template.render() diff --git a/moto/emr/utils.py b/moto/emr/utils.py index 0f75995b8..fb33214c8 100644 --- a/moto/emr/utils.py +++ b/moto/emr/utils.py @@ -22,22 +22,6 @@ def random_instance_group_id(size=13): return "i-{0}".format(random_id()) -def tags_from_query_string(querystring_dict): - prefix = "Tags" - suffix = "Key" - response_values = {} - for key, value in querystring_dict.items(): - if key.startswith(prefix) and key.endswith(suffix): - tag_index = key.replace(prefix + ".", "").replace("." + suffix, "") - tag_key = querystring_dict.get("Tags.{0}.Key".format(tag_index))[0] - tag_value_key = "Tags.{0}.Value".format(tag_index) - if tag_value_key in querystring_dict: - response_values[tag_key] = querystring_dict.get(tag_value_key)[0] - else: - response_values[tag_key] = None - return response_values - - def steps_from_query_string(querystring_dict): steps = [] for step in querystring_dict: From 7d524eaec9bb49f8d3e8e55a7f84c1876cd4e3d1 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 14:08:22 +0100 Subject: [PATCH 19/46] Elastic Beanstalk - Rename and Add Implementation Coverage --- IMPLEMENTATION_COVERAGE.md | 14 +++--- moto/__init__.py | 2 +- moto/{eb => elasticbeanstalk}/__init__.py | 2 +- moto/{eb => elasticbeanstalk}/exceptions.py | 0 moto/{eb => elasticbeanstalk}/models.py | 50 ++++++++++++++++++++- moto/{eb => elasticbeanstalk}/responses.py | 41 +++-------------- moto/{eb => elasticbeanstalk}/urls.py | 0 tests/test_eb/test_eb.py | 18 ++++---- 8 files changed, 74 insertions(+), 53 deletions(-) rename moto/{eb => elasticbeanstalk}/__init__.py (59%) rename moto/{eb => elasticbeanstalk}/exceptions.py (100%) rename moto/{eb => elasticbeanstalk}/models.py (61%) rename moto/{eb => elasticbeanstalk}/responses.py (97%) rename moto/{eb => elasticbeanstalk}/urls.py (100%) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 705618524..bd9e9a4cd 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2878,15 +2878,15 @@ - [ ] test_failover ## elasticbeanstalk -0% implemented +13% implemented - [ ] abort_environment_update - [ ] apply_environment_managed_action - [ ] check_dns_availability - [ ] compose_environments -- [ ] create_application +- [X] create_application - [ ] create_application_version - [ ] create_configuration_template -- [ ] create_environment +- [X] create_environment - [ ] create_platform_version - [ ] create_storage_location - [ ] delete_application @@ -2903,13 +2903,13 @@ - [ ] describe_environment_managed_action_history - [ ] describe_environment_managed_actions - [ ] describe_environment_resources -- [ ] describe_environments +- [X] describe_environments - [ ] describe_events - [ ] describe_instances_health - [ ] describe_platform_version -- [ ] list_available_solution_stacks +- [X] list_available_solution_stacks - [ ] list_platform_versions -- [ ] list_tags_for_resource +- [X] list_tags_for_resource - [ ] rebuild_environment - [ ] request_environment_info - [ ] restart_app_server @@ -2921,7 +2921,7 @@ - [ ] update_application_version - [ ] update_configuration_template - [ ] update_environment -- [ ] update_tags_for_resource +- [X] update_tags_for_resource - [ ] validate_configuration_settings ## elastictranscoder diff --git a/moto/__init__.py b/moto/__init__.py index 9b59f18eb..4c9d4753c 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -21,7 +21,7 @@ from .datasync import mock_datasync # noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # noqa from .dynamodbstreams import mock_dynamodbstreams # noqa -from .eb import mock_eb # noqa +from .elasticbeanstalk import mock_elasticbeanstalk # noqa from .ec2 import mock_ec2, mock_ec2_deprecated # noqa from .ec2_instance_connect import mock_ec2_instance_connect # noqa from .ecr import mock_ecr, mock_ecr_deprecated # noqa diff --git a/moto/eb/__init__.py b/moto/elasticbeanstalk/__init__.py similarity index 59% rename from moto/eb/__init__.py rename to moto/elasticbeanstalk/__init__.py index 3e06e9595..851fa445b 100644 --- a/moto/eb/__init__.py +++ b/moto/elasticbeanstalk/__init__.py @@ -1,4 +1,4 @@ from .models import eb_backends from moto.core.models import base_decorator -mock_eb = base_decorator(eb_backends) +mock_elasticbeanstalk = base_decorator(eb_backends) diff --git a/moto/eb/exceptions.py b/moto/elasticbeanstalk/exceptions.py similarity index 100% rename from moto/eb/exceptions.py rename to moto/elasticbeanstalk/exceptions.py diff --git a/moto/eb/models.py b/moto/elasticbeanstalk/models.py similarity index 61% rename from moto/eb/models.py rename to moto/elasticbeanstalk/models.py index 71873f30c..83ad65ab0 100644 --- a/moto/eb/models.py +++ b/moto/elasticbeanstalk/models.py @@ -3,7 +3,7 @@ import weakref import boto.beanstalk from moto.core import BaseBackend, BaseModel -from .exceptions import InvalidParameterValueError +from .exceptions import InvalidParameterValueError, ResourceNotFoundException class FakeEnvironment(BaseModel): @@ -90,6 +90,54 @@ class EBBackend(BaseBackend): self.applications[application_name] = new_app return new_app + def create_environment(self, app, environment_name, stack_name, tags): + return app.create_environment( + environment_name=environment_name, + solution_stack_name=stack_name, + tags=tags, + ) + + def describe_environments(self): + envs = [] + for app in self.applications.values(): + for env in app.environments.values(): + envs.append(env) + return envs + + def list_available_solution_stacks(self): + # Implemented in response.py + pass + + def update_tags_for_resource(self, resource_arn, tags_to_add, tags_to_remove): + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN '{}'.".format(resource_arn) + ) + + for key, value in tags_to_add.items(): + res.tags[key] = value + + for key in tags_to_remove: + del res.tags[key] + + def list_tags_for_resource(self, resource_arn): + try: + res = self._find_environment_by_arn(resource_arn) + except KeyError: + raise ResourceNotFoundException( + "Resource not found for ARN '{}'.".format(resource_arn) + ) + return res.tags + + def _find_environment_by_arn(self, arn): + for app in self.applications.keys(): + for env in self.applications[app].environments.values(): + if env.environment_arn == arn: + return env + raise KeyError() + eb_backends = dict( (region.name, EBBackend(region.name)) for region in boto.beanstalk.regions() diff --git a/moto/eb/responses.py b/moto/elasticbeanstalk/responses.py similarity index 97% rename from moto/eb/responses.py rename to moto/elasticbeanstalk/responses.py index 6178e4a7f..0416121b2 100644 --- a/moto/eb/responses.py +++ b/moto/elasticbeanstalk/responses.py @@ -1,7 +1,7 @@ from moto.core.responses import BaseResponse from moto.core.utils import tags_from_query_string from .models import eb_backends -from .exceptions import InvalidParameterValueError, ResourceNotFoundException +from .exceptions import InvalidParameterValueError class EBResponse(BaseResponse): @@ -34,9 +34,10 @@ class EBResponse(BaseResponse): ) tags = tags_from_query_string(self.querystring, prefix="Tags.member") - env = app.create_environment( + env = self.backend.create_environment( + app, environment_name=self._get_param("EnvironmentName"), - solution_stack_name=self._get_param("SolutionStackName"), + stack_name=self._get_param("SolutionStackName"), tags=tags, ) @@ -44,11 +45,7 @@ class EBResponse(BaseResponse): return template.render(environment=env, region=self.backend.region,) def describe_environments(self): - envs = [] - - for app in self.backend.applications.values(): - for env in app.environments.values(): - envs.append(env) + envs = self.backend.describe_environments() template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) return template.render(environments=envs,) @@ -57,43 +54,19 @@ class EBResponse(BaseResponse): def list_available_solution_stacks(): return EB_LIST_AVAILABLE_SOLUTION_STACKS - def _find_environment_by_arn(self, arn): - for app in self.backend.applications.keys(): - for env in self.backend.applications[app].environments.values(): - if env.environment_arn == arn: - return env - raise KeyError() - def update_tags_for_resource(self): resource_arn = self._get_param("ResourceArn") - try: - res = self._find_environment_by_arn(resource_arn) - except KeyError: - raise ResourceNotFoundException( - "Resource not found for ARN '{}'.".format(resource_arn) - ) - tags_to_add = tags_from_query_string( self.querystring, prefix="TagsToAdd.member" ) - for key, value in tags_to_add.items(): - res.tags[key] = value - tags_to_remove = self._get_multi_param("TagsToRemove.member") - for key in tags_to_remove: - del res.tags[key] + self.backend.update_tags_for_resource(resource_arn, tags_to_add, tags_to_remove) return EB_UPDATE_TAGS_FOR_RESOURCE def list_tags_for_resource(self): resource_arn = self._get_param("ResourceArn") - try: - res = self._find_environment_by_arn(resource_arn) - except KeyError: - raise ResourceNotFoundException( - "Resource not found for ARN '{}'.".format(resource_arn) - ) - tags = res.tags + tags = self.backend.list_tags_for_resource(resource_arn) template = self.response_template(EB_LIST_TAGS_FOR_RESOURCE) return template.render(tags=tags, arn=resource_arn,) diff --git a/moto/eb/urls.py b/moto/elasticbeanstalk/urls.py similarity index 100% rename from moto/eb/urls.py rename to moto/elasticbeanstalk/urls.py diff --git a/tests/test_eb/test_eb.py b/tests/test_eb/test_eb.py index 1064bf31a..42eb09be3 100644 --- a/tests/test_eb/test_eb.py +++ b/tests/test_eb/test_eb.py @@ -2,10 +2,10 @@ import boto3 import sure # noqa from botocore.exceptions import ClientError -from moto import mock_eb +from moto import mock_elasticbeanstalk -@mock_eb +@mock_elasticbeanstalk def test_create_application(): # Create Elastic Beanstalk Application conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -13,7 +13,7 @@ def test_create_application(): app["Application"]["ApplicationName"].should.equal("myapp") -@mock_eb +@mock_elasticbeanstalk def test_create_application_dup(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") conn.create_application(ApplicationName="myapp",) @@ -22,7 +22,7 @@ def test_create_application_dup(): ) -@mock_eb +@mock_elasticbeanstalk def test_describe_applications(): # Create Elastic Beanstalk Application conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -33,7 +33,7 @@ def test_describe_applications(): apps["Applications"][0]["ApplicationName"].should.equal("myapp") -@mock_eb +@mock_elasticbeanstalk def test_create_environment(): # Create Elastic Beanstalk Environment conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -42,7 +42,7 @@ def test_create_environment(): env["EnvironmentName"].should.equal("myenv") -@mock_eb +@mock_elasticbeanstalk def test_describe_environments(): # List Elastic Beanstalk Envs conn = boto3.client("elasticbeanstalk", region_name="us-east-1") @@ -72,7 +72,7 @@ def tags_list_to_dict(tag_list): return tag_dict -@mock_eb +@mock_elasticbeanstalk def test_create_environment_tags(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") conn.create_application(ApplicationName="myapp",) @@ -88,7 +88,7 @@ def test_create_environment_tags(): tags_list_to_dict(tags["ResourceTags"]).should.equal(env_tags) -@mock_eb +@mock_elasticbeanstalk def test_update_tags(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") conn.create_application(ApplicationName="myapp",) @@ -122,7 +122,7 @@ def test_update_tags(): tags_list_to_dict(tags["ResourceTags"]).should.equal(total_env_tags) -@mock_eb +@mock_elasticbeanstalk def test_list_available_solution_stacks(): conn = boto3.client("elasticbeanstalk", region_name="us-east-1") stacks = conn.list_available_solution_stacks() From 551dc024595cc602091ecf640311a3db4a52d6ca Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 30 Mar 2020 16:28:36 +0100 Subject: [PATCH 20/46] ElasticBeanstalk - Fix tests in Python2 and ServerMode --- moto/backends.py | 2 ++ moto/elasticbeanstalk/models.py | 16 ++++++++++++---- moto/elasticbeanstalk/responses.py | 3 +-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/moto/backends.py b/moto/backends.py index a358b8fd2..a48df74a4 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -23,6 +23,7 @@ from moto.ec2 import ec2_backends from moto.ec2_instance_connect import ec2_instance_connect_backends from moto.ecr import ecr_backends from moto.ecs import ecs_backends +from moto.elasticbeanstalk import eb_backends from moto.elb import elb_backends from moto.elbv2 import elbv2_backends from moto.emr import emr_backends @@ -77,6 +78,7 @@ BACKENDS = { "ec2_instance_connect": ec2_instance_connect_backends, "ecr": ecr_backends, "ecs": ecs_backends, + "elasticbeanstalk": eb_backends, "elb": elb_backends, "elbv2": elbv2_backends, "events": events_backends, diff --git a/moto/elasticbeanstalk/models.py b/moto/elasticbeanstalk/models.py index 83ad65ab0..3767846c1 100644 --- a/moto/elasticbeanstalk/models.py +++ b/moto/elasticbeanstalk/models.py @@ -1,6 +1,6 @@ import weakref -import boto.beanstalk +from boto3 import Session from moto.core import BaseBackend, BaseModel from .exceptions import InvalidParameterValueError, ResourceNotFoundException @@ -139,6 +139,14 @@ class EBBackend(BaseBackend): raise KeyError() -eb_backends = dict( - (region.name, EBBackend(region.name)) for region in boto.beanstalk.regions() -) +eb_backends = {} +for region in Session().get_available_regions("elasticbeanstalk"): + eb_backends[region] = EBBackend(region) +for region in Session().get_available_regions( + "elasticbeanstalk", partition_name="aws-us-gov" +): + eb_backends[region] = EBBackend(region) +for region in Session().get_available_regions( + "elasticbeanstalk", partition_name="aws-cn" +): + eb_backends[region] = EBBackend(region) diff --git a/moto/elasticbeanstalk/responses.py b/moto/elasticbeanstalk/responses.py index 0416121b2..387cbb3ea 100644 --- a/moto/elasticbeanstalk/responses.py +++ b/moto/elasticbeanstalk/responses.py @@ -50,8 +50,7 @@ class EBResponse(BaseResponse): template = self.response_template(EB_DESCRIBE_ENVIRONMENTS) return template.render(environments=envs,) - @staticmethod - def list_available_solution_stacks(): + def list_available_solution_stacks(self): return EB_LIST_AVAILABLE_SOLUTION_STACKS def update_tags_for_resource(self): From 6dd6686afcc5c9dc40a9ba90b5b853b2a9f60e48 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 31 Mar 2020 11:10:38 +0100 Subject: [PATCH 21/46] Use TaggingService for S3 Buckets --- moto/s3/models.py | 23 ++++++++++++++++++----- moto/s3/responses.py | 8 ++++---- moto/utilities/tagging_service.py | 12 ++++++++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 8c2a86f41..aede52d26 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -22,6 +22,7 @@ import six from bisect import insort from moto.core import ACCOUNT_ID, BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime +from moto.utilities.tagging_service import TaggingService from .exceptions import ( BucketAlreadyExists, MissingBucket, @@ -787,7 +788,6 @@ class FakeBucket(BaseModel): self.policy = None self.website_configuration = None self.acl = get_canned_acl("private") - self.tags = FakeTagging() self.cors = [] self.logging = {} self.notification_configuration = None @@ -1085,6 +1085,10 @@ class FakeBucket(BaseModel): def set_acl(self, acl): self.acl = acl + @property + def arn(self): + return "arn:aws:s3:::{}".format(self.name) + @property def physical_resource_id(self): return self.name @@ -1110,7 +1114,7 @@ class FakeBucket(BaseModel): int(time.mktime(self.creation_date.timetuple())) ), # PY2 and 3 compatible "configurationItemMD5Hash": "", - "arn": "arn:aws:s3:::{}".format(self.name), + "arn": self.arn, "resourceType": "AWS::S3::Bucket", "resourceId": self.name, "resourceName": self.name, @@ -1119,7 +1123,7 @@ class FakeBucket(BaseModel): "resourceCreationTime": str(self.creation_date), "relatedEvents": [], "relationships": [], - "tags": {tag.key: tag.value for tag in self.tagging.tag_set.tags}, + "tags": s3_backend.tagger.get_tag_dict_for_resource(self.arn), "configuration": { "name": self.name, "owner": {"id": OWNER}, @@ -1181,6 +1185,7 @@ class S3Backend(BaseBackend): def __init__(self): self.buckets = {} self.account_public_access_block = None + self.tagger = TaggingService() def create_bucket(self, bucket_name, region_name): if bucket_name in self.buckets: @@ -1357,16 +1362,24 @@ class S3Backend(BaseBackend): key.set_tagging(tagging) return key + def get_bucket_tags(self, bucket_name): + bucket = self.get_bucket(bucket_name) + return self.tagger.list_tags_for_resource(bucket.arn) + def put_bucket_tagging(self, bucket_name, tagging): tag_keys = [tag.key for tag in tagging.tag_set.tags] if len(tag_keys) != len(set(tag_keys)): raise DuplicateTagKeys() bucket = self.get_bucket(bucket_name) - bucket.set_tags(tagging) + self.tagger.delete_all_tags_for_resource(bucket.arn) + self.tagger.tag_resource( + bucket.arn, + [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + ) def delete_bucket_tagging(self, bucket_name): bucket = self.get_bucket(bucket_name) - bucket.delete_tags() + self.tagger.delete_all_tags_for_resource(bucket.arn) def put_bucket_cors(self, bucket_name, cors_rules): bucket = self.get_bucket(bucket_name) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 197cd9080..f3a5eeaac 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -378,13 +378,13 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_OBJECT_ACL_RESPONSE) return template.render(obj=bucket) elif "tagging" in querystring: - bucket = self.backend.get_bucket(bucket_name) + tags = self.backend.get_bucket_tags(bucket_name)["Tags"] # "Special Error" if no tags: - if len(bucket.tagging.tag_set.tags) == 0: + if len(tags) == 0: template = self.response_template(S3_NO_BUCKET_TAGGING) return 404, {}, template.render(bucket_name=bucket_name) template = self.response_template(S3_BUCKET_TAGGING_RESPONSE) - return template.render(bucket=bucket) + return template.render(tags=tags) elif "logging" in querystring: bucket = self.backend.get_bucket(bucket_name) if not bucket.logging: @@ -1929,7 +1929,7 @@ S3_OBJECT_TAGGING_RESPONSE = """\ S3_BUCKET_TAGGING_RESPONSE = """ - {% for tag in bucket.tagging.tag_set.tags %} + {% for tag in tags %} {{ tag.key }} {{ tag.value }} diff --git a/moto/utilities/tagging_service.py b/moto/utilities/tagging_service.py index 89b857277..8c3228552 100644 --- a/moto/utilities/tagging_service.py +++ b/moto/utilities/tagging_service.py @@ -5,15 +5,23 @@ class TaggingService: self.valueName = valueName self.tags = {} + def get_tag_dict_for_resource(self, arn): + result = {} + if self.has_tags(arn): + for k, v in self.tags[arn].items(): + result[k] = v + return result + def list_tags_for_resource(self, arn): result = [] - if arn in self.tags: + if self.has_tags(arn): for k, v in self.tags[arn].items(): result.append({self.keyName: k, self.valueName: v}) return {self.tagName: result} def delete_all_tags_for_resource(self, arn): - del self.tags[arn] + if self.has_tags(arn): + del self.tags[arn] def has_tags(self, arn): return arn in self.tags From f7ad4cbc09164205d9f216355cec6c921170d3f9 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 31 Mar 2020 12:04:04 +0100 Subject: [PATCH 22/46] Use TaggingService for S3 Objects --- moto/s3/models.py | 30 ++++++++++++------- moto/s3/responses.py | 28 ++++++------------ moto/utilities/tagging_service.py | 6 ++++ tests/test_s3/test_s3.py | 9 ++++-- tests/test_utilities/test_tagging_service.py | 31 ++++++++++++++++++++ 5 files changed, 71 insertions(+), 33 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index aede52d26..b5224b64a 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -95,6 +95,7 @@ class FakeKey(BaseModel): version_id=0, max_buffer_size=DEFAULT_KEY_BUFFER_SIZE, multipart=None, + bucket_name=None, ): self.name = name self.last_modified = datetime.datetime.utcnow() @@ -106,8 +107,8 @@ class FakeKey(BaseModel): self._etag = etag self._version_id = version_id self._is_versioned = is_versioned - self._tagging = FakeTagging() self.multipart = multipart + self.bucket_name = bucket_name self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size) self._max_buffer_size = max_buffer_size @@ -127,6 +128,13 @@ class FakeKey(BaseModel): self.lock.release() return r + @property + def arn(self): + # S3 Objects don't have an ARN, but we do need something unique when creating tags against this resource + return "arn:aws:s3:::{}/{}/{}".format( + self.bucket_name, self.name, self.version_id + ) + @value.setter def value(self, new_value): self._value_buffer.seek(0) @@ -153,9 +161,6 @@ class FakeKey(BaseModel): self._metadata = {} self._metadata.update(metadata) - def set_tagging(self, tagging): - self._tagging = tagging - def set_storage_class(self, storage): if storage is not None and storage not in STORAGE_CLASS: raise InvalidStorageClass(storage=storage) @@ -211,10 +216,6 @@ class FakeKey(BaseModel): def metadata(self): return self._metadata - @property - def tagging(self): - return self._tagging - @property def response_dict(self): res = { @@ -1355,11 +1356,17 @@ class S3Backend(BaseBackend): else: return None - def set_key_tagging(self, bucket_name, key_name, tagging, version_id=None): - key = self.get_key(bucket_name, key_name, version_id) + def get_key_tags(self, key): + return self.tagger.list_tags_for_resource(key.arn) + + def set_key_tags(self, key, tagging, key_name=None): if key is None: raise MissingKey(key_name) - key.set_tagging(tagging) + self.tagger.delete_all_tags_for_resource(key.arn) + self.tagger.tag_resource( + key.arn, + [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + ) return key def get_bucket_tags(self, bucket_name): @@ -1587,6 +1594,7 @@ class S3Backend(BaseBackend): key = self.get_key(src_bucket_name, src_key_name, version_id=src_version_id) new_key = key.copy(dest_key_name, dest_bucket.is_versioned) + self.tagger.copy_tags(key.arn, new_key.arn) if storage is not None: new_key.set_storage_class(storage) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f3a5eeaac..4e3b9a67b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -383,7 +383,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if len(tags) == 0: template = self.response_template(S3_NO_BUCKET_TAGGING) return 404, {}, template.render(bucket_name=bucket_name) - template = self.response_template(S3_BUCKET_TAGGING_RESPONSE) + template = self.response_template(S3_OBJECT_TAGGING_RESPONSE) return template.render(tags=tags) elif "logging" in querystring: bucket = self.backend.get_bucket(bucket_name) @@ -1091,8 +1091,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): template = self.response_template(S3_OBJECT_ACL_RESPONSE) return 200, response_headers, template.render(obj=key) if "tagging" in query: + tags = self.backend.get_key_tags(key)["Tags"] template = self.response_template(S3_OBJECT_TAGGING_RESPONSE) - return 200, response_headers, template.render(obj=key) + return 200, response_headers, template.render(tags=tags) response_headers.update(key.metadata) response_headers.update(key.response_dict) @@ -1164,8 +1165,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): version_id = query["versionId"][0] else: version_id = None + key = self.backend.get_key(bucket_name, key_name, version_id=version_id) tagging = self._tagging_from_xml(body) - self.backend.set_key_tagging(bucket_name, key_name, tagging, version_id) + self.backend.set_key_tags(key, tagging, key_name) return 200, response_headers, "" if "x-amz-copy-source" in request.headers: @@ -1206,7 +1208,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): tdirective = request.headers.get("x-amz-tagging-directive") if tdirective == "REPLACE": tagging = self._tagging_from_headers(request.headers) - new_key.set_tagging(tagging) + self.backend.set_key_tags(new_key, tagging) template = self.response_template(S3_OBJECT_COPY_RESPONSE) response_headers.update(new_key.response_dict) return 200, response_headers, template.render(key=new_key) @@ -1230,7 +1232,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): new_key.website_redirect_location = request.headers.get( "x-amz-website-redirect-location" ) - new_key.set_tagging(tagging) + self.backend.set_key_tags(new_key, tagging) template = self.response_template(S3_OBJECT_RESPONSE) response_headers.update(new_key.response_dict) @@ -1916,23 +1918,11 @@ S3_OBJECT_ACL_RESPONSE = """ S3_OBJECT_TAGGING_RESPONSE = """\ - - {% for tag in obj.tagging.tag_set.tags %} - - {{ tag.key }} - {{ tag.value }} - - {% endfor %} - -""" - -S3_BUCKET_TAGGING_RESPONSE = """ - {% for tag in tags %} - {{ tag.key }} - {{ tag.value }} + {{ tag.Key }} + {{ tag.Value }} {% endfor %} diff --git a/moto/utilities/tagging_service.py b/moto/utilities/tagging_service.py index 8c3228552..2d6ac99c9 100644 --- a/moto/utilities/tagging_service.py +++ b/moto/utilities/tagging_service.py @@ -35,6 +35,12 @@ class TaggingService: else: self.tags[arn][t[self.keyName]] = None + def copy_tags(self, from_arn, to_arn): + if self.has_tags(from_arn): + self.tag_resource( + to_arn, self.list_tags_for_resource(from_arn)[self.tagName] + ) + def untag_resource_using_names(self, arn, tag_names): for name in tag_names: if name in self.tags.get(arn, {}): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 303ed523d..4ddc160a8 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -3255,7 +3255,8 @@ def test_boto3_put_object_tagging_on_earliest_version(): # Older version has tags while the most recent does not resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) resp["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - resp["TagSet"].should.equal( + sorted_tagset = sorted(resp["TagSet"], key=lambda t: t["Key"]) + sorted_tagset.should.equal( [{"Key": "item1", "Value": "foo"}, {"Key": "item2", "Value": "bar"}] ) @@ -3333,7 +3334,8 @@ def test_boto3_put_object_tagging_on_both_version(): resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) resp["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - resp["TagSet"].should.equal( + sorted_tagset = sorted(resp["TagSet"], key=lambda t: t["Key"]) + sorted_tagset.should.equal( [{"Key": "item1", "Value": "foo"}, {"Key": "item2", "Value": "bar"}] ) @@ -3341,7 +3343,8 @@ def test_boto3_put_object_tagging_on_both_version(): Bucket=bucket_name, Key=key, VersionId=second_object.id ) resp["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) - resp["TagSet"].should.equal( + sorted_tagset = sorted(resp["TagSet"], key=lambda t: t["Key"]) + sorted_tagset.should.equal( [{"Key": "item1", "Value": "baz"}, {"Key": "item2", "Value": "bin"}] ) diff --git a/tests/test_utilities/test_tagging_service.py b/tests/test_utilities/test_tagging_service.py index 249e903fe..1eac276a1 100644 --- a/tests/test_utilities/test_tagging_service.py +++ b/tests/test_utilities/test_tagging_service.py @@ -77,3 +77,34 @@ def test_extract_tag_names(): expected = ["key1", "key2"] expected.should.be.equal(actual) + + +def test_copy_non_existing_arn(): + svc = TaggingService() + tags = [{"Key": "key1", "Value": "value1"}, {"Key": "key2", "Value": "value2"}] + svc.tag_resource("new_arn", tags) + # + svc.copy_tags("non_existing_arn", "new_arn") + # Copying from a non-existing ARN should a NOOP + # Assert the old tags still exist + actual = sorted( + svc.list_tags_for_resource("new_arn")["Tags"], key=lambda t: t["Key"] + ) + actual.should.equal(tags) + + +def test_copy_existing_arn(): + svc = TaggingService() + tags_old_arn = [{"Key": "key1", "Value": "value1"}] + tags_new_arn = [{"Key": "key2", "Value": "value2"}] + svc.tag_resource("old_arn", tags_old_arn) + svc.tag_resource("new_arn", tags_new_arn) + # + svc.copy_tags("old_arn", "new_arn") + # Assert the old tags still exist + actual = sorted( + svc.list_tags_for_resource("new_arn")["Tags"], key=lambda t: t["Key"] + ) + actual.should.equal( + [{"Key": "key1", "Value": "value1"}, {"Key": "key2", "Value": "value2"}] + ) From 8dbfd43c5c7556af546764d16b2f49d57a3127c4 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 1 Apr 2020 15:35:25 +0100 Subject: [PATCH 23/46] Use TaggingService for S3 - Cleanup --- moto/s3/models.py | 59 +++++++------------------------ moto/s3/responses.py | 60 +++++++++++++------------------- tests/test_config/test_config.py | 2 ++ tests/test_s3/test_s3.py | 11 ++---- 4 files changed, 40 insertions(+), 92 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index b5224b64a..44a94e7a3 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -35,7 +35,6 @@ from .exceptions import ( MalformedXML, InvalidStorageClass, InvalidTargetBucketForLogging, - DuplicateTagKeys, CrossLocationLoggingProhibitted, NoSuchPublicAccessBlockConfiguration, InvalidPublicAccessBlockConfiguration, @@ -473,26 +472,10 @@ def get_canned_acl(acl): return FakeAcl(grants=grants) -class FakeTagging(BaseModel): - def __init__(self, tag_set=None): - self.tag_set = tag_set or FakeTagSet() - - -class FakeTagSet(BaseModel): - def __init__(self, tags=None): - self.tags = tags or [] - - -class FakeTag(BaseModel): - def __init__(self, key, value=None): - self.key = key - self.value = value - - class LifecycleFilter(BaseModel): def __init__(self, prefix=None, tag=None, and_filter=None): self.prefix = prefix - self.tag = tag + (self.tag_key, self.tag_value) = tag if tag else (None, None) self.and_filter = and_filter def to_config_dict(self): @@ -501,11 +484,11 @@ class LifecycleFilter(BaseModel): "predicate": {"type": "LifecyclePrefixPredicate", "prefix": self.prefix} } - elif self.tag: + elif self.tag_key: return { "predicate": { "type": "LifecycleTagPredicate", - "tag": {"key": self.tag.key, "value": self.tag.value}, + "tag": {"key": self.tag_key, "value": self.tag_value}, } } @@ -529,12 +512,9 @@ class LifecycleAndFilter(BaseModel): if self.prefix is not None: data.append({"type": "LifecyclePrefixPredicate", "prefix": self.prefix}) - for tag in self.tags: + for key, value in self.tags.items(): data.append( - { - "type": "LifecycleTagPredicate", - "tag": {"key": tag.key, "value": tag.value}, - } + {"type": "LifecycleTagPredicate", "tag": {"key": key, "value": value},} ) return data @@ -880,7 +860,7 @@ class FakeBucket(BaseModel): and_filter = None if rule["Filter"].get("And"): filters += 1 - and_tags = [] + and_tags = {} if rule["Filter"]["And"].get("Tag"): if not isinstance(rule["Filter"]["And"]["Tag"], list): rule["Filter"]["And"]["Tag"] = [ @@ -888,7 +868,7 @@ class FakeBucket(BaseModel): ] for t in rule["Filter"]["And"]["Tag"]: - and_tags.append(FakeTag(t["Key"], t.get("Value", ""))) + and_tags[t["Key"]] = t.get("Value", "") try: and_prefix = ( @@ -902,7 +882,7 @@ class FakeBucket(BaseModel): filter_tag = None if rule["Filter"].get("Tag"): filters += 1 - filter_tag = FakeTag( + filter_tag = ( rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", ""), ) @@ -989,16 +969,6 @@ class FakeBucket(BaseModel): def delete_cors(self): self.cors = [] - def set_tags(self, tagging): - self.tags = tagging - - def delete_tags(self): - self.tags = FakeTagging() - - @property - def tagging(self): - return self.tags - def set_logging(self, logging_config, bucket_backend): if not logging_config: self.logging = {} @@ -1359,13 +1329,12 @@ class S3Backend(BaseBackend): def get_key_tags(self, key): return self.tagger.list_tags_for_resource(key.arn) - def set_key_tags(self, key, tagging, key_name=None): + def set_key_tags(self, key, tags, key_name=None): if key is None: raise MissingKey(key_name) self.tagger.delete_all_tags_for_resource(key.arn) self.tagger.tag_resource( - key.arn, - [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + key.arn, [{"Key": key, "Value": value} for key, value in tags.items()], ) return key @@ -1373,15 +1342,11 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) return self.tagger.list_tags_for_resource(bucket.arn) - def put_bucket_tagging(self, bucket_name, tagging): - tag_keys = [tag.key for tag in tagging.tag_set.tags] - if len(tag_keys) != len(set(tag_keys)): - raise DuplicateTagKeys() + def put_bucket_tags(self, bucket_name, tags): bucket = self.get_bucket(bucket_name) self.tagger.delete_all_tags_for_resource(bucket.arn) self.tagger.tag_resource( - bucket.arn, - [{"Key": tag.key, "Value": tag.value} for tag in tagging.tag_set.tags], + bucket.arn, [{"Key": key, "Value": value} for key, value in tags.items()], ) def delete_bucket_tagging(self, bucket_name): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 4e3b9a67b..913b20861 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -24,6 +24,7 @@ from moto.s3bucket_path.utils import ( from .exceptions import ( BucketAlreadyExists, + DuplicateTagKeys, S3ClientError, MissingBucket, MissingKey, @@ -43,9 +44,6 @@ from .models import ( FakeGrant, FakeAcl, FakeKey, - FakeTagging, - FakeTagSet, - FakeTag, ) from .utils import ( bucket_name_from_url, @@ -652,7 +650,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return "" elif "tagging" in querystring: tagging = self._bucket_tagging_from_xml(body) - self.backend.put_bucket_tagging(bucket_name, tagging) + self.backend.put_bucket_tags(bucket_name, tagging) return "" elif "website" in querystring: self.backend.set_bucket_website_configuration(bucket_name, body) @@ -1361,55 +1359,45 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return None def _tagging_from_headers(self, headers): + tags = {} if headers.get("x-amz-tagging"): parsed_header = parse_qs(headers["x-amz-tagging"], keep_blank_values=True) - tags = [] for tag in parsed_header.items(): - tags.append(FakeTag(tag[0], tag[1][0])) - - tag_set = FakeTagSet(tags) - tagging = FakeTagging(tag_set) - return tagging - else: - return FakeTagging() + tags[tag[0]] = tag[1][0] + return tags def _tagging_from_xml(self, xml): parsed_xml = xmltodict.parse(xml, force_list={"Tag": True}) - tags = [] + tags = {} for tag in parsed_xml["Tagging"]["TagSet"]["Tag"]: - tags.append(FakeTag(tag["Key"], tag["Value"])) + tags[tag["Key"]] = tag["Value"] - tag_set = FakeTagSet(tags) - tagging = FakeTagging(tag_set) - return tagging + return tags def _bucket_tagging_from_xml(self, xml): parsed_xml = xmltodict.parse(xml) - tags = [] + tags = {} # Optional if no tags are being sent: if parsed_xml["Tagging"].get("TagSet"): # If there is only 1 tag, then it's not a list: if not isinstance(parsed_xml["Tagging"]["TagSet"]["Tag"], list): - tags.append( - FakeTag( - parsed_xml["Tagging"]["TagSet"]["Tag"]["Key"], - parsed_xml["Tagging"]["TagSet"]["Tag"]["Value"], - ) - ) + tags[parsed_xml["Tagging"]["TagSet"]["Tag"]["Key"]] = parsed_xml[ + "Tagging" + ]["TagSet"]["Tag"]["Value"] else: for tag in parsed_xml["Tagging"]["TagSet"]["Tag"]: - tags.append(FakeTag(tag["Key"], tag["Value"])) + if tag["Key"] in tags: + raise DuplicateTagKeys() + tags[tag["Key"]] = tag["Value"] # Verify that "aws:" is not in the tags. If so, then this is a problem: - for tag in tags: - if tag.key.startswith("aws:"): + for key, _ in tags.items(): + if key.startswith("aws:"): raise NoSystemTags() - tag_set = FakeTagSet(tags) - tagging = FakeTagging(tag_set) - return tagging + return tags def _cors_from_xml(self, xml): parsed_xml = xmltodict.parse(xml) @@ -1730,10 +1718,10 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% if rule.filter.prefix != None %} {{ rule.filter.prefix }} {% endif %} - {% if rule.filter.tag %} + {% if rule.filter.tag_key %} - {{ rule.filter.tag.key }} - {{ rule.filter.tag.value }} + {{ rule.filter.tag_key }} + {{ rule.filter.tag_value }} {% endif %} {% if rule.filter.and_filter %} @@ -1741,10 +1729,10 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% if rule.filter.and_filter.prefix != None %} {{ rule.filter.and_filter.prefix }} {% endif %} - {% for tag in rule.filter.and_filter.tags %} + {% for key, value in rule.filter.and_filter.tags.items() %} - {{ tag.key }} - {{ tag.value }} + {{ key }} + {{ value }} {% endfor %} diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 1ffd52a2c..1bf39428e 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -11,6 +11,8 @@ from moto import mock_s3 from moto.config import mock_config from moto.core import ACCOUNT_ID +import sure # noqa + @mock_config def test_put_configuration_recorder(): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 4ddc160a8..e2acf32f2 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4295,24 +4295,17 @@ def test_s3_config_dict(): FakeAcl, FakeGrant, FakeGrantee, - FakeTag, - FakeTagging, - FakeTagSet, OWNER, ) # Without any buckets: assert not s3_config_query.get_config_resource("some_bucket") - tags = FakeTagging( - FakeTagSet( - [FakeTag("someTag", "someValue"), FakeTag("someOtherTag", "someOtherValue")] - ) - ) + tags = {"someTag": "someValue", "someOtherTag": "someOtherValue"} # With 1 bucket in us-west-2: s3_config_query.backends["global"].create_bucket("bucket1", "us-west-2") - s3_config_query.backends["global"].put_bucket_tagging("bucket1", tags) + s3_config_query.backends["global"].put_bucket_tags("bucket1", tags) # With a log bucket: s3_config_query.backends["global"].create_bucket("logbucket", "us-west-2") From dff1ab580b20a22a0b12f8731e86d9468f42cf4b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 1 Apr 2020 16:15:03 +0100 Subject: [PATCH 24/46] Extend new S3 tag structure to ResourceGroupStaging API --- moto/resourcegroupstaggingapi/models.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/moto/resourcegroupstaggingapi/models.py b/moto/resourcegroupstaggingapi/models.py index d05a53f81..b6e35d586 100644 --- a/moto/resourcegroupstaggingapi/models.py +++ b/moto/resourcegroupstaggingapi/models.py @@ -145,10 +145,7 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Do S3, resource type s3 if not resource_type_filters or "s3" in resource_type_filters: for bucket in self.s3_backend.buckets.values(): - tags = [] - for tag in bucket.tags.tag_set.tags: - tags.append({"Key": tag.key, "Value": tag.value}) - + tags = self.s3_backend.tagger.list_tags_for_resource(bucket.arn)["Tags"] if not tags or not tag_filter( tags ): # Skip if no tags, or invalid filter @@ -362,8 +359,9 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Do S3, resource type s3 for bucket in self.s3_backend.buckets.values(): - for tag in bucket.tags.tag_set.tags: - yield tag.key + tags = self.s3_backend.tagger.get_tag_dict_for_resource(bucket.arn) + for key, _ in tags.items(): + yield key # EC2 tags def get_ec2_keys(res_id): @@ -414,9 +412,10 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Do S3, resource type s3 for bucket in self.s3_backend.buckets.values(): - for tag in bucket.tags.tag_set.tags: - if tag.key == tag_key: - yield tag.value + tags = self.s3_backend.tagger.get_tag_dict_for_resource(bucket.arn) + for key, value in tags.items(): + if key == tag_key: + yield value # EC2 tags def get_ec2_values(res_id): From 9ab02e17d528d461b5a05b7aafa623addae65966 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 3 Apr 2020 10:30:05 +0100 Subject: [PATCH 25/46] #883 - Lambda - Add test to verify remove_permission functinonality --- moto/awslambda/models.py | 4 ++-- moto/awslambda/responses.py | 6 ++--- tests/test_awslambda/test_lambda.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 9cdf2397c..589a790ae 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -1006,11 +1006,11 @@ class LambdaBackend(BaseBackend): return True return False - def add_policy_statement(self, function_name, raw): + def add_permission(self, function_name, raw): fn = self.get_function(function_name) fn.policy.add_statement(raw) - def del_policy_statement(self, function_name, sid, revision=""): + def remove_permission(self, function_name, sid, revision=""): fn = self.get_function(function_name) fn.policy.del_statement(sid, revision) diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index ce6c93f16..4213840f6 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -146,7 +146,7 @@ class LambdaResponse(BaseResponse): function_name = path.split("/")[-2] if self.lambda_backend.get_function(function_name): statement = self.body - self.lambda_backend.add_policy_statement(function_name, statement) + self.lambda_backend.add_permission(function_name, statement) return 200, {}, json.dumps({"Statement": statement}) else: return 404, {}, "{}" @@ -166,9 +166,7 @@ class LambdaResponse(BaseResponse): statement_id = path.split("/")[-1].split("?")[0] revision = querystring.get("RevisionId", "") if self.lambda_backend.get_function(function_name): - self.lambda_backend.del_policy_statement( - function_name, statement_id, revision - ) + self.lambda_backend.remove_permission(function_name, statement_id, revision) return 204, {}, "{}" else: return 404, {}, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index eb8453e43..e67576518 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1677,6 +1677,42 @@ def test_create_function_with_unknown_arn(): ) +@mock_lambda +def test_remove_function_permission(): + conn = boto3.client("lambda", _lambda_region) + zip_content = get_test_zip_file1() + conn.create_function( + FunctionName="testFunction", + Runtime="python2.7", + Role=(get_role_name()), + Handler="lambda_function.handler", + Code={"ZipFile": zip_content}, + Description="test lambda function", + Timeout=3, + MemorySize=128, + Publish=True, + ) + + conn.add_permission( + FunctionName="testFunction", + StatementId="1", + Action="lambda:InvokeFunction", + Principal="432143214321", + SourceArn="arn:aws:lambda:us-west-2:account-id:function:helloworld", + SourceAccount="123412341234", + EventSourceToken="blah", + Qualifier="2", + ) + + remove = conn.remove_permission( + FunctionName="testFunction", StatementId="1", Qualifier="2", + ) + remove["ResponseMetadata"]["HTTPStatusCode"].should.equal(204) + policy = conn.get_policy(FunctionName="testFunction", Qualifier="2")["Policy"] + policy = json.loads(policy) + policy["Statement"].should.equal([]) + + def create_invalid_lambda(role): conn = boto3.client("lambda", _lambda_region) zip_content = get_test_zip_file1() From 280db9df6c43f606721d07b51e104eda8e065313 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 4 Apr 2020 14:09:38 +0100 Subject: [PATCH 26/46] #2800 - CognitoIdentity - Fix format of Identity ID --- moto/cognitoidentity/utils.py | 4 ++-- tests/test_cognitoidentity/test_cognitoidentity.py | 8 +++++--- tests/test_cognitoidentity/test_server.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/moto/cognitoidentity/utils.py b/moto/cognitoidentity/utils.py index 6143d5121..54016ad17 100644 --- a/moto/cognitoidentity/utils.py +++ b/moto/cognitoidentity/utils.py @@ -1,5 +1,5 @@ -from moto.core.utils import get_random_hex +from uuid import uuid4 def get_random_identity_id(region): - return "{0}:{1}".format(region, get_random_hex(length=19)) + return "{0}:{1}".format(region, uuid4()) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index 8eae183c6..0ec7acfb0 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -7,6 +7,7 @@ from nose.tools import assert_raises from moto import mock_cognitoidentity from moto.cognitoidentity.utils import get_random_identity_id from moto.core import ACCOUNT_ID +from uuid import UUID @mock_cognitoidentity @@ -83,8 +84,10 @@ def test_describe_identity_pool_with_invalid_id_raises_error(): # testing a helper function def test_get_random_identity_id(): - assert len(get_random_identity_id("us-west-2")) > 0 - assert len(get_random_identity_id("us-west-2").split(":")[1]) == 19 + identity_id = get_random_identity_id("us-west-2") + region, id = identity_id.split(":") + region.should.equal("us-west-2") + UUID(id, version=4) # Will throw an error if it's not a valid UUID @mock_cognitoidentity @@ -96,7 +99,6 @@ def test_get_id(): IdentityPoolId="us-west-2:12345", Logins={"someurl": "12345"}, ) - print(result) assert ( result.get("IdentityId", "").startswith("us-west-2") or result.get("ResponseMetadata").get("HTTPStatusCode") == 200 diff --git a/tests/test_cognitoidentity/test_server.py b/tests/test_cognitoidentity/test_server.py index 903dae290..8c4229f06 100644 --- a/tests/test_cognitoidentity/test_server.py +++ b/tests/test_cognitoidentity/test_server.py @@ -48,6 +48,5 @@ def test_get_id(): }, ) - print(res.data) json_data = json.loads(res.data.decode("utf-8")) assert ":" in json_data["IdentityId"] From 1e1fe3ee4bffb5470795d8aab16fa3de1145f5a6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:13 +0100 Subject: [PATCH 27/46] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a35eded61..de2a06fd4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -286,7 +286,7 @@ class Item(BaseModel): return "Item: {0}".format(self.to_json()) def size(self): - return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + return sum(bytesize(key) + value.size() for key, value in self.attrs.items()) def to_json(self): attributes = {} From 8122a40be064f28dd0aa2ea8567cc2fb2ce4dea8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:20 +0100 Subject: [PATCH 28/46] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index de2a06fd4..62c60efb0 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1127,7 +1127,7 @@ class Table(BaseModel): last_evaluated_key = None size_limit = 1000000 # DynamoDB has a 1MB size limit - item_size = sum([res.size() for res in results]) + item_size = sum(res.size() for res in results) if item_size > size_limit: item_size = idx = 0 while item_size + results[idx].size() < size_limit: From c2b4c397f272f80684c395150159028ed265fd20 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:53:15 +0100 Subject: [PATCH 29/46] DDB test - Fix KeySchema, and set BillingMode for easier online testing --- tests/test_dynamodb2/test_dynamodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index daae79232..e00a45e1d 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4143,13 +4143,13 @@ def test_dynamodb_max_1mb_limit(): TableName=table_name, KeySchema=[ {"AttributeName": "partition_key", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "SORT"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, ], AttributeDefinitions=[ {"AttributeName": "partition_key", "AttributeType": "S"}, {"AttributeName": "sort_key", "AttributeType": "S"}, ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + BillingMode="PAY_PER_REQUEST", ) # Populate the table @@ -4170,3 +4170,4 @@ def test_dynamodb_max_1mb_limit(): # We shouldn't get everything back - the total result set is well over 1MB assert response["Count"] < len(items) response["LastEvaluatedKey"].shouldnt.be(None) + From a6902e87137da229d63d138b898a63ffd12fe326 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 15 Apr 2020 07:26:09 +0100 Subject: [PATCH 30/46] Update tests/test_dynamodb2/test_dynamodb.py Co-Authored-By: Guilherme Martins Crocetti --- tests/test_dynamodb2/test_dynamodb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e00a45e1d..2b4c0969c 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4168,6 +4168,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - assert response["Count"] < len(items) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) - From 870b34ba7693e88df38d2f2765b972cfda955cee Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 16 Apr 2020 07:09:50 +0100 Subject: [PATCH 31/46] Spacing --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 8a8c69a8c..a0a5e6406 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4213,5 +4213,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - len(items).should.be.greater_than(response["Count"]) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) From ed5e0b586c50955515fb2ed72a7c028ece91a9d3 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Mon, 20 Apr 2020 00:15:00 +0300 Subject: [PATCH 32/46] Handle ValueError raised on missing table name --- moto/dynamodb2/responses.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 65484aa08..32f10abc0 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -92,16 +92,24 @@ class DynamoHandler(BaseResponse): def list_tables(self): body = self.body limit = body.get("Limit", 100) - if body.get("ExclusiveStartTableName"): - last = body.get("ExclusiveStartTableName") - start = list(self.dynamodb_backend.tables.keys()).index(last) + 1 + all_tables = list(self.dynamodb_backend.tables.keys()) + + exclusive_start_table_name = body.get("ExclusiveStartTableName") + if exclusive_start_table_name: + try: + last_table_index = all_tables.index(exclusive_start_table_name) + except ValueError: + start = len(all_tables) + else: + start = last_table_index + 1 else: start = 0 - all_tables = list(self.dynamodb_backend.tables.keys()) + if limit: tables = all_tables[start : start + limit] else: tables = all_tables[start:] + response = {"TableNames": tables} if limit and len(all_tables) > start + limit: response["LastEvaluatedTableName"] = tables[-1] From 1a3a7d6a92619ec4142f5d4263f4b308b15e4209 Mon Sep 17 00:00:00 2001 From: Dmitry Ryzhikov Date: Mon, 20 Apr 2020 20:23:37 +0300 Subject: [PATCH 33/46] Add test for missing table name --- tests/test_dynamodb2/test_dynamodb.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index bec24c966..cb9230a4a 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4177,3 +4177,12 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_list_tables_exclusive_start_table_name_empty(): + client = boto3.client("dynamodb", region_name="us-east-1") + + resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever") + + len(resp["TableNames"]).should.equal(0) From 753a39ed0d195c3f3092d2f22fa361f93711f57b Mon Sep 17 00:00:00 2001 From: MarcosBernal Date: Tue, 21 Apr 2020 20:10:39 +0200 Subject: [PATCH 34/46] Add get_databases method to glue moto client. Update IMPLEMENTATION_COVERAGE.md with methods that were covered previously --- IMPLEMENTATION_COVERAGE.md | 18 +++++++++--------- moto/glue/models.py | 3 +++ moto/glue/responses.py | 4 ++++ tests/test_glue/test_datacatalog.py | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 705618524..82ee2f046 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3351,11 +3351,11 @@ - [ ] update_listener ## glue -4% implemented -- [ ] batch_create_partition +14/123 = 11% implemented +- [X] batch_create_partition - [ ] batch_delete_connection -- [ ] batch_delete_partition -- [ ] batch_delete_table +- [X] batch_delete_partition +- [X] batch_delete_table - [ ] batch_delete_table_version - [ ] batch_get_crawlers - [ ] batch_get_dev_endpoints @@ -3372,7 +3372,7 @@ - [ ] create_dev_endpoint - [ ] create_job - [ ] create_ml_transform -- [ ] create_partition +- [X] create_partition - [ ] create_script - [ ] create_security_configuration - [X] create_table @@ -3404,7 +3404,7 @@ - [ ] get_crawlers - [ ] get_data_catalog_encryption_settings - [X] get_database -- [ ] get_databases +- [X] get_databases - [ ] get_dataflow_graph - [ ] get_dev_endpoint - [ ] get_dev_endpoints @@ -3418,7 +3418,7 @@ - [ ] get_ml_task_runs - [ ] get_ml_transform - [ ] get_ml_transforms -- [ ] get_partition +- [X] get_partition - [ ] get_partitions - [ ] get_plan - [ ] get_resource_policy @@ -3470,8 +3470,8 @@ - [ ] update_dev_endpoint - [ ] update_job - [ ] update_ml_transform -- [ ] update_partition -- [ ] update_table +- [X] update_partition +- [X] update_table - [ ] update_trigger - [ ] update_user_defined_function - [ ] update_workflow diff --git a/moto/glue/models.py b/moto/glue/models.py index 8f3396d9a..cf930cfb2 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -34,6 +34,9 @@ class GlueBackend(BaseBackend): except KeyError: raise DatabaseNotFoundException(database_name) + def get_databases(self): + return [self.databases[key] for key in self.databases] if self.databases else [] + def create_table(self, database_name, table_name, table_input): database = self.get_database(database_name) diff --git a/moto/glue/responses.py b/moto/glue/responses.py index bf7b5776b..4fb144bba 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -30,6 +30,10 @@ class GlueResponse(BaseResponse): database = self.glue_backend.get_database(database_name) return json.dumps({"Database": {"Name": database.name}}) + def get_databases(self): + database_list = self.glue_backend.get_databases() + return json.dumps({"DatabaseList": [{"Name": database.name} for database in database_list]}) + def create_table(self): database_name = self.parameters.get("DatabaseName") table_input = self.parameters.get("TableInput") diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 31731e598..54fb17451 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -52,6 +52,27 @@ def test_get_database_not_exits(): ) +@mock_glue +def test_get_databases_empty(): + client = boto3.client("glue", region_name="us-east-1") + response = client.get_databases() + response["DatabaseList"].should.have.length_of(0) + + +@mock_glue +def test_get_databases_several_items(): + client = boto3.client("glue", region_name="us-east-1") + database_name_1, database_name_2 = "firstdatabase", "seconddatabase" + + helpers.create_database(client, database_name_1) + helpers.create_database(client, database_name_2) + + database_list = sorted(client.get_databases()["DatabaseList"], key=lambda x: x["Name"]) + database_list.should.have.length_of(2) + database_list[0].should.equal({"Name": database_name_1}) + database_list[1].should.equal({"Name": database_name_2}) + + @mock_glue def test_create_table(): client = boto3.client("glue", region_name="us-east-1") From 9381c670ab5d9ab169b071c4cf7580d04dfb4636 Mon Sep 17 00:00:00 2001 From: MarcosBernal Date: Tue, 21 Apr 2020 22:33:55 +0200 Subject: [PATCH 35/46] change code style to pass black --check --- moto/glue/responses.py | 4 +++- tests/test_glue/test_datacatalog.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moto/glue/responses.py b/moto/glue/responses.py index 4fb144bba..66185e099 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -32,7 +32,9 @@ class GlueResponse(BaseResponse): def get_databases(self): database_list = self.glue_backend.get_databases() - return json.dumps({"DatabaseList": [{"Name": database.name} for database in database_list]}) + return json.dumps( + {"DatabaseList": [{"Name": database.name} for database in database_list]} + ) def create_table(self): database_name = self.parameters.get("DatabaseName") diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 54fb17451..bc68b48f6 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -67,7 +67,9 @@ def test_get_databases_several_items(): helpers.create_database(client, database_name_1) helpers.create_database(client, database_name_2) - database_list = sorted(client.get_databases()["DatabaseList"], key=lambda x: x["Name"]) + database_list = sorted( + client.get_databases()["DatabaseList"], key=lambda x: x["Name"] + ) database_list.should.have.length_of(2) database_list[0].should.equal({"Name": database_name_1}) database_list[1].should.equal({"Name": database_name_2}) From 50a147592debbbb5e887d40d34ae146dbb266cdd Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 22 Apr 2020 09:08:30 -0500 Subject: [PATCH 36/46] Make all CallbackResponse requests into a Werkzeug Request The "request" object in CallbackResponse is the PreparedRequest send by whatever client is used to contact the mocked moto service. This can end up with unparsed bodies, as we added for processing presigned post requests in #2155. This will make sure that all of the requests comming in from mocked functions also get processed by werkzeug as if it was running a live server. --- moto/core/models.py | 20 ++++++++++++++++++++ moto/s3/responses.py | 9 --------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 73942c669..460823bd6 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -12,6 +12,8 @@ from io import BytesIO from collections import defaultdict from botocore.handlers import BUILTIN_HANDLERS from botocore.awsrequest import AWSResponse +from six.moves.urllib.parse import urlparse +from werkzeug.wrappers import Request import mock from moto import settings @@ -175,6 +177,24 @@ class CallbackResponse(responses.CallbackResponse): """ Need to override this so we can pass decode_content=False """ + if not isinstance(request, Request): + url = urlparse(request.url) + if request.body is None: + body = None + elif isinstance(request.body, six.text_type): + body = six.BytesIO(six.b(request.body)) + else: + body = six.BytesIO(request.body) + req = Request.from_values( + path='?'.join([url.path, url.query]), + input_stream=body, + content_length=request.headers.get("Content-Length"), + content_type=request.headers.get("Content-Type"), + method=request.method, + base_url='{scheme}://{netloc}'.format(scheme=url.scheme, netloc=url.netloc), + headers=[(k, v) for k, v in six.iteritems(request.headers)] + ) + request = req headers = self.get_headers() result = self.callback(request) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6ac139a14..442489a8a 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -5,7 +5,6 @@ import sys import six from botocore.awsrequest import AWSPreparedRequest -from werkzeug.wrappers import Request from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl @@ -797,14 +796,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if hasattr(request, "form"): # Not HTTPretty form = request.form - elif request.headers.get("Content-Type").startswith("multipart/form-data"): - request = Request.from_values( - input_stream=six.BytesIO(request.body), - content_length=request.headers["Content-Length"], - content_type=request.headers["Content-Type"], - method="POST", - ) - form = request.form else: # HTTPretty, build new form object body = body.decode() From 4cd2b201b5cb07165816b1e3a0c453e7a5410d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20Bernal=20Espa=C3=B1a?= Date: Wed, 22 Apr 2020 16:44:25 +0200 Subject: [PATCH 37/46] Update IMPLEMENTATION_COVERAGE.md Co-Authored-By: Bert Blommers --- IMPLEMENTATION_COVERAGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 82ee2f046..78c7ba0e4 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3351,7 +3351,7 @@ - [ ] update_listener ## glue -14/123 = 11% implemented +11% implemented - [X] batch_create_partition - [ ] batch_delete_connection - [X] batch_delete_partition From d9e2aeed5856ef762779d8920572398a1ed6c4c1 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Wed, 22 Apr 2020 10:02:25 -0500 Subject: [PATCH 38/46] blacken --- moto/core/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 460823bd6..1ee11607a 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -186,13 +186,15 @@ class CallbackResponse(responses.CallbackResponse): else: body = six.BytesIO(request.body) req = Request.from_values( - path='?'.join([url.path, url.query]), + path="?".join([url.path, url.query]), input_stream=body, content_length=request.headers.get("Content-Length"), content_type=request.headers.get("Content-Type"), method=request.method, - base_url='{scheme}://{netloc}'.format(scheme=url.scheme, netloc=url.netloc), - headers=[(k, v) for k, v in six.iteritems(request.headers)] + base_url="{scheme}://{netloc}".format( + scheme=url.scheme, netloc=url.netloc + ), + headers=[(k, v) for k, v in six.iteritems(request.headers)], ) request = req headers = self.get_headers() From 343b20a5fbfebac4cebf3f1dbd6e794084fb65fe Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 22 Apr 2020 16:36:41 +0100 Subject: [PATCH 39/46] Update CONTRIBUTING to add Linting info --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40da55ccf..941fc0624 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,10 @@ Moto has a [Code of Conduct](https://github.com/spulec/moto/blob/master/CODE_OF_ ## Running the tests locally -Moto has a Makefile which has some helpful commands for getting setup. You should be able to run `make init` to install the dependencies and then `make test` to run the tests. +Moto has a Makefile which has some helpful commands for getting setup. You should be able to run `make init` to install the dependencies and then `make test` to run the tests. + +## Linting +Run `make lint` or `black --check moto tests` to verify whether your code confirms to the guidelines. ## Is there a missing feature? From 194de2b6eaf5d884b6bbce8856d6f0e21eb45149 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Wed, 22 Apr 2020 13:32:12 -0700 Subject: [PATCH 40/46] Add af-south-1 The new version of botcore adds this region: https://github.com/boto/botocore/commit/f7dc4730ad34c6c3322da7d43ba64452bb3ae0d8#diff-9dfab05d4ba739e097a193e8b5fa61caR13 Which in turn, breaks moto: ``` /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/__init__.py:3: in from .acm import mock_acm # noqa /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/acm/__init__.py:2: in from .models import acm_backends /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/acm/models.py:7: in from moto.ec2 import ec2_backends /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/__init__.py:2: in from .models import ec2_backends /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:5169: in ec2_backends = { /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:5170: in region.name: EC2Backend(region.name) for region in RegionsAndZonesBackend.regions /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:5083: in __init__ super(EC2Backend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:852: in __init__ super(InstanceBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1127: in __init__ super(TagBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:2368: in __init__ super(EBSBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1904: in __init__ super(SecurityGroupBackend, self).__init__() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1355: in __init__ self._load_amis() /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1362: in _load_amis self.amis[ami_id] = Ami(self, **ami) /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1313: in __init__ volume = self.ec2_backend.create_volume(15, region_name) /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:2372: in create_volume zone = self.get_zone_by_name(zone_name) /moto-1.3.15.dev640-py2.py3-none-any.whl/moto/ec2/models.py:1713: in get_zone_by_name for zone in self.zones[self.region_name]: E KeyError: 'af-south-1' ``` --- moto/ec2/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index ee9b0fcc4..dc8e617e0 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1512,6 +1512,9 @@ class RegionsAndZonesBackend(object): regions.append(Region(region, "ec2.{}.amazonaws.com.cn".format(region))) zones = { + "af-south-1": [ + Zone(region_name="af-south-1", name="af-south-1a", zone_id="afs1-az1"), + ], "ap-south-1": [ Zone(region_name="ap-south-1", name="ap-south-1a", zone_id="aps1-az1"), Zone(region_name="ap-south-1", name="ap-south-1b", zone_id="aps1-az3"), From 1d31ea6397ef2349ea89ba481b1c2af23bde9d05 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Thu, 23 Apr 2020 08:25:14 -0700 Subject: [PATCH 41/46] add two more zones. --- moto/ec2/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index dc8e617e0..332c8f030 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1514,6 +1514,8 @@ class RegionsAndZonesBackend(object): zones = { "af-south-1": [ Zone(region_name="af-south-1", name="af-south-1a", zone_id="afs1-az1"), + Zone(region_name="af-south-1", name="af-south-1b", zone_id="afs1-az2"), + Zone(region_name="af-south-1", name="af-south-1c", zone_id="afs1-az3"), ], "ap-south-1": [ Zone(region_name="ap-south-1", name="ap-south-1a", zone_id="aps1-az1"), From a658900d69ca4ae36a4b265161809a529aabb211 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 03:13:36 -0500 Subject: [PATCH 42/46] Add policy to apigateway --- moto/apigateway/models.py | 4 ++++ moto/apigateway/responses.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index e5e5e3bfd..e011af601 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -461,6 +461,7 @@ class RestAPI(BaseModel): self.description = description self.create_date = int(time.time()) self.api_key_source = kwargs.get("api_key_source") or "HEADER" + self.policy = kwargs.get("policy") or None self.endpoint_configuration = kwargs.get("endpoint_configuration") or { "types": ["EDGE"] } @@ -485,6 +486,7 @@ class RestAPI(BaseModel): "apiKeySource": self.api_key_source, "endpointConfiguration": self.endpoint_configuration, "tags": self.tags, + "policy": self.policy, } def add_child(self, path, parent_id=None): @@ -713,6 +715,7 @@ class APIGatewayBackend(BaseBackend): api_key_source=None, endpoint_configuration=None, tags=None, + policy=None, ): api_id = create_id() rest_api = RestAPI( @@ -723,6 +726,7 @@ class APIGatewayBackend(BaseBackend): api_key_source=api_key_source, endpoint_configuration=endpoint_configuration, tags=tags, + policy=policy, ) self.apis[api_id] = rest_api return rest_api diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 822d4c0ce..a3c41a6d4 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -59,6 +59,7 @@ class APIGatewayResponse(BaseResponse): api_key_source = self._get_param("apiKeySource") endpoint_configuration = self._get_param("endpointConfiguration") tags = self._get_param("tags") + policy = self._get_param("policy") # Param validation if api_key_source and api_key_source not in API_KEY_SOURCES: @@ -94,6 +95,7 @@ class APIGatewayResponse(BaseResponse): api_key_source=api_key_source, endpoint_configuration=endpoint_configuration, tags=tags, + policy=policy, ) return 200, {}, json.dumps(rest_api.to_dict()) From 0828c5af9dfff7430537cfb26cc62a8523d9cef3 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 03:27:59 -0500 Subject: [PATCH 43/46] Add unit test for add apigateway with policy --- tests/test_apigateway/test_apigateway.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 596ed2dd4..107dc5d05 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -69,6 +69,24 @@ def test_create_rest_api_with_tags(): response["tags"].should.equal({"MY_TAG1": "MY_VALUE1"}) +@mock_apigateway +def test_create_rest_api_with_policy(): + client = boto3.client("apigateway", region_name="us-west-2") + + policy = "{\"Version\": \"2012-10-17\",\"Statement\": []}" + response = client.create_rest_api( + name="my_api", + description="this is my api", + policy=policy + ) + api_id = response["id"] + + response = client.get_rest_api(restApiId=api_id) + + assert "policy" in response + response["policy"].should.equal(policy) + + @mock_apigateway def test_create_rest_api_invalid_apikeysource(): client = boto3.client("apigateway", region_name="us-west-2") From 637e0188a2ab81bf3a72b7ddae2677f235c50973 Mon Sep 17 00:00:00 2001 From: Olivier Parent Colombel Date: Mon, 20 Apr 2020 20:54:31 +0200 Subject: [PATCH 44/46] Allow S3 keys to start with leading slashes. --- moto/s3/responses.py | 3 ++- moto/s3/urls.py | 2 +- tests/test_s3/test_s3.py | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 442489a8a..ce1e6128d 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -134,7 +134,8 @@ ACTION_MAP = { def parse_key_name(pth): - return pth.lstrip("/") + # strip the first '/' left by urlparse + return pth[1:] if pth.startswith('/') else pth def is_delete_keys(request, path, bucket_name): diff --git a/moto/s3/urls.py b/moto/s3/urls.py index 752762184..4c4e9ea76 100644 --- a/moto/s3/urls.py +++ b/moto/s3/urls.py @@ -15,5 +15,5 @@ url_paths = { # path-based bucket + key "{0}/(?P[^/]+)/(?P.+)": S3ResponseInstance.key_or_control_response, # subdomain bucket + key with empty first part of path - "{0}//(?P.*)$": S3ResponseInstance.key_or_control_response, + "{0}/(?P/.*)$": S3ResponseInstance.key_or_control_response, } diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index ffbd73966..3048f6507 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -3744,6 +3744,28 @@ def test_root_dir_with_empty_name_works(): store_and_read_back_a_key("/") +@parameterized(['mybucket', 'my.bucket']) +@mock_s3 +def test_leading_slashes_not_removed(bucket_name): + """Make sure that leading slashes are not removed internally.""" + s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME) + s3.create_bucket(Bucket=bucket_name) + + uploaded_key = '/key' + invalid_key_1 = 'key' + invalid_key_2 = '//key' + + s3.put_object(Bucket=bucket_name, Key=uploaded_key, Body=b'Some body') + + with assert_raises(ClientError) as e: + s3.get_object(Bucket=bucket_name, Key=invalid_key_1) + e.exception.response["Error"]["Code"].should.equal("NoSuchKey") + + with assert_raises(ClientError) as e: + s3.get_object(Bucket=bucket_name, Key=invalid_key_2) + e.exception.response["Error"]["Code"].should.equal("NoSuchKey") + + @parameterized( [("foo/bar/baz",), ("foo",), ("foo/run_dt%3D2019-01-01%252012%253A30%253A00",)] ) From d852f7dd063ae17cc1fa7f97bc3510e3daef55e9 Mon Sep 17 00:00:00 2001 From: Olivier Parent Colombel Date: Sat, 25 Apr 2020 15:10:23 +0200 Subject: [PATCH 45/46] Fixing lint errors. --- moto/s3/responses.py | 2 +- tests/test_s3/test_s3.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ce1e6128d..71c424244 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -135,7 +135,7 @@ ACTION_MAP = { def parse_key_name(pth): # strip the first '/' left by urlparse - return pth[1:] if pth.startswith('/') else pth + return pth[1:] if pth.startswith("/") else pth def is_delete_keys(request, path, bucket_name): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 3048f6507..fea76b9e3 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -3744,18 +3744,18 @@ def test_root_dir_with_empty_name_works(): store_and_read_back_a_key("/") -@parameterized(['mybucket', 'my.bucket']) +@parameterized(["mybucket", "my.bucket"]) @mock_s3 def test_leading_slashes_not_removed(bucket_name): """Make sure that leading slashes are not removed internally.""" s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME) s3.create_bucket(Bucket=bucket_name) - uploaded_key = '/key' - invalid_key_1 = 'key' - invalid_key_2 = '//key' + uploaded_key = "/key" + invalid_key_1 = "key" + invalid_key_2 = "//key" - s3.put_object(Bucket=bucket_name, Key=uploaded_key, Body=b'Some body') + s3.put_object(Bucket=bucket_name, Key=uploaded_key, Body=b"Some body") with assert_raises(ClientError) as e: s3.get_object(Bucket=bucket_name, Key=invalid_key_1) From 4a800d8f2c8677b098ea0a2c41deface8236c267 Mon Sep 17 00:00:00 2001 From: JohnWC Date: Sat, 25 Apr 2020 11:24:54 -0500 Subject: [PATCH 46/46] Updated for black --- tests/test_apigateway/test_apigateway.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 107dc5d05..b04328a03 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -73,11 +73,9 @@ def test_create_rest_api_with_tags(): def test_create_rest_api_with_policy(): client = boto3.client("apigateway", region_name="us-west-2") - policy = "{\"Version\": \"2012-10-17\",\"Statement\": []}" + policy = '{"Version": "2012-10-17","Statement": []}' response = client.create_rest_api( - name="my_api", - description="this is my api", - policy=policy + name="my_api", description="this is my api", policy=policy ) api_id = response["id"]