From 725ad7571d6446d6b989008df8c885d6e48cce86 Mon Sep 17 00:00:00 2001 From: pwrmiller Date: Fri, 6 Nov 2020 03:23:47 -0500 Subject: [PATCH] Adds some basic endpoints for Amazon Forecast (#3434) * Adding some basic endpoints for Amazon Forecast, including all dataset group related endpoints * Adds better testing around exception handling in forecast endpoint, removes some unused code, and cleans up validation code * Fix unused imports, optimize imports, code style fixes Co-authored-by: Paul Miller --- IMPLEMENTATION_COVERAGE.md | 12 +- README.md | 1 + docs/index.rst | 2 + moto/__init__.py | 1 + moto/backends.py | 1 + moto/forecast/__init__.py | 7 + moto/forecast/exceptions.py | 43 ++++++ moto/forecast/models.py | 173 +++++++++++++++++++++ moto/forecast/responses.py | 92 +++++++++++ moto/forecast/urls.py | 7 + tests/test_forecast/__init__.py | 0 tests/test_forecast/test_forecast.py | 222 +++++++++++++++++++++++++++ 12 files changed, 555 insertions(+), 6 deletions(-) create mode 100644 moto/forecast/__init__.py create mode 100644 moto/forecast/exceptions.py create mode 100644 moto/forecast/models.py create mode 100644 moto/forecast/responses.py create mode 100644 moto/forecast/urls.py create mode 100644 tests/test_forecast/__init__.py create mode 100644 tests/test_forecast/test_forecast.py diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 110cd7b6b..9ea4330fa 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3518,34 +3518,34 @@ ## forecast
-0% implemented +19% implemented - [ ] create_dataset -- [ ] create_dataset_group +- [X] create_dataset_group - [ ] create_dataset_import_job - [ ] create_forecast - [ ] create_forecast_export_job - [ ] create_predictor - [ ] delete_dataset -- [ ] delete_dataset_group +- [X] delete_dataset_group - [ ] delete_dataset_import_job - [ ] delete_forecast - [ ] delete_forecast_export_job - [ ] delete_predictor - [ ] describe_dataset -- [ ] describe_dataset_group +- [X] describe_dataset_group - [ ] describe_dataset_import_job - [ ] describe_forecast - [ ] describe_forecast_export_job - [ ] describe_predictor - [ ] get_accuracy_metrics -- [ ] list_dataset_groups +- [X] list_dataset_groups - [ ] list_dataset_import_jobs - [ ] list_datasets - [ ] list_forecast_export_jobs - [ ] list_forecasts - [ ] list_predictors -- [ ] update_dataset_group +- [X] update_dataset_group
## forecastquery diff --git a/README.md b/README.md index 3915a85cd..784976a4a 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L | ELB | @mock_elb | core endpoints done | | | ELBv2 | @mock_elbv2 | all endpoints done | | | EMR | @mock_emr | core endpoints done | | +| Forecast | @mock_forecast | some core endpoints done | | | Glacier | @mock_glacier | core endpoints done | | | IAM | @mock_iam | core endpoints done | | | IoT | @mock_iot | core endpoints done | | diff --git a/docs/index.rst b/docs/index.rst index 22ac97228..4f2d7e090 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,6 +60,8 @@ Currently implemented Services: +---------------------------+-----------------------+------------------------------------+ | EMR | @mock_emr | core endpoints done | +---------------------------+-----------------------+------------------------------------+ +| Forecast | @mock_forecast | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ | Glacier | @mock_glacier | core endpoints done | +---------------------------+-----------------------+------------------------------------+ | IAM | @mock_iam | core endpoints done | diff --git a/moto/__init__.py b/moto/__init__.py index c73e111a0..fd467cbf8 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -63,6 +63,7 @@ mock_elbv2 = lazy_load(".elbv2", "mock_elbv2") mock_emr = lazy_load(".emr", "mock_emr") mock_emr_deprecated = lazy_load(".emr", "mock_emr_deprecated") mock_events = lazy_load(".events", "mock_events") +mock_forecast = lazy_load(".forecast", "mock_forecast") mock_glacier = lazy_load(".glacier", "mock_glacier") mock_glacier_deprecated = lazy_load(".glacier", "mock_glacier_deprecated") mock_glue = lazy_load(".glue", "mock_glue") diff --git a/moto/backends.py b/moto/backends.py index e76a89ccb..c8bac72fc 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -75,6 +75,7 @@ BACKENDS = { "kinesisvideoarchivedmedia", "kinesisvideoarchivedmedia_backends", ), + "forecast": ("forecast", "forecast_backends"), } diff --git a/moto/forecast/__init__.py b/moto/forecast/__init__.py new file mode 100644 index 000000000..75b23b94a --- /dev/null +++ b/moto/forecast/__init__.py @@ -0,0 +1,7 @@ +from __future__ import unicode_literals + +from .models import forecast_backends +from ..core.models import base_decorator + +forecast_backend = forecast_backends["us-east-1"] +mock_forecast = base_decorator(forecast_backends) diff --git a/moto/forecast/exceptions.py b/moto/forecast/exceptions.py new file mode 100644 index 000000000..ad86e90fc --- /dev/null +++ b/moto/forecast/exceptions.py @@ -0,0 +1,43 @@ +from __future__ import unicode_literals + +import json + + +class AWSError(Exception): + TYPE = None + STATUS = 400 + + def __init__(self, message, type=None, status=None): + self.message = message + self.type = type if type is not None else self.TYPE + self.status = status if status is not None else self.STATUS + + def response(self): + return ( + json.dumps({"__type": self.type, "message": self.message}), + dict(status=self.status), + ) + + +class InvalidInputException(AWSError): + TYPE = "InvalidInputException" + + +class ResourceAlreadyExistsException(AWSError): + TYPE = "ResourceAlreadyExistsException" + + +class ResourceNotFoundException(AWSError): + TYPE = "ResourceNotFoundException" + + +class ResourceInUseException(AWSError): + TYPE = "ResourceInUseException" + + +class LimitExceededException(AWSError): + TYPE = "LimitExceededException" + + +class ValidationException(AWSError): + TYPE = "ValidationException" diff --git a/moto/forecast/models.py b/moto/forecast/models.py new file mode 100644 index 000000000..c7b18618c --- /dev/null +++ b/moto/forecast/models.py @@ -0,0 +1,173 @@ +import re +from datetime import datetime + +from boto3 import Session +from future.utils import iteritems + +from moto.core import ACCOUNT_ID, BaseBackend +from moto.core.utils import iso_8601_datetime_without_milliseconds +from .exceptions import ( + InvalidInputException, + ResourceAlreadyExistsException, + ResourceNotFoundException, + ValidationException, +) + + +class DatasetGroup: + accepted_dataset_group_name_format = re.compile(r"^[a-zA-Z][a-z-A-Z0-9_]*") + accepted_dataset_group_arn_format = re.compile(r"^[a-zA-Z0-9\-\_\.\/\:]+$") + accepted_dataset_types = [ + "INVENTORY_PLANNING", + "METRICS", + "RETAIL", + "EC2_CAPACITY", + "CUSTOM", + "WEB_TRAFFIC", + "WORK_FORCE", + ] + + def __init__( + self, region_name, dataset_arns, dataset_group_name, domain, tags=None + ): + self.creation_date = iso_8601_datetime_without_milliseconds(datetime.now()) + self.modified_date = self.creation_date + + self.arn = ( + "arn:aws:forecast:" + + region_name + + ":" + + str(ACCOUNT_ID) + + ":dataset-group/" + + dataset_group_name + ) + self.dataset_arns = dataset_arns if dataset_arns else [] + self.dataset_group_name = dataset_group_name + self.domain = domain + self.tags = tags + self._validate() + + def update(self, dataset_arns): + self.dataset_arns = dataset_arns + self.last_modified_date = iso_8601_datetime_without_milliseconds(datetime.now()) + + def _validate(self): + errors = [] + + errors.extend(self._validate_dataset_group_name()) + errors.extend(self._validate_dataset_group_name_len()) + errors.extend(self._validate_dataset_group_domain()) + + if errors: + err_count = len(errors) + message = str(err_count) + " validation error" + message += "s" if err_count > 1 else "" + message += " detected: " + message += "; ".join(errors) + raise ValidationException(message) + + def _validate_dataset_group_name(self): + errors = [] + if not re.match( + self.accepted_dataset_group_name_format, self.dataset_group_name + ): + errors.append( + "Value '" + + self.dataset_group_name + + "' at 'datasetGroupName' failed to satisfy constraint: Member must satisfy regular expression pattern " + + self.accepted_dataset_group_name_format.pattern + ) + return errors + + def _validate_dataset_group_name_len(self): + errors = [] + if len(self.dataset_group_name) >= 64: + errors.append( + "Value '" + + self.dataset_group_name + + "' at 'datasetGroupName' failed to satisfy constraint: Member must have length less than or equal to 63" + ) + return errors + + def _validate_dataset_group_domain(self): + errors = [] + if self.domain not in self.accepted_dataset_types: + errors.append( + "Value '" + + self.domain + + "' at 'domain' failed to satisfy constraint: Member must satisfy enum value set " + + str(self.accepted_dataset_types) + ) + return errors + + +class ForecastBackend(BaseBackend): + def __init__(self, region_name): + super(ForecastBackend, self).__init__() + self.dataset_groups = {} + self.datasets = {} + self.region_name = region_name + + def create_dataset_group(self, dataset_group_name, domain, dataset_arns, tags): + dataset_group = DatasetGroup( + region_name=self.region_name, + dataset_group_name=dataset_group_name, + domain=domain, + dataset_arns=dataset_arns, + tags=tags, + ) + + if dataset_arns: + for dataset_arn in dataset_arns: + if dataset_arn not in self.datasets: + raise InvalidInputException( + "Dataset arns: [" + dataset_arn + "] are not found" + ) + + if self.dataset_groups.get(dataset_group.arn): + raise ResourceAlreadyExistsException( + "A dataset group already exists with the arn: " + dataset_group.arn + ) + + self.dataset_groups[dataset_group.arn] = dataset_group + return dataset_group + + def describe_dataset_group(self, dataset_group_arn): + try: + dataset_group = self.dataset_groups[dataset_group_arn] + except KeyError: + raise ResourceNotFoundException("No resource found " + dataset_group_arn) + return dataset_group + + def delete_dataset_group(self, dataset_group_arn): + try: + del self.dataset_groups[dataset_group_arn] + except KeyError: + raise ResourceNotFoundException("No resource found " + dataset_group_arn) + + def update_dataset_group(self, dataset_group_arn, dataset_arns): + try: + dsg = self.dataset_groups[dataset_group_arn] + except KeyError: + raise ResourceNotFoundException("No resource found " + dataset_group_arn) + + for dataset_arn in dataset_arns: + if dataset_arn not in dsg.dataset_arns: + raise InvalidInputException( + "Dataset arns: [" + dataset_arn + "] are not found" + ) + + dsg.update(dataset_arns) + + def list_dataset_groups(self): + return [v for (_, v) in iteritems(self.dataset_groups)] + + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + + +forecast_backends = {} +for region in Session().get_available_regions("forecast"): + forecast_backends[region] = ForecastBackend(region) diff --git a/moto/forecast/responses.py b/moto/forecast/responses.py new file mode 100644 index 000000000..09d55b0d8 --- /dev/null +++ b/moto/forecast/responses.py @@ -0,0 +1,92 @@ +from __future__ import unicode_literals + +import json + +from moto.core.responses import BaseResponse +from moto.core.utils import amzn_request_id +from .exceptions import AWSError +from .models import forecast_backends + + +class ForecastResponse(BaseResponse): + @property + def forecast_backend(self): + return forecast_backends[self.region] + + @amzn_request_id + def create_dataset_group(self): + dataset_group_name = self._get_param("DatasetGroupName") + domain = self._get_param("Domain") + dataset_arns = self._get_param("DatasetArns") + tags = self._get_param("Tags") + + try: + dataset_group = self.forecast_backend.create_dataset_group( + dataset_group_name=dataset_group_name, + domain=domain, + dataset_arns=dataset_arns, + tags=tags, + ) + response = {"DatasetGroupArn": dataset_group.arn} + return 200, {}, json.dumps(response) + except AWSError as err: + return err.response() + + @amzn_request_id + def describe_dataset_group(self): + dataset_group_arn = self._get_param("DatasetGroupArn") + + try: + dataset_group = self.forecast_backend.describe_dataset_group( + dataset_group_arn=dataset_group_arn + ) + response = { + "CreationTime": dataset_group.creation_date, + "DatasetArns": dataset_group.dataset_arns, + "DatasetGroupArn": dataset_group.arn, + "DatasetGroupName": dataset_group.dataset_group_name, + "Domain": dataset_group.domain, + "LastModificationTime": dataset_group.modified_date, + "Status": "ACTIVE", + } + return 200, {}, json.dumps(response) + except AWSError as err: + return err.response() + + @amzn_request_id + def delete_dataset_group(self): + dataset_group_arn = self._get_param("DatasetGroupArn") + try: + self.forecast_backend.delete_dataset_group(dataset_group_arn) + return 200, {}, None + except AWSError as err: + return err.response() + + @amzn_request_id + def update_dataset_group(self): + dataset_group_arn = self._get_param("DatasetGroupArn") + dataset_arns = self._get_param("DatasetArns") + try: + self.forecast_backend.update_dataset_group(dataset_group_arn, dataset_arns) + return 200, {}, None + except AWSError as err: + return err.response() + + @amzn_request_id + def list_dataset_groups(self): + list_all = self.forecast_backend.list_dataset_groups() + list_all = sorted( + [ + { + "DatasetGroupArn": dsg.arn, + "DatasetGroupName": dsg.dataset_group_name, + "CreationTime": dsg.creation_date, + "LastModificationTime": dsg.creation_date, + } + for dsg in list_all + ], + key=lambda x: x["LastModificationTime"], + reverse=True, + ) + response = {"DatasetGroups": list_all} + return 200, {}, json.dumps(response) diff --git a/moto/forecast/urls.py b/moto/forecast/urls.py new file mode 100644 index 000000000..221659e6f --- /dev/null +++ b/moto/forecast/urls.py @@ -0,0 +1,7 @@ +from __future__ import unicode_literals + +from .responses import ForecastResponse + +url_bases = ["https?://forecast.(.+).amazonaws.com"] + +url_paths = {"{0}/$": ForecastResponse.dispatch} diff --git a/tests/test_forecast/__init__.py b/tests/test_forecast/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_forecast/test_forecast.py b/tests/test_forecast/test_forecast.py new file mode 100644 index 000000000..32af519c7 --- /dev/null +++ b/tests/test_forecast/test_forecast.py @@ -0,0 +1,222 @@ +from __future__ import unicode_literals + +import boto3 +import sure # noqa +from botocore.exceptions import ClientError +from nose.tools import assert_raises +from parameterized import parameterized + +from moto import mock_forecast +from moto.core import ACCOUNT_ID + +region = "us-east-1" +account_id = None +valid_domains = [ + "RETAIL", + "CUSTOM", + "INVENTORY_PLANNING", + "EC2_CAPACITY", + "WORK_FORCE", + "WEB_TRAFFIC", + "METRICS", +] + + +@parameterized(valid_domains) +@mock_forecast +def test_forecast_dataset_group_create(domain): + name = "example_dataset_group" + client = boto3.client("forecast", region_name=region) + response = client.create_dataset_group(DatasetGroupName=name, Domain=domain) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["DatasetGroupArn"].should.equal( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/" + name + ) + + +@mock_forecast +def test_forecast_dataset_group_create_invalid_domain(): + name = "example_dataset_group" + client = boto3.client("forecast", region_name=region) + invalid_domain = "INVALID" + + with assert_raises(ClientError) as exc: + client.create_dataset_group(DatasetGroupName=name, Domain=invalid_domain) + exc.exception.response["Error"]["Code"].should.equal("ValidationException") + exc.exception.response["Error"]["Message"].should.equal( + "1 validation error detected: Value '" + + invalid_domain + + "' at 'domain' failed to satisfy constraint: Member must satisfy enum value set ['INVENTORY_PLANNING', 'METRICS', 'RETAIL', 'EC2_CAPACITY', 'CUSTOM', 'WEB_TRAFFIC', 'WORK_FORCE']" + ) + + +@parameterized([" ", "a" * 64]) +@mock_forecast +def test_forecast_dataset_group_create_invalid_name(name): + client = boto3.client("forecast", region_name=region) + + with assert_raises(ClientError) as exc: + client.create_dataset_group(DatasetGroupName=name, Domain="CUSTOM") + exc.exception.response["Error"]["Code"].should.equal("ValidationException") + exc.exception.response["Error"]["Message"].should.contain( + "1 validation error detected: Value '" + + name + + "' at 'datasetGroupName' failed to satisfy constraint: Member must" + ) + + +@mock_forecast +def test_forecast_dataset_group_create_duplicate_fails(): + client = boto3.client("forecast", region_name=region) + client.create_dataset_group(DatasetGroupName="name", Domain="RETAIL") + + with assert_raises(ClientError) as exc: + client.create_dataset_group(DatasetGroupName="name", Domain="RETAIL") + + exc.exception.response["Error"]["Code"].should.equal( + "ResourceAlreadyExistsException" + ) + + +@mock_forecast +def test_forecast_dataset_group_list_default_empty(): + client = boto3.client("forecast", region_name=region) + + list = client.list_dataset_groups() + list["DatasetGroups"].should.be.empty + + +@mock_forecast +def test_forecast_dataset_group_list_some(): + client = boto3.client("forecast", region_name=region) + + client.create_dataset_group(DatasetGroupName="hello", Domain="CUSTOM") + result = client.list_dataset_groups() + + assert len(result["DatasetGroups"]) == 1 + result["DatasetGroups"][0]["DatasetGroupArn"].should.equal( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/hello" + ) + + +@mock_forecast +def test_forecast_delete_dataset_group(): + dataset_group_name = "name" + dataset_group_arn = ( + "arn:aws:forecast:" + + region + + ":" + + ACCOUNT_ID + + ":dataset-group/" + + dataset_group_name + ) + client = boto3.client("forecast", region_name=region) + client.create_dataset_group(DatasetGroupName=dataset_group_name, Domain="CUSTOM") + client.delete_dataset_group(DatasetGroupArn=dataset_group_arn) + + +@mock_forecast +def test_forecast_delete_dataset_group_missing(): + client = boto3.client("forecast", region_name=region) + missing_dsg_arn = ( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/missing" + ) + + with assert_raises(ClientError) as exc: + client.delete_dataset_group(DatasetGroupArn=missing_dsg_arn) + exc.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") + exc.exception.response["Error"]["Message"].should.equal( + "No resource found " + missing_dsg_arn + ) + + +@mock_forecast +def test_forecast_update_dataset_arns_empty(): + dataset_group_name = "name" + dataset_group_arn = ( + "arn:aws:forecast:" + + region + + ":" + + ACCOUNT_ID + + ":dataset-group/" + + dataset_group_name + ) + client = boto3.client("forecast", region_name=region) + client.create_dataset_group(DatasetGroupName=dataset_group_name, Domain="CUSTOM") + client.update_dataset_group(DatasetGroupArn=dataset_group_arn, DatasetArns=[]) + + +@mock_forecast +def test_forecast_update_dataset_group_not_found(): + client = boto3.client("forecast", region_name=region) + dataset_group_arn = ( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/" + "test" + ) + with assert_raises(ClientError) as exc: + client.update_dataset_group(DatasetGroupArn=dataset_group_arn, DatasetArns=[]) + exc.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") + exc.exception.response["Error"]["Message"].should.equal( + "No resource found " + dataset_group_arn + ) + + +@mock_forecast +def test_describe_dataset_group(): + name = "test" + client = boto3.client("forecast", region_name=region) + dataset_group_arn = ( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/" + name + ) + client.create_dataset_group(DatasetGroupName=name, Domain="CUSTOM") + result = client.describe_dataset_group(DatasetGroupArn=dataset_group_arn) + assert result.get("DatasetGroupArn") == dataset_group_arn + assert result.get("Domain") == "CUSTOM" + assert result.get("DatasetArns") == [] + + +@mock_forecast +def test_describe_dataset_group_missing(): + client = boto3.client("forecast", region_name=region) + dataset_group_arn = ( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/name" + ) + with assert_raises(ClientError) as exc: + client.describe_dataset_group(DatasetGroupArn=dataset_group_arn) + exc.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException") + exc.exception.response["Error"]["Message"].should.equal( + "No resource found " + dataset_group_arn + ) + + +@mock_forecast +def test_create_dataset_group_missing_datasets(): + client = boto3.client("forecast", region_name=region) + dataset_arn = "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset/name" + with assert_raises(ClientError) as exc: + client.create_dataset_group( + DatasetGroupName="name", Domain="CUSTOM", DatasetArns=[dataset_arn] + ) + exc.exception.response["Error"]["Code"].should.equal("InvalidInputException") + exc.exception.response["Error"]["Message"].should.equal( + "Dataset arns: [" + dataset_arn + "] are not found" + ) + + +@mock_forecast +def test_update_dataset_group_missing_datasets(): + name = "test" + client = boto3.client("forecast", region_name=region) + dataset_group_arn = ( + "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset-group/" + name + ) + client.create_dataset_group(DatasetGroupName=name, Domain="CUSTOM") + dataset_arn = "arn:aws:forecast:" + region + ":" + ACCOUNT_ID + ":dataset/name" + + with assert_raises(ClientError) as exc: + client.update_dataset_group( + DatasetGroupArn=dataset_group_arn, DatasetArns=[dataset_arn] + ) + exc.exception.response["Error"]["Code"].should.equal("InvalidInputException") + exc.exception.response["Error"]["Message"].should.equal( + "Dataset arns: [" + dataset_arn + "] are not found" + )