From a460adc9405e87b2df6708f02a8586ada6b7c7e2 Mon Sep 17 00:00:00 2001 From: John-Paul Stanford Date: Mon, 7 Mar 2022 20:53:45 +0000 Subject: [PATCH] This adds a new service "databrew", mostly around managing databrew (#4898) --- moto/__init__.py | 1 + moto/backend_index.py | 3 +- moto/databrew/__init__.py | 4 + moto/databrew/exceptions.py | 25 ++++++ moto/databrew/models.py | 71 ++++++++++++++++ moto/databrew/responses.py | 69 +++++++++++++++ moto/databrew/urls.py | 8 ++ tests/test_databrew/__init__.py | 0 tests/test_databrew/test_databrew.py | 120 +++++++++++++++++++++++++++ 9 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 moto/databrew/__init__.py create mode 100644 moto/databrew/exceptions.py create mode 100644 moto/databrew/models.py create mode 100644 moto/databrew/responses.py create mode 100644 moto/databrew/urls.py create mode 100644 tests/test_databrew/__init__.py create mode 100644 tests/test_databrew/test_databrew.py diff --git a/moto/__init__.py b/moto/__init__.py index 65d543249..01b2cda49 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -49,6 +49,7 @@ mock_cognitoidentity = lazy_load( ) mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp", boto3_name="cognito-idp") mock_config = lazy_load(".config", "mock_config") +mock_databrew = lazy_load(".databrew", "mock_databrew") mock_datapipeline = lazy_load(".datapipeline", "mock_datapipeline") mock_datasync = lazy_load(".datasync", "mock_datasync") mock_dax = lazy_load(".dax", "mock_dax") diff --git a/moto/backend_index.py b/moto/backend_index.py index f748157be..5ddfe8b0a 100644 --- a/moto/backend_index.py +++ b/moto/backend_index.py @@ -1,4 +1,4 @@ -# autogenerated by scripts/update_backend_index.py +# autogenerated by ./scripts/update_backend_index.py import re backend_url_patterns = [ @@ -25,6 +25,7 @@ backend_url_patterns = [ ), ("cognito-idp", re.compile("https?://cognito-idp\\.(.+)\\.amazonaws.com")), ("config", re.compile("https?://config\\.(.+)\\.amazonaws\\.com")), + ("databrew", re.compile("https?://databrew\\.(.+)\\.amazonaws.com")), ("datapipeline", re.compile("https?://datapipeline\\.(.+)\\.amazonaws\\.com")), ("datasync", re.compile("https?://(.*\\.)?(datasync)\\.(.+)\\.amazonaws.com")), ("dax", re.compile("https?://dax\\.(.+)\\.amazonaws\\.com")), diff --git a/moto/databrew/__init__.py b/moto/databrew/__init__.py new file mode 100644 index 000000000..34fff819d --- /dev/null +++ b/moto/databrew/__init__.py @@ -0,0 +1,4 @@ +from .models import databrew_backends +from ..core.models import base_decorator + +mock_databrew = base_decorator(databrew_backends) diff --git a/moto/databrew/exceptions.py b/moto/databrew/exceptions.py new file mode 100644 index 000000000..bf2b726e4 --- /dev/null +++ b/moto/databrew/exceptions.py @@ -0,0 +1,25 @@ +from moto.core.exceptions import JsonRESTError + + +class DataBrewClientError(JsonRESTError): + code = 400 + + +class AlreadyExistsException(DataBrewClientError): + def __init__(self, typ): + super().__init__("AlreadyExistsException", "%s already exists." % (typ)) + + +class RecipeAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super().__init__("Recipe") + + +class EntityNotFoundException(DataBrewClientError): + def __init__(self, msg): + super().__init__("EntityNotFoundException", msg) + + +class RecipeNotFoundException(EntityNotFoundException): + def __init__(self, recipe_name): + super().__init__("Recipe %s not found." % recipe_name) diff --git a/moto/databrew/models.py b/moto/databrew/models.py new file mode 100644 index 000000000..92a3ff586 --- /dev/null +++ b/moto/databrew/models.py @@ -0,0 +1,71 @@ +from collections import OrderedDict +from datetime import datetime + +from moto.core import BaseBackend, BaseModel +from moto.core.utils import BackendDict +from moto.utilities.paginator import paginate +from .exceptions import RecipeAlreadyExistsException, RecipeNotFoundException + + +class DataBrewBackend(BaseBackend): + PAGINATION_MODEL = { + "list_recipes": { + "input_token": "next_token", + "limit_key": "max_results", + "limit_default": 100, + "unique_attribute": "name", + }, + } + + def __init__(self, region_name): + self.region_name = region_name + self.recipes = OrderedDict() + + def reset(self): + """Re-initialize all attributes for this instance.""" + region_name = self.region_name + self.__init__(region_name) + + def create_recipe(self, recipe_name, recipe_description, recipe_steps, tags): + # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipe.html + if recipe_name in self.recipes: + raise RecipeAlreadyExistsException() + + recipe = FakeRecipe( + self.region_name, recipe_name, recipe_description, recipe_steps, tags + ) + self.recipes[recipe_name] = recipe + return recipe + + @paginate(pagination_model=PAGINATION_MODEL) + def list_recipes(self): + return [self.recipes[key] for key in self.recipes] if self.recipes else [] + + def get_recipe(self, recipe_name, version): + if recipe_name not in self.recipes: + raise RecipeNotFoundException(recipe_name) + return self.recipes[recipe_name] + + +class FakeRecipe(BaseModel): + def __init__( + self, region_name, recipe_name, recipe_description, recipe_steps, tags + ): + self.region_name = region_name + self.name = recipe_name + self.description = recipe_description + self.steps = recipe_steps + self.created_time = datetime.now() + self.tags = tags + + def as_dict(self): + return { + "Name": self.name, + "Steps": self.steps, + "Description": self.description, + "CreateTime": self.created_time.isoformat(), + "Tags": self.tags or dict(), + } + + +databrew_backends = BackendDict(DataBrewBackend, "databrew") diff --git a/moto/databrew/responses.py b/moto/databrew/responses.py new file mode 100644 index 000000000..ae83b982c --- /dev/null +++ b/moto/databrew/responses.py @@ -0,0 +1,69 @@ +import json +from urllib.parse import urlparse + +from moto.core.responses import BaseResponse +from moto.core.utils import amzn_request_id +from .exceptions import DataBrewClientError +from .models import databrew_backends + + +class DataBrewResponse(BaseResponse): + SERVICE_NAME = "databrew" + + @property + def databrew_backend(self): + """Return backend instance specific for this region.""" + return databrew_backends[self.region] + + @property + def parameters(self): + return json.loads(self.body) + + @amzn_request_id + def create_recipe(self): + # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipe.html + recipe_description = self.parameters.get("Description") + recipe_steps = self.parameters.get("Steps") + recipe_name = self.parameters.get("Name") + tags = self.parameters.get("Tags") + return json.dumps( + self.databrew_backend.create_recipe( + recipe_name, recipe_description, recipe_steps, tags + ).as_dict() + ) + + @amzn_request_id + def list_recipes(self): + # https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipes.html + next_token = self._get_param("NextToken", self._get_param("nextToken")) + max_results = self._get_int_param( + "MaxResults", self._get_int_param("maxResults") + ) + + # pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking + recipe_list, next_token = self.databrew_backend.list_recipes( + next_token=next_token, max_results=max_results + ) + return json.dumps( + { + "Recipes": [recipe.as_dict() for recipe in recipe_list], + "NextToken": next_token, + } + ) + + @amzn_request_id + def describe_recipe_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + parsed_url = urlparse(full_url) + + recipe_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1] + + try: + return json.dumps( + self.databrew_backend.get_recipe(recipe_name, None).as_dict() + ) + except DataBrewClientError as e: + return e.code, e.get_headers(), e.get_body() + + +# 'DescribeRecipe' diff --git a/moto/databrew/urls.py b/moto/databrew/urls.py new file mode 100644 index 000000000..0394fa8ce --- /dev/null +++ b/moto/databrew/urls.py @@ -0,0 +1,8 @@ +from .responses import DataBrewResponse + +url_bases = [r"https?://databrew\.(.+)\.amazonaws.com"] + +url_paths = { + "{0}/recipes$": DataBrewResponse.dispatch, + "{0}/recipes/(?P[^/]+)$": DataBrewResponse().describe_recipe_response, +} diff --git a/tests/test_databrew/__init__.py b/tests/test_databrew/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_databrew/test_databrew.py b/tests/test_databrew/test_databrew.py new file mode 100644 index 000000000..b32c26b38 --- /dev/null +++ b/tests/test_databrew/test_databrew.py @@ -0,0 +1,120 @@ +import uuid + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from moto import mock_databrew + + +def _create_databrew_client(): + client = boto3.client("databrew", region_name="us-west-1") + return client + + +def _create_test_recipe(client, tags=None, recipe_name=None): + if recipe_name is None: + recipe_name = str(uuid.uuid4()) + + return client.create_recipe( + Name=recipe_name, + Steps=[ + { + "Action": { + "Operation": "REMOVE_COMBINED", + "Parameters": { + "collapseConsecutiveWhitespace": "false", + "removeAllPunctuation": "false", + "removeAllQuotes": "false", + "removeAllWhitespace": "false", + "removeCustomCharacters": "false", + "removeCustomValue": "false", + "removeLeadingAndTrailingPunctuation": "false", + "removeLeadingAndTrailingQuotes": "false", + "removeLeadingAndTrailingWhitespace": "false", + "removeLetters": "false", + "removeNumbers": "false", + "removeSpecialCharacters": "true", + "sourceColumn": "FakeColumn", + }, + } + } + ], + Tags=tags or {}, + ) + + +def _create_test_recipes(client, count): + for _ in range(count): + _create_test_recipe(client) + + +@mock_databrew +def test_recipe_list_when_empty(): + client = _create_databrew_client() + + response = client.list_recipes() + response.should.have.key("Recipes") + response["Recipes"].should.have.length_of(0) + + +@mock_databrew +def test_list_recipes_with_max_results(): + client = _create_databrew_client() + + _create_test_recipes(client, 4) + response = client.list_recipes(MaxResults=2) + response["Recipes"].should.have.length_of(2) + response.should.have.key("NextToken") + + +@mock_databrew +def test_list_recipes_from_next_token(): + client = _create_databrew_client() + _create_test_recipes(client, 10) + first_response = client.list_recipes(MaxResults=3) + response = client.list_recipes(NextToken=first_response["NextToken"]) + response["Recipes"].should.have.length_of(7) + + +@mock_databrew +def test_list_recipes_with_max_results_greater_than_actual_results(): + client = _create_databrew_client() + _create_test_recipes(client, 4) + response = client.list_recipes(MaxResults=10) + response["Recipes"].should.have.length_of(4) + + +@mock_databrew +def test_describe_recipe(): + client = _create_databrew_client() + response = _create_test_recipe(client) + + recipe = client.describe_recipe(Name=response["Name"]) + + recipe["Name"].should.equal(response["Name"]) + recipe["Steps"].should.have.length_of(1) + + +@mock_databrew +def test_describe_recipe_that_does_not_exist(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name="DoseNotExist") + err = exc.value.response["Error"] + err["Code"].should.equal("EntityNotFoundException") + err["Message"].should.equal("Recipe DoseNotExist not found.") + + +@mock_databrew +def test_create_recipe_that_already_exists(): + client = _create_databrew_client() + + response = _create_test_recipe(client) + + with pytest.raises(ClientError) as exc: + _create_test_recipe(client, recipe_name=response["Name"]) + err = exc.value.response["Error"] + err["Code"].should.equal("AlreadyExistsException") + err["Message"].should.equal("Recipe already exists.")