From 3f89b98889682f9df772b38867efa628fc3519be Mon Sep 17 00:00:00 2001 From: Michael Sanders Date: Thu, 5 May 2022 21:33:33 +0100 Subject: [PATCH] Databrew: add recipe version support (#5094) --- IMPLEMENTATION_COVERAGE.md | 10 +- docs/docs/services/databrew.rst | 8 +- moto/databrew/exceptions.py | 21 +- moto/databrew/models.py | 232 +++++++++-- moto/databrew/responses.py | 71 +++- moto/databrew/urls.py | 3 + tests/test_databrew/test_databrew_recipes.py | 399 ++++++++++++++++++- 7 files changed, 685 insertions(+), 59 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index ea4bf0c41..4bf741c86 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -1069,7 +1069,7 @@ ## databrew
-15% implemented +22% implemented - [ ] batch_delete_recipe_version - [ ] create_dataset @@ -1082,26 +1082,26 @@ - [ ] delete_dataset - [ ] delete_job - [ ] delete_project -- [ ] delete_recipe_version +- [X] delete_recipe_version - [X] delete_ruleset - [ ] delete_schedule - [ ] describe_dataset - [ ] describe_job - [ ] describe_job_run - [ ] describe_project -- [ ] describe_recipe +- [X] describe_recipe - [ ] describe_ruleset - [ ] describe_schedule - [ ] list_datasets - [ ] list_job_runs - [ ] list_jobs - [ ] list_projects -- [ ] list_recipe_versions +- [X] list_recipe_versions - [X] list_recipes - [X] list_rulesets - [ ] list_schedules - [ ] list_tags_for_resource -- [ ] publish_recipe +- [X] publish_recipe - [ ] send_project_session_action - [ ] start_job_run - [ ] start_project_session diff --git a/docs/docs/services/databrew.rst b/docs/docs/services/databrew.rst index 4fe6e9fc4..2bb2e3fbc 100644 --- a/docs/docs/services/databrew.rst +++ b/docs/docs/services/databrew.rst @@ -36,26 +36,26 @@ databrew - [ ] delete_dataset - [ ] delete_job - [ ] delete_project -- [ ] delete_recipe_version +- [X] delete_recipe_version - [X] delete_ruleset - [ ] delete_schedule - [ ] describe_dataset - [ ] describe_job - [ ] describe_job_run - [ ] describe_project -- [ ] describe_recipe +- [X] describe_recipe - [ ] describe_ruleset - [ ] describe_schedule - [ ] list_datasets - [ ] list_job_runs - [ ] list_jobs - [ ] list_projects -- [ ] list_recipe_versions +- [X] list_recipe_versions - [X] list_recipes - [X] list_rulesets - [ ] list_schedules - [ ] list_tags_for_resource -- [ ] publish_recipe +- [X] publish_recipe - [ ] send_project_session_action - [ ] start_job_run - [ ] start_project_session diff --git a/moto/databrew/exceptions.py b/moto/databrew/exceptions.py index bcc208502..6a3a839ab 100644 --- a/moto/databrew/exceptions.py +++ b/moto/databrew/exceptions.py @@ -10,9 +10,16 @@ class AlreadyExistsException(DataBrewClientError): super().__init__("AlreadyExistsException", "%s already exists." % (typ)) -class RecipeAlreadyExistsException(AlreadyExistsException): - def __init__(self): - super().__init__("Recipe") +class ConflictException(DataBrewClientError): + code = 409 + + def __init__(self, message, **kwargs): + super().__init__("ConflictException", message, **kwargs) + + +class ValidationException(DataBrewClientError): + def __init__(self, message, **kwargs): + super().__init__("ValidationException", message, **kwargs) class RulesetAlreadyExistsException(AlreadyExistsException): @@ -25,9 +32,11 @@ class EntityNotFoundException(DataBrewClientError): super().__init__("EntityNotFoundException", msg) -class RecipeNotFoundException(EntityNotFoundException): - def __init__(self, recipe_name): - super().__init__("Recipe %s not found." % recipe_name) +class ResourceNotFoundException(DataBrewClientError): + code = 404 + + def __init__(self, message, **kwargs): + super().__init__("ResourceNotFoundException", message, **kwargs) class RulesetNotFoundException(EntityNotFoundException): diff --git a/moto/databrew/models.py b/moto/databrew/models.py index cc27bcc97..0787cf2f3 100644 --- a/moto/databrew/models.py +++ b/moto/databrew/models.py @@ -1,10 +1,16 @@ from collections import OrderedDict +from copy import deepcopy +import math from datetime import datetime from moto.core import BaseBackend, BaseModel from moto.core.utils import BackendDict from moto.utilities.paginator import paginate -from .exceptions import RecipeAlreadyExistsException, RecipeNotFoundException +from .exceptions import ( + ConflictException, + ResourceNotFoundException, + ValidationException, +) from .exceptions import RulesetAlreadyExistsException, RulesetNotFoundException @@ -16,6 +22,12 @@ class DataBrewBackend(BaseBackend): "limit_default": 100, "unique_attribute": "name", }, + "list_recipe_versions": { + "input_token": "next_token", + "limit_key": "max_results", + "limit_default": 100, + "unique_attribute": "name", + }, "list_rulesets": { "input_token": "next_token", "limit_key": "max_results", @@ -34,42 +46,134 @@ class DataBrewBackend(BaseBackend): region_name = self.region_name self.__init__(region_name) + @staticmethod + def validate_length(param, param_name, max_length): + if len(param) > max_length: + raise ValidationException( + f"1 validation error detected: Value '{param}' at '{param_name}' failed to " + f"satisfy constraint: Member must have length less than or equal to {max_length}" + ) + def create_recipe(self, recipe_name, recipe_description, recipe_steps, tags): # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipe.html if recipe_name in self.recipes: - raise RecipeAlreadyExistsException() + raise ConflictException(f"The recipe {recipe_name} already exists") recipe = FakeRecipe( self.region_name, recipe_name, recipe_description, recipe_steps, tags ) self.recipes[recipe_name] = recipe - return recipe + return recipe.latest_working - def update_recipe(self, recipe_name, recipe_description, recipe_steps, tags): + def delete_recipe_version(self, recipe_name, recipe_version): + if not FakeRecipe.version_is_valid(recipe_version, latest_published=False): + raise ValidationException( + f"Recipe {recipe_name} version {recipe_version} is invalid." + ) + + try: + recipe = self.recipes[recipe_name] + except KeyError: + raise ResourceNotFoundException(f"The recipe {recipe_name} wasn't found") + + if ( + recipe_version != FakeRecipe.LATEST_WORKING + and float(recipe_version) not in recipe.versions + ): + raise ResourceNotFoundException( + f"The recipe {recipe_name} version {recipe_version } wasn't found." + ) + + if recipe_version in ( + FakeRecipe.LATEST_WORKING, + str(recipe.latest_working.version), + ): + if recipe.latest_published is not None: + # Can only delete latest working version when there are no others + raise ValidationException( + f"Recipe version {recipe_version} is not allowed to be deleted" + ) + else: + del self.recipes[recipe_name] + else: + recipe.delete_published_version(recipe_version) + + def update_recipe(self, recipe_name, recipe_description, recipe_steps): if recipe_name not in self.recipes: - raise RecipeNotFoundException(recipe_name) + raise ResourceNotFoundException(f"The recipe {recipe_name} wasn't found") recipe = self.recipes[recipe_name] - if recipe_description is not None: - recipe.description = recipe_description - if recipe_steps is not None: - recipe.steps = recipe_steps - if tags is not None: - recipe.tags = tags + recipe.update(recipe_description, recipe_steps) - return recipe + return recipe.latest_working @paginate(pagination_model=PAGINATION_MODEL) - def list_recipes(self): - return [self.recipes[key] for key in self.recipes] if self.recipes else [] + def list_recipes(self, recipe_version=None): + # https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipes.html + if recipe_version == FakeRecipe.LATEST_WORKING: + version = "latest_working" + elif recipe_version in (None, FakeRecipe.LATEST_PUBLISHED): + version = "latest_published" + else: + raise ValidationException( + f"Invalid version {recipe_version}. " + "Valid versions are LATEST_PUBLISHED and LATEST_WORKING." + ) + recipes = [getattr(self.recipes[key], version) for key in self.recipes] + return [r for r in recipes if r is not None] - def get_recipe(self, recipe_name): - """ - The Version-parameter has not yet been implemented - """ - if recipe_name not in self.recipes: - raise RecipeNotFoundException(recipe_name) - return self.recipes[recipe_name] + @paginate(pagination_model=PAGINATION_MODEL) + def list_recipe_versions(self, recipe_name): + # https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipeVersions.html + self.validate_length(recipe_name, "name", 255) + + recipe = self.recipes.get(recipe_name) + if recipe is None: + return [] + + latest_working = recipe.latest_working + + recipe_versions = [ + recipe_version + for recipe_version in recipe.versions.values() + if recipe_version is not latest_working + ] + return [r for r in recipe_versions if r is not None] + + def get_recipe(self, recipe_name, recipe_version=None): + # https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeRecipe.html + self.validate_length(recipe_name, "name", 255) + + if recipe_version is None: + recipe_version = FakeRecipe.LATEST_PUBLISHED + else: + self.validate_length(recipe_version, "recipeVersion", 16) + if not FakeRecipe.version_is_valid(recipe_version): + raise ValidationException( + f"Recipe {recipe_name} version {recipe_version} isn't valid." + ) + + recipe = None + if recipe_name in self.recipes: + if recipe_version == FakeRecipe.LATEST_PUBLISHED: + recipe = self.recipes[recipe_name].latest_published + elif recipe_version == FakeRecipe.LATEST_WORKING: + recipe = self.recipes[recipe_name].latest_working + else: + recipe = self.recipes[recipe_name].versions.get(float(recipe_version)) + if recipe is None: + raise ResourceNotFoundException( + f"The recipe {recipe_name} for version {recipe_version} wasn't found." + ) + return recipe + + def publish_recipe(self, recipe_name, description=None): + # https://docs.aws.amazon.com/databrew/latest/dg/API_PublishRecipe.html + self.validate_length(recipe_name, "name", 255) + try: + self.recipes[recipe_name].publish(description) + except KeyError: + raise ResourceNotFoundException(f"Recipe {recipe_name} wasn't found") def create_ruleset( self, ruleset_name, ruleset_description, ruleset_rules, ruleset_target_arn, tags @@ -119,8 +223,77 @@ class DataBrewBackend(BaseBackend): class FakeRecipe(BaseModel): + INITIAL_VERSION = 0.1 + LATEST_WORKING = "LATEST_WORKING" + LATEST_PUBLISHED = "LATEST_PUBLISHED" + + @classmethod + def version_is_valid(cls, version, latest_working=True, latest_published=True): + validity = True + + if len(version) < 1 or len(version) > 16: + validity = False + else: + try: + version = float(version) + except ValueError: + if not ( + (version == cls.LATEST_WORKING and latest_working) + or (version == cls.LATEST_PUBLISHED and latest_published) + ): + validity = False + return validity + def __init__( self, region_name, recipe_name, recipe_description, recipe_steps, tags + ): + self.versions = OrderedDict() + self.versions[self.INITIAL_VERSION] = FakeRecipeVersion( + region_name, + recipe_name, + recipe_description, + recipe_steps, + tags, + version=self.INITIAL_VERSION, + ) + self.latest_working = self.versions[self.INITIAL_VERSION] + self.latest_published = None + + def publish(self, description=None): + self.latest_published = self.latest_working + self.latest_working = deepcopy(self.latest_working) + self.latest_published.publish(description) + del self.versions[self.latest_working.version] + self.versions[self.latest_published.version] = self.latest_published + self.latest_working.version = self.latest_published.version + 0.1 + self.versions[self.latest_working.version] = self.latest_working + + def update(self, description, steps): + if description is not None: + self.latest_working.description = description + if steps is not None: + self.latest_working.steps = steps + + def delete_published_version(self, version): + version = float(version) + assert version.is_integer() + if version == self.latest_published.version: + prev_version = version - 1.0 + # Iterate back through the published versions until we find one that exists + while prev_version >= 1.0: + if prev_version in self.versions: + self.latest_published = self.versions[prev_version] + break + prev_version -= 1.0 + else: + # Didn't find an earlier published version + self.latest_published = None + del self.versions[version] + + +class FakeRecipeVersion(BaseModel): + def __init__( + self, region_name, recipe_name, recipe_description, recipe_steps, tags, version ): self.region_name = region_name self.name = recipe_name @@ -128,15 +301,28 @@ class FakeRecipe(BaseModel): self.steps = recipe_steps self.created_time = datetime.now() self.tags = tags + self.published_date = None + self.version = version def as_dict(self): - return { + dict_recipe = { "Name": self.name, "Steps": self.steps, "Description": self.description, - "CreateTime": self.created_time.isoformat(), + "CreateDate": "%.3f" % self.created_time.timestamp(), "Tags": self.tags or dict(), + "RecipeVersion": str(self.version), } + if self.published_date is not None: + dict_recipe["PublishedDate"] = "%.3f" % self.published_date.timestamp() + + return dict_recipe + + def publish(self, description): + self.version = float(math.ceil(self.version)) + self.published_date = datetime.now() + if description is not None: + self.description = description class FakeRuleset(BaseModel): diff --git a/moto/databrew/responses.py b/moto/databrew/responses.py index 216d63ea2..89100e488 100644 --- a/moto/databrew/responses.py +++ b/moto/databrew/responses.py @@ -31,6 +31,22 @@ class DataBrewResponse(BaseResponse): ).as_dict() ) + @amzn_request_id + def delete_recipe_version(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + # https://docs.aws.amazon.com/databrew/latest/dg/API_DeleteRecipeVersion.html + if request.method == "DELETE": + parsed_url = urlparse(full_url) + split_path = parsed_url.path.strip("/").split("/") + recipe_name = split_path[1] + recipe_version = split_path[3] + self.databrew_backend.delete_recipe_version(recipe_name, recipe_version) + return ( + 200, + {}, + json.dumps({"Name": recipe_name, "RecipeVersion": recipe_version}), + ) + @amzn_request_id def list_recipes(self): # https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipes.html @@ -38,10 +54,15 @@ class DataBrewResponse(BaseResponse): max_results = self._get_int_param( "MaxResults", self._get_int_param("maxResults") ) + recipe_version = self._get_param( + "RecipeVersion", self._get_param("recipeVersion") + ) # pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking recipe_list, next_token = self.databrew_backend.list_recipes( - next_token=next_token, max_results=max_results + next_token=next_token, + max_results=max_results, + recipe_version=recipe_version, ) return json.dumps( { @@ -50,19 +71,55 @@ class DataBrewResponse(BaseResponse): } ) + @amzn_request_id + def list_recipe_versions(self, request, full_url, headers): + # https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipeVersions.html + self.setup_class(request, full_url, headers) + recipe_name = self._get_param("Name", self._get_param("name")) + next_token = self._get_param("NextToken", self._get_param("nextToken")) + max_results = self._get_int_param( + "MaxResults", self._get_int_param("maxResults") + ) + + # pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking + recipe_list, next_token = self.databrew_backend.list_recipe_versions( + recipe_name=recipe_name, next_token=next_token, max_results=max_results + ) + return json.dumps( + { + "Recipes": [recipe.as_dict() for recipe in recipe_list], + "NextToken": next_token, + } + ) + + @amzn_request_id + def publish_recipe(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + if request.method == "POST": + parsed_url = urlparse(full_url) + recipe_name = parsed_url.path.strip("/").split("/", 2)[1] + recipe_description = self.parameters.get("Description") + self.databrew_backend.publish_recipe(recipe_name, recipe_description) + return 200, {}, json.dumps({"Name": recipe_name}) + def put_recipe_response(self, recipe_name): recipe_description = self.parameters.get("Description") recipe_steps = self.parameters.get("Steps") - tags = self.parameters.get("Tags") - recipe = self.databrew_backend.update_recipe( - recipe_name, recipe_description, recipe_steps, tags + self.databrew_backend.update_recipe( + recipe_name, recipe_description, recipe_steps ) - return 200, {}, json.dumps(recipe.as_dict()) + return 200, {}, json.dumps({"Name": recipe_name}) def get_recipe_response(self, recipe_name): - recipe = self.databrew_backend.get_recipe(recipe_name) - return 201, {}, json.dumps(recipe.as_dict()) + # https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeRecipe.html + recipe_version = self._get_param( + "RecipeVersion", self._get_param("recipeVersion") + ) + recipe = self.databrew_backend.get_recipe( + recipe_name, recipe_version=recipe_version + ) + return 200, {}, json.dumps(recipe.as_dict()) @amzn_request_id def recipe_response(self, request, full_url, headers): diff --git a/moto/databrew/urls.py b/moto/databrew/urls.py index b3130c603..d3b441be3 100644 --- a/moto/databrew/urls.py +++ b/moto/databrew/urls.py @@ -3,8 +3,11 @@ from .responses import DataBrewResponse url_bases = [r"https?://databrew\.(.+)\.amazonaws.com"] url_paths = { + "{0}/recipeVersions$": DataBrewResponse().list_recipe_versions, "{0}/recipes$": DataBrewResponse.dispatch, "{0}/recipes/(?P[^/]+)$": DataBrewResponse().recipe_response, + "{0}/recipes/(?P[^/]+)/recipeVersion/(?P[^/]+)": DataBrewResponse().delete_recipe_version, + "{0}/recipes/(?P[^/]+)/publishRecipe$": DataBrewResponse().publish_recipe, "{0}/rulesets$": DataBrewResponse.dispatch, "{0}/rulesets/(?P[^/]+)$": DataBrewResponse().ruleset_response, } diff --git a/tests/test_databrew/test_databrew_recipes.py b/tests/test_databrew/test_databrew_recipes.py index f0590cfc1..60e486cc8 100644 --- a/tests/test_databrew/test_databrew_recipes.py +++ b/tests/test_databrew/test_databrew_recipes.py @@ -3,6 +3,7 @@ import uuid import boto3 import pytest from botocore.exceptions import ClientError +from datetime import datetime from moto import mock_databrew @@ -58,12 +59,30 @@ def test_recipe_list_when_empty(): response["Recipes"].should.have.length_of(0) +@mock_databrew +def test_recipe_list_with_invalid_version(): + client = _create_databrew_client() + + recipe_version = "1.1" + with pytest.raises(ClientError) as exc: + client.list_recipes(RecipeVersion=recipe_version) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"Invalid version {recipe_version}. " + "Valid versions are LATEST_PUBLISHED and LATEST_WORKING." + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + + @mock_databrew def test_list_recipes_with_max_results(): client = _create_databrew_client() _create_test_recipes(client, 4) - response = client.list_recipes(MaxResults=2) + response = client.list_recipes(MaxResults=2, RecipeVersion="LATEST_WORKING") response["Recipes"].should.have.length_of(2) response.should.have.key("NextToken") @@ -72,8 +91,10 @@ def test_list_recipes_with_max_results(): def test_list_recipes_from_next_token(): client = _create_databrew_client() _create_test_recipes(client, 10) - first_response = client.list_recipes(MaxResults=3) - response = client.list_recipes(NextToken=first_response["NextToken"]) + first_response = client.list_recipes(MaxResults=3, RecipeVersion="LATEST_WORKING") + response = client.list_recipes( + NextToken=first_response["NextToken"], RecipeVersion="LATEST_WORKING" + ) response["Recipes"].should.have.length_of(7) @@ -81,19 +102,160 @@ def test_list_recipes_from_next_token(): def test_list_recipes_with_max_results_greater_than_actual_results(): client = _create_databrew_client() _create_test_recipes(client, 4) - response = client.list_recipes(MaxResults=10) + response = client.list_recipes(MaxResults=10, RecipeVersion="LATEST_WORKING") response["Recipes"].should.have.length_of(4) @mock_databrew -def test_describe_recipe(): +def test_list_recipe_versions_no_recipe(): + client = _create_databrew_client() + recipe_name = "NotExist" + response = client.list_recipe_versions(Name=recipe_name) + response["Recipes"].should.have.length_of(0) + + +@mock_databrew +def test_list_recipe_versions_none_published(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + response = client.list_recipe_versions(Name=recipe_name) + response["Recipes"].should.have.length_of(0) + + +@mock_databrew +def test_list_recipe_versions_one_published(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + client.publish_recipe(Name=recipe_name) + response = client.list_recipe_versions(Name=recipe_name) + response["Recipes"].should.have.length_of(1) + response["Recipes"][0]["RecipeVersion"].should.equal("1.0") + + +@mock_databrew +def test_list_recipe_versions_two_published(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + client.publish_recipe(Name=recipe_name) + client.publish_recipe(Name=recipe_name) + response = client.list_recipe_versions(Name=recipe_name) + response["Recipes"].should.have.length_of(2) + response["Recipes"][0]["RecipeVersion"].should.equal("1.0") + response["Recipes"][1]["RecipeVersion"].should.equal("2.0") + + +@mock_databrew +def test_describe_recipe_latest_working(): client = _create_databrew_client() response = _create_test_recipe(client) + recipe = client.describe_recipe( + Name=response["Name"], RecipeVersion="LATEST_WORKING" + ) + + recipe["Name"].should.equal(response["Name"]) + recipe["Steps"].should.have.length_of(1) + recipe["RecipeVersion"].should.equal("0.1") + + +@mock_databrew +def test_describe_recipe_with_version(): + client = _create_databrew_client() + response = _create_test_recipe(client) + + recipe = client.describe_recipe(Name=response["Name"], RecipeVersion="0.1") + + recipe["Name"].should.equal(response["Name"]) + recipe["Steps"].should.have.length_of(1) + recipe["RecipeVersion"].should.equal("0.1") + + +@mock_databrew +def test_describe_recipe_latest_published(): + client = _create_databrew_client() + response = _create_test_recipe(client) + + client.publish_recipe(Name=response["Name"]) + recipe = client.describe_recipe( + Name=response["Name"], RecipeVersion="LATEST_PUBLISHED" + ) + + recipe["Name"].should.equal(response["Name"]) + recipe["Steps"].should.have.length_of(1) + recipe["RecipeVersion"].should.equal("1.0") + + +@mock_databrew +def test_describe_recipe_implicit_latest_published(): + client = _create_databrew_client() + response = _create_test_recipe(client) + + client.publish_recipe(Name=response["Name"]) recipe = client.describe_recipe(Name=response["Name"]) recipe["Name"].should.equal(response["Name"]) recipe["Steps"].should.have.length_of(1) + recipe["RecipeVersion"].should.equal("1.0") + + +@mock_databrew +def test_describe_recipe_that_does_not_exist(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name="DoseNotExist") + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal( + "The recipe DoseNotExist for version LATEST_PUBLISHED wasn't found." + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_describe_recipe_with_long_name(): + client = _create_databrew_client() + name = "a" * 256 + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name=name) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: " + f"Member must have length less than or equal to 255" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_describe_recipe_with_long_version(): + client = _create_databrew_client() + version = "1" * 17 + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name="AnyName", RecipeVersion=version) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"1 validation error detected: Value '{version}' at 'recipeVersion' failed to satisfy constraint: " + f"Member must have length less than or equal to 16" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_describe_recipe_with_invalid_version(): + client = _create_databrew_client() + name = "AnyName" + version = "invalid" + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name=name, RecipeVersion=version) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal(f"Recipe {name} version {version} isn't valid.") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) @mock_databrew @@ -129,22 +291,46 @@ def test_update_recipe(): recipe["Name"].should.equal(response["Name"]) - # Describe the recipe and change the changes - recipe = client.describe_recipe(Name=response["Name"]) + # Describe the recipe and check the changes + recipe = client.describe_recipe( + Name=response["Name"], RecipeVersion="LATEST_WORKING" + ) recipe["Name"].should.equal(response["Name"]) recipe["Steps"].should.have.length_of(1) recipe["Steps"][0]["Action"]["Parameters"]["removeCustomValue"].should.equal("true") @mock_databrew -def test_describe_recipe_that_does_not_exist(): +def test_update_recipe_description(): + client = _create_databrew_client() + response = _create_test_recipe(client) + + description = "NewDescription" + recipe = client.update_recipe( + Name=response["Name"], Steps=[], Description=description + ) + + recipe["Name"].should.equal(response["Name"]) + + # Describe the recipe and check the changes + recipe = client.describe_recipe( + Name=response["Name"], RecipeVersion="LATEST_WORKING" + ) + recipe["Name"].should.equal(response["Name"]) + recipe["Description"].should.equal(description) + + +@mock_databrew +def test_update_recipe_invalid(): client = _create_databrew_client() + recipe_name = "NotFound" with pytest.raises(ClientError) as exc: - client.describe_recipe(Name="DoseNotExist") + client.update_recipe(Name=recipe_name) err = exc.value.response["Error"] - err["Code"].should.equal("EntityNotFoundException") - err["Message"].should.equal("Recipe DoseNotExist not found.") + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal(f"The recipe {recipe_name} wasn't found") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) @mock_databrew @@ -152,9 +338,194 @@ def test_create_recipe_that_already_exists(): client = _create_databrew_client() response = _create_test_recipe(client) - + recipe_name = response["Name"] with pytest.raises(ClientError) as exc: _create_test_recipe(client, recipe_name=response["Name"]) err = exc.value.response["Error"] - err["Code"].should.equal("AlreadyExistsException") - err["Message"].should.equal("Recipe already exists.") + err["Code"].should.equal("ConflictException") + err["Message"].should.equal(f"The recipe {recipe_name} already exists") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409) + + +@mock_databrew +def test_publish_recipe(): + client = _create_databrew_client() + + response = _create_test_recipe(client) + recipe_name = response["Name"] + + # Before a recipe is published, we should not be able to retrieve a published version + with pytest.raises(ClientError) as exc: + recipe = client.describe_recipe(Name=recipe_name) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + + dt_before_publish = datetime.now().astimezone() + + # Publish the recipe + publish_response = client.publish_recipe(Name=recipe_name, Description="1st desc") + publish_response["Name"].should.equal(recipe_name) + + # Recipe is now published, so check we can retrieve the published version + recipe = client.describe_recipe(Name=recipe_name) + recipe["Description"].should.equal("1st desc") + recipe["RecipeVersion"].should.equal("1.0") + recipe["PublishedDate"].should.be.greater_than(dt_before_publish) + first_published_date = recipe["PublishedDate"] + + # Publish the recipe a 2nd time + publish_response = client.publish_recipe(Name=recipe_name, Description="2nd desc") + publish_response["Name"].should.equal(recipe_name) + + recipe = client.describe_recipe(Name=recipe_name) + recipe["Description"].should.equal("2nd desc") + recipe["RecipeVersion"].should.equal("2.0") + recipe["PublishedDate"].should.be.greater_than(first_published_date) + + +@mock_databrew +def test_publish_recipe_that_does_not_exist(): + client = _create_databrew_client() + with pytest.raises(ClientError) as exc: + client.publish_recipe(Name="DoesNotExist") + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_publish_long_recipe_name(): + client = _create_databrew_client() + name = "a" * 256 + with pytest.raises(ClientError) as exc: + client.publish_recipe(Name=name) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: " + f"Member must have length less than or equal to 255" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + + +@mock_databrew +def test_delete_recipe_version(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + client.delete_recipe_version(Name=recipe_name, RecipeVersion="LATEST_WORKING") + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name=recipe_name) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_delete_recipe_version_published(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + client.publish_recipe(Name=recipe_name) + client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.0") + with pytest.raises(ClientError) as exc: + client.describe_recipe(Name=recipe_name) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + recipe = client.describe_recipe(Name=recipe_name, RecipeVersion="1.1") + recipe["RecipeVersion"].should.equal("1.1") + + +@mock_databrew +def test_delete_recipe_version_latest_working_after_publish(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + client.publish_recipe(Name=recipe_name) + with pytest.raises(ClientError) as exc: + client.delete_recipe_version(Name=recipe_name, RecipeVersion="LATEST_WORKING") + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + "Recipe version LATEST_WORKING is not allowed to be deleted" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_delete_recipe_version_latest_working_numeric_after_publish(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + client.publish_recipe(Name=recipe_name) + with pytest.raises(ClientError) as exc: + client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.1") + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal("Recipe version 1.1 is not allowed to be deleted") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_delete_recipe_version_invalid_version_string(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + recipe_version = "NotValid" + client.publish_recipe(Name=recipe_name) + with pytest.raises(ClientError) as exc: + client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"Recipe {recipe_name} version {recipe_version} is invalid." + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_delete_recipe_version_invalid_version_length(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + recipe_version = "1" * 17 + client.publish_recipe(Name=recipe_name) + with pytest.raises(ClientError) as exc: + client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"Recipe {recipe_name} version {recipe_version} is invalid." + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_delete_recipe_version_unknown_recipe(): + client = _create_databrew_client() + recipe_name = "Unknown" + with pytest.raises(ClientError) as exc: + client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.1") + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal(f"The recipe {recipe_name} wasn't found") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_delete_recipe_version_unknown_version(): + client = _create_databrew_client() + response = _create_test_recipe(client) + recipe_name = response["Name"] + recipe_version = "1.1" + with pytest.raises(ClientError) as exc: + client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal( + f"The recipe {recipe_name} version {recipe_version} wasn't found." + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)