Databrew: add recipe version support (#5094)
This commit is contained in:
Michael Sanders
GitHub
parent
a666d59b58
commit
3f89b98889
@ -1069,7 +1069,7 @@
|
||||
|
||||
## databrew
|
||||
<details>
|
||||
<summary>15% implemented</summary>
|
||||
<summary>22% implemented</summary>
|
||||
|
||||
- [ ] batch_delete_recipe_version
|
||||
- [ ] create_dataset
|
||||
@ -1082,26 +1082,26 @@
|
||||
- [ ] delete_dataset
|
||||
- [ ] delete_job
|
||||
- [ ] delete_project
|
||||
- [ ] delete_recipe_version
|
||||
- [X] delete_recipe_version
|
||||
- [X] delete_ruleset
|
||||
- [ ] delete_schedule
|
||||
- [ ] describe_dataset
|
||||
- [ ] describe_job
|
||||
- [ ] describe_job_run
|
||||
- [ ] describe_project
|
||||
- [ ] describe_recipe
|
||||
- [X] describe_recipe
|
||||
- [ ] describe_ruleset
|
||||
- [ ] describe_schedule
|
||||
- [ ] list_datasets
|
||||
- [ ] list_job_runs
|
||||
- [ ] list_jobs
|
||||
- [ ] list_projects
|
||||
- [ ] list_recipe_versions
|
||||
- [X] list_recipe_versions
|
||||
- [X] list_recipes
|
||||
- [X] list_rulesets
|
||||
- [ ] list_schedules
|
||||
- [ ] list_tags_for_resource
|
||||
- [ ] publish_recipe
|
||||
- [X] publish_recipe
|
||||
- [ ] send_project_session_action
|
||||
- [ ] start_job_run
|
||||
- [ ] start_project_session
|
||||
|
||||
@ -36,26 +36,26 @@ databrew
|
||||
- [ ] delete_dataset
|
||||
- [ ] delete_job
|
||||
- [ ] delete_project
|
||||
- [ ] delete_recipe_version
|
||||
- [X] delete_recipe_version
|
||||
- [X] delete_ruleset
|
||||
- [ ] delete_schedule
|
||||
- [ ] describe_dataset
|
||||
- [ ] describe_job
|
||||
- [ ] describe_job_run
|
||||
- [ ] describe_project
|
||||
- [ ] describe_recipe
|
||||
- [X] describe_recipe
|
||||
- [ ] describe_ruleset
|
||||
- [ ] describe_schedule
|
||||
- [ ] list_datasets
|
||||
- [ ] list_job_runs
|
||||
- [ ] list_jobs
|
||||
- [ ] list_projects
|
||||
- [ ] list_recipe_versions
|
||||
- [X] list_recipe_versions
|
||||
- [X] list_recipes
|
||||
- [X] list_rulesets
|
||||
- [ ] list_schedules
|
||||
- [ ] list_tags_for_resource
|
||||
- [ ] publish_recipe
|
||||
- [X] publish_recipe
|
||||
- [ ] send_project_session_action
|
||||
- [ ] start_job_run
|
||||
- [ ] start_project_session
|
||||
|
||||
@ -10,9 +10,16 @@ class AlreadyExistsException(DataBrewClientError):
|
||||
super().__init__("AlreadyExistsException", "%s already exists." % (typ))
|
||||
|
||||
|
||||
class RecipeAlreadyExistsException(AlreadyExistsException):
|
||||
def __init__(self):
|
||||
super().__init__("Recipe")
|
||||
class ConflictException(DataBrewClientError):
|
||||
code = 409
|
||||
|
||||
def __init__(self, message, **kwargs):
|
||||
super().__init__("ConflictException", message, **kwargs)
|
||||
|
||||
|
||||
class ValidationException(DataBrewClientError):
|
||||
def __init__(self, message, **kwargs):
|
||||
super().__init__("ValidationException", message, **kwargs)
|
||||
|
||||
|
||||
class RulesetAlreadyExistsException(AlreadyExistsException):
|
||||
@ -25,9 +32,11 @@ class EntityNotFoundException(DataBrewClientError):
|
||||
super().__init__("EntityNotFoundException", msg)
|
||||
|
||||
|
||||
class RecipeNotFoundException(EntityNotFoundException):
|
||||
def __init__(self, recipe_name):
|
||||
super().__init__("Recipe %s not found." % recipe_name)
|
||||
class ResourceNotFoundException(DataBrewClientError):
|
||||
code = 404
|
||||
|
||||
def __init__(self, message, **kwargs):
|
||||
super().__init__("ResourceNotFoundException", message, **kwargs)
|
||||
|
||||
|
||||
class RulesetNotFoundException(EntityNotFoundException):
|
||||
|
||||
@ -1,10 +1,16 @@
|
||||
from collections import OrderedDict
|
||||
from copy import deepcopy
|
||||
import math
|
||||
from datetime import datetime
|
||||
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.core.utils import BackendDict
|
||||
from moto.utilities.paginator import paginate
|
||||
from .exceptions import RecipeAlreadyExistsException, RecipeNotFoundException
|
||||
from .exceptions import (
|
||||
ConflictException,
|
||||
ResourceNotFoundException,
|
||||
ValidationException,
|
||||
)
|
||||
from .exceptions import RulesetAlreadyExistsException, RulesetNotFoundException
|
||||
|
||||
|
||||
@ -16,6 +22,12 @@ class DataBrewBackend(BaseBackend):
|
||||
"limit_default": 100,
|
||||
"unique_attribute": "name",
|
||||
},
|
||||
"list_recipe_versions": {
|
||||
"input_token": "next_token",
|
||||
"limit_key": "max_results",
|
||||
"limit_default": 100,
|
||||
"unique_attribute": "name",
|
||||
},
|
||||
"list_rulesets": {
|
||||
"input_token": "next_token",
|
||||
"limit_key": "max_results",
|
||||
@ -34,42 +46,134 @@ class DataBrewBackend(BaseBackend):
|
||||
region_name = self.region_name
|
||||
self.__init__(region_name)
|
||||
|
||||
@staticmethod
|
||||
def validate_length(param, param_name, max_length):
|
||||
if len(param) > max_length:
|
||||
raise ValidationException(
|
||||
f"1 validation error detected: Value '{param}' at '{param_name}' failed to "
|
||||
f"satisfy constraint: Member must have length less than or equal to {max_length}"
|
||||
)
|
||||
|
||||
def create_recipe(self, recipe_name, recipe_description, recipe_steps, tags):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipe.html
|
||||
if recipe_name in self.recipes:
|
||||
raise RecipeAlreadyExistsException()
|
||||
raise ConflictException(f"The recipe {recipe_name} already exists")
|
||||
|
||||
recipe = FakeRecipe(
|
||||
self.region_name, recipe_name, recipe_description, recipe_steps, tags
|
||||
)
|
||||
self.recipes[recipe_name] = recipe
|
||||
return recipe
|
||||
return recipe.latest_working
|
||||
|
||||
def update_recipe(self, recipe_name, recipe_description, recipe_steps, tags):
|
||||
def delete_recipe_version(self, recipe_name, recipe_version):
|
||||
if not FakeRecipe.version_is_valid(recipe_version, latest_published=False):
|
||||
raise ValidationException(
|
||||
f"Recipe {recipe_name} version {recipe_version} is invalid."
|
||||
)
|
||||
|
||||
try:
|
||||
recipe = self.recipes[recipe_name]
|
||||
except KeyError:
|
||||
raise ResourceNotFoundException(f"The recipe {recipe_name} wasn't found")
|
||||
|
||||
if (
|
||||
recipe_version != FakeRecipe.LATEST_WORKING
|
||||
and float(recipe_version) not in recipe.versions
|
||||
):
|
||||
raise ResourceNotFoundException(
|
||||
f"The recipe {recipe_name} version {recipe_version } wasn't found."
|
||||
)
|
||||
|
||||
if recipe_version in (
|
||||
FakeRecipe.LATEST_WORKING,
|
||||
str(recipe.latest_working.version),
|
||||
):
|
||||
if recipe.latest_published is not None:
|
||||
# Can only delete latest working version when there are no others
|
||||
raise ValidationException(
|
||||
f"Recipe version {recipe_version} is not allowed to be deleted"
|
||||
)
|
||||
else:
|
||||
del self.recipes[recipe_name]
|
||||
else:
|
||||
recipe.delete_published_version(recipe_version)
|
||||
|
||||
def update_recipe(self, recipe_name, recipe_description, recipe_steps):
|
||||
if recipe_name not in self.recipes:
|
||||
raise RecipeNotFoundException(recipe_name)
|
||||
raise ResourceNotFoundException(f"The recipe {recipe_name} wasn't found")
|
||||
|
||||
recipe = self.recipes[recipe_name]
|
||||
if recipe_description is not None:
|
||||
recipe.description = recipe_description
|
||||
if recipe_steps is not None:
|
||||
recipe.steps = recipe_steps
|
||||
if tags is not None:
|
||||
recipe.tags = tags
|
||||
recipe.update(recipe_description, recipe_steps)
|
||||
|
||||
return recipe
|
||||
return recipe.latest_working
|
||||
|
||||
@paginate(pagination_model=PAGINATION_MODEL)
|
||||
def list_recipes(self):
|
||||
return [self.recipes[key] for key in self.recipes] if self.recipes else []
|
||||
def list_recipes(self, recipe_version=None):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipes.html
|
||||
if recipe_version == FakeRecipe.LATEST_WORKING:
|
||||
version = "latest_working"
|
||||
elif recipe_version in (None, FakeRecipe.LATEST_PUBLISHED):
|
||||
version = "latest_published"
|
||||
else:
|
||||
raise ValidationException(
|
||||
f"Invalid version {recipe_version}. "
|
||||
"Valid versions are LATEST_PUBLISHED and LATEST_WORKING."
|
||||
)
|
||||
recipes = [getattr(self.recipes[key], version) for key in self.recipes]
|
||||
return [r for r in recipes if r is not None]
|
||||
|
||||
def get_recipe(self, recipe_name):
|
||||
"""
|
||||
The Version-parameter has not yet been implemented
|
||||
"""
|
||||
if recipe_name not in self.recipes:
|
||||
raise RecipeNotFoundException(recipe_name)
|
||||
return self.recipes[recipe_name]
|
||||
@paginate(pagination_model=PAGINATION_MODEL)
|
||||
def list_recipe_versions(self, recipe_name):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipeVersions.html
|
||||
self.validate_length(recipe_name, "name", 255)
|
||||
|
||||
recipe = self.recipes.get(recipe_name)
|
||||
if recipe is None:
|
||||
return []
|
||||
|
||||
latest_working = recipe.latest_working
|
||||
|
||||
recipe_versions = [
|
||||
recipe_version
|
||||
for recipe_version in recipe.versions.values()
|
||||
if recipe_version is not latest_working
|
||||
]
|
||||
return [r for r in recipe_versions if r is not None]
|
||||
|
||||
def get_recipe(self, recipe_name, recipe_version=None):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeRecipe.html
|
||||
self.validate_length(recipe_name, "name", 255)
|
||||
|
||||
if recipe_version is None:
|
||||
recipe_version = FakeRecipe.LATEST_PUBLISHED
|
||||
else:
|
||||
self.validate_length(recipe_version, "recipeVersion", 16)
|
||||
if not FakeRecipe.version_is_valid(recipe_version):
|
||||
raise ValidationException(
|
||||
f"Recipe {recipe_name} version {recipe_version} isn't valid."
|
||||
)
|
||||
|
||||
recipe = None
|
||||
if recipe_name in self.recipes:
|
||||
if recipe_version == FakeRecipe.LATEST_PUBLISHED:
|
||||
recipe = self.recipes[recipe_name].latest_published
|
||||
elif recipe_version == FakeRecipe.LATEST_WORKING:
|
||||
recipe = self.recipes[recipe_name].latest_working
|
||||
else:
|
||||
recipe = self.recipes[recipe_name].versions.get(float(recipe_version))
|
||||
if recipe is None:
|
||||
raise ResourceNotFoundException(
|
||||
f"The recipe {recipe_name} for version {recipe_version} wasn't found."
|
||||
)
|
||||
return recipe
|
||||
|
||||
def publish_recipe(self, recipe_name, description=None):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_PublishRecipe.html
|
||||
self.validate_length(recipe_name, "name", 255)
|
||||
try:
|
||||
self.recipes[recipe_name].publish(description)
|
||||
except KeyError:
|
||||
raise ResourceNotFoundException(f"Recipe {recipe_name} wasn't found")
|
||||
|
||||
def create_ruleset(
|
||||
self, ruleset_name, ruleset_description, ruleset_rules, ruleset_target_arn, tags
|
||||
@ -119,8 +223,77 @@ class DataBrewBackend(BaseBackend):
|
||||
|
||||
|
||||
class FakeRecipe(BaseModel):
|
||||
INITIAL_VERSION = 0.1
|
||||
LATEST_WORKING = "LATEST_WORKING"
|
||||
LATEST_PUBLISHED = "LATEST_PUBLISHED"
|
||||
|
||||
@classmethod
|
||||
def version_is_valid(cls, version, latest_working=True, latest_published=True):
|
||||
validity = True
|
||||
|
||||
if len(version) < 1 or len(version) > 16:
|
||||
validity = False
|
||||
else:
|
||||
try:
|
||||
version = float(version)
|
||||
except ValueError:
|
||||
if not (
|
||||
(version == cls.LATEST_WORKING and latest_working)
|
||||
or (version == cls.LATEST_PUBLISHED and latest_published)
|
||||
):
|
||||
validity = False
|
||||
return validity
|
||||
|
||||
def __init__(
|
||||
self, region_name, recipe_name, recipe_description, recipe_steps, tags
|
||||
):
|
||||
self.versions = OrderedDict()
|
||||
self.versions[self.INITIAL_VERSION] = FakeRecipeVersion(
|
||||
region_name,
|
||||
recipe_name,
|
||||
recipe_description,
|
||||
recipe_steps,
|
||||
tags,
|
||||
version=self.INITIAL_VERSION,
|
||||
)
|
||||
self.latest_working = self.versions[self.INITIAL_VERSION]
|
||||
self.latest_published = None
|
||||
|
||||
def publish(self, description=None):
|
||||
self.latest_published = self.latest_working
|
||||
self.latest_working = deepcopy(self.latest_working)
|
||||
self.latest_published.publish(description)
|
||||
del self.versions[self.latest_working.version]
|
||||
self.versions[self.latest_published.version] = self.latest_published
|
||||
self.latest_working.version = self.latest_published.version + 0.1
|
||||
self.versions[self.latest_working.version] = self.latest_working
|
||||
|
||||
def update(self, description, steps):
|
||||
if description is not None:
|
||||
self.latest_working.description = description
|
||||
if steps is not None:
|
||||
self.latest_working.steps = steps
|
||||
|
||||
def delete_published_version(self, version):
|
||||
version = float(version)
|
||||
assert version.is_integer()
|
||||
if version == self.latest_published.version:
|
||||
prev_version = version - 1.0
|
||||
# Iterate back through the published versions until we find one that exists
|
||||
while prev_version >= 1.0:
|
||||
if prev_version in self.versions:
|
||||
self.latest_published = self.versions[prev_version]
|
||||
break
|
||||
prev_version -= 1.0
|
||||
else:
|
||||
# Didn't find an earlier published version
|
||||
self.latest_published = None
|
||||
del self.versions[version]
|
||||
|
||||
|
||||
class FakeRecipeVersion(BaseModel):
|
||||
def __init__(
|
||||
self, region_name, recipe_name, recipe_description, recipe_steps, tags, version
|
||||
):
|
||||
self.region_name = region_name
|
||||
self.name = recipe_name
|
||||
@ -128,15 +301,28 @@ class FakeRecipe(BaseModel):
|
||||
self.steps = recipe_steps
|
||||
self.created_time = datetime.now()
|
||||
self.tags = tags
|
||||
self.published_date = None
|
||||
self.version = version
|
||||
|
||||
def as_dict(self):
|
||||
return {
|
||||
dict_recipe = {
|
||||
"Name": self.name,
|
||||
"Steps": self.steps,
|
||||
"Description": self.description,
|
||||
"CreateTime": self.created_time.isoformat(),
|
||||
"CreateDate": "%.3f" % self.created_time.timestamp(),
|
||||
"Tags": self.tags or dict(),
|
||||
"RecipeVersion": str(self.version),
|
||||
}
|
||||
if self.published_date is not None:
|
||||
dict_recipe["PublishedDate"] = "%.3f" % self.published_date.timestamp()
|
||||
|
||||
return dict_recipe
|
||||
|
||||
def publish(self, description):
|
||||
self.version = float(math.ceil(self.version))
|
||||
self.published_date = datetime.now()
|
||||
if description is not None:
|
||||
self.description = description
|
||||
|
||||
|
||||
class FakeRuleset(BaseModel):
|
||||
|
||||
@ -31,6 +31,22 @@ class DataBrewResponse(BaseResponse):
|
||||
).as_dict()
|
||||
)
|
||||
|
||||
@amzn_request_id
|
||||
def delete_recipe_version(self, request, full_url, headers):
|
||||
self.setup_class(request, full_url, headers)
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_DeleteRecipeVersion.html
|
||||
if request.method == "DELETE":
|
||||
parsed_url = urlparse(full_url)
|
||||
split_path = parsed_url.path.strip("/").split("/")
|
||||
recipe_name = split_path[1]
|
||||
recipe_version = split_path[3]
|
||||
self.databrew_backend.delete_recipe_version(recipe_name, recipe_version)
|
||||
return (
|
||||
200,
|
||||
{},
|
||||
json.dumps({"Name": recipe_name, "RecipeVersion": recipe_version}),
|
||||
)
|
||||
|
||||
@amzn_request_id
|
||||
def list_recipes(self):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipes.html
|
||||
@ -38,10 +54,15 @@ class DataBrewResponse(BaseResponse):
|
||||
max_results = self._get_int_param(
|
||||
"MaxResults", self._get_int_param("maxResults")
|
||||
)
|
||||
recipe_version = self._get_param(
|
||||
"RecipeVersion", self._get_param("recipeVersion")
|
||||
)
|
||||
|
||||
# pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking
|
||||
recipe_list, next_token = self.databrew_backend.list_recipes(
|
||||
next_token=next_token, max_results=max_results
|
||||
next_token=next_token,
|
||||
max_results=max_results,
|
||||
recipe_version=recipe_version,
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
@ -50,19 +71,55 @@ class DataBrewResponse(BaseResponse):
|
||||
}
|
||||
)
|
||||
|
||||
@amzn_request_id
|
||||
def list_recipe_versions(self, request, full_url, headers):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_ListRecipeVersions.html
|
||||
self.setup_class(request, full_url, headers)
|
||||
recipe_name = self._get_param("Name", self._get_param("name"))
|
||||
next_token = self._get_param("NextToken", self._get_param("nextToken"))
|
||||
max_results = self._get_int_param(
|
||||
"MaxResults", self._get_int_param("maxResults")
|
||||
)
|
||||
|
||||
# pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking
|
||||
recipe_list, next_token = self.databrew_backend.list_recipe_versions(
|
||||
recipe_name=recipe_name, next_token=next_token, max_results=max_results
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"Recipes": [recipe.as_dict() for recipe in recipe_list],
|
||||
"NextToken": next_token,
|
||||
}
|
||||
)
|
||||
|
||||
@amzn_request_id
|
||||
def publish_recipe(self, request, full_url, headers):
|
||||
self.setup_class(request, full_url, headers)
|
||||
if request.method == "POST":
|
||||
parsed_url = urlparse(full_url)
|
||||
recipe_name = parsed_url.path.strip("/").split("/", 2)[1]
|
||||
recipe_description = self.parameters.get("Description")
|
||||
self.databrew_backend.publish_recipe(recipe_name, recipe_description)
|
||||
return 200, {}, json.dumps({"Name": recipe_name})
|
||||
|
||||
def put_recipe_response(self, recipe_name):
|
||||
recipe_description = self.parameters.get("Description")
|
||||
recipe_steps = self.parameters.get("Steps")
|
||||
tags = self.parameters.get("Tags")
|
||||
|
||||
recipe = self.databrew_backend.update_recipe(
|
||||
recipe_name, recipe_description, recipe_steps, tags
|
||||
self.databrew_backend.update_recipe(
|
||||
recipe_name, recipe_description, recipe_steps
|
||||
)
|
||||
return 200, {}, json.dumps(recipe.as_dict())
|
||||
return 200, {}, json.dumps({"Name": recipe_name})
|
||||
|
||||
def get_recipe_response(self, recipe_name):
|
||||
recipe = self.databrew_backend.get_recipe(recipe_name)
|
||||
return 201, {}, json.dumps(recipe.as_dict())
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeRecipe.html
|
||||
recipe_version = self._get_param(
|
||||
"RecipeVersion", self._get_param("recipeVersion")
|
||||
)
|
||||
recipe = self.databrew_backend.get_recipe(
|
||||
recipe_name, recipe_version=recipe_version
|
||||
)
|
||||
return 200, {}, json.dumps(recipe.as_dict())
|
||||
|
||||
@amzn_request_id
|
||||
def recipe_response(self, request, full_url, headers):
|
||||
|
||||
@ -3,8 +3,11 @@ from .responses import DataBrewResponse
|
||||
url_bases = [r"https?://databrew\.(.+)\.amazonaws.com"]
|
||||
|
||||
url_paths = {
|
||||
"{0}/recipeVersions$": DataBrewResponse().list_recipe_versions,
|
||||
"{0}/recipes$": DataBrewResponse.dispatch,
|
||||
"{0}/recipes/(?P<recipe_name>[^/]+)$": DataBrewResponse().recipe_response,
|
||||
"{0}/recipes/(?P<recipe_name>[^/]+)/recipeVersion/(?P<recipe_version>[^/]+)": DataBrewResponse().delete_recipe_version,
|
||||
"{0}/recipes/(?P<recipe_name>[^/]+)/publishRecipe$": DataBrewResponse().publish_recipe,
|
||||
"{0}/rulesets$": DataBrewResponse.dispatch,
|
||||
"{0}/rulesets/(?P<ruleset_name>[^/]+)$": DataBrewResponse().ruleset_response,
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ import uuid
|
||||
import boto3
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
from datetime import datetime
|
||||
|
||||
from moto import mock_databrew
|
||||
|
||||
@ -58,12 +59,30 @@ def test_recipe_list_when_empty():
|
||||
response["Recipes"].should.have.length_of(0)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_recipe_list_with_invalid_version():
|
||||
client = _create_databrew_client()
|
||||
|
||||
recipe_version = "1.1"
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.list_recipes(RecipeVersion=recipe_version)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"Invalid version {recipe_version}. "
|
||||
"Valid versions are LATEST_PUBLISHED and LATEST_WORKING."
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_recipes_with_max_results():
|
||||
client = _create_databrew_client()
|
||||
|
||||
_create_test_recipes(client, 4)
|
||||
response = client.list_recipes(MaxResults=2)
|
||||
response = client.list_recipes(MaxResults=2, RecipeVersion="LATEST_WORKING")
|
||||
response["Recipes"].should.have.length_of(2)
|
||||
response.should.have.key("NextToken")
|
||||
|
||||
@ -72,8 +91,10 @@ def test_list_recipes_with_max_results():
|
||||
def test_list_recipes_from_next_token():
|
||||
client = _create_databrew_client()
|
||||
_create_test_recipes(client, 10)
|
||||
first_response = client.list_recipes(MaxResults=3)
|
||||
response = client.list_recipes(NextToken=first_response["NextToken"])
|
||||
first_response = client.list_recipes(MaxResults=3, RecipeVersion="LATEST_WORKING")
|
||||
response = client.list_recipes(
|
||||
NextToken=first_response["NextToken"], RecipeVersion="LATEST_WORKING"
|
||||
)
|
||||
response["Recipes"].should.have.length_of(7)
|
||||
|
||||
|
||||
@ -81,19 +102,160 @@ def test_list_recipes_from_next_token():
|
||||
def test_list_recipes_with_max_results_greater_than_actual_results():
|
||||
client = _create_databrew_client()
|
||||
_create_test_recipes(client, 4)
|
||||
response = client.list_recipes(MaxResults=10)
|
||||
response = client.list_recipes(MaxResults=10, RecipeVersion="LATEST_WORKING")
|
||||
response["Recipes"].should.have.length_of(4)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe():
|
||||
def test_list_recipe_versions_no_recipe():
|
||||
client = _create_databrew_client()
|
||||
recipe_name = "NotExist"
|
||||
response = client.list_recipe_versions(Name=recipe_name)
|
||||
response["Recipes"].should.have.length_of(0)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_recipe_versions_none_published():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
response = client.list_recipe_versions(Name=recipe_name)
|
||||
response["Recipes"].should.have.length_of(0)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_recipe_versions_one_published():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
response = client.list_recipe_versions(Name=recipe_name)
|
||||
response["Recipes"].should.have.length_of(1)
|
||||
response["Recipes"][0]["RecipeVersion"].should.equal("1.0")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_recipe_versions_two_published():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
response = client.list_recipe_versions(Name=recipe_name)
|
||||
response["Recipes"].should.have.length_of(2)
|
||||
response["Recipes"][0]["RecipeVersion"].should.equal("1.0")
|
||||
response["Recipes"][1]["RecipeVersion"].should.equal("2.0")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_latest_working():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
|
||||
recipe = client.describe_recipe(
|
||||
Name=response["Name"], RecipeVersion="LATEST_WORKING"
|
||||
)
|
||||
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
recipe["Steps"].should.have.length_of(1)
|
||||
recipe["RecipeVersion"].should.equal("0.1")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_with_version():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
|
||||
recipe = client.describe_recipe(Name=response["Name"], RecipeVersion="0.1")
|
||||
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
recipe["Steps"].should.have.length_of(1)
|
||||
recipe["RecipeVersion"].should.equal("0.1")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_latest_published():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
|
||||
client.publish_recipe(Name=response["Name"])
|
||||
recipe = client.describe_recipe(
|
||||
Name=response["Name"], RecipeVersion="LATEST_PUBLISHED"
|
||||
)
|
||||
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
recipe["Steps"].should.have.length_of(1)
|
||||
recipe["RecipeVersion"].should.equal("1.0")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_implicit_latest_published():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
|
||||
client.publish_recipe(Name=response["Name"])
|
||||
recipe = client.describe_recipe(Name=response["Name"])
|
||||
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
recipe["Steps"].should.have.length_of(1)
|
||||
recipe["RecipeVersion"].should.equal("1.0")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_that_does_not_exist():
|
||||
client = _create_databrew_client()
|
||||
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name="DoseNotExist")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal(
|
||||
"The recipe DoseNotExist for version LATEST_PUBLISHED wasn't found."
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_with_long_name():
|
||||
client = _create_databrew_client()
|
||||
name = "a" * 256
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name=name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
|
||||
f"Member must have length less than or equal to 255"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_with_long_version():
|
||||
client = _create_databrew_client()
|
||||
version = "1" * 17
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name="AnyName", RecipeVersion=version)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"1 validation error detected: Value '{version}' at 'recipeVersion' failed to satisfy constraint: "
|
||||
f"Member must have length less than or equal to 16"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_with_invalid_version():
|
||||
client = _create_databrew_client()
|
||||
name = "AnyName"
|
||||
version = "invalid"
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name=name, RecipeVersion=version)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(f"Recipe {name} version {version} isn't valid.")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
@ -129,22 +291,46 @@ def test_update_recipe():
|
||||
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
|
||||
# Describe the recipe and change the changes
|
||||
recipe = client.describe_recipe(Name=response["Name"])
|
||||
# Describe the recipe and check the changes
|
||||
recipe = client.describe_recipe(
|
||||
Name=response["Name"], RecipeVersion="LATEST_WORKING"
|
||||
)
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
recipe["Steps"].should.have.length_of(1)
|
||||
recipe["Steps"][0]["Action"]["Parameters"]["removeCustomValue"].should.equal("true")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_that_does_not_exist():
|
||||
def test_update_recipe_description():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
|
||||
description = "NewDescription"
|
||||
recipe = client.update_recipe(
|
||||
Name=response["Name"], Steps=[], Description=description
|
||||
)
|
||||
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
|
||||
# Describe the recipe and check the changes
|
||||
recipe = client.describe_recipe(
|
||||
Name=response["Name"], RecipeVersion="LATEST_WORKING"
|
||||
)
|
||||
recipe["Name"].should.equal(response["Name"])
|
||||
recipe["Description"].should.equal(description)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_update_recipe_invalid():
|
||||
client = _create_databrew_client()
|
||||
|
||||
recipe_name = "NotFound"
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name="DoseNotExist")
|
||||
client.update_recipe(Name=recipe_name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("EntityNotFoundException")
|
||||
err["Message"].should.equal("Recipe DoseNotExist not found.")
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal(f"The recipe {recipe_name} wasn't found")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
@ -152,9 +338,194 @@ def test_create_recipe_that_already_exists():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = _create_test_recipe(client)
|
||||
|
||||
recipe_name = response["Name"]
|
||||
with pytest.raises(ClientError) as exc:
|
||||
_create_test_recipe(client, recipe_name=response["Name"])
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("AlreadyExistsException")
|
||||
err["Message"].should.equal("Recipe already exists.")
|
||||
err["Code"].should.equal("ConflictException")
|
||||
err["Message"].should.equal(f"The recipe {recipe_name} already exists")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_publish_recipe():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
|
||||
# Before a recipe is published, we should not be able to retrieve a published version
|
||||
with pytest.raises(ClientError) as exc:
|
||||
recipe = client.describe_recipe(Name=recipe_name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
|
||||
dt_before_publish = datetime.now().astimezone()
|
||||
|
||||
# Publish the recipe
|
||||
publish_response = client.publish_recipe(Name=recipe_name, Description="1st desc")
|
||||
publish_response["Name"].should.equal(recipe_name)
|
||||
|
||||
# Recipe is now published, so check we can retrieve the published version
|
||||
recipe = client.describe_recipe(Name=recipe_name)
|
||||
recipe["Description"].should.equal("1st desc")
|
||||
recipe["RecipeVersion"].should.equal("1.0")
|
||||
recipe["PublishedDate"].should.be.greater_than(dt_before_publish)
|
||||
first_published_date = recipe["PublishedDate"]
|
||||
|
||||
# Publish the recipe a 2nd time
|
||||
publish_response = client.publish_recipe(Name=recipe_name, Description="2nd desc")
|
||||
publish_response["Name"].should.equal(recipe_name)
|
||||
|
||||
recipe = client.describe_recipe(Name=recipe_name)
|
||||
recipe["Description"].should.equal("2nd desc")
|
||||
recipe["RecipeVersion"].should.equal("2.0")
|
||||
recipe["PublishedDate"].should.be.greater_than(first_published_date)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_publish_recipe_that_does_not_exist():
|
||||
client = _create_databrew_client()
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.publish_recipe(Name="DoesNotExist")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_publish_long_recipe_name():
|
||||
client = _create_databrew_client()
|
||||
name = "a" * 256
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.publish_recipe(Name=name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
|
||||
f"Member must have length less than or equal to 255"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion="LATEST_WORKING")
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name=recipe_name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_published():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.0")
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_recipe(Name=recipe_name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
recipe = client.describe_recipe(Name=recipe_name, RecipeVersion="1.1")
|
||||
recipe["RecipeVersion"].should.equal("1.1")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_latest_working_after_publish():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion="LATEST_WORKING")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
"Recipe version LATEST_WORKING is not allowed to be deleted"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_latest_working_numeric_after_publish():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.1")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal("Recipe version 1.1 is not allowed to be deleted")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_invalid_version_string():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
recipe_version = "NotValid"
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"Recipe {recipe_name} version {recipe_version} is invalid."
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_invalid_version_length():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
recipe_version = "1" * 17
|
||||
client.publish_recipe(Name=recipe_name)
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"Recipe {recipe_name} version {recipe_version} is invalid."
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_unknown_recipe():
|
||||
client = _create_databrew_client()
|
||||
recipe_name = "Unknown"
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.1")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal(f"The recipe {recipe_name} wasn't found")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_recipe_version_unknown_version():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_recipe(client)
|
||||
recipe_name = response["Name"]
|
||||
recipe_version = "1.1"
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal(
|
||||
f"The recipe {recipe_name} version {recipe_version} wasn't found."
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user