Adds CRUD support for databrew recipe jobs and profile jobs (#5244)
This commit is contained in:
Michael Sanders
GitHub
parent
d3cee3c6fa
commit
e004c6d218
@ -1069,32 +1069,32 @@
|
||||
|
||||
## databrew
|
||||
<details>
|
||||
<summary>34% implemented</summary>
|
||||
<summary>54% implemented</summary>
|
||||
|
||||
- [ ] batch_delete_recipe_version
|
||||
- [X] create_dataset
|
||||
- [ ] create_profile_job
|
||||
- [X] create_profile_job
|
||||
- [ ] create_project
|
||||
- [X] create_recipe
|
||||
- [ ] create_recipe_job
|
||||
- [X] create_recipe_job
|
||||
- [X] create_ruleset
|
||||
- [ ] create_schedule
|
||||
- [X] delete_dataset
|
||||
- [ ] delete_job
|
||||
- [X] delete_job
|
||||
- [ ] delete_project
|
||||
- [X] delete_recipe_version
|
||||
- [X] delete_ruleset
|
||||
- [ ] delete_schedule
|
||||
- [X] describe_dataset
|
||||
- [ ] describe_job
|
||||
- [X] describe_job
|
||||
- [ ] describe_job_run
|
||||
- [ ] describe_project
|
||||
- [ ] describe_recipe
|
||||
- [ ] describe_ruleset
|
||||
- [X] describe_recipe
|
||||
- [X] describe_ruleset
|
||||
- [ ] describe_schedule
|
||||
- [X] list_datasets
|
||||
- [ ] list_job_runs
|
||||
- [ ] list_jobs
|
||||
- [X] list_jobs
|
||||
- [ ] list_projects
|
||||
- [X] list_recipe_versions
|
||||
- [X] list_recipes
|
||||
@ -1109,10 +1109,10 @@
|
||||
- [ ] tag_resource
|
||||
- [ ] untag_resource
|
||||
- [X] update_dataset
|
||||
- [ ] update_profile_job
|
||||
- [X] update_profile_job
|
||||
- [ ] update_project
|
||||
- [X] update_recipe
|
||||
- [ ] update_recipe_job
|
||||
- [X] update_recipe_job
|
||||
- [X] update_ruleset
|
||||
- [ ] update_schedule
|
||||
</details>
|
||||
@ -6163,4 +6163,4 @@
|
||||
- workspaces
|
||||
- workspaces-web
|
||||
- xray
|
||||
</details>
|
||||
</details>
|
||||
|
||||
@ -27,28 +27,28 @@ databrew
|
||||
|
||||
- [ ] batch_delete_recipe_version
|
||||
- [X] create_dataset
|
||||
- [ ] create_profile_job
|
||||
- [X] create_profile_job
|
||||
- [ ] create_project
|
||||
- [X] create_recipe
|
||||
- [ ] create_recipe_job
|
||||
- [X] create_recipe_job
|
||||
- [X] create_ruleset
|
||||
- [ ] create_schedule
|
||||
- [X] delete_dataset
|
||||
- [ ] delete_job
|
||||
- [X] delete_job
|
||||
- [ ] delete_project
|
||||
- [X] delete_recipe_version
|
||||
- [X] delete_ruleset
|
||||
- [ ] delete_schedule
|
||||
- [X] describe_dataset
|
||||
- [ ] describe_job
|
||||
- [X] describe_job
|
||||
- [ ] describe_job_run
|
||||
- [ ] describe_project
|
||||
- [ ] describe_recipe
|
||||
- [ ] describe_ruleset
|
||||
- [X] describe_recipe
|
||||
- [X] describe_ruleset
|
||||
- [ ] describe_schedule
|
||||
- [X] list_datasets
|
||||
- [ ] list_job_runs
|
||||
- [ ] list_jobs
|
||||
- [X] list_jobs
|
||||
- [ ] list_projects
|
||||
- [X] list_recipe_versions
|
||||
- [X] list_recipes
|
||||
@ -63,10 +63,10 @@ databrew
|
||||
- [ ] tag_resource
|
||||
- [ ] untag_resource
|
||||
- [X] update_dataset
|
||||
- [ ] update_profile_job
|
||||
- [X] update_profile_job
|
||||
- [ ] update_project
|
||||
- [X] update_recipe
|
||||
- [ ] update_recipe_job
|
||||
- [X] update_recipe_job
|
||||
- [X] update_ruleset
|
||||
- [ ] update_schedule
|
||||
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
from abc import ABCMeta
|
||||
from abc import abstractmethod
|
||||
|
||||
from collections import OrderedDict
|
||||
from copy import deepcopy
|
||||
import math
|
||||
@ -5,6 +8,9 @@ from datetime import datetime
|
||||
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.core.utils import BackendDict
|
||||
from moto.core.utils import underscores_to_camelcase
|
||||
from moto.core.utils import camelcase_to_pascal
|
||||
|
||||
from moto.utilities.paginator import paginate
|
||||
|
||||
from .exceptions import (
|
||||
@ -43,6 +49,12 @@ class DataBrewBackend(BaseBackend):
|
||||
"limit_default": 100,
|
||||
"unique_attribute": "name",
|
||||
},
|
||||
"list_jobs": {
|
||||
"input_token": "next_token",
|
||||
"limit_key": "max_results",
|
||||
"limit_default": 100,
|
||||
"unique_attribute": "name",
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, region_name, account_id):
|
||||
@ -50,6 +62,7 @@ class DataBrewBackend(BaseBackend):
|
||||
self.recipes = OrderedDict()
|
||||
self.rulesets = OrderedDict()
|
||||
self.datasets = OrderedDict()
|
||||
self.jobs = OrderedDict()
|
||||
|
||||
@staticmethod
|
||||
def validate_length(param, param_name, max_length):
|
||||
@ -145,7 +158,7 @@ class DataBrewBackend(BaseBackend):
|
||||
]
|
||||
return [r for r in recipe_versions if r is not None]
|
||||
|
||||
def get_recipe(self, recipe_name, recipe_version=None):
|
||||
def describe_recipe(self, recipe_name, recipe_version=None):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeRecipe.html
|
||||
self.validate_length(recipe_name, "name", 255)
|
||||
|
||||
@ -211,7 +224,7 @@ class DataBrewBackend(BaseBackend):
|
||||
|
||||
return ruleset
|
||||
|
||||
def get_ruleset(self, ruleset_name):
|
||||
def describe_ruleset(self, ruleset_name):
|
||||
if ruleset_name not in self.rulesets:
|
||||
raise RulesetNotFoundException(ruleset_name)
|
||||
return self.rulesets[ruleset_name]
|
||||
@ -295,6 +308,99 @@ class DataBrewBackend(BaseBackend):
|
||||
|
||||
return self.datasets[dataset_name]
|
||||
|
||||
def describe_job(self, job_name):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeJob.html
|
||||
self.validate_length(job_name, "name", 240)
|
||||
|
||||
if job_name not in self.jobs:
|
||||
raise ResourceNotFoundException(f"Job {job_name} wasn't found.")
|
||||
|
||||
return self.jobs[job_name]
|
||||
|
||||
def delete_job(self, job_name):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_DeleteJob.html
|
||||
self.validate_length(job_name, "name", 240)
|
||||
|
||||
if job_name not in self.jobs:
|
||||
raise ResourceNotFoundException(f"The job {job_name} wasn't found.")
|
||||
|
||||
del self.jobs[job_name]
|
||||
|
||||
def create_profile_job(self, **kwargs):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_CreateProfileJob.html
|
||||
job_name = kwargs["name"]
|
||||
self.validate_length(job_name, "name", 240)
|
||||
|
||||
if job_name in self.jobs:
|
||||
raise ConflictException(
|
||||
f"The job {job_name} {self.jobs[job_name].job_type.lower()} job already exists."
|
||||
)
|
||||
|
||||
job = FakeProfileJob(
|
||||
account_id=self.account_id, region_name=self.region_name, **kwargs
|
||||
)
|
||||
|
||||
self.jobs[job_name] = job
|
||||
return job
|
||||
|
||||
def create_recipe_job(self, **kwargs):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipeJob.html
|
||||
job_name = kwargs["name"]
|
||||
self.validate_length(job_name, "name", 240)
|
||||
|
||||
if job_name in self.jobs:
|
||||
raise ConflictException(
|
||||
f"The job {job_name} {self.jobs[job_name].job_type.lower()} job already exists."
|
||||
)
|
||||
|
||||
job = FakeRecipeJob(
|
||||
account_id=self.account_id, region_name=self.region_name, **kwargs
|
||||
)
|
||||
|
||||
self.jobs[job_name] = job
|
||||
return job
|
||||
|
||||
def update_job(self, **kwargs):
|
||||
job_name = kwargs["name"]
|
||||
self.validate_length(job_name, "name", 240)
|
||||
|
||||
if job_name not in self.jobs:
|
||||
raise ResourceNotFoundException(f"The job {job_name} wasn't found")
|
||||
|
||||
job = self.jobs[job_name]
|
||||
|
||||
for param, value in kwargs.items():
|
||||
setattr(job, param, value)
|
||||
return job
|
||||
|
||||
def update_recipe_job(self, **kwargs):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateRecipeJob.html
|
||||
return self.update_job(**kwargs)
|
||||
|
||||
def update_profile_job(self, **kwargs):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateProfileJob.html
|
||||
return self.update_job(**kwargs)
|
||||
|
||||
@paginate(pagination_model=PAGINATION_MODEL)
|
||||
def list_jobs(self, dataset_name=None, project_name=None):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_ListJobs.html
|
||||
if dataset_name is not None:
|
||||
self.validate_length(dataset_name, "datasetName", 255)
|
||||
if project_name is not None:
|
||||
self.validate_length(project_name, "projectName", 255)
|
||||
|
||||
def filter_jobs(job):
|
||||
if dataset_name is not None and job.dataset_name != dataset_name:
|
||||
return False
|
||||
if (
|
||||
project_name is not None
|
||||
and getattr(job, "project_name", None) != project_name
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
return list(filter(filter_jobs, self.jobs.values()))
|
||||
|
||||
|
||||
class FakeRecipe(BaseModel):
|
||||
INITIAL_VERSION = 0.1
|
||||
@ -424,7 +530,7 @@ class FakeRuleset(BaseModel):
|
||||
"Rules": self.rules,
|
||||
"Description": self.description,
|
||||
"TargetArn": self.target_arn,
|
||||
"CreateTime": self.created_time.isoformat(),
|
||||
"CreateDate": "%.3f" % self.created_time.timestamp(),
|
||||
"Tags": self.tags or dict(),
|
||||
}
|
||||
|
||||
@ -464,10 +570,106 @@ class FakeDataset(BaseModel):
|
||||
"FormatOptions": self.format_options,
|
||||
"Input": self.input,
|
||||
"PathOptions": self.path_options,
|
||||
"CreateTime": self.created_time.isoformat(),
|
||||
"CreateDate": "%.3f" % self.created_time.timestamp(),
|
||||
"Tags": self.tags or dict(),
|
||||
"ResourceArn": self.resource_arn,
|
||||
}
|
||||
|
||||
|
||||
class BaseModelABCMeta(ABCMeta, type(BaseModel)):
|
||||
pass
|
||||
|
||||
|
||||
class FakeJob(BaseModel, metaclass=BaseModelABCMeta):
|
||||
|
||||
ENCRYPTION_MODES = ("SSE-S3", "SSE-KMS")
|
||||
LOG_SUBSCRIPTION_VALUES = ("ENABLE", "DISABLE")
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def local_attrs(self) -> tuple:
|
||||
raise NotImplementedError
|
||||
|
||||
def __init__(self, account_id, region_name, **kwargs):
|
||||
self.account_id = account_id
|
||||
self.region_name = region_name
|
||||
self.name = kwargs.get("name")
|
||||
self.created_time = datetime.now()
|
||||
self.dataset_name = kwargs.get("dataset_name")
|
||||
self.encryption_mode = kwargs.get("encryption_mode")
|
||||
self.log_subscription = kwargs.get("log_subscription")
|
||||
self.max_capacity = kwargs.get("max_capacity")
|
||||
self.max_retries = kwargs.get("max_retries")
|
||||
self.role_arn = kwargs.get("role_arn")
|
||||
self.tags = kwargs.get("tags")
|
||||
self.validate()
|
||||
# Set attributes specific to subclass
|
||||
for k in self.local_attrs:
|
||||
setattr(self, k, kwargs.get(k))
|
||||
|
||||
def validate(self):
|
||||
if self.encryption_mode is not None:
|
||||
if self.encryption_mode not in FakeJob.ENCRYPTION_MODES:
|
||||
raise ValidationException(
|
||||
f"1 validation error detected: Value '{self.encryption_mode}' at 'encryptionMode' failed to satisfy constraint: Member must satisfy enum value set: [{', '.join(self.ENCRYPTION_MODES)}]"
|
||||
)
|
||||
if self.log_subscription is not None:
|
||||
if self.log_subscription not in FakeJob.LOG_SUBSCRIPTION_VALUES:
|
||||
raise ValidationException(
|
||||
f"1 validation error detected: Value '{self.log_subscription}' at 'logSubscription' failed to satisfy constraint: Member must satisfy enum value set: [{', '.join(self.LOG_SUBSCRIPTION_VALUES)}]"
|
||||
)
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def job_type(self) -> str:
|
||||
pass
|
||||
|
||||
@property
|
||||
def resource_arn(self):
|
||||
return f"arn:aws:databrew:{self.region_name}:{self.account_id}:job/{self.name}"
|
||||
|
||||
def as_dict(self):
|
||||
rtn_dict = {
|
||||
"Name": self.name,
|
||||
"AccountId": self.account_id,
|
||||
"CreateDate": "%.3f" % self.created_time.timestamp(),
|
||||
"DatasetName": self.dataset_name,
|
||||
"EncryptionMode": self.encryption_mode,
|
||||
"Tags": self.tags or dict(),
|
||||
"LogSubscription": self.log_subscription,
|
||||
"MaxCapacity": self.max_capacity,
|
||||
"MaxRetries": self.max_retries,
|
||||
"ResourceArn": self.resource_arn,
|
||||
"RoleArn": self.role_arn,
|
||||
"Type": self.job_type,
|
||||
}
|
||||
|
||||
# Add in subclass attributes
|
||||
for k in self.local_attrs:
|
||||
rtn_dict[camelcase_to_pascal(underscores_to_camelcase(k))] = getattr(
|
||||
self, k
|
||||
)
|
||||
|
||||
# Remove items that have a value of None
|
||||
rtn_dict = {k: v for k, v in rtn_dict.items() if v is not None}
|
||||
|
||||
return rtn_dict
|
||||
|
||||
|
||||
class FakeProfileJob(FakeJob):
|
||||
job_type = "PROFILE"
|
||||
local_attrs = ("output_location", "configuration", "validation_configurations")
|
||||
|
||||
|
||||
class FakeRecipeJob(FakeJob):
|
||||
local_attrs = (
|
||||
"database_outputs",
|
||||
"data_catalog_outputs",
|
||||
"outputs",
|
||||
"project_name",
|
||||
"recipe_reference",
|
||||
)
|
||||
job_type = "RECIPE"
|
||||
|
||||
|
||||
databrew_backends = BackendDict(DataBrewBackend, "databrew")
|
||||
|
||||
@ -117,7 +117,7 @@ class DataBrewResponse(BaseResponse):
|
||||
recipe_version = self._get_param(
|
||||
"RecipeVersion", self._get_param("recipeVersion")
|
||||
)
|
||||
recipe = self.databrew_backend.get_recipe(
|
||||
recipe = self.databrew_backend.describe_recipe(
|
||||
recipe_name, recipe_version=recipe_version
|
||||
)
|
||||
return 200, {}, json.dumps(recipe.as_dict())
|
||||
@ -167,12 +167,12 @@ class DataBrewResponse(BaseResponse):
|
||||
return 200, {}, json.dumps(ruleset.as_dict())
|
||||
|
||||
def get_ruleset_response(self, ruleset_name):
|
||||
ruleset = self.databrew_backend.get_ruleset(ruleset_name)
|
||||
return 201, {}, json.dumps(ruleset.as_dict())
|
||||
ruleset = self.databrew_backend.describe_ruleset(ruleset_name)
|
||||
return 200, {}, json.dumps(ruleset.as_dict())
|
||||
|
||||
def delete_ruleset_response(self, ruleset_name):
|
||||
self.databrew_backend.delete_ruleset(ruleset_name)
|
||||
return 204, {}, ""
|
||||
return 200, {}, json.dumps({"Name": ruleset_name})
|
||||
|
||||
@amzn_request_id
|
||||
def ruleset_response(self, request, full_url, headers):
|
||||
@ -298,3 +298,150 @@ class DataBrewResponse(BaseResponse):
|
||||
return self.update_dataset(dataset_name)
|
||||
|
||||
# endregion
|
||||
|
||||
# region Jobs
|
||||
@amzn_request_id
|
||||
def list_jobs(self, request, full_url, headers):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_ListJobs.html
|
||||
self.setup_class(request, full_url, headers)
|
||||
dataset_name = self._get_param("datasetName")
|
||||
project_name = self._get_param("projectName")
|
||||
next_token = self._get_param("NextToken", self._get_param("nextToken"))
|
||||
max_results = self._get_int_param(
|
||||
"MaxResults", self._get_int_param("maxResults")
|
||||
)
|
||||
|
||||
# pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking
|
||||
job_list, next_token = self.databrew_backend.list_jobs(
|
||||
dataset_name=dataset_name,
|
||||
project_name=project_name,
|
||||
next_token=next_token,
|
||||
max_results=max_results,
|
||||
)
|
||||
return json.dumps(
|
||||
{
|
||||
"Jobs": [job.as_dict() for job in job_list],
|
||||
"NextToken": next_token,
|
||||
}
|
||||
)
|
||||
|
||||
def get_job_response(self, job_name):
|
||||
job = self.databrew_backend.describe_job(job_name)
|
||||
return 200, {}, json.dumps(job.as_dict())
|
||||
|
||||
def delete_job_response(self, job_name):
|
||||
self.databrew_backend.delete_job(job_name)
|
||||
return 200, {}, json.dumps({"Name": job_name})
|
||||
|
||||
@amzn_request_id
|
||||
def job_response(self, request, full_url, headers):
|
||||
self.setup_class(request, full_url, headers)
|
||||
parsed_url = urlparse(full_url)
|
||||
|
||||
job_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1]
|
||||
|
||||
if request.method == "GET":
|
||||
return self.get_job_response(job_name)
|
||||
elif request.method == "DELETE":
|
||||
return self.delete_job_response(job_name)
|
||||
|
||||
@amzn_request_id
|
||||
def create_profile_job(self):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_CreateProfileJob.html
|
||||
kwargs = {
|
||||
"dataset_name": self._get_param("DatasetName"),
|
||||
"name": self._get_param("Name"),
|
||||
"output_location": self._get_param("OutputLocation"),
|
||||
"role_arn": self._get_param("RoleArn"),
|
||||
"configuration": self._get_param("Configuration"),
|
||||
"encryption_key_arn": self._get_param("EncryptionKeyArn"),
|
||||
"encryption_mode": self._get_param("EncryptionMode"),
|
||||
"job_sample": self._get_param("JobSample"),
|
||||
"log_subscription": self._get_param("LogSubscription"),
|
||||
"max_capacity": self._get_int_param("MaxCapacity"),
|
||||
"max_retries": self._get_int_param("MaxRetries"),
|
||||
"tags": self._get_param("Tags"),
|
||||
"timeout": self._get_int_param("Timeout"),
|
||||
"validation_configurations": self._get_param("ValidationConfigurations"),
|
||||
}
|
||||
return json.dumps(self.databrew_backend.create_profile_job(**kwargs).as_dict())
|
||||
|
||||
def update_profile_job_response(self, name):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateProfileJob.html
|
||||
kwargs = {
|
||||
"name": name,
|
||||
"output_location": self._get_param("OutputLocation"),
|
||||
"role_arn": self._get_param("RoleArn"),
|
||||
"configuration": self._get_param("Configuration"),
|
||||
"encryption_key_arn": self._get_param("EncryptionKeyArn"),
|
||||
"encryption_mode": self._get_param("EncryptionMode"),
|
||||
"job_sample": self._get_param("JobSample"),
|
||||
"log_subscription": self._get_param("LogSubscription"),
|
||||
"max_capacity": self._get_int_param("MaxCapacity"),
|
||||
"max_retries": self._get_int_param("MaxRetries"),
|
||||
"timeout": self._get_int_param("Timeout"),
|
||||
"validation_configurations": self._get_param("ValidationConfigurations"),
|
||||
}
|
||||
return json.dumps(self.databrew_backend.update_profile_job(**kwargs).as_dict())
|
||||
|
||||
@amzn_request_id
|
||||
def create_recipe_job(self):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipeJob.html
|
||||
kwargs = {
|
||||
"name": self._get_param("Name"),
|
||||
"role_arn": self._get_param("RoleArn"),
|
||||
"database_outputs": self._get_param("DatabaseOutputs"),
|
||||
"data_catalog_outputs": self._get_param("DataCatalogOutputs"),
|
||||
"dataset_name": self._get_param("DatasetName"),
|
||||
"encryption_key_arn": self._get_param("EncryptionKeyArn"),
|
||||
"encryption_mode": self._get_param("EncryptionMode"),
|
||||
"log_subscription": self._get_param("LogSubscription"),
|
||||
"max_capacity": self._get_int_param("MaxCapacity"),
|
||||
"max_retries": self._get_int_param("MaxRetries"),
|
||||
"outputs": self._get_param("Outputs"),
|
||||
"project_name": self._get_param("ProjectName"),
|
||||
"recipe_reference": self._get_param("RecipeReference"),
|
||||
"tags": self._get_param("Tags"),
|
||||
"timeout": self._get_int_param("Timeout"),
|
||||
}
|
||||
return json.dumps(self.databrew_backend.create_recipe_job(**kwargs).as_dict())
|
||||
|
||||
@amzn_request_id
|
||||
def update_recipe_job_response(self, name):
|
||||
# https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateRecipeJob.html
|
||||
kwargs = {
|
||||
"name": name,
|
||||
"role_arn": self._get_param("RoleArn"),
|
||||
"database_outputs": self._get_param("DatabaseOutputs"),
|
||||
"data_catalog_outputs": self._get_param("DataCatalogOutputs"),
|
||||
"encryption_key_arn": self._get_param("EncryptionKeyArn"),
|
||||
"encryption_mode": self._get_param("EncryptionMode"),
|
||||
"log_subscription": self._get_param("LogSubscription"),
|
||||
"max_capacity": self._get_int_param("MaxCapacity"),
|
||||
"max_retries": self._get_int_param("MaxRetries"),
|
||||
"outputs": self._get_param("Outputs"),
|
||||
"timeout": self._get_int_param("Timeout"),
|
||||
}
|
||||
return json.dumps(self.databrew_backend.update_recipe_job(**kwargs).as_dict())
|
||||
|
||||
@amzn_request_id
|
||||
def profile_job_response(self, request, full_url, headers):
|
||||
self.setup_class(request, full_url, headers)
|
||||
parsed_url = urlparse(full_url)
|
||||
|
||||
job_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1]
|
||||
|
||||
if request.method == "PUT":
|
||||
return self.update_profile_job_response(job_name)
|
||||
|
||||
@amzn_request_id
|
||||
def recipe_job_response(self, request, full_url, headers):
|
||||
self.setup_class(request, full_url, headers)
|
||||
parsed_url = urlparse(full_url)
|
||||
|
||||
job_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1]
|
||||
|
||||
if request.method == "PUT":
|
||||
return self.update_recipe_job_response(job_name)
|
||||
|
||||
# endregion
|
||||
|
||||
@ -12,4 +12,10 @@ url_paths = {
|
||||
"{0}/rulesets/(?P<ruleset_name>[^/]+)$": DataBrewResponse().ruleset_response,
|
||||
"{0}/datasets$": DataBrewResponse.dispatch,
|
||||
"{0}/datasets/(?P<dataset_name>[^/]+)$": DataBrewResponse().dataset_response,
|
||||
"{0}/jobs$": DataBrewResponse().list_jobs,
|
||||
"{0}/jobs/(?P<job_name>[^/]+)$": DataBrewResponse().job_response,
|
||||
"{0}/profileJobs$": DataBrewResponse.dispatch,
|
||||
"{0}/recipeJobs$": DataBrewResponse.dispatch,
|
||||
"{0}/profileJobs/(?P<job_name>[^/]+)$": DataBrewResponse().profile_job_response,
|
||||
"{0}/recipeJobs/(?P<job_name>[^/]+)$": DataBrewResponse().recipe_job_response,
|
||||
}
|
||||
|
||||
395
tests/test_databrew/test_databrew_jobs.py
Normal file
395
tests/test_databrew/test_databrew_jobs.py
Normal file
@ -0,0 +1,395 @@
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
import pytest
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from moto import mock_databrew
|
||||
from moto.core import ACCOUNT_ID
|
||||
|
||||
|
||||
def _create_databrew_client():
|
||||
client = boto3.client("databrew", region_name="us-west-1")
|
||||
return client
|
||||
|
||||
|
||||
def _create_test_profile_job(
|
||||
client,
|
||||
dataset_name=None,
|
||||
job_name=None,
|
||||
output_location=None,
|
||||
role_arn=None,
|
||||
tags=None,
|
||||
):
|
||||
kwargs = {}
|
||||
kwargs["Name"] = job_name or str(uuid.uuid4())
|
||||
kwargs["RoleArn"] = role_arn or str(uuid.uuid4())
|
||||
kwargs["DatasetName"] = dataset_name or str(uuid.uuid4())
|
||||
kwargs["OutputLocation"] = output_location or {"Bucket": str(uuid.uuid4())}
|
||||
if tags is not None:
|
||||
kwargs["Tags"] = tags
|
||||
|
||||
return client.create_profile_job(**kwargs)
|
||||
|
||||
|
||||
def _create_test_recipe_job(
|
||||
client,
|
||||
job_name=None,
|
||||
role_arn=None,
|
||||
tags=None,
|
||||
encryption_mode=None,
|
||||
log_subscription=None,
|
||||
dataset_name=None,
|
||||
project_name=None,
|
||||
):
|
||||
kwargs = {}
|
||||
kwargs["Name"] = job_name or str(uuid.uuid4())
|
||||
kwargs["RoleArn"] = role_arn or str(uuid.uuid4())
|
||||
if tags is not None:
|
||||
kwargs["Tags"] = tags
|
||||
if encryption_mode is not None:
|
||||
kwargs["EncryptionMode"] = encryption_mode
|
||||
if log_subscription is not None:
|
||||
kwargs["LogSubscription"] = log_subscription
|
||||
if dataset_name is not None:
|
||||
kwargs["DatasetName"] = dataset_name
|
||||
if project_name is not None:
|
||||
kwargs["ProjectName"] = project_name
|
||||
|
||||
return client.create_recipe_job(**kwargs)
|
||||
|
||||
|
||||
def _create_test_recipe_jobs(client, count, **kwargs):
|
||||
for _ in range(count):
|
||||
_create_test_recipe_job(client, **kwargs)
|
||||
|
||||
|
||||
def _create_test_profile_jobs(client, count, **kwargs):
|
||||
for _ in range(count):
|
||||
_create_test_profile_job(client, **kwargs)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_create_profile_job_that_already_exists():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = _create_test_profile_job(client)
|
||||
job_name = response["Name"]
|
||||
with pytest.raises(ClientError) as exc:
|
||||
_create_test_profile_job(client, job_name=response["Name"])
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ConflictException")
|
||||
err["Message"].should.equal(f"The job {job_name} profile job already exists.")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_create_recipe_job_that_already_exists():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = _create_test_recipe_job(client)
|
||||
job_name = response["Name"]
|
||||
with pytest.raises(ClientError) as exc:
|
||||
_create_test_recipe_job(client, job_name=response["Name"])
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ConflictException")
|
||||
err["Message"].should.equal(f"The job {job_name} recipe job already exists.")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_create_recipe_job_with_invalid_encryption_mode():
|
||||
client = _create_databrew_client()
|
||||
|
||||
with pytest.raises(ClientError) as exc:
|
||||
_create_test_recipe_job(client, encryption_mode="INVALID")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
"1 validation error detected: Value 'INVALID' at 'encryptionMode' failed to satisfy constraint: "
|
||||
"Member must satisfy enum value set: [SSE-S3, SSE-KMS]"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_create_recipe_job_with_invalid_log_subscription_value():
|
||||
client = _create_databrew_client()
|
||||
|
||||
with pytest.raises(ClientError) as exc:
|
||||
_create_test_recipe_job(client, log_subscription="INVALID")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
"1 validation error detected: Value 'INVALID' at 'logSubscription' failed to satisfy constraint: "
|
||||
"Member must satisfy enum value set: [ENABLE, DISABLE]"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_create_recipe_job_with_same_name_as_profile_job():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = _create_test_profile_job(client)
|
||||
job_name = response["Name"]
|
||||
with pytest.raises(ClientError) as exc:
|
||||
_create_test_recipe_job(client, job_name=response["Name"])
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ConflictException")
|
||||
err["Message"].should.equal(f"The job {job_name} profile job already exists.")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_recipe_job():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = _create_test_recipe_job(client)
|
||||
job_name = response["Name"]
|
||||
job = client.describe_job(Name=job_name)
|
||||
job.should.have.key("Name").equal(response["Name"])
|
||||
job.should.have.key("Type").equal("RECIPE")
|
||||
job.should.have.key("ResourceArn").equal(
|
||||
f"arn:aws:databrew:us-west-1:{ACCOUNT_ID}:job/{job_name}"
|
||||
)
|
||||
job["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_job_that_does_not_exist():
|
||||
client = _create_databrew_client()
|
||||
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_job(Name="DoesNotExist")
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal("Job DoesNotExist wasn't found.")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_describe_job_with_long_name():
|
||||
client = _create_databrew_client()
|
||||
name = "a" * 241
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_job(Name=name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
|
||||
f"Member must have length less than or equal to 240"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_update_profile_job():
|
||||
client = _create_databrew_client()
|
||||
|
||||
# Create the job
|
||||
response = _create_test_profile_job(client)
|
||||
job_name = response["Name"]
|
||||
|
||||
# Update the job by changing RoleArn
|
||||
update_response = client.update_profile_job(
|
||||
Name=job_name, RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20}
|
||||
)
|
||||
update_response.should.have.key("Name").equals(job_name)
|
||||
update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
# Describe the job to check that RoleArn was updated
|
||||
job = client.describe_job(Name=job_name)
|
||||
job.should.have.key("Name").equal(response["Name"])
|
||||
job.should.have.key("RoleArn").equal("a" * 20)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_update_recipe_job():
|
||||
client = _create_databrew_client()
|
||||
|
||||
# Create the job
|
||||
response = _create_test_recipe_job(client)
|
||||
job_name = response["Name"]
|
||||
|
||||
# Update the job by changing RoleArn
|
||||
update_response = client.update_recipe_job(Name=job_name, RoleArn="a" * 20)
|
||||
update_response.should.have.key("Name").equals(job_name)
|
||||
update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
# Describe the job to check that RoleArn was updated
|
||||
job = client.describe_job(Name=job_name)
|
||||
job.should.have.key("Name").equal(response["Name"])
|
||||
job.should.have.key("RoleArn").equal("a" * 20)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_update_profile_job_does_not_exist():
|
||||
client = _create_databrew_client()
|
||||
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.update_profile_job(
|
||||
Name="DoesNotExist", RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20}
|
||||
)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal("The job DoesNotExist wasn't found")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_update_recipe_job_does_not_exist():
|
||||
client = _create_databrew_client()
|
||||
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.update_recipe_job(Name="DoesNotExist", RoleArn="a" * 20)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal("The job DoesNotExist wasn't found")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_job():
|
||||
client = _create_databrew_client()
|
||||
|
||||
# Create the job
|
||||
response = _create_test_recipe_job(client)
|
||||
job_name = response["Name"]
|
||||
|
||||
# Delete the job
|
||||
response = client.delete_job(Name=job_name)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
response["Name"].should.equal(job_name)
|
||||
|
||||
# Check the job does not exist anymore
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_job(Name=job_name)
|
||||
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal(f"Job {job_name} wasn't found.")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_job_does_not_exist():
|
||||
client = _create_databrew_client()
|
||||
|
||||
# Delete the job
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_job(Name="DoesNotExist")
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
||||
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ResourceNotFoundException")
|
||||
err["Message"].should.equal("The job DoesNotExist wasn't found.")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_delete_job_with_long_name():
|
||||
client = _create_databrew_client()
|
||||
name = "a" * 241
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_job(Name=name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("ValidationException")
|
||||
err["Message"].should.equal(
|
||||
f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
|
||||
f"Member must have length less than or equal to 240"
|
||||
)
|
||||
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_job_list_when_empty():
|
||||
client = _create_databrew_client()
|
||||
|
||||
response = client.list_jobs()
|
||||
response.should.have.key("Jobs")
|
||||
response["Jobs"].should.have.length_of(0)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_with_max_results():
|
||||
client = _create_databrew_client()
|
||||
|
||||
_create_test_recipe_jobs(client, 4)
|
||||
response = client.list_jobs(MaxResults=2)
|
||||
response["Jobs"].should.have.length_of(2)
|
||||
response.should.have.key("NextToken")
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_from_next_token():
|
||||
client = _create_databrew_client()
|
||||
_create_test_recipe_jobs(client, 10)
|
||||
first_response = client.list_jobs(MaxResults=3)
|
||||
response = client.list_jobs(NextToken=first_response["NextToken"])
|
||||
response["Jobs"].should.have.length_of(7)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_with_max_results_greater_than_actual_results():
|
||||
client = _create_databrew_client()
|
||||
_create_test_recipe_jobs(client, 4)
|
||||
response = client.list_jobs(MaxResults=10)
|
||||
response["Jobs"].should.have.length_of(4)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_recipe_and_profile():
|
||||
client = _create_databrew_client()
|
||||
|
||||
_create_test_recipe_jobs(client, 4)
|
||||
_create_test_profile_jobs(client, 2)
|
||||
response = client.list_jobs()
|
||||
response["Jobs"].should.have.length_of(6)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_dataset_name_filter():
|
||||
client = _create_databrew_client()
|
||||
|
||||
_create_test_recipe_jobs(client, 3, dataset_name="TEST")
|
||||
_create_test_recipe_jobs(client, 1)
|
||||
_create_test_profile_jobs(client, 4, dataset_name="TEST")
|
||||
_create_test_profile_jobs(client, 1)
|
||||
|
||||
response = client.list_jobs(DatasetName="TEST")
|
||||
response["Jobs"].should.have.length_of(7)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_project_name_filter():
|
||||
client = _create_databrew_client()
|
||||
|
||||
_create_test_recipe_jobs(client, 3, project_name="TEST_PROJECT")
|
||||
_create_test_recipe_jobs(client, 1)
|
||||
_create_test_profile_jobs(client, 1)
|
||||
|
||||
response = client.list_jobs(ProjectName="TEST_PROJECT")
|
||||
response["Jobs"].should.have.length_of(3)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
def test_list_jobs_dataset_name_and_project_name_filter():
|
||||
client = _create_databrew_client()
|
||||
|
||||
_create_test_recipe_jobs(client, 1, dataset_name="TEST")
|
||||
_create_test_recipe_jobs(client, 1, project_name="TEST_PROJECT")
|
||||
_create_test_recipe_jobs(
|
||||
client, 10, dataset_name="TEST", project_name="TEST_PROJECT"
|
||||
)
|
||||
_create_test_recipe_jobs(client, 1)
|
||||
_create_test_profile_jobs(client, 1)
|
||||
|
||||
response = client.list_jobs(DatasetName="TEST", ProjectName="TEST_PROJECT")
|
||||
response["Jobs"].should.have.length_of(10)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
@ -86,6 +86,7 @@ def test_describe_ruleset():
|
||||
|
||||
ruleset["Name"].should.equal(response["Name"])
|
||||
ruleset["Rules"].should.have.length_of(1)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
|
||||
|
||||
@mock_databrew
|
||||
@ -116,28 +117,31 @@ def test_create_ruleset_that_already_exists():
|
||||
def test_delete_ruleset():
|
||||
client = _create_databrew_client()
|
||||
response = _create_test_ruleset(client)
|
||||
ruleset_name = response["Name"]
|
||||
|
||||
# Check ruleset exists
|
||||
ruleset = client.describe_ruleset(Name=response["Name"])
|
||||
ruleset = client.describe_ruleset(Name=ruleset_name)
|
||||
ruleset["Name"].should.equal(response["Name"])
|
||||
|
||||
# Delete the ruleset
|
||||
client.delete_ruleset(Name=response["Name"])
|
||||
response = client.delete_ruleset(Name=ruleset_name)
|
||||
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
||||
response["Name"].should.equal(ruleset_name)
|
||||
|
||||
# Check it does not exist anymore
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.describe_ruleset(Name=response["Name"])
|
||||
client.describe_ruleset(Name=ruleset_name)
|
||||
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("EntityNotFoundException")
|
||||
err["Message"].should.equal(f"Ruleset {response['Name']} not found.")
|
||||
err["Message"].should.equal(f"Ruleset {ruleset_name} not found.")
|
||||
|
||||
# Check that a ruleset that does not exist errors
|
||||
with pytest.raises(ClientError) as exc:
|
||||
client.delete_ruleset(Name=response["Name"])
|
||||
client.delete_ruleset(Name=ruleset_name)
|
||||
err = exc.value.response["Error"]
|
||||
err["Code"].should.equal("EntityNotFoundException")
|
||||
err["Message"].should.equal(f"Ruleset {response['Name']} not found.")
|
||||
err["Message"].should.equal(f"Ruleset {ruleset_name} not found.")
|
||||
|
||||
|
||||
@mock_databrew
|
||||
|
||||
Loading…
Reference in New Issue
Block a user