diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index b1d02ff9b..29ab81ad2 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -1069,32 +1069,32 @@ ## databrew
-34% implemented +54% implemented - [ ] batch_delete_recipe_version - [X] create_dataset -- [ ] create_profile_job +- [X] create_profile_job - [ ] create_project - [X] create_recipe -- [ ] create_recipe_job +- [X] create_recipe_job - [X] create_ruleset - [ ] create_schedule - [X] delete_dataset -- [ ] delete_job +- [X] delete_job - [ ] delete_project - [X] delete_recipe_version - [X] delete_ruleset - [ ] delete_schedule - [X] describe_dataset -- [ ] describe_job +- [X] describe_job - [ ] describe_job_run - [ ] describe_project -- [ ] describe_recipe -- [ ] describe_ruleset +- [X] describe_recipe +- [X] describe_ruleset - [ ] describe_schedule - [X] list_datasets - [ ] list_job_runs -- [ ] list_jobs +- [X] list_jobs - [ ] list_projects - [X] list_recipe_versions - [X] list_recipes @@ -1109,10 +1109,10 @@ - [ ] tag_resource - [ ] untag_resource - [X] update_dataset -- [ ] update_profile_job +- [X] update_profile_job - [ ] update_project - [X] update_recipe -- [ ] update_recipe_job +- [X] update_recipe_job - [X] update_ruleset - [ ] update_schedule
@@ -6163,4 +6163,4 @@ - workspaces - workspaces-web - xray - \ No newline at end of file + diff --git a/docs/docs/services/databrew.rst b/docs/docs/services/databrew.rst index 8715cae99..7b8dea091 100644 --- a/docs/docs/services/databrew.rst +++ b/docs/docs/services/databrew.rst @@ -27,28 +27,28 @@ databrew - [ ] batch_delete_recipe_version - [X] create_dataset -- [ ] create_profile_job +- [X] create_profile_job - [ ] create_project - [X] create_recipe -- [ ] create_recipe_job +- [X] create_recipe_job - [X] create_ruleset - [ ] create_schedule - [X] delete_dataset -- [ ] delete_job +- [X] delete_job - [ ] delete_project - [X] delete_recipe_version - [X] delete_ruleset - [ ] delete_schedule - [X] describe_dataset -- [ ] describe_job +- [X] describe_job - [ ] describe_job_run - [ ] describe_project -- [ ] describe_recipe -- [ ] describe_ruleset +- [X] describe_recipe +- [X] describe_ruleset - [ ] describe_schedule - [X] list_datasets - [ ] list_job_runs -- [ ] list_jobs +- [X] list_jobs - [ ] list_projects - [X] list_recipe_versions - [X] list_recipes @@ -63,10 +63,10 @@ databrew - [ ] tag_resource - [ ] untag_resource - [X] update_dataset -- [ ] update_profile_job +- [X] update_profile_job - [ ] update_project - [X] update_recipe -- [ ] update_recipe_job +- [X] update_recipe_job - [X] update_ruleset - [ ] update_schedule diff --git a/moto/databrew/models.py b/moto/databrew/models.py index 60976f199..1138cc38a 100644 --- a/moto/databrew/models.py +++ b/moto/databrew/models.py @@ -1,3 +1,6 @@ +from abc import ABCMeta +from abc import abstractmethod + from collections import OrderedDict from copy import deepcopy import math @@ -5,6 +8,9 @@ from datetime import datetime from moto.core import BaseBackend, BaseModel from moto.core.utils import BackendDict +from moto.core.utils import underscores_to_camelcase +from moto.core.utils import camelcase_to_pascal + from moto.utilities.paginator import paginate from .exceptions import ( @@ -43,6 +49,12 @@ class DataBrewBackend(BaseBackend): "limit_default": 100, "unique_attribute": "name", }, + "list_jobs": { + "input_token": "next_token", + "limit_key": "max_results", + "limit_default": 100, + "unique_attribute": "name", + }, } def __init__(self, region_name, account_id): @@ -50,6 +62,7 @@ class DataBrewBackend(BaseBackend): self.recipes = OrderedDict() self.rulesets = OrderedDict() self.datasets = OrderedDict() + self.jobs = OrderedDict() @staticmethod def validate_length(param, param_name, max_length): @@ -145,7 +158,7 @@ class DataBrewBackend(BaseBackend): ] return [r for r in recipe_versions if r is not None] - def get_recipe(self, recipe_name, recipe_version=None): + def describe_recipe(self, recipe_name, recipe_version=None): # https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeRecipe.html self.validate_length(recipe_name, "name", 255) @@ -211,7 +224,7 @@ class DataBrewBackend(BaseBackend): return ruleset - def get_ruleset(self, ruleset_name): + def describe_ruleset(self, ruleset_name): if ruleset_name not in self.rulesets: raise RulesetNotFoundException(ruleset_name) return self.rulesets[ruleset_name] @@ -295,6 +308,99 @@ class DataBrewBackend(BaseBackend): return self.datasets[dataset_name] + def describe_job(self, job_name): + # https://docs.aws.amazon.com/databrew/latest/dg/API_DescribeJob.html + self.validate_length(job_name, "name", 240) + + if job_name not in self.jobs: + raise ResourceNotFoundException(f"Job {job_name} wasn't found.") + + return self.jobs[job_name] + + def delete_job(self, job_name): + # https://docs.aws.amazon.com/databrew/latest/dg/API_DeleteJob.html + self.validate_length(job_name, "name", 240) + + if job_name not in self.jobs: + raise ResourceNotFoundException(f"The job {job_name} wasn't found.") + + del self.jobs[job_name] + + def create_profile_job(self, **kwargs): + # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateProfileJob.html + job_name = kwargs["name"] + self.validate_length(job_name, "name", 240) + + if job_name in self.jobs: + raise ConflictException( + f"The job {job_name} {self.jobs[job_name].job_type.lower()} job already exists." + ) + + job = FakeProfileJob( + account_id=self.account_id, region_name=self.region_name, **kwargs + ) + + self.jobs[job_name] = job + return job + + def create_recipe_job(self, **kwargs): + # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipeJob.html + job_name = kwargs["name"] + self.validate_length(job_name, "name", 240) + + if job_name in self.jobs: + raise ConflictException( + f"The job {job_name} {self.jobs[job_name].job_type.lower()} job already exists." + ) + + job = FakeRecipeJob( + account_id=self.account_id, region_name=self.region_name, **kwargs + ) + + self.jobs[job_name] = job + return job + + def update_job(self, **kwargs): + job_name = kwargs["name"] + self.validate_length(job_name, "name", 240) + + if job_name not in self.jobs: + raise ResourceNotFoundException(f"The job {job_name} wasn't found") + + job = self.jobs[job_name] + + for param, value in kwargs.items(): + setattr(job, param, value) + return job + + def update_recipe_job(self, **kwargs): + # https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateRecipeJob.html + return self.update_job(**kwargs) + + def update_profile_job(self, **kwargs): + # https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateProfileJob.html + return self.update_job(**kwargs) + + @paginate(pagination_model=PAGINATION_MODEL) + def list_jobs(self, dataset_name=None, project_name=None): + # https://docs.aws.amazon.com/databrew/latest/dg/API_ListJobs.html + if dataset_name is not None: + self.validate_length(dataset_name, "datasetName", 255) + if project_name is not None: + self.validate_length(project_name, "projectName", 255) + + def filter_jobs(job): + if dataset_name is not None and job.dataset_name != dataset_name: + return False + if ( + project_name is not None + and getattr(job, "project_name", None) != project_name + ): + return False + return True + + return list(filter(filter_jobs, self.jobs.values())) + class FakeRecipe(BaseModel): INITIAL_VERSION = 0.1 @@ -424,7 +530,7 @@ class FakeRuleset(BaseModel): "Rules": self.rules, "Description": self.description, "TargetArn": self.target_arn, - "CreateTime": self.created_time.isoformat(), + "CreateDate": "%.3f" % self.created_time.timestamp(), "Tags": self.tags or dict(), } @@ -464,10 +570,106 @@ class FakeDataset(BaseModel): "FormatOptions": self.format_options, "Input": self.input, "PathOptions": self.path_options, - "CreateTime": self.created_time.isoformat(), + "CreateDate": "%.3f" % self.created_time.timestamp(), "Tags": self.tags or dict(), "ResourceArn": self.resource_arn, } +class BaseModelABCMeta(ABCMeta, type(BaseModel)): + pass + + +class FakeJob(BaseModel, metaclass=BaseModelABCMeta): + + ENCRYPTION_MODES = ("SSE-S3", "SSE-KMS") + LOG_SUBSCRIPTION_VALUES = ("ENABLE", "DISABLE") + + @property + @abstractmethod + def local_attrs(self) -> tuple: + raise NotImplementedError + + def __init__(self, account_id, region_name, **kwargs): + self.account_id = account_id + self.region_name = region_name + self.name = kwargs.get("name") + self.created_time = datetime.now() + self.dataset_name = kwargs.get("dataset_name") + self.encryption_mode = kwargs.get("encryption_mode") + self.log_subscription = kwargs.get("log_subscription") + self.max_capacity = kwargs.get("max_capacity") + self.max_retries = kwargs.get("max_retries") + self.role_arn = kwargs.get("role_arn") + self.tags = kwargs.get("tags") + self.validate() + # Set attributes specific to subclass + for k in self.local_attrs: + setattr(self, k, kwargs.get(k)) + + def validate(self): + if self.encryption_mode is not None: + if self.encryption_mode not in FakeJob.ENCRYPTION_MODES: + raise ValidationException( + f"1 validation error detected: Value '{self.encryption_mode}' at 'encryptionMode' failed to satisfy constraint: Member must satisfy enum value set: [{', '.join(self.ENCRYPTION_MODES)}]" + ) + if self.log_subscription is not None: + if self.log_subscription not in FakeJob.LOG_SUBSCRIPTION_VALUES: + raise ValidationException( + f"1 validation error detected: Value '{self.log_subscription}' at 'logSubscription' failed to satisfy constraint: Member must satisfy enum value set: [{', '.join(self.LOG_SUBSCRIPTION_VALUES)}]" + ) + + @property + @abstractmethod + def job_type(self) -> str: + pass + + @property + def resource_arn(self): + return f"arn:aws:databrew:{self.region_name}:{self.account_id}:job/{self.name}" + + def as_dict(self): + rtn_dict = { + "Name": self.name, + "AccountId": self.account_id, + "CreateDate": "%.3f" % self.created_time.timestamp(), + "DatasetName": self.dataset_name, + "EncryptionMode": self.encryption_mode, + "Tags": self.tags or dict(), + "LogSubscription": self.log_subscription, + "MaxCapacity": self.max_capacity, + "MaxRetries": self.max_retries, + "ResourceArn": self.resource_arn, + "RoleArn": self.role_arn, + "Type": self.job_type, + } + + # Add in subclass attributes + for k in self.local_attrs: + rtn_dict[camelcase_to_pascal(underscores_to_camelcase(k))] = getattr( + self, k + ) + + # Remove items that have a value of None + rtn_dict = {k: v for k, v in rtn_dict.items() if v is not None} + + return rtn_dict + + +class FakeProfileJob(FakeJob): + job_type = "PROFILE" + local_attrs = ("output_location", "configuration", "validation_configurations") + + +class FakeRecipeJob(FakeJob): + local_attrs = ( + "database_outputs", + "data_catalog_outputs", + "outputs", + "project_name", + "recipe_reference", + ) + job_type = "RECIPE" + + databrew_backends = BackendDict(DataBrewBackend, "databrew") diff --git a/moto/databrew/responses.py b/moto/databrew/responses.py index 422dc74a5..cc5a57885 100644 --- a/moto/databrew/responses.py +++ b/moto/databrew/responses.py @@ -117,7 +117,7 @@ class DataBrewResponse(BaseResponse): recipe_version = self._get_param( "RecipeVersion", self._get_param("recipeVersion") ) - recipe = self.databrew_backend.get_recipe( + recipe = self.databrew_backend.describe_recipe( recipe_name, recipe_version=recipe_version ) return 200, {}, json.dumps(recipe.as_dict()) @@ -167,12 +167,12 @@ class DataBrewResponse(BaseResponse): return 200, {}, json.dumps(ruleset.as_dict()) def get_ruleset_response(self, ruleset_name): - ruleset = self.databrew_backend.get_ruleset(ruleset_name) - return 201, {}, json.dumps(ruleset.as_dict()) + ruleset = self.databrew_backend.describe_ruleset(ruleset_name) + return 200, {}, json.dumps(ruleset.as_dict()) def delete_ruleset_response(self, ruleset_name): self.databrew_backend.delete_ruleset(ruleset_name) - return 204, {}, "" + return 200, {}, json.dumps({"Name": ruleset_name}) @amzn_request_id def ruleset_response(self, request, full_url, headers): @@ -298,3 +298,150 @@ class DataBrewResponse(BaseResponse): return self.update_dataset(dataset_name) # endregion + + # region Jobs + @amzn_request_id + def list_jobs(self, request, full_url, headers): + # https://docs.aws.amazon.com/databrew/latest/dg/API_ListJobs.html + self.setup_class(request, full_url, headers) + dataset_name = self._get_param("datasetName") + project_name = self._get_param("projectName") + next_token = self._get_param("NextToken", self._get_param("nextToken")) + max_results = self._get_int_param( + "MaxResults", self._get_int_param("maxResults") + ) + + # pylint: disable=unexpected-keyword-arg, unbalanced-tuple-unpacking + job_list, next_token = self.databrew_backend.list_jobs( + dataset_name=dataset_name, + project_name=project_name, + next_token=next_token, + max_results=max_results, + ) + return json.dumps( + { + "Jobs": [job.as_dict() for job in job_list], + "NextToken": next_token, + } + ) + + def get_job_response(self, job_name): + job = self.databrew_backend.describe_job(job_name) + return 200, {}, json.dumps(job.as_dict()) + + def delete_job_response(self, job_name): + self.databrew_backend.delete_job(job_name) + return 200, {}, json.dumps({"Name": job_name}) + + @amzn_request_id + def job_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + parsed_url = urlparse(full_url) + + job_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1] + + if request.method == "GET": + return self.get_job_response(job_name) + elif request.method == "DELETE": + return self.delete_job_response(job_name) + + @amzn_request_id + def create_profile_job(self): + # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateProfileJob.html + kwargs = { + "dataset_name": self._get_param("DatasetName"), + "name": self._get_param("Name"), + "output_location": self._get_param("OutputLocation"), + "role_arn": self._get_param("RoleArn"), + "configuration": self._get_param("Configuration"), + "encryption_key_arn": self._get_param("EncryptionKeyArn"), + "encryption_mode": self._get_param("EncryptionMode"), + "job_sample": self._get_param("JobSample"), + "log_subscription": self._get_param("LogSubscription"), + "max_capacity": self._get_int_param("MaxCapacity"), + "max_retries": self._get_int_param("MaxRetries"), + "tags": self._get_param("Tags"), + "timeout": self._get_int_param("Timeout"), + "validation_configurations": self._get_param("ValidationConfigurations"), + } + return json.dumps(self.databrew_backend.create_profile_job(**kwargs).as_dict()) + + def update_profile_job_response(self, name): + # https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateProfileJob.html + kwargs = { + "name": name, + "output_location": self._get_param("OutputLocation"), + "role_arn": self._get_param("RoleArn"), + "configuration": self._get_param("Configuration"), + "encryption_key_arn": self._get_param("EncryptionKeyArn"), + "encryption_mode": self._get_param("EncryptionMode"), + "job_sample": self._get_param("JobSample"), + "log_subscription": self._get_param("LogSubscription"), + "max_capacity": self._get_int_param("MaxCapacity"), + "max_retries": self._get_int_param("MaxRetries"), + "timeout": self._get_int_param("Timeout"), + "validation_configurations": self._get_param("ValidationConfigurations"), + } + return json.dumps(self.databrew_backend.update_profile_job(**kwargs).as_dict()) + + @amzn_request_id + def create_recipe_job(self): + # https://docs.aws.amazon.com/databrew/latest/dg/API_CreateRecipeJob.html + kwargs = { + "name": self._get_param("Name"), + "role_arn": self._get_param("RoleArn"), + "database_outputs": self._get_param("DatabaseOutputs"), + "data_catalog_outputs": self._get_param("DataCatalogOutputs"), + "dataset_name": self._get_param("DatasetName"), + "encryption_key_arn": self._get_param("EncryptionKeyArn"), + "encryption_mode": self._get_param("EncryptionMode"), + "log_subscription": self._get_param("LogSubscription"), + "max_capacity": self._get_int_param("MaxCapacity"), + "max_retries": self._get_int_param("MaxRetries"), + "outputs": self._get_param("Outputs"), + "project_name": self._get_param("ProjectName"), + "recipe_reference": self._get_param("RecipeReference"), + "tags": self._get_param("Tags"), + "timeout": self._get_int_param("Timeout"), + } + return json.dumps(self.databrew_backend.create_recipe_job(**kwargs).as_dict()) + + @amzn_request_id + def update_recipe_job_response(self, name): + # https://docs.aws.amazon.com/databrew/latest/dg/API_UpdateRecipeJob.html + kwargs = { + "name": name, + "role_arn": self._get_param("RoleArn"), + "database_outputs": self._get_param("DatabaseOutputs"), + "data_catalog_outputs": self._get_param("DataCatalogOutputs"), + "encryption_key_arn": self._get_param("EncryptionKeyArn"), + "encryption_mode": self._get_param("EncryptionMode"), + "log_subscription": self._get_param("LogSubscription"), + "max_capacity": self._get_int_param("MaxCapacity"), + "max_retries": self._get_int_param("MaxRetries"), + "outputs": self._get_param("Outputs"), + "timeout": self._get_int_param("Timeout"), + } + return json.dumps(self.databrew_backend.update_recipe_job(**kwargs).as_dict()) + + @amzn_request_id + def profile_job_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + parsed_url = urlparse(full_url) + + job_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1] + + if request.method == "PUT": + return self.update_profile_job_response(job_name) + + @amzn_request_id + def recipe_job_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + parsed_url = urlparse(full_url) + + job_name = parsed_url.path.rstrip("/").rsplit("/", 1)[1] + + if request.method == "PUT": + return self.update_recipe_job_response(job_name) + + # endregion diff --git a/moto/databrew/urls.py b/moto/databrew/urls.py index d8508e55c..85da69108 100644 --- a/moto/databrew/urls.py +++ b/moto/databrew/urls.py @@ -12,4 +12,10 @@ url_paths = { "{0}/rulesets/(?P[^/]+)$": DataBrewResponse().ruleset_response, "{0}/datasets$": DataBrewResponse.dispatch, "{0}/datasets/(?P[^/]+)$": DataBrewResponse().dataset_response, + "{0}/jobs$": DataBrewResponse().list_jobs, + "{0}/jobs/(?P[^/]+)$": DataBrewResponse().job_response, + "{0}/profileJobs$": DataBrewResponse.dispatch, + "{0}/recipeJobs$": DataBrewResponse.dispatch, + "{0}/profileJobs/(?P[^/]+)$": DataBrewResponse().profile_job_response, + "{0}/recipeJobs/(?P[^/]+)$": DataBrewResponse().recipe_job_response, } diff --git a/tests/test_databrew/test_databrew_jobs.py b/tests/test_databrew/test_databrew_jobs.py new file mode 100644 index 000000000..06a5acfed --- /dev/null +++ b/tests/test_databrew/test_databrew_jobs.py @@ -0,0 +1,395 @@ +import uuid + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from moto import mock_databrew +from moto.core import ACCOUNT_ID + + +def _create_databrew_client(): + client = boto3.client("databrew", region_name="us-west-1") + return client + + +def _create_test_profile_job( + client, + dataset_name=None, + job_name=None, + output_location=None, + role_arn=None, + tags=None, +): + kwargs = {} + kwargs["Name"] = job_name or str(uuid.uuid4()) + kwargs["RoleArn"] = role_arn or str(uuid.uuid4()) + kwargs["DatasetName"] = dataset_name or str(uuid.uuid4()) + kwargs["OutputLocation"] = output_location or {"Bucket": str(uuid.uuid4())} + if tags is not None: + kwargs["Tags"] = tags + + return client.create_profile_job(**kwargs) + + +def _create_test_recipe_job( + client, + job_name=None, + role_arn=None, + tags=None, + encryption_mode=None, + log_subscription=None, + dataset_name=None, + project_name=None, +): + kwargs = {} + kwargs["Name"] = job_name or str(uuid.uuid4()) + kwargs["RoleArn"] = role_arn or str(uuid.uuid4()) + if tags is not None: + kwargs["Tags"] = tags + if encryption_mode is not None: + kwargs["EncryptionMode"] = encryption_mode + if log_subscription is not None: + kwargs["LogSubscription"] = log_subscription + if dataset_name is not None: + kwargs["DatasetName"] = dataset_name + if project_name is not None: + kwargs["ProjectName"] = project_name + + return client.create_recipe_job(**kwargs) + + +def _create_test_recipe_jobs(client, count, **kwargs): + for _ in range(count): + _create_test_recipe_job(client, **kwargs) + + +def _create_test_profile_jobs(client, count, **kwargs): + for _ in range(count): + _create_test_profile_job(client, **kwargs) + + +@mock_databrew +def test_create_profile_job_that_already_exists(): + client = _create_databrew_client() + + response = _create_test_profile_job(client) + job_name = response["Name"] + with pytest.raises(ClientError) as exc: + _create_test_profile_job(client, job_name=response["Name"]) + err = exc.value.response["Error"] + err["Code"].should.equal("ConflictException") + err["Message"].should.equal(f"The job {job_name} profile job already exists.") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409) + + +@mock_databrew +def test_create_recipe_job_that_already_exists(): + client = _create_databrew_client() + + response = _create_test_recipe_job(client) + job_name = response["Name"] + with pytest.raises(ClientError) as exc: + _create_test_recipe_job(client, job_name=response["Name"]) + err = exc.value.response["Error"] + err["Code"].should.equal("ConflictException") + err["Message"].should.equal(f"The job {job_name} recipe job already exists.") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409) + + +@mock_databrew +def test_create_recipe_job_with_invalid_encryption_mode(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + _create_test_recipe_job(client, encryption_mode="INVALID") + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + "1 validation error detected: Value 'INVALID' at 'encryptionMode' failed to satisfy constraint: " + "Member must satisfy enum value set: [SSE-S3, SSE-KMS]" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_create_recipe_job_with_invalid_log_subscription_value(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + _create_test_recipe_job(client, log_subscription="INVALID") + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + "1 validation error detected: Value 'INVALID' at 'logSubscription' failed to satisfy constraint: " + "Member must satisfy enum value set: [ENABLE, DISABLE]" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_create_recipe_job_with_same_name_as_profile_job(): + client = _create_databrew_client() + + response = _create_test_profile_job(client) + job_name = response["Name"] + with pytest.raises(ClientError) as exc: + _create_test_recipe_job(client, job_name=response["Name"]) + err = exc.value.response["Error"] + err["Code"].should.equal("ConflictException") + err["Message"].should.equal(f"The job {job_name} profile job already exists.") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409) + + +@mock_databrew +def test_describe_recipe_job(): + client = _create_databrew_client() + + response = _create_test_recipe_job(client) + job_name = response["Name"] + job = client.describe_job(Name=job_name) + job.should.have.key("Name").equal(response["Name"]) + job.should.have.key("Type").equal("RECIPE") + job.should.have.key("ResourceArn").equal( + f"arn:aws:databrew:us-west-1:{ACCOUNT_ID}:job/{job_name}" + ) + job["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_describe_job_that_does_not_exist(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + client.describe_job(Name="DoesNotExist") + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal("Job DoesNotExist wasn't found.") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_describe_job_with_long_name(): + client = _create_databrew_client() + name = "a" * 241 + with pytest.raises(ClientError) as exc: + client.describe_job(Name=name) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: " + f"Member must have length less than or equal to 240" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_update_profile_job(): + client = _create_databrew_client() + + # Create the job + response = _create_test_profile_job(client) + job_name = response["Name"] + + # Update the job by changing RoleArn + update_response = client.update_profile_job( + Name=job_name, RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20} + ) + update_response.should.have.key("Name").equals(job_name) + update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + # Describe the job to check that RoleArn was updated + job = client.describe_job(Name=job_name) + job.should.have.key("Name").equal(response["Name"]) + job.should.have.key("RoleArn").equal("a" * 20) + + +@mock_databrew +def test_update_recipe_job(): + client = _create_databrew_client() + + # Create the job + response = _create_test_recipe_job(client) + job_name = response["Name"] + + # Update the job by changing RoleArn + update_response = client.update_recipe_job(Name=job_name, RoleArn="a" * 20) + update_response.should.have.key("Name").equals(job_name) + update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + # Describe the job to check that RoleArn was updated + job = client.describe_job(Name=job_name) + job.should.have.key("Name").equal(response["Name"]) + job.should.have.key("RoleArn").equal("a" * 20) + + +@mock_databrew +def test_update_profile_job_does_not_exist(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + client.update_profile_job( + Name="DoesNotExist", RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20} + ) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal("The job DoesNotExist wasn't found") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_update_recipe_job_does_not_exist(): + client = _create_databrew_client() + + with pytest.raises(ClientError) as exc: + client.update_recipe_job(Name="DoesNotExist", RoleArn="a" * 20) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal("The job DoesNotExist wasn't found") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + +@mock_databrew +def test_delete_job(): + client = _create_databrew_client() + + # Create the job + response = _create_test_recipe_job(client) + job_name = response["Name"] + + # Delete the job + response = client.delete_job(Name=job_name) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["Name"].should.equal(job_name) + + # Check the job does not exist anymore + with pytest.raises(ClientError) as exc: + client.describe_job(Name=job_name) + + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal(f"Job {job_name} wasn't found.") + + +@mock_databrew +def test_delete_job_does_not_exist(): + client = _create_databrew_client() + + # Delete the job + with pytest.raises(ClientError) as exc: + client.delete_job(Name="DoesNotExist") + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404) + + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNotFoundException") + err["Message"].should.equal("The job DoesNotExist wasn't found.") + + +@mock_databrew +def test_delete_job_with_long_name(): + client = _create_databrew_client() + name = "a" * 241 + with pytest.raises(ClientError) as exc: + client.delete_job(Name=name) + err = exc.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: " + f"Member must have length less than or equal to 240" + ) + exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + +@mock_databrew +def test_job_list_when_empty(): + client = _create_databrew_client() + + response = client.list_jobs() + response.should.have.key("Jobs") + response["Jobs"].should.have.length_of(0) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_with_max_results(): + client = _create_databrew_client() + + _create_test_recipe_jobs(client, 4) + response = client.list_jobs(MaxResults=2) + response["Jobs"].should.have.length_of(2) + response.should.have.key("NextToken") + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_from_next_token(): + client = _create_databrew_client() + _create_test_recipe_jobs(client, 10) + first_response = client.list_jobs(MaxResults=3) + response = client.list_jobs(NextToken=first_response["NextToken"]) + response["Jobs"].should.have.length_of(7) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_with_max_results_greater_than_actual_results(): + client = _create_databrew_client() + _create_test_recipe_jobs(client, 4) + response = client.list_jobs(MaxResults=10) + response["Jobs"].should.have.length_of(4) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_recipe_and_profile(): + client = _create_databrew_client() + + _create_test_recipe_jobs(client, 4) + _create_test_profile_jobs(client, 2) + response = client.list_jobs() + response["Jobs"].should.have.length_of(6) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_dataset_name_filter(): + client = _create_databrew_client() + + _create_test_recipe_jobs(client, 3, dataset_name="TEST") + _create_test_recipe_jobs(client, 1) + _create_test_profile_jobs(client, 4, dataset_name="TEST") + _create_test_profile_jobs(client, 1) + + response = client.list_jobs(DatasetName="TEST") + response["Jobs"].should.have.length_of(7) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_project_name_filter(): + client = _create_databrew_client() + + _create_test_recipe_jobs(client, 3, project_name="TEST_PROJECT") + _create_test_recipe_jobs(client, 1) + _create_test_profile_jobs(client, 1) + + response = client.list_jobs(ProjectName="TEST_PROJECT") + response["Jobs"].should.have.length_of(3) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + + +@mock_databrew +def test_list_jobs_dataset_name_and_project_name_filter(): + client = _create_databrew_client() + + _create_test_recipe_jobs(client, 1, dataset_name="TEST") + _create_test_recipe_jobs(client, 1, project_name="TEST_PROJECT") + _create_test_recipe_jobs( + client, 10, dataset_name="TEST", project_name="TEST_PROJECT" + ) + _create_test_recipe_jobs(client, 1) + _create_test_profile_jobs(client, 1) + + response = client.list_jobs(DatasetName="TEST", ProjectName="TEST_PROJECT") + response["Jobs"].should.have.length_of(10) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) diff --git a/tests/test_databrew/test_databrew_rulesets.py b/tests/test_databrew/test_databrew_rulesets.py index c95de802b..a59a38c01 100644 --- a/tests/test_databrew/test_databrew_rulesets.py +++ b/tests/test_databrew/test_databrew_rulesets.py @@ -86,6 +86,7 @@ def test_describe_ruleset(): ruleset["Name"].should.equal(response["Name"]) ruleset["Rules"].should.have.length_of(1) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) @mock_databrew @@ -116,28 +117,31 @@ def test_create_ruleset_that_already_exists(): def test_delete_ruleset(): client = _create_databrew_client() response = _create_test_ruleset(client) + ruleset_name = response["Name"] # Check ruleset exists - ruleset = client.describe_ruleset(Name=response["Name"]) + ruleset = client.describe_ruleset(Name=ruleset_name) ruleset["Name"].should.equal(response["Name"]) # Delete the ruleset - client.delete_ruleset(Name=response["Name"]) + response = client.delete_ruleset(Name=ruleset_name) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["Name"].should.equal(ruleset_name) # Check it does not exist anymore with pytest.raises(ClientError) as exc: - client.describe_ruleset(Name=response["Name"]) + client.describe_ruleset(Name=ruleset_name) err = exc.value.response["Error"] err["Code"].should.equal("EntityNotFoundException") - err["Message"].should.equal(f"Ruleset {response['Name']} not found.") + err["Message"].should.equal(f"Ruleset {ruleset_name} not found.") # Check that a ruleset that does not exist errors with pytest.raises(ClientError) as exc: - client.delete_ruleset(Name=response["Name"]) + client.delete_ruleset(Name=ruleset_name) err = exc.value.response["Error"] err["Code"].should.equal("EntityNotFoundException") - err["Message"].should.equal(f"Ruleset {response['Name']} not found.") + err["Message"].should.equal(f"Ruleset {ruleset_name} not found.") @mock_databrew