396 lines
13 KiB
Python
396 lines
13 KiB
Python
import uuid
|
|
|
|
import boto3
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
|
|
from moto import mock_databrew
|
|
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
|
|
|
|
|
|
def _create_databrew_client():
|
|
client = boto3.client("databrew", region_name="us-west-1")
|
|
return client
|
|
|
|
|
|
def _create_test_profile_job(
|
|
client,
|
|
dataset_name=None,
|
|
job_name=None,
|
|
output_location=None,
|
|
role_arn=None,
|
|
tags=None,
|
|
):
|
|
kwargs = {}
|
|
kwargs["Name"] = job_name or str(uuid.uuid4())
|
|
kwargs["RoleArn"] = role_arn or str(uuid.uuid4())
|
|
kwargs["DatasetName"] = dataset_name or str(uuid.uuid4())
|
|
kwargs["OutputLocation"] = output_location or {"Bucket": str(uuid.uuid4())}
|
|
if tags is not None:
|
|
kwargs["Tags"] = tags
|
|
|
|
return client.create_profile_job(**kwargs)
|
|
|
|
|
|
def _create_test_recipe_job(
|
|
client,
|
|
job_name=None,
|
|
role_arn=None,
|
|
tags=None,
|
|
encryption_mode=None,
|
|
log_subscription=None,
|
|
dataset_name=None,
|
|
project_name=None,
|
|
):
|
|
kwargs = {}
|
|
kwargs["Name"] = job_name or str(uuid.uuid4())
|
|
kwargs["RoleArn"] = role_arn or str(uuid.uuid4())
|
|
if tags is not None:
|
|
kwargs["Tags"] = tags
|
|
if encryption_mode is not None:
|
|
kwargs["EncryptionMode"] = encryption_mode
|
|
if log_subscription is not None:
|
|
kwargs["LogSubscription"] = log_subscription
|
|
if dataset_name is not None:
|
|
kwargs["DatasetName"] = dataset_name
|
|
if project_name is not None:
|
|
kwargs["ProjectName"] = project_name
|
|
|
|
return client.create_recipe_job(**kwargs)
|
|
|
|
|
|
def _create_test_recipe_jobs(client, count, **kwargs):
|
|
for _ in range(count):
|
|
_create_test_recipe_job(client, **kwargs)
|
|
|
|
|
|
def _create_test_profile_jobs(client, count, **kwargs):
|
|
for _ in range(count):
|
|
_create_test_profile_job(client, **kwargs)
|
|
|
|
|
|
@mock_databrew
|
|
def test_create_profile_job_that_already_exists():
|
|
client = _create_databrew_client()
|
|
|
|
response = _create_test_profile_job(client)
|
|
job_name = response["Name"]
|
|
with pytest.raises(ClientError) as exc:
|
|
_create_test_profile_job(client, job_name=response["Name"])
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ConflictException")
|
|
err["Message"].should.equal(f"The job {job_name} profile job already exists.")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
|
|
|
|
|
@mock_databrew
|
|
def test_create_recipe_job_that_already_exists():
|
|
client = _create_databrew_client()
|
|
|
|
response = _create_test_recipe_job(client)
|
|
job_name = response["Name"]
|
|
with pytest.raises(ClientError) as exc:
|
|
_create_test_recipe_job(client, job_name=response["Name"])
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ConflictException")
|
|
err["Message"].should.equal(f"The job {job_name} recipe job already exists.")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
|
|
|
|
|
@mock_databrew
|
|
def test_create_recipe_job_with_invalid_encryption_mode():
|
|
client = _create_databrew_client()
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
_create_test_recipe_job(client, encryption_mode="INVALID")
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ValidationException")
|
|
err["Message"].should.equal(
|
|
"1 validation error detected: Value 'INVALID' at 'encryptionMode' failed to satisfy constraint: "
|
|
"Member must satisfy enum value set: [SSE-S3, SSE-KMS]"
|
|
)
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
|
|
|
|
|
@mock_databrew
|
|
def test_create_recipe_job_with_invalid_log_subscription_value():
|
|
client = _create_databrew_client()
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
_create_test_recipe_job(client, log_subscription="INVALID")
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ValidationException")
|
|
err["Message"].should.equal(
|
|
"1 validation error detected: Value 'INVALID' at 'logSubscription' failed to satisfy constraint: "
|
|
"Member must satisfy enum value set: [ENABLE, DISABLE]"
|
|
)
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
|
|
|
|
|
@mock_databrew
|
|
def test_create_recipe_job_with_same_name_as_profile_job():
|
|
client = _create_databrew_client()
|
|
|
|
response = _create_test_profile_job(client)
|
|
job_name = response["Name"]
|
|
with pytest.raises(ClientError) as exc:
|
|
_create_test_recipe_job(client, job_name=response["Name"])
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ConflictException")
|
|
err["Message"].should.equal(f"The job {job_name} profile job already exists.")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)
|
|
|
|
|
|
@mock_databrew
|
|
def test_describe_recipe_job():
|
|
client = _create_databrew_client()
|
|
|
|
response = _create_test_recipe_job(client)
|
|
job_name = response["Name"]
|
|
job = client.describe_job(Name=job_name)
|
|
job.should.have.key("Name").equal(response["Name"])
|
|
job.should.have.key("Type").equal("RECIPE")
|
|
job.should.have.key("ResourceArn").equal(
|
|
f"arn:aws:databrew:us-west-1:{ACCOUNT_ID}:job/{job_name}"
|
|
)
|
|
job["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_describe_job_that_does_not_exist():
|
|
client = _create_databrew_client()
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.describe_job(Name="DoesNotExist")
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ResourceNotFoundException")
|
|
err["Message"].should.equal("Job DoesNotExist wasn't found.")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
|
|
|
|
|
@mock_databrew
|
|
def test_describe_job_with_long_name():
|
|
client = _create_databrew_client()
|
|
name = "a" * 241
|
|
with pytest.raises(ClientError) as exc:
|
|
client.describe_job(Name=name)
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ValidationException")
|
|
err["Message"].should.equal(
|
|
f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
|
|
f"Member must have length less than or equal to 240"
|
|
)
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
|
|
|
|
|
@mock_databrew
|
|
def test_update_profile_job():
|
|
client = _create_databrew_client()
|
|
|
|
# Create the job
|
|
response = _create_test_profile_job(client)
|
|
job_name = response["Name"]
|
|
|
|
# Update the job by changing RoleArn
|
|
update_response = client.update_profile_job(
|
|
Name=job_name, RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20}
|
|
)
|
|
update_response.should.have.key("Name").equals(job_name)
|
|
update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
# Describe the job to check that RoleArn was updated
|
|
job = client.describe_job(Name=job_name)
|
|
job.should.have.key("Name").equal(response["Name"])
|
|
job.should.have.key("RoleArn").equal("a" * 20)
|
|
|
|
|
|
@mock_databrew
|
|
def test_update_recipe_job():
|
|
client = _create_databrew_client()
|
|
|
|
# Create the job
|
|
response = _create_test_recipe_job(client)
|
|
job_name = response["Name"]
|
|
|
|
# Update the job by changing RoleArn
|
|
update_response = client.update_recipe_job(Name=job_name, RoleArn="a" * 20)
|
|
update_response.should.have.key("Name").equals(job_name)
|
|
update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
# Describe the job to check that RoleArn was updated
|
|
job = client.describe_job(Name=job_name)
|
|
job.should.have.key("Name").equal(response["Name"])
|
|
job.should.have.key("RoleArn").equal("a" * 20)
|
|
|
|
|
|
@mock_databrew
|
|
def test_update_profile_job_does_not_exist():
|
|
client = _create_databrew_client()
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.update_profile_job(
|
|
Name="DoesNotExist", RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20}
|
|
)
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ResourceNotFoundException")
|
|
err["Message"].should.equal("The job DoesNotExist wasn't found")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
|
|
|
|
|
@mock_databrew
|
|
def test_update_recipe_job_does_not_exist():
|
|
client = _create_databrew_client()
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.update_recipe_job(Name="DoesNotExist", RoleArn="a" * 20)
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ResourceNotFoundException")
|
|
err["Message"].should.equal("The job DoesNotExist wasn't found")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
|
|
|
|
|
@mock_databrew
|
|
def test_delete_job():
|
|
client = _create_databrew_client()
|
|
|
|
# Create the job
|
|
response = _create_test_recipe_job(client)
|
|
job_name = response["Name"]
|
|
|
|
# Delete the job
|
|
response = client.delete_job(Name=job_name)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
response["Name"].should.equal(job_name)
|
|
|
|
# Check the job does not exist anymore
|
|
with pytest.raises(ClientError) as exc:
|
|
client.describe_job(Name=job_name)
|
|
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ResourceNotFoundException")
|
|
err["Message"].should.equal(f"Job {job_name} wasn't found.")
|
|
|
|
|
|
@mock_databrew
|
|
def test_delete_job_does_not_exist():
|
|
client = _create_databrew_client()
|
|
|
|
# Delete the job
|
|
with pytest.raises(ClientError) as exc:
|
|
client.delete_job(Name="DoesNotExist")
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
|
|
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ResourceNotFoundException")
|
|
err["Message"].should.equal("The job DoesNotExist wasn't found.")
|
|
|
|
|
|
@mock_databrew
|
|
def test_delete_job_with_long_name():
|
|
client = _create_databrew_client()
|
|
name = "a" * 241
|
|
with pytest.raises(ClientError) as exc:
|
|
client.delete_job(Name=name)
|
|
err = exc.value.response["Error"]
|
|
err["Code"].should.equal("ValidationException")
|
|
err["Message"].should.equal(
|
|
f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
|
|
f"Member must have length less than or equal to 240"
|
|
)
|
|
exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
|
|
|
|
|
|
@mock_databrew
|
|
def test_job_list_when_empty():
|
|
client = _create_databrew_client()
|
|
|
|
response = client.list_jobs()
|
|
response.should.have.key("Jobs")
|
|
response["Jobs"].should.have.length_of(0)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_with_max_results():
|
|
client = _create_databrew_client()
|
|
|
|
_create_test_recipe_jobs(client, 4)
|
|
response = client.list_jobs(MaxResults=2)
|
|
response["Jobs"].should.have.length_of(2)
|
|
response.should.have.key("NextToken")
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_from_next_token():
|
|
client = _create_databrew_client()
|
|
_create_test_recipe_jobs(client, 10)
|
|
first_response = client.list_jobs(MaxResults=3)
|
|
response = client.list_jobs(NextToken=first_response["NextToken"])
|
|
response["Jobs"].should.have.length_of(7)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_with_max_results_greater_than_actual_results():
|
|
client = _create_databrew_client()
|
|
_create_test_recipe_jobs(client, 4)
|
|
response = client.list_jobs(MaxResults=10)
|
|
response["Jobs"].should.have.length_of(4)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_recipe_and_profile():
|
|
client = _create_databrew_client()
|
|
|
|
_create_test_recipe_jobs(client, 4)
|
|
_create_test_profile_jobs(client, 2)
|
|
response = client.list_jobs()
|
|
response["Jobs"].should.have.length_of(6)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_dataset_name_filter():
|
|
client = _create_databrew_client()
|
|
|
|
_create_test_recipe_jobs(client, 3, dataset_name="TEST")
|
|
_create_test_recipe_jobs(client, 1)
|
|
_create_test_profile_jobs(client, 4, dataset_name="TEST")
|
|
_create_test_profile_jobs(client, 1)
|
|
|
|
response = client.list_jobs(DatasetName="TEST")
|
|
response["Jobs"].should.have.length_of(7)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_project_name_filter():
|
|
client = _create_databrew_client()
|
|
|
|
_create_test_recipe_jobs(client, 3, project_name="TEST_PROJECT")
|
|
_create_test_recipe_jobs(client, 1)
|
|
_create_test_profile_jobs(client, 1)
|
|
|
|
response = client.list_jobs(ProjectName="TEST_PROJECT")
|
|
response["Jobs"].should.have.length_of(3)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|
|
|
|
|
|
@mock_databrew
|
|
def test_list_jobs_dataset_name_and_project_name_filter():
|
|
client = _create_databrew_client()
|
|
|
|
_create_test_recipe_jobs(client, 1, dataset_name="TEST")
|
|
_create_test_recipe_jobs(client, 1, project_name="TEST_PROJECT")
|
|
_create_test_recipe_jobs(
|
|
client, 10, dataset_name="TEST", project_name="TEST_PROJECT"
|
|
)
|
|
_create_test_recipe_jobs(client, 1)
|
|
_create_test_profile_jobs(client, 1)
|
|
|
|
response = client.list_jobs(DatasetName="TEST", ProjectName="TEST_PROJECT")
|
|
response["Jobs"].should.have.length_of(10)
|
|
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
|