moto/tests/test_databrew/test_databrew_jobs.py

import uuid

import boto3
import pytest
from botocore.exceptions import ClientError

from moto import mock_databrew
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID


def _create_databrew_client():
    client = boto3.client("databrew", region_name="us-west-1")
    return client


def _create_test_profile_job(
    client,
    dataset_name=None,
    job_name=None,
    output_location=None,
    role_arn=None,
    tags=None,
):
    kwargs = {}
    kwargs["Name"] = job_name or str(uuid.uuid4())
    kwargs["RoleArn"] = role_arn or str(uuid.uuid4())
    kwargs["DatasetName"] = dataset_name or str(uuid.uuid4())
    kwargs["OutputLocation"] = output_location or {"Bucket": str(uuid.uuid4())}
    if tags is not None:
        kwargs["Tags"] = tags

    return client.create_profile_job(**kwargs)


def _create_test_recipe_job(
    client,
    job_name=None,
    role_arn=None,
    tags=None,
    encryption_mode=None,
    log_subscription=None,
    dataset_name=None,
    project_name=None,
):
    kwargs = {}
    kwargs["Name"] = job_name or str(uuid.uuid4())
    kwargs["RoleArn"] = role_arn or str(uuid.uuid4())
    if tags is not None:
        kwargs["Tags"] = tags
    if encryption_mode is not None:
        kwargs["EncryptionMode"] = encryption_mode
    if log_subscription is not None:
        kwargs["LogSubscription"] = log_subscription
    if dataset_name is not None:
        kwargs["DatasetName"] = dataset_name
    if project_name is not None:
        kwargs["ProjectName"] = project_name

    return client.create_recipe_job(**kwargs)


def _create_test_recipe_jobs(client, count, **kwargs):
    for _ in range(count):
        _create_test_recipe_job(client, **kwargs)


def _create_test_profile_jobs(client, count, **kwargs):
    for _ in range(count):
        _create_test_profile_job(client, **kwargs)


@mock_databrew
def test_create_profile_job_that_already_exists():
    client = _create_databrew_client()

    response = _create_test_profile_job(client)
    job_name = response["Name"]
    with pytest.raises(ClientError) as exc:
        _create_test_profile_job(client, job_name=response["Name"])
    err = exc.value.response["Error"]
    err["Code"].should.equal("ConflictException")
    err["Message"].should.equal(f"The job {job_name} profile job already exists.")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)


@mock_databrew
def test_create_recipe_job_that_already_exists():
    client = _create_databrew_client()

    response = _create_test_recipe_job(client)
    job_name = response["Name"]
    with pytest.raises(ClientError) as exc:
        _create_test_recipe_job(client, job_name=response["Name"])
    err = exc.value.response["Error"]
    err["Code"].should.equal("ConflictException")
    err["Message"].should.equal(f"The job {job_name} recipe job already exists.")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)


@mock_databrew
def test_create_recipe_job_with_invalid_encryption_mode():
    client = _create_databrew_client()

    with pytest.raises(ClientError) as exc:
        _create_test_recipe_job(client, encryption_mode="INVALID")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        "1 validation error detected: Value 'INVALID' at 'encryptionMode' failed to satisfy constraint: "
        "Member must satisfy enum value set: [SSE-S3, SSE-KMS]"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_create_recipe_job_with_invalid_log_subscription_value():
    client = _create_databrew_client()

    with pytest.raises(ClientError) as exc:
        _create_test_recipe_job(client, log_subscription="INVALID")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        "1 validation error detected: Value 'INVALID' at 'logSubscription' failed to satisfy constraint: "
        "Member must satisfy enum value set: [ENABLE, DISABLE]"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_create_recipe_job_with_same_name_as_profile_job():
    client = _create_databrew_client()

    response = _create_test_profile_job(client)
    job_name = response["Name"]
    with pytest.raises(ClientError) as exc:
        _create_test_recipe_job(client, job_name=response["Name"])
    err = exc.value.response["Error"]
    err["Code"].should.equal("ConflictException")
    err["Message"].should.equal(f"The job {job_name} profile job already exists.")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)


@mock_databrew
def test_describe_recipe_job():
    client = _create_databrew_client()

    response = _create_test_recipe_job(client)
    job_name = response["Name"]
    job = client.describe_job(Name=job_name)
    job.should.have.key("Name").equal(response["Name"])
    job.should.have.key("Type").equal("RECIPE")
    job.should.have.key("ResourceArn").equal(
        f"arn:aws:databrew:us-west-1:{ACCOUNT_ID}:job/{job_name}"
    )
    job["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_describe_job_that_does_not_exist():
    client = _create_databrew_client()

    with pytest.raises(ClientError) as exc:
        client.describe_job(Name="DoesNotExist")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal("Job DoesNotExist wasn't found.")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_describe_job_with_long_name():
    client = _create_databrew_client()
    name = "a" * 241
    with pytest.raises(ClientError) as exc:
        client.describe_job(Name=name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
        f"Member must have length less than or equal to 240"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_update_profile_job():
    client = _create_databrew_client()

    # Create the job
    response = _create_test_profile_job(client)
    job_name = response["Name"]

    # Update the job by changing RoleArn
    update_response = client.update_profile_job(
        Name=job_name, RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20}
    )
    update_response.should.have.key("Name").equals(job_name)
    update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)

    # Describe the job to check that RoleArn was updated
    job = client.describe_job(Name=job_name)
    job.should.have.key("Name").equal(response["Name"])
    job.should.have.key("RoleArn").equal("a" * 20)


@mock_databrew
def test_update_recipe_job():
    client = _create_databrew_client()

    # Create the job
    response = _create_test_recipe_job(client)
    job_name = response["Name"]

    # Update the job by changing RoleArn
    update_response = client.update_recipe_job(Name=job_name, RoleArn="a" * 20)
    update_response.should.have.key("Name").equals(job_name)
    update_response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)

    # Describe the job to check that RoleArn was updated
    job = client.describe_job(Name=job_name)
    job.should.have.key("Name").equal(response["Name"])
    job.should.have.key("RoleArn").equal("a" * 20)


@mock_databrew
def test_update_profile_job_does_not_exist():
    client = _create_databrew_client()

    with pytest.raises(ClientError) as exc:
        client.update_profile_job(
            Name="DoesNotExist", RoleArn="a" * 20, OutputLocation={"Bucket": "b" * 20}
        )
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal("The job DoesNotExist wasn't found")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_update_recipe_job_does_not_exist():
    client = _create_databrew_client()

    with pytest.raises(ClientError) as exc:
        client.update_recipe_job(Name="DoesNotExist", RoleArn="a" * 20)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal("The job DoesNotExist wasn't found")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_delete_job():
    client = _create_databrew_client()

    # Create the job
    response = _create_test_recipe_job(client)
    job_name = response["Name"]

    # Delete the job
    response = client.delete_job(Name=job_name)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
    response["Name"].should.equal(job_name)

    # Check the job does not exist anymore
    with pytest.raises(ClientError) as exc:
        client.describe_job(Name=job_name)

    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal(f"Job {job_name} wasn't found.")


@mock_databrew
def test_delete_job_does_not_exist():
    client = _create_databrew_client()

    # Delete the job
    with pytest.raises(ClientError) as exc:
        client.delete_job(Name="DoesNotExist")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)

    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal("The job DoesNotExist wasn't found.")


@mock_databrew
def test_delete_job_with_long_name():
    client = _create_databrew_client()
    name = "a" * 241
    with pytest.raises(ClientError) as exc:
        client.delete_job(Name=name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
        f"Member must have length less than or equal to 240"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_job_list_when_empty():
    client = _create_databrew_client()

    response = client.list_jobs()
    response.should.have.key("Jobs")
    response["Jobs"].should.have.length_of(0)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_with_max_results():
    client = _create_databrew_client()

    _create_test_recipe_jobs(client, 4)
    response = client.list_jobs(MaxResults=2)
    response["Jobs"].should.have.length_of(2)
    response.should.have.key("NextToken")
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_from_next_token():
    client = _create_databrew_client()
    _create_test_recipe_jobs(client, 10)
    first_response = client.list_jobs(MaxResults=3)
    response = client.list_jobs(NextToken=first_response["NextToken"])
    response["Jobs"].should.have.length_of(7)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_with_max_results_greater_than_actual_results():
    client = _create_databrew_client()
    _create_test_recipe_jobs(client, 4)
    response = client.list_jobs(MaxResults=10)
    response["Jobs"].should.have.length_of(4)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_recipe_and_profile():
    client = _create_databrew_client()

    _create_test_recipe_jobs(client, 4)
    _create_test_profile_jobs(client, 2)
    response = client.list_jobs()
    response["Jobs"].should.have.length_of(6)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_dataset_name_filter():
    client = _create_databrew_client()

    _create_test_recipe_jobs(client, 3, dataset_name="TEST")
    _create_test_recipe_jobs(client, 1)
    _create_test_profile_jobs(client, 4, dataset_name="TEST")
    _create_test_profile_jobs(client, 1)

    response = client.list_jobs(DatasetName="TEST")
    response["Jobs"].should.have.length_of(7)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_project_name_filter():
    client = _create_databrew_client()

    _create_test_recipe_jobs(client, 3, project_name="TEST_PROJECT")
    _create_test_recipe_jobs(client, 1)
    _create_test_profile_jobs(client, 1)

    response = client.list_jobs(ProjectName="TEST_PROJECT")
    response["Jobs"].should.have.length_of(3)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)


@mock_databrew
def test_list_jobs_dataset_name_and_project_name_filter():
    client = _create_databrew_client()

    _create_test_recipe_jobs(client, 1, dataset_name="TEST")
    _create_test_recipe_jobs(client, 1, project_name="TEST_PROJECT")
    _create_test_recipe_jobs(
        client, 10, dataset_name="TEST", project_name="TEST_PROJECT"
    )
    _create_test_recipe_jobs(client, 1)
    _create_test_profile_jobs(client, 1)

    response = client.list_jobs(DatasetName="TEST", ProjectName="TEST_PROJECT")
    response["Jobs"].should.have.length_of(10)
    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)