moto/tests/test_databrew/test_databrew_recipes.py

import uuid

import boto3
import pytest
from botocore.exceptions import ClientError
from datetime import datetime

from moto import mock_databrew


def _create_databrew_client():
    client = boto3.client("databrew", region_name="us-west-1")
    return client


def _create_test_recipe(client, tags=None, recipe_name=None):
    if recipe_name is None:
        recipe_name = str(uuid.uuid4())

    return client.create_recipe(
        Name=recipe_name,
        Steps=[
            {
                "Action": {
                    "Operation": "REMOVE_COMBINED",
                    "Parameters": {
                        "collapseConsecutiveWhitespace": "false",
                        "removeAllPunctuation": "false",
                        "removeAllQuotes": "false",
                        "removeAllWhitespace": "false",
                        "removeCustomCharacters": "false",
                        "removeCustomValue": "false",
                        "removeLeadingAndTrailingPunctuation": "false",
                        "removeLeadingAndTrailingQuotes": "false",
                        "removeLeadingAndTrailingWhitespace": "false",
                        "removeLetters": "false",
                        "removeNumbers": "false",
                        "removeSpecialCharacters": "true",
                        "sourceColumn": "FakeColumn",
                    },
                }
            }
        ],
        Tags=tags or {},
    )


def _create_test_recipes(client, count):
    for _ in range(count):
        _create_test_recipe(client)


@mock_databrew
def test_recipe_list_when_empty():
    client = _create_databrew_client()

    response = client.list_recipes()
    response.should.have.key("Recipes")
    response["Recipes"].should.have.length_of(0)


@mock_databrew
def test_recipe_list_with_invalid_version():
    client = _create_databrew_client()

    recipe_version = "1.1"
    with pytest.raises(ClientError) as exc:
        client.list_recipes(RecipeVersion=recipe_version)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"Invalid version {recipe_version}. "
        "Valid versions are LATEST_PUBLISHED and LATEST_WORKING."
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")


@mock_databrew
def test_list_recipes_with_max_results():
    client = _create_databrew_client()

    _create_test_recipes(client, 4)
    response = client.list_recipes(MaxResults=2, RecipeVersion="LATEST_WORKING")
    response["Recipes"].should.have.length_of(2)
    response.should.have.key("NextToken")


@mock_databrew
def test_list_recipes_from_next_token():
    client = _create_databrew_client()
    _create_test_recipes(client, 10)
    first_response = client.list_recipes(MaxResults=3, RecipeVersion="LATEST_WORKING")
    response = client.list_recipes(
        NextToken=first_response["NextToken"], RecipeVersion="LATEST_WORKING"
    )
    response["Recipes"].should.have.length_of(7)


@mock_databrew
def test_list_recipes_with_max_results_greater_than_actual_results():
    client = _create_databrew_client()
    _create_test_recipes(client, 4)
    response = client.list_recipes(MaxResults=10, RecipeVersion="LATEST_WORKING")
    response["Recipes"].should.have.length_of(4)


@mock_databrew
def test_list_recipe_versions_no_recipe():
    client = _create_databrew_client()
    recipe_name = "NotExist"
    response = client.list_recipe_versions(Name=recipe_name)
    response["Recipes"].should.have.length_of(0)


@mock_databrew
def test_list_recipe_versions_none_published():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    response = client.list_recipe_versions(Name=recipe_name)
    response["Recipes"].should.have.length_of(0)


@mock_databrew
def test_list_recipe_versions_one_published():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    client.publish_recipe(Name=recipe_name)
    response = client.list_recipe_versions(Name=recipe_name)
    response["Recipes"].should.have.length_of(1)
    response["Recipes"][0]["RecipeVersion"].should.equal("1.0")


@mock_databrew
def test_list_recipe_versions_two_published():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    client.publish_recipe(Name=recipe_name)
    client.publish_recipe(Name=recipe_name)
    response = client.list_recipe_versions(Name=recipe_name)
    response["Recipes"].should.have.length_of(2)
    response["Recipes"][0]["RecipeVersion"].should.equal("1.0")
    response["Recipes"][1]["RecipeVersion"].should.equal("2.0")


@mock_databrew
def test_describe_recipe_latest_working():
    client = _create_databrew_client()
    response = _create_test_recipe(client)

    recipe = client.describe_recipe(
        Name=response["Name"], RecipeVersion="LATEST_WORKING"
    )

    recipe["Name"].should.equal(response["Name"])
    recipe["Steps"].should.have.length_of(1)
    recipe["RecipeVersion"].should.equal("0.1")


@mock_databrew
def test_describe_recipe_with_version():
    client = _create_databrew_client()
    response = _create_test_recipe(client)

    recipe = client.describe_recipe(Name=response["Name"], RecipeVersion="0.1")

    recipe["Name"].should.equal(response["Name"])
    recipe["Steps"].should.have.length_of(1)
    recipe["RecipeVersion"].should.equal("0.1")


@mock_databrew
def test_describe_recipe_latest_published():
    client = _create_databrew_client()
    response = _create_test_recipe(client)

    client.publish_recipe(Name=response["Name"])
    recipe = client.describe_recipe(
        Name=response["Name"], RecipeVersion="LATEST_PUBLISHED"
    )

    recipe["Name"].should.equal(response["Name"])
    recipe["Steps"].should.have.length_of(1)
    recipe["RecipeVersion"].should.equal("1.0")


@mock_databrew
def test_describe_recipe_implicit_latest_published():
    client = _create_databrew_client()
    response = _create_test_recipe(client)

    client.publish_recipe(Name=response["Name"])
    recipe = client.describe_recipe(Name=response["Name"])

    recipe["Name"].should.equal(response["Name"])
    recipe["Steps"].should.have.length_of(1)
    recipe["RecipeVersion"].should.equal("1.0")


@mock_databrew
def test_describe_recipe_that_does_not_exist():
    client = _create_databrew_client()

    with pytest.raises(ClientError) as exc:
        client.describe_recipe(Name="DoseNotExist")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal(
        "The recipe DoseNotExist for version LATEST_PUBLISHED wasn't found."
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_describe_recipe_with_long_name():
    client = _create_databrew_client()
    name = "a" * 256
    with pytest.raises(ClientError) as exc:
        client.describe_recipe(Name=name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
        f"Member must have length less than or equal to 255"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_describe_recipe_with_long_version():
    client = _create_databrew_client()
    version = "1" * 17
    with pytest.raises(ClientError) as exc:
        client.describe_recipe(Name="AnyName", RecipeVersion=version)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"1 validation error detected: Value '{version}' at 'recipeVersion' failed to satisfy constraint: "
        f"Member must have length less than or equal to 16"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_describe_recipe_with_invalid_version():
    client = _create_databrew_client()
    name = "AnyName"
    version = "invalid"
    with pytest.raises(ClientError) as exc:
        client.describe_recipe(Name=name, RecipeVersion=version)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(f"Recipe {name} version {version} isn't valid.")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_update_recipe():
    client = _create_databrew_client()
    response = _create_test_recipe(client)

    recipe = client.update_recipe(
        Name=response["Name"],
        Steps=[
            {
                "Action": {
                    "Operation": "REMOVE_COMBINED",
                    "Parameters": {
                        "collapseConsecutiveWhitespace": "false",
                        "removeAllPunctuation": "false",
                        "removeAllQuotes": "false",
                        "removeAllWhitespace": "false",
                        "removeCustomCharacters": "true",
                        "removeCustomValue": "true",
                        "removeLeadingAndTrailingPunctuation": "false",
                        "removeLeadingAndTrailingQuotes": "false",
                        "removeLeadingAndTrailingWhitespace": "false",
                        "removeLetters": "false",
                        "removeNumbers": "false",
                        "removeSpecialCharacters": "true",
                        "sourceColumn": "FakeColumn",
                    },
                }
            }
        ],
    )

    recipe["Name"].should.equal(response["Name"])

    # Describe the recipe and check the changes
    recipe = client.describe_recipe(
        Name=response["Name"], RecipeVersion="LATEST_WORKING"
    )
    recipe["Name"].should.equal(response["Name"])
    recipe["Steps"].should.have.length_of(1)
    recipe["Steps"][0]["Action"]["Parameters"]["removeCustomValue"].should.equal("true")


@mock_databrew
def test_update_recipe_description():
    client = _create_databrew_client()
    response = _create_test_recipe(client)

    description = "NewDescription"
    recipe = client.update_recipe(
        Name=response["Name"], Steps=[], Description=description
    )

    recipe["Name"].should.equal(response["Name"])

    # Describe the recipe and check the changes
    recipe = client.describe_recipe(
        Name=response["Name"], RecipeVersion="LATEST_WORKING"
    )
    recipe["Name"].should.equal(response["Name"])
    recipe["Description"].should.equal(description)


@mock_databrew
def test_update_recipe_invalid():
    client = _create_databrew_client()

    recipe_name = "NotFound"
    with pytest.raises(ClientError) as exc:
        client.update_recipe(Name=recipe_name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal(f"The recipe {recipe_name} wasn't found")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_create_recipe_that_already_exists():
    client = _create_databrew_client()

    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    with pytest.raises(ClientError) as exc:
        _create_test_recipe(client, recipe_name=response["Name"])
    err = exc.value.response["Error"]
    err["Code"].should.equal("ConflictException")
    err["Message"].should.equal(f"The recipe {recipe_name} already exists")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(409)


@mock_databrew
def test_publish_recipe():
    client = _create_databrew_client()

    response = _create_test_recipe(client)
    recipe_name = response["Name"]

    # Before a recipe is published, we should not be able to retrieve a published version
    with pytest.raises(ClientError) as exc:
        recipe = client.describe_recipe(Name=recipe_name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")

    dt_before_publish = datetime.now().astimezone()

    # Publish the recipe
    publish_response = client.publish_recipe(Name=recipe_name, Description="1st desc")
    publish_response["Name"].should.equal(recipe_name)

    # Recipe is now published, so check we can retrieve the published version
    recipe = client.describe_recipe(Name=recipe_name)
    recipe["Description"].should.equal("1st desc")
    recipe["RecipeVersion"].should.equal("1.0")
    recipe["PublishedDate"].should.be.greater_than(dt_before_publish)
    first_published_date = recipe["PublishedDate"]

    # Publish the recipe a 2nd time
    publish_response = client.publish_recipe(Name=recipe_name, Description="2nd desc")
    publish_response["Name"].should.equal(recipe_name)

    recipe = client.describe_recipe(Name=recipe_name)
    recipe["Description"].should.equal("2nd desc")
    recipe["RecipeVersion"].should.equal("2.0")
    recipe["PublishedDate"].should.be.greater_than(first_published_date)


@mock_databrew
def test_publish_recipe_that_does_not_exist():
    client = _create_databrew_client()
    with pytest.raises(ClientError) as exc:
        client.publish_recipe(Name="DoesNotExist")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_publish_long_recipe_name():
    client = _create_databrew_client()
    name = "a" * 256
    with pytest.raises(ClientError) as exc:
        client.publish_recipe(Name=name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"1 validation error detected: Value '{name}' at 'name' failed to satisfy constraint: "
        f"Member must have length less than or equal to 255"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")


@mock_databrew
def test_delete_recipe_version():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    client.delete_recipe_version(Name=recipe_name, RecipeVersion="LATEST_WORKING")
    with pytest.raises(ClientError) as exc:
        client.describe_recipe(Name=recipe_name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_delete_recipe_version_published():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    client.publish_recipe(Name=recipe_name)
    client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.0")
    with pytest.raises(ClientError) as exc:
        client.describe_recipe(Name=recipe_name)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)
    recipe = client.describe_recipe(Name=recipe_name, RecipeVersion="1.1")
    recipe["RecipeVersion"].should.equal("1.1")


@mock_databrew
def test_delete_recipe_version_latest_working_after_publish():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    client.publish_recipe(Name=recipe_name)
    with pytest.raises(ClientError) as exc:
        client.delete_recipe_version(Name=recipe_name, RecipeVersion="LATEST_WORKING")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        "Recipe version LATEST_WORKING is not allowed to be deleted"
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_delete_recipe_version_latest_working_numeric_after_publish():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    client.publish_recipe(Name=recipe_name)
    with pytest.raises(ClientError) as exc:
        client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.1")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal("Recipe version 1.1 is not allowed to be deleted")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_delete_recipe_version_invalid_version_string():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    recipe_version = "NotValid"
    client.publish_recipe(Name=recipe_name)
    with pytest.raises(ClientError) as exc:
        client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"Recipe {recipe_name} version {recipe_version} is invalid."
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_delete_recipe_version_invalid_version_length():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    recipe_version = "1" * 17
    client.publish_recipe(Name=recipe_name)
    with pytest.raises(ClientError) as exc:
        client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ValidationException")
    err["Message"].should.equal(
        f"Recipe {recipe_name} version {recipe_version} is invalid."
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)


@mock_databrew
def test_delete_recipe_version_unknown_recipe():
    client = _create_databrew_client()
    recipe_name = "Unknown"
    with pytest.raises(ClientError) as exc:
        client.delete_recipe_version(Name=recipe_name, RecipeVersion="1.1")
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal(f"The recipe {recipe_name} wasn't found")
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)


@mock_databrew
def test_delete_recipe_version_unknown_version():
    client = _create_databrew_client()
    response = _create_test_recipe(client)
    recipe_name = response["Name"]
    recipe_version = "1.1"
    with pytest.raises(ClientError) as exc:
        client.delete_recipe_version(Name=recipe_name, RecipeVersion=recipe_version)
    err = exc.value.response["Error"]
    err["Code"].should.equal("ResourceNotFoundException")
    err["Message"].should.equal(
        f"The recipe {recipe_name} version {recipe_version} wasn't found."
    )
    exc.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(404)