From 6343d24c929c8179df3753be35d36da39f8f74d5 Mon Sep 17 00:00:00 2001 From: Himani Patel Date: Tue, 21 Jun 2022 13:31:28 -0700 Subject: [PATCH] Implemented Glue Schema Registry CreateRegistry API (#5234) --- moto/glue/exceptions.py | 46 +++++++++++ moto/glue/models.py | 84 +++++++++++++++++++ moto/glue/responses.py | 7 ++ tests/test_glue/test_glue.py | 156 +++++++++++++++++++++++++++++++++++ 4 files changed, 293 insertions(+) diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py index 003734b11..bb54071a3 100644 --- a/moto/glue/exceptions.py +++ b/moto/glue/exceptions.py @@ -97,3 +97,49 @@ class InvalidInputException(_InvalidOperationException): class InvalidStateException(_InvalidOperationException): def __init__(self, op, msg): super().__init__("InvalidStateException", op, msg) + + +class ResourceNumberLimitExceededException(_InvalidOperationException): + def __init__(self, op, resource): + super().__init__( + "ResourceNumberLimitExceededException", + op, + "More " + + resource + + " cannot be created. The maximum limit has been reached.", + ) + + +class GSRAlreadyExistsException(_InvalidOperationException): + def __init__(self, op, resource, param_name, param_value): + super().__init__( + "AlreadyExistsException", + op, + resource + " already exists. " + param_name + ": " + param_value, + ) + + +class ResourceNameTooLongException(InvalidInputException): + def __init__(self, op, param_name): + super().__init__( + op, + "The resource name contains too many or too few characters. Parameter Name: " + + param_name, + ) + + +class ParamValueContainsInvalidCharactersException(InvalidInputException): + def __init__(self, op, param_name): + super().__init__( + op, + "The parameter value contains one or more characters that are not valid. Parameter Name: " + + param_name, + ) + + +class InvalidNumberOfTagsException(InvalidInputException): + def __init__(self, op): + super().__init__( + op, + "New Tags cannot be empty or more than 50", + ) diff --git a/moto/glue/models.py b/moto/glue/models.py index 2b8648310..48d884320 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -1,4 +1,5 @@ import time +import re from collections import OrderedDict from datetime import datetime @@ -19,6 +20,11 @@ from .exceptions import ( VersionNotFoundException, JobNotFoundException, ConcurrentRunsExceededException, + GSRAlreadyExistsException, + ResourceNumberLimitExceededException, + ResourceNameTooLongException, + ParamValueContainsInvalidCharactersException, + InvalidNumberOfTagsException, ) from .utils import PartitionFilter from ..utilities.paginator import paginate @@ -48,6 +54,7 @@ class GlueBackend(BaseBackend): self.jobs = OrderedDict() self.job_runs = OrderedDict() self.tagger = TaggingService() + self.registries = OrderedDict() @staticmethod def default_vpc_endpoint_service(service_region, zones): @@ -247,6 +254,63 @@ class GlueBackend(BaseBackend): def untag_resource(self, resource_arn, tag_keys): self.tagger.untag_resource_using_names(resource_arn, tag_keys) + # TODO: @Himani. Will Refactor validation logic as I find the common validation required for other APIs + def create_registry(self, registry_name, description, tags): + operation_name = "CreateRegistry" + + registry_name_pattern = re.compile(r"^[a-zA-Z0-9-_$#.]+$") + registry_description_pattern = re.compile( + r"[\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\r\\n\\t]*" + ) + + max_registry_name_length = 255 + max_registries_allowed = 10 + max_description_length = 2048 + max_tags_allowed = 50 + + if len(self.registries) >= max_registries_allowed: + raise ResourceNumberLimitExceededException( + operation_name, resource="registries" + ) + + if ( + registry_name == "" + or len(registry_name.encode("utf-8")) > max_registry_name_length + ): + param_name = "registryName" + raise ResourceNameTooLongException(operation_name, param_name) + + if re.match(registry_name_pattern, registry_name) is None: + param_name = "registryName" + raise ParamValueContainsInvalidCharactersException( + operation_name, param_name + ) + + if registry_name in self.registries: + raise GSRAlreadyExistsException( + operation_name, + resource="Registry", + param_name="RegistryName", + param_value=registry_name, + ) + + if description and len(description.encode("utf-8")) > max_description_length: + param_name = "description" + raise ResourceNameTooLongException(operation_name, param_name) + + if description and re.match(registry_description_pattern, description) is None: + param_name = "description" + raise ParamValueContainsInvalidCharactersException( + operation_name, param_name + ) + + if tags and len(tags) > max_tags_allowed: + raise InvalidNumberOfTagsException(operation_name) + + registry = FakeRegistry(registry_name, description, tags) + self.registries[registry_name] = registry + return registry + class FakeDatabase(BaseModel): def __init__(self, database_name, database_input): @@ -631,6 +695,26 @@ class FakeJobRun: } +class FakeRegistry(BaseModel): + def __init__(self, registry_name, description=None, tags=None): + self.name = registry_name + self.description = description + self.tags = tags + self.created_time = datetime.utcnow() + self.updated_time = datetime.utcnow() + self.registry_arn = ( + f"arn:aws:glue:us-east-1:{get_account_id()}:registry/{self.name}" + ) + + def as_dict(self): + return { + "RegistryArn": self.registry_arn, + "RegistryName": self.name, + "Description": self.description, + "Tags": self.tags, + } + + glue_backends = BackendDict( GlueBackend, "glue", use_boto3_regions=False, additional_regions=["global"] ) diff --git a/moto/glue/responses.py b/moto/glue/responses.py index 4c6b86fd4..ea71c7c60 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -452,3 +452,10 @@ class GlueResponse(BaseResponse): if glue_resource_tags[key] == tags[key]: return True return False + + def create_registry(self): + registry_name = self._get_param("RegistryName") + description = self._get_param("Description") + tags = self._get_param("Tags") + registry = self.glue_backend.create_registry(registry_name, description, tags) + return json.dumps(registry.as_dict()) diff --git a/tests/test_glue/test_glue.py b/tests/test_glue/test_glue.py index 7adf95d4f..c66345c7b 100644 --- a/tests/test_glue/test_glue.py +++ b/tests/test_glue/test_glue.py @@ -7,6 +7,7 @@ import pytest import sure # noqa # pylint: disable=unused-import from botocore.exceptions import ParamValidationError from botocore.client import ClientError +from moto.core import ACCOUNT_ID from moto import mock_glue @@ -449,3 +450,158 @@ def test_untag_glue_crawler(): resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key3": "value3"}) + + +@mock_glue +def test_create_registry_valid_input(): + client = create_glue_client() + registry_name = "TestRegistry" + response = client.create_registry( + RegistryName=registry_name, + Description="test_create_registry_description", + Tags={"key1": "value1", "key2": "value2"}, + ) + response.should.have.key("RegistryName").equals("TestRegistry") + response.should.have.key("Description").equals("test_create_registry_description") + response.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) + response.should.have.key("RegistryArn").equals( + f"arn:aws:glue:us-east-1:{ACCOUNT_ID}:registry/" + registry_name + ) + + +@mock_glue +def test_create_registry_valid_partial_input(): + client = create_glue_client() + registry_name = "TestRegistry" + response = client.create_registry(RegistryName=registry_name) + response.should.have.key("RegistryName").equals("TestRegistry") + response.should.have.key("RegistryArn").equals( + f"arn:aws:glue:us-east-1:{ACCOUNT_ID}:registry/" + registry_name + ) + + +@mock_glue +def test_create_registry_invalid_input_registry_name_too_long(): + client = create_glue_client() + registry_name = "" + for _ in range(90): + registry_name = registry_name + "foobar" + + with pytest.raises(ClientError) as exc: + client.create_registry( + RegistryName=registry_name, + Description="test_create_registry_description", + Tags={"key1": "value1", "key2": "value2"}, + ) + err = exc.value.response["Error"] + err["Code"].should.equal("InvalidInputException") + err["Message"].should.equal( + "An error occurred (InvalidInputException) when calling the CreateRegistry operation: The resource name contains too many or too few characters. Parameter Name: registryName" + ) + + +@mock_glue +def test_create_registry_more_than_allowed(): + client = create_glue_client() + + for i in range(10): + registry_name = "TestRegistry" + str(i) + client.create_registry( + RegistryName=registry_name, + Description="test_create_registry_description", + Tags={"key1": "value1", "key2": "value2"}, + ) + + with pytest.raises(ClientError) as exc: + client.create_registry( + RegistryName="TestRegistry10", + Description="test_create_registry_description10", + Tags={"key1": "value1", "key2": "value2"}, + ) + err = exc.value.response["Error"] + err["Code"].should.equal("ResourceNumberLimitExceededException") + err["Message"].should.equal( + "An error occurred (ResourceNumberLimitExceededException) when calling the CreateRegistry operation: More registries cannot be created. The maximum limit has been reached." + ) + + +@mock_glue +def test_create_registry_invalid_registry_name(): + client = create_glue_client() + + with pytest.raises(ClientError) as exc: + client.create_registry( + RegistryName="A,B,C", + Description="test_create_registry_description", + Tags={"key1": "value1", "key2": "value2"}, + ) + err = exc.value.response["Error"] + err["Code"].should.equal("InvalidInputException") + err["Message"].should.equal( + "An error occurred (InvalidInputException) when calling the CreateRegistry operation: The parameter value contains one or more characters that are not valid. Parameter Name: registryName" + ) + + +@mock_glue +def test_create_registry_already_exists(): + client = create_glue_client() + + client.create_registry( + RegistryName="TestRegistry1", + Description="test_create_registry_description1", + Tags={"key1": "value1", "key2": "value2"}, + ) + + with pytest.raises(ClientError) as exc: + client.create_registry( + RegistryName="TestRegistry1", + Description="test_create_registry_description1", + Tags={"key1": "value1", "key2": "value2"}, + ) + err = exc.value.response["Error"] + err["Code"].should.equal("AlreadyExistsException") + err["Message"].should.equal( + "An error occurred (AlreadyExistsException) when calling the CreateRegistry operation: Registry already exists. RegistryName: TestRegistry1" + ) + + +@mock_glue +def test_create_registry_invalid_description_too_long(): + client = create_glue_client() + description = "" + for _ in range(300): + description = description + "foobar, " + + with pytest.raises(ClientError) as exc: + client.create_registry( + RegistryName="TestRegistry1", + Description=description, + Tags={"key1": "value1", "key2": "value2"}, + ) + err = exc.value.response["Error"] + err["Code"].should.equal("InvalidInputException") + err["Message"].should.equal( + "An error occurred (InvalidInputException) when calling the CreateRegistry operation: The resource name contains too many or too few characters. Parameter Name: description" + ) + + +@mock_glue +def test_create_registry_invalid_number_of_tags(): + tags = {} + for i in range(51): + key = "k" + str(i) + val = "v" + str(i) + tags[key] = val + + client = create_glue_client() + with pytest.raises(ClientError) as exc: + client.create_registry( + RegistryName="TestRegistry1", + Description="test_create_registry_description", + Tags=tags, + ) + err = exc.value.response["Error"] + err["Code"].should.equal("InvalidInputException") + err["Message"].should.equal( + "An error occurred (InvalidInputException) when calling the CreateRegistry operation: New Tags cannot be empty or more than 50" + )