Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Glue jobs initial implementation (#4606)

Browse Source
This commit is contained in:
Dan Aronson 2021-11-21 15:17:52 +02:00 committed by GitHub
parent c7a9dc32e0
commit c7fc9f490d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 163 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
moto/glue/models.py
View File

@ -4,7 +4,6 @@ from datetime import datetime
from moto.core import BaseBackend, BaseModel
from moto.glue.exceptions import CrawlerRunningException, CrawlerNotRunningException
from .exceptions import (
JsonRESTError,
CrawlerAlreadyExistsException,
@ -23,6 +22,7 @@ class GlueBackend(BaseBackend):
def __init__(self):
self.databases = OrderedDict()
self.crawlers = OrderedDict()
self.jobs = OrderedDict()
@staticmethod
def default_vpc_endpoint_service(service_region, zones):
@ -138,6 +138,51 @@ class GlueBackend(BaseBackend):
except KeyError:
raise CrawlerNotFoundException(name)
def create_job(
self,
name,
role,
command,
description,
log_uri,
execution_property,
default_arguments,
non_overridable_arguments,
connections,
max_retries,
allocated_capacity,
timeout,
max_capacity,
security_configuration,
tags,
notification_property,
glue_version,
number_of_workers,
worker_type,
):
self.jobs[name] = FakeJob(
name,
role,
command,
description,
log_uri,
execution_property,
default_arguments,
non_overridable_arguments,
connections,
max_retries,
allocated_capacity,
timeout,
max_capacity,
security_configuration,
tags,
notification_property,
glue_version,
number_of_workers,
worker_type,
)
return name
class FakeDatabase(BaseModel):
def __init__(self, database_name, database_input):
@ -358,4 +403,50 @@ class LastCrawlInfo(BaseModel):
}
class FakeJob:
def __init__(
self,
name,
role,
command,
description=None,
log_uri=None,
execution_property=None,
default_arguments=None,
non_overridable_arguments=None,
connections=None,
max_retries=None,
allocated_capacity=None,
timeout=None,
max_capacity=None,
security_configuration=None,
tags=None,
notification_property=None,
glue_version=None,
number_of_workers=None,
worker_type=None,
):
self.name = name
self.description = description
self.log_uri = log_uri
self.role = role
self.execution_property = execution_property
self.command = command
self.default_arguments = default_arguments
self.non_overridable_arguments = non_overridable_arguments
self.connections = connections
self.max_retries = max_retries
self.allocated_capacity = allocated_capacity
self.timeout = timeout
self.max_capacity = max_capacity
self.security_configuration = security_configuration
self.tags = tags
self.notification_property = notification_property
self.glue_version = glue_version
self.number_of_workers = number_of_workers
self.worker_type = worker_type
self.created_on = datetime.utcnow()
self.last_modified_on = datetime.utcnow()
glue_backend = GlueBackend()

45
moto/glue/responses.py
View File

@ -1,12 +1,12 @@
import json
from moto.core.responses import BaseResponse
from .models import glue_backend
from .exceptions import (
PartitionAlreadyExistsException,
PartitionNotFoundException,
TableNotFoundException,
)
from .models import glue_backend
class GlueResponse(BaseResponse):
@ -317,3 +317,46 @@ class GlueResponse(BaseResponse):
name = self.parameters.get("Name")
self.glue_backend.delete_crawler(name)
return ""
def create_job(self):
name = self._get_param("Name")
description = self._get_param("Description")
log_uri = self._get_param("LogUri")
role = self._get_param("Role")
execution_property = self._get_param("ExecutionProperty")
command = self._get_param("Command")
default_arguments = self._get_param("DefaultArguments")
non_overridable_arguments = self._get_param("NonOverridableArguments")
connections = self._get_param("Connections")
max_retries = self._get_int_param("MaxRetries")
allocated_capacity = self._get_int_param("AllocatedCapacity")
timeout = self._get_int_param("Timeout")
max_capacity = self._get_param("MaxCapacity")
security_configuration = self._get_param("SecurityConfiguration")
tags = self._get_param("Tags")
notification_property = self._get_param("NotificationProperty")
glue_version = self._get_param("GlueVersion")
number_of_workers = self._get_int_param("NumberOfWorkers")
worker_type = self._get_param("WorkerType")
name = self.glue_backend.create_job(
name=name,
description=description,
log_uri=log_uri,
role=role,
execution_property=execution_property,
command=command,
default_arguments=default_arguments,
non_overridable_arguments=non_overridable_arguments,
connections=connections,
max_retries=max_retries,
allocated_capacity=allocated_capacity,
timeout=timeout,
max_capacity=max_capacity,
security_configuration=security_configuration,
tags=tags,
notification_property=notification_property,
glue_version=glue_version,
number_of_workers=number_of_workers,
worker_type=worker_type,
)
return json.dumps(dict(Name=name))

27
tests/test_glue/test_glue.py Normal file
View File

@ -0,0 +1,27 @@
"""Unit tests for glue-supported APIs."""
import boto3
import pytest
import sure # noqa # pylint: disable=unused-import
from botocore.exceptions import ParamValidationError
from moto import mock_glue
@mock_glue
def test_create_job():
client = boto3.client("glue", region_name="us-east-1")
response = client.create_job(
Name="test_name", Role="test_role", Command=dict(Name="test_command")
)
assert response["Name"] == "test_name"
@mock_glue
def test_create_job_default_argument_not_provided():
client = boto3.client("glue", region_name="us-east-1")
with pytest.raises(ParamValidationError) as exc:
client.create_job(Role="test_role", Command=dict(Name="test_command"))
exc.value.kwargs["report"].should.equal(
'Missing required parameter in input: "Name"'
)