From aa9bba72af9bdc0c45260cb5c9662426019ef8bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20D=C4=85bek?= <373530+szemek@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:45:16 +0100 Subject: [PATCH] [Glue] Add support to more fields in Glue job (#6045) --- moto/glue/models.py | 15 +++++++++++ moto/glue/responses.py | 6 +++++ tests/test_glue/test_glue.py | 48 ++++++++++++++++++++---------------- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/moto/glue/models.py b/moto/glue/models.py index a0b3566aa..6abbf4aa9 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -324,6 +324,9 @@ class GlueBackend(BaseBackend): glue_version, number_of_workers, worker_type, + code_gen_configuration_nodes, + execution_class, + source_control_details, ): self.jobs[name] = FakeJob( name, @@ -345,6 +348,9 @@ class GlueBackend(BaseBackend): glue_version, number_of_workers, worker_type, + code_gen_configuration_nodes, + execution_class, + source_control_details, backend=self, ) return name @@ -1070,6 +1076,9 @@ class FakeJob: glue_version=None, number_of_workers=None, worker_type=None, + code_gen_configuration_nodes=None, + execution_class=None, + source_control_details=None, backend=None, ): self.name = name @@ -1090,6 +1099,9 @@ class FakeJob: self.glue_version = glue_version self.number_of_workers = number_of_workers self.worker_type = worker_type + self.code_gen_configuration_nodes = code_gen_configuration_nodes + self.execution_class = execution_class or "STANDARD" + self.source_control_details = source_control_details self.created_on = datetime.utcnow() self.last_modified_on = datetime.utcnow() self.arn = ( @@ -1125,6 +1137,9 @@ class FakeJob: "SecurityConfiguration": self.security_configuration, "NotificationProperty": self.notification_property, "GlueVersion": self.glue_version, + "CodeGenConfigurationNodes": self.code_gen_configuration_nodes, + "ExecutionClass": self.execution_class, + "SourceControlDetails": self.source_control_details, } def start_job_run(self): diff --git a/moto/glue/responses.py b/moto/glue/responses.py index 7439dbbd7..934b1ed76 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -309,6 +309,9 @@ class GlueResponse(BaseResponse): glue_version = self._get_param("GlueVersion") number_of_workers = self._get_int_param("NumberOfWorkers") worker_type = self._get_param("WorkerType") + code_gen_configuration_nodes = self._get_param("CodeGenConfigurationNodes") + execution_class = self._get_param("ExecutionClass") + source_control_details = self._get_param("SourceControlDetails") name = self.glue_backend.create_job( name=name, description=description, @@ -329,6 +332,9 @@ class GlueResponse(BaseResponse): glue_version=glue_version, number_of_workers=number_of_workers, worker_type=worker_type, + code_gen_configuration_nodes=code_gen_configuration_nodes, + execution_class=execution_class, + source_control_details=source_control_details, ) return json.dumps(dict(Name=name)) diff --git a/tests/test_glue/test_glue.py b/tests/test_glue/test_glue.py index 820bbc14e..03d7a54e8 100644 --- a/tests/test_glue/test_glue.py +++ b/tests/test_glue/test_glue.py @@ -83,30 +83,36 @@ def test_get_job_exists(): "SecurityConfiguration": "test_config", "NotificationProperty": {"NotifyDelayAfter": 123}, "GlueVersion": "string", + "CodeGenConfigurationNodes": {}, + "ExecutionClass": "string", + "SourceControlDetails": {}, } job_name = create_test_job_w_all_attributes(client, **job_attributes) job = client.get_job(JobName=job_name)["Job"] - job.should.have.key("Name").equals(job_name) - job.should.have.key("Description") - job.should.have.key("LogUri") - job.should.have.key("Role") - job.should.have.key("ExecutionProperty").equals({"MaxConcurrentRuns": 123}) - job.should.have.key("CreatedOn") - job.should.have.key("LastModifiedOn") - job.should.have.key("ExecutionProperty") - job.should.have.key("Command") - job.should.have.key("DefaultArguments") - job.should.have.key("NonOverridableArguments") - job.should.have.key("Connections") - job.should.have.key("MaxRetries") - job.should.have.key("AllocatedCapacity") - job.should.have.key("Timeout") - job.should.have.key("MaxCapacity") - job.should.have.key("WorkerType") - job.should.have.key("NumberOfWorkers") - job.should.have.key("SecurityConfiguration") - job.should.have.key("NotificationProperty") - job.should.have.key("GlueVersion") + assert job["Name"] == job_name + assert "Description" in job + assert "LogUri" in job + assert "Role" in job + assert job["ExecutionProperty"] == {"MaxConcurrentRuns": 123} + assert "CreatedOn" in job + assert "LastModifiedOn" in job + assert "ExecutionProperty" in job + assert "Command" in job + assert "DefaultArguments" in job + assert "NonOverridableArguments" in job + assert "Connections" in job + assert "MaxRetries" in job + assert "AllocatedCapacity" in job + assert "Timeout" in job + assert "MaxCapacity" in job + assert "WorkerType" in job + assert "NumberOfWorkers" in job + assert "SecurityConfiguration" in job + assert "NotificationProperty" in job + assert "GlueVersion" in job + assert "CodeGenConfigurationNodes" in job + assert "ExecutionClass" in job + assert "SourceControlDetails" in job @mock_glue