"""Unit tests for glue-supported APIs.""" from random import randint from uuid import uuid4 import boto3 import pytest import sure # noqa # pylint: disable=unused-import from botocore.exceptions import ParamValidationError from botocore.client import ClientError from moto import mock_glue @mock_glue def test_create_job(): client = create_glue_client() job_name = str(uuid4()) response = client.create_job( Name=job_name, Role="test_role", Command=dict(Name="test_command") ) assert response["Name"] == job_name @mock_glue def test_create_job_default_argument_not_provided(): client = create_glue_client() with pytest.raises(ParamValidationError) as exc: client.create_job(Role="test_role", Command=dict(Name="test_command")) exc.value.kwargs["report"].should.equal( 'Missing required parameter in input: "Name"' ) @mock_glue def test_list_jobs(): client = create_glue_client() expected_jobs = randint(1, 15) create_test_jobs(client, expected_jobs) response = client.list_jobs() response["JobNames"].should.have.length_of(expected_jobs) response.shouldnt.have.key("NextToken") @mock_glue def test_get_job_not_exists(): client = create_glue_client() name = "my_job_name" with pytest.raises(ClientError) as exc: client.get_job(JobName=name) exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") exc.value.response["Error"]["Message"].should.match("Job my_job_name not found") @mock_glue def test_get_job_exists(): client = create_glue_client() job_attributes = { "Description": "test_description", "LogUri": "test_log/", "Role": "test_role", "ExecutionProperty": {"MaxConcurrentRuns": 123}, "Command": { "Name": "test_command", "ScriptLocation": "test_s3_path", "PythonVersion": "3.6", }, "DefaultArguments": {"string": "string"}, "NonOverridableArguments": {"string": "string"}, "Connections": { "Connections": [ "string", ] }, "MaxRetries": 123, "AllocatedCapacity": 123, "Timeout": 123, "MaxCapacity": 123.0, "WorkerType": "G.2X", "NumberOfWorkers": 123, "SecurityConfiguration": "test_config", "NotificationProperty": {"NotifyDelayAfter": 123}, "GlueVersion": "string", "CodeGenConfigurationNodes": {}, "ExecutionClass": "string", "SourceControlDetails": {}, } job_name = create_test_job_w_all_attributes(client, **job_attributes) job = client.get_job(JobName=job_name)["Job"] assert job["Name"] == job_name assert "Description" in job assert "LogUri" in job assert "Role" in job assert job["ExecutionProperty"] == {"MaxConcurrentRuns": 123} assert "CreatedOn" in job assert "LastModifiedOn" in job assert "ExecutionProperty" in job assert "Command" in job assert "DefaultArguments" in job assert "NonOverridableArguments" in job assert "Connections" in job assert "MaxRetries" in job assert "AllocatedCapacity" in job assert "Timeout" in job assert "MaxCapacity" in job assert "WorkerType" in job assert "NumberOfWorkers" in job assert "SecurityConfiguration" in job assert "NotificationProperty" in job assert "GlueVersion" in job assert "CodeGenConfigurationNodes" in job assert "ExecutionClass" in job assert "SourceControlDetails" in job @mock_glue def test_get_jobs_job_name_exists(): client = create_glue_client() test_job_name = create_test_job(client) response = client.get_jobs() response["Jobs"].should.have.length_of(1) response["Jobs"][0].should.have.key("Name").equals(test_job_name) @mock_glue def test_get_jobs_with_max_results(): client = create_glue_client() create_test_jobs(client, 4) response = client.get_jobs(MaxResults=2) response["Jobs"].should.have.length_of(2) response.should.have.key("NextToken") @mock_glue def test_get_jobs_from_next_token(): client = create_glue_client() create_test_jobs(client, 10) first_response = client.get_jobs(MaxResults=3) response = client.get_jobs(NextToken=first_response["NextToken"]) response["Jobs"].should.have.length_of(7) @mock_glue def test_get_jobs_with_max_results_greater_than_actual_results(): client = create_glue_client() create_test_jobs(client, 4) response = client.get_jobs(MaxResults=10) response["Jobs"].should.have.length_of(4) @mock_glue def test_get_jobs_next_token_logic_does_not_create_infinite_loop(): client = create_glue_client() create_test_jobs(client, 4) first_response = client.get_jobs(MaxResults=1) next_token = first_response["NextToken"] while next_token: response = client.get_jobs(NextToken=next_token) next_token = response.get("NextToken") assert not next_token @mock_glue def test_list_jobs_with_max_results(): client = create_glue_client() create_test_jobs(client, 4) response = client.list_jobs(MaxResults=2) response["JobNames"].should.have.length_of(2) response.should.have.key("NextToken") @mock_glue def test_list_jobs_from_next_token(): client = create_glue_client() create_test_jobs(client, 10) first_response = client.list_jobs(MaxResults=3) response = client.list_jobs(NextToken=first_response["NextToken"]) response["JobNames"].should.have.length_of(7) @mock_glue def test_list_jobs_with_max_results_greater_than_actual_results(): client = create_glue_client() create_test_jobs(client, 4) response = client.list_jobs(MaxResults=10) response["JobNames"].should.have.length_of(4) @mock_glue def test_list_jobs_with_tags(): client = create_glue_client() create_test_job(client) create_test_job(client, {"string": "string"}) response = client.list_jobs(Tags={"string": "string"}) response["JobNames"].should.have.length_of(1) @mock_glue def test_list_jobs_after_tagging(): client = create_glue_client() job_name = create_test_job(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:job/{job_name}" client.tag_resource(ResourceArn=resource_arn, TagsToAdd={"key1": "value1"}) response = client.list_jobs(Tags={"key1": "value1"}) response["JobNames"].should.have.length_of(1) @mock_glue def test_list_jobs_after_removing_tag(): client = create_glue_client() job_name = create_test_job(client, {"key1": "value1"}) resource_arn = f"arn:aws:glue:us-east-1:123456789012:job/{job_name}" client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key1"]) response = client.list_jobs(Tags={"key1": "value1"}) response["JobNames"].should.have.length_of(0) @mock_glue def test_list_jobs_next_token_logic_does_not_create_infinite_loop(): client = create_glue_client() create_test_jobs(client, 4) first_response = client.list_jobs(MaxResults=1) next_token = first_response["NextToken"] while next_token: response = client.list_jobs(NextToken=next_token) next_token = response.get("NextToken") assert not next_token def create_glue_client(): return boto3.client("glue", region_name="us-east-1") def create_test_job(client, tags=None): job_name = str(uuid4()) client.create_job( Name=job_name, Role="test_role", Command=dict(Name="test_command"), Tags=tags or {}, ) return job_name def create_test_job_w_all_attributes(client, **job_attributes): job_name = str(uuid4()) client.create_job(Name=job_name, **job_attributes) return job_name def create_test_jobs(client, number_of_jobs): for _ in range(number_of_jobs): create_test_job(client) def create_test_crawler(client, tags=None): crawler_name = str(uuid4()) client.create_crawler( Name=crawler_name, Role="test_role", Targets={"S3Targets": [{"Path": "s3://tests3target"}]}, Tags=tags or {}, ) return crawler_name def create_test_crawlers(client, number_of_crawlers): for _ in range(number_of_crawlers): create_test_crawler(client) @mock_glue def test_list_crawlers_with_max_results(): client = create_glue_client() create_test_crawlers(client, 4) response = client.list_crawlers(MaxResults=2) response["CrawlerNames"].should.have.length_of(2) response.should.have.key("NextToken") @mock_glue def test_list_crawlers_from_next_token(): client = create_glue_client() create_test_crawlers(client, 10) first_response = client.list_crawlers(MaxResults=3) response = client.list_crawlers(NextToken=first_response["NextToken"]) response["CrawlerNames"].should.have.length_of(7) @mock_glue def test_list_crawlers_with_max_results_greater_than_actual_results(): client = create_glue_client() create_test_crawlers(client, 4) response = client.list_crawlers(MaxResults=10) response["CrawlerNames"].should.have.length_of(4) @mock_glue def test_list_crawlers_with_tags(): client = create_glue_client() create_test_crawler(client) create_test_crawler(client, {"string": "string"}) response = client.list_crawlers(Tags={"string": "string"}) response["CrawlerNames"].should.have.length_of(1) @mock_glue def test_list_crawlers_after_tagging(): client = create_glue_client() crawler_name = create_test_crawler(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:crawler/{crawler_name}" client.tag_resource(ResourceArn=resource_arn, TagsToAdd={"key1": "value1"}) response = client.list_crawlers(Tags={"key1": "value1"}) response["CrawlerNames"].should.have.length_of(1) @mock_glue def test_list_crawlers_after_removing_tag(): client = create_glue_client() crawler_name = create_test_crawler(client, {"key1": "value1"}) resource_arn = f"arn:aws:glue:us-east-1:123456789012:crawler/{crawler_name}" client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key1"]) response = client.list_crawlers(Tags={"key1": "value1"}) response["CrawlerNames"].should.have.length_of(0) @mock_glue def test_list_crawlers_next_token_logic_does_not_create_infinite_loop(): client = create_glue_client() create_test_crawlers(client, 4) first_response = client.list_crawlers(MaxResults=1) next_token = first_response["NextToken"] while next_token: response = client.list_crawlers(NextToken=next_token) next_token = response.get("NextToken") assert not next_token @mock_glue def test_get_tags_job(): client = create_glue_client() job_name = create_test_job(client, {"key1": "value1", "key2": "value2"}) resource_arn = f"arn:aws:glue:us-east-1:123456789012:job/{job_name}" resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) @mock_glue def test_get_tags_jobs_no_tags(): client = create_glue_client() job_name = create_test_job(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:job/{job_name}" resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({}) @mock_glue def test_tag_glue_job(): client = create_glue_client() job_name = create_test_job(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:job/{job_name}" client.tag_resource( ResourceArn=resource_arn, TagsToAdd={"key1": "value1", "key2": "value2"} ) resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) @mock_glue def test_untag_glue_job(): client = create_glue_client() job_name = create_test_job(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:job/{job_name}" client.tag_resource( ResourceArn=resource_arn, TagsToAdd={"key1": "value1", "key2": "value2", "key3": "value3"}, ) client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key2"]) resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key3": "value3"}) @mock_glue def test_get_tags_crawler(): client = create_glue_client() crawler_name = create_test_crawler(client, {"key1": "value1", "key2": "value2"}) resource_arn = f"arn:aws:glue:us-east-1:123456789012:crawler/{crawler_name}" resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) @mock_glue def test_get_tags_crawler_no_tags(): client = create_glue_client() crawler_name = create_test_crawler(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:crawler/{crawler_name}" resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({}) @mock_glue def test_tag_glue_crawler(): client = create_glue_client() crawler_name = create_test_crawler(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:crawler/{crawler_name}" client.tag_resource( ResourceArn=resource_arn, TagsToAdd={"key1": "value1", "key2": "value2"} ) resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) @mock_glue def test_untag_glue_crawler(): client = create_glue_client() crawler_name = create_test_crawler(client) resource_arn = f"arn:aws:glue:us-east-1:123456789012:crawler/{crawler_name}" client.tag_resource( ResourceArn=resource_arn, TagsToAdd={"key1": "value1", "key2": "value2", "key3": "value3"}, ) client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key2"]) resp = client.get_tags(ResourceArn=resource_arn) resp.should.have.key("Tags").equals({"key1": "value1", "key3": "value3"}) @mock_glue def test_batch_get_crawlers(): client = create_glue_client() crawler_name = create_test_crawler(client) response = client.batch_get_crawlers( CrawlerNames=[crawler_name, "crawler-not-found"] ) response["Crawlers"].should.have.length_of(1) response["CrawlersNotFound"].should.have.length_of(1)