294 lines
9.1 KiB
Python
294 lines
9.1 KiB
Python
"""Unit tests for glue-supported APIs."""
|
|
from random import randint
|
|
from uuid import uuid4
|
|
|
|
import boto3
|
|
import pytest
|
|
import sure # noqa # pylint: disable=unused-import
|
|
from botocore.exceptions import ParamValidationError
|
|
from botocore.client import ClientError
|
|
|
|
from moto import mock_glue
|
|
|
|
|
|
@mock_glue
|
|
def test_create_job():
|
|
client = create_glue_client()
|
|
job_name = str(uuid4())
|
|
response = client.create_job(
|
|
Name=job_name, Role="test_role", Command=dict(Name="test_command")
|
|
)
|
|
assert response["Name"] == job_name
|
|
|
|
|
|
@mock_glue
|
|
def test_create_job_default_argument_not_provided():
|
|
client = create_glue_client()
|
|
with pytest.raises(ParamValidationError) as exc:
|
|
client.create_job(Role="test_role", Command=dict(Name="test_command"))
|
|
|
|
exc.value.kwargs["report"].should.equal(
|
|
'Missing required parameter in input: "Name"'
|
|
)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_jobs():
|
|
client = create_glue_client()
|
|
expected_jobs = randint(1, 15)
|
|
create_test_jobs(client, expected_jobs)
|
|
response = client.list_jobs()
|
|
response["JobNames"].should.have.length_of(expected_jobs)
|
|
response.shouldnt.have.key("NextToken")
|
|
|
|
|
|
@mock_glue
|
|
def test_get_job_not_exists():
|
|
client = create_glue_client()
|
|
name = "my_job_name"
|
|
|
|
with pytest.raises(ClientError) as exc:
|
|
client.get_job(JobName=name)
|
|
|
|
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException")
|
|
exc.value.response["Error"]["Message"].should.match("Job my_job_name not found")
|
|
|
|
|
|
@mock_glue
|
|
def test_get_job_exists():
|
|
client = create_glue_client()
|
|
job_attributes = {
|
|
"Description": "test_description",
|
|
"LogUri": "test_log/",
|
|
"Role": "test_role",
|
|
"ExecutionProperty": {"MaxConcurrentRuns": 123},
|
|
"Command": {
|
|
"Name": "test_command",
|
|
"ScriptLocation": "test_s3_path",
|
|
"PythonVersion": "3.6",
|
|
},
|
|
"DefaultArguments": {"string": "string"},
|
|
"NonOverridableArguments": {"string": "string"},
|
|
"Connections": {
|
|
"Connections": [
|
|
"string",
|
|
]
|
|
},
|
|
"MaxRetries": 123,
|
|
"AllocatedCapacity": 123,
|
|
"Timeout": 123,
|
|
"MaxCapacity": 123.0,
|
|
"WorkerType": "G.2X",
|
|
"NumberOfWorkers": 123,
|
|
"SecurityConfiguration": "test_config",
|
|
"NotificationProperty": {"NotifyDelayAfter": 123},
|
|
"GlueVersion": "string",
|
|
}
|
|
job_name = create_test_job_w_all_attributes(client, **job_attributes)
|
|
response = client.get_job(JobName=job_name)
|
|
assert response["Job"]["Name"] == job_name
|
|
assert response["Job"]["Description"]
|
|
assert response["Job"]["LogUri"]
|
|
assert response["Job"]["Role"]
|
|
assert response["Job"]["CreatedOn"]
|
|
assert response["Job"]["LastModifiedOn"]
|
|
assert response["Job"]["ExecutionProperty"]
|
|
assert response["Job"]["Command"]
|
|
assert response["Job"]["DefaultArguments"]
|
|
assert response["Job"]["NonOverridableArguments"]
|
|
assert response["Job"]["Connections"]
|
|
assert response["Job"]["MaxRetries"]
|
|
assert response["Job"]["AllocatedCapacity"]
|
|
assert response["Job"]["Timeout"]
|
|
assert response["Job"]["MaxCapacity"]
|
|
assert response["Job"]["WorkerType"]
|
|
assert response["Job"]["NumberOfWorkers"]
|
|
assert response["Job"]["SecurityConfiguration"]
|
|
assert response["Job"]["NotificationProperty"]
|
|
assert response["Job"]["GlueVersion"]
|
|
|
|
|
|
@mock_glue
|
|
def test_start_job_run():
|
|
client = create_glue_client()
|
|
job_name = create_test_job(client)
|
|
response = client.start_job_run(JobName=job_name)
|
|
assert response["JobRunId"]
|
|
|
|
|
|
@mock_glue
|
|
def test_start_job_run_already_running():
|
|
client = create_glue_client()
|
|
job_name = create_test_job(client)
|
|
client.start_job_run(JobName=job_name)
|
|
with pytest.raises(ClientError) as exc:
|
|
client.start_job_run(JobName=job_name)
|
|
exc.value.response["Error"]["Code"].should.equal("ConcurrentRunsExceededException")
|
|
exc.value.response["Error"]["Message"].should.match(
|
|
f"Job with name {job_name} already running"
|
|
)
|
|
|
|
|
|
@mock_glue
|
|
def test_get_job_run():
|
|
client = create_glue_client()
|
|
job_name = create_test_job(client)
|
|
response = client.get_job_run(JobName=job_name, RunId="01")
|
|
assert response["JobRun"]["Id"]
|
|
assert response["JobRun"]["Attempt"]
|
|
assert response["JobRun"]["PreviousRunId"]
|
|
assert response["JobRun"]["TriggerName"]
|
|
assert response["JobRun"]["StartedOn"]
|
|
assert response["JobRun"]["LastModifiedOn"]
|
|
assert response["JobRun"]["CompletedOn"]
|
|
assert response["JobRun"]["JobRunState"]
|
|
assert response["JobRun"]["Arguments"]
|
|
assert response["JobRun"]["ErrorMessage"] == ""
|
|
assert response["JobRun"]["PredecessorRuns"]
|
|
assert response["JobRun"]["AllocatedCapacity"]
|
|
assert response["JobRun"]["ExecutionTime"]
|
|
assert response["JobRun"]["Timeout"]
|
|
assert response["JobRun"]["MaxCapacity"]
|
|
assert response["JobRun"]["WorkerType"]
|
|
assert response["JobRun"]["NumberOfWorkers"]
|
|
assert response["JobRun"]["SecurityConfiguration"]
|
|
assert response["JobRun"]["LogGroupName"]
|
|
assert response["JobRun"]["NotificationProperty"]
|
|
assert response["JobRun"]["GlueVersion"]
|
|
|
|
|
|
@mock_glue
|
|
def test_list_jobs_with_max_results():
|
|
client = create_glue_client()
|
|
create_test_jobs(client, 4)
|
|
response = client.list_jobs(MaxResults=2)
|
|
response["JobNames"].should.have.length_of(2)
|
|
response.should.have.key("NextToken")
|
|
|
|
|
|
@mock_glue
|
|
def test_list_jobs_from_next_token():
|
|
client = create_glue_client()
|
|
create_test_jobs(client, 10)
|
|
first_response = client.list_jobs(MaxResults=3)
|
|
response = client.list_jobs(NextToken=first_response["NextToken"])
|
|
response["JobNames"].should.have.length_of(7)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_jobs_with_max_results_greater_than_actual_results():
|
|
client = create_glue_client()
|
|
create_test_jobs(client, 4)
|
|
response = client.list_jobs(MaxResults=10)
|
|
response["JobNames"].should.have.length_of(4)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_jobs_with_tags():
|
|
client = create_glue_client()
|
|
create_test_job(client)
|
|
create_test_job(client, {"string": "string"})
|
|
response = client.list_jobs(Tags={"string": "string"})
|
|
response["JobNames"].should.have.length_of(1)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_jobs_next_token_logic_does_not_create_infinite_loop():
|
|
client = create_glue_client()
|
|
create_test_jobs(client, 4)
|
|
first_response = client.list_jobs(MaxResults=1)
|
|
next_token = first_response["NextToken"]
|
|
while next_token:
|
|
response = client.list_jobs(NextToken=next_token)
|
|
next_token = response.get("NextToken")
|
|
assert not next_token
|
|
|
|
|
|
def create_glue_client():
|
|
return boto3.client("glue", region_name="us-east-1")
|
|
|
|
|
|
def create_test_job(client, tags=None):
|
|
job_name = str(uuid4())
|
|
client.create_job(
|
|
Name=job_name,
|
|
Role="test_role",
|
|
Command=dict(Name="test_command"),
|
|
Tags=tags or {},
|
|
)
|
|
return job_name
|
|
|
|
|
|
def create_test_job_w_all_attributes(client, **job_attributes):
|
|
job_name = str(uuid4())
|
|
client.create_job(Name=job_name, **job_attributes)
|
|
return job_name
|
|
|
|
|
|
def create_test_jobs(client, number_of_jobs):
|
|
for _ in range(number_of_jobs):
|
|
create_test_job(client)
|
|
|
|
|
|
def create_test_crawler(client, tags=None):
|
|
crawler_name = str(uuid4())
|
|
client.create_crawler(
|
|
Name=crawler_name,
|
|
Role="test_role",
|
|
Targets={"S3Targets": [{"Path": "s3://tests3target"}]},
|
|
Tags=tags or {},
|
|
)
|
|
return crawler_name
|
|
|
|
|
|
def create_test_crawlers(client, number_of_crawlers):
|
|
for _ in range(number_of_crawlers):
|
|
create_test_crawler(client)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_crawlers_with_max_results():
|
|
client = create_glue_client()
|
|
create_test_crawlers(client, 4)
|
|
response = client.list_crawlers(MaxResults=2)
|
|
response["CrawlerNames"].should.have.length_of(2)
|
|
response.should.have.key("NextToken")
|
|
|
|
|
|
@mock_glue
|
|
def test_list_crawlers_from_next_token():
|
|
client = create_glue_client()
|
|
create_test_crawlers(client, 10)
|
|
first_response = client.list_crawlers(MaxResults=3)
|
|
response = client.list_crawlers(NextToken=first_response["NextToken"])
|
|
response["CrawlerNames"].should.have.length_of(7)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_crawlers_with_max_results_greater_than_actual_results():
|
|
client = create_glue_client()
|
|
create_test_crawlers(client, 4)
|
|
response = client.list_crawlers(MaxResults=10)
|
|
response["CrawlerNames"].should.have.length_of(4)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_crawlers_with_tags():
|
|
client = create_glue_client()
|
|
create_test_crawler(client)
|
|
create_test_crawler(client, {"string": "string"})
|
|
response = client.list_crawlers(Tags={"string": "string"})
|
|
response["CrawlerNames"].should.have.length_of(1)
|
|
|
|
|
|
@mock_glue
|
|
def test_list_crawlers_next_token_logic_does_not_create_infinite_loop():
|
|
client = create_glue_client()
|
|
create_test_crawlers(client, 4)
|
|
first_response = client.list_crawlers(MaxResults=1)
|
|
next_token = first_response["NextToken"]
|
|
while next_token:
|
|
response = client.list_crawlers(NextToken=next_token)
|
|
next_token = response.get("NextToken")
|
|
assert not next_token
|