Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Techdebt: Replace sure with regular assertions in Glue (#6568)

Browse Source
This commit is contained in:
Bert Blommers 2023-07-29 22:26:04 +00:00 committed by GitHub
parent 35ab6cefe8
commit 414a9e6c86
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 607 additions and 637 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

419
tests/test_glue/test_datacatalog.py
View File

@ -1,5 +1,3 @@
import sure # noqa # pylint: disable=unused-import
import re
import pytest import pytest
import json import json
import boto3 import boto3
@ -29,19 +27,17 @@ def test_create_database():
response = helpers.get_database(client, database_name) response = helpers.get_database(client, database_name)
database = response["Database"] database = response["Database"]
database.get("Name").should.equal(database_name) assert database.get("Name") == database_name
database.get("Description").should.equal(database_input.get("Description")) assert database.get("Description") == database_input.get("Description")
database.get("LocationUri").should.equal(database_input.get("LocationUri")) assert database.get("LocationUri") == database_input.get("LocationUri")
database.get("Parameters").should.equal(database_input.get("Parameters")) assert database.get("Parameters") == database_input.get("Parameters")
if not settings.TEST_SERVER_MODE: if not settings.TEST_SERVER_MODE:
database.get("CreateTime").timestamp().should.equal( assert database["CreateTime"].timestamp() == FROZEN_CREATE_TIME.timestamp()
FROZEN_CREATE_TIME.timestamp() assert database["CreateTableDefaultPermissions"] == database_input.get(
) "CreateTableDefaultPermissions"
database.get("CreateTableDefaultPermissions").should.equal(
database_input.get("CreateTableDefaultPermissions")
) )
database.get("TargetDatabase").should.equal(database_input.get("TargetDatabase")) assert database.get("TargetDatabase") == database_input.get("TargetDatabase")
database.get("CatalogId").should.equal(database_catalog_id) assert database.get("CatalogId") == database_catalog_id
@mock_glue @mock_glue
@ -53,7 +49,7 @@ def test_create_database_already_exists():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.create_database(client, database_name) helpers.create_database(client, database_name)
exc.value.response["Error"]["Code"].should.equal("AlreadyExistsException") assert exc.value.response["Error"]["Code"] == "AlreadyExistsException"
@mock_glue @mock_glue
@ -64,9 +60,9 @@ def test_get_database_not_exits():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_database(client, database_name) helpers.get_database(client, database_name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert (
"Database nosuchdatabase not found" exc.value.response["Error"]["Message"] == "Database nosuchdatabase not found."
) )
@ -74,7 +70,7 @@ def test_get_database_not_exits():
def test_get_databases_empty(): def test_get_databases_empty():
client = boto3.client("glue", region_name="us-east-1") client = boto3.client("glue", region_name="us-east-1")
response = client.get_databases() response = client.get_databases()
response["DatabaseList"].should.have.length_of(0) assert len(response["DatabaseList"]) == 0
@mock_glue @mock_glue
@ -88,9 +84,9 @@ def test_get_databases_several_items():
database_list = sorted( database_list = sorted(
client.get_databases()["DatabaseList"], key=lambda x: x["Name"] client.get_databases()["DatabaseList"], key=lambda x: x["Name"]
) )
database_list.should.have.length_of(2) assert len(database_list) == 2
database_list[0]["Name"].should.equal(database_name_1) assert database_list[0]["Name"] == database_name_1
database_list[1]["Name"].should.equal(database_name_2) assert database_list[1]["Name"] == database_name_2
@mock_glue @mock_glue
@ -104,9 +100,9 @@ def test_update_database():
response = helpers.get_database(client, database_name) response = helpers.get_database(client, database_name)
database = response["Database"] database = response["Database"]
database.get("CatalogId").should.equal(database_catalog_id) assert database.get("CatalogId") == database_catalog_id
database.get("Description").should.be.none assert database.get("Description") is None
database.get("LocationUri").should.be.none assert database.get("LocationUri") is None
database_input = { database_input = {
"Name": database_name, "Name": database_name,
@ -119,9 +115,9 @@ def test_update_database():
response = helpers.get_database(client, database_name) response = helpers.get_database(client, database_name)
database = response["Database"] database = response["Database"]
database.get("CatalogId").should.equal(database_catalog_id) assert database.get("CatalogId") == database_catalog_id
database.get("Description").should.equal("desc") assert database.get("Description") == "desc"
database.get("LocationUri").should.equal("s3://bucket/existingdatabase/") assert database.get("LocationUri") == "s3://bucket/existingdatabase/"
@mock_glue @mock_glue
@ -131,8 +127,8 @@ def test_update_unknown_database():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.update_database(Name="x", DatabaseInput={"Name": "x"}) client.update_database(Name="x", DatabaseInput={"Name": "x"})
err = exc.value.response["Error"] err = exc.value.response["Error"]
err["Code"].should.equal("EntityNotFoundException") assert err["Code"] == "EntityNotFoundException"
err["Message"].should.equal("Database x not found.") assert err["Message"] == "Database x not found."
@mock_glue @mock_glue
@ -148,7 +144,7 @@ def test_delete_database():
database_list = sorted( database_list = sorted(
client.get_databases()["DatabaseList"], key=lambda x: x["Name"] client.get_databases()["DatabaseList"], key=lambda x: x["Name"]
) )
[db["Name"] for db in database_list].should.equal([database_name_2]) assert [db["Name"] for db in database_list] == [database_name_2]
@mock_glue @mock_glue
@ -158,8 +154,8 @@ def test_delete_unknown_database():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.delete_database(Name="x") client.delete_database(Name="x")
err = exc.value.response["Error"] err = exc.value.response["Error"]
err["Code"].should.equal("EntityNotFoundException") assert err["Code"] == "EntityNotFoundException"
err["Message"].should.equal("Database x not found.") assert err["Message"] == "Database x not found."
@mock_glue @mock_glue
@ -177,11 +173,11 @@ def test_create_table():
table = response["Table"] table = response["Table"]
if not settings.TEST_SERVER_MODE: if not settings.TEST_SERVER_MODE:
table["CreateTime"].timestamp().should.equal(FROZEN_CREATE_TIME.timestamp()) assert table["CreateTime"].timestamp() == FROZEN_CREATE_TIME.timestamp()
table["Name"].should.equal(table_input["Name"]) assert table["Name"] == table_input["Name"]
table["StorageDescriptor"].should.equal(table_input["StorageDescriptor"]) assert table["StorageDescriptor"] == table_input["StorageDescriptor"]
table["PartitionKeys"].should.equal(table_input["PartitionKeys"]) assert table["PartitionKeys"] == table_input["PartitionKeys"]
@mock_glue @mock_glue
@ -196,7 +192,7 @@ def test_create_table_already_exists():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.create_table(client, database_name, table_name) helpers.create_table(client, database_name, table_name)
exc.value.response["Error"]["Code"].should.equal("AlreadyExistsException") assert exc.value.response["Error"]["Code"] == "AlreadyExistsException"
@mock_glue @mock_glue
@ -217,15 +213,15 @@ def test_get_tables():
tables = response["TableList"] tables = response["TableList"]
tables.should.have.length_of(3) assert len(tables) == 3
for table in tables: for table in tables:
table_name = table["Name"] table_name = table["Name"]
table_name.should.equal(table_inputs[table_name]["Name"]) assert table_name == table_inputs[table_name]["Name"]
table["StorageDescriptor"].should.equal( assert (
table_inputs[table_name]["StorageDescriptor"] table["StorageDescriptor"] == table_inputs[table_name]["StorageDescriptor"]
) )
table["PartitionKeys"].should.equal(table_inputs[table_name]["PartitionKeys"]) assert table["PartitionKeys"] == table_inputs[table_name]["PartitionKeys"]
@mock_glue @mock_glue
@ -295,15 +291,15 @@ def test_get_tables_expression():
tables_star_expression5 = response_star_expression5["TableList"] tables_star_expression5 = response_star_expression5["TableList"]
tables_star_expression6 = response_star_expression6["TableList"] tables_star_expression6 = response_star_expression6["TableList"]
tables_prefix.should.have.length_of(2) assert len(tables_prefix) == 2
tables_postfix.should.have.length_of(1) assert len(tables_postfix) == 1
tables_string_match.should.have.length_of(3) assert len(tables_string_match) == 3
tables_star_expression1.should.have.length_of(8) assert len(tables_star_expression1) == 8
tables_star_expression2.should.have.length_of(2) assert len(tables_star_expression2) == 2
tables_star_expression3.should.have.length_of(3) assert len(tables_star_expression3) == 3
tables_star_expression4.should.have.length_of(3) assert len(tables_star_expression4) == 3
tables_star_expression5.should.have.length_of(3) assert len(tables_star_expression5) == 3
tables_star_expression6.should.have.length_of(2) assert len(tables_star_expression6) == 2
@mock_glue @mock_glue
@ -321,8 +317,8 @@ def test_get_table_versions():
# Get table should retrieve the first version # Get table should retrieve the first version
table = client.get_table(DatabaseName=database_name, Name=table_name)["Table"] table = client.get_table(DatabaseName=database_name, Name=table_name)["Table"]
table["StorageDescriptor"]["Columns"].should.equal([]) assert table["StorageDescriptor"]["Columns"] == []
table["VersionId"].should.equal("1") assert table["VersionId"] == "1"
columns = [{"Name": "country", "Type": "string"}] columns = [{"Name": "country", "Type": "string"}]
table_input = helpers.create_table_input(database_name, table_name, columns=columns) table_input = helpers.create_table_input(database_name, table_name, columns=columns)
@ -337,36 +333,36 @@ def test_get_table_versions():
vers = response["TableVersions"] vers = response["TableVersions"]
vers.should.have.length_of(3) assert len(vers) == 3
vers[0]["Table"]["StorageDescriptor"]["Columns"].should.equal([]) assert vers[0]["Table"]["StorageDescriptor"]["Columns"] == []
vers[-1]["Table"]["StorageDescriptor"]["Columns"].should.equal(columns) assert vers[-1]["Table"]["StorageDescriptor"]["Columns"] == columns
for n, ver in enumerate(vers): for n, ver in enumerate(vers):
n = str(n + 1) n = str(n + 1)
ver["VersionId"].should.equal(n) assert ver["VersionId"] == n
ver["Table"]["VersionId"].should.equal(n) assert ver["Table"]["VersionId"] == n
ver["Table"]["Name"].should.equal(table_name) assert ver["Table"]["Name"] == table_name
ver["Table"]["StorageDescriptor"].should.equal( assert (
version_inputs[n]["StorageDescriptor"] ver["Table"]["StorageDescriptor"] == version_inputs[n]["StorageDescriptor"]
) )
ver["Table"]["PartitionKeys"].should.equal(version_inputs[n]["PartitionKeys"]) assert ver["Table"]["PartitionKeys"] == version_inputs[n]["PartitionKeys"]
ver["Table"].should.have.key("UpdateTime") assert "UpdateTime" in ver["Table"]
response = helpers.get_table_version(client, database_name, table_name, "3") response = helpers.get_table_version(client, database_name, table_name, "3")
ver = response["TableVersion"] ver = response["TableVersion"]
ver["VersionId"].should.equal("3") assert ver["VersionId"] == "3"
ver["Table"]["Name"].should.equal(table_name) assert ver["Table"]["Name"] == table_name
ver["Table"]["StorageDescriptor"]["Columns"].should.equal(columns) assert ver["Table"]["StorageDescriptor"]["Columns"] == columns
# get_table should retrieve the latest version # get_table should retrieve the latest version
table = client.get_table(DatabaseName=database_name, Name=table_name)["Table"] table = client.get_table(DatabaseName=database_name, Name=table_name)["Table"]
table["StorageDescriptor"]["Columns"].should.equal(columns) assert table["StorageDescriptor"]["Columns"] == columns
table["VersionId"].should.equal("3") assert table["VersionId"] == "3"
table = client.get_tables(DatabaseName=database_name)["TableList"][0] table = client.get_tables(DatabaseName=database_name)["TableList"][0]
table["StorageDescriptor"]["Columns"].should.equal(columns) assert table["StorageDescriptor"]["Columns"] == columns
table["VersionId"].should.equal("3") assert table["VersionId"] == "3"
@mock_glue @mock_glue
@ -380,8 +376,8 @@ def test_get_table_version_not_found():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_table_version(client, database_name, "myfirsttable", "20") helpers.get_table_version(client, database_name, "myfirsttable", "20")
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match("version", re.I) assert exc.value.response["Error"]["Message"] == "Version not found."
@mock_glue @mock_glue
@ -395,7 +391,7 @@ def test_get_table_version_invalid_input():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_table_version(client, database_name, "myfirsttable", "10not-an-int") helpers.get_table_version(client, database_name, "myfirsttable", "10not-an-int")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
@mock_glue @mock_glue
@ -422,7 +418,7 @@ def test_delete_table_version():
response = helpers.get_table_versions(client, database_name, table_name) response = helpers.get_table_versions(client, database_name, table_name)
vers = response["TableVersions"] vers = response["TableVersions"]
vers.should.have.length_of(3) assert len(vers) == 3
client.delete_table_version( client.delete_table_version(
DatabaseName=database_name, TableName=table_name, VersionId="2" DatabaseName=database_name, TableName=table_name, VersionId="2"
@ -430,8 +426,8 @@ def test_delete_table_version():
response = helpers.get_table_versions(client, database_name, table_name) response = helpers.get_table_versions(client, database_name, table_name)
vers = response["TableVersions"] vers = response["TableVersions"]
vers.should.have.length_of(2) assert len(vers) == 2
[v["VersionId"] for v in vers].should.equal(["1", "3"]) assert [v["VersionId"] for v in vers] == ["1", "3"]
@mock_glue @mock_glue
@ -443,8 +439,8 @@ def test_get_table_not_exits():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_table(client, database_name, "myfirsttable") helpers.get_table(client, database_name, "myfirsttable")
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match("Table myfirsttable not found") assert exc.value.response["Error"]["Message"] == "Table myfirsttable not found."
@mock_glue @mock_glue
@ -455,9 +451,9 @@ def test_get_table_when_database_not_exits():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_table(client, database_name, "myfirsttable") helpers.get_table(client, database_name, "myfirsttable")
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert (
"Database nosuchdatabase not found" exc.value.response["Error"]["Message"] == "Database nosuchdatabase not found."
) )
@ -472,16 +468,14 @@ def test_delete_table():
helpers.create_table(client, database_name, table_name, table_input) helpers.create_table(client, database_name, table_name, table_input)
result = client.delete_table(DatabaseName=database_name, Name=table_name) result = client.delete_table(DatabaseName=database_name, Name=table_name)
result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) assert result["ResponseMetadata"]["HTTPStatusCode"] == 200
# confirm table is deleted # confirm table is deleted
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_table(client, database_name, table_name) helpers.get_table(client, database_name, table_name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert exc.value.response["Error"]["Message"] == "Table myspecialtable not found."
"Table myspecialtable not found"
)
@mock_glue @mock_glue
@ -497,16 +491,14 @@ def test_batch_delete_table():
result = client.batch_delete_table( result = client.batch_delete_table(
DatabaseName=database_name, TablesToDelete=[table_name] DatabaseName=database_name, TablesToDelete=[table_name]
) )
result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) assert result["ResponseMetadata"]["HTTPStatusCode"] == 200
# confirm table is deleted # confirm table is deleted
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_table(client, database_name, table_name) helpers.get_table(client, database_name, table_name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert exc.value.response["Error"]["Message"] == "Table myspecialtable not found."
"Table myspecialtable not found"
)
@mock_glue @mock_glue
@ -520,7 +512,7 @@ def test_get_partitions_empty():
response = client.get_partitions(DatabaseName=database_name, TableName=table_name) response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
response["Partitions"].should.have.length_of(0) assert len(response["Partitions"]) == 0
@mock_glue @mock_glue
@ -546,15 +538,15 @@ def test_create_partition():
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["StorageDescriptor"].should.equal(part_input["StorageDescriptor"]) assert partition["StorageDescriptor"] == part_input["StorageDescriptor"]
partition["Values"].should.equal(values) assert partition["Values"] == values
partition["CreationTime"].should.be.greater_than(before) assert partition["CreationTime"] > before
partition["CreationTime"].should.be.lower_than(after) assert partition["CreationTime"] < after
@mock_glue @mock_glue
@ -572,7 +564,7 @@ def test_create_partition_already_exist():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.create_partition(client, database_name, table_name, values=values) helpers.create_partition(client, database_name, table_name, values=values)
exc.value.response["Error"]["Code"].should.equal("AlreadyExistsException") assert exc.value.response["Error"]["Code"] == "AlreadyExistsException"
@mock_glue @mock_glue
@ -588,8 +580,8 @@ def test_get_partition_not_found():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_partition(client, database_name, table_name, values) helpers.get_partition(client, database_name, table_name, values)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match("partition") assert "partition" in exc.value.response["Error"]["Message"]
@mock_glue @mock_glue
@ -623,18 +615,16 @@ def test_batch_create_partition():
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(20) assert len(partitions) == 20
for idx, partition in enumerate(partitions): for idx, partition in enumerate(partitions):
partition_input = partition_inputs[idx] partition_input = partition_inputs[idx]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["StorageDescriptor"].should.equal( assert partition["StorageDescriptor"] == partition_input["StorageDescriptor"]
partition_input["StorageDescriptor"] assert partition["Values"] == partition_input["Values"]
) assert partition["CreationTime"] > before
partition["Values"].should.equal(partition_input["Values"]) assert partition["CreationTime"] < after
partition["CreationTime"].should.be.greater_than(before)
partition["CreationTime"].should.be.lower_than(after)
@mock_glue @mock_glue
@ -659,12 +649,9 @@ def test_batch_create_partition_already_exist():
PartitionInputList=[partition_input], PartitionInputList=[partition_input],
) )
response.should.have.key("Errors") assert len(response["Errors"]) == 1
response["Errors"].should.have.length_of(1) assert response["Errors"][0]["PartitionValues"] == values
response["Errors"][0]["PartitionValues"].should.equal(values) assert response["Errors"][0]["ErrorDetail"]["ErrorCode"] == "AlreadyExistsException"
response["Errors"][0]["ErrorDetail"]["ErrorCode"].should.equal(
"AlreadyExistsException"
)
@mock_glue @mock_glue
@ -687,8 +674,8 @@ def test_get_partition():
partition = response["Partition"] partition = response["Partition"]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["Values"].should.equal(values[1]) assert partition["Values"] == values[1]
@mock_glue @mock_glue
@ -713,11 +700,11 @@ def test_batch_get_partition():
) )
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(2) assert len(partitions) == 2
partition = partitions[1] partition = partitions[1]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["Values"].should.equal(values[1]) assert partition["Values"] == values[1]
@mock_glue @mock_glue
@ -746,10 +733,10 @@ def test_batch_get_partition_missing_partition():
) )
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(2) assert len(partitions) == 2
partitions[0]["Values"].should.equal(values[0]) assert partitions[0]["Values"] == values[0]
partitions[1]["Values"].should.equal(values[2]) assert partitions[1]["Values"] == values[2]
@mock_glue @mock_glue
@ -770,8 +757,8 @@ def test_update_partition_not_found_moving():
values=["2018-10-02"], values=["2018-10-02"],
) )
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match("partition") assert "partition" in exc.value.response["Error"]["Message"]
@mock_glue @mock_glue
@ -789,8 +776,8 @@ def test_update_partition_not_found_change_in_place():
client, database_name, table_name, old_values=values, values=values client, database_name, table_name, old_values=values, values=values
) )
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match("partition") assert "partition" in exc.value.response["Error"]["Message"]
@mock_glue @mock_glue
@ -812,7 +799,7 @@ def test_update_partition_cannot_overwrite():
client, database_name, table_name, old_values=values[0], values=values[1] client, database_name, table_name, old_values=values[0], values=values[1]
) )
exc.value.response["Error"]["Code"].should.equal("AlreadyExistsException") assert exc.value.response["Error"]["Code"] == "AlreadyExistsException"
@mock_glue @mock_glue
@ -840,10 +827,10 @@ def test_update_partition():
) )
partition = response["Partition"] partition = response["Partition"]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["StorageDescriptor"]["Columns"].should.equal( assert partition["StorageDescriptor"]["Columns"] == [
[{"Name": "country", "Type": "string"}] {"Name": "country", "Type": "string"}
) ]
@mock_glue @mock_glue
@ -871,17 +858,17 @@ def test_update_partition_move():
helpers.get_partition(client, database_name, table_name, values) helpers.get_partition(client, database_name, table_name, values)
# Old partition shouldn't exist anymore # Old partition shouldn't exist anymore
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
response = client.get_partition( response = client.get_partition(
DatabaseName=database_name, TableName=table_name, PartitionValues=new_values DatabaseName=database_name, TableName=table_name, PartitionValues=new_values
) )
partition = response["Partition"] partition = response["Partition"]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["StorageDescriptor"]["Columns"].should.equal( assert partition["StorageDescriptor"]["Columns"] == [
[{"Name": "country", "Type": "string"}] {"Name": "country", "Type": "string"}
) ]
@mock_glue @mock_glue
@ -927,7 +914,7 @@ def test_batch_update_partition():
for value in values: for value in values:
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.get_partition(client, database_name, table_name, value) helpers.get_partition(client, database_name, table_name, value)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
for value in new_values: for value in new_values:
response = client.get_partition( response = client.get_partition(
@ -935,10 +922,10 @@ def test_batch_update_partition():
) )
partition = response["Partition"] partition = response["Partition"]
partition["TableName"].should.equal(table_name) assert partition["TableName"] == table_name
partition["StorageDescriptor"]["Columns"].should.equal( assert partition["StorageDescriptor"]["Columns"] == [
[{"Name": "country", "Type": "string"}] {"Name": "country", "Type": "string"}
) ]
@mock_glue @mock_glue
@ -992,9 +979,8 @@ def test_batch_update_partition_missing_partition():
DatabaseName=database_name, TableName=table_name, Entries=batch_update_values DatabaseName=database_name, TableName=table_name, Entries=batch_update_values
) )
response.should.have.key("Errors") assert len(response["Errors"]) == 1
response["Errors"].should.have.length_of(1) assert response["Errors"][0]["PartitionValueList"] == ["2020-10-10"]
response["Errors"][0]["PartitionValueList"].should.equal(["2020-10-10"])
@mock_glue @mock_glue
@ -1017,7 +1003,7 @@ def test_delete_partition():
response = client.get_partitions(DatabaseName=database_name, TableName=table_name) response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.equal([]) assert partitions == []
@mock_glue @mock_glue
@ -1034,7 +1020,7 @@ def test_delete_partition_bad_partition():
DatabaseName=database_name, TableName=table_name, PartitionValues=values DatabaseName=database_name, TableName=table_name, PartitionValues=values
) )
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
@mock_glue @mock_glue
@ -1067,7 +1053,7 @@ def test_batch_delete_partition():
PartitionsToDelete=partition_values, PartitionsToDelete=partition_values,
) )
response.should_not.have.key("Errors") assert "Errors" not in response
@mock_glue @mock_glue
@ -1104,12 +1090,11 @@ def test_batch_delete_partition_with_bad_partitions():
PartitionsToDelete=partition_values, PartitionsToDelete=partition_values,
) )
response.should.have.key("Errors") assert len(response["Errors"]) == 3
response["Errors"].should.have.length_of(3)
error_partitions = map(lambda x: x["PartitionValues"], response["Errors"]) error_partitions = map(lambda x: x["PartitionValues"], response["Errors"])
["2018-11-01"].should.be.within(error_partitions) assert ["2018-11-01"] in error_partitions
["2018-11-02"].should.be.within(error_partitions) assert ["2018-11-02"] in error_partitions
["2018-11-03"].should.be.within(error_partitions) assert ["2018-11-03"] in error_partitions
@mock_glue @mock_glue
@ -1168,36 +1153,28 @@ def test_create_crawler_scheduled():
response = client.get_crawler(Name=name) response = client.get_crawler(Name=name)
crawler = response["Crawler"] crawler = response["Crawler"]
crawler.get("Name").should.equal(name) assert crawler.get("Name") == name
crawler.get("Role").should.equal(role) assert crawler.get("Role") == role
crawler.get("DatabaseName").should.equal(database_name) assert crawler.get("DatabaseName") == database_name
crawler.get("Description").should.equal(description) assert crawler.get("Description") == description
crawler.get("Targets").should.equal(targets) assert crawler.get("Targets") == targets
crawler.get("Schedule").should.equal( assert crawler["Schedule"] == {"ScheduleExpression": schedule, "State": "SCHEDULED"}
{"ScheduleExpression": schedule, "State": "SCHEDULED"} assert crawler.get("Classifiers") == classifiers
) assert crawler.get("TablePrefix") == table_prefix
crawler.get("Classifiers").should.equal(classifiers) assert crawler.get("SchemaChangePolicy") == schema_change_policy
crawler.get("TablePrefix").should.equal(table_prefix) assert crawler.get("RecrawlPolicy") == recrawl_policy
crawler.get("SchemaChangePolicy").should.equal(schema_change_policy) assert crawler.get("LineageConfiguration") == lineage_configuration
crawler.get("RecrawlPolicy").should.equal(recrawl_policy) assert crawler.get("Configuration") == configuration
crawler.get("LineageConfiguration").should.equal(lineage_configuration) assert crawler["CrawlerSecurityConfiguration"] == crawler_security_configuration
crawler.get("Configuration").should.equal(configuration)
crawler.get("CrawlerSecurityConfiguration").should.equal(
crawler_security_configuration
)
crawler.get("State").should.equal("READY") assert crawler.get("State") == "READY"
crawler.get("CrawlElapsedTime").should.equal(0) assert crawler.get("CrawlElapsedTime") == 0
crawler.get("Version").should.equal(1) assert crawler.get("Version") == 1
if not settings.TEST_SERVER_MODE: if not settings.TEST_SERVER_MODE:
crawler.get("CreationTime").timestamp().should.equal( assert crawler["CreationTime"].timestamp() == FROZEN_CREATE_TIME.timestamp()
FROZEN_CREATE_TIME.timestamp() assert crawler["LastUpdated"].timestamp() == FROZEN_CREATE_TIME.timestamp()
)
crawler.get("LastUpdated").timestamp().should.equal(
FROZEN_CREATE_TIME.timestamp()
)
crawler.should.not_have.key("LastCrawl") assert "LastCrawl" not in crawler
@mock_glue @mock_glue
@ -1254,34 +1231,28 @@ def test_create_crawler_unscheduled():
response = client.get_crawler(Name=name) response = client.get_crawler(Name=name)
crawler = response["Crawler"] crawler = response["Crawler"]
crawler.get("Name").should.equal(name) assert crawler.get("Name") == name
crawler.get("Role").should.equal(role) assert crawler.get("Role") == role
crawler.get("DatabaseName").should.equal(database_name) assert crawler.get("DatabaseName") == database_name
crawler.get("Description").should.equal(description) assert crawler.get("Description") == description
crawler.get("Targets").should.equal(targets) assert crawler.get("Targets") == targets
crawler.should.not_have.key("Schedule") assert "Schedule" not in crawler
crawler.get("Classifiers").should.equal(classifiers) assert crawler.get("Classifiers") == classifiers
crawler.get("TablePrefix").should.equal(table_prefix) assert crawler.get("TablePrefix") == table_prefix
crawler.get("SchemaChangePolicy").should.equal(schema_change_policy) assert crawler.get("SchemaChangePolicy") == schema_change_policy
crawler.get("RecrawlPolicy").should.equal(recrawl_policy) assert crawler.get("RecrawlPolicy") == recrawl_policy
crawler.get("LineageConfiguration").should.equal(lineage_configuration) assert crawler.get("LineageConfiguration") == lineage_configuration
crawler.get("Configuration").should.equal(configuration) assert crawler.get("Configuration") == configuration
crawler.get("CrawlerSecurityConfiguration").should.equal( assert crawler["CrawlerSecurityConfiguration"] == crawler_security_configuration
crawler_security_configuration
)
crawler.get("State").should.equal("READY") assert crawler.get("State") == "READY"
crawler.get("CrawlElapsedTime").should.equal(0) assert crawler.get("CrawlElapsedTime") == 0
crawler.get("Version").should.equal(1) assert crawler.get("Version") == 1
if not settings.TEST_SERVER_MODE: if not settings.TEST_SERVER_MODE:
crawler.get("CreationTime").timestamp().should.equal( assert crawler["CreationTime"].timestamp() == FROZEN_CREATE_TIME.timestamp()
FROZEN_CREATE_TIME.timestamp() assert crawler["LastUpdated"].timestamp() == FROZEN_CREATE_TIME.timestamp()
)
crawler.get("LastUpdated").timestamp().should.equal(
FROZEN_CREATE_TIME.timestamp()
)
crawler.should.not_have.key("LastCrawl") assert "LastCrawl" not in crawler
@mock_glue @mock_glue
@ -1293,7 +1264,7 @@ def test_create_crawler_already_exists():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
helpers.create_crawler(client, name) helpers.create_crawler(client, name)
exc.value.response["Error"]["Code"].should.equal("AlreadyExistsException") assert exc.value.response["Error"]["Code"] == "AlreadyExistsException"
@mock_glue @mock_glue
@ -1304,9 +1275,9 @@ def test_get_crawler_not_exits():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_crawler(Name=name) client.get_crawler(Name=name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert (
"Crawler my_crawler_name not found" exc.value.response["Error"]["Message"] == "Crawler my_crawler_name not found."
) )
@ -1314,7 +1285,7 @@ def test_get_crawler_not_exits():
def test_get_crawlers_empty(): def test_get_crawlers_empty():
client = boto3.client("glue", region_name="us-east-1") client = boto3.client("glue", region_name="us-east-1")
response = client.get_crawlers() response = client.get_crawlers()
response["Crawlers"].should.have.length_of(0) assert len(response["Crawlers"]) == 0
@mock_glue @mock_glue
@ -1326,9 +1297,9 @@ def test_get_crawlers_several_items():
helpers.create_crawler(client, name_2) helpers.create_crawler(client, name_2)
crawlers = sorted(client.get_crawlers()["Crawlers"], key=lambda x: x["Name"]) crawlers = sorted(client.get_crawlers()["Crawlers"], key=lambda x: x["Name"])
crawlers.should.have.length_of(2) assert len(crawlers) == 2
crawlers[0].get("Name").should.equal(name_1) assert crawlers[0].get("Name") == name_1
crawlers[1].get("Name").should.equal(name_2) assert crawlers[1].get("Name") == name_2
@mock_glue @mock_glue
@ -1342,7 +1313,7 @@ def test_start_crawler():
response = client.get_crawler(Name=name) response = client.get_crawler(Name=name)
crawler = response["Crawler"] crawler = response["Crawler"]
crawler.get("State").should.equal("RUNNING") assert crawler.get("State") == "RUNNING"
@mock_glue @mock_glue
@ -1355,7 +1326,7 @@ def test_start_crawler_should_raise_exception_if_already_running():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.start_crawler(Name=name) client.start_crawler(Name=name)
exc.value.response["Error"]["Code"].should.equal("CrawlerRunningException") assert exc.value.response["Error"]["Code"] == "CrawlerRunningException"
@mock_glue @mock_glue
@ -1370,7 +1341,7 @@ def test_stop_crawler():
response = client.get_crawler(Name=name) response = client.get_crawler(Name=name)
crawler = response["Crawler"] crawler = response["Crawler"]
crawler.get("State").should.equal("STOPPING") assert crawler.get("State") == "STOPPING"
@mock_glue @mock_glue
@ -1382,7 +1353,7 @@ def test_stop_crawler_should_raise_exception_if_not_running():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.stop_crawler(Name=name) client.stop_crawler(Name=name)
exc.value.response["Error"]["Code"].should.equal("CrawlerNotRunningException") assert exc.value.response["Error"]["Code"] == "CrawlerNotRunningException"
@mock_glue @mock_glue
@ -1392,15 +1363,15 @@ def test_delete_crawler():
helpers.create_crawler(client, name) helpers.create_crawler(client, name)
result = client.delete_crawler(Name=name) result = client.delete_crawler(Name=name)
result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) assert result["ResponseMetadata"]["HTTPStatusCode"] == 200
# confirm crawler is deleted # confirm crawler is deleted
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_crawler(Name=name) client.get_crawler(Name=name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert (
"Crawler my_crawler_name not found" exc.value.response["Error"]["Message"] == "Crawler my_crawler_name not found."
) )
@ -1412,7 +1383,7 @@ def test_delete_crawler_not_exists():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.delete_crawler(Name=name) client.delete_crawler(Name=name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match( assert (
"Crawler my_crawler_name not found" exc.value.response["Error"]["Message"] == "Crawler my_crawler_name not found."
) )

73
tests/test_glue/test_glue.py
View File

@ -4,7 +4,6 @@ from uuid import uuid4
import boto3 import boto3
import pytest import pytest
import sure # noqa # pylint: disable=unused-import
from botocore.exceptions import ParamValidationError from botocore.exceptions import ParamValidationError
from botocore.client import ClientError from botocore.client import ClientError
@ -42,9 +41,7 @@ def test_create_job_default_argument_not_provided():
with pytest.raises(ParamValidationError) as exc: with pytest.raises(ParamValidationError) as exc:
client.create_job(Role="test_role", Command=dict(Name="test_command")) client.create_job(Role="test_role", Command=dict(Name="test_command"))
exc.value.kwargs["report"].should.equal( assert exc.value.kwargs["report"] == 'Missing required parameter in input: "Name"'
'Missing required parameter in input: "Name"'
)
@mock_glue @mock_glue
@ -53,8 +50,8 @@ def test_list_jobs():
expected_jobs = randint(1, 15) expected_jobs = randint(1, 15)
create_test_jobs(client, expected_jobs) create_test_jobs(client, expected_jobs)
response = client.list_jobs() response = client.list_jobs()
response["JobNames"].should.have.length_of(expected_jobs) assert len(response["JobNames"]) == expected_jobs
response.shouldnt.have.key("NextToken") assert "NextToken" not in response
@mock_glue @mock_glue
@ -65,8 +62,8 @@ def test_get_job_not_exists():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_job(JobName=name) client.get_job(JobName=name)
exc.value.response["Error"]["Code"].should.equal("EntityNotFoundException") assert exc.value.response["Error"]["Code"] == "EntityNotFoundException"
exc.value.response["Error"]["Message"].should.match("Job my_job_name not found") assert exc.value.response["Error"]["Message"] == "Job my_job_name not found."
@mock_glue @mock_glue
@ -135,8 +132,8 @@ def test_get_jobs_job_name_exists():
client = create_glue_client() client = create_glue_client()
test_job_name = create_test_job(client) test_job_name = create_test_job(client)
response = client.get_jobs() response = client.get_jobs()
response["Jobs"].should.have.length_of(1) assert len(response["Jobs"]) == 1
response["Jobs"][0].should.have.key("Name").equals(test_job_name) assert response["Jobs"][0]["Name"] == test_job_name
@mock_glue @mock_glue
@ -144,8 +141,8 @@ def test_get_jobs_with_max_results():
client = create_glue_client() client = create_glue_client()
create_test_jobs(client, 4) create_test_jobs(client, 4)
response = client.get_jobs(MaxResults=2) response = client.get_jobs(MaxResults=2)
response["Jobs"].should.have.length_of(2) assert len(response["Jobs"]) == 2
response.should.have.key("NextToken") assert "NextToken" in response
@mock_glue @mock_glue
@ -154,7 +151,7 @@ def test_get_jobs_from_next_token():
create_test_jobs(client, 10) create_test_jobs(client, 10)
first_response = client.get_jobs(MaxResults=3) first_response = client.get_jobs(MaxResults=3)
response = client.get_jobs(NextToken=first_response["NextToken"]) response = client.get_jobs(NextToken=first_response["NextToken"])
response["Jobs"].should.have.length_of(7) assert len(response["Jobs"]) == 7
@mock_glue @mock_glue
@ -162,7 +159,7 @@ def test_get_jobs_with_max_results_greater_than_actual_results():
client = create_glue_client() client = create_glue_client()
create_test_jobs(client, 4) create_test_jobs(client, 4)
response = client.get_jobs(MaxResults=10) response = client.get_jobs(MaxResults=10)
response["Jobs"].should.have.length_of(4) assert len(response["Jobs"]) == 4
@mock_glue @mock_glue
@ -182,8 +179,8 @@ def test_list_jobs_with_max_results():
client = create_glue_client() client = create_glue_client()
create_test_jobs(client, 4) create_test_jobs(client, 4)
response = client.list_jobs(MaxResults=2) response = client.list_jobs(MaxResults=2)
response["JobNames"].should.have.length_of(2) assert len(response["JobNames"]) == 2
response.should.have.key("NextToken") assert "NextToken" in response
@mock_glue @mock_glue
@ -192,7 +189,7 @@ def test_list_jobs_from_next_token():
create_test_jobs(client, 10) create_test_jobs(client, 10)
first_response = client.list_jobs(MaxResults=3) first_response = client.list_jobs(MaxResults=3)
response = client.list_jobs(NextToken=first_response["NextToken"]) response = client.list_jobs(NextToken=first_response["NextToken"])
response["JobNames"].should.have.length_of(7) assert len(response["JobNames"]) == 7
@mock_glue @mock_glue
@ -200,7 +197,7 @@ def test_list_jobs_with_max_results_greater_than_actual_results():
client = create_glue_client() client = create_glue_client()
create_test_jobs(client, 4) create_test_jobs(client, 4)
response = client.list_jobs(MaxResults=10) response = client.list_jobs(MaxResults=10)
response["JobNames"].should.have.length_of(4) assert len(response["JobNames"]) == 4
@mock_glue @mock_glue
@ -209,7 +206,7 @@ def test_list_jobs_with_tags():
create_test_job(client) create_test_job(client)
create_test_job(client, {"string": "string"}) create_test_job(client, {"string": "string"})
response = client.list_jobs(Tags={"string": "string"}) response = client.list_jobs(Tags={"string": "string"})
response["JobNames"].should.have.length_of(1) assert len(response["JobNames"]) == 1
@mock_glue @mock_glue
@ -221,7 +218,7 @@ def test_list_jobs_after_tagging():
client.tag_resource(ResourceArn=resource_arn, TagsToAdd={"key1": "value1"}) client.tag_resource(ResourceArn=resource_arn, TagsToAdd={"key1": "value1"})
response = client.list_jobs(Tags={"key1": "value1"}) response = client.list_jobs(Tags={"key1": "value1"})
response["JobNames"].should.have.length_of(1) assert len(response["JobNames"]) == 1
@mock_glue @mock_glue
@ -233,7 +230,7 @@ def test_list_jobs_after_removing_tag():
client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key1"]) client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key1"])
response = client.list_jobs(Tags={"key1": "value1"}) response = client.list_jobs(Tags={"key1": "value1"})
response["JobNames"].should.have.length_of(0) assert len(response["JobNames"]) == 0
@mock_glue @mock_glue
@ -306,8 +303,8 @@ def test_list_crawlers_with_max_results():
client = create_glue_client() client = create_glue_client()
create_test_crawlers(client, 4) create_test_crawlers(client, 4)
response = client.list_crawlers(MaxResults=2) response = client.list_crawlers(MaxResults=2)
response["CrawlerNames"].should.have.length_of(2) assert len(response["CrawlerNames"]) == 2
response.should.have.key("NextToken") assert "NextToken" in response
@mock_glue @mock_glue
@ -316,7 +313,7 @@ def test_list_crawlers_from_next_token():
create_test_crawlers(client, 10) create_test_crawlers(client, 10)
first_response = client.list_crawlers(MaxResults=3) first_response = client.list_crawlers(MaxResults=3)
response = client.list_crawlers(NextToken=first_response["NextToken"]) response = client.list_crawlers(NextToken=first_response["NextToken"])
response["CrawlerNames"].should.have.length_of(7) assert len(response["CrawlerNames"]) == 7
@mock_glue @mock_glue
@ -324,7 +321,7 @@ def test_list_crawlers_with_max_results_greater_than_actual_results():
client = create_glue_client() client = create_glue_client()
create_test_crawlers(client, 4) create_test_crawlers(client, 4)
response = client.list_crawlers(MaxResults=10) response = client.list_crawlers(MaxResults=10)
response["CrawlerNames"].should.have.length_of(4) assert len(response["CrawlerNames"]) == 4
@mock_glue @mock_glue
@ -333,7 +330,7 @@ def test_list_crawlers_with_tags():
create_test_crawler(client) create_test_crawler(client)
create_test_crawler(client, {"string": "string"}) create_test_crawler(client, {"string": "string"})
response = client.list_crawlers(Tags={"string": "string"}) response = client.list_crawlers(Tags={"string": "string"})
response["CrawlerNames"].should.have.length_of(1) assert len(response["CrawlerNames"]) == 1
@mock_glue @mock_glue
@ -345,7 +342,7 @@ def test_list_crawlers_after_tagging():
client.tag_resource(ResourceArn=resource_arn, TagsToAdd={"key1": "value1"}) client.tag_resource(ResourceArn=resource_arn, TagsToAdd={"key1": "value1"})
response = client.list_crawlers(Tags={"key1": "value1"}) response = client.list_crawlers(Tags={"key1": "value1"})
response["CrawlerNames"].should.have.length_of(1) assert len(response["CrawlerNames"]) == 1
@mock_glue @mock_glue
@ -357,7 +354,7 @@ def test_list_crawlers_after_removing_tag():
client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key1"]) client.untag_resource(ResourceArn=resource_arn, TagsToRemove=["key1"])
response = client.list_crawlers(Tags={"key1": "value1"}) response = client.list_crawlers(Tags={"key1": "value1"})
response["CrawlerNames"].should.have.length_of(0) assert len(response["CrawlerNames"]) == 0
@mock_glue @mock_glue
@ -380,7 +377,7 @@ def test_get_tags_job():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) assert resp["Tags"] == {"key1": "value1", "key2": "value2"}
@mock_glue @mock_glue
@ -391,7 +388,7 @@ def test_get_tags_jobs_no_tags():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({}) assert resp["Tags"] == {}
@mock_glue @mock_glue
@ -406,7 +403,7 @@ def test_tag_glue_job():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) assert resp["Tags"] == {"key1": "value1", "key2": "value2"}
@mock_glue @mock_glue
@ -424,7 +421,7 @@ def test_untag_glue_job():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({"key1": "value1", "key3": "value3"}) assert resp["Tags"] == {"key1": "value1", "key3": "value3"}
@mock_glue @mock_glue
@ -435,7 +432,7 @@ def test_get_tags_crawler():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) assert resp["Tags"] == {"key1": "value1", "key2": "value2"}
@mock_glue @mock_glue
@ -446,7 +443,7 @@ def test_get_tags_crawler_no_tags():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({}) assert resp["Tags"] == {}
@mock_glue @mock_glue
@ -461,7 +458,7 @@ def test_tag_glue_crawler():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({"key1": "value1", "key2": "value2"}) assert resp["Tags"] == {"key1": "value1", "key2": "value2"}
@mock_glue @mock_glue
@ -479,7 +476,7 @@ def test_untag_glue_crawler():
resp = client.get_tags(ResourceArn=resource_arn) resp = client.get_tags(ResourceArn=resource_arn)
resp.should.have.key("Tags").equals({"key1": "value1", "key3": "value3"}) assert resp["Tags"] == {"key1": "value1", "key3": "value3"}
@mock_glue @mock_glue
@ -491,8 +488,8 @@ def test_batch_get_crawlers():
CrawlerNames=[crawler_name, "crawler-not-found"] CrawlerNames=[crawler_name, "crawler-not-found"]
) )
response["Crawlers"].should.have.length_of(1) assert len(response["Crawlers"]) == 1
response["CrawlersNotFound"].should.have.length_of(1) assert len(response["CrawlersNotFound"]) == 1
@mock_glue @mock_glue

28
tests/test_glue/test_glue_job_runs.py
View File

@ -1,5 +1,4 @@
import pytest import pytest
import sure # noqa # pylint: disable=unused-import
from botocore.client import ClientError from botocore.client import ClientError
from unittest import SkipTest from unittest import SkipTest
@ -42,9 +41,10 @@ def test_start_job_run__multiple_runs_allowed():
# The 6th should fail # The 6th should fail
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
glue.start_job_run(JobName="somejobname") glue.start_job_run(JobName="somejobname")
exc.value.response["Error"]["Code"].should.equal("ConcurrentRunsExceededException") assert exc.value.response["Error"]["Code"] == "ConcurrentRunsExceededException"
exc.value.response["Error"]["Message"].should.match( assert (
"Job with name somejobname already running" exc.value.response["Error"]["Message"]
== "Job with name somejobname already running"
) )
@ -62,9 +62,10 @@ def test_start_job_run__single_run_allowed():
client.start_job_run(JobName=job_name) client.start_job_run(JobName=job_name)
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.start_job_run(JobName=job_name) client.start_job_run(JobName=job_name)
exc.value.response["Error"]["Code"].should.equal("ConcurrentRunsExceededException") assert exc.value.response["Error"]["Code"] == "ConcurrentRunsExceededException"
exc.value.response["Error"]["Message"].should.match( assert (
f"Job with name {job_name} already running" exc.value.response["Error"]["Message"]
== f"Job with name {job_name} already running"
) )
@ -76,14 +77,14 @@ def test_get_job_run():
job_run_id = client.start_job_run(JobName=job_name)["JobRunId"] job_run_id = client.start_job_run(JobName=job_name)["JobRunId"]
response = client.get_job_run(JobName=job_name, RunId=job_run_id) response = client.get_job_run(JobName=job_name, RunId=job_run_id)
response["JobRun"].should.have.key("Id").equals(job_run_id) assert response["JobRun"]["Id"] == job_run_id
assert response["JobRun"]["Attempt"] assert response["JobRun"]["Attempt"]
assert response["JobRun"]["PreviousRunId"] assert response["JobRun"]["PreviousRunId"]
assert response["JobRun"]["TriggerName"] assert response["JobRun"]["TriggerName"]
assert response["JobRun"]["StartedOn"] assert response["JobRun"]["StartedOn"]
assert response["JobRun"]["LastModifiedOn"] assert response["JobRun"]["LastModifiedOn"]
assert response["JobRun"]["CompletedOn"] assert response["JobRun"]["CompletedOn"]
response["JobRun"].should.have.key("JobRunState").equals("SUCCEEDED") assert response["JobRun"]["JobRunState"] == "SUCCEEDED"
assert response["JobRun"]["Arguments"] assert response["JobRun"]["Arguments"]
assert response["JobRun"]["ErrorMessage"] == "" assert response["JobRun"]["ErrorMessage"] == ""
assert response["JobRun"]["PredecessorRuns"] assert response["JobRun"]["PredecessorRuns"]
@ -106,7 +107,7 @@ def test_get_job_run_that_doesnt_exist():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_job_run(JobName=job_name, RunId="unknown") client.get_job_run(JobName=job_name, RunId="unknown")
err = exc.value.response["Error"] err = exc.value.response["Error"]
err["Code"].should.equal("EntityNotFoundException") assert err["Code"] == "EntityNotFoundException"
@mock_glue @mock_glue
@ -135,6 +136,7 @@ def test_job_run_transition():
def expect_job_state(client, job_name, run_id, expected_state): def expect_job_state(client, job_name, run_id, expected_state):
client.get_job_run(JobName=job_name, RunId=run_id)["JobRun"][ assert (
"JobRunState" client.get_job_run(JobName=job_name, RunId=run_id)["JobRun"]["JobRunState"]
].should.equal(expected_state) == expected_state
)

95
tests/test_glue/test_partition_filter.py
View File

@ -2,7 +2,6 @@ from unittest import SkipTest
import boto3 import boto3
import pytest import pytest
import sure # noqa # pylint: disable=unused-import
from botocore.client import ClientError from botocore.client import ClientError
from moto import mock_glue, settings from moto import mock_glue, settings
@ -32,8 +31,8 @@ def test_get_partitions_expression_unknown_column():
Expression="unknown_col IS NULL", Expression="unknown_col IS NULL",
) )
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("Unknown column 'unknown_col'") assert "Unknown column 'unknown_col'" in exc.value.response["Error"]["Message"]
@mock_glue @mock_glue
@ -55,7 +54,7 @@ def test_get_partitions_expression_int_column():
response = client.get_partitions(**kwargs) response = client.get_partitions(**kwargs)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(3) assert len(partitions) == 3
int_col_is_two_expressions = ( int_col_is_two_expressions = (
"int_col = 2", "int_col = 2",
@ -75,24 +74,25 @@ def test_get_partitions_expression_int_column():
for expression in int_col_is_two_expressions: for expression in int_col_is_two_expressions:
response = client.get_partitions(**kwargs, Expression=expression) response = client.get_partitions(**kwargs, Expression=expression)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["Values"].should.equal(["2"]) assert partition["Values"] == ["2"]
bad_int_expressions = ("int_col = 'test'", "int_col in (2.5)") bad_int_expressions = ("int_col = 'test'", "int_col in (2.5)")
for expression in bad_int_expressions: for expression in bad_int_expressions:
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression=expression) client.get_partitions(**kwargs, Expression=expression)
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("is not an integer") assert "is not an integer" in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="int_col LIKE '2'") client.get_partitions(**kwargs, Expression="int_col LIKE '2'")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match( assert (
"Integral data type doesn't support operation 'LIKE'" "Integral data type doesn't support operation 'LIKE'"
in exc.value.response["Error"]["Message"]
) )
@ -115,7 +115,7 @@ def test_get_partitions_expression_decimal_column():
response = client.get_partitions(**kwargs) response = client.get_partitions(**kwargs)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(3) assert len(partitions) == 3
decimal_col_is_two_point_six_expressions = ( decimal_col_is_two_point_six_expressions = (
"decimal_col = 2.6", "decimal_col = 2.6",
@ -130,24 +130,25 @@ def test_get_partitions_expression_decimal_column():
for expression in decimal_col_is_two_point_six_expressions: for expression in decimal_col_is_two_point_six_expressions:
response = client.get_partitions(**kwargs, Expression=expression) response = client.get_partitions(**kwargs, Expression=expression)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["Values"].should.equal(["2.6"]) assert partition["Values"] == ["2.6"]
bad_decimal_expressions = ("decimal_col = 'test'",) bad_decimal_expressions = ("decimal_col = 'test'",)
for expression in bad_decimal_expressions: for expression in bad_decimal_expressions:
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression=expression) client.get_partitions(**kwargs, Expression=expression)
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("is not a decimal") assert "is not a decimal" in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="decimal_col LIKE '2'") client.get_partitions(**kwargs, Expression="decimal_col LIKE '2'")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match( assert (
"Decimal data type doesn't support operation 'LIKE'" "Decimal data type doesn't support operation 'LIKE'"
in exc.value.response["Error"]["Message"]
) )
@ -171,7 +172,7 @@ def test_get_partitions_expression_string_column():
response = client.get_partitions(**kwargs) response = client.get_partitions(**kwargs)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(4) assert len(partitions) == 4
string_col_is_two_expressions = ( string_col_is_two_expressions = (
"string_col = 'two'", "string_col = 'two'",
@ -191,15 +192,15 @@ def test_get_partitions_expression_string_column():
for expression in string_col_is_two_expressions: for expression in string_col_is_two_expressions:
response = client.get_partitions(**kwargs, Expression=expression) response = client.get_partitions(**kwargs, Expression=expression)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["Values"].should.be.within((["two"], ["2"])) assert partition["Values"] in [["two"], ["2"]]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="unknown_col LIKE 'two'") client.get_partitions(**kwargs, Expression="unknown_col LIKE 'two'")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("Unknown column 'unknown_col'") assert "Unknown column 'unknown_col'" in exc.value.response["Error"]["Message"]
@mock_glue @mock_glue
@ -221,7 +222,7 @@ def test_get_partitions_expression_date_column():
response = client.get_partitions(**kwargs) response = client.get_partitions(**kwargs)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(3) assert len(partitions) == 3
date_col_is_february_expressions = ( date_col_is_february_expressions = (
"date_col = '2022-02-01'", "date_col = '2022-02-01'",
@ -234,24 +235,25 @@ def test_get_partitions_expression_date_column():
for expression in date_col_is_february_expressions: for expression in date_col_is_february_expressions:
response = client.get_partitions(**kwargs, Expression=expression) response = client.get_partitions(**kwargs, Expression=expression)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["Values"].should.equal(["2022-02-01"]) assert partition["Values"] == ["2022-02-01"]
bad_date_expressions = ("date_col = 'test'", "date_col = '2022-02-32'") bad_date_expressions = ("date_col = 'test'", "date_col = '2022-02-32'")
for expression in bad_date_expressions: for expression in bad_date_expressions:
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression=expression) client.get_partitions(**kwargs, Expression=expression)
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("is not a date") assert "is not a date" in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="date_col LIKE '2022-02-01'") client.get_partitions(**kwargs, Expression="date_col LIKE '2022-02-01'")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match( assert (
"Date data type doesn't support operation 'LIKE'" "Date data type doesn't support operation 'LIKE'"
in exc.value.response["Error"]["Message"]
) )
@ -278,7 +280,7 @@ def test_get_partitions_expression_timestamp_column():
response = client.get_partitions(**kwargs) response = client.get_partitions(**kwargs)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(3) assert len(partitions) == 3
timestamp_col_is_february_expressions = ( timestamp_col_is_february_expressions = (
"timestamp_col = '2022-02-01 00:00:00'", "timestamp_col = '2022-02-01 00:00:00'",
@ -297,9 +299,9 @@ def test_get_partitions_expression_timestamp_column():
for expression in timestamp_col_is_february_expressions: for expression in timestamp_col_is_february_expressions:
response = client.get_partitions(**kwargs, Expression=expression) response = client.get_partitions(**kwargs, Expression=expression)
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["Values"].should.equal(["2022-02-01 00:00:00.000000"]) assert partition["Values"] == ["2022-02-01 00:00:00.000000"]
bad_timestamp_expressions = ( bad_timestamp_expressions = (
"timestamp_col = '2022-02-01'", "timestamp_col = '2022-02-01'",
@ -310,17 +312,18 @@ def test_get_partitions_expression_timestamp_column():
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression=expression) client.get_partitions(**kwargs, Expression=expression)
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("is not a timestamp") assert "is not a timestamp" in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions( client.get_partitions(
**kwargs, Expression="timestamp_col LIKE '2022-02-01 00:00:00'" **kwargs, Expression="timestamp_col LIKE '2022-02-01 00:00:00'"
) )
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match( assert (
"Timestamp data type doesn't support operation 'LIKE'" "Timestamp data type doesn't support operation 'LIKE'"
in exc.value.response["Error"]["Message"]
) )
@ -348,32 +351,32 @@ def test_get_partition_expression_warnings_and_exceptions():
response = client.get_partitions(**kwargs, Expression="string_col = 'test'") response = client.get_partitions(**kwargs, Expression="string_col = 'test'")
partitions = response["Partitions"] partitions = response["Partitions"]
partitions.should.have.length_of(1) assert len(partitions) == 1
partition = partitions[0] partition = partitions[0]
partition["Values"].should.equal(["test", "int", "3.14"]) assert partition["Values"] == ["test", "int", "3.14"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="float_col = 3.14") client.get_partitions(**kwargs, Expression="float_col = 3.14")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("Unknown type : 'float'") assert "Unknown type : 'float'" in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="int_col = 2") client.get_partitions(**kwargs, Expression="int_col = 2")
exc.value.response["Error"]["Code"].should.equal("InvalidStateException") assert exc.value.response["Error"]["Code"] == "InvalidStateException"
exc.value.response["Error"]["Message"].should.match('"int" is not an integer') assert '"int" is not an integer' in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions(**kwargs, Expression="unknown_col = 'test'") client.get_partitions(**kwargs, Expression="unknown_col = 'test'")
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("Unknown column 'unknown_col'") assert "Unknown column 'unknown_col'" in exc.value.response["Error"]["Message"]
with pytest.raises(ClientError) as exc: with pytest.raises(ClientError) as exc:
client.get_partitions( client.get_partitions(
**kwargs, Expression="string_col IS test' AND not parsable" **kwargs, Expression="string_col IS test' AND not parsable"
) )
exc.value.response["Error"]["Code"].should.equal("InvalidInputException") assert exc.value.response["Error"]["Code"] == "InvalidInputException"
exc.value.response["Error"]["Message"].should.match("Unsupported expression") assert "Unsupported expression" in exc.value.response["Error"]["Message"]

629
tests/test_glue/test_schema_registry.py
View File

File diff suppressed because it is too large Load Diff