Mock more of the Glue Data Catalog APIs
This adds some of the missing Get/Update/Create APIs relating to the Glue data catalog -- but not yet all of them, and none of the Batch* API calls.
This commit is contained in:
Ash Berlin-Taylor
parent
dfa7935e13
commit
5783d66206
@ -6,19 +6,56 @@ class GlueClientError(JsonRESTError):
|
||||
code = 400
|
||||
|
||||
|
||||
class DatabaseAlreadyExistsException(GlueClientError):
|
||||
def __init__(self):
|
||||
self.code = 400
|
||||
super(DatabaseAlreadyExistsException, self).__init__(
|
||||
'DatabaseAlreadyExistsException',
|
||||
'Database already exists.'
|
||||
class AlreadyExistsException(GlueClientError):
|
||||
def __init__(self, typ):
|
||||
super(GlueClientError, self).__init__(
|
||||
'AlreadyExistsException',
|
||||
'%s already exists.' % (typ),
|
||||
)
|
||||
|
||||
|
||||
class TableAlreadyExistsException(GlueClientError):
|
||||
class DatabaseAlreadyExistsException(AlreadyExistsException):
|
||||
def __init__(self):
|
||||
self.code = 400
|
||||
super(TableAlreadyExistsException, self).__init__(
|
||||
'TableAlreadyExistsException',
|
||||
'Table already exists.'
|
||||
super(DatabaseAlreadyExistsException, self).__init__('Database')
|
||||
|
||||
|
||||
class TableAlreadyExistsException(AlreadyExistsException):
|
||||
def __init__(self):
|
||||
super(TableAlreadyExistsException, self).__init__('Table')
|
||||
|
||||
|
||||
class PartitionAlreadyExistsException(AlreadyExistsException):
|
||||
def __init__(self):
|
||||
super(PartitionAlreadyExistsException, self).__init__('Partition')
|
||||
|
||||
|
||||
class EntityNotFoundException(GlueClientError):
|
||||
def __init__(self, msg):
|
||||
super(GlueClientError, self).__init__(
|
||||
'EntityNotFoundException',
|
||||
msg,
|
||||
)
|
||||
|
||||
|
||||
class DatabaseNotFoundException(EntityNotFoundException):
|
||||
def __init__(self, db):
|
||||
super(DatabaseNotFoundException, self).__init__(
|
||||
'Database %s not found.' % db,
|
||||
)
|
||||
|
||||
|
||||
class TableNotFoundException(EntityNotFoundException):
|
||||
def __init__(self, tbl):
|
||||
super(TableNotFoundException, self).__init__(
|
||||
'Table %s not found.' % tbl,
|
||||
)
|
||||
|
||||
|
||||
class PartitionNotFoundException(EntityNotFoundException):
|
||||
def __init__(self):
|
||||
super(PartitionNotFoundException, self).__init__("Cannot find partition.")
|
||||
|
||||
|
||||
class VersionNotFoundException(EntityNotFoundException):
|
||||
def __init__(self):
|
||||
super(VersionNotFoundException, self).__init__("Version not found.")
|
||||
|
||||
@ -1,8 +1,19 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import time
|
||||
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.compat import OrderedDict
|
||||
from.exceptions import DatabaseAlreadyExistsException, TableAlreadyExistsException
|
||||
from.exceptions import (
|
||||
JsonRESTError,
|
||||
DatabaseAlreadyExistsException,
|
||||
DatabaseNotFoundException,
|
||||
TableAlreadyExistsException,
|
||||
TableNotFoundException,
|
||||
PartitionAlreadyExistsException,
|
||||
PartitionNotFoundException,
|
||||
VersionNotFoundException,
|
||||
)
|
||||
|
||||
|
||||
class GlueBackend(BaseBackend):
|
||||
@ -19,7 +30,10 @@ class GlueBackend(BaseBackend):
|
||||
return database
|
||||
|
||||
def get_database(self, database_name):
|
||||
return self.databases[database_name]
|
||||
try:
|
||||
return self.databases[database_name]
|
||||
except KeyError:
|
||||
raise DatabaseNotFoundException(database_name)
|
||||
|
||||
def create_table(self, database_name, table_name, table_input):
|
||||
database = self.get_database(database_name)
|
||||
@ -33,7 +47,10 @@ class GlueBackend(BaseBackend):
|
||||
|
||||
def get_table(self, database_name, table_name):
|
||||
database = self.get_database(database_name)
|
||||
return database.tables[table_name]
|
||||
try:
|
||||
return database.tables[table_name]
|
||||
except KeyError:
|
||||
raise TableNotFoundException(table_name)
|
||||
|
||||
def get_tables(self, database_name):
|
||||
database = self.get_database(database_name)
|
||||
@ -52,9 +69,84 @@ class FakeTable(BaseModel):
|
||||
def __init__(self, database_name, table_name, table_input):
|
||||
self.database_name = database_name
|
||||
self.name = table_name
|
||||
self.table_input = table_input
|
||||
self.storage_descriptor = self.table_input.get('StorageDescriptor', {})
|
||||
self.partition_keys = self.table_input.get('PartitionKeys', [])
|
||||
self.partitions = OrderedDict()
|
||||
self.versions = []
|
||||
self.update(table_input)
|
||||
|
||||
def update(self, table_input):
|
||||
self.versions.append(table_input)
|
||||
|
||||
def get_version(self, ver):
|
||||
try:
|
||||
if not isinstance(ver, int):
|
||||
# "1" goes to [0]
|
||||
ver = int(ver) - 1
|
||||
except ValueError as e:
|
||||
raise JsonRESTError("InvalidInputException", str(e))
|
||||
|
||||
try:
|
||||
return self.versions[ver]
|
||||
except IndexError:
|
||||
raise VersionNotFoundException()
|
||||
|
||||
def as_dict(self, version=-1):
|
||||
obj = {
|
||||
'DatabaseName': self.database_name,
|
||||
'Name': self.name,
|
||||
}
|
||||
obj.update(self.get_version(version))
|
||||
return obj
|
||||
|
||||
def create_partition(self, partiton_input):
|
||||
partition = FakePartition(self.database_name, self.name, partiton_input)
|
||||
key = str(partition.values)
|
||||
if key in self.partitions:
|
||||
raise PartitionAlreadyExistsException()
|
||||
self.partitions[str(partition.values)] = partition
|
||||
|
||||
def get_partitions(self):
|
||||
return [p for str_part_values, p in self.partitions.items()]
|
||||
|
||||
def get_partition(self, values):
|
||||
try:
|
||||
return self.partitions[str(values)]
|
||||
except KeyError:
|
||||
raise PartitionNotFoundException()
|
||||
|
||||
def update_partition(self, old_values, partiton_input):
|
||||
partition = FakePartition(self.database_name, self.name, partiton_input)
|
||||
key = str(partition.values)
|
||||
if old_values == partiton_input['Values']:
|
||||
# Altering a partition in place. Don't remove it so the order of
|
||||
# returned partitions doesn't change
|
||||
if key not in self.partitions:
|
||||
raise PartitionNotFoundException()
|
||||
else:
|
||||
removed = self.partitions.pop(str(old_values), None)
|
||||
if removed is None:
|
||||
raise PartitionNotFoundException()
|
||||
if key in self.partitions:
|
||||
# Trying to update to overwrite a partition that exists
|
||||
raise PartitionAlreadyExistsException()
|
||||
self.partitions[key] = partition
|
||||
|
||||
|
||||
class FakePartition(BaseModel):
|
||||
def __init__(self, database_name, table_name, partiton_input):
|
||||
self.creation_time = time.time()
|
||||
self.database_name = database_name
|
||||
self.table_name = table_name
|
||||
self.partition_input = partiton_input
|
||||
self.values = self.partition_input.get('Values', [])
|
||||
|
||||
def as_dict(self):
|
||||
obj = {
|
||||
'DatabaseName': self.database_name,
|
||||
'TableName': self.table_name,
|
||||
'CreationTime': self.creation_time,
|
||||
}
|
||||
obj.update(self.partition_input)
|
||||
return obj
|
||||
|
||||
|
||||
glue_backend = GlueBackend()
|
||||
|
||||
@ -37,27 +37,94 @@ class GlueResponse(BaseResponse):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('Name')
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
|
||||
return json.dumps({'Table': table.as_dict()})
|
||||
|
||||
def update_table(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_input = self.parameters.get('TableInput')
|
||||
table_name = table_input.get('Name')
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
table.update(table_input)
|
||||
return ""
|
||||
|
||||
def get_table_versions(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
|
||||
return json.dumps({
|
||||
'Table': {
|
||||
'DatabaseName': table.database_name,
|
||||
'Name': table.name,
|
||||
'PartitionKeys': table.partition_keys,
|
||||
'StorageDescriptor': table.storage_descriptor
|
||||
}
|
||||
"TableVersions": [
|
||||
{
|
||||
"Table": table.as_dict(version=n),
|
||||
"VersionId": str(n + 1),
|
||||
} for n in range(len(table.versions))
|
||||
],
|
||||
})
|
||||
|
||||
def get_table_version(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
ver_id = self.parameters.get('VersionId')
|
||||
|
||||
return json.dumps({
|
||||
"TableVersion": {
|
||||
"Table": table.as_dict(version=ver_id),
|
||||
"VersionId": ver_id,
|
||||
},
|
||||
})
|
||||
|
||||
def get_tables(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
tables = self.glue_backend.get_tables(database_name)
|
||||
return json.dumps(
|
||||
{
|
||||
'TableList': [
|
||||
{
|
||||
'DatabaseName': table.database_name,
|
||||
'Name': table.name,
|
||||
'PartitionKeys': table.partition_keys,
|
||||
'StorageDescriptor': table.storage_descriptor
|
||||
} for table in tables
|
||||
]
|
||||
}
|
||||
)
|
||||
return json.dumps({
|
||||
'TableList': [
|
||||
table.as_dict() for table in tables
|
||||
]
|
||||
})
|
||||
|
||||
def get_partitions(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
if 'Expression' in self.parameters:
|
||||
raise NotImplementedError("Expression filtering in get_partitions is not implemented in moto")
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
|
||||
return json.dumps({
|
||||
'Partitions': [
|
||||
p.as_dict() for p in table.get_partitions()
|
||||
]
|
||||
})
|
||||
|
||||
def get_partition(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
values = self.parameters.get('PartitionValues')
|
||||
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
|
||||
p = table.get_partition(values)
|
||||
|
||||
return json.dumps({'Partition': p.as_dict()})
|
||||
|
||||
def create_partition(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
part_input = self.parameters.get('PartitionInput')
|
||||
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
table.create_partition(part_input)
|
||||
|
||||
return ""
|
||||
|
||||
def update_partition(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
part_input = self.parameters.get('PartitionInput')
|
||||
part_to_update = self.parameters.get('PartitionValueList')
|
||||
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
table.update_partition(part_to_update, part_input)
|
||||
|
||||
return ""
|
||||
|
||||
@ -29,3 +29,28 @@ TABLE_INPUT = {
|
||||
},
|
||||
'TableType': 'EXTERNAL_TABLE',
|
||||
}
|
||||
|
||||
|
||||
PARTITION_INPUT = {
|
||||
# 'DatabaseName': 'dbname',
|
||||
'StorageDescriptor': {
|
||||
'BucketColumns': [],
|
||||
'Columns': [],
|
||||
'Compressed': False,
|
||||
'InputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat',
|
||||
'Location': 's3://.../partition=value',
|
||||
'NumberOfBuckets': -1,
|
||||
'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat',
|
||||
'Parameters': {},
|
||||
'SerdeInfo': {
|
||||
'Parameters': {'path': 's3://...', 'serialization.format': '1'},
|
||||
'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'},
|
||||
'SkewedInfo': {'SkewedColumnNames': [],
|
||||
'SkewedColumnValueLocationMaps': {},
|
||||
'SkewedColumnValues': []},
|
||||
'SortColumns': [],
|
||||
'StoredAsSubDirectories': False,
|
||||
},
|
||||
# 'TableName': 'source_table',
|
||||
# 'Values': ['2018-06-26'],
|
||||
}
|
||||
|
||||
@ -2,7 +2,7 @@ from __future__ import unicode_literals
|
||||
|
||||
import copy
|
||||
|
||||
from .fixtures.datacatalog import TABLE_INPUT
|
||||
from .fixtures.datacatalog import TABLE_INPUT, PARTITION_INPUT
|
||||
|
||||
|
||||
def create_database(client, database_name):
|
||||
@ -17,22 +17,38 @@ def get_database(client, database_name):
|
||||
return client.get_database(Name=database_name)
|
||||
|
||||
|
||||
def create_table_input(table_name, s3_location, columns=[], partition_keys=[]):
|
||||
def create_table_input(database_name, table_name, columns=[], partition_keys=[]):
|
||||
table_input = copy.deepcopy(TABLE_INPUT)
|
||||
table_input['Name'] = table_name
|
||||
table_input['PartitionKeys'] = partition_keys
|
||||
table_input['StorageDescriptor']['Columns'] = columns
|
||||
table_input['StorageDescriptor']['Location'] = s3_location
|
||||
table_input['StorageDescriptor']['Location'] = 's3://my-bucket/{database_name}/{table_name}'.format(
|
||||
database_name=database_name,
|
||||
table_name=table_name
|
||||
)
|
||||
return table_input
|
||||
|
||||
|
||||
def create_table(client, database_name, table_name, table_input):
|
||||
def create_table(client, database_name, table_name, table_input=None, **kwargs):
|
||||
if table_input is None:
|
||||
table_input = create_table_input(database_name, table_name, **kwargs)
|
||||
|
||||
return client.create_table(
|
||||
DatabaseName=database_name,
|
||||
TableInput=table_input
|
||||
)
|
||||
|
||||
|
||||
def update_table(client, database_name, table_name, table_input=None, **kwargs):
|
||||
if table_input is None:
|
||||
table_input = create_table_input(database_name, table_name, **kwargs)
|
||||
|
||||
return client.update_table(
|
||||
DatabaseName=database_name,
|
||||
TableInput=table_input,
|
||||
)
|
||||
|
||||
|
||||
def get_table(client, database_name, table_name):
|
||||
return client.get_table(
|
||||
DatabaseName=database_name,
|
||||
@ -44,3 +60,60 @@ def get_tables(client, database_name):
|
||||
return client.get_tables(
|
||||
DatabaseName=database_name
|
||||
)
|
||||
|
||||
|
||||
def get_table_versions(client, database_name, table_name):
|
||||
return client.get_table_versions(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name
|
||||
)
|
||||
|
||||
|
||||
def get_table_version(client, database_name, table_name, version_id):
|
||||
return client.get_table_version(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
VersionId=version_id,
|
||||
)
|
||||
|
||||
|
||||
def create_partition_input(database_name, table_name, values=[], columns=[]):
|
||||
root_path = 's3://my-bucket/{database_name}/{table_name}'.format(
|
||||
database_name=database_name,
|
||||
table_name=table_name
|
||||
)
|
||||
|
||||
part_input = copy.deepcopy(PARTITION_INPUT)
|
||||
part_input['Values'] = values
|
||||
part_input['StorageDescriptor']['Columns'] = columns
|
||||
part_input['StorageDescriptor']['SerdeInfo']['Parameters']['path'] = root_path
|
||||
return part_input
|
||||
|
||||
|
||||
def create_partition(client, database_name, table_name, partiton_input=None, **kwargs):
|
||||
if partiton_input is None:
|
||||
partiton_input = create_partition_input(database_name, table_name, **kwargs)
|
||||
return client.create_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionInput=partiton_input
|
||||
)
|
||||
|
||||
|
||||
def update_partition(client, database_name, table_name, old_values=[], partiton_input=None, **kwargs):
|
||||
if partiton_input is None:
|
||||
partiton_input = create_partition_input(database_name, table_name, **kwargs)
|
||||
return client.update_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionInput=partiton_input,
|
||||
PartitionValueList=old_values,
|
||||
)
|
||||
|
||||
|
||||
def get_partition(client, database_name, table_name, values):
|
||||
return client.get_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionValues=values,
|
||||
)
|
||||
|
||||
@ -1,10 +1,15 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import sure # noqa
|
||||
import re
|
||||
from nose.tools import assert_raises
|
||||
import boto3
|
||||
from botocore.client import ClientError
|
||||
|
||||
|
||||
from datetime import datetime
|
||||
import pytz
|
||||
|
||||
from moto import mock_glue
|
||||
from . import helpers
|
||||
|
||||
@ -30,7 +35,19 @@ def test_create_database_already_exists():
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('DatabaseAlreadyExistsException')
|
||||
exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_database_not_exits():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'nosuchdatabase'
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_database(client, database_name)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found')
|
||||
|
||||
|
||||
@mock_glue
|
||||
@ -40,12 +57,7 @@ def test_create_table():
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
table_name = 'myspecialtable'
|
||||
s3_location = 's3://my-bucket/{database_name}/{table_name}'.format(
|
||||
database_name=database_name,
|
||||
table_name=table_name
|
||||
)
|
||||
|
||||
table_input = helpers.create_table_input(table_name, s3_location)
|
||||
table_input = helpers.create_table_input(database_name, table_name)
|
||||
helpers.create_table(client, database_name, table_name, table_input)
|
||||
|
||||
response = helpers.get_table(client, database_name, table_name)
|
||||
@ -63,18 +75,12 @@ def test_create_table_already_exists():
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
table_name = 'cantcreatethistabletwice'
|
||||
s3_location = 's3://my-bucket/{database_name}/{table_name}'.format(
|
||||
database_name=database_name,
|
||||
table_name=table_name
|
||||
)
|
||||
|
||||
table_input = helpers.create_table_input(table_name, s3_location)
|
||||
helpers.create_table(client, database_name, table_name, table_input)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.create_table(client, database_name, table_name, table_input)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('TableAlreadyExistsException')
|
||||
exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException')
|
||||
|
||||
|
||||
@mock_glue
|
||||
@ -87,11 +93,7 @@ def test_get_tables():
|
||||
table_inputs = {}
|
||||
|
||||
for table_name in table_names:
|
||||
s3_location = 's3://my-bucket/{database_name}/{table_name}'.format(
|
||||
database_name=database_name,
|
||||
table_name=table_name
|
||||
)
|
||||
table_input = helpers.create_table_input(table_name, s3_location)
|
||||
table_input = helpers.create_table_input(database_name, table_name)
|
||||
table_inputs[table_name] = table_input
|
||||
helpers.create_table(client, database_name, table_name, table_input)
|
||||
|
||||
@ -99,10 +101,326 @@ def test_get_tables():
|
||||
|
||||
tables = response['TableList']
|
||||
|
||||
assert len(tables) == 3
|
||||
tables.should.have.length_of(3)
|
||||
|
||||
for table in tables:
|
||||
table_name = table['Name']
|
||||
table_name.should.equal(table_inputs[table_name]['Name'])
|
||||
table['StorageDescriptor'].should.equal(table_inputs[table_name]['StorageDescriptor'])
|
||||
table['PartitionKeys'].should.equal(table_inputs[table_name]['PartitionKeys'])
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_table_versions():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
table_name = 'myfirsttable'
|
||||
version_inputs = {}
|
||||
|
||||
table_input = helpers.create_table_input(database_name, table_name)
|
||||
helpers.create_table(client, database_name, table_name, table_input)
|
||||
version_inputs["1"] = table_input
|
||||
|
||||
columns = [{'Name': 'country', 'Type': 'string'}]
|
||||
table_input = helpers.create_table_input(database_name, table_name, columns=columns)
|
||||
helpers.update_table(client, database_name, table_name, table_input)
|
||||
version_inputs["2"] = table_input
|
||||
|
||||
# Updateing with an indentical input should still create a new version
|
||||
helpers.update_table(client, database_name, table_name, table_input)
|
||||
version_inputs["3"] = table_input
|
||||
|
||||
response = helpers.get_table_versions(client, database_name, table_name)
|
||||
|
||||
vers = response['TableVersions']
|
||||
|
||||
vers.should.have.length_of(3)
|
||||
vers[0]['Table']['StorageDescriptor']['Columns'].should.equal([])
|
||||
vers[-1]['Table']['StorageDescriptor']['Columns'].should.equal(columns)
|
||||
|
||||
for n, ver in enumerate(vers):
|
||||
n = str(n + 1)
|
||||
ver['VersionId'].should.equal(n)
|
||||
ver['Table']['Name'].should.equal(table_name)
|
||||
ver['Table']['StorageDescriptor'].should.equal(version_inputs[n]['StorageDescriptor'])
|
||||
ver['Table']['PartitionKeys'].should.equal(version_inputs[n]['PartitionKeys'])
|
||||
|
||||
response = helpers.get_table_version(client, database_name, table_name, "3")
|
||||
ver = response['TableVersion']
|
||||
|
||||
ver['VersionId'].should.equal("3")
|
||||
ver['Table']['Name'].should.equal(table_name)
|
||||
ver['Table']['StorageDescriptor']['Columns'].should.equal(columns)
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_table_version_not_found():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_table_version(client, database_name, 'myfirsttable', "20")
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('version', re.I)
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_table_version_invalid_input():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_table_version(client, database_name, 'myfirsttable', "10not-an-int")
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('InvalidInputException')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_table_not_exits():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_table(client, database_name, 'myfirsttable')
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('Table myfirsttable not found')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_table_when_database_not_exits():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'nosuchdatabase'
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_table(client, database_name, 'myfirsttable')
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_partitions_empty():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
|
||||
|
||||
response['Partitions'].should.have.length_of(0)
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_create_partition():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
before = datetime.now(pytz.utc)
|
||||
|
||||
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||
helpers.create_partition(client, database_name, table_name, part_input)
|
||||
|
||||
after = datetime.now(pytz.utc)
|
||||
|
||||
response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
|
||||
|
||||
partitions = response['Partitions']
|
||||
|
||||
partitions.should.have.length_of(1)
|
||||
|
||||
partition = partitions[0]
|
||||
|
||||
partition['TableName'].should.equal(table_name)
|
||||
partition['StorageDescriptor'].should.equal(part_input['StorageDescriptor'])
|
||||
partition['Values'].should.equal(values)
|
||||
partition['CreationTime'].should.be.greater_than(before)
|
||||
partition['CreationTime'].should.be.lower_than(after)
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_create_partition_already_exist():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
helpers.create_partition(client, database_name, table_name, values=values)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.create_partition(client, database_name, table_name, values=values)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_partition_not_found():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_partition(client, database_name, table_name, values)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('partition')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_get_partition():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
values = [['2018-10-01'], ['2018-09-01']]
|
||||
|
||||
helpers.create_partition(client, database_name, table_name, values=values[0])
|
||||
helpers.create_partition(client, database_name, table_name, values=values[1])
|
||||
|
||||
response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values[1])
|
||||
|
||||
partition = response['Partition']
|
||||
|
||||
partition['TableName'].should.equal(table_name)
|
||||
partition['Values'].should.equal(values[1])
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_update_partition_not_found_moving():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.update_partition(client, database_name, table_name, old_values=['0000-00-00'], values=['2018-10-02'])
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('partition')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_update_partition_not_found_change_in_place():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.update_partition(client, database_name, table_name, old_values=values, values=values)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
exc.exception.response['Error']['Message'].should.match('partition')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_update_partition_cannot_overwrite():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
values = [['2018-10-01'], ['2018-09-01']]
|
||||
|
||||
helpers.create_partition(client, database_name, table_name, values=values[0])
|
||||
helpers.create_partition(client, database_name, table_name, values=values[1])
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.update_partition(client, database_name, table_name, old_values=values[0], values=values[1])
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException')
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_update_partition():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
helpers.create_partition(client, database_name, table_name, values=values)
|
||||
|
||||
response = helpers.update_partition(
|
||||
client,
|
||||
database_name,
|
||||
table_name,
|
||||
old_values=values,
|
||||
values=values,
|
||||
columns=[{'Name': 'country', 'Type': 'string'}],
|
||||
)
|
||||
|
||||
response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values)
|
||||
partition = response['Partition']
|
||||
|
||||
partition['TableName'].should.equal(table_name)
|
||||
partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}])
|
||||
|
||||
|
||||
@mock_glue
|
||||
def test_update_partition_move():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
new_values = ['2018-09-01']
|
||||
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
helpers.create_partition(client, database_name, table_name, values=values)
|
||||
|
||||
response = helpers.update_partition(
|
||||
client,
|
||||
database_name,
|
||||
table_name,
|
||||
old_values=values,
|
||||
values=new_values,
|
||||
columns=[{'Name': 'country', 'Type': 'string'}],
|
||||
)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
helpers.get_partition(client, database_name, table_name, values)
|
||||
|
||||
# Old partition shouldn't exist anymore
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
|
||||
response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=new_values)
|
||||
partition = response['Partition']
|
||||
|
||||
partition['TableName'].should.equal(table_name)
|
||||
partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user