Add endpoints to glue for deleting partitions
Specifically add glue.delete_partition and glue.batch_delete_partition.
This commit is contained in:
parent
df493ea18d
commit
df2d2ac6b4
@ -138,6 +138,12 @@ class FakeTable(BaseModel):
|
|||||||
raise PartitionAlreadyExistsException()
|
raise PartitionAlreadyExistsException()
|
||||||
self.partitions[key] = partition
|
self.partitions[key] = partition
|
||||||
|
|
||||||
|
def delete_partition(self, values):
|
||||||
|
try:
|
||||||
|
del self.partitions[str(values)]
|
||||||
|
except KeyError:
|
||||||
|
raise PartitionNotFoundException()
|
||||||
|
|
||||||
|
|
||||||
class FakePartition(BaseModel):
|
class FakePartition(BaseModel):
|
||||||
def __init__(self, database_name, table_name, partiton_input):
|
def __init__(self, database_name, table_name, partiton_input):
|
||||||
|
@ -6,6 +6,7 @@ from moto.core.responses import BaseResponse
|
|||||||
from .models import glue_backend
|
from .models import glue_backend
|
||||||
from .exceptions import (
|
from .exceptions import (
|
||||||
PartitionAlreadyExistsException,
|
PartitionAlreadyExistsException,
|
||||||
|
PartitionNotFoundException,
|
||||||
TableNotFoundException
|
TableNotFoundException
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -184,3 +185,38 @@ class GlueResponse(BaseResponse):
|
|||||||
table.update_partition(part_to_update, part_input)
|
table.update_partition(part_to_update, part_input)
|
||||||
|
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
def delete_partition(self):
|
||||||
|
database_name = self.parameters.get('DatabaseName')
|
||||||
|
table_name = self.parameters.get('TableName')
|
||||||
|
part_to_delete = self.parameters.get('PartitionValues')
|
||||||
|
|
||||||
|
table = self.glue_backend.get_table(database_name, table_name)
|
||||||
|
table.delete_partition(part_to_delete)
|
||||||
|
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def batch_delete_partition(self):
|
||||||
|
database_name = self.parameters.get('DatabaseName')
|
||||||
|
table_name = self.parameters.get('TableName')
|
||||||
|
table = self.glue_backend.get_table(database_name, table_name)
|
||||||
|
|
||||||
|
errors_output = []
|
||||||
|
for part_input in self.parameters.get('PartitionsToDelete'):
|
||||||
|
values = part_input.get('Values')
|
||||||
|
try:
|
||||||
|
table.delete_partition(values)
|
||||||
|
except PartitionNotFoundException:
|
||||||
|
errors_output.append({
|
||||||
|
'PartitionValues': values,
|
||||||
|
'ErrorDetail': {
|
||||||
|
'ErrorCode': 'EntityNotFoundException',
|
||||||
|
'ErrorMessage': 'Partition not found',
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
out = {}
|
||||||
|
if errors_output:
|
||||||
|
out['Errors'] = errors_output
|
||||||
|
|
||||||
|
return json.dumps(out)
|
||||||
|
@ -531,3 +531,112 @@ def test_update_partition_move():
|
|||||||
|
|
||||||
partition['TableName'].should.equal(table_name)
|
partition['TableName'].should.equal(table_name)
|
||||||
partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}])
|
partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}])
|
||||||
|
|
||||||
|
@mock_glue
|
||||||
|
def test_delete_partition():
|
||||||
|
client = boto3.client('glue', region_name='us-east-1')
|
||||||
|
database_name = 'myspecialdatabase'
|
||||||
|
table_name = 'myfirsttable'
|
||||||
|
values = ['2018-10-01']
|
||||||
|
helpers.create_database(client, database_name)
|
||||||
|
helpers.create_table(client, database_name, table_name)
|
||||||
|
|
||||||
|
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||||
|
helpers.create_partition(client, database_name, table_name, part_input)
|
||||||
|
|
||||||
|
client.delete_partition(
|
||||||
|
DatabaseName=database_name,
|
||||||
|
TableName=table_name,
|
||||||
|
PartitionValues=values,
|
||||||
|
)
|
||||||
|
|
||||||
|
response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
|
||||||
|
partitions = response['Partitions']
|
||||||
|
partitions.should.be.empty
|
||||||
|
|
||||||
|
@mock_glue
|
||||||
|
def test_delete_partition_bad_partition():
|
||||||
|
client = boto3.client('glue', region_name='us-east-1')
|
||||||
|
database_name = 'myspecialdatabase'
|
||||||
|
table_name = 'myfirsttable'
|
||||||
|
values = ['2018-10-01']
|
||||||
|
helpers.create_database(client, database_name)
|
||||||
|
helpers.create_table(client, database_name, table_name)
|
||||||
|
|
||||||
|
with assert_raises(ClientError) as exc:
|
||||||
|
client.delete_partition(
|
||||||
|
DatabaseName=database_name,
|
||||||
|
TableName=table_name,
|
||||||
|
PartitionValues=values,
|
||||||
|
)
|
||||||
|
|
||||||
|
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||||
|
|
||||||
|
@mock_glue
|
||||||
|
def test_batch_delete_partition():
|
||||||
|
client = boto3.client('glue', region_name='us-east-1')
|
||||||
|
database_name = 'myspecialdatabase'
|
||||||
|
table_name = 'myfirsttable'
|
||||||
|
helpers.create_database(client, database_name)
|
||||||
|
helpers.create_table(client, database_name, table_name)
|
||||||
|
|
||||||
|
partition_inputs = []
|
||||||
|
for i in range(0, 20):
|
||||||
|
values = ["2018-10-{:2}".format(i)]
|
||||||
|
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||||
|
partition_inputs.append(part_input)
|
||||||
|
|
||||||
|
client.batch_create_partition(
|
||||||
|
DatabaseName=database_name,
|
||||||
|
TableName=table_name,
|
||||||
|
PartitionInputList=partition_inputs
|
||||||
|
)
|
||||||
|
|
||||||
|
partition_values = [{"Values": p["Values"]} for p in partition_inputs]
|
||||||
|
|
||||||
|
response = client.batch_delete_partition(
|
||||||
|
DatabaseName=database_name,
|
||||||
|
TableName=table_name,
|
||||||
|
PartitionsToDelete=partition_values,
|
||||||
|
)
|
||||||
|
|
||||||
|
response.should_not.have.key('Errors')
|
||||||
|
|
||||||
|
@mock_glue
|
||||||
|
def test_batch_delete_partition_with_bad_partitions():
|
||||||
|
client = boto3.client('glue', region_name='us-east-1')
|
||||||
|
database_name = 'myspecialdatabase'
|
||||||
|
table_name = 'myfirsttable'
|
||||||
|
helpers.create_database(client, database_name)
|
||||||
|
helpers.create_table(client, database_name, table_name)
|
||||||
|
|
||||||
|
partition_inputs = []
|
||||||
|
for i in range(0, 20):
|
||||||
|
values = ["2018-10-{:2}".format(i)]
|
||||||
|
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||||
|
partition_inputs.append(part_input)
|
||||||
|
|
||||||
|
client.batch_create_partition(
|
||||||
|
DatabaseName=database_name,
|
||||||
|
TableName=table_name,
|
||||||
|
PartitionInputList=partition_inputs
|
||||||
|
)
|
||||||
|
|
||||||
|
partition_values = [{"Values": p["Values"]} for p in partition_inputs]
|
||||||
|
|
||||||
|
partition_values.insert(5, {"Values": ["2018-11-01"]})
|
||||||
|
partition_values.insert(10, {"Values": ["2018-11-02"]})
|
||||||
|
partition_values.insert(15, {"Values": ["2018-11-03"]})
|
||||||
|
|
||||||
|
response = client.batch_delete_partition(
|
||||||
|
DatabaseName=database_name,
|
||||||
|
TableName=table_name,
|
||||||
|
PartitionsToDelete=partition_values,
|
||||||
|
)
|
||||||
|
|
||||||
|
response.should.have.key('Errors')
|
||||||
|
response['Errors'].should.have.length_of(3)
|
||||||
|
error_partitions = map(lambda x: x['PartitionValues'], response['Errors'])
|
||||||
|
['2018-11-01'].should.be.within(error_partitions)
|
||||||
|
['2018-11-02'].should.be.within(error_partitions)
|
||||||
|
['2018-11-03'].should.be.within(error_partitions)
|
||||||
|
Loading…
Reference in New Issue
Block a user