Add endpoints to glue for deleting partitions
Specifically add glue.delete_partition and glue.batch_delete_partition.
This commit is contained in:
parent
df493ea18d
commit
df2d2ac6b4
@ -138,6 +138,12 @@ class FakeTable(BaseModel):
|
||||
raise PartitionAlreadyExistsException()
|
||||
self.partitions[key] = partition
|
||||
|
||||
def delete_partition(self, values):
|
||||
try:
|
||||
del self.partitions[str(values)]
|
||||
except KeyError:
|
||||
raise PartitionNotFoundException()
|
||||
|
||||
|
||||
class FakePartition(BaseModel):
|
||||
def __init__(self, database_name, table_name, partiton_input):
|
||||
|
@ -6,6 +6,7 @@ from moto.core.responses import BaseResponse
|
||||
from .models import glue_backend
|
||||
from .exceptions import (
|
||||
PartitionAlreadyExistsException,
|
||||
PartitionNotFoundException,
|
||||
TableNotFoundException
|
||||
)
|
||||
|
||||
@ -184,3 +185,38 @@ class GlueResponse(BaseResponse):
|
||||
table.update_partition(part_to_update, part_input)
|
||||
|
||||
return ""
|
||||
|
||||
def delete_partition(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
part_to_delete = self.parameters.get('PartitionValues')
|
||||
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
table.delete_partition(part_to_delete)
|
||||
|
||||
return ""
|
||||
|
||||
def batch_delete_partition(self):
|
||||
database_name = self.parameters.get('DatabaseName')
|
||||
table_name = self.parameters.get('TableName')
|
||||
table = self.glue_backend.get_table(database_name, table_name)
|
||||
|
||||
errors_output = []
|
||||
for part_input in self.parameters.get('PartitionsToDelete'):
|
||||
values = part_input.get('Values')
|
||||
try:
|
||||
table.delete_partition(values)
|
||||
except PartitionNotFoundException:
|
||||
errors_output.append({
|
||||
'PartitionValues': values,
|
||||
'ErrorDetail': {
|
||||
'ErrorCode': 'EntityNotFoundException',
|
||||
'ErrorMessage': 'Partition not found',
|
||||
}
|
||||
})
|
||||
|
||||
out = {}
|
||||
if errors_output:
|
||||
out['Errors'] = errors_output
|
||||
|
||||
return json.dumps(out)
|
||||
|
@ -531,3 +531,112 @@ def test_update_partition_move():
|
||||
|
||||
partition['TableName'].should.equal(table_name)
|
||||
partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}])
|
||||
|
||||
@mock_glue
|
||||
def test_delete_partition():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||
helpers.create_partition(client, database_name, table_name, part_input)
|
||||
|
||||
client.delete_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionValues=values,
|
||||
)
|
||||
|
||||
response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
|
||||
partitions = response['Partitions']
|
||||
partitions.should.be.empty
|
||||
|
||||
@mock_glue
|
||||
def test_delete_partition_bad_partition():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
values = ['2018-10-01']
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
with assert_raises(ClientError) as exc:
|
||||
client.delete_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionValues=values,
|
||||
)
|
||||
|
||||
exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
|
||||
|
||||
@mock_glue
|
||||
def test_batch_delete_partition():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
partition_inputs = []
|
||||
for i in range(0, 20):
|
||||
values = ["2018-10-{:2}".format(i)]
|
||||
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||
partition_inputs.append(part_input)
|
||||
|
||||
client.batch_create_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionInputList=partition_inputs
|
||||
)
|
||||
|
||||
partition_values = [{"Values": p["Values"]} for p in partition_inputs]
|
||||
|
||||
response = client.batch_delete_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionsToDelete=partition_values,
|
||||
)
|
||||
|
||||
response.should_not.have.key('Errors')
|
||||
|
||||
@mock_glue
|
||||
def test_batch_delete_partition_with_bad_partitions():
|
||||
client = boto3.client('glue', region_name='us-east-1')
|
||||
database_name = 'myspecialdatabase'
|
||||
table_name = 'myfirsttable'
|
||||
helpers.create_database(client, database_name)
|
||||
helpers.create_table(client, database_name, table_name)
|
||||
|
||||
partition_inputs = []
|
||||
for i in range(0, 20):
|
||||
values = ["2018-10-{:2}".format(i)]
|
||||
part_input = helpers.create_partition_input(database_name, table_name, values=values)
|
||||
partition_inputs.append(part_input)
|
||||
|
||||
client.batch_create_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionInputList=partition_inputs
|
||||
)
|
||||
|
||||
partition_values = [{"Values": p["Values"]} for p in partition_inputs]
|
||||
|
||||
partition_values.insert(5, {"Values": ["2018-11-01"]})
|
||||
partition_values.insert(10, {"Values": ["2018-11-02"]})
|
||||
partition_values.insert(15, {"Values": ["2018-11-03"]})
|
||||
|
||||
response = client.batch_delete_partition(
|
||||
DatabaseName=database_name,
|
||||
TableName=table_name,
|
||||
PartitionsToDelete=partition_values,
|
||||
)
|
||||
|
||||
response.should.have.key('Errors')
|
||||
response['Errors'].should.have.length_of(3)
|
||||
error_partitions = map(lambda x: x['PartitionValues'], response['Errors'])
|
||||
['2018-11-01'].should.be.within(error_partitions)
|
||||
['2018-11-02'].should.be.within(error_partitions)
|
||||
['2018-11-03'].should.be.within(error_partitions)
|
||||
|
Loading…
Reference in New Issue
Block a user