From df2d2ac6b4c57baa00a4fe35e6441b3d60a559bd Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Tue, 11 Jun 2019 14:14:28 -0500 Subject: [PATCH] Add endpoints to glue for deleting partitions Specifically add glue.delete_partition and glue.batch_delete_partition. --- moto/glue/models.py | 6 ++ moto/glue/responses.py | 36 +++++++++ tests/test_glue/test_datacatalog.py | 109 ++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) diff --git a/moto/glue/models.py b/moto/glue/models.py index 407c3a020..0989e0e9b 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -138,6 +138,12 @@ class FakeTable(BaseModel): raise PartitionAlreadyExistsException() self.partitions[key] = partition + def delete_partition(self, values): + try: + del self.partitions[str(values)] + except KeyError: + raise PartitionNotFoundException() + class FakePartition(BaseModel): def __init__(self, database_name, table_name, partiton_input): diff --git a/moto/glue/responses.py b/moto/glue/responses.py index 5c001cac6..cb1ecf519 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -6,6 +6,7 @@ from moto.core.responses import BaseResponse from .models import glue_backend from .exceptions import ( PartitionAlreadyExistsException, + PartitionNotFoundException, TableNotFoundException ) @@ -184,3 +185,38 @@ class GlueResponse(BaseResponse): table.update_partition(part_to_update, part_input) return "" + + def delete_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_to_delete = self.parameters.get('PartitionValues') + + table = self.glue_backend.get_table(database_name, table_name) + table.delete_partition(part_to_delete) + + return "" + + def batch_delete_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + + errors_output = [] + for part_input in self.parameters.get('PartitionsToDelete'): + values = part_input.get('Values') + try: + table.delete_partition(values) + except PartitionNotFoundException: + errors_output.append({ + 'PartitionValues': values, + 'ErrorDetail': { + 'ErrorCode': 'EntityNotFoundException', + 'ErrorMessage': 'Partition not found', + } + }) + + out = {} + if errors_output: + out['Errors'] = errors_output + + return json.dumps(out) diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index a02266560..232ab3019 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -531,3 +531,112 @@ def test_update_partition_move(): partition['TableName'].should.equal(table_name) partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) + +@mock_glue +def test_delete_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + part_input = helpers.create_partition_input(database_name, table_name, values=values) + helpers.create_partition(client, database_name, table_name, part_input) + + client.delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + partitions = response['Partitions'] + partitions.should.be.empty + +@mock_glue +def test_delete_partition_bad_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + client.delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + +@mock_glue +def test_batch_delete_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + partition_inputs = [] + for i in range(0, 20): + values = ["2018-10-{:2}".format(i)] + part_input = helpers.create_partition_input(database_name, table_name, values=values) + partition_inputs.append(part_input) + + client.batch_create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInputList=partition_inputs + ) + + partition_values = [{"Values": p["Values"]} for p in partition_inputs] + + response = client.batch_delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionsToDelete=partition_values, + ) + + response.should_not.have.key('Errors') + +@mock_glue +def test_batch_delete_partition_with_bad_partitions(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + partition_inputs = [] + for i in range(0, 20): + values = ["2018-10-{:2}".format(i)] + part_input = helpers.create_partition_input(database_name, table_name, values=values) + partition_inputs.append(part_input) + + client.batch_create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInputList=partition_inputs + ) + + partition_values = [{"Values": p["Values"]} for p in partition_inputs] + + partition_values.insert(5, {"Values": ["2018-11-01"]}) + partition_values.insert(10, {"Values": ["2018-11-02"]}) + partition_values.insert(15, {"Values": ["2018-11-03"]}) + + response = client.batch_delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionsToDelete=partition_values, + ) + + response.should.have.key('Errors') + response['Errors'].should.have.length_of(3) + error_partitions = map(lambda x: x['PartitionValues'], response['Errors']) + ['2018-11-01'].should.be.within(error_partitions) + ['2018-11-02'].should.be.within(error_partitions) + ['2018-11-03'].should.be.within(error_partitions)