Merge pull request #2236 from dkuntz2/glue-endpoints

Add glue.delete_partition and glue.batch_delete_partition

IMPLEMENTATION_COVERAGE.md

@@ -2012,23 +2012,23 @@
 - [ ] upload_archive
 - [ ] upload_multipart_part
 
-## glue - 0% implemented
-- [ ] batch_create_partition
+## glue - 23% implemented
+- [x] batch_create_partition
 - [ ] batch_delete_connection
-- [ ] batch_delete_partition
-- [ ] batch_delete_table
+- [x] batch_delete_partition
+- [x] batch_delete_table
 - [ ] batch_delete_table_version
 - [ ] batch_get_partition
 - [ ] batch_stop_job_run
 - [ ] create_classifier
 - [ ] create_connection
 - [ ] create_crawler
-- [ ] create_database
+- [x] create_database
 - [ ] create_dev_endpoint
 - [ ] create_job
-- [ ] create_partition
+- [x] create_partition
 - [ ] create_script
-- [ ] create_table
+- [x] create_table
 - [ ] create_trigger
 - [ ] create_user_defined_function
 - [ ] delete_classifier
@@ -2037,8 +2037,8 @@
 - [ ] delete_database
 - [ ] delete_dev_endpoint
 - [ ] delete_job
-- [ ] delete_partition
-- [ ] delete_table
+- [x] delete_partition
+- [x] delete_table
 - [ ] delete_table_version
 - [ ] delete_trigger
 - [ ] delete_user_defined_function
@@ -2050,7 +2050,7 @@
 - [ ] get_crawler
 - [ ] get_crawler_metrics
 - [ ] get_crawlers
-- [ ] get_database
+- [x] get_database
 - [ ] get_databases
 - [ ] get_dataflow_graph
 - [ ] get_dev_endpoint
@@ -2060,13 +2060,13 @@
 - [ ] get_job_runs
 - [ ] get_jobs
 - [ ] get_mapping
-- [ ] get_partition
-- [ ] get_partitions
+- [x] get_partition
+- [x] get_partitions
 - [ ] get_plan
-- [ ] get_table
-- [ ] get_table_version
-- [ ] get_table_versions
-- [ ] get_tables
+- [x] get_table
+- [x] get_table_version
+- [x] get_table_versions
+- [x] get_tables
 - [ ] get_trigger
 - [ ] get_triggers
 - [ ] get_user_defined_function
@@ -2087,8 +2087,8 @@
 - [ ] update_database
 - [ ] update_dev_endpoint
 - [ ] update_job
-- [ ] update_partition
-- [ ] update_table
+- [x] update_partition
+- [x] update_table
 - [ ] update_trigger
 - [ ] update_user_defined_function
 

moto/glue/models.py

@@ -138,6 +138,12 @@ class FakeTable(BaseModel):
                 raise PartitionAlreadyExistsException()
         self.partitions[key] = partition
 
+    def delete_partition(self, values):
+        try:
+            del self.partitions[str(values)]
+        except KeyError:
+            raise PartitionNotFoundException()
+
 
 class FakePartition(BaseModel):
     def __init__(self, database_name, table_name, partiton_input):

moto/glue/responses.py

@@ -6,6 +6,7 @@ from moto.core.responses import BaseResponse
 from .models import glue_backend
 from .exceptions import (
     PartitionAlreadyExistsException,
+    PartitionNotFoundException,
     TableNotFoundException
 )
 
@@ -184,3 +185,38 @@ class GlueResponse(BaseResponse):
         table.update_partition(part_to_update, part_input)
 
         return ""
+
+    def delete_partition(self):
+        database_name = self.parameters.get('DatabaseName')
+        table_name = self.parameters.get('TableName')
+        part_to_delete = self.parameters.get('PartitionValues')
+
+        table = self.glue_backend.get_table(database_name, table_name)
+        table.delete_partition(part_to_delete)
+
+        return ""
+
+    def batch_delete_partition(self):
+        database_name = self.parameters.get('DatabaseName')
+        table_name = self.parameters.get('TableName')
+        table = self.glue_backend.get_table(database_name, table_name)
+
+        errors_output = []
+        for part_input in self.parameters.get('PartitionsToDelete'):
+            values = part_input.get('Values')
+            try:
+                table.delete_partition(values)
+            except PartitionNotFoundException:
+                errors_output.append({
+                    'PartitionValues': values,
+                    'ErrorDetail': {
+                        'ErrorCode': 'EntityNotFoundException',
+                        'ErrorMessage': 'Partition not found',
+                    }
+                })
+
+        out = {}
+        if errors_output:
+            out['Errors'] = errors_output
+
+        return json.dumps(out)

tests/test_glue/test_datacatalog.py

@@ -531,3 +531,112 @@ def test_update_partition_move():
 
     partition['TableName'].should.equal(table_name)
     partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}])
+
+@mock_glue
+def test_delete_partition():
+    client = boto3.client('glue', region_name='us-east-1')
+    database_name = 'myspecialdatabase'
+    table_name = 'myfirsttable'
+    values = ['2018-10-01']
+    helpers.create_database(client, database_name)
+    helpers.create_table(client, database_name, table_name)
+
+    part_input = helpers.create_partition_input(database_name, table_name, values=values)
+    helpers.create_partition(client, database_name, table_name, part_input)
+
+    client.delete_partition(
+        DatabaseName=database_name,
+        TableName=table_name,
+        PartitionValues=values,
+    )
+
+    response = client.get_partitions(DatabaseName=database_name, TableName=table_name)
+    partitions = response['Partitions']
+    partitions.should.be.empty
+
+@mock_glue
+def test_delete_partition_bad_partition():
+    client = boto3.client('glue', region_name='us-east-1')
+    database_name = 'myspecialdatabase'
+    table_name = 'myfirsttable'
+    values = ['2018-10-01']
+    helpers.create_database(client, database_name)
+    helpers.create_table(client, database_name, table_name)
+
+    with assert_raises(ClientError) as exc:
+        client.delete_partition(
+            DatabaseName=database_name,
+            TableName=table_name,
+            PartitionValues=values,
+        )
+
+    exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException')
+
+@mock_glue
+def test_batch_delete_partition():
+    client = boto3.client('glue', region_name='us-east-1')
+    database_name = 'myspecialdatabase'
+    table_name = 'myfirsttable'
+    helpers.create_database(client, database_name)
+    helpers.create_table(client, database_name, table_name)
+
+    partition_inputs = []
+    for i in range(0, 20):
+        values = ["2018-10-{:2}".format(i)]
+        part_input = helpers.create_partition_input(database_name, table_name, values=values)
+        partition_inputs.append(part_input)
+
+    client.batch_create_partition(
+        DatabaseName=database_name,
+        TableName=table_name,
+        PartitionInputList=partition_inputs
+    )
+
+    partition_values = [{"Values": p["Values"]} for p in partition_inputs]
+
+    response = client.batch_delete_partition(
+        DatabaseName=database_name,
+        TableName=table_name,
+        PartitionsToDelete=partition_values,
+    )
+
+    response.should_not.have.key('Errors')
+
+@mock_glue
+def test_batch_delete_partition_with_bad_partitions():
+    client = boto3.client('glue', region_name='us-east-1')
+    database_name = 'myspecialdatabase'
+    table_name = 'myfirsttable'
+    helpers.create_database(client, database_name)
+    helpers.create_table(client, database_name, table_name)
+
+    partition_inputs = []
+    for i in range(0, 20):
+        values = ["2018-10-{:2}".format(i)]
+        part_input = helpers.create_partition_input(database_name, table_name, values=values)
+        partition_inputs.append(part_input)
+
+    client.batch_create_partition(
+        DatabaseName=database_name,
+        TableName=table_name,
+        PartitionInputList=partition_inputs
+    )
+
+    partition_values = [{"Values": p["Values"]} for p in partition_inputs]
+
+    partition_values.insert(5, {"Values": ["2018-11-01"]})
+    partition_values.insert(10, {"Values": ["2018-11-02"]})
+    partition_values.insert(15, {"Values": ["2018-11-03"]})
+
+    response = client.batch_delete_partition(
+        DatabaseName=database_name,
+        TableName=table_name,
+        PartitionsToDelete=partition_values,
+    )
+
+    response.should.have.key('Errors')
+    response['Errors'].should.have.length_of(3)
+    error_partitions = map(lambda x: x['PartitionValues'], response['Errors'])
+    ['2018-11-01'].should.be.within(error_partitions)
+    ['2018-11-02'].should.be.within(error_partitions)
+    ['2018-11-03'].should.be.within(error_partitions)