| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | from __future__ import unicode_literals | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | import copy | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-01-11 13:10:18 +00:00
										 |  |  | from .fixtures.datacatalog import TABLE_INPUT, PARTITION_INPUT, DATABASE_INPUT | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-01-11 13:10:18 +00:00
										 |  |  | def create_database_input(database_name): | 
					
						
							|  |  |  |     database_input = copy.deepcopy(DATABASE_INPUT) | 
					
						
							|  |  |  |     database_input["Name"] = database_name | 
					
						
							|  |  |  |     database_input["LocationUri"] = "s3://my-bucket/{database_name}".format( | 
					
						
							|  |  |  |         database_name=database_name | 
					
						
							|  |  |  |     ) | 
					
						
							|  |  |  |     return database_input | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def create_database(client, database_name, database_input=None): | 
					
						
							|  |  |  |     if database_input is None: | 
					
						
							|  |  |  |         database_input = create_database_input(database_name) | 
					
						
							|  |  |  |     return client.create_database(DatabaseInput=database_input) | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def get_database(client, database_name): | 
					
						
							|  |  |  |     return client.get_database(Name=database_name) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def create_table_input(database_name, table_name, columns=[], partition_keys=[]): | 
					
						
							|  |  |  |     table_input = copy.deepcopy(TABLE_INPUT) | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     table_input["Name"] = table_name | 
					
						
							|  |  |  |     table_input["PartitionKeys"] = partition_keys | 
					
						
							|  |  |  |     table_input["StorageDescriptor"]["Columns"] = columns | 
					
						
							|  |  |  |     table_input["StorageDescriptor"][ | 
					
						
							|  |  |  |         "Location" | 
					
						
							|  |  |  |     ] = "s3://my-bucket/{database_name}/{table_name}".format( | 
					
						
							|  |  |  |         database_name=database_name, table_name=table_name | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     ) | 
					
						
							|  |  |  |     return table_input | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def create_table(client, database_name, table_name, table_input=None, **kwargs): | 
					
						
							|  |  |  |     if table_input is None: | 
					
						
							|  |  |  |         table_input = create_table_input(database_name, table_name, **kwargs) | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     return client.create_table(DatabaseName=database_name, TableInput=table_input) | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def update_table(client, database_name, table_name, table_input=None, **kwargs): | 
					
						
							|  |  |  |     if table_input is None: | 
					
						
							|  |  |  |         table_input = create_table_input(database_name, table_name, **kwargs) | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     return client.update_table(DatabaseName=database_name, TableInput=table_input) | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def get_table(client, database_name, table_name): | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     return client.get_table(DatabaseName=database_name, Name=table_name) | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def get_tables(client, database_name): | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     return client.get_tables(DatabaseName=database_name) | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def get_table_versions(client, database_name, table_name): | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     return client.get_table_versions(DatabaseName=database_name, TableName=table_name) | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def get_table_version(client, database_name, table_name, version_id): | 
					
						
							|  |  |  |     return client.get_table_version( | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |         DatabaseName=database_name, TableName=table_name, VersionId=version_id | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def create_partition_input(database_name, table_name, values=[], columns=[]): | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     root_path = "s3://my-bucket/{database_name}/{table_name}".format( | 
					
						
							|  |  |  |         database_name=database_name, table_name=table_name | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     part_input = copy.deepcopy(PARTITION_INPUT) | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |     part_input["Values"] = values | 
					
						
							|  |  |  |     part_input["StorageDescriptor"]["Columns"] = columns | 
					
						
							|  |  |  |     part_input["StorageDescriptor"]["SerdeInfo"]["Parameters"]["path"] = root_path | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     return part_input | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def create_partition(client, database_name, table_name, partiton_input=None, **kwargs): | 
					
						
							|  |  |  |     if partiton_input is None: | 
					
						
							|  |  |  |         partiton_input = create_partition_input(database_name, table_name, **kwargs) | 
					
						
							|  |  |  |     return client.create_partition( | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |         DatabaseName=database_name, TableName=table_name, PartitionInput=partiton_input | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  | def update_partition( | 
					
						
							|  |  |  |     client, database_name, table_name, old_values=[], partiton_input=None, **kwargs | 
					
						
							|  |  |  | ): | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     if partiton_input is None: | 
					
						
							|  |  |  |         partiton_input = create_partition_input(database_name, table_name, **kwargs) | 
					
						
							|  |  |  |     return client.update_partition( | 
					
						
							|  |  |  |         DatabaseName=database_name, | 
					
						
							|  |  |  |         TableName=table_name, | 
					
						
							|  |  |  |         PartitionInput=partiton_input, | 
					
						
							|  |  |  |         PartitionValueList=old_values, | 
					
						
							|  |  |  |     ) | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | def get_partition(client, database_name, table_name, values): | 
					
						
							|  |  |  |     return client.get_partition( | 
					
						
							| 
									
										
										
										
											2019-12-23 08:38:53 +01:00
										 |  |  |         DatabaseName=database_name, TableName=table_name, PartitionValues=values | 
					
						
							| 
									
										
										
										
											2018-12-21 12:28:56 +01:00
										 |  |  |     ) |