From c02c0e40338c041b2b13030eebb9c707dbf9684c Mon Sep 17 00:00:00 2001 From: Bjorn Olsen Date: Fri, 1 Nov 2019 21:16:59 +0200 Subject: [PATCH] added several basic tests --- moto/datasync/models.py | 94 +++++++++++++++++------- moto/datasync/responses.py | 81 ++++++++++++++++---- tests/test_datasync/test_datasync.py | 106 +++++++++++++++++++++++++-- 3 files changed, 232 insertions(+), 49 deletions(-) diff --git a/moto/datasync/models.py b/moto/datasync/models.py index 0ecc3a748..7eb839a82 100644 --- a/moto/datasync/models.py +++ b/moto/datasync/models.py @@ -8,34 +8,49 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel -''' -Endpoints I need to test: - -list_locations -list_tasks -start_task_execution -cancel_task_execution -describe_task -describe_task_execution -''' - - -def datasync_json_dump(datasync_object): - return json.dumps(datasync_object) - class Location(BaseModel): - def __init__(self, location_uri, region_name): - self.location_uri = location_uri - self.region_name = region_name - loc = ''.join([random.choice(string.ascii_lowercase + string.digits) for _ in range(17)]) - self.arn = 'arn:aws:datasync:{0}:111222333444:location/loc-{1}'.format(region_name, loc) + def __init__(self, + location_uri, + region_name, + arn_counter=0): + self.uri = location_uri + self.region_name = region_name + # Generate ARN + self.arn = 'arn:aws:datasync:{0}:111222333444:location/loc-{1}'.format(region_name, str(arn_counter).zfill(17)) + + +class Task(BaseModel): + def __init__(self, + source_location_arn, + destination_location_arn, + name, + region_name, + arn_counter=0): + self.source_location_arn = source_location_arn + self.destination_location_arn = destination_location_arn + self.status = 'AVAILABLE' + self.name = name + # Generate ARN + self.arn = 'arn:aws:datasync:{0}:111222333444:task/task-{1}'.format(region_name, str(arn_counter).zfill(17)) + +class TaskExecution(BaseModel): + def __init__(self, + task_arn, + arn_counter=0): + self.task_arn = task_arn + self.arn = '{0}/execution/exec-{1}'.format(task_arn, str(arn_counter).zfill(17)) class DataSyncBackend(BaseBackend): def __init__(self, region_name): self.region_name = region_name - self.locations = OrderedDict() - + # Always increase when new things are created + # This ensures uniqueness + self.arn_counter = 0 + self.locations = dict() + self.tasks = dict() + self.task_executions = dict() + def reset(self): region_name = self.region_name self._reset_model_refs() @@ -43,13 +58,38 @@ class DataSyncBackend(BaseBackend): self.__init__(region_name) def create_location(self, location_uri): - if location_uri in self.locations: - raise Exception('Location already exists') - - location = Location(location_uri, region_name=self.region_name) - self.locations['location_uri'] = location + # TODO BJORN figure out exception + # TODO BJORN test for exception + for arn, location in self.locations.items(): + if location.uri == location_uri: + raise Exception('Location already exists') + self.arn_counter = self.arn_counter + 1 + location = Location(location_uri, + region_name=self.region_name, + arn_counter=self.arn_counter) + self.locations[location.arn] = location return location.arn + def create_task(self, + source_location_arn, + destination_location_arn, + name): + self.arn_counter = self.arn_counter + 1 + task = Task(source_location_arn, + destination_location_arn, + name, + region_name=self.region_name, + arn_counter=self.arn_counter + ) + self.tasks[task.arn] = task + return task.arn + + def start_task_execution(self, task_arn): + self.arn_counter = self.arn_counter + 1 + task_execution = TaskExecution(task_arn, + arn_counter=self.arn_counter) + self.task_executions[task_execution.arn] = task_execution + return task_execution.arn datasync_backends = {} for region in boto3.Session().get_available_regions("datasync"): diff --git a/moto/datasync/responses.py b/moto/datasync/responses.py index bf4790b55..e2f6e2f77 100644 --- a/moto/datasync/responses.py +++ b/moto/datasync/responses.py @@ -10,6 +10,7 @@ from .models import datasync_backends class DataSyncResponse(BaseResponse): + # TODO BJORN check datasync rege region_regex = re.compile(r"://(.+?)\.datasync\.amazonaws\.com") @property @@ -17,25 +18,27 @@ class DataSyncResponse(BaseResponse): return datasync_backends[self.region] def list_locations(self): - locations = self.datasync_backend.locations - logging.info('FOUND SOME SHIT {0}'.format(locations)) - - template = self.response_template(LIST_LOCATIONS_RESPONSE) - r = template.render(locations=locations) - logging.info('list_locations r={0}'.format(r)) - return 'GARBAGE' - return r + locations = list() + for arn, location in self.datasync_backend.locations.items(): + locations.append( { + 'LocationArn': location.arn, + 'LocationUri': location.uri + }) + return json.dumps({"Locations": locations}) def create_location_s3(self): # s3://bucket_name/folder/ s3_bucket_arn = self._get_param("S3BucketArn") + subdirectory = self._get_param("Subdirectory") - bucket_and_path = s3_bucket_arn.split(':')[-1] - location_uri='/'.join(['s3:/', bucket_and_path]) - location = self.datasync_backend.create_location(location_uri) + location_uri_elts = ['s3:/', s3_bucket_arn.split(':')[-1]] + if subdirectory: + location_uri_elts.append(subdirectory) + location_uri='/'.join(location_uri_elts) + arn = self.datasync_backend.create_location(location_uri) - return json.dumps({'LocationArn':location}) + return json.dumps({'LocationArn':arn}) def create_location_smb(self): @@ -44,6 +47,56 @@ class DataSyncResponse(BaseResponse): server_hostname = self._get_param("ServerHostname") location_uri = '/'.join(['smb:/', server_hostname, subdirectory]) - location = self.datasync_backend.create_location(location_uri) + arn = self.datasync_backend.create_location(location_uri) - return json.dumps({'LocationArn':location}) + return json.dumps({'LocationArn':arn}) + + + def create_task(self): + destination_location_arn = self._get_param("DestinationLocationArn") + source_location_arn = self._get_param("SourceLocationArn") + name = self._get_param("Name") + + arn = self.datasync_backend.create_task( + source_location_arn, + destination_location_arn, + name + ) + + return json.dumps({'TaskArn':arn}) + + def list_tasks(self): + tasks = list() + for arn, task in self.datasync_backend.tasks.items(): + tasks.append( { + 'Name': task.name, + 'Status': task.status, + 'TaskArn': task.arn + }) + + return json.dumps({"Tasks": tasks}) + + def describe_task(self): + task_arn = self._get_param("TaskArn") + if task_arn in self.datasync_backend.tasks: + task = self.datasync_backend.tasks[task_arn] + return json.dumps({ + 'TaskArn': task.arn, + 'Name': task.name, + 'Status': task.status, + 'SourceLocationArn': task.source_location_arn, + 'DestinationLocationArn': task.destination_location_arn + }) + # TODO BJORN exception if task_arn not found? + return None + + def start_task_execution(self): + task_arn = self._get_param("TaskArn") + if task_arn in self.datasync_backend.tasks: + arn = self.datasync_backend.start_task_execution( + task_arn + ) + return json.dumps({'TaskExecutionArn':arn}) + + # TODO BJORN exception if task_arn not found? + return None diff --git a/tests/test_datasync/test_datasync.py b/tests/test_datasync/test_datasync.py index b7ecc3ff0..280434791 100644 --- a/tests/test_datasync/test_datasync.py +++ b/tests/test_datasync/test_datasync.py @@ -5,6 +5,15 @@ import boto3 from moto import mock_datasync +''' +Endpoints I need to test: +start_task_execution +cancel_task_execution +describe_task_execution +''' + + + @mock_datasync def test_create_location_smb(): client = boto3.client("datasync", region_name="us-east-1") @@ -19,27 +28,108 @@ def test_create_location_smb(): @mock_datasync def test_create_location_s3(): client = boto3.client("datasync", region_name="us-east-1") - response = client.create_location_s3(S3BucketArn='my_bucket', + response = client.create_location_s3(S3BucketArn='arn:aws:s3:::my_bucket', Subdirectory='dir', S3Config={'BucketAccessRoleArn':'role'}) assert 'LocationArn' in response -''' @mock_datasync def test_list_locations(): client = boto3.client("datasync", region_name="us-east-1") response = client.list_locations() - logging.info ('No locations: {0}'.format(response)) + # TODO BJORN check if Locations exists when there are none + assert len(response['Locations']) == 0 response = client.create_location_smb(ServerHostname='host', Subdirectory='somewhere', User='', Password='', AgentArns=['stuff']) - logging.info ('A location 1 : {0}'.format(response)) response = client.list_locations() - logging.info ('A location 2 : {0}'.format(response)) + assert len(response['Locations']) == 1 + assert response['Locations'][0]['LocationUri'] == 'smb://host/somewhere' - assert 1 == 0 - #assert response == ["TestLocation"] -''' + response = client.create_location_s3(S3BucketArn='arn:aws:s3:::my_bucket', + S3Config={'BucketAccessRoleArn':'role'}) + + response = client.list_locations() + assert len(response['Locations']) == 2 + assert response['Locations'][1]['LocationUri'] == 's3://my_bucket' + + response = client.create_location_s3(S3BucketArn='arn:aws:s3:::my_bucket', + Subdirectory='subdir', + S3Config={'BucketAccessRoleArn':'role'}) + + response = client.list_locations() + assert len(response['Locations']) == 3 + assert response['Locations'][2]['LocationUri'] == 's3://my_bucket/subdir' + +@mock_datasync +def test_create_task(): + client = boto3.client("datasync", region_name="us-east-1") + # TODO BJORN check if task can be created when there are no locations + response = client.create_task( + SourceLocationArn='1', + DestinationLocationArn='2' + ) + assert 'TaskArn' in response + +@mock_datasync +def test_list_tasks(): + client = boto3.client("datasync", region_name="us-east-1") + response = client.create_task( + SourceLocationArn='1', + DestinationLocationArn='2', + ) + response = client.create_task( + SourceLocationArn='3', + DestinationLocationArn='4', + Name='task_name' + ) + response = client.list_tasks() + tasks = response['Tasks'] + assert len(tasks) == 2 + + task = tasks[0] + assert task['Status'] == 'AVAILABLE' + assert 'Name' not in task + + task = tasks[1] + assert task['Status'] == 'AVAILABLE' + assert task['Name'] == 'task_name' + +@mock_datasync +def test_describe_task(): + client = boto3.client("datasync", region_name="us-east-1") + + response = client.create_task( + SourceLocationArn='3', + DestinationLocationArn='4', + Name='task_name' + ) + task_arn = response['TaskArn'] + + response = client.describe_task( + TaskArn=task_arn + ) + + assert 'TaskArn' in response + assert 'Status' in response + assert 'SourceLocationArn' in response + assert 'DestinationLocationArn' in response + +@mock_datasync +def test_start_task_execution(): + client = boto3.client("datasync", region_name="us-east-1") + + response = client.create_task( + SourceLocationArn='3', + DestinationLocationArn='4', + Name='task_name' + ) + task_arn = response['TaskArn'] + + response = client.start_task_execution( + TaskArn=task_arn + ) + assert 'TaskExecutionArn' in response