From fa3fd729d13ccbfe3934d500436139ac41d5d7fd Mon Sep 17 00:00:00 2001 From: stephane soulier Date: Fri, 8 Mar 2019 16:27:24 +0100 Subject: [PATCH 01/31] fix bug in Stream init (wrong number of shards) --- moto/kinesis/models.py | 14 ++++---------- tests/test_kinesis/test_kinesis.py | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index d9a47ea87..21773c5a1 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -121,17 +121,11 @@ class Stream(BaseModel): self.shards = {} self.tags = {} - if six.PY3: - izip_longest = itertools.zip_longest - else: - izip_longest = itertools.izip_longest + step = 2**128 // shard_count + for index, start, end in itertools.chain( + map(lambda i: (i, i*step, (i+1) * step), range(shard_count - 1)), + [(shard_count - 1, (shard_count -1) * step, 2**128)]): - for index, start, end in izip_longest(range(shard_count), - range(0, 2**128, 2 ** - 128 // shard_count), - range(2**128 // shard_count, 2 ** - 128, 2**128 // shard_count), - fillvalue=2**128): shard = Shard(index, start, end) self.shards[shard.shard_id] = shard diff --git a/tests/test_kinesis/test_kinesis.py b/tests/test_kinesis/test_kinesis.py index c70236978..aba8e8916 100644 --- a/tests/test_kinesis/test_kinesis.py +++ b/tests/test_kinesis/test_kinesis.py @@ -14,7 +14,7 @@ from moto import mock_kinesis, mock_kinesis_deprecated def test_create_cluster(): conn = boto.kinesis.connect_to_region("us-west-2") - conn.create_stream("my_stream", 2) + conn.create_stream("my_stream", 3) stream_response = conn.describe_stream("my_stream") @@ -26,7 +26,7 @@ def test_create_cluster(): stream["StreamStatus"].should.equal("ACTIVE") shards = stream['Shards'] - shards.should.have.length_of(2) + shards.should.have.length_of(3) @mock_kinesis_deprecated From 92ca7aee12303496c610de677fe550b77c5245e3 Mon Sep 17 00:00:00 2001 From: stephane soulier Date: Fri, 8 Mar 2019 17:03:56 +0100 Subject: [PATCH 02/31] comply with coding style --- moto/kinesis/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index 21773c5a1..57bc0a103 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -122,9 +122,10 @@ class Stream(BaseModel): self.tags = {} step = 2**128 // shard_count - for index, start, end in itertools.chain( - map(lambda i: (i, i*step, (i+1) * step), range(shard_count - 1)), - [(shard_count - 1, (shard_count -1) * step, 2**128)]): + hash_ranges = itertools.chain(map(lambda i: (i, i * step, (i + 1) * step), + range(shard_count - 1)), + [(shard_count - 1, (shard_count - 1) * step, 2**128)]) + for index, start, end in hash_ranges: shard = Shard(index, start, end) self.shards[shard.shard_id] = shard From df2d2ac6b4c57baa00a4fe35e6441b3d60a559bd Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Tue, 11 Jun 2019 14:14:28 -0500 Subject: [PATCH 03/31] Add endpoints to glue for deleting partitions Specifically add glue.delete_partition and glue.batch_delete_partition. --- moto/glue/models.py | 6 ++ moto/glue/responses.py | 36 +++++++++ tests/test_glue/test_datacatalog.py | 109 ++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+) diff --git a/moto/glue/models.py b/moto/glue/models.py index 407c3a020..0989e0e9b 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -138,6 +138,12 @@ class FakeTable(BaseModel): raise PartitionAlreadyExistsException() self.partitions[key] = partition + def delete_partition(self, values): + try: + del self.partitions[str(values)] + except KeyError: + raise PartitionNotFoundException() + class FakePartition(BaseModel): def __init__(self, database_name, table_name, partiton_input): diff --git a/moto/glue/responses.py b/moto/glue/responses.py index 5c001cac6..cb1ecf519 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -6,6 +6,7 @@ from moto.core.responses import BaseResponse from .models import glue_backend from .exceptions import ( PartitionAlreadyExistsException, + PartitionNotFoundException, TableNotFoundException ) @@ -184,3 +185,38 @@ class GlueResponse(BaseResponse): table.update_partition(part_to_update, part_input) return "" + + def delete_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_to_delete = self.parameters.get('PartitionValues') + + table = self.glue_backend.get_table(database_name, table_name) + table.delete_partition(part_to_delete) + + return "" + + def batch_delete_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + + errors_output = [] + for part_input in self.parameters.get('PartitionsToDelete'): + values = part_input.get('Values') + try: + table.delete_partition(values) + except PartitionNotFoundException: + errors_output.append({ + 'PartitionValues': values, + 'ErrorDetail': { + 'ErrorCode': 'EntityNotFoundException', + 'ErrorMessage': 'Partition not found', + } + }) + + out = {} + if errors_output: + out['Errors'] = errors_output + + return json.dumps(out) diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index a02266560..232ab3019 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -531,3 +531,112 @@ def test_update_partition_move(): partition['TableName'].should.equal(table_name) partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) + +@mock_glue +def test_delete_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + part_input = helpers.create_partition_input(database_name, table_name, values=values) + helpers.create_partition(client, database_name, table_name, part_input) + + client.delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + partitions = response['Partitions'] + partitions.should.be.empty + +@mock_glue +def test_delete_partition_bad_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + client.delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + +@mock_glue +def test_batch_delete_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + partition_inputs = [] + for i in range(0, 20): + values = ["2018-10-{:2}".format(i)] + part_input = helpers.create_partition_input(database_name, table_name, values=values) + partition_inputs.append(part_input) + + client.batch_create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInputList=partition_inputs + ) + + partition_values = [{"Values": p["Values"]} for p in partition_inputs] + + response = client.batch_delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionsToDelete=partition_values, + ) + + response.should_not.have.key('Errors') + +@mock_glue +def test_batch_delete_partition_with_bad_partitions(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + partition_inputs = [] + for i in range(0, 20): + values = ["2018-10-{:2}".format(i)] + part_input = helpers.create_partition_input(database_name, table_name, values=values) + partition_inputs.append(part_input) + + client.batch_create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInputList=partition_inputs + ) + + partition_values = [{"Values": p["Values"]} for p in partition_inputs] + + partition_values.insert(5, {"Values": ["2018-11-01"]}) + partition_values.insert(10, {"Values": ["2018-11-02"]}) + partition_values.insert(15, {"Values": ["2018-11-03"]}) + + response = client.batch_delete_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionsToDelete=partition_values, + ) + + response.should.have.key('Errors') + response['Errors'].should.have.length_of(3) + error_partitions = map(lambda x: x['PartitionValues'], response['Errors']) + ['2018-11-01'].should.be.within(error_partitions) + ['2018-11-02'].should.be.within(error_partitions) + ['2018-11-03'].should.be.within(error_partitions) From 17b8ce7df0a61549a9ef8696abaa7886d853818c Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Tue, 11 Jun 2019 14:28:20 -0500 Subject: [PATCH 04/31] Update implementation coverage with current glue coverage --- IMPLEMENTATION_COVERAGE.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7c379d8a6..685db7ec4 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2012,23 +2012,23 @@ - [ ] upload_archive - [ ] upload_multipart_part -## glue - 0% implemented -- [ ] batch_create_partition +## glue - 23% implemented +- [x] batch_create_partition - [ ] batch_delete_connection -- [ ] batch_delete_partition -- [ ] batch_delete_table +- [x] batch_delete_partition +- [x] batch_delete_table - [ ] batch_delete_table_version - [ ] batch_get_partition - [ ] batch_stop_job_run - [ ] create_classifier - [ ] create_connection - [ ] create_crawler -- [ ] create_database +- [x] create_database - [ ] create_dev_endpoint - [ ] create_job -- [ ] create_partition +- [x] create_partition - [ ] create_script -- [ ] create_table +- [x] create_table - [ ] create_trigger - [ ] create_user_defined_function - [ ] delete_classifier @@ -2037,8 +2037,8 @@ - [ ] delete_database - [ ] delete_dev_endpoint - [ ] delete_job -- [ ] delete_partition -- [ ] delete_table +- [x] delete_partition +- [x] delete_table - [ ] delete_table_version - [ ] delete_trigger - [ ] delete_user_defined_function @@ -2050,7 +2050,7 @@ - [ ] get_crawler - [ ] get_crawler_metrics - [ ] get_crawlers -- [ ] get_database +- [x] get_database - [ ] get_databases - [ ] get_dataflow_graph - [ ] get_dev_endpoint @@ -2060,13 +2060,13 @@ - [ ] get_job_runs - [ ] get_jobs - [ ] get_mapping -- [ ] get_partition -- [ ] get_partitions +- [x] get_partition +- [x] get_partitions - [ ] get_plan -- [ ] get_table -- [ ] get_table_version -- [ ] get_table_versions -- [ ] get_tables +- [x] get_table +- [x] get_table_version +- [x] get_table_versions +- [x] get_tables - [ ] get_trigger - [ ] get_triggers - [ ] get_user_defined_function @@ -2087,8 +2087,8 @@ - [ ] update_database - [ ] update_dev_endpoint - [ ] update_job -- [ ] update_partition -- [ ] update_table +- [x] update_partition +- [x] update_table - [ ] update_trigger - [ ] update_user_defined_function From a9319fad04b2466930c5764ffd83bc5fd4349a9a Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Fri, 14 Jun 2019 16:15:14 +0200 Subject: [PATCH 05/31] Route53 get_record_sets: filter type after name According to the documentation [1], name should be filtered first, followed by type. > If you specify both Name and Type > The results begin with the first resource record set in the list > whose name is greater than or equal to Name, and whose type is > greater than or equal to Type. [1]: https://docs.aws.amazon.com/Route53/latest/APIReference/API_ListResourceRecordSets.html --- moto/route53/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 3760d3817..4e85c172b 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -200,12 +200,12 @@ class FakeZone(BaseModel): def get_record_sets(self, start_type, start_name): record_sets = list(self.rrsets) # Copy the list - if start_type: - record_sets = [ - record_set for record_set in record_sets if record_set.type_ >= start_type] if start_name: record_sets = [ record_set for record_set in record_sets if record_set.name >= start_name] + if start_type: + record_sets = [ + record_set for record_set in record_sets if record_set.type_ >= start_type] return record_sets From 9ef69a617a73659442807bb6e015756d09f880e1 Mon Sep 17 00:00:00 2001 From: Niels Laukens Date: Fri, 14 Jun 2019 16:17:50 +0200 Subject: [PATCH 06/31] Route53 get_record_sets: sort names lexicographically --- moto/route53/models.py | 11 ++++++++++- tests/test_route53/test_route53.py | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 4e85c172b..d70307036 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -165,6 +165,12 @@ class RecordSet(BaseModel): hosted_zone.delete_rrset_by_name(self.name) +def reverse_domain_name(domain_name): + if domain_name.endswith('.'): # normalize without trailing dot + domain_name = domain_name[:-1] + return '.'.join(reversed(domain_name.split('.'))) + + class FakeZone(BaseModel): def __init__(self, name, id_, private_zone, comment=None): @@ -202,7 +208,10 @@ class FakeZone(BaseModel): record_sets = list(self.rrsets) # Copy the list if start_name: record_sets = [ - record_set for record_set in record_sets if record_set.name >= start_name] + record_set + for record_set in record_sets + if reverse_domain_name(record_set.name) >= reverse_domain_name(start_name) + ] if start_type: record_sets = [ record_set for record_set in record_sets if record_set.type_ >= start_type] diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index d730f8dcf..e174e1c26 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -123,12 +123,12 @@ def test_rrset(): rrsets.should.have.length_of(2) rrsets = conn.get_all_rrsets( - zoneid, name="foo.bar.testdns.aws.com", type="A") + zoneid, name="bar.foo.testdns.aws.com", type="A") rrsets.should.have.length_of(1) - rrsets[0].resource_records[0].should.equal('1.2.3.4') + rrsets[0].resource_records[0].should.equal('5.6.7.8') rrsets = conn.get_all_rrsets( - zoneid, name="bar.foo.testdns.aws.com", type="A") + zoneid, name="foo.bar.testdns.aws.com", type="A") rrsets.should.have.length_of(2) resource_records = [rr for rr_set in rrsets for rr in rr_set.resource_records] resource_records.should.contain('1.2.3.4') From 9a26c92e72007356280f8ba8bc900872d1543cb8 Mon Sep 17 00:00:00 2001 From: Juan Martinez Date: Mon, 17 Jun 2019 13:41:35 -0400 Subject: [PATCH 07/31] Delete ECR image when it has no tags This is a bug fix to my initial work when implementing batch_delete_image. Deleting the last tag for a given image should delete the image from the backend. I also cleaned up the tests previously-added in the initial implementation. --- moto/ecr/models.py | 5 ++- tests/test_ecr/test_ecr_boto3.py | 71 +++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index 9ff37b7d6..b03f25dee 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -403,7 +403,10 @@ class ECRBackend(BaseBackend): image_found = True repository.images[num].image_tag = image_id["imageTag"] response["imageIds"].append(image.response_batch_delete_image) - repository.images[num].remove_tag(image_id["imageTag"]) + if len(image.image_tags) > 1: + repository.images[num].remove_tag(image_id["imageTag"]) + else: + repository.images.remove(image) if not image_found: failure_response = { diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 221eba842..ec0e4e732 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -740,7 +740,7 @@ def test_batch_get_image_no_tags(): @mock_ecr def test_batch_delete_image_by_tag(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -748,14 +748,13 @@ def test_batch_delete_image_by_tag(): tags = ['v1', 'v1.0', 'latest'] for tag in tags: - put_response = client.put_image( + client.put_image( repositoryName='test_repository', imageManifest=json.dumps(manifest), imageTag=tag, ) describe_response1 = client.describe_images(repositoryName='test_repository') - image_digest = describe_response1['imageDetails'][0]['imageDigest'] batch_delete_response = client.batch_delete_image( registryId='012345678910', @@ -784,10 +783,52 @@ def test_batch_delete_image_by_tag(): len(batch_delete_response['failures']).should.be(0) +@mock_ecr +def test_batch_delete_image_delete_last_tag(): + client = boto3.client('ecr', region_name='us-east-1') + client.create_repository( + repositoryName='test_repository' + ) + + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='v1', + ) + + describe_response1 = client.describe_images(repositoryName='test_repository') + + batch_delete_response = client.batch_delete_image( + registryId='012345678910', + repositoryName='test_repository', + imageIds=[ + { + 'imageTag': 'v1' + }, + ], + ) + + describe_response2 = client.describe_images(repositoryName='test_repository') + + type(describe_response1['imageDetails'][0]['imageTags']).should.be(list) + len(describe_response1['imageDetails'][0]['imageTags']).should.be(1) + + type(describe_response2['imageDetails']).should.be(list) + len(describe_response2['imageDetails']).should.be(0) + + type(batch_delete_response['imageIds']).should.be(list) + len(batch_delete_response['imageIds']).should.be(1) + + batch_delete_response['imageIds'][0]['imageTag'].should.equal("v1") + + type(batch_delete_response['failures']).should.be(list) + len(batch_delete_response['failures']).should.be(0) + + @mock_ecr def test_batch_delete_image_with_nonexistent_tag(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -795,14 +836,13 @@ def test_batch_delete_image_with_nonexistent_tag(): tags = ['v1', 'v1.0', 'latest'] for tag in tags: - put_response = client.put_image( + client.put_image( repositoryName='test_repository', imageManifest=json.dumps(manifest), imageTag=tag, ) describe_response = client.describe_images(repositoryName='test_repository') - image_digest = describe_response['imageDetails'][0]['imageDigest'] missing_tag = "missing-tag" batch_delete_response = client.batch_delete_image( @@ -832,7 +872,7 @@ def test_batch_delete_image_with_nonexistent_tag(): @mock_ecr def test_batch_delete_image_by_digest(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -840,7 +880,7 @@ def test_batch_delete_image_by_digest(): tags = ['v1', 'v2', 'latest'] for tag in tags: - put_response = client.put_image( + client.put_image( repositoryName='test_repository', imageManifest=json.dumps(manifest), imageTag=tag @@ -883,7 +923,7 @@ def test_batch_delete_image_by_digest(): @mock_ecr def test_batch_delete_image_with_invalid_digest(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -891,13 +931,12 @@ def test_batch_delete_image_with_invalid_digest(): tags = ['v1', 'v2', 'latest'] for tag in tags: - put_response = client.put_image( + client.put_image( repositoryName='test_repository', imageManifest=json.dumps(manifest), imageTag=tag ) - describe_response = client.describe_images(repositoryName='test_repository') invalid_image_digest = 'sha256:invalid-digest' batch_delete_response = client.batch_delete_image( @@ -924,7 +963,7 @@ def test_batch_delete_image_with_invalid_digest(): @mock_ecr def test_batch_delete_image_with_missing_parameters(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -950,7 +989,7 @@ def test_batch_delete_image_with_missing_parameters(): @mock_ecr def test_batch_delete_image_with_matching_digest_and_tag(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -958,7 +997,7 @@ def test_batch_delete_image_with_matching_digest_and_tag(): tags = ['v1', 'v1.0', 'latest'] for tag in tags: - put_response = client.put_image( + client.put_image( repositoryName='test_repository', imageManifest=json.dumps(manifest), imageTag=tag @@ -1002,7 +1041,7 @@ def test_batch_delete_image_with_matching_digest_and_tag(): @mock_ecr def test_batch_delete_image_with_mismatched_digest_and_tag(): client = boto3.client('ecr', region_name='us-east-1') - _ = client.create_repository( + client.create_repository( repositoryName='test_repository' ) @@ -1010,7 +1049,7 @@ def test_batch_delete_image_with_mismatched_digest_and_tag(): tags = ['v1', 'latest'] for tag in tags: - put_response = client.put_image( + client.put_image( repositoryName='test_repository', imageManifest=json.dumps(manifest), imageTag=tag From e0078a781ada92a16392f0b172e9160aa972866e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bendeg=C3=BAz=20=C3=81cs?= <30595431+acsbendi@users.noreply.github.com> Date: Tue, 18 Jun 2019 15:27:07 +0200 Subject: [PATCH 08/31] Fixed random_suffix() --- moto/cloudformation/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/cloudformation/utils.py b/moto/cloudformation/utils.py index de75d2c15..e4290ce1a 100644 --- a/moto/cloudformation/utils.py +++ b/moto/cloudformation/utils.py @@ -4,6 +4,7 @@ import six import random import yaml import os +import string from cfnlint import decode, core @@ -29,7 +30,7 @@ def generate_stackset_arn(stackset_id, region_name): def random_suffix(): size = 12 - chars = list(range(10)) + ['A-Z'] + chars = list(range(10)) + list(string.ascii_uppercase) return ''.join(six.text_type(random.choice(chars)) for x in range(size)) From 2275c53b3e4c6d21bc3de6d2dee9a02bad7a3491 Mon Sep 17 00:00:00 2001 From: Juan Martinez Date: Mon, 24 Jun 2019 19:43:35 -0400 Subject: [PATCH 09/31] Update lists of implemented services and endpoints (#2258) --- README.md | 178 ++++++++++++++++++++++++------------------------- docs/index.rst | 149 ++++++++++++++++++++++++----------------- 2 files changed, 178 insertions(+), 149 deletions(-) diff --git a/README.md b/README.md index 70faee2c8..55f2551e9 100644 --- a/README.md +++ b/README.md @@ -55,95 +55,95 @@ With the decorator wrapping the test, all the calls to s3 are automatically mock It gets even better! Moto isn't just for Python code and it isn't just for S3. Look at the [standalone server mode](https://github.com/spulec/moto#stand-alone-server-mode) for more information about running Moto with other languages. Here's the status of the other AWS services implemented: ```gherkin -|------------------------------------------------------------------------------| -| Service Name | Decorator | Development Status | -|------------------------------------------------------------------------------| -| ACM | @mock_acm | all endpoints done | -|------------------------------------------------------------------------------| -| API Gateway | @mock_apigateway | core endpoints done | -|------------------------------------------------------------------------------| -| Autoscaling | @mock_autoscaling| core endpoints done | -|------------------------------------------------------------------------------| -| Cloudformation | @mock_cloudformation| core endpoints done | -|------------------------------------------------------------------------------| -| Cloudwatch | @mock_cloudwatch | basic endpoints done | -|------------------------------------------------------------------------------| -| CloudwatchEvents | @mock_events | all endpoints done | -|------------------------------------------------------------------------------| -| Cognito Identity | @mock_cognitoidentity| basic endpoints done | -|------------------------------------------------------------------------------| -| Cognito Identity Provider | @mock_cognitoidp| basic endpoints done | -|------------------------------------------------------------------------------| -| Config | @mock_config | basic endpoints done | -|------------------------------------------------------------------------------| -| Data Pipeline | @mock_datapipeline| basic endpoints done | -|------------------------------------------------------------------------------| -| DynamoDB | @mock_dynamodb | core endpoints done | -| DynamoDB2 | @mock_dynamodb2 | all endpoints + partial indexes | -|------------------------------------------------------------------------------| -| EC2 | @mock_ec2 | core endpoints done | -| - AMI | | core endpoints done | -| - EBS | | core endpoints done | -| - Instances | | all endpoints done | -| - Security Groups | | core endpoints done | -| - Tags | | all endpoints done | -|------------------------------------------------------------------------------| -| ECR | @mock_ecr | basic endpoints done | -|------------------------------------------------------------------------------| -| ECS | @mock_ecs | basic endpoints done | -|------------------------------------------------------------------------------| -| ELB | @mock_elb | core endpoints done | -|------------------------------------------------------------------------------| -| ELBv2 | @mock_elbv2 | all endpoints done | -|------------------------------------------------------------------------------| -| EMR | @mock_emr | core endpoints done | -|------------------------------------------------------------------------------| -| Glacier | @mock_glacier | core endpoints done | -|------------------------------------------------------------------------------| -| IAM | @mock_iam | core endpoints done | -|------------------------------------------------------------------------------| -| IoT | @mock_iot | core endpoints done | -| | @mock_iotdata | core endpoints done | -|------------------------------------------------------------------------------| -| Lambda | @mock_lambda | basic endpoints done, requires | -| | | docker | -|------------------------------------------------------------------------------| -| Logs | @mock_logs | basic endpoints done | -|------------------------------------------------------------------------------| -| Kinesis | @mock_kinesis | core endpoints done | -|------------------------------------------------------------------------------| -| KMS | @mock_kms | basic endpoints done | -|------------------------------------------------------------------------------| -| Organizations | @mock_organizations | some core endpoints done | -|------------------------------------------------------------------------------| -| Polly | @mock_polly | all endpoints done | -|------------------------------------------------------------------------------| -| RDS | @mock_rds | core endpoints done | -|------------------------------------------------------------------------------| -| RDS2 | @mock_rds2 | core endpoints done | -|------------------------------------------------------------------------------| -| Redshift | @mock_redshift | core endpoints done | -|------------------------------------------------------------------------------| -| Route53 | @mock_route53 | core endpoints done | -|------------------------------------------------------------------------------| -| S3 | @mock_s3 | core endpoints done | -|------------------------------------------------------------------------------| -| SecretsManager | @mock_secretsmanager | basic endpoints done -|------------------------------------------------------------------------------| -| SES | @mock_ses | all endpoints done | -|------------------------------------------------------------------------------| -| SNS | @mock_sns | all endpoints done | -|------------------------------------------------------------------------------| -| SQS | @mock_sqs | core endpoints done | -|------------------------------------------------------------------------------| -| SSM | @mock_ssm | core endpoints done | -|------------------------------------------------------------------------------| -| STS | @mock_sts | core endpoints done | -|------------------------------------------------------------------------------| -| SWF | @mock_swf | basic endpoints done | -|------------------------------------------------------------------------------| -| X-Ray | @mock_xray | all endpoints done | -|------------------------------------------------------------------------------| +|-------------------------------------------------------------------------------------| +| Service Name | Decorator | Development Status | +|-------------------------------------------------------------------------------------| +| ACM | @mock_acm | all endpoints done | +|-------------------------------------------------------------------------------------| +| API Gateway | @mock_apigateway | core endpoints done | +|-------------------------------------------------------------------------------------| +| Autoscaling | @mock_autoscaling | core endpoints done | +|-------------------------------------------------------------------------------------| +| Cloudformation | @mock_cloudformation | core endpoints done | +|-------------------------------------------------------------------------------------| +| Cloudwatch | @mock_cloudwatch | basic endpoints done | +|-------------------------------------------------------------------------------------| +| CloudwatchEvents | @mock_events | all endpoints done | +|-------------------------------------------------------------------------------------| +| Cognito Identity | @mock_cognitoidentity | basic endpoints done | +|-------------------------------------------------------------------------------------| +| Cognito Identity Provider | @mock_cognitoidp | basic endpoints done | +|-------------------------------------------------------------------------------------| +| Config | @mock_config | basic endpoints done | +|-------------------------------------------------------------------------------------| +| Data Pipeline | @mock_datapipeline | basic endpoints done | +|-------------------------------------------------------------------------------------| +| DynamoDB | @mock_dynamodb | core endpoints done | +| DynamoDB2 | @mock_dynamodb2 | all endpoints + partial indexes | +|-------------------------------------------------------------------------------------| +| EC2 | @mock_ec2 | core endpoints done | +| - AMI | | core endpoints done | +| - EBS | | core endpoints done | +| - Instances | | all endpoints done | +| - Security Groups | | core endpoints done | +| - Tags | | all endpoints done | +|-------------------------------------------------------------------------------------| +| ECR | @mock_ecr | basic endpoints done | +|-------------------------------------------------------------------------------------| +| ECS | @mock_ecs | basic endpoints done | +|-------------------------------------------------------------------------------------| +| ELB | @mock_elb | core endpoints done | +|-------------------------------------------------------------------------------------| +| ELBv2 | @mock_elbv2 | all endpoints done | +|-------------------------------------------------------------------------------------| +| EMR | @mock_emr | core endpoints done | +|-------------------------------------------------------------------------------------| +| Glacier | @mock_glacier | core endpoints done | +|-------------------------------------------------------------------------------------| +| IAM | @mock_iam | core endpoints done | +|-------------------------------------------------------------------------------------| +| IoT | @mock_iot | core endpoints done | +| | @mock_iotdata | core endpoints done | +|-------------------------------------------------------------------------------------| +| Kinesis | @mock_kinesis | core endpoints done | +|-------------------------------------------------------------------------------------| +| KMS | @mock_kms | basic endpoints done | +|-------------------------------------------------------------------------------------| +| Lambda | @mock_lambda | basic endpoints done, requires | +| | | docker | +|-------------------------------------------------------------------------------------| +| Logs | @mock_logs | basic endpoints done | +|-------------------------------------------------------------------------------------| +| Organizations | @mock_organizations | some core endpoints done | +|-------------------------------------------------------------------------------------| +| Polly | @mock_polly | all endpoints done | +|-------------------------------------------------------------------------------------| +| RDS | @mock_rds | core endpoints done | +|-------------------------------------------------------------------------------------| +| RDS2 | @mock_rds2 | core endpoints done | +|-------------------------------------------------------------------------------------| +| Redshift | @mock_redshift | core endpoints done | +|-------------------------------------------------------------------------------------| +| Route53 | @mock_route53 | core endpoints done | +|-------------------------------------------------------------------------------------| +| S3 | @mock_s3 | core endpoints done | +|-------------------------------------------------------------------------------------| +| SecretsManager | @mock_secretsmanager | basic endpoints done | +|-------------------------------------------------------------------------------------| +| SES | @mock_ses | all endpoints done | +|-------------------------------------------------------------------------------------| +| SNS | @mock_sns | all endpoints done | +|-------------------------------------------------------------------------------------| +| SQS | @mock_sqs | core endpoints done | +|-------------------------------------------------------------------------------------| +| SSM | @mock_ssm | core endpoints done | +|-------------------------------------------------------------------------------------| +| STS | @mock_sts | core endpoints done | +|-------------------------------------------------------------------------------------| +| SWF | @mock_swf | basic endpoints done | +|-------------------------------------------------------------------------------------| +| X-Ray | @mock_xray | all endpoints done | +|-------------------------------------------------------------------------------------| ``` For a full list of endpoint [implementation coverage](https://github.com/spulec/moto/blob/master/IMPLEMENTATION_COVERAGE.md) diff --git a/docs/index.rst b/docs/index.rst index 66e12e4bd..4811fb797 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,66 +17,95 @@ with ``moto`` and its usage. Currently implemented Services: ------------------------------- -+-----------------------+---------------------+-----------------------------------+ -| Service Name | Decorator | Development Status | -+=======================+=====================+===================================+ -| API Gateway | @mock_apigateway | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Autoscaling | @mock_autoscaling | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Cloudformation | @mock_cloudformation| core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Cloudwatch | @mock_cloudwatch | basic endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Data Pipeline | @mock_datapipeline | basic endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| - DynamoDB | - @mock_dynamodb | - core endpoints done | -| - DynamoDB2 | - @mock_dynamodb2 | - core endpoints + partial indexes| -+-----------------------+---------------------+-----------------------------------+ -| EC2 | @mock_ec2 | core endpoints done | -| - AMI | | - core endpoints done | -| - EBS | | - core endpoints done | -| - Instances | | - all endpoints done | -| - Security Groups | | - core endpoints done | -| - Tags | | - all endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| ECS | @mock_ecs | basic endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| ELB | @mock_elb | core endpoints done | -| | @mock_elbv2 | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| EMR | @mock_emr | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Glacier | @mock_glacier | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| IAM | @mock_iam | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Lambda | @mock_lambda | basic endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Kinesis | @mock_kinesis | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| KMS | @mock_kms | basic endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| RDS | @mock_rds | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| RDS2 | @mock_rds2 | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Redshift | @mock_redshift | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| Route53 | @mock_route53 | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| S3 | @mock_s3 | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| SES | @mock_ses | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| SNS | @mock_sns | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| SQS | @mock_sqs | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| STS | @mock_sts | core endpoints done | -+-----------------------+---------------------+-----------------------------------+ -| SWF | @mock_swf | basic endpoints done | -+-----------------------+---------------------+-----------------------------------+ ++---------------------------+-----------------------+------------------------------------+ +| Service Name | Decorator | Development Status | ++===========================+=======================+====================================+ +| ACM | @mock_acm | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| API Gateway | @mock_apigateway | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Autoscaling | @mock_autoscaling | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Cloudformation | @mock_cloudformation | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Cloudwatch | @mock_cloudwatch | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| CloudwatchEvents | @mock_events | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Cognito Identity | @mock_cognitoidentity | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Cognito Identity Provider | @mock_cognitoidp | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Config | @mock_config | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Data Pipeline | @mock_datapipeline | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| DynamoDB | - @mock_dynamodb | - core endpoints done | +| DynamoDB2 | - @mock_dynamodb2 | - core endpoints + partial indexes | ++---------------------------+-----------------------+------------------------------------+ +| EC2 | @mock_ec2 | core endpoints done | +| - AMI | | - core endpoints done | +| - EBS | | - core endpoints done | +| - Instances | | - all endpoints done | +| - Security Groups | | - core endpoints done | +| - Tags | | - all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| ECR | @mock_ecr | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| ECS | @mock_ecs | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| ELB | @mock_elb | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| ELBv2 | @mock_elbv2 | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| EMR | @mock_emr | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Glacier | @mock_glacier | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| IAM | @mock_iam | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| IoT | @mock_iot | core endpoints done | +| | @mock_iotdata | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Kinesis | @mock_kinesis | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| KMS | @mock_kms | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Lambda | @mock_lambda | basic endpoints done, | +| | | requires docker | ++---------------------------+-----------------------+------------------------------------+ +| Logs | @mock_logs | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Organizations | @mock_organizations | some core edpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Polly | @mock_polly | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| RDS | @mock_rds | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| RDS2 | @mock_rds2 | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Redshift | @mock_redshift | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| Route53 | @mock_route53 | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| S3 | @mock_s3 | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| SecretsManager | @mock_secretsmanager | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| SES | @mock_ses | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| SNS | @mock_sns | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| SQS | @mock_sqs | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| SSM | @mock_ssm | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| STS | @mock_sts | core endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| SWF | @mock_swf | basic endpoints done | ++---------------------------+-----------------------+------------------------------------+ +| X-Ray | @mock_xray | all endpoints done | ++---------------------------+-----------------------+------------------------------------+ From 7cc1afa25f5bdddf93da8dce2b57f9990603452e Mon Sep 17 00:00:00 2001 From: Adam Bruehl Date: Wed, 26 Jun 2019 11:56:17 -0400 Subject: [PATCH 10/31] ELBv2 LBs names must be 32 char or shorter --- moto/cloudformation/parsing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 3bf994bed..f2e03bd81 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -246,7 +246,8 @@ def resource_name_property_from_type(resource_type): def generate_resource_name(resource_type, stack_name, logical_id): - if resource_type == "AWS::ElasticLoadBalancingV2::TargetGroup": + if resource_type in ["AWS::ElasticLoadBalancingV2::TargetGroup", + "AWS::ElasticLoadBalancingV2::LoadBalancer"]: # Target group names need to be less than 32 characters, so when cloudformation creates a name for you # it makes sure to stay under that limit name_prefix = '{0}-{1}'.format(stack_name, logical_id) From e50ce7287dedabdb477e726d2289ff0e17e97189 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Wed, 26 Jun 2019 21:54:48 +0100 Subject: [PATCH 11/31] ProjectionExpression works with table.scan() --- moto/dynamodb2/models.py | 14 ++++- moto/dynamodb2/responses.py | 5 +- tests/test_dynamodb2/test_dynamodb.py | 84 +++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 6bcde41b2..f04bb830f 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -724,7 +724,7 @@ class Table(BaseModel): if idx_col_set.issubset(set(hash_set.attrs)): yield hash_set - def scan(self, filters, limit, exclusive_start_key, filter_expression=None, index_name=None): + def scan(self, filters, limit, exclusive_start_key, filter_expression=None, index_name=None, projection_expression=None): results = [] scanned_count = 0 all_indexes = self.all_indexes() @@ -763,6 +763,14 @@ class Table(BaseModel): if passes_all_conditions: results.append(item) + if projection_expression: + expressions = [x.strip() for x in projection_expression.split(',')] + results = copy.deepcopy(results) + for result in results: + for attr in list(result.attrs): + if attr not in expressions: + result.attrs.pop(attr) + results, last_evaluated_key = self._trim_results(results, limit, exclusive_start_key, index_name) return results, scanned_count, last_evaluated_key @@ -962,7 +970,7 @@ class DynamoDBBackend(BaseBackend): return table.query(hash_key, range_comparison, range_values, limit, exclusive_start_key, scan_index_forward, projection_expression, index_name, filter_expression, **filter_kwargs) - def scan(self, table_name, filters, limit, exclusive_start_key, filter_expression, expr_names, expr_values, index_name): + def scan(self, table_name, filters, limit, exclusive_start_key, filter_expression, expr_names, expr_values, index_name, projection_expression): table = self.tables.get(table_name) if not table: return None, None, None @@ -977,7 +985,7 @@ class DynamoDBBackend(BaseBackend): else: filter_expression = Op(None, None) # Will always eval to true - return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name) + return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name, projection_expression) def update_item(self, table_name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values, expected=None): diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 7eb565747..01f2df088 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -558,7 +558,7 @@ class DynamoHandler(BaseResponse): filter_expression = self.body.get('FilterExpression') expression_attribute_values = self.body.get('ExpressionAttributeValues', {}) expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) - + projection_expression = self.body.get('ProjectionExpression') exclusive_start_key = self.body.get('ExclusiveStartKey') limit = self.body.get("Limit") index_name = self.body.get('IndexName') @@ -570,7 +570,8 @@ class DynamoHandler(BaseResponse): filter_expression, expression_attribute_names, expression_attribute_values, - index_name) + index_name, + projection_expression) except InvalidIndexNameError as err: er = 'com.amazonaws.dynamodb.v20111205#ValidationException' return self.error(er, str(err)) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 77846de04..67610ad72 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -452,6 +452,90 @@ def test_basic_projection_expressions(): assert 'body' in results['Items'][1] assert 'forum_name' in results['Items'][1] +@mock_dynamodb2 +def test_basic_projection_expressions_using_scan(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'forum_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'subject', + 'KeyType': 'RANGE' + }, + ], + AttributeDefinitions=[ + { + 'AttributeName': 'forum_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'subject', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) + table = dynamodb.Table('users') + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '123', + 'body': 'some test message' + }) + + table.put_item(Item={ + 'forum_name': 'not-the-key', + 'subject': '123', + 'body': 'some other test message' + }) + # Test a query returning all items + results = table.scan( + FilterExpression=Key('forum_name').eq( + 'the-key'), + ProjectionExpression='body, subject' + ) + + assert 'body' in results['Items'][0] + assert results['Items'][0]['body'] == 'some test message' + assert 'subject' in results['Items'][0] + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '1234', + 'body': 'yet another test message' + }) + + results = table.scan( + FilterExpression=Key('forum_name').eq( + 'the-key'), + ProjectionExpression='body' + ) + + assert 'body' in results['Items'][0] + assert 'subject' not in results['Items'][0] + assert results['Items'][0]['body'] == 'some test message' + assert 'body' in results['Items'][1] + assert 'subject' not in results['Items'][1] + assert results['Items'][1]['body'] == 'yet another test message' + + # The projection expression should not remove data from storage + results = table.query( + KeyConditionExpression=Key('forum_name').eq( + 'the-key'), + ) + assert 'subject' in results['Items'][0] + assert 'body' in results['Items'][1] + assert 'forum_name' in results['Items'][1] + @mock_dynamodb2 def test_basic_projection_expressions_with_attr_expression_names(): From b60097fab2603420ed8dc3b793b77d1f83284f32 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 19:08:32 +0100 Subject: [PATCH 12/31] improve test case --- tests/test_dynamodb2/test_dynamodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 67610ad72..532ee4a62 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -522,10 +522,10 @@ def test_basic_projection_expressions_using_scan(): assert 'body' in results['Items'][0] assert 'subject' not in results['Items'][0] - assert results['Items'][0]['body'] == 'some test message' + assert 'forum_name' not in results['Items'][0] assert 'body' in results['Items'][1] assert 'subject' not in results['Items'][1] - assert results['Items'][1]['body'] == 'yet another test message' + assert 'forum_name' not in results['Items'][1] # The projection expression should not remove data from storage results = table.query( @@ -536,6 +536,7 @@ def test_basic_projection_expressions_using_scan(): assert 'body' in results['Items'][1] assert 'forum_name' in results['Items'][1] +test_basic_projection_expressions_using_scan() @mock_dynamodb2 def test_basic_projection_expressions_with_attr_expression_names(): From 949637a14cf407d91bddef94b07187a7583f782e Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 19:12:53 +0100 Subject: [PATCH 13/31] remove debug --- tests/test_dynamodb2/test_dynamodb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 532ee4a62..e0d44fb82 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -536,7 +536,6 @@ def test_basic_projection_expressions_using_scan(): assert 'body' in results['Items'][1] assert 'forum_name' in results['Items'][1] -test_basic_projection_expressions_using_scan() @mock_dynamodb2 def test_basic_projection_expressions_with_attr_expression_names(): From 4ce0e6bbcb51763cb281977fd3be7c937da23df5 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 19:37:46 +0100 Subject: [PATCH 14/31] add extra test for ProjectionExpression using scan with ExpressionAttributes --- moto/dynamodb2/models.py | 3 + moto/dynamodb2/responses.py | 2 +- tests/test_dynamodb2/test_dynamodb.py | 80 ++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index f04bb830f..fb34e19cc 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -985,6 +985,9 @@ class DynamoDBBackend(BaseBackend): else: filter_expression = Op(None, None) # Will always eval to true + projection_expression = ','.join([expr_names[attr] if attr in expr_names else attr + for attr in projection_expression.replace(' ', '').split(',')]) + return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name, projection_expression) def update_item(self, table_name, key, update_expression, attribute_updates, expression_attribute_names, diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 01f2df088..943340438 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -558,7 +558,7 @@ class DynamoHandler(BaseResponse): filter_expression = self.body.get('FilterExpression') expression_attribute_values = self.body.get('ExpressionAttributeValues', {}) expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) - projection_expression = self.body.get('ProjectionExpression') + projection_expression = self.body.get('ProjectionExpression', '') exclusive_start_key = self.body.get('ExclusiveStartKey') limit = self.body.get("Limit") index_name = self.body.get('IndexName') diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e0d44fb82..faa467aab 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -497,7 +497,7 @@ def test_basic_projection_expressions_using_scan(): 'subject': '123', 'body': 'some other test message' }) - # Test a query returning all items + # Test a scan returning all items results = table.scan( FilterExpression=Key('forum_name').eq( 'the-key'), @@ -603,6 +603,84 @@ def test_basic_projection_expressions_with_attr_expression_names(): assert 'attachment' in results['Items'][0] assert results['Items'][0]['attachment'] == 'something' +@mock_dynamodb2 +def test_basic_projection_expressions_using_scan_with_attr_expression_names(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'forum_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'subject', + 'KeyType': 'RANGE' + }, + ], + AttributeDefinitions=[ + { + 'AttributeName': 'forum_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'subject', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) + table = dynamodb.Table('users') + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '123', + 'body': 'some test message', + 'attachment': 'something' + }) + + table.put_item(Item={ + 'forum_name': 'not-the-key', + 'subject': '123', + 'body': 'some other test message', + 'attachment': 'something' + }) + # Test a scan returning all items + + results = table.scan( + FilterExpression=Key('forum_name').eq( + 'the-key'), + ProjectionExpression='#rl, #rt, subject', + ExpressionAttributeNames={ + '#rl': 'body', + '#rt': 'attachment' + }, + ) + + assert 'body' in results['Items'][0] + assert 'attachment' in results['Items'][0] + assert 'subject' in results['Items'][0] + assert 'form_name' not in results['Items'][0] + + # Test without a FilterExpression + results = table.scan( + ProjectionExpression='#rl, #rt, subject', + ExpressionAttributeNames={ + '#rl': 'body', + '#rt': 'attachment' + }, + ) + + assert 'body' in results['Items'][0] + assert 'attachment' in results['Items'][0] + assert 'subject' in results['Items'][0] + assert 'form_name' not in results['Items'][0] + @mock_dynamodb2 def test_put_item_returns_consumed_capacity(): From cd666758f609e1067cf182880e7c4dd7525e4732 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 20:13:36 +0100 Subject: [PATCH 15/31] one liner --- moto/dynamodb2/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index fb34e19cc..bfbb654b4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -985,8 +985,7 @@ class DynamoDBBackend(BaseBackend): else: filter_expression = Op(None, None) # Will always eval to true - projection_expression = ','.join([expr_names[attr] if attr in expr_names else attr - for attr in projection_expression.replace(' ', '').split(',')]) + projection_expression = ','.join([expr_names.get(attr, attr) for attr in projection_expression.replace(' ', '').split(',')]) return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name, projection_expression) From 6fa51ac3b423cf8942bb14db2c7de23237cca19d Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 22:31:21 -0500 Subject: [PATCH 16/31] Setup pypi automatic publishing. --- .travis.yml | 67 +++++++++++++-------- README.md | 8 +++ setup.py | 11 +++- update_version_from_git.py | 119 +++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 26 deletions(-) create mode 100644 update_version_from_git.py diff --git a/.travis.yml b/.travis.yml index 5bc9779f3..fd1c31bdb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,36 +2,53 @@ dist: xenial language: python sudo: false services: - - docker +- docker python: - - 2.7 - - 3.6 - - 3.7 +- 2.7 +- 3.6 +- 3.7 env: - - TEST_SERVER_MODE=false - - TEST_SERVER_MODE=true +- TEST_SERVER_MODE=false +- TEST_SERVER_MODE=true before_install: - - export BOTO_CONFIG=/dev/null +- export BOTO_CONFIG=/dev/null install: - # We build moto first so the docker container doesn't try to compile it as well, also note we don't use - # -d for docker run so the logs show up in travis - # Python images come from here: https://hub.docker.com/_/python/ - - | - python setup.py sdist +- | + python setup.py sdist - if [ "$TEST_SERVER_MODE" = "true" ]; then - docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh & - fi - travis_retry pip install boto==2.45.0 - travis_retry pip install boto3 - travis_retry pip install dist/moto*.gz - travis_retry pip install coveralls==1.1 - travis_retry pip install -r requirements-dev.txt + if [ "$TEST_SERVER_MODE" = "true" ]; then + docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh & + fi + travis_retry pip install boto==2.45.0 + travis_retry pip install boto3 + travis_retry pip install dist/moto*.gz + travis_retry pip install coveralls==1.1 + travis_retry pip install -r requirements-dev.txt - if [ "$TEST_SERVER_MODE" = "true" ]; then - python wait_for.py - fi + if [ "$TEST_SERVER_MODE" = "true" ]; then + python wait_for.py + fi script: - - make test +- make test after_success: - - coveralls +- coveralls +before_deploy: +- git checkout $TRAVIS_BRANCH +- python update_version_from_git.py +deploy: + - provider: pypi + distributions: sdist bdist_wheel + user: spulec + password: + secure: NxnPylnTfekJmGyoufCw0lMoYRskSMJzvAIyAlJJVYKwEhmiCPOrdy5qV8i8mRZ1AkUsqU3jBZ/PD56n96clHW0E3d080UleRDj6JpyALVdeLfMqZl9kLmZ8bqakWzYq3VSJKw2zGP/L4tPGf8wTK1SUv9yl/YNDsBdCkjDverw= + on: + branch: + - master + skip_cleanup: true + - provider: pypi + distributions: sdist bdist_wheel + user: spulec + password: + secure: NxnPylnTfekJmGyoufCw0lMoYRskSMJzvAIyAlJJVYKwEhmiCPOrdy5qV8i8mRZ1AkUsqU3jBZ/PD56n96clHW0E3d080UleRDj6JpyALVdeLfMqZl9kLmZ8bqakWzYq3VSJKw2zGP/L4tPGf8wTK1SUv9yl/YNDsBdCkjDverw= + on: + tags: true diff --git a/README.md b/README.md index 55f2551e9..e4c88dec8 100644 --- a/README.md +++ b/README.md @@ -318,3 +318,11 @@ boto3.resource( ```console $ pip install moto ``` + +## Releases + +Releases are done from travisci. Fairly closely following this: +https://docs.travis-ci.com/user/deployment/pypi/ + +- Commits to `master` branch do a dev deploy to pypi. +- Commits to a tag do a real deploy to pypi. diff --git a/setup.py b/setup.py index bc53ff6bb..fcb9b6d17 100755 --- a/setup.py +++ b/setup.py @@ -18,6 +18,15 @@ def read(*parts): return fp.read() +def get_version(): + version_file = read('moto', '__init__.py') + version_match = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', + version_file, re.MULTILINE) + if version_match: + return version_match.group(1) + raise RuntimeError('Unable to find version string.') + + install_requires = [ "Jinja2>=2.10.1", "boto>=2.36.0", @@ -57,7 +66,7 @@ else: setup( name='moto', - version='1.3.8', + version=get_version(), description='A library that allows your python tests to easily' ' mock out the boto library', long_description=read('README.md'), diff --git a/update_version_from_git.py b/update_version_from_git.py new file mode 100644 index 000000000..a48ce5945 --- /dev/null +++ b/update_version_from_git.py @@ -0,0 +1,119 @@ +""" +Adapted from https://github.com/pygame/pygameweb/blob/master/pygameweb/builds/update_version_from_git.py + +For updating the version from git. +__init__.py contains a __version__ field. +Update that. +If we are on master, we want to update the version as a pre-release. +git describe --tags +With these: + __init__.py + __version__= '0.0.2' + git describe --tags + 0.0.1-22-g729a5ae +We want this: + __init__.py + __version__= '0.0.2.dev22.g729a5ae' +Get the branch/tag name with this. + git symbolic-ref -q --short HEAD || git describe --tags --exact-match +""" + +import io +import os +import re +import subprocess + + +def migrate_source_attribute(attr, to_this, target_file, regex): + """Updates __magic__ attributes in the source file""" + change_this = re.compile(regex, re.S) + new_file = [] + found = False + + with open(target_file, 'r') as fp: + lines = fp.readlines() + + for line in lines: + if line.startswith(attr): + found = True + line = re.sub(change_this, to_this, line) + new_file.append(line) + + if found: + with open(target_file, 'w') as fp: + fp.writelines(new_file) + +def migrate_version(target_file, new_version): + """Updates __version__ in the source file""" + regex = r"['\"](.*)['\"]" + migrate_source_attribute('__version__', f"'{new_version}'", target_file, regex) + + +def is_master_branch(): + cmd = ('git rev-parse --abbrev-ref HEAD') + tag_branch = subprocess.check_output(cmd, shell=True) + return tag_branch in [b'master\n'] + +def git_tag_name(): + cmd = ('git describe --tags') + tag_branch = subprocess.check_output(cmd, shell=True) + tag_branch = tag_branch.decode().strip() + return tag_branch + +def get_git_version_info(): + cmd = 'git describe --tags' + ver_str = subprocess.check_output(cmd, shell=True) + ver, commits_since, githash = ver_str.decode().strip().split('-') + return ver, commits_since, githash + +def prerelease_version(): + """ return what the prerelease version should be. + https://packaging.python.org/tutorials/distributing-packages/#pre-release-versioning + 0.0.2.dev22 + """ + ver, commits_since, githash = get_git_version_info() + initpy_ver = get_version() + + assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2 or 0.0.2.dev' + assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' + # return f'{initpy_ver}.dev{commits_since}+git.{commits_since}.{githash}' + return f'{initpy_ver}{commits_since}' + +def read(*parts): + """ Reads in file from *parts. + """ + try: + return io.open(os.path.join(*parts), 'r', encoding='utf-8').read() + except IOError: + return '' + +def get_version(): + """ Returns version from moto/__init__.py + """ + version_file = read('moto', '__init__.py') + version_match = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', + version_file, re.MULTILINE) + if version_match: + return version_match.group(1) + raise RuntimeError('Unable to find version string.') + + +def release_version_correct(): + """Makes sure the: + - prerelease verion for master is correct. + - release version is correct for tags. + """ + if is_master_branch(): + # update for a pre release version. + initpy = os.path.abspath("moto/__init__.py") + + new_version = prerelease_version() + print(f'updating version in __init__.py to {new_version}') + migrate_version(initpy, new_version) + else: + # check that we are a tag with the same version as in __init__.py + assert get_version() == git_tag_name(), 'git tag/branch name not the same as moto/__init__.py __verion__' + + +if __name__ == '__main__': + release_version_correct() From 0a2bf3a26288f3a9f90ffa2e7a8dd87c998a01aa Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 22:51:38 -0500 Subject: [PATCH 17/31] Fallback on descibing tags. --- update_version_from_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index a48ce5945..31fa08139 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -61,7 +61,7 @@ def git_tag_name(): return tag_branch def get_git_version_info(): - cmd = 'git describe --tags' + cmd = 'git describe --tags --always' ver_str = subprocess.check_output(cmd, shell=True) ver, commits_since, githash = ver_str.decode().strip().split('-') return ver, commits_since, githash From 73f726fffe186d71d6507be8129edb1873c0e1f3 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 23:03:06 -0500 Subject: [PATCH 18/31] Need to do a git fetch so we have full depth of tag history. --- .travis.yml | 1 + update_version_from_git.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd1c31bdb..0bbc06261 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ after_success: - coveralls before_deploy: - git checkout $TRAVIS_BRANCH +- git fetch --unshallow - python update_version_from_git.py deploy: - provider: pypi diff --git a/update_version_from_git.py b/update_version_from_git.py index 31fa08139..a48ce5945 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -61,7 +61,7 @@ def git_tag_name(): return tag_branch def get_git_version_info(): - cmd = 'git describe --tags --always' + cmd = 'git describe --tags' ver_str = subprocess.check_output(cmd, shell=True) ver, commits_since, githash = ver_str.decode().strip().split('-') return ver, commits_since, githash From c4da5632ab5a9ff412bef04174dd7522d0570a93 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 23:18:14 -0500 Subject: [PATCH 19/31] Bump version number for next release. --- moto/__init__.py | 2 +- update_version_from_git.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 8c51bab27..9c974f00d 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.8' +__version__ = '1.3.9' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/update_version_from_git.py b/update_version_from_git.py index a48ce5945..3a2964722 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -76,8 +76,7 @@ def prerelease_version(): assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2 or 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' - # return f'{initpy_ver}.dev{commits_since}+git.{commits_since}.{githash}' - return f'{initpy_ver}{commits_since}' + return f'{initpy_ver}.dev{commits_since}.{githash}' def read(*parts): """ Reads in file from *parts. From 4f86cad21e973fac83b8ff52ce87477dc9bd4957 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 23:29:23 -0500 Subject: [PATCH 20/31] Simplify version numbers for dev to make pypi happy. --- update_version_from_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index 3a2964722..c300a2870 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -76,7 +76,7 @@ def prerelease_version(): assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2 or 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' - return f'{initpy_ver}.dev{commits_since}.{githash}' + return f'{initpy_ver}.dev{commits_since}' def read(*parts): """ Reads in file from *parts. From 15d596ce7517d123a18b5f726184dc085ec32b3e Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 23:37:47 -0500 Subject: [PATCH 21/31] Dont fail on duplicate upload. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0bbc06261..8145cfb46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,7 @@ deploy: branch: - master skip_cleanup: true + skip_existing: true - provider: pypi distributions: sdist bdist_wheel user: spulec @@ -53,3 +54,4 @@ deploy: secure: NxnPylnTfekJmGyoufCw0lMoYRskSMJzvAIyAlJJVYKwEhmiCPOrdy5qV8i8mRZ1AkUsqU3jBZ/PD56n96clHW0E3d080UleRDj6JpyALVdeLfMqZl9kLmZ8bqakWzYq3VSJKw2zGP/L4tPGf8wTK1SUv9yl/YNDsBdCkjDverw= on: tags: true + skip_existing: true From 06483932748742b75fffc167861fd1fcdc5c37d0 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 30 Jun 2019 23:53:31 -0500 Subject: [PATCH 22/31] Fix updating version for py2. --- update_version_from_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index c300a2870..b2880f088 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -46,7 +46,7 @@ def migrate_source_attribute(attr, to_this, target_file, regex): def migrate_version(target_file, new_version): """Updates __version__ in the source file""" regex = r"['\"](.*)['\"]" - migrate_source_attribute('__version__', f"'{new_version}'", target_file, regex) + migrate_source_attribute('__version__', "'%s'" % new_version, target_file, regex) def is_master_branch(): From 940b4a954264e3d5f338fce59734a8aa89995b31 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 1 Jul 2019 00:13:16 -0500 Subject: [PATCH 23/31] Cleanup string formatting. --- update_version_from_git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index b2880f088..925673862 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -46,7 +46,7 @@ def migrate_source_attribute(attr, to_this, target_file, regex): def migrate_version(target_file, new_version): """Updates __version__ in the source file""" regex = r"['\"](.*)['\"]" - migrate_source_attribute('__version__', "'%s'" % new_version, target_file, regex) + migrate_source_attribute('__version__', "'{new_version}'".format(new_version=new_version), target_file, regex) def is_master_branch(): @@ -76,7 +76,7 @@ def prerelease_version(): assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2 or 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' - return f'{initpy_ver}.dev{commits_since}' + return '{initpy_ver}.dev{commits_since}'.format(initpy_ver=initpy_ver, commits_since=commits_since) def read(*parts): """ Reads in file from *parts. From 1d890099c32ba63f58e09413ba853af76ca29e9a Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 1 Jul 2019 08:58:24 -0500 Subject: [PATCH 24/31] More string formatting. --- update_version_from_git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index 925673862..355bc2ba9 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -107,7 +107,7 @@ def release_version_correct(): initpy = os.path.abspath("moto/__init__.py") new_version = prerelease_version() - print(f'updating version in __init__.py to {new_version}') + print('updating version in __init__.py to {new_version}'.format(new_version=new_version)) migrate_version(initpy, new_version) else: # check that we are a tag with the same version as in __init__.py From e9dfa890f4dc989152d7898fb709205598fdc3a2 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Mon, 1 Jul 2019 19:07:22 +0200 Subject: [PATCH 25/31] Fixed linting errors. --- moto/iam/policy_validation.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/moto/iam/policy_validation.py b/moto/iam/policy_validation.py index 91ee79dab..6ee286072 100644 --- a/moto/iam/policy_validation.py +++ b/moto/iam/policy_validation.py @@ -88,9 +88,9 @@ VALID_RESOURCE_PATH_STARTING_VALUES = { class IAMPolicyDocumentValidator: - def __init__(self, policy_document: str): - self._policy_document: str = policy_document - self._policy_json: dict = {} + def __init__(self, policy_document): + self._policy_document = policy_document + self._policy_json = {} self._statements = [] self._resource_error = "" # the first resource error found that does not generate a legacy parsing error @@ -308,7 +308,7 @@ class IAMPolicyDocumentValidator: for resource in sorted(statement[key], reverse=True): self._validate_resource_format(resource) if self._resource_error == "": - IAMPolicyDocumentValidator._legacy_parse_resource_like(statement, key) + IAMPolicyDocumentValidator._legacy_parse_resource_like(statement, key) def _validate_resource_format(self, resource): if resource != "*": @@ -327,12 +327,12 @@ class IAMPolicyDocumentValidator: arn3 = remaining_resource_parts[2] if len(remaining_resource_parts) > 2 else "*" arn4 = ":".join(remaining_resource_parts[3:]) if len(remaining_resource_parts) > 3 else "*" self._resource_error = 'Partition "{partition}" is not valid for resource "arn:{partition}:{arn1}:{arn2}:{arn3}:{arn4}".'.format( - partition=resource_partitions[0], - arn1=arn1, - arn2=arn2, - arn3=arn3, - arn4=arn4 - ) + partition=resource_partitions[0], + arn1=arn1, + arn2=arn2, + arn3=arn3, + arn4=arn4 + ) return if resource_partitions[1] != ":": From 35e7f1b4e90e774a1aed475c45ea47f28f35da81 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 1 Jul 2019 20:47:02 -0500 Subject: [PATCH 26/31] Remove pyyaml pin. CC #2271. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fcb9b6d17..593d248e9 100755 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ install_requires = [ "xmltodict", "six>1.9", "werkzeug", - "PyYAML==3.13", + "PyYAML", "pytz", "python-dateutil<3.0.0,>=2.1", "python-jose<4.0.0", From f6dd3ab959218948ae292267059ae3494c0c3c66 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Thu, 4 Jul 2019 03:13:25 -0700 Subject: [PATCH 27/31] not requiring the provisioned throughput key (#2278) --- moto/dynamodb2/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 943340438..5dde432d5 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -166,7 +166,7 @@ class DynamoHandler(BaseResponse): when BillingMode is PAY_PER_REQUEST') throughput = None else: # Provisioned (default billing mode) - throughput = body["ProvisionedThroughput"] + throughput = body.get("ProvisionedThroughput") # getting the schema key_schema = body['KeySchema'] # getting attribute definition From ef97e0f28f9f2bd33749a7e171b887b4745fbe7d Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 20:04:27 +0200 Subject: [PATCH 28/31] Extended create_access_key test. --- tests/test_iam/test_iam.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 0d96fd1b1..df084c9e2 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -577,13 +577,16 @@ def test_delete_login_profile(): conn.delete_login_profile('my-user') -@mock_iam_deprecated() +@mock_iam() def test_create_access_key(): - conn = boto.connect_iam() - with assert_raises(BotoServerError): - conn.create_access_key('my-user') - conn.create_user('my-user') - conn.create_access_key('my-user') + conn = boto3.client('iam', region_name='us-east-1') + with assert_raises(ClientError): + conn.create_access_key(UserName='my-user') + conn.create_user(UserName='my-user') + access_key = conn.create_access_key(UserName='my-user') + (datetime.utcnow() - access_key["CreateDate"]).seconds.should.be.within(0, 2) + access_key["AccessKeyId"].should.have.length_of(20) + assert access_key["AccessKeyId"].startswith("AKIA") @mock_iam_deprecated() From 0bd2c5adb401a84e0d7afc2cbb3eaa1d9d273793 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 20:12:53 +0200 Subject: [PATCH 29/31] Fixed minor errors in test. --- tests/test_iam/test_iam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index df084c9e2..d1326712e 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -583,8 +583,8 @@ def test_create_access_key(): with assert_raises(ClientError): conn.create_access_key(UserName='my-user') conn.create_user(UserName='my-user') - access_key = conn.create_access_key(UserName='my-user') - (datetime.utcnow() - access_key["CreateDate"]).seconds.should.be.within(0, 2) + access_key = conn.create_access_key(UserName='my-user')["AccessKey"] + (datetime.utcnow() - access_key["CreateDate"].replace(tzinfo=None)).seconds.should.be.within(0, 2) access_key["AccessKeyId"].should.have.length_of(20) assert access_key["AccessKeyId"].startswith("AKIA") From 9382c40c37fe2b7074feb85a39972aaf2aacbfc4 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 20:13:38 +0200 Subject: [PATCH 30/31] Return CreateDate in CreateAccessKey response. --- moto/iam/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 05624101a..3ee3377c5 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1493,6 +1493,7 @@ CREATE_ACCESS_KEY_TEMPLATE = """ {{ key.access_key_id }} {{ key.status }} {{ key.secret_access_key }} + {{ key.created_iso_8601 }} From 48f0c6f194175f515738768bd0c996082e5188ef Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 4 Jul 2019 20:20:08 +0200 Subject: [PATCH 31/31] Fixed format of access key ID and secret access key. --- moto/iam/models.py | 4 ++-- moto/iam/utils.py | 2 +- tests/test_iam/test_iam.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4bc7a5447..98abc538d 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -292,8 +292,8 @@ class AccessKey(BaseModel): def __init__(self, user_name): self.user_name = user_name - self.access_key_id = random_access_key() - self.secret_access_key = random_alphanumeric(32) + self.access_key_id = "AKIA" + random_access_key() + self.secret_access_key = random_alphanumeric(40) self.status = 'Active' self.create_date = datetime.utcnow() self.last_used = datetime.utcnow() diff --git a/moto/iam/utils.py b/moto/iam/utils.py index f59bdfffe..2bd6448f9 100644 --- a/moto/iam/utils.py +++ b/moto/iam/utils.py @@ -7,7 +7,7 @@ import six def random_alphanumeric(length): return ''.join(six.text_type( random.choice( - string.ascii_letters + string.digits + string.ascii_letters + string.digits + "+" + "/" )) for _ in range(length) ) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index d1326712e..679d04d9c 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -584,8 +584,9 @@ def test_create_access_key(): conn.create_access_key(UserName='my-user') conn.create_user(UserName='my-user') access_key = conn.create_access_key(UserName='my-user')["AccessKey"] - (datetime.utcnow() - access_key["CreateDate"].replace(tzinfo=None)).seconds.should.be.within(0, 2) + (datetime.utcnow() - access_key["CreateDate"].replace(tzinfo=None)).seconds.should.be.within(0, 10) access_key["AccessKeyId"].should.have.length_of(20) + access_key["SecretAccessKey"].should.have.length_of(40) assert access_key["AccessKeyId"].startswith("AKIA")