From 172396e6a8e19a8142964ca112a56beaf4e87554 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 20 Nov 2017 13:17:24 -0800 Subject: [PATCH 01/25] Updating CONTRIBUTING with release instructions --- CONTRIBUTING.md | 25 +++++++++++++++++++++++-- Makefile | 2 +- scripts/bump_version | 22 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100755 scripts/bump_version diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1266d508e..f28083221 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,25 @@ ### Contributing code -If you have improvements to Moto, send us your pull requests! For those -just getting started, Github has a [howto](https://help.github.com/articles/using-pull-requests/). +Moto has a [Code of Conduct](https://github.com/spulec/moto/blob/master/CODE_OF_CONDUCT.md), you can expect to be treated with respect at all times when interacting with this project. + +## Is there a missing feature? + +Moto is easier to contribute to than you probably think. There's [a list of which endpoints have been implemented](https://github.com/spulec/moto/blob/master/IMPLEMENTATION_COVERAGE.md) and we invite you to add new endpoints to existing services or to add new services. + +How to teach Moto to support a new AWS endpoint: + +* Create an issue describing what's missing. This is where we'll all talk about the new addition and help you get it done. +* Create a [pull request](https://help.github.com/articles/using-pull-requests/) and mention the issue # in the PR description. +* Try to add a failing test case. For example, if you're trying to implement `boto3.client('acm').import_certificate()` you'll want to add a new method called `def test_import_certificate` to `tests/test_acm/test_acm.py`. +* If you can also implement the code that gets that test passing that's great. If not, just ask the community for a hand and somebody will assist you. + +# Maintainers + +## Releasing a new version of Moto + +You'll need a PyPi account and a Dockerhub account to release Moto. After we release a new PyPi package we build and push the [motoserver/moto](https://hub.docker.com/r/motoserver/moto/) Docker image. + +* First, `scripts/bump_version` modifies the version and opens a PR +* Then, merge the new pull request +* Finally, generate and ship the new artifacts with `make publish` + diff --git a/Makefile b/Makefile index 99b7f2620..9324a61cd 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ tag_github_release: git tag `python setup.py --version` git push origin `python setup.py --version` -publish: implementation_coverage \ +publish: upload_pypi_artifact \ tag_github_release \ push_dockerhub_image diff --git a/scripts/bump_version b/scripts/bump_version new file mode 100755 index 000000000..53030700e --- /dev/null +++ b/scripts/bump_version @@ -0,0 +1,22 @@ +#!/bin/bash + +main() { + local version=$1 + if [[ -z "${version}" ]]; then + echo "USAGE: $0 1.3.2" + echo "Provide a new version number as an argument to bump the version" + echo -n "Current:" + grep version= setup.py + return 1 + fi + sed -i '' "s/version=.*$/version='${version}',/g" setup.py + git checkout -b version-${version} + # Commit the new version + git commit setup.py -m "bumping to version ${version}" + # Commit an updated IMPLEMENTATION_COVERAGE.md + make implementation_coverage || true + # Open a PR + open https://github.com/spulec/moto/compare/master...version-${version} +} + +main $@ From 7a4e48e8df2957fdadb7c6f91184e24a39407ed4 Mon Sep 17 00:00:00 2001 From: Semyon Maryasin Date: Sat, 16 Dec 2017 05:16:45 +0300 Subject: [PATCH 02/25] mock_xray_client: do return what f() returned fixes #1399 this won't help with fixtures though --- moto/xray/mock_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/xray/mock_client.py b/moto/xray/mock_client.py index 6e2164d63..135796054 100644 --- a/moto/xray/mock_client.py +++ b/moto/xray/mock_client.py @@ -51,7 +51,7 @@ def mock_xray_client(f): aws_xray_sdk.core.xray_recorder._emitter = MockEmitter() try: - f(*args, **kwargs) + return f(*args, **kwargs) finally: if old_xray_context_var is None: From 21606bc8aedf29d09892c080de924fcd83a9b7ba Mon Sep 17 00:00:00 2001 From: NimbusScale Date: Mon, 18 Dec 2017 20:44:04 -0800 Subject: [PATCH 03/25] update support JSON or YAML --- moto/cloudformation/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 70c15d697..b89d76605 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -107,7 +107,7 @@ class FakeStack(BaseModel): def update(self, template, role_arn=None, parameters=None, tags=None): self._add_stack_event("UPDATE_IN_PROGRESS", resource_status_reason="User Initiated") self.template = template - self.resource_map.update(json.loads(template), parameters) + self.resource_map.update(self.template_dict, parameters) self.output_map = self._create_output_map() self._add_stack_event("UPDATE_COMPLETE") self.status = "UPDATE_COMPLETE" From bb4bc01999ba47e3210b44b89153093bb421c6dd Mon Sep 17 00:00:00 2001 From: Joe Keegan Date: Thu, 21 Dec 2017 12:10:27 -0800 Subject: [PATCH 04/25] update self.template_dict based on new template --- moto/cloudformation/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index b89d76605..57f42df56 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -107,6 +107,7 @@ class FakeStack(BaseModel): def update(self, template, role_arn=None, parameters=None, tags=None): self._add_stack_event("UPDATE_IN_PROGRESS", resource_status_reason="User Initiated") self.template = template + self._parse_template() self.resource_map.update(self.template_dict, parameters) self.output_map = self._create_output_map() self._add_stack_event("UPDATE_COMPLETE") From 6f6a881e52632a48abd1f1ac6836cd1a86d09996 Mon Sep 17 00:00:00 2001 From: Joe Keegan Date: Thu, 21 Dec 2017 14:12:43 -0800 Subject: [PATCH 05/25] rerun tests From a4d1319821986536443efa8b0981a3e777ecc963 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 27 Dec 2017 11:06:26 -0800 Subject: [PATCH 06/25] Adding comment inviting a future person to help use bumpversion --- scripts/bump_version | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/bump_version b/scripts/bump_version index 53030700e..fe7ec1970 100755 --- a/scripts/bump_version +++ b/scripts/bump_version @@ -9,7 +9,12 @@ main() { grep version= setup.py return 1 fi + + # TODO: replace this with the bumpversion pip package, I couldn't + # figure out how to use that for these files sed -i '' "s/version=.*$/version='${version}',/g" setup.py + sed -i '' "s/__version__ = .*$/__version__ = '${version}',/g" moto/__init__.py + git checkout -b version-${version} # Commit the new version git commit setup.py -m "bumping to version ${version}" From 24f83e91f26f95c27877a7049ecceda422f0944a Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Wed, 27 Dec 2017 22:58:24 -0500 Subject: [PATCH 07/25] return 404 error on missing action --- moto/core/responses.py | 3 +++ moto/core/utils.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/moto/core/responses.py b/moto/core/responses.py index 52be602f6..ae91cdc02 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -272,6 +272,9 @@ class BaseResponse(_TemplateEnvironmentMixin): headers['status'] = str(headers['status']) return status, headers, body + if not action: + return 404, headers, '' + raise NotImplementedError( "The {0} action has not been implemented".format(action)) diff --git a/moto/core/utils.py b/moto/core/utils.py index 43f05672e..86e7632b0 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -18,6 +18,8 @@ def camelcase_to_underscores(argument): python underscore variable like the_new_attribute''' result = '' prev_char_title = True + if not argument: + return argument for index, char in enumerate(argument): try: next_char_title = argument[index + 1].istitle() From 6da22f9fa41b1f5d2c661f4d69369270b86b5ee7 Mon Sep 17 00:00:00 2001 From: Gordon Irving Date: Thu, 28 Dec 2017 19:04:37 +0000 Subject: [PATCH 08/25] fix adding tags to vpc created by cloudformation --- moto/ec2/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 932f535a1..1f372b57a 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2004,6 +2004,11 @@ class VPC(TaggedEC2Resource): cidr_block=properties['CidrBlock'], instance_tenancy=properties.get('InstanceTenancy', 'default') ) + for tag in properties.get("Tags", []): + tag_key = tag["Key"] + tag_value = tag["Value"] + vpc.add_tag(tag_key, tag_value) + return vpc @property From 5fed6988da49516d8214722a477c760876c2f5ee Mon Sep 17 00:00:00 2001 From: Gordon Irving Date: Thu, 28 Dec 2017 17:16:49 +0000 Subject: [PATCH 09/25] describe_regions: handle region-names parameter --- moto/ec2/models.py | 11 +++++++++-- moto/ec2/responses/availability_zones_and_regions.py | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 932f535a1..b9759099b 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -1261,8 +1261,15 @@ class RegionsAndZonesBackend(object): (region, [Zone(region + c, region) for c in 'abc']) for region in [r.name for r in regions]) - def describe_regions(self): - return self.regions + def describe_regions(self, region_names=[]): + if len(region_names) == 0: + return self.regions + ret = [] + for name in region_names: + for region in self.regions: + if region.name == name: + ret.append(region) + return ret def describe_availability_zones(self): return self.zones[self.region_name] diff --git a/moto/ec2/responses/availability_zones_and_regions.py b/moto/ec2/responses/availability_zones_and_regions.py index 3d0a5ab05..a6e35a89c 100644 --- a/moto/ec2/responses/availability_zones_and_regions.py +++ b/moto/ec2/responses/availability_zones_and_regions.py @@ -10,7 +10,8 @@ class AvailabilityZonesAndRegions(BaseResponse): return template.render(zones=zones) def describe_regions(self): - regions = self.ec2_backend.describe_regions() + region_names = self._get_multi_param('RegionName') + regions = self.ec2_backend.describe_regions(region_names) template = self.response_template(DESCRIBE_REGIONS_RESPONSE) return template.render(regions=regions) From e9b81bb3253cd7375617a733a5761752d645ee66 Mon Sep 17 00:00:00 2001 From: Gordon Irving Date: Thu, 28 Dec 2017 19:27:53 +0000 Subject: [PATCH 10/25] add test for vpc tags --- .../test_cloudformation_stack_integration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 051d8bed7..3a7525585 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -752,6 +752,9 @@ def test_vpc_single_instance_in_subnet(): security_group.vpc_id.should.equal(vpc.id) stack = conn.describe_stacks()[0] + + vpc.tags.should.have.key('Application').which.should.equal(stack.stack_id) + resources = stack.describe_resources() vpc_resource = [ resource for resource in resources if resource.resource_type == 'AWS::EC2::VPC'][0] From 4d9833b972440cb003a042906a1789fb20ec2fde Mon Sep 17 00:00:00 2001 From: Gordon Irving Date: Thu, 28 Dec 2017 21:02:58 +0000 Subject: [PATCH 11/25] add test for descrie_regions with args --- tests/test_ec2/test_availability_zones_and_regions.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_ec2/test_availability_zones_and_regions.py b/tests/test_ec2/test_availability_zones_and_regions.py index 7226cacaf..c64f075ca 100644 --- a/tests/test_ec2/test_availability_zones_and_regions.py +++ b/tests/test_ec2/test_availability_zones_and_regions.py @@ -36,6 +36,11 @@ def test_boto3_describe_regions(): for rec in resp['Regions']: rec['Endpoint'].should.contain(rec['RegionName']) + test_region = 'us-east-1' + resp = ec2.describe_regions(RegionNames=[test_region]) + resp['Regions'].should.have.length_of(1) + resp['Regions'][0].should.have.key('RegionName').which.should.equal(test_region) + @mock_ec2 def test_boto3_availability_zones(): From b855fee2e4109eab04a9d55699ffeffb1ca51df6 Mon Sep 17 00:00:00 2001 From: Mike Bjerkness Date: Sat, 30 Dec 2017 20:39:23 -0600 Subject: [PATCH 12/25] Add batch_get_image support for ECR (#1406) * Add batch_get_image for ECR * Add tests for batch_get_image * Add tests for batch_get_image * Undo local commits * Undo local commits * Adding object representation for batch_get_image * Update responses. Add a couple more tests. --- moto/ecr/models.py | 52 ++++++++++++-- moto/ecr/responses.py | 10 ++- tests/test_ecr/test_ecr_boto3.py | 116 ++++++++++++++++++++++++++++++- 3 files changed, 170 insertions(+), 8 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index f5b6f24e4..e20c550c9 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -1,14 +1,14 @@ from __future__ import unicode_literals -# from datetime import datetime + +import hashlib +from copy import copy from random import random from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends -from copy import copy -import hashlib - from moto.ecr.exceptions import ImageNotFoundException, RepositoryNotFoundException +from botocore.exceptions import ParamValidationError DEFAULT_REGISTRY_ID = '012345678910' @@ -145,6 +145,17 @@ class Image(BaseObject): response_object['imagePushedAt'] = '2017-05-09' return response_object + @property + def response_batch_get_image(self): + response_object = {} + response_object['imageId'] = {} + response_object['imageId']['imageTag'] = self.image_tag + response_object['imageId']['imageDigest'] = self.get_image_digest() + response_object['imageManifest'] = self.image_manifest + response_object['repositoryName'] = self.repository + response_object['registryId'] = self.registry_id + return response_object + class ECRBackend(BaseBackend): @@ -245,6 +256,39 @@ class ECRBackend(BaseBackend): repository.images.append(image) return image + def batch_get_image(self, repository_name, registry_id=None, image_ids=None, accepted_media_types=None): + if repository_name in self.repositories: + repository = self.repositories[repository_name] + else: + raise RepositoryNotFoundException(repository_name, registry_id or DEFAULT_REGISTRY_ID) + + if not image_ids: + raise ParamValidationError(msg='Missing required parameter in input: "imageIds"') + + response = { + 'images': [], + 'failures': [], + } + + for image_id in image_ids: + found = False + for image in repository.images: + if (('imageDigest' in image_id and image.get_image_digest() == image_id['imageDigest']) or + ('imageTag' in image_id and image.image_tag == image_id['imageTag'])): + found = True + response['images'].append(image.response_batch_get_image) + + if not found: + response['failures'].append({ + 'imageId': { + 'imageTag': image_id.get('imageTag', 'null') + }, + 'failureCode': 'ImageNotFound', + 'failureReason': 'Requested image not found' + }) + + return response + ecr_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/ecr/responses.py b/moto/ecr/responses.py index 6207de4eb..ca45c63c9 100644 --- a/moto/ecr/responses.py +++ b/moto/ecr/responses.py @@ -89,9 +89,13 @@ class ECRResponse(BaseResponse): 'ECR.batch_delete_image is not yet implemented') def batch_get_image(self): - if self.is_not_dryrun('BatchGetImage'): - raise NotImplementedError( - 'ECR.batch_get_image is not yet implemented') + repository_str = self._get_param('repositoryName') + registry_id = self._get_param('registryId') + image_ids = self._get_param('imageIds') + accepted_media_types = self._get_param('acceptedMediaTypes') + + response = self.ecr_backend.batch_get_image(repository_str, registry_id, image_ids, accepted_media_types) + return json.dumps(response) def can_paginate(self): if self.is_not_dryrun('CanPaginate'): diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 00628e22f..b4497ef60 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -9,7 +9,7 @@ import re import sure # noqa import boto3 -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, ParamValidationError from dateutil.tz import tzlocal from moto import mock_ecr @@ -445,3 +445,117 @@ def test_get_authorization_token_explicit_regions(): } ]) + + +@mock_ecr +def test_batch_get_image(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='latest' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='v1' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='v2' + ) + + response = client.batch_get_image( + repositoryName='test_repository', + imageIds=[ + { + 'imageTag': 'v2' + }, + ], + ) + + type(response['images']).should.be(list) + len(response['images']).should.be(1) + + response['images'][0]['imageManifest'].should.contain("vnd.docker.distribution.manifest.v2+json") + response['images'][0]['registryId'].should.equal("012345678910") + response['images'][0]['repositoryName'].should.equal("test_repository") + + response['images'][0]['imageId']['imageTag'].should.equal("v2") + response['images'][0]['imageId']['imageDigest'].should.contain("sha") + + type(response['failures']).should.be(list) + len(response['failures']).should.be(0) + + +@mock_ecr +def test_batch_get_image_that_doesnt_exist(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='latest' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='v1' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='v2' + ) + + response = client.batch_get_image( + repositoryName='test_repository', + imageIds=[ + { + 'imageTag': 'v5' + }, + ], + ) + + type(response['images']).should.be(list) + len(response['images']).should.be(0) + + type(response['failures']).should.be(list) + len(response['failures']).should.be(1) + response['failures'][0]['failureReason'].should.equal("Requested image not found") + response['failures'][0]['failureCode'].should.equal("ImageNotFound") + response['failures'][0]['imageId']['imageTag'].should.equal("v5") + + +@mock_ecr +def test_batch_get_image_no_tags(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()), + imageTag='latest' + ) + + error_msg = re.compile( + r".*Missing required parameter in input: \"imageIds\".*", + re.MULTILINE) + + client.batch_get_image.when.called_with( + repositoryName='test_repository').should.throw( + ParamValidationError, error_msg) From 770281aef2132de8b4d8abb7ec3325a3038b6f09 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Tue, 2 Jan 2018 23:47:57 -0500 Subject: [PATCH 13/25] Added put_bucket_logging support (#1401) - Also added put acl for XML - Put logging will also verify that the destination bucket exists in the same region with the proper ACLs attached. --- moto/s3/exceptions.py | 27 +++++ moto/s3/models.py | 39 ++++++ moto/s3/responses.py | 163 +++++++++++++++++++++++-- tests/test_s3/test_s3.py | 254 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 468 insertions(+), 15 deletions(-) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 24704e7ef..08dd02313 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -111,3 +111,30 @@ class MalformedXML(S3ClientError): "MalformedXML", "The XML you provided was not well-formed or did not validate against our published schema", *args, **kwargs) + + +class MalformedACLError(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(MalformedACLError, self).__init__( + "MalformedACLError", + "The XML you provided was not well-formed or did not validate against our published schema", + *args, **kwargs) + + +class InvalidTargetBucketForLogging(S3ClientError): + code = 400 + + def __init__(self, msg): + super(InvalidTargetBucketForLogging, self).__init__("InvalidTargetBucketForLogging", msg) + + +class CrossLocationLoggingProhibitted(S3ClientError): + code = 403 + + def __init__(self): + super(CrossLocationLoggingProhibitted, self).__init__( + "CrossLocationLoggingProhibitted", + "Cross S3 location logging not allowed." + ) diff --git a/moto/s3/models.py b/moto/s3/models.py index 91d3c1e2d..7eb89531f 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -347,6 +347,7 @@ class FakeBucket(BaseModel): self.acl = get_canned_acl('private') self.tags = FakeTagging() self.cors = [] + self.logging = {} @property def location(self): @@ -422,6 +423,40 @@ class FakeBucket(BaseModel): def tagging(self): return self.tags + def set_logging(self, logging_config, bucket_backend): + if not logging_config: + self.logging = {} + else: + from moto.s3.exceptions import InvalidTargetBucketForLogging, CrossLocationLoggingProhibitted + # Target bucket must exist in the same account (assuming all moto buckets are in the same account): + if not bucket_backend.buckets.get(logging_config["TargetBucket"]): + raise InvalidTargetBucketForLogging("The target bucket for logging does not exist.") + + # Does the target bucket have the log-delivery WRITE and READ_ACP permissions? + write = read_acp = False + for grant in bucket_backend.buckets[logging_config["TargetBucket"]].acl.grants: + # Must be granted to: http://acs.amazonaws.com/groups/s3/LogDelivery + for grantee in grant.grantees: + if grantee.uri == "http://acs.amazonaws.com/groups/s3/LogDelivery": + if "WRITE" in grant.permissions or "FULL_CONTROL" in grant.permissions: + write = True + + if "READ_ACP" in grant.permissions or "FULL_CONTROL" in grant.permissions: + read_acp = True + + break + + if not write or not read_acp: + raise InvalidTargetBucketForLogging("You must give the log-delivery group WRITE and READ_ACP" + " permissions to the target bucket") + + # Buckets must also exist within the same region: + if bucket_backend.buckets[logging_config["TargetBucket"]].region_name != self.region_name: + raise CrossLocationLoggingProhibitted() + + # Checks pass -- set the logging config: + self.logging = logging_config + def set_website_configuration(self, website_configuration): self.website_configuration = website_configuration @@ -608,6 +643,10 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) bucket.set_cors(cors_rules) + def put_bucket_logging(self, bucket_name, logging_config): + bucket = self.get_bucket(bucket_name) + bucket.set_logging(logging_config, self) + def delete_bucket_cors(self, bucket_name): bucket = self.get_bucket(bucket_name) bucket.delete_cors() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6abb4f2d1..8d2caf098 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -11,11 +11,13 @@ import xmltodict from moto.packages.httpretty.core import HTTPrettyRequest from moto.core.responses import _TemplateEnvironmentMixin -from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys +from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, \ + parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys - -from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, MissingKey, InvalidPartOrder -from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, FakeTag +from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, MissingKey, InvalidPartOrder, MalformedXML, \ + MalformedACLError +from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, \ + FakeTag from .utils import bucket_name_from_url, metadata_from_headers from xml.dom import minidom @@ -70,8 +72,9 @@ class ResponseObject(_TemplateEnvironmentMixin): match = re.match(r'^\[(.+)\](:\d+)?$', host) if match: - match = re.match(r'^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z', - match.groups()[0], re.IGNORECASE) + match = re.match( + r'^(((?=.*(::))(?!.*\3.+\3))\3?|[\dA-F]{1,4}:)([\dA-F]{1,4}(\3|:\b)|\2){5}(([\dA-F]{1,4}(\3|:\b|$)|\2){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})\Z', + match.groups()[0], re.IGNORECASE) if match: return False @@ -229,6 +232,13 @@ class ResponseObject(_TemplateEnvironmentMixin): return 404, {}, template.render(bucket_name=bucket_name) template = self.response_template(S3_BUCKET_TAGGING_RESPONSE) return template.render(bucket=bucket) + elif 'logging' in querystring: + bucket = self.backend.get_bucket(bucket_name) + if not bucket.logging: + template = self.response_template(S3_NO_LOGGING_CONFIG) + return 200, {}, template.render() + template = self.response_template(S3_LOGGING_CONFIG) + return 200, {}, template.render(logging=bucket.logging) elif "cors" in querystring: bucket = self.backend.get_bucket(bucket_name) if len(bucket.cors) == 0: @@ -324,8 +334,7 @@ class ResponseObject(_TemplateEnvironmentMixin): limit = continuation_token or start_after result_keys = self._get_results_from_token(result_keys, limit) - result_keys, is_truncated, \ - next_continuation_token = self._truncate_result(result_keys, max_keys) + result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys) return template.render( bucket=bucket, @@ -380,8 +389,11 @@ class ResponseObject(_TemplateEnvironmentMixin): self.backend.set_bucket_policy(bucket_name, body) return 'True' elif 'acl' in querystring: - # TODO: Support the XML-based ACL format - self.backend.set_bucket_acl(bucket_name, self._acl_from_headers(request.headers)) + # Headers are first. If not set, then look at the body (consistent with the documentation): + acls = self._acl_from_headers(request.headers) + if not acls: + acls = self._acl_from_xml(body) + self.backend.set_bucket_acl(bucket_name, acls) return "" elif "tagging" in querystring: tagging = self._bucket_tagging_from_xml(body) @@ -391,12 +403,18 @@ class ResponseObject(_TemplateEnvironmentMixin): self.backend.set_bucket_website_configuration(bucket_name, body) return "" elif "cors" in querystring: - from moto.s3.exceptions import MalformedXML try: self.backend.put_bucket_cors(bucket_name, self._cors_from_xml(body)) return "" except KeyError: raise MalformedXML() + elif "logging" in querystring: + try: + self.backend.put_bucket_logging(bucket_name, self._logging_from_xml(body)) + return "" + except KeyError: + raise MalformedXML() + else: if body: try: @@ -515,6 +533,7 @@ class ResponseObject(_TemplateEnvironmentMixin): def toint(i): return int(i) if i else None + begin, end = map(toint, rspec.split('-')) if begin is not None: # byte range end = last if end is None else min(end, last) @@ -731,6 +750,58 @@ class ResponseObject(_TemplateEnvironmentMixin): else: return 404, response_headers, "" + def _acl_from_xml(self, xml): + parsed_xml = xmltodict.parse(xml) + if not parsed_xml.get("AccessControlPolicy"): + raise MalformedACLError() + + # The owner is needed for some reason... + if not parsed_xml["AccessControlPolicy"].get("Owner"): + # TODO: Validate that the Owner is actually correct. + raise MalformedACLError() + + # If empty, then no ACLs: + if parsed_xml["AccessControlPolicy"].get("AccessControlList") is None: + return [] + + if not parsed_xml["AccessControlPolicy"]["AccessControlList"].get("Grant"): + raise MalformedACLError() + + permissions = [ + "READ", + "WRITE", + "READ_ACP", + "WRITE_ACP", + "FULL_CONTROL" + ] + + if not isinstance(parsed_xml["AccessControlPolicy"]["AccessControlList"]["Grant"], list): + parsed_xml["AccessControlPolicy"]["AccessControlList"]["Grant"] = \ + [parsed_xml["AccessControlPolicy"]["AccessControlList"]["Grant"]] + + grants = self._get_grants_from_xml(parsed_xml["AccessControlPolicy"]["AccessControlList"]["Grant"], + MalformedACLError, permissions) + return FakeAcl(grants) + + def _get_grants_from_xml(self, grant_list, exception_type, permissions): + grants = [] + for grant in grant_list: + if grant.get("Permission", "") not in permissions: + raise exception_type() + + if grant["Grantee"].get("@xsi:type", "") not in ["CanonicalUser", "AmazonCustomerByEmail", "Group"]: + raise exception_type() + + # TODO: Verify that the proper grantee data is supplied based on the type. + + grants.append(FakeGrant( + [FakeGrantee(id=grant["Grantee"].get("ID", ""), display_name=grant["Grantee"].get("DisplayName", ""), + uri=grant["Grantee"].get("URI", ""))], + [grant["Permission"]]) + ) + + return grants + def _acl_from_headers(self, headers): canned_acl = headers.get('x-amz-acl', '') if canned_acl: @@ -814,6 +885,42 @@ class ResponseObject(_TemplateEnvironmentMixin): return [parsed_xml["CORSConfiguration"]["CORSRule"]] + def _logging_from_xml(self, xml): + parsed_xml = xmltodict.parse(xml) + + if not parsed_xml["BucketLoggingStatus"].get("LoggingEnabled"): + return {} + + if not parsed_xml["BucketLoggingStatus"]["LoggingEnabled"].get("TargetBucket"): + raise MalformedXML() + + if not parsed_xml["BucketLoggingStatus"]["LoggingEnabled"].get("TargetPrefix"): + parsed_xml["BucketLoggingStatus"]["LoggingEnabled"]["TargetPrefix"] = "" + + # Get the ACLs: + if parsed_xml["BucketLoggingStatus"]["LoggingEnabled"].get("TargetGrants"): + permissions = [ + "READ", + "WRITE", + "FULL_CONTROL" + ] + if not isinstance(parsed_xml["BucketLoggingStatus"]["LoggingEnabled"]["TargetGrants"]["Grant"], list): + target_grants = self._get_grants_from_xml( + [parsed_xml["BucketLoggingStatus"]["LoggingEnabled"]["TargetGrants"]["Grant"]], + MalformedXML, + permissions + ) + else: + target_grants = self._get_grants_from_xml( + parsed_xml["BucketLoggingStatus"]["LoggingEnabled"]["TargetGrants"]["Grant"], + MalformedXML, + permissions + ) + + parsed_xml["BucketLoggingStatus"]["LoggingEnabled"]["TargetGrants"] = target_grants + + return parsed_xml["BucketLoggingStatus"]["LoggingEnabled"] + def _key_response_delete(self, bucket_name, query, key_name, headers): if query.get('uploadId'): upload_id = query['uploadId'][0] @@ -1322,3 +1429,37 @@ S3_NO_CORS_CONFIG = """ 9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg= """ + +S3_LOGGING_CONFIG = """ + + + {{ logging["TargetBucket"] }} + {{ logging["TargetPrefix"] }} + {% if logging.get("TargetGrants") %} + + {% for grant in logging["TargetGrants"] %} + + + {% if grant.grantees[0].uri %} + {{ grant.grantees[0].uri }} + {% endif %} + {% if grant.grantees[0].id %} + {{ grant.grantees[0].id }} + {% endif %} + {% if grant.grantees[0].display_name %} + {{ grant.grantees[0].display_name }} + {% endif %} + + {{ grant.permissions[0] }} + + {% endfor %} + + {% endif %} + + +""" + +S3_NO_LOGGING_CONFIG = """ + +""" diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 829941d79..33752af60 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -50,6 +50,7 @@ def reduced_min_part_size(f): return f(*args, **kwargs) finally: s3model.UPLOAD_PART_MIN_SIZE = orig_size + return wrapped @@ -883,11 +884,12 @@ def test_s3_object_in_public_bucket(): s3_anonymous.Object(key='file.txt', bucket_name='test-bucket').get() exc.exception.response['Error']['Code'].should.equal('403') - params = {'Bucket': 'test-bucket','Key': 'file.txt'} + params = {'Bucket': 'test-bucket', 'Key': 'file.txt'} presigned_url = boto3.client('s3').generate_presigned_url('get_object', params, ExpiresIn=900) response = requests.get(presigned_url) assert response.status_code == 200 + @mock_s3 def test_s3_object_in_private_bucket(): s3 = boto3.resource('s3') @@ -1102,6 +1104,7 @@ def test_boto3_key_etag(): resp = s3.get_object(Bucket='mybucket', Key='steve') resp['ETag'].should.equal('"d32bda93738f7e03adb22e66c90fbc04"') + @mock_s3 def test_website_redirect_location(): s3 = boto3.client('s3', region_name='us-east-1') @@ -1116,6 +1119,7 @@ def test_website_redirect_location(): resp = s3.get_object(Bucket='mybucket', Key='steve') resp['WebsiteRedirectLocation'].should.equal(url) + @mock_s3 def test_boto3_list_keys_xml_escaped(): s3 = boto3.client('s3', region_name='us-east-1') @@ -1627,7 +1631,7 @@ def test_boto3_put_bucket_cors(): }) e = err.exception e.response["Error"]["Code"].should.equal("InvalidRequest") - e.response["Error"]["Message"].should.equal("Found unsupported HTTP method in CORS config. " + e.response["Error"]["Message"].should.equal("Found unsupported HTTP method in CORS config. " "Unsupported method is NOTREAL") with assert_raises(ClientError) as err: @@ -1732,6 +1736,249 @@ def test_boto3_delete_bucket_cors(): e.response["Error"]["Message"].should.equal("The CORS configuration does not exist") +@mock_s3 +def test_put_bucket_acl_body(): + s3 = boto3.client("s3", region_name="us-east-1") + s3.create_bucket(Bucket="bucket") + bucket_owner = s3.get_bucket_acl(Bucket="bucket")["Owner"] + s3.put_bucket_acl(Bucket="bucket", AccessControlPolicy={ + "Grants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "WRITE" + }, + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "READ_ACP" + } + ], + "Owner": bucket_owner + }) + + result = s3.get_bucket_acl(Bucket="bucket") + assert len(result["Grants"]) == 2 + for g in result["Grants"]: + assert g["Grantee"]["URI"] == "http://acs.amazonaws.com/groups/s3/LogDelivery" + assert g["Grantee"]["Type"] == "Group" + assert g["Permission"] in ["WRITE", "READ_ACP"] + + # With one: + s3.put_bucket_acl(Bucket="bucket", AccessControlPolicy={ + "Grants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "WRITE" + } + ], + "Owner": bucket_owner + }) + result = s3.get_bucket_acl(Bucket="bucket") + assert len(result["Grants"]) == 1 + + # With no owner: + with assert_raises(ClientError) as err: + s3.put_bucket_acl(Bucket="bucket", AccessControlPolicy={ + "Grants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "WRITE" + } + ] + }) + assert err.exception.response["Error"]["Code"] == "MalformedACLError" + + # With incorrect permission: + with assert_raises(ClientError) as err: + s3.put_bucket_acl(Bucket="bucket", AccessControlPolicy={ + "Grants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "lskjflkasdjflkdsjfalisdjflkdsjf" + } + ], + "Owner": bucket_owner + }) + assert err.exception.response["Error"]["Code"] == "MalformedACLError" + + # Clear the ACLs: + result = s3.put_bucket_acl(Bucket="bucket", AccessControlPolicy={"Grants": [], "Owner": bucket_owner}) + assert not result.get("Grants") + + +@mock_s3 +def test_boto3_put_bucket_logging(): + s3 = boto3.client("s3", region_name="us-east-1") + bucket_name = "mybucket" + log_bucket = "logbucket" + wrong_region_bucket = "wrongregionlogbucket" + s3.create_bucket(Bucket=bucket_name) + s3.create_bucket(Bucket=log_bucket) # Adding the ACL for log-delivery later... + s3.create_bucket(Bucket=wrong_region_bucket, CreateBucketConfiguration={"LocationConstraint": "us-west-2"}) + + # No logging config: + result = s3.get_bucket_logging(Bucket=bucket_name) + assert not result.get("LoggingEnabled") + + # A log-bucket that doesn't exist: + with assert_raises(ClientError) as err: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": "IAMNOTREAL", + "TargetPrefix": "" + } + }) + assert err.exception.response["Error"]["Code"] == "InvalidTargetBucketForLogging" + + # A log-bucket that's missing the proper ACLs for LogDelivery: + with assert_raises(ClientError) as err: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": log_bucket, + "TargetPrefix": "" + } + }) + assert err.exception.response["Error"]["Code"] == "InvalidTargetBucketForLogging" + assert "log-delivery" in err.exception.response["Error"]["Message"] + + # Add the proper "log-delivery" ACL to the log buckets: + bucket_owner = s3.get_bucket_acl(Bucket=log_bucket)["Owner"] + for bucket in [log_bucket, wrong_region_bucket]: + s3.put_bucket_acl(Bucket=bucket, AccessControlPolicy={ + "Grants": [ + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "WRITE" + }, + { + "Grantee": { + "URI": "http://acs.amazonaws.com/groups/s3/LogDelivery", + "Type": "Group" + }, + "Permission": "READ_ACP" + }, + { + "Grantee": { + "Type": "CanonicalUser", + "ID": bucket_owner["ID"] + }, + "Permission": "FULL_CONTROL" + } + ], + "Owner": bucket_owner + }) + + # A log-bucket that's in the wrong region: + with assert_raises(ClientError) as err: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": wrong_region_bucket, + "TargetPrefix": "" + } + }) + assert err.exception.response["Error"]["Code"] == "CrossLocationLoggingProhibitted" + + # Correct logging: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": log_bucket, + "TargetPrefix": "{}/".format(bucket_name) + } + }) + result = s3.get_bucket_logging(Bucket=bucket_name) + assert result["LoggingEnabled"]["TargetBucket"] == log_bucket + assert result["LoggingEnabled"]["TargetPrefix"] == "{}/".format(bucket_name) + assert not result["LoggingEnabled"].get("TargetGrants") + + # And disabling: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={}) + assert not s3.get_bucket_logging(Bucket=bucket_name).get("LoggingEnabled") + + # And enabling with multiple target grants: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": log_bucket, + "TargetPrefix": "{}/".format(bucket_name), + "TargetGrants": [ + { + "Grantee": { + "ID": "SOMEIDSTRINGHERE9238748923734823917498237489237409123840983274", + "Type": "CanonicalUser" + }, + "Permission": "READ" + }, + { + "Grantee": { + "ID": "SOMEIDSTRINGHERE9238748923734823917498237489237409123840983274", + "Type": "CanonicalUser" + }, + "Permission": "WRITE" + } + ] + } + }) + + result = s3.get_bucket_logging(Bucket=bucket_name) + assert len(result["LoggingEnabled"]["TargetGrants"]) == 2 + assert result["LoggingEnabled"]["TargetGrants"][0]["Grantee"]["ID"] == \ + "SOMEIDSTRINGHERE9238748923734823917498237489237409123840983274" + + # Test with just 1 grant: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": log_bucket, + "TargetPrefix": "{}/".format(bucket_name), + "TargetGrants": [ + { + "Grantee": { + "ID": "SOMEIDSTRINGHERE9238748923734823917498237489237409123840983274", + "Type": "CanonicalUser" + }, + "Permission": "READ" + } + ] + } + }) + result = s3.get_bucket_logging(Bucket=bucket_name) + assert len(result["LoggingEnabled"]["TargetGrants"]) == 1 + + # With an invalid grant: + with assert_raises(ClientError) as err: + s3.put_bucket_logging(Bucket=bucket_name, BucketLoggingStatus={ + "LoggingEnabled": { + "TargetBucket": log_bucket, + "TargetPrefix": "{}/".format(bucket_name), + "TargetGrants": [ + { + "Grantee": { + "ID": "SOMEIDSTRINGHERE9238748923734823917498237489237409123840983274", + "Type": "CanonicalUser" + }, + "Permission": "NOTAREALPERM" + } + ] + } + }) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + @mock_s3 def test_boto3_put_object_tagging(): s3 = boto3.client('s3', region_name='us-east-1') @@ -1939,11 +2186,10 @@ def test_get_stream_gzipped(): Bucket='moto-tests', Key='keyname', ) - res = zlib.decompress(obj['Body'].read(), 16+zlib.MAX_WBITS) + res = zlib.decompress(obj['Body'].read(), 16 + zlib.MAX_WBITS) assert res == payload - TEST_XML = """\ From 71af9317f236a5fb884fa6aec1a31e8abced26b8 Mon Sep 17 00:00:00 2001 From: Toshiya Kawasaki Date: Thu, 4 Jan 2018 18:59:37 +0900 Subject: [PATCH 14/25] Add group features to iot (#1402) * Add thing group features * thing thing-group relation * clean up comments --- moto/iot/exceptions.py | 12 ++- moto/iot/models.py | 150 ++++++++++++++++++++++++++++- moto/iot/responses.py | 136 ++++++++++++++++++++++++-- tests/test_iot/test_iot.py | 189 +++++++++++++++++++++++++++++++++++++ 4 files changed, 475 insertions(+), 12 deletions(-) diff --git a/moto/iot/exceptions.py b/moto/iot/exceptions.py index 4bb01c095..47435eeb5 100644 --- a/moto/iot/exceptions.py +++ b/moto/iot/exceptions.py @@ -16,9 +16,17 @@ class ResourceNotFoundException(IoTClientError): class InvalidRequestException(IoTClientError): - def __init__(self): + def __init__(self, msg=None): self.code = 400 super(InvalidRequestException, self).__init__( "InvalidRequestException", - "The request is not valid." + msg or "The request is not valid." + ) + + +class VersionConflictException(IoTClientError): + def __init__(self, name): + self.code = 409 + super(VersionConflictException, self).__init__( + 'The version for thing %s does not match the expected version.' % name ) diff --git a/moto/iot/models.py b/moto/iot/models.py index 1efa6690e..77b0dde08 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -9,7 +9,8 @@ from moto.core import BaseBackend, BaseModel from collections import OrderedDict from .exceptions import ( ResourceNotFoundException, - InvalidRequestException + InvalidRequestException, + VersionConflictException ) @@ -44,6 +45,7 @@ class FakeThingType(BaseModel): self.region_name = region_name self.thing_type_name = thing_type_name self.thing_type_properties = thing_type_properties + self.thing_type_id = str(uuid.uuid4()) # I don't know the rule of id t = time.time() self.metadata = { 'deprecated': False, @@ -54,11 +56,37 @@ class FakeThingType(BaseModel): def to_dict(self): return { 'thingTypeName': self.thing_type_name, + 'thingTypeId': self.thing_type_id, 'thingTypeProperties': self.thing_type_properties, 'thingTypeMetadata': self.metadata } +class FakeThingGroup(BaseModel): + def __init__(self, thing_group_name, parent_group_name, thing_group_properties, region_name): + self.region_name = region_name + self.thing_group_name = thing_group_name + self.thing_group_id = str(uuid.uuid4()) # I don't know the rule of id + self.version = 1 # TODO: tmp + self.parent_group_name = parent_group_name + self.thing_group_properties = thing_group_properties or {} + t = time.time() + self.metadata = { + 'creationData': int(t * 1000) / 1000.0 + } + self.arn = 'arn:aws:iot:%s:1:thinggroup/%s' % (self.region_name, thing_group_name) + self.things = OrderedDict() + + def to_dict(self): + return { + 'thingGroupName': self.thing_group_name, + 'thingGroupId': self.thing_group_id, + 'version': self.version, + 'thingGroupProperties': self.thing_group_properties, + 'thingGroupMetadata': self.metadata + } + + class FakeCertificate(BaseModel): def __init__(self, certificate_pem, status, region_name): m = hashlib.sha256() @@ -137,6 +165,7 @@ class IoTBackend(BaseBackend): self.region_name = region_name self.things = OrderedDict() self.thing_types = OrderedDict() + self.thing_groups = OrderedDict() self.certificates = OrderedDict() self.policies = OrderedDict() self.principal_policies = OrderedDict() @@ -359,6 +388,125 @@ class IoTBackend(BaseBackend): principals = [k[0] for k, v in self.principal_things.items() if k[1] == thing_name] return principals + def describe_thing_group(self, thing_group_name): + thing_groups = [_ for _ in self.thing_groups.values() if _.thing_group_name == thing_group_name] + if len(thing_groups) == 0: + raise ResourceNotFoundException() + return thing_groups[0] + + def create_thing_group(self, thing_group_name, parent_group_name, thing_group_properties): + thing_group = FakeThingGroup(thing_group_name, parent_group_name, thing_group_properties, self.region_name) + self.thing_groups[thing_group.arn] = thing_group + return thing_group.thing_group_name, thing_group.arn, thing_group.thing_group_id + + def delete_thing_group(self, thing_group_name, expected_version): + thing_group = self.describe_thing_group(thing_group_name) + del self.thing_groups[thing_group.arn] + + def list_thing_groups(self, parent_group, name_prefix_filter, recursive): + thing_groups = self.thing_groups.values() + return thing_groups + + def update_thing_group(self, thing_group_name, thing_group_properties, expected_version): + thing_group = self.describe_thing_group(thing_group_name) + if expected_version and expected_version != thing_group.version: + raise VersionConflictException(thing_group_name) + attribute_payload = thing_group_properties.get('attributePayload', None) + if attribute_payload is not None and 'attributes' in attribute_payload: + do_merge = attribute_payload.get('merge', False) + attributes = attribute_payload['attributes'] + if not do_merge: + thing_group.thing_group_properties['attributePayload']['attributes'] = attributes + else: + thing_group.thing_group_properties['attributePayload']['attributes'].update(attributes) + elif attribute_payload is not None and 'attributes' not in attribute_payload: + thing_group.attributes = {} + thing_group.version = thing_group.version + 1 + return thing_group.version + + def _identify_thing_group(self, thing_group_name, thing_group_arn): + # identify thing group + if thing_group_name is None and thing_group_arn is None: + raise InvalidRequestException( + ' Both thingGroupArn and thingGroupName are empty. Need to specify at least one of them' + ) + if thing_group_name is not None: + thing_group = self.describe_thing_group(thing_group_name) + if thing_group_arn and thing_group.arn != thing_group_arn: + raise InvalidRequestException( + 'ThingGroupName thingGroupArn does not match specified thingGroupName in request' + ) + elif thing_group_arn is not None: + if thing_group_arn not in self.thing_groups: + raise InvalidRequestException() + thing_group = self.thing_groups[thing_group_arn] + return thing_group + + def _identify_thing(self, thing_name, thing_arn): + # identify thing + if thing_name is None and thing_arn is None: + raise InvalidRequestException( + 'Both thingArn and thingName are empty. Need to specify at least one of them' + ) + if thing_name is not None: + thing = self.describe_thing(thing_name) + if thing_arn and thing.arn != thing_arn: + raise InvalidRequestException( + 'ThingName thingArn does not match specified thingName in request' + ) + elif thing_arn is not None: + if thing_arn not in self.things: + raise InvalidRequestException() + thing = self.things[thing_arn] + return thing + + def add_thing_to_thing_group(self, thing_group_name, thing_group_arn, thing_name, thing_arn): + thing_group = self._identify_thing_group(thing_group_name, thing_group_arn) + thing = self._identify_thing(thing_name, thing_arn) + if thing.arn in thing_group.things: + # aws ignores duplicate registration + return + thing_group.things[thing.arn] = thing + + def remove_thing_from_thing_group(self, thing_group_name, thing_group_arn, thing_name, thing_arn): + thing_group = self._identify_thing_group(thing_group_name, thing_group_arn) + thing = self._identify_thing(thing_name, thing_arn) + if thing.arn not in thing_group.things: + # aws ignores non-registered thing + return + del thing_group.things[thing.arn] + + def list_things_in_thing_group(self, thing_group_name, recursive): + thing_group = self.describe_thing_group(thing_group_name) + return thing_group.things.values() + + def list_thing_groups_for_thing(self, thing_name): + thing = self.describe_thing(thing_name) + all_thing_groups = self.list_thing_groups(None, None, None) + ret = [] + for thing_group in all_thing_groups: + if thing.arn in thing_group.things: + ret.append({ + 'groupName': thing_group.thing_group_name, + 'groupArn': thing_group.arn + }) + return ret + + def update_thing_groups_for_thing(self, thing_name, thing_groups_to_add, thing_groups_to_remove): + thing = self.describe_thing(thing_name) + for thing_group_name in thing_groups_to_add: + thing_group = self.describe_thing_group(thing_group_name) + self.add_thing_to_thing_group( + thing_group.thing_group_name, None, + thing.thing_name, None + ) + for thing_group_name in thing_groups_to_remove: + thing_group = self.describe_thing_group(thing_group_name) + self.remove_thing_from_thing_group( + thing_group.thing_group_name, None, + thing.thing_name, None + ) + available_regions = boto3.session.Session().get_available_regions("iot") iot_backends = {region: IoTBackend(region) for region in available_regions} diff --git a/moto/iot/responses.py b/moto/iot/responses.py index bbe2bb016..f59c105da 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -38,8 +38,7 @@ class IoTResponse(BaseResponse): thing_types = self.iot_backend.list_thing_types( thing_type_name=thing_type_name ) - - # TODO: support next_token and max_results + # TODO: implement pagination in the future next_token = None return json.dumps(dict(thingTypes=[_.to_dict() for _ in thing_types], nextToken=next_token)) @@ -54,7 +53,7 @@ class IoTResponse(BaseResponse): attribute_value=attribute_value, thing_type_name=thing_type_name, ) - # TODO: support next_token and max_results + # TODO: implement pagination in the future next_token = None return json.dumps(dict(things=[_.to_dict() for _ in things], nextToken=next_token)) @@ -63,7 +62,6 @@ class IoTResponse(BaseResponse): thing = self.iot_backend.describe_thing( thing_name=thing_name, ) - print(thing.to_dict(include_default_client_id=True)) return json.dumps(thing.to_dict(include_default_client_id=True)) def describe_thing_type(self): @@ -135,7 +133,7 @@ class IoTResponse(BaseResponse): # marker = self._get_param("marker") # ascending_order = self._get_param("ascendingOrder") certificates = self.iot_backend.list_certificates() - # TODO: handle pagination + # TODO: implement pagination in the future return json.dumps(dict(certificates=[_.to_dict() for _ in certificates])) def update_certificate(self): @@ -162,7 +160,7 @@ class IoTResponse(BaseResponse): # ascending_order = self._get_param("ascendingOrder") policies = self.iot_backend.list_policies() - # TODO: handle pagination + # TODO: implement pagination in the future return json.dumps(dict(policies=[_.to_dict() for _ in policies])) def get_policy(self): @@ -205,7 +203,7 @@ class IoTResponse(BaseResponse): policies = self.iot_backend.list_principal_policies( principal_arn=principal ) - # TODO: handle pagination + # TODO: implement pagination in the future next_marker = None return json.dumps(dict(policies=[_.to_dict() for _ in policies], nextMarker=next_marker)) @@ -217,7 +215,7 @@ class IoTResponse(BaseResponse): principals = self.iot_backend.list_policy_principals( policy_name=policy_name, ) - # TODO: handle pagination + # TODO: implement pagination in the future next_marker = None return json.dumps(dict(principals=principals, nextMarker=next_marker)) @@ -246,7 +244,7 @@ class IoTResponse(BaseResponse): things = self.iot_backend.list_principal_things( principal_arn=principal, ) - # TODO: handle pagination + # TODO: implement pagination in the future next_token = None return json.dumps(dict(things=things, nextToken=next_token)) @@ -256,3 +254,123 @@ class IoTResponse(BaseResponse): thing_name=thing_name, ) return json.dumps(dict(principals=principals)) + + def describe_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + thing_group = self.iot_backend.describe_thing_group( + thing_group_name=thing_group_name, + ) + return json.dumps(thing_group.to_dict()) + + def create_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + parent_group_name = self._get_param("parentGroupName") + thing_group_properties = self._get_param("thingGroupProperties") + thing_group_name, thing_group_arn, thing_group_id = self.iot_backend.create_thing_group( + thing_group_name=thing_group_name, + parent_group_name=parent_group_name, + thing_group_properties=thing_group_properties, + ) + return json.dumps(dict( + thingGroupName=thing_group_name, + thingGroupArn=thing_group_arn, + thingGroupId=thing_group_id) + ) + + def delete_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + expected_version = self._get_param("expectedVersion") + self.iot_backend.delete_thing_group( + thing_group_name=thing_group_name, + expected_version=expected_version, + ) + return json.dumps(dict()) + + def list_thing_groups(self): + # next_token = self._get_param("nextToken") + # max_results = self._get_int_param("maxResults") + parent_group = self._get_param("parentGroup") + name_prefix_filter = self._get_param("namePrefixFilter") + recursive = self._get_param("recursive") + thing_groups = self.iot_backend.list_thing_groups( + parent_group=parent_group, + name_prefix_filter=name_prefix_filter, + recursive=recursive, + ) + next_token = None + rets = [{'groupName': _.thing_group_name, 'groupArn': _.arn} for _ in thing_groups] + # TODO: implement pagination in the future + return json.dumps(dict(thingGroups=rets, nextToken=next_token)) + + def update_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + thing_group_properties = self._get_param("thingGroupProperties") + expected_version = self._get_param("expectedVersion") + version = self.iot_backend.update_thing_group( + thing_group_name=thing_group_name, + thing_group_properties=thing_group_properties, + expected_version=expected_version, + ) + return json.dumps(dict(version=version)) + + def add_thing_to_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + thing_group_arn = self._get_param("thingGroupArn") + thing_name = self._get_param("thingName") + thing_arn = self._get_param("thingArn") + self.iot_backend.add_thing_to_thing_group( + thing_group_name=thing_group_name, + thing_group_arn=thing_group_arn, + thing_name=thing_name, + thing_arn=thing_arn, + ) + return json.dumps(dict()) + + def remove_thing_from_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + thing_group_arn = self._get_param("thingGroupArn") + thing_name = self._get_param("thingName") + thing_arn = self._get_param("thingArn") + self.iot_backend.remove_thing_from_thing_group( + thing_group_name=thing_group_name, + thing_group_arn=thing_group_arn, + thing_name=thing_name, + thing_arn=thing_arn, + ) + return json.dumps(dict()) + + def list_things_in_thing_group(self): + thing_group_name = self._get_param("thingGroupName") + recursive = self._get_param("recursive") + # next_token = self._get_param("nextToken") + # max_results = self._get_int_param("maxResults") + things = self.iot_backend.list_things_in_thing_group( + thing_group_name=thing_group_name, + recursive=recursive, + ) + next_token = None + thing_names = [_.thing_name for _ in things] + # TODO: implement pagination in the future + return json.dumps(dict(things=thing_names, nextToken=next_token)) + + def list_thing_groups_for_thing(self): + thing_name = self._get_param("thingName") + # next_token = self._get_param("nextToken") + # max_results = self._get_int_param("maxResults") + thing_groups = self.iot_backend.list_thing_groups_for_thing( + thing_name=thing_name + ) + next_token = None + # TODO: implement pagination in the future + return json.dumps(dict(thingGroups=thing_groups, nextToken=next_token)) + + def update_thing_groups_for_thing(self): + thing_name = self._get_param("thingName") + thing_groups_to_add = self._get_param("thingGroupsToAdd") or [] + thing_groups_to_remove = self._get_param("thingGroupsToRemove") or [] + self.iot_backend.update_thing_groups_for_thing( + thing_name=thing_name, + thing_groups_to_add=thing_groups_to_add, + thing_groups_to_remove=thing_groups_to_remove, + ) + return json.dumps(dict()) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 31631e459..7c01934d3 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -177,3 +177,192 @@ def test_principal_thing(): res.should.have.key('things').which.should.have.length_of(0) res = client.list_thing_principals(thingName=thing_name) res.should.have.key('principals').which.should.have.length_of(0) + + +@mock_iot +def test_thing_groups(): + client = boto3.client('iot', region_name='ap-northeast-1') + name = 'my-thing' + group_name = 'my-group-name' + + # thing group + thing_group = client.create_thing_group(thingGroupName=group_name) + thing_group.should.have.key('thingGroupName').which.should.equal(group_name) + thing_group.should.have.key('thingGroupArn') + + res = client.list_thing_groups() + res.should.have.key('thingGroups').which.should.have.length_of(1) + for thing_group in res['thingGroups']: + thing_group.should.have.key('groupName').which.should_not.be.none + thing_group.should.have.key('groupArn').which.should_not.be.none + + thing_group = client.describe_thing_group(thingGroupName=group_name) + thing_group.should.have.key('thingGroupName').which.should.equal(group_name) + thing_group.should.have.key('thingGroupProperties') + thing_group.should.have.key('thingGroupMetadata') + thing_group.should.have.key('version') + + # delete thing group + client.delete_thing_group(thingGroupName=group_name) + res = client.list_thing_groups() + res.should.have.key('thingGroups').which.should.have.length_of(0) + + # props create test + props = { + 'thingGroupDescription': 'my first thing group', + 'attributePayload': { + 'attributes': { + 'key1': 'val01', + 'Key02': 'VAL2' + } + } + } + thing_group = client.create_thing_group(thingGroupName=group_name, thingGroupProperties=props) + thing_group.should.have.key('thingGroupName').which.should.equal(group_name) + thing_group.should.have.key('thingGroupArn') + + thing_group = client.describe_thing_group(thingGroupName=group_name) + thing_group.should.have.key('thingGroupProperties')\ + .which.should.have.key('attributePayload')\ + .which.should.have.key('attributes') + res_props = thing_group['thingGroupProperties']['attributePayload']['attributes'] + res_props.should.have.key('key1').which.should.equal('val01') + res_props.should.have.key('Key02').which.should.equal('VAL2') + + # props update test with merge + new_props = { + 'attributePayload': { + 'attributes': { + 'k3': 'v3' + }, + 'merge': True + } + } + client.update_thing_group( + thingGroupName=group_name, + thingGroupProperties=new_props + ) + thing_group = client.describe_thing_group(thingGroupName=group_name) + thing_group.should.have.key('thingGroupProperties')\ + .which.should.have.key('attributePayload')\ + .which.should.have.key('attributes') + res_props = thing_group['thingGroupProperties']['attributePayload']['attributes'] + res_props.should.have.key('key1').which.should.equal('val01') + res_props.should.have.key('Key02').which.should.equal('VAL2') + + res_props.should.have.key('k3').which.should.equal('v3') + + # props update test + new_props = { + 'attributePayload': { + 'attributes': { + 'k4': 'v4' + } + } + } + client.update_thing_group( + thingGroupName=group_name, + thingGroupProperties=new_props + ) + thing_group = client.describe_thing_group(thingGroupName=group_name) + thing_group.should.have.key('thingGroupProperties')\ + .which.should.have.key('attributePayload')\ + .which.should.have.key('attributes') + res_props = thing_group['thingGroupProperties']['attributePayload']['attributes'] + res_props.should.have.key('k4').which.should.equal('v4') + res_props.should_not.have.key('key1') + + +@mock_iot +def test_thing_group_relations(): + client = boto3.client('iot', region_name='ap-northeast-1') + name = 'my-thing' + group_name = 'my-group-name' + + # thing group + thing_group = client.create_thing_group(thingGroupName=group_name) + thing_group.should.have.key('thingGroupName').which.should.equal(group_name) + thing_group.should.have.key('thingGroupArn') + + # thing + thing = client.create_thing(thingName=name) + thing.should.have.key('thingName').which.should.equal(name) + thing.should.have.key('thingArn') + + # add in 4 way + client.add_thing_to_thing_group( + thingGroupName=group_name, + thingName=name + ) + client.add_thing_to_thing_group( + thingGroupArn=thing_group['thingGroupArn'], + thingArn=thing['thingArn'] + ) + client.add_thing_to_thing_group( + thingGroupName=group_name, + thingArn=thing['thingArn'] + ) + client.add_thing_to_thing_group( + thingGroupArn=thing_group['thingGroupArn'], + thingName=name + ) + + things = client.list_things_in_thing_group( + thingGroupName=group_name + ) + things.should.have.key('things') + things['things'].should.have.length_of(1) + + thing_groups = client.list_thing_groups_for_thing( + thingName=name + ) + thing_groups.should.have.key('thingGroups') + thing_groups['thingGroups'].should.have.length_of(1) + + # remove in 4 way + client.remove_thing_from_thing_group( + thingGroupName=group_name, + thingName=name + ) + client.remove_thing_from_thing_group( + thingGroupArn=thing_group['thingGroupArn'], + thingArn=thing['thingArn'] + ) + client.remove_thing_from_thing_group( + thingGroupName=group_name, + thingArn=thing['thingArn'] + ) + client.remove_thing_from_thing_group( + thingGroupArn=thing_group['thingGroupArn'], + thingName=name + ) + things = client.list_things_in_thing_group( + thingGroupName=group_name + ) + things.should.have.key('things') + things['things'].should.have.length_of(0) + + # update thing group for thing + client.update_thing_groups_for_thing( + thingName=name, + thingGroupsToAdd=[ + group_name + ] + ) + things = client.list_things_in_thing_group( + thingGroupName=group_name + ) + things.should.have.key('things') + things['things'].should.have.length_of(1) + + client.update_thing_groups_for_thing( + thingName=name, + thingGroupsToRemove=[ + group_name + ] + ) + things = client.list_things_in_thing_group( + thingGroupName=group_name + ) + things.should.have.key('things') + things['things'].should.have.length_of(0) From 56ce26a72809a7e7b56c56002e843e406b2dfd46 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera Date: Thu, 4 Jan 2018 15:31:17 +0530 Subject: [PATCH 15/25] Added support for filtering AMIs by self (#1398) * Added support for filtering AMIs by self Closes: https://github.com/spulec/moto/issues/1396 * Adjusted regex to also match signature v4 and fixed py3 compatibility --- moto/core/responses.py | 16 ++++++++++++++++ moto/ec2/models.py | 15 +++++++++++---- moto/ec2/responses/amis.py | 5 +++-- tests/test_ec2/test_amis.py | 14 ++++++++++++++ 4 files changed, 44 insertions(+), 6 deletions(-) diff --git a/moto/core/responses.py b/moto/core/responses.py index ae91cdc02..5afe5e168 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -108,6 +108,7 @@ class BaseResponse(_TemplateEnvironmentMixin): # to extract region, use [^.] region_regex = re.compile(r'\.(?P[a-z]{2}-[a-z]+-\d{1})\.amazonaws\.com') param_list_regex = re.compile(r'(.*)\.(\d+)\.') + access_key_regex = re.compile(r'AWS.*(?P(? Date: Wed, 10 Jan 2018 15:29:08 -0800 Subject: [PATCH 16/25] Add update_access_key endpoint (#1423) --- moto/iam/models.py | 12 ++++++++++++ moto/iam/responses.py | 8 ++++++++ tests/test_iam/test_iam.py | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/moto/iam/models.py b/moto/iam/models.py index 57d24826d..32ca144c3 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -349,6 +349,14 @@ class User(BaseModel): raise IAMNotFoundException( "Key {0} not found".format(access_key_id)) + def update_access_key(self, access_key_id, status): + for key in self.access_keys: + if key.access_key_id == access_key_id: + key.status = status + break + else: + raise IAMNotFoundException("The Access Key with id {0} cannot be found".format(access_key_id)) + def get_cfn_attribute(self, attribute_name): from moto.cloudformation.exceptions import UnformattedGetAttTemplateException if attribute_name == 'Arn': @@ -817,6 +825,10 @@ class IAMBackend(BaseBackend): key = user.create_access_key() return key + def update_access_key(self, user_name, access_key_id, status): + user = self.get_user(user_name) + user.update_access_key(access_key_id, status) + def get_all_access_keys(self, user_name, marker=None, max_items=None): user = self.get_user(user_name) keys = user.get_all_access_keys() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 0e11c09d5..9931cb8d0 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -440,6 +440,14 @@ class IamResponse(BaseResponse): template = self.response_template(CREATE_ACCESS_KEY_TEMPLATE) return template.render(key=key) + def update_access_key(self): + user_name = self._get_param('UserName') + access_key_id = self._get_param('AccessKeyId') + status = self._get_param('Status') + iam_backend.update_access_key(user_name, access_key_id, status) + template = self.response_template(GENERIC_EMPTY_TEMPLATE) + return template.render(name='UpdateAccessKey') + def list_access_keys(self): user_name = self._get_param('UserName') diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index d50f6999e..b4dfe532d 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -651,3 +651,21 @@ def test_attach_detach_user_policy(): resp = client.list_attached_user_policies(UserName=user.name) resp['AttachedPolicies'].should.have.length_of(0) + + +@mock_iam +def test_update_access_key(): + iam = boto3.resource('iam', region_name='us-east-1') + client = iam.meta.client + username = 'test-user' + iam.create_user(UserName=username) + with assert_raises(ClientError): + client.update_access_key(UserName=username, + AccessKeyId='non-existent-key', + Status='Inactive') + key = client.create_access_key(UserName=username)['AccessKey'] + client.update_access_key(UserName=username, + AccessKeyId=key['AccessKeyId'], + Status='Inactive') + resp = client.list_access_keys(UserName=username) + resp['AccessKeyMetadata'][0]['Status'].should.equal('Inactive') From 681726b82679dadefbbba2f30f267ac0c754eeb4 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 27 Dec 2017 11:08:09 -0800 Subject: [PATCH 17/25] Including the in-source version number --- moto/__init__.py | 2 +- scripts/bump_version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index 3508dfeda..9d292a3e1 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.0.1' +__version__ = '1.2.0', from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/scripts/bump_version b/scripts/bump_version index fe7ec1970..b5dc43562 100755 --- a/scripts/bump_version +++ b/scripts/bump_version @@ -17,7 +17,7 @@ main() { git checkout -b version-${version} # Commit the new version - git commit setup.py -m "bumping to version ${version}" + git commit setup.py moto/__init__.py -m "bumping to version ${version}" # Commit an updated IMPLEMENTATION_COVERAGE.md make implementation_coverage || true # Open a PR From fbaca6a130a232d9c1b49b37633b334675befd30 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 27 Dec 2017 11:12:47 -0800 Subject: [PATCH 18/25] Updating CHANGELOG for 1.2.0 --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b10967f64..15ddbec45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,17 @@ Moto Changelog =================== -Latest +1.2.0 ------ + * Implemented signal_workflow_execution for SWF * Wired SWF backend to the moto server - * Fixed incorrect handling of task list parameter on start_workflow_execution + * Revamped lambda function storage to do versioning + * IOT improvements + * RDS improvements + * Implemented CloudWatch get_metric_statistics + * Improved Cloudformation EC2 support + * Implemented Cloudformation change_set endpoints 1.1.25 ----- From c348fd25018589d7a964ea011e108c80e3775203 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:01:40 -0800 Subject: [PATCH 19/25] Adding .bumpversion.cfg --- .bumpversion.cfg | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 000000000..b775ca46c --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,8 @@ +[bumpversion] +current_version = 1.1.25 + +[bumpversion:file:setup.py] + +[bumpversion:file:moto/__init__.py] + +[bumpversion:file:setup.cfg] From 38711b398cc148c350a2eeb0545562a80178aaf0 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:02:41 -0800 Subject: [PATCH 20/25] bringing old version number into line --- moto/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/__init__.py b/moto/__init__.py index 9d292a3e1..12a4edc6d 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.2.0', +__version__ = '1.1.25', from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa From 85e0e2d2c0b962768a750edb01f7aa27e2074f0e Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:03:04 -0800 Subject: [PATCH 21/25] not committing version to setup.cfg --- .bumpversion.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b775ca46c..add9882e0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -5,4 +5,3 @@ current_version = 1.1.25 [bumpversion:file:moto/__init__.py] -[bumpversion:file:setup.cfg] From 24fee6726af5ebc85bb66e6fd9f390feff41b9d2 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:04:32 -0800 Subject: [PATCH 22/25] bumping version to 1.2.0 --- .bumpversion.cfg | 2 +- moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index add9882e0..32a01af8f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.25 +current_version = 1.2.0 [bumpversion:file:setup.py] diff --git a/moto/__init__.py b/moto/__init__.py index 12a4edc6d..9d292a3e1 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.1.25', +__version__ = '1.2.0', from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 201622627..27c635944 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ else: setup( name='moto', - version='1.1.25', + version='1.2.0', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 58c37c6fdfc1cabca09f956792caa5237164f4f5 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:06:32 -0800 Subject: [PATCH 23/25] using bumpversion package for scripts/bumpversion --- scripts/bump_version | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/bump_version b/scripts/bump_version index b5dc43562..5315f26f0 100755 --- a/scripts/bump_version +++ b/scripts/bump_version @@ -10,10 +10,8 @@ main() { return 1 fi - # TODO: replace this with the bumpversion pip package, I couldn't - # figure out how to use that for these files - sed -i '' "s/version=.*$/version='${version}',/g" setup.py - sed -i '' "s/__version__ = .*$/__version__ = '${version}',/g" moto/__init__.py + &>/dev/null which bumpversion || pip install bumpversion + bumpversion --new-version ${version} patch git checkout -b version-${version} # Commit the new version From 021303a2af40fc648975cdfc3e2393178ab70add Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:07:40 -0800 Subject: [PATCH 24/25] simplifying committing of changed versioned files --- scripts/bump_version | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/bump_version b/scripts/bump_version index 5315f26f0..d1af3a84b 100755 --- a/scripts/bump_version +++ b/scripts/bump_version @@ -1,6 +1,8 @@ #!/bin/bash main() { + set -euo pipefail # Bash safemode + local version=$1 if [[ -z "${version}" ]]; then echo "USAGE: $0 1.3.2" @@ -15,7 +17,7 @@ main() { git checkout -b version-${version} # Commit the new version - git commit setup.py moto/__init__.py -m "bumping to version ${version}" + git commit -a -m "bumping to version ${version}" # Commit an updated IMPLEMENTATION_COVERAGE.md make implementation_coverage || true # Open a PR From 738dc083c813857d565ea823ba0ca7754a0e4632 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 10 Jan 2018 15:32:16 -0800 Subject: [PATCH 25/25] updating CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15ddbec45..4dac737b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Moto Changelog 1.2.0 ------ + * Supports filtering AMIs by self * Implemented signal_workflow_execution for SWF * Wired SWF backend to the moto server * Revamped lambda function storage to do versioning