From 23ad62a4f9951f9027a9b019bf7f9158b977e73e Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Fri, 27 Oct 2017 03:16:06 +0200 Subject: [PATCH 1/6] validate protocol on target group creation --- moto/elbv2/models.py | 10 ++++++++++ tests/test_elbv2/test_elbv2.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 8aa9ee9f0..bb1a195d3 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -300,6 +300,16 @@ class ELBv2Backend(BaseBackend): if target_group.name == name: raise DuplicateTargetGroupName() + valid_protocols = ['HTTPS', 'HTTP', 'TCP'] + if kwargs['healthcheck_protocol'] not in valid_protocols: + raise InvalidConditionValueError( + "Value {} at 'healthCheckProtocol' failed to satisfy constraint: " + "Member must satisfy enum value set: {}".format(kwargs['healthcheck_protocol'], valid_protocols)) + if kwargs['protocol'] not in valid_protocols: + raise InvalidConditionValueError( + "Value {} at 'protocol' failed to satisfy constraint: " + "Member must satisfy enum value set: {}".format(kwargs['protocol'], valid_protocols)) + arn = "arn:aws:elasticloadbalancing:%s:1:targetgroup/%s/50dc6c495c0c9188" % (self.region_name, name) target_group = FakeTargetGroup(name, arn, **kwargs) self.target_groups[target_group.arn] = target_group diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 98634c677..a33cfeec6 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -283,6 +283,21 @@ def test_create_target_group_and_listeners(): load_balancer_arn = response.get('LoadBalancers')[0].get('LoadBalancerArn') + # Can't create a target group with an invalid protocol + with assert_raises(ClientError): + conn.create_target_group( + Name='a-target', + Protocol='HTTP', + Port=8080, + VpcId=vpc.id, + HealthCheckProtocol='/HTTP', + HealthCheckPort='8080', + HealthCheckPath='/', + HealthCheckIntervalSeconds=5, + HealthCheckTimeoutSeconds=5, + HealthyThresholdCount=5, + UnhealthyThresholdCount=2, + Matcher={'HttpCode': '200'}) response = conn.create_target_group( Name='a-target', Protocol='HTTP', From 70e7f08d8f126424856bcf0e65a71d4b44251f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20Xia=28=E5=A4=8F=E6=81=BA=29?= Date: Mon, 6 Nov 2017 20:44:54 +1100 Subject: [PATCH 2/6] Added support for SSM get_parameters_by_path. (#1299) * Added support for SSM get_parameters_by_path. Signed-off-by: Kai Xia * add logic to handle trailing '/'. Also, fix pep8. Signed-off-by: Kai Xia * add tests for parameter value in response. Signed-off-by: Kai Xia --- moto/ssm/models.py | 15 +++++++++++ moto/ssm/responses.py | 19 ++++++++++++++ tests/test_ssm/test_ssm_boto3.py | 45 ++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index a0e4a2155..c8c428b64 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -75,6 +75,21 @@ class SimpleSystemManagerBackend(BaseBackend): result.append(self._parameters[name]) return result + def get_parameters_by_path(self, path, with_decryption, recursive): + """Implement the get-parameters-by-path-API in the backend.""" + result = [] + # path could be with or without a trailing /. we handle this + # difference here. + path = path.rstrip('/') + '/' + for param in self._parameters: + if not param.startswith(path): + continue + if '/' in param[len(path) + 1:] and not recursive: + continue + result.append(self._parameters[param]) + + return result + def get_parameter(self, name, with_decryption): if name in self._parameters: return self._parameters[name] diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index 3b75ada09..3227839b9 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -81,6 +81,25 @@ class SimpleSystemManagerResponse(BaseResponse): response['InvalidParameters'].append(name) return json.dumps(response) + def get_parameters_by_path(self): + path = self._get_param('Path') + with_decryption = self._get_param('WithDecryption') + recursive = self._get_param('Recursive', False) + + result = self.ssm_backend.get_parameters_by_path( + path, with_decryption, recursive + ) + + response = { + 'Parameters': [], + } + + for parameter in result: + param_data = parameter.response_object(with_decryption) + response['Parameters'].append(param_data) + + return json.dumps(response) + def describe_parameters(self): page_size = 10 filters = self._get_param('Filters') diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 7f4aca533..781727c26 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -47,6 +47,51 @@ def test_delete_parameters(): len(response['Parameters']).should.equal(0) +@mock_ssm +def test_get_parameters_by_path(): + client = boto3.client('ssm', region_name='us-east-1') + + client.put_parameter( + Name='/foo/name1', + Description='A test parameter', + Value='value1', + Type='String') + + client.put_parameter( + Name='/foo/name2', + Description='A test parameter', + Value='value2', + Type='String') + + client.put_parameter( + Name='/bar/name3', + Description='A test parameter', + Value='value3', + Type='String') + + client.put_parameter( + Name='/bar/name3/name4', + Description='A test parameter', + Value='value4', + Type='String') + + response = client.get_parameters_by_path(Path='/foo') + len(response['Parameters']).should.equal(2) + {p['Value'] for p in response['Parameters']}.should.equal( + set(['value1', 'value2']) + ) + + response = client.get_parameters_by_path(Path='/bar', Recursive=False) + len(response['Parameters']).should.equal(1) + response['Parameters'][0]['Value'].should.equal('value3') + + response = client.get_parameters_by_path(Path='/bar', Recursive=True) + len(response['Parameters']).should.equal(2) + {p['Value'] for p in response['Parameters']}.should.equal( + set(['value3', 'value4']) + ) + + @mock_ssm def test_put_parameter(): client = boto3.client('ssm', region_name='us-east-1') From d0a285536da3ead5df0414bcdb6b36afebc562b6 Mon Sep 17 00:00:00 2001 From: Adam Gilman Date: Mon, 6 Nov 2017 19:06:55 +0000 Subject: [PATCH 3/6] Makes Creating SNS topics idempotent (#1324) Make creating SNS topics idempotent Closes #1323 --- moto/sns/models.py | 9 ++++++--- tests/test_sns/test_topics_boto3.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 856255be5..bf7c605e4 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -193,9 +193,12 @@ class SNSBackend(BaseBackend): self.sms_attributes.update(attrs) def create_topic(self, name): - topic = Topic(name, self) - self.topics[topic.arn] = topic - return topic + candidate_topic = Topic(name, self) + if candidate_topic.arn in self.topics: + return self.topics[candidate_topic.arn] + else: + self.topics[candidate_topic.arn] = candidate_topic + return candidate_topic def _get_values_nexttoken(self, values_map, next_token=None): if next_token is None: diff --git a/tests/test_sns/test_topics_boto3.py b/tests/test_sns/test_topics_boto3.py index a9c2a2904..495e63e72 100644 --- a/tests/test_sns/test_topics_boto3.py +++ b/tests/test_sns/test_topics_boto3.py @@ -31,6 +31,29 @@ def test_create_and_delete_topic(): topics = topics_json["Topics"] topics.should.have.length_of(0) +@mock_sns +def test_create_topic_should_be_indempodent(): + conn = boto3.client("sns", region_name="us-east-1") + topic_arn = conn.create_topic(Name="some-topic")['TopicArn'] + conn.set_topic_attributes( + TopicArn=topic_arn, + AttributeName="DisplayName", + AttributeValue="should_be_set" + ) + topic_display_name = conn.get_topic_attributes( + TopicArn=topic_arn + )['Attributes']['DisplayName'] + topic_display_name.should.be.equal("should_be_set") + + #recreate topic to prove indempodentcy + topic_arn = conn.create_topic(Name="some-topic")['TopicArn'] + topic_display_name = conn.get_topic_attributes( + TopicArn=topic_arn + )['Attributes']['DisplayName'] + + topic_display_name.should.be.equal("should_be_set") + + @mock_sns def test_get_missing_topic(): From d4745a575b1bed852fd753e281a3ce2b096cb044 Mon Sep 17 00:00:00 2001 From: Terry Cain Date: Mon, 6 Nov 2017 21:39:08 +0000 Subject: [PATCH 4/6] utf 8 in key deletions V2 (#1321) * supporting utf-8 in key deletions * Fixed decoding of version body when regexing * Fixed some more random errors * Possible fix * Fixed unused import * Added UTF comment Py2 --- moto/core/models.py | 38 ++++++++++++++++++++++++++++++++++++-- moto/s3/responses.py | 20 ++++++++++++++++---- tests/test_s3/test_s3.py | 28 ++++++++++++++++------------ 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 6e93f911a..c6fb72ffa 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals from __future__ import absolute_import @@ -176,16 +177,49 @@ class ServerModeMockAWS(BaseMockAWS): if 'endpoint_url' not in kwargs: kwargs['endpoint_url'] = "http://localhost:5000" return real_boto3_resource(*args, **kwargs) + + def fake_httplib_send_output(self, message_body=None, *args, **kwargs): + def _convert_to_bytes(mixed_buffer): + bytes_buffer = [] + for chunk in mixed_buffer: + if isinstance(chunk, six.text_type): + bytes_buffer.append(chunk.encode('utf-8')) + else: + bytes_buffer.append(chunk) + msg = b"\r\n".join(bytes_buffer) + return msg + + self._buffer.extend((b"", b"")) + msg = _convert_to_bytes(self._buffer) + del self._buffer[:] + if isinstance(message_body, bytes): + msg += message_body + message_body = None + self.send(msg) + # if self._expect_header_set: + # read, write, exc = select.select([self.sock], [], [self.sock], 1) + # if read: + # self._handle_expect_response(message_body) + # return + if message_body is not None: + self.send(message_body) + self._client_patcher = mock.patch('boto3.client', fake_boto3_client) - self._resource_patcher = mock.patch( - 'boto3.resource', fake_boto3_resource) + self._resource_patcher = mock.patch('boto3.resource', fake_boto3_resource) + if six.PY2: + self._httplib_patcher = mock.patch('httplib.HTTPConnection._send_output', fake_httplib_send_output) + self._client_patcher.start() self._resource_patcher.start() + if six.PY2: + self._httplib_patcher.start() def disable_patching(self): if self._client_patcher: self._client_patcher.stop() self._resource_patcher.stop() + if six.PY2: + self._httplib_patcher.stop() class Model(type): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index d5e15aead..fb1735a5c 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -8,6 +8,7 @@ from six.moves.urllib.parse import parse_qs, urlparse import xmltodict +from moto.packages.httpretty.core import HTTPrettyRequest from moto.core.responses import _TemplateEnvironmentMixin from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys @@ -113,7 +114,10 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, {}, response.encode("utf-8") else: status_code, headers, response_content = response - return status_code, headers, response_content.encode("utf-8") + if not isinstance(response_content, six.binary_type): + response_content = response_content.encode("utf-8") + + return status_code, headers, response_content def _bucket_response(self, request, full_url, headers): parsed_url = urlparse(full_url) @@ -139,6 +143,7 @@ class ResponseObject(_TemplateEnvironmentMixin): body = b'' if isinstance(body, six.binary_type): body = body.decode('utf-8') + body = u'{0}'.format(body).encode('utf-8') if method == 'HEAD': return self._bucket_response_head(bucket_name, headers) @@ -209,7 +214,7 @@ class ResponseObject(_TemplateEnvironmentMixin): if not website_configuration: template = self.response_template(S3_NO_BUCKET_WEBSITE_CONFIG) return 404, {}, template.render(bucket_name=bucket_name) - return website_configuration + return 200, {}, website_configuration elif 'acl' in querystring: bucket = self.backend.get_bucket(bucket_name) template = self.response_template(S3_OBJECT_ACL_RESPONSE) @@ -355,7 +360,7 @@ class ResponseObject(_TemplateEnvironmentMixin): if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" if 'versioning' in querystring: - ver = re.search('([A-Za-z]+)', body) + ver = re.search('([A-Za-z]+)', body.decode()) if ver: self.backend.set_bucket_versioning(bucket_name, ver.group(1)) template = self.response_template(S3_BUCKET_VERSIONING) @@ -444,7 +449,12 @@ class ResponseObject(_TemplateEnvironmentMixin): def _bucket_response_post(self, request, body, bucket_name, headers): if not request.headers.get('Content-Length'): return 411, {}, "Content-Length required" - path = request.path if hasattr(request, 'path') else request.path_url + + if isinstance(request, HTTPrettyRequest): + path = request.path + else: + path = request.full_path if hasattr(request, 'full_path') else request.path_url + if self.is_delete_keys(request, path, bucket_name): return self._bucket_response_delete_keys(request, body, bucket_name, headers) @@ -454,6 +464,8 @@ class ResponseObject(_TemplateEnvironmentMixin): form = request.form else: # HTTPretty, build new form object + body = body.decode() + form = {} for kv in body.split('&'): k, v = kv.split('=') diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 32ccd1b9c..829941d79 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - from __future__ import unicode_literals import datetime @@ -1865,7 +1864,7 @@ def test_boto3_list_object_versions(): def test_boto3_delete_markers(): s3 = boto3.client('s3', region_name='us-east-1') bucket_name = 'mybucket' - key = 'key-with-versions' + key = u'key-with-versions-and-unicode-ó' s3.create_bucket(Bucket=bucket_name) s3.put_bucket_versioning( Bucket=bucket_name, @@ -1880,10 +1879,9 @@ def test_boto3_delete_markers(): Key=key, Body=body ) - s3.delete_object( - Bucket=bucket_name, - Key=key - ) + + s3.delete_objects(Bucket=bucket_name, Delete={'Objects': [{'Key': key}]}) + with assert_raises(ClientError) as e: s3.get_object( Bucket=bucket_name, @@ -1905,12 +1903,18 @@ def test_boto3_delete_markers(): Bucket=bucket_name ) response['Versions'].should.have.length_of(2) - response['Versions'][-1]['IsLatest'].should.be.true - response['Versions'][0]['IsLatest'].should.be.false - [(key_metadata['Key'], key_metadata['VersionId']) - for key_metadata in response['Versions']].should.equal( - [('key-with-versions', '0'), ('key-with-versions', '1')] - ) + + # We've asserted there is only 2 records so one is newest, one is oldest + latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0] + oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0] + + # Double check ordering of version ID's + latest['VersionId'].should.equal('1') + oldest['VersionId'].should.equal('0') + + # Double check the name is still unicode + latest['Key'].should.equal('key-with-versions-and-unicode-ó') + oldest['Key'].should.equal('key-with-versions-and-unicode-ó') @mock_s3 From 7c3d78e7c565751feb45af7b4b58834900dad989 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Fri, 27 Oct 2017 03:16:06 +0200 Subject: [PATCH 5/6] validate protocol on target group creation --- moto/elbv2/models.py | 10 ++++++++++ tests/test_elbv2/test_elbv2.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 40bfdd067..9571eeea1 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -443,6 +443,16 @@ class ELBv2Backend(BaseBackend): if FakeTargetGroup.HTTP_CODE_REGEX.match(kwargs['matcher']['HttpCode']) is None: raise RESTError('InvalidParameterValue', 'HttpCode must be like 200 | 200-399 | 200,201 ...') + valid_protocols = ['HTTPS', 'HTTP', 'TCP'] + if kwargs['healthcheck_protocol'] not in valid_protocols: + raise InvalidConditionValueError( + "Value {} at 'healthCheckProtocol' failed to satisfy constraint: " + "Member must satisfy enum value set: {}".format(kwargs['healthcheck_protocol'], valid_protocols)) + if kwargs['protocol'] not in valid_protocols: + raise InvalidConditionValueError( + "Value {} at 'protocol' failed to satisfy constraint: " + "Member must satisfy enum value set: {}".format(kwargs['protocol'], valid_protocols)) + arn = "arn:aws:elasticloadbalancing:%s:1:targetgroup/%s/50dc6c495c0c9188" % (self.region_name, name) target_group = FakeTargetGroup(name, arn, **kwargs) self.target_groups[target_group.arn] = target_group diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index d030925e6..4f0b1a9cd 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -740,6 +740,21 @@ def test_handle_listener_rules(): load_balancer_arn = response.get('LoadBalancers')[0].get('LoadBalancerArn') + # Can't create a target group with an invalid protocol + with assert_raises(ClientError): + conn.create_target_group( + Name='a-target', + Protocol='HTTP', + Port=8080, + VpcId=vpc.id, + HealthCheckProtocol='/HTTP', + HealthCheckPort='8080', + HealthCheckPath='/', + HealthCheckIntervalSeconds=5, + HealthCheckTimeoutSeconds=5, + HealthyThresholdCount=5, + UnhealthyThresholdCount=2, + Matcher={'HttpCode': '200'}) response = conn.create_target_group( Name='a-target', Protocol='HTTP', From 73f28d6ba8992886343fcecf79e57f2d69f31406 Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Mon, 6 Nov 2017 14:17:10 -0800 Subject: [PATCH 6/6] adding elbv2 utils.py for arn generation --- moto/elbv2/models.py | 6 ++++-- moto/elbv2/utils.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 moto/elbv2/utils.py diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 9571eeea1..c3bf77534 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -7,6 +7,8 @@ from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel from moto.ec2.models import ec2_backends from moto.acm.models import acm_backends +from .utils import make_arn_for_target_group +from .utils import make_arn_for_load_balancer from .exceptions import ( DuplicateLoadBalancerName, DuplicateListenerError, @@ -342,7 +344,7 @@ class ELBv2Backend(BaseBackend): subnets.append(subnet) vpc_id = subnets[0].vpc_id - arn = "arn:aws:elasticloadbalancing:%s:1:loadbalancer/%s/50dc6c495c0c9188" % (self.region_name, name) + arn = make_arn_for_load_balancer(account_id=1, name=name, region_name=self.region_name) dns_name = "%s-1.%s.elb.amazonaws.com" % (name, self.region_name) if arn in self.load_balancers: @@ -453,7 +455,7 @@ class ELBv2Backend(BaseBackend): "Value {} at 'protocol' failed to satisfy constraint: " "Member must satisfy enum value set: {}".format(kwargs['protocol'], valid_protocols)) - arn = "arn:aws:elasticloadbalancing:%s:1:targetgroup/%s/50dc6c495c0c9188" % (self.region_name, name) + arn = make_arn_for_target_group(account_id=1, name=name, region_name=self.region_name) target_group = FakeTargetGroup(name, arn, **kwargs) self.target_groups[target_group.arn] = target_group return target_group diff --git a/moto/elbv2/utils.py b/moto/elbv2/utils.py new file mode 100644 index 000000000..47a3e66d5 --- /dev/null +++ b/moto/elbv2/utils.py @@ -0,0 +1,8 @@ +def make_arn_for_load_balancer(account_id, name, region_name): + return "arn:aws:elasticloadbalancing:{}:{}:loadbalancer/{}/50dc6c495c0c9188".format( + region_name, account_id, name) + + +def make_arn_for_target_group(account_id, name, region_name): + return "arn:aws:elasticloadbalancing:{}:{}:targetgroup/{}/50dc6c495c0c9188".format( + region_name, account_id, name)