Merge pull request #736 from okomestudio/ts/botocore_update

Remove botocore version pinning

moto/core/responses.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 import datetime
 import json
+import logging
 import re
 
 import pytz
@@ -18,6 +19,9 @@ from moto.compat import OrderedDict
 from moto.core.utils import camelcase_to_underscores, method_names_from_class
 
 
+log = logging.getLogger(__name__)
+
+
 def _decode_dict(d):
     decoded = {}
     for key, value in d.items():
@@ -550,11 +554,20 @@ def xml_to_json_response(service_spec, operation, xml, result_node=None):
 
         od = OrderedDict()
         for k, v in value.items():
-            if k.startswith('@') or v is None:
+            if k.startswith('@'):
+                continue
+
+            if k not in spec:
+                # this can happen when with an older version of
+                # botocore for which the node in XML template is not
+                # defined in service spec.
+                log.warning('Field %s is not defined by the botocore version in use', k)
                 continue
 
             if spec[k]['type'] == 'list':
-                if len(spec[k]['member']) == 1:
+                if v is None:
+                    od[k] = []
+                elif len(spec[k]['member']) == 1:
                     if isinstance(v['member'], list):
                         od[k] = transform(v['member'], spec[k]['member'])
                     else:
@@ -566,9 +579,20 @@ def xml_to_json_response(service_spec, operation, xml, result_node=None):
                 else:
                     raise ValueError('Malformatted input')
             elif spec[k]['type'] == 'map':
-                key = from_str(v['entry']['key'], spec[k]['key'])
+                if v is None:
-                val = from_str(v['entry']['value'], spec[k]['value'])
+                    od[k] = {}
-                od[k] = {key: val}
+                else:
+                    items = ([v['entry']] if not isinstance(v['entry'], list) else
+                             v['entry'])
+                    for item in items:
+                        key = from_str(item['key'], spec[k]['key'])
+                        val = from_str(item['value'], spec[k]['value'])
+                        if k not in od:
+                            od[k] = {}
+                        od[k][key] = val
+            else:
+                if v is None:
+                    od[k] = None
                 else:
                     od[k] = transform(v, spec[k])
         return od

moto/emr/models.py

@@ -26,12 +26,19 @@ class FakeBootstrapAction(object):
 
 
 class FakeInstanceGroup(object):
-    def __init__(self, instance_count, instance_role, instance_type, market, name,
+    def __init__(self, instance_count, instance_role, instance_type,
-                 id=None, bid_price=None):
+                 market='ON_DEMAND', name=None, id=None, bid_price=None):
         self.id = id or random_instance_group_id()
 
         self.bid_price = bid_price
         self.market = market
+        if name is None:
+            if instance_role == 'MASTER':
+                name = 'master'
+            elif instance_role == 'CORE':
+                name = 'slave'
+            else:
+                name = 'Task instance group'
         self.name = name
         self.num_instances = instance_count
         self.role = instance_role

requirements-dev.txt

@@ -5,21 +5,6 @@ sure==1.2.24
 coverage
 freezegun
 flask
-# botocore 1.4.29 breaks the following tests:
+boto3>=1.3.1
-# test_nat_gateway.test_delete_nat_gateway
+botocore>=1.4.28
-# test `list -> create -> list -> get -> delete -> list` integration
-# test_lambda.test_get_function
-# test_lambda.test_delete_function
-# test_lambda.test_create_function_from_zipfile
-# test_lambda.test_create_function_from_aws_bucket
-# test_apigateway.test_integrations
-# test_apigateway.test_integration_response
-# test_apigateway.test_deployment
-# test_apigateway.test_create_resource
-# test_apigateway.test_create_method_response
-# test_apigateway.test_create_method
-# test_apigateway.test_child_resource
-# so we need to pin a boto3 and botocore revision pair that we know works
-boto3==1.3.1
-botocore==1.4.28
 six

tests/test_apigateway/test_apigateway.py

@@ -79,6 +79,7 @@ def test_create_resource():
         resourceId=root_id,
     )
     root_resource['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    root_resource['ResponseMetadata'].pop('RetryAttempts', None)
     root_resource.should.equal({
         'path': '/',
         'id': root_id,
@@ -137,6 +138,7 @@ def test_child_resource():
         resourceId=tags_id,
     )
     child_resource['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    child_resource['ResponseMetadata'].pop('RetryAttempts', None)
     child_resource.should.equal({
         'path': '/users/tags',
         'pathPart': 'tags',
@@ -173,6 +175,7 @@ def test_create_method():
     )
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'httpMethod': 'GET',
         'authorizationType': 'none',
@@ -212,6 +215,7 @@ def test_create_method_response():
         statusCode='200',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'ResponseMetadata': {'HTTPStatusCode': 200},
         'statusCode': '200'
@@ -224,6 +228,7 @@ def test_create_method_response():
         statusCode='200',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'ResponseMetadata': {'HTTPStatusCode': 200},
         'statusCode': '200'
@@ -236,6 +241,7 @@ def test_create_method_response():
         statusCode='200',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({'ResponseMetadata': {'HTTPStatusCode': 200}})
 
 
@@ -273,6 +279,7 @@ def test_integrations():
         uri='http://httpbin.org/robots.txt',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'ResponseMetadata': {'HTTPStatusCode': 200},
         'httpMethod': 'GET',
@@ -294,6 +301,7 @@ def test_integrations():
         httpMethod='GET'
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'ResponseMetadata': {'HTTPStatusCode': 200},
         'httpMethod': 'GET',
@@ -314,6 +322,7 @@ def test_integrations():
         resourceId=root_id,
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response['resourceMethods']['GET']['methodIntegration'].should.equal({
         'httpMethod': 'GET',
         'integrationResponses': {
@@ -363,6 +372,7 @@ def test_integrations():
         requestTemplates=templates
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response['ResponseMetadata'].should.equal({'HTTPStatusCode': 200})
 
     response = client.get_integration(
@@ -416,6 +426,7 @@ def test_integration_response():
         selectionPattern='foobar',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'statusCode': '200',
         'selectionPattern': 'foobar',
@@ -432,6 +443,7 @@ def test_integration_response():
         statusCode='200',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'statusCode': '200',
         'selectionPattern': 'foobar',
@@ -447,6 +459,7 @@ def test_integration_response():
         httpMethod='GET',
     )
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response['methodIntegration']['integrationResponses'].should.equal({
         '200': {
             'responseTemplates': {
@@ -495,6 +508,7 @@ def test_update_stage_configuration():
     )
     response.pop('createdDate',None) # createdDate is hard to match against, remove it
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'id': deployment_id,
         'ResponseMetadata': {'HTTPStatusCode': 200},
@@ -658,6 +672,7 @@ def test_create_stage():
     )
     response.pop('createdDate',None) # createdDate is hard to match against, remove it
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'id': deployment_id,
         'ResponseMetadata': {'HTTPStatusCode': 200},
@@ -677,6 +692,7 @@ def test_create_stage():
     )
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
 
     response['items'][0].pop('createdDate')
     response['items'][1].pop('createdDate')
@@ -688,6 +704,7 @@ def test_create_stage():
     response = client.create_stage(restApiId=api_id,stageName=new_stage_name,deploymentId=deployment_id2)
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
 
     response.should.equal({
         'stageName':new_stage_name,
@@ -712,6 +729,7 @@ def test_create_stage():
     })
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
 
     response.should.equal({
         'stageName':new_stage_name_with_vars,
@@ -737,6 +755,7 @@ def test_create_stage():
     }, cacheClusterEnabled=True,description="hello moto")
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
 
     response.should.equal({
         'stageName':new_stage_name,
@@ -762,6 +781,7 @@ def test_create_stage():
     }, cacheClusterEnabled=True,cacheClusterSize="1.6",description="hello moto")
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
 
     response.should.equal({
         'stageName':new_stage_name,
@@ -807,6 +827,7 @@ def test_deployment():
     )
     response.pop('createdDate',None) # createdDate is hard to match against, remove it
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'id': deployment_id,
         'ResponseMetadata': {'HTTPStatusCode': 200},

tests/test_ec2/test_nat_gateway.py

@@ -57,6 +57,7 @@ def test_delete_nat_gateway():
     response = conn.delete_nat_gateway(NatGatewayId=nat_gateway_id)
 
     response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it
+    response['ResponseMetadata'].pop('RetryAttempts', None)
     response.should.equal({
         'NatGatewayId': nat_gateway_id,
         'ResponseMetadata': {

tests/test_emr/test_emr.py

@@ -46,6 +46,8 @@ def test_describe_cluster():
             'Configurations.member.1.Classification': 'yarn-site',
             'Configurations.member.1.Properties.entry.1.key': 'someproperty',
             'Configurations.member.1.Properties.entry.1.value': 'somevalue',
+            'Configurations.member.1.Properties.entry.2.key': 'someotherproperty',
+            'Configurations.member.1.Properties.entry.2.value': 'someothervalue',
             'Instances.EmrManagedMasterSecurityGroup': 'master-security-group',
             'Instances.Ec2SubnetId': 'subnet-8be41cec',
         },

tests/test_emr/test_emr_boto3.py

@@ -63,7 +63,8 @@ def test_describe_cluster():
     args['Applications'] = [{'Name': 'Spark', 'Version': '2.4.2'}]
     args['Configurations'] = [
         {'Classification': 'yarn-site',
-         'Properties': {'someproperty': 'somevalue'}}]
+         'Properties': {'someproperty': 'somevalue',
+                        'someotherproperty': 'someothervalue'}}]
     args['Instances']['AdditionalMasterSecurityGroups'] = ['additional-master']
     args['Instances']['AdditionalSlaveSecurityGroups'] = ['additional-slave']
     args['Instances']['Ec2KeyName'] = 'mykey'
@@ -232,8 +233,8 @@ def test_describe_job_flow():
     jf['LogUri'].should.equal(args['LogUri'])
     jf['Name'].should.equal(args['Name'])
     jf['ServiceRole'].should.equal(args['ServiceRole'])
-    jf.shouldnt.have.key('Steps')
+    jf['Steps'].should.equal([])
-    jf.shouldnt.have.key('SupportedProducts')
+    jf['SupportedProducts'].should.equal([])
     jf['VisibleToAllUsers'].should.equal(True)
 
 
@@ -320,7 +321,7 @@ def test_run_job_flow():
     resp['LogUri'].should.equal(args['LogUri'])
     resp['VisibleToAllUsers'].should.equal(args['VisibleToAllUsers'])
     resp['Instances']['NormalizedInstanceHours'].should.equal(0)
-    resp.shouldnt.have.key('Steps')
+    resp['Steps'].should.equal([])
 
 
 @mock_emr
@@ -445,10 +446,11 @@ def test_bootstrap_actions():
         {'Name': 'bs1',
          'ScriptBootstrapAction': {
              'Args': ['arg1', 'arg2'],
-             'Path': 'path/to/script'}},
+             'Path': 's3://path/to/script'}},
         {'Name': 'bs2',
          'ScriptBootstrapAction': {
-             'Path': 'path/to/anotherscript'}}
+             'Args': [],
+             'Path': 's3://path/to/anotherscript'}}
     ]
 
     client = boto3.client('emr', region_name='us-east-1')
@@ -658,4 +660,4 @@ def test_tags():
 
     client.remove_tags(ResourceId=cluster_id, TagKeys=[t['Key'] for t in input_tags])
     resp = client.describe_cluster(ClusterId=cluster_id)['Cluster']
-    resp.shouldnt.have.key('Tags')
+    resp['Tags'].should.equal([])