Merge branch 'master' of https://github.com/spulec/moto

Dockerfile

@@ -1,11 +1,15 @@
-FROM python:2
+FROM alpine:3.6
 
 ADD . /moto/
 ENV PYTHONUNBUFFERED 1
 
 WORKDIR /moto/
-RUN pip install ".[server]"
+RUN  apk add --no-cache python3 && \
+     python3 -m ensurepip && \
+     rm -r /usr/lib/python*/ensurepip && \
+     pip3 --no-cache-dir install --upgrade pip setuptools && \
+     pip3 --no-cache-dir install ".[server]"
 
-CMD ["moto_server"]
+ENTRYPOINT ["/usr/bin/moto_server", "-H", "0.0.0.0"]
 
 EXPOSE 5000

Makefile

@@ -15,6 +15,9 @@ test: lint
 test_server:
 	@TEST_SERVER_MODE=true nosetests -sv --with-coverage --cover-html ./tests/
 
+aws_managed_policies:
+	scripts/update_managed_policies.py
+
 publish:
 	python setup.py sdist bdist_wheel upload
 	git tag `python setup.py --version`

moto/dynamodb2/models.py

@@ -484,7 +484,6 @@ class Table(BaseModel):
         else:
             results.sort(key=lambda item: item.range_key)
 
-        
         if projection_expression:
             expressions = [x.strip() for x in projection_expression.split(',')]
             for result in possible_results:
@@ -499,7 +498,6 @@ class Table(BaseModel):
 
         results, last_evaluated_key = self._trim_results(results, limit,
                                                          exclusive_start_key)
-
         return results, scanned_count, last_evaluated_key
 
     def all_items(self):

moto/iam/models.py

@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 import base64
 from datetime import datetime
+import json
 
 import pytz
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_without_milliseconds
 
+from .aws_managed_policies import aws_managed_policies_data
 from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException
 from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id
 
@@ -92,6 +94,20 @@ class ManagedPolicy(Policy):
 class AWSManagedPolicy(ManagedPolicy):
     """AWS-managed policy."""
 
+    @classmethod
+    def from_data(cls, name, data):
+        return cls(name,
+                   default_version_id=data.get('DefaultVersionId'),
+                   path=data.get('Path'),
+                   document=data.get('Document'))
+
+
+# AWS defines some of its own managed policies and we periodically
+# import them via `make aws_managed_policies`
+aws_managed_policies = [
+    AWSManagedPolicy.from_data(name, d) for name, d
+    in json.loads(aws_managed_policies_data).items()]
+
 
 class InlinePolicy(Policy):
     """TODO: is this needed?"""
@@ -388,115 +404,6 @@ class User(BaseModel):
                                                                                                         )
 
 
-# predefine AWS managed policies
-aws_managed_policies = [
-    AWSManagedPolicy(
-        'AmazonElasticMapReduceRole',
-        default_version_id='v6',
-        path='/service-role/',
-        document={
-            "Version": "2012-10-17",
-            "Statement": [{
-                "Effect": "Allow",
-                "Resource": "*",
-                "Action": [
-                    "ec2:AuthorizeSecurityGroupEgress",
-                    "ec2:AuthorizeSecurityGroupIngress",
-                    "ec2:CancelSpotInstanceRequests",
-                    "ec2:CreateNetworkInterface",
-                    "ec2:CreateSecurityGroup",
-                    "ec2:CreateTags",
-                    "ec2:DeleteNetworkInterface",
-                    "ec2:DeleteSecurityGroup",
-                    "ec2:DeleteTags",
-                    "ec2:DescribeAvailabilityZones",
-                    "ec2:DescribeAccountAttributes",
-                    "ec2:DescribeDhcpOptions",
-                    "ec2:DescribeInstanceStatus",
-                    "ec2:DescribeInstances",
-                    "ec2:DescribeKeyPairs",
-                    "ec2:DescribeNetworkAcls",
-                    "ec2:DescribeNetworkInterfaces",
-                    "ec2:DescribePrefixLists",
-                    "ec2:DescribeRouteTables",
-                    "ec2:DescribeSecurityGroups",
-                    "ec2:DescribeSpotInstanceRequests",
-                    "ec2:DescribeSpotPriceHistory",
-                    "ec2:DescribeSubnets",
-                    "ec2:DescribeVpcAttribute",
-                    "ec2:DescribeVpcEndpoints",
-                    "ec2:DescribeVpcEndpointServices",
-                    "ec2:DescribeVpcs",
-                    "ec2:DetachNetworkInterface",
-                    "ec2:ModifyImageAttribute",
-                    "ec2:ModifyInstanceAttribute",
-                    "ec2:RequestSpotInstances",
-                    "ec2:RevokeSecurityGroupEgress",
-                    "ec2:RunInstances",
-                    "ec2:TerminateInstances",
-                    "ec2:DeleteVolume",
-                    "ec2:DescribeVolumeStatus",
-                    "ec2:DescribeVolumes",
-                    "ec2:DetachVolume",
-                    "iam:GetRole",
-                    "iam:GetRolePolicy",
-                    "iam:ListInstanceProfiles",
-                    "iam:ListRolePolicies",
-                    "iam:PassRole",
-                    "s3:CreateBucket",
-                    "s3:Get*",
-                    "s3:List*",
-                    "sdb:BatchPutAttributes",
-                    "sdb:Select",
-                    "sqs:CreateQueue",
-                    "sqs:Delete*",
-                    "sqs:GetQueue*",
-                    "sqs:PurgeQueue",
-                    "sqs:ReceiveMessage"
-                ]
-            }]
-        }
-    ),
-    AWSManagedPolicy(
-        'AmazonElasticMapReduceforEC2Role',
-        default_version_id='v2',
-        path='/service-role/',
-        document={
-            "Version": "2012-10-17",
-            "Statement": [{
-                "Effect": "Allow",
-                "Resource": "*",
-                "Action": [
-                    "cloudwatch:*",
-                    "dynamodb:*",
-                    "ec2:Describe*",
-                    "elasticmapreduce:Describe*",
-                    "elasticmapreduce:ListBootstrapActions",
-                    "elasticmapreduce:ListClusters",
-                    "elasticmapreduce:ListInstanceGroups",
-                    "elasticmapreduce:ListInstances",
-                    "elasticmapreduce:ListSteps",
-                    "kinesis:CreateStream",
-                    "kinesis:DeleteStream",
-                    "kinesis:DescribeStream",
-                    "kinesis:GetRecords",
-                    "kinesis:GetShardIterator",
-                    "kinesis:MergeShards",
-                    "kinesis:PutRecord",
-                    "kinesis:SplitShard",
-                    "rds:Describe*",
-                    "s3:*",
-                    "sdb:*",
-                    "sns:*",
-                    "sqs:*"
-                ]
-            }]
-        }
-    )
-]
-# TODO: add more predefined AWS managed policies
-
-
 class IAMBackend(BaseBackend):
 
     def __init__(self):

moto/s3/responses.py

@@ -636,6 +636,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
 
         storage_class = request.headers.get('x-amz-storage-class', 'STANDARD')
         acl = self._acl_from_headers(request.headers)
+        if acl is None:
+            acl = self.backend.get_bucket(bucket_name).acl
         tagging = self._tagging_from_headers(request.headers)
 
         if 'acl' in query:
@@ -740,7 +742,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
         if grants:
             return FakeAcl(grants)
         else:
-            return get_canned_acl('private')
+            return None
 
     def _tagging_from_headers(self, headers):
         if headers.get('x-amz-tagging'):

scripts/update_managed_policies.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# This updates our local copies of AWS' managed policies
+# Invoked via `make update_managed_policies`
+#
+# Credit goes to
+#   https://gist.github.com/gene1wood/55b358748be3c314f956
+
+from botocore.exceptions import NoCredentialsError
+from datetime import datetime
+import boto3
+import json
+import sys
+
+output_file = "./moto/iam/aws_managed_policies.py"
+
+
+def json_serial(obj):
+    """JSON serializer for objects not serializable by default json code"""
+
+    if isinstance(obj, datetime):
+        serial = obj.isoformat()
+        return serial
+    raise TypeError("Type not serializable")
+
+
+client = boto3.client('iam')
+
+policies = {}
+
+paginator = client.get_paginator('list_policies')
+try:
+    response_iterator = paginator.paginate(Scope='AWS')
+    for response in response_iterator:
+        for policy in response['Policies']:
+            policies[policy['PolicyName']] = policy
+except NoCredentialsError:
+    print("USAGE:")
+    print("Put your AWS credentials into ~/.aws/credentials and run:")
+    print(__file__)
+    print("")
+    print("Or specify them on the command line:")
+    print("AWS_ACCESS_KEY_ID=your_personal_access_key AWS_SECRET_ACCESS_KEY=your_personal_secret {}".format(__file__))
+    print("")
+    sys.exit(1)
+
+for policy_name in policies:
+    response = client.get_policy_version(
+        PolicyArn=policies[policy_name]['Arn'],
+        VersionId=policies[policy_name]['DefaultVersionId'])
+    for key in response['PolicyVersion']:
+        policies[policy_name][key] = response['PolicyVersion'][key]
+
+with open(output_file, 'w') as f:
+    triple_quote = '\"\"\"'
+
+    f.write("# Imported via `make aws_managed_policies`\n")
+    f.write('aws_managed_policies_data = {}\n'.format(triple_quote))
+    f.write(json.dumps(policies,
+                     sort_keys=True,
+                     indent=4,
+                     separators=(',', ': '),
+                     default=json_serial))
+    f.write('{}\n'.format(triple_quote))

setup.py

@@ -24,7 +24,7 @@ extras_require = {
 
 setup(
     name='moto',
-    version='1.1.11',
+    version='1.1.12',
     description='A library that allows your python tests to easily'
                 ' mock out the boto library',
     author='Steve Pulec',

tests/test_iam/test_iam.py

@@ -525,8 +525,14 @@ def test_managed_policy():
                        path='/mypolicy/',
                        description='my user managed policy')
 
-    aws_policies = conn.list_policies(scope='AWS')['list_policies_response'][
+    marker = 0
-        'list_policies_result']['policies']
+    aws_policies = []
+    while marker is not None:
+        response = conn.list_policies(scope='AWS', marker=marker)[
+                'list_policies_response']['list_policies_result']
+        for policy in response['policies']:
+            aws_policies.append(policy)
+        marker = response.get('marker')
     set(p.name for p in aws_managed_policies).should.equal(
         set(p['policy_name'] for p in aws_policies))
 
@@ -535,8 +541,14 @@ def test_managed_policy():
     set(['UserManagedPolicy']).should.equal(
         set(p['policy_name'] for p in user_policies))
 
-    all_policies = conn.list_policies()['list_policies_response'][
+    marker = 0
-        'list_policies_result']['policies']
+    all_policies = []
+    while marker is not None:
+        response = conn.list_policies(marker=marker)[
+                'list_policies_response']['list_policies_result']
+        for policy in response['policies']:
+            all_policies.append(policy)
+        marker = response.get('marker')
     set(p['policy_name'] for p in aws_policies +
         user_policies).should.equal(set(p['policy_name'] for p in all_policies))
 

tests/test_s3/test_s3.py

@@ -870,7 +870,7 @@ def test_s3_object_in_public_bucket():
     s3 = boto3.resource('s3')
     bucket = s3.Bucket('test-bucket')
     bucket.create(ACL='public-read')
-    bucket.put_object(ACL='public-read', Body=b'ABCD', Key='file.txt')
+    bucket.put_object(Body=b'ABCD', Key='file.txt')
 
     s3_anonymous = boto3.resource('s3')
     s3_anonymous.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)