Adding a script to import all of AWS' managed policies

fixes #1118

Makefile

@@ -15,6 +15,9 @@ test: lint
 test_server:
 	@TEST_SERVER_MODE=true nosetests -sv --with-coverage --cover-html ./tests/
 
+aws_managed_policies:
+	scripts/update_managed_policies.py
+
 publish:
 	python setup.py sdist bdist_wheel upload
 	git tag `python setup.py --version`

moto/iam/models.py

@@ -1,11 +1,13 @@
 from __future__ import unicode_literals
 import base64
 from datetime import datetime
+import json
 
 import pytz
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_without_milliseconds
 
+from .aws_managed_policies import aws_managed_policies_data
 from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException
 from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id
 
@@ -92,6 +94,18 @@ class ManagedPolicy(Policy):
 class AWSManagedPolicy(ManagedPolicy):
     """AWS-managed policy."""
 
+    @classmethod
+    def from_data(cls, name, data):
+        return cls(name,
+                   default_version_id=data.get('DefaultVersionId'),
+                   path=data.get('Path'),
+                   document=data.get('Document'))
+
+# AWS defines some of its own managed policies and we periodically
+# import them via `make aws_managed_policies`
+aws_managed_policies = [
+        AWSManagedPolicy.from_data(name, d) for name, d
+        in json.loads(aws_managed_policies_data).items()]
 
 class InlinePolicy(Policy):
     """TODO: is this needed?"""
@@ -388,115 +402,6 @@ class User(BaseModel):
                                                                                                         )
 
 
-# predefine AWS managed policies
-aws_managed_policies = [
-    AWSManagedPolicy(
-        'AmazonElasticMapReduceRole',
-        default_version_id='v6',
-        path='/service-role/',
-        document={
-            "Version": "2012-10-17",
-            "Statement": [{
-                "Effect": "Allow",
-                "Resource": "*",
-                "Action": [
-                    "ec2:AuthorizeSecurityGroupEgress",
-                    "ec2:AuthorizeSecurityGroupIngress",
-                    "ec2:CancelSpotInstanceRequests",
-                    "ec2:CreateNetworkInterface",
-                    "ec2:CreateSecurityGroup",
-                    "ec2:CreateTags",
-                    "ec2:DeleteNetworkInterface",
-                    "ec2:DeleteSecurityGroup",
-                    "ec2:DeleteTags",
-                    "ec2:DescribeAvailabilityZones",
-                    "ec2:DescribeAccountAttributes",
-                    "ec2:DescribeDhcpOptions",
-                    "ec2:DescribeInstanceStatus",
-                    "ec2:DescribeInstances",
-                    "ec2:DescribeKeyPairs",
-                    "ec2:DescribeNetworkAcls",
-                    "ec2:DescribeNetworkInterfaces",
-                    "ec2:DescribePrefixLists",
-                    "ec2:DescribeRouteTables",
-                    "ec2:DescribeSecurityGroups",
-                    "ec2:DescribeSpotInstanceRequests",
-                    "ec2:DescribeSpotPriceHistory",
-                    "ec2:DescribeSubnets",
-                    "ec2:DescribeVpcAttribute",
-                    "ec2:DescribeVpcEndpoints",
-                    "ec2:DescribeVpcEndpointServices",
-                    "ec2:DescribeVpcs",
-                    "ec2:DetachNetworkInterface",
-                    "ec2:ModifyImageAttribute",
-                    "ec2:ModifyInstanceAttribute",
-                    "ec2:RequestSpotInstances",
-                    "ec2:RevokeSecurityGroupEgress",
-                    "ec2:RunInstances",
-                    "ec2:TerminateInstances",
-                    "ec2:DeleteVolume",
-                    "ec2:DescribeVolumeStatus",
-                    "ec2:DescribeVolumes",
-                    "ec2:DetachVolume",
-                    "iam:GetRole",
-                    "iam:GetRolePolicy",
-                    "iam:ListInstanceProfiles",
-                    "iam:ListRolePolicies",
-                    "iam:PassRole",
-                    "s3:CreateBucket",
-                    "s3:Get*",
-                    "s3:List*",
-                    "sdb:BatchPutAttributes",
-                    "sdb:Select",
-                    "sqs:CreateQueue",
-                    "sqs:Delete*",
-                    "sqs:GetQueue*",
-                    "sqs:PurgeQueue",
-                    "sqs:ReceiveMessage"
-                ]
-            }]
-        }
-    ),
-    AWSManagedPolicy(
-        'AmazonElasticMapReduceforEC2Role',
-        default_version_id='v2',
-        path='/service-role/',
-        document={
-            "Version": "2012-10-17",
-            "Statement": [{
-                "Effect": "Allow",
-                "Resource": "*",
-                "Action": [
-                    "cloudwatch:*",
-                    "dynamodb:*",
-                    "ec2:Describe*",
-                    "elasticmapreduce:Describe*",
-                    "elasticmapreduce:ListBootstrapActions",
-                    "elasticmapreduce:ListClusters",
-                    "elasticmapreduce:ListInstanceGroups",
-                    "elasticmapreduce:ListInstances",
-                    "elasticmapreduce:ListSteps",
-                    "kinesis:CreateStream",
-                    "kinesis:DeleteStream",
-                    "kinesis:DescribeStream",
-                    "kinesis:GetRecords",
-                    "kinesis:GetShardIterator",
-                    "kinesis:MergeShards",
-                    "kinesis:PutRecord",
-                    "kinesis:SplitShard",
-                    "rds:Describe*",
-                    "s3:*",
-                    "sdb:*",
-                    "sns:*",
-                    "sqs:*"
-                ]
-            }]
-        }
-    )
-]
-# TODO: add more predefined AWS managed policies
-
-
 class IAMBackend(BaseBackend):
 
     def __init__(self):

scripts/update_managed_policies.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# This updates our local copies of AWS' managed policies
+# Invoked via `make update_managed_policies`
+#
+# Credit goes to
+#   https://gist.github.com/gene1wood/55b358748be3c314f956
+
+from botocore.exceptions import NoCredentialsError
+from datetime import datetime
+import boto3
+import json
+import sys
+
+output_file = "./moto/iam/aws_managed_policies.py"
+
+
+def json_serial(obj):
+    """JSON serializer for objects not serializable by default json code"""
+
+    if isinstance(obj, datetime):
+        serial = obj.isoformat()
+        return serial
+    raise TypeError("Type not serializable")
+
+
+client = boto3.client('iam')
+
+policies = {}
+
+paginator = client.get_paginator('list_policies')
+try:
+    response_iterator = paginator.paginate(Scope='AWS')
+    for response in response_iterator:
+        for policy in response['Policies']:
+            policies[policy['PolicyName']] = policy
+except NoCredentialsError:
+    print("USAGE:")
+    print("Put your AWS credentials into ~/.aws/credentials and run:")
+    print(__file__)
+    print("")
+    print("Or specify them on the command line:")
+    print("AWS_ACCESS_KEY_ID=your_personal_access_key AWS_SECRET_ACCESS_KEY=your_personal_secret {}".format(__file__))
+    print("")
+    sys.exit(1)
+
+for policy_name in policies:
+    response = client.get_policy_version(
+        PolicyArn=policies[policy_name]['Arn'],
+        VersionId=policies[policy_name]['DefaultVersionId'])
+    for key in response['PolicyVersion']:
+        policies[policy_name][key] = response['PolicyVersion'][key]
+
+with open(output_file, 'w') as f:
+    triple_quote = '\"\"\"'
+
+    f.write("# Imported via `make aws_managed_policies`\n")
+    f.write('aws_managed_policies_data = {}\n'.format(triple_quote))
+    f.write(json.dumps(policies,
+                     sort_keys=True,
+                     indent=4,
+                     separators=(',', ': '),
+                     default=json_serial))
+    f.write(triple_quote)

tests/test_iam/test_iam.py

@@ -525,8 +525,14 @@ def test_managed_policy():
                        path='/mypolicy/',
                        description='my user managed policy')
 
-    aws_policies = conn.list_policies(scope='AWS')['list_policies_response'][
-        'list_policies_result']['policies']
+    marker = 0
+    aws_policies = []
+    while marker is not None:
+        response = conn.list_policies(scope='AWS', marker=marker)[
+                'list_policies_response']['list_policies_result']
+        for policy in response['policies']:
+            aws_policies.append(policy)
+        marker = response.get('marker')
     set(p.name for p in aws_managed_policies).should.equal(
         set(p['policy_name'] for p in aws_policies))
 
@@ -535,8 +541,14 @@ def test_managed_policy():
     set(['UserManagedPolicy']).should.equal(
         set(p['policy_name'] for p in user_policies))
 
-    all_policies = conn.list_policies()['list_policies_response'][
-        'list_policies_result']['policies']
+    marker = 0
+    all_policies = []
+    while marker is not None:
+        response = conn.list_policies(marker=marker)[
+                'list_policies_response']['list_policies_result']
+        for policy in response['policies']:
+            all_policies.append(policy)
+        marker = response.get('marker')
     set(p['policy_name'] for p in aws_policies +
         user_policies).should.equal(set(p['policy_name'] for p in all_policies))