Merge pull request #43 from spulec/master

Merge upstream
This commit is contained in:
Bert Blommers 2020-05-01 12:49:42 +01:00 committed by GitHub
commit c39924501b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 414 additions and 21 deletions

1
.gitignore vendored
View File

@ -22,3 +22,4 @@ tests/file.tmp
.eggs/
.mypy_cache/
*.tmp
.venv/

View File

@ -11,7 +11,6 @@ import requests
import pytz
from moto.core.access_control import IAMRequest, S3IAMRequest
from moto.core.exceptions import DryRunClientError
from jinja2 import Environment, DictLoader, TemplateNotFound
@ -134,9 +133,13 @@ class ActionAuthenticatorMixin(object):
ActionAuthenticatorMixin.request_count += 1
def _authenticate_and_authorize_normal_action(self):
from moto.iam.access_control import IAMRequest
self._authenticate_and_authorize_action(IAMRequest)
def _authenticate_and_authorize_s3_action(self):
from moto.iam.access_control import S3IAMRequest
self._authenticate_and_authorize_action(S3IAMRequest)
@staticmethod

View File

@ -187,7 +187,7 @@ def iso_8601_datetime_with_milliseconds(datetime):
def iso_8601_datetime_without_milliseconds(datetime):
return datetime.strftime("%Y-%m-%dT%H:%M:%S") + "Z"
return None if datetime is None else datetime.strftime("%Y-%m-%dT%H:%M:%S") + "Z"
RFC1123 = "%a, %d %b %Y %H:%M:%S GMT"

View File

@ -104,6 +104,7 @@ from .utils import (
random_internet_gateway_id,
random_ip,
random_ipv6_cidr,
randor_ipv4_cidr,
random_launch_template_id,
random_nat_gateway_id,
random_key_pair,
@ -112,6 +113,8 @@ from .utils import (
random_reservation_id,
random_route_table_id,
generate_route_id,
generate_vpc_end_point_id,
create_dns_entries,
split_route_id,
random_security_group_id,
random_snapshot_id,
@ -2741,6 +2744,7 @@ class VPCBackend(object):
def __init__(self):
self.vpcs = {}
self.vpc_end_points = {}
self.vpc_refs[self.__class__].add(weakref.ref(self))
super(VPCBackend, self).__init__()
@ -2883,6 +2887,66 @@ class VPCBackend(object):
vpc = self.get_vpc(vpc_id)
return vpc.associate_vpc_cidr_block(cidr_block, amazon_provided_ipv6_cidr_block)
def create_vpc_endpoint(
self,
vpc_id,
service_name,
type=None,
policy_document=False,
route_table_ids=None,
subnet_ids=[],
network_interface_ids=[],
dns_entries=None,
client_token=None,
security_group=None,
tag_specifications=None,
private_dns_enabled=None,
):
vpc_endpoint_id = generate_vpc_end_point_id(vpc_id)
# validates if vpc is present or not.
self.get_vpc(vpc_id)
if type and type.lower() == "interface":
network_interface_ids = []
for subnet_id in subnet_ids:
self.get_subnet(subnet_id)
eni = self.create_network_interface(subnet_id, random_private_ip())
network_interface_ids.append(eni.id)
dns_entries = create_dns_entries(service_name, vpc_endpoint_id)
else:
# considering gateway if type is not mentioned.
service_destination_cidr = randor_ipv4_cidr()
for route_table_id in route_table_ids:
self.create_route(route_table_id, service_destination_cidr)
if dns_entries:
dns_entries = [dns_entries]
vpc_end_point = VPCEndPoint(
vpc_endpoint_id,
vpc_id,
service_name,
type,
policy_document,
route_table_ids,
subnet_ids,
network_interface_ids,
dns_entries,
client_token,
security_group,
tag_specifications,
private_dns_enabled,
)
self.vpc_end_points[vpc_endpoint_id] = vpc_end_point
return vpc_end_point
class VPCPeeringConnectionStatus(object):
def __init__(self, code="initiating-request", message=""):
@ -3491,6 +3555,40 @@ class Route(object):
return route_table
class VPCEndPoint(TaggedEC2Resource):
def __init__(
self,
id,
vpc_id,
service_name,
type=None,
policy_document=False,
route_table_ids=None,
subnet_ids=None,
network_interface_ids=None,
dns_entries=None,
client_token=None,
security_group=None,
tag_specifications=None,
private_dns_enabled=None,
):
self.id = id
self.vpc_id = vpc_id
self.service_name = service_name
self.type = type
self.policy_document = policy_document
self.route_table_ids = route_table_ids
self.network_interface_ids = network_interface_ids
self.subnet_ids = subnet_ids
self.client_token = client_token
self.security_group = security_group
self.tag_specifications = tag_specifications
self.private_dns_enabled = private_dns_enabled
self.created_at = datetime.utcnow()
self.dns_entries = dns_entries
class RouteBackend(object):
def __init__(self):
super(RouteBackend, self).__init__()

View File

@ -163,6 +163,34 @@ class VPCs(BaseResponse):
cidr_block_state="disassociating",
)
def create_vpc_endpoint(self):
vpc_id = self._get_param("VpcId")
service_name = self._get_param("ServiceName")
route_table_ids = self._get_multi_param("RouteTableId")
subnet_ids = self._get_multi_param("SubnetId")
type = self._get_param("VpcEndpointType")
policy_document = self._get_param("PolicyDocument")
client_token = self._get_param("ClientToken")
tag_specifications = self._get_param("TagSpecifications")
private_dns_enabled = self._get_param("PrivateDNSEnabled")
security_group = self._get_param("SecurityGroup")
vpc_end_point = self.ec2_backend.create_vpc_endpoint(
vpc_id=vpc_id,
service_name=service_name,
type=type,
policy_document=policy_document,
route_table_ids=route_table_ids,
subnet_ids=subnet_ids,
client_token=client_token,
security_group=security_group,
tag_specifications=tag_specifications,
private_dns_enabled=private_dns_enabled,
)
template = self.response_template(CREATE_VPC_END_POINT)
return template.render(vpc_end_point=vpc_end_point)
CREATE_VPC_RESPONSE = """
<CreateVpcResponse xmlns="http://ec2.amazonaws.com/doc/{{doc_date}}/">
@ -384,3 +412,40 @@ IPV6_DISASSOCIATE_VPC_CIDR_BLOCK_RESPONSE = """
</ipv6CidrBlockState>
</ipv6CidrBlockAssociation>
</DisassociateVpcCidrBlockResponse>"""
CREATE_VPC_END_POINT = """ <CreateVpcEndpointResponse xmlns="http://monitoring.amazonaws.com/doc/2010-08-01/">
<vpcEndpoint>
<policyDocument>{{ vpc_end_point.policy_document }}</policyDocument>
<state> available </state>
<vpcEndpointPolicySupported> false </vpcEndpointPolicySupported>
<serviceName>{{ vpc_end_point.service_name }}</serviceName>
<vpcId>{{ vpc_end_point.vpc_id }}</vpcId>
<vpcEndpointId>{{ vpc_end_point.id }}</vpcEndpointId>
<routeTableIdSet>
{% for routeid in vpc_end_point.route_table_ids %}
<item>{{ routeid }}</item>
{% endfor %}
</routeTableIdSet>
<networkInterfaceIdSet>
{% for network_interface_id in vpc_end_point.network_interface_ids %}
<item>{{ network_interface_id }}</item>
{% endfor %}
</networkInterfaceIdSet>
<subnetIdSet>
{% for subnetId in vpc_end_point.subnet_ids %}
<item>{{ subnetId }}</item>
{% endfor %}
</subnetIdSet>
<dnsEntrySet>
{% if vpc_end_point.dns_entries %}
{% for entry in vpc_end_point.dns_entries %}
<item>
<hostedZoneId>{{ entry["hosted_zone_id"] }}</hostedZoneId>
<dnsName>{{ entry["dns_name"] }}</dnsName>
</item>
{% endfor %}
{% endif %}
</dnsEntrySet>
<creationTimestamp>{{ vpc_end_point.created_at }}</creationTimestamp>
</vpcEndpoint>
</CreateVpcEndpointResponse>"""

View File

@ -181,6 +181,10 @@ def random_ip():
)
def randor_ipv4_cidr():
return "10.0.{}.{}/16".format(random.randint(0, 255), random.randint(0, 255))
def random_ipv6_cidr():
return "2400:6500:{}:{}::/56".format(random_resource_id(4), random_resource_id(4))
@ -189,6 +193,19 @@ def generate_route_id(route_table_id, cidr_block):
return "%s~%s" % (route_table_id, cidr_block)
def generate_vpc_end_point_id(vpc_id):
return "%s-%s" % ("vpce", vpc_id[4:])
def create_dns_entries(service_name, vpc_endpoint_id):
dns_entries = {}
dns_entries["dns_name"] = "{}-{}.{}".format(
vpc_endpoint_id, random_resource_id(8), service_name
)
dns_entries["hosted_zone_id"] = random_resource_id(13).upper()
return dns_entries
def split_route_id(route_id):
values = route_id.split("~")
return values[0], values[1]

View File

@ -25,8 +25,6 @@ from botocore.credentials import Credentials
from six import string_types
from moto.core import ACCOUNT_ID
from moto.iam.models import Policy
from moto.iam import iam_backend
from moto.core.exceptions import (
SignatureDoesNotMatchError,
AccessDeniedError,
@ -44,6 +42,7 @@ from moto.s3.exceptions import (
S3SignatureDoesNotMatchError,
)
from moto.sts import sts_backend
from .models import iam_backend, Policy
log = logging.getLogger(__name__)

View File

@ -464,7 +464,7 @@ class AccessKey(BaseModel):
self.secret_access_key = random_alphanumeric(40)
self.status = "Active"
self.create_date = datetime.utcnow()
self.last_used = datetime.utcnow()
self.last_used = None
@property
def created_iso_8601(self):
@ -676,20 +676,50 @@ class User(BaseModel):
if len(self.access_keys) == 0:
access_key_1_active = "false"
access_key_1_last_rotated = "N/A"
access_key_1_last_used = "N/A"
access_key_2_active = "false"
access_key_2_last_rotated = "N/A"
access_key_2_last_used = "N/A"
elif len(self.access_keys) == 1:
access_key_1_active = "true"
access_key_1_last_rotated = date_created.strftime(date_format)
access_key_1_active = (
"true" if self.access_keys[0].status == "Active" else "false"
)
access_key_1_last_rotated = self.access_keys[0].create_date.strftime(
date_format
)
access_key_1_last_used = (
"N/A"
if self.access_keys[0].last_used is None
else self.access_keys[0].last_used.strftime(date_format)
)
access_key_2_active = "false"
access_key_2_last_rotated = "N/A"
access_key_2_last_used = "N/A"
else:
access_key_1_active = "true"
access_key_1_last_rotated = date_created.strftime(date_format)
access_key_2_active = "true"
access_key_2_last_rotated = date_created.strftime(date_format)
access_key_1_active = (
"true" if self.access_keys[0].status == "Active" else "false"
)
access_key_1_last_rotated = self.access_keys[0].create_date.strftime(
date_format
)
access_key_1_last_used = (
"N/A"
if self.access_keys[0].last_used is None
else self.access_keys[0].last_used.strftime(date_format)
)
access_key_2_active = (
"true" if self.access_keys[1].status == "Active" else "false"
)
access_key_2_last_rotated = self.access_keys[1].create_date.strftime(
date_format
)
access_key_2_last_used = (
"N/A"
if self.access_keys[1].last_used is None
else self.access_keys[1].last_used.strftime(date_format)
)
return "{0},{1},{2},{3},{4},{5},not_supported,false,{6},{7},{8},{9},false,N/A,false,N/A".format(
return "{0},{1},{2},{3},{4},{5},not_supported,false,{6},{7},{8},not_supported,not_supported,{9},{10},{11},not_supported,not_supported,false,N/A,false,N/A\n".format(
self.name,
self.arn,
date_created.strftime(date_format),
@ -698,8 +728,10 @@ class User(BaseModel):
date_created.strftime(date_format),
access_key_1_active,
access_key_1_last_rotated,
access_key_1_last_used,
access_key_2_active,
access_key_2_last_rotated,
access_key_2_last_used,
)
@ -1799,7 +1831,7 @@ class IAMBackend(BaseBackend):
def get_credential_report(self):
if not self.credential_report:
raise IAMReportNotPresentException("Credential report not present")
report = "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_2_active,access_key_2_last_rotated,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\n"
report = "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\n"
for user in self.users:
report += self.users[user].to_csv()
return base64.b64encode(report.encode("ascii")).decode("ascii")

View File

@ -1779,7 +1779,11 @@ GET_ACCESS_KEY_LAST_USED_TEMPLATE = """
<GetAccessKeyLastUsedResult>
<UserName>{{ user_name }}</UserName>
<AccessKeyLastUsed>
<LastUsedDate>{{ last_used }}</LastUsedDate>
{% if last_used %}
<LastUsedDate>{{ last_used }}</LastUsedDate>
{% endif %}
<ServiceName>N/A</ServiceName>
<Region>N/A</Region>
</AccessKeyLastUsed>
</GetAccessKeyLastUsedResult>
</GetAccessKeyLastUsedResponse>

View File

@ -842,6 +842,17 @@ class IoTBackend(BaseBackend):
return thing_group.thing_group_name, thing_group.arn, thing_group.thing_group_id
def delete_thing_group(self, thing_group_name, expected_version):
child_groups = [
thing_group
for _, thing_group in self.thing_groups.items()
if thing_group.parent_group_name == thing_group_name
]
if len(child_groups) > 0:
raise InvalidRequestException(
" Cannot delete thing group : "
+ thing_group_name
+ " when there are still child groups attached to it"
)
thing_group = self.describe_thing_group(thing_group_name)
del self.thing_groups[thing_group.arn]

View File

@ -618,3 +618,63 @@ def test_describe_route_tables_with_nat_gateway():
nat_gw_routes[0]["DestinationCidrBlock"].should.equal("0.0.0.0/0")
nat_gw_routes[0]["NatGatewayId"].should.equal(nat_gw_id)
nat_gw_routes[0]["State"].should.equal("active")
@mock_ec2
def test_create_vpc_end_point():
ec2 = boto3.client("ec2", region_name="us-west-1")
vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")
subnet = ec2.create_subnet(VpcId=vpc["Vpc"]["VpcId"], CidrBlock="10.0.0.0/24")
route_table = ec2.create_route_table(VpcId=vpc["Vpc"]["VpcId"])
# test without any end point type specified
vpc_end_point = ec2.create_vpc_endpoint(
VpcId=vpc["Vpc"]["VpcId"],
ServiceName="com.amazonaws.us-east-1.s3",
RouteTableIds=[route_table["RouteTable"]["RouteTableId"]],
)
vpc_end_point["VpcEndpoint"]["ServiceName"].should.equal(
"com.amazonaws.us-east-1.s3"
)
vpc_end_point["VpcEndpoint"]["RouteTableIds"][0].should.equal(
route_table["RouteTable"]["RouteTableId"]
)
vpc_end_point["VpcEndpoint"]["VpcId"].should.equal(vpc["Vpc"]["VpcId"])
vpc_end_point["VpcEndpoint"]["DnsEntries"].should.have.length_of(0)
# test with any end point type as gateway
vpc_end_point = ec2.create_vpc_endpoint(
VpcId=vpc["Vpc"]["VpcId"],
ServiceName="com.amazonaws.us-east-1.s3",
RouteTableIds=[route_table["RouteTable"]["RouteTableId"]],
VpcEndpointType="gateway",
)
vpc_end_point["VpcEndpoint"]["ServiceName"].should.equal(
"com.amazonaws.us-east-1.s3"
)
vpc_end_point["VpcEndpoint"]["RouteTableIds"][0].should.equal(
route_table["RouteTable"]["RouteTableId"]
)
vpc_end_point["VpcEndpoint"]["VpcId"].should.equal(vpc["Vpc"]["VpcId"])
vpc_end_point["VpcEndpoint"]["DnsEntries"].should.have.length_of(0)
# test with end point type as interface
vpc_end_point = ec2.create_vpc_endpoint(
VpcId=vpc["Vpc"]["VpcId"],
ServiceName="com.amazonaws.us-east-1.s3",
SubnetIds=[subnet["Subnet"]["SubnetId"]],
VpcEndpointType="interface",
)
vpc_end_point["VpcEndpoint"]["ServiceName"].should.equal(
"com.amazonaws.us-east-1.s3"
)
vpc_end_point["VpcEndpoint"]["SubnetIds"][0].should.equal(
subnet["Subnet"]["SubnetId"]
)
vpc_end_point["VpcEndpoint"]["VpcId"].should.equal(vpc["Vpc"]["VpcId"])
len(vpc_end_point["VpcEndpoint"]["DnsEntries"]).should.be.greater_than(0)

View File

@ -4,6 +4,7 @@ import json
import boto
import boto3
import csv
import os
import sure # noqa
import sys
@ -11,9 +12,10 @@ from boto.exception import BotoServerError
from botocore.exceptions import ClientError
from dateutil.tz import tzutc
from moto import mock_iam, mock_iam_deprecated
from moto.iam.models import aws_managed_policies
from moto import mock_iam, mock_iam_deprecated, settings
from moto.core import ACCOUNT_ID
from moto.iam.models import aws_managed_policies
from moto.backends import get_backend
from nose.tools import assert_raises, assert_equals
from nose.tools import raises
@ -1215,6 +1217,69 @@ def test_boto3_get_credential_report():
report.should.match(r".*my-user.*")
@mock_iam
def test_boto3_get_credential_report_content():
conn = boto3.client("iam", region_name="us-east-1")
username = "my-user"
conn.create_user(UserName=username)
key1 = conn.create_access_key(UserName=username)["AccessKey"]
conn.update_access_key(
UserName=username, AccessKeyId=key1["AccessKeyId"], Status="Inactive"
)
key1 = conn.create_access_key(UserName=username)["AccessKey"]
timestamp = datetime.utcnow()
if not settings.TEST_SERVER_MODE:
iam_backend = get_backend("iam")["global"]
iam_backend.users[username].access_keys[1].last_used = timestamp
with assert_raises(ClientError):
conn.get_credential_report()
result = conn.generate_credential_report()
while result["State"] != "COMPLETE":
result = conn.generate_credential_report()
result = conn.get_credential_report()
report = result["Content"].decode("utf-8")
header = report.split("\n")[0]
header.should.equal(
"user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated"
)
report_dict = csv.DictReader(report.split("\n"))
user = next(report_dict)
user["user"].should.equal("my-user")
user["access_key_1_active"].should.equal("false")
user["access_key_1_last_rotated"].should.match(timestamp.strftime("%Y-%m-%d"))
user["access_key_1_last_used_date"].should.equal("N/A")
user["access_key_2_active"].should.equal("true")
if not settings.TEST_SERVER_MODE:
user["access_key_2_last_used_date"].should.match(timestamp.strftime("%Y-%m-%d"))
else:
user["access_key_2_last_used_date"].should.equal("N/A")
@mock_iam
def test_get_access_key_last_used_when_used():
iam = boto3.resource("iam", region_name="us-east-1")
client = iam.meta.client
username = "test-user"
iam.create_user(UserName=username)
with assert_raises(ClientError):
client.get_access_key_last_used(AccessKeyId="non-existent-key-id")
create_key_response = client.create_access_key(UserName=username)["AccessKey"]
# Set last used date using the IAM backend. Moto currently does not have a mechanism for tracking usage of access keys
if not settings.TEST_SERVER_MODE:
timestamp = datetime.utcnow()
iam_backend = get_backend("iam")["global"]
iam_backend.users[username].access_keys[0].last_used = timestamp
resp = client.get_access_key_last_used(
AccessKeyId=create_key_response["AccessKeyId"]
)
if not settings.TEST_SERVER_MODE:
datetime.strftime(
resp["AccessKeyLastUsed"]["LastUsedDate"], "%Y-%m-%d"
).should.equal(timestamp.strftime("%Y-%m-%d"))
else:
resp["AccessKeyLastUsed"].should_not.contain("LastUsedDate")
@requires_boto_gte("2.39")
@mock_iam_deprecated()
def test_managed_policy():
@ -1382,7 +1447,7 @@ def test_update_access_key():
@mock_iam
def test_get_access_key_last_used():
def test_get_access_key_last_used_when_unused():
iam = boto3.resource("iam", region_name="us-east-1")
client = iam.meta.client
username = "test-user"
@ -1393,10 +1458,7 @@ def test_get_access_key_last_used():
resp = client.get_access_key_last_used(
AccessKeyId=create_key_response["AccessKeyId"]
)
datetime.strftime(
resp["AccessKeyLastUsed"]["LastUsedDate"], "%Y-%m-%d"
).should.equal(datetime.strftime(datetime.utcnow(), "%Y-%m-%d"))
resp["AccessKeyLastUsed"].should_not.contain("LastUsedDate")
resp["UserName"].should.equal(create_key_response["UserName"])

View File

@ -756,6 +756,47 @@ def test_delete_principal_thing():
client.delete_certificate(certificateId=cert_id)
@mock_iot
def test_delete_thing_group():
client = boto3.client("iot", region_name="ap-northeast-1")
group_name_1a = "my-group-name-1a"
group_name_2a = "my-group-name-2a"
# --1a
# |--2a
# create thing groups tree
# 1
thing_group1a = client.create_thing_group(thingGroupName=group_name_1a)
thing_group1a.should.have.key("thingGroupName").which.should.equal(group_name_1a)
thing_group1a.should.have.key("thingGroupArn")
# 2
thing_group2a = client.create_thing_group(
thingGroupName=group_name_2a, parentGroupName=group_name_1a
)
thing_group2a.should.have.key("thingGroupName").which.should.equal(group_name_2a)
thing_group2a.should.have.key("thingGroupArn")
# delete group with child
try:
client.delete_thing_group(thingGroupName=group_name_1a)
except client.exceptions.InvalidRequestException as exc:
error_code = exc.response["Error"]["Code"]
error_code.should.equal("InvalidRequestException")
else:
raise Exception("Should have raised error")
# delete child group
client.delete_thing_group(thingGroupName=group_name_2a)
res = client.list_thing_groups()
res.should.have.key("thingGroups").which.should.have.length_of(1)
res["thingGroups"].should_not.have.key(group_name_2a)
# now that there is no child group, we can delete the previus group safely
client.delete_thing_group(thingGroupName=group_name_1a)
res = client.list_thing_groups()
res.should.have.key("thingGroups").which.should.have.length_of(0)
@mock_iot
def test_describe_thing_group_metadata_hierarchy():
client = boto3.client("iot", region_name="ap-northeast-1")