diff --git a/.gitignore b/.gitignore index fb9bd51de..02e812c5b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ tests/file.tmp .eggs/ .mypy_cache/ *.tmp +.venv/ diff --git a/moto/core/responses.py b/moto/core/responses.py index 9a46f8ac5..508bd8c59 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -11,7 +11,6 @@ import requests import pytz -from moto.core.access_control import IAMRequest, S3IAMRequest from moto.core.exceptions import DryRunClientError from jinja2 import Environment, DictLoader, TemplateNotFound @@ -134,9 +133,13 @@ class ActionAuthenticatorMixin(object): ActionAuthenticatorMixin.request_count += 1 def _authenticate_and_authorize_normal_action(self): + from moto.iam.access_control import IAMRequest + self._authenticate_and_authorize_action(IAMRequest) def _authenticate_and_authorize_s3_action(self): + from moto.iam.access_control import S3IAMRequest + self._authenticate_and_authorize_action(S3IAMRequest) @staticmethod diff --git a/moto/core/utils.py b/moto/core/utils.py index dce9f675c..921f64be2 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -187,7 +187,7 @@ def iso_8601_datetime_with_milliseconds(datetime): def iso_8601_datetime_without_milliseconds(datetime): - return datetime.strftime("%Y-%m-%dT%H:%M:%S") + "Z" + return None if datetime is None else datetime.strftime("%Y-%m-%dT%H:%M:%S") + "Z" RFC1123 = "%a, %d %b %Y %H:%M:%S GMT" diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 120cda7e4..e94d2877c 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -104,6 +104,7 @@ from .utils import ( random_internet_gateway_id, random_ip, random_ipv6_cidr, + randor_ipv4_cidr, random_launch_template_id, random_nat_gateway_id, random_key_pair, @@ -112,6 +113,8 @@ from .utils import ( random_reservation_id, random_route_table_id, generate_route_id, + generate_vpc_end_point_id, + create_dns_entries, split_route_id, random_security_group_id, random_snapshot_id, @@ -2741,6 +2744,7 @@ class VPCBackend(object): def __init__(self): self.vpcs = {} + self.vpc_end_points = {} self.vpc_refs[self.__class__].add(weakref.ref(self)) super(VPCBackend, self).__init__() @@ -2883,6 +2887,66 @@ class VPCBackend(object): vpc = self.get_vpc(vpc_id) return vpc.associate_vpc_cidr_block(cidr_block, amazon_provided_ipv6_cidr_block) + def create_vpc_endpoint( + self, + vpc_id, + service_name, + type=None, + policy_document=False, + route_table_ids=None, + subnet_ids=[], + network_interface_ids=[], + dns_entries=None, + client_token=None, + security_group=None, + tag_specifications=None, + private_dns_enabled=None, + ): + + vpc_endpoint_id = generate_vpc_end_point_id(vpc_id) + + # validates if vpc is present or not. + self.get_vpc(vpc_id) + + if type and type.lower() == "interface": + + network_interface_ids = [] + for subnet_id in subnet_ids: + self.get_subnet(subnet_id) + eni = self.create_network_interface(subnet_id, random_private_ip()) + network_interface_ids.append(eni.id) + + dns_entries = create_dns_entries(service_name, vpc_endpoint_id) + + else: + # considering gateway if type is not mentioned. + service_destination_cidr = randor_ipv4_cidr() + + for route_table_id in route_table_ids: + self.create_route(route_table_id, service_destination_cidr) + if dns_entries: + dns_entries = [dns_entries] + + vpc_end_point = VPCEndPoint( + vpc_endpoint_id, + vpc_id, + service_name, + type, + policy_document, + route_table_ids, + subnet_ids, + network_interface_ids, + dns_entries, + client_token, + security_group, + tag_specifications, + private_dns_enabled, + ) + + self.vpc_end_points[vpc_endpoint_id] = vpc_end_point + + return vpc_end_point + class VPCPeeringConnectionStatus(object): def __init__(self, code="initiating-request", message=""): @@ -3491,6 +3555,40 @@ class Route(object): return route_table +class VPCEndPoint(TaggedEC2Resource): + def __init__( + self, + id, + vpc_id, + service_name, + type=None, + policy_document=False, + route_table_ids=None, + subnet_ids=None, + network_interface_ids=None, + dns_entries=None, + client_token=None, + security_group=None, + tag_specifications=None, + private_dns_enabled=None, + ): + + self.id = id + self.vpc_id = vpc_id + self.service_name = service_name + self.type = type + self.policy_document = policy_document + self.route_table_ids = route_table_ids + self.network_interface_ids = network_interface_ids + self.subnet_ids = subnet_ids + self.client_token = client_token + self.security_group = security_group + self.tag_specifications = tag_specifications + self.private_dns_enabled = private_dns_enabled + self.created_at = datetime.utcnow() + self.dns_entries = dns_entries + + class RouteBackend(object): def __init__(self): super(RouteBackend, self).__init__() diff --git a/moto/ec2/responses/vpcs.py b/moto/ec2/responses/vpcs.py index 0fd198378..59222207d 100644 --- a/moto/ec2/responses/vpcs.py +++ b/moto/ec2/responses/vpcs.py @@ -163,6 +163,34 @@ class VPCs(BaseResponse): cidr_block_state="disassociating", ) + def create_vpc_endpoint(self): + vpc_id = self._get_param("VpcId") + service_name = self._get_param("ServiceName") + route_table_ids = self._get_multi_param("RouteTableId") + subnet_ids = self._get_multi_param("SubnetId") + type = self._get_param("VpcEndpointType") + policy_document = self._get_param("PolicyDocument") + client_token = self._get_param("ClientToken") + tag_specifications = self._get_param("TagSpecifications") + private_dns_enabled = self._get_param("PrivateDNSEnabled") + security_group = self._get_param("SecurityGroup") + + vpc_end_point = self.ec2_backend.create_vpc_endpoint( + vpc_id=vpc_id, + service_name=service_name, + type=type, + policy_document=policy_document, + route_table_ids=route_table_ids, + subnet_ids=subnet_ids, + client_token=client_token, + security_group=security_group, + tag_specifications=tag_specifications, + private_dns_enabled=private_dns_enabled, + ) + + template = self.response_template(CREATE_VPC_END_POINT) + return template.render(vpc_end_point=vpc_end_point) + CREATE_VPC_RESPONSE = """ @@ -384,3 +412,40 @@ IPV6_DISASSOCIATE_VPC_CIDR_BLOCK_RESPONSE = """ """ + +CREATE_VPC_END_POINT = """ + + {{ vpc_end_point.policy_document }} + available + false + {{ vpc_end_point.service_name }} + {{ vpc_end_point.vpc_id }} + {{ vpc_end_point.id }} + + {% for routeid in vpc_end_point.route_table_ids %} + {{ routeid }} + {% endfor %} + + + {% for network_interface_id in vpc_end_point.network_interface_ids %} + {{ network_interface_id }} + {% endfor %} + + + {% for subnetId in vpc_end_point.subnet_ids %} + {{ subnetId }} + {% endfor %} + + + {% if vpc_end_point.dns_entries %} + {% for entry in vpc_end_point.dns_entries %} + + {{ entry["hosted_zone_id"] }} + {{ entry["dns_name"] }} + + {% endfor %} + {% endif %} + + {{ vpc_end_point.created_at }} + +""" diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 3b363e45d..c07c470a9 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -181,6 +181,10 @@ def random_ip(): ) +def randor_ipv4_cidr(): + return "10.0.{}.{}/16".format(random.randint(0, 255), random.randint(0, 255)) + + def random_ipv6_cidr(): return "2400:6500:{}:{}::/56".format(random_resource_id(4), random_resource_id(4)) @@ -189,6 +193,19 @@ def generate_route_id(route_table_id, cidr_block): return "%s~%s" % (route_table_id, cidr_block) +def generate_vpc_end_point_id(vpc_id): + return "%s-%s" % ("vpce", vpc_id[4:]) + + +def create_dns_entries(service_name, vpc_endpoint_id): + dns_entries = {} + dns_entries["dns_name"] = "{}-{}.{}".format( + vpc_endpoint_id, random_resource_id(8), service_name + ) + dns_entries["hosted_zone_id"] = random_resource_id(13).upper() + return dns_entries + + def split_route_id(route_id): values = route_id.split("~") return values[0], values[1] diff --git a/moto/core/access_control.py b/moto/iam/access_control.py similarity index 99% rename from moto/core/access_control.py rename to moto/iam/access_control.py index 8ba0c3ba1..bcde25d9e 100644 --- a/moto/core/access_control.py +++ b/moto/iam/access_control.py @@ -25,8 +25,6 @@ from botocore.credentials import Credentials from six import string_types from moto.core import ACCOUNT_ID -from moto.iam.models import Policy -from moto.iam import iam_backend from moto.core.exceptions import ( SignatureDoesNotMatchError, AccessDeniedError, @@ -44,6 +42,7 @@ from moto.s3.exceptions import ( S3SignatureDoesNotMatchError, ) from moto.sts import sts_backend +from .models import iam_backend, Policy log = logging.getLogger(__name__) diff --git a/moto/iam/models.py b/moto/iam/models.py index 08a1eb36a..d3907da26 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -464,7 +464,7 @@ class AccessKey(BaseModel): self.secret_access_key = random_alphanumeric(40) self.status = "Active" self.create_date = datetime.utcnow() - self.last_used = datetime.utcnow() + self.last_used = None @property def created_iso_8601(self): @@ -676,20 +676,50 @@ class User(BaseModel): if len(self.access_keys) == 0: access_key_1_active = "false" access_key_1_last_rotated = "N/A" + access_key_1_last_used = "N/A" access_key_2_active = "false" access_key_2_last_rotated = "N/A" + access_key_2_last_used = "N/A" elif len(self.access_keys) == 1: - access_key_1_active = "true" - access_key_1_last_rotated = date_created.strftime(date_format) + access_key_1_active = ( + "true" if self.access_keys[0].status == "Active" else "false" + ) + access_key_1_last_rotated = self.access_keys[0].create_date.strftime( + date_format + ) + access_key_1_last_used = ( + "N/A" + if self.access_keys[0].last_used is None + else self.access_keys[0].last_used.strftime(date_format) + ) access_key_2_active = "false" access_key_2_last_rotated = "N/A" + access_key_2_last_used = "N/A" else: - access_key_1_active = "true" - access_key_1_last_rotated = date_created.strftime(date_format) - access_key_2_active = "true" - access_key_2_last_rotated = date_created.strftime(date_format) + access_key_1_active = ( + "true" if self.access_keys[0].status == "Active" else "false" + ) + access_key_1_last_rotated = self.access_keys[0].create_date.strftime( + date_format + ) + access_key_1_last_used = ( + "N/A" + if self.access_keys[0].last_used is None + else self.access_keys[0].last_used.strftime(date_format) + ) + access_key_2_active = ( + "true" if self.access_keys[1].status == "Active" else "false" + ) + access_key_2_last_rotated = self.access_keys[1].create_date.strftime( + date_format + ) + access_key_2_last_used = ( + "N/A" + if self.access_keys[1].last_used is None + else self.access_keys[1].last_used.strftime(date_format) + ) - return "{0},{1},{2},{3},{4},{5},not_supported,false,{6},{7},{8},{9},false,N/A,false,N/A".format( + return "{0},{1},{2},{3},{4},{5},not_supported,false,{6},{7},{8},not_supported,not_supported,{9},{10},{11},not_supported,not_supported,false,N/A,false,N/A\n".format( self.name, self.arn, date_created.strftime(date_format), @@ -698,8 +728,10 @@ class User(BaseModel): date_created.strftime(date_format), access_key_1_active, access_key_1_last_rotated, + access_key_1_last_used, access_key_2_active, access_key_2_last_rotated, + access_key_2_last_used, ) @@ -1799,7 +1831,7 @@ class IAMBackend(BaseBackend): def get_credential_report(self): if not self.credential_report: raise IAMReportNotPresentException("Credential report not present") - report = "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_2_active,access_key_2_last_rotated,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\n" + report = "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\n" for user in self.users: report += self.users[user].to_csv() return base64.b64encode(report.encode("ascii")).decode("ascii") diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 12501769e..667a6d13b 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1779,7 +1779,11 @@ GET_ACCESS_KEY_LAST_USED_TEMPLATE = """ {{ user_name }} - {{ last_used }} + {% if last_used %} + {{ last_used }} + {% endif %} + N/A + N/A diff --git a/moto/iot/models.py b/moto/iot/models.py index 51a23b6c6..2e9979bda 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -842,6 +842,17 @@ class IoTBackend(BaseBackend): return thing_group.thing_group_name, thing_group.arn, thing_group.thing_group_id def delete_thing_group(self, thing_group_name, expected_version): + child_groups = [ + thing_group + for _, thing_group in self.thing_groups.items() + if thing_group.parent_group_name == thing_group_name + ] + if len(child_groups) > 0: + raise InvalidRequestException( + " Cannot delete thing group : " + + thing_group_name + + " when there are still child groups attached to it" + ) thing_group = self.describe_thing_group(thing_group_name) del self.thing_groups[thing_group.arn] diff --git a/tests/test_ec2/test_route_tables.py b/tests/test_ec2/test_route_tables.py index 347464691..a64fbae1a 100644 --- a/tests/test_ec2/test_route_tables.py +++ b/tests/test_ec2/test_route_tables.py @@ -618,3 +618,63 @@ def test_describe_route_tables_with_nat_gateway(): nat_gw_routes[0]["DestinationCidrBlock"].should.equal("0.0.0.0/0") nat_gw_routes[0]["NatGatewayId"].should.equal(nat_gw_id) nat_gw_routes[0]["State"].should.equal("active") + + +@mock_ec2 +def test_create_vpc_end_point(): + + ec2 = boto3.client("ec2", region_name="us-west-1") + vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") + subnet = ec2.create_subnet(VpcId=vpc["Vpc"]["VpcId"], CidrBlock="10.0.0.0/24") + + route_table = ec2.create_route_table(VpcId=vpc["Vpc"]["VpcId"]) + + # test without any end point type specified + vpc_end_point = ec2.create_vpc_endpoint( + VpcId=vpc["Vpc"]["VpcId"], + ServiceName="com.amazonaws.us-east-1.s3", + RouteTableIds=[route_table["RouteTable"]["RouteTableId"]], + ) + + vpc_end_point["VpcEndpoint"]["ServiceName"].should.equal( + "com.amazonaws.us-east-1.s3" + ) + vpc_end_point["VpcEndpoint"]["RouteTableIds"][0].should.equal( + route_table["RouteTable"]["RouteTableId"] + ) + vpc_end_point["VpcEndpoint"]["VpcId"].should.equal(vpc["Vpc"]["VpcId"]) + vpc_end_point["VpcEndpoint"]["DnsEntries"].should.have.length_of(0) + + # test with any end point type as gateway + vpc_end_point = ec2.create_vpc_endpoint( + VpcId=vpc["Vpc"]["VpcId"], + ServiceName="com.amazonaws.us-east-1.s3", + RouteTableIds=[route_table["RouteTable"]["RouteTableId"]], + VpcEndpointType="gateway", + ) + + vpc_end_point["VpcEndpoint"]["ServiceName"].should.equal( + "com.amazonaws.us-east-1.s3" + ) + vpc_end_point["VpcEndpoint"]["RouteTableIds"][0].should.equal( + route_table["RouteTable"]["RouteTableId"] + ) + vpc_end_point["VpcEndpoint"]["VpcId"].should.equal(vpc["Vpc"]["VpcId"]) + vpc_end_point["VpcEndpoint"]["DnsEntries"].should.have.length_of(0) + + # test with end point type as interface + vpc_end_point = ec2.create_vpc_endpoint( + VpcId=vpc["Vpc"]["VpcId"], + ServiceName="com.amazonaws.us-east-1.s3", + SubnetIds=[subnet["Subnet"]["SubnetId"]], + VpcEndpointType="interface", + ) + + vpc_end_point["VpcEndpoint"]["ServiceName"].should.equal( + "com.amazonaws.us-east-1.s3" + ) + vpc_end_point["VpcEndpoint"]["SubnetIds"][0].should.equal( + subnet["Subnet"]["SubnetId"] + ) + vpc_end_point["VpcEndpoint"]["VpcId"].should.equal(vpc["Vpc"]["VpcId"]) + len(vpc_end_point["VpcEndpoint"]["DnsEntries"]).should.be.greater_than(0) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 995895437..6792d8f52 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -4,6 +4,7 @@ import json import boto import boto3 +import csv import os import sure # noqa import sys @@ -11,9 +12,10 @@ from boto.exception import BotoServerError from botocore.exceptions import ClientError from dateutil.tz import tzutc -from moto import mock_iam, mock_iam_deprecated -from moto.iam.models import aws_managed_policies +from moto import mock_iam, mock_iam_deprecated, settings from moto.core import ACCOUNT_ID +from moto.iam.models import aws_managed_policies +from moto.backends import get_backend from nose.tools import assert_raises, assert_equals from nose.tools import raises @@ -1215,6 +1217,69 @@ def test_boto3_get_credential_report(): report.should.match(r".*my-user.*") +@mock_iam +def test_boto3_get_credential_report_content(): + conn = boto3.client("iam", region_name="us-east-1") + username = "my-user" + conn.create_user(UserName=username) + key1 = conn.create_access_key(UserName=username)["AccessKey"] + conn.update_access_key( + UserName=username, AccessKeyId=key1["AccessKeyId"], Status="Inactive" + ) + key1 = conn.create_access_key(UserName=username)["AccessKey"] + timestamp = datetime.utcnow() + if not settings.TEST_SERVER_MODE: + iam_backend = get_backend("iam")["global"] + iam_backend.users[username].access_keys[1].last_used = timestamp + with assert_raises(ClientError): + conn.get_credential_report() + result = conn.generate_credential_report() + while result["State"] != "COMPLETE": + result = conn.generate_credential_report() + result = conn.get_credential_report() + report = result["Content"].decode("utf-8") + header = report.split("\n")[0] + header.should.equal( + "user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_1_last_used_date,access_key_1_last_used_region,access_key_1_last_used_service,access_key_2_active,access_key_2_last_rotated,access_key_2_last_used_date,access_key_2_last_used_region,access_key_2_last_used_service,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated" + ) + report_dict = csv.DictReader(report.split("\n")) + user = next(report_dict) + user["user"].should.equal("my-user") + user["access_key_1_active"].should.equal("false") + user["access_key_1_last_rotated"].should.match(timestamp.strftime("%Y-%m-%d")) + user["access_key_1_last_used_date"].should.equal("N/A") + user["access_key_2_active"].should.equal("true") + if not settings.TEST_SERVER_MODE: + user["access_key_2_last_used_date"].should.match(timestamp.strftime("%Y-%m-%d")) + else: + user["access_key_2_last_used_date"].should.equal("N/A") + + +@mock_iam +def test_get_access_key_last_used_when_used(): + iam = boto3.resource("iam", region_name="us-east-1") + client = iam.meta.client + username = "test-user" + iam.create_user(UserName=username) + with assert_raises(ClientError): + client.get_access_key_last_used(AccessKeyId="non-existent-key-id") + create_key_response = client.create_access_key(UserName=username)["AccessKey"] + # Set last used date using the IAM backend. Moto currently does not have a mechanism for tracking usage of access keys + if not settings.TEST_SERVER_MODE: + timestamp = datetime.utcnow() + iam_backend = get_backend("iam")["global"] + iam_backend.users[username].access_keys[0].last_used = timestamp + resp = client.get_access_key_last_used( + AccessKeyId=create_key_response["AccessKeyId"] + ) + if not settings.TEST_SERVER_MODE: + datetime.strftime( + resp["AccessKeyLastUsed"]["LastUsedDate"], "%Y-%m-%d" + ).should.equal(timestamp.strftime("%Y-%m-%d")) + else: + resp["AccessKeyLastUsed"].should_not.contain("LastUsedDate") + + @requires_boto_gte("2.39") @mock_iam_deprecated() def test_managed_policy(): @@ -1382,7 +1447,7 @@ def test_update_access_key(): @mock_iam -def test_get_access_key_last_used(): +def test_get_access_key_last_used_when_unused(): iam = boto3.resource("iam", region_name="us-east-1") client = iam.meta.client username = "test-user" @@ -1393,10 +1458,7 @@ def test_get_access_key_last_used(): resp = client.get_access_key_last_used( AccessKeyId=create_key_response["AccessKeyId"] ) - - datetime.strftime( - resp["AccessKeyLastUsed"]["LastUsedDate"], "%Y-%m-%d" - ).should.equal(datetime.strftime(datetime.utcnow(), "%Y-%m-%d")) + resp["AccessKeyLastUsed"].should_not.contain("LastUsedDate") resp["UserName"].should.equal(create_key_response["UserName"]) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 2f43de5b9..58a820fee 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -756,6 +756,47 @@ def test_delete_principal_thing(): client.delete_certificate(certificateId=cert_id) +@mock_iot +def test_delete_thing_group(): + client = boto3.client("iot", region_name="ap-northeast-1") + group_name_1a = "my-group-name-1a" + group_name_2a = "my-group-name-2a" + # --1a + # |--2a + + # create thing groups tree + # 1 + thing_group1a = client.create_thing_group(thingGroupName=group_name_1a) + thing_group1a.should.have.key("thingGroupName").which.should.equal(group_name_1a) + thing_group1a.should.have.key("thingGroupArn") + # 2 + thing_group2a = client.create_thing_group( + thingGroupName=group_name_2a, parentGroupName=group_name_1a + ) + thing_group2a.should.have.key("thingGroupName").which.should.equal(group_name_2a) + thing_group2a.should.have.key("thingGroupArn") + + # delete group with child + try: + client.delete_thing_group(thingGroupName=group_name_1a) + except client.exceptions.InvalidRequestException as exc: + error_code = exc.response["Error"]["Code"] + error_code.should.equal("InvalidRequestException") + else: + raise Exception("Should have raised error") + + # delete child group + client.delete_thing_group(thingGroupName=group_name_2a) + res = client.list_thing_groups() + res.should.have.key("thingGroups").which.should.have.length_of(1) + res["thingGroups"].should_not.have.key(group_name_2a) + + # now that there is no child group, we can delete the previus group safely + client.delete_thing_group(thingGroupName=group_name_1a) + res = client.list_thing_groups() + res.should.have.key("thingGroups").which.should.have.length_of(0) + + @mock_iot def test_describe_thing_group_metadata_hierarchy(): client = boto3.client("iot", region_name="ap-northeast-1")