From 8526013e6173e35874eb7606609817d4e4110730 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 5 Oct 2021 17:11:07 +0000 Subject: [PATCH] Parallelize tests - Part 1 (#4368) --- .github/workflows/build.yml | 26 +- CONTRIBUTING.md | 4 + Makefile | 27 +- moto/core/models.py | 6 +- moto/ec2/models.py | 107 +++-- moto/ec2/responses/customer_gateways.py | 5 +- moto/ec2/responses/spot_instances.py | 5 +- .../responses/transit_gateway_attachments.py | 8 + moto/ec2/responses/transit_gateways.py | 5 +- moto/ec2/responses/vpcs.py | 2 +- moto/sqs/models.py | 3 + requirements-tests.txt | 1 + tests/test_batch/__init__.py | 10 +- tests/test_batch/test_batch_cloudformation.py | 30 +- tests/test_batch/test_batch_compute_envs.py | 109 +++-- tests/test_batch/test_batch_job_queue.py | 59 +-- tests/test_batch/test_batch_jobs.py | 65 ++- .../test_batch/test_batch_task_definition.py | 56 ++- tests/test_ec2/test_amis.py | 199 +++++--- tests/test_ec2/test_carrier_gateways.py | 16 +- tests/test_ec2/test_customer_gateways.py | 31 +- tests/test_ec2/test_dhcp_options.py | 109 +++-- tests/test_ec2/test_ec2_cloudformation.py | 262 ++++++----- tests/test_ec2/test_ec2_integration.py | 9 +- tests/test_ec2/test_egress_only_igw.py | 14 +- tests/test_ec2/test_elastic_block_store.py | 143 ++++-- tests/test_ec2/test_elastic_ip_addresses.py | 80 +++- .../test_elastic_network_interfaces.py | 123 +++-- tests/test_ec2/test_flow_logs.py | 197 ++++---- .../test_ec2/test_flow_logs_cloudformation.py | 41 +- tests/test_ec2/test_iam_integration.py | 110 ++--- tests/test_ec2/test_instances.py | 426 +++++++++++------- tests/test_ec2/test_internet_gateways.py | 52 ++- tests/test_ec2/test_key_pairs.py | 87 ++-- tests/test_ec2/test_launch_templates.py | 131 +++--- tests/test_ec2/test_nat_gateway.py | 73 +-- tests/test_ec2/test_network_acls.py | 53 ++- tests/test_ec2/test_prefix_lists.py | 17 +- tests/test_ec2/test_regions.py | 51 ++- tests/test_ec2/test_route_tables.py | 27 +- tests/test_ec2/test_security_groups.py | 293 +++++++----- .../test_security_groups_cloudformation.py | 59 ++- tests/test_ec2/test_spot_instances.py | 77 +++- tests/test_ec2/test_subnets.py | 123 +++-- tests/test_ec2/test_tags.py | 124 +++-- tests/test_ec2/test_transit_gateway.py | 157 +++++-- .../test_transit_gateway_cloudformation.py | 31 +- ...est_transit_gateway_peering_attachments.py | 62 ++- .../test_ec2/test_virtual_private_gateways.py | 63 ++- tests/test_ec2/test_vpc_endpoint_services.py | 7 +- tests/test_ec2/test_vpc_peering.py | 57 ++- tests/test_ec2/test_vpcs.py | 287 +++++++----- tests/test_ec2/test_vpn_connections.py | 39 +- tests/test_sqs/test_sqs.py | 354 ++++++++------- tests/test_sqs/test_sqs_cloudformation.py | 110 +++-- travis_moto_server.sh | 2 +- 56 files changed, 2872 insertions(+), 1752 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2877a35ea..06c0f5142 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -143,7 +143,7 @@ jobs: pip install "coverage<=4.5.4" - name: Test with pytest run: | - make test-coverage + make test-only - name: "Upload coverage to Codecov" if: ${{ github.repository == 'spulec/moto'}} uses: codecov/codecov-action@v1 @@ -192,13 +192,28 @@ jobs: env: TEST_SERVER_MODE: ${{ true }} run: | - make test-coverage + make test-only - name: "Upload coverage to Codecov" if: ${{ github.repository == 'spulec/moto'}} uses: codecov/codecov-action@v1 with: fail_ci_if_error: false flags: servertests + - name: "Stop MotoServer" + if: always() + run: | + mkdir serverlogs + pwd + ls -la + cp server_output.log serverlogs/server_output.log + docker stop motoserver + - name: Archive TF logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: motoserver-${{ matrix.python-version }} + path: | + serverlogs/* test_responses: name: Test Responses==0.12.0 @@ -283,11 +298,14 @@ jobs: # So we simply split the list of tests, and ask our CI for separate VM's to run them in parallel - name: Get list of tests run: | - split -n l/3 tests/terraform-tests.success.txt tf-split- + cd moto-terraform-tests + bin/list-tests -i ../tests/terraform-tests.success.txt -e ../tests/terraform-tests.failures.txt > tftestlist.txt + split -n l/3 tftestlist.txt tf-split- + cd .. - name: Run Terraform Tests run: | cd moto-terraform-tests - AWS_DEFAULT_REGION=us-east-1 AWS_ALTERNATE_REGION=eu-west-1 bin/run-tests -t -i ../tf-split-${{ matrix.part }} -e ../tests/terraform-tests.failures.txt + AWS_DEFAULT_REGION=us-east-1 AWS_ALTERNATE_REGION=eu-west-1 bin/run-tests -t -i tf-split-${{ matrix.part }} -e ../tests/terraform-tests.failures.txt cd .. - name: "Create report" run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13f77a382..4e62ce8e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,10 @@ TEST_SERVER_MODE=true pytest -sv tests/test_service/.. import warnings warnings.warn("The Filters-parameter is not yet implemented for client.method()") ``` + - To speed up our CI, the ServerMode tests for the `awslambda`, `batch`, `ec2` and `sqs` services will run in parallel. + This means the following: + - Make sure you use unique names for functions/queues/etc + - Calls to `describe_reservations()`/`list_queues()`/etc might return unexpected results # Missing features diff --git a/Makefile b/Makefile index e0fd4e6c0..7804f7c4f 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,12 @@ SHELL := /bin/bash ifeq ($(TEST_SERVER_MODE), true) # exclude test_kinesisvideoarchivedmedia # because testing with moto_server is difficult with data-endpoint - TEST_EXCLUDE := -k 'not test_kinesisvideoarchivedmedia' + TEST_EXCLUDE := -k 'not (test_kinesisvideoarchivedmedia or test_awslambda or test_batch or test_ec2 or test_sqs)' + # Parallel tests will be run separate + PARALLEL_TESTS := ./tests/test_awslambda ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs else TEST_EXCLUDE := + PARALLEL_TESTS := ./tests/test_core endif init: @@ -21,14 +24,10 @@ format: black moto/ tests/ test-only: - rm -f .coverage - rm -rf cover - pytest -sv ./tests/ $(TEST_EXCLUDE) - -test-coverage: rm -f .coverage rm -rf cover pytest -sv --cov=moto --cov-report xml ./tests/ $(TEST_EXCLUDE) + MOTO_CALL_RESET_API=false pytest -n 4 $(PARALLEL_TESTS) test: lint test-only @@ -38,22 +37,6 @@ test_server: aws_managed_policies: scripts/update_managed_policies.py -upload_pypi_artifact: - python setup.py sdist bdist_wheel - twine upload dist/* - -push_dockerhub_image: - docker build -t motoserver/moto . --tag moto:`python setup.py --version` - docker push motoserver/moto - -tag_github_release: - git tag `python setup.py --version` - git push origin `python setup.py --version` - -publish: upload_pypi_artifact \ - tag_github_release \ - push_dockerhub_image - implementation_coverage: ./scripts/implementation_coverage.py git commit IMPLEMENTATION_COVERAGE.md -m "Updating implementation coverage" || true diff --git a/moto/core/models.py b/moto/core/models.py index 2482f4702..307f738ca 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -481,9 +481,11 @@ MockAWS = BotocoreEventMockAWS class ServerModeMockAWS(BaseMockAWS): def reset(self): - import requests + call_reset_api = os.environ.get("MOTO_CALL_RESET_API") + if not call_reset_api or call_reset_api.lower() != "false": + import requests - requests.post("http://localhost:5000/moto-api/reset") + requests.post("http://localhost:5000/moto-api/reset") def enable_patching(self): if self.__class__.nested_count == 1: diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 6bbc12799..c2952f875 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -358,7 +358,9 @@ class NetworkInterface(TaggedEC2Resource, CloudFormationModel): def association(self): association = {} if self.public_ip: - eips = self.ec2_backend.address_by_ip([self.public_ip]) + eips = self.ec2_backend.address_by_ip( + [self.public_ip], fail_if_not_found=False + ) eip = eips[0] if len(eips) > 0 else None if eip: association["allocationId"] = eip.allocation_id or None @@ -538,7 +540,7 @@ class NetworkInterfaceBackend(object): eni.source_dest_check = source_dest_check def get_all_network_interfaces(self, eni_ids=None, filters=None): - enis = self.enis.values() + enis = self.enis.copy().values() if eni_ids: enis = [eni for eni in enis if eni.id in eni_ids] @@ -1234,7 +1236,7 @@ class InstanceBackend(object): def all_reservations(self, filters=None): reservations = [ - copy.copy(reservation) for reservation in self.reservations.values() + copy.copy(reservation) for reservation in self.reservations.copy().values() ] if filters is not None: reservations = filter_reservations(reservations, filters) @@ -1441,7 +1443,7 @@ class TagBackend(object): value_filters.append( re.compile(simple_aws_filter_to_re(value)) ) - for resource_id, tags in self.tags.items(): + for resource_id, tags in self.tags.copy().items(): for key, value in tags.items(): add_result = False if filters is None: @@ -1486,9 +1488,9 @@ class TagBackend(object): "resource_id": resource_id, "key": key, "value": value, - "resource_type": EC2_PREFIX_TO_RESOURCE[ - get_prefix(resource_id) - ], + "resource_type": EC2_PREFIX_TO_RESOURCE.get( + get_prefix(resource_id), "" + ), } results.append(result) return results @@ -1677,7 +1679,7 @@ class AmiBackend(object): def describe_images( self, ami_ids=(), filters=None, exec_users=None, owners=None, context=None ): - images = self.amis.values() + images = self.amis.copy().values() if len(ami_ids): # boto3 seems to default to just searching based on ami ids if that parameter is passed @@ -2536,7 +2538,8 @@ class SecurityGroupBackend(object): return group def describe_security_groups(self, group_ids=None, groupnames=None, filters=None): - matches = itertools.chain(*[x.values() for x in self.groups.values()]) + all_groups = self.groups.copy() + matches = itertools.chain(*[x.copy().values() for x in all_groups.values()]) if group_ids: matches = [grp for grp in matches if grp.id in group_ids] if len(group_ids) > len(matches): @@ -2578,7 +2581,7 @@ class SecurityGroupBackend(object): def get_security_group_from_id(self, group_id): # 2 levels of chaining necessary since it's a complex structure all_groups = itertools.chain.from_iterable( - [x.values() for x in self.groups.values()] + [x.copy().values() for x in self.groups.copy().values()] ) for group in all_groups: if group.id == group_id: @@ -3273,7 +3276,7 @@ class Volume(TaggedEC2Resource, CloudFormationModel): elif filter_name == "encrypted": return str(self.encrypted).lower() elif filter_name == "availability-zone": - return self.zone.name + return self.zone.name if self.zone else None else: return super().get_filter_value(filter_name, "DescribeVolumes") @@ -3347,7 +3350,7 @@ class EBSBackend(object): return volume def describe_volumes(self, volume_ids=None, filters=None): - matches = self.volumes.values() + matches = self.volumes.copy().values() if volume_ids: matches = [vol for vol in matches if vol.id in volume_ids] if len(volume_ids) > len(matches): @@ -3422,7 +3425,7 @@ class EBSBackend(object): return snapshot def describe_snapshots(self, snapshot_ids=None, filters=None): - matches = self.snapshots.values() + matches = self.snapshots.copy().values() if snapshot_ids: matches = [snap for snap in matches if snap.id in snapshot_ids] if len(snapshot_ids) > len(matches): @@ -3776,7 +3779,7 @@ class VPCBackend(object): return match_vpc def describe_vpcs(self, vpc_ids=None, filters=None): - matches = self.vpcs.values() + matches = self.vpcs.copy().values() if vpc_ids: matches = [vpc for vpc in matches if vpc.id in vpc_ids] if len(vpc_ids) > len(matches): @@ -3859,9 +3862,9 @@ class VPCBackend(object): raise InvalidParameterValueError(attr_name) def disassociate_vpc_cidr_block(self, association_id): - for vpc in self.vpcs.values(): + for vpc in self.vpcs.copy().values(): response = vpc.disassociate_vpc_cidr_block(association_id) - for route_table in self.route_tables.values(): + for route_table in self.route_tables.copy().values(): if route_table.vpc_id == response.get("vpc_id"): if "::/" in response.get("cidr_block"): self.delete_route( @@ -4277,9 +4280,10 @@ class VPCPeeringConnectionBackend(object): return vpc_pcx def describe_vpc_peering_connections(self, vpc_peering_ids=None): + all_pcxs = self.vpc_pcxs.copy().values() if vpc_peering_ids: - return [pcx for pcx in self.vpc_pcxs.values() if pcx.id in vpc_peering_ids] - return self.vpc_pcxs.values() + return [pcx for pcx in all_pcxs if pcx.id in vpc_peering_ids] + return all_pcxs def get_vpc_peering_connection(self, vpc_pcx_id): if vpc_pcx_id not in self.vpc_pcxs: @@ -4346,9 +4350,7 @@ class Subnet(TaggedEC2Resource, CloudFormationModel): self.vpc_id = vpc_id self.cidr_block = cidr_block self.cidr = ipaddress.IPv4Network(str(self.cidr_block), strict=False) - self._available_ip_addresses = ( - ipaddress.IPv4Network(str(self.cidr_block)).num_addresses - 5 - ) + self._available_ip_addresses = self.cidr.num_addresses - 5 self._availability_zone = availability_zone self.default_for_az = default_for_az self.map_public_ip_on_launch = map_public_ip_on_launch @@ -5111,7 +5113,7 @@ class RouteTableBackend(object): return route_table def describe_route_tables(self, route_table_ids=None, filters=None): - route_tables = self.route_tables.values() + route_tables = self.route_tables.copy().values() if route_table_ids: route_tables = [ @@ -5380,7 +5382,7 @@ class ManagedPrefixListBackend(object): return managed_prefix_list def describe_managed_prefix_lists(self, prefix_list_ids=None, filters=None): - managed_prefix_lists = list(self.managed_prefix_lists.values()) + managed_prefix_lists = list(self.managed_prefix_lists.copy().values()) attr_pairs = ( ("owner-id", "owner_id"), ("prefix-list-id", "id"), @@ -6036,8 +6038,11 @@ class SpotRequestBackend(object, metaclass=Model): return requests @Model.prop("SpotInstanceRequest") - def describe_spot_instance_requests(self, filters=None): - requests = self.spot_instance_requests.values() + def describe_spot_instance_requests(self, filters=None, spot_instance_ids=[]): + requests = self.spot_instance_requests.copy().values() + + if spot_instance_ids: + requests = [i for i in requests if i.id in spot_instance_ids] return generic_filter(filters, requests) @@ -6447,12 +6452,18 @@ class ElasticAddress(TaggedEC2Resource, CloudFormationModel): return self.association_id elif filter_name == "domain": return self.domain - elif filter_name == "instance-id" and self.instance: - return self.instance.id - elif filter_name == "network-interface-id" and self.eni: - return self.eni.id - elif filter_name == "private-ip-address" and self.eni: - return self.eni.private_ip_address + elif filter_name == "instance-id": + if self.instance: + return self.instance.id + return None + elif filter_name == "network-interface-id": + if self.eni: + return self.eni.id + return None + elif filter_name == "private-ip-address": + if self.eni: + return self.eni.private_ip_address + return None elif filter_name == "public-ip": return self.public_ip elif filter_name == "network-interface-owner-id": @@ -6477,11 +6488,13 @@ class ElasticAddressBackend(object): self.addresses.append(address) return address - def address_by_ip(self, ips): - eips = [address for address in self.addresses if address.public_ip in ips] + def address_by_ip(self, ips, fail_if_not_found=True): + eips = [ + address for address in self.addresses.copy() if address.public_ip in ips + ] # TODO: Trim error message down to specific invalid address. - if not eips or len(ips) > len(eips): + if (not eips or len(ips) > len(eips)) and fail_if_not_found: raise InvalidAddressError(ips) return eips @@ -6548,7 +6561,7 @@ class ElasticAddressBackend(object): raise ResourceAlreadyAssociatedError(eip.public_ip) def describe_addresses(self, allocation_ids=None, public_ips=None, filters=None): - matches = self.addresses + matches = self.addresses.copy() if allocation_ids: matches = [addr for addr in matches if addr.allocation_id in allocation_ids] if len(allocation_ids) > len(matches): @@ -6557,7 +6570,7 @@ class ElasticAddressBackend(object): if public_ips: matches = [addr for addr in matches if addr.public_ip in public_ips] if len(public_ips) > len(matches): - unknown_ips = set(allocation_ids) - set(matches) + unknown_ips = set(public_ips) - set(matches) raise InvalidAddressError(unknown_ips) if filters: matches = generic_filter(filters, matches) @@ -6690,7 +6703,7 @@ class DHCPOptionsSetBackend(object): options_sets.append(self.dhcp_options_sets[option_id]) else: raise InvalidDHCPOptionsIdError(option_id) - return options_sets or self.dhcp_options_sets.values() + return options_sets or self.dhcp_options_sets.copy().values() def delete_dhcp_options_set(self, options_id): if not (options_id and options_id.startswith("dopt-")): @@ -6988,7 +7001,7 @@ class NetworkAclBackend(object): ) def describe_network_acls(self, network_acl_ids=None, filters=None): - network_acls = self.network_acls.values() + network_acls = self.network_acls.copy().values() if network_acl_ids: network_acls = [ @@ -7199,8 +7212,13 @@ class CustomerGatewayBackend(object): self.customer_gateways[customer_gateway_id] = customer_gateway return customer_gateway - def get_all_customer_gateways(self, filters=None): - customer_gateways = self.customer_gateways.values() + def get_all_customer_gateways(self, filters=None, customer_gateway_ids=None): + customer_gateways = self.customer_gateways.copy().values() + if customer_gateway_ids: + customer_gateways = [ + cg for cg in customer_gateways if cg.id in customer_gateway_ids + ] + if filters is not None: if filters.get("customer-gateway-id") is not None: customer_gateways = [ @@ -7324,8 +7342,13 @@ class TransitGatewayBackend(object): self.transit_gateways[transit_gateway.id] = transit_gateway return transit_gateway - def describe_transit_gateways(self, filters): - transit_gateways = list(self.transit_gateways.values()) + def describe_transit_gateways(self, filters, transit_gateway_ids): + transit_gateways = list(self.transit_gateways.copy().values()) + + if transit_gateway_ids: + transit_gateways = [ + item for item in transit_gateways if item.id in transit_gateway_ids + ] attr_pairs = ( ("transit-gateway-id", "id"), diff --git a/moto/ec2/responses/customer_gateways.py b/moto/ec2/responses/customer_gateways.py index f2b2ea1f2..0192a2ccf 100644 --- a/moto/ec2/responses/customer_gateways.py +++ b/moto/ec2/responses/customer_gateways.py @@ -27,7 +27,10 @@ class CustomerGateways(BaseResponse): def describe_customer_gateways(self): filters = filters_from_querystring(self.querystring) - customer_gateways = self.ec2_backend.get_all_customer_gateways(filters) + customer_gateway_ids = self._get_multi_param("CustomerGatewayId") + customer_gateways = self.ec2_backend.get_all_customer_gateways( + filters, customer_gateway_ids + ) template = self.response_template(DESCRIBE_CUSTOMER_GATEWAYS_RESPONSE) return template.render(customer_gateways=customer_gateways) diff --git a/moto/ec2/responses/spot_instances.py b/moto/ec2/responses/spot_instances.py index 7bb286667..59f4da327 100644 --- a/moto/ec2/responses/spot_instances.py +++ b/moto/ec2/responses/spot_instances.py @@ -29,8 +29,11 @@ class SpotInstances(BaseResponse): ) def describe_spot_instance_requests(self): + spot_instance_ids = self._get_multi_param("SpotInstanceRequestId") filters = filters_from_querystring(self.querystring) - requests = self.ec2_backend.describe_spot_instance_requests(filters=filters) + requests = self.ec2_backend.describe_spot_instance_requests( + filters=filters, spot_instance_ids=spot_instance_ids + ) template = self.response_template(DESCRIBE_SPOT_INSTANCES_TEMPLATE) return template.render(requests=requests) diff --git a/moto/ec2/responses/transit_gateway_attachments.py b/moto/ec2/responses/transit_gateway_attachments.py index 106a568df..5d6b201fd 100644 --- a/moto/ec2/responses/transit_gateway_attachments.py +++ b/moto/ec2/responses/transit_gateway_attachments.py @@ -256,11 +256,13 @@ DESCRIBE_TRANSIT_GATEWAY_VPC_ATTACHMENTS = """ 2021-07-18T08:57:21.000Z + {% if transit_gateway_vpc_attachment.options %} {{ transit_gateway_vpc_attachment.options.ApplianceModeSupport }} {{ transit_gateway_vpc_attachment.options.DnsSupport }} {{ transit_gateway_vpc_attachment.options.Ipv6Support }} + {% endif %} {{ transit_gateway_vpc_attachment.state }} {% for id in transit_gateway_vpc_attachment.subnet_ids %} @@ -423,17 +425,23 @@ DESCRIBE_TRANSIT_GATEWAY_PEERING_ATTACHMENTS = """ {{ transit_gateway_peering_attachment.create_time }} {{ transit_gateway_peering_attachment.state }} + {% if transit_gateway_peering_attachment.accepter_tgw_info %} {{ transit_gateway_peering_attachment.accepter_tgw_info.ownerId or '' }} {{ transit_gateway_peering_attachment.accepter_tgw_info.region or '' }} {{ transit_gateway_peering_attachment.accepter_tgw_info.transitGatewayId or '' }} + {% endif %} + {% if transit_gateway_peering_attachment.requester_tgw_info %} {{ transit_gateway_peering_attachment.requester_tgw_info.ownerId or '' }} {{ transit_gateway_peering_attachment.requester_tgw_info.region or '' }} {{ transit_gateway_peering_attachment.requester_tgw_info.transitGatewayId or '' }} + {% endif %} + {% if transit_gateway_peering_attachment.status %} {{ transit_gateway_peering_attachment.status.code }} + {% endif %} {% for tag in transit_gateway_peering_attachment.get_tags() %} diff --git a/moto/ec2/responses/transit_gateways.py b/moto/ec2/responses/transit_gateways.py index fa3449bd5..c8b013b1f 100644 --- a/moto/ec2/responses/transit_gateways.py +++ b/moto/ec2/responses/transit_gateways.py @@ -39,8 +39,11 @@ class TransitGateways(BaseResponse): return template.render(transit_gateway=transit_gateway) def describe_transit_gateways(self): + transit_gateway_ids = self._get_multi_param("TransitGatewayIds") filters = filters_from_querystring(self.querystring) - transit_gateways = self.ec2_backend.describe_transit_gateways(filters) + transit_gateways = self.ec2_backend.describe_transit_gateways( + filters, transit_gateway_ids + ) template = self.response_template(DESCRIBE_TRANSIT_GATEWAY_RESPONSE) return template.render(transit_gateways=transit_gateways) diff --git a/moto/ec2/responses/vpcs.py b/moto/ec2/responses/vpcs.py index ccbf63676..a0eac2cc0 100644 --- a/moto/ec2/responses/vpcs.py +++ b/moto/ec2/responses/vpcs.py @@ -826,7 +826,7 @@ DESCRIBE_PREFIX_LIST = """ {% for entry in pl.entries.1 %} diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 6c74f1ece..56a2988dd 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -402,6 +402,9 @@ class Queue(CloudFormationModel): tags = properties.pop("Tags", []) tags_dict = tags_from_cloudformation_tags_list(tags) + # Could be passed as an integer - just treat it as a string + resource_name = str(resource_name) + sqs_backend = sqs_backends[region_name] return sqs_backend.create_queue( name=resource_name, tags=tags_dict, region=region_name, **properties diff --git a/requirements-tests.txt b/requirements-tests.txt index 1efb50046..0496fb9fd 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,5 +1,6 @@ pytest pytest-cov +pytest-xdist sure freezegun pylint diff --git a/tests/test_batch/__init__.py b/tests/test_batch/__init__.py index 52713db50..b3320fd02 100644 --- a/tests/test_batch/__init__.py +++ b/tests/test_batch/__init__.py @@ -1,4 +1,5 @@ import boto3 +from uuid import uuid4 DEFAULT_REGION = "eu-central-1" @@ -27,17 +28,18 @@ def _setup(ec2_client, iam_client): ) subnet_id = resp["Subnet"]["SubnetId"] resp = ec2_client.create_security_group( - Description="test_sg_desc", GroupName="test_sg", VpcId=vpc_id + Description="test_sg_desc", GroupName=str(uuid4())[0:6], VpcId=vpc_id ) sg_id = resp["GroupId"] + role_name = f"{str(uuid4())[0:6]}" resp = iam_client.create_role( - RoleName="TestRole", AssumeRolePolicyDocument="some_policy" + RoleName=role_name, AssumeRolePolicyDocument="some_policy" ) iam_arn = resp["Role"]["Arn"] - iam_client.create_instance_profile(InstanceProfileName="TestRole") + iam_client.create_instance_profile(InstanceProfileName=role_name) iam_client.add_role_to_instance_profile( - InstanceProfileName="TestRole", RoleName="TestRole" + InstanceProfileName=role_name, RoleName=role_name ) return vpc_id, subnet_id, sg_id, iam_arn diff --git a/tests/test_batch/test_batch_cloudformation.py b/tests/test_batch/test_batch_cloudformation.py index 7935f3fe9..c6a35ec53 100644 --- a/tests/test_batch/test_batch_cloudformation.py +++ b/tests/test_batch/test_batch_cloudformation.py @@ -1,20 +1,16 @@ from __future__ import unicode_literals -import time -import datetime import boto3 -from botocore.exceptions import ClientError import sure # noqa from moto import ( mock_batch, mock_iam, mock_ec2, mock_ecs, - mock_logs, mock_cloudformation, ) -import functools import json +from uuid import uuid4 DEFAULT_REGION = "eu-central-1" @@ -42,17 +38,18 @@ def _setup(ec2_client, iam_client): ) subnet_id = resp["Subnet"]["SubnetId"] resp = ec2_client.create_security_group( - Description="test_sg_desc", GroupName="test_sg", VpcId=vpc_id + Description="test_sg_desc", GroupName=str(uuid4())[0:6], VpcId=vpc_id ) sg_id = resp["GroupId"] + role_name = str(uuid4())[0:6] resp = iam_client.create_role( - RoleName="TestRole", AssumeRolePolicyDocument="some_policy" + RoleName=role_name, AssumeRolePolicyDocument="some_policy" ) iam_arn = resp["Role"]["Arn"] - iam_client.create_instance_profile(InstanceProfileName="TestRole") + iam_client.create_instance_profile(InstanceProfileName=role_name) iam_client.add_role_to_instance_profile( - InstanceProfileName="TestRole", RoleName="TestRole" + InstanceProfileName=role_name, RoleName=role_name ) return vpc_id, subnet_id, sg_id, iam_arn @@ -91,7 +88,8 @@ def test_create_env_cf(): cf_json = json.dumps(create_environment_template) cf_conn = boto3.client("cloudformation", DEFAULT_REGION) - stack_id = cf_conn.create_stack(StackName="test_stack", TemplateBody=cf_json)[ + stack_name = str(uuid4())[0:6] + stack_id = cf_conn.create_stack(StackName=stack_name, TemplateBody=cf_json)[ "StackId" ] @@ -105,7 +103,7 @@ def test_create_env_cf(): "arn:aws:batch:" ) stack_resources["StackResourceSummaries"][0]["PhysicalResourceId"].should.contain( - "test_stack" + stack_name ) @@ -154,7 +152,8 @@ def test_create_job_queue_cf(): cf_json = json.dumps(create_environment_template) cf_conn = boto3.client("cloudformation", DEFAULT_REGION) - stack_id = cf_conn.create_stack(StackName="test_stack", TemplateBody=cf_json)[ + stack_name = str(uuid4())[0:6] + stack_id = cf_conn.create_stack(StackName=stack_name, TemplateBody=cf_json)[ "StackId" ] @@ -171,7 +170,7 @@ def test_create_job_queue_cf(): job_queue_resource["ResourceStatus"].should.equal("CREATE_COMPLETE") # Spot checks on the ARN job_queue_resource["PhysicalResourceId"].startswith("arn:aws:batch:") - job_queue_resource["PhysicalResourceId"].should.contain("test_stack") + job_queue_resource["PhysicalResourceId"].should.contain(stack_name) job_queue_resource["PhysicalResourceId"].should.contain("job-queue/") @@ -243,7 +242,8 @@ def test_create_job_def_cf(): cf_json = json.dumps(create_environment_template) cf_conn = boto3.client("cloudformation", DEFAULT_REGION) - stack_id = cf_conn.create_stack(StackName="test_stack", TemplateBody=cf_json)[ + stack_name = str(uuid4())[0:6] + stack_id = cf_conn.create_stack(StackName=stack_name, TemplateBody=cf_json)[ "StackId" ] @@ -260,7 +260,7 @@ def test_create_job_def_cf(): job_def_resource["ResourceStatus"].should.equal("CREATE_COMPLETE") # Spot checks on the ARN job_def_resource["PhysicalResourceId"].startswith("arn:aws:batch:") - job_def_resource["PhysicalResourceId"].should.contain("test_stack-JobDef") + job_def_resource["PhysicalResourceId"].should.contain(f"{stack_name}-JobDef") job_def_resource["PhysicalResourceId"].should.contain("job-definition/") # Test the linux parameter device host path diff --git a/tests/test_batch/test_batch_compute_envs.py b/tests/test_batch/test_batch_compute_envs.py index 2e577b89f..77a574f86 100644 --- a/tests/test_batch/test_batch_compute_envs.py +++ b/tests/test_batch/test_batch_compute_envs.py @@ -2,7 +2,8 @@ from __future__ import unicode_literals from . import _get_clients, _setup import sure # noqa -from moto import mock_batch, mock_iam, mock_ec2, mock_ecs, mock_logs +from moto import mock_batch, mock_iam, mock_ec2, mock_ecs, settings +from uuid import uuid4 # Yes, yes it talks to all the things @@ -14,7 +15,7 @@ def test_create_managed_compute_environment(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="MANAGED", @@ -39,15 +40,20 @@ def test_create_managed_compute_environment(): resp.should.contain("computeEnvironmentArn") resp["computeEnvironmentName"].should.equal(compute_name) + our_env = batch_client.describe_compute_environments( + computeEnvironments=[compute_name] + )["computeEnvironments"][0] + # Given a t2.medium is 2 vcpu and t2.small is 1, therefore 2 mediums and 1 small should be created - resp = ec2_client.describe_instances() - resp.should.contain("Reservations") - len(resp["Reservations"]).should.equal(3) + if not settings.TEST_SERVER_MODE: + # Can't verify this in ServerMode, as other tests may have created instances + resp = ec2_client.describe_instances() + resp.should.contain("Reservations") + len(resp["Reservations"]).should.equal(3) # Should have created 1 ECS cluster - resp = ecs_client.list_clusters() - resp.should.contain("clusterArns") - len(resp["clusterArns"]).should.equal(1) + all_clusters = ecs_client.list_clusters()["clusterArns"] + all_clusters.should.contain(our_env["ecsClusterArn"]) @mock_ec2 @@ -58,7 +64,7 @@ def test_create_unmanaged_compute_environment(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -68,15 +74,21 @@ def test_create_unmanaged_compute_environment(): resp.should.contain("computeEnvironmentArn") resp["computeEnvironmentName"].should.equal(compute_name) + our_env = batch_client.describe_compute_environments( + computeEnvironments=[compute_name] + )["computeEnvironments"][0] + our_env.should.have.key("ecsClusterArn") + # Its unmanaged so no instances should be created - resp = ec2_client.describe_instances() - resp.should.contain("Reservations") - len(resp["Reservations"]).should.equal(0) + if not settings.TEST_SERVER_MODE: + # Can't verify this in ServerMode, as other tests may have created instances + resp = ec2_client.describe_instances() + resp.should.contain("Reservations") + len(resp["Reservations"]).should.equal(0) # Should have created 1 ECS cluster - resp = ecs_client.list_clusters() - resp.should.contain("clusterArns") - len(resp["clusterArns"]).should.equal(1) + all_clusters = ecs_client.list_clusters()["clusterArns"] + all_clusters.should.contain(our_env["ecsClusterArn"]) # TODO create 1000s of tests to test complex option combinations of create environment @@ -90,7 +102,10 @@ def test_describe_compute_environment(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) + compute_arn = ( + f"arn:aws:batch:eu-central-1:123456789012:compute-environment/{compute_name}" + ) batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -98,9 +113,14 @@ def test_describe_compute_environment(): serviceRole=iam_arn, ) - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(1) - resp["computeEnvironments"][0]["computeEnvironmentName"].should.equal(compute_name) + all_envs = batch_client.describe_compute_environments()["computeEnvironments"] + our_envs = [e for e in all_envs if e["computeEnvironmentName"] == compute_name] + our_envs.should.have.length_of(1) + our_envs[0]["computeEnvironmentName"].should.equal(compute_name) + our_envs[0]["computeEnvironmentArn"].should.equal(compute_arn) + our_envs[0].should.have.key("ecsClusterArn") + our_envs[0].should.have.key("state").equal("ENABLED") + our_envs[0].should.have.key("status").equal("VALID") # Test filtering resp = batch_client.describe_compute_environments(computeEnvironments=["test1"]) @@ -115,7 +135,7 @@ def test_delete_unmanaged_compute_environment(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -123,13 +143,18 @@ def test_delete_unmanaged_compute_environment(): serviceRole=iam_arn, ) + our_env = batch_client.describe_compute_environments( + computeEnvironments=[compute_name] + )["computeEnvironments"][0] + batch_client.delete_compute_environment(computeEnvironment=compute_name) - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(0) + all_envs = batch_client.describe_compute_environments()["computeEnvironments"] + all_names = [e["computeEnvironmentName"] for e in all_envs] + all_names.shouldnt.contain(compute_name) - resp = ecs_client.list_clusters() - len(resp.get("clusterArns", [])).should.equal(0) + all_clusters = ecs_client.list_clusters()["clusterArns"] + all_clusters.shouldnt.contain(our_env["ecsClusterArn"]) @mock_ec2 @@ -140,7 +165,7 @@ def test_delete_managed_compute_environment(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="MANAGED", @@ -163,19 +188,26 @@ def test_delete_managed_compute_environment(): serviceRole=iam_arn, ) + our_env = batch_client.describe_compute_environments( + computeEnvironments=[compute_name] + )["computeEnvironments"][0] + batch_client.delete_compute_environment(computeEnvironment=compute_name) - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(0) + all_envs = batch_client.describe_compute_environments()["computeEnvironments"] + all_names = [e["computeEnvironmentName"] for e in all_envs] + all_names.shouldnt.contain(compute_name) - resp = ec2_client.describe_instances() - resp.should.contain("Reservations") - len(resp["Reservations"]).should.equal(3) - for reservation in resp["Reservations"]: - reservation["Instances"][0]["State"]["Name"].should.equal("terminated") + if not settings.TEST_SERVER_MODE: + # Too many instances to know which one is ours in ServerMode + resp = ec2_client.describe_instances() + resp.should.contain("Reservations") + len(resp["Reservations"]).should.equal(3) + for reservation in resp["Reservations"]: + reservation["Instances"][0]["State"]["Name"].should.equal("terminated") - resp = ecs_client.list_clusters() - len(resp.get("clusterArns", [])).should.equal(0) + all_clusters = ecs_client.list_clusters()["clusterArns"] + all_clusters.shouldnt.contain(our_env["ecsClusterArn"]) @mock_ec2 @@ -186,7 +218,7 @@ def test_update_unmanaged_compute_environment_state(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -198,6 +230,7 @@ def test_update_unmanaged_compute_environment_state(): computeEnvironment=compute_name, state="DISABLED" ) - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(1) - resp["computeEnvironments"][0]["state"].should.equal("DISABLED") + all_envs = batch_client.describe_compute_environments()["computeEnvironments"] + our_envs = [e for e in all_envs if e["computeEnvironmentName"] == compute_name] + our_envs.should.have.length_of(1) + our_envs[0]["state"].should.equal("DISABLED") diff --git a/tests/test_batch/test_batch_job_queue.py b/tests/test_batch/test_batch_job_queue.py index 41612baf8..df1d692a8 100644 --- a/tests/test_batch/test_batch_job_queue.py +++ b/tests/test_batch/test_batch_job_queue.py @@ -4,6 +4,7 @@ from botocore.exceptions import ClientError import pytest import sure # noqa from moto import mock_batch, mock_iam, mock_ec2, mock_ecs +from uuid import uuid4 @mock_ec2 @@ -14,7 +15,7 @@ def test_create_job_queue(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -23,8 +24,9 @@ def test_create_job_queue(): ) arn = resp["computeEnvironmentArn"] + jq_name = str(uuid4())[0:6] resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=jq_name, state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], @@ -33,9 +35,10 @@ def test_create_job_queue(): resp.should.contain("jobQueueName") queue_arn = resp["jobQueueArn"] - resp = batch_client.describe_job_queues() - resp.should.have.key("jobQueues").being.length_of(1) - resp["jobQueues"][0]["jobQueueArn"].should.equal(queue_arn) + all_queues = batch_client.describe_job_queues()["jobQueues"] + our_queues = [q for q in all_queues if q["jobQueueName"] == jq_name] + our_queues.should.have.length_of(1) + our_queues[0]["jobQueueArn"].should.equal(queue_arn) @mock_ec2 @@ -57,7 +60,7 @@ def test_create_job_queue_twice(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -66,15 +69,16 @@ def test_create_job_queue_twice(): ) compute_env_arn = resp["computeEnvironmentArn"] + jq_name = str(uuid4())[0:6] batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=jq_name, state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": compute_env_arn}], ) with pytest.raises(ClientError) as ex: batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=jq_name, state="ENABLED", priority=123, computeEnvironmentOrder=[ @@ -84,7 +88,7 @@ def test_create_job_queue_twice(): err = ex.value.response["Error"] err["Code"].should.equal("ClientException") - err["Message"].should.equal("Job queue test_job_queue already exists") + err["Message"].should.equal(f"Job queue {jq_name} already exists") @mock_ec2 @@ -96,7 +100,7 @@ def test_create_job_queue_incorrect_state(): with pytest.raises(ClientError) as ex: batch_client.create_job_queue( - jobQueueName="test_job_queue2", + jobQueueName=str(uuid4()), state="JUNK", priority=123, computeEnvironmentOrder=[], @@ -115,7 +119,7 @@ def test_create_job_queue_without_compute_environment(): with pytest.raises(ClientError) as ex: batch_client.create_job_queue( - jobQueueName="test_job_queue3", + jobQueueName=str(uuid4()), state="ENABLED", priority=123, computeEnvironmentOrder=[], @@ -133,7 +137,7 @@ def test_job_queue_bad_arn(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -144,7 +148,7 @@ def test_job_queue_bad_arn(): with pytest.raises(ClientError) as ex: batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=str(uuid4()), state="ENABLED", priority=123, computeEnvironmentOrder=[ @@ -164,7 +168,7 @@ def test_update_job_queue(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -173,8 +177,9 @@ def test_update_job_queue(): ) arn = resp["computeEnvironmentArn"] + jq_name = str(uuid4()) resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=jq_name, state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], @@ -183,15 +188,16 @@ def test_update_job_queue(): batch_client.update_job_queue(jobQueue=queue_arn, priority=5) - resp = batch_client.describe_job_queues() - resp.should.have.key("jobQueues").being.length_of(1) - resp["jobQueues"][0]["priority"].should.equal(5) + all_queues = batch_client.describe_job_queues()["jobQueues"] + our_queues = [q for q in all_queues if q["jobQueueName"] == jq_name] + our_queues[0]["priority"].should.equal(5) - batch_client.update_job_queue(jobQueue="test_job_queue", priority=5) + batch_client.update_job_queue(jobQueue=jq_name, priority=15) - resp = batch_client.describe_job_queues() - resp.should.have.key("jobQueues").being.length_of(1) - resp["jobQueues"][0]["priority"].should.equal(5) + all_queues = batch_client.describe_job_queues()["jobQueues"] + our_queues = [q for q in all_queues if q["jobQueueName"] == jq_name] + our_queues.should.have.length_of(1) + our_queues[0]["priority"].should.equal(15) @mock_ec2 @@ -202,7 +208,7 @@ def test_delete_job_queue(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -211,8 +217,9 @@ def test_delete_job_queue(): ) arn = resp["computeEnvironmentArn"] + jq_name = str(uuid4()) resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=jq_name, state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], @@ -221,5 +228,5 @@ def test_delete_job_queue(): batch_client.delete_job_queue(jobQueue=queue_arn) - resp = batch_client.describe_job_queues() - resp.should.have.key("jobQueues").being.length_of(0) + all_queues = batch_client.describe_job_queues()["jobQueues"] + [q["jobQueueName"] for q in all_queues].shouldnt.contain(jq_name) diff --git a/tests/test_batch/test_batch_jobs.py b/tests/test_batch/test_batch_jobs.py index 6ef5bf519..918c81108 100644 --- a/tests/test_batch/test_batch_jobs.py +++ b/tests/test_batch/test_batch_jobs.py @@ -5,6 +5,7 @@ import sure # noqa from moto import mock_batch, mock_iam, mock_ec2, mock_ecs, mock_logs import pytest import time +from uuid import uuid4 @mock_logs @@ -16,7 +17,7 @@ def test_submit_job_by_name(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4()) resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -26,14 +27,14 @@ def test_submit_job_by_name(): arn = resp["computeEnvironmentArn"] resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=str(uuid4()), state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], ) queue_arn = resp["jobQueueArn"] - job_definition_name = "sleep10" + job_definition_name = f"sleep10_{str(uuid4())[0:6]}" batch_client.register_job_definition( jobDefinitionName=job_definition_name, @@ -95,12 +96,12 @@ def test_submit_job(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - job_def_name = "sayhellotomylittlefriend" + job_def_name = str(uuid4())[0:6] commands = ["echo", "hello"] job_def_arn, queue_arn = prepare_job(batch_client, commands, iam_arn, job_def_name) resp = batch_client.submit_job( - jobName="test1", jobQueue=queue_arn, jobDefinition=job_def_arn + jobName=str(uuid4())[0:6], jobQueue=queue_arn, jobDefinition=job_def_arn ) job_id = resp["jobId"] @@ -163,7 +164,7 @@ def test_terminate_job(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - job_def_name = "echo-sleep-echo" + job_def_name = f"echo-sleep-echo-{str(uuid4())[0:6]}" commands = ["sh", "-c", "echo start && sleep 30 && echo stop"] job_def_arn, queue_arn = prepare_job(batch_client, commands, iam_arn, job_def_name) @@ -183,11 +184,7 @@ def test_terminate_job(): resp["jobs"][0]["status"].should.equal("FAILED") resp["jobs"][0]["statusReason"].should.equal("test_terminate") - resp = logs_client.describe_log_streams( - logGroupName="/aws/batch/job", logStreamNamePrefix=job_def_name - ) - resp["logStreams"].should.have.length_of(1) - ls_name = resp["logStreams"][0]["logStreamName"] + ls_name = f"{job_def_name}/default/{job_id}" resp = logs_client.get_log_events( logGroupName="/aws/batch/job", logStreamName=ls_name @@ -365,16 +362,40 @@ def test_dependencies(): else: raise RuntimeError("Batch job timed out") - resp = logs_client.describe_log_streams(logGroupName="/aws/batch/job") - len(resp["logStreams"]).should.equal(3) - for log_stream in resp["logStreams"]: + log_stream_name = "/aws/batch/job" + all_streams = retrieve_all_streams(log_stream_name, logs_client) + + nr_logstreams_found = 0 + expected_logstream_names = [ + f"dependencytest/default/{_id}" for _id in [job_id1, job_id2, job_id3] + ] + for log_stream in all_streams: ls_name = log_stream["logStreamName"] + if ls_name not in expected_logstream_names: + continue + resp = logs_client.get_log_events( - logGroupName="/aws/batch/job", logStreamName=ls_name + logGroupName=log_stream_name, logStreamName=ls_name ) [event["message"] for event in resp["events"]].should.equal(["hello"]) + nr_logstreams_found = nr_logstreams_found + 1 + nr_logstreams_found.should.equal(3) + + +def retrieve_all_streams(log_stream_name, logs_client): + resp = logs_client.describe_log_streams(logGroupName=log_stream_name) + all_streams = resp["logStreams"] + token = resp.get("nextToken") + while token: + resp = logs_client.describe_log_streams( + logGroupName=log_stream_name, nextToken=token + ) + all_streams.extend(resp["logStreams"]) + token = resp.get("nextToken") + return all_streams + @mock_logs @mock_ec2 @@ -385,7 +406,7 @@ def test_failed_dependencies(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4())[0:6] resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -395,7 +416,7 @@ def test_failed_dependencies(): arn = resp["computeEnvironmentArn"] resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=str(uuid4())[0:6], state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], @@ -485,7 +506,7 @@ def test_container_overrides(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - compute_name = "test_compute_env" + compute_name = str(uuid4())[0:6] resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -495,14 +516,14 @@ def test_container_overrides(): arn = resp["computeEnvironmentArn"] resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=str(uuid4())[0:6], state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], ) queue_arn = resp["jobQueueArn"] - job_definition_name = "sleep10" + job_definition_name = f"sleep10_{str(uuid4())[0:6]}" # Set up Job Definition # We will then override the container properties in the actual job @@ -600,7 +621,7 @@ def test_container_overrides(): def prepare_job(batch_client, commands, iam_arn, job_def_name): - compute_name = "test_compute_env" + compute_name = str(uuid4())[0:6] resp = batch_client.create_compute_environment( computeEnvironmentName=compute_name, type="UNMANAGED", @@ -610,7 +631,7 @@ def prepare_job(batch_client, commands, iam_arn, job_def_name): arn = resp["computeEnvironmentArn"] resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", + jobQueueName=str(uuid4())[0:6], state="ENABLED", priority=123, computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], diff --git a/tests/test_batch/test_batch_task_definition.py b/tests/test_batch/test_batch_task_definition.py index cf6135f0c..4f1a10d4a 100644 --- a/tests/test_batch/test_batch_task_definition.py +++ b/tests/test_batch/test_batch_task_definition.py @@ -2,6 +2,7 @@ from . import _get_clients, _setup import random import sure # noqa from moto import mock_batch, mock_iam, mock_ec2, mock_ecs +from uuid import uuid4 @mock_ec2 @@ -51,10 +52,11 @@ def test_reregister_task_definition(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() _setup(ec2_client, iam_client) - resp1 = register_job_def(batch_client) + job_def_name = str(uuid4())[0:6] + resp1 = register_job_def(batch_client, definition_name=job_def_name) resp1.should.contain("jobDefinitionArn") - resp1.should.contain("jobDefinitionName") + resp1.should.have.key("jobDefinitionName").equals(job_def_name) resp1.should.contain("revision") assert resp1["jobDefinitionArn"].endswith( @@ -62,18 +64,18 @@ def test_reregister_task_definition(): ) resp1["revision"].should.equal(1) - resp2 = register_job_def(batch_client) + resp2 = register_job_def(batch_client, definition_name=job_def_name) resp2["revision"].should.equal(2) resp2["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) - resp3 = register_job_def(batch_client) + resp3 = register_job_def(batch_client, definition_name=job_def_name) resp3["revision"].should.equal(3) resp3["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) resp3["jobDefinitionArn"].should_not.equal(resp2["jobDefinitionArn"]) - resp4 = register_job_def(batch_client) + resp4 = register_job_def(batch_client, definition_name=job_def_name) resp4["revision"].should.equal(4) resp4["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) @@ -89,12 +91,13 @@ def test_delete_task_definition(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() _setup(ec2_client, iam_client) - resp = register_job_def(batch_client) + resp = register_job_def(batch_client, definition_name=str(uuid4())) + name = resp["jobDefinitionName"] batch_client.deregister_job_definition(jobDefinition=resp["jobDefinitionArn"]) - resp = batch_client.describe_job_definitions() - len(resp["jobDefinitions"]).should.equal(0) + all_defs = batch_client.describe_job_definitions()["jobDefinitions"] + [jobdef["jobDefinitionName"] for jobdef in all_defs].shouldnt.contain(name) @mock_ec2 @@ -105,14 +108,13 @@ def test_delete_task_definition_by_name(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() _setup(ec2_client, iam_client) - resp = register_job_def(batch_client) + resp = register_job_def(batch_client, definition_name=str(uuid4())) + name = resp["jobDefinitionName"] - batch_client.deregister_job_definition( - jobDefinition=f"{resp['jobDefinitionName']}:{resp['revision']}" - ) + batch_client.deregister_job_definition(jobDefinition=f"{name}:{resp['revision']}") - resp = batch_client.describe_job_definitions() - len(resp["jobDefinitions"]).should.equal(0) + all_defs = batch_client.describe_job_definitions()["jobDefinitions"] + [jobdef["jobDefinitionName"] for jobdef in all_defs].shouldnt.contain(name) @mock_ec2 @@ -123,22 +125,30 @@ def test_describe_task_definition(): ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() _setup(ec2_client, iam_client) - register_job_def(batch_client, definition_name="sleep10") - register_job_def(batch_client, definition_name="sleep10") - register_job_def(batch_client, definition_name="test1") - register_job_def_with_tags(batch_client, definition_name="tagged_def") + sleep_def_name = f"sleep10_{str(uuid4())[0:6]}" + other_name = str(uuid4())[0:6] + tagged_name = str(uuid4())[0:6] + register_job_def(batch_client, definition_name=sleep_def_name) + register_job_def(batch_client, definition_name=sleep_def_name) + register_job_def(batch_client, definition_name=other_name) + register_job_def_with_tags(batch_client, definition_name=tagged_name) - resp = batch_client.describe_job_definitions(jobDefinitionName="sleep10") + resp = batch_client.describe_job_definitions(jobDefinitionName=sleep_def_name) len(resp["jobDefinitions"]).should.equal(2) - resp = batch_client.describe_job_definitions() - len(resp["jobDefinitions"]).should.equal(4) + job_defs = batch_client.describe_job_definitions()["jobDefinitions"] + all_names = [jd["jobDefinitionName"] for jd in job_defs] + all_names.should.contain(sleep_def_name) + all_names.should.contain(other_name) + all_names.should.contain(tagged_name) - resp = batch_client.describe_job_definitions(jobDefinitions=["sleep10", "test1"]) + resp = batch_client.describe_job_definitions( + jobDefinitions=[sleep_def_name, other_name] + ) len(resp["jobDefinitions"]).should.equal(3) resp["jobDefinitions"][0]["tags"].should.equal({}) - resp = batch_client.describe_job_definitions(jobDefinitionName="tagged_def") + resp = batch_client.describe_job_definitions(jobDefinitionName=tagged_name) resp["jobDefinitions"][0]["tags"].should.equal( {"foo": "123", "bar": "456",} ) diff --git a/tests/test_ec2/test_amis.py b/tests/test_ec2/test_amis.py index aa176bb5e..d66c1f5fd 100644 --- a/tests/test_ec2/test_amis.py +++ b/tests/test_ec2/test_amis.py @@ -7,13 +7,15 @@ from boto.exception import EC2ResponseError from botocore.exceptions import ClientError import pytest +import random import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 from moto.ec2.models import AMIS, OWNER_ID from moto.core import ACCOUNT_ID -from tests import EXAMPLE_AMI_ID, EXAMPLE_AMI_PARAVIRTUAL, EXAMPLE_AMI_WINDOWS +from tests import EXAMPLE_AMI_ID, EXAMPLE_AMI_PARAVIRTUAL from tests.helpers import requires_boto_gte +from uuid import uuid4 # Has boto3 equivalent @@ -95,16 +97,31 @@ def test_ami_create_and_delete(): cm.value.request_id.should_not.be.none +@mock_ec2 +def test_snapshots_for_initial_amis(): + ec2 = boto3.client("ec2", region_name="us-east-1") + + snapshots = ec2.describe_snapshots()["Snapshots"] + snapshot_descs = [s["Description"] for s in snapshots] + initial_ami_count = len(AMIS) + + assert ( + len(snapshots) >= initial_ami_count + ), "Should have at least as many snapshots as AMIs" + + for ami in AMIS: + ami_id = ami["ami_id"] + expected_description = f"Auto-created snapshot for AMI {ami_id}" + snapshot_descs.should.contain(expected_description) + + @mock_ec2 def test_ami_create_and_delete_boto3(): ec2 = boto3.client("ec2", region_name="us-east-1") - initial_ami_count = len(AMIS) - ec2.describe_volumes()["Volumes"].should.have.length_of(0) - ec2.describe_snapshots()["Snapshots"].should.have.length_of(initial_ami_count) - reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) instance = reservation["Instances"][0] + instance_id = instance["InstanceId"] with pytest.raises(ClientError) as ex: ec2.create_image( @@ -118,15 +135,14 @@ def test_ami_create_and_delete_boto3(): ) image_id = ec2.create_image( - InstanceId=instance["InstanceId"], - Name="test-ami", - Description="this is a test ami", + InstanceId=instance_id, Name="test-ami", Description="this is a test ami", )["ImageId"] all_images = ec2.describe_images()["Images"] set([i["ImageId"] for i in all_images]).should.contain(image_id) retrieved_image = [i for i in all_images if i["ImageId"] == image_id][0] + created_snapshot_id = retrieved_image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"] retrieved_image.should.have.key("ImageId").equal(image_id) retrieved_image.should.have.key("VirtualizationType").equal( @@ -136,14 +152,18 @@ def test_ami_create_and_delete_boto3(): retrieved_image.should.have.key("KernelId").equal(instance["KernelId"]) retrieved_image.should.have.key("Platform").equal(instance["Platform"]) retrieved_image.should.have.key("CreationDate") - ec2.terminate_instances(InstanceIds=[instance["InstanceId"]]) + ec2.terminate_instances(InstanceIds=[instance_id]) # Ensure we're no longer creating a volume - ec2.describe_volumes()["Volumes"].should.have.length_of(0) + volumes_for_instance = [ + v + for v in ec2.describe_volumes()["Volumes"] + if "Attachment" in v and v["Attachment"][0]["InstanceId"] == instance_id + ] + volumes_for_instance.should.have.length_of(0) # Validate auto-created snapshot snapshots = ec2.describe_snapshots()["Snapshots"] - snapshots.should.have.length_of(initial_ami_count + 1) retrieved_image_snapshot_id = retrieved_image["BlockDeviceMappings"][0]["Ebs"][ "SnapshotId" @@ -269,13 +289,9 @@ def test_ami_copy(): @mock_ec2 -def test_ami_copy_boto3(): +def test_ami_copy_boto3_dryrun(): ec2 = boto3.client("ec2", region_name="us-west-1") - initial_ami_count = len(AMIS) - ec2.describe_volumes()["Volumes"].should.have.length_of(0) - ec2.describe_snapshots()["Snapshots"].should.have.length_of(initial_ami_count) - reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) instance = reservation["Instances"][0] @@ -284,7 +300,6 @@ def test_ami_copy_boto3(): Name="test-ami", Description="this is a test ami", )["ImageId"] - ec2.terminate_instances(InstanceIds=[instance["InstanceId"]]) source_image = ec2.describe_images(ImageIds=[source_image_id])["Images"][0] with pytest.raises(ClientError) as ex: @@ -302,6 +317,22 @@ def test_ami_copy_boto3(): "An error occurred (DryRunOperation) when calling the CopyImage operation: Request would have succeeded, but DryRun flag is set" ) + +@mock_ec2 +def test_ami_copy_boto3(): + ec2 = boto3.client("ec2", region_name="us-west-1") + + reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) + instance = reservation["Instances"][0] + + source_image_id = ec2.create_image( + InstanceId=instance["InstanceId"], + Name="test-ami", + Description="this is a test ami", + )["ImageId"] + ec2.terminate_instances(InstanceIds=[instance["InstanceId"]]) + source_image = ec2.describe_images(ImageIds=[source_image_id])["Images"][0] + copy_image_ref = ec2.copy_image( SourceRegion="us-west-1", SourceImageId=source_image["ImageId"], @@ -319,15 +350,22 @@ def test_ami_copy_boto3(): copy_image["KernelId"].should.equal(source_image["KernelId"]) copy_image["Platform"].should.equal(source_image["Platform"]) - # Ensure we're no longer creating a volume - ec2.describe_volumes()["Volumes"].should.have.length_of(0) - # Validate auto-created snapshot - ec2.describe_snapshots()["Snapshots"].should.have.length_of(initial_ami_count + 2) + source_image_snapshot_id = source_image["BlockDeviceMappings"][0]["Ebs"][ + "SnapshotId" + ] + copied_image_snapshot_id = copy_image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"] - copy_image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"].shouldnt.equal( - source_image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"] - ) + snapshot_ids = [s["SnapshotId"] for s in ec2.describe_snapshots()["Snapshots"]] + snapshot_ids.should.contain(source_image_snapshot_id) + snapshot_ids.should.contain(copied_image_snapshot_id) + + copied_image_snapshot_id.shouldnt.equal(source_image_snapshot_id) + + +@mock_ec2 +def test_ami_copy_nonexistent_source_id(): + ec2 = boto3.client("ec2", region_name="us-west-1") # Copy from non-existent source ID. with pytest.raises(ClientError) as ex: @@ -341,6 +379,21 @@ def test_ami_copy_boto3(): ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidAMIID.NotFound") + +@mock_ec2 +def test_ami_copy_nonexisting_source_region(): + ec2 = boto3.client("ec2", region_name="us-west-1") + + reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) + instance = reservation["Instances"][0] + + source_image_id = ec2.create_image( + InstanceId=instance["InstanceId"], + Name="test-ami", + Description="this is a test ami", + )["ImageId"] + source_image = ec2.describe_images(ImageIds=[source_image_id])["Images"][0] + # Copy from non-existent source region. with pytest.raises(ClientError) as ex: ec2.copy_image( @@ -366,16 +419,21 @@ def test_copy_image_changes_owner_id(): check_resp = conn.describe_images(ImageIds=[source_ami_id]) check_resp["Images"][0]["OwnerId"].should_not.equal(OWNER_ID) + new_image_name = str(uuid4())[0:6] + copy_resp = conn.copy_image( SourceImageId=source_ami_id, - Name="new-image", + Name=new_image_name, Description="a copy of an image", SourceRegion="us-east-1", ) - describe_resp = conn.describe_images(Owners=["self"]) - describe_resp["Images"][0]["OwnerId"].should.equal(OWNER_ID) - describe_resp["Images"][0]["ImageId"].should.equal(copy_resp["ImageId"]) + describe_resp = conn.describe_images( + Owners=["self"], Filters=[{"Name": "name", "Values": [new_image_name]}] + )["Images"] + describe_resp.should.have.length_of(1) + describe_resp[0]["OwnerId"].should.equal(OWNER_ID) + describe_resp[0]["ImageId"].should.equal(copy_resp["ImageId"]) # Has boto3 equivalent @@ -510,20 +568,16 @@ def test_ami_uses_account_id_if_valid_access_key_is_supplied_boto3(): # The boto-equivalent required an access_key to be passed in, but Moto will always mock this in boto3 # So the only thing we're testing here, really.. is whether OwnerId is equal to ACCOUNT_ID? # TODO: Maybe patch account_id with multiple values, and verify it always matches with OwnerId - # TODO: And probably remove the modify_instance_attribute, as it looks like it was a unnecessary copy-paste ec2 = boto3.client("ec2", region_name="us-east-1") reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) instance = reservation["Instances"][0] - ec2.modify_instance_attribute( - InstanceId=instance["InstanceId"], Kernel={"Value": "test-kernel"} - ) image_id = ec2.create_image(InstanceId=instance["InstanceId"], Name="test-ami")[ "ImageId" ] images = ec2.describe_images(Owners=["self"])["Images"] - [(ami["ImageId"], ami["OwnerId"]) for ami in images].should.equal( - [(image_id, ACCOUNT_ID)] + [(ami["ImageId"], ami["OwnerId"]) for ami in images].should.contain( + (image_id, ACCOUNT_ID) ) @@ -591,14 +645,18 @@ def test_ami_filters(): @mock_ec2 def test_ami_filters_boto3(): + image_name_A = f"test-ami-{str(uuid4())[0:6]}" + kernel_value_A = f"k-{str(uuid4())[0:6]}" + kernel_value_B = f"k-{str(uuid4())[0:6]}" + ec2 = boto3.client("ec2", region_name="us-east-1") reservationA = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) instanceA = reservationA["Instances"][0] ec2.modify_instance_attribute( - InstanceId=instanceA["InstanceId"], Kernel={"Value": "k-1234abcd"} + InstanceId=instanceA["InstanceId"], Kernel={"Value": kernel_value_A} ) - imageA_id = ec2.create_image(InstanceId=instanceA["InstanceId"], Name="test-ami-A")[ + imageA_id = ec2.create_image(InstanceId=instanceA["InstanceId"], Name=image_name_A)[ "ImageId" ] imageA = boto3.resource("ec2", region_name="us-east-1").Image(imageA_id) @@ -608,7 +666,7 @@ def test_ami_filters_boto3(): ) instanceB = reservationB["Instances"][0] ec2.modify_instance_attribute( - InstanceId=instanceB["InstanceId"], Kernel={"Value": "k-abcd1234"} + InstanceId=instanceB["InstanceId"], Kernel={"Value": kernel_value_B} ) imageB_id = ec2.create_image(InstanceId=instanceB["InstanceId"], Name="test-ami-B")[ "ImageId" @@ -620,10 +678,12 @@ def test_ami_filters_boto3(): Filters=[{"Name": "architecture", "Values": ["x86_64"]}] )["Images"] [ami["ImageId"] for ami in amis_by_architecture].should.contain(imageB_id) - amis_by_architecture.should.have.length_of(40) + assert ( + len(amis_by_architecture) >= 40 + ), "Should have at least 40 AMI's of type x86_64" amis_by_kernel = ec2.describe_images( - Filters=[{"Name": "kernel-id", "Values": ["k-abcd1234"]}] + Filters=[{"Name": "kernel-id", "Values": [kernel_value_B]}] )["Images"] [ami["ImageId"] for ami in amis_by_kernel].should.equal([imageB.id]) @@ -631,13 +691,13 @@ def test_ami_filters_boto3(): Filters=[{"Name": "virtualization-type", "Values": ["paravirtual"]}] )["Images"] [ami["ImageId"] for ami in amis_by_virtualization].should.contain(imageB.id) - amis_by_virtualization.should.have.length_of(3) + assert len(amis_by_virtualization) >= 3, "Should have at least 3 paravirtual AMI's" amis_by_platform = ec2.describe_images( Filters=[{"Name": "platform", "Values": ["windows"]}] )["Images"] [ami["ImageId"] for ami in amis_by_platform].should.contain(imageA_id) - amis_by_platform.should.have.length_of(25) + assert len(amis_by_platform) >= 25, "Should have at least 25 Windows images" amis_by_id = ec2.describe_images( Filters=[{"Name": "image-id", "Values": [imageA_id]}] @@ -650,7 +710,7 @@ def test_ami_filters_boto3(): ami_ids_by_state = [ami["ImageId"] for ami in amis_by_state] ami_ids_by_state.should.contain(imageA_id) ami_ids_by_state.should.contain(imageB.id) - amis_by_state.should.have.length_of(40) + assert len(amis_by_state) >= 40, "Should have at least 40 images available" amis_by_name = ec2.describe_images( Filters=[{"Name": "name", "Values": [imageA.name]}] @@ -660,13 +720,13 @@ def test_ami_filters_boto3(): amis_by_public = ec2.describe_images( Filters=[{"Name": "is-public", "Values": ["true"]}] )["Images"] - amis_by_public.should.have.length_of(38) + assert len(amis_by_public) >= 38, "Should have at least 38 public images" amis_by_nonpublic = ec2.describe_images( Filters=[{"Name": "is-public", "Values": ["false"]}] )["Images"] [ami["ImageId"] for ami in amis_by_nonpublic].should.contain(imageA.id) - amis_by_nonpublic.should.have.length_of(2) + assert len(amis_by_nonpublic) >= 2, "Should have at least 2 non-public images" # Has boto3 equivalent @@ -695,6 +755,8 @@ def test_ami_filtering_via_tag(): @mock_ec2 def test_ami_filtering_via_tag_boto3(): + tag_value = f"value {str(uuid4())}" + other_value = f"value {str(uuid4())}" ec2 = boto3.client("ec2", region_name="us-east-1") reservationA = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) instanceA = reservationA["Instances"][0] @@ -703,7 +765,7 @@ def test_ami_filtering_via_tag_boto3(): "ImageId" ] imageA = boto3.resource("ec2", region_name="us-east-1").Image(imageA_id) - imageA.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) + imageA.create_tags(Tags=[{"Key": "a key", "Value": tag_value}]) reservationB = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) instanceB = reservationB["Instances"][0] @@ -711,15 +773,15 @@ def test_ami_filtering_via_tag_boto3(): "ImageId" ] imageB = boto3.resource("ec2", region_name="us-east-1").Image(imageB_id) - imageB.create_tags(Tags=[{"Key": "another key", "Value": "some other value"}]) + imageB.create_tags(Tags=[{"Key": "another key", "Value": other_value}]) amis_by_tagA = ec2.describe_images( - Filters=[{"Name": "tag:a key", "Values": ["some value"]}] + Filters=[{"Name": "tag:a key", "Values": [tag_value]}] )["Images"] [ami["ImageId"] for ami in amis_by_tagA].should.equal([imageA_id]) amis_by_tagB = ec2.describe_images( - Filters=[{"Name": "tag:another key", "Values": ["some other value"]}] + Filters=[{"Name": "tag:another key", "Values": [other_value]}] )["Images"] [ami["ImageId"] for ami in amis_by_tagB].should.equal([imageB_id]) @@ -993,8 +1055,8 @@ def test_ami_attribute_user_permissions_boto3(): ] permissions.should.equal([]) - USER1 = "123456789011" - USER2 = "123456789022" + USER1 = "".join(["{}".format(random.randint(0, 9)) for _ in range(0, 12)]) + USER2 = "".join(["{}".format(random.randint(0, 9)) for _ in range(0, 12)]) ADD_USERS_ARGS = { "ImageId": image.id, @@ -1067,7 +1129,7 @@ def test_ami_describe_executable_users(): instance_id = response["Reservations"][0]["Instances"][0]["InstanceId"] image_id = conn.create_image(InstanceId=instance_id, Name="TestImage")["ImageId"] - USER1 = "123456789011" + USER1 = "".join(["{}".format(random.randint(0, 9)) for _ in range(0, 12)]) ADD_USER_ARGS = { "ImageId": image_id, @@ -1100,8 +1162,8 @@ def test_ami_describe_executable_users_negative(): instance_id = response["Reservations"][0]["Instances"][0]["InstanceId"] image_id = conn.create_image(InstanceId=instance_id, Name="TestImage")["ImageId"] - USER1 = "123456789011" - USER2 = "113355789012" + USER1 = "".join(["{}".format(random.randint(0, 9)) for _ in range(0, 12)]) + USER2 = "".join(["{}".format(random.randint(0, 9)) for _ in range(0, 12)]) ADD_USER_ARGS = { "ImageId": image_id, @@ -1136,7 +1198,7 @@ def test_ami_describe_executable_users_and_filter(): "ImageId" ] - USER1 = "123456789011" + USER1 = "".join(["{}".format(random.randint(0, 9)) for _ in range(0, 12)]) ADD_USER_ARGS = { "ImageId": image_id, @@ -1504,16 +1566,19 @@ def test_ami_filter_wildcard(): ec2_resource = boto3.resource("ec2", region_name="us-west-1") ec2_client = boto3.client("ec2", region_name="us-west-1") + image_name = str(uuid4())[0:6] + instance = ec2_resource.create_instances( ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1 )[0] - instance.create_image(Name="test-image") + instance.create_image(Name=image_name) # create an image with the same owner but will not match the filter - instance.create_image(Name="not-matching-image") + instance.create_image(Name=str(uuid4())[0:6]) my_images = ec2_client.describe_images( - Owners=[ACCOUNT_ID], Filters=[{"Name": "name", "Values": ["test*"]}] + Owners=[ACCOUNT_ID], + Filters=[{"Name": "name", "Values": [f"{image_name[0:4]}*"]}], )["Images"] my_images.should.have.length_of(1) @@ -1541,17 +1606,21 @@ def test_ami_filter_by_self(): ec2_resource = boto3.resource("ec2", region_name="us-west-1") ec2_client = boto3.client("ec2", region_name="us-west-1") - my_images = ec2_client.describe_images(Owners=["self"])["Images"] - my_images.should.have.length_of(0) + unique_name = str(uuid4())[0:6] + + images = ec2_client.describe_images(Owners=["self"])["Images"] + image_names = [i["Name"] for i in images] + image_names.shouldnt.contain(unique_name) # Create a new image instance = ec2_resource.create_instances( ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1 )[0] - instance.create_image(Name="test-image") + instance.create_image(Name=unique_name) - my_images = ec2_client.describe_images(Owners=["self"])["Images"] - my_images.should.have.length_of(1) + images = ec2_client.describe_images(Owners=["self"])["Images"] + image_names = [i["Name"] for i in images] + image_names.should.contain(unique_name) @mock_ec2 @@ -1654,9 +1723,11 @@ def test_ami_filter_by_empty_tag(): ) fake_images.append(image) # Add release tags to some of the images in the middle + release_version = str(uuid4())[0:6] for image in fake_images[3:6]: ec2.create_tags( - Resources=[image["ImageId"]], Tags=[{"Key": "RELEASE", "Value": ""}] + Resources=[image["ImageId"]], + Tags=[{"Key": "RELEASE", "Value": release_version}], ) images_filter = [ { @@ -1664,7 +1735,7 @@ def test_ami_filter_by_empty_tag(): "Values": ["Deep Learning Base AMI (Amazon Linux 2) Version 31.0"], }, {"Name": "tag:OS_Version", "Values": ["AWS Linux 2"]}, - {"Name": "tag:RELEASE", "Values": [""]}, + {"Name": "tag:RELEASE", "Values": [release_version]}, ] assert len(client.describe_images(Filters=images_filter)["Images"]) == 3 diff --git a/tests/test_ec2/test_carrier_gateways.py b/tests/test_ec2/test_carrier_gateways.py index c01cafa72..f41e54b0e 100644 --- a/tests/test_ec2/test_carrier_gateways.py +++ b/tests/test_ec2/test_carrier_gateways.py @@ -2,12 +2,15 @@ import boto3 import sure # noqa import pytest from botocore.exceptions import ClientError -from moto import mock_ec2 +from moto import mock_ec2, settings from moto.core import ACCOUNT_ID +from unittest import SkipTest @mock_ec2 def test_describe_carrier_gateways_none(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.client("ec2", region_name="us-east-1") ec2.describe_carrier_gateways()["CarrierGateways"].should.equal([]) @@ -20,7 +23,11 @@ def test_describe_carrier_gateways_multiple(): cg1 = client.create_carrier_gateway(VpcId=vpc.id)["CarrierGateway"] cg2 = client.create_carrier_gateway(VpcId=vpc.id)["CarrierGateway"] - client.describe_carrier_gateways()["CarrierGateways"].should.have.length_of(2) + gateways = client.describe_carrier_gateways()["CarrierGateways"] + gateway_ids = [g["CarrierGatewayId"] for g in gateways] + + gateway_ids.should.contain(cg1["CarrierGatewayId"]) + gateway_ids.should.contain(cg2["CarrierGatewayId"]) find_one = client.describe_carrier_gateways( CarrierGatewayIds=[cg1["CarrierGatewayId"]] @@ -86,4 +93,7 @@ def test_delete_carrier_gateways(): cg = client.create_carrier_gateway(VpcId=vpc.id)["CarrierGateway"] client.delete_carrier_gateway(CarrierGatewayId=cg["CarrierGatewayId"]) - client.describe_carrier_gateways()["CarrierGateways"].should.equal([]) + gateways = client.describe_carrier_gateways()["CarrierGateways"] + gateway_ids = [g["CarrierGatewayId"] for g in gateways] + + gateway_ids.shouldnt.contain(cg["CarrierGatewayId"]) diff --git a/tests/test_ec2/test_customer_gateways.py b/tests/test_ec2/test_customer_gateways.py index dcaa95ad8..b5039bd4c 100644 --- a/tests/test_ec2/test_customer_gateways.py +++ b/tests/test_ec2/test_customer_gateways.py @@ -48,9 +48,25 @@ def test_describe_customer_gateways_boto3(): ec2 = boto3.client("ec2", region_name="us-east-1") customer_gateway = create_customer_gateway(ec2) + cg_id = customer_gateway["CustomerGatewayId"] + cgws = ec2.describe_customer_gateways()["CustomerGateways"] - cgws.should.have.length_of(1) - cgws[0]["CustomerGatewayId"].should.match(customer_gateway["CustomerGatewayId"]) + cg_ids = [cg["CustomerGatewayId"] for cg in cgws] + cg_ids.should.contain(cg_id) + + cgw = ec2.describe_customer_gateways(CustomerGatewayIds=[cg_id])[ + "CustomerGateways" + ][0] + cgw.should.have.key("BgpAsn") + cgw.should.have.key("CustomerGatewayId").equal(cg_id) + cgw.should.have.key("IpAddress") + cgw.should.have.key("State").equal("available") + cgw.should.have.key("Type").equal("ipsec.1") + + all_cgws = ec2.describe_customer_gateways()["CustomerGateways"] + assert ( + len(all_cgws) >= 1 + ), "Should have at least the one CustomerGateway we just created" # Has boto3 equivalent @@ -73,12 +89,19 @@ def test_delete_customer_gateways_boto3(): ec2 = boto3.client("ec2", region_name="us-east-1") customer_gateway = create_customer_gateway(ec2) + cg_id = customer_gateway["CustomerGatewayId"] - cgws = ec2.describe_customer_gateways()["CustomerGateways"] + cgws = ec2.describe_customer_gateways(CustomerGatewayIds=[cg_id])[ + "CustomerGateways" + ] cgws.should.have.length_of(1) + cgws[0].should.have.key("State").equal("available") ec2.delete_customer_gateway(CustomerGatewayId=customer_gateway["CustomerGatewayId"]) - cgws = ec2.describe_customer_gateways()["CustomerGateways"] + + cgws = ec2.describe_customer_gateways(CustomerGatewayIds=[cg_id])[ + "CustomerGateways" + ] cgws.should.have.length_of(1) cgws[0].should.have.key("State").equal("deleted") diff --git a/tests/test_ec2/test_dhcp_options.py b/tests/test_ec2/test_dhcp_options.py index 96ffed11d..8890de178 100644 --- a/tests/test_ec2/test_dhcp_options.py +++ b/tests/test_ec2/test_dhcp_options.py @@ -8,6 +8,8 @@ from boto.exception import EC2ResponseError from botocore.exceptions import ClientError import sure # noqa +import random +import uuid from moto import mock_ec2, mock_ec2_deprecated, settings from unittest import SkipTest @@ -269,9 +271,6 @@ def test_describe_dhcp_options_boto3(): ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") - all_options = client.describe_dhcp_options()["DhcpOptions"] - all_options.should.have.length_of(0) - dhcp_options = ec2.create_dhcp_options( DhcpConfigurations=[ {"Key": "domain-name", "Values": [SAMPLE_DOMAIN_NAME]}, @@ -284,9 +283,12 @@ def test_describe_dhcp_options_boto3(): all_options.should.have.length_of(1) all_options = client.describe_dhcp_options()["DhcpOptions"] - all_options.should.have.length_of(1) - all_options[0]["DhcpOptionsId"].should.equal(dhcp_options.id) - config = all_options[0]["DhcpConfigurations"] + assert len(all_options) >= 1, "Should have recently created DHCP option" + recently_created = [ + o for o in all_options if o["DhcpOptionsId"] == dhcp_options.id + ][0] + recently_created["DhcpOptionsId"].should.equal(dhcp_options.id) + config = recently_created["DhcpConfigurations"] config.should.have.length_of(2) config.should.contain( { @@ -445,17 +447,22 @@ def test_dhcp_tagging_boto3(): ] ) - dhcp_option.create_tags(Tags=[{"Key": "a tag", "Value": "some value"}]) + tag_value = str(uuid.uuid4()) + dhcp_option.create_tags(Tags=[{"Key": "a tag", "Value": tag_value}]) - tag = client.describe_tags()["Tags"][0] + tag = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [dhcp_option.id]}] + )["Tags"][0] tag.should.have.key("ResourceId").equal(dhcp_option.id) tag.should.have.key("ResourceType").equal("dhcp-options") tag.should.have.key("Key").equal("a tag") - tag.should.have.key("Value").equal("some value") + tag.should.have.key("Value").equal(tag_value) # Refresh the DHCP options - dhcp_option = client.describe_dhcp_options()["DhcpOptions"][0] - dhcp_option["Tags"].should.equal([{"Key": "a tag", "Value": "some value"}]) + dhcp_option = client.describe_dhcp_options(DhcpOptionsIds=[dhcp_option.id])[ + "DhcpOptions" + ][0] + dhcp_option["Tags"].should.equal([{"Key": "a tag", "Value": tag_value}]) # Has boto3 equivalent @@ -500,16 +507,19 @@ def test_dhcp_options_get_by_tag_boto3(): ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") + dhcp_tag_value = str(uuid.uuid4()) + dhcp1 = ec2.create_dhcp_options( DhcpConfigurations=[ {"Key": "domain-name", "Values": ["example.com"]}, {"Key": "domain-name-servers", "Values": ["10.0.10.2"]}, ] ) + dhcp1_tag_name = str(uuid.uuid4()) dhcp1.create_tags( Tags=[ - {"Key": "Name", "Value": "TestDhcpOptions1"}, - {"Key": "test-tag", "Value": "test-value"}, + {"Key": "Name", "Value": dhcp1_tag_name}, + {"Key": "test-tag", "Value": dhcp_tag_value}, ] ) @@ -519,17 +529,18 @@ def test_dhcp_options_get_by_tag_boto3(): {"Key": "domain-name-servers", "Values": ["10.0.20.2"]}, ] ) + dhcp2_tag_name = str(uuid.uuid4()) dhcp2.create_tags( Tags=[ - {"Key": "Name", "Value": "TestDhcpOptions2"}, - {"Key": "test-tag", "Value": "test-value"}, + {"Key": "Name", "Value": dhcp2_tag_name}, + {"Key": "test-tag", "Value": dhcp_tag_value}, ] ) dhcp_options_sets = client.describe_dhcp_options( Filters=[ - {"Name": "tag:Name", "Values": ["TestDhcpOptions1"]}, - {"Name": "tag:test-tag", "Values": ["test-value"]}, + {"Name": "tag:Name", "Values": [dhcp1_tag_name]}, + {"Name": "tag:test-tag", "Values": [dhcp_tag_value]}, ] )["DhcpOptions"] @@ -542,13 +553,13 @@ def test_dhcp_options_get_by_tag_boto3(): ) tags = dhcp_options_sets[0]["Tags"] tags.should.have.length_of(2) - tags.should.contain({"Key": "Name", "Value": "TestDhcpOptions1"}) - tags.should.contain({"Key": "test-tag", "Value": "test-value"}) + tags.should.contain({"Key": "Name", "Value": dhcp1_tag_name}) + tags.should.contain({"Key": "test-tag", "Value": dhcp_tag_value}) dhcp_options_sets = client.describe_dhcp_options( Filters=[ - {"Name": "tag:Name", "Values": ["TestDhcpOptions2"]}, - {"Name": "tag:test-tag", "Values": ["test-value"]}, + {"Name": "tag:Name", "Values": [dhcp2_tag_name]}, + {"Name": "tag:test-tag", "Values": [dhcp_tag_value]}, ] )["DhcpOptions"] @@ -561,11 +572,11 @@ def test_dhcp_options_get_by_tag_boto3(): ) tags = dhcp_options_sets[0]["Tags"] tags.should.have.length_of(2) - tags.should.contain({"Key": "Name", "Value": "TestDhcpOptions2"}) - tags.should.contain({"Key": "test-tag", "Value": "test-value"}) + tags.should.contain({"Key": "Name", "Value": dhcp2_tag_name}) + tags.should.contain({"Key": "test-tag", "Value": dhcp_tag_value}) dhcp_options_sets = client.describe_dhcp_options( - Filters=[{"Name": "tag:test-tag", "Values": ["test-value"]}] + Filters=[{"Name": "tag:test-tag", "Values": [dhcp_tag_value]}] )["DhcpOptions"] dhcp_options_sets.should.have.length_of(2) @@ -625,8 +636,10 @@ def test_dhcp_options_get_by_id_boto3(): dhcp1.create_tags(Tags=[{"Key": "Name", "Value": "TestDhcpOptions2"}]) dhcp1.create_tags(Tags=[{"Key": "test-tag", "Value": "test-value"}]) - d = client.describe_dhcp_options()["DhcpOptions"] - d.should.have.length_of(2) + options = client.describe_dhcp_options()["DhcpOptions"] + d_ids = [d["DhcpOptionsId"] for d in options] + d_ids.should.contain(dhcp1.id) + d_ids.should.contain(dhcp2.id) d = client.describe_dhcp_options( Filters=[{"Name": "dhcp-options-id", "Values": [dhcp1.id]}] @@ -647,28 +660,32 @@ def test_dhcp_options_get_by_id_boto3(): def test_dhcp_options_get_by_value_filter(): ec2 = boto3.resource("ec2", region_name="us-west-1") + random_server_1 = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + random_server_2 = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + random_server_3 = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + ec2.create_dhcp_options( DhcpConfigurations=[ {"Key": "domain-name", "Values": ["example.com"]}, - {"Key": "domain-name-servers", "Values": ["10.0.10.2"]}, + {"Key": "domain-name-servers", "Values": [random_server_1]}, ] ) ec2.create_dhcp_options( DhcpConfigurations=[ {"Key": "domain-name", "Values": ["example.com"]}, - {"Key": "domain-name-servers", "Values": ["10.0.20.2"]}, + {"Key": "domain-name-servers", "Values": [random_server_2]}, ] ) ec2.create_dhcp_options( DhcpConfigurations=[ {"Key": "domain-name", "Values": ["example.com"]}, - {"Key": "domain-name-servers", "Values": ["10.0.30.2"]}, + {"Key": "domain-name-servers", "Values": [random_server_3]}, ] ) - filters = [{"Name": "value", "Values": ["10.0.10.2"]}] + filters = [{"Name": "value", "Values": [random_server_2]}] dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters)) dhcp_options_sets.should.have.length_of(1) @@ -677,30 +694,36 @@ def test_dhcp_options_get_by_value_filter(): def test_dhcp_options_get_by_key_filter(): ec2 = boto3.resource("ec2", region_name="us-west-1") + random_domain_name = str(uuid.uuid4())[0:6] + ec2.create_dhcp_options( - DhcpConfigurations=[ - {"Key": "domain-name", "Values": ["example.com"]}, - {"Key": "domain-name-servers", "Values": ["10.0.10.2"]}, - ] + DhcpConfigurations=[{"Key": "domain-name", "Values": [random_domain_name]},] ) ec2.create_dhcp_options( - DhcpConfigurations=[ - {"Key": "domain-name", "Values": ["example.com"]}, - {"Key": "domain-name-servers", "Values": ["10.0.20.2"]}, - ] + DhcpConfigurations=[{"Key": "domain-name", "Values": ["example.com"]},] ) ec2.create_dhcp_options( - DhcpConfigurations=[ - {"Key": "domain-name", "Values": ["example.com"]}, - {"Key": "domain-name-servers", "Values": ["10.0.30.2"]}, - ] + DhcpConfigurations=[{"Key": "domain-name", "Values": [random_domain_name]},] ) filters = [{"Name": "key", "Values": ["domain-name"]}] dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters)) - dhcp_options_sets.should.have.length_of(3) + assert ( + len(dhcp_options_sets) >= 3 + ), "Should have at least 3 DHCP options just created" + + configs = [] + for d in dhcp_options_sets: + configs.extend(d.dhcp_configurations) + + servers = [] + for config in configs: + if config["Key"] == "domain-name": + servers.extend(config["Values"]) + servers.should.contain({"Value": random_domain_name}) + servers.should.contain({"Value": "example.com"}) # Has boto3 equivalent diff --git a/tests/test_ec2/test_ec2_cloudformation.py b/tests/test_ec2/test_ec2_cloudformation.py index df862d5cb..ddf26b528 100644 --- a/tests/test_ec2/test_ec2_cloudformation.py +++ b/tests/test_ec2/test_ec2_cloudformation.py @@ -7,6 +7,7 @@ from tests.test_cloudformation.fixtures import single_instance_with_ebs_volume from tests.test_cloudformation.fixtures import vpc_eip from tests.test_cloudformation.fixtures import vpc_eni from tests.test_cloudformation.fixtures import vpc_single_instance_in_subnet +from uuid import uuid4 import boto import boto.ec2 import boto.cloudformation @@ -31,67 +32,59 @@ template_vpc = { def test_vpc_single_instance_in_subnet(): template_json = json.dumps(vpc_single_instance_in_subnet.template) cf = boto3.client("cloudformation", region_name="us-west-1") + stack_name = str(uuid4())[0:6] cf.create_stack( - StackName="test_stack", + StackName=stack_name, TemplateBody=template_json, Parameters=[{"ParameterKey": "KeyName", "ParameterValue": "my_key"}], ) ec2 = boto3.client("ec2", region_name="us-west-1") - vpc = ec2.describe_vpcs(Filters=[{"Name": "cidrBlock", "Values": ["10.0.0.0/16"]}])[ - "Vpcs" - ][0] + stack = cf.describe_stacks(StackName=stack_name)["Stacks"][0] + + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] + vpc_id = [ + resource + for resource in resources + if resource["ResourceType"] == "AWS::EC2::VPC" + ][0]["PhysicalResourceId"] + + vpc = ec2.describe_vpcs(VpcIds=[vpc_id])["Vpcs"][0] vpc["CidrBlock"].should.equal("10.0.0.0/16") - - ec2.describe_internet_gateways()["InternetGateways"].should.have.length_of(1) - - subnet = ec2.describe_subnets( - Filters=[{"Name": "vpcId", "Values": [vpc["VpcId"]]}] - )["Subnets"][0] - subnet["VpcId"].should.equal(vpc["VpcId"]) - - ec2 = boto3.client("ec2", region_name="us-west-1") - reservation = ec2.describe_instances()["Reservations"][0] - instance = reservation["Instances"][0] - instance["Tags"].should.contain({"Key": "Foo", "Value": "Bar"}) - # Check that the EIP is attached the the EC2 instance - eip = ec2.describe_addresses()["Addresses"][0] - eip["Domain"].should.equal("vpc") - eip["InstanceId"].should.equal(instance["InstanceId"]) + vpc["Tags"].should.contain({"Key": "Application", "Value": stack["StackId"]}) security_group = ec2.describe_security_groups( Filters=[{"Name": "vpc-id", "Values": [vpc["VpcId"]]}] )["SecurityGroups"][0] security_group["VpcId"].should.equal(vpc["VpcId"]) - stack = cf.describe_stacks(StackName="test_stack")["Stacks"][0] - - vpc["Tags"].should.contain({"Key": "Application", "Value": stack["StackId"]}) - - resources = cf.list_stack_resources(StackName="test_stack")[ - "StackResourceSummaries" - ] - vpc_resource = [ - resource - for resource in resources - if resource["ResourceType"] == "AWS::EC2::VPC" - ][0] - vpc_resource["PhysicalResourceId"].should.equal(vpc["VpcId"]) - - subnet_resource = [ + subnet_id = [ resource for resource in resources if resource["ResourceType"] == "AWS::EC2::Subnet" - ][0] - subnet_resource["PhysicalResourceId"].should.equal(subnet["SubnetId"]) + ][0]["PhysicalResourceId"] - eip_resource = [ + subnet = ec2.describe_subnets(SubnetIds=[subnet_id])["Subnets"][0] + subnet["VpcId"].should.equal(vpc["VpcId"]) + + instance_id = [ + resource + for resource in resources + if resource["ResourceType"] == "AWS::EC2::Instance" + ][0]["PhysicalResourceId"] + res = ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0] + instance = res["Instances"][0] + instance["Tags"].should.contain({"Key": "Foo", "Value": "Bar"}) + + eip_id = [ resource for resource in resources if resource["ResourceType"] == "AWS::EC2::EIP" - ][0] - eip_resource["PhysicalResourceId"].should.equal(eip["PublicIp"]) + ][0]["PhysicalResourceId"] + eip = ec2.describe_addresses(PublicIps=[eip_id])["Addresses"][0] + eip["Domain"].should.equal("vpc") + eip["InstanceId"].should.equal(instance["InstanceId"]) @mock_cloudformation @@ -99,11 +92,13 @@ def test_vpc_single_instance_in_subnet(): def test_delete_stack_with_resource_missing_delete_attr(): cf = boto3.client("cloudformation", region_name="us-east-1") ec2 = boto3.client("ec2", region_name="us-east-1") - name = "test_stack" + name = str(uuid4())[0:6] cf.create_stack(StackName=name, TemplateBody=json.dumps(template_vpc)) cf.describe_stacks(StackName=name)["Stacks"].should.have.length_of(1) - ec2.describe_vpcs()["Vpcs"].should.have.length_of(2) + + resources = cf.list_stack_resources(StackName=name)["StackResourceSummaries"] + vpc_id = resources[0]["PhysicalResourceId"] cf.delete_stack( StackName=name @@ -112,10 +107,10 @@ def test_delete_stack_with_resource_missing_delete_attr(): cf.describe_stacks(StackName=name) err = exc.value.response["Error"] err.should.have.key("Code").equals("ValidationError") - err.should.have.key("Message").equals("Stack with id test_stack does not exist") + err.should.have.key("Message").equals(f"Stack with id {name} does not exist") - # We still have two VPCs, as the VPC-object does not have a delete-method yet - ec2.describe_vpcs()["Vpcs"].should.have.length_of(2) + # We still have our VPC, as the VPC-object does not have a delete-method yet + ec2.describe_vpcs(VpcIds=[vpc_id])["Vpcs"].should.have.length_of(1) # Has boto3 equivalent @@ -149,26 +144,26 @@ def test_elastic_network_interfaces_cloudformation_boto3(): template = vpc_eni.template template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) ec2 = boto3.client("ec2", region_name="us-west-1") - eni = ec2.describe_network_interfaces()["NetworkInterfaces"][0] - eni["PrivateIpAddresses"].should.have.length_of(1) - private_ip_address = eni["PrivateIpAddresses"][0]["PrivateIpAddress"] + all_enis = ec2.describe_network_interfaces()["NetworkInterfaces"] + all_eni_ids = [eni["NetworkInterfaceId"] for eni in all_enis] + all_ips = [eni["PrivateIpAddresses"][0]["PrivateIpAddress"] for eni in all_enis] - resources = cf.list_stack_resources(StackName="test_stack")[ - "StackResourceSummaries" - ] + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] cfn_eni = [ resource for resource in resources if resource["ResourceType"] == "AWS::EC2::NetworkInterface" ][0] - cfn_eni["PhysicalResourceId"].should.equal(eni["NetworkInterfaceId"]) + all_eni_ids.should.contain(cfn_eni["PhysicalResourceId"]) - outputs = cf.describe_stacks(StackName="test_stack")["Stacks"][0]["Outputs"] - outputs.should.contain( - {"OutputKey": "ENIIpAddress", "OutputValue": private_ip_address} - ) + outputs = cf.describe_stacks(StackName=stack_name)["Stacks"][0]["Outputs"] + received_ip = [ + o["OutputValue"] for o in outputs if o["OutputKey"] == "ENIIpAddress" + ][0] + all_ips.should.contain(received_ip) @mock_ec2 @@ -177,6 +172,7 @@ def test_volume_size_through_cloudformation(): ec2 = boto3.client("ec2", region_name="us-east-1") cf = boto3.client("cloudformation", region_name="us-east-1") + tag_value = str(uuid4()) volume_template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { @@ -191,23 +187,26 @@ def test_volume_size_through_cloudformation(): ], "Tags": [ {"Key": "foo", "Value": "bar"}, - {"Key": "blah", "Value": "baz"}, + {"Key": "blah", "Value": tag_value}, ], }, } }, } template_json = json.dumps(volume_template) - cf.create_stack(StackName="test_stack", TemplateBody=template_json) - resource = cf.list_stack_resources(StackName="test_stack")[ - "StackResourceSummaries" - ][0] + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) + + resource = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"][ + 0 + ] resource.should.have.key("LogicalResourceId").being.equal("testInstance") resource.should.have.key("PhysicalResourceId").shouldnt.be.none resource.should.have.key("ResourceType").being.equal("AWS::EC2::Instance") instances = ec2.describe_instances(InstanceIds=[resource["PhysicalResourceId"]]) + volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][ "Ebs" ] @@ -275,11 +274,13 @@ def test_subnet_tags_through_cloudformation_boto3(): } template_json = json.dumps(subnet_template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] + subnet_id = resources[0]["PhysicalResourceId"] - subnet = ec2.describe_subnets( - Filters=[{"Name": "cidrBlock", "Values": ["10.0.0.0/24"]}] - )["Subnets"][0] + subnet = ec2.describe_subnets(SubnetIds=[subnet_id])["Subnets"][0] + subnet["CidrBlock"].should.equal("10.0.0.0/24") subnet["Tags"].should.contain({"Key": "foo", "Value": "bar"}) subnet["Tags"].should.contain({"Key": "blah", "Value": "baz"}) @@ -289,16 +290,30 @@ def test_subnet_tags_through_cloudformation_boto3(): def test_single_instance_with_ebs_volume(): template_json = json.dumps(single_instance_with_ebs_volume.template) cf = boto3.client("cloudformation", region_name="us-west-1") + stack_name = str(uuid4())[0:6] cf.create_stack( - StackName="test_stack", + StackName=stack_name, TemplateBody=template_json, Parameters=[{"ParameterKey": "KeyName", "ParameterValue": "key_name"}], ) + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] + instance_id = [ + r["PhysicalResourceId"] + for r in resources + if r["ResourceType"] == "AWS::EC2::Instance" + ][0] + volume_id = [ + r["PhysicalResourceId"] + for r in resources + if r["ResourceType"] == "AWS::EC2::Volume" + ][0] ec2 = boto3.client("ec2", region_name="us-west-1") - ec2_instance = ec2.describe_instances()["Reservations"][0]["Instances"][0] + ec2_instance = ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0][ + "Instances" + ][0] - volumes = ec2.describe_volumes()["Volumes"] + volumes = ec2.describe_volumes(VolumeIds=[volume_id])["Volumes"] # Grab the mounted drive volume = [ volume for volume in volumes if volume["Attachments"][0]["Device"] == "/dev/sdh" @@ -306,35 +321,24 @@ def test_single_instance_with_ebs_volume(): volume["State"].should.equal("in-use") volume["Attachments"][0]["InstanceId"].should.equal(ec2_instance["InstanceId"]) - resources = cf.list_stack_resources(StackName="test_stack")[ - "StackResourceSummaries" - ] - ebs_volumes = [ - resource - for resource in resources - if resource["ResourceType"] == "AWS::EC2::Volume" - ] - ebs_volumes[0]["PhysicalResourceId"].should.equal(volume["VolumeId"]) - @mock_ec2 @mock_cloudformation def test_classic_eip(): template_json = json.dumps(ec2_classic_eip.template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) ec2 = boto3.client("ec2", region_name="us-west-1") - eip = ec2.describe_addresses()["Addresses"][0] + all_ips = [eip["PublicIp"] for eip in ec2.describe_addresses()["Addresses"]] - resources = cf.list_stack_resources(StackName="test_stack")[ - "StackResourceSummaries" - ] + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] cfn_eip = [ resource for resource in resources if resource["ResourceType"] == "AWS::EC2::EIP" ][0] - cfn_eip["PhysicalResourceId"].should.equal(eip["PublicIp"]) + all_ips.should.contain(cfn_eip["PhysicalResourceId"]) @mock_ec2 @@ -342,19 +346,19 @@ def test_classic_eip(): def test_vpc_eip(): template_json = json.dumps(vpc_eip.template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) - ec2 = boto3.client("ec2", region_name="us-west-1") - eip = ec2.describe_addresses()["Addresses"][0] + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) - resources = cf.list_stack_resources(StackName="test_stack")[ - "StackResourceSummaries" - ] + ec2 = boto3.client("ec2", region_name="us-west-1") + all_ips = [eip["PublicIp"] for eip in ec2.describe_addresses()["Addresses"]] + + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] cfn_eip = [ resource for resource in resources if resource["ResourceType"] == "AWS::EC2::EIP" ][0] - cfn_eip["PhysicalResourceId"].should.equal(eip["PublicIp"]) + all_ips.should.contain(cfn_eip["PhysicalResourceId"]) @mock_cloudformation @@ -385,12 +389,15 @@ def test_vpc_gateway_attachment_creation_should_attach_itself_to_vpc(): template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) + + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] + vpc_id = resources[1]["PhysicalResourceId"] ec2 = boto3.client("ec2", region_name="us-west-1") - vpc = ec2.describe_vpcs(Filters=[{"Name": "cidrBlock", "Values": ["10.0.0.0/16"]}])[ - "Vpcs" - ][0] + vpc = ec2.describe_vpcs(VpcIds=[vpc_id])["Vpcs"][0] + vpc["CidrBlock"].should.equal("10.0.0.0/16") igws = ec2.describe_internet_gateways( Filters=[{"Name": "attachment.vpc-id", "Values": [vpc["VpcId"]]}] @@ -417,17 +424,22 @@ def test_vpc_peering_creation(): template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] + our_vpx_id = resources[0]["PhysicalResourceId"] - peering_connections = ec2_client.describe_vpc_peering_connections()[ - "VpcPeeringConnections" - ] + peering_connections = ec2_client.describe_vpc_peering_connections( + VpcPeeringConnectionIds=[our_vpx_id] + )["VpcPeeringConnections"] peering_connections.should.have.length_of(1) @mock_cloudformation @mock_ec2 def test_multiple_security_group_ingress_separate_from_security_group_by_id(): + sg1 = str(uuid4())[0:6] + sg2 = str(uuid4())[0:6] template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { @@ -435,14 +447,14 @@ def test_multiple_security_group_ingress_separate_from_security_group_by_id(): "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "test security group", - "Tags": [{"Key": "sg-name", "Value": "sg1"}], + "Tags": [{"Key": "sg-name", "Value": sg1}], }, }, "test-security-group2": { "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "test security group", - "Tags": [{"Key": "sg-name", "Value": "sg2"}], + "Tags": [{"Key": "sg-name", "Value": sg2}], }, }, "test-sg-ingress": { @@ -460,11 +472,11 @@ def test_multiple_security_group_ingress_separate_from_security_group_by_id(): template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + cf.create_stack(StackName=str(uuid4())[0:6], TemplateBody=template_json) ec2 = boto3.client("ec2", region_name="us-west-1") - security_group1 = get_secgroup_by_tag(ec2, "sg1") - security_group2 = get_secgroup_by_tag(ec2, "sg2") + security_group1 = get_secgroup_by_tag(ec2, sg1) + security_group2 = get_secgroup_by_tag(ec2, sg2) security_group1["IpPermissions"].should.have.length_of(1) security_group1["IpPermissions"][0]["UserIdGroupPairs"].should.have.length_of(1) @@ -480,10 +492,10 @@ def test_multiple_security_group_ingress_separate_from_security_group_by_id(): @mock_ec2 def test_security_group_ingress_separate_from_security_group_by_id(): ec2 = boto3.client("ec2", region_name="us-west-1") - ec2.create_security_group( - GroupName="test-security-group1", Description="test security group" - ) + sg_name = str(uuid4()) + ec2.create_security_group(GroupName=sg_name, Description="test security group") + sg_2 = str(uuid4())[0:6] template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { @@ -491,13 +503,13 @@ def test_security_group_ingress_separate_from_security_group_by_id(): "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "test security group", - "Tags": [{"Key": "sg-name", "Value": "sg2"}], + "Tags": [{"Key": "sg-name", "Value": sg_2}], }, }, "test-sg-ingress": { "Type": "AWS::EC2::SecurityGroupIngress", "Properties": { - "GroupName": "test-security-group1", + "GroupName": sg_name, "IpProtocol": "tcp", "FromPort": "80", "ToPort": "8080", @@ -509,11 +521,11 @@ def test_security_group_ingress_separate_from_security_group_by_id(): template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) - security_group1 = ec2.describe_security_groups(GroupNames=["test-security-group1"])[ + cf.create_stack(StackName=str(uuid4())[0:6], TemplateBody=template_json) + security_group1 = ec2.describe_security_groups(GroupNames=[sg_name])[ "SecurityGroups" ][0] - security_group2 = get_secgroup_by_tag(ec2, "sg2") + security_group2 = get_secgroup_by_tag(ec2, sg_2) security_group1["IpPermissions"].should.have.length_of(1) security_group1["IpPermissions"][0]["UserIdGroupPairs"].should.have.length_of(1) @@ -567,7 +579,7 @@ def test_security_group_ingress_separate_from_security_group_by_id_using_vpc(): template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) + cf.create_stack(StackName=str(uuid4())[0:6], TemplateBody=template_json) security_group1 = get_secgroup_by_tag(ec2_client, "sg1") security_group2 = get_secgroup_by_tag(ec2_client, "sg2") @@ -589,6 +601,7 @@ def test_security_group_with_update(): vpc1 = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc2 = ec2.create_vpc(CidrBlock="10.1.0.0/16") + sg = str(uuid4())[0:6] template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { @@ -597,7 +610,7 @@ def test_security_group_with_update(): "Properties": { "GroupDescription": "test security group", "VpcId": vpc1.id, - "Tags": [{"Key": "sg-name", "Value": "sg"}], + "Tags": [{"Key": "sg-name", "Value": sg}], }, } }, @@ -605,14 +618,15 @@ def test_security_group_with_update(): template_json = json.dumps(template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=template_json) - security_group = get_secgroup_by_tag(ec2_client, "sg") + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) + security_group = get_secgroup_by_tag(ec2_client, sg) security_group["VpcId"].should.equal(vpc1.id) template["Resources"]["test-security-group"]["Properties"]["VpcId"] = vpc2.id template_json = json.dumps(template) - cf.update_stack(StackName="test_stack", TemplateBody=template_json) - security_group = get_secgroup_by_tag(ec2_client, "sg") + cf.update_stack(StackName=stack_name, TemplateBody=template_json) + security_group = get_secgroup_by_tag(ec2_client, sg) security_group["VpcId"].should.equal(vpc2.id) @@ -638,10 +652,12 @@ def test_subnets_should_be_created_with_availability_zone(): } cf = boto3.client("cloudformation", region_name="us-west-1") template_json = json.dumps(subnet_template) - cf.create_stack(StackName="test_stack", TemplateBody=template_json) - subnet = ec2_client.describe_subnets( - Filters=[{"Name": "cidrBlock", "Values": ["10.0.0.0/24"]}] - )["Subnets"][0] + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=template_json) + resources = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"] + subnet_id = resources[0]["PhysicalResourceId"] + subnet = ec2_client.describe_subnets(SubnetIds=[subnet_id])["Subnets"][0] + subnet["CidrBlock"].should.equal("10.0.0.0/24") subnet["AvailabilityZone"].should.equal("us-west-1b") diff --git a/tests/test_ec2/test_ec2_integration.py b/tests/test_ec2/test_ec2_integration.py index e128224e5..5a7fb99c4 100644 --- a/tests/test_ec2/test_ec2_integration.py +++ b/tests/test_ec2/test_ec2_integration.py @@ -33,9 +33,14 @@ def test_run_instance_with_encrypted_ebs(): } ], } - ec2.run_instances(**kwargs) + instance = ec2.run_instances(**kwargs) + instance_id = instance["Instances"][0]["InstanceId"] - instances = ec2.describe_instances().get("Reservations")[0].get("Instances") + instances = ( + ec2.describe_instances(InstanceIds=[instance_id]) + .get("Reservations")[0] + .get("Instances") + ) volume = instances[0]["BlockDeviceMappings"][0]["Ebs"] volumes = ec2.describe_volumes(VolumeIds=[volume["VolumeId"]]) diff --git a/tests/test_ec2/test_egress_only_igw.py b/tests/test_ec2/test_egress_only_igw.py index dad614f07..491ed15ef 100644 --- a/tests/test_ec2/test_egress_only_igw.py +++ b/tests/test_ec2/test_egress_only_igw.py @@ -48,7 +48,7 @@ def test_describe_all(): gateways = client.describe_egress_only_internet_gateways()[ "EgressOnlyInternetGateways" ] - gateways.should.have.length_of(2) + assert len(gateways) >= 2, "Should have two recently created gateways" gateways.should.contain(gw1) gateways.should.contain(gw2) @@ -90,12 +90,12 @@ def test_create_and_delete(): ] gw1_id = gw1["EgressOnlyInternetGatewayId"] - client.describe_egress_only_internet_gateways()[ - "EgressOnlyInternetGateways" - ].should.have.length_of(1) + client.describe_egress_only_internet_gateways( + EgressOnlyInternetGatewayIds=[gw1_id] + )["EgressOnlyInternetGateways"].should.have.length_of(1) client.delete_egress_only_internet_gateway(EgressOnlyInternetGatewayId=gw1_id) - client.describe_egress_only_internet_gateways()[ - "EgressOnlyInternetGateways" - ].should.have.length_of(0) + client.describe_egress_only_internet_gateways( + EgressOnlyInternetGatewayIds=[gw1_id] + )["EgressOnlyInternetGateways"].should.have.length_of(0) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 50a96f855..f3c7f908f 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -13,6 +13,7 @@ from moto.ec2 import ec2_backends from moto.ec2.models import OWNER_ID from moto.kms import mock_kms from tests import EXAMPLE_AMI_ID +from uuid import uuid4 # Has boto3 equivalent @@ -391,9 +392,9 @@ def test_volume_filters_boto3(): Size=25, AvailabilityZone="us-east-1a", SnapshotId=snapshot.id ) - ec2.create_tags( - Resources=[volume1.id], Tags=[{"Key": "testkey1", "Value": "testvalue1"}] - ) + tag_key1 = str(uuid4())[0:6] + tag_val1 = str(uuid4()) + ec2.create_tags(Resources=[volume1.id], Tags=[{"Key": tag_key1, "Value": tag_val1}]) ec2.create_tags( Resources=[volume2.id], Tags=[{"Key": "testkey2", "Value": "testvalue2"}] ) @@ -411,30 +412,84 @@ def test_volume_filters_boto3(): ][0] block_volume = block_mapping["Ebs"]["VolumeId"] - def verify_filter(name, value, expected=None): + def verify_filter(name, value, expected=None, not_expected=None): + multiple_results = not_expected is not None expected = expected or block_volume expected = expected if type(expected) == list else [expected] volumes = client.describe_volumes(Filters=[{"Name": name, "Values": [value]}])[ "Volumes" ] - set([vol["VolumeId"] for vol in volumes]).should.equal(set(expected)) + actual = [vol["VolumeId"] for vol in volumes] + if multiple_results: + for e in expected: + actual.should.contain(e) + for e in not_expected: + actual.shouldnt.contain(e) + else: + set(actual).should.equal(set(expected)) # We should probably make this less strict, i.e. figure out which formats AWS expects/approves of attach_time = block_mapping["Ebs"]["AttachTime"].strftime("%Y-%m-%dT%H:%M:%S.000Z") - verify_filter("attachment.attach-time", attach_time) - verify_filter("attachment.device", "/dev/sda1") + verify_filter( + "attachment.attach-time", + attach_time, + not_expected=[volume1.id, volume2.id, volume3.id, volume4.id], + ) + verify_filter( + "attachment.device", + "/dev/sda1", + not_expected=[volume1.id, volume2.id, volume3.id, volume4.id], + ) verify_filter("attachment.instance-id", instance.id) - verify_filter("attachment.status", "attached") - verify_filter("size", str(volume2.size), expected=volume2.id) - verify_filter("snapshot-id", snapshot.id, expected=volume4.id) - verify_filter("status", "in-use") - verify_filter("volume-id", volume1.id, expected=volume1.id) - verify_filter("tag-key", "testkey1", expected=volume1.id) - verify_filter("tag-value", "testvalue1", expected=volume1.id) - verify_filter("tag:testkey1", "testvalue1", expected=volume1.id) - verify_filter("encrypted", "false", expected=[block_volume, volume2.id]) - verify_filter("encrypted", "true", expected=[volume1.id, volume3.id, volume4.id]) - verify_filter("availability-zone", "us-east-1b", expected=volume2.id) + verify_filter( + "attachment.status", + "attached", + not_expected=[volume1.id, volume2.id, volume3.id, volume4.id], + ) + verify_filter( + "size", + str(volume2.size), + expected=volume2.id, + not_expected=[volume1.id, volume3.id, volume4.id], + ) + verify_filter( + "snapshot-id", + snapshot.id, + expected=volume4.id, + not_expected=[volume1.id, volume2.id, volume3.id], + ) + verify_filter( + "status", + "in-use", + not_expected=[volume1.id, volume2.id, volume3.id, volume4.id], + ) + verify_filter( + "volume-id", + volume1.id, + expected=volume1.id, + not_expected=[volume2.id, volume3.id, volume4.id], + ) + verify_filter("tag-key", tag_key1, expected=volume1.id) + verify_filter("tag-value", tag_val1, expected=volume1.id) + verify_filter(f"tag:{tag_key1}", tag_val1, expected=volume1.id) + verify_filter( + "encrypted", + "false", + expected=[block_volume, volume2.id], + not_expected=[volume1.id, volume3.id, volume4.id], + ) + verify_filter( + "encrypted", + "true", + expected=[volume1.id, volume3.id, volume4.id], + not_expected=[block_volume, volume2.id], + ) + verify_filter( + "availability-zone", + "us-east-1b", + expected=volume2.id, + not_expected=[volume1.id, volume3.id, volume4.id], + ) # create_time = volume4.create_time.strftime("%Y-%m-%dT%H:%M:%S.000Z") volumes_by_attach_device = client.describe_volumes( @@ -634,10 +689,12 @@ def test_create_snapshot_boto3(): num_snapshots = len(client.describe_snapshots()["Snapshots"]) snapshot = volume.create_snapshot() - client.describe_snapshots()["Snapshots"].should.have.length_of(num_snapshots + 1) + current_snapshots = client.describe_snapshots()["Snapshots"] + [s["SnapshotId"] for s in current_snapshots].should.contain(snapshot.id) snapshot.delete() - client.describe_snapshots()["Snapshots"].should.have.length_of(num_snapshots) + current_snapshots = client.describe_snapshots()["Snapshots"] + [s["SnapshotId"] for s in current_snapshots].shouldnt.contain(snapshot.id) # Deleting something that was already deleted should throw an error with pytest.raises(ClientError) as ex: @@ -814,36 +871,53 @@ def test_snapshot_filters_boto3(): volume1 = ec2.create_volume(Size=20, AvailabilityZone="us-east-1a", Encrypted=False) volume2 = ec2.create_volume(Size=25, AvailabilityZone="us-east-1a", Encrypted=True) - snapshot1 = volume1.create_snapshot(Description="testsnapshot1") + snapshot1_desc = str(uuid4()) + + snapshot1 = volume1.create_snapshot(Description=snapshot1_desc) snapshot2 = volume1.create_snapshot(Description="testsnapshot2") snapshot3 = volume2.create_snapshot(Description="testsnapshot3") + key_name_1 = str(uuid4())[0:6] + key_value_1 = str(uuid4())[0:6] + key_name_2 = str(uuid4())[0:6] + key_value_2 = str(uuid4())[0:6] ec2.create_tags( - Resources=[snapshot1.id], Tags=[{"Key": "testkey1", "Value": "testvalue1"}] + Resources=[snapshot1.id], Tags=[{"Key": key_name_1, "Value": key_value_1}] ) ec2.create_tags( - Resources=[snapshot2.id], Tags=[{"Key": "testkey2", "Value": "testvalue2"}] + Resources=[snapshot2.id], Tags=[{"Key": key_name_2, "Value": key_value_2}] ) - def verify_filter(name, value, expected): + def verify_filter(name, value, expected, others=False): expected = expected if type(expected) == list else [expected] snapshots = client.describe_snapshots( Filters=[{"Name": name, "Values": [value]}] )["Snapshots"] - set([s["SnapshotId"] for s in snapshots]).should.equal(set(expected)) + if others: + actual = set([s["SnapshotId"] for s in snapshots]) + for e in expected: + actual.should.contain(e) + else: + set([s["SnapshotId"] for s in snapshots]).should.equal(set(expected)) - verify_filter("description", "testsnapshot1", expected=snapshot1.id) + verify_filter("description", snapshot1_desc, expected=snapshot1.id) verify_filter("snapshot-id", snapshot1.id, expected=snapshot1.id) verify_filter("volume-id", volume1.id, expected=[snapshot1.id, snapshot2.id]) verify_filter( - "volume-size", str(volume1.size), expected=[snapshot1.id, snapshot2.id] + "volume-size", + str(volume1.size), + expected=[snapshot1.id, snapshot2.id], + others=True, ) - verify_filter("tag-key", "testkey1", expected=snapshot1.id) - verify_filter("tag-value", "testvalue1", expected=snapshot1.id) - verify_filter("tag:testkey2", "testvalue2", expected=snapshot2.id) - verify_filter("encrypted", "true", expected=snapshot3.id) + verify_filter("tag-key", key_name_1, expected=snapshot1.id) + verify_filter("tag-value", key_value_1, expected=snapshot1.id) + verify_filter(f"tag:{key_name_2}", key_value_2, expected=snapshot2.id) + verify_filter("encrypted", "true", expected=snapshot3.id, others=True) verify_filter( - "owner-id", OWNER_ID, expected=[snapshot1.id, snapshot2.id, snapshot3.id] + "owner-id", + OWNER_ID, + expected=[snapshot1.id, snapshot2.id, snapshot3.id], + others=True, ) # # We should probably make this less strict, i.e. figure out which formats AWS expects/approves of @@ -1427,10 +1501,7 @@ def test_create_unencrypted_volume_with_kms_key_fails(): @mock_ec2 def test_create_encrypted_volume_without_kms_key_should_use_default_key(): kms = boto3.client("kms", region_name="us-east-1") - # Default master key for EBS does not exist until needed. - with pytest.raises(ClientError) as ex: - kms.describe_key(KeyId="alias/aws/ebs") - ex.value.response["Error"]["Code"].should.equal("NotFoundException") + # Creating an encrypted volume should create (and use) the default key. resource = boto3.resource("ec2", region_name="us-east-1") volume = resource.create_volume( diff --git a/tests/test_ec2/test_elastic_ip_addresses.py b/tests/test_ec2/test_elastic_ip_addresses.py index 81631eb39..4d8bbb124 100644 --- a/tests/test_ec2/test_elastic_ip_addresses.py +++ b/tests/test_ec2/test_elastic_ip_addresses.py @@ -6,6 +6,7 @@ import boto import boto3 from boto.exception import EC2ResponseError from botocore.exceptions import ClientError +from uuid import uuid4 import sure # noqa @@ -65,7 +66,9 @@ def test_eip_allocate_classic_boto3(): standard.should.have.key("PublicIp") standard.should.have.key("Domain").equal("standard") - standard = ec2.ClassicAddress(standard["PublicIp"]) + public_ip = standard["PublicIp"] + + standard = ec2.ClassicAddress(public_ip) standard.load() with pytest.raises(ClientError) as ex: @@ -77,7 +80,9 @@ def test_eip_allocate_classic_boto3(): ) standard.release() - client.describe_addresses()["Addresses"].should.be.empty + + all_addresses = client.describe_addresses()["Addresses"] + [a["PublicIp"] for a in all_addresses].shouldnt.contain(public_ip) # Has boto3 equivalent @@ -119,12 +124,20 @@ def test_eip_allocate_vpc_boto3(): vpc.should.have.key("AllocationId") vpc.should.have.key("Domain").equal("vpc") - client.describe_addresses()["Addresses"].should.have.length_of(1) + allocation_id = vpc["AllocationId"] - vpc = ec2.VpcAddress(vpc["AllocationId"]) + all_addresses = client.describe_addresses()["Addresses"] + [a["AllocationId"] for a in all_addresses if "AllocationId" in a].should.contain( + allocation_id + ) + + vpc = ec2.VpcAddress(allocation_id) vpc.release() - client.describe_addresses()["Addresses"].should.be.empty + all_addresses = client.describe_addresses()["Addresses"] + [a["AllocationId"] for a in all_addresses if "AllocationId" in a].shouldnt.contain( + allocation_id + ) @mock_ec2 @@ -264,7 +277,12 @@ def test_eip_associate_classic_boto3(): eip.reload() eip.instance_id.should.be.equal("") eip.release() - client.describe_addresses()["Addresses"].should.be.empty + + with pytest.raises(ClientError) as ex: + client.describe_addresses(PublicIps=[eip.public_ip]) + err = ex.value.response["Error"] + err["Code"].should.equal("InvalidAddress.NotFound") + err["Message"].should.equal("Address '{'" + eip.public_ip + "'}' not found.") instance.terminate() @@ -835,7 +853,9 @@ def test_eip_describe_boto3(): # Release all IPs for eip in eips: eip.release() - client.describe_addresses()["Addresses"].should.have.length_of(0) + all_addresses = client.describe_addresses()["Addresses"] + [a["PublicIp"] for a in all_addresses].shouldnt.contain(eips[0].public_ip) + [a["PublicIp"] for a in all_addresses].shouldnt.contain(eips[1].public_ip) # Has boto3 equivalent @@ -908,16 +928,21 @@ def test_eip_filters(): inst3.public_ip_address.should.equal(addresses[0].public_ip) # Param search by Filter - def check_vpc_filter_valid(filter_name, filter_values): + def check_vpc_filter_valid(filter_name, filter_values, all_values=True): addresses = list( service.vpc_addresses.filter( Filters=[{"Name": filter_name, "Values": filter_values}] ) ) - len(addresses).should.equal(2) - ips = [addr.public_ip for addr in addresses] - set(ips).should.equal(set([eip1.public_ip, eip2.public_ip])) - ips.should.contain(inst1.public_ip_address) + if all_values: + len(addresses).should.equal(2) + ips = [addr.public_ip for addr in addresses] + set(ips).should.equal(set([eip1.public_ip, eip2.public_ip])) + ips.should.contain(inst1.public_ip_address) + else: + ips = [addr.public_ip for addr in addresses] + ips.should.contain(eip1.public_ip) + ips.should.contain(eip2.public_ip) def check_vpc_filter_invalid(filter_name): addresses = list( @@ -927,8 +952,8 @@ def test_eip_filters(): ) len(addresses).should.equal(0) - def check_vpc_filter(filter_name, filter_values): - check_vpc_filter_valid(filter_name, filter_values) + def check_vpc_filter(filter_name, filter_values, all_values=True): + check_vpc_filter_valid(filter_name, filter_values, all_values) check_vpc_filter_invalid(filter_name) check_vpc_filter("allocation-id", [eip1.allocation_id, eip2.allocation_id]) @@ -947,6 +972,7 @@ def test_eip_filters(): inst1.network_interfaces_attribute[0].get("PrivateIpAddress"), inst2.network_interfaces_attribute[0].get("PrivateIpAddress"), ], + all_values=False, # Other ENI's may have the same ip address ) check_vpc_filter("public-ip", [inst1.public_ip_address, inst2.public_ip_address]) @@ -954,7 +980,10 @@ def test_eip_filters(): addresses = list( service.vpc_addresses.filter(Filters=[{"Name": "domain", "Values": ["vpc"]}]) ) - len(addresses).should.equal(3) + public_ips = [a.public_ip for a in addresses] + public_ips.should.contain(eip1.public_ip) + public_ips.should.contain(eip1.public_ip) + public_ips.should.contain(inst1.public_ip_address) @mock_ec2 @@ -963,17 +992,19 @@ def test_eip_tags(): client = boto3.client("ec2", region_name="us-west-1") # Allocate one address without tags - client.allocate_address(Domain="vpc") + no_tags = client.allocate_address(Domain="vpc") + # Allocate one address and add tags alloc_tags = client.allocate_address(Domain="vpc") + managed_by = str(uuid4()) with_tags = client.create_tags( Resources=[alloc_tags["AllocationId"]], - Tags=[{"Key": "ManagedBy", "Value": "MyCode"}], + Tags=[{"Key": "ManagedBy", "Value": managed_by}], ) addresses_with_tags = client.describe_addresses( Filters=[ {"Name": "domain", "Values": ["vpc"]}, - {"Name": "tag:ManagedBy", "Values": ["MyCode"]}, + {"Name": "tag:ManagedBy", "Values": [managed_by]}, ] ) len(addresses_with_tags["Addresses"]).should.equal(1) @@ -981,7 +1012,7 @@ def test_eip_tags(): service.vpc_addresses.filter( Filters=[ {"Name": "domain", "Values": ["vpc"]}, - {"Name": "tag:ManagedBy", "Values": ["MyCode"]}, + {"Name": "tag:ManagedBy", "Values": [managed_by]}, ] ) ) @@ -998,8 +1029,10 @@ def test_eip_tags(): addresses = list( service.vpc_addresses.filter(Filters=[{"Name": "domain", "Values": ["vpc"]}]) ) - # Expected total is 2, one with and one without tags - len(addresses).should.equal(2) + # Expected at least 2, one with and one without tags + assert len(addresses) >= 2, "Should find our two created addresses" + [a.allocation_id for a in addresses].should.contain(no_tags["AllocationId"]) + [a.allocation_id for a in addresses].should.contain(alloc_tags["AllocationId"]) @mock_ec2 @@ -1011,7 +1044,10 @@ def test_describe_addresses_tags(): Resources=[alloc_tags["AllocationId"]], Tags=[{"Key": "ManagedBy", "Value": "MyCode"}], ) - addresses_with_tags = client.describe_addresses() + + addresses_with_tags = client.describe_addresses( + AllocationIds=[alloc_tags["AllocationId"]] + ) assert addresses_with_tags.get("Addresses")[0].get("Tags") == [ {"Key": "ManagedBy", "Value": "MyCode"} ] diff --git a/tests/test_ec2/test_elastic_network_interfaces.py b/tests/test_ec2/test_elastic_network_interfaces.py index 863da88e3..d28201f21 100644 --- a/tests/test_ec2/test_elastic_network_interfaces.py +++ b/tests/test_ec2/test_elastic_network_interfaces.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import pytest +import random import boto3 from botocore.exceptions import ClientError @@ -11,6 +12,7 @@ import sure # noqa from moto import mock_ec2, mock_ec2_deprecated, settings from tests.helpers import requires_boto_gte +from uuid import uuid4 # Has boto3 equivalent @@ -75,9 +77,11 @@ def test_elastic_network_interfaces_boto3(): eni_id = ec2.create_network_interface(SubnetId=subnet.id).id - all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(1) - eni = all_enis[0] + my_enis = client.describe_network_interfaces(NetworkInterfaceIds=[eni_id])[ + "NetworkInterfaces" + ] + my_enis.should.have.length_of(1) + eni = my_enis[0] eni["Groups"].should.have.length_of(0) eni["PrivateIpAddresses"].should.have.length_of(1) eni["PrivateIpAddresses"][0]["PrivateIpAddress"].startswith("10.").should.be.true @@ -93,7 +97,15 @@ def test_elastic_network_interfaces_boto3(): client.delete_network_interface(NetworkInterfaceId=eni_id) all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(0) + [eni["NetworkInterfaceId"] for eni in all_enis].shouldnt.contain(eni_id) + + with pytest.raises(ClientError) as ex: + client.describe_network_interfaces(NetworkInterfaceIds=[eni_id]) + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["ResponseMetadata"].should.have.key("RequestId") + ex.value.response["Error"]["Code"].should.equal( + "InvalidNetworkInterfaceID.NotFound" + ) with pytest.raises(ClientError) as ex: client.delete_network_interface(NetworkInterfaceId=eni_id) @@ -155,12 +167,16 @@ def test_elastic_network_interfaces_with_private_ip_boto3(): subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") private_ip = "54.0.0.1" - ec2.create_network_interface(SubnetId=subnet.id, PrivateIpAddress=private_ip) + eni = ec2.create_network_interface(SubnetId=subnet.id, PrivateIpAddress=private_ip) all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(1) + [eni["NetworkInterfaceId"] for eni in all_enis].should.contain(eni.id) - eni = all_enis[0] + my_enis = client.describe_network_interfaces(NetworkInterfaceIds=[eni.id])[ + "NetworkInterfaces" + ] + + eni = my_enis[0] eni["Groups"].should.have.length_of(0) eni["PrivateIpAddresses"].should.have.length_of(1) @@ -201,16 +217,16 @@ def test_elastic_network_interfaces_with_groups_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") - sec_group1 = ec2.create_security_group(GroupName="group #1", Description="n/a") - sec_group2 = ec2.create_security_group(GroupName="group #2", Description="n/a") - subnet.create_network_interface(Groups=[sec_group1.id, sec_group2.id]) + sec_group1 = ec2.create_security_group(GroupName=str(uuid4()), Description="n/a") + sec_group2 = ec2.create_security_group(GroupName=str(uuid4()), Description="n/a") + my_eni = subnet.create_network_interface(Groups=[sec_group1.id, sec_group2.id]) all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(1) + [eni["NetworkInterfaceId"] for eni in all_enis].should.contain(my_eni.id) - eni = all_enis[0] - eni["Groups"].should.have.length_of(2) - set([group["GroupId"] for group in eni["Groups"]]).should.equal( + my_eni = [eni for eni in all_enis if eni["NetworkInterfaceId"] == my_eni.id][0] + my_eni["Groups"].should.have.length_of(2) + set([group["GroupId"] for group in my_eni["Groups"]]).should.equal( set([sec_group1.id, sec_group2.id]) ) @@ -267,16 +283,16 @@ def test_elastic_network_interfaces_modify_attribute_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") - sec_group1 = ec2.create_security_group(GroupName="group #1", Description="n/a") - sec_group2 = ec2.create_security_group(GroupName="group #2", Description="n/a") + sec_group1 = ec2.create_security_group(GroupName=str(uuid4()), Description="n/a") + sec_group2 = ec2.create_security_group(GroupName=str(uuid4()), Description="n/a") eni_id = subnet.create_network_interface(Groups=[sec_group1.id]).id - all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(1) + my_eni = client.describe_network_interfaces(NetworkInterfaceIds=[eni_id])[ + "NetworkInterfaces" + ][0] - eni = all_enis[0] - eni["Groups"].should.have.length_of(1) - eni["Groups"][0]["GroupId"].should.equal(sec_group1.id) + my_eni["Groups"].should.have.length_of(1) + my_eni["Groups"][0]["GroupId"].should.equal(sec_group1.id) with pytest.raises(ClientError) as ex: client.modify_network_interface_attribute( @@ -292,12 +308,11 @@ def test_elastic_network_interfaces_modify_attribute_boto3(): NetworkInterfaceId=eni_id, Groups=[sec_group2.id] ) - all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(1) - - eni = all_enis[0] - eni["Groups"].should.have.length_of(1) - eni["Groups"][0]["GroupId"].should.equal(sec_group2.id) + my_eni = client.describe_network_interfaces(NetworkInterfaceIds=[eni_id])[ + "NetworkInterfaces" + ][0] + my_eni["Groups"].should.have.length_of(1) + my_eni["Groups"][0]["GroupId"].should.equal(sec_group2.id) # Has boto3 equivalent @@ -370,15 +385,17 @@ def test_elastic_network_interfaces_filtering_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") - sec_group1 = ec2.create_security_group(GroupName="group #1", Description="n/a") - sec_group2 = ec2.create_security_group(GroupName="group #2", Description="n/a") + sec_group1 = ec2.create_security_group(GroupName=str(uuid4()), Description="n/a") + sec_group2 = ec2.create_security_group(GroupName=str(uuid4()), Description="n/a") eni1 = subnet.create_network_interface(Groups=[sec_group1.id, sec_group2.id]) eni2 = subnet.create_network_interface(Groups=[sec_group1.id]) - eni3 = subnet.create_network_interface(Description="test description") + eni3 = subnet.create_network_interface(Description=str(uuid4())) all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(3) + [eni["NetworkInterfaceId"] for eni in all_enis].should.contain(eni1.id) + [eni["NetworkInterfaceId"] for eni in all_enis].should.contain(eni2.id) + [eni["NetworkInterfaceId"] for eni in all_enis].should.contain(eni3.id) # Filter by NetworkInterfaceId enis_by_id = client.describe_network_interfaces(NetworkInterfaceIds=[eni1.id])[ @@ -453,13 +470,14 @@ def test_elastic_network_interfaces_get_by_tag_name(): "An error occurred (DryRunOperation) when calling the CreateTags operation: Request would have succeeded, but DryRun flag is set" ) - eni1.create_tags(Tags=[{"Key": "Name", "Value": "eni1"}]) + tag_value = str(uuid4()) + eni1.create_tags(Tags=[{"Key": "Name", "Value": tag_value}]) # The status of the new interface should be 'available' waiter = ec2_client.get_waiter("network_interface_available") waiter.wait(NetworkInterfaceIds=[eni1.id]) - filters = [{"Name": "tag:Name", "Values": ["eni1"]}] + filters = [{"Name": "tag:Name", "Values": [tag_value]}] enis = list(ec2.network_interfaces.filter(Filters=filters)) enis.should.have.length_of(1) @@ -496,32 +514,33 @@ def test_elastic_network_interfaces_get_by_availability_zone(): filters = [{"Name": "availability-zone", "Values": ["us-west-2a"]}] enis = list(ec2.network_interfaces.filter(Filters=filters)) - enis.should.have.length_of(1) + [eni.id for eni in enis].should.contain(eni1.id) + [eni.id for eni in enis].shouldnt.contain(eni2.id) filters = [{"Name": "availability-zone", "Values": ["us-west-2c"]}] enis = list(ec2.network_interfaces.filter(Filters=filters)) - enis.should.have.length_of(0) + [eni.id for eni in enis].shouldnt.contain(eni1.id) + [eni.id for eni in enis].shouldnt.contain(eni2.id) @mock_ec2 def test_elastic_network_interfaces_get_by_private_ip(): ec2 = boto3.resource("ec2", region_name="us-west-2") ec2_client = boto3.client("ec2", region_name="us-west-2") + random_ip = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet( VpcId=vpc.id, CidrBlock="10.0.0.0/24", AvailabilityZone="us-west-2a" ) - eni1 = ec2.create_network_interface( - SubnetId=subnet.id, PrivateIpAddress="10.0.10.5" - ) + eni1 = ec2.create_network_interface(SubnetId=subnet.id, PrivateIpAddress=random_ip) # The status of the new interface should be 'available' waiter = ec2_client.get_waiter("network_interface_available") waiter.wait(NetworkInterfaceIds=[eni1.id]) - filters = [{"Name": "private-ip-address", "Values": ["10.0.10.5"]}] + filters = [{"Name": "private-ip-address", "Values": [random_ip]}] enis = list(ec2.network_interfaces.filter(Filters=filters)) enis.should.have.length_of(1) @@ -529,7 +548,7 @@ def test_elastic_network_interfaces_get_by_private_ip(): enis = list(ec2.network_interfaces.filter(Filters=filters)) enis.should.have.length_of(0) - filters = [{"Name": "addresses.private-ip-address", "Values": ["10.0.10.5"]}] + filters = [{"Name": "addresses.private-ip-address", "Values": [random_ip]}] enis = list(ec2.network_interfaces.filter(Filters=filters)) enis.should.have.length_of(1) @@ -602,8 +621,9 @@ def test_elastic_network_interfaces_get_by_description(): VpcId=vpc.id, CidrBlock="10.0.0.0/24", AvailabilityZone="us-west-2a" ) + desc = str(uuid4()) eni1 = ec2.create_network_interface( - SubnetId=subnet.id, PrivateIpAddress="10.0.10.5", Description="test interface" + SubnetId=subnet.id, PrivateIpAddress="10.0.10.5", Description=desc ) # The status of the new interface should be 'available' @@ -624,18 +644,20 @@ def test_elastic_network_interfaces_describe_network_interfaces_with_filter(): ec2 = boto3.resource("ec2", region_name="us-west-2") ec2_client = boto3.client("ec2", region_name="us-west-2") + random_ip = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet( VpcId=vpc.id, CidrBlock="10.0.0.0/24", AvailabilityZone="us-west-2a" ) - sg = ec2_client.create_security_group(Description="test", GroupName="test_sg") + sg = ec2_client.create_security_group(Description="test", GroupName=str(uuid4())) sg_id = sg["GroupId"] eni1 = ec2.create_network_interface( SubnetId=subnet.id, - PrivateIpAddress="10.0.10.5", - Description="test interface", + PrivateIpAddress=random_ip, + Description=str(uuid4()), Groups=[sg_id], ) @@ -740,6 +762,9 @@ def test_elastic_network_interfaces_filter_by_tag(): VpcId=vpc.id, CidrBlock="10.0.0.0/24", AvailabilityZone="us-west-2a" ) + dev_env = f"dev-{str(uuid4())[0:4]}" + prod_env = f"prod-{str(uuid4())[0:4]}" + eni_dev = ec2.create_network_interface( SubnetId=subnet.id, PrivateIpAddress="10.0.10.5", @@ -747,7 +772,7 @@ def test_elastic_network_interfaces_filter_by_tag(): TagSpecifications=[ { "ResourceType": "network-interface", - "Tags": [{"Key": "environment", "Value": "dev"}], + "Tags": [{"Key": "environment", "Value": dev_env}], }, ], ) @@ -759,7 +784,7 @@ def test_elastic_network_interfaces_filter_by_tag(): TagSpecifications=[ { "ResourceType": "network-interface", - "Tags": [{"Key": "environment", "Value": "prod"}], + "Tags": [{"Key": "environment", "Value": prod_env}], }, ], ) @@ -774,19 +799,19 @@ def test_elastic_network_interfaces_filter_by_tag(): resp["NetworkInterfaces"].should.have.length_of(0) resp = ec2_client.describe_network_interfaces( - Filters=[{"Name": "tag:environment", "Values": ["dev"]}] + Filters=[{"Name": "tag:environment", "Values": [dev_env]}] ) resp["NetworkInterfaces"].should.have.length_of(1) resp["NetworkInterfaces"][0]["Description"].should.equal("dev interface") resp = ec2_client.describe_network_interfaces( - Filters=[{"Name": "tag:environment", "Values": ["prod"]}] + Filters=[{"Name": "tag:environment", "Values": [prod_env]}] ) resp["NetworkInterfaces"].should.have.length_of(1) resp["NetworkInterfaces"][0]["Description"].should.equal("prod interface") resp = ec2_client.describe_network_interfaces( - Filters=[{"Name": "tag:environment", "Values": ["dev", "prod"]}] + Filters=[{"Name": "tag:environment", "Values": [dev_env, prod_env]}] ) resp["NetworkInterfaces"].should.have.length_of(2) diff --git a/tests/test_ec2/test_flow_logs.py b/tests/test_ec2/test_flow_logs.py index 6ea72dcf0..5debda70b 100644 --- a/tests/test_ec2/test_flow_logs.py +++ b/tests/test_ec2/test_flow_logs.py @@ -16,6 +16,7 @@ from moto import ( ) from moto.core import ACCOUNT_ID from moto.ec2.exceptions import FilterNotImplementedError +from uuid import uuid4 @mock_s3 @@ -26,8 +27,9 @@ def test_create_flow_logs_s3(): vpc = client.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] + bucket_name = str(uuid4()) bucket = s3.create_bucket( - Bucket="test-flow-logs", + Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "us-west-1"}, ) @@ -55,7 +57,7 @@ def test_create_flow_logs_s3(): )["FlowLogIds"] response.should.have.length_of(1) - flow_logs = client.describe_flow_logs()["FlowLogs"] + flow_logs = client.describe_flow_logs(FlowLogIds=[response[0]])["FlowLogs"] flow_logs.should.have.length_of(1) flow_log = flow_logs[0] @@ -80,7 +82,8 @@ def test_create_flow_logs_cloud_watch(): logs_client = boto3.client("logs", region_name="us-west-1") vpc = client.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] - logs_client.create_log_group(logGroupName="test-group") + lg_name = str(uuid4()) + logs_client.create_log_group(logGroupName=lg_name) with pytest.raises(ClientError) as ex: client.create_flow_logs( @@ -88,7 +91,7 @@ def test_create_flow_logs_cloud_watch(): ResourceIds=[vpc["VpcId"]], TrafficType="ALL", LogDestinationType="cloud-watch-logs", - LogGroupName="test-group", + LogGroupName=lg_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role", DryRun=True, ) @@ -103,12 +106,14 @@ def test_create_flow_logs_cloud_watch(): ResourceIds=[vpc["VpcId"]], TrafficType="ALL", LogDestinationType="cloud-watch-logs", - LogGroupName="test-group", + LogGroupName=lg_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role", )["FlowLogIds"] response.should.have.length_of(1) - flow_logs = client.describe_flow_logs()["FlowLogs"] + flow_logs = client.describe_flow_logs( + Filters=[{"Name": "resource-id", "Values": [vpc["VpcId"]]}] + )["FlowLogs"] flow_logs.should.have.length_of(1) flow_log = flow_logs[0] @@ -119,7 +124,7 @@ def test_create_flow_logs_cloud_watch(): flow_log["ResourceId"].should.equal(vpc["VpcId"]) flow_log["TrafficType"].should.equal("ALL") flow_log["LogDestinationType"].should.equal("cloud-watch-logs") - flow_log["LogGroupName"].should.equal("test-group") + flow_log["LogGroupName"].should.equal(lg_name) flow_log["DeliverLogsPermissionArn"].should.equal( "arn:aws:iam::" + ACCOUNT_ID + ":role/test-role" ) @@ -139,7 +144,7 @@ def test_create_flow_log_create(): vpc2 = client.create_vpc(CidrBlock="10.1.0.0/16")["Vpc"] bucket = s3.create_bucket( - Bucket="test-flow-logs", + Bucket=str(uuid4()), CreateBucketConfiguration={"LocationConstraint": "us-west-1",}, ) @@ -153,7 +158,7 @@ def test_create_flow_log_create(): )["FlowLogIds"] response.should.have.length_of(2) - flow_logs = client.describe_flow_logs()["FlowLogs"] + flow_logs = client.describe_flow_logs(FlowLogIds=response)["FlowLogs"] flow_logs.should.have.length_of(2) flow_logs[0]["LogFormat"].should.equal( @@ -174,7 +179,7 @@ def test_delete_flow_logs(): vpc2 = client.create_vpc(CidrBlock="10.1.0.0/16")["Vpc"] bucket = s3.create_bucket( - Bucket="test-flow-logs", + Bucket=str(uuid4()), CreateBucketConfiguration={"LocationConstraint": "us-west-1"}, ) @@ -187,18 +192,19 @@ def test_delete_flow_logs(): )["FlowLogIds"] response.should.have.length_of(2) - flow_logs = client.describe_flow_logs()["FlowLogs"] + filters = [{"Name": "resource-id", "Values": [vpc1["VpcId"], vpc2["VpcId"]]}] + flow_logs = client.describe_flow_logs(Filters=filters)["FlowLogs"] flow_logs.should.have.length_of(2) client.delete_flow_logs(FlowLogIds=[response[0]]) - flow_logs = client.describe_flow_logs()["FlowLogs"] + flow_logs = client.describe_flow_logs(Filters=filters)["FlowLogs"] flow_logs.should.have.length_of(1) flow_logs[0]["FlowLogId"].should.equal(response[1]) client.delete_flow_logs(FlowLogIds=[response[1]]) - flow_logs = client.describe_flow_logs()["FlowLogs"] + flow_logs = client.describe_flow_logs(Filters=filters)["FlowLogs"] flow_logs.should.have.length_of(0) @@ -212,7 +218,7 @@ def test_delete_flow_logs_delete_many(): vpc2 = client.create_vpc(CidrBlock="10.1.0.0/16")["Vpc"] bucket = s3.create_bucket( - Bucket="test-flow-logs", + Bucket=str(uuid4()), CreateBucketConfiguration={"LocationConstraint": "us-west-1"}, ) @@ -225,13 +231,15 @@ def test_delete_flow_logs_delete_many(): )["FlowLogIds"] response.should.have.length_of(2) - flow_logs = client.describe_flow_logs()["FlowLogs"] - flow_logs.should.have.length_of(2) + all_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client)] + for fl_id in response: + all_ids.should.contain(fl_id) client.delete_flow_logs(FlowLogIds=response) - flow_logs = client.describe_flow_logs()["FlowLogs"] - flow_logs.should.have.length_of(0) + all_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client)] + for fl_id in response: + all_ids.shouldnt.contain(fl_id) @mock_ec2 @@ -295,7 +303,7 @@ def test_create_flow_logs_invalid_parameters(): vpc = client.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] bucket = s3.create_bucket( - Bucket="test-flow-logs", + Bucket=str(uuid4()), CreateBucketConfiguration={"LocationConstraint": "us-west-1"}, ) @@ -377,11 +385,12 @@ def test_create_flow_logs_invalid_parameters(): "Error. There is an existing Flow Log with the same configuration and log destination." ) + lg_name = str(uuid4()) response = client.create_flow_logs( ResourceType="VPC", ResourceIds=[vpc["VpcId"]], TrafficType="ALL", - LogGroupName="test-group", + LogGroupName=lg_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role", )["FlowLogIds"] response.should.have.length_of(1) @@ -391,7 +400,7 @@ def test_create_flow_logs_invalid_parameters(): ResourceType="VPC", ResourceIds=[vpc["VpcId"]], TrafficType="ALL", - LogGroupName="test-group", + LogGroupName=lg_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role", ) ex.value.response["Error"]["Code"].should.equal("FlowLogAlreadyExists") @@ -400,9 +409,6 @@ def test_create_flow_logs_invalid_parameters(): "Error. There is an existing Flow Log with the same configuration and log destination." ) - flow_logs = client.describe_flow_logs()["FlowLogs"] - flow_logs.should.have.length_of(2) - @mock_s3 @mock_ec2 @@ -421,20 +427,22 @@ def test_describe_flow_logs_filtering(): ] bucket1 = s3.create_bucket( - Bucket="test-flow-logs-1", + Bucket=str(uuid4()), CreateBucketConfiguration={"LocationConstraint": "us-west-1"}, ) - logs_client.create_log_group(logGroupName="test-group") + lg_name = str(uuid4()) + logs_client.create_log_group(logGroupName=lg_name) fl1 = client.create_flow_logs( ResourceType="Subnet", ResourceIds=[subnet1["SubnetId"]], TrafficType="ALL", - LogGroupName="test-group", + LogGroupName=lg_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role", )["FlowLogIds"][0] + tag_key = str(uuid4())[0:6] fl2 = client.create_flow_logs( ResourceType="VPC", ResourceIds=[vpc2["VpcId"]], @@ -442,56 +450,60 @@ def test_describe_flow_logs_filtering(): LogDestinationType="s3", LogDestination="arn:aws:s3:::" + bucket1.name, TagSpecifications=[ - {"ResourceType": "vpc-flow-log", "Tags": [{"Key": "foo", "Value": "bar"}]} + {"ResourceType": "vpc-flow-log", "Tags": [{"Key": tag_key, "Value": "bar"}]} ], )["FlowLogIds"][0] + non_existing_group = str(uuid4()) fl3 = client.create_flow_logs( ResourceType="VPC", ResourceIds=[vpc3["VpcId"]], TrafficType="Reject", - LogGroupName="non-existing-group", + LogGroupName=non_existing_group, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role", )["FlowLogIds"][0] - all_flow_logs = client.describe_flow_logs()["FlowLogs"] - all_flow_logs.should.have.length_of(3) + all_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client)] + all_ids.should.contain(fl1) + all_ids.should.contain(fl2) + all_ids.should.contain(fl3) - fl_by_deliver_status = client.describe_flow_logs( - Filters=[{"Name": "deliver-log-status", "Values": ["SUCCESS"]}], - )["FlowLogs"] - fl_by_deliver_status.should.have.length_of(3) + filters = [{"Name": "deliver-log-status", "Values": ["SUCCESS"]}] + success_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client, filters)] + success_ids.should.contain(fl1) + success_ids.should.contain(fl2) + success_ids.should.contain(fl3) - fl_by_s3_bucket = client.describe_flow_logs( - Filters=[{"Name": "log-destination-type", "Values": ["s3"]}], - )["FlowLogs"] - fl_by_s3_bucket.should.have.length_of(1) - fl_by_s3_bucket[0]["FlowLogId"].should.equal(fl2) - fl_by_s3_bucket[0]["ResourceId"].should.equal(vpc2["VpcId"]) + filters = [{"Name": "log-destination-type", "Values": ["s3"]}] + all_s3_logs = retrieve_all_logs(client, filters) + s3_ids = [fl["FlowLogId"] for fl in all_s3_logs] + s3_ids.shouldnt.contain(fl1) + s3_ids.should.contain(fl2) + s3_ids.shouldnt.contain(fl3) + our_flow_log = [fl for fl in all_s3_logs if fl["FlowLogId"] == fl2][0] + our_flow_log["ResourceId"].should.equal(vpc2["VpcId"]) - fl_by_cloud_watch = client.describe_flow_logs( - Filters=[{"Name": "log-destination-type", "Values": ["cloud-watch-logs"]}], - )["FlowLogs"] - fl_by_cloud_watch.should.have.length_of(2) + filters = [{"Name": "log-destination-type", "Values": ["cloud-watch-logs"]}] + all_cw_logs = retrieve_all_logs(client, filters) + cw_ids = [fl["FlowLogId"] for fl in all_cw_logs] - flow_logs_ids = tuple(map(lambda fl: fl["FlowLogId"], fl_by_cloud_watch)) - fl1.should.be.within(flow_logs_ids) - fl3.should.be.within(flow_logs_ids) + fl1.should.be.within(cw_ids) + fl2.shouldnt.be.within(cw_ids) + fl3.should.be.within(cw_ids) - flow_logs_resource_ids = tuple(map(lambda fl: fl["ResourceId"], fl_by_cloud_watch)) + flow_logs_resource_ids = tuple(map(lambda fl: fl["ResourceId"], all_cw_logs)) subnet1["SubnetId"].should.be.within(flow_logs_resource_ids) vpc3["VpcId"].should.be.within(flow_logs_resource_ids) - test_fl3 = next(fl for fl in fl_by_cloud_watch if fl["FlowLogId"] == fl3) + test_fl3 = next(fl for fl in all_cw_logs if fl["FlowLogId"] == fl3) test_fl3["DeliverLogsStatus"].should.equal("FAILED") test_fl3["DeliverLogsErrorMessage"].should.equal("Access error") - fl_by_both = client.describe_flow_logs( - Filters=[ - {"Name": "log-destination-type", "Values": ["cloud-watch-logs", "s3"]} - ], - )["FlowLogs"] - fl_by_both.should.have.length_of(3) + filters = [{"Name": "log-destination-type", "Values": ["cloud-watch-logs", "s3"]}] + cw_s3_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client, filters)] + cw_s3_ids.should.contain(fl1) + cw_s3_ids.should.contain(fl2) + cw_s3_ids.should.contain(fl3) fl_by_flow_log_ids = client.describe_flow_logs( Filters=[{"Name": "flow-log-id", "Values": [fl1, fl3]}], @@ -506,14 +518,14 @@ def test_describe_flow_logs_filtering(): vpc3["VpcId"].should.be.within(flow_logs_resource_ids) fl_by_group_name = client.describe_flow_logs( - Filters=[{"Name": "log-group-name", "Values": ["test-group"]}], + Filters=[{"Name": "log-group-name", "Values": [lg_name]}], )["FlowLogs"] fl_by_group_name.should.have.length_of(1) fl_by_group_name[0]["FlowLogId"].should.equal(fl1) fl_by_group_name[0]["ResourceId"].should.equal(subnet1["SubnetId"]) fl_by_group_name = client.describe_flow_logs( - Filters=[{"Name": "log-group-name", "Values": ["non-existing-group"]}], + Filters=[{"Name": "log-group-name", "Values": [non_existing_group]}], )["FlowLogs"] fl_by_group_name.should.have.length_of(1) fl_by_group_name[0]["FlowLogId"].should.equal(fl3) @@ -526,29 +538,26 @@ def test_describe_flow_logs_filtering(): fl_by_resource_id[0]["FlowLogId"].should.equal(fl2) fl_by_resource_id[0]["ResourceId"].should.equal(vpc2["VpcId"]) - fl_by_traffic_type = client.describe_flow_logs( - Filters=[{"Name": "traffic-type", "Values": ["ALL"]}], - )["FlowLogs"] - fl_by_traffic_type.should.have.length_of(1) - fl_by_traffic_type[0]["FlowLogId"].should.equal(fl1) - fl_by_traffic_type[0]["ResourceId"].should.equal(subnet1["SubnetId"]) + filters = [{"Name": "traffic-type", "Values": ["ALL"]}] + traffic_all = retrieve_all_logs(client, filters) + [fl["FlowLogId"] for fl in traffic_all].should.contain(fl1) + our_flow_log = [fl for fl in traffic_all if fl["FlowLogId"] == fl1][0] + our_flow_log["ResourceId"].should.equal(subnet1["SubnetId"]) - fl_by_traffic_type = client.describe_flow_logs( - Filters=[{"Name": "traffic-type", "Values": ["Reject"]}], - )["FlowLogs"] - fl_by_traffic_type.should.have.length_of(1) - fl_by_traffic_type[0]["FlowLogId"].should.equal(fl3) - fl_by_traffic_type[0]["ResourceId"].should.equal(vpc3["VpcId"]) + filters = [{"Name": "traffic-type", "Values": ["Reject"]}] + traffic_reject = retrieve_all_logs(client, filters) + [fl["FlowLogId"] for fl in traffic_reject].should.contain(fl3) + our_flow_log = [fl for fl in traffic_reject if fl["FlowLogId"] == fl3][0] + our_flow_log["ResourceId"].should.equal(vpc3["VpcId"]) - fl_by_traffic_type = client.describe_flow_logs( - Filters=[{"Name": "traffic-type", "Values": ["Accept"]}], - )["FlowLogs"] - fl_by_traffic_type.should.have.length_of(1) - fl_by_traffic_type[0]["FlowLogId"].should.equal(fl2) - fl_by_traffic_type[0]["ResourceId"].should.equal(vpc2["VpcId"]) + filters = [{"Name": "traffic-type", "Values": ["Accept"]}] + traffic_accept = retrieve_all_logs(client, filters) + [fl["FlowLogId"] for fl in traffic_accept].should.contain(fl2) + our_flow_log = [fl for fl in traffic_accept if fl["FlowLogId"] == fl2][0] + our_flow_log["ResourceId"].should.equal(vpc2["VpcId"]) fl_by_tag_key = client.describe_flow_logs( - Filters=[{"Name": "tag-key", "Values": ["foo"]}], + Filters=[{"Name": "tag-key", "Values": [tag_key]}], )["FlowLogs"] fl_by_tag_key.should.have.length_of(1) fl_by_tag_key[0]["FlowLogId"].should.equal(fl2) @@ -579,19 +588,21 @@ def test_flow_logs_by_ids(): vpc2 = client.create_vpc(CidrBlock="10.1.0.0/16")["Vpc"] vpc3 = client.create_vpc(CidrBlock="10.2.0.0/16")["Vpc"] + lg1_name = str(uuid4()) fl1 = client.create_flow_logs( ResourceType="VPC", ResourceIds=[vpc1["VpcId"]], TrafficType="Reject", - LogGroupName="test-group-1", + LogGroupName=lg1_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role-1", )["FlowLogIds"][0] + lg3_name = str(uuid4()) fl2 = client.create_flow_logs( ResourceType="VPC", ResourceIds=[vpc2["VpcId"]], TrafficType="Reject", - LogGroupName="test-group-3", + LogGroupName=lg3_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role-3", )["FlowLogIds"][0] @@ -599,7 +610,7 @@ def test_flow_logs_by_ids(): ResourceType="VPC", ResourceIds=[vpc3["VpcId"]], TrafficType="Reject", - LogGroupName="test-group-3", + LogGroupName=lg3_name, DeliverLogsPermissionArn="arn:aws:iam::" + ACCOUNT_ID + ":role/test-role-3", )["FlowLogIds"][0] @@ -618,11 +629,25 @@ def test_flow_logs_by_ids(): flow_logs = client.describe_flow_logs(FlowLogIds=[fl1, fl3])["FlowLogs"] flow_logs.should.have.length_of(0) - flow_logs = client.describe_flow_logs()["FlowLogs"] - flow_logs.should.have.length_of(1) - flow_logs[0]["FlowLogId"].should.equal(fl2) - flow_logs[0]["ResourceId"].should.equal(vpc2["VpcId"]) + all_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client)] + all_ids.shouldnt.contain(fl1) + all_ids.should.contain(fl2) + all_ids.shouldnt.contain(fl3) flow_logs = client.delete_flow_logs(FlowLogIds=[fl2]) - flow_logs = client.describe_flow_logs()["FlowLogs"] - flow_logs.should.have.length_of(0) + + all_ids = [fl["FlowLogId"] for fl in retrieve_all_logs(client)] + all_ids.shouldnt.contain(fl1) + all_ids.shouldnt.contain(fl2) + all_ids.shouldnt.contain(fl3) + + +def retrieve_all_logs(client, filters=[]): + resp = client.describe_flow_logs(Filters=filters) + all_logs = resp["FlowLogs"] + token = resp.get("NextToken") + while token: + resp = client.describe_flow_logs(Filters=filters, NextToken=token) + all_logs.extend(resp["FlowLogs"]) + token = resp.get("NextToken") + return all_logs diff --git a/tests/test_ec2/test_flow_logs_cloudformation.py b/tests/test_ec2/test_flow_logs_cloudformation.py index baa86cd1d..af69a6f1c 100644 --- a/tests/test_ec2/test_flow_logs_cloudformation.py +++ b/tests/test_ec2/test_flow_logs_cloudformation.py @@ -10,7 +10,9 @@ from moto import ( mock_ec2, mock_s3, ) +from moto.core import ACCOUNT_ID from tests import EXAMPLE_AMI_ID +from uuid import uuid4 @mock_cloudformation @@ -23,8 +25,9 @@ def test_flow_logs_by_cloudformation(): vpc = client.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] + bucket_name = str(uuid4()) bucket = s3.create_bucket( - Bucket="test-flow-logs", + Bucket=bucket_name, CreateBucketConfiguration={"LocationConstraint": "us-west-1"}, ) @@ -47,11 +50,14 @@ def test_flow_logs_by_cloudformation(): }, } flow_log_template_json = json.dumps(flow_log_template) + stack_name = str(uuid4()) stack_id = cf_client.create_stack( - StackName="test_stack", TemplateBody=flow_log_template_json + StackName=stack_name, TemplateBody=flow_log_template_json )["StackId"] - flow_logs = client.describe_flow_logs()["FlowLogs"] + flow_logs = client.describe_flow_logs( + Filters=[{"Name": "resource-id", "Values": [vpc["VpcId"]]}] + )["FlowLogs"] flow_logs.should.have.length_of(1) flow_logs[0]["ResourceId"].should.equal(vpc["VpcId"]) flow_logs[0]["LogDestination"].should.equal("arn:aws:s3:::" + bucket.name) @@ -81,15 +87,26 @@ def test_cloudformation(): client = boto3.client("ec2", region_name="us-east-1") cf_conn = boto3.client("cloudformation", region_name="us-east-1") + stack_name = str(uuid4()) cf_conn.create_stack( - StackName="test_stack", TemplateBody=json.dumps(dummy_template_json) + StackName=stack_name, TemplateBody=json.dumps(dummy_template_json) ) - associations = client.describe_iam_instance_profile_associations() - associations["IamInstanceProfileAssociations"].should.have.length_of(1) - associations["IamInstanceProfileAssociations"][0]["IamInstanceProfile"][ - "Arn" - ].should.contain("test_stack") - cf_conn.delete_stack(StackName="test_stack") - associations = client.describe_iam_instance_profile_associations() - associations["IamInstanceProfileAssociations"].should.have.length_of(0) + resources = cf_conn.list_stack_resources(StackName=stack_name)[ + "StackResourceSummaries" + ] + iam_id = resources[0]["PhysicalResourceId"] + iam_ip_arn = f"arn:aws:iam::{ACCOUNT_ID}:instance-profile/{iam_id}" + + all_assocs = client.describe_iam_instance_profile_associations()[ + "IamInstanceProfileAssociations" + ] + our_assoc = [a for a in all_assocs if a["IamInstanceProfile"]["Arn"] == iam_ip_arn] + our_assoc[0]["IamInstanceProfile"]["Arn"].should.contain(stack_name) + our_assoc_id = our_assoc[0]["AssociationId"] + + cf_conn.delete_stack(StackName=stack_name) + associations = client.describe_iam_instance_profile_associations()[ + "IamInstanceProfileAssociations" + ] + [a["AssociationId"] for a in associations].shouldnt.contain(our_assoc_id) diff --git a/tests/test_ec2/test_iam_integration.py b/tests/test_ec2/test_iam_integration.py index 0e7b0f142..207901ffe 100644 --- a/tests/test_ec2/test_iam_integration.py +++ b/tests/test_ec2/test_iam_integration.py @@ -10,6 +10,7 @@ import sure # noqa from moto import mock_ec2, mock_iam, mock_cloudformation from tests import EXAMPLE_AMI_ID +from uuid import uuid4 def quick_instance_creation(): @@ -35,7 +36,7 @@ def test_associate(): client = boto3.client("ec2", region_name="us-east-1") instance_id = quick_instance_creation() instance_profile_arn, instance_profile_name = quick_instance_profile_creation( - "test_profile" + str(uuid4()) ) association = client.associate_iam_instance_profile( @@ -58,7 +59,7 @@ def test_invalid_associate(): client = boto3.client("ec2", region_name="us-east-1") instance_id = quick_instance_creation() instance_profile_arn, instance_profile_name = quick_instance_profile_creation( - "test_profile" + str(uuid4()) ) client.associate_iam_instance_profile( @@ -109,81 +110,74 @@ def test_invalid_associate(): def test_describe(): client = boto3.client("ec2", region_name="us-east-1") - instance_id = quick_instance_creation() - instance_profile_arn, instance_profile_name = quick_instance_profile_creation( - "test_profile" + instance_id1 = quick_instance_creation() + instance_profile_arn1, instance_profile_name1 = quick_instance_profile_creation( + str(uuid4()) ) client.associate_iam_instance_profile( IamInstanceProfile={ - "Arn": instance_profile_arn, - "Name": instance_profile_name, + "Arn": instance_profile_arn1, + "Name": instance_profile_name1, }, - InstanceId=instance_id, + InstanceId=instance_id1, ) associations = client.describe_iam_instance_profile_associations() - associations["IamInstanceProfileAssociations"].should.have.length_of(1) - associations["IamInstanceProfileAssociations"][0]["InstanceId"].should.equal( - instance_id - ) - associations["IamInstanceProfileAssociations"][0]["IamInstanceProfile"][ - "Arn" - ].should.equal(instance_profile_arn) - associations["IamInstanceProfileAssociations"][0]["State"].should.equal( - "associated" + associations = associations["IamInstanceProfileAssociations"] + [a["IamInstanceProfile"]["Arn"] for a in associations].should.contain( + instance_profile_arn1 ) + my_assoc = [ + a + for a in associations + if a["IamInstanceProfile"]["Arn"] == instance_profile_arn1 + ][0] + my_assoc["InstanceId"].should.equal(instance_id1) + my_assoc["State"].should.equal("associated") - instance_id = quick_instance_creation() - instance_profile_arn, instance_profile_name = quick_instance_profile_creation( - "test_profile1" + instance_id2 = quick_instance_creation() + instance_profile_arn2, instance_profile_name2 = quick_instance_profile_creation( + str(uuid4()) ) client.associate_iam_instance_profile( IamInstanceProfile={ - "Arn": instance_profile_arn, - "Name": instance_profile_name, + "Arn": instance_profile_arn2, + "Name": instance_profile_name2, }, - InstanceId=instance_id, + InstanceId=instance_id2, ) - next_test_associations = client.describe_iam_instance_profile_associations() - next_test_associations["IamInstanceProfileAssociations"].should.have.length_of(2) + associations = client.describe_iam_instance_profile_associations() + associations = associations["IamInstanceProfileAssociations"] + [a["IamInstanceProfile"]["Arn"] for a in associations].should.contain( + instance_profile_arn1 + ) + [a["IamInstanceProfile"]["Arn"] for a in associations].should.contain( + instance_profile_arn2 + ) + my_assoc = [ + a + for a in associations + if a["IamInstanceProfile"]["Arn"] == instance_profile_arn1 + ][0] associations = client.describe_iam_instance_profile_associations( - AssociationIds=[ - next_test_associations["IamInstanceProfileAssociations"][0][ - "AssociationId" - ], - ] + AssociationIds=[my_assoc["AssociationId"],] ) associations["IamInstanceProfileAssociations"].should.have.length_of(1) associations["IamInstanceProfileAssociations"][0]["IamInstanceProfile"][ "Arn" - ].should.equal( - next_test_associations["IamInstanceProfileAssociations"][0][ - "IamInstanceProfile" - ]["Arn"] - ) + ].should.equal(my_assoc["IamInstanceProfile"]["Arn"]) associations = client.describe_iam_instance_profile_associations( Filters=[ - { - "Name": "instance-id", - "Values": [ - next_test_associations["IamInstanceProfileAssociations"][0][ - "InstanceId" - ], - ], - }, + {"Name": "instance-id", "Values": [my_assoc["InstanceId"],],}, {"Name": "state", "Values": ["associated"]}, ] ) associations["IamInstanceProfileAssociations"].should.have.length_of(1) associations["IamInstanceProfileAssociations"][0]["IamInstanceProfile"][ "Arn" - ].should.equal( - next_test_associations["IamInstanceProfileAssociations"][0][ - "IamInstanceProfile" - ]["Arn"] - ) + ].should.equal(my_assoc["IamInstanceProfile"]["Arn"]) @mock_ec2 @@ -192,10 +186,10 @@ def test_replace(): client = boto3.client("ec2", region_name="us-east-1") instance_id1 = quick_instance_creation() instance_profile_arn1, instance_profile_name1 = quick_instance_profile_creation( - "test_profile1" + str(uuid4()) ) instance_profile_arn2, instance_profile_name2 = quick_instance_profile_creation( - "test_profile2" + str(uuid4()) ) association = client.associate_iam_instance_profile( @@ -226,10 +220,10 @@ def test_invalid_replace(): client = boto3.client("ec2", region_name="us-east-1") instance_id = quick_instance_creation() instance_profile_arn, instance_profile_name = quick_instance_profile_creation( - "test_profile" + str(uuid4()) ) instance_profile_arn2, instance_profile_name2 = quick_instance_profile_creation( - "test_profile2" + str(uuid4()) ) association = client.associate_iam_instance_profile( @@ -268,7 +262,7 @@ def test_disassociate(): client = boto3.client("ec2", region_name="us-east-1") instance_id = quick_instance_creation() instance_profile_arn, instance_profile_name = quick_instance_profile_creation( - "test_profile" + str(uuid4()) ) association = client.associate_iam_instance_profile( @@ -280,7 +274,10 @@ def test_disassociate(): ) associations = client.describe_iam_instance_profile_associations() - associations["IamInstanceProfileAssociations"].should.have.length_of(1) + associations = associations["IamInstanceProfileAssociations"] + [a["IamInstanceProfile"]["Arn"] for a in associations].should.contain( + instance_profile_arn + ) disassociation = client.disassociate_iam_instance_profile( AssociationId=association["IamInstanceProfileAssociation"]["AssociationId"], @@ -294,7 +291,10 @@ def test_disassociate(): ) associations = client.describe_iam_instance_profile_associations() - associations["IamInstanceProfileAssociations"].should.have.length_of(0) + associations = associations["IamInstanceProfileAssociations"] + [a["IamInstanceProfile"]["Arn"] for a in associations].shouldnt.contain( + instance_profile_arn + ) @mock_ec2 diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index b9d539090..260c74a21 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -16,8 +16,10 @@ from freezegun import freeze_time import sure # noqa from moto import mock_ec2_deprecated, mock_ec2, settings +from moto.core import ACCOUNT_ID from tests import EXAMPLE_AMI_ID from tests.helpers import requires_boto_gte +from uuid import uuid4 decode_method = base64.decodebytes @@ -44,10 +46,13 @@ def test_add_servers(): @mock_ec2 def test_add_servers_boto3(): client = boto3.client("ec2", region_name="us-east-1") - client.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=2, MaxCount=2) + resp = client.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=2, MaxCount=2) + for i in resp["Instances"]: + i["ImageId"].should.equal(EXAMPLE_AMI_ID) - reservations = client.describe_instances()["Reservations"] - instances = reservations[0]["Instances"] + instances = client.describe_instances( + InstanceIds=[i["InstanceId"] for i in resp["Instances"]] + )["Reservations"][0]["Instances"] instances.should.have.length_of(2) for i in instances: i["ImageId"].should.equal(EXAMPLE_AMI_ID) @@ -133,7 +138,7 @@ def test_instance_launch_and_terminate_boto3(): instance["State"].should.equal({"Code": 0, "Name": "pending"}) instance_id = instance["InstanceId"] - reservations = client.describe_instances()["Reservations"] + reservations = client.describe_instances(InstanceIds=[instance_id])["Reservations"] reservations.should.have.length_of(1) reservations[0]["ReservationId"].should.equal(reservation["ReservationId"]) instances = reservations[0]["Instances"] @@ -171,7 +176,7 @@ def test_instance_launch_and_terminate_boto3(): client.terminate_instances(InstanceIds=[instance_id]) - reservations = client.describe_instances()["Reservations"] + reservations = client.describe_instances(InstanceIds=[instance_id])["Reservations"] instance = reservations[0]["Instances"][0] instance["State"].should.equal({"Code": 48, "Name": "terminated"}) @@ -201,7 +206,9 @@ def test_instance_terminate_discard_volumes(): instance.terminate() instance.wait_until_terminated() - assert not list(ec2_resource.volumes.all()) + all_volumes_ids = [v.id for v in list(ec2_resource.volumes.all())] + for my_id in instance_volume_ids: + all_volumes_ids.shouldnt.contain(my_id) @mock_ec2 @@ -229,7 +236,9 @@ def test_instance_terminate_keep_volumes_explicit(): instance.terminate() instance.wait_until_terminated() - assert len(list(ec2_resource.volumes.all())) == 1 + all_volumes_ids = [v.id for v in list(ec2_resource.volumes.all())] + for my_id in instance_volume_ids: + all_volumes_ids.should.contain(my_id) @mock_ec2 @@ -269,14 +278,18 @@ def test_instance_terminate_detach_volumes(): ], ) instance = result[0] + my_volume_ids = [] for volume in instance.volumes.all(): + my_volume_ids.append(volume.volume_id) response = instance.detach_volume(VolumeId=volume.volume_id) response["State"].should.equal("detaching") instance.terminate() instance.wait_until_terminated() - assert len(list(ec2_resource.volumes.all())) == 2 + all_volumes_ids = [v.id for v in list(ec2_resource.volumes.all())] + for my_id in my_volume_ids: + all_volumes_ids.should.contain(my_id) @mock_ec2 @@ -452,17 +465,25 @@ def test_get_instances_by_id_boto3(): def test_get_paginated_instances(): client = boto3.client("ec2", region_name="us-east-1") conn = boto3.resource("ec2", "us-east-1") - for i in range(100): - conn.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) - resp = client.describe_instances(MaxResults=50) - reservations = resp["Reservations"] - reservations.should.have.length_of(50) - next_token = resp["NextToken"] + instances = [] + for i in range(12): + instances.extend( + conn.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) + ) + instance_ids = [i.id for i in instances] + + resp1 = client.describe_instances(InstanceIds=instance_ids, MaxResults=5) + res1 = resp1["Reservations"] + res1.should.have.length_of(5) + next_token = resp1["NextToken"] next_token.should_not.be.none - resp2 = client.describe_instances(NextToken=next_token) - reservations.extend(resp2["Reservations"]) - reservations.should.have.length_of(100) - assert "NextToken" not in resp2.keys() + + resp2 = client.describe_instances(InstanceIds=instance_ids, NextToken=next_token) + resp2["Reservations"].should.have.length_of(7) # 12 total - 5 from the first call + assert "NextToken" not in resp2 # This is it - no more pages + + for i in instances: + i.terminate() @mock_ec2 @@ -566,14 +587,15 @@ def test_get_instances_filtering_by_state_boto3(): client.terminate_instances(InstanceIds=[instance1.id]) - reservations = client.describe_instances( - Filters=[{"Name": "instance-state-name", "Values": ["running"]}] - )["Reservations"] - reservations.should.have.length_of(1) + instances = retrieve_all_instances( + client, [{"Name": "instance-state-name", "Values": ["running"]}] + ) + instance_ids = [i["InstanceId"] for i in instances] # Since we terminated instance1, only instance2 and instance3 should be # returned - instance_ids = [instance["InstanceId"] for instance in reservations[0]["Instances"]] - set(instance_ids).should.equal(set([instance2.id, instance3.id])) + instance_ids.shouldnt.contain(instance1.id) + instance_ids.should.contain(instance2.id) + instance_ids.should.contain(instance3.id) reservations = client.describe_instances( InstanceIds=[instance2.id], @@ -590,8 +612,11 @@ def test_get_instances_filtering_by_state_boto3(): reservations.should.equal([]) # get_all_reservations should still return all 3 - reservations = client.describe_instances()["Reservations"] - reservations[0]["Instances"].should.have.length_of(3) + instances = retrieve_all_instances(client, filters=[]) + instance_ids = [i["InstanceId"] for i in instances] + instance_ids.should.contain(instance1.id) + instance_ids.should.contain(instance2.id) + instance_ids.should.contain(instance3.id) if not settings.TEST_SERVER_MODE: # ServerMode will just throw a generic 500 @@ -703,33 +728,26 @@ def test_get_instances_filtering_by_instance_type_boto3(): ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, InstanceType="t1.micro" )[0] - res = client.describe_instances( - Filters=[{"Name": "instance-type", "Values": ["m1.small"]}] - )["Reservations"] - # get_all_reservations should return instance1,2 - res.should.have.length_of(2) - res[0]["Instances"].should.have.length_of(1) - res[1]["Instances"].should.have.length_of(1) - instance_ids = [r["Instances"][0]["InstanceId"] for r in res] - set(instance_ids).should.equal(set([instance1.id, instance2.id])) + instances = retrieve_all_instances( + client, [{"Name": "instance-type", "Values": ["m1.small"]}] + ) + instance_ids = [i["InstanceId"] for i in instances] + set(instance_ids).should.contain(instance1.id) + set(instance_ids).should.contain(instance2.id) - res = client.describe_instances( - Filters=[{"Name": "instance-type", "Values": ["t1.micro"]}] - )["Reservations"] - # get_all_reservations should return one - res.should.have.length_of(1) - res[0]["Instances"].should.have.length_of(1) - res[0]["Instances"][0]["InstanceId"].should.equal(instance3.id) + instances = retrieve_all_instances( + client, [{"Name": "instance-type", "Values": ["t1.micro"]}] + ) + instance_ids = [i["InstanceId"] for i in instances] + instance_ids.should.contain(instance3.id) - res = client.describe_instances( - Filters=[{"Name": "instance-type", "Values": ["t1.micro", "m1.small"]}] - )["Reservations"] - res.should.have.length_of(3) - res[0]["Instances"].should.have.length_of(1) - res[1]["Instances"].should.have.length_of(1) - res[2]["Instances"].should.have.length_of(1) - instance_ids = [r["Instances"][0]["InstanceId"] for r in res] - set(instance_ids).should.equal(set([instance1.id, instance2.id, instance3.id])) + instances = retrieve_all_instances( + client, [{"Name": "instance-type", "Values": ["t1.micro", "m1.small"]}] + ) + instance_ids = [i["InstanceId"] for i in instances] + instance_ids.should.contain(instance1.id) + instance_ids.should.contain(instance2.id) + instance_ids.should.contain(instance3.id) res = client.describe_instances( Filters=[{"Name": "instance-type", "Values": ["bogus"]}] @@ -770,23 +788,22 @@ def test_get_instances_filtering_by_reason_code_boto3(): instance1.stop() instance2.terminate() - res = client.describe_instances( - Filters=[ - {"Name": "state-reason-code", "Values": ["Client.UserInitiatedShutdown"]} - ] - )["Reservations"] - # get_all_reservations should return instance1 and instance2 - res[0]["Instances"].should.have.length_of(2) - set([instance1.id, instance2.id]).should.equal( - set([i["InstanceId"] for i in res[0]["Instances"]]) - ) + filters = [ + {"Name": "state-reason-code", "Values": ["Client.UserInitiatedShutdown"]} + ] + instances = retrieve_all_instances(client, filters) + instance_ids = [i["InstanceId"] for i in instances] - res = client.describe_instances( - Filters=[{"Name": "state-reason-code", "Values": [""]}] - )["Reservations"] - # get_all_reservations should return instance 3 - res[0]["Instances"].should.have.length_of(1) - res[0]["Instances"][0]["InstanceId"].should.equal(instance3.id) + instance_ids.should.contain(instance1.id) + instance_ids.should.contain(instance2.id) + instance_ids.shouldnt.contain(instance3.id) + + filters = [{"Name": "state-reason-code", "Values": [""]}] + instances = retrieve_all_instances(client, filters) + instance_ids = [i["InstanceId"] for i in instances] + instance_ids.should.contain(instance3.id) + instance_ids.shouldnt.contain(instance1.id) + instance_ids.shouldnt.contain(instance2.id) # Has boto3 equivalent @@ -823,18 +840,18 @@ def test_get_instances_filtering_by_source_dest_check_boto3(): InstanceId=instance1.id, SourceDestCheck={"Value": False} ) - check_false = client.describe_instances( - Filters=[{"Name": "source-dest-check", "Values": ["false"]}] - )["Reservations"] - check_true = client.describe_instances( - Filters=[{"Name": "source-dest-check", "Values": ["true"]}] - )["Reservations"] + instances_false = retrieve_all_instances( + client, [{"Name": "source-dest-check", "Values": ["false"]}] + ) + instances_true = retrieve_all_instances( + client, [{"Name": "source-dest-check", "Values": ["true"]}] + ) - check_false[0]["Instances"].should.have.length_of(1) - check_false[0]["Instances"][0]["InstanceId"].should.equal(instance1.id) + [i["InstanceId"] for i in instances_false].should.contain(instance1.id) + [i["InstanceId"] for i in instances_false].shouldnt.contain(instance2.id) - check_true[0]["Instances"].should.have.length_of(1) - check_true[0]["Instances"][0]["InstanceId"].should.equal(instance2.id) + [i["InstanceId"] for i in instances_true].shouldnt.contain(instance1.id) + [i["InstanceId"] for i in instances_true].should.contain(instance2.id) # Has boto3 equivalent @@ -937,20 +954,20 @@ def test_get_instances_filtering_by_image_id(): reservations = client.describe_instances( Filters=[{"Name": "image-id", "Values": [EXAMPLE_AMI_ID]}] )["Reservations"] - reservations[0]["Instances"].should.have.length_of(1) + assert len(reservations[0]["Instances"]) >= 1, "Should return just created instance" @mock_ec2 def test_get_instances_filtering_by_account_id(): client = boto3.client("ec2", region_name="us-east-1") conn = boto3.resource("ec2", "us-east-1") - conn.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) + instance = conn.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] - reservations = client.describe_instances( - Filters=[{"Name": "owner-id", "Values": ["123456789012"]}] - )["Reservations"] + instances = retrieve_all_instances( + client, filters=[{"Name": "owner-id", "Values": [ACCOUNT_ID]}] + ) - reservations[0]["Instances"].should.have.length_of(1) + [i["InstanceId"] for i in instances].should.contain(instance.id) @mock_ec2 @@ -987,12 +1004,13 @@ def test_get_instances_filtering_by_ni_private_dns(): @mock_ec2 def test_get_instances_filtering_by_instance_group_name(): client = boto3.client("ec2", region_name="us-east-1") - client.create_security_group(Description="test", GroupName="test_sg") + sec_group_name = str(uuid4())[0:6] + client.create_security_group(Description="test", GroupName=sec_group_name) client.run_instances( - ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=["test_sg"] + ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=[sec_group_name] ) reservations = client.describe_instances( - Filters=[{"Name": "instance.group-name", "Values": ["test_sg"]}] + Filters=[{"Name": "instance.group-name", "Values": [sec_group_name]}] )["Reservations"] reservations[0]["Instances"].should.have.length_of(1) @@ -1000,10 +1018,13 @@ def test_get_instances_filtering_by_instance_group_name(): @mock_ec2 def test_get_instances_filtering_by_instance_group_id(): client = boto3.client("ec2", region_name="us-east-1") - create_sg = client.create_security_group(Description="test", GroupName="test_sg") + sec_group_name = str(uuid4())[0:6] + create_sg = client.create_security_group( + Description="test", GroupName=sec_group_name + ) group_id = create_sg["GroupId"] client.run_instances( - ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=["test_sg"] + ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=[sec_group_name] ) reservations = client.describe_instances( Filters=[{"Name": "instance.group-id", "Values": [group_id]}] @@ -1088,11 +1109,17 @@ def test_get_instances_filtering_by_tag_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") reservation = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=3, MaxCount=3) instance1, instance2, instance3 = reservation - instance1.create_tags(Tags=[{"Key": "tag1", "Value": "value1"}]) - instance1.create_tags(Tags=[{"Key": "tag2", "Value": "value2"}]) - instance2.create_tags(Tags=[{"Key": "tag1", "Value": "value1"}]) - instance2.create_tags(Tags=[{"Key": "tag2", "Value": "wrong value"}]) - instance3.create_tags(Tags=[{"Key": "tag2", "Value": "value2"}]) + + tag1_name = str(uuid4())[0:6] + tag1_val = str(uuid4()) + tag2_name = str(uuid4())[0:6] + tag2_val = str(uuid4()) + + instance1.create_tags(Tags=[{"Key": tag1_name, "Value": tag1_val}]) + instance1.create_tags(Tags=[{"Key": tag2_name, "Value": tag2_val}]) + instance2.create_tags(Tags=[{"Key": tag1_name, "Value": tag1_val}]) + instance2.create_tags(Tags=[{"Key": tag2_name, "Value": "wrong value"}]) + instance3.create_tags(Tags=[{"Key": tag2_name, "Value": tag2_val}]) res = client.describe_instances( Filters=[{"Name": "tag:tag0", "Values": ["value0"]}] @@ -1101,7 +1128,7 @@ def test_get_instances_filtering_by_tag_boto3(): res["Reservations"].should.have.length_of(0) res = client.describe_instances( - Filters=[{"Name": "tag:tag1", "Values": ["value1"]}] + Filters=[{"Name": f"tag:{tag1_name}", "Values": [tag1_val]}] ) # describe_instances should return both instances with this tag value res["Reservations"].should.have.length_of(1) @@ -1111,8 +1138,8 @@ def test_get_instances_filtering_by_tag_boto3(): res = client.describe_instances( Filters=[ - {"Name": "tag:tag1", "Values": ["value1"]}, - {"Name": "tag:tag2", "Values": ["value2"]}, + {"Name": f"tag:{tag1_name}", "Values": [tag1_val]}, + {"Name": f"tag:{tag2_name}", "Values": [tag2_val]}, ] ) # describe_instances should return the instance with both tag values @@ -1121,7 +1148,7 @@ def test_get_instances_filtering_by_tag_boto3(): res["Reservations"][0]["Instances"][0]["InstanceId"].should.equal(instance1.id) res = client.describe_instances( - Filters=[{"Name": "tag:tag2", "Values": ["value2", "bogus"]}] + Filters=[{"Name": f"tag:{tag2_name}", "Values": [tag2_val, "bogus"]}] ) # describe_instances should return both instances with one of the # acceptable tag values @@ -1180,11 +1207,16 @@ def test_get_instances_filtering_by_tag_value_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") reservation = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=3, MaxCount=3) instance1, instance2, instance3 = reservation - instance1.create_tags(Tags=[{"Key": "tag1", "Value": "value1"}]) - instance1.create_tags(Tags=[{"Key": "tag2", "Value": "value2"}]) - instance2.create_tags(Tags=[{"Key": "tag1", "Value": "value1"}]) - instance2.create_tags(Tags=[{"Key": "tag2", "Value": "wrong value"}]) - instance3.create_tags(Tags=[{"Key": "tag2", "Value": "value2"}]) + + tag1_name = str(uuid4())[0:6] + tag1_val = str(uuid4()) + tag2_name = str(uuid4())[0:6] + tag2_val = str(uuid4()) + instance1.create_tags(Tags=[{"Key": tag1_name, "Value": tag1_val}]) + instance1.create_tags(Tags=[{"Key": tag2_name, "Value": tag2_val}]) + instance2.create_tags(Tags=[{"Key": tag1_name, "Value": tag1_val}]) + instance2.create_tags(Tags=[{"Key": tag2_name, "Value": "wrong value"}]) + instance3.create_tags(Tags=[{"Key": tag2_name, "Value": tag2_val}]) res = client.describe_instances( Filters=[{"Name": "tag-value", "Values": ["value0"]}] @@ -1193,7 +1225,7 @@ def test_get_instances_filtering_by_tag_value_boto3(): res["Reservations"].should.have.length_of(0) res = client.describe_instances( - Filters=[{"Name": "tag-value", "Values": ["value1"]}] + Filters=[{"Name": "tag-value", "Values": [tag1_val]}] ) # describe_instances should return both instances with this tag value res["Reservations"].should.have.length_of(1) @@ -1202,7 +1234,7 @@ def test_get_instances_filtering_by_tag_value_boto3(): res["Reservations"][0]["Instances"][1]["InstanceId"].should.equal(instance2.id) res = client.describe_instances( - Filters=[{"Name": "tag-value", "Values": ["value2", "value1"]}] + Filters=[{"Name": "tag-value", "Values": [tag2_val, tag1_val]}] ) # describe_instances should return both instances with one of the # acceptable tag values @@ -1213,7 +1245,7 @@ def test_get_instances_filtering_by_tag_value_boto3(): res["Reservations"][0]["Instances"][2]["InstanceId"].should.equal(instance3.id) res = client.describe_instances( - Filters=[{"Name": "tag-value", "Values": ["value2", "bogus"]}] + Filters=[{"Name": "tag-value", "Values": [tag2_val, "bogus"]}] ) # describe_instances should return both instances with one of the # acceptable tag values @@ -1262,17 +1294,21 @@ def test_get_instances_filtering_by_tag_name_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") reservation = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=3, MaxCount=3) instance1, instance2, instance3 = reservation - instance1.create_tags(Tags=[{"Key": "tag1", "Value": ""}]) + + tag1 = str(uuid4()) + tag3 = str(uuid4()) + + instance1.create_tags(Tags=[{"Key": tag1, "Value": ""}]) instance1.create_tags(Tags=[{"Key": "tag2", "Value": ""}]) - instance2.create_tags(Tags=[{"Key": "tag1", "Value": ""}]) + instance2.create_tags(Tags=[{"Key": tag1, "Value": ""}]) instance2.create_tags(Tags=[{"Key": "tag2X", "Value": ""}]) - instance3.create_tags(Tags=[{"Key": "tag3", "Value": ""}]) + instance3.create_tags(Tags=[{"Key": tag3, "Value": ""}]) res = client.describe_instances(Filters=[{"Name": "tag-key", "Values": ["tagX"]}]) # describe_instances should return no instances res["Reservations"].should.have.length_of(0) - res = client.describe_instances(Filters=[{"Name": "tag-key", "Values": ["tag1"]}]) + res = client.describe_instances(Filters=[{"Name": "tag-key", "Values": [tag1]}]) # describe_instances should return both instances with this tag value res["Reservations"].should.have.length_of(1) res["Reservations"][0]["Instances"].should.have.length_of(2) @@ -1280,7 +1316,7 @@ def test_get_instances_filtering_by_tag_name_boto3(): res["Reservations"][0]["Instances"][1]["InstanceId"].should.equal(instance2.id) res = client.describe_instances( - Filters=[{"Name": "tag-key", "Values": ["tag1", "tag3"]}] + Filters=[{"Name": "tag-key", "Values": [tag1, tag3]}] ) # describe_instances should return both instances with one of the # acceptable tag values @@ -1501,10 +1537,10 @@ def test_modify_instance_attribute_security_groups_boto3(): old_groups.should.equal([]) sg_id = ec2.create_security_group( - GroupName="test security group", Description="this is a test security group" + GroupName=str(uuid4()), Description="this is a test security group" ).id sg_id2 = ec2.create_security_group( - GroupName="test security group 2", Description="this is a test security group 2" + GroupName=str(uuid4()), Description="this is a test security group 2" ).id with pytest.raises(ClientError) as ex: @@ -1697,8 +1733,12 @@ def test_run_instance_with_security_group_name(): def test_run_instance_with_security_group_name_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") + sec_group_name = str(uuid4())[0:6] + with pytest.raises(ClientError) as ex: - ec2.create_security_group(GroupName="group1", Description="d", DryRun=True) + ec2.create_security_group( + GroupName=sec_group_name, Description="d", DryRun=True + ) ex.value.response["Error"]["Code"].should.equal("DryRunOperation") ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(412) ex.value.response["Error"]["Message"].should.equal( @@ -1706,16 +1746,16 @@ def test_run_instance_with_security_group_name_boto3(): ) group = ec2.create_security_group( - GroupName="group1", Description="some description" + GroupName=sec_group_name, Description="some description" ) res = ec2.create_instances( - ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=["group1"] + ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=[sec_group_name] ) instance = res[0] instance.security_groups.should.equal( - [{"GroupName": "group1", "GroupId": group.id}] + [{"GroupName": sec_group_name, "GroupId": group.id}] ) @@ -1734,15 +1774,16 @@ def test_run_instance_with_security_group_id(): @mock_ec2 def test_run_instance_with_security_group_id_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") + sec_group_name = str(uuid4()) group = ec2.create_security_group( - GroupName="group1", Description="some description" + GroupName=sec_group_name, Description="some description" ) instance = ec2.create_instances( ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroupIds=[group.id] )[0] instance.security_groups.should.equal( - [{"GroupName": "group1", "GroupId": group.id}] + [{"GroupName": sec_group_name, "GroupId": group.id}] ) @@ -1955,10 +1996,10 @@ def test_run_instance_with_nic_autocreated_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") security_group1 = ec2.create_security_group( - GroupName="test security group #1", Description="n/a" + GroupName=str(uuid4()), Description="n/a" ) security_group2 = ec2.create_security_group( - GroupName="test security group #2", Description="n/a" + GroupName=str(uuid4()), Description="n/a" ) private_ip = "10.0.0.1" @@ -1972,13 +2013,16 @@ def test_run_instance_with_nic_autocreated_boto3(): PrivateIpAddress=private_ip, )[0] - all_enis = client.describe_network_interfaces()["NetworkInterfaces"] - all_enis.should.have.length_of(1) - eni = all_enis[0] - instance_eni = instance.network_interfaces_attribute instance_eni.should.have.length_of(1) - instance_eni[0]["NetworkInterfaceId"].should.equal(eni["NetworkInterfaceId"]) + + nii = instance_eni[0]["NetworkInterfaceId"] + + my_enis = client.describe_network_interfaces(NetworkInterfaceIds=[nii])[ + "NetworkInterfaces" + ] + my_enis.should.have.length_of(1) + eni = my_enis[0] instance.subnet_id.should.equal(subnet.id) instance.security_groups.should.have.length_of(2) @@ -2057,10 +2101,10 @@ def test_run_instance_with_nic_preexisting_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") security_group1 = ec2.create_security_group( - GroupName="test security group #1", Description="n/a" + GroupName=str(uuid4()), Description="n/a" ) security_group2 = ec2.create_security_group( - GroupName="test security group #2", Description="n/a" + GroupName=str(uuid4()), Description="n/a" ) private_ip = "54.0.0.1" eni = ec2.create_network_interface( @@ -2080,7 +2124,11 @@ def test_run_instance_with_nic_preexisting_boto3(): instance.subnet_id.should.equal(subnet.id) - all_enis = client.describe_network_interfaces()["NetworkInterfaces"] + nii = instance.network_interfaces_attribute[0]["NetworkInterfaceId"] + + all_enis = client.describe_network_interfaces(NetworkInterfaceIds=[nii])[ + "NetworkInterfaces" + ] all_enis.should.have.length_of(1) instance_enis = instance.network_interfaces_attribute @@ -2186,10 +2234,10 @@ def test_instance_with_nic_attach_detach_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") security_group1 = ec2.create_security_group( - GroupName="test security group #1", Description="n/a" + GroupName=str(uuid4()), Description="n/a" ) security_group2 = ec2.create_security_group( - GroupName="test security group #2", Description="n/a" + GroupName=str(uuid4()), Description="n/a" ) instance = ec2.create_instances( @@ -2342,9 +2390,9 @@ def test_run_instance_with_block_device_mappings(): "BlockDeviceMappings": [{"DeviceName": "/dev/sda2", "Ebs": {"VolumeSize": 50}}], } - ec2_client.run_instances(**kwargs) + instance_id = ec2_client.run_instances(**kwargs)["Instances"][0]["InstanceId"] - instances = ec2_client.describe_instances() + instances = ec2_client.describe_instances(InstanceIds=[instance_id]) volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][ "Ebs" ] @@ -2421,9 +2469,10 @@ def test_run_instance_with_block_device_mappings_from_snapshot(): ], } - ec2_client.run_instances(**kwargs) + resp = ec2_client.run_instances(**kwargs) + instance_id = resp["Instances"][0]["InstanceId"] - instances = ec2_client.describe_instances() + instances = ec2_client.describe_instances(InstanceIds=[instance_id]) volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][ "Ebs" ] @@ -2444,6 +2493,8 @@ def test_describe_instance_status_no_instances(): @mock_ec2 def test_describe_instance_status_no_instances_boto3(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") client = boto3.client("ec2", region_name="us-east-1") all_status = client.describe_instance_status()["InstanceStatuses"] all_status.should.have.length_of(0) @@ -2465,12 +2516,15 @@ def test_describe_instance_status_with_instances(): def test_describe_instance_status_with_instances_boto3(): client = boto3.client("ec2", region_name="us-east-1") ec2 = boto3.resource("ec2", region_name="us-east-1") - ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] + instance = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] all_status = client.describe_instance_status()["InstanceStatuses"] - all_status.should.have.length_of(1) - all_status[0]["InstanceStatus"]["Status"].should.equal("ok") - all_status[0]["SystemStatus"]["Status"].should.equal("ok") + instance_ids = [s["InstanceId"] for s in all_status] + instance_ids.should.contain(instance.id) + + my_status = [s for s in all_status if s["InstanceId"] == instance.id][0] + my_status["InstanceStatus"]["Status"].should.equal("ok") + my_status["SystemStatus"]["Status"].should.equal("ok") # Has boto3 equivalent @@ -2564,19 +2618,26 @@ def test_describe_instance_status_with_instance_filter(): IncludeAllInstances=True, Filters=state_name_filter["running_and_stopped"] )["InstanceStatuses"] found_instance_ids = [status["InstanceId"] for status in found_statuses] - sorted(found_instance_ids).should.equal(all_instance_ids) + for _id in all_instance_ids: + found_instance_ids.should.contain(_id) found_statuses = conn.describe_instance_status( IncludeAllInstances=True, Filters=state_name_filter["running"] )["InstanceStatuses"] found_instance_ids = [status["InstanceId"] for status in found_statuses] - sorted(found_instance_ids).should.equal(running_instance_ids) + for _id in stopped_instance_ids: + found_instance_ids.shouldnt.contain(_id) + for _id in running_instance_ids: + found_instance_ids.should.contain(_id) found_statuses = conn.describe_instance_status( IncludeAllInstances=True, Filters=state_name_filter["stopped"] )["InstanceStatuses"] found_instance_ids = [status["InstanceId"] for status in found_statuses] - sorted(found_instance_ids).should.equal(stopped_instance_ids) + for _id in stopped_instance_ids: + found_instance_ids.should.contain(_id) + for _id in running_instance_ids: + found_instance_ids.shouldnt.contain(_id) # Filter instance using the state code state_code_filter = { @@ -2591,19 +2652,26 @@ def test_describe_instance_status_with_instance_filter(): IncludeAllInstances=True, Filters=state_code_filter["running_and_stopped"] )["InstanceStatuses"] found_instance_ids = [status["InstanceId"] for status in found_statuses] - sorted(found_instance_ids).should.equal(all_instance_ids) + for _id in all_instance_ids: + found_instance_ids.should.contain(_id) found_statuses = conn.describe_instance_status( IncludeAllInstances=True, Filters=state_code_filter["running"] )["InstanceStatuses"] found_instance_ids = [status["InstanceId"] for status in found_statuses] - sorted(found_instance_ids).should.equal(running_instance_ids) + for _id in stopped_instance_ids: + found_instance_ids.shouldnt.contain(_id) + for _id in running_instance_ids: + found_instance_ids.should.contain(_id) found_statuses = conn.describe_instance_status( IncludeAllInstances=True, Filters=state_code_filter["stopped"] )["InstanceStatuses"] found_instance_ids = [status["InstanceId"] for status in found_statuses] - sorted(found_instance_ids).should.equal(stopped_instance_ids) + for _id in stopped_instance_ids: + found_instance_ids.should.contain(_id) + for _id in running_instance_ids: + found_instance_ids.shouldnt.contain(_id) # Has boto3 equivalent @@ -2644,14 +2712,23 @@ def test_describe_instance_status_with_non_running_instances_boto3(): instance2.terminate() all_running_status = client.describe_instance_status()["InstanceStatuses"] - all_running_status.should.have.length_of(1) - all_running_status[0]["InstanceId"].should.equal(instance3.id) - all_running_status[0]["InstanceState"].should.equal({"Code": 16, "Name": "running"}) + [status["InstanceId"] for status in all_running_status].shouldnt.contain( + instance1.id + ) + [status["InstanceId"] for status in all_running_status].shouldnt.contain( + instance2.id + ) + [status["InstanceId"] for status in all_running_status].should.contain(instance3.id) + + my_status = [s for s in all_running_status if s["InstanceId"] == instance3.id][0] + my_status["InstanceState"].should.equal({"Code": 16, "Name": "running"}) all_status = client.describe_instance_status(IncludeAllInstances=True)[ "InstanceStatuses" ] - all_status.should.have.length_of(3) + [status["InstanceId"] for status in all_status].should.contain(instance1.id) + [status["InstanceId"] for status in all_status].should.contain(instance2.id) + [status["InstanceId"] for status in all_status].should.contain(instance3.id) status1 = next((s for s in all_status if s["InstanceId"] == instance1.id), None) status1["InstanceState"].should.equal({"Code": 80, "Name": "stopped"}) @@ -2698,7 +2775,9 @@ def test_get_instance_by_security_group_boto3(): instance = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] - security_group = ec2.create_security_group(GroupName="test", Description="test") + security_group = ec2.create_security_group( + GroupName=str(uuid4())[0:6], Description="test" + ) with pytest.raises(ClientError) as ex: client.modify_instance_attribute( @@ -2762,14 +2841,21 @@ def test_create_instance_ebs_optimized(): def test_run_multiple_instances_in_same_command(): instance_count = 4 client = boto3.client("ec2", region_name="us-east-1") - client.run_instances( + instances = client.run_instances( ImageId=EXAMPLE_AMI_ID, MinCount=instance_count, MaxCount=instance_count ) - reservations = client.describe_instances()["Reservations"] + reservation_id = instances["ReservationId"] - reservations[0]["Instances"].should.have.length_of(instance_count) + # TODO: use this filter when implemented + # client.describe_instances(Filters=[{"Name": "reservation-id", "Values": [instances["ReservationId"]]}])["Reservations"] + all_reservations = retrieve_all_reservations(client) + my_reservation = [ + r for r in all_reservations if r["ReservationId"] == reservation_id + ][0] - instances = reservations[0]["Instances"] + my_reservation["Instances"].should.have.length_of(instance_count) + + instances = my_reservation["Instances"] for i in range(0, instance_count): instances[i]["AmiLaunchIndex"].should.be(i) @@ -2778,17 +2864,15 @@ def test_run_multiple_instances_in_same_command(): def test_describe_instance_attribute(): client = boto3.client("ec2", region_name="us-east-1") security_group_id = client.create_security_group( - GroupName="test security group", Description="this is a test security group" + GroupName=str(uuid4()), Description="this is a test security group" )["GroupId"] - client.run_instances( + resp = client.run_instances( ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroupIds=[security_group_id], ) - instance_id = client.describe_instances()["Reservations"][0]["Instances"][0][ - "InstanceId" - ] + instance_id = resp["Instances"][0]["InstanceId"] valid_instance_attributes = [ "instanceType", @@ -2855,7 +2939,8 @@ def test_warn_on_invalid_ami(): def test_filter_wildcard_in_specified_tag_only(): ec2_client = boto3.client("ec2", region_name="us-west-1") - tags_name = [{"Key": "Name", "Value": "alice in wonderland"}] + name = str(uuid4())[0:6] + tags_name = [{"Key": "Name", "Value": f"{name} in wonderland"}] ec2_client.run_instances( ImageId=EXAMPLE_AMI_ID, MaxCount=1, @@ -2863,7 +2948,7 @@ def test_filter_wildcard_in_specified_tag_only(): TagSpecifications=[{"ResourceType": "instance", "Tags": tags_name}], ) - tags_owner = [{"Key": "Owner", "Value": "alice in wonderland"}] + tags_owner = [{"Key": "Owner", "Value": f"{name} in wonderland"}] ec2_client.run_instances( ImageId=EXAMPLE_AMI_ID, MaxCount=1, @@ -2873,7 +2958,7 @@ def test_filter_wildcard_in_specified_tag_only(): # should only match the Name tag response = ec2_client.describe_instances( - Filters=[{"Name": "tag:Name", "Values": ["*alice*"]}] + Filters=[{"Name": "tag:Name", "Values": [f"*{name}*"]}] ) instances = [i for r in response["Reservations"] for i in r["Instances"]] instances.should.have.length_of(1) @@ -2946,8 +3031,7 @@ def test_create_instance_with_launch_template_id_produces_no_warning( ) template = client.create_launch_template( - LaunchTemplateName="test-template", - LaunchTemplateData={"ImageId": EXAMPLE_AMI_ID}, + LaunchTemplateName=str(uuid4()), LaunchTemplateData={"ImageId": EXAMPLE_AMI_ID}, )["LaunchTemplate"] with pytest.warns(None) as captured_warnings: @@ -2958,3 +3042,19 @@ def test_create_instance_with_launch_template_id_produces_no_warning( ) assert len(captured_warnings) == 0 + + +def retrieve_all_reservations(client, filters=[]): + resp = client.describe_instances(Filters=filters) + all_reservations = resp["Reservations"] + next_token = resp.get("NextToken") + while next_token: + resp = client.describe_instances(Filters=filters, NextToken=next_token) + all_reservations.extend(resp["Reservations"]) + next_token = resp.get("NextToken") + return all_reservations + + +def retrieve_all_instances(client, filters=[]): + reservations = retrieve_all_reservations(client, filters) + return [i for r in reservations for i in r["Instances"]] diff --git a/tests/test_ec2/test_internet_gateways.py b/tests/test_ec2/test_internet_gateways.py index 11471935c..18056bb00 100644 --- a/tests/test_ec2/test_internet_gateways.py +++ b/tests/test_ec2/test_internet_gateways.py @@ -13,6 +13,7 @@ from botocore.exceptions import ClientError import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 +from uuid import uuid4 VPC_CIDR = "10.0.0.0/16" @@ -50,8 +51,6 @@ def test_igw_create_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - client.describe_internet_gateways()["InternetGateways"].should.have.length_of(0) - with pytest.raises(ClientError) as ex: client.create_internet_gateway(DryRun=True) ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(412) @@ -61,10 +60,11 @@ def test_igw_create_boto3(): ) igw = ec2.create_internet_gateway() - client.describe_internet_gateways()["InternetGateways"].should.have.length_of(1) igw.id.should.match(r"igw-[0-9a-f]+") - igw = client.describe_internet_gateways()["InternetGateways"][0] + igw = client.describe_internet_gateways(InternetGatewayIds=[igw.id])[ + "InternetGateways" + ][0] igw["Attachments"].should.have.length_of(0) @@ -109,7 +109,9 @@ def test_igw_attach_boto3(): vpc.attach_internet_gateway(InternetGatewayId=igw.id) - igw = client.describe_internet_gateways()["InternetGateways"][0] + igw = client.describe_internet_gateways(InternetGatewayIds=[igw.id])[ + "InternetGateways" + ][0] igw["Attachments"].should.equal([{"State": "available", "VpcId": vpc.id}]) @@ -216,7 +218,9 @@ def test_igw_detach_boto3(): ) client.detach_internet_gateway(InternetGatewayId=igw.id, VpcId=vpc.id) - igw = client.describe_internet_gateways()["InternetGateways"][0] + igw = igw = client.describe_internet_gateways(InternetGatewayIds=[igw.id])[ + "InternetGateways" + ][0] igw["Attachments"].should.have.length_of(0) @@ -345,9 +349,8 @@ def test_igw_delete_boto3(): client = boto3.client("ec2", region_name="us-west-1") ec2.create_vpc(CidrBlock=VPC_CIDR) - client.describe_internet_gateways()["InternetGateways"].should.have.length_of(0) igw = ec2.create_internet_gateway() - client.describe_internet_gateways()["InternetGateways"].should.have.length_of(1) + [i["InternetGatewayId"] for i in (retrieve_all(client))].should.contain(igw.id) with pytest.raises(ClientError) as ex: client.delete_internet_gateway(InternetGatewayId=igw.id, DryRun=True) @@ -358,7 +361,7 @@ def test_igw_delete_boto3(): ) client.delete_internet_gateway(InternetGatewayId=igw.id) - client.describe_internet_gateways()["InternetGateways"].should.have.length_of(0) + [i["InternetGatewayId"] for i in (retrieve_all(client))].shouldnt.contain(igw.id) # Has boto3 equivalent @@ -495,13 +498,12 @@ def test_igw_filter_by_tags_boto3(): igw1 = ec2.create_internet_gateway() igw2 = ec2.create_internet_gateway() - igw1.create_tags(Tags=[{"Key": "tests", "Value": "yes"}]) + tag_value = str(uuid4()) + igw1.create_tags(Tags=[{"Key": "tests", "Value": tag_value}]) - result = client.describe_internet_gateways( - Filters=[{"Name": "tag:tests", "Values": ["yes"]}] - ) - result["InternetGateways"].should.have.length_of(1) - result["InternetGateways"][0]["InternetGatewayId"].should.equal(igw1.id) + result = retrieve_all(client, [{"Name": "tag:tests", "Values": [tag_value]}]) + result.should.have.length_of(1) + result[0]["InternetGatewayId"].should.equal(igw1.id) # Has boto3 equivalent @@ -561,11 +563,10 @@ def test_igw_filter_by_attachment_state_boto3(): vpc = ec2.create_vpc(CidrBlock=VPC_CIDR) client.attach_internet_gateway(InternetGatewayId=igw1.id, VpcId=vpc.id) - result = client.describe_internet_gateways( - Filters=[{"Name": "attachment.state", "Values": ["available"]}] - ) - result["InternetGateways"].should.have.length_of(1) - result["InternetGateways"][0]["InternetGatewayId"].should.equal(igw1.id) + filters = [{"Name": "attachment.state", "Values": ["available"]}] + all_ids = [igw["InternetGatewayId"] for igw in (retrieve_all(client, filters))] + all_ids.should.contain(igw1.id) + all_ids.shouldnt.contain(igw2.id) @mock_ec2 @@ -582,3 +583,14 @@ def test_create_internet_gateway_with_tags(): ) igw.tags.should.have.length_of(1) igw.tags.should.equal([{"Key": "test", "Value": "TestRouteTable"}]) + + +def retrieve_all(client, filters=[]): + resp = client.describe_internet_gateways(Filters=filters) + all_igws = resp["InternetGateways"] + token = resp.get("NextToken") + while token: + resp = client.describe_internet_gateways(NextToken=token, Filters=filters) + all_igws.extend(resp["InternetGateways"]) + token = resp.get("NextToken") + return all_igws diff --git a/tests/test_ec2/test_key_pairs.py b/tests/test_ec2/test_key_pairs.py index 3c5889993..b9ef6e258 100644 --- a/tests/test_ec2/test_key_pairs.py +++ b/tests/test_ec2/test_key_pairs.py @@ -8,7 +8,9 @@ import sure # noqa from boto.exception import EC2ResponseError from botocore.exceptions import ClientError -from moto import mock_ec2, mock_ec2_deprecated +from moto import mock_ec2, mock_ec2_deprecated, settings +from uuid import uuid4 +from unittest import SkipTest from .helpers import rsa_check_private_key @@ -55,6 +57,8 @@ def test_key_pairs_empty(): @mock_ec2 def test_key_pairs_empty_boto3(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") client.describe_key_pairs()["KeyPairs"].should.be.empty @@ -78,7 +82,7 @@ def test_key_pairs_invalid_id_boto3(): client = boto3.client("ec2", "us-west-1") with pytest.raises(ClientError) as ex: - client.describe_key_pairs(KeyNames=["foo"]) + client.describe_key_pairs(KeyNames=[str(uuid4())]) ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidKeyPair.NotFound") @@ -145,21 +149,24 @@ def test_key_pairs_create_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - kp = ec2.create_key_pair(KeyName="foo") + key_name = str(uuid4())[0:6] + kp = ec2.create_key_pair(KeyName=key_name) rsa_check_private_key(kp.key_material) # Verify the client can create a key_pair as well - should behave the same - kp2 = client.create_key_pair(KeyName="foo2") + key_name2 = str(uuid4()) + kp2 = client.create_key_pair(KeyName=key_name2) rsa_check_private_key(kp2["KeyMaterial"]) kp.key_material.shouldnt.equal(kp2["KeyMaterial"]) kps = client.describe_key_pairs()["KeyPairs"] - kps.should.have.length_of(2) - set([k["KeyName"] for k in kps]).should.equal(set(["foo", "foo2"])) + all_names = set([k["KeyName"] for k in kps]) + all_names.should.contain(key_name) + all_names.should.contain(key_name2) - kps = client.describe_key_pairs(KeyNames=["foo"])["KeyPairs"] + kps = client.describe_key_pairs(KeyNames=[key_name])["KeyPairs"] kps.should.have.length_of(1) - kps[0].should.have.key("KeyName").equal("foo") + kps[0].should.have.key("KeyName").equal(key_name) kps[0].should.have.key("KeyFingerprint") @@ -180,10 +187,11 @@ def test_key_pairs_create_exist(): @mock_ec2 def test_key_pairs_create_exist_boto3(): client = boto3.client("ec2", "us-west-1") - client.create_key_pair(KeyName="foo") + key_name = str(uuid4())[0:6] + client.create_key_pair(KeyName=key_name) with pytest.raises(ClientError) as ex: - client.create_key_pair(KeyName="foo") + client.create_key_pair(KeyName=key_name) ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidKeyPair.Duplicate") @@ -201,8 +209,7 @@ def test_key_pairs_delete_no_exist(): @mock_ec2 def test_key_pairs_delete_no_exist_boto3(): client = boto3.client("ec2", "us-west-1") - client.describe_key_pairs()["KeyPairs"].should.have.length_of(0) - client.delete_key_pair(KeyName="non-existing") + client.delete_key_pair(KeyName=str(uuid4())[0:6]) # Has boto3 equivalent @@ -228,18 +235,21 @@ def test_key_pairs_delete_exist(): def test_key_pairs_delete_exist_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - client.create_key_pair(KeyName="foo") + key_name = str(uuid4())[0:6] + client.create_key_pair(KeyName=key_name) with pytest.raises(ClientError) as ex: - client.delete_key_pair(KeyName="foo", DryRun=True) + client.delete_key_pair(KeyName=key_name, DryRun=True) ex.value.response["Error"]["Code"].should.equal("DryRunOperation") ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(412) ex.value.response["Error"]["Message"].should.equal( "An error occurred (DryRunOperation) when calling the DeleteKeyPair operation: Request would have succeeded, but DryRun flag is set" ) - client.delete_key_pair(KeyName="foo") - client.describe_key_pairs()["KeyPairs"].should.equal([]) + client.delete_key_pair(KeyName=key_name) + [kp.get("Name") for kp in client.describe_key_pairs()["KeyPairs"]].shouldnt.contain( + key_name + ) # Has boto3 equivalent @@ -274,9 +284,10 @@ def test_key_pairs_import_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") + key_name = str(uuid4())[0:6] with pytest.raises(ClientError) as ex: client.import_key_pair( - KeyName="foo", PublicKeyMaterial=RSA_PUBLIC_KEY_OPENSSH, DryRun=True + KeyName=key_name, PublicKeyMaterial=RSA_PUBLIC_KEY_OPENSSH, DryRun=True ) ex.value.response["Error"]["Code"].should.equal("DryRunOperation") ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(412) @@ -285,22 +296,23 @@ def test_key_pairs_import_boto3(): ) kp1 = client.import_key_pair( - KeyName="foo", PublicKeyMaterial=RSA_PUBLIC_KEY_OPENSSH + KeyName=key_name, PublicKeyMaterial=RSA_PUBLIC_KEY_OPENSSH ) - print(kp1) - kp1.should.have.key("KeyName").equal("foo") + + kp1.should.have.key("KeyName").equal(key_name) kp1.should.have.key("KeyFingerprint").equal(RSA_PUBLIC_KEY_FINGERPRINT) + key_name2 = str(uuid4()) kp2 = client.import_key_pair( - KeyName="foo2", PublicKeyMaterial=RSA_PUBLIC_KEY_RFC4716 + KeyName=key_name2, PublicKeyMaterial=RSA_PUBLIC_KEY_RFC4716 ) - kp2.should.have.key("KeyName").equal("foo2") + kp2.should.have.key("KeyName").equal(key_name2) kp2.should.have.key("KeyFingerprint").equal(RSA_PUBLIC_KEY_FINGERPRINT) - kps = client.describe_key_pairs()["KeyPairs"] - kps.should.have.length_of(2) - kps[0]["KeyName"].should.equal(kp1["KeyName"]) - kps[1]["KeyName"].should.equal(kp2["KeyName"]) + all_kps = client.describe_key_pairs()["KeyPairs"] + all_names = [kp["KeyName"] for kp in all_kps] + all_names.should.contain(kp1["KeyName"]) + all_names.should.contain(kp2["KeyName"]) # Has boto3 equivalent @@ -323,12 +335,16 @@ def test_key_pairs_import_exist_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - kp = client.import_key_pair(KeyName="foo", PublicKeyMaterial=RSA_PUBLIC_KEY_OPENSSH) - kp["KeyName"].should.equal("foo") - client.describe_key_pairs()["KeyPairs"].should.have.length_of(1) + key_name = str(uuid4())[0:6] + kp = client.import_key_pair( + KeyName=key_name, PublicKeyMaterial=RSA_PUBLIC_KEY_OPENSSH + ) + kp["KeyName"].should.equal(key_name) + + client.describe_key_pairs(KeyNames=[key_name])["KeyPairs"].should.have.length_of(1) with pytest.raises(ClientError) as ex: - client.create_key_pair(KeyName="foo") + client.create_key_pair(KeyName=key_name) ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidKeyPair.Duplicate") @@ -406,12 +422,15 @@ def test_key_pair_filters_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - _ = ec2.create_key_pair(KeyName="kpfltr1") - kp2 = ec2.create_key_pair(KeyName="kpfltr2") - kp3 = ec2.create_key_pair(KeyName="kpfltr3") + key_name_1 = str(uuid4())[0:6] + key_name_2 = str(uuid4())[0:6] + key_name_3 = str(uuid4())[0:6] + _ = ec2.create_key_pair(KeyName=key_name_1) + kp2 = ec2.create_key_pair(KeyName=key_name_2) + kp3 = ec2.create_key_pair(KeyName=key_name_3) kp_by_name = client.describe_key_pairs( - Filters=[{"Name": "key-name", "Values": ["kpfltr2"]}] + Filters=[{"Name": "key-name", "Values": [key_name_2]}] )["KeyPairs"] set([kp["KeyName"] for kp in kp_by_name]).should.equal(set([kp2.name])) diff --git a/tests/test_ec2/test_launch_templates.py b/tests/test_ec2/test_launch_templates.py index 41896be96..27570cbf8 100644 --- a/tests/test_ec2/test_launch_templates.py +++ b/tests/test_ec2/test_launch_templates.py @@ -4,15 +4,17 @@ import sure # noqa import pytest from botocore.client import ClientError -from moto import mock_ec2 +from moto import mock_ec2, settings +from uuid import uuid4 @mock_ec2 def test_launch_template_create(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) resp = cli.create_launch_template( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, # the absolute minimum needed to create a template without other resources LaunchTemplateData={ "TagSpecifications": [ @@ -26,13 +28,13 @@ def test_launch_template_create(): resp.should.have.key("LaunchTemplate") lt = resp["LaunchTemplate"] - lt["LaunchTemplateName"].should.equal("test-template") + lt["LaunchTemplateName"].should.equal(template_name) lt["DefaultVersionNumber"].should.equal(1) lt["LatestVersionNumber"].should.equal(1) with pytest.raises(ClientError) as ex: cli.create_launch_template( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={ "TagSpecifications": [ { @@ -61,13 +63,14 @@ def test_describe_launch_template_versions(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) create_resp = cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData=template_data + LaunchTemplateName=template_name, LaunchTemplateData=template_data ) # test using name resp = cli.describe_launch_template_versions( - LaunchTemplateName="test-template", Versions=["1"] + LaunchTemplateName=template_name, Versions=["1"] ) templ = resp["LaunchTemplateVersions"][0]["LaunchTemplateData"] @@ -87,12 +90,13 @@ def test_describe_launch_template_versions(): def test_create_launch_template_version(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) create_resp = cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) version_resp = cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-def456"}, VersionDescription="new ami", ) @@ -111,8 +115,9 @@ def test_create_launch_template_version(): def test_create_launch_template_version_by_id(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) create_resp = cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) version_resp = cli.create_launch_template_version( @@ -135,17 +140,18 @@ def test_create_launch_template_version_by_id(): def test_describe_launch_template_versions_with_multiple_versions(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-def456"}, VersionDescription="new ami", ) - resp = cli.describe_launch_template_versions(LaunchTemplateName="test-template") + resp = cli.describe_launch_template_versions(LaunchTemplateName=template_name) resp["LaunchTemplateVersions"].should.have.length_of(2) resp["LaunchTemplateVersions"][0]["LaunchTemplateData"]["ImageId"].should.equal( @@ -160,24 +166,25 @@ def test_describe_launch_template_versions_with_multiple_versions(): def test_describe_launch_template_versions_with_versions_option(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-def456"}, VersionDescription="new ami", ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-hij789"}, VersionDescription="new ami, again", ) resp = cli.describe_launch_template_versions( - LaunchTemplateName="test-template", Versions=["2", "3"] + LaunchTemplateName=template_name, Versions=["2", "3"] ) resp["LaunchTemplateVersions"].should.have.length_of(2) @@ -193,24 +200,25 @@ def test_describe_launch_template_versions_with_versions_option(): def test_describe_launch_template_versions_with_min(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-def456"}, VersionDescription="new ami", ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-hij789"}, VersionDescription="new ami, again", ) resp = cli.describe_launch_template_versions( - LaunchTemplateName="test-template", MinVersion="2" + LaunchTemplateName=template_name, MinVersion="2" ) resp["LaunchTemplateVersions"].should.have.length_of(2) @@ -226,24 +234,25 @@ def test_describe_launch_template_versions_with_min(): def test_describe_launch_template_versions_with_max(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-def456"}, VersionDescription="new ami", ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-hij789"}, VersionDescription="new ami, again", ) resp = cli.describe_launch_template_versions( - LaunchTemplateName="test-template", MaxVersion="2" + LaunchTemplateName=template_name, MaxVersion="2" ) resp["LaunchTemplateVersions"].should.have.length_of(2) @@ -259,30 +268,31 @@ def test_describe_launch_template_versions_with_max(): def test_describe_launch_template_versions_with_min_and_max(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-def456"}, VersionDescription="new ami", ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-hij789"}, VersionDescription="new ami, again", ) cli.create_launch_template_version( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-345abc"}, VersionDescription="new ami, because why not", ) resp = cli.describe_launch_template_versions( - LaunchTemplateName="test-template", MinVersion="2", MaxVersion="3" + LaunchTemplateName=template_name, MinVersion="2", MaxVersion="3" ) resp["LaunchTemplateVersions"].should.have.length_of(2) @@ -299,81 +309,89 @@ def test_describe_launch_templates(): cli = boto3.client("ec2", region_name="us-east-1") lt_ids = [] + template_name = str(uuid4()) r = cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) lt_ids.append(r["LaunchTemplate"]["LaunchTemplateId"]) + template_name2 = str(uuid4()) r = cli.create_launch_template( - LaunchTemplateName="test-template2", - LaunchTemplateData={"ImageId": "ami-abc123"}, + LaunchTemplateName=template_name2, LaunchTemplateData={"ImageId": "ami-abc123"}, ) lt_ids.append(r["LaunchTemplate"]["LaunchTemplateId"]) # general call, all templates - resp = cli.describe_launch_templates() - resp.should.have.key("LaunchTemplates") - resp["LaunchTemplates"].should.have.length_of(2) - resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal("test-template") - resp["LaunchTemplates"][1]["LaunchTemplateName"].should.equal("test-template2") + if not settings.TEST_SERVER_MODE: + # Bug: Only 15 launch templates are returned, ever + # ServerMode may have many more templates, created in parallel + all_templates = retrieve_all_templates(cli) + my_templates = [t for t in all_templates if t["LaunchTemplateId"] in lt_ids] + my_templates[0]["LaunchTemplateName"].should.equal(template_name) + my_templates[1]["LaunchTemplateName"].should.equal(template_name2) # filter by names resp = cli.describe_launch_templates( - LaunchTemplateNames=["test-template2", "test-template"] + LaunchTemplateNames=[template_name2, template_name] ) resp.should.have.key("LaunchTemplates") resp["LaunchTemplates"].should.have.length_of(2) - resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal("test-template2") - resp["LaunchTemplates"][1]["LaunchTemplateName"].should.equal("test-template") + resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal(template_name2) + resp["LaunchTemplates"][1]["LaunchTemplateName"].should.equal(template_name) # filter by ids resp = cli.describe_launch_templates(LaunchTemplateIds=lt_ids) resp.should.have.key("LaunchTemplates") resp["LaunchTemplates"].should.have.length_of(2) - resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal("test-template") - resp["LaunchTemplates"][1]["LaunchTemplateName"].should.equal("test-template2") + resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal(template_name) + resp["LaunchTemplates"][1]["LaunchTemplateName"].should.equal(template_name2) @mock_ec2 def test_describe_launch_templates_with_filters(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) r = cli.create_launch_template( - LaunchTemplateName="test-template", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"} ) + tag_value = str(uuid4()) cli.create_tags( Resources=[r["LaunchTemplate"]["LaunchTemplateId"]], Tags=[ - {"Key": "tag1", "Value": "a value"}, + {"Key": "tag1", "Value": tag_value}, {"Key": "another-key", "Value": "this value"}, ], ) + template_name_no_tags = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="no-tags", LaunchTemplateData={"ImageId": "ami-abc123"} + LaunchTemplateName=template_name_no_tags, + LaunchTemplateData={"ImageId": "ami-abc123"}, ) resp = cli.describe_launch_templates( - Filters=[{"Name": "tag:tag1", "Values": ["a value"]}] + Filters=[{"Name": "tag:tag1", "Values": [tag_value]}] ) resp["LaunchTemplates"].should.have.length_of(1) - resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal("test-template") + resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal(template_name) resp = cli.describe_launch_templates( - Filters=[{"Name": "launch-template-name", "Values": ["no-tags"]}] + Filters=[{"Name": "launch-template-name", "Values": [template_name_no_tags]}] ) resp["LaunchTemplates"].should.have.length_of(1) - resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal("no-tags") + resp["LaunchTemplates"][0]["LaunchTemplateName"].should.equal(template_name_no_tags) @mock_ec2 def test_create_launch_template_with_tag_spec(): cli = boto3.client("ec2", region_name="us-east-1") + template_name = str(uuid4()) cli.create_launch_template( - LaunchTemplateName="test-template", + LaunchTemplateName=template_name, LaunchTemplateData={"ImageId": "ami-abc123"}, TagSpecifications=[ {"ResourceType": "instance", "Tags": [{"Key": "key", "Value": "value"}]} @@ -381,7 +399,7 @@ def test_create_launch_template_with_tag_spec(): ) resp = cli.describe_launch_template_versions( - LaunchTemplateName="test-template", Versions=["1"] + LaunchTemplateName=template_name, Versions=["1"] ) version = resp["LaunchTemplateVersions"][0] @@ -390,3 +408,14 @@ def test_create_launch_template_with_tag_spec(): version["LaunchTemplateData"]["TagSpecifications"][0].should.equal( {"ResourceType": "instance", "Tags": [{"Key": "key", "Value": "value"}]} ) + + +def retrieve_all_templates(client, filters=[]): + resp = client.describe_launch_templates(Filters=filters) + all_templates = resp["LaunchTemplates"] + next_token = resp.get("NextToken") + while next_token: + resp = client.describe_launch_templates(Filters=filters, NextToken=next_token) + all_templates.extend(resp["LaunchTemplates"]) + next_token = resp.get("NextToken") + return all_templates diff --git a/tests/test_ec2/test_nat_gateway.py b/tests/test_ec2/test_nat_gateway.py index ede98c332..094dae984 100644 --- a/tests/test_ec2/test_nat_gateway.py +++ b/tests/test_ec2/test_nat_gateway.py @@ -2,11 +2,14 @@ from __future__ import unicode_literals import boto3 import sure # noqa -from moto import mock_ec2 +from moto import mock_ec2, settings +from unittest import SkipTest @mock_ec2 def test_describe_nat_gateways(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to have no resources") conn = boto3.client("ec2", "us-east-1") response = conn.describe_nat_gateways() @@ -43,7 +46,7 @@ def test_describe_nat_gateway_tags(): allocation_id = conn.allocate_address(Domain="vpc")["AllocationId"] subnet_id = subnet["Subnet"]["SubnetId"] - conn.create_nat_gateway( + gateway = conn.create_nat_gateway( SubnetId=subnet_id, AllocationId=allocation_id, TagSpecifications=[ @@ -55,12 +58,15 @@ def test_describe_nat_gateway_tags(): ], } ], - ) + )["NatGateway"] - describe_response = conn.describe_nat_gateways() + describe_all = retrieve_all_gateways(conn) + [gw["VpcId"] for gw in describe_all].should.contain(vpc_id) + describe_gateway = [gw for gw in describe_all if gw["VpcId"] == vpc_id] - assert describe_response["NatGateways"][0]["VpcId"] == vpc_id - assert describe_response["NatGateways"][0]["Tags"] == [ + assert describe_gateway[0]["NatGatewayId"] == gateway["NatGatewayId"] + assert describe_gateway[0]["VpcId"] == vpc_id + assert describe_gateway[0]["Tags"] == [ {"Key": "name", "Value": "some-nat-gateway"}, {"Key": "name1", "Value": "some-nat-gateway-1"}, ] @@ -112,31 +118,27 @@ def test_create_and_describe_nat_gateway(): SubnetId=subnet_id, AllocationId=allocation_id ) nat_gateway_id = create_response["NatGateway"]["NatGatewayId"] - describe_response = conn.describe_nat_gateways() + net_interface_id = create_response["NatGateway"]["NatGatewayAddresses"][0][ + "NetworkInterfaceId" + ] - enis = conn.describe_network_interfaces()["NetworkInterfaces"] - eni_id = enis[0]["NetworkInterfaceId"] public_ip = conn.describe_addresses(AllocationIds=[allocation_id])["Addresses"][0][ "PublicIp" ] - describe_response["NatGateways"].should.have.length_of(1) - describe_response["NatGateways"][0]["NatGatewayId"].should.equal(nat_gateway_id) - describe_response["NatGateways"][0]["State"].should.equal("available") - describe_response["NatGateways"][0]["SubnetId"].should.equal(subnet_id) - describe_response["NatGateways"][0]["VpcId"].should.equal(vpc_id) - describe_response["NatGateways"][0]["NatGatewayAddresses"][0][ - "AllocationId" - ].should.equal(allocation_id) - describe_response["NatGateways"][0]["NatGatewayAddresses"][0][ - "NetworkInterfaceId" - ].should.equal(eni_id) - assert describe_response["NatGateways"][0]["NatGatewayAddresses"][0][ - "PrivateIp" - ].startswith("10.") - describe_response["NatGateways"][0]["NatGatewayAddresses"][0][ - "PublicIp" - ].should.equal(public_ip) + describe = conn.describe_nat_gateways(NatGatewayIds=[nat_gateway_id])["NatGateways"] + + describe.should.have.length_of(1) + describe[0]["NatGatewayId"].should.equal(nat_gateway_id) + describe[0]["State"].should.equal("available") + describe[0]["SubnetId"].should.equal(subnet_id) + describe[0]["VpcId"].should.equal(vpc_id) + describe[0]["NatGatewayAddresses"][0]["AllocationId"].should.equal(allocation_id) + describe[0]["NatGatewayAddresses"][0]["NetworkInterfaceId"].should.equal( + net_interface_id + ) + assert describe[0]["NatGatewayAddresses"][0]["PrivateIp"].startswith("10.") + describe[0]["NatGatewayAddresses"][0]["PublicIp"].should.equal(public_ip) @mock_ec2 @@ -204,8 +206,9 @@ def test_describe_nat_gateway_filter_by_subnet_id(): nat_gateway_id_1 = create_response_1["NatGateway"]["NatGatewayId"] # nat_gateway_id_2 = create_response_2["NatGateway"]["NatGatewayId"] - describe_response = conn.describe_nat_gateways() - describe_response["NatGateways"].should.have.length_of(2) + all_gws = retrieve_all_gateways(conn) + all_gw_ids = [gw["NatGatewayId"] for gw in all_gws] + all_gw_ids.should.contain(nat_gateway_id_1) describe_response = conn.describe_nat_gateways( Filters=[{"Name": "subnet-id", "Values": [subnet_id_1]}] @@ -244,9 +247,6 @@ def test_describe_nat_gateway_filter_vpc_id(): conn.create_nat_gateway(SubnetId=subnet_id_2, AllocationId=allocation_id_2) nat_gateway_id_1 = create_response_1["NatGateway"]["NatGatewayId"] - describe_response = conn.describe_nat_gateways() - describe_response["NatGateways"].should.have.length_of(2) - describe_response = conn.describe_nat_gateways( Filters=[{"Name": "vpc-id", "Values": [vpc_id_1]}] ) @@ -258,3 +258,14 @@ def test_describe_nat_gateway_filter_vpc_id(): describe_response["NatGateways"][0]["NatGatewayAddresses"][0][ "AllocationId" ].should.equal(allocation_id_1) + + +def retrieve_all_gateways(client, filters=[]): + resp = client.describe_nat_gateways(Filters=filters) + all_gws = resp["NatGateways"] + token = resp.get("NextToken") + while token: + resp = client.describe_nat_gateways(Filters=filters, NextToken=token) + all_gws.extend(resp["NatGateways"]) + token = resp.get("NextToken") + return all_gws diff --git a/tests/test_ec2/test_network_acls.py b/tests/test_ec2/test_network_acls.py index d049584b6..359ff1548 100644 --- a/tests/test_ec2/test_network_acls.py +++ b/tests/test_ec2/test_network_acls.py @@ -5,8 +5,10 @@ import sure # noqa import pytest from botocore.exceptions import ClientError -from moto import mock_ec2_deprecated, mock_ec2 +from moto import mock_ec2_deprecated, mock_ec2, settings from moto.ec2.models import OWNER_ID +from random import randint +from unittest import SkipTest # Has boto3 equivalent @@ -22,10 +24,12 @@ def test_default_network_acl_created_with_vpc(): def test_default_network_acl_created_with_vpc_boto3(): client = boto3.client("ec2", region_name="us-east-1") ec2 = boto3.resource("ec2", region_name="us-east-1") - ec2.create_vpc(CidrBlock="10.0.0.0/16") + vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") all_network_acls = client.describe_network_acls()["NetworkAcls"] - all_network_acls.should.have.length_of(2) + our_acl = [a for a in all_network_acls if a["VpcId"] == vpc.id] + our_acl.should.have.length_of(1) + our_acl[0].should.have.key("IsDefault").equals(True) # Has boto3 equivalent @@ -46,7 +50,6 @@ def test_network_create_and_list_acls_boto3(): created_acl = ec2.create_network_acl(VpcId=vpc.id) all_network_acls = client.describe_network_acls()["NetworkAcls"] - all_network_acls.should.have.length_of(3) acl_found = [ a for a in all_network_acls if a["NetworkAclId"] == created_acl.network_acl_id @@ -73,6 +76,8 @@ def test_new_subnet_associates_with_default_network_acl(): @mock_ec2 def test_new_subnet_associates_with_default_network_acl_boto3(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode will have conflicting CidrBlocks") client = boto3.client("ec2", region_name="us-east-1") ec2 = boto3.resource("ec2", region_name="us-east-1") default_vpc = client.describe_vpcs()["Vpcs"][0] @@ -135,7 +140,6 @@ def test_network_acl_entries_boto3(): ) all_network_acls = client.describe_network_acls()["NetworkAcls"] - all_network_acls.should.have.length_of(3) test_network_acl = next( na for na in all_network_acls if na["NetworkAclId"] == network_acl.id @@ -335,14 +339,12 @@ def test_delete_network_acl_boto3(): network_acl = ec2.create_network_acl(VpcId=vpc.id) all_network_acls = client.describe_network_acls()["NetworkAcls"] - all_network_acls.should.have.length_of(3) any(acl["NetworkAclId"] == network_acl.id for acl in all_network_acls).should.be.ok client.delete_network_acl(NetworkAclId=network_acl.id) updated_network_acls = client.describe_network_acls()["NetworkAcls"] - updated_network_acls.should.have.length_of(2) any( acl["NetworkAclId"] == network_acl.id for acl in updated_network_acls @@ -377,7 +379,9 @@ def test_network_acl_tagging_boto3(): network_acl.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - tag = client.describe_tags()["Tags"][0] + tag = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [network_acl.id]}] + )["Tags"][0] tag.should.have.key("ResourceId").equal(network_acl.id) tag.should.have.key("Key").equal("a key") tag.should.have.key("Value").equal("some value") @@ -408,12 +412,27 @@ def test_new_subnet_in_new_vpc_associates_with_default_network_acl(): @mock_ec2 def test_default_network_acl_default_entries(): ec2 = boto3.resource("ec2", region_name="us-west-1") - default_network_acl = next(iter(ec2.network_acls.all()), None) - default_network_acl.is_default.should.be.ok + client = boto3.client("ec2", region_name="us-west-1") + + if not settings.TEST_SERVER_MODE: + # Can't know whether the first ACL is the default in ServerMode + default_network_acl = next(iter(ec2.network_acls.all()), None) + default_network_acl.is_default.should.be.ok + default_network_acl.entries.should.have.length_of(4) + + vpc = client.create_vpc(CidrBlock="10.0.0.0/16") + vpc_id = vpc["Vpc"]["VpcId"] + + default_acls = client.describe_network_acls( + Filters=[ + {"Name": "default", "Values": ["true"]}, + {"Name": "vpc-id", "Values": [vpc_id]}, + ] + )["NetworkAcls"] + default_acl = default_acls[0] - default_network_acl.entries.should.have.length_of(4) unique_entries = [] - for entry in default_network_acl.entries: + for entry in default_acl["Entries"]: entry["CidrBlock"].should.equal("0.0.0.0/0") entry["Protocol"].should.equal("-1") entry["RuleNumber"].should.be.within([100, 32767]) @@ -431,6 +450,10 @@ def test_default_network_acl_default_entries(): @mock_ec2 def test_delete_default_network_acl_default_entry(): + if settings.TEST_SERVER_MODE: + raise SkipTest( + "Don't want to mess with default ACLs in ServerMode, as other tests may need them" + ) ec2 = boto3.resource("ec2", region_name="us-west-1") default_network_acl = next(iter(ec2.network_acls.all()), None) default_network_acl.is_default.should.be.ok @@ -452,7 +475,7 @@ def test_duplicate_network_acl_entry(): default_network_acl = next(iter(ec2.network_acls.all()), None) default_network_acl.is_default.should.be.ok - rule_number = 200 + rule_number = randint(0, 9999) egress = True default_network_acl.create_entry( CidrBlock="0.0.0.0/0", @@ -496,12 +519,12 @@ def test_describe_network_acls(): result[0]["NetworkAclId"].should.equal(network_acl_id) resp2 = conn.describe_network_acls()["NetworkAcls"] - resp2.should.have.length_of(3) + [na["NetworkAclId"] for na in resp2].should.contain(network_acl_id) resp3 = conn.describe_network_acls( Filters=[{"Name": "owner-id", "Values": [OWNER_ID]}] )["NetworkAcls"] - resp3.should.have.length_of(3) + [na["NetworkAclId"] for na in resp3].should.contain(network_acl_id) with pytest.raises(ClientError) as ex: conn.describe_network_acls(NetworkAclIds=["1"]) diff --git a/tests/test_ec2/test_prefix_lists.py b/tests/test_ec2/test_prefix_lists.py index fad644a2b..cd8629b5d 100644 --- a/tests/test_ec2/test_prefix_lists.py +++ b/tests/test_ec2/test_prefix_lists.py @@ -1,7 +1,7 @@ import boto3 import sure # noqa -from moto import mock_ec2 +from moto import mock_ec2, settings from moto.core import ACCOUNT_ID @@ -47,15 +47,13 @@ def test_create_with_tags(): def test_describe_managed_prefix_lists(): ec2 = boto3.client("ec2", region_name="us-west-1") - default_lists = ec2.describe_managed_prefix_lists()["PrefixLists"] - set([l["OwnerId"] for l in default_lists]).should.equal({"aws"}) - - ec2.create_managed_prefix_list( + prefix_list = ec2.create_managed_prefix_list( PrefixListName="examplelist", MaxEntries=2, AddressFamily="?" ) + pl_id = prefix_list["PrefixList"]["PrefixListId"] all_lists = ec2.describe_managed_prefix_lists()["PrefixLists"] - all_lists.should.have.length_of(len(default_lists) + 1) + [pl["PrefixListId"] for pl in all_lists].should.contain(pl_id) set([l["OwnerId"] for l in all_lists]).should.equal({"aws", ACCOUNT_ID}) @@ -64,7 +62,9 @@ def test_describe_managed_prefix_lists_with_prefix(): ec2 = boto3.client("ec2", region_name="us-west-1") default_lists = ec2.describe_managed_prefix_lists()["PrefixLists"] - set([l["OwnerId"] for l in default_lists]).should.equal({"aws"}) + if not settings.TEST_SERVER_MODE: + # ServerMode is not guaranteed to only have AWS prefix lists + set([l["OwnerId"] for l in default_lists]).should.equal({"aws"}) random_list_id = default_lists[0]["PrefixListId"] @@ -72,7 +72,8 @@ def test_describe_managed_prefix_lists_with_prefix(): "PrefixLists" ] lists_by_id.should.have.length_of(1) - lists_by_id[0]["OwnerId"].should.equal("aws") + if not settings.TEST_SERVER_MODE: + lists_by_id[0]["OwnerId"].should.equal("aws") @mock_ec2 diff --git a/tests/test_ec2/test_regions.py b/tests/test_ec2/test_regions.py index e7d1cfe03..1633ce939 100644 --- a/tests/test_ec2/test_regions.py +++ b/tests/test_ec2/test_regions.py @@ -11,6 +11,8 @@ from moto import mock_autoscaling, mock_ec2, mock_elb from moto.ec2 import ec2_backends from tests import EXAMPLE_AMI_ID, EXAMPLE_AMI_ID2 +from uuid import uuid4 +from .test_instances import retrieve_all_instances def test_use_boto_regions(): @@ -34,7 +36,7 @@ def add_servers_to_region(ami_id, count, region): def add_servers_to_region_boto3(ami_id, count, region): ec2 = boto3.resource("ec2", region_name=region) - ec2.create_instances(ImageId=ami_id, MinCount=count, MaxCount=count) + return ec2.create_instances(ImageId=ami_id, MinCount=count, MaxCount=count) # Has boto3 equivalent @@ -55,15 +57,16 @@ def test_add_servers_to_a_single_region(): @mock_ec2 def test_add_servers_to_a_single_region_boto3(): region = "ap-northeast-1" - add_servers_to_region_boto3(EXAMPLE_AMI_ID, 1, region) - add_servers_to_region_boto3(EXAMPLE_AMI_ID2, 1, region) + id_1 = add_servers_to_region_boto3(EXAMPLE_AMI_ID, 1, region)[0].id + id_2 = add_servers_to_region_boto3(EXAMPLE_AMI_ID2, 1, region)[0].id client = boto3.client("ec2", region_name=region) - reservations = client.describe_instances()["Reservations"] - reservations.should.have.length_of(2) + instances = retrieve_all_instances(client) - image_ids = [r["Instances"][0]["ImageId"] for r in reservations] - image_ids.should.equal([EXAMPLE_AMI_ID, EXAMPLE_AMI_ID2]) + instance1 = [i for i in instances if i["InstanceId"] == id_1][0] + instance1["ImageId"].should.equal(EXAMPLE_AMI_ID) + instance2 = [i for i in instances if i["InstanceId"] == id_2][0] + instance2["ImageId"].should.equal(EXAMPLE_AMI_ID2) # Has boto3 equivalent @@ -90,19 +93,27 @@ def test_add_servers_to_multiple_regions(): def test_add_servers_to_multiple_regions_boto3(): region1 = "us-east-1" region2 = "ap-northeast-1" - add_servers_to_region_boto3(EXAMPLE_AMI_ID, 1, region1) - add_servers_to_region_boto3(EXAMPLE_AMI_ID2, 1, region2) + us_id = add_servers_to_region_boto3(EXAMPLE_AMI_ID, 1, region1)[0].id + ap_id = add_servers_to_region_boto3(EXAMPLE_AMI_ID2, 1, region2)[0].id us_client = boto3.client("ec2", region_name=region1) ap_client = boto3.client("ec2", region_name=region2) - us_reservations = us_client.describe_instances()["Reservations"] - ap_reservations = ap_client.describe_instances()["Reservations"] + us_instances = retrieve_all_instances(us_client) + ap_instances = retrieve_all_instances(ap_client) - us_reservations.should.have.length_of(1) - ap_reservations.should.have.length_of(1) + [r["InstanceId"] for r in us_instances].should.contain(us_id) + [r["InstanceId"] for r in us_instances].shouldnt.contain(ap_id) + [r["InstanceId"] for r in ap_instances].should.contain(ap_id) + [r["InstanceId"] for r in ap_instances].shouldnt.contain(us_id) - us_reservations[0]["Instances"][0]["ImageId"].should.equal(EXAMPLE_AMI_ID) - ap_reservations[0]["Instances"][0]["ImageId"].should.equal(EXAMPLE_AMI_ID2) + us_instance = us_client.describe_instances(InstanceIds=[us_id])["Reservations"][0][ + "Instances" + ][0] + us_instance["ImageId"].should.equal(EXAMPLE_AMI_ID) + ap_instance = ap_client.describe_instances(InstanceIds=[ap_id])["Reservations"][0][ + "Instances" + ][0] + ap_instance["ImageId"].should.equal(EXAMPLE_AMI_ID2) # Has boto3 equivalent @@ -213,9 +224,9 @@ def test_create_autoscaling_group_boto3(): regions = [("us-east-1", "c"), ("ap-northeast-1", "a")] for region, zone in regions: a_zone = "{}{}".format(region, zone) - asg_name = "{}_tester_group".format(region) - lb_name = "{}_lb".format(region) - config_name = "{}_tester".format(region) + asg_name = "{}_tester_group_{}".format(region, str(uuid4())[0:6]) + lb_name = "{}_lb_{}".format(region, str(uuid4())[0:6]) + config_name = "{}_tester_{}".format(region, str(uuid4())[0:6]) elb_client = boto3.client("elb", region_name=region) elb_client.create_load_balancer( @@ -254,7 +265,9 @@ def test_create_autoscaling_group_boto3(): TerminationPolicies=["OldestInstance", "NewestInstance"], ) - groups = as_client.describe_auto_scaling_groups()["AutoScalingGroups"] + groups = as_client.describe_auto_scaling_groups( + AutoScalingGroupNames=[asg_name] + )["AutoScalingGroups"] groups.should.have.length_of(1) group = groups[0] diff --git a/tests/test_ec2/test_route_tables.py b/tests/test_ec2/test_route_tables.py index 767036874..044c38191 100644 --- a/tests/test_ec2/test_route_tables.py +++ b/tests/test_ec2/test_route_tables.py @@ -11,6 +11,7 @@ import sure # noqa from moto import mock_ec2, mock_ec2_deprecated, settings from tests import EXAMPLE_AMI_ID from tests.helpers import requires_boto_gte +from uuid import uuid4 # Has boto3 equivalent @@ -212,13 +213,14 @@ def test_route_tables_filters_standard_boto3(): route_table2 = ec2.create_route_table(VpcId=vpc2.id) all_route_tables = client.describe_route_tables()["RouteTables"] - all_route_tables.should.have.length_of(5) + all_ids = [rt["RouteTableId"] for rt in all_route_tables] + all_ids.should.contain(route_table1.id) + all_ids.should.contain(route_table2.id) # Filter by main route table main_route_tables = client.describe_route_tables( Filters=[{"Name": "association.main", "Values": ["true"]}] )["RouteTables"] - main_route_tables.should.have.length_of(3) main_route_table_ids = [ route_table["RouteTableId"] for route_table in main_route_tables ] @@ -321,9 +323,6 @@ def test_route_tables_filters_associations_boto3(): client.associate_route_table(RouteTableId=route_table1.id, SubnetId=subnet2.id) client.associate_route_table(RouteTableId=route_table2.id, SubnetId=subnet3.id) - all_route_tables = client.describe_route_tables()["RouteTables"] - all_route_tables.should.have.length_of(4) - # Filter by association ID association1_route_tables = client.describe_route_tables( Filters=[ @@ -430,9 +429,6 @@ def test_route_table_associations_boto3(): subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") route_table = ec2.create_route_table(VpcId=vpc.id) - all_route_tables = client.describe_route_tables()["RouteTables"] - all_route_tables.should.have.length_of(3) - # Refresh r = client.describe_route_tables(RouteTableIds=[route_table.id])["RouteTables"][0] r["Associations"].should.have.length_of(0) @@ -584,7 +580,9 @@ def test_route_table_replace_route_table_association_boto3(): route_table2_id = ec2.create_route_table(VpcId=vpc.id).id all_route_tables = client.describe_route_tables()["RouteTables"] - all_route_tables.should.have.length_of(4) + all_ids = [rt["RouteTableId"] for rt in all_route_tables] + all_ids.should.contain(route_table1_id) + all_ids.should.contain(route_table2_id) # Refresh route_table1 = client.describe_route_tables(RouteTableIds=[route_table1_id])[ @@ -691,16 +689,17 @@ def test_route_table_get_by_tag_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") route_table = ec2.create_route_table(VpcId=vpc.id) - route_table.create_tags(Tags=[{"Key": "Name", "Value": "TestRouteTable"}]) + tag_value = str(uuid4()) + route_table.create_tags(Tags=[{"Key": "Name", "Value": tag_value}]) - filters = [{"Name": "tag:Name", "Values": ["TestRouteTable"]}] + filters = [{"Name": "tag:Name", "Values": [tag_value]}] route_tables = list(ec2.route_tables.filter(Filters=filters)) route_tables.should.have.length_of(1) route_tables[0].vpc_id.should.equal(vpc.id) route_tables[0].id.should.equal(route_table.id) route_tables[0].tags.should.have.length_of(1) - route_tables[0].tags[0].should.equal({"Key": "Name", "Value": "TestRouteTable"}) + route_tables[0].tags[0].should.equal({"Key": "Name", "Value": tag_value}) # Has boto3 equivalent @@ -1163,7 +1162,9 @@ def test_network_acl_tagging_boto3(): route_table = ec2.create_route_table(VpcId=vpc.id) route_table.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - tag = client.describe_tags()["Tags"][0] + tag = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [route_table.id]}] + )["Tags"][0] tag.should.have.key("ResourceType").equal("route-table") tag.should.have.key("Key").equal("a key") tag.should.have.key("Value").equal("some value") diff --git a/tests/test_ec2/test_security_groups.py b/tests/test_ec2/test_security_groups.py index f25080b49..f91933a38 100644 --- a/tests/test_ec2/test_security_groups.py +++ b/tests/test_ec2/test_security_groups.py @@ -8,12 +8,16 @@ import pytest import boto3 import boto +import boto.ec2 from botocore.exceptions import ClientError from boto.exception import EC2ResponseError import sure # noqa from moto import mock_ec2, mock_ec2_deprecated, settings from moto.ec2 import ec2_backend +from random import randint +from uuid import uuid4 +from unittest import SkipTest # Has boto3 equivalent @@ -67,25 +71,25 @@ def test_create_and_describe_security_group_boto3(): "An error occurred (DryRunOperation) when calling the CreateSecurityGroup operation: Request would have succeeded, but DryRun flag is set" ) - security_group = ec2.create_security_group( - GroupName="test security group", Description="test" - ) + sec_name = str(uuid4()) + security_group = ec2.create_security_group(GroupName=sec_name, Description="test") - security_group.group_name.should.equal("test security group") + security_group.group_name.should.equal(sec_name) security_group.description.should.equal("test") # Trying to create another group with the same name should throw an error with pytest.raises(ClientError) as ex: - client.create_security_group(GroupName="test security group", Description="n/a") + client.create_security_group(GroupName=sec_name, Description="n/a") ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidGroup.Duplicate") - all_groups = client.describe_security_groups()["SecurityGroups"] + all_groups = retrieve_all_sgs(client) # The default group gets created automatically - all_groups.should.have.length_of(2) - group_names = [group["GroupName"] for group in all_groups] - set(group_names).should.equal(set(["default", "test security group"])) + [g["GroupId"] for g in all_groups].should.contain(security_group.id) + group_names = set([group["GroupName"] for group in all_groups]) + group_names.should.contain("default") + group_names.should.contain(sec_name) # Has boto3 equivalent @@ -123,9 +127,8 @@ def test_default_security_group(): @mock_ec2 def test_default_security_group_boto3(): client = boto3.client("ec2", "us-west-1") - groups = client.describe_security_groups()["SecurityGroups"] - groups.should.have.length_of(1) - groups[0]["GroupName"].should.equal("default") + groups = retrieve_all_sgs(client) + [g["GroupName"] for g in groups].should.contain("default") # Has boto3 equivalent @@ -165,16 +168,16 @@ def test_create_and_describe_vpc_security_group_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - name = "test secgr" - vpc_id = "vpc-5300000c" - security_group = ec2.create_security_group( + name = str(uuid4()) + vpc_id = f"vpc-{str(uuid4())[0:6]}" + group_with = ec2.create_security_group( GroupName=name, Description="test", VpcId=vpc_id ) - security_group.vpc_id.should.equal(vpc_id) + group_with.vpc_id.should.equal(vpc_id) - security_group.group_name.should.equal(name) - security_group.description.should.equal("test") + group_with.group_name.should.equal(name) + group_with.description.should.equal("test") # Trying to create another group with the same name in the same VPC should # throw an error @@ -185,10 +188,13 @@ def test_create_and_describe_vpc_security_group_boto3(): ex.value.response["Error"]["Code"].should.equal("InvalidGroup.Duplicate") # Trying to create another group in the same name without VPC should pass - ec2.create_security_group(GroupName=name, Description="non-vpc-group") + group_without = ec2.create_security_group( + GroupName=name, Description="non-vpc-group" + ) - all_groups = client.describe_security_groups()["SecurityGroups"] - all_groups.should.have.length_of(3) # 1 default, 1 vpc, 1 no-vpc + all_groups = retrieve_all_sgs(client) + [a["GroupId"] for a in all_groups].should.contain(group_with.id) + [a["GroupId"] for a in all_groups].should.contain(group_without.id) all_groups = client.describe_security_groups( Filters=[{"Name": "vpc-id", "Values": [vpc_id]}] @@ -226,19 +232,20 @@ def test_create_two_security_groups_with_same_name_in_different_vpc_boto3(): ec2 = boto3.resource("ec2", "us-east-1") client = boto3.client("ec2", "us-east-1") - name = "test security group" + name = str(uuid4()) vpc_id = "vpc-5300000c" vpc_id2 = "vpc-5300000d" - ec2.create_security_group(GroupName=name, Description="n/a 1", VpcId=vpc_id) - ec2.create_security_group(GroupName=name, Description="n/a 2", VpcId=vpc_id2) + sg1 = ec2.create_security_group(GroupName=name, Description="n/a 1", VpcId=vpc_id) + sg2 = ec2.create_security_group(GroupName=name, Description="n/a 2", VpcId=vpc_id2) - all_groups = client.describe_security_groups()["SecurityGroups"] + all_groups = retrieve_all_sgs(client) + group_ids = [group["GroupId"] for group in all_groups] + group_ids.should.contain(sg1.id) + group_ids.should.contain(sg2.id) - all_groups.should.have.length_of(3) group_names = [group["GroupName"] for group in all_groups] - # The default group is created automatically - set(group_names).should.equal(set(["default", name])) + group_names.should.contain(name) @mock_ec2 @@ -247,7 +254,7 @@ def test_create_two_security_groups_in_vpc_with_ipv6_enabled(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16", AmazonProvidedIpv6CidrBlock=True) security_group = ec2.create_security_group( - GroupName="sg01", Description="Test security group sg01", VpcId=vpc.id + GroupName=str(uuid4()), Description="Test security group sg01", VpcId=vpc.id ) # The security group must have two defaul egress rules (one for ipv4 and aonther for ipv6) @@ -291,10 +298,14 @@ def test_deleting_security_groups(): def test_deleting_security_groups_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - security_group1 = ec2.create_security_group(GroupName="test1", Description="test1") - ec2.create_security_group(GroupName="test2", Description="test2") + sg_name1 = str(uuid4()) + sg_name2 = str(uuid4()) + group1 = ec2.create_security_group(GroupName=sg_name1, Description="test desc 1") + group2 = ec2.create_security_group(GroupName=sg_name2, Description="test desc 2") - client.describe_security_groups()["SecurityGroups"].should.have.length_of(3) + all_groups = retrieve_all_sgs(client) + [g["GroupId"] for g in all_groups].should.contain(group1.id) + [g["GroupId"] for g in all_groups].should.contain(group2.id) # Deleting a group that doesn't exist should throw an error with pytest.raises(ClientError) as ex: @@ -305,19 +316,24 @@ def test_deleting_security_groups_boto3(): # Delete by name with pytest.raises(ClientError) as ex: - client.delete_security_group(GroupName="test2", DryRun=True) + client.delete_security_group(GroupName=sg_name2, DryRun=True) ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(412) ex.value.response["Error"]["Code"].should.equal("DryRunOperation") ex.value.response["Error"]["Message"].should.equal( "An error occurred (DryRunOperation) when calling the DeleteSecurityGroup operation: Request would have succeeded, but DryRun flag is set" ) - client.delete_security_group(GroupName="test2") - client.describe_security_groups()["SecurityGroups"].should.have.length_of(2) + client.delete_security_group(GroupName=sg_name2) + + all_groups = retrieve_all_sgs(client) + [g["GroupId"] for g in all_groups].should.contain(group1.id) + [g["GroupId"] for g in all_groups].shouldnt.contain(group2.id) # Delete by group id - client.delete_security_group(GroupId=security_group1.id) - client.describe_security_groups()["SecurityGroups"].should.have.length_of(1) + client.delete_security_group(GroupId=group1.id) + + all_groups = retrieve_all_sgs(client) + [g["GroupId"] for g in all_groups].shouldnt.contain(group1.id) # Has boto3 equivalent @@ -336,18 +352,18 @@ def test_delete_security_group_in_vpc_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - client.describe_security_groups()["SecurityGroups"].should.have.length_of(1) - group = ec2.create_security_group( - GroupName="test1", Description="test1", VpcId="vpc-12345" + GroupName=str(uuid4()), Description="test1", VpcId="vpc-12345" ) - client.describe_security_groups()["SecurityGroups"].should.have.length_of(2) + all_groups = retrieve_all_sgs(client) + [g["GroupId"] for g in all_groups].should.contain(group.id) # this should not throw an exception client.delete_security_group(GroupId=group.id) - client.describe_security_groups()["SecurityGroups"].should.have.length_of(1) + all_groups = retrieve_all_sgs(client) + [g["GroupId"] for g in all_groups].shouldnt.contain(group.id) # Has boto3 equivalent @@ -492,7 +508,9 @@ def test_authorize_ip_range_and_revoke(): def test_authorize_ip_range_and_revoke_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - security_group = ec2.create_security_group(GroupName="test", Description="test") + security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="test" + ) with pytest.raises(ClientError) as ex: security_group.authorize_ingress( @@ -550,7 +568,7 @@ def test_authorize_ip_range_and_revoke_boto3(): # Test for egress as well egress_security_group = ec2.create_security_group( - GroupName="testegress", Description="testegress", VpcId="vpc-3432589" + GroupName=str(uuid4()), Description="desc", VpcId="vpc-3432589" ) egress_permissions = [ { @@ -656,11 +674,14 @@ def test_authorize_other_group_and_revoke(): def test_authorize_other_group_and_revoke_boto3(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - security_group = ec2.create_security_group(GroupName="test", Description="test") - other_security_group = ec2.create_security_group( - GroupName="other", Description="other" + sg_name = str(uuid4()) + security_group = ec2.create_security_group( + GroupName=sg_name, Description="test desc" ) - ec2.create_security_group(GroupName="wrong", Description="wrong") + other_security_group = ec2.create_security_group( + GroupName=str(uuid4()), Description="other" + ) + ec2.create_security_group(GroupName=str(uuid4()), Description="wrong") # Note: Should be easier to use the SourceSecurityGroupNames-parameter, but that's not supported atm permissions = [ @@ -679,7 +700,7 @@ def test_authorize_other_group_and_revoke_boto3(): ] security_group.authorize_ingress(IpPermissions=permissions) - found_sec_group = client.describe_security_groups(GroupNames=["test"])[ + found_sec_group = client.describe_security_groups(GroupNames=[sg_name])[ "SecurityGroups" ][0] found_sec_group["IpPermissions"][0]["ToPort"].should.equal(2222) @@ -699,7 +720,7 @@ def test_authorize_other_group_and_revoke_boto3(): # Actually revoke security_group.revoke_ingress(IpPermissions=permissions) - found_sec_group = client.describe_security_groups(GroupNames=["test"])[ + found_sec_group = client.describe_security_groups(GroupNames=[sg_name])[ "SecurityGroups" ][0] found_sec_group["IpPermissions"].should.have.length_of(0) @@ -712,10 +733,10 @@ def test_authorize_other_group_egress_and_revoke(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") sg01 = ec2.create_security_group( - GroupName="sg01", Description="Test security group sg01", VpcId=vpc.id + GroupName=str(uuid4()), Description="Test security group sg01", VpcId=vpc.id ) sg02 = ec2.create_security_group( - GroupName="sg02", Description="Test security group sg02", VpcId=vpc.id + GroupName=str(uuid4()), Description="Test security group sg02", VpcId=vpc.id ) ip_permission = { @@ -788,11 +809,13 @@ def test_authorize_group_in_vpc_boto3(): vpc_id = "vpc-12345" # create 2 groups in a vpc + sec_name1 = str(uuid4()) + sec_name2 = str(uuid4()) security_group = ec2.create_security_group( - GroupName="test1", Description="test1", VpcId=vpc_id + GroupName=sec_name1, Description="test desc 1", VpcId=vpc_id ) other_security_group = ec2.create_security_group( - GroupName="test2", Description="test2", VpcId=vpc_id + GroupName=sec_name2, Description="test desc 2", VpcId=vpc_id ) permissions = [ @@ -812,7 +835,7 @@ def test_authorize_group_in_vpc_boto3(): security_group.authorize_ingress(IpPermissions=permissions) # Check that the rule is accurate - found_sec_group = client.describe_security_groups(GroupNames=["test1"])[ + found_sec_group = client.describe_security_groups(GroupNames=[sec_name1])[ "SecurityGroups" ][0] found_sec_group["IpPermissions"][0]["ToPort"].should.equal(2222) @@ -824,7 +847,7 @@ def test_authorize_group_in_vpc_boto3(): security_group.revoke_ingress(IpPermissions=permissions) # And check that it gets revoked - found_sec_group = client.describe_security_groups(GroupNames=["test1"])[ + found_sec_group = client.describe_security_groups(GroupNames=[sec_name1])[ "SecurityGroups" ][0] found_sec_group["IpPermissions"].should.have.length_of(0) @@ -872,13 +895,13 @@ def test_get_all_security_groups(): def test_describe_security_groups(): ec2 = boto3.resource("ec2", "us-west-1") client = boto3.client("ec2", "us-west-1") - vpc_id = "vpc-mjm05d27" - sg1 = ec2.create_security_group( - GroupName="test1", Description="test1", VpcId=vpc_id - ) - ec2.create_security_group(GroupName="test2", Description="test2") + vpc_id = f"vpc-{str(uuid4())[0:6]}" + name_1 = str(uuid4()) + desc_1 = str(uuid4()) + sg1 = ec2.create_security_group(GroupName=name_1, Description=desc_1, VpcId=vpc_id) + sg2 = ec2.create_security_group(GroupName=str(uuid4()), Description="test desc 2") - resp = client.describe_security_groups(GroupNames=["test1"])["SecurityGroups"] + resp = client.describe_security_groups(GroupNames=[name_1])["SecurityGroups"] resp.should.have.length_of(1) resp[0].should.have.key("GroupId").equal(sg1.id) @@ -895,13 +918,15 @@ def test_describe_security_groups(): resp[0].should.have.key("GroupId").equal(sg1.id) resp = client.describe_security_groups( - Filters=[{"Name": "description", "Values": ["test1"]}] + Filters=[{"Name": "description", "Values": [desc_1]}] )["SecurityGroups"] resp.should.have.length_of(1) resp[0].should.have.key("GroupId").equal(sg1.id) - resp = client.describe_security_groups()["SecurityGroups"] - resp.should.have.length_of(3) + all_sgs = retrieve_all_sgs(client) + sg_ids = [sg["GroupId"] for sg in all_sgs] + sg_ids.should.contain(sg1.id) + sg_ids.should.contain(sg2.id) # Has boto3 equivalent @@ -921,7 +946,7 @@ def test_authorize_bad_cidr_throws_invalid_parameter_value(): @mock_ec2 def test_authorize_bad_cidr_throws_invalid_parameter_value_boto3(): ec2 = boto3.resource("ec2", "us-west-1") - sec_group = ec2.create_security_group(GroupName="test", Description="test") + sec_group = ec2.create_security_group(GroupName=str(uuid4()), Description="test") with pytest.raises(ClientError) as ex: permissions = [ { @@ -979,16 +1004,18 @@ def test_security_group_tag_filtering(): def test_security_group_tag_filtering_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") client = boto3.client("ec2", region_name="us-east-1") - sg = ec2.create_security_group(GroupName="test-sg", Description="Test SG") - sg.create_tags(Tags=[{"Key": "test-tag", "Value": "test-value"}]) + sg = ec2.create_security_group(GroupName=str(uuid4()), Description="Test SG") + tag_name = str(uuid4())[0:6] + tag_val = str(uuid4()) + sg.create_tags(Tags=[{"Key": tag_name, "Value": tag_val}]) groups = client.describe_security_groups( - Filters=[{"Name": "tag:test-tag", "Values": ["test-value"]}] + Filters=[{"Name": f"tag:{tag_name}", "Values": [tag_val]}] )["SecurityGroups"] groups.should.have.length_of(1) groups = client.describe_security_groups( - Filters=[{"Name": "tag:test-tag", "Values": ["unknown"]}] + Filters=[{"Name": f"tag:{tag_name}", "Values": ["unknown"]}] )["SecurityGroups"] groups.should.have.length_of(0) @@ -1011,12 +1038,13 @@ def test_authorize_all_protocols_with_no_port_specification(): def test_authorize_all_protocols_with_no_port_specification_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") client = boto3.client("ec2", region_name="us-east-1") - sg = ec2.create_security_group(GroupName="test", Description="test") + sg_name = str(uuid4()) + sg = ec2.create_security_group(GroupName=sg_name, Description="test desc") permissions = [{"IpProtocol": "-1", "IpRanges": [{"CidrIp": "0.0.0.0/0"}]}] sg.authorize_ingress(IpPermissions=permissions) - sg = client.describe_security_groups(GroupNames=["test"])["SecurityGroups"][0] + sg = client.describe_security_groups(GroupNames=[sg_name])["SecurityGroups"][0] permission = sg["IpPermissions"][0] permission.should.have.key("IpProtocol").equal("-1") permission.should.have.key("IpRanges").equal([{"CidrIp": "0.0.0.0/0"}]) @@ -1102,15 +1130,15 @@ def test_sec_group_rule_limit_boto3(use_vpc): if use_vpc: vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") sg = ec2.create_security_group( - GroupName="test", Description="test", VpcId=vpc.id + GroupName=str(uuid4()), Description="test", VpcId=vpc.id ) other_sg = ec2.create_security_group( - GroupName="test_2", Description="test_other", VpcId=vpc.id + GroupName=str(uuid4()), Description="test_other", VpcId=vpc.id ) else: - sg = ec2.create_security_group(GroupName="test", Description="test") + sg = ec2.create_security_group(GroupName=str(uuid4()), Description="test") other_sg = ec2.create_security_group( - GroupName="test_2", Description="test_other" + GroupName=str(uuid4()), Description="test_other" ) # INGRESS @@ -1383,7 +1411,7 @@ def test_description_in_ip_permissions(): def test_security_group_tagging_boto3(): conn = boto3.client("ec2", region_name="us-east-1") - sg = conn.create_security_group(GroupName="test-sg", Description="Test SG") + sg = conn.create_security_group(GroupName=str(uuid4()), Description="Test SG") with pytest.raises(ClientError) as ex: conn.create_tags( @@ -1397,26 +1425,34 @@ def test_security_group_tagging_boto3(): "An error occurred (DryRunOperation) when calling the CreateTags operation: Request would have succeeded, but DryRun flag is set" ) - conn.create_tags(Resources=[sg["GroupId"]], Tags=[{"Key": "Test", "Value": "Tag"}]) + tag_val = str(uuid4()) + conn.create_tags( + Resources=[sg["GroupId"]], Tags=[{"Key": "Test", "Value": tag_val}] + ) describe = conn.describe_security_groups( - Filters=[{"Name": "tag-value", "Values": ["Tag"]}] + Filters=[{"Name": "tag-value", "Values": [tag_val]}] ) tag = describe["SecurityGroups"][0]["Tags"][0] - tag["Value"].should.equal("Tag") + tag["Value"].should.equal(tag_val) tag["Key"].should.equal("Test") @mock_ec2 def test_security_group_wildcard_tag_filter_boto3(): conn = boto3.client("ec2", region_name="us-east-1") - sg = conn.create_security_group(GroupName="test-sg", Description="Test SG") - conn.create_tags(Resources=[sg["GroupId"]], Tags=[{"Key": "Test", "Value": "Tag"}]) + sg = conn.create_security_group(GroupName=str(uuid4()), Description="Test SG") + + rand_name = str(uuid4())[0:6] + tag_val = f"random {rand_name} things" + conn.create_tags( + Resources=[sg["GroupId"]], Tags=[{"Key": "Test", "Value": tag_val}] + ) describe = conn.describe_security_groups( - Filters=[{"Name": "tag-value", "Values": ["*"]}] + Filters=[{"Name": "tag-value", "Values": [f"*{rand_name}*"]}] ) tag = describe["SecurityGroups"][0]["Tags"][0] - tag["Value"].should.equal("Tag") + tag["Value"].should.equal(tag_val) tag["Key"].should.equal("Test") @@ -1426,22 +1462,40 @@ def test_security_group_filter_ip_permission(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") conn = boto3.client("ec2", region_name="us-east-1") + sg_name = str(uuid4())[0:6] sg = ec2.create_security_group( - GroupName="test-sg", Description="Test SG", VpcId=vpc.id + GroupName=sg_name, Description="Test SG", VpcId=vpc.id ) + from_port = randint(0, 65535) + to_port = randint(0, 65535) ip_permissions = [ - {"IpProtocol": "tcp", "FromPort": 27017, "ToPort": 27017, "IpRanges": [],}, + { + "IpProtocol": "tcp", + "FromPort": from_port, + "ToPort": to_port, + "IpRanges": [], + }, ] sg.authorize_ingress(IpPermissions=ip_permissions) - describe = conn.describe_security_groups( - Filters=[{"Name": "ip-permission.from-port", "Values": ["27017"]}] - )["SecurityGroups"] + filters = [{"Name": "ip-permission.from-port", "Values": [f"{from_port}"]}] + describe = retrieve_all_sgs(conn, filters) describe.should.have.length_of(1) - describe[0]["GroupName"].should.equal("test-sg") + describe[0]["GroupName"].should.equal(sg_name) + + +def retrieve_all_sgs(conn, filters=[]): + res = conn.describe_security_groups(Filters=filters) + all_groups = res["SecurityGroups"] + next_token = res.get("NextToken") + while next_token: + res = conn.describe_security_groups(Filters=filters) + all_groups.extend(res["SecurityGroups"]) + next_token = res.get("NextToken") + return all_groups @mock_ec2 @@ -1451,17 +1505,13 @@ def test_authorize_and_revoke_in_bulk(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") sg01 = ec2.create_security_group( - GroupName="sg01", Description="Test security group sg01", VpcId=vpc.id + GroupName=str(uuid4()), Description="Test sg01", VpcId=vpc.id ) sg02 = ec2.create_security_group( - GroupName="sg02", Description="Test security group sg02", VpcId=vpc.id - ) - sg03 = ec2.create_security_group( - GroupName="sg03", Description="Test security group sg03" - ) - sg04 = ec2.create_security_group( - GroupName="sg04", Description="Test security group sg04" + GroupName=str(uuid4()), Description="Test sg02", VpcId=vpc.id ) + sg03 = ec2.create_security_group(GroupName=str(uuid4()), Description="Test sg03") + sg04 = ec2.create_security_group(GroupName=str(uuid4()), Description="Test sg04") ip_permissions = [ { "IpProtocol": "tcp", @@ -1595,7 +1645,7 @@ def test_authorize_and_revoke_in_bulk(): @mock_ec2 def test_security_group_ingress_without_multirule(): ec2 = boto3.resource("ec2", "ca-central-1") - sg = ec2.create_security_group(Description="Test SG", GroupName="test-sg") + sg = ec2.create_security_group(Description="Test SG", GroupName=str(uuid4())) assert len(sg.ip_permissions) == 0 sg.authorize_ingress( @@ -1609,7 +1659,7 @@ def test_security_group_ingress_without_multirule(): @mock_ec2 def test_security_group_ingress_without_multirule_after_reload(): ec2 = boto3.resource("ec2", "ca-central-1") - sg = ec2.create_security_group(Description="Test SG", GroupName="test-sg") + sg = ec2.create_security_group(Description="Test SG", GroupName=str(uuid4())) assert len(sg.ip_permissions) == 0 sg.authorize_ingress( @@ -1650,10 +1700,10 @@ def test_get_all_security_groups_filter_with_same_vpc_id_boto3(): client = boto3.client("ec2", region_name="us-east-1") vpc_id = "vpc-5300000c" security_group = ec2.create_security_group( - GroupName="test1", Description="test1", VpcId=vpc_id + GroupName=str(uuid4()), Description="test1", VpcId=vpc_id ) security_group2 = ec2.create_security_group( - GroupName="test2", Description="test2", VpcId=vpc_id + GroupName=str(uuid4()), Description="test2", VpcId=vpc_id ) security_group.vpc_id.should.equal(vpc_id) @@ -1676,7 +1726,7 @@ def test_revoke_security_group_egress(): ec2 = boto3.resource("ec2", "us-east-1") vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") sg = ec2.create_security_group( - Description="Test SG", GroupName="test-sg", VpcId=vpc.id + Description="Test SG", GroupName=str(uuid4()), VpcId=vpc.id ) sg.ip_permissions_egress.should.equal( @@ -1712,8 +1762,9 @@ def test_update_security_group_rule_descriptions_egress(): ec2 = boto3.resource("ec2", "us-east-1") client = boto3.client("ec2", "us-east-1") vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") + sg_name = str(uuid4()) sg = ec2.create_security_group( - Description="Test SG", GroupName="test-sg", VpcId=vpc.id + Description="Test SG", GroupName=sg_name, VpcId=vpc.id ) sg_id = sg.id @@ -1724,7 +1775,7 @@ def test_update_security_group_rule_descriptions_egress(): ip_ranges[0].should.equal({"CidrIp": "0.0.0.0/0"}) client.update_security_group_rule_descriptions_egress( - GroupName="test-sg", + GroupName=sg_name, IpPermissions=[ { "IpProtocol": "-1", @@ -1748,8 +1799,9 @@ def test_update_security_group_rule_descriptions_ingress(): ec2 = boto3.resource("ec2", "us-east-1") client = boto3.client("ec2", "us-east-1") vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") + sg_name = str(uuid4()) sg = ec2.create_security_group( - Description="Test SG", GroupName="test-sg", VpcId=vpc.id + Description="Test SG", GroupName=sg_name, VpcId=vpc.id ) sg_id = sg.id @@ -1770,7 +1822,7 @@ def test_update_security_group_rule_descriptions_ingress(): ip_ranges[0].should.equal({"CidrIp": "1.2.3.4/32", "Description": "first desc"}) client.update_security_group_rule_descriptions_ingress( - GroupName="test-sg", + GroupName=sg_name, IpPermissions=[ { "IpProtocol": "tcp", @@ -1811,7 +1863,7 @@ def test_security_group_rules_added_via_the_backend_can_be_revoked_via_the_api() ec2_resource = boto3.resource("ec2", region_name="us-east-1") ec2_client = boto3.client("ec2", region_name="us-east-1") vpc = ec2_resource.create_vpc(CidrBlock="10.0.0.0/16") - group_name = "test-backend-authorize" + group_name = str(uuid4()) sg = ec2_resource.create_security_group( GroupName=group_name, Description="test", VpcId=vpc.id ) @@ -1859,8 +1911,9 @@ def test_filter_description(): ec2r = boto3.resource("ec2", region_name="us-west-1") vpc = ec2r.create_vpc(CidrBlock="10.250.0.0/16") + unique = str(uuid4()) sg1 = vpc.create_security_group( - Description="An Excellent Description", GroupName="test-1" + Description=(f"A {unique} Description"), GroupName="test-1" ) sg2 = vpc.create_security_group( Description="Another Description That Awes The Human Mind", GroupName="test-2" @@ -1868,7 +1921,7 @@ def test_filter_description(): filter_to_match_group_1_description = { "Name": "description", - "Values": ["Excellent"], + "Values": [unique], } security_groups = ec2r.security_groups.filter( @@ -1882,6 +1935,10 @@ def test_filter_description(): @mock_ec2 def test_filter_egress__ip_permission__cidr(): + if settings.TEST_SERVER_MODE: + raise SkipTest( + "CIDR's might already exist due to other tests creating IP ranges" + ) ec2r = boto3.resource("ec2", region_name="us-west-1") vpc = ec2r.create_vpc(CidrBlock="10.250.1.0/16") @@ -1935,10 +1992,11 @@ def test_filter_egress__ip_permission__from_port(): Description="Another Description That Awes The Human Mind", GroupName="test-2" ) + from_port = randint(1, 9999) sg1.authorize_egress( IpPermissions=[ { - "FromPort": 7357, + "FromPort": from_port, "ToPort": 7359, "IpProtocol": "tcp", "IpRanges": [{"CidrIp": "10.250.0.0/16"}, {"CidrIp": "10.251.0.0/16"}], @@ -1957,7 +2015,7 @@ def test_filter_egress__ip_permission__from_port(): ) filter_to_match_group_1 = { "Name": "egress.ip-permission.from-port", - "Values": ["7357"], + "Values": [f"{from_port}"], } security_groups = ec2r.security_groups.filter(Filters=[filter_to_match_group_1]) @@ -2162,9 +2220,9 @@ def test_filter_egress__ip_permission__protocol(): } security_groups = ec2r.security_groups.filter(Filters=[filter_to_match_group_1]) - security_groups = list(security_groups) - assert len(security_groups) == 1 - assert security_groups[0].group_id == sg1.group_id + group_ids = [sg.group_id for sg in security_groups] + group_ids.should.contain(sg1.group_id) + group_ids.shouldnt.contain(sg2.group_id) @mock_ec2 @@ -2179,11 +2237,12 @@ def test_filter_egress__ip_permission__to_port(): Description="Another Description That Awes The Human Mind", GroupName="test-2" ) + to_port = randint(1, 9999) sg1.authorize_egress( IpPermissions=[ { "FromPort": 7357, - "ToPort": 7359, + "ToPort": to_port, "IpProtocol": "tcp", "IpRanges": [{"CidrIp": "10.250.0.0/16"}, {"CidrIp": "10.251.0.0/16"}], } @@ -2201,7 +2260,7 @@ def test_filter_egress__ip_permission__to_port(): ) filter_to_match_group_1 = { "Name": "egress.ip-permission.to-port", - "Values": ["7359"], + "Values": [f"{to_port}"], } security_groups = ec2r.security_groups.filter(Filters=[filter_to_match_group_1]) diff --git a/tests/test_ec2/test_security_groups_cloudformation.py b/tests/test_ec2/test_security_groups_cloudformation.py index e3cdafc07..e06c11ddc 100644 --- a/tests/test_ec2/test_security_groups_cloudformation.py +++ b/tests/test_ec2/test_security_groups_cloudformation.py @@ -3,9 +3,13 @@ import json import sure # noqa from moto import mock_cloudformation, mock_ec2 from tests import EXAMPLE_AMI_ID +from string import Template +from uuid import uuid4 +from .test_instances import retrieve_all_instances -SEC_GROUP_INGRESS = """{ +SEC_GROUP_INGRESS = Template( + """{ "AWSTemplateFormatVersion": "2010-09-09", "Description": "AWS CloudFormation Template to create an EC2 instance", "Parameters": { @@ -20,7 +24,7 @@ SEC_GROUP_INGRESS = """{ "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "Test VPC security group", - "GroupName": "My-SG", + "GroupName": $group_name, "VpcId": { "Ref": "VPCId" } @@ -45,9 +49,11 @@ SEC_GROUP_INGRESS = """{ } } """ +) -SEC_GROUP_INGRESS_WITHOUT_DESC = """{ +SEC_GROUP_INGRESS_WITHOUT_DESC = Template( + """{ "AWSTemplateFormatVersion": "2010-09-09", "Description": "AWS CloudFormation Template to create an EC2 instance", "Parameters": { @@ -62,7 +68,7 @@ SEC_GROUP_INGRESS_WITHOUT_DESC = """{ "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "Test VPC security group", - "GroupName": "My-SG", + "GroupName": "$group_name", "VpcId": { "Ref": "VPCId" } @@ -86,6 +92,7 @@ SEC_GROUP_INGRESS_WITHOUT_DESC = """{ } } """ +) SEC_GROUP_SOURCE = { "AWSTemplateFormatVersion": "2010-09-09", @@ -133,17 +140,18 @@ def test_security_group_ingress(): ec2 = boto3.resource("ec2", region_name="us-west-1") ec2_client = boto3.client("ec2", region_name="us-east-1") + group_name = str(uuid4()) vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") cf_client.create_stack( - StackName="test_stack", - TemplateBody=SEC_GROUP_INGRESS, + StackName=str(uuid4()), + TemplateBody=SEC_GROUP_INGRESS.substitute(group_name=group_name), Parameters=[{"ParameterKey": "VPCId", "ParameterValue": vpc.id}], Capabilities=["CAPABILITY_NAMED_IAM"], OnFailure="DELETE", ) groups = ec2_client.describe_security_groups()["SecurityGroups"] - group = [g for g in groups if g["GroupName"] == "My-SG"][0] + group = [g for g in groups if g["GroupName"] == group_name][0] group["Description"].should.equal("Test VPC security group") len(group["IpPermissions"]).should.be(1) ingress = group["IpPermissions"][0] @@ -162,17 +170,18 @@ def test_security_group_ingress_without_description(): ec2 = boto3.resource("ec2", region_name="us-west-1") ec2_client = boto3.client("ec2", region_name="us-east-1") + group_name = str(uuid4()) vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") cf_client.create_stack( - StackName="test_stack", - TemplateBody=SEC_GROUP_INGRESS_WITHOUT_DESC, + StackName=str(uuid4()), + TemplateBody=SEC_GROUP_INGRESS_WITHOUT_DESC.substitute(group_name=group_name), Parameters=[{"ParameterKey": "VPCId", "ParameterValue": vpc.id}], Capabilities=["CAPABILITY_NAMED_IAM"], OnFailure="DELETE", ) groups = ec2_client.describe_security_groups()["SecurityGroups"] - group = [g for g in groups if g["GroupName"] == "My-SG"][0] + group = [g for g in groups if g["GroupName"] == group_name][0] group["Description"].should.equal("Test VPC security group") len(group["IpPermissions"]).should.be(1) ingress = group["IpPermissions"][0] @@ -183,28 +192,46 @@ def test_security_group_ingress_without_description(): @mock_cloudformation def test_stack_security_groups(): - template = json.dumps(SEC_GROUP_SOURCE) + first_desc = str(uuid4()) + second_desc = str(uuid4()) + our_template = SEC_GROUP_SOURCE.copy() + our_template["Resources"]["my-security-group"]["Properties"][ + "GroupDescription" + ] = second_desc + our_template["Resources"]["InstanceSecurityGroup"]["Properties"][ + "GroupDescription" + ] = first_desc + + template = json.dumps(our_template) cf = boto3.client("cloudformation", region_name="us-west-1") + stack_name = str(uuid4())[0:6] cf.create_stack( - StackName="security_group_stack", + StackName=stack_name, TemplateBody=template, Tags=[{"Key": "foo", "Value": "bar"}], ) ec2 = boto3.client("ec2", region_name="us-west-1") instance_group = ec2.describe_security_groups( - Filters=[{"Name": "description", "Values": ["My security group"]}] + Filters=[{"Name": "description", "Values": [first_desc]}] )["SecurityGroups"][0] - instance_group.should.have.key("Description").equal("My security group") + instance_group.should.have.key("Description").equal(first_desc) instance_group.should.have.key("Tags") instance_group["Tags"].should.contain({"Key": "bar", "Value": "baz"}) instance_group["Tags"].should.contain({"Key": "foo", "Value": "bar"}) other_group = ec2.describe_security_groups( - Filters=[{"Name": "description", "Values": ["My other group"]}] + Filters=[{"Name": "description", "Values": [second_desc]}] )["SecurityGroups"][0] - ec2_instance = ec2.describe_instances()["Reservations"][0]["Instances"][0] + instance = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"][ + 1 + ] + instance_id = instance["PhysicalResourceId"] + + ec2_instance = ec2.describe_instances(InstanceIds=[instance_id])["Reservations"][0][ + "Instances" + ][0] ec2_instance["NetworkInterfaces"][0]["Groups"][0]["GroupId"].should.equal( instance_group["GroupId"] diff --git a/tests/test_ec2/test_spot_instances.py b/tests/test_ec2/test_spot_instances.py index 179bbaa82..3bd4e5e02 100644 --- a/tests/test_ec2/test_spot_instances.py +++ b/tests/test_ec2/test_spot_instances.py @@ -3,6 +3,7 @@ import pytest import datetime import boto +import boto.ec2 import boto3 from boto.exception import EC2ResponseError from botocore.exceptions import ClientError @@ -13,6 +14,8 @@ from moto import mock_ec2, mock_ec2_deprecated, settings from moto.ec2.models import ec2_backends from moto.core.utils import iso_8601_datetime_with_milliseconds from tests import EXAMPLE_AMI_ID +from uuid import uuid4 +from unittest import SkipTest @mock_ec2 @@ -24,8 +27,10 @@ def test_request_spot_instances(): )["Subnet"] subnet_id = subnet["SubnetId"] - conn.create_security_group(GroupName="group1", Description="description") - conn.create_security_group(GroupName="group2", Description="description") + sec_name_1 = str(uuid4()) + sec_name_2 = str(uuid4()) + conn.create_security_group(GroupName=sec_name_1, Description="description") + conn.create_security_group(GroupName=sec_name_2, Description="description") start_dt = datetime.datetime(2013, 1, 1).replace(tzinfo=pytz.utc) end_dt = datetime.datetime(2013, 1, 2).replace(tzinfo=pytz.utc) @@ -44,7 +49,7 @@ def test_request_spot_instances(): LaunchSpecification={ "ImageId": EXAMPLE_AMI_ID, "KeyName": "test", - "SecurityGroups": ["group1", "group2"], + "SecurityGroups": [sec_name_1, sec_name_2], "UserData": "some test data", "InstanceType": "m1.small", "Placement": {"AvailabilityZone": "us-east-1c"}, @@ -72,7 +77,7 @@ def test_request_spot_instances(): LaunchSpecification={ "ImageId": EXAMPLE_AMI_ID, "KeyName": "test", - "SecurityGroups": ["group1", "group2"], + "SecurityGroups": [sec_name_1, sec_name_2], "UserData": "some test data", "InstanceType": "m1.small", "Placement": {"AvailabilityZone": "us-east-1c"}, @@ -82,8 +87,10 @@ def test_request_spot_instances(): "SubnetId": subnet_id, }, ) + request_id = request["SpotInstanceRequests"][0]["SpotInstanceRequestId"] - requests = conn.describe_spot_instance_requests()["SpotInstanceRequests"] + all_requests = conn.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = [r for r in all_requests if r["SpotInstanceRequestId"] == request_id] requests.should.have.length_of(1) request = requests[0] @@ -99,7 +106,7 @@ def test_request_spot_instances(): security_group_names = [ group["GroupName"] for group in launch_spec["SecurityGroups"] ] - set(security_group_names).should.equal(set(["group1", "group2"])) + set(security_group_names).should.equal(set([sec_name_1, sec_name_2])) launch_spec["ImageId"].should.equal(EXAMPLE_AMI_ID) launch_spec["KeyName"].should.equal("test") @@ -119,8 +126,10 @@ def test_request_spot_instances_default_arguments(): request = conn.request_spot_instances( SpotPrice="0.5", LaunchSpecification={"ImageId": EXAMPLE_AMI_ID} ) + request_id = request["SpotInstanceRequests"][0]["SpotInstanceRequestId"] - requests = conn.describe_spot_instance_requests()["SpotInstanceRequests"] + all_requests = conn.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = [r for r in all_requests if r["SpotInstanceRequestId"] == request_id] requests.should.have.length_of(1) request = requests[0] @@ -175,11 +184,14 @@ def test_cancel_spot_instance_request(): def test_cancel_spot_instance_request_boto3(): client = boto3.client("ec2", region_name="us-west-1") - client.request_spot_instances( + rsi = client.request_spot_instances( SpotPrice="0.5", LaunchSpecification={"ImageId": EXAMPLE_AMI_ID} ) + spot_id = rsi["SpotInstanceRequests"][0]["SpotInstanceRequestId"] - requests = client.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = client.describe_spot_instance_requests(SpotInstanceRequestIds=[spot_id])[ + "SpotInstanceRequests" + ] requests.should.have.length_of(1) request = requests[0] request.should.have.key("CreateTime") @@ -202,7 +214,9 @@ def test_cancel_spot_instance_request_boto3(): SpotInstanceRequestIds=[request["SpotInstanceRequestId"]] ) - requests = client.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = client.describe_spot_instance_requests(SpotInstanceRequestIds=[spot_id])[ + "SpotInstanceRequests" + ] requests.should.have.length_of(0) @@ -245,7 +259,9 @@ def test_request_spot_instances_fulfilled_boto3(): ) request_id = request["SpotInstanceRequests"][0]["SpotInstanceRequestId"] - requests = client.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = client.describe_spot_instance_requests( + SpotInstanceRequestIds=[request_id] + )["SpotInstanceRequests"] requests.should.have.length_of(1) request = requests[0] @@ -254,7 +270,9 @@ def test_request_spot_instances_fulfilled_boto3(): if not settings.TEST_SERVER_MODE: ec2_backends["us-east-1"].spot_instance_requests[request_id].state = "active" - requests = client.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = client.describe_spot_instance_requests( + SpotInstanceRequestIds=[request_id] + )["SpotInstanceRequests"] requests.should.have.length_of(1) request = requests[0] @@ -297,7 +315,9 @@ def test_tag_spot_instance_request_boto3(): Tags=[{"Key": "tag1", "Value": "value1"}, {"Key": "tag2", "Value": "value2"}], ) - requests = client.describe_spot_instance_requests()["SpotInstanceRequests"] + requests = client.describe_spot_instance_requests( + SpotInstanceRequestIds=[request_id] + )["SpotInstanceRequests"] requests.should.have.length_of(1) request = requests[0] @@ -355,33 +375,38 @@ def test_get_all_spot_instance_requests_filtering_boto3(): client.request_spot_instances( SpotPrice="0.5", LaunchSpecification={"ImageId": EXAMPLE_AMI_ID} ) + tag_value1 = str(uuid4()) client.create_tags( Resources=[request1_id], - Tags=[{"Key": "tag1", "Value": "value1"}, {"Key": "tag2", "Value": "value2"}], + Tags=[{"Key": "tag1", "Value": tag_value1}, {"Key": "tag2", "Value": "value2"}], ) client.create_tags( Resources=[request2_id], - Tags=[{"Key": "tag1", "Value": "value1"}, {"Key": "tag2", "Value": "wrong"}], + Tags=[{"Key": "tag1", "Value": tag_value1}, {"Key": "tag2", "Value": "wrong"}], ) requests = client.describe_spot_instance_requests( Filters=[{"Name": "state", "Values": ["active"]}] )["SpotInstanceRequests"] - requests.should.have.length_of(0) + r_ids = [r["SpotInstanceRequestId"] for r in requests] + r_ids.shouldnt.contain(request1_id) + r_ids.shouldnt.contain(request2_id) requests = client.describe_spot_instance_requests( Filters=[{"Name": "state", "Values": ["open"]}] )["SpotInstanceRequests"] - requests.should.have.length_of(3) + r_ids = [r["SpotInstanceRequestId"] for r in requests] + r_ids.should.contain(request1_id) + r_ids.should.contain(request2_id) requests = client.describe_spot_instance_requests( - Filters=[{"Name": "tag:tag1", "Values": ["value1"]}] + Filters=[{"Name": "tag:tag1", "Values": [tag_value1]}] )["SpotInstanceRequests"] requests.should.have.length_of(2) requests = client.describe_spot_instance_requests( Filters=[ - {"Name": "tag:tag1", "Values": ["value1"]}, + {"Name": "tag:tag1", "Values": [tag_value1]}, {"Name": "tag:tag2", "Values": ["value2"]}, ] )["SpotInstanceRequests"] @@ -406,6 +431,10 @@ def test_request_spot_instances_setting_instance_id(): @mock_ec2 def test_request_spot_instances_instance_lifecycle(): + if settings.TEST_SERVER_MODE: + # Currently no easy way to check which instance was created by request_spot_instance + # And we can't just pick the first instance in ServerMode and expect it to be the right one + raise SkipTest("ServerMode is not guaranteed to be empty") client = boto3.client("ec2", region_name="us-east-1") request = client.request_spot_instances(SpotPrice="0.5") @@ -431,9 +460,10 @@ def test_launch_spot_instance_instance_lifecycle(): "InstanceMarketOptions": {"MarketType": "spot"}, } - client.run_instances(**kwargs)["Instances"][0] + instance = client.run_instances(**kwargs)["Instances"][0] + instance_id = instance["InstanceId"] - response = client.describe_instances() + response = client.describe_instances(InstanceIds=[instance_id]) instance = response["Reservations"][0]["Instances"][0] instance["InstanceLifecycle"].should.equal("spot") @@ -453,9 +483,10 @@ def test_launch_instance_instance_lifecycle(): ], } - client.run_instances(**kwargs)["Instances"][0] + instance = client.run_instances(**kwargs)["Instances"][0] + instance_id = instance["InstanceId"] - response = client.describe_instances() + response = client.describe_instances(InstanceIds=[instance_id]) instance = response["Reservations"][0]["Instances"][0] instance.get("InstanceLifecycle").should.equal(None) diff --git a/tests/test_ec2/test_subnets.py b/tests/test_ec2/test_subnets.py index 786eef428..1fbb0f892 100644 --- a/tests/test_ec2/test_subnets.py +++ b/tests/test_ec2/test_subnets.py @@ -12,6 +12,8 @@ from boto.exception import EC2ResponseError from botocore.exceptions import ClientError from moto import mock_ec2, mock_ec2_deprecated, settings from tests import EXAMPLE_AMI_ID +from uuid import uuid4 +from unittest import SkipTest # Has boto3 equivalent @@ -44,14 +46,15 @@ def test_subnets_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18") - all_subnets = client.describe_subnets()["Subnets"] - nr_of_a_zones = len(client.describe_availability_zones()["AvailabilityZones"]) - all_subnets.should.have.length_of(1 + nr_of_a_zones) + ours = client.describe_subnets(SubnetIds=[subnet.id])["Subnets"] + ours.should.have.length_of(1) client.delete_subnet(SubnetId=subnet.id) - all_subnets = client.describe_subnets()["Subnets"] - all_subnets.should.have.length_of(nr_of_a_zones) + with pytest.raises(ClientError) as ex: + client.describe_subnets(SubnetIds=[subnet.id]) + err = ex.value.response["Error"] + err["Code"].should.equal("InvalidSubnetID.NotFound") with pytest.raises(ClientError) as ex: client.delete_subnet(SubnetId=subnet.id) @@ -111,7 +114,9 @@ def test_subnet_tagging_boto3(): subnet.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - tag = client.describe_tags()["Tags"][0] + tag = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [subnet.id]}] + )["Tags"][0] tag["Key"].should.equal("a key") tag["Value"].should.equal("some value") @@ -153,6 +158,8 @@ def test_availability_zone_in_create_subnet(): @mock_ec2 def test_default_subnet(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode will have conflicting CidrBlocks") ec2 = boto3.resource("ec2", region_name="us-west-1") default_vpc = list(ec2.vpcs.all())[0] @@ -202,11 +209,13 @@ def test_modify_subnet_attribute_public_ip_on_launch(): ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") - # Get the default VPC - vpc = list(ec2.vpcs.all())[0] + random_ip = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + vpc = ec2.create_vpc(CidrBlock=f"{random_ip}/16") + + random_subnet_cidr = f"{random_ip}/20" # Same block as the VPC subnet = ec2.create_subnet( - VpcId=vpc.id, CidrBlock="172.31.48.0/20", AvailabilityZone="us-west-1a" + VpcId=vpc.id, CidrBlock=random_subnet_cidr, AvailabilityZone="us-west-1a" ) # 'map_public_ip_on_launch' is set when calling 'DescribeSubnets' action @@ -233,11 +242,13 @@ def test_modify_subnet_attribute_assign_ipv6_address_on_creation(): ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") - # Get the default VPC - vpc = list(ec2.vpcs.all())[0] + random_ip = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + vpc = ec2.create_vpc(CidrBlock=f"{random_ip}/16") + + random_subnet_cidr = f"{random_ip}/20" # Same block as the VPC subnet = ec2.create_subnet( - VpcId=vpc.id, CidrBlock="172.31.112.0/20", AvailabilityZone="us-west-1a" + VpcId=vpc.id, CidrBlock=random_subnet_cidr, AvailabilityZone="us-west-1a" ) # 'map_public_ip_on_launch' is set when calling 'DescribeSubnets' action @@ -423,7 +434,14 @@ def test_get_subnets_filtering_boto3(): nr_of_a_zones = len(client.describe_availability_zones()["AvailabilityZones"]) all_subnets = client.describe_subnets()["Subnets"] - all_subnets.should.have.length_of(3 + nr_of_a_zones) + if settings.TEST_SERVER_MODE: + # ServerMode may have other tests running that are creating subnets + all_subnet_ids = [s["SubnetId"] for s in all_subnets] + all_subnet_ids.should.contain(subnetA.id) + all_subnet_ids.should.contain(subnetB1.id) + all_subnet_ids.should.contain(subnetB2.id) + else: + all_subnets.should.have.length_of(3 + nr_of_a_zones) # Filter by VPC ID subnets_by_vpc = client.describe_subnets( @@ -438,26 +456,26 @@ def test_get_subnets_filtering_boto3(): subnets_by_cidr1 = client.describe_subnets( Filters=[{"Name": "cidr", "Values": ["10.0.0.0/24"]}] )["Subnets"] - subnets_by_cidr1.should.have.length_of(2) - set([subnet["SubnetId"] for subnet in subnets_by_cidr1]).should.equal( - set([subnetA.id, subnetB1.id]) - ) + subnets_by_cidr1 = [s["SubnetId"] for s in subnets_by_cidr1] + subnets_by_cidr1.should.contain(subnetA.id) + subnets_by_cidr1.should.contain(subnetB1.id) + subnets_by_cidr1.shouldnt.contain(subnetB2.id) subnets_by_cidr2 = client.describe_subnets( Filters=[{"Name": "cidr-block", "Values": ["10.0.0.0/24"]}] )["Subnets"] - subnets_by_cidr2.should.have.length_of(2) - set([subnet["SubnetId"] for subnet in subnets_by_cidr2]).should.equal( - set([subnetA.id, subnetB1.id]) - ) + subnets_by_cidr2 = [s["SubnetId"] for s in subnets_by_cidr2] + subnets_by_cidr2.should.contain(subnetA.id) + subnets_by_cidr2.should.contain(subnetB1.id) + subnets_by_cidr2.shouldnt.contain(subnetB2.id) subnets_by_cidr3 = client.describe_subnets( Filters=[{"Name": "cidrBlock", "Values": ["10.0.0.0/24"]}] )["Subnets"] - subnets_by_cidr3.should.have.length_of(2) - set([subnet["SubnetId"] for subnet in subnets_by_cidr3]).should.equal( - set([subnetA.id, subnetB1.id]) - ) + subnets_by_cidr3 = [s["SubnetId"] for s in subnets_by_cidr3] + subnets_by_cidr3.should.contain(subnetA.id) + subnets_by_cidr3.should.contain(subnetB1.id) + subnets_by_cidr3.shouldnt.contain(subnetB2.id) # Filter by VPC ID and CIDR subnets_by_vpc_and_cidr = client.describe_subnets( @@ -486,14 +504,14 @@ def test_get_subnets_filtering_boto3(): subnets_by_az.should.have.length_of(1) subnets_by_az[0]["SubnetId"].should.equal(subnetB1.id) - # Filter by defaultForAz - subnets_by_az = client.describe_subnets( - Filters=[{"Name": "defaultForAz", "Values": ["true"]}] - )["Subnets"] - subnets_by_az.should.have.length_of(nr_of_a_zones) - - # Unsupported filter if not settings.TEST_SERVER_MODE: + # Filter by defaultForAz + subnets_by_az = client.describe_subnets( + Filters=[{"Name": "defaultForAz", "Values": ["true"]}] + )["Subnets"] + subnets_by_az.should.have.length_of(nr_of_a_zones) + + # Unsupported filter filters = [{"Name": "not-implemented-filter", "Values": ["foobar"]}] client.describe_subnets.when.called_with(Filters=filters).should.throw( NotImplementedError @@ -709,9 +727,14 @@ def test_create_subnet_with_tags(): ec2 = boto3.resource("ec2", region_name="us-west-1") vpc = ec2.create_vpc(CidrBlock="172.31.0.0/16") + random_ip = "172.31." + ".".join( + map(str, (random.randint(10, 40) for _ in range(2))) + ) + random_cidr = f"{random_ip}/20" + subnet = ec2.create_subnet( VpcId=vpc.id, - CidrBlock="172.31.48.0/20", + CidrBlock=random_cidr, AvailabilityZoneId="use1-az6", TagSpecifications=[ {"ResourceType": "subnet", "Tags": [{"Key": "name", "Value": "some-vpc"}]} @@ -723,6 +746,10 @@ def test_create_subnet_with_tags(): @mock_ec2 def test_available_ip_addresses_in_subnet(): + if settings.TEST_SERVER_MODE: + raise SkipTest( + "ServerMode is not guaranteed to be empty - other subnets will affect the count" + ) ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") @@ -748,6 +775,10 @@ def test_available_ip_addresses_in_subnet(): @mock_ec2 def test_available_ip_addresses_in_subnet_with_enis(): + if settings.TEST_SERVER_MODE: + raise SkipTest( + "ServerMode is not guaranteed to be empty - other ENI's will affect the count" + ) ec2 = boto3.resource("ec2", region_name="us-west-1") client = boto3.client("ec2", region_name="us-west-1") @@ -848,15 +879,20 @@ def validate_subnet_details_after_creating_eni( @mock_ec2 def test_run_instances_should_attach_to_default_subnet(): # https://github.com/spulec/moto/issues/2877 - ec2 = boto3.resource("ec2", region_name="us-west-1") - client = boto3.client("ec2", region_name="us-west-1") - ec2.create_security_group(GroupName="sg01", Description="Test security group sg01") + ec2 = boto3.resource("ec2", region_name="sa-east-1") + client = boto3.client("ec2", region_name="sa-east-1") + sec_group_name = str(uuid4())[0:6] + ec2.create_security_group( + GroupName=sec_group_name, Description="Test security group sg01" + ) # run_instances instances = client.run_instances( - ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=["sg01"], + ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1, SecurityGroups=[sec_group_name], ) # Assert subnet is created appropriately - subnets = client.describe_subnets()["Subnets"] + subnets = client.describe_subnets( + Filters=[{"Name": "defaultForAz", "Values": ["true"]}] + )["Subnets"] default_subnet_id = subnets[0]["SubnetId"] if len(subnets) > 1: default_subnet_id1 = subnets[1]["SubnetId"] @@ -866,10 +902,13 @@ def test_run_instances_should_attach_to_default_subnet(): or instances["Instances"][0]["NetworkInterfaces"][0]["SubnetId"] == default_subnet_id1 ) - assert ( - subnets[0]["AvailableIpAddressCount"] == 4090 - or subnets[1]["AvailableIpAddressCount"] == 4090 - ) + + if not settings.TEST_SERVER_MODE: + # Available IP addresses will depend on other resources that might be created in parallel + assert ( + subnets[0]["AvailableIpAddressCount"] == 4090 + or subnets[1]["AvailableIpAddressCount"] == 4090 + ) @mock_ec2 diff --git a/tests/test_ec2/test_tags.py b/tests/test_ec2/test_tags.py index ab6d04248..3d0790271 100644 --- a/tests/test_ec2/test_tags.py +++ b/tests/test_ec2/test_tags.py @@ -13,6 +13,8 @@ import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 import pytest from tests import EXAMPLE_AMI_ID +from .test_instances import retrieve_all_instances +from uuid import uuid4 # Has boto3 equivalent @@ -57,13 +59,9 @@ def test_instance_create_tags(): ) instance.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - chain = itertools.chain.from_iterable - existing_instances = list( - chain([res["Instances"] for res in client.describe_instances()["Reservations"]]) - ) - existing_instances.should.have.length_of(1) - existing_instance = existing_instances[0] - existing_instance["Tags"].should.equal([{"Key": "a key", "Value": "some value"}]) + existing_instances = retrieve_all_instances(client) + ours = [i for i in existing_instances if i["InstanceId"] == instance.id][0] + ours["Tags"].should.equal([{"Key": "a key", "Value": "some value"}]) # Has boto3 equivalent @@ -104,7 +102,9 @@ def test_instance_delete_tags(): instance.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - tags = client.describe_tags()["Tags"] + tags = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [instance.id]}] + )["Tags"] tag = tags[0] tag.should.have.key("Key").equal("a key") tag.should.have.key("Value").equal("some value") @@ -119,14 +119,20 @@ def test_instance_delete_tags(): # Specifying key only instance.delete_tags(Tags=[{"Key": "a key"}]) - client.describe_tags()["Tags"].should.have.length_of(0) + client.describe_tags(Filters=[{"Name": "resource-id", "Values": [instance.id]}])[ + "Tags" + ].should.have.length_of(0) instance.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - client.describe_tags()["Tags"].should.have.length_of(1) + client.describe_tags(Filters=[{"Name": "resource-id", "Values": [instance.id]}])[ + "Tags" + ].should.have.length_of(1) # Specifying key and value instance.delete_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - client.describe_tags()["Tags"].should.have.length_of(0) + client.describe_tags(Filters=[{"Name": "resource-id", "Values": [instance.id]}])[ + "Tags" + ].should.have.length_of(0) # Has boto3 equivalent @@ -165,10 +171,13 @@ def test_get_all_tags_with_special_characters_boto3(): client = boto3.client("ec2", region_name="us-east-1") instance = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] - instance.create_tags(Tags=[{"Key": "a key", "Value": "some<> value"}]) + tag_key = str(uuid4()) + instance.create_tags(Tags=[{"Key": tag_key, "Value": "some<> value"}]) - tag = client.describe_tags()["Tags"][0] - tag.should.have.key("Key").equal("a key") + tag = client.describe_tags(Filters=[{"Name": "key", "Values": [tag_key]}])["Tags"][ + 0 + ] + tag.should.have.key("Key").equal(tag_key) tag.should.have.key("Value").equal("some<> value") @@ -220,7 +229,9 @@ def test_create_tags_boto3(): ) client.create_tags(Resources=[instance.id], Tags=tag_list) - tags = client.describe_tags()["Tags"] + tags = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [instance.id]}] + )["Tags"] tags.should.have.length_of(3) for expected_tag in tag_list: tags.should.contain( @@ -285,7 +296,9 @@ def test_tag_limit_exceeded_boto3(): ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("TagLimitExceeded") - tags = client.describe_tags()["Tags"] + tags = client.describe_tags( + Filters=[{"Name": "resource-id", "Values": [instance.id]}] + )["Tags"] tags.should.have.length_of(1) tags[0].should.have.key("Key").equal("a key") tags[0].should.have.key("Value").equal("a value") @@ -375,30 +388,30 @@ def test_get_all_tags_resource_filter_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") client = boto3.client("ec2", region_name="us-east-1") instance = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] + inst_tag_key = str(uuid4())[0:6] client.create_tags( - Resources=[instance.id], - Tags=[{"Key": "an instance key", "Value": "some value"}], + Resources=[instance.id], Tags=[{"Key": inst_tag_key, "Value": "some value"}], ) image = instance.create_image(Name="test-ami", Description="this is a test ami") image.create_tags(Tags=[{"Key": "an image key", "Value": "some value"}]) expected = { - "Key": "an instance key", + "Key": inst_tag_key, "ResourceId": instance.id, "ResourceType": "instance", "Value": "some value", } - tags = client.describe_tags( + our_tags = client.describe_tags( Filters=[{"Name": "resource-id", "Values": [instance.id]}] )["Tags"] - tags.should.equal([expected]) - tags = client.describe_tags( + our_tags.should.equal([expected]) + instances = client.describe_tags( Filters=[{"Name": "resource-type", "Values": ["instance"]}] )["Tags"] - tags.should.equal([expected]) - tags = client.describe_tags( - Filters=[{"Name": "key", "Values": ["an instance key"]}] - )["Tags"] + instances.should.contain(expected) + tags = client.describe_tags(Filters=[{"Name": "key", "Values": [inst_tag_key]}])[ + "Tags" + ] tags.should.equal([expected]) expected = { @@ -407,14 +420,14 @@ def test_get_all_tags_resource_filter_boto3(): "ResourceType": "image", "Value": "some value", } - tags = client.describe_tags( + my_image = client.describe_tags( Filters=[{"Name": "resource-id", "Values": [image.id]}] )["Tags"] - tags.should.equal([expected]) - tags = client.describe_tags( + my_image.should.equal([expected]) + all_images = client.describe_tags( Filters=[{"Name": "resource-type", "Values": ["image"]}] )["Tags"] - tags.should.equal([expected]) + all_images.should.contain(expected) tags = client.describe_tags( Filters=[{"Name": "resource-type", "Values": ["unknown"]}] @@ -538,9 +551,11 @@ def test_get_all_tags_value_filter_boto3(): image.create_tags(Tags=[{"Key": "an image key", "Value": "some value"}]) def filter_by_value(query, expected): - filter = {"Name": "value", "Values": [query]} - tags = client.describe_tags(Filters=[filter])["Tags"] - set([t["ResourceId"] for t in tags]).should.equal(set(expected)) + filters = [{"Name": "value", "Values": [query]}] + tags = retrieve_all_tagged(client, filters) + actual = set([t["ResourceId"] for t in tags]) + for e in expected: + actual.should.contain(e) filter_by_value("some value", [instance_a.id, image.id]) filter_by_value("some*value", [instance_a.id, instance_b.id, image.id]) @@ -593,17 +608,17 @@ def test_retrieved_instances_must_contain_their_tags_boto3(): client = boto3.client("ec2", region_name="us-east-1") instance = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)[0] - reservations = client.describe_instances()["Reservations"] - reservations.should.have.length_of(1) - instances = reservations[0]["Instances"] - instances.should.have.length_of(1) - instances[0]["InstanceId"].should.equal(instance.id) - instances[0].shouldnt.have.key("Tags") + all_instances = retrieve_all_instances(client) + ours = [i for i in all_instances if i["InstanceId"] == instance.id] + ours.should.have.length_of(1) + ours[0]["InstanceId"].should.equal(instance.id) + ours[0].shouldnt.have.key("Tags") client.create_tags(Resources=[instance.id], Tags=[tags_to_be_set]) - reservations = client.describe_instances()["Reservations"] - instance = reservations[0]["Instances"][0] - retrieved_tags = instance["Tags"] + + all_instances = retrieve_all_instances(client) + ours = [i for i in all_instances if i["InstanceId"] == instance.id] + retrieved_tags = ours[0]["Tags"] # Check whether tag is present with correct value retrieved_tags.should.equal([{"Key": tag_key, "Value": tag_value}]) @@ -822,30 +837,43 @@ def test_delete_tag_empty_resource(): def test_retrieve_resource_with_multiple_tags(): ec2 = boto3.resource("ec2", region_name="us-west-1") blue, green = ec2.create_instances(ImageId=EXAMPLE_AMI_ID, MinCount=2, MaxCount=2) + tag_val1 = str(uuid4()) ec2.create_tags( Resources=[blue.instance_id], Tags=[ - {"Key": "environment", "Value": "blue"}, + {"Key": "environment", "Value": tag_val1}, {"Key": "application", "Value": "api"}, ], ) + tag_val2 = str(uuid4()) ec2.create_tags( Resources=[green.instance_id], Tags=[ - {"Key": "environment", "Value": "green"}, + {"Key": "environment", "Value": tag_val2}, {"Key": "application", "Value": "api"}, ], ) - green_instances = list(ec2.instances.filter(Filters=(get_filter("green")))) + green_instances = list(ec2.instances.filter(Filters=(get_filter(tag_val2)))) green_instances.should.equal([green]) - blue_instances = list(ec2.instances.filter(Filters=(get_filter("blue")))) + blue_instances = list(ec2.instances.filter(Filters=(get_filter(tag_val1)))) blue_instances.should.equal([blue]) -def get_filter(color): +def get_filter(tag_val): return [ {"Name": "tag-key", "Values": ["application"]}, {"Name": "tag-value", "Values": ["api"]}, {"Name": "tag-key", "Values": ["environment"]}, - {"Name": "tag-value", "Values": [color]}, + {"Name": "tag-value", "Values": [tag_val]}, ] + + +def retrieve_all_tagged(client, filters=[]): + resp = client.describe_tags(Filters=filters) + tags = resp["Tags"] + token = resp.get("NextToken") + while token: + resp = client.describe_tags(Filters=filters, NextToken=token) + tags.extend(resp["Tags"]) + token = resp.get("Token") + return tags diff --git a/tests/test_ec2/test_transit_gateway.py b/tests/test_ec2/test_transit_gateway.py index cbda02e3d..622f11433 100644 --- a/tests/test_ec2/test_transit_gateway.py +++ b/tests/test_ec2/test_transit_gateway.py @@ -1,11 +1,14 @@ import boto3 import sure # noqa -from moto import mock_ec2 +from moto import mock_ec2, settings from moto.core import ACCOUNT_ID +from unittest import SkipTest @mock_ec2 def test_describe_transit_gateways(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.client("ec2", region_name="us-west-1") response = ec2.describe_transit_gateways() response.should.have.key("TransitGateways").equal([]) @@ -34,8 +37,14 @@ def test_create_transit_gateway(): options.should.have.key("DnsSupport").equal("disable") # # Verify we can retrieve it - response = ec2.describe_transit_gateways() - gateways = response["TransitGateways"] + all_gateways = retrieve_all_transit_gateways(ec2) + [gw["TransitGatewayId"] for gw in all_gateways].should.contain( + gateway["TransitGatewayId"] + ) + gateways = ec2.describe_transit_gateways( + TransitGatewayIds=[gateway["TransitGatewayId"]] + )["TransitGateways"] + gateways.should.have.length_of(1) gateways[0].should.have.key("CreationTime") gateways[0].should.have.key("TransitGatewayArn").equal( @@ -79,32 +88,57 @@ def test_create_transit_gateway_with_tags(): def test_delete_transit_gateway(): ec2 = boto3.client("ec2", region_name="us-west-1") g = ec2.create_transit_gateway(Description="my first gateway")["TransitGateway"] - ec2.describe_transit_gateways()["TransitGateways"].should.have.length_of(1) + g_id = g["TransitGatewayId"] - ec2.delete_transit_gateway(TransitGatewayId=g["TransitGatewayId"]) - ec2.describe_transit_gateways()["TransitGateways"].should.have.length_of(0) + all_gateways = retrieve_all_transit_gateways(ec2) + [g["TransitGatewayId"] for g in all_gateways].should.contain(g_id) + + ec2.delete_transit_gateway(TransitGatewayId=g_id) + + all_gateways = retrieve_all_transit_gateways(ec2) + [g["TransitGatewayId"] for g in all_gateways].shouldnt.contain(g_id) + + +@mock_ec2 +def test_describe_transit_gateway_by_id(): + ec2 = boto3.client("ec2", region_name="us-west-1") + g1 = ec2.create_transit_gateway(Description="my first gatway")["TransitGateway"] + g2 = ec2.create_transit_gateway(Description="my second gatway")["TransitGateway"] + g2_id = g2["TransitGatewayId"] + g3 = ec2.create_transit_gateway(Description="my third gatway")["TransitGateway"] + + my_gateway = ec2.describe_transit_gateways(TransitGatewayIds=[g2_id])[ + "TransitGateways" + ][0] + my_gateway["TransitGatewayId"].should.equal(g2_id) + my_gateway["Description"].should.equal("my second gatway") @mock_ec2 def test_modify_transit_gateway(): ec2 = boto3.client("ec2", region_name="us-west-1") g = ec2.create_transit_gateway(Description="my first gatway")["TransitGateway"] - ec2.describe_transit_gateways()["TransitGateways"].should.have.length_of(1) - ec2.describe_transit_gateways()["TransitGateways"][0]["Description"].should.equal( - "my first gatway" + g_id = g["TransitGatewayId"] + + my_gateway = ec2.describe_transit_gateways(TransitGatewayIds=[g_id])[ + "TransitGateways" + ][0] + my_gateway["Description"].should.equal("my first gatway") + + resp = ec2.modify_transit_gateway( + TransitGatewayId=g_id, Description="my first gateway" ) - ec2.modify_transit_gateway( - TransitGatewayId=g["TransitGatewayId"], Description="my first gateway" - ) - ec2.describe_transit_gateways()["TransitGateways"].should.have.length_of(1) - ec2.describe_transit_gateways()["TransitGateways"][0]["Description"].should.equal( - "my first gateway" - ) + my_gateway = ec2.describe_transit_gateways(TransitGatewayIds=[g_id])[ + "TransitGateways" + ][0] + my_gateway["Description"].should.equal("my first gateway") @mock_ec2 def test_describe_transit_gateway_vpc_attachments(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.client("ec2", region_name="us-west-1") response = ec2.describe_transit_gateway_vpc_attachments() response.should.have.key("TransitGatewayVpcAttachments").equal([]) @@ -112,11 +146,24 @@ def test_describe_transit_gateway_vpc_attachments(): @mock_ec2 def test_describe_transit_gateway_attachments(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.client("ec2", region_name="us-west-1") response = ec2.describe_transit_gateway_attachments() response.should.have.key("TransitGatewayAttachments").equal([]) +def retrieve_all_transit_gateways(ec2): + resp = ec2.describe_transit_gateways() + all_tg = resp["TransitGateways"] + token = resp.get("NextToken") + while token: + resp = ec2.describe_transit_gateways(NextToken=token) + all_tg.extend(resp["TransitGateways"]) + token = resp.get("NextToken") + return all_tg + + @mock_ec2 def test_create_transit_gateway_vpn_attachment(): ec2 = boto3.client("ec2", region_name="us-west-1") @@ -130,19 +177,29 @@ def test_create_transit_gateway_vpn_attachment(): VpnGatewayId=vpn_gateway["VpnGatewayId"], CustomerGatewayId=customer_gateway["CustomerGatewayId"], TransitGatewayId="gateway_id", - ).get("VpnConnection", {}) + )["VpnConnection"] + vpn_conn_id = vpn_connection["VpnConnectionId"] # # Verify we can retrieve it as a general attachment - attachments = ec2.describe_transit_gateway_attachments()[ - "TransitGatewayAttachments" - ] - attachments.should.have.length_of(1) + attachments = retrieve_all_attachments(ec2) + [a["ResourceId"] for a in attachments].should.contain(vpn_conn_id) - attachments[0].should.have.key("ResourceType").equal("vpn") - attachments[0].should.have.key("ResourceId").equal( - vpn_connection["VpnConnectionId"] - ) + my_attachments = [a for a in attachments if a["ResourceId"] == vpn_conn_id] + my_attachments.should.have.length_of(1) + + my_attachments[0].should.have.key("ResourceType").equal("vpn") + + +def retrieve_all_attachments(client): + resp = client.describe_transit_gateway_attachments() + att = resp["TransitGatewayAttachments"] + token = resp.get("NextToken") + while token: + resp = client.describe_transit_gateway_attachments(NextToken=token) + att.extend(resp["TransitGatewayAttachments"]) + token = resp.get("NextToken") + return att @mock_ec2 @@ -168,18 +225,18 @@ def test_create_and_describe_transit_gateway_vpc_attachment(): attachment.should.have.key("Tags").equal([]) # # Verify we can retrieve it as a VPC attachment - attachments = ec2.describe_transit_gateway_vpc_attachments()[ - "TransitGatewayVpcAttachments" - ] + attachments = ec2.describe_transit_gateway_vpc_attachments( + TransitGatewayAttachmentIds=[attachment["TransitGatewayAttachmentId"]] + )["TransitGatewayVpcAttachments"] attachments.should.have.length_of(1) attachments[0].should.have.key("CreationTime") del attachments[0]["CreationTime"] attachment.should.equal(attachments[0]) # # Verify we can retrieve it as a general attachment - attachments = ec2.describe_transit_gateway_attachments()[ - "TransitGatewayAttachments" - ] + attachments = ec2.describe_transit_gateway_attachments( + TransitGatewayAttachmentIds=[attachment["TransitGatewayAttachmentId"]] + )["TransitGatewayAttachments"] attachments.should.have.length_of(1) attachments[0].should.have.key("CreationTime") attachments[0].should.have.key("TransitGatewayOwnerId").equal(ACCOUNT_ID) @@ -196,6 +253,8 @@ def test_create_and_describe_transit_gateway_vpc_attachment(): @mock_ec2 def test_describe_transit_gateway_route_tables(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.client("ec2", region_name="us-west-1") response = ec2.describe_transit_gateway_route_tables() response.should.have.key("TransitGatewayRouteTables").equal([]) @@ -204,8 +263,6 @@ def test_describe_transit_gateway_route_tables(): @mock_ec2 def test_create_transit_gateway_route_table(): ec2 = boto3.client("ec2", region_name="us-west-1") - tables = ec2.describe_transit_gateway_route_tables()["TransitGatewayRouteTables"] - tables.should.equal([]) gateway_id = ec2.create_transit_gateway(Description="g")["TransitGateway"][ "TransitGatewayId" @@ -221,8 +278,10 @@ def test_create_transit_gateway_route_table(): table.should.have.key("CreationTime") table.should.have.key("Tags").equals([]) - tables = ec2.describe_transit_gateway_route_tables()["TransitGatewayRouteTables"] - tables.should.have.length_of(2) + tables = ec2.describe_transit_gateway_route_tables( + TransitGatewayRouteTableIds=[table["TransitGatewayRouteTableId"]] + )["TransitGatewayRouteTables"] + tables.should.have.length_of(1) @mock_ec2 @@ -260,18 +319,23 @@ def test_delete_transit_gateway_route_table(): table = ec2.create_transit_gateway_route_table(TransitGatewayId=gateway_id)[ "TransitGatewayRouteTable" ] + table_id = table["TransitGatewayRouteTableId"] - tables = ec2.describe_transit_gateway_route_tables()["TransitGatewayRouteTables"] - tables.should.have.length_of(2) + tables = ec2.describe_transit_gateway_route_tables( + TransitGatewayRouteTableIds=[table_id] + )["TransitGatewayRouteTables"] + tables.should.have.length_of(1) + tables[0]["State"].should.equal("available") - table = ec2.delete_transit_gateway_route_table( - TransitGatewayRouteTableId=table["TransitGatewayRouteTableId"] - ) + table = ec2.delete_transit_gateway_route_table(TransitGatewayRouteTableId=table_id) table["TransitGatewayRouteTable"].should.have.key("State").equals("deleted") - tables = ec2.describe_transit_gateway_route_tables()["TransitGatewayRouteTables"] - tables.should.have.length_of(2) + tables = ec2.describe_transit_gateway_route_tables( + TransitGatewayRouteTableIds=[table_id] + )["TransitGatewayRouteTables"] + tables.should.have.length_of(1) + tables[0]["State"].should.equal("deleted") @mock_ec2 @@ -528,7 +592,8 @@ def test_delete_transit_gateway_vpc_attachment(): )["TransitGatewayVpcAttachment"]["TransitGatewayAttachmentId"] available = ec2.describe_transit_gateway_vpc_attachments( - Filters=[{"Name": "state", "Values": ["available"]}] + TransitGatewayAttachmentIds=[a1, a2], + Filters=[{"Name": "state", "Values": ["available"]}], )["TransitGatewayVpcAttachments"] available.should.have.length_of(2) @@ -538,9 +603,9 @@ def test_delete_transit_gateway_vpc_attachment(): a1_removed.should.have.key("TransitGatewayAttachmentId").equal(a1) a1_removed.should.have.key("State").equal("deleted") - all_attchmnts = ec2.describe_transit_gateway_vpc_attachments()[ - "TransitGatewayVpcAttachments" - ] + all_attchmnts = ec2.describe_transit_gateway_vpc_attachments( + TransitGatewayAttachmentIds=[a1, a2] + )["TransitGatewayVpcAttachments"] all_attchmnts.should.have.length_of(1) [a["TransitGatewayAttachmentId"] for a in all_attchmnts].should.equal([a2]) diff --git a/tests/test_ec2/test_transit_gateway_cloudformation.py b/tests/test_ec2/test_transit_gateway_cloudformation.py index 4899a361e..528db9d4f 100644 --- a/tests/test_ec2/test_transit_gateway_cloudformation.py +++ b/tests/test_ec2/test_transit_gateway_cloudformation.py @@ -9,6 +9,7 @@ from moto import ( mock_cloudformation, mock_ec2, ) +from uuid import uuid4 @mock_cloudformation @@ -17,8 +18,6 @@ def test_transit_gateway_by_cloudformation_simple(): ec2 = boto3.client("ec2", region_name="us-east-1") cf_client = boto3.client("cloudformation", "us-east-1") - ec2.describe_transit_gateways()["TransitGateways"].should.have.length_of(0) - template = { "AWSTemplateFormatVersion": "2010-09-09", "Description": "Template for Transit Gateway creation.", @@ -30,9 +29,17 @@ def test_transit_gateway_by_cloudformation_simple(): }, } template = json.dumps(template) - cf_client.create_stack(StackName="test_stack", TemplateBody=template) + stack_name = str(uuid4()) + cf_client.create_stack(StackName=stack_name, TemplateBody=template) - gateways = ec2.describe_transit_gateways()["TransitGateways"] + resources = cf_client.list_stack_resources(StackName=stack_name)[ + "StackResourceSummaries" + ] + gateway_id = resources[0]["PhysicalResourceId"] + + gateways = ec2.describe_transit_gateways(TransitGatewayIds=[gateway_id])[ + "TransitGateways" + ] gateways.should.have.length_of(1) gateways[0]["TransitGatewayId"].should.match("tgw-[0-9a-z]+") gateways[0]["State"].should.equal("available") @@ -50,8 +57,6 @@ def test_transit_gateway_by_cloudformation(): ec2 = boto3.client("ec2", region_name="us-east-1") cf_client = boto3.client("cloudformation", "us-east-1") - ec2.describe_transit_gateways()["TransitGateways"].should.have.length_of(0) - template = { "AWSTemplateFormatVersion": "2010-09-09", "Description": "Template for Transit Gateway creation.", @@ -69,9 +74,17 @@ def test_transit_gateway_by_cloudformation(): }, } template = json.dumps(template) - cf_client.create_stack(StackName="test_stack", TemplateBody=template) + stack_name = str(uuid4()) + cf_client.create_stack(StackName=stack_name, TemplateBody=template) - gateways = ec2.describe_transit_gateways()["TransitGateways"] + resources = cf_client.list_stack_resources(StackName=stack_name)[ + "StackResourceSummaries" + ] + gateway_id = resources[0]["PhysicalResourceId"] + + gateways = ec2.describe_transit_gateways(TransitGatewayIds=[gateway_id])[ + "TransitGateways" + ] gateways.should.have.length_of(1) gateways[0]["TransitGatewayId"].should.match("tgw-[0-9a-z]+") gateways[0]["State"].should.equal("available") @@ -82,5 +95,5 @@ def test_transit_gateway_by_cloudformation(): tags = gateways[0].get("Tags", {}) tags.should.have.length_of(4) tags.should.contain({"Key": "foo", "Value": "bar"}) - tags.should.contain({"Key": "aws:cloudformation:stack-name", "Value": "test_stack"}) + tags.should.contain({"Key": "aws:cloudformation:stack-name", "Value": stack_name}) tags.should.contain({"Key": "aws:cloudformation:logical-id", "Value": "ttg"}) diff --git a/tests/test_ec2/test_transit_gateway_peering_attachments.py b/tests/test_ec2/test_transit_gateway_peering_attachments.py index ed404ca0e..cbdc9223e 100644 --- a/tests/test_ec2/test_transit_gateway_peering_attachments.py +++ b/tests/test_ec2/test_transit_gateway_peering_attachments.py @@ -1,11 +1,14 @@ import boto3 import sure # noqa -from moto import mock_ec2 +from moto import mock_ec2, settings from moto.core import ACCOUNT_ID +from unittest import SkipTest @mock_ec2 def test_describe_transit_gateway_peering_attachment_empty(): + if settings.TEST_SERVER_MODE: + raise SkipTest("ServerMode is not guaranteed to be empty") ec2 = boto3.client("ec2", region_name="us-west-1") all_attachments = ec2.describe_transit_gateway_peering_attachments()[ @@ -40,7 +43,12 @@ def test_create_and_describe_transit_gateway_peering_attachment(): all_attachments = ec2.describe_transit_gateway_peering_attachments()[ "TransitGatewayPeeringAttachments" ] - all_attachments.should.equal([attachment]) + our_attachment = [ + att + for att in all_attachments + if att["TransitGatewayAttachmentId"] == attachment["TransitGatewayAttachmentId"] + ] + our_attachment.should.equal([attachment]) @mock_ec2 @@ -62,7 +70,12 @@ def test_describe_transit_gateway_peering_attachment_by_filters(): all_attachments = ec2.describe_transit_gateway_peering_attachments()[ "TransitGatewayPeeringAttachments" ] - all_attachments.should.have.length_of(3) + ours = [ + a + for a in all_attachments + if a["TransitGatewayAttachmentId"] in [attchmnt1, attchmnt2, attchmnt3] + ] + ours.should.have.length_of(3) find_1 = ec2.describe_transit_gateway_peering_attachments( TransitGatewayAttachmentIds=[attchmnt1] @@ -81,12 +94,12 @@ def test_describe_transit_gateway_peering_attachment_by_filters(): )["TransitGatewayPeeringAttachments"] [a["TransitGatewayAttachmentId"] for a in find_3].should.equal([attchmnt3]) - find_all = ec2.describe_transit_gateway_peering_attachments( - Filters=[{"Name": "state", "Values": ["available"]}] - )["TransitGatewayPeeringAttachments"] - [a["TransitGatewayAttachmentId"] for a in find_all].should.equal( - [attchmnt1, attchmnt2, attchmnt3] - ) + filters = [{"Name": "state", "Values": ["available"]}] + find_all = retrieve_all_attachments(ec2, filters) + all_ids = [a["TransitGatewayAttachmentId"] for a in find_all] + all_ids.should.contain(attchmnt1) + all_ids.should.contain(attchmnt2) + all_ids.should.contain(attchmnt3) find_none = ec2.describe_transit_gateway_peering_attachments( Filters=[{"Name": "state", "Values": ["unknown"]}] @@ -117,9 +130,9 @@ def test_create_and_accept_transit_gateway_peering_attachment(): TransitGatewayAttachmentId=attchment_id ) - attachment = ec2.describe_transit_gateway_peering_attachments()[ - "TransitGatewayPeeringAttachments" - ][0] + attachment = ec2.describe_transit_gateway_peering_attachments( + TransitGatewayAttachmentIds=[attchment_id] + )["TransitGatewayPeeringAttachments"][0] attachment.should.have.key("TransitGatewayAttachmentId").equal(attchment_id) attachment.should.have.key("State").equal("available") @@ -139,9 +152,9 @@ def test_create_and_reject_transit_gateway_peering_attachment(): TransitGatewayAttachmentId=attchment_id ) - attachment = ec2.describe_transit_gateway_peering_attachments()[ - "TransitGatewayPeeringAttachments" - ][0] + attachment = ec2.describe_transit_gateway_peering_attachments( + TransitGatewayAttachmentIds=[attchment_id] + )["TransitGatewayPeeringAttachments"][0] attachment.should.have.key("TransitGatewayAttachmentId").equal(attchment_id) attachment.should.have.key("State").equal("rejected") @@ -161,9 +174,9 @@ def test_create_and_delete_transit_gateway_peering_attachment(): TransitGatewayAttachmentId=attchment_id ) - attachment = ec2.describe_transit_gateway_peering_attachments()[ - "TransitGatewayPeeringAttachments" - ][0] + attachment = ec2.describe_transit_gateway_peering_attachments( + TransitGatewayAttachmentIds=[attchment_id] + )["TransitGatewayPeeringAttachments"][0] attachment.should.have.key("TransitGatewayAttachmentId").equal(attchment_id) attachment.should.have.key("State").equal("deleted") @@ -175,3 +188,16 @@ def create_peering_attachment(ec2, gateway_id1, gateway_id2): PeerAccountId=ACCOUNT_ID, PeerRegion="us-east-1", )["TransitGatewayPeeringAttachment"]["TransitGatewayAttachmentId"] + + +def retrieve_all_attachments(client, filters=[]): + resp = client.describe_transit_gateway_peering_attachments(Filters=filters) + attmnts = resp["TransitGatewayPeeringAttachments"] + token = resp.get("NextToken") + while token: + resp = client.describe_transit_gateway_peering_attachments( + Filters=filters, NextToken=token + ) + attmnts.extend(resp["TransitGatewayPeeringAttachments"]) + token = resp.get("NextToken") + return attmnts diff --git a/tests/test_ec2/test_virtual_private_gateways.py b/tests/test_ec2/test_virtual_private_gateways.py index 2f12afc81..2d45d6338 100644 --- a/tests/test_ec2/test_virtual_private_gateways.py +++ b/tests/test_ec2/test_virtual_private_gateways.py @@ -6,6 +6,7 @@ import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 from botocore.exceptions import ClientError +from .test_tags import retrieve_all_tagged @mock_ec2 @@ -146,7 +147,9 @@ def test_describe_vpn_gateway_boto3(): Type="ipsec.1", AvailabilityZone="us-east-1a" )["VpnGateway"] - vgws = client.describe_vpn_gateways()["VpnGateways"] + vgws = client.describe_vpn_gateways(VpnGatewayIds=[vpn_gateway["VpnGatewayId"]])[ + "VpnGateways" + ] vgws.should.have.length_of(1) gateway = vgws[0] @@ -171,15 +174,15 @@ def test_describe_vpn_connections_state_filter_attached(): ec2.attach_vpn_gateway(VpcId=vpc_id, VpnGatewayId=gateway_id) - gateways = ec2.describe_vpn_gateways( - Filters=[{"Name": "attachment.state", "Values": ["attached"]}] + all_gateways = retrieve_all( + ec2, [{"Name": "attachment.state", "Values": ["attached"]}] ) - gateways["VpnGateways"].should.have.length_of(1) - gateways["VpnGateways"][0]["VpnGatewayId"].should.equal(gateway_id) - gateways["VpnGateways"][0]["VpcAttachments"].should.contain( - {"State": "attached", "VpcId": vpc_id} - ) + [gw["VpnGatewayId"] for gw in all_gateways].should.contain(gateway_id) + + my_gateway = [gw for gw in all_gateways if gw["VpnGatewayId"] == gateway_id][0] + + my_gateway["VpcAttachments"].should.contain({"State": "attached", "VpcId": vpc_id}) @mock_ec2 @@ -223,12 +226,9 @@ def test_describe_vpn_connections_type_filter_match(): gateway = ec2.create_vpn_gateway(AvailabilityZone="us-east-1a", Type="ipsec.1") gateway_id = gateway["VpnGateway"]["VpnGatewayId"] - gateways = ec2.describe_vpn_gateways( - Filters=[{"Name": "type", "Values": ["ipsec.1"]}] - ) + my_gateways = retrieve_all(ec2, [{"Name": "type", "Values": ["ipsec.1"]}]) - gateways["VpnGateways"].should.have.length_of(1) - gateways["VpnGateways"][0]["VpnGatewayId"].should.equal(gateway_id) + [gw["VpnGatewayId"] for gw in my_gateways].should.contain(gateway_id) @mock_ec2 @@ -270,10 +270,11 @@ def test_vpn_gateway_vpc_attachment_boto3(): vpn_gateway = client.create_vpn_gateway( Type="ipsec.1", AvailabilityZone="us-east-1a" )["VpnGateway"] + vpng_id = vpn_gateway["VpnGatewayId"] - client.attach_vpn_gateway(VpnGatewayId=vpn_gateway["VpnGatewayId"], VpcId=vpc.id) + client.attach_vpn_gateway(VpnGatewayId=vpng_id, VpcId=vpc.id) - gateway = client.describe_vpn_gateways()["VpnGateways"][0] + gateway = client.describe_vpn_gateways(VpnGatewayIds=[vpng_id])["VpnGateways"][0] attachments = gateway["VpcAttachments"] attachments.should.equal([{"State": "attached", "VpcId": vpc.id}]) @@ -297,9 +298,10 @@ def test_delete_vpn_gateway_boto3(): vpn_gateway = client.create_vpn_gateway( Type="ipsec.1", AvailabilityZone="us-east-1a" )["VpnGateway"] + vpng_id = vpn_gateway["VpnGatewayId"] - client.delete_vpn_gateway(VpnGatewayId=vpn_gateway["VpnGatewayId"]) - gateways = client.describe_vpn_gateways()["VpnGateways"] + client.delete_vpn_gateway(VpnGatewayId=vpng_id) + gateways = client.describe_vpn_gateways(VpnGatewayIds=[vpng_id])["VpnGateways"] gateways.should.have.length_of(1) gateways[0].should.have.key("State").equal("deleted") @@ -333,9 +335,10 @@ def test_vpn_gateway_tagging_boto3(): Tags=[{"Key": "a key", "Value": "some value"}], ) - tag = client.describe_tags()["Tags"][0] - tag.should.have.key("Key").equal("a key") - tag.should.have.key("Value").equal("some value") + all_tags = retrieve_all_tagged(client) + ours = [a for a in all_tags if a["ResourceId"] == vpn_gateway["VpnGatewayId"]][0] + ours.should.have.key("Key").equal("a key") + ours.should.have.key("Value").equal("some value") vpn_gateway = client.describe_vpn_gateways()["VpnGateways"][0] # TODO: Fixme: Tags is currently empty @@ -376,15 +379,27 @@ def test_detach_vpn_gateway_boto3(): Type="ipsec.1", AvailabilityZone="us-east-1a" ) vpn_gateway = vpn_gateway["VpnGateway"] + vpng_id = vpn_gateway["VpnGatewayId"] - client.attach_vpn_gateway(VpnGatewayId=vpn_gateway["VpnGatewayId"], VpcId=vpc.id) + client.attach_vpn_gateway(VpnGatewayId=vpng_id, VpcId=vpc.id) - gateway = client.describe_vpn_gateways()["VpnGateways"][0] + gateway = client.describe_vpn_gateways(VpnGatewayIds=[vpng_id])["VpnGateways"][0] attachments = gateway["VpcAttachments"] attachments.should.equal([{"State": "attached", "VpcId": vpc.id}]) - client.detach_vpn_gateway(VpnGatewayId=vpn_gateway["VpnGatewayId"], VpcId=vpc.id) + client.detach_vpn_gateway(VpnGatewayId=vpng_id, VpcId=vpc.id) - gateway = client.describe_vpn_gateways()["VpnGateways"][0] + gateway = client.describe_vpn_gateways(VpnGatewayIds=[vpng_id])["VpnGateways"][0] attachments = gateway["VpcAttachments"] attachments.should.equal([{"State": "detached", "VpcId": vpc.id}]) + + +def retrieve_all(client, filters=[]): + resp = client.describe_vpn_gateways(Filters=filters) + all_gateways = resp["VpnGateways"] + token = resp.get("NextToken") + while token: + resp = client.describe_vpn_gateways(Filters=filters) + all_gateways.extend(resp["VpnGateways"]) + token = resp.get("NextToken") + return all_gateways diff --git a/tests/test_ec2/test_vpc_endpoint_services.py b/tests/test_ec2/test_vpc_endpoint_services.py index bd0422278..b8980ceb3 100644 --- a/tests/test_ec2/test_vpc_endpoint_services.py +++ b/tests/test_ec2/test_vpc_endpoint_services.py @@ -4,11 +4,16 @@ import pytest import boto3 from botocore.exceptions import ClientError -from moto import mock_ec2 +from moto import mock_ec2, settings +from unittest import SkipTest @mock_ec2 def test_describe_vpc_endpoint_services_bad_args(): + if settings.TEST_SERVER_MODE: + # Long-running operation - doesn't quite work in ServerMode, with parallel tests + # Probably needs some locking to force the initialization to only occur once + raise SkipTest("Can't run in ServerMode") """Verify exceptions are raised for bad arguments.""" ec2 = boto3.client("ec2", region_name="us-west-1") diff --git a/tests/test_ec2/test_vpc_peering.py b/tests/test_ec2/test_vpc_peering.py index 522d4118c..35ad07bf1 100644 --- a/tests/test_ec2/test_vpc_peering.py +++ b/tests/test_ec2/test_vpc_peering.py @@ -64,10 +64,18 @@ def test_vpc_peering_connections_get_all_boto3(): ec2 = boto3.resource("ec2", region_name="us-east-1") client = boto3.client("ec2", region_name="us-east-1") vpc_pcx = create_vpx_pcx(ec2, client) + vpc_pcx_id = vpc_pcx["VpcPeeringConnectionId"] - all_vpc_pcxs = client.describe_vpc_peering_connections()["VpcPeeringConnections"] - all_vpc_pcxs.should.have.length_of(1) - all_vpc_pcxs[0]["Status"]["Code"].should.equal("pending-acceptance") + all_vpc_pcxs = retrieve_all(client) + [vpc_pcx["VpcPeeringConnectionId"] for vpc_pcx in all_vpc_pcxs].should.contain( + vpc_pcx_id + ) + my_vpc_pcx = [ + vpc_pcx + for vpc_pcx in all_vpc_pcxs + if vpc_pcx["VpcPeeringConnectionId"] == vpc_pcx_id + ][0] + my_vpc_pcx["Status"]["Code"].should.equal("pending-acceptance") # Has boto3 equivalent @@ -108,9 +116,11 @@ def test_vpc_peering_connections_accept_boto3(): ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidStateTransition") - all_vpc_pcxs = client.describe_vpc_peering_connections()["VpcPeeringConnections"] - all_vpc_pcxs.should.have.length_of(1) - all_vpc_pcxs[0]["Status"]["Code"].should.equal("active") + my_vpc_pcxs = client.describe_vpc_peering_connections( + VpcPeeringConnectionIds=[vpc_pcx_id] + )["VpcPeeringConnections"] + my_vpc_pcxs.should.have.length_of(1) + my_vpc_pcxs[0]["Status"]["Code"].should.equal("active") # Has boto3 equivalent @@ -149,9 +159,11 @@ def test_vpc_peering_connections_reject_boto3(): ex.value.response["ResponseMetadata"].should.have.key("RequestId") ex.value.response["Error"]["Code"].should.equal("InvalidStateTransition") - all_vpc_pcxs = client.describe_vpc_peering_connections()["VpcPeeringConnections"] - all_vpc_pcxs.should.have.length_of(1) - all_vpc_pcxs[0]["Status"]["Code"].should.equal("rejected") + my_pcxs = client.describe_vpc_peering_connections( + VpcPeeringConnectionIds=[vpc_pcx_id] + )["VpcPeeringConnections"] + my_pcxs.should.have.length_of(1) + my_pcxs[0]["Status"]["Code"].should.equal("rejected") # Has boto3 equivalent @@ -184,9 +196,13 @@ def test_vpc_peering_connections_delete_boto3(): client.delete_vpc_peering_connection(VpcPeeringConnectionId=vpc_pcx_id) - all_vpc_pcxs = client.describe_vpc_peering_connections()["VpcPeeringConnections"] - all_vpc_pcxs.should.have.length_of(1) - all_vpc_pcxs[0]["Status"]["Code"].should.equal("deleted") + all_vpc_pcxs = retrieve_all(client) + [vpcx["VpcPeeringConnectionId"] for vpcx in all_vpc_pcxs].should.contain(vpc_pcx_id) + + my_vpcx = [ + vpcx for vpcx in all_vpc_pcxs if vpcx["VpcPeeringConnectionId"] == vpc_pcx_id + ][0] + my_vpcx["Status"]["Code"].should.equal("deleted") with pytest.raises(ClientError) as ex: client.delete_vpc_peering_connection(VpcPeeringConnectionId="pcx-1234abcd") @@ -342,8 +358,10 @@ def test_describe_vpc_peering_connections_only_returns_requested_id(): ) # describe peering ec2_usw1 = boto3.client("ec2", region_name="us-west-1") - all_pcx = ec2_usw1.describe_vpc_peering_connections()["VpcPeeringConnections"] - all_pcx.should.have.length_of(2) + our_vpcx = [vpcx["VpcPeeringConnectionId"] for vpcx in retrieve_all(ec2_usw1)] + our_vpcx.should.contain(vpc_pcx_usw1.id) + our_vpcx.should.contain(vpc_pcx_usw2.id) + our_vpcx.shouldnt.contain(vpc_apn1.id) both_pcx = ec2_usw1.describe_vpc_peering_connections( VpcPeeringConnectionIds=[vpc_pcx_usw1.id, vpc_pcx_usw2.id] @@ -507,3 +525,14 @@ def test_vpc_peering_connections_cross_region_reject_wrong_region(): "rejected in region {2}".format("us-west-1", vpc_pcx_usw1.id, "ap-northeast-1") ) cm.value.response["Error"]["Message"].should.equal(exp_msg) + + +def retrieve_all(client): + resp = client.describe_vpc_peering_connections() + all_vpx = resp["VpcPeeringConnections"] + token = resp.get("NextToken") + while token: + resp = client.describe_vpc_peering_connections(NextToken=token) + all_vpx.extend(resp["VpcPeeringConnections"]) + token = resp.get("NextToken") + return all_vpx diff --git a/tests/test_ec2/test_vpcs.py b/tests/test_ec2/test_vpcs.py index 324786a4d..d888e372d 100644 --- a/tests/test_ec2/test_vpcs.py +++ b/tests/test_ec2/test_vpcs.py @@ -9,8 +9,11 @@ import boto from boto.exception import EC2ResponseError import sure # noqa +import random from moto import mock_ec2, mock_ec2_deprecated +from uuid import uuid4 +from .test_tags import retrieve_all_tagged SAMPLE_DOMAIN_NAME = "example.com" SAMPLE_NAME_SERVERS = ["10.0.0.6", "10.0.0.7"] @@ -45,13 +48,13 @@ def test_create_and_delete_vpc(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc.cidr_block.should.equal("10.0.0.0/16") - all_vpcs = client.describe_vpcs()["Vpcs"] - all_vpcs.should.have.length_of(2) + all_vpcs = retrieve_all_vpcs(client) + [v["VpcId"] for v in all_vpcs].should.contain(vpc.id) vpc.delete() - all_vpcs = client.describe_vpcs()["Vpcs"] - all_vpcs.should.have.length_of(1) + all_vpcs = retrieve_all_vpcs(client) + [v["VpcId"] for v in all_vpcs].shouldnt.contain(vpc.id) with pytest.raises(ClientError) as ex: client.delete_vpc(VpcId="vpc-1234abcd") @@ -83,17 +86,21 @@ def test_vpc_defaults_boto3(): client = boto3.client("ec2", region_name="eu-north-1") vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") - client.describe_vpcs()["Vpcs"].should.have.length_of(2) - client.describe_route_tables()["RouteTables"].should.have.length_of(2) - client.describe_security_groups(Filters=[{"Name": "vpc-id", "Values": [vpc.id]}])[ + filters = [{"Name": "vpc-id", "Values": [vpc.id]}] + + client.describe_route_tables(Filters=filters)["RouteTables"].should.have.length_of( + 1 + ) + client.describe_security_groups(Filters=filters)[ "SecurityGroups" ].should.have.length_of(1) vpc.delete() - client.describe_vpcs()["Vpcs"].should.have.length_of(1) - client.describe_route_tables()["RouteTables"].should.have.length_of(1) - client.describe_security_groups(Filters=[{"Name": "vpc-id", "Values": [vpc.id]}])[ + client.describe_route_tables(Filters=filters)["RouteTables"].should.have.length_of( + 0 + ) + client.describe_security_groups(Filters=filters)[ "SecurityGroups" ].should.have.length_of(0) @@ -144,12 +151,10 @@ def test_multiple_vpcs_default_filter_boto3(): ec2.create_vpc(CidrBlock="10.8.0.0/16") ec2.create_vpc(CidrBlock="10.0.0.0/16") ec2.create_vpc(CidrBlock="192.168.0.0/16") - client.describe_vpcs()["Vpcs"].should.have.length_of(4) - vpc = client.describe_vpcs(Filters=[{"Name": "isDefault", "Values": ["true"]}])[ - "Vpcs" - ] - vpc.should.have.length_of(1) - vpc[0]["CidrBlock"].should.equal("172.31.0.0/16") + default_vpcs = retrieve_all_vpcs( + client, [{"Name": "isDefault", "Values": ["true"]}] + ) + [v["CidrBlock"] for v in default_vpcs].should.contain("172.31.0.0/16") # Has boto3 equivalent @@ -167,15 +172,29 @@ def test_vpc_state_available_filter(): def test_vpc_state_available_filter_boto3(): ec2 = boto3.resource("ec2", region_name="eu-west-1") client = boto3.client("ec2", region_name="eu-west-1") - vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") - ec2.create_vpc(CidrBlock="10.1.0.0/16") - client.describe_vpcs(Filters=[{"Name": "state", "Values": ["available"]}])[ - "Vpcs" - ].should.have.length_of(3) - vpc.delete() - client.describe_vpcs(Filters=[{"Name": "state", "Values": ["available"]}])[ - "Vpcs" - ].should.have.length_of(2) + vpc1 = ec2.create_vpc(CidrBlock="10.0.0.0/16") + vpc2 = ec2.create_vpc(CidrBlock="10.1.0.0/16") + + available = retrieve_all_vpcs(client, [{"Name": "state", "Values": ["available"]}]) + [v["VpcId"] for v in available].should.contain(vpc1.id) + [v["VpcId"] for v in available].should.contain(vpc2.id) + + vpc1.delete() + + available = retrieve_all_vpcs(client, [{"Name": "state", "Values": ["available"]}]) + [v["VpcId"] for v in available].shouldnt.contain(vpc1.id) + [v["VpcId"] for v in available].should.contain(vpc2.id) + + +def retrieve_all_vpcs(client, filters=[]): + resp = client.describe_vpcs(Filters=filters) + all_vpcs = resp["Vpcs"] + token = resp.get("NextToken") + while token: + resp = client.describe_vpcs(Filters=filters, NextToken=token) + all_vpcs.extend(resp["Vpcs"]) + token = resp.get("NextToken") + return all_vpcs # Has boto3 equivalent @@ -202,9 +221,11 @@ def test_vpc_tagging_boto3(): vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc.create_tags(Tags=[{"Key": "a key", "Value": "some value"}]) - tag = client.describe_tags()["Tags"][0] - tag.should.have.key("Key").equal("a key") - tag.should.have.key("Value").equal("some value") + + all_tags = retrieve_all_tagged(client) + ours = [t for t in all_tags if t["ResourceId"] == vpc.id][0] + ours.should.have.key("Key").equal("a key") + ours.should.have.key("Value").equal("some value") # Refresh the vpc vpc = client.describe_vpcs(VpcIds=[vpc.id])["Vpcs"][0] @@ -272,17 +293,16 @@ def test_vpc_get_by_cidr_block(): def test_vpc_get_by_cidr_block_boto3(): ec2 = boto3.resource("ec2", region_name="eu-west-1") client = boto3.client("ec2", region_name="eu-west-1") - vpc1 = ec2.create_vpc(CidrBlock="10.0.0.0/16") - vpc2 = ec2.create_vpc(CidrBlock="10.0.0.0/16") + random_ip = ".".join(map(str, (random.randint(0, 99) for _ in range(4)))) + random_cidr = f"{random_ip}/16" + vpc1 = ec2.create_vpc(CidrBlock=random_cidr) + vpc2 = ec2.create_vpc(CidrBlock=random_cidr) ec2.create_vpc(CidrBlock="10.0.0.0/24") - vpcs = client.describe_vpcs(Filters=[{"Name": "cidr", "Values": ["10.0.0.0/16"]}])[ + vpcs = client.describe_vpcs(Filters=[{"Name": "cidr", "Values": [random_cidr]}])[ "Vpcs" ] - vpcs.should.have.length_of(2) - vpc_ids = tuple(map(lambda v: v["VpcId"], vpcs)) - vpc1.id.should.be.within(vpc_ids) - vpc2.id.should.be.within(vpc_ids) + set([vpc["VpcId"] for vpc in vpcs]).should.equal(set([vpc1.id, vpc2.id])) # Has boto3 equivalent @@ -357,17 +377,16 @@ def test_vpc_get_by_tag_boto3(): vpc2 = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc3 = ec2.create_vpc(CidrBlock="10.0.0.0/24") - vpc1.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) - vpc2.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) + value1 = str(uuid4()) + vpc1.create_tags(Tags=[{"Key": "Name", "Value": value1}]) + vpc2.create_tags(Tags=[{"Key": "Name", "Value": value1}]) vpc3.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC2"}]) - vpcs = client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": ["TestVPC"]}])[ + vpcs = client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [value1]}])[ "Vpcs" ] vpcs.should.have.length_of(2) - vpc_ids = tuple(map(lambda v: v["VpcId"], vpcs)) - vpc1.id.should.be.within(vpc_ids) - vpc2.id.should.be.within(vpc_ids) + set([vpc["VpcId"] for vpc in vpcs]).should.equal(set([vpc1.id, vpc2.id])) # Has boto3 equivalent @@ -399,19 +418,18 @@ def test_vpc_get_by_tag_key_superset_boto3(): vpc2 = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc3 = ec2.create_vpc(CidrBlock="10.0.0.0/24") - vpc1.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) + tag_key = str(uuid4())[0:6] + vpc1.create_tags(Tags=[{"Key": tag_key, "Value": "TestVPC"}]) vpc1.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) - vpc2.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) + vpc2.create_tags(Tags=[{"Key": tag_key, "Value": "TestVPC"}]) vpc2.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) vpc3.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) - vpcs = client.describe_vpcs(Filters=[{"Name": "tag-key", "Values": ["Name"]}])[ + vpcs = client.describe_vpcs(Filters=[{"Name": "tag-key", "Values": [tag_key]}])[ "Vpcs" ] vpcs.should.have.length_of(2) - vpc_ids = tuple(map(lambda v: v["VpcId"], vpcs)) - vpc1.id.should.be.within(vpc_ids) - vpc2.id.should.be.within(vpc_ids) + set([vpc["VpcId"] for vpc in vpcs]).should.equal(set([vpc1.id, vpc2.id])) # Has boto3 equivalent @@ -443,19 +461,19 @@ def test_vpc_get_by_tag_key_subset_boto3(): vpc2 = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc3 = ec2.create_vpc(CidrBlock="10.0.0.0/24") - vpc1.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) - vpc1.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) - vpc2.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) - vpc2.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) + tag_key1 = str(uuid4())[0:6] + tag_key2 = str(uuid4())[0:6] + vpc1.create_tags(Tags=[{"Key": tag_key1, "Value": "TestVPC"}]) + vpc1.create_tags(Tags=[{"Key": tag_key2, "Value": "TestVPC2"}]) + vpc2.create_tags(Tags=[{"Key": tag_key1, "Value": "TestVPC"}]) + vpc2.create_tags(Tags=[{"Key": tag_key2, "Value": "TestVPC2"}]) vpc3.create_tags(Tags=[{"Key": "Test", "Value": "TestVPC2"}]) vpcs = client.describe_vpcs( - Filters=[{"Name": "tag-key", "Values": ["Name", "Key"]}] + Filters=[{"Name": "tag-key", "Values": [tag_key1, tag_key2]}] )["Vpcs"] vpcs.should.have.length_of(2) - vpc_ids = tuple(map(lambda v: v["VpcId"], vpcs)) - vpc1.id.should.be.within(vpc_ids) - vpc2.id.should.be.within(vpc_ids) + set([vpc["VpcId"] for vpc in vpcs]).should.equal(set([vpc1.id, vpc2.id])) # Has boto3 equivalent @@ -487,19 +505,18 @@ def test_vpc_get_by_tag_value_superset_boto3(): vpc2 = ec2.create_vpc(CidrBlock="10.0.0.0/16") vpc3 = ec2.create_vpc(CidrBlock="10.0.0.0/24") - vpc1.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) + tag_value = str(uuid4()) + vpc1.create_tags(Tags=[{"Key": "Name", "Value": tag_value}]) vpc1.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) - vpc2.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) + vpc2.create_tags(Tags=[{"Key": "Name", "Value": tag_value}]) vpc2.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) vpc3.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) - vpcs = client.describe_vpcs(Filters=[{"Name": "tag-value", "Values": ["TestVPC"]}])[ + vpcs = client.describe_vpcs(Filters=[{"Name": "tag-value", "Values": [tag_value]}])[ "Vpcs" ] vpcs.should.have.length_of(2) - vpc_ids = tuple(map(lambda v: v["VpcId"], vpcs)) - vpc1.id.should.be.within(vpc_ids) - vpc2.id.should.be.within(vpc_ids) + set([vpc["VpcId"] for vpc in vpcs]).should.equal(set([vpc1.id, vpc2.id])) # Has boto3 equivalent @@ -530,13 +547,15 @@ def test_vpc_get_by_tag_value_subset_boto3(): vpc2 = ec2.create_vpc(CidrBlock="10.0.0.0/16") ec2.create_vpc(CidrBlock="10.0.0.0/24") - vpc1.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) - vpc1.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) - vpc2.create_tags(Tags=[{"Key": "Name", "Value": "TestVPC"}]) - vpc2.create_tags(Tags=[{"Key": "Key", "Value": "TestVPC2"}]) + value1 = str(uuid4())[0:6] + value2 = str(uuid4())[0:6] + vpc1.create_tags(Tags=[{"Key": "Name", "Value": value1}]) + vpc1.create_tags(Tags=[{"Key": "Key", "Value": value2}]) + vpc2.create_tags(Tags=[{"Key": "Name", "Value": value1}]) + vpc2.create_tags(Tags=[{"Key": "Key", "Value": value2}]) vpcs = client.describe_vpcs( - Filters=[{"Name": "tag-value", "Values": ["TestVPC", "TestVPC2"]}] + Filters=[{"Name": "tag-value", "Values": [value1, value2]}] )["Vpcs"] vpcs.should.have.length_of(2) vpc_ids = tuple(map(lambda v: v["VpcId"], vpcs)) @@ -842,8 +861,9 @@ def test_cidr_block_association_filters(): ] ) ) - filtered_vpcs.should.be.length_of(1) - filtered_vpcs[0].id.should.equal(vpc2.id) + [vpc.id for vpc in filtered_vpcs].shouldnt.contain(vpc1.id) + [vpc.id for vpc in filtered_vpcs].should.contain(vpc2.id) + [vpc.id for vpc in filtered_vpcs].shouldnt.contain(vpc3.id) # Test filter for association id in VPCs association_id = vpc3_assoc_response["CidrBlockAssociation"]["AssociationId"] @@ -980,14 +1000,18 @@ def test_ipv6_cidr_block_association_filters(): filtered_vpcs[0].id.should.equal(vpc2.id) # Test filter for association state in VPC - this will never show anything in this test - filtered_vpcs = list( - ec2.vpcs.filter( + assoc_vpcs = [ + vpc.id + for vpc in ec2.vpcs.filter( Filters=[ {"Name": "ipv6-cidr-block-association.state", "Values": ["associated"]} ] ) - ) - filtered_vpcs.should.be.length_of(2) # 2 of 4 VPCs + ] + assoc_vpcs.shouldnt.contain(vpc1.id) + assoc_vpcs.should.contain(vpc2.id) + assoc_vpcs.should.contain(vpc3.id) + assoc_vpcs.shouldnt.contain(vpc4.id) @mock_ec2 @@ -1181,81 +1205,94 @@ def test_describe_classic_link_dns_support_multiple(): @mock_ec2 def test_describe_vpc_end_points(): ec2 = boto3.client("ec2", region_name="us-west-1") - vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") + vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] - route_table = ec2.create_route_table(VpcId=vpc["Vpc"]["VpcId"]) + route_table = ec2.create_route_table(VpcId=vpc["VpcId"])["RouteTable"] vpc_end_point = ec2.create_vpc_endpoint( - VpcId=vpc["Vpc"]["VpcId"], + VpcId=vpc["VpcId"], ServiceName="com.amazonaws.us-east-1.s3", - RouteTableIds=[route_table["RouteTable"]["RouteTableId"]], + RouteTableIds=[route_table["RouteTableId"]], VpcEndpointType="gateway", - ) + )["VpcEndpoint"] + our_id = vpc_end_point["VpcEndpointId"] - vpc_endpoints = ec2.describe_vpc_endpoints() - assert ( - vpc_endpoints.get("VpcEndpoints")[0].get("PrivateDnsEnabled") - is vpc_end_point.get("VpcEndpoint").get("PrivateDnsEnabled") - is True - ) - assert vpc_endpoints.get("VpcEndpoints")[0].get( - "VpcEndpointId" - ) == vpc_end_point.get("VpcEndpoint").get("VpcEndpointId") - assert vpc_endpoints.get("VpcEndpoints")[0].get("VpcId") == vpc["Vpc"]["VpcId"] - assert vpc_endpoints.get("VpcEndpoints")[0].get("RouteTableIds") == [ - route_table.get("RouteTable").get("RouteTableId") - ] - assert "VpcEndpointType" in vpc_endpoints.get("VpcEndpoints")[0] - assert "ServiceName" in vpc_endpoints.get("VpcEndpoints")[0] - assert "State" in vpc_endpoints.get("VpcEndpoints")[0] + all_endpoints = retrieve_all_endpoints(ec2) + [e["VpcEndpointId"] for e in all_endpoints].should.contain(our_id) + our_endpoint = [e for e in all_endpoints if e["VpcEndpointId"] == our_id][0] + vpc_end_point["PrivateDnsEnabled"].should.be.true + our_endpoint["PrivateDnsEnabled"].should.be.true - vpc_endpoints = ec2.describe_vpc_endpoints( - VpcEndpointIds=[vpc_end_point.get("VpcEndpoint").get("VpcEndpointId")] - ) - assert vpc_endpoints.get("VpcEndpoints")[0].get( - "VpcEndpointId" - ) == vpc_end_point.get("VpcEndpoint").get("VpcEndpointId") - assert vpc_endpoints.get("VpcEndpoints")[0].get("VpcId") == vpc["Vpc"]["VpcId"] - assert vpc_endpoints.get("VpcEndpoints")[0].get("RouteTableIds") == [ - route_table.get("RouteTable").get("RouteTableId") - ] - assert "VpcEndpointType" in vpc_endpoints.get("VpcEndpoints")[0] - assert "ServiceName" in vpc_endpoints.get("VpcEndpoints")[0] - assert "State" in vpc_endpoints.get("VpcEndpoints")[0] + our_endpoint["VpcId"].should.equal(vpc["VpcId"]) + our_endpoint["RouteTableIds"].should.equal([route_table["RouteTableId"]]) - try: - ec2.describe_vpc_endpoints( - VpcEndpointIds=[route_table.get("RouteTable").get("RouteTableId")] - ) - except ClientError as err: - assert err.response["Error"]["Code"] == "InvalidVpcEndpointId.NotFound" + our_endpoint.should.have.key("VpcEndpointType").equal("gateway") + our_endpoint.should.have.key("ServiceName").equal("com.amazonaws.us-east-1.s3") + our_endpoint.should.have.key("State").equal("available") + + endpoint_by_id = ec2.describe_vpc_endpoints(VpcEndpointIds=[our_id])[ + "VpcEndpoints" + ][0] + endpoint_by_id["VpcEndpointId"].should.equal(our_id) + endpoint_by_id["VpcId"].should.equal(vpc["VpcId"]) + endpoint_by_id["RouteTableIds"].should.equal([route_table["RouteTableId"]]) + endpoint_by_id["VpcEndpointType"].should.equal("gateway") + endpoint_by_id["ServiceName"].should.equal("com.amazonaws.us-east-1.s3") + endpoint_by_id["State"].should.equal("available") + + with pytest.raises(ClientError) as ex: + ec2.describe_vpc_endpoints(VpcEndpointIds=[route_table["RouteTableId"]]) + err = ex.value.response["Error"] + err["Code"].should.equal("InvalidVpcEndpointId.NotFound") + + +def retrieve_all_endpoints(ec2): + resp = ec2.describe_vpc_endpoints() + all_endpoints = resp["VpcEndpoints"] + next_token = resp.get("NextToken") + while next_token: + resp = ec2.describe_vpc_endpoints(NextToken=next_token) + all_endpoints.extend(resp["VpcEndpoints"]) + next_token = resp.get("NextToken") + return all_endpoints @mock_ec2 def test_delete_vpc_end_points(): ec2 = boto3.client("ec2", region_name="us-west-1") - vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") + vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")["Vpc"] - route_table = ec2.create_route_table(VpcId=vpc["Vpc"]["VpcId"]) + route_table = ec2.create_route_table(VpcId=vpc["VpcId"])["RouteTable"] vpc_end_point1 = ec2.create_vpc_endpoint( - VpcId=vpc["Vpc"]["VpcId"], + VpcId=vpc["VpcId"], ServiceName="com.amazonaws.us-west-1.s3", - RouteTableIds=[route_table["RouteTable"]["RouteTableId"]], + RouteTableIds=[route_table["RouteTableId"]], VpcEndpointType="gateway", )["VpcEndpoint"] vpc_end_point2 = ec2.create_vpc_endpoint( - VpcId=vpc["Vpc"]["VpcId"], + VpcId=vpc["VpcId"], ServiceName="com.amazonaws.us-west-1.s3", - RouteTableIds=[route_table["RouteTable"]["RouteTableId"]], + RouteTableIds=[route_table["RouteTableId"]], VpcEndpointType="gateway", - ) + )["VpcEndpoint"] - vpc_endpoints = ec2.describe_vpc_endpoints()["VpcEndpoints"] - vpc_endpoints.should.have.length_of(2) + vpc_endpoints = retrieve_all_endpoints(ec2) + all_ids = [e["VpcEndpointId"] for e in vpc_endpoints] + all_ids.should.contain(vpc_end_point1["VpcEndpointId"]) + all_ids.should.contain(vpc_end_point2["VpcEndpointId"]) ec2.delete_vpc_endpoints(VpcEndpointIds=[vpc_end_point1["VpcEndpointId"]]) - vpc_endpoints = ec2.describe_vpc_endpoints()["VpcEndpoints"] - vpc_endpoints.should.have.length_of(2) + vpc_endpoints = retrieve_all_endpoints(ec2) + all_ids = [e["VpcEndpointId"] for e in vpc_endpoints] + all_ids.should.contain(vpc_end_point1["VpcEndpointId"]) + all_ids.should.contain(vpc_end_point2["VpcEndpointId"]) - states = set([vpce["State"] for vpce in vpc_endpoints]) - states.should.equal({"available", "deleted"}) + ep1 = ec2.describe_vpc_endpoints(VpcEndpointIds=[vpc_end_point1["VpcEndpointId"]])[ + "VpcEndpoints" + ][0] + ep1["State"].should.equal("deleted") + + ep2 = ec2.describe_vpc_endpoints(VpcEndpointIds=[vpc_end_point2["VpcEndpointId"]])[ + "VpcEndpoints" + ][0] + ep2["State"].should.equal("available") diff --git a/tests/test_ec2/test_vpn_connections.py b/tests/test_ec2/test_vpn_connections.py index 9c251df90..2fc076a3e 100644 --- a/tests/test_ec2/test_vpn_connections.py +++ b/tests/test_ec2/test_vpn_connections.py @@ -52,14 +52,21 @@ def test_delete_vpn_connections_boto3(): Type="ipsec.1", VpnGatewayId="vgw-0123abcd", CustomerGatewayId="cgw-0123abcd" )["VpnConnection"] - cnx = client.describe_vpn_connections()["VpnConnections"] - cnx.should.have.length_of(1) + conns = retrieve_all_vpncs(client) + [c["VpnConnectionId"] for c in conns].should.contain( + vpn_connection["VpnConnectionId"] + ) client.delete_vpn_connection(VpnConnectionId=vpn_connection["VpnConnectionId"]) - cnx = client.describe_vpn_connections()["VpnConnections"] - cnx.should.have.length_of(1) - cnx[0].should.have.key("State").equal("deleted") + conns = retrieve_all_vpncs(client) + [c["VpnConnectionId"] for c in conns].should.contain( + vpn_connection["VpnConnectionId"] + ) + my_cnx = [ + c for c in conns if c["VpnConnectionId"] == vpn_connection["VpnConnectionId"] + ][0] + my_cnx.should.have.key("State").equal("deleted") # Has boto3 equivalent @@ -125,7 +132,7 @@ def test_describe_vpn_connections_boto3(): customer_gateway = client.create_customer_gateway( Type="ipsec.1", PublicIp="205.251.242.54", BgpAsn=65534, ).get("CustomerGateway", {}) - client.create_vpn_connection( + vpn_connection1 = client.create_vpn_connection( Type="ipsec.1", VpnGatewayId=vpn_gateway["VpnGatewayId"], CustomerGatewayId=customer_gateway["CustomerGatewayId"], @@ -136,8 +143,13 @@ def test_describe_vpn_connections_boto3(): CustomerGatewayId=customer_gateway["CustomerGatewayId"], )["VpnConnection"] - conns = client.describe_vpn_connections()["VpnConnections"] - conns.should.have.length_of(2) + conns = retrieve_all_vpncs(client) + [c["VpnConnectionId"] for c in conns].should.contain( + vpn_connection1["VpnConnectionId"] + ) + [c["VpnConnectionId"] for c in conns].should.contain( + vpn_connection2["VpnConnectionId"] + ) conns = client.describe_vpn_connections( VpnConnectionIds=[vpn_connection2["VpnConnectionId"]] @@ -159,3 +171,14 @@ def test_describe_vpn_connections_unknown(): err = ex.value.response["Error"] err["Message"].should.equal("The vpnConnection ID '?' does not exist") err["Code"].should.equal("InvalidVpnConnectionID.NotFound") + + +def retrieve_all_vpncs(client, filters=[]): + resp = client.describe_vpn_connections(Filters=filters) + all_vpncs = resp["VpnConnections"] + token = resp.get("NextToken") + while token: + resp = client.describe_vpn_connections(NextToken=token, Filters=filters) + all_vpncs.extend(resp["VpnConnections"]) + token = resp.get("NextToken") + return all_vpncs diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 33560daef..1f234833e 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -27,6 +27,7 @@ from moto.sqs.models import ( MAXIMUM_MESSAGE_SIZE_ATTR_UPPER_BOUND, MAXIMUM_MESSAGE_LENGTH, ) +from uuid import uuid4 TEST_POLICY = """ { @@ -55,7 +56,7 @@ def test_create_fifo_queue_fail(): sqs = boto3.client("sqs", region_name="us-east-1") try: - sqs.create_queue(QueueName="test-queue", Attributes={"FifoQueue": "true"}) + sqs.create_queue(QueueName=str(uuid4())[0:6], Attributes={"FifoQueue": "true"}) except botocore.exceptions.ClientError as err: err.response["Error"]["Code"].should.equal("InvalidParameterValue") else: @@ -66,7 +67,7 @@ def test_create_fifo_queue_fail(): def test_create_queue_with_same_attributes(): sqs = boto3.client("sqs", region_name="us-east-1") - dlq_url = sqs.create_queue(QueueName="test-queue-dlq")["QueueUrl"] + dlq_url = sqs.create_queue(QueueName=str(uuid4()))["QueueUrl"] dlq_arn = sqs.get_queue_attributes(QueueUrl=dlq_url, AttributeNames=["All"])[ "Attributes" ]["QueueArn"] @@ -81,33 +82,32 @@ def test_create_queue_with_same_attributes(): "VisibilityTimeout": "43200", } - sqs.create_queue(QueueName="test-queue", Attributes=attributes) + q_name = str(uuid4())[0:6] + sqs.create_queue(QueueName=q_name, Attributes=attributes) - sqs.create_queue(QueueName="test-queue", Attributes=attributes) + sqs.create_queue(QueueName=q_name, Attributes=attributes) @mock_sqs def test_create_queue_with_different_attributes_fail(): sqs = boto3.client("sqs", region_name="us-east-1") - sqs.create_queue(QueueName="test-queue", Attributes={"VisibilityTimeout": "10"}) + q_name = str(uuid4())[0:6] + sqs.create_queue(QueueName=q_name, Attributes={"VisibilityTimeout": "10"}) try: - sqs.create_queue(QueueName="test-queue", Attributes={"VisibilityTimeout": "60"}) + sqs.create_queue(QueueName=q_name, Attributes={"VisibilityTimeout": "60"}) except botocore.exceptions.ClientError as err: err.response["Error"]["Code"].should.equal("QueueAlreadyExists") else: raise RuntimeError("Should of raised QueueAlreadyExists Exception") - response = sqs.create_queue( - QueueName="test-queue1", Attributes={"FifoQueue": "True"} - ) + q_name2 = str(uuid4())[0:6] + response = sqs.create_queue(QueueName=q_name2, Attributes={"FifoQueue": "True"}) attributes = {"VisibilityTimeout": "60"} sqs.set_queue_attributes(QueueUrl=response.get("QueueUrl"), Attributes=attributes) - new_response = sqs.create_queue( - QueueName="test-queue1", Attributes={"FifoQueue": "True"} - ) + new_response = sqs.create_queue(QueueName=q_name2, Attributes={"FifoQueue": "True"}) new_response["QueueUrl"].should.equal(response.get("QueueUrl")) @@ -116,7 +116,7 @@ def test_create_fifo_queue(): # given region_name = "us-east-1" sqs = boto3.client("sqs", region_name=region_name) - queue_name = "test-queue.fifo" + queue_name = f"{str(uuid4())[0:6]}.fifo" # when queue_url = sqs.create_queue( @@ -152,7 +152,7 @@ def test_create_fifo_queue(): def test_create_fifo_queue_with_high_throughput(): # given sqs = boto3.client("sqs", region_name="us-east-1") - queue_name = "test-queue.fifo" + queue_name = f"{str(uuid4())[0:6]}.fifo" # when queue_url = sqs.create_queue( @@ -179,13 +179,14 @@ def test_create_fifo_queue_with_high_throughput(): def test_create_queue(): sqs = boto3.resource("sqs", region_name="us-east-1") - new_queue = sqs.create_queue(QueueName="test-queue") + q_name = str(uuid4())[0:6] + new_queue = sqs.create_queue(QueueName=q_name) new_queue.should_not.be.none - new_queue.should.have.property("url").should.contain("test-queue") + new_queue.should.have.property("url").should.contain(q_name) - queue = sqs.get_queue_by_name(QueueName="test-queue") + queue = sqs.get_queue_by_name(QueueName=q_name) queue.attributes.get("QueueArn").should_not.be.none - queue.attributes.get("QueueArn").split(":")[-1].should.equal("test-queue") + queue.attributes.get("QueueArn").split(":")[-1].should.equal(q_name) queue.attributes.get("QueueArn").split(":")[3].should.equal("us-east-1") queue.attributes.get("VisibilityTimeout").should_not.be.none queue.attributes.get("VisibilityTimeout").should.equal("30") @@ -195,8 +196,9 @@ def test_create_queue(): def test_create_queue_kms(): sqs = boto3.resource("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] new_queue = sqs.create_queue( - QueueName="test-queue", + QueueName=q_name, Attributes={ "KmsMasterKeyId": "master-key-id", "KmsDataKeyReusePeriodSeconds": "600", @@ -204,7 +206,7 @@ def test_create_queue_kms(): ) new_queue.should_not.be.none - queue = sqs.get_queue_by_name(QueueName="test-queue") + queue = sqs.get_queue_by_name(QueueName=q_name) queue.attributes.get("KmsMasterKeyId").should.equal("master-key-id") queue.attributes.get("KmsDataKeyReusePeriodSeconds").should.equal("600") @@ -213,9 +215,9 @@ def test_create_queue_kms(): @mock_sqs def test_create_queue_with_tags(): client = boto3.client("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] response = client.create_queue( - QueueName="test-queue-with-tags", - tags={"tag_key_1": "tag_value_1", "tag_key_2": ""}, + QueueName=q_name, tags={"tag_key_1": "tag_value_1", "tag_key_2": ""}, ) queue_url = response["QueueUrl"] @@ -227,8 +229,9 @@ def test_create_queue_with_tags(): @mock_sqs def test_create_queue_with_policy(): client = boto3.client("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] response = client.create_queue( - QueueName="test-queue", + QueueName=q_name, Attributes={ "Policy": json.dumps( { @@ -256,8 +259,9 @@ def test_create_queue_with_policy(): @mock_sqs def test_set_queue_attribute_empty_policy_removes_attr(): client = boto3.client("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] response = client.create_queue( - QueueName="test-queue", + QueueName=q_name, Attributes={ "Policy": json.dumps( { @@ -281,11 +285,12 @@ def test_set_queue_attribute_empty_policy_removes_attr(): @mock_sqs def test_get_queue_url(): client = boto3.client("sqs", region_name="us-east-1") - client.create_queue(QueueName="test-queue") + q_name = str(uuid4())[0:6] + client.create_queue(QueueName=q_name) - response = client.get_queue_url(QueueName="test-queue") + response = client.get_queue_url(QueueName=q_name) - response.should.have.key("QueueUrl").which.should.contain("test-queue") + response.should.have.key("QueueUrl").which.should.contain(q_name) @mock_sqs @@ -326,7 +331,7 @@ def test_get_nonexistent_queue(): @mock_sqs def test_message_send_without_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) msg = queue.send_message(MessageBody="derp") msg.get("MD5OfMessageBody").should.equal("58fd9edd83341c29f1aebba81c31e257") msg.shouldnt.have.key("MD5OfMessageAttributes") @@ -339,7 +344,7 @@ def test_message_send_without_attributes(): @mock_sqs def test_message_send_with_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) msg = queue.send_message( MessageBody="derp", MessageAttributes={ @@ -361,7 +366,7 @@ def test_message_send_with_attributes(): def test_message_retention_period(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="blah", Attributes={"MessageRetentionPeriod": "3"} + QueueName=str(uuid4())[0:6], Attributes={"MessageRetentionPeriod": "3"} ) queue.send_message( MessageBody="derp", @@ -395,7 +400,7 @@ def test_message_retention_period(): def test_queue_retention_period(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="blah", Attributes={"MessageRetentionPeriod": "3"} + QueueName=str(uuid4())[0:6], Attributes={"MessageRetentionPeriod": "3"} ) time.sleep(5) @@ -417,7 +422,7 @@ def test_queue_retention_period(): @mock_sqs def test_message_with_invalid_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) with pytest.raises(ClientError) as e: queue.send_message( MessageBody="derp", @@ -436,7 +441,7 @@ def test_message_with_invalid_attributes(): @mock_sqs def test_message_with_string_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) msg = queue.send_message( MessageBody="derp", MessageAttributes={ @@ -462,7 +467,7 @@ def test_message_with_string_attributes(): @mock_sqs def test_message_with_binary_attribute(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) msg = queue.send_message( MessageBody="derp", MessageAttributes={ @@ -489,7 +494,7 @@ def test_message_with_binary_attribute(): @mock_sqs def test_message_with_attributes_have_labels(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) msg = queue.send_message( MessageBody="derp", MessageAttributes={ @@ -510,7 +515,7 @@ def test_message_with_attributes_have_labels(): @mock_sqs def test_message_with_attributes_invalid_datatype(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) with pytest.raises(ClientError) as e: queue.send_message( @@ -534,7 +539,7 @@ def test_message_with_attributes_invalid_datatype(): def test_send_message_with_message_group_id(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-group-id.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) sent = queue.send_message( @@ -560,7 +565,7 @@ def test_send_message_with_unicode_characters(): body_one = "Héllo!😀" sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) msg = queue.send_message(MessageBody=body_one) messages = queue.receive_messages() @@ -572,7 +577,7 @@ def test_send_message_with_unicode_characters(): @mock_sqs def test_set_queue_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) queue.attributes["VisibilityTimeout"].should.equal("30") @@ -582,62 +587,74 @@ def test_set_queue_attributes(): @mock_sqs def test_create_queues_in_multiple_region(): - west1_conn = boto3.client("sqs", region_name="us-west-1") - west1_conn.create_queue(QueueName="blah") + w1 = boto3.client("sqs", region_name="us-west-1") + w1_name = str(uuid4())[0:6] + w1.create_queue(QueueName=w1_name) - west2_conn = boto3.client("sqs", region_name="us-west-2") - west2_conn.create_queue(QueueName="test-queue") + w2 = boto3.client("sqs", region_name="us-west-2") + w2_name = str(uuid4())[0:6] + w2.create_queue(QueueName=w2_name) - list(west1_conn.list_queues()["QueueUrls"]).should.have.length_of(1) - list(west2_conn.list_queues()["QueueUrls"]).should.have.length_of(1) - - if settings.TEST_SERVER_MODE: - base_url = "http://localhost:5000" - else: - base_url = "https://us-west-1.queue.amazonaws.com" - - west1_conn.list_queues()["QueueUrls"][0].should.equal( - "{base_url}/{AccountId}/blah".format(base_url=base_url, AccountId=ACCOUNT_ID) + base_url = ( + "http://localhost:5000" + if settings.TEST_SERVER_MODE + else "https://us-west-1.queue.amazonaws.com" ) + w1.list_queues()["QueueUrls"].should.contain(f"{base_url}/{ACCOUNT_ID}/{w1_name}") + w1.list_queues()["QueueUrls"].shouldnt.contain(f"{base_url}/{ACCOUNT_ID}/{w2_name}") + + base_url = ( + "http://localhost:5000" + if settings.TEST_SERVER_MODE + else "https://us-west-2.queue.amazonaws.com" + ) + w2.list_queues()["QueueUrls"].shouldnt.contain(f"{base_url}/{ACCOUNT_ID}/{w1_name}") + w2.list_queues()["QueueUrls"].should.contain(f"{base_url}/{ACCOUNT_ID}/{w2_name}") @mock_sqs def test_get_queue_with_prefix(): conn = boto3.client("sqs", region_name="us-west-1") - conn.create_queue(QueueName="prefixa-queue") - conn.create_queue(QueueName="prefixb-queue") - conn.create_queue(QueueName="test-queue") + conn.create_queue(QueueName=str(uuid4())[0:6]) + q_name1 = str(uuid4())[0:6] + conn.create_queue(QueueName=q_name1) + prefix = str(uuid4())[0:6] + q_name2 = f"{prefix}-test" + conn.create_queue(QueueName=q_name2) - conn.list_queues()["QueueUrls"].should.have.length_of(3) + base_url = ( + "http://localhost:5000" + if settings.TEST_SERVER_MODE + else "https://us-west-1.queue.amazonaws.com" + ) + expected_url1 = f"{base_url}/{ACCOUNT_ID}/{q_name1}" + expected_url2 = f"{base_url}/{ACCOUNT_ID}/{q_name2}" - queue = conn.list_queues(QueueNamePrefix="test-")["QueueUrls"] + all_urls = conn.list_queues()["QueueUrls"] + all_urls.should.contain(expected_url1) + all_urls.should.contain(expected_url2) + + queue = conn.list_queues(QueueNamePrefix=prefix)["QueueUrls"] queue.should.have.length_of(1) - if settings.TEST_SERVER_MODE: - base_url = "http://localhost:5000" - else: - base_url = "https://us-west-1.queue.amazonaws.com" - - queue[0].should.equal( - "{base_url}/{AccountId}/test-queue".format( - base_url=base_url, AccountId=ACCOUNT_ID - ) - ) + queue[0].should.equal(expected_url2) @mock_sqs def test_delete_queue(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - q_resp = conn.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout": "3"} - ) + q_name = str(uuid4())[0:6] + q_resp = conn.create_queue(QueueName=q_name, Attributes={"VisibilityTimeout": "3"}) queue = sqs.Queue(q_resp["QueueUrl"]) - conn.list_queues()["QueueUrls"].should.have.length_of(1) + all_urls = conn.list_queues()["QueueUrls"] + [u[u.rfind("/") + 1 :] for u in all_urls].should.contain(q_name) queue.delete() - conn.list_queues().get("QueueUrls").should.equal(None) + + all_urls = conn.list_queues().get("QueueUrls", []) + [u[u.rfind("/") + 1 :] for u in all_urls].shouldnt.contain(q_name) @mock_sqs @@ -669,8 +686,9 @@ def test_get_queue_attributes(): QueueUrl=dlq_resp["QueueUrl"], AttributeNames=["QueueArn"] )["Attributes"]["QueueArn"] + q_name = str(uuid4())[0:6] response = client.create_queue( - QueueName="test-queue", + QueueName=q_name, Attributes={ "RedrivePolicy": json.dumps( {"deadLetterTargetArn": dlq_arn1, "maxReceiveCount": 2} @@ -690,7 +708,7 @@ def test_get_queue_attributes(): response["Attributes"]["MaximumMessageSize"].should.equal("262144") response["Attributes"]["MessageRetentionPeriod"].should.equal("345600") response["Attributes"]["QueueArn"].should.equal( - "arn:aws:sqs:us-east-1:{}:test-queue".format(ACCOUNT_ID) + "arn:aws:sqs:us-east-1:{}:{}".format(ACCOUNT_ID, q_name) ) response["Attributes"]["ReceiveMessageWaitTimeSeconds"].should.equal("0") response["Attributes"]["VisibilityTimeout"].should.equal("30") @@ -710,7 +728,7 @@ def test_get_queue_attributes(): { "ApproximateNumberOfMessages": "0", "MaximumMessageSize": "262144", - "QueueArn": "arn:aws:sqs:us-east-1:{}:test-queue".format(ACCOUNT_ID), + "QueueArn": "arn:aws:sqs:us-east-1:{}:{}".format(ACCOUNT_ID, q_name), "VisibilityTimeout": "30", "RedrivePolicy": json.dumps( {"deadLetterTargetArn": dlq_arn1, "maxReceiveCount": 2} @@ -729,7 +747,7 @@ def test_get_queue_attributes(): @mock_sqs def test_get_queue_attributes_errors(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + response = client.create_queue(QueueName=str(uuid4())[0:6]) queue_url = response["QueueUrl"] client.get_queue_attributes.when.called_with( @@ -774,7 +792,7 @@ def test_set_queue_attribute(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") q_resp = conn.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout": "3"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout": "3"} ) queue = sqs.Queue(q_resp["QueueUrl"]) @@ -789,7 +807,7 @@ def test_set_queue_attribute(): def test_send_receive_message_without_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - q_resp = conn.create_queue(QueueName="test-queue") + q_resp = conn.create_queue(QueueName=str(uuid4())[0:6]) queue = sqs.Queue(q_resp["QueueUrl"]) body_one = "this is a test message" @@ -819,7 +837,7 @@ def test_send_receive_message_without_attributes(): def test_send_receive_message_with_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - q_resp = conn.create_queue(QueueName="test-queue") + q_resp = conn.create_queue(QueueName=str(uuid4())[0:6]) queue = sqs.Queue(q_resp["QueueUrl"]) body_one = "this is a test message" @@ -861,7 +879,7 @@ def test_send_receive_message_with_attributes(): def test_send_receive_message_with_attributes_with_labels(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - q_resp = conn.create_queue(QueueName="test-queue") + q_resp = conn.create_queue(QueueName=str(uuid4())[0:6]) queue = sqs.Queue(q_resp["QueueUrl"]) body_one = "this is a test message" @@ -919,7 +937,7 @@ def test_send_receive_message_with_attributes_with_labels(): @mock_sqs def test_receive_message_with_xml_content(): sqs = boto3.client("sqs", region_name="eu-west-2") - queue_url = sqs.create_queue(QueueName="test-queue")["QueueUrl"] + queue_url = sqs.create_queue(QueueName=str(uuid4())[0:6])["QueueUrl"] original_payload = '' data = {"Payload": {"DataType": "String", "StringValue": original_payload}} @@ -977,7 +995,7 @@ def test_change_message_visibility_than_permitted(): def test_send_receive_message_timestamps(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - q_resp = conn.create_queue(QueueName="test-queue") + q_resp = conn.create_queue(QueueName=str(uuid4())[0:6]) queue = sqs.Queue(q_resp["QueueUrl"]) response = queue.send_message(MessageBody="derp") @@ -1075,7 +1093,7 @@ def test_send_receive_message_timestamps(): def test_send_receive_message_with_attribute_name(attribute_name, expected): sqs = boto3.resource("sqs", region_name="us-east-1") client = boto3.client("sqs", region_name="us-east-1") - q_resp = client.create_queue(QueueName="test-queue") + q_resp = client.create_queue(QueueName=str(uuid4())[0:6]) queue = sqs.Queue(q_resp["QueueUrl"]) body_one = "this is a test message" @@ -1241,7 +1259,7 @@ def test_send_receive_message_with_attribute_name(attribute_name, expected): def test_fifo_send_receive_message_with_attribute_name(attribute_name, expected): client = boto3.client("sqs", region_name="us-east-1") queue_url = client.create_queue( - QueueName="test-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} )["QueueUrl"] body = "this is a test message" @@ -1282,7 +1300,7 @@ def test_get_queue_attributes_no_param(): AWS does not return the Attributes-key when omitting the AttributeNames-parameter """ sqs = boto3.client("sqs", region_name="ap-northeast-3") - queue_url = sqs.create_queue(QueueName="test-queue")["QueueUrl"] + queue_url = sqs.create_queue(QueueName=str(uuid4())[0:6])["QueueUrl"] queue_attrs = sqs.get_queue_attributes(QueueUrl=queue_url) queue_attrs.shouldnt.have.key("Attributes") @@ -1294,7 +1312,7 @@ def test_get_queue_attributes_no_param(): @mock_sqs def test_max_number_of_messages_invalid_param(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) with pytest.raises(ClientError): queue.receive_messages(MaxNumberOfMessages=11) @@ -1309,7 +1327,7 @@ def test_max_number_of_messages_invalid_param(): @mock_sqs def test_wait_time_seconds_invalid_param(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) with pytest.raises(ClientError): queue.receive_messages(WaitTimeSeconds=-1) @@ -1330,7 +1348,7 @@ def test_receive_messages_with_wait_seconds_timeout_of_zero(): """ sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="blah") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) messages = queue.receive_messages(WaitTimeSeconds=0) messages.should.equal([]) @@ -1356,7 +1374,7 @@ def test_send_message_with_xml_characters(): def test_send_message_with_xml_characters_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") client = boto3.client("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) body_one = "< & >" @@ -1433,7 +1451,7 @@ def test_send_message_with_delay(): @mock_sqs def test_send_message_with_delay_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) body_one = "this is a test message" body_two = "this is another test message" @@ -1466,7 +1484,7 @@ def test_send_large_message_fails(): @mock_sqs def test_send_large_message_fails_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") + queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) body = "test message" * 200000 with pytest.raises(ClientError) as ex: @@ -1504,7 +1522,7 @@ def test_message_becomes_inflight_when_received(): def test_message_becomes_inflight_when_received_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "1"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "1"} ) queue.attributes["ApproximateNumberOfMessages"].should.equal("0") @@ -1551,7 +1569,7 @@ def test_receive_message_with_explicit_visibility_timeout(): def test_receive_message_with_explicit_visibility_timeout_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "1"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "1"} ) queue.attributes["ApproximateNumberOfMessages"].should.equal("0") @@ -1608,7 +1626,7 @@ def test_change_message_visibility(): def test_change_message_visibility_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "2"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "2"} ) body = "this is a test message" @@ -1698,7 +1716,7 @@ def test_queue_length(): def test_queue_length_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "2"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "2"} ) queue.send_message(MessageBody="this is a test message") @@ -1802,7 +1820,7 @@ def test_delete_batch_operation(): def test_delete_batch_operation_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "2"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "2"} ) queue.send_message(MessageBody="test message 1") @@ -1886,7 +1904,7 @@ def test_change_message_visibility_on_invalid_receipt(): def test_change_message_visibility_on_invalid_receipt_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "1"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "1"} ) queue.send_message(MessageBody="test message 1") @@ -1944,7 +1962,7 @@ def test_change_message_visibility_on_visible_message(): def test_change_message_visibility_on_visible_message_boto3(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"VisibilityTimeout ": "1"} + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": "1"} ) queue.send_message(MessageBody="test message") @@ -1986,7 +2004,7 @@ def test_purge_queue_before_delete_message(): client = boto3.client("sqs", region_name="us-east-1") create_resp = client.create_queue( - QueueName="test-dlr-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"dlr-{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) queue_url = create_resp["QueueUrl"] @@ -2040,7 +2058,7 @@ def test_delete_message_after_visibility_timeout_boto3(): VISIBILITY_TIMEOUT = 1 sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="new-queue", + QueueName=str(uuid4())[0:6], Attributes={"VisibilityTimeout ": f"{VISIBILITY_TIMEOUT}"}, ) @@ -2062,7 +2080,7 @@ def test_delete_message_after_visibility_timeout_boto3(): @mock_sqs def test_delete_message_errors(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + response = client.create_queue(QueueName=str(uuid4())[0:6]) queue_url = response["QueueUrl"] client.send_message(QueueUrl=queue_url, MessageBody="body") response = client.receive_message(QueueUrl=queue_url) @@ -2082,7 +2100,7 @@ def test_delete_message_errors(): @mock_sqs def test_send_message_batch(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + response = client.create_queue(QueueName=str(uuid4())[0:6]) queue_url = response["QueueUrl"] response = client.send_message_batch( @@ -2150,7 +2168,7 @@ def test_send_message_batch(): @mock_sqs def test_delete_message_batch_with_duplicates(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + response = client.create_queue(QueueName=str(uuid4())[0:6]) queue_url = response["QueueUrl"] client.send_message(QueueUrl=queue_url, MessageBody="coucou") @@ -2390,9 +2408,8 @@ def test_batch_change_message_visibility(): def test_permissions(): client = boto3.client("sqs", region_name="us-east-1") - resp = client.create_queue( - QueueName="test-dlr-queue.fifo", Attributes={"FifoQueue": "true"} - ) + q_name = f"{str(uuid4())[0:6]}.fifo" + resp = client.create_queue(QueueName=q_name, Attributes={"FifoQueue": "true"}) queue_url = resp["QueueUrl"] client.add_permission( @@ -2414,7 +2431,7 @@ def test_permissions(): policy = json.loads(response["Attributes"]["Policy"]) policy["Version"].should.equal("2012-10-17") policy["Id"].should.equal( - "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo/SQSDefaultPolicy" + f"arn:aws:sqs:us-east-1:123456789012:{q_name}/SQSDefaultPolicy" ) sorted(policy["Statement"], key=lambda x: x["Sid"]).should.equal( [ @@ -2423,14 +2440,14 @@ def test_permissions(): "Effect": "Allow", "Principal": {"AWS": "arn:aws:iam::111111111111:root"}, "Action": "SQS:*", - "Resource": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo", + "Resource": f"arn:aws:sqs:us-east-1:123456789012:{q_name}", }, { "Sid": "account2", "Effect": "Allow", "Principal": {"AWS": "arn:aws:iam::222211111111:root"}, "Action": "SQS:SendMessage", - "Resource": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo", + "Resource": f"arn:aws:sqs:us-east-1:123456789012:{q_name}", }, ] ) @@ -2443,14 +2460,14 @@ def test_permissions(): json.loads(response["Attributes"]["Policy"]).should.equal( { "Version": "2012-10-17", - "Id": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo/SQSDefaultPolicy", + "Id": f"arn:aws:sqs:us-east-1:123456789012:{q_name}/SQSDefaultPolicy", "Statement": [ { "Sid": "account1", "Effect": "Allow", "Principal": {"AWS": "arn:aws:iam::111111111111:root"}, "Action": "SQS:*", - "Resource": "arn:aws:sqs:us-east-1:123456789012:test-dlr-queue.fifo", + "Resource": f"arn:aws:sqs:us-east-1:123456789012:{q_name}", }, ], } @@ -2462,7 +2479,7 @@ def test_get_queue_attributes_template_response_validation(): client = boto3.client("sqs", region_name="us-east-1") resp = client.create_queue( - QueueName="test-dlr-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) queue_url = resp["QueueUrl"] @@ -2490,7 +2507,7 @@ def test_get_queue_attributes_template_response_validation(): @mock_sqs def test_add_permission_errors(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + response = client.create_queue(QueueName=str(uuid4())[0:6]) queue_url = response["QueueUrl"] client.add_permission( QueueUrl=queue_url, @@ -2588,7 +2605,7 @@ def test_add_permission_errors(): @mock_sqs def test_remove_permission_errors(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue(QueueName="test-queue") + response = client.create_queue(QueueName=str(uuid4())[0:6]) queue_url = response["QueueUrl"] with pytest.raises(ClientError) as e: @@ -2634,7 +2651,7 @@ def test_list_queue_tags_errors(): client = boto3.client("sqs", region_name="us-east-1") response = client.create_queue( - QueueName="test-queue-with-tags", tags={"tag_key_1": "tag_value_X"} + QueueName=str(uuid4())[0:6], tags={"tag_key_1": "tag_value_X"} ) queue_url = response["QueueUrl"] @@ -2649,9 +2666,8 @@ def test_list_queue_tags_errors(): def test_tag_queue_errors(): client = boto3.client("sqs", region_name="us-east-1") - response = client.create_queue( - QueueName="test-queue-with-tags", tags={"tag_key_1": "tag_value_X"} - ) + q_name = str(uuid4())[0:6] + response = client.create_queue(QueueName=q_name, tags={"tag_key_1": "tag_value_X"}) queue_url = response["QueueUrl"] client.tag_queue.when.called_with( @@ -2669,7 +2685,7 @@ def test_tag_queue_errors(): } client.tag_queue.when.called_with( QueueUrl=queue_url, Tags=too_many_tags - ).should.throw(ClientError, "Too many tags added for queue test-queue-with-tags.") + ).should.throw(ClientError, f"Too many tags added for queue {q_name}.") # when the request fails, the tags should not be updated client.list_queue_tags(QueueUrl=queue_url)["Tags"].should.equal( @@ -2682,7 +2698,7 @@ def test_untag_queue_errors(): client = boto3.client("sqs", region_name="us-east-1") response = client.create_queue( - QueueName="test-queue-with-tags", tags={"tag_key_1": "tag_value_1"} + QueueName=str(uuid4())[0:6], tags={"tag_key_1": "tag_value_1"} ) queue_url = response["QueueUrl"] @@ -2701,7 +2717,7 @@ def test_untag_queue_errors(): def test_create_fifo_queue_with_dlq(): sqs = boto3.client("sqs", region_name="us-east-1") resp = sqs.create_queue( - QueueName="test-dlr-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) queue_url1 = resp["QueueUrl"] queue_arn1 = sqs.get_queue_attributes( @@ -2709,7 +2725,7 @@ def test_create_fifo_queue_with_dlq(): )["Attributes"]["QueueArn"] resp = sqs.create_queue( - QueueName="test-dlr-queue", Attributes={"FifoQueue": "false"} + QueueName=str(uuid4())[0:6], Attributes={"FifoQueue": "false"} ) queue_url2 = resp["QueueUrl"] queue_arn2 = sqs.get_queue_attributes( @@ -2717,7 +2733,7 @@ def test_create_fifo_queue_with_dlq(): )["Attributes"]["QueueArn"] sqs.create_queue( - QueueName="test-queue.fifo", + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={ "FifoQueue": "true", "RedrivePolicy": json.dumps( @@ -2729,7 +2745,7 @@ def test_create_fifo_queue_with_dlq(): # Cant have fifo queue with non fifo DLQ with pytest.raises(ClientError): sqs.create_queue( - QueueName="test-queue2.fifo", + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={ "FifoQueue": "true", "RedrivePolicy": json.dumps( @@ -2748,7 +2764,7 @@ def test_queue_with_dlq(): with freeze_time("2015-01-01 12:00:00"): resp = sqs.create_queue( - QueueName="test-dlr-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) queue_url1 = resp["QueueUrl"] queue_arn1 = sqs.get_queue_attributes( @@ -2756,7 +2772,7 @@ def test_queue_with_dlq(): )["Attributes"]["QueueArn"] resp = sqs.create_queue( - QueueName="test-queue.fifo", + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={ "FifoQueue": "true", "RedrivePolicy": json.dumps( @@ -2807,7 +2823,7 @@ def test_queue_with_dlq(): def test_redrive_policy_available(): sqs = boto3.client("sqs", region_name="us-east-1") - resp = sqs.create_queue(QueueName="test-deadletter") + resp = sqs.create_queue(QueueName=str(uuid4())[0:6]) queue_url1 = resp["QueueUrl"] queue_arn1 = sqs.get_queue_attributes( QueueUrl=queue_url1, AttributeNames=["QueueArn"] @@ -2815,7 +2831,8 @@ def test_redrive_policy_available(): redrive_policy = {"deadLetterTargetArn": queue_arn1, "maxReceiveCount": 1} resp = sqs.create_queue( - QueueName="test-queue", Attributes={"RedrivePolicy": json.dumps(redrive_policy)} + QueueName=str(uuid4())[0:6], + Attributes={"RedrivePolicy": json.dumps(redrive_policy)}, ) queue_url2 = resp["QueueUrl"] @@ -2828,7 +2845,7 @@ def test_redrive_policy_available(): # Cant have redrive policy without maxReceiveCount with pytest.raises(ClientError): sqs.create_queue( - QueueName="test-queue2", + QueueName=str(uuid4())[0:6], Attributes={ "FifoQueue": "true", "RedrivePolicy": json.dumps({"deadLetterTargetArn": queue_arn1}), @@ -2855,8 +2872,9 @@ def test_redrive_policy_non_existent_queue(): def test_redrive_policy_set_attributes(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") - deadletter_queue = sqs.create_queue(QueueName="test-deadletter") + q_name = str(uuid4())[0:6] + queue = sqs.create_queue(QueueName=q_name) + deadletter_queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) redrive_policy = { "deadLetterTargetArn": deadletter_queue.attributes["QueueArn"], @@ -2865,7 +2883,7 @@ def test_redrive_policy_set_attributes(): queue.set_attributes(Attributes={"RedrivePolicy": json.dumps(redrive_policy)}) - copy = sqs.get_queue_by_name(QueueName="test-queue") + copy = sqs.get_queue_by_name(QueueName=q_name) assert "RedrivePolicy" in copy.attributes copy_policy = json.loads(copy.attributes["RedrivePolicy"]) assert copy_policy == redrive_policy @@ -2875,8 +2893,9 @@ def test_redrive_policy_set_attributes(): def test_redrive_policy_set_attributes_with_string_value(): sqs = boto3.resource("sqs", region_name="us-east-1") - queue = sqs.create_queue(QueueName="test-queue") - deadletter_queue = sqs.create_queue(QueueName="test-deadletter") + q_name = str(uuid4())[0:6] + queue = sqs.create_queue(QueueName=q_name) + deadletter_queue = sqs.create_queue(QueueName=str(uuid4())[0:6]) queue.set_attributes( Attributes={ @@ -2889,7 +2908,7 @@ def test_redrive_policy_set_attributes_with_string_value(): } ) - copy = sqs.get_queue_by_name(QueueName="test-queue") + copy = sqs.get_queue_by_name(QueueName=q_name) assert "RedrivePolicy" in copy.attributes copy_policy = json.loads(copy.attributes["RedrivePolicy"]) assert copy_policy == { @@ -2902,7 +2921,7 @@ def test_redrive_policy_set_attributes_with_string_value(): def test_receive_messages_with_message_group_id(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) queue.set_attributes(Attributes={"VisibilityTimeout": "3600"}) queue.send_message(MessageBody="message-1", MessageGroupId="group") @@ -2933,7 +2952,7 @@ def test_receive_messages_with_message_group_id(): def test_receive_messages_with_message_group_id_on_requeue(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue.fifo", Attributes={"FifoQueue": "true"} + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true"} ) queue.set_attributes(Attributes={"VisibilityTimeout": "3600"}) queue.send_message(MessageBody="message-1", MessageGroupId="group") @@ -2997,7 +3016,7 @@ def test_receive_message_for_queue_with_receive_message_wait_time_seconds_set(): sqs = boto3.resource("sqs", region_name="us-east-1") queue = sqs.create_queue( - QueueName="test-queue", Attributes={"ReceiveMessageWaitTimeSeconds": "2"} + QueueName=str(uuid4())[0:6], Attributes={"ReceiveMessageWaitTimeSeconds": "2"} ) queue.receive_messages() @@ -3005,29 +3024,39 @@ def test_receive_message_for_queue_with_receive_message_wait_time_seconds_set(): @mock_sqs def test_list_queues_limits_to_1000_queues(): + if settings.TEST_SERVER_MODE: + # Re-visit once we have a NextToken-implementation for list_queues + raise SkipTest("Too many queues for a persistent mode") client = boto3.client("sqs", region_name="us-east-1") + prefix_name = str(uuid4())[0:6] + queue_urls = [] for i in range(1001): - client.create_queue(QueueName="test-queue-{0}".format(i)) + queue = client.create_queue(QueueName=f"{prefix_name}-{i}") + queue_urls.append(queue["QueueUrl"]) client.list_queues()["QueueUrls"].should.have.length_of(1000) - client.list_queues(QueueNamePrefix="test-queue")["QueueUrls"].should.have.length_of( + client.list_queues(QueueNamePrefix=prefix_name)["QueueUrls"].should.have.length_of( 1000 ) resource = boto3.resource("sqs", region_name="us-east-1") list(resource.queues.all()).should.have.length_of(1000) - list(resource.queues.filter(QueueNamePrefix="test-queue")).should.have.length_of( + list(resource.queues.filter(QueueNamePrefix=prefix_name)).should.have.length_of( 1000 ) + # Delete this again, to not hog all the resources + for url in queue_urls: + client.delete_queue(QueueUrl=url) + @mock_sqs def test_send_message_to_fifo_without_message_group_id(): sqs = boto3.resource("sqs", region_name="eu-west-3") queue = sqs.create_queue( - QueueName="blah.fifo", + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"}, ) @@ -3044,7 +3073,7 @@ def test_send_message_to_fifo_without_message_group_id(): def test_send_messages_to_fifo_without_message_group_id(): sqs = boto3.resource("sqs", region_name="eu-west-3") queue = sqs.create_queue( - QueueName="blah.fifo", + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"}, ) @@ -3062,7 +3091,7 @@ def test_send_messages_to_fifo_without_message_group_id(): @mock_sqs def test_maximum_message_size_attribute_default(): sqs = boto3.resource("sqs", region_name="eu-west-3") - queue = sqs.create_queue(QueueName="test-queue",) + queue = sqs.create_queue(QueueName=str(uuid4()),) int(queue.attributes["MaximumMessageSize"]).should.equal(MAXIMUM_MESSAGE_LENGTH) with pytest.raises(Exception) as e: queue.send_message(MessageBody="a" * (MAXIMUM_MESSAGE_LENGTH + 1)) @@ -3080,7 +3109,7 @@ def test_maximum_message_size_attribute_fails_for_invalid_values(): for message_size in invalid_values: with pytest.raises(ClientError) as e: sqs.create_queue( - QueueName="test-queue", + QueueName=str(uuid4()), Attributes={"MaximumMessageSize": str(message_size)}, ) ex = e.value @@ -3092,7 +3121,7 @@ def test_send_message_fails_when_message_size_greater_than_max_message_size(): sqs = boto3.resource("sqs", region_name="eu-west-3") message_size_limit = 12345 queue = sqs.create_queue( - QueueName="test-queue", + QueueName=str(uuid4()), Attributes={"MaximumMessageSize": str(message_size_limit)}, ) int(queue.attributes["MaximumMessageSize"]).should.equal(message_size_limit) @@ -3120,8 +3149,9 @@ def test_fifo_queue_deduplication_with_id( ): sqs = boto3.resource("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] msg_queue = sqs.create_queue( - QueueName="test-queue-dlq.fifo", + QueueName=(f"{q_name}-dlq.fifo"), Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"}, ) @@ -3142,8 +3172,9 @@ def test_fifo_queue_deduplication_with_id( def test_fifo_queue_deduplication_withoutid(msg_1, msg_2, expected_count): sqs = boto3.resource("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] msg_queue = sqs.create_queue( - QueueName="test-queue-dlq.fifo", + QueueName=f"{q_name}-dlq.fifo", Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"}, ) @@ -3178,8 +3209,9 @@ def test_fifo_queue_send_duplicate_messages_after_deduplication_time_limit(): def test_fifo_queue_send_deduplicationid_same_as_sha256_of_old_message(): sqs = boto3.resource("sqs", region_name="us-east-1") + q_name = str(uuid4())[0:6] msg_queue = sqs.create_queue( - QueueName="test-queue-dlq.fifo", + QueueName=f"{q_name}-dlq.fifo", Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"}, ) @@ -3200,15 +3232,14 @@ def test_fifo_queue_send_deduplicationid_same_as_sha256_of_old_message(): def test_fifo_send_message_when_same_group_id_is_in_dlq(): sqs = boto3.resource("sqs", region_name="us-east-1") - dlq = sqs.create_queue( - QueueName="test-queue-dlq.fifo", Attributes={"FifoQueue": "true"} - ) + q_name = f"{str(uuid4())[0:6]}-dlq.fifo" + dlq = sqs.create_queue(QueueName=q_name, Attributes={"FifoQueue": "true"}) - queue = sqs.get_queue_by_name(QueueName="test-queue-dlq.fifo") + queue = sqs.get_queue_by_name(QueueName=q_name) dead_letter_queue_arn = queue.attributes.get("QueueArn") msg_queue = sqs.create_queue( - QueueName="test-queue.fifo", + QueueName=f"{str(uuid4())[0:6]}.fifo", Attributes={ "FifoQueue": "true", "RedrivePolicy": json.dumps( @@ -3239,7 +3270,7 @@ def test_fifo_send_message_when_same_group_id_is_in_dlq(): def test_receive_message_should_not_accept_invalid_urls(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - name = "test-queue" + name = str(uuid4())[0:6] q_response = conn.create_queue(QueueName=name) working_url = q_response[ "QueueUrl" @@ -3250,24 +3281,21 @@ def test_receive_message_should_not_accept_invalid_urls(): queue.send_message(MessageBody="this is a test message") err = e.value.response["Error"] err["Code"].should.equal("InvalidAddress") - err["Message"].should.equal( - "The address test-queue is not valid for this endpoint." - ) + err["Message"].should.equal(f"The address {name} is not valid for this endpoint.") with pytest.raises(ClientError) as e: conn.receive_message(QueueUrl=name) err = e.value.response["Error"] err["Code"].should.equal("InvalidAddress") - err["Message"].should.equal( - "The address test-queue is not valid for this endpoint." - ) + err["Message"].should.equal(f"The address {name} is not valid for this endpoint.") @mock_sqs def test_message_attributes_contains_trace_header(): sqs = boto3.resource("sqs", region_name="us-east-1") conn = boto3.client("sqs", region_name="us-east-1") - q_resp = conn.create_queue(QueueName="test-queue") + q_name = str(uuid4())[0:6] + q_resp = conn.create_queue(QueueName=q_name) queue = sqs.Queue(q_resp["QueueUrl"]) body_one = "this is a test message" diff --git a/tests/test_sqs/test_sqs_cloudformation.py b/tests/test_sqs/test_sqs_cloudformation.py index fd4af186d..8f9bc5cad 100644 --- a/tests/test_sqs/test_sqs_cloudformation.py +++ b/tests/test_sqs/test_sqs_cloudformation.py @@ -1,21 +1,25 @@ import boto3 + import json import sure # noqa - from moto import mock_sqs, mock_cloudformation from moto.core import ACCOUNT_ID +from string import Template +from random import randint +from uuid import uuid4 - -simple_queue = { +simple_queue = Template( + """{ "AWSTemplateFormatVersion": "2010-09-09", "Resources": { "QueueGroup": { "Type": "AWS::SQS::Queue", - "Properties": {"QueueName": "my-queue", "VisibilityTimeout": 60}, + "Properties": {"QueueName": $q_name, "VisibilityTimeout": 60}, } }, -} -simple_queue_json = json.dumps(simple_queue) +}""" +) + sqs_template_with_tags = """ { "AWSTemplateFormatVersion": "2010-09-09", @@ -46,16 +50,19 @@ def test_describe_stack_subresources(): cf = boto3.client("cloudformation", region_name="us-east-1") client = boto3.client("sqs", region_name="us-east-1") - cf.create_stack(StackName="test_sqs", TemplateBody=simple_queue_json) + stack_name = str(uuid4())[0:6] + q_name = str(uuid4())[0:6] + template_body = simple_queue.substitute(q_name=q_name) + cf.create_stack(StackName=stack_name, TemplateBody=template_body) - queue_url = client.list_queues()["QueueUrls"][0] - queue_url.should.contain("{}/{}".format(ACCOUNT_ID, "my-queue")) + queue_urls = client.list_queues()["QueueUrls"] + assert any(["{}/{}".format(ACCOUNT_ID, q_name) in url for url in queue_urls]) - stack = res.Stack("test_sqs") + stack = res.Stack(stack_name) for s in stack.resource_summaries.all(): s.resource_type.should.equal("AWS::SQS::Queue") s.logical_id.should.equal("QueueGroup") - s.physical_resource_id.should.equal("my-queue") + s.physical_resource_id.should.equal(q_name) @mock_sqs @@ -64,16 +71,19 @@ def test_list_stack_resources(): cf = boto3.client("cloudformation", region_name="us-east-1") client = boto3.client("sqs", region_name="us-east-1") - cf.create_stack(StackName="test_sqs", TemplateBody=simple_queue_json) + stack_name = str(uuid4())[0:6] + q_name = str(uuid4())[0:6] + template_body = simple_queue.substitute(q_name=q_name) + cf.create_stack(StackName=stack_name, TemplateBody=template_body) - queue_url = client.list_queues()["QueueUrls"][0] - queue_url.should.contain("{}/{}".format(ACCOUNT_ID, "my-queue")) + queue_urls = client.list_queues()["QueueUrls"] + assert any(["{}/{}".format(ACCOUNT_ID, q_name) in url for url in queue_urls]) - queue = cf.list_stack_resources(StackName="test_sqs")["StackResourceSummaries"][0] + queue = cf.list_stack_resources(StackName=stack_name)["StackResourceSummaries"][0] queue.should.have.key("ResourceType").equal("AWS::SQS::Queue") queue.should.have.key("LogicalResourceId").should.equal("QueueGroup") - queue.should.have.key("PhysicalResourceId").should.equal("my-queue") + queue.should.have.key("PhysicalResourceId").should.equal(q_name) @mock_sqs @@ -82,9 +92,14 @@ def test_create_from_cloudformation_json_with_tags(): cf = boto3.client("cloudformation", region_name="us-east-1") client = boto3.client("sqs", region_name="us-east-1") - cf.create_stack(StackName="test-sqs", TemplateBody=sqs_template_with_tags) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=sqs_template_with_tags) - queue_url = client.list_queues()["QueueUrls"][0] + response = cf.describe_stack_resources(StackName=stack_name) + q_name = response["StackResources"][0]["PhysicalResourceId"] + + all_urls = client.list_queues()["QueueUrls"] + queue_url = [url for url in all_urls if url.endswith(q_name)][0] queue_tags = client.list_queue_tags(QueueUrl=queue_url)["Tags"] queue_tags.should.equal({"keyname1": "value1", "keyname2": "value2"}) @@ -93,22 +108,24 @@ def test_create_from_cloudformation_json_with_tags(): @mock_cloudformation @mock_sqs def test_update_stack(): + q_name = str(uuid4())[0:6] sqs_template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { "QueueGroup": { "Type": "AWS::SQS::Queue", - "Properties": {"QueueName": "my-queue", "VisibilityTimeout": 60}, + "Properties": {"QueueName": q_name, "VisibilityTimeout": 60}, } }, } sqs_template_json = json.dumps(sqs_template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=sqs_template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=sqs_template_json) client = boto3.client("sqs", region_name="us-west-1") - queues = client.list_queues()["QueueUrls"] + queues = client.list_queues(QueueNamePrefix=q_name)["QueueUrls"] queues.should.have.length_of(1) attrs = client.get_queue_attributes(QueueUrl=queues[0], AttributeNames=["All"])[ "Attributes" @@ -118,10 +135,10 @@ def test_update_stack(): # when updating sqs_template["Resources"]["QueueGroup"]["Properties"]["VisibilityTimeout"] = 100 sqs_template_json = json.dumps(sqs_template) - cf.update_stack(StackName="test_stack", TemplateBody=sqs_template_json) + cf.update_stack(StackName=stack_name, TemplateBody=sqs_template_json) # then the attribute should be updated - queues = client.list_queues()["QueueUrls"] + queues = client.list_queues(QueueNamePrefix=q_name)["QueueUrls"] queues.should.have.length_of(1) attrs = client.get_queue_attributes(QueueUrl=queues[0], AttributeNames=["All"])[ "Attributes" @@ -132,30 +149,31 @@ def test_update_stack(): @mock_cloudformation @mock_sqs def test_update_stack_and_remove_resource(): + q_name = str(uuid4())[0:6] sqs_template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { "QueueGroup": { "Type": "AWS::SQS::Queue", - "Properties": {"QueueName": "my-queue", "VisibilityTimeout": 60}, + "Properties": {"QueueName": q_name, "VisibilityTimeout": 60}, } }, } sqs_template_json = json.dumps(sqs_template) cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=sqs_template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=sqs_template_json) client = boto3.client("sqs", region_name="us-west-1") - client.list_queues()["QueueUrls"].should.have.length_of(1) + client.list_queues(QueueNamePrefix=q_name)["QueueUrls"].should.have.length_of(1) sqs_template["Resources"].pop("QueueGroup") sqs_template_json = json.dumps(sqs_template) - cf.update_stack(StackName="test_stack", TemplateBody=sqs_template_json) + cf.update_stack(StackName=stack_name, TemplateBody=sqs_template_json) - client.list_queues().shouldnt.have.key( - "QueueUrls" - ) # No queues exist, so the key is not passed through + # No queues exist, so the key is not passed through + client.list_queues(QueueNamePrefix=q_name).shouldnt.have.key("QueueUrls") @mock_cloudformation @@ -164,22 +182,44 @@ def test_update_stack_and_add_resource(): sqs_template = {"AWSTemplateFormatVersion": "2010-09-09", "Resources": {}} sqs_template_json = json.dumps(sqs_template) + q_name = str(uuid4())[0:6] + cf = boto3.client("cloudformation", region_name="us-west-1") - cf.create_stack(StackName="test_stack", TemplateBody=sqs_template_json) + stack_name = str(uuid4())[0:6] + cf.create_stack(StackName=stack_name, TemplateBody=sqs_template_json) client = boto3.client("sqs", region_name="us-west-1") - client.list_queues().shouldnt.have.key("QueueUrls") + client.list_queues(QueueNamePrefix=q_name).shouldnt.have.key("QueueUrls") sqs_template = { "AWSTemplateFormatVersion": "2010-09-09", "Resources": { "QueueGroup": { "Type": "AWS::SQS::Queue", - "Properties": {"QueueName": "my-queue", "VisibilityTimeout": 60}, + "Properties": {"QueueName": q_name, "VisibilityTimeout": 60}, } }, } sqs_template_json = json.dumps(sqs_template) - cf.update_stack(StackName="test_stack", TemplateBody=sqs_template_json) + cf.update_stack(StackName=stack_name, TemplateBody=sqs_template_json) - client.list_queues()["QueueUrls"].should.have.length_of(1) + client.list_queues(QueueNamePrefix=q_name)["QueueUrls"].should.have.length_of(1) + + +@mock_sqs +@mock_cloudformation +def test_create_queue_passing_integer_as_name(): + """ + Passing the queue name as an Integer in the CloudFormation template + This works in AWS - it simply converts the integer to a string, and treats it as any other name + """ + cf = boto3.client("cloudformation", region_name="us-east-1") + client = boto3.client("sqs", region_name="us-east-1") + + stack_name = str(uuid4())[0:6] + q_name = f"{randint(10000000, 99999999)}" + template_body = simple_queue.substitute(q_name=q_name) + cf.create_stack(StackName=stack_name, TemplateBody=template_body) + + queue_urls = client.list_queues(QueueNamePrefix=q_name[0:6])["QueueUrls"] + queue_urls.should.have.length_of(1) diff --git a/travis_moto_server.sh b/travis_moto_server.sh index a3233a294..6f6d29a21 100755 --- a/travis_moto_server.sh +++ b/travis_moto_server.sh @@ -4,4 +4,4 @@ export MOTO_PORT=${MOTO_PORT:-5000} set -e pip install $(ls /moto/dist/moto*.gz)[server,all] -moto_server -H 0.0.0.0 -p ${MOTO_PORT} +moto_server -H 0.0.0.0 -p ${MOTO_PORT} > /moto/server_output.log 2>&1