269 lines
9.4 KiB
Python
269 lines
9.4 KiB
Python
import random
|
|
import time
|
|
import pytest
|
|
|
|
from moto.core import BaseBackend, DEFAULT_ACCOUNT_ID
|
|
from moto.core.utils import AccountSpecificBackend, BackendDict
|
|
|
|
from moto.autoscaling.models import AutoScalingBackend
|
|
from moto.ec2.models import EC2Backend
|
|
from moto.elbv2.models import ELBv2Backend
|
|
|
|
from threading import Thread
|
|
|
|
|
|
class ExampleBackend(BaseBackend):
|
|
def __init__(self, region_name, account_id):
|
|
super().__init__(region_name, account_id)
|
|
|
|
|
|
def test_backend_dict_returns_nothing_by_default():
|
|
backend_dict = BackendDict(ExampleBackend, "ebs")
|
|
list(backend_dict.items()).should.equal([])
|
|
|
|
|
|
def test_account_specific_dict_contains_known_regions():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2")
|
|
backend_dict["account"].should.have.key("eu-north-1")
|
|
backend_dict["account"]["eu-north-1"].should.be.a(ExampleBackend)
|
|
|
|
|
|
def test_backend_dict_known_regions_can_be_retrieved_directly():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2")
|
|
backend_dict["account"]["eu-west-1"].should.be.a(ExampleBackend)
|
|
|
|
|
|
def test_backend_dict_can_get_known_regions():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2")["12345"]
|
|
backend_dict["us-east-1"].should.be.a(ExampleBackend)
|
|
|
|
|
|
def test_backend_dict_does_not_contain_unknown_regions():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2")
|
|
backend_dict["account"].shouldnt.have.key("mars-south-1")
|
|
|
|
|
|
def test_backend_dict_fails_when_retrieving_unknown_regions():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2")
|
|
with pytest.raises(KeyError):
|
|
backend_dict["account"]["mars-south-1"] # pylint: disable=pointless-statement
|
|
|
|
|
|
def test_backend_dict_can_retrieve_for_specific_account():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2")
|
|
|
|
# Random account does not exist
|
|
backend_dict.shouldnt.have.key("000000")
|
|
|
|
# Retrieve AccountSpecificBackend by assuming it exists
|
|
backend = backend_dict["012345"]
|
|
backend.should.be.a(AccountSpecificBackend)
|
|
|
|
backend.should.have.key("eu-north-1")
|
|
regional_backend = backend["eu-north-1"]
|
|
regional_backend.should.be.a(ExampleBackend)
|
|
regional_backend.region_name.should.equal("eu-north-1")
|
|
# We always return a fixed account_id for now, until we have proper multi-account support
|
|
regional_backend.account_id.should.equal("012345")
|
|
|
|
|
|
def test_backend_dict_can_ignore_boto3_regions():
|
|
backend_dict = BackendDict(ExampleBackend, "ec2", use_boto3_regions=False)
|
|
backend_dict["account"].get("us-east-1").should.equal(None)
|
|
|
|
|
|
def test_backend_dict_can_specify_additional_regions():
|
|
backend_dict = BackendDict(
|
|
ExampleBackend, "ec2", additional_regions=["region1", "global"]
|
|
)["123456"]
|
|
backend_dict["us-east-1"].should.be.a(ExampleBackend)
|
|
backend_dict["region1"].should.be.a(ExampleBackend)
|
|
backend_dict["global"].should.be.a(ExampleBackend)
|
|
|
|
# Unknown regions still do not exist
|
|
backend_dict.get("us-east-3").should.equal(None)
|
|
|
|
|
|
class TestMultiThreadedAccess:
|
|
class SlowExampleBackend(BaseBackend):
|
|
def __init__(self, region_name, account_id):
|
|
super().__init__(region_name, account_id)
|
|
time.sleep(0.1)
|
|
self.data = []
|
|
|
|
def setup(self):
|
|
self.backend = BackendDict(TestMultiThreadedAccess.SlowExampleBackend, "ec2")
|
|
|
|
def test_access_a_slow_backend_concurrently(self):
|
|
"""None
|
|
Usecase that we want to avoid:
|
|
|
|
Thread 1 comes in, and sees the backend does not exist for this region
|
|
Thread 1 starts creating the backend
|
|
Thread 2 comes in, and sees the backend does not exist for this region
|
|
Thread 2 starts creating the backend
|
|
Thread 1 finishes creating the backend, initializes the list and adds a new value to it
|
|
Thread 2 finishes creating the backend, re-initializes the list and adds a new value to it
|
|
|
|
Creating the Backend for a region should only be invoked once at a time, and the execution flow should look like:
|
|
|
|
Thread 1 comes in, and sees the backend does not exist for this region
|
|
Thread 1 starts creating the backend
|
|
Thread 2 comes in and is blocked
|
|
Thread 1 finishes creating the backend, initializes the list and adds a new value to it
|
|
Thread 2 gains access, and re-uses the created backend
|
|
Thread 2 adds a new value to the list
|
|
"""
|
|
|
|
def access(random_number):
|
|
self.backend["123456789012"]["us-east-1"].data.append(random_number)
|
|
|
|
threads = []
|
|
|
|
for _ in range(0, 15):
|
|
x = Thread(target=access, args=(random.randint(100, 200),))
|
|
x.start()
|
|
threads.append(x)
|
|
|
|
for x in threads:
|
|
x.join()
|
|
|
|
self.backend["123456789012"]["us-east-1"].data.should.have.length_of(15)
|
|
|
|
|
|
def test_backend_dict_can_be_hashed():
|
|
hashes = []
|
|
for backend in [ExampleBackend, set, list, BaseBackend]:
|
|
hashes.append(BackendDict(backend, "n/a").__hash__())
|
|
# Hash is different for different backends
|
|
set(hashes).should.have.length_of(4)
|
|
|
|
|
|
def test_account_specific_dict_can_be_hashed():
|
|
hashes = []
|
|
ids = ["01234567912", "01234567911", "01234567913", "000000000000", "0"]
|
|
for accnt_id in ids:
|
|
asb = _create_asb(accnt_id)
|
|
hashes.append(asb.__hash__())
|
|
# Hash is different for different accounts
|
|
set(hashes).should.have.length_of(5)
|
|
|
|
|
|
def _create_asb(account_id, backend=None, use_boto3_regions=False, regions=None):
|
|
return AccountSpecificBackend(
|
|
service_name="ec2",
|
|
account_id=account_id,
|
|
backend=backend or ExampleBackend,
|
|
use_boto3_regions=use_boto3_regions,
|
|
additional_regions=regions,
|
|
)
|
|
|
|
|
|
def test_multiple_backends_cache_behaviour():
|
|
|
|
ec2 = BackendDict(EC2Backend, "ec2")
|
|
ec2_useast1 = ec2[DEFAULT_ACCOUNT_ID]["us-east-1"]
|
|
assert type(ec2_useast1) == EC2Backend
|
|
|
|
autoscaling = BackendDict(AutoScalingBackend, "autoscaling")
|
|
as_1 = autoscaling[DEFAULT_ACCOUNT_ID]["us-east-1"]
|
|
assert type(as_1) == AutoScalingBackend
|
|
|
|
from moto.elbv2 import elbv2_backends
|
|
|
|
elbv2_useast = elbv2_backends["00000000"]["us-east-1"]
|
|
assert type(elbv2_useast) == ELBv2Backend
|
|
elbv2_useast2 = elbv2_backends[DEFAULT_ACCOUNT_ID]["us-east-2"]
|
|
assert type(elbv2_useast2) == ELBv2Backend
|
|
|
|
ec2_useast1 = ec2[DEFAULT_ACCOUNT_ID]["us-east-1"]
|
|
assert type(ec2_useast1) == EC2Backend
|
|
ec2_useast2 = ec2[DEFAULT_ACCOUNT_ID]["us-east-2"]
|
|
assert type(ec2_useast2) == EC2Backend
|
|
|
|
as_1 = autoscaling[DEFAULT_ACCOUNT_ID]["us-east-1"]
|
|
assert type(as_1) == AutoScalingBackend
|
|
|
|
|
|
def test_backenddict_cache_hits_and_misses():
|
|
backend = BackendDict(ExampleBackend, "ebs")
|
|
backend.__getitem__.cache_clear()
|
|
|
|
assert backend.__getitem__.cache_info().hits == 0
|
|
assert backend.__getitem__.cache_info().misses == 0
|
|
assert backend.__getitem__.cache_info().currsize == 0
|
|
|
|
# Create + Retrieve an account - verify it is stored in cache
|
|
accnt_1 = backend["accnt1"]
|
|
assert accnt_1.account_id == "accnt1"
|
|
|
|
assert backend.__getitem__.cache_info().hits == 0
|
|
assert backend.__getitem__.cache_info().misses == 1
|
|
assert backend.__getitem__.cache_info().currsize == 1
|
|
|
|
# Creating + Retrieving a second account
|
|
accnt_2 = backend["accnt2"]
|
|
assert accnt_2.account_id == "accnt2"
|
|
|
|
assert backend.__getitem__.cache_info().hits == 0
|
|
assert backend.__getitem__.cache_info().misses == 2
|
|
assert backend.__getitem__.cache_info().currsize == 2
|
|
|
|
# Retrieving the first account from cache
|
|
accnt_1_again = backend["accnt1"]
|
|
assert accnt_1_again.account_id == "accnt1"
|
|
|
|
assert backend.__getitem__.cache_info().hits == 1
|
|
assert backend.__getitem__.cache_info().misses == 2
|
|
assert backend.__getitem__.cache_info().currsize == 2
|
|
|
|
# Retrieving the second account from cache
|
|
accnt_2_again = backend["accnt2"]
|
|
assert accnt_2_again.account_id == "accnt2"
|
|
|
|
assert backend.__getitem__.cache_info().hits == 2
|
|
assert backend.__getitem__.cache_info().misses == 2
|
|
assert backend.__getitem__.cache_info().currsize == 2
|
|
|
|
|
|
def test_asb_cache_hits_and_misses():
|
|
backend = BackendDict(ExampleBackend, "ebs")
|
|
acb = backend["accnt_id"]
|
|
acb.__getitem__.cache_clear()
|
|
|
|
assert acb.__getitem__.cache_info().hits == 0
|
|
assert acb.__getitem__.cache_info().misses == 0
|
|
assert acb.__getitem__.cache_info().currsize == 0
|
|
|
|
# Create + Retrieve an account - verify it is stored in cache
|
|
region_1 = acb["us-east-1"]
|
|
assert region_1.region_name == "us-east-1"
|
|
|
|
assert acb.__getitem__.cache_info().hits == 0
|
|
assert acb.__getitem__.cache_info().misses == 1
|
|
assert acb.__getitem__.cache_info().currsize == 1
|
|
|
|
# Creating + Retrieving a second account
|
|
region_2 = acb["us-east-2"]
|
|
assert region_2.region_name == "us-east-2"
|
|
|
|
assert acb.__getitem__.cache_info().hits == 0
|
|
assert acb.__getitem__.cache_info().misses == 2
|
|
assert acb.__getitem__.cache_info().currsize == 2
|
|
|
|
# Retrieving the first account from cache
|
|
region_1_again = acb["us-east-1"]
|
|
assert region_1_again.region_name == "us-east-1"
|
|
|
|
assert acb.__getitem__.cache_info().hits == 1
|
|
assert acb.__getitem__.cache_info().misses == 2
|
|
assert acb.__getitem__.cache_info().currsize == 2
|
|
|
|
# Retrieving the second account from cache
|
|
region_2_again = acb["us-east-2"]
|
|
assert region_2_again.region_name == "us-east-2"
|
|
|
|
assert acb.__getitem__.cache_info().hits == 2
|
|
assert acb.__getitem__.cache_info().misses == 2
|
|
assert acb.__getitem__.cache_info().currsize == 2
|