diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 286574553..ecb25030a 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1134,3 +1134,39 @@ def test_multiple_qualifiers(): fn["FunctionArn"].should.equal( f"arn:aws:lambda:us-east-1:{ACCOUNT_ID}:function:{fn_name}:6" ) + + +def test_get_role_name_utility_race_condition(): + # Play with these variables as needed to reproduce the error. + max_workers, num_threads = 3, 15 + + errors = [] + roles = [] + + def thread_function(_): + while True: + # noinspection PyBroadException + try: + role = get_role_name() + except ClientError as e: + errors.append(str(e)) + break + except Exception: + # boto3 and our own IAMBackend are not thread-safe, + # and occasionally throw weird errors, so we just + # pass and retry. + # https://github.com/boto/boto3/issues/1592 + pass + else: + roles.append(role) + break + + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + executor.map(thread_function, range(num_threads)) + # Check all threads are accounted for, all roles are the same entity, + # and there are no client errors. + assert len(errors) + len(roles) == num_threads + assert roles.count(roles[0]) == len(roles) + assert len(errors) == 0 diff --git a/tests/test_awslambda/utilities.py b/tests/test_awslambda/utilities.py index 124af4805..8c0f35764 100644 --- a/tests/test_awslambda/utilities.py +++ b/tests/test_awslambda/utilities.py @@ -166,14 +166,18 @@ def create_invalid_lambda(role): def get_role_name(): with mock_iam(): iam = boto3.client("iam", region_name=_lambda_region) - try: - return iam.get_role(RoleName="my-role")["Role"]["Arn"] - except ClientError: - return iam.create_role( - RoleName="my-role", - AssumeRolePolicyDocument="some policy", - Path="/my-path/", - )["Role"]["Arn"] + while True: + try: + return iam.get_role(RoleName="my-role")["Role"]["Arn"] + except ClientError: + try: + return iam.create_role( + RoleName="my-role", + AssumeRolePolicyDocument="some policy", + Path="/my-path/", + )["Role"]["Arn"] + except ClientError: + pass def wait_for_log_msg(expected_msg, log_group, wait_time=30):