Fix Race Condition in Parallel AWS Lambda Server Mode Tests (#5643)

tests/test_awslambda/test_lambda.py

@@ -1134,3 +1134,39 @@ def test_multiple_qualifiers():
     fn["FunctionArn"].should.equal(
         f"arn:aws:lambda:us-east-1:{ACCOUNT_ID}:function:{fn_name}:6"
     )
+
+
+def test_get_role_name_utility_race_condition():
+    # Play with these variables as needed to reproduce the error.
+    max_workers, num_threads = 3, 15
+
+    errors = []
+    roles = []
+
+    def thread_function(_):
+        while True:
+            # noinspection PyBroadException
+            try:
+                role = get_role_name()
+            except ClientError as e:
+                errors.append(str(e))
+                break
+            except Exception:
+                # boto3 and our own IAMBackend are not thread-safe,
+                # and occasionally throw weird errors, so we just
+                # pass and retry.
+                # https://github.com/boto/boto3/issues/1592
+                pass
+            else:
+                roles.append(role)
+                break
+
+    import concurrent.futures
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        executor.map(thread_function, range(num_threads))
+    # Check all threads are accounted for, all roles are the same entity,
+    # and there are no client errors.
+    assert len(errors) + len(roles) == num_threads
+    assert roles.count(roles[0]) == len(roles)
+    assert len(errors) == 0

tests/test_awslambda/utilities.py

@@ -166,14 +166,18 @@ def create_invalid_lambda(role):
 def get_role_name():
     with mock_iam():
         iam = boto3.client("iam", region_name=_lambda_region)
-        try:
-            return iam.get_role(RoleName="my-role")["Role"]["Arn"]
-        except ClientError:
-            return iam.create_role(
-                RoleName="my-role",
-                AssumeRolePolicyDocument="some policy",
-                Path="/my-path/",
-            )["Role"]["Arn"]
+        while True:
+            try:
+                return iam.get_role(RoleName="my-role")["Role"]["Arn"]
+            except ClientError:
+                try:
+                    return iam.create_role(
+                        RoleName="my-role",
+                        AssumeRolePolicyDocument="some policy",
+                        Path="/my-path/",
+                    )["Role"]["Arn"]
+                except ClientError:
+                    pass
 
 
 def wait_for_log_msg(expected_msg, log_group, wait_time=30):