Remove microsleep (#1903)

Use `threading.Condition` instead of `sleep()` to prevent high cpu usage while SQS long polling on an empty queue. 

Ref: #871
Ref: #1916

Co-authored-by: Brian Pandola <bpandola@gmail.com>
This commit is contained in:
David Laban 2022-11-05 13:09:58 +00:00 committed by GitHub
parent 37a127acd5
commit d560ff002d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 5 deletions

View File

@ -7,6 +7,7 @@ import string
import struct import struct
from copy import deepcopy from copy import deepcopy
from typing import Dict from typing import Dict
from threading import Condition
from xml.sax.saxutils import escape from xml.sax.saxutils import escape
from moto.core.exceptions import RESTError from moto.core.exceptions import RESTError
@ -257,6 +258,7 @@ class Queue(CloudFormationModel):
self._messages = [] self._messages = []
self._pending_messages = set() self._pending_messages = set()
self.deleted_messages = set() self.deleted_messages = set()
self._messages_lock = Condition()
now = unix_time() now = unix_time()
self.created_timestamp = now self.created_timestamp = now
@ -536,7 +538,9 @@ class Queue(CloudFormationModel):
if diff / 1000 < DEDUPLICATION_TIME_IN_SECONDS: if diff / 1000 < DEDUPLICATION_TIME_IN_SECONDS:
return return
self._messages.append(message) with self._messages_lock:
self._messages.append(message)
self._messages_lock.notify_all()
for arn, esm in self.lambda_event_source_mappings.items(): for arn, esm in self.lambda_event_source_mappings.items():
backend = sqs_backends[self.account_id][self.region] backend = sqs_backends[self.account_id][self.region]
@ -591,6 +595,10 @@ class Queue(CloudFormationModel):
new_messages.append(message) new_messages.append(message)
self._messages = new_messages self._messages = new_messages
def wait_for_messages(self, timeout):
with self._messages_lock:
self._messages_lock.wait_for(lambda: self.messages, timeout=timeout)
@classmethod @classmethod
def has_cfn_attr(cls, attr): def has_cfn_attr(cls, attr):
return attr in ["Arn", "QueueName"] return attr in ["Arn", "QueueName"]
@ -930,13 +938,11 @@ class SQSBackend(BaseBackend):
if previous_result_count == len(result): if previous_result_count == len(result):
if wait_seconds_timeout == 0: if wait_seconds_timeout == 0:
# There is timeout and we have added no additional results, # There is no timeout and no additional results,
# so break to avoid an infinite loop. # so break to avoid an infinite loop.
break break
import time queue.wait_for_messages(wait_seconds_timeout)
time.sleep(0.01)
continue continue
previous_result_count = len(result) previous_result_count = len(result)

View File

@ -1,3 +1,4 @@
import datetime
import re import re
import sure # noqa # pylint: disable=unused-import import sure # noqa # pylint: disable=unused-import
import threading import threading
@ -80,3 +81,19 @@ def test_messages_polling():
# got each message in a separate call to ReceiveMessage, despite the long # got each message in a separate call to ReceiveMessage, despite the long
# WaitTimeSeconds # WaitTimeSeconds
assert len(messages) == 5 assert len(messages) == 5
def test_no_messages_polling_timeout():
backend = server.create_backend_app("sqs")
queue_name = "test-queue"
test_client = backend.test_client()
test_client.put(f"/?Action=CreateQueue&QueueName={queue_name}")
wait_seconds = 5
start = datetime.datetime.utcnow()
test_client.get(
f"/123/{queue_name}?Action=ReceiveMessage&MaxNumberOfMessages=1&WaitTimeSeconds={wait_seconds}"
)
end = datetime.datetime.utcnow()
duration = end - start
assert duration.seconds >= wait_seconds
assert duration.seconds <= wait_seconds + (wait_seconds / 2)