diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py
index 29804256c..5cc77e9fb 100644
--- a/moto/sqs/responses.py
+++ b/moto/sqs/responses.py
@@ -70,7 +70,10 @@ class SQSResponse(BaseResponse):
     def call_action(self):
         status_code, headers, body = super(SQSResponse, self).call_action()
         if status_code == 404:
-            return 404, headers, ERROR_INEXISTENT_QUEUE
+            queue_name = self.querystring.get("QueueName", [""])[0]
+            template = self.response_template(ERROR_INEXISTENT_QUEUE)
+            response = template.render(queue_name=queue_name)
+            return 404, headers, response
         return status_code, headers, body
 
     def _error(self, code, message, status=400):
@@ -718,7 +721,11 @@ ERROR_INEXISTENT_QUEUE = """<ErrorResponse xmlns="http://queue.amazonaws.com/doc
     <Error>
         <Type>Sender</Type>
         <Code>AWS.SimpleQueueService.NonExistentQueue</Code>
-        <Message>The specified queue does not exist for this wsdl version.</Message>
+         {% if queue_name %}
+            <Message>The specified queue {{queue_name}} does not exist for this wsdl version.</Message>
+        {% else %}
+            <Message>The specified queue does not exist for this wsdl version.</Message>
+        {% endif %}
         <Detail/>
     </Error>
     <RequestId>b8bc806b-fa6b-53b5-8be8-cfa2f9836bc3</RequestId>
diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py
index 4de5b9018..945fe86ae 100644
--- a/tests/test_sqs/test_sqs.py
+++ b/tests/test_sqs/test_sqs.py
@@ -198,7 +198,8 @@ def test_get_queue_url_errors():
     client = boto3.client("sqs", region_name="us-east-1")
 
     client.get_queue_url.when.called_with(QueueName="non-existing-queue").should.throw(
-        ClientError, "The specified queue does not exist for this wsdl version."
+        ClientError,
+        "The specified queue non-existing-queue does not exist for this wsdl version.",
     )
 
 
@@ -206,10 +207,13 @@ def test_get_queue_url_errors():
 def test_get_nonexistent_queue():
     sqs = boto3.resource("sqs", region_name="us-east-1")
     with assert_raises(ClientError) as err:
-        sqs.get_queue_by_name(QueueName="nonexisting-queue")
+        sqs.get_queue_by_name(QueueName="non-existing-queue")
     ex = err.exception
     ex.operation_name.should.equal("GetQueueUrl")
     ex.response["Error"]["Code"].should.equal("AWS.SimpleQueueService.NonExistentQueue")
+    ex.response["Error"]["Message"].should.equal(
+        "The specified queue non-existing-queue does not exist for this wsdl version."
+    )
 
     with assert_raises(ClientError) as err:
         sqs.Queue("http://whatever-incorrect-queue-address").load()