diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py
index 44ed64611..68c4abaae 100644
--- a/moto/sqs/exceptions.py
+++ b/moto/sqs/exceptions.py
@@ -7,9 +7,14 @@ class MessageNotInflight(Exception):
     status_code = 400
 
 
-class ReceiptHandleIsInvalid(Exception):
-    description = "The receipt handle provided is not valid."
-    status_code = 400
+class ReceiptHandleIsInvalid(RESTError):
+    code = 400
+
+    def __init__(self):
+        super(ReceiptHandleIsInvalid, self).__init__(
+            'ReceiptHandleIsInvalid',
+            'The input receipt handle is invalid.'
+        )
 
 
 class MessageAttributesInvalid(Exception):
diff --git a/moto/sqs/models.py b/moto/sqs/models.py
index 0df5fb7c4..8d02fe529 100644
--- a/moto/sqs/models.py
+++ b/moto/sqs/models.py
@@ -669,6 +669,10 @@ class SQSBackend(BaseBackend):
 
     def delete_message(self, queue_name, receipt_handle):
         queue = self.get_queue(queue_name)
+
+        if not any(message.receipt_handle == receipt_handle for message in queue._messages):
+            raise ReceiptHandleIsInvalid()
+
         new_messages = []
         for message in queue._messages:
             # Only delete message if it is not visible and the reciept_handle
diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py
index 3d5ee1fea..e6982b795 100644
--- a/moto/sqs/responses.py
+++ b/moto/sqs/responses.py
@@ -118,7 +118,7 @@ class SQSResponse(BaseResponse):
                 receipt_handle=receipt_handle,
                 visibility_timeout=visibility_timeout
             )
-        except (ReceiptHandleIsInvalid, MessageNotInflight) as e:
+        except MessageNotInflight as e:
             return "Invalid request: {0}".format(e.description), dict(status=e.status_code)
 
         template = self.response_template(CHANGE_MESSAGE_VISIBILITY_RESPONSE)
diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py
index 42a540d5b..2db457f72 100644
--- a/tests/test_sqs/test_sqs.py
+++ b/tests/test_sqs/test_sqs.py
@@ -999,6 +999,36 @@ def test_delete_message_after_visibility_timeout():
     assert new_queue.count() == 0
 
 
+@mock_sqs
+def test_delete_message_errors():
+    client = boto3.client('sqs', region_name='us-east-1')
+    response = client.create_queue(QueueName='test-queue')
+    queue_url = response['QueueUrl']
+    client.send_message(
+        QueueUrl=queue_url,
+        MessageBody='body'
+    )
+    response = client.receive_message(
+        QueueUrl=queue_url
+    )
+    receipt_handle = response['Messages'][0]['ReceiptHandle']
+
+    client.delete_message.when.called_with(
+        QueueUrl=queue_url + '-not-existing',
+        ReceiptHandle=receipt_handle
+    ).should.throw(
+        ClientError,
+        'The specified queue does not exist for this wsdl version.'
+    )
+
+    client.delete_message.when.called_with(
+        QueueUrl=queue_url,
+        ReceiptHandle='not-existing'
+    ).should.throw(
+        ClientError,
+        'The input receipt handle is invalid.'
+    )
+
 @mock_sqs
 def test_send_message_batch():
     client = boto3.client('sqs', region_name='us-east-1')