diff --git a/moto/ssm/models.py b/moto/ssm/models.py
index 60c47f021..a7518d405 100644
--- a/moto/ssm/models.py
+++ b/moto/ssm/models.py
@@ -278,10 +278,7 @@ class SimpleSystemManagerBackend(BaseBackend):
                 self._region = region
 
     def delete_parameter(self, name):
-        try:
-            del self._parameters[name]
-        except KeyError:
-            pass
+        return self._parameters.pop(name, None)
 
     def delete_parameters(self, names):
         result = []
diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py
index 1b13780a8..831737848 100644
--- a/moto/ssm/responses.py
+++ b/moto/ssm/responses.py
@@ -22,7 +22,13 @@ class SimpleSystemManagerResponse(BaseResponse):
 
     def delete_parameter(self):
         name = self._get_param("Name")
-        self.ssm_backend.delete_parameter(name)
+        result = self.ssm_backend.delete_parameter(name)
+        if result is None:
+            error = {
+                "__type": "ParameterNotFound",
+                "message": "Parameter {0} not found.".format(name),
+            }
+            return json.dumps(error), dict(status=400)
         return json.dumps({})
 
     def delete_parameters(self):
diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py
index 5b978520d..bb674fb65 100644
--- a/tests/test_ssm/test_ssm_boto3.py
+++ b/tests/test_ssm/test_ssm_boto3.py
@@ -30,6 +30,18 @@ def test_delete_parameter():
     len(response["Parameters"]).should.equal(0)
 
 
+@mock_ssm
+def test_delete_nonexistent_parameter():
+    client = boto3.client("ssm", region_name="us-east-1")
+
+    with assert_raises(ClientError) as ex:
+        client.delete_parameter(Name="test_noexist")
+    ex.exception.response["Error"]["Code"].should.equal("ParameterNotFound")
+    ex.exception.response["Error"]["Message"].should.equal(
+        "Parameter test_noexist not found."
+    )
+
+
 @mock_ssm
 def test_delete_parameters():
     client = boto3.client("ssm", region_name="us-east-1")