diff --git a/moto/kms/models.py b/moto/kms/models.py
index be8c52162..ca27f030a 100644
--- a/moto/kms/models.py
+++ b/moto/kms/models.py
@@ -103,8 +103,10 @@ class KmsBackend(BaseBackend):
         self.key_to_aliases[target_key_id].add(alias_name)
 
     def delete_alias(self, alias_name):
+        """Delete the alias."""
         for aliases in self.key_to_aliases.values():
-            aliases.remove(alias_name)
+            if alias_name in aliases:
+                aliases.remove(alias_name)
 
     def get_all_aliases(self):
         return self.key_to_aliases
diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py
index 8d034c7ff..96715de71 100644
--- a/tests/test_kms/test_kms.py
+++ b/tests/test_kms/test_kms.py
@@ -491,7 +491,14 @@ def test__delete_alias():
     key_id = create_resp['KeyMetadata']['KeyId']
     alias = 'alias/my-alias'
 
+    # added another alias here to make sure that the deletion of the alias can
+    # be done when there are multiple existing aliases.
+    another_create_resp = kms.create_key()
+    another_key_id = create_resp['KeyMetadata']['KeyId']
+    another_alias = 'alias/another-alias'
+
     kms.create_alias(alias, key_id)
+    kms.create_alias(another_alias, another_key_id)
 
     resp = kms.delete_alias(alias)