diff --git a/grub-core/commands/minicmd.c b/grub-core/commands/minicmd.c
index 6bbce3128..fa498931e 100644
--- a/grub-core/commands/minicmd.c
+++ b/grub-core/commands/minicmd.c
@@ -140,8 +140,11 @@ grub_mini_cmd_rmmod (struct grub_command *cmd __attribute__ ((unused)),
   if (grub_dl_is_persistent (mod))
     return grub_error (GRUB_ERR_BAD_ARGUMENT, "cannot unload persistent module");
 
-  if (grub_dl_unref (mod) <= 0)
-    grub_dl_unload (mod);
+  if (grub_dl_ref_count (mod) > 1)
+    return grub_error (GRUB_ERR_BAD_ARGUMENT, "cannot unload referenced module");
+
+  grub_dl_unref (mod);
+  grub_dl_unload (mod);
 
   return 0;
 }
diff --git a/grub-core/kern/dl.c b/grub-core/kern/dl.c
index 48eb5e7b6..48f8a7907 100644
--- a/grub-core/kern/dl.c
+++ b/grub-core/kern/dl.c
@@ -549,6 +549,15 @@ grub_dl_unref (grub_dl_t mod)
   return --mod->ref_count;
 }
 
+int
+grub_dl_ref_count (grub_dl_t mod)
+{
+  if (mod == NULL)
+    return 0;
+
+  return mod->ref_count;
+}
+
 static void
 grub_dl_flush_cache (grub_dl_t mod)
 {
diff --git a/include/grub/dl.h b/include/grub/dl.h
index f03c03561..b3753c9ca 100644
--- a/include/grub/dl.h
+++ b/include/grub/dl.h
@@ -203,9 +203,11 @@ grub_dl_t EXPORT_FUNC(grub_dl_load) (const char *name);
 grub_dl_t grub_dl_load_core (void *addr, grub_size_t size);
 grub_dl_t EXPORT_FUNC(grub_dl_load_core_noinit) (void *addr, grub_size_t size);
 int EXPORT_FUNC(grub_dl_unload) (grub_dl_t mod);
-void grub_dl_unload_unneeded (void);
-int EXPORT_FUNC(grub_dl_ref) (grub_dl_t mod);
-int EXPORT_FUNC(grub_dl_unref) (grub_dl_t mod);
+extern void grub_dl_unload_unneeded (void);
+extern int EXPORT_FUNC(grub_dl_ref) (grub_dl_t mod);
+extern int EXPORT_FUNC(grub_dl_unref) (grub_dl_t mod);
+extern int EXPORT_FUNC(grub_dl_ref_count) (grub_dl_t mod);
+
 extern grub_dl_t EXPORT_VAR(grub_dl_head);
 
 #ifndef GRUB_UTIL