datastore/api: add error message on failed removal due to old locking

group or namespace removal can fail if the old locking mechanism is
still in use, as it is unsafe to properly clean up in that scenario.
return an error message that explains how to rectify that situation.

Signed-off-by: Shannon Sterz <s.sterz@proxmox.com>
 [TL: address simple merge conflict and fine tune message to admins]
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>

pbs-datastore/src/backup_info.rs

@@ -27,7 +27,7 @@ pub const DATASTORE_LOCKS_DIR: &str = "/run/proxmox-backup/locks";
 // of the file. this should only happen if a user messes with the `/run/proxmox-backup` directory.
 // if that happens, a lot more should fail as we rely on the existence of the directory throughout
 // the code. so just panic with a reasonable message.
-static OLD_LOCKING: LazyLock<bool> = LazyLock::new(|| {
+pub(crate) static OLD_LOCKING: LazyLock<bool> = LazyLock::new(|| {
     std::fs::exists("/run/proxmox-backup/old-locking")
         .expect("cannot read `/run/proxmox-backup`, please check permissions")
 });

pbs-datastore/src/datastore.rs

@@ -26,7 +26,7 @@ use pbs_api_types::{
 };
 use pbs_config::BackupLockGuard;
 
-use crate::backup_info::{BackupDir, BackupGroup, BackupInfo};
+use crate::backup_info::{BackupDir, BackupGroup, BackupInfo, OLD_LOCKING};
 use crate::chunk_store::ChunkStore;
 use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
 use crate::fixed_index::{FixedIndexReader, FixedIndexWriter};
@@ -1631,4 +1631,8 @@ impl DataStore {
 
         Ok(())
     }
+
+    pub fn old_locking(&self) -> bool {
+        *OLD_LOCKING
+    }
 }

src/api2/admin/datastore.rs

@@ -313,13 +313,23 @@ pub async fn delete_group(
         )?;
 
         let delete_stats = datastore.remove_backup_group(&ns, &group)?;
-        if !delete_stats.all_removed() {
+
-            if error_on_protected {
+        let error_msg = if datastore.old_locking() {
-                bail!("group only partially deleted due to protected snapshots");
+            "could not remove empty groups directories due to old locking mechanism.\n\
+            If you are an admin, please reboot PBS or ensure no old backup job is running anymore, \
+            then remove the file '/run/proxmox-backup/old-locking', and reload all PBS daemons"
+        } else if !delete_stats.all_removed() {
+            "group only partially deleted due to protected snapshots"
         } else {
-                warn!("group only partially deleted due to protected snapshots");
+            return Ok(delete_stats);
-            }
+        };
+
+        if error_on_protected {
+            bail!(error_msg);
+        } else {
+            warn!(error_msg);
         }
+
         Ok(delete_stats)
     })
     .await?

src/api2/admin/namespace.rs

@@ -167,7 +167,14 @@ pub fn delete_namespace(
     let (removed_all, stats) = datastore.remove_namespace_recursive(&ns, delete_groups)?;
     if !removed_all {
         let err_msg = if delete_groups {
+            if datastore.old_locking() {
+                "could not remove empty group directoriess due to old locking mechanism.\n\
+                If you are an admin, please reboot PBS or ensure no old backup job is running \
+                anymore, then remove the file '/run/proxmox-backup/old-locking', and reload all \
+                PBS daemons"
+            } else {
                 "group only partially deleted due to protected snapshots"
+            }
         } else {
             "only partially deleted due to existing groups but `delete-groups` not true"
         };