diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs index 039cd71a..39e44de6 100644 --- a/pbs-api-types/src/datastore.rs +++ b/pbs-api-types/src/datastore.rs @@ -119,6 +119,52 @@ pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = .minimum(1) .schema(); +#[api( + properties: { + "keep-last": { + schema: PRUNE_SCHEMA_KEEP_LAST, + optional: true, + }, + "keep-hourly": { + schema: PRUNE_SCHEMA_KEEP_HOURLY, + optional: true, + }, + "keep-daily": { + schema: PRUNE_SCHEMA_KEEP_DAILY, + optional: true, + }, + "keep-weekly": { + schema: PRUNE_SCHEMA_KEEP_WEEKLY, + optional: true, + }, + "keep-monthly": { + schema: PRUNE_SCHEMA_KEEP_MONTHLY, + optional: true, + }, + "keep-yearly": { + schema: PRUNE_SCHEMA_KEEP_YEARLY, + optional: true, + }, + } +)] +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +/// Common pruning options +pub struct PruneOptions { + #[serde(skip_serializing_if="Option::is_none")] + pub keep_last: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub keep_hourly: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub keep_daily: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub keep_weekly: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub keep_monthly: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub keep_yearly: Option, +} + #[api( properties: { name: { diff --git a/pbs-datastore/src/prune.rs b/pbs-datastore/src/prune.rs index 4605e26f..3b4cf4f2 100644 --- a/pbs-datastore/src/prune.rs +++ b/pbs-datastore/src/prune.rs @@ -2,18 +2,8 @@ use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use anyhow::{Error}; -use serde::{Deserialize, Serialize}; -use proxmox::api::api; - -use pbs_api_types::{ - PRUNE_SCHEMA_KEEP_LAST, - PRUNE_SCHEMA_KEEP_HOURLY, - PRUNE_SCHEMA_KEEP_DAILY, - PRUNE_SCHEMA_KEEP_WEEKLY, - PRUNE_SCHEMA_KEEP_MONTHLY, - PRUNE_SCHEMA_KEEP_YEARLY, -}; +use pbs_api_types::PruneOptions; use super::BackupInfo; @@ -80,142 +70,52 @@ fn remove_incomplete_snapshots( } } -#[api( - properties: { - "keep-last": { - schema: PRUNE_SCHEMA_KEEP_LAST, - optional: true, - }, - "keep-hourly": { - schema: PRUNE_SCHEMA_KEEP_HOURLY, - optional: true, - }, - "keep-daily": { - schema: PRUNE_SCHEMA_KEEP_DAILY, - optional: true, - }, - "keep-weekly": { - schema: PRUNE_SCHEMA_KEEP_WEEKLY, - optional: true, - }, - "keep-monthly": { - schema: PRUNE_SCHEMA_KEEP_MONTHLY, - optional: true, - }, - "keep-yearly": { - schema: PRUNE_SCHEMA_KEEP_YEARLY, - optional: true, - }, - } -)] -#[derive(Serialize, Deserialize, Default)] -#[serde(rename_all = "kebab-case")] -/// Common pruning options -pub struct PruneOptions { - #[serde(skip_serializing_if="Option::is_none")] - pub keep_last: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub keep_hourly: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub keep_daily: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub keep_weekly: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub keep_monthly: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub keep_yearly: Option, +pub fn keeps_something(options: &PruneOptions) -> bool { + let mut keep_something = false; + if let Some(count) = options.keep_last { if count > 0 { keep_something = true; } } + if let Some(count) = options.keep_hourly { if count > 0 { keep_something = true; } } + if let Some(count) = options.keep_daily { if count > 0 { keep_something = true; } } + if let Some(count) = options.keep_weekly { if count > 0 { keep_something = true; } } + if let Some(count) = options.keep_monthly { if count > 0 { keep_something = true; } } + if let Some(count) = options.keep_yearly { if count > 0 { keep_something = true; } } + keep_something } -impl PruneOptions { +pub fn cli_options_string(options: &PruneOptions) -> String { + let mut opts = Vec::new(); - pub fn new() -> Self { - Self { - keep_last: None, - keep_hourly: None, - keep_daily: None, - keep_weekly: None, - keep_monthly: None, - keep_yearly: None, + if let Some(count) = options.keep_last { + if count > 0 { + opts.push(format!("--keep-last {}", count)); + } + } + if let Some(count) = options.keep_hourly { + if count > 0 { + opts.push(format!("--keep-hourly {}", count)); + } + } + if let Some(count) = options.keep_daily { + if count > 0 { + opts.push(format!("--keep-daily {}", count)); + } + } + if let Some(count) = options.keep_weekly { + if count > 0 { + opts.push(format!("--keep-weekly {}", count)); + } + } + if let Some(count) = options.keep_monthly { + if count > 0 { + opts.push(format!("--keep-monthly {}", count)); + } + } + if let Some(count) = options.keep_yearly { + if count > 0 { + opts.push(format!("--keep-yearly {}", count)); } } - pub fn keep_hourly(mut self, value: Option) -> Self { - self.keep_hourly = value; - self - } - - pub fn keep_last(mut self, value: Option) -> Self { - self.keep_last = value; - self - } - - pub fn keep_daily(mut self, value: Option) -> Self { - self.keep_daily = value; - self - } - - pub fn keep_weekly(mut self, value: Option) -> Self { - self.keep_weekly = value; - self - } - - pub fn keep_monthly(mut self, value: Option) -> Self { - self.keep_monthly = value; - self - } - - pub fn keep_yearly(mut self, value: Option) -> Self { - self.keep_yearly = value; - self - } - - pub fn keeps_something(&self) -> bool { - let mut keep_something = false; - if let Some(count) = self.keep_last { if count > 0 { keep_something = true; } } - if let Some(count) = self.keep_hourly { if count > 0 { keep_something = true; } } - if let Some(count) = self.keep_daily { if count > 0 { keep_something = true; } } - if let Some(count) = self.keep_weekly { if count > 0 { keep_something = true; } } - if let Some(count) = self.keep_monthly { if count > 0 { keep_something = true; } } - if let Some(count) = self.keep_yearly { if count > 0 { keep_something = true; } } - keep_something - } - - pub fn cli_options_string(&self) -> String { - let mut opts = Vec::new(); - - if let Some(count) = self.keep_last { - if count > 0 { - opts.push(format!("--keep-last {}", count)); - } - } - if let Some(count) = self.keep_hourly { - if count > 0 { - opts.push(format!("--keep-hourly {}", count)); - } - } - if let Some(count) = self.keep_daily { - if count > 0 { - opts.push(format!("--keep-daily {}", count)); - } - } - if let Some(count) = self.keep_weekly { - if count > 0 { - opts.push(format!("--keep-weekly {}", count)); - } - } - if let Some(count) = self.keep_monthly { - if count > 0 { - opts.push(format!("--keep-monthly {}", count)); - } - } - if let Some(count) = self.keep_yearly { - if count > 0 { - opts.push(format!("--keep-yearly {}", count)); - } - } - - opts.join(" ") - } + opts.join(" ") } pub fn compute_prune_info( diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index 857fdb62..52cff217 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -29,7 +29,7 @@ use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation}; use pbs_api_types::{ BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, Authid, CryptMode, GroupListItem, - PruneListItem, SnapshotListItem, StorageStatus, Fingerprint, + PruneListItem, SnapshotListItem, StorageStatus, Fingerprint, PruneOptions, }; use pbs_client::{ BACKUP_SOURCE_SCHEMA, @@ -72,7 +72,6 @@ use pbs_datastore::manifest::{ ENCRYPTED_KEY_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type, }; use pbs_datastore::read_chunk::AsyncReadChunk; -use pbs_datastore::prune::PruneOptions; use pbs_tools::sync::StdChannelWriter; use pbs_tools::tokio::TokioWriterAdapter; use pbs_tools::json; diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index a04bb472..45d5ba7d 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -26,12 +26,14 @@ use proxmox::{http_err, identity, list_subdirs_api_method, sortable}; use pxar::accessor::aio::Accessor; use pxar::EntryKind; -use pbs_api_types::{ - Authid, BackupContent, Counts, CryptMode, DataStoreListItem, GarbageCollectionStatus, - GroupListItem, SnapshotListItem, SnapshotVerifyState, BACKUP_ARCHIVE_NAME_SCHEMA, - BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA, - IGNORE_VERIFIED_BACKUPS_SCHEMA, UPID_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA, - PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY, PRIV_DATASTORE_READ, PRIV_DATASTORE_PRUNE, +use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode, + DataStoreListItem, GarbageCollectionStatus, GroupListItem, + SnapshotListItem, SnapshotVerifyState, PruneOptions, + BACKUP_ARCHIVE_NAME_SCHEMA, BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, + BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA, + IGNORE_VERIFIED_BACKUPS_SCHEMA, UPID_SCHEMA, + VERIFICATION_OUTDATED_AFTER_SCHEMA, PRIV_DATASTORE_AUDIT, + PRIV_DATASTORE_MODIFY, PRIV_DATASTORE_READ, PRIV_DATASTORE_PRUNE, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_VERIFY, }; @@ -46,7 +48,7 @@ use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader, Lo use pbs_datastore::fixed_index::{FixedIndexReader}; use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::{BackupManifest, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME}; -use pbs_datastore::prune::{compute_prune_info, PruneOptions}; +use pbs_datastore::prune::compute_prune_info; use pbs_tools::blocking::WrappedReaderStream; use pbs_tools::stream::{AsyncReaderStream, AsyncChannelWriter}; use pbs_tools::json::{required_integer_param, required_string_param}; @@ -838,7 +840,7 @@ pub fn prune( prune_info.reverse(); // delete older snapshots first - let keep_all = !prune_options.keeps_something(); + let keep_all = !pbs_datastore::prune::keeps_something(&prune_options); if dry_run { for (info, mut keep) in prune_info { @@ -864,7 +866,7 @@ pub fn prune( if keep_all { worker.log("No prune selection - keeping all files."); } else { - worker.log(format!("retention options: {}", prune_options.cli_options_string())); + worker.log(format!("retention options: {}", pbs_datastore::prune::cli_options_string(&prune_options))); worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"", store, backup_type, backup_id)); } diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 8a3a5eb8..4240711f 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -32,8 +32,10 @@ use pbs_buildcfg::configdir; use pbs_systemd::time::{compute_next_event, parse_calendar_event}; use pbs_tools::logrotate::LogRotate; -use pbs_api_types::{Authid, TapeBackupJobConfig, VerificationJobConfig, SyncJobConfig, DataStoreConfig}; -use pbs_datastore::prune::PruneOptions; +use pbs_api_types::{ + Authid, TapeBackupJobConfig, VerificationJobConfig, SyncJobConfig, DataStoreConfig, + PruneOptions, +}; use proxmox_backup::server; use proxmox_backup::auth_helpers::*; @@ -487,7 +489,7 @@ async fn schedule_datastore_prune() { keep_yearly: store_config.keep_yearly, }; - if !prune_options.keeps_something() { // no prune settings - keep all + if !pbs_datastore::prune::keeps_something(&prune_options) { // no prune settings - keep all continue; } diff --git a/src/server/prune_job.rs b/src/server/prune_job.rs index 5475d18f..f298a74c 100644 --- a/src/server/prune_job.rs +++ b/src/server/prune_job.rs @@ -4,8 +4,8 @@ use anyhow::Error; use pbs_datastore::{task_log, task_warn}; use pbs_datastore::backup_info::BackupInfo; -use pbs_datastore::prune::{compute_prune_info, PruneOptions}; -use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY}; +use pbs_datastore::prune::compute_prune_info; +use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY, PruneOptions}; use pbs_config::CachedUserInfo; use crate::{ @@ -28,7 +28,7 @@ pub fn prune_datastore( task_log!(worker, "(dry test run)"); } - let keep_all = !prune_options.keeps_something(); + let keep_all = !pbs_datastore::prune::keeps_something(&prune_options); if keep_all { task_log!(worker, "No prune selection - keeping all files."); @@ -36,7 +36,7 @@ pub fn prune_datastore( task_log!( worker, "retention options: {}", - prune_options.cli_options_string() + pbs_datastore::prune::cli_options_string(&prune_options) ); } diff --git a/tests/prune.rs b/tests/prune.rs index 3297b031..35b40ad7 100644 --- a/tests/prune.rs +++ b/tests/prune.rs @@ -1,8 +1,9 @@ use anyhow::{Error}; use std::path::PathBuf; +use pbs_api_types::PruneOptions; use pbs_datastore::manifest::MANIFEST_BLOB_NAME; -use pbs_datastore::prune::{compute_prune_info, PruneOptions}; +use pbs_datastore::prune::compute_prune_info; use pbs_datastore::{BackupDir, BackupInfo}; fn get_prune_list( @@ -56,7 +57,8 @@ fn test_prune_hourly() -> Result<(), Error> { orig_list.push(create_info("host/elsa/2019-11-15T11:59:15Z", false)); let list = orig_list.clone(); - let options = PruneOptions::new().keep_hourly(Some(3)); + let mut options = PruneOptions::default(); + options.keep_hourly = Some(3); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-11-15T10:49:15Z"), @@ -66,7 +68,8 @@ fn test_prune_hourly() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list; - let options = PruneOptions::new().keep_hourly(Some(2)); + let mut options = PruneOptions::default(); + options.keep_hourly = Some(2); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-11-15T10:59:15Z"), @@ -77,7 +80,7 @@ fn test_prune_hourly() -> Result<(), Error> { Ok(()) } - #[test] +#[test] fn test_prune_simple2() -> Result<(), Error> { let mut orig_list = Vec::new(); @@ -93,7 +96,8 @@ fn test_prune_simple2() -> Result<(), Error> { orig_list.push(create_info("host/elsa/2019-12-04T11:59:15Z", false)); let list = orig_list.clone(); - let options = PruneOptions::new().keep_daily(Some(1)); + let mut options = PruneOptions::default(); + options.keep_daily = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-04T11:59:15Z"), @@ -101,7 +105,9 @@ fn test_prune_simple2() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(1)).keep_daily(Some(1)); + let mut options = PruneOptions::default(); + options.keep_last = Some(1); + options.keep_daily = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-03T11:59:15Z"), @@ -110,7 +116,9 @@ fn test_prune_simple2() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_daily(Some(1)).keep_weekly(Some(1)); + let mut options = PruneOptions::default(); + options.keep_daily = Some(1); + options.keep_weekly = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-01T11:59:15Z"), @@ -119,7 +127,10 @@ fn test_prune_simple2() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_daily(Some(1)).keep_weekly(Some(1)).keep_monthly(Some(1)); + let mut options = PruneOptions::default(); + options.keep_daily = Some(1); + options.keep_weekly = Some(1); + options.keep_monthly = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-11-22T11:59:15Z"), @@ -129,7 +140,9 @@ fn test_prune_simple2() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list; - let options = PruneOptions::new().keep_monthly(Some(1)).keep_yearly(Some(1)); + let mut options = PruneOptions::default(); + options.keep_monthly = Some(1); + options.keep_yearly = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2018-11-15T11:59:15Z"), @@ -153,13 +166,15 @@ fn test_prune_simple() -> Result<(), Error> { // keep-last tests let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(4)); + let mut options = PruneOptions::default(); + options.keep_last = Some(4); let remove_list = get_prune_list(list, false, &options); let expect: Vec = Vec::new(); assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(3)); + let mut options = PruneOptions::default(); + options.keep_last = Some(3); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -167,7 +182,8 @@ fn test_prune_simple() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(2)); + let mut options = PruneOptions::default(); + options.keep_last = Some(2); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -176,7 +192,8 @@ fn test_prune_simple() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(1)); + let mut options = PruneOptions::default(); + options.keep_last = Some(1); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -186,7 +203,8 @@ fn test_prune_simple() -> Result<(), Error> { assert_eq!(remove_list, expect); let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(0)); + let mut options = PruneOptions::default(); + options.keep_last = Some(0); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -198,21 +216,25 @@ fn test_prune_simple() -> Result<(), Error> { // keep-last, keep-daily mixed let list = orig_list.clone(); - let options = PruneOptions::new().keep_last(Some(2)).keep_daily(Some(2)); + let mut options = PruneOptions::default(); + options.keep_last = Some(2); + options.keep_daily = Some(2); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![]; assert_eq!(remove_list, expect); // keep-daily test let list = orig_list.clone(); - let options = PruneOptions::new().keep_daily(Some(3)); + let mut options = PruneOptions::default(); + options.keep_daily = Some(3); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")]; assert_eq!(remove_list, expect); // keep-daily test let list = orig_list.clone(); - let options = PruneOptions::new().keep_daily(Some(2)); + let mut options = PruneOptions::default(); + options.keep_daily = Some(2); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -222,7 +244,8 @@ fn test_prune_simple() -> Result<(), Error> { // keep-weekly let list = orig_list.clone(); - let options = PruneOptions::new().keep_weekly(Some(5)); + let mut options = PruneOptions::default(); + options.keep_weekly = Some(5); let remove_list = get_prune_list(list, false, &options); // all backup are within the same week, so we only keep a single file let expect: Vec = vec![ @@ -234,7 +257,9 @@ fn test_prune_simple() -> Result<(), Error> { // keep-daily + keep-weekly let list = orig_list.clone(); - let options = PruneOptions::new().keep_daily(Some(1)).keep_weekly(Some(5)); + let mut options = PruneOptions::default(); + options.keep_daily = Some(1); + options.keep_weekly = Some(5); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -245,7 +270,8 @@ fn test_prune_simple() -> Result<(), Error> { // keep-monthly let list = orig_list.clone(); - let options = PruneOptions::new().keep_monthly(Some(6)); + let mut options = PruneOptions::default(); + options.keep_monthly = Some(6); let remove_list = get_prune_list(list, false, &options); // all backup are within the same month, so we only keep a single file let expect: Vec = vec![ @@ -257,7 +283,8 @@ fn test_prune_simple() -> Result<(), Error> { // keep-yearly let list = orig_list.clone(); - let options = PruneOptions::new().keep_yearly(Some(7)); + let mut options = PruneOptions::default(); + options.keep_yearly = Some(7); let remove_list = get_prune_list(list, false, &options); // all backup are within the same year, so we only keep a single file let expect: Vec = vec![ @@ -269,7 +296,10 @@ fn test_prune_simple() -> Result<(), Error> { // keep-weekly + keep-monthly + keep-yearly let list = orig_list; - let options = PruneOptions::new().keep_weekly(Some(5)).keep_monthly(Some(6)).keep_yearly(Some(7)); + let mut options = PruneOptions::default(); + options.keep_weekly = Some(5); + options.keep_monthly = Some(6); + options.keep_yearly = Some(7); let remove_list = get_prune_list(list, false, &options); // all backup are within one week, so we only keep a single file let expect: Vec = vec![