From d420962fbcf5eaa1ea28bdbfe926c72ee09e19d7 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 6 Jul 2021 12:08:44 +0200 Subject: [PATCH] split out pbs-runtime module These are mostly tokio specific "hacks" or "workarounds" we only really need/want in our binaries without pulling it in via our library crates. Signed-off-by: Wolfgang Bumiller --- Cargo.toml | 4 +++- Makefile | 3 ++- pbs-runtime/Cargo.toml | 11 +++++++++++ src/tools/runtime.rs => pbs-runtime/src/lib.rs | 0 src/api2/backup/mod.rs | 6 +++--- src/api2/backup/upload_chunk.rs | 2 +- src/api2/node/apt.rs | 4 ++-- src/api2/reader/mod.rs | 2 +- src/backup/catalog_shell.rs | 2 +- src/bin/proxmox-backup-api.rs | 2 +- src/bin/proxmox-backup-client.rs | 2 +- src/bin/proxmox-backup-manager.rs | 4 ++-- src/bin/proxmox-backup-proxy.rs | 6 +++--- src/bin/proxmox-daily-update.rs | 2 +- src/bin/proxmox-file-restore.rs | 2 +- src/bin/proxmox-restore-daemon.rs | 2 +- src/bin/proxmox-tape.rs | 2 +- src/bin/proxmox_backup_client/mount.rs | 4 ++-- src/bin/proxmox_client_tools/mod.rs | 10 +++++----- src/bin/pxar.rs | 2 +- src/client/pull.rs | 4 ++-- src/client/pxar_backup_stream.rs | 2 +- src/client/remote_chunk_reader.rs | 2 +- src/tools/mod.rs | 1 - src/tools/subscription.rs | 2 +- src/tools/wrapped_reader_stream.rs | 4 ++-- 26 files changed, 50 insertions(+), 37 deletions(-) create mode 100644 pbs-runtime/Cargo.toml rename src/tools/runtime.rs => pbs-runtime/src/lib.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index fa6d5019..b9c31550 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ exclude = [ "build", "debian", "tests/catar_data/test_symlink/symlink1"] [workspace] members = [ "pbs-buildcfg", + "pbs-runtime", ] [lib] @@ -91,7 +92,8 @@ proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "webs #proxmox-http = { version = "0.2.0", path = "../proxmox/proxmox-http", features = [ "client", "http-helpers", "websocket" ] } proxmox-openid = "0.6.0" -pbs-buildcfg = { path = "pbs-buildcfg", version = "0.1" } +pbs-buildcfg = { path = "pbs-buildcfg" } +pbs-runtime = { path = "pbs-runtime" } [features] default = [] diff --git a/Makefile b/Makefile index 48296f93..f6915c5f 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,8 @@ RESTORE_BIN := \ proxmox-restore-daemon SUBCRATES := \ - pbs-buildcfg + pbs-buildcfg \ + pbs-runtime ifeq ($(BUILD_MODE), release) CARGO_BUILD_ARGS += --release diff --git a/pbs-runtime/Cargo.toml b/pbs-runtime/Cargo.toml new file mode 100644 index 00000000..72c25ac2 --- /dev/null +++ b/pbs-runtime/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "pbs-runtime" +version = "0.1.0" +authors = ["Proxmox Support Team "] +edition = "2018" +description = "tokio runtime related helpers required for binaries" + +[dependencies] +lazy_static = "1.4" +pin-utils = "0.1.0" +tokio = { version = "1.6", features = [ "rt", "rt-multi-thread" ] } diff --git a/src/tools/runtime.rs b/pbs-runtime/src/lib.rs similarity index 100% rename from src/tools/runtime.rs rename to pbs-runtime/src/lib.rs diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs index 9962cdcb..36fdc1b2 100644 --- a/src/api2/backup/mod.rs +++ b/src/api2/backup/mod.rs @@ -218,7 +218,7 @@ async move { }; if benchmark { env.log("benchmark finished successfully"); - tools::runtime::block_in_place(|| env.remove_backup())?; + pbs_runtime::block_in_place(|| env.remove_backup())?; return Ok(()); } @@ -246,13 +246,13 @@ async move { (Ok(_), Err(err)) => { env.log(format!("backup ended and finish failed: {}", err)); env.log("removing unfinished backup"); - tools::runtime::block_in_place(|| env.remove_backup())?; + pbs_runtime::block_in_place(|| env.remove_backup())?; Err(err) }, (Err(err), Err(_)) => { env.log(format!("backup failed: {}", err)); env.log("removing failed backup"); - tools::runtime::block_in_place(|| env.remove_backup())?; + pbs_runtime::block_in_place(|| env.remove_backup())?; Err(err) }, } diff --git a/src/api2/backup/upload_chunk.rs b/src/api2/backup/upload_chunk.rs index cdd38c87..1e8f3a79 100644 --- a/src/api2/backup/upload_chunk.rs +++ b/src/api2/backup/upload_chunk.rs @@ -61,7 +61,7 @@ impl Future for UploadChunk { let (is_duplicate, compressed_size) = match proxmox::try_block! { let mut chunk = DataBlob::from_raw(raw_data)?; - tools::runtime::block_in_place(|| { + pbs_runtime::block_in_place(|| { chunk.verify_unencrypted(this.size as usize, &this.digest)?; // always comput CRC at server side diff --git a/src/api2/node/apt.rs b/src/api2/node/apt.rs index 35c9a79a..054c8813 100644 --- a/src/api2/node/apt.rs +++ b/src/api2/node/apt.rs @@ -236,7 +236,7 @@ fn apt_get_changelog( let changelog_url = &pkg_info[0].change_log_url; // FIXME: use 'apt-get changelog' for proxmox packages as well, once repo supports it if changelog_url.starts_with("http://download.proxmox.com/") { - let changelog = crate::tools::runtime::block_on(client.get_string(changelog_url, None)) + let changelog = pbs_runtime::block_on(client.get_string(changelog_url, None)) .map_err(|err| format_err!("Error downloading changelog from '{}': {}", changelog_url, err))?; Ok(json!(changelog)) @@ -260,7 +260,7 @@ fn apt_get_changelog( auth_header.insert("Authorization".to_owned(), format!("Basic {}", base64::encode(format!("{}:{}", key, id)))); - let changelog = crate::tools::runtime::block_on(client.get_string(changelog_url, Some(&auth_header))) + let changelog = pbs_runtime::block_on(client.get_string(changelog_url, Some(&auth_header))) .map_err(|err| format_err!("Error downloading changelog from '{}': {}", changelog_url, err))?; Ok(json!(changelog)) diff --git a/src/api2/reader/mod.rs b/src/api2/reader/mod.rs index 648fe73d..d0081fad 100644 --- a/src/api2/reader/mod.rs +++ b/src/api2/reader/mod.rs @@ -322,7 +322,7 @@ fn download_chunk( env.debug(format!("download chunk {:?}", path)); - let data = tools::runtime::block_in_place(|| std::fs::read(path)) + let data = pbs_runtime::block_in_place(|| std::fs::read(path)) .map_err(move |err| http_err!(BAD_REQUEST, "reading file {:?} failed: {}", path2, err))?; let body = Body::from(data); diff --git a/src/backup/catalog_shell.rs b/src/backup/catalog_shell.rs index 11866c30..b186ac6c 100644 --- a/src/backup/catalog_shell.rs +++ b/src/backup/catalog_shell.rs @@ -21,7 +21,7 @@ use pxar::{EntryKind, Metadata}; use crate::backup::catalog::{self, DirEntryAttribute}; use crate::pxar::fuse::{Accessor, FileEntry}; use crate::pxar::Flags; -use crate::tools::runtime::block_in_place; +use pbs_runtime::block_in_place; use crate::tools::ControlFlow; type CatalogReader = crate::backup::CatalogReader; diff --git a/src/bin/proxmox-backup-api.rs b/src/bin/proxmox-backup-api.rs index 625b7232..25ed030a 100644 --- a/src/bin/proxmox-backup-api.rs +++ b/src/bin/proxmox-backup-api.rs @@ -18,7 +18,7 @@ use proxmox_backup::config; fn main() { proxmox_backup::tools::setup_safe_path_env(); - if let Err(err) = proxmox_backup::tools::runtime::main(run()) { + if let Err(err) = pbs_runtime::main(run()) { eprintln!("Error: {}", err); std::process::exit(-1); } diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index 578b2680..bd52abbc 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -1494,6 +1494,6 @@ fn main() { let rpcenv = CliEnvironment::new(); run_cli_command(cmd_def, rpcenv, Some(|future| { - proxmox_backup::tools::runtime::main(future) + pbs_runtime::main(future) })); } diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 60b4c6c6..a7e7eecf 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -397,7 +397,7 @@ fn main() { let mut rpcenv = CliEnvironment::new(); rpcenv.set_auth_id(Some(String::from("root@pam"))); - proxmox_backup::tools::runtime::main(run_async_cli_command(cmd_def, rpcenv)); + pbs_runtime::main(run_async_cli_command(cmd_def, rpcenv)); } // shell completion helper @@ -408,7 +408,7 @@ pub fn complete_remote_datastore_name(_arg: &str, param: &HashMap Result<(), Error> { bail!("proxy not running as backup user or group (got uid {} gid {})", running_uid, running_gid); } - proxmox_backup::tools::runtime::main(run()) + pbs_runtime::main(run()) } async fn run() -> Result<(), Error> { @@ -700,7 +700,7 @@ async fn schedule_task_log_rotate() { if logrotate.rotate(max_size, None, Some(max_files))? { println!("rotated access log, telling daemons to re-open log file"); - proxmox_backup::tools::runtime::block_on(command_reopen_logfiles())?; + pbs_runtime::block_on(command_reopen_logfiles())?; worker.log("API access log was rotated".to_string()); } else { worker.log("API access log was not rotated".to_string()); @@ -787,7 +787,7 @@ async fn generate_host_stats(save: bool) { use proxmox_backup::config::datastore; - proxmox_backup::tools::runtime::block_in_place(move || { + pbs_runtime::block_in_place(move || { match read_proc_stat() { Ok(stat) => { diff --git a/src/bin/proxmox-daily-update.rs b/src/bin/proxmox-daily-update.rs index be3bfe44..b962b67f 100644 --- a/src/bin/proxmox-daily-update.rs +++ b/src/bin/proxmox-daily-update.rs @@ -91,7 +91,7 @@ fn main() { let mut rpcenv = CliEnvironment::new(); rpcenv.set_auth_id(Some(String::from("root@pam"))); - if let Err(err) = proxmox_backup::tools::runtime::main(do_update(&mut rpcenv)) { + if let Err(err) = pbs_runtime::main(do_update(&mut rpcenv)) { eprintln!("error during update: {}", err); std::process::exit(1); } diff --git a/src/bin/proxmox-file-restore.rs b/src/bin/proxmox-file-restore.rs index 5766b279..17033cf5 100644 --- a/src/bin/proxmox-file-restore.rs +++ b/src/bin/proxmox-file-restore.rs @@ -473,6 +473,6 @@ fn main() { run_cli_command( cmd_def, rpcenv, - Some(|future| proxmox_backup::tools::runtime::main(future)), + Some(|future| pbs_runtime::main(future)), ); } diff --git a/src/bin/proxmox-restore-daemon.rs b/src/bin/proxmox-restore-daemon.rs index 420c668f..3e59ccfa 100644 --- a/src/bin/proxmox-restore-daemon.rs +++ b/src/bin/proxmox-restore-daemon.rs @@ -62,7 +62,7 @@ fn main() -> Result<(), Error> { info!("disk scan complete, starting main runtime..."); - proxmox_backup::tools::runtime::main(run()) + pbs_runtime::main(run()) } async fn run() -> Result<(), Error> { diff --git a/src/bin/proxmox-tape.rs b/src/bin/proxmox-tape.rs index 46bd4ecc..c095fe43 100644 --- a/src/bin/proxmox-tape.rs +++ b/src/bin/proxmox-tape.rs @@ -1101,5 +1101,5 @@ fn main() { let mut rpcenv = CliEnvironment::new(); rpcenv.set_auth_id(Some(String::from("root@pam"))); - proxmox_backup::tools::runtime::main(run_async_cli_command(cmd_def, rpcenv)); + pbs_runtime::main(run_async_cli_command(cmd_def, rpcenv)); } diff --git a/src/bin/proxmox_backup_client/mount.rs b/src/bin/proxmox_backup_client/mount.rs index d0f04f89..21f78c32 100644 --- a/src/bin/proxmox_backup_client/mount.rs +++ b/src/bin/proxmox_backup_client/mount.rs @@ -139,7 +139,7 @@ fn mount( if verbose { // This will stay in foreground with debug output enabled as None is // passed for the RawFd. - return proxmox_backup::tools::runtime::main(mount_do(param, None)); + return pbs_runtime::main(mount_do(param, None)); } // Process should be daemonized. @@ -155,7 +155,7 @@ fn mount( Ok(ForkResult::Child) => { drop(pr); nix::unistd::setsid().unwrap(); - proxmox_backup::tools::runtime::main(mount_do(param, Some(pw))) + pbs_runtime::main(mount_do(param, Some(pw))) } Err(_) => bail!("failed to daemonize process"), } diff --git a/src/bin/proxmox_client_tools/mod.rs b/src/bin/proxmox_client_tools/mod.rs index 24d9fa63..00e5155a 100644 --- a/src/bin/proxmox_client_tools/mod.rs +++ b/src/bin/proxmox_client_tools/mod.rs @@ -107,7 +107,7 @@ pub async fn try_get(repo: &BackupRepository, url: &str) -> Value { } pub fn complete_backup_group(_arg: &str, param: &HashMap) -> Vec { - proxmox_backup::tools::runtime::main(async { complete_backup_group_do(param).await }) + pbs_runtime::main(async { complete_backup_group_do(param).await }) } pub async fn complete_backup_group_do(param: &HashMap) -> Vec { @@ -137,7 +137,7 @@ pub async fn complete_backup_group_do(param: &HashMap) -> Vec) -> Vec { - proxmox_backup::tools::runtime::main(async { complete_group_or_snapshot_do(arg, param).await }) + pbs_runtime::main(async { complete_group_or_snapshot_do(arg, param).await }) } pub async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap) -> Vec { @@ -156,7 +156,7 @@ pub async fn complete_group_or_snapshot_do(arg: &str, param: &HashMap) -> Vec { - proxmox_backup::tools::runtime::main(async { complete_backup_snapshot_do(param).await }) + pbs_runtime::main(async { complete_backup_snapshot_do(param).await }) } pub async fn complete_backup_snapshot_do(param: &HashMap) -> Vec { @@ -188,7 +188,7 @@ pub async fn complete_backup_snapshot_do(param: &HashMap) -> Vec } pub fn complete_server_file_name(_arg: &str, param: &HashMap) -> Vec { - proxmox_backup::tools::runtime::main(async { complete_server_file_name_do(param).await }) + pbs_runtime::main(async { complete_server_file_name_do(param).await }) } pub async fn complete_server_file_name_do(param: &HashMap) -> Vec { @@ -279,7 +279,7 @@ pub fn complete_chunk_size(_arg: &str, _param: &HashMap) -> Vec< } pub fn complete_auth_id(_arg: &str, param: &HashMap) -> Vec { - proxmox_backup::tools::runtime::main(async { complete_auth_id_do(param).await }) + pbs_runtime::main(async { complete_auth_id_do(param).await }) } pub async fn complete_auth_id_do(param: &HashMap) -> Vec { diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs index 0bd17ee1..34cae30c 100644 --- a/src/bin/pxar.rs +++ b/src/bin/pxar.rs @@ -491,6 +491,6 @@ fn main() { let rpcenv = CliEnvironment::new(); run_cli_command(cmd_def, rpcenv, Some(|future| { - proxmox_backup::tools::runtime::main(future) + pbs_runtime::main(future) })); } diff --git a/src/client/pull.rs b/src/client/pull.rs index 1ee0e0d1..19f91961 100644 --- a/src/client/pull.rs +++ b/src/client/pull.rs @@ -73,7 +73,7 @@ async fn pull_index_chunks( let verify_and_write_channel = verify_and_write_channel.clone(); Ok::<_, Error>(async move { - let chunk_exists = crate::tools::runtime::block_in_place(|| { + let chunk_exists = pbs_runtime::block_in_place(|| { target.cond_touch_chunk(&info.digest, false) })?; if chunk_exists { @@ -85,7 +85,7 @@ async fn pull_index_chunks( let raw_size = chunk.raw_size() as usize; // decode, verify and write in a separate threads to maximize throughput - crate::tools::runtime::block_in_place(|| { + pbs_runtime::block_in_place(|| { verify_and_write_channel.send((chunk, info.digest, info.size())) })?; diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs index 035f735c..1d3fc228 100644 --- a/src/client/pxar_backup_stream.rs +++ b/src/client/pxar_backup_stream.rs @@ -113,7 +113,7 @@ impl Stream for PxarBackupStream { } } - match crate::tools::runtime::block_in_place(|| self.rx.as_ref().unwrap().recv()) { + match pbs_runtime::block_in_place(|| self.rx.as_ref().unwrap().recv()) { Ok(data) => Poll::Ready(Some(data)), Err(_) => { let error = self.error.lock().unwrap(); diff --git a/src/client/remote_chunk_reader.rs b/src/client/remote_chunk_reader.rs index 06f693a2..62e6feaa 100644 --- a/src/client/remote_chunk_reader.rs +++ b/src/client/remote_chunk_reader.rs @@ -7,7 +7,7 @@ use anyhow::{bail, Error}; use super::BackupReader; use crate::backup::{AsyncReadChunk, CryptConfig, CryptMode, DataBlob, ReadChunk}; -use crate::tools::runtime::block_on; +use pbs_runtime::block_on; /// Read chunks from remote host using ``BackupReader`` #[derive(Clone)] diff --git a/src/tools/mod.rs b/src/tools/mod.rs index 716b9593..23943990 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -47,7 +47,6 @@ pub mod loopdev; pub mod lru_cache; pub mod async_lru_cache; pub mod nom; -pub mod runtime; pub mod serde_filter; pub mod statistics; pub mod subscription; diff --git a/src/tools/subscription.rs b/src/tools/subscription.rs index 08917284..425468f1 100644 --- a/src/tools/subscription.rs +++ b/src/tools/subscription.rs @@ -228,7 +228,7 @@ pub fn check_subscription(key: String, server_id: String) -> Result { @@ -108,7 +108,7 @@ mod test { #[test] fn test_wrapped_stream_reader() -> Result<(), Error> { - crate::tools::runtime::main(async { + pbs_runtime::main(async { run_wrapped_stream_reader_test().await }) }