From 56e887fd178cc859416d19e0275c5a31b572be59 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 9 Jul 2020 12:18:46 +0200 Subject: [PATCH] produce correct manifest --- Cargo.toml | 3 +-- src/backup.rs | 30 ++++++++++++++++++----------- src/commands.rs | 50 +++++++++++++++++-------------------------------- src/registry.rs | 19 +------------------ 4 files changed, 38 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9e27800..15bc06f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,7 @@ cbindgen = "0.14.2" libc = "0.2" bytes = "0.5" proxmox = { version = "0.1.42", features = [ "sortable-macro", "api-macro" ] } -proxmox-backup = { git = "git://git.proxmox.com/git/proxmox-backup.git", commit = "8b2ad84a251bd4af33d8c2cddd4f40e26a99e84b" } -#proxmox-backup = { git = "git://git.proxmox.com/git/proxmox-backup.git", tag = "v0.8.0" } +proxmox-backup = { git = "git://git.proxmox.com/git/proxmox-backup.git", tag = "v0.8.2" } #proxmox-backup = { path = "../proxmox-backup" } chrono = "0.4" # Date and time library for Rust anyhow = "1.0" diff --git a/src/backup.rs b/src/backup.rs index 77fb611..717e099 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -8,7 +8,7 @@ use futures::future::{Future, Either, FutureExt}; use tokio::runtime::Runtime; use proxmox_backup::tools::runtime::get_runtime_with_builder; -use proxmox_backup::backup::{CryptConfig, CryptMode, BackupManifest, load_and_decrypt_key}; +use proxmox_backup::backup::{CryptConfig, CryptMode, BackupDir, BackupManifest, load_and_decrypt_key}; use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupWriter}; use super::BackupSetup; @@ -24,6 +24,7 @@ pub(crate) struct BackupTask { crypt_config: Option>, writer: OnceCell>, last_manifest: OnceCell>, + manifest: Arc>, registry: Arc>>, known_chunks: Arc>>, abort: tokio::sync::broadcast::Sender<()>, @@ -58,10 +59,14 @@ impl BackupTask { let (abort, _) = tokio::sync::broadcast::channel(16); + let snapshot = BackupDir::new(&setup.backup_type, &setup.backup_id, setup.backup_time.timestamp()); + let manifest = Arc::new(Mutex::new(BackupManifest::new(snapshot))); + let registry = Arc::new(Mutex::new(Registry::::new())); let known_chunks = Arc::new(Mutex::new(HashSet::new())); - Ok(Self { runtime, setup, compress, crypt_mode, crypt_config, abort, registry, known_chunks, + Ok(Self { runtime, setup, compress, crypt_mode, crypt_config, abort, + registry, manifest, known_chunks, writer: OnceCell::new(), last_manifest: OnceCell::new(), aborted: OnceCell::new() }) } @@ -145,11 +150,11 @@ impl BackupTask { let command_future = add_config( writer, - self.registry.clone(), + self.manifest.clone(), name, data, self.compress, - self.crypt_mode == CryptMode::Encrypt, + self.crypt_mode, ); let mut abort_rx = self.abort.subscribe(); @@ -240,7 +245,14 @@ impl BackupTask { None => bail!("not connected"), }; - let command_future = close_image(writer, self.registry.clone(), dev_id); + let command_future = close_image( + writer, + self.manifest.clone(), + self.registry.clone(), + dev_id, + self.crypt_mode, + ); + let mut abort_rx = self.abort.subscribe(); abortable_command(command_future, abort_rx.recv()).await } @@ -254,12 +266,8 @@ impl BackupTask { None => bail!("not connected"), }; - let command_future = finish_backup( - writer, - self.crypt_mode, - self.registry.clone(), - self.setup.clone(), - ); + let command_future = finish_backup(writer, self.crypt_config.clone(), self.manifest.clone()); + let mut abort_rx = self.abort.subscribe(); abortable_command(command_future, abort_rx.recv()).await } diff --git a/src/commands.rs b/src/commands.rs index bf8b8a0..9c21702 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -9,7 +9,6 @@ use serde_json::json; use proxmox_backup::backup::*; use proxmox_backup::client::*; -use super::BackupSetup; use crate::registry::Registry; use crate::capi_types::*; use crate::upload_queue::*; @@ -61,24 +60,20 @@ async fn register_zero_chunk( pub(crate) async fn add_config( client: Arc, - registry: Arc>>, + manifest: Arc>, name: String, data: Vec, compress: bool, - encrypt: bool, + crypt_mode: CryptMode, ) -> Result { //println!("add config {} size {}", name, size); let blob_name = format!("{}.blob", name); - let stats = client.upload_blob_from_data(data, &blob_name, compress, encrypt).await?; + let stats = client.upload_blob_from_data(data, &blob_name, compress, crypt_mode == CryptMode::Encrypt).await?; - let mut guard = registry.lock().unwrap(); - guard.add_file_info(json!({ - "filename": blob_name, - "size": stats.size, - "csum": proxmox::tools::digest_to_hex(&stats.csum), - })); + let mut guard = manifest.lock().unwrap(); + guard.add_file(blob_name, stats.size, stats.csum, crypt_mode)?; Ok(0) } @@ -194,8 +189,10 @@ pub(crate) async fn register_image( pub(crate) async fn close_image( client: Arc, + manifest: Arc>, registry: Arc>>, dev_id: u8, + crypt_mode: CryptMode, ) -> Result { //println!("close image {}", dev_id); @@ -235,11 +232,9 @@ pub(crate) async fn close_image( let mut prev_csum_guard = PREVIOUS_CSUMS.lock().unwrap(); prev_csum_guard.insert(info.device_name.clone(), upload_result.csum); - guard.add_file_info(json!({ - "filename": format!("{}.img.fidx", device_name), - "size": device_size, - "csum": csum.clone(), - })); + + let mut guard = manifest.lock().unwrap(); + guard.add_file(format!("{}.img.fidx", device_name), device_size, upload_result.csum, crypt_mode)?; Ok(0) } @@ -366,29 +361,18 @@ pub(crate) async fn write_data( pub(crate) async fn finish_backup( client: Arc, - crypt_mode: CryptMode, - registry: Arc>>, - setup: BackupSetup, + crypt_config: Option>, + manifest: Arc>, ) -> Result { - //println!("call finish"); - - let index_data = { - let guard = registry.lock().unwrap(); - - let index = json!({ - "backup-type": "vm", - "backup-id": setup.backup_id, - "backup-time": setup.backup_time.timestamp(), - "files": &guard.file_list(), - }); - - //println!("Upload index.json"); - serde_json::to_string_pretty(&index)?.into() + let manifest = { + let guard = manifest.lock().unwrap(); + guard.to_string(crypt_config.as_ref().map(Arc::as_ref)) + .map_err(|err| format_err!("unable to format manifest - {}", err))? }; client - .upload_blob_from_data(index_data, "index.json.blob", true, crypt_mode == CryptMode::Encrypt) + .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, true, false) .await?; client.finish().await?; diff --git a/src/registry.rs b/src/registry.rs index f584c87..a84f9db 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -1,5 +1,4 @@ use anyhow::{bail, Error}; -use serde_json::Value; /// Helper to store data, accessible by integer ID /// @@ -7,17 +6,13 @@ use serde_json::Value; /// generate integer handles to pass them to "C" code. pub struct Registry { info_list: Vec, - file_list: Vec, } impl Registry { /// Create a new instance pub fn new() -> Self { - Self { - info_list: Vec::new(), - file_list: Vec::new(), - } + Self { info_list: Vec::new() } } /// Register data, returns associated ID @@ -37,16 +32,4 @@ impl Registry { } Ok(&mut self.info_list[id as usize]) } - - /// Store info in a list - /// - /// We use this to store data for the backup manifest. - pub fn add_file_info(&mut self, info: Value) { - self.file_list.push(info); - } - - /// Returns the list produced with add_file_info() - pub fn file_list(&self) -> &[Value] { - &self.file_list - } }