diff --git a/src/api2/backup.rs b/src/api2/backup.rs index 8177bab5..e60c2078 100644 --- a/src/api2/backup.rs +++ b/src/api2/backup.rs @@ -284,6 +284,8 @@ pub const API_METHOD_CREATE_FIXED_INDEX: ApiMethod = ApiMethod::new( .minimum(1) .schema() ), + ("reuse-csum", true, &StringSchema::new("If set, compare last backup's \ + csum and reuse index for incremental backup if it matches.").schema()), ]), ) ); @@ -298,6 +300,7 @@ fn create_fixed_index( let name = tools::required_string_param(¶m, "archive-name")?.to_owned(); let size = tools::required_integer_param(¶m, "size")? as usize; + let reuse_csum = param["reuse-csum"].as_str(); let archive_name = name.clone(); if !archive_name.ends_with(".fidx") { @@ -305,12 +308,49 @@ fn create_fixed_index( } let mut path = env.backup_dir.relative_path(); - path.push(archive_name); + path.push(&archive_name); let chunk_size = 4096*1024; // todo: ?? - let index = env.datastore.create_fixed_writer(&path, size, chunk_size)?; - let wid = env.register_fixed_writer(index, name, size, chunk_size as u32)?; + // do incremental backup if csum is set + let mut reader = None; + let mut incremental = false; + if let Some(csum) = reuse_csum { + incremental = true; + let last_backup = match &env.last_backup { + Some(info) => info, + None => { + bail!("cannot reuse index - no previous backup exists"); + } + }; + + let mut last_path = last_backup.backup_dir.relative_path(); + last_path.push(&archive_name); + + let index = match env.datastore.open_fixed_reader(last_path) { + Ok(index) => index, + Err(_) => { + bail!("cannot reuse index - no previous backup exists for archive"); + } + }; + + let (old_csum, _) = index.compute_csum(); + let old_csum = proxmox::tools::digest_to_hex(&old_csum); + if old_csum != csum { + bail!("expected csum ({}) doesn't match last backup's ({}), cannot do incremental backup", + csum, old_csum); + } + + reader = Some(index); + } + + let mut writer = env.datastore.create_fixed_writer(&path, size, chunk_size)?; + + if let Some(reader) = reader { + writer.clone_data_from(&reader)?; + } + + let wid = env.register_fixed_writer(writer, name, size, chunk_size as u32, incremental)?; env.log(format!("created new fixed index {} ({:?})", wid, path)); @@ -518,15 +558,15 @@ pub const API_METHOD_CLOSE_FIXED_INDEX: ApiMethod = ApiMethod::new( ( "chunk-count", false, - &IntegerSchema::new("Chunk count. This is used to verify that the server got all chunks.") - .minimum(1) + &IntegerSchema::new("Chunk count. This is used to verify that the server got all chunks. Ignored for incremental backups.") + .minimum(0) .schema() ), ( "size", false, - &IntegerSchema::new("File size. This is used to verify that the server got all data.") - .minimum(1) + &IntegerSchema::new("File size. This is used to verify that the server got all data. Ignored for incremental backups.") + .minimum(0) .schema() ), ("csum", false, &StringSchema::new("Digest list checksum.").schema()), diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 735798ac..8d3d4279 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -47,6 +47,7 @@ struct FixedWriterState { chunk_count: u64, small_chunk_count: usize, // allow 0..1 small chunks (last chunk may be smaller) upload_stat: UploadStatistic, + incremental: bool, } struct SharedBackupState { @@ -237,7 +238,7 @@ impl BackupEnvironment { } /// Store the writer with an unique ID - pub fn register_fixed_writer(&self, index: FixedIndexWriter, name: String, size: usize, chunk_size: u32) -> Result { + pub fn register_fixed_writer(&self, index: FixedIndexWriter, name: String, size: usize, chunk_size: u32, incremental: bool) -> Result { let mut state = self.state.lock().unwrap(); state.ensure_unfinished()?; @@ -245,7 +246,7 @@ impl BackupEnvironment { let uid = state.next_uid(); state.fixed_writers.insert(uid, FixedWriterState { - index, name, chunk_count: 0, size, chunk_size, small_chunk_count: 0, upload_stat: UploadStatistic::new(), + index, name, chunk_count: 0, size, chunk_size, small_chunk_count: 0, upload_stat: UploadStatistic::new(), incremental, }); Ok(uid) @@ -379,21 +380,22 @@ impl BackupEnvironment { bail!("fixed writer '{}' close failed - received wrong number of chunk ({} != {})", data.name, data.chunk_count, chunk_count); } - let expected_count = data.index.index_length(); + if !data.incremental { + let expected_count = data.index.index_length(); - if chunk_count != (expected_count as u64) { - bail!("fixed writer '{}' close failed - unexpected chunk count ({} != {})", data.name, expected_count, chunk_count); - } + if chunk_count != (expected_count as u64) { + bail!("fixed writer '{}' close failed - unexpected chunk count ({} != {})", data.name, expected_count, chunk_count); + } - if size != (data.size as u64) { - bail!("fixed writer '{}' close failed - unexpected file size ({} != {})", data.name, data.size, size); + if size != (data.size as u64) { + bail!("fixed writer '{}' close failed - unexpected file size ({} != {})", data.name, data.size, size); + } } let uuid = data.index.uuid; - let expected_csum = data.index.close()?; - println!("server checksum {:?} client: {:?}", expected_csum, csum); + println!("server checksum: {:?} client: {:?} (incremental: {})", expected_csum, csum, data.incremental); if csum != expected_csum { bail!("fixed writer '{}' close failed - got unexpected checksum", data.name); } diff --git a/src/backup/fixed_index.rs b/src/backup/fixed_index.rs index fdaef35d..56daaedc 100644 --- a/src/backup/fixed_index.rs +++ b/src/backup/fixed_index.rs @@ -467,6 +467,18 @@ impl FixedIndexWriter { Ok(()) } + + pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> { + if self.index_length != reader.index_count() { + bail!("clone_data_from failed - index sizes not equal"); + } + + for i in 0..self.index_length { + self.add_digest(i, reader.index_digest(i).unwrap())?; + } + + Ok(()) + } } pub struct BufferedFixedReader {