From fdaab0df4e5cc385b15f556b8a3c2396197a9e88 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 26 Jun 2020 08:14:45 +0200 Subject: [PATCH] src/backup/index.rs: add chunk_info method --- src/backup/dynamic_index.rs | 39 +++++++++++----------- src/backup/fixed_index.rs | 64 +++++++++++++++++-------------------- src/backup/index.rs | 1 + src/backup/verify.rs | 43 ++++++++++++------------- 4 files changed, 70 insertions(+), 77 deletions(-) diff --git a/src/backup/dynamic_index.rs b/src/backup/dynamic_index.rs index ae8427a4..b30b1bce 100644 --- a/src/backup/dynamic_index.rs +++ b/src/backup/dynamic_index.rs @@ -124,25 +124,6 @@ impl DynamicIndexReader { }) } - #[allow(clippy::cast_ptr_alignment)] - pub fn chunk_info(&self, pos: usize) -> Result { - if pos >= self.index.len() { - bail!("chunk index out of range"); - } - let start = if pos == 0 { - 0 - } else { - self.index[pos - 1].end() - }; - - let end = self.index[pos].end(); - - Ok(ChunkReadInfo { - range: start..end, - digest: self.index[pos].digest.clone(), - }) - } - #[inline] #[allow(clippy::cast_ptr_alignment)] fn chunk_end(&self, pos: usize) -> u64 { @@ -225,6 +206,21 @@ impl IndexFile for DynamicIndexReader { self.chunk_end(self.index.len() - 1) } } + + #[allow(clippy::cast_ptr_alignment)] + fn chunk_info(&self, pos: usize) -> Option { + if pos >= self.index.len() { + return None; + } + let start = if pos == 0 { 0 } else { self.index[pos - 1].end() }; + + let end = self.index[pos].end(); + + Some(ChunkReadInfo { + range: start..end, + digest: self.index[pos].digest.clone(), + }) + } } struct CachedChunk { @@ -264,7 +260,10 @@ struct ChunkCacher<'a, S> { impl<'a, S: ReadChunk> crate::tools::lru_cache::Cacher for ChunkCacher<'a, S> { fn fetch(&mut self, index: usize) -> Result, Error> { - let info = self.index.chunk_info(index)?; + let info = match self.index.chunk_info(index) { + Some(info) => info, + None => bail!("chunk index out of range"), + }; let range = info.range; let data = self.store.read_chunk(&info.digest)?; CachedChunk::new(range, data).map(Some) diff --git a/src/backup/fixed_index.rs b/src/backup/fixed_index.rs index 56daaedc..d671e4eb 100644 --- a/src/backup/fixed_index.rs +++ b/src/backup/fixed_index.rs @@ -4,7 +4,7 @@ use std::io::{Seek, SeekFrom}; use super::chunk_stat::*; use super::chunk_store::*; -use super::IndexFile; +use super::{IndexFile, ChunkReadInfo}; use crate::tools::{self, epoch_now_u64}; use chrono::{Local, TimeZone}; @@ -147,29 +147,6 @@ impl FixedIndexReader { Ok(()) } - pub fn chunk_info(&self, pos: usize) -> Result<(u64, u64, [u8; 32]), Error> { - if pos >= self.index_length { - bail!("chunk index out of range"); - } - let start = (pos * self.chunk_size) as u64; - let mut end = start + self.chunk_size as u64; - - if end > self.size { - end = self.size; - } - - let mut digest = std::mem::MaybeUninit::<[u8; 32]>::uninit(); - unsafe { - std::ptr::copy_nonoverlapping( - self.index.add(pos * 32), - (*digest.as_mut_ptr()).as_mut_ptr(), - 32, - ); - } - - Ok((start, end, unsafe { digest.assume_init() })) - } - #[inline] fn chunk_digest(&self, pos: usize) -> &[u8; 32] { if pos >= self.index_length { @@ -234,6 +211,25 @@ impl IndexFile for FixedIndexReader { fn index_bytes(&self) -> u64 { self.size } + + fn chunk_info(&self, pos: usize) -> Option { + if pos >= self.index_length { + return None; + } + + let start = (pos * self.chunk_size) as u64; + let mut end = start + self.chunk_size as u64; + + if end > self.size { + end = self.size; + } + + let digest = self.index_digest(pos).unwrap(); + Some(ChunkReadInfo { + range: start..end, + digest: *digest, + }) + } } pub struct FixedIndexWriter { @@ -511,18 +507,17 @@ impl BufferedFixedReader { fn buffer_chunk(&mut self, idx: usize) -> Result<(), Error> { let index = &self.index; - let (start, end, digest) = index.chunk_info(idx)?; + let info = match index.chunk_info(idx) { + Some(info) => info, + None => bail!("chunk index out of range"), + }; // fixme: avoid copy - let data = self.store.read_chunk(&digest)?; - - if (end - start) != data.len() as u64 { - bail!( - "read chunk with wrong size ({} != {}", - (end - start), - data.len() - ); + let data = self.store.read_chunk(&info.digest)?; + let size = info.range.end - info.range.start; + if size != data.len() as u64 { + bail!("read chunk with wrong size ({} != {}", size, data.len()); } self.read_buffer.clear(); @@ -530,8 +525,7 @@ impl BufferedFixedReader { self.buffered_chunk_idx = idx; - self.buffered_chunk_start = start as u64; - //println!("BUFFER {} {}", self.buffered_chunk_start, end); + self.buffered_chunk_start = info.range.start as u64; Ok(()) } } diff --git a/src/backup/index.rs b/src/backup/index.rs index 2f362421..4195b3f1 100644 --- a/src/backup/index.rs +++ b/src/backup/index.rs @@ -20,6 +20,7 @@ pub trait IndexFile { fn index_count(&self) -> usize; fn index_digest(&self, pos: usize) -> Option<&[u8; 32]>; fn index_bytes(&self) -> u64; + fn chunk_info(&self, pos: usize) -> Option; /// Returns most often used chunks fn find_most_used_chunks(&self, max: usize) -> HashMap<[u8; 32], usize> { diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 026c5e6b..77c58b0c 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -34,6 +34,25 @@ fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) - Ok(()) } +fn verify_index_chunks( + datastore: &DataStore, + index: Box, + worker: &WorkerTask, +) -> Result<(), Error> { + + for pos in 0..index.index_count() { + + worker.fail_on_abort()?; + crate::tools::fail_on_shutdown()?; + + let info = index.chunk_info(pos).unwrap(); + let size = info.range.end - info.range.start; + datastore.verify_stored_chunk(&info.digest, size)?; + } + + Ok(()) +} + fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> { let mut path = backup_dir.relative_path(); @@ -50,17 +69,7 @@ fn verify_fixed_index(datastore: &DataStore, backup_dir: &BackupDir, info: &File bail!("wrong index checksum"); } - for pos in 0..index.index_count() { - - worker.fail_on_abort()?; - crate::tools::fail_on_shutdown()?; - - let (start, end, digest) = index.chunk_info(pos).unwrap(); - let size = end - start; - datastore.verify_stored_chunk(&digest, size)?; - } - - Ok(()) + verify_index_chunks(datastore, Box::new(index), worker) } fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo, worker: &WorkerTask) -> Result<(), Error> { @@ -78,17 +87,7 @@ fn verify_dynamic_index(datastore: &DataStore, backup_dir: &BackupDir, info: &Fi bail!("wrong index checksum"); } - for pos in 0..index.index_count() { - - worker.fail_on_abort()?; - crate::tools::fail_on_shutdown()?; - - let chunk_info = index.chunk_info(pos).unwrap(); - let size = chunk_info.range.end - chunk_info.range.start; - datastore.verify_stored_chunk(&chunk_info.digest, size)?; - } - - Ok(()) + verify_index_chunks(datastore, Box::new(index), worker) } /// Verify a single backup snapshot