diff --git a/src/backup/chunk_stat.rs b/pbs-datastore/src/chunk_stat.rs similarity index 100% rename from src/backup/chunk_stat.rs rename to pbs-datastore/src/chunk_stat.rs diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index cae66905..3a5b15a9 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -182,6 +182,7 @@ pub mod backup_info; pub mod catalog; pub mod checksum_reader; pub mod checksum_writer; +pub mod chunk_stat; pub mod chunk_store; pub mod chunker; pub mod crypt_config; @@ -194,6 +195,7 @@ pub mod file_formats; pub mod index; pub mod key_derivation; pub mod manifest; +pub mod read_chunk; pub mod task; pub use backup_info::{BackupDir, BackupGroup, BackupInfo}; diff --git a/pbs-datastore/src/read_chunk.rs b/pbs-datastore/src/read_chunk.rs new file mode 100644 index 00000000..c04a7431 --- /dev/null +++ b/pbs-datastore/src/read_chunk.rs @@ -0,0 +1,29 @@ +use std::future::Future; +use std::pin::Pin; + +use anyhow::Error; + +use crate::data_blob::DataBlob; + +/// The ReadChunk trait allows reading backup data chunks (local or remote) +pub trait ReadChunk { + /// Returns the encoded chunk data + fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result; + + /// Returns the decoded chunk data + fn read_chunk(&self, digest: &[u8; 32]) -> Result, Error>; +} + +pub trait AsyncReadChunk: Send { + /// Returns the encoded chunk data + fn read_raw_chunk<'a>( + &'a self, + digest: &'a [u8; 32], + ) -> Pin> + Send + 'a>>; + + /// Returns the decoded chunk data + fn read_chunk<'a>( + &'a self, + digest: &'a [u8; 32], + ) -> Pin, Error>> + Send + 'a>>; +} diff --git a/src/backup/cached_chunk_reader.rs b/src/backup/cached_chunk_reader.rs index c9ca4773..3798ae4a 100644 --- a/src/backup/cached_chunk_reader.rs +++ b/src/backup/cached_chunk_reader.rs @@ -1,20 +1,22 @@ //! An async and concurrency safe data reader backed by a local LRU cache. -use anyhow::Error; -use futures::future::Future; -use futures::ready; -use tokio::io::{AsyncRead, AsyncSeek, ReadBuf}; - +use std::future::Future; use std::io::SeekFrom; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use super::{AsyncReadChunk, IndexFile}; -use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache}; +use anyhow::Error; +use futures::ready; +use tokio::io::{AsyncRead, AsyncSeek, ReadBuf}; + use proxmox::io_format_err; use proxmox::sys::error::io_err_other; +use pbs_datastore::read_chunk::AsyncReadChunk; +use super::IndexFile; +use crate::tools::async_lru_cache::{AsyncCacher, AsyncLruCache}; + struct AsyncChunkCacher { reader: Arc, } diff --git a/src/backup/dynamic_index.rs b/src/backup/dynamic_index.rs index 8321b295..4bba6a29 100644 --- a/src/backup/dynamic_index.rs +++ b/src/backup/dynamic_index.rs @@ -14,14 +14,13 @@ use proxmox::tools::uuid::Uuid; use proxmox::tools::mmap::Mmap; use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation}; -use super::chunk_stat::ChunkStat; -use super::chunk_store::ChunkStore; -use super::index::ChunkReadInfo; -use super::read_chunk::ReadChunk; -use super::Chunker; -use super::IndexFile; -use super::{DataBlob, DataChunkBuilder}; -use crate::tools; +use pbs_datastore::Chunker; +use pbs_datastore::index::{IndexFile, ChunkReadInfo}; +use pbs_datastore::chunk_stat::ChunkStat; +use pbs_datastore::data_blob::{DataBlob, DataChunkBuilder}; +use pbs_datastore::chunk_store::ChunkStore; +use pbs_datastore::read_chunk::ReadChunk; +use pbs_tools::process_locker::ProcessLockSharedGuard; /// Header format definition for dynamic index files (`.dixd`) #[repr(C)] @@ -480,7 +479,7 @@ impl ReadAt for LocalDynamicReadAt { /// Create dynamic index files (`.dixd`) pub struct DynamicIndexWriter { store: Arc, - _lock: tools::ProcessLockSharedGuard, + _lock: ProcessLockSharedGuard, writer: BufWriter, closed: bool, filename: PathBuf, diff --git a/src/backup/fixed_index.rs b/src/backup/fixed_index.rs index ebf64456..518a5b0d 100644 --- a/src/backup/fixed_index.rs +++ b/src/backup/fixed_index.rs @@ -1,18 +1,17 @@ -use anyhow::{bail, format_err, Error}; -use std::io::{Seek, SeekFrom}; - -use super::chunk_stat::*; -use super::chunk_store::*; -use super::{ChunkReadInfo, IndexFile}; -use crate::tools; - use std::fs::File; use std::io::Write; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::io::{Seek, SeekFrom}; -use super::ChunkInfo; +use anyhow::{bail, format_err, Error}; + +use pbs_datastore::chunk_stat::ChunkStat; +use pbs_datastore::chunk_store::ChunkStore; +use pbs_datastore::data_blob::ChunkInfo; +use pbs_datastore::index::{ChunkReadInfo, IndexFile}; +use pbs_tools::process_locker::ProcessLockSharedGuard; use proxmox::tools::io::ReadExt; use proxmox::tools::Uuid; @@ -229,7 +228,7 @@ impl IndexFile for FixedIndexReader { pub struct FixedIndexWriter { store: Arc, file: File, - _lock: tools::ProcessLockSharedGuard, + _lock: ProcessLockSharedGuard, filename: PathBuf, tmp_filename: PathBuf, chunk_size: usize, diff --git a/src/backup/mod.rs b/src/backup/mod.rs index 34d1c5ac..5b597265 100644 --- a/src/backup/mod.rs +++ b/src/backup/mod.rs @@ -149,12 +149,16 @@ pub const CATALOG_NAME: &str = "catalog.pcat1.didx"; #[macro_export] macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 { - () => { "proxmox-backup-protocol-v1" } + () => { + "proxmox-backup-protocol-v1" + }; } #[macro_export] macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 { - () => { "proxmox-backup-reader-protocol-v1" } + () => { + "proxmox-backup-reader-protocol-v1" + }; } /// Unix system user used by proxmox-backup-proxy @@ -186,6 +190,8 @@ pub use pbs_datastore::checksum_reader; pub use pbs_datastore::checksum_reader::*; pub use pbs_datastore::checksum_writer; pub use pbs_datastore::checksum_writer::*; +pub use pbs_datastore::chunk_stat; +pub use pbs_datastore::chunk_stat::*; pub use pbs_datastore::chunk_store; pub use pbs_datastore::chunk_store::*; pub use pbs_datastore::chunker; @@ -210,13 +216,12 @@ pub use pbs_datastore::key_derivation; pub use pbs_datastore::key_derivation::*; pub use pbs_datastore::manifest; pub use pbs_datastore::manifest::*; +pub use pbs_datastore::read_chunk::*; mod chunk_stream; pub use chunk_stream::*; -mod chunk_stat; -pub use chunk_stat::*; - +// Split mod read_chunk; pub use read_chunk::*; @@ -238,6 +243,7 @@ pub use store_progress::*; mod verify; pub use verify::*; +// Move to client mod catalog_shell; pub use catalog_shell::*; diff --git a/src/backup/read_chunk.rs b/src/backup/read_chunk.rs index 078779de..588563c5 100644 --- a/src/backup/read_chunk.rs +++ b/src/backup/read_chunk.rs @@ -4,19 +4,12 @@ use std::sync::Arc; use anyhow::{bail, Error}; -use super::crypt_config::{CryptConfig, CryptMode}; -use super::data_blob::DataBlob; +use pbs_datastore::crypt_config::{CryptConfig, CryptMode}; +use pbs_datastore::data_blob::DataBlob; +use pbs_datastore::read_chunk::{ReadChunk, AsyncReadChunk}; + use super::datastore::DataStore; -/// The ReadChunk trait allows reading backup data chunks (local or remote) -pub trait ReadChunk { - /// Returns the encoded chunk data - fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result; - - /// Returns the decoded chunk data - fn read_chunk(&self, digest: &[u8; 32]) -> Result, Error>; -} - #[derive(Clone)] pub struct LocalChunkReader { store: Arc, @@ -67,20 +60,6 @@ impl ReadChunk for LocalChunkReader { } } -pub trait AsyncReadChunk: Send { - /// Returns the encoded chunk data - fn read_raw_chunk<'a>( - &'a self, - digest: &'a [u8; 32], - ) -> Pin> + Send + 'a>>; - - /// Returns the decoded chunk data - fn read_chunk<'a>( - &'a self, - digest: &'a [u8; 32], - ) -> Pin, Error>> + Send + 'a>>; -} - impl AsyncReadChunk for LocalChunkReader { fn read_raw_chunk<'a>( &'a self,