diff --git a/src/backup.rs b/src/backup.rs index 0f3ac666..ab7e400a 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -3,4 +3,5 @@ pub mod chunker; pub mod chunk_store; pub mod image_index; +pub mod archive_index; pub mod datastore; diff --git a/src/backup/archive_index.rs b/src/backup/archive_index.rs new file mode 100644 index 00000000..24500da2 --- /dev/null +++ b/src/backup/archive_index.rs @@ -0,0 +1,133 @@ +use failure::*; + +use super::chunk_store::*; +use super::chunker::*; + +use std::io::{Read, Write}; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::os::unix::io::AsRawFd; +use uuid::Uuid; +use chrono::{Local, TimeZone}; + +#[repr(C)] +pub struct ArchiveIndexHeader { + pub magic: [u8; 12], + pub version: u32, + pub uuid: [u8; 16], + pub ctime: u64, + reserved: [u8; 4056], // oversall size is one page (4096 bytes) +} + +pub struct ArchiveIndexWriter<'a> { + store: &'a ChunkStore, + chunker: Chunker, + file: File, + filename: PathBuf, + tmp_filename: PathBuf, + uuid: [u8; 16], + ctime: u64, + + chunk_offset: usize, + last_chunk: usize, + chunk_buffer: Vec, +} + +impl <'a> ArchiveIndexWriter<'a> { + + pub fn create(store: &'a ChunkStore, path: &Path, chunk_size: usize) -> Result { + + let full_path = store.relative_path(path); + let mut tmp_path = full_path.clone(); + tmp_path.set_extension("tmp_aidx"); + + let mut file = std::fs::OpenOptions::new() + .create(true).truncate(true) + .read(true) + .write(true) + .open(&tmp_path)?; + + let header_size = std::mem::size_of::(); + + // todo: use static assertion when available in rust + if header_size != 4096 { panic!("got unexpected header size"); } + + let ctime = std::time::SystemTime::now().duration_since( + std::time::SystemTime::UNIX_EPOCH)?.as_secs(); + + let uuid = Uuid::new_v4(); + + let mut buffer = vec![0u8; header_size]; + let header = crate::tools::map_struct_mut::(&mut buffer)?; + + header.magic = *b"PROXMOX-AIDX"; + header.version = u32::to_le(1); + header.ctime = u64::to_le(ctime); + header.uuid = *uuid.as_bytes(); + + file.write_all(&buffer)?; + + Ok(Self { + store, + chunker: Chunker::new(chunk_size), + file: file, + filename: full_path, + tmp_filename: tmp_path, + ctime, + uuid: *uuid.as_bytes(), + + chunk_offset: 0, + last_chunk: 0, + chunk_buffer: Vec::with_capacity(chunk_size*4), + }) + } +} + +impl <'a> Write for ArchiveIndexWriter<'a> { + + fn write(&mut self, data: &[u8]) -> std::result::Result { + + use std::io::{Error, ErrorKind}; + + let chunker = &mut self.chunker; + + let pos = chunker.scan(data); + + if pos > 0 { + self.chunk_buffer.extend(&data[0..pos]); + self.chunk_offset += pos; + + let chunk_size = self.chunk_buffer.len(); + + let expected_chunk_size = self.chunk_offset - self.last_chunk; + if expected_chunk_size != self.chunk_buffer.len() { + panic!("wrong chunk size {} != {}", + expected_chunk_size, chunk_size); + } + + self.last_chunk = self.chunk_offset; + + match self.store.insert_chunk(&self.chunk_buffer) { + Ok((is_duplicate, digest)) => { + println!("ADD CHUNK {} {} {} {}", self.chunk_offset, chunk_size, is_duplicate, digest_to_hex(&digest)); + self.chunk_buffer.truncate(0); + return Ok(pos); + } + Err(err) => { + self.chunk_buffer.truncate(0); + return Err(Error::new(ErrorKind::Other, err.to_string())); + } + } + + } else { + self.chunk_offset += data.len(); + self.chunk_buffer.extend(data); + return Ok(data.len()); + } + } + + fn flush(&mut self) -> std::result::Result<(), std::io::Error> { + + Ok(()) + } +} diff --git a/src/backup/chunker.rs b/src/backup/chunker.rs index 1c724385..f652deb0 100644 --- a/src/backup/chunker.rs +++ b/src/backup/chunker.rs @@ -205,26 +205,3 @@ impl Chunker { self.h ^= BUZHASH_TABLE[(byte as usize)]; } } - -impl Write for Chunker { - - fn write(&mut self, data: &[u8]) -> std::result::Result { - - let pos = self.scan(data); - - if pos > 0 { - self.offset += pos; - println!("BOUND {} size {}", self.offset, self.offset - self.last_offset); - self.last_offset = self.offset; - return Ok(pos); - } else { - self.offset += data.len(); - return Ok(data.len()); - } - } - - fn flush(&mut self) -> std::result::Result<(), std::io::Error> { - - Ok(()) - } -} diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs index 355f6526..db78c5c4 100644 --- a/src/backup/datastore.rs +++ b/src/backup/datastore.rs @@ -8,6 +8,7 @@ use std::sync::{Mutex, Arc}; use crate::config::datastore; use super::chunk_store::*; use super::image_index::*; +use super::archive_index::*; pub struct DataStore { chunk_store: ChunkStore, @@ -76,6 +77,16 @@ impl DataStore { Ok(index) } + pub fn create_archive_writer>( + &self, filename: P, + chunk_size: usize + ) -> Result { + + let index = ArchiveIndexWriter::create(&self.chunk_store, filename.as_ref(), chunk_size)?; + + Ok(index) + } + pub fn list_images(&self) -> Result, Error> { let base = self.chunk_store.base_path(); diff --git a/src/bin/backup-client.rs b/src/bin/backup-client.rs index 8797591e..56b517f7 100644 --- a/src/bin/backup-client.rs +++ b/src/bin/backup-client.rs @@ -44,11 +44,11 @@ fn backup_dir( // .truncate(true) // .open("mytest.catar")?; - let writer = Chunker::new(chunk_size); + let mut index = datastore.create_archive_writer(&target, chunk_size)?; let path = std::path::PathBuf::from(path); - CaTarEncoder::encode(path, dir, writer)?; + CaTarEncoder::encode(path, dir, index)?; Ok(()) } diff --git a/src/catar/encoder.rs b/src/catar/encoder.rs index 4cf21837..2116663e 100644 --- a/src/catar/encoder.rs +++ b/src/catar/encoder.rs @@ -163,7 +163,7 @@ impl CaTarEncoder { fn encode_dir(&mut self, dir: &mut nix::dir::Dir) -> Result<(), Error> { - println!("encode_dir: {:?} start {}", self.current_path, self.writer_pos); + //println!("encode_dir: {:?} start {}", self.current_path, self.writer_pos); let mut name_list = vec![]; @@ -270,7 +270,7 @@ impl CaTarEncoder { self.current_path.pop(); } - println!("encode_dir: {:?} end {}", self.current_path, self.writer_pos); + //println!("encode_dir: {:?} end {}", self.current_path, self.writer_pos); // fixup goodby item offsets let goodbye_start = self.writer_pos as u64; @@ -282,13 +282,13 @@ impl CaTarEncoder { self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?; - println!("encode_dir: {:?} end1 {}", self.current_path, self.writer_pos); + //println!("encode_dir: {:?} end1 {}", self.current_path, self.writer_pos); Ok(()) } fn encode_file(&mut self, filefd: RawFd) -> Result<(), Error> { - println!("encode_file: {:?}", self.current_path); + //println!("encode_file: {:?}", self.current_path); let stat = match nix::sys::stat::fstat(filefd) { Ok(stat) => stat, @@ -338,7 +338,7 @@ impl CaTarEncoder { fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> { - println!("encode_symlink: {:?} -> {:?}", self.current_path, target); + //println!("encode_symlink: {:?} -> {:?}", self.current_path, target); self.write_entry(stat)?;