From 7c72ae04f16dc2b4c95ddeb1618fe1d1797a7825 Mon Sep 17 00:00:00 2001 From: Hannes Laimer Date: Mon, 30 Aug 2021 10:53:37 +0200 Subject: [PATCH] add chunk inspection to pb-debug Adds possibility to inspect chunks and find indexes that reference the chunk. Options: - chunk: path to the chunk file - [opt] decode: path to a file or to stdout(-), if specified, the chunk will be decoded into the specified location - [opt] digest: needed when searching for references, if set, it will be used for verification when decoding - [opt] keyfile: path to a keyfile, needed if decode is specified and the data was encrypted - [opt] reference-filter: path in which indexes that reference the chunk should be searched, can be a group, snapshot or the whole datastore, if not specified no references will be searched - [default=true] use-filename-as-digest: use chunk-filename as digest, if no digest is specified Signed-off-by: Hannes Laimer Signed-off-by: Wolfgang Bumiller --- Makefile | 3 +- src/bin/proxmox-backup-debug.rs | 13 ++ src/bin/proxmox_backup_debug/inspect.rs | 213 ++++++++++++++++++++++++ src/bin/proxmox_backup_debug/mod.rs | 2 + src/tools/mod.rs | 13 ++ 5 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 src/bin/proxmox-backup-debug.rs create mode 100644 src/bin/proxmox_backup_debug/inspect.rs create mode 100644 src/bin/proxmox_backup_debug/mod.rs diff --git a/Makefile b/Makefile index b2a8e5b4..dbbe5492 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ USR_BIN := \ # Binaries usable by admins USR_SBIN := \ - proxmox-backup-manager + proxmox-backup-manager \ + proxmox-backup-debug \ # Binaries for services: SERVICE_BIN := \ diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs new file mode 100644 index 00000000..897fa221 --- /dev/null +++ b/src/bin/proxmox-backup-debug.rs @@ -0,0 +1,13 @@ +use proxmox::api::cli::*; + +mod proxmox_backup_debug; +use proxmox_backup_debug::inspect_commands; + +fn main() { + proxmox_backup::tools::setup_safe_path_env(); + + let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands()); + + let rpcenv = CliEnvironment::new(); + run_cli_command(cmd_def, rpcenv, Some(|future| pbs_runtime::main(future))); +} diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs new file mode 100644 index 00000000..5a0979d2 --- /dev/null +++ b/src/bin/proxmox_backup_debug/inspect.rs @@ -0,0 +1,213 @@ +use std::path::Path; + +use anyhow::{format_err, Error}; +use proxmox::api::cli::{ + format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface, +}; +use proxmox::api::{api, cli::*}; +use serde_json::{json, Value}; +use walkdir::WalkDir; + +use proxmox_backup::backup::{ + load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile, +}; + +use pbs_client::tools::key_source::get_encryption_key_password; + +use proxmox_backup::tools::outfile_or_stdout; + +/// Decodes a blob and writes its content either to stdout or into a file +fn decode_blob( + mut output_path: Option<&Path>, + key_file: Option<&Path>, + digest: Option<&[u8; 32]>, + blob: &DataBlob, +) -> Result<(), Error> { + let mut crypt_conf_opt = None; + let crypt_conf; + + if blob.is_encrypted() && key_file.is_some() { + let (key, _created, _fingerprint) = + load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?; + crypt_conf = CryptConfig::new(key)?; + crypt_conf_opt = Some(&crypt_conf); + } + + output_path = match output_path { + Some(path) if path.eq(Path::new("-")) => None, + _ => output_path, + }; + + outfile_or_stdout(output_path)?.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?; + Ok(()) +} + +#[api( + input: { + properties: { + chunk: { + description: "The chunk file.", + type: String, + }, + "reference-filter": { + description: "Path to the directory that should be searched for references.", + type: String, + optional: true, + }, + "digest": { + description: "Needed when searching for references, if set, it will be used for verification when decoding.", + type: String, + optional: true, + }, + "decode": { + description: "Path to the file to which the chunk should be decoded, '-' -> decode to stdout.", + type: String, + optional: true, + }, + "keyfile": { + description: "Path to the keyfile with which the chunk was encrypted.", + type: String, + optional: true, + }, + "use-filename-as-digest": { + description: "The filename should be used as digest for reference search and decode verification, if no digest is specified.", + type: bool, + optional: true, + default: true, + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + } + } +)] +/// Inspect a chunk +fn inspect_chunk( + chunk: String, + reference_filter: Option, + mut digest: Option, + decode: Option, + keyfile: Option, + use_filename_as_digest: bool, + param: Value, +) -> Result<(), Error> { + let output_format = get_output_format(¶m); + let chunk_path = Path::new(&chunk); + + if digest.is_none() && use_filename_as_digest { + digest = Some(if let Some((_, filename)) = chunk.rsplit_once("/") { + String::from(filename) + } else { + chunk.clone() + }); + }; + + let digest_raw: Option<[u8; 32]> = digest + .map(|ref d| { + proxmox::tools::hex_to_digest(d) + .map_err(|e| format_err!("could not parse chunk - {}", e)) + }) + .map_or(Ok(None), |r| r.map(Some))?; + + let search_path = reference_filter.as_ref().map(Path::new); + let key_file_path = keyfile.as_ref().map(Path::new); + let decode_output_path = decode.as_ref().map(Path::new); + + let blob = DataBlob::load_from_reader( + &mut std::fs::File::open(&chunk_path) + .map_err(|e| format_err!("could not open chunk file - {}", e))?, + )?; + + let referenced_by = if let (Some(search_path), Some(digest_raw)) = (search_path, digest_raw) { + let mut references = Vec::new(); + for entry in WalkDir::new(search_path) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + { + use std::os::unix::ffi::OsStrExt; + let file_name = entry.file_name().as_bytes(); + + let index: Box = if file_name.ends_with(b".fidx") { + match FixedIndexReader::open(entry.path()) { + Ok(index) => Box::new(index), + Err(_) => continue, + } + } else if file_name.ends_with(b".didx") { + match DynamicIndexReader::open(entry.path()) { + Ok(index) => Box::new(index), + Err(_) => continue, + } + } else { + continue; + }; + + for pos in 0..index.index_count() { + if let Some(index_chunk_digest) = index.index_digest(pos) { + if digest_raw.eq(index_chunk_digest) { + references.push(entry.path().to_string_lossy().into_owned()); + break; + } + } + } + } + if !references.is_empty() { + Some(references) + } else { + None + } + } else { + None + }; + + if decode_output_path.is_some() { + decode_blob( + decode_output_path, + key_file_path, + digest_raw.as_ref(), + &blob, + )?; + } + + let crc_status = format!( + "{}({})", + blob.compute_crc(), + blob.verify_crc().map_or("BAD", |_| "OK") + ); + + let val = match referenced_by { + Some(references) => json!({ + "crc": crc_status, + "encryption": blob.crypt_mode()?, + "referenced-by": references + }), + None => json!({ + "crc": crc_status, + "encryption": blob.crypt_mode()?, + }), + }; + + if output_format == "text" { + println!("CRC: {}", val["crc"]); + println!("encryption: {}", val["encryption"]); + if let Some(refs) = val["referenced-by"].as_array() { + println!("referenced by:"); + for reference in refs { + println!(" {}", reference); + } + } + } else { + format_and_print_result(&val, &output_format); + } + Ok(()) +} + +pub fn inspect_commands() -> CommandLineInterface { + let cmd_def = CliCommandMap::new().insert( + "chunk", + CliCommand::new(&API_METHOD_INSPECT_CHUNK).arg_param(&["chunk"]), + ); + + cmd_def.into() +} diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs new file mode 100644 index 00000000..644583db --- /dev/null +++ b/src/bin/proxmox_backup_debug/mod.rs @@ -0,0 +1,2 @@ +mod inspect; +pub use inspect::*; diff --git a/src/tools/mod.rs b/src/tools/mod.rs index b6c55ac2..dfff8c10 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -2,7 +2,10 @@ //! //! This is a collection of small and useful tools. use std::any::Any; +use std::fs::File; +use std::io::{stdout, Write}; use std::os::unix::io::RawFd; +use std::path::Path; use anyhow::{bail, format_err, Error}; use openssl::hash::{hash, DigestBytes, MessageDigest}; @@ -224,3 +227,13 @@ pub fn create_run_dir() -> Result<(), Error> { let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?; Ok(()) } + +/// Returns either a new file, if a path is given, or stdout, if no path is given. +pub fn outfile_or_stdout>(path: Option

) -> Result, Error> { + if let Some(path) = path { + let f = File::create(path)?; + Ok(Box::new(f) as Box) + } else { + Ok(Box::new(stdout()) as Box) + } +}