From d0b7e1e299352be2035faf7edca6a9c65a134ada Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 22 Nov 2021 13:30:59 +0100 Subject: [PATCH] proxmox-sys: imported pbs-tools/src/fs.rs to src/fs/read_dir.rs Signed-off-by: Dietmar Maurer --- proxmox-sys/Cargo.toml | 2 + proxmox-sys/src/fs/mod.rs | 3 + proxmox-sys/src/fs/read_dir.rs | 371 +++++++++++++++++++++++++++++++++ 3 files changed, 376 insertions(+) create mode 100644 proxmox-sys/src/fs/read_dir.rs diff --git a/proxmox-sys/Cargo.toml b/proxmox-sys/Cargo.toml index 8a78cb62..85632cfc 100644 --- a/proxmox-sys/Cargo.toml +++ b/proxmox-sys/Cargo.toml @@ -15,9 +15,11 @@ lazy_static = "1.4" libc = "0.2.107" log = "0.4" nix = "0.19.1" +regex = "1.2" serde_json = "1.0" zstd = { version = "0.6", features = [ "bindgen" ] } proxmox = { path = "../proxmox", version = "0.15", default-features = false } proxmox-io = { path = "../proxmox-io", version = "1.0.0" } proxmox-lang = { path = "../proxmox-lang", version = "1.0.0" } +proxmox-borrow = { path = "../proxmox-borrow", version = "1.0.0" } diff --git a/proxmox-sys/src/fs/mod.rs b/proxmox-sys/src/fs/mod.rs index 049c9bdc..1cb1681b 100644 --- a/proxmox-sys/src/fs/mod.rs +++ b/proxmox-sys/src/fs/mod.rs @@ -15,6 +15,9 @@ pub use file::*; mod dir; pub use dir::*; +mod read_dir; +pub use read_dir::*; + /// Change ownership of an open file handle pub fn fchown(fd: RawFd, owner: Option, group: Option) -> Result<(), Error> { // According to the POSIX specification, -1 is used to indicate that owner and group diff --git a/proxmox-sys/src/fs/read_dir.rs b/proxmox-sys/src/fs/read_dir.rs new file mode 100644 index 00000000..8cfc5eff --- /dev/null +++ b/proxmox-sys/src/fs/read_dir.rs @@ -0,0 +1,371 @@ +use std::borrow::{Borrow, BorrowMut}; +use std::fs::File; +use std::io::{self, BufRead}; +use std::ops::{Deref, DerefMut}; +use std::os::unix::io::{AsRawFd, RawFd}; +use std::path::Path; + +use anyhow::{bail, format_err, Error}; +use nix::dir; +use nix::dir::Dir; +use nix::fcntl::OFlag; +use nix::sys::stat::Mode; + +use regex::Regex; + +use proxmox_borrow::Tied; + +use crate::error::SysError; + +pub type DirLockGuard = Dir; + +/// This wraps nix::dir::Entry with the parent directory's file descriptor. +pub struct ReadDirEntry { + entry: dir::Entry, + parent_fd: RawFd, +} + +impl Into for ReadDirEntry { + fn into(self) -> dir::Entry { + self.entry + } +} + +impl Deref for ReadDirEntry { + type Target = dir::Entry; + + fn deref(&self) -> &Self::Target { + &self.entry + } +} + +impl DerefMut for ReadDirEntry { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.entry + } +} + +impl AsRef for ReadDirEntry { + fn as_ref(&self) -> &dir::Entry { + &self.entry + } +} + +impl AsMut for ReadDirEntry { + fn as_mut(&mut self) -> &mut dir::Entry { + &mut self.entry + } +} + +impl Borrow for ReadDirEntry { + fn borrow(&self) -> &dir::Entry { + &self.entry + } +} + +impl BorrowMut for ReadDirEntry { + fn borrow_mut(&mut self) -> &mut dir::Entry { + &mut self.entry + } +} + +impl ReadDirEntry { + #[inline] + pub fn parent_fd(&self) -> RawFd { + self.parent_fd + } + + pub unsafe fn file_name_utf8_unchecked(&self) -> &str { + std::str::from_utf8_unchecked(self.file_name().to_bytes()) + } +} + +// Since Tied implements Deref to U, a Tied already implements Iterator. +// This is simply a wrapper with a shorter type name mapping nix::Error to anyhow::Error. +/// Wrapper over a pair of `nix::dir::Dir` and `nix::dir::Iter`, returned by `read_subdir()`. +pub struct ReadDir { + iter: Tied> + Send>, + dir_fd: RawFd, +} + +impl Iterator for ReadDir { + type Item = Result; + + fn next(&mut self) -> Option { + self.iter.next().map(|res| { + res.map(|entry| ReadDirEntry { entry, parent_fd: self.dir_fd }) + .map_err(Error::from) + }) + } +} + +/// Create an iterator over sub directory entries. +/// This uses `openat` on `dirfd`, so `path` can be relative to that or an absolute path. +pub fn read_subdir(dirfd: RawFd, path: &P) -> nix::Result { + let dir = Dir::openat(dirfd, path, OFlag::O_RDONLY, Mode::empty())?; + let fd = dir.as_raw_fd(); + let iter = Tied::new(dir, |dir| { + Box::new(unsafe { (*dir).iter() }) + as Box> + Send> + }); + Ok(ReadDir { iter, dir_fd: fd }) +} + +/// Scan through a directory with a regular expression. This is simply a shortcut filtering the +/// results of `read_subdir`. Non-UTF8 compatible file names are silently ignored. +pub fn scan_subdir<'a, P: ?Sized + nix::NixPath>( + dirfd: RawFd, + path: &P, + regex: &'a regex::Regex, +) -> Result> + 'a, nix::Error> { + Ok(read_subdir(dirfd, path)?.filter_file_name_regex(regex)) +} + +/// Scan directory for matching file names with a callback. +/// +/// Scan through all directory entries and call `callback()` function +/// if the entry name matches the regular expression. This function +/// used unix `openat()`, so you can pass absolute or relative file +/// names. This function simply skips non-UTF8 encoded names. +pub fn scandir( + dirfd: RawFd, + path: &P, + regex: ®ex::Regex, + mut callback: F, +) -> Result<(), Error> +where + F: FnMut(RawFd, &str, nix::dir::Type) -> Result<(), Error>, + P: ?Sized + nix::NixPath, +{ + for entry in scan_subdir(dirfd, path, regex)? { + let entry = entry?; + let file_type = match entry.file_type() { + Some(file_type) => file_type, + None => bail!("unable to detect file type"), + }; + + callback( + entry.parent_fd(), + unsafe { entry.file_name_utf8_unchecked() }, + file_type, + )?; + } + Ok(()) +} + + +/// Helper trait to provide a combinators for directory entry iterators. +pub trait FileIterOps +where + Self: Sized + Iterator>, + T: Borrow, + E: Into + Send + Sync, +{ + /// Filter by file type. This is more convenient than using the `filter` method alone as this + /// also includes error handling and handling of files without a type (via an error). + fn filter_file_type(self, ty: dir::Type) -> FileTypeFilter { + FileTypeFilter { inner: self, ty } + } + + /// Filter by file name. Note that file names which aren't valid utf-8 will be treated as if + /// they do not match the pattern. + fn filter_file_name_regex(self, regex: &Regex) -> FileNameRegexFilter { + FileNameRegexFilter { inner: self, regex } + } +} + +impl FileIterOps for I +where + I: Iterator>, + T: Borrow, + E: Into + Send + Sync, +{ +} + +/// This filters files from its inner iterator by a file type. Files with no type produce an error. +pub struct FileTypeFilter +where + I: Iterator>, + T: Borrow, + E: Into + Send + Sync, +{ + inner: I, + ty: nix::dir::Type, +} + +impl Iterator for FileTypeFilter +where + I: Iterator>, + T: Borrow, + E: Into + Send + Sync, +{ + type Item = Result; + + fn next(&mut self) -> Option { + loop { + let item = self.inner.next()?.map_err(|e| e.into()); + match item { + Ok(ref entry) => match entry.borrow().file_type() { + Some(ty) => { + if ty == self.ty { + return Some(item); + } else { + continue; + } + } + None => return Some(Err(format_err!("unable to detect file type"))), + }, + Err(_) => return Some(item), + } + } + } +} + +/// This filters files by name via a Regex. Files whose file name aren't valid utf-8 are skipped +/// silently. +pub struct FileNameRegexFilter<'a, I, T, E> +where + I: Iterator>, + T: Borrow, +{ + inner: I, + regex: &'a Regex, +} + +impl Iterator for FileNameRegexFilter<'_, I, T, E> +where + I: Iterator>, + T: Borrow, +{ + type Item = Result; + + fn next(&mut self) -> Option { + loop { + let item = self.inner.next()?; + match item { + Ok(ref entry) => { + if let Ok(name) = entry.borrow().file_name().to_str() { + if self.regex.is_match(name) { + return Some(item); + } + } + // file did not match regex or isn't valid utf-8 + continue; + }, + Err(_) => return Some(item), + } + } + } +} + +// /usr/include/linux/fs.h: #define FS_IOC_GETFLAGS _IOR('f', 1, long) +// read Linux file system attributes (see man chattr) +nix::ioctl_read!(read_attr_fd, b'f', 1, libc::c_long); +nix::ioctl_write_ptr!(write_attr_fd, b'f', 2, libc::c_long); + +// /usr/include/linux/msdos_fs.h: #define FAT_IOCTL_GET_ATTRIBUTES _IOR('r', 0x10, __u32) +// read FAT file system attributes +nix::ioctl_read!(read_fat_attr_fd, b'r', 0x10, u32); +nix::ioctl_write_ptr!(write_fat_attr_fd, b'r', 0x11, u32); + +// From /usr/include/linux/fs.h +// #define FS_IOC_FSGETXATTR _IOR('X', 31, struct fsxattr) +// #define FS_IOC_FSSETXATTR _IOW('X', 32, struct fsxattr) +nix::ioctl_read!(fs_ioc_fsgetxattr, b'X', 31, FSXAttr); +nix::ioctl_write_ptr!(fs_ioc_fssetxattr, b'X', 32, FSXAttr); + +#[repr(C)] +#[derive(Debug)] +pub struct FSXAttr { + pub fsx_xflags: u32, + pub fsx_extsize: u32, + pub fsx_nextents: u32, + pub fsx_projid: u32, + pub fsx_cowextsize: u32, + pub fsx_pad: [u8; 8], +} + +impl Default for FSXAttr { + fn default() -> Self { + FSXAttr { + fsx_xflags: 0u32, + fsx_extsize: 0u32, + fsx_nextents: 0u32, + fsx_projid: 0u32, + fsx_cowextsize: 0u32, + fsx_pad: [0u8; 8], + } + } +} + +/// Attempt to acquire a shared flock on the given path, 'what' and +/// 'would_block_message' are used for error formatting. +pub fn lock_dir_noblock_shared( + path: &std::path::Path, + what: &str, + would_block_msg: &str, +) -> Result { + do_lock_dir_noblock(path, what, would_block_msg, false) +} + +/// Attempt to acquire an exclusive flock on the given path, 'what' and +/// 'would_block_message' are used for error formatting. +pub fn lock_dir_noblock( + path: &std::path::Path, + what: &str, + would_block_msg: &str, +) -> Result { + do_lock_dir_noblock(path, what, would_block_msg, true) +} + +fn do_lock_dir_noblock( + path: &std::path::Path, + what: &str, + would_block_msg: &str, + exclusive: bool, +) -> Result { + let mut handle = Dir::open(path, OFlag::O_RDONLY, Mode::empty()) + .map_err(|err| { + format_err!("unable to open {} directory {:?} for locking - {}", what, path, err) + })?; + + // acquire in non-blocking mode, no point in waiting here since other + // backups could still take a very long time + crate::fs::lock_file(&mut handle, exclusive, Some(std::time::Duration::from_nanos(0))) + .map_err(|err| { + format_err!( + "unable to acquire lock on {} directory {:?} - {}", what, path, + if err.would_block() { + String::from(would_block_msg) + } else { + err.to_string() + } + ) + })?; + + Ok(handle) +} + +/// Get an iterator over lines of a file, skipping empty lines and comments (lines starting with a +/// `#`). +pub fn file_get_non_comment_lines>( + path: P, +) -> Result>, Error> { + let path = path.as_ref(); + + Ok(io::BufReader::new( + File::open(path).map_err(|err| format_err!("error opening {:?}: {}", path, err))?, + ) + .lines() + .filter_map(|line| match line { + Ok(line) => { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + None + } else { + Some(Ok(line.to_string())) + } + } + Err(err) => Some(Err(err)), + })) +}