commit 2e6520a987d2714ab8bfd5daa1e90155932efc68 Author: Wolfgang Bumiller Date: Thu Jun 6 13:47:38 2019 +0200 initial import, starting with vec & io helpers Signed-off-by: Wolfgang Bumiller diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..aa085cd8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +Cargo.lock +/target +**/*.rs.bk diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..62cc191e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +members = [ + "proxmox-tools", + "proxmox", +] diff --git a/proxmox-tools/Cargo.toml b/proxmox-tools/Cargo.toml new file mode 100644 index 00000000..7de28a8a --- /dev/null +++ b/proxmox-tools/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "proxmox-tools" +edition = "2018" +version = "0.1.0" +authors = [ + "Dietmar Maurer ", + "Wolfgang Bumiller ", +] + +[dependencies] +endian_trait = { version = "0.6", features = ["arrays"] } +libc = "0.2" +valgrind_request = { version = "1.1.0", optional = true } + +[features] +default = [ "valgrind" ] +valgrind = [ "valgrind_request" ] + +# Docs should be able to reference the proxmox crate. +[dev-dependencies] +proxmox = { path = "../proxmox" } diff --git a/proxmox-tools/src/io.rs b/proxmox-tools/src/io.rs new file mode 100644 index 00000000..878b5b5a --- /dev/null +++ b/proxmox-tools/src/io.rs @@ -0,0 +1,10 @@ +//! Module providing I/O helpers (sync and async). +//! +//! The [`ops`](io::ops) module provides helper traits for types implementing [`Read`](std::io::Read). +//! +//! The top level functions in of this module here are used for standalone implementations of +//! various functionality which is actually intended to be available as methods to types +//! implementing `AsyncRead`, which, however, without async/await cannot be methods due to them +//! having non-static lifetimes in that case. + +pub mod ops; diff --git a/proxmox-tools/src/io/ops.rs b/proxmox-tools/src/io/ops.rs new file mode 100644 index 00000000..b30ea68b --- /dev/null +++ b/proxmox-tools/src/io/ops.rs @@ -0,0 +1,226 @@ +//! This module provides additional operations for handling byte buffers for types implementing +//! [`Read`](std::io::Read). +//! +//! See the [`ReadExtOps`](ops::ReadExtOps) trait for examples. + +use std::io; + +use endian_trait::Endian; + +use crate::vec::{self, ops::*}; + +/// Adds some additional related functionality for types implementing [`Read`](std::io::Read). +/// +/// Particularly for reading into a newly allocated buffer, appending to a `Vec` or reading +/// values of a specific endianess (types implementing [`Endian`]). +/// +/// Examples: +/// ```no_run +/// use proxmox::tools::io::ops::*; +/// +/// # fn code() -> std::io::Result<()> { +/// let mut file = std::fs::File::open("some.data")?; +/// +/// // read some bytes into a newly allocated Vec: +/// let mut data = file.read_exact_allocated(64)?; +/// +/// // appending data to a vector: +/// let actually_appended = file.append_to_vec(&mut data, 64)?; // .read() version +/// file.append_exact_to_vec(&mut data, 64)?; // .read_exact() version +/// # Ok(()) +/// # } +/// ``` +/// +/// Or for reading values of a defined representation and endianess: +/// +/// ```no_run +/// # use endian_trait::Endian; +/// # use proxmox::tools::io::ops::*; +/// +/// #[derive(Endian)] +/// #[repr(C)] +/// struct Header { +/// version: u16, +/// data_size: u16, +/// } +/// +/// # fn code(mut file: std::fs::File) -> std::io::Result<()> { +/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe: +/// let header: Header = unsafe { file.read_le_value()? }; +/// let mut blob = file.read_exact_allocated(header.data_size as usize)?; +/// # Ok(()) +/// # } +/// ``` +/// +/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html +pub trait ReadExtOps { + /// Read data into a newly allocated vector. This is a shortcut for: + /// ```ignore + /// let mut data = Vec::with_capacity(len); + /// unsafe { + /// data.set_len(len); + /// } + /// reader.read_exact(&mut data)?; + /// ``` + /// + /// With this trait, we just use: + /// ```no_run + /// use proxmox::tools::io::ops::*; + /// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> { + /// let data = reader.read_exact_allocated(len)?; + /// # Ok(()) + /// # } + /// ``` + fn read_exact_allocated(&mut self, size: usize) -> io::Result>; + + /// Append data to a vector, growing it as necessary. Returns the amount of data appended. + fn append_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result; + + /// Append an exact amount of data to a vector, growing it as necessary. + fn append_exact_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result<()>; + + /// Read a value with host endianess. + /// + /// This is limited to types implementing the [`Endian`] trait under the assumption that + /// this is only done for types which are supposed to be read/writable directly. + /// + /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore + /// this is considered unsafe. + /// + /// ```no_run + /// # use endian_trait::Endian; + /// use proxmox::tools::io::ops::*; + /// + /// #[derive(Endian)] + /// #[repr(C, packed)] + /// struct Data { + /// value: u16, + /// count: u32, + /// } + /// + /// # fn code() -> std::io::Result<()> { + /// let mut file = std::fs::File::open("my-raw.dat")?; + /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can + /// // safely use our helper: + /// let data: Data = unsafe { file.read_host_value()? }; + /// # Ok(()) + /// # } + /// ``` + /// + /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html + unsafe fn read_host_value(&mut self) -> io::Result; + + /// Read a little endian value. + /// + /// The return type is required to implement the [`Endian`] trait, and we make the + /// assumption that this is only done for types which are supposed to be read/writable + /// directly. + /// + /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore + /// this is considered unsafe. + /// + /// ```no_run + /// # use endian_trait::Endian; + /// use proxmox::tools::io::ops::*; + /// + /// #[derive(Endian)] + /// #[repr(C, packed)] + /// struct Data { + /// value: u16, + /// count: u32, + /// } + /// + /// # fn code() -> std::io::Result<()> { + /// let mut file = std::fs::File::open("my-little-endian.dat")?; + /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can + /// // safely use our helper: + /// let data: Data = unsafe { file.read_le_value()? }; + /// # Ok(()) + /// # } + /// ``` + /// + /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html + unsafe fn read_le_value(&mut self) -> io::Result; + + /// Read a big endian value. + /// + /// The return type is required to implement the [`Endian`] trait, and we make the + /// assumption that this is only done for types which are supposed to be read/writable + /// directly. + /// + /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore + /// this is considered unsafe. + /// + /// ```no_run + /// # use endian_trait::Endian; + /// use proxmox::tools::io::ops::*; + /// + /// #[derive(Endian)] + /// #[repr(C, packed)] + /// struct Data { + /// value: u16, + /// count: u32, + /// } + /// + /// # fn code() -> std::io::Result<()> { + /// let mut file = std::fs::File::open("my-big-endian.dat")?; + /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can + /// // safely use our helper: + /// let data: Data = unsafe { file.read_be_value()? }; + /// # Ok(()) + /// # } + /// ``` + /// + /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html + unsafe fn read_be_value(&mut self) -> io::Result; +} + +impl ReadExtOps for R { + fn read_exact_allocated(&mut self, size: usize) -> io::Result> { + let mut out = unsafe { vec::uninitialized(size) }; + self.read_exact(&mut out)?; + Ok(out) + } + + fn append_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result { + let pos = out.len(); + unsafe { + out.grow_uninitialized(size); + } + let got = self.read(&mut out[pos..])?; + unsafe { + out.set_len(pos + got); + } + Ok(got) + } + + fn append_exact_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result<()> { + let pos = out.len(); + unsafe { + out.grow_uninitialized(size); + } + self.read_exact(&mut out[pos..])?; + Ok(()) + } + + unsafe fn read_host_value(&mut self) -> io::Result { + let mut value: T = std::mem::uninitialized(); + self.read_exact(std::slice::from_raw_parts_mut( + &mut value as *mut T as *mut u8, + std::mem::size_of::(), + ))?; + Ok(value) + } + + unsafe fn read_le_value(&mut self) -> io::Result { + Ok(self.read_host_value::()?. + from_le() + ) + } + + unsafe fn read_be_value(&mut self) -> io::Result { + Ok(self.read_host_value::()? + .from_be() + ) + } +} diff --git a/proxmox-tools/src/lib.rs b/proxmox-tools/src/lib.rs new file mode 100644 index 00000000..4df7ecd7 --- /dev/null +++ b/proxmox-tools/src/lib.rs @@ -0,0 +1,4 @@ +//! This is a general utility crate used by all our rust projects. + +pub mod vec; +pub mod io; diff --git a/proxmox-tools/src/vec.rs b/proxmox-tools/src/vec.rs new file mode 100644 index 00000000..f1869a69 --- /dev/null +++ b/proxmox-tools/src/vec.rs @@ -0,0 +1,121 @@ +//! Byte vector helpers. +//! +//! We have a lot of I/O code such as: +//! ```ignore +//! let mut buffer = vec![0u8; header_size]; +//! file.read_exact(&mut buffer)?; +//! ``` +//! (We even have this case with a 4M buffer!) +//! +//! This needlessly initializes the buffer to zero (which not only wastes time (an insane amount of +//! time on debug builds, actually) but also prevents tools such as valgrind from pointing out +//! access to actually uninitialized data, which may hide bugs...) +//! +//! This module provides some helpers for this kind of code. Many of these are supposed to stay on +//! a lower level, with I/O helpers for types implementing [`Read`](std::io::Read) being available +//! in the [`tools::io`](crate::io) module. +//! +//! Examples: +//! ```no_run +//! use proxmox::tools::vec::{self, ops::*}; +//! +//! # let size = 64usize; +//! # let more = 64usize; +//! let mut buffer = vec::undefined(size); // A zero-initialized buffer with valgrind support +//! +//! let mut buffer = unsafe { vec::uninitialized(size) }; // an actually uninitialized buffer +//! vec::clear(&mut buffer); // zero out an &mut [u8] +//! +//! vec::clear(unsafe { +//! buffer.grow_uninitialized(more) // grow the buffer with uninitialized bytes +//! }); +//! ``` + +pub mod ops; + +/// Create an uninitialized byte vector of a specific size. +/// +/// This is just a shortcut for: +/// ```no_run +/// # let len = 64usize; +/// let mut v = Vec::::with_capacity(len); +/// unsafe { +/// v.set_len(len); +/// } +/// ``` +#[inline] +pub unsafe fn uninitialized(len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + out.set_len(len); + out +} + +/// Shortcut to zero out a slice of bytes. +#[inline] +pub fn clear(data: &mut [u8]) { + unsafe { + std::ptr::write_bytes(data.as_mut_ptr(), 0, data.len()); + } +} + +/// Create a newly allocated, zero initialized byte vector. +#[inline] +pub fn zeroed(len: usize) -> Vec { + unsafe { + let mut out = uninitialized(len); + clear(&mut out); + out + } +} + +/// Create a newly allocated byte vector of a specific size with "undefined" content. +/// +/// The data will be zero initialized, but, if the `valgrind` feature is activated, it will be +/// marked as uninitialized for debugging. +#[inline] +pub fn undefined(len: usize) -> Vec { + undefined_impl(len) +} + +#[cfg(not(feature = "valgrind"))] +fn undefined_impl(len: usize) -> Vec { + zeroed(len) +} + +#[cfg(feature = "valgrind")] +fn undefined_impl(len: usize) -> Vec { + let out = zeroed(len); + vg::make_slice_undefined(&out[..]); + out +} + +#[cfg(feature = "valgrind")] +mod vg { + type ValgrindValue = valgrind_request::Value; + + /// Mark a memory region as undefined when using valgrind, causing it to treat read access to + /// it as error. + #[inline] + pub(crate) fn make_mem_undefined(addr: *const u8, len: usize) -> ValgrindValue { + const MAKE_MEM_UNDEFINED: ValgrindValue = + (((b'M' as ValgrindValue) << 24) | ((b'C' as ValgrindValue) << 16)) + 1; + unsafe { + valgrind_request::do_client_request( + 0, + &[ + MAKE_MEM_UNDEFINED, + addr as usize as ValgrindValue, + len as ValgrindValue, + 0, 0, 0, + ], + ) + } + } + + /// Mark a slice of bytes as undefined when using valgrind, causing it to treat read access to + /// it as error. + #[inline] + pub(crate) fn make_slice_undefined(data: &[u8]) -> ValgrindValue { + make_mem_undefined(data.as_ptr(), data.len()) + } +} diff --git a/proxmox-tools/src/vec/ops.rs b/proxmox-tools/src/vec/ops.rs new file mode 100644 index 00000000..4813ef2c --- /dev/null +++ b/proxmox-tools/src/vec/ops.rs @@ -0,0 +1,99 @@ +//! This module provides additional operations for `Vec`. +//! +//! Example: +//! ``` +//! # use std::io::Read; +//! use proxmox::tools::vec::{self, ops::*}; +//! +//! fn append_1024_to_vec(mut input: T, buffer: &mut Vec) -> std::io::Result<()> { +//! input.read_exact(unsafe { buffer.grow_uninitialized(1024) }) +//! } +//! ``` + +/// Some additional byte vector operations useful for I/O code. +/// Example: +/// ``` +/// # use std::io::Read; +/// # use proxmox::tools::io::{self, ops::*}; +/// use proxmox::tools::vec::{self, ops::*}; +/// +/// # fn code(mut file: std::fs::File, mut data: Vec) -> std::io::Result<()> { +/// file.read_exact(unsafe { +/// data.grow_uninitialized(1024) +/// })?; +/// # Ok(()) +/// # } +/// ``` +/// +/// Note that this module also provides a safe alternative for the case where +/// `grow_uninitialized()` is directly followed by a `read_exact()` call via the [`ReadExtOps`] +/// trait: +/// ```ignore +/// file.append_to_vec(&mut data, 1024)?; +/// ``` +/// +/// [`ReadExtOps`]: crate::io::ops::ReadExtOps +pub trait VecU8ExtOps { + /// Grow a vector without initializing its elements. The difference to simply using `reserve` + /// is that it also updates the actual length, making the newly allocated data part of the + /// slice. + /// + /// This is a shortcut for: + /// ```ignore + /// vec.reserve(more); + /// let total = vec.len() + more; + /// unsafe { + /// vec.set_len(total); + /// } + /// ``` + /// + /// This returns a mutable slice to the newly allocated space, so it can be used inline: + /// ``` + /// # use std::io::Read; + /// # use proxmox::tools::vec::ops::*; + /// # fn test(mut file: std::fs::File, buffer: &mut Vec) -> std::io::Result<()> { + /// file.read_exact(unsafe { buffer.grow_uninitialized(1024) })?; + /// # Ok(()) + /// # } + /// ``` + /// + /// Although for the above case it is recommended to use the even shorter version from the + /// [`ReadExtOps`] trait: + /// ```ignore + /// // use crate::tools::vec::ops::ReadExtOps; + /// file.append_to_vec(&mut buffer, 1024)?; + /// ``` + /// + /// [`ReadExtOps`]: crate::io::ops::ReadExtOps + unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8]; + + /// Resize a vector to a specific size without initializing its data. This is a shortcut for: + /// ```ignore + /// if new_size <= vec.len() { + /// vec.truncate(new_size); + /// } else { + /// unsafe { + /// vec.grow_uninitialized(new_size - vec.len()); + /// } + /// } + /// ``` + unsafe fn resize_uninitialized(&mut self, total: usize); +} + +impl VecU8ExtOps for Vec { + unsafe fn grow_uninitialized(&mut self, more: usize) -> &mut [u8] { + let old_len = self.len(); + self.reserve(more); + let total = old_len + more; + self.set_len(total); + &mut self[old_len..] + } + + unsafe fn resize_uninitialized(&mut self, new_size: usize) { + if new_size <= self.len() { + self.truncate(new_size); + } else { + self.grow_uninitialized(new_size - self.len()); + } + } +} diff --git a/proxmox/Cargo.toml b/proxmox/Cargo.toml new file mode 100644 index 00000000..d09bb26d --- /dev/null +++ b/proxmox/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "proxmox" +edition = "2018" +version = "0.1.0" +authors = [ + "Dietmar Maurer ", + "Wolfgang Bumiller ", +] + +[dependencies] +proxmox-tools = { path = "../proxmox-tools" } diff --git a/proxmox/src/lib.rs b/proxmox/src/lib.rs new file mode 100644 index 00000000..5818021d --- /dev/null +++ b/proxmox/src/lib.rs @@ -0,0 +1 @@ +pub use proxmox_tools as tools; diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 00000000..bf867e0a --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +nightly diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..32a9786f --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +edition = "2018"