diff --git a/src/tools.rs b/src/tools.rs index 74885d35..d288e976 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -37,7 +37,6 @@ pub mod procfs; pub mod acl; pub mod xattr; pub mod vec; -pub mod io; pub mod futures; mod process_locker; diff --git a/src/tools/io.rs b/src/tools/io.rs deleted file mode 100644 index f5c390dd..00000000 --- a/src/tools/io.rs +++ /dev/null @@ -1,311 +0,0 @@ -//! Module providing I/O helpers (sync and async). -//! -//! The [`ops`](io::ops) module provides helper traits for types implementing [`Read`](std::io::Read). -//! -//! The top level functions in of this module here are used for standalone implementations of -//! various functionality which is actually intended to be available as methods to types -//! implementing `AsyncRead`, which, however, without async/await cannot be methods due to them -//! having non-static lifetimes in that case. -//! -//! ```ignore -//! use std::io; -//! -//! use crate::tools::io::read_exact_allocated; -//! use crate::tools::vec::{self, ops::*}; -//! -//! // Currently usable: -//! fn do_something() -> impl Future, Error = io::Error> { -//! tokio::fs::File::open("some.file") -//! .and_then(|file| read_exact_allocated(file, unsafe { vec::uninitialized(1024) })) -//! .and_then(|(file, mut buffer)| { -//! so_something_with(&buffer); -//! // append more data: -//! tokio::io::read_exact(file, unsafe { buffer.grow_uninitialized(1024) }) -//! }) -//! .and_then(|(_file, bigger_buffer)| { -//! // use bigger_buffer -//! Ok(()) -//! }); -//! } -//! -//! // Future async/await variant: -//! async fn do_something() -> Vec { -//! let mut file = tokio::fs::File::open("some.file").await?; -//! let mut buffer = file.read_exact_allocated(1024).await?; -//! do_something_with(buffer); -//! file.append_to_vec(&mut buffer, 1024).await?; -//! buffer -//! } -//! ``` - -use std::io; - -use futures::Future; -use futures::{Async, Poll}; -use tokio::io::AsyncRead; - -use crate::tools::vec::{self, ops::*}; - -pub mod ops; - -/// Create a future which reads an exact amount of bytes from an input. -/// -/// The future's output is a tuple containing the input and a newly allocated `Vec` containing -/// the data. -/// -/// Example: -/// ``` -/// # use futures::future::Future; -/// # use proxmox_backup::tools::io::*; -/// tokio::fs::File::open("some.file") -/// .and_then(|file| read_exact_allocated(file, 1024)) -/// .and_then(|(_file, data)| { -/// // use data -/// Ok(()) -/// }); -/// ``` -pub fn read_exact_allocated(reader: R, size: usize) -> ReadExactAllocated { - ReadExactAllocated(Some(reader), None, size) -} - -/// A future returned by [`read_exact_allocated`]. -pub struct ReadExactAllocated(Option, Option>, usize); - -impl Future for ReadExactAllocated { - type Item = (R, Vec); - type Error = io::Error; - - fn poll(&mut self) -> Poll { - assert!(self.0.is_some(), "polled after ready"); - - // allocation happens on first poll: - if self.1.is_none() { - self.1 = Some(unsafe { vec::uninitialized(self.2) }); - // now self.2 is the position: - self.2 = 0; - } - - let mut buffer = self.1.take().unwrap(); - - loop { - match self.0.as_mut().unwrap().poll_read(&mut buffer[self.2..]) { - Ok(Async::Ready(0)) => { - self.0 = None; - return Err(io::Error::from(io::ErrorKind::UnexpectedEof)); - } - Ok(Async::Ready(some)) => { - self.2 += some; - if self.2 == buffer.len() { - self.0 = None; - return Ok(Async::Ready((self.0.take().unwrap(), buffer))); - } - continue; - } - Ok(Async::NotReady) => { - self.1 = Some(buffer); - return Ok(Async::NotReady); - } - Err(err) => { - self.0 = None; - return Err(err); - } - } - } - } -} - -/// Create a future which appends up to at most `size` bytes to a vector, growing it as needed. -/// -/// This will grow the vector as if a single `.reserve(amount_to_read)` call was made and fill it -/// with as much data as a single read call will provide. -/// -/// The future's output is a tuple containing the input, the vector and the number of bytes -/// actually read. -/// -/// Example: -/// ``` -/// # use futures::future::Future; -/// # use proxmox_backup::tools::io::*; -/// tokio::fs::File::open("some.file") -/// .and_then(|file| append_to_vec(file, Vec::new(), 1024)) -/// .and_then(|(_file, data, size)| { -/// assert!(data.len() == size); -/// println!("Actually got {} bytes of data.", size); -/// // use the data -/// Ok(()) -/// }); -/// ``` -pub fn append_to_vec(reader: R, mut vector: V, size: usize) -> AppendToVec -where - R: AsyncRead, - V: AsMut>, -{ - let pos = vector.as_mut().len(); - unsafe { - vector.as_mut().grow_uninitialized(size); - } - AppendToVec(Some(reader), Some(vector), pos) -} - -pub struct AppendToVec(Option, Option, usize) -where - R: AsyncRead, - V: AsMut>; - -impl Future for AppendToVec -where - R: AsyncRead, - V: AsMut>, -{ - type Item = (R, V, usize); - type Error = io::Error; - - fn poll(&mut self) -> Poll { - assert!(self.0.is_some() && self.1.is_some(), "polled after ready"); - - let mut output = self.1.take().unwrap(); - match self.0.as_mut().unwrap().poll_read(&mut output.as_mut()[self.2..]) { - Ok(Async::Ready(some)) => { - unsafe { - output.as_mut().set_len(self.2 + some); - } - return Ok(Async::Ready((self.0.take().unwrap(), output, some))); - } - Ok(Async::NotReady) => { - self.1 = Some(output); - return Ok(Async::NotReady); - } - Err(err) => { - self.0 = None; - return Err(err); - } - } - } -} - -/// Create a future which appends an exact amount of bytes to a vector, growing it as needed. -/// -/// This will grow the vector as if a single `.reserve(amount_to_read)` call was made and fill it -/// as much data as requested. If not enough data is available, this produces an -/// [`io::Error`](std::io::Error) of kind -/// [`ErrorKind::UnexpectedEof`](std::io::ErrorKind::UnexpectedEof). -/// -/// The future's output is a tuple containing the input and the vector. -/// -/// Example: -/// ``` -/// # use futures::future::Future; -/// # use proxmox_backup::tools::io::*; -/// tokio::fs::File::open("some.file") -/// .and_then(|file| append_exact_to_vec(file, Vec::new(), 1024)) -/// .and_then(|(_file, data)| { -/// assert!(data.len() == 1024); -/// // use data -/// Ok(()) -/// }); -/// ``` -pub fn append_exact_to_vec(reader: R, mut vector: V, size: usize) -> AppendExactToVec -where - R: AsyncRead, - V: AsMut>, -{ - let pos = vector.as_mut().len(); - unsafe { - vector.as_mut().grow_uninitialized(size); - } - AppendExactToVec(Some(reader), Some(vector), pos) -} - -pub struct AppendExactToVec(Option, Option, usize) -where - R: AsyncRead, - V: AsMut>; - -impl Future for AppendExactToVec -where - R: AsyncRead, - V: AsMut>, -{ - type Item = (R, V); - type Error = io::Error; - - fn poll(&mut self) -> Poll { - assert!(self.0.is_some() && self.1.is_some(), "polled after ready"); - - let mut output = self.1.take().unwrap(); - loop { - match self.0.as_mut().unwrap().poll_read(&mut output.as_mut()[self.2..]) { - Ok(Async::Ready(0)) => { - self.0 = None; - return Err(io::Error::from(io::ErrorKind::UnexpectedEof)); - } - Ok(Async::Ready(some)) => { - self.2 += some; - if self.2 == output.as_mut().len() { - self.0 = None; - return Ok(Async::Ready((self.0.take().unwrap(), output))); - } - continue; - } - Ok(Async::NotReady) => { - self.1 = Some(output); - return Ok(Async::NotReady); - } - Err(err) => { - self.0 = None; - return Err(err); - } - } - } - } -} - - -/* - * TODO: A trait such as the one below is only useful inside `async fn`, so this partwill have to - * wait... - * - * When we have async/await we can finish this and move it into io/async_read.rs - -/// Some additional related functionality for types implementing `AsyncRead`. Note that most of -/// these methods map to functions from the [`io`](super::io) module, which are standalone -/// variants. -/// -/// This trait only works with standard futures or as part of `poll_fn` bodies, due to it requiring -/// non-static lifetimes on futures. -pub trait AsyncReadExtOps: AsyncRead + Sized { - /// Read data into a newly allocated vector. This is a shortcut for: - /// ``` - /// let mut data = Vec::with_capacity(len); - /// unsafe { - /// data.set_len(len); - /// } - /// reader.read_exact(&mut data) - /// ``` - /// - /// With this trait, we just use: - /// ``` - /// use crate::tools::vec::ops::*; - /// - /// let data = reader.read_exact_allocated(len).await?; - /// ``` - fn read_exact_allocated(&mut self, size: usize) -> ReadExactAllocated<&mut Self> { - ReadExactAllocated(crate::tools::io::read_exact_allocated(self, size)) - } -} - -impl AsyncReadExtOps for T { -} - -pub struct ReadExactAllocated(crate::tools::io::ReadExactAllocated); - -impl futures::Future for ReadExactAllocated { - type Item = Vec; - type Error = io::Error; - - fn poll(&mut self) -> futures::Poll { - let (_this, data) = futures::try_ready!(self.0.poll()); - Ok(futures::Async::Ready(data)) - } -} - */ diff --git a/src/tools/io/ops.rs b/src/tools/io/ops.rs deleted file mode 100644 index 39d4772e..00000000 --- a/src/tools/io/ops.rs +++ /dev/null @@ -1,226 +0,0 @@ -//! This module provides additional operations for handling byte buffers for types implementing -//! [`Read`](std::io::Read). -//! -//! See the [`ReadExtOps`](ops::ReadExtOps) trait for examples. - -use std::io; - -use endian_trait::Endian; - -use crate::tools::vec::{self, ops::*}; - -/// Adds some additional related functionality for types implementing [`Read`](std::io::Read). -/// -/// Particularly for reading into a newly allocated buffer, appending to a `Vec` or reading -/// values of a specific endianess (types implementing [`Endian`]). -/// -/// Examples: -/// ```no_run -/// use proxmox_backup::tools::io::ops::*; -/// -/// # fn code() -> std::io::Result<()> { -/// let mut file = std::fs::File::open("some.data")?; -/// -/// // read some bytes into a newly allocated Vec: -/// let mut data = file.read_exact_allocated(64)?; -/// -/// // appending data to a vector: -/// let actually_appended = file.append_to_vec(&mut data, 64)?; // .read() version -/// file.append_exact_to_vec(&mut data, 64)?; // .read_exact() version -/// # Ok(()) -/// # } -/// ``` -/// -/// Or for reading values of a defined representation and endianess: -/// -/// ```no_run -/// # use endian_trait::Endian; -/// # use proxmox_backup::tools::io::ops::*; -/// -/// #[derive(Endian)] -/// #[repr(C)] -/// struct Header { -/// version: u16, -/// data_size: u16, -/// } -/// -/// # fn code(mut file: std::fs::File) -> std::io::Result<()> { -/// // We have given `Header` a proper binary representation via `#[repr]`, so this is safe: -/// let header: Header = unsafe { file.read_le_value()? }; -/// let mut blob = file.read_exact_allocated(header.data_size as usize)?; -/// # Ok(()) -/// # } -/// ``` -/// -/// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html -pub trait ReadExtOps { - /// Read data into a newly allocated vector. This is a shortcut for: - /// ```ignore - /// let mut data = Vec::with_capacity(len); - /// unsafe { - /// data.set_len(len); - /// } - /// reader.read_exact(&mut data)?; - /// ``` - /// - /// With this trait, we just use: - /// ```no_run - /// use proxmox_backup::tools::io::ops::*; - /// # fn code(mut reader: std::fs::File, len: usize) -> std::io::Result<()> { - /// let data = reader.read_exact_allocated(len)?; - /// # Ok(()) - /// # } - /// ``` - fn read_exact_allocated(&mut self, size: usize) -> io::Result>; - - /// Append data to a vector, growing it as necessary. Returns the amount of data appended. - fn append_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result; - - /// Append an exact amount of data to a vector, growing it as necessary. - fn append_exact_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result<()>; - - /// Read a value with host endianess. - /// - /// This is limited to types implementing the [`Endian`] trait under the assumption that - /// this is only done for types which are supposed to be read/writable directly. - /// - /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore - /// this is considered unsafe. - /// - /// ```no_run - /// # use endian_trait::Endian; - /// use proxmox_backup::tools::io::ops::*; - /// - /// #[derive(Endian)] - /// #[repr(C, packed)] - /// struct Data { - /// value: u16, - /// count: u32, - /// } - /// - /// # fn code() -> std::io::Result<()> { - /// let mut file = std::fs::File::open("my-raw.dat")?; - /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can - /// // safely use our helper: - /// let data: Data = unsafe { file.read_host_value()? }; - /// # Ok(()) - /// # } - /// ``` - /// - /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html - unsafe fn read_host_value(&mut self) -> io::Result; - - /// Read a little endian value. - /// - /// The return type is required to implement the [`Endian`] trait, and we make the - /// assumption that this is only done for types which are supposed to be read/writable - /// directly. - /// - /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore - /// this is considered unsafe. - /// - /// ```no_run - /// # use endian_trait::Endian; - /// use proxmox_backup::tools::io::ops::*; - /// - /// #[derive(Endian)] - /// #[repr(C, packed)] - /// struct Data { - /// value: u16, - /// count: u32, - /// } - /// - /// # fn code() -> std::io::Result<()> { - /// let mut file = std::fs::File::open("my-little-endian.dat")?; - /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can - /// // safely use our helper: - /// let data: Data = unsafe { file.read_le_value()? }; - /// # Ok(()) - /// # } - /// ``` - /// - /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html - unsafe fn read_le_value(&mut self) -> io::Result; - - /// Read a big endian value. - /// - /// The return type is required to implement the [`Endian`] trait, and we make the - /// assumption that this is only done for types which are supposed to be read/writable - /// directly. - /// - /// There's no way to directly depend on a type having a specific `#[repr(...)]`, therefore - /// this is considered unsafe. - /// - /// ```no_run - /// # use endian_trait::Endian; - /// use proxmox_backup::tools::io::ops::*; - /// - /// #[derive(Endian)] - /// #[repr(C, packed)] - /// struct Data { - /// value: u16, - /// count: u32, - /// } - /// - /// # fn code() -> std::io::Result<()> { - /// let mut file = std::fs::File::open("my-big-endian.dat")?; - /// // We know `Data` has a safe binary representation (#[repr(C, packed)]), so we can - /// // safely use our helper: - /// let data: Data = unsafe { file.read_be_value()? }; - /// # Ok(()) - /// # } - /// ``` - /// - /// [`Endian`]: https://docs.rs/endian_trait/0.6/endian_trait/trait.Endian.html - unsafe fn read_be_value(&mut self) -> io::Result; -} - -impl ReadExtOps for R { - fn read_exact_allocated(&mut self, size: usize) -> io::Result> { - let mut out = unsafe { vec::uninitialized(size) }; - self.read_exact(&mut out)?; - Ok(out) - } - - fn append_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result { - let pos = out.len(); - unsafe { - out.grow_uninitialized(size); - } - let got = self.read(&mut out[pos..])?; - unsafe { - out.set_len(pos + got); - } - Ok(got) - } - - fn append_exact_to_vec(&mut self, out: &mut Vec, size: usize) -> io::Result<()> { - let pos = out.len(); - unsafe { - out.grow_uninitialized(size); - } - self.read_exact(&mut out[pos..])?; - Ok(()) - } - - unsafe fn read_host_value(&mut self) -> io::Result { - let mut value: T = std::mem::uninitialized(); - self.read_exact(std::slice::from_raw_parts_mut( - &mut value as *mut T as *mut u8, - std::mem::size_of::(), - ))?; - Ok(value) - } - - unsafe fn read_le_value(&mut self) -> io::Result { - Ok(self.read_host_value::()?. - from_le() - ) - } - - unsafe fn read_be_value(&mut self) -> io::Result { - Ok(self.read_host_value::()? - .from_be() - ) - } -}