Updated version 1.19.0+dfsg2 from 'upstream/1.19.0+dfsg2'

with Debian dir b98cddaff1
This commit is contained in:
Ximin Luo 2017-07-25 11:28:40 +02:00
commit c2a9bcb1b4
1588 changed files with 4475 additions and 340692 deletions

View File

@ -0,0 +1,4 @@
language: rust
script:
- cargo build
- ./test.sh

View File

@ -0,0 +1,25 @@
[package]
authors = ["The Rust Project Developers"]
name = "installer"
version = "0.0.0"
[[bin]]
doc = false
name = "rust-installer"
path = "src/main.rs"
[dependencies]
error-chain = "0.10.0"
flate2 = "0.2.19"
tar = "0.4.13"
walkdir = "1.0.7"
xz2 = "0.1.3"
[dependencies.clap]
features = ["yaml"]
version = "2.19.0"
[target."cfg(windows)".dependencies]
lazy_static = "0.2.8"
kernel32-sys = "0.2.2"
winapi = "0.2.8"

View File

@ -0,0 +1,71 @@
[![Build Status](https://travis-ci.org/rust-lang/rust-installer.svg?branch=master)](https://travis-ci.org/rust-lang/rust-installer)
A generator for the install.sh script commonly used to install Rust in
Unix environments. It is used By Rust, Cargo, and is intended to be
used by a future combined installer of Rust + Cargo.
# Usage
```
./gen-installer.sh --product-name=Rust \
--rel-manifest-dir=rustlib \
--success-message=Rust-is-ready-to-roll. \
--image-dir=./install-image \
--work-dir=./temp \
--output-dir=./dist \
--non-installed-overlay=./overlay \
--package-name=rustc-nightly-i686-apple-darwin \
--component-name=rustc \
--legacy-manifest-dirs=rustlib \
--bulk-dirs=share/doc
```
Or, to just generate the script.
```
./gen-install-script.sh --product-name=Rust \
--rel-manifest-dir=rustlib \
--success-message=Rust-is-ready-to-roll. \
--output-script=install.sh \
--legacy-manifest-dirs=rustlib
```
*Note: the dashes in `success-message` are converted to spaces. The
script's argument handling is broken with spaces.*
To combine installers.
```
./combine-installers.sh --product-name=Rust \
--rel-manifest-dir=rustlib \
--success-message=Rust-is-ready-to-roll. \
--work-dir=./temp \
--output-dir=./dist \
--non-installed-overlay=./overlay \
--package-name=rustc-nightly-i686-apple-darwin \
--legacy-manifest-dirs=rustlib \
--input-tarballs=./rustc.tar.gz,cargo.tar.gz
```
# Future work
* Make install.sh not have to be customized, pull it's data from a
config file.
* Be more resiliant to installation failures, particularly if the disk
is full.
* Pre-install and post-uninstall scripts.
* Allow components to depend on or contradict other components.
* Sanity check that expected destination dirs (bin, lib, share exist)?
* Add --docdir flag. Is there a standard name for this?
* Remove empty directories on uninstall.
* Detect mismatches in --prefix, --mandir, etc. in follow-on
installs/uninstalls.
* Fix argument handling for spaces.
* Add --bindir.
# License
This software is distributed under the terms of both the MIT license
and/or the Apache License (Version 2.0), at your option.
See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details.

View File

@ -0,0 +1,24 @@
#!/bin/sh
# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
set -ue
# Prints the absolute path of a directory to stdout
abs_path() {
local path="$1"
# Unset CDPATH because it causes havok: it makes the destination unpredictable
# and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
# for good measure.
(unset CDPATH && cd "$path" > /dev/null && pwd)
}
src_dir="$(abs_path $(dirname "$0"))"
cargo run --manifest-path="$src_dir/Cargo.toml" -- combine "$@"

View File

@ -0,0 +1,24 @@
#!/bin/sh
# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
set -ue
# Prints the absolute path of a directory to stdout
abs_path() {
local path="$1"
# Unset CDPATH because it causes havok: it makes the destination unpredictable
# and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
# for good measure.
(unset CDPATH && cd "$path" > /dev/null && pwd)
}
src_dir="$(abs_path $(dirname "$0"))"
cargo run --manifest-path="$src_dir/Cargo.toml" -- script "$@"

View File

@ -0,0 +1,24 @@
#!/bin/sh
# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
set -ue
# Prints the absolute path of a directory to stdout
abs_path() {
local path="$1"
# Unset CDPATH because it causes havok: it makes the destination unpredictable
# and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
# for good measure.
(unset CDPATH && cd "$path" > /dev/null && pwd)
}
src_dir="$(abs_path $(dirname "$0"))"
cargo run --manifest-path="$src_dir/Cargo.toml" -- generate "$@"

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,24 @@
#!/bin/sh
# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
set -ue
# Prints the absolute path of a directory to stdout
abs_path() {
local path="$1"
# Unset CDPATH because it causes havok: it makes the destination unpredictable
# and triggers 'cd' to print the path to stdout. Route `cd`'s output to /dev/null
# for good measure.
(unset CDPATH && cd "$path" > /dev/null && pwd)
}
src_dir="$(abs_path $(dirname "$0"))"
cargo run --manifest-path="$src_dir/Cargo.toml" -- tarball "$@"

View File

@ -0,0 +1 @@
3

View File

@ -0,0 +1,137 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::io::{Read, Write};
use std::path::Path;
use flate2::read::GzDecoder;
use tar::Archive;
use errors::*;
use super::Scripter;
use super::Tarballer;
use util::*;
actor!{
#[derive(Debug)]
pub struct Combiner {
/// The name of the product, for display
product_name: String = "Product",
/// The name of the package, tarball
package_name: String = "package",
/// The directory under lib/ where the manifest lives
rel_manifest_dir: String = "packagelib",
/// The string to print after successful installation
success_message: String = "Installed.",
/// Places to look for legacy manifests to uninstall
legacy_manifest_dirs: String = "",
/// Installers to combine
input_tarballs: String = "",
/// Directory containing files that should not be installed
non_installed_overlay: String = "",
/// The directory to do temporary work
work_dir: String = "./workdir",
/// The location to put the final image and tarball
output_dir: String = "./dist",
}
}
impl Combiner {
/// Combine the installer tarballs
pub fn run(self) -> Result<()> {
create_dir_all(&self.work_dir)?;
let package_dir = Path::new(&self.work_dir).join(&self.package_name);
if package_dir.exists() {
remove_dir_all(&package_dir)?;
}
create_dir_all(&package_dir)?;
// Merge each installer into the work directory of the new installer
let components = create_new_file(package_dir.join("components"))?;
for input_tarball in self.input_tarballs.split(',').map(str::trim).filter(|s| !s.is_empty()) {
// Extract the input tarballs
GzDecoder::new(open_file(&input_tarball)?)
.and_then(|tar| Archive::new(tar).unpack(&self.work_dir))
.chain_err(|| format!("unable to extract '{}' into '{}'",
&input_tarball, self.work_dir))?;
let pkg_name = input_tarball.trim_right_matches(".tar.gz");
let pkg_name = Path::new(pkg_name).file_name().unwrap();
let pkg_dir = Path::new(&self.work_dir).join(&pkg_name);
// Verify the version number
let mut version = String::new();
open_file(pkg_dir.join("rust-installer-version"))
.and_then(|mut file| file.read_to_string(&mut version).map_err(Error::from))
.chain_err(|| format!("failed to read version in '{}'", input_tarball))?;
if version.trim().parse() != Ok(::RUST_INSTALLER_VERSION) {
bail!("incorrect installer version in {}", input_tarball);
}
// Copy components to the new combined installer
let mut pkg_components = String::new();
open_file(pkg_dir.join("components"))
.and_then(|mut file| file.read_to_string(&mut pkg_components).map_err(Error::from))
.chain_err(|| format!("failed to read components in '{}'", input_tarball))?;
for component in pkg_components.split_whitespace() {
// All we need to do is copy the component directory. We could
// move it, but rustbuild wants to reuse the unpacked package
// dir for OS-specific installers on macOS and Windows.
let component_dir = package_dir.join(&component);
create_dir(&component_dir)?;
copy_recursive(&pkg_dir.join(&component), &component_dir)?;
// Merge the component name
writeln!(&components, "{}", component)
.chain_err(|| "failed to write new components")?;
}
}
drop(components);
// Write the installer version
let version = package_dir.join("rust-installer-version");
writeln!(create_new_file(version)?, "{}", ::RUST_INSTALLER_VERSION)
.chain_err(|| "failed to write new installer version")?;
// Copy the overlay
if !self.non_installed_overlay.is_empty() {
copy_recursive(self.non_installed_overlay.as_ref(), &package_dir)?;
}
// Generate the install script
let output_script = package_dir.join("install.sh");
let mut scripter = Scripter::default();
scripter.product_name(self.product_name)
.rel_manifest_dir(self.rel_manifest_dir)
.success_message(self.success_message)
.legacy_manifest_dirs(self.legacy_manifest_dirs)
.output_script(path_to_str(&output_script)?);
scripter.run()?;
// Make the tarballs
create_dir_all(&self.output_dir)?;
let output = Path::new(&self.output_dir).join(&self.package_name);
let mut tarballer = Tarballer::default();
tarballer.work_dir(self.work_dir)
.input(self.package_name)
.output(path_to_str(&output)?);
tarballer.run()?;
Ok(())
}
}

View File

@ -0,0 +1,146 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::io::Write;
use std::path::Path;
use errors::*;
use super::Scripter;
use super::Tarballer;
use util::*;
actor!{
#[derive(Debug)]
pub struct Generator {
/// The name of the product, for display
product_name: String = "Product",
/// The name of the component, distinct from other installed components
component_name: String = "component",
/// The name of the package, tarball
package_name: String = "package",
/// The directory under lib/ where the manifest lives
rel_manifest_dir: String = "packagelib",
/// The string to print after successful installation
success_message: String = "Installed.",
/// Places to look for legacy manifests to uninstall
legacy_manifest_dirs: String = "",
/// Directory containing files that should not be installed
non_installed_overlay: String = "",
/// Path prefixes of directories that should be installed/uninstalled in bulk
bulk_dirs: String = "",
/// The directory containing the installation medium
image_dir: String = "./install_image",
/// The directory to do temporary work
work_dir: String = "./workdir",
/// The location to put the final image and tarball
output_dir: String = "./dist",
}
}
impl Generator {
/// Generate the actual installer tarball
pub fn run(self) -> Result<()> {
create_dir_all(&self.work_dir)?;
let package_dir = Path::new(&self.work_dir).join(&self.package_name);
if package_dir.exists() {
remove_dir_all(&package_dir)?;
}
// Copy the image and write the manifest
let component_dir = package_dir.join(&self.component_name);
create_dir_all(&component_dir)?;
copy_and_manifest(self.image_dir.as_ref(), &component_dir, &self.bulk_dirs)?;
// Write the component name
let components = package_dir.join("components");
writeln!(create_new_file(components)?, "{}", self.component_name)
.chain_err(|| "failed to write the component file")?;
// Write the installer version (only used by combine-installers.sh)
let version = package_dir.join("rust-installer-version");
writeln!(create_new_file(version)?, "{}", ::RUST_INSTALLER_VERSION)
.chain_err(|| "failed to write new installer version")?;
// Copy the overlay
if !self.non_installed_overlay.is_empty() {
copy_recursive(self.non_installed_overlay.as_ref(), &package_dir)?;
}
// Generate the install script
let output_script = package_dir.join("install.sh");
let mut scripter = Scripter::default();
scripter.product_name(self.product_name)
.rel_manifest_dir(self.rel_manifest_dir)
.success_message(self.success_message)
.legacy_manifest_dirs(self.legacy_manifest_dirs)
.output_script(path_to_str(&output_script)?);
scripter.run()?;
// Make the tarballs
create_dir_all(&self.output_dir)?;
let output = Path::new(&self.output_dir).join(&self.package_name);
let mut tarballer = Tarballer::default();
tarballer.work_dir(self.work_dir)
.input(self.package_name)
.output(path_to_str(&output)?);
tarballer.run()?;
Ok(())
}
}
/// Copies the `src` directory recursively to `dst`, writing `manifest.in` too.
fn copy_and_manifest(src: &Path, dst: &Path, bulk_dirs: &str) -> Result<()> {
let manifest = create_new_file(dst.join("manifest.in"))?;
let bulk_dirs: Vec<_> = bulk_dirs.split(',')
.filter(|s| !s.is_empty())
.map(Path::new).collect();
copy_with_callback(src, dst, |path, file_type| {
// We need paths to be compatible with both Unix and Windows.
if path.components().filter_map(|c| c.as_os_str().to_str()).any(|s| s.contains('\\')) {
bail!("rust-installer doesn't support '\\' in path components: {:?}", path);
}
// Normalize to Unix-style path separators.
let normalized_string;
let mut string = path.to_str().ok_or_else(|| {
format!("rust-installer doesn't support non-Unicode paths: {:?}", path)
})?;
if string.contains('\\') {
normalized_string = string.replace('\\', "/");
string = &normalized_string;
}
if file_type.is_dir() {
// Only manifest directories that are explicitly bulk.
if bulk_dirs.contains(&path) {
writeln!(&manifest, "dir:{}", string)?;
}
} else {
// Only manifest files that aren't under bulk directories.
if !bulk_dirs.iter().any(|d| path.starts_with(d)) {
writeln!(&manifest, "file:{}", string)?;
}
}
Ok(())
})
}

View File

@ -0,0 +1,55 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#[macro_use]
extern crate error_chain;
extern crate flate2;
extern crate tar;
extern crate walkdir;
extern crate xz2;
#[cfg(windows)]
extern crate winapi;
#[cfg(windows)]
extern crate kernel32;
#[cfg(windows)]
#[macro_use]
extern crate lazy_static;
mod errors {
error_chain!{
foreign_links {
Io(::std::io::Error);
StripPrefix(::std::path::StripPrefixError);
WalkDir(::walkdir::Error);
}
}
}
#[macro_use]
mod util;
// deal with OS complications (cribbed from rustup.rs)
mod remove_dir_all;
mod combiner;
mod generator;
mod scripter;
mod tarballer;
pub use errors::{Result, Error, ErrorKind};
pub use combiner::Combiner;
pub use generator::Generator;
pub use scripter::Scripter;
pub use tarballer::Tarballer;
/// The installer version, output only to be used by combine-installers.sh.
/// (should match `SOURCE_DIRECTORY/rust_installer_version`)
pub const RUST_INSTALLER_VERSION: u32 = 3;

View File

@ -0,0 +1,98 @@
#[macro_use]
extern crate clap;
#[macro_use]
extern crate error_chain;
extern crate installer;
use errors::*;
use clap::{App, ArgMatches};
mod errors {
error_chain!{
links {
Installer(::installer::Error, ::installer::ErrorKind);
}
}
}
quick_main!(run);
fn run() -> Result<()> {
let yaml = load_yaml!("main.yml");
let matches = App::from_yaml(yaml).get_matches();
match matches.subcommand() {
("combine", Some(matches)) => combine(matches),
("generate", Some(matches)) => generate(matches),
("script", Some(matches)) => script(matches),
("tarball", Some(matches)) => tarball(matches),
_ => unreachable!(),
}
}
/// Parse clap arguements into the type constructor.
macro_rules! parse(
($matches:expr => $type:ty { $( $option:tt => $setter:ident, )* }) => {
{
let mut command: $type = Default::default();
$( $matches.value_of($option).map(|s| command.$setter(s)); )*
command
}
}
);
fn combine(matches: &ArgMatches) -> Result<()> {
let combiner = parse!(matches => installer::Combiner {
"product-name" => product_name,
"package-name" => package_name,
"rel-manifest-dir" => rel_manifest_dir,
"success-message" => success_message,
"legacy-manifest-dirs" => legacy_manifest_dirs,
"input-tarballs" => input_tarballs,
"non-installed-overlay" => non_installed_overlay,
"work-dir" => work_dir,
"output-dir" => output_dir,
});
combiner.run().chain_err(|| "failed to combine installers")
}
fn generate(matches: &ArgMatches) -> Result<()> {
let generator = parse!(matches => installer::Generator {
"product-name" => product_name,
"component-name" => component_name,
"package-name" => package_name,
"rel-manifest-dir" => rel_manifest_dir,
"success-message" => success_message,
"legacy-manifest-dirs" => legacy_manifest_dirs,
"non-installed-overlay" => non_installed_overlay,
"bulk-dirs" => bulk_dirs,
"image-dir" => image_dir,
"work-dir" => work_dir,
"output-dir" => output_dir,
});
generator.run().chain_err(|| "failed to generate installer")
}
fn script(matches: &ArgMatches) -> Result<()> {
let scripter = parse!(matches => installer::Scripter {
"product-name" => product_name,
"rel-manifest-dir" => rel_manifest_dir,
"success-message" => success_message,
"legacy-manifest-dirs" => legacy_manifest_dirs,
"output-script" => output_script,
});
scripter.run().chain_err(|| "failed to generate installation script")
}
fn tarball(matches: &ArgMatches) -> Result<()> {
let tarballer = parse!(matches => installer::Tarballer {
"input" => input,
"output" => output,
"work-dir" => work_dir,
});
tarballer.run().chain_err(|| "failed to generate tarballs")
}

View File

@ -0,0 +1,157 @@
name: installer
settings:
- ArgRequiredElseHelp
subcommands:
- generate:
about: Generate a complete installer tarball
args:
- product-name:
help: The name of the product, for display
long: product-name
takes_value: true
value_name: NAME
- component-name:
help: The name of the component, distinct from other installed components
long: component-name
takes_value: true
value_name: NAME
- package-name:
help: The name of the package, tarball
long: package-name
takes_value: true
value_name: NAME
- rel-manifest-dir:
help: The directory under lib/ where the manifest lives
long: rel-manifest-dir
takes_value: true
value_name: DIR
- success-message:
help: The string to print after successful installation
long: success-message
takes_value: true
value_name: MESSAGE
- legacy-manifest-dirs:
help: Places to look for legacy manifests to uninstall
long: legacy-manifest-dirs
takes_value: true
value_name: DIRS
- non-installed-overlay:
help: Directory containing files that should not be installed
long: non-installed-overlay
takes_value: true
value_name: DIR
- bulk-dirs:
help: Path prefixes of directories that should be installed/uninstalled in bulk
long: bulk-dirs
takes_value: true
value_name: DIRS
- image-dir:
help: The directory containing the installation medium
long: image-dir
takes_value: true
value_name: DIR
- work-dir:
help: The directory to do temporary work
long: work-dir
takes_value: true
value_name: DIR
- output-dir:
help: The location to put the final image and tarball
long: output-dir
takes_value: true
value_name: DIR
- combine:
about: Combine installer tarballs
args:
- product-name:
help: The name of the product, for display
long: product-name
takes_value: true
value_name: NAME
- package-name:
help: The name of the package, tarball
long: package-name
takes_value: true
value_name: NAME
- rel-manifest-dir:
help: The directory under lib/ where the manifest lives
long: rel-manifest-dir
takes_value: true
value_name: DIR
- success-message:
help: The string to print after successful installation
long: success-message
takes_value: true
value_name: MESSAGE
- legacy-manifest-dirs:
help: Places to look for legacy manifests to uninstall
long: legacy-manifest-dirs
takes_value: true
value_name: DIRS
- input-tarballs:
help: Installers to combine
long: input-tarballs
takes_value: true
value_name: FILE,FILE
- non-installed-overlay:
help: Directory containing files that should not be installed
long: non-installed-overlay
takes_value: true
value_name: DIR
- work-dir:
help: The directory to do temporary work
long: work-dir
takes_value: true
value_name: DIR
- output-dir:
help: The location to put the final image and tarball
long: output-dir
takes_value: true
value_name: DIR
- script:
about: Generate an installation script
args:
- product-name:
help: The name of the product, for display
long: product-name
takes_value: true
value_name: NAME
- rel-manifest-dir:
help: The directory under lib/ where the manifest lives
long: rel-manifest-dir
takes_value: true
value_name: DIR
- success-message:
help: The string to print after successful installation
long: success-message
takes_value: true
value_name: MESSAGE
- legacy-manifest-dirs:
help: Places to look for legacy manifests to uninstall
long: legacy-manifest-dirs
takes_value: true
value_name: DIRS
- output-script:
help: The name of the output script
long: output-script
takes_value: true
value_name: FILE
- tarball:
about: Generate package tarballs
args:
- input:
help: The input folder to be compressed
long: input
takes_value: true
value_name: NAME
- output:
help: The prefix of the tarballs
long: output
takes_value: true
value_name: PATH
- work-dir:
help: The folder in which the input is to be found
long: work-dir
takes_value: true
value_name: DIR

View File

@ -0,0 +1,835 @@
// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
#![allow(non_snake_case)]
use std::path::Path;
use std::io;
#[cfg(not(windows))]
pub fn remove_dir_all(path: &Path) -> io::Result<()> {
::std::fs::remove_dir_all(path)
}
#[cfg(windows)]
pub fn remove_dir_all(path: &Path) -> io::Result<()> {
win::remove_dir_all(path)
}
#[cfg(windows)]
mod win {
use winapi::{
FileBasicInfo,
FILE_BASIC_INFO,
FALSE,
FileRenameInfo,
FILE_RENAME_INFO,
c_ushort,
c_uint,
FILETIME,
FILE_ATTRIBUTE_READONLY,
FILE_ATTRIBUTE_REPARSE_POINT,
FILE_ATTRIBUTE_DIRECTORY,
WIN32_FIND_DATAW,
ERROR_NO_MORE_FILES,
OPEN_EXISTING,
OPEN_ALWAYS,
TRUNCATE_EXISTING,
CREATE_ALWAYS,
CREATE_NEW,
GENERIC_READ,
GENERIC_WRITE,
FILE_GENERIC_WRITE,
FILE_WRITE_DATA,
FILE_SHARE_READ,
FILE_SHARE_WRITE,
FILE_SHARE_DELETE,
FILE_FLAG_DELETE_ON_CLOSE,
DELETE,
FILE_WRITE_ATTRIBUTES,
FILE_INFO_BY_HANDLE_CLASS,
HANDLE,
ERROR_INSUFFICIENT_BUFFER,
FILE_READ_ATTRIBUTES,
FILE_FLAG_BACKUP_SEMANTICS,
FILE_FLAG_OPEN_REPARSE_POINT,
ERROR_CALL_NOT_IMPLEMENTED,
DWORD,
BOOL,
LPVOID,
INVALID_HANDLE_VALUE,
LPCWSTR,
SECURITY_SQOS_PRESENT,
FSCTL_GET_REPARSE_POINT,
BY_HANDLE_FILE_INFORMATION,
IO_REPARSE_TAG_SYMLINK,
IO_REPARSE_TAG_MOUNT_POINT,
};
use kernel32::{
CreateFileW,
GetFileInformationByHandle,
CloseHandle,
GetLastError,
SetLastError,
DeviceIoControl,
GetModuleHandleW,
GetProcAddress,
FindNextFileW,
FindFirstFileW,
};
use std::ptr;
use std::sync::Arc;
use std::path::{PathBuf, Path};
use std::mem;
use std::io;
use std::ffi::{OsStr, OsString};
use std::os::windows::ffi::{OsStrExt, OsStringExt};
pub fn remove_dir_all(path: &Path) -> io::Result<()> {
// On Windows it is not enough to just recursively remove the contents of a
// directory and then the directory itself. Deleting does not happen
// instantaneously, but is scheduled.
// To work around this, we move the file or directory to some `base_dir`
// right before deletion to avoid races.
//
// As `base_dir` we choose the parent dir of the directory we want to
// remove. We very probably have permission to create files here, as we
// already need write permission in this dir to delete the directory. And it
// should be on the same volume.
//
// To handle files with names like `CON` and `morse .. .`, and when a
// directory structure is so deep it needs long path names the path is first
// converted to a `//?/`-path with `get_path()`.
//
// To make sure we don't leave a moved file laying around if the process
// crashes before we can delete the file, we do all operations on an file
// handle. By opening a file with `FILE_FLAG_DELETE_ON_CLOSE` Windows will
// always delete the file when the handle closes.
//
// All files are renamed to be in the `base_dir`, and have their name
// changed to "rm-<counter>". After every rename the counter is increased.
// Rename should not overwrite possibly existing files in the base dir. So
// if it fails with `AlreadyExists`, we just increase the counter and try
// again.
//
// For read-only files and directories we first have to remove the read-only
// attribute before we can move or delete them. This also removes the
// attribute from possible hardlinks to the file, so just before closing we
// restore the read-only attribute.
//
// If 'path' points to a directory symlink or junction we should not
// recursively remove the target of the link, but only the link itself.
//
// Moving and deleting is guaranteed to succeed if we are able to open the
// file with `DELETE` permission. If others have the file open we only have
// `DELETE` permission if they have specified `FILE_SHARE_DELETE`. We can
// also delete the file now, but it will not disappear until all others have
// closed the file. But no-one can open the file after we have flagged it
// for deletion.
// Open the path once to get the canonical path, file type and attributes.
let (path, metadata) = {
let mut opts = OpenOptions::new();
opts.access_mode(FILE_READ_ATTRIBUTES);
opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS |
FILE_FLAG_OPEN_REPARSE_POINT);
let file = try!(File::open(path, &opts));
(try!(get_path(&file)), try!(file.file_attr()))
};
let mut ctx = RmdirContext {
base_dir: match path.parent() {
Some(dir) => dir,
None => return Err(io::Error::new(io::ErrorKind::PermissionDenied,
"can't delete root directory"))
},
readonly: metadata.perm().readonly(),
counter: 0,
};
let filetype = metadata.file_type();
if filetype.is_dir() {
remove_dir_all_recursive(path.as_ref(), &mut ctx)
} else if filetype.is_symlink_dir() {
remove_item(path.as_ref(), &mut ctx)
} else {
Err(io::Error::new(io::ErrorKind::PermissionDenied, "Not a directory"))
}
}
fn readdir(p: &Path) -> io::Result<ReadDir> {
let root = p.to_path_buf();
let star = p.join("*");
let path = try!(to_u16s(&star));
unsafe {
let mut wfd = mem::zeroed();
let find_handle = FindFirstFileW(path.as_ptr(), &mut wfd);
if find_handle != INVALID_HANDLE_VALUE {
Ok(ReadDir {
handle: FindNextFileHandle(find_handle),
root: Arc::new(root),
first: Some(wfd),
})
} else {
Err(io::Error::last_os_error())
}
}
}
struct RmdirContext<'a> {
base_dir: &'a Path,
readonly: bool,
counter: u64,
}
fn remove_dir_all_recursive(path: &Path, ctx: &mut RmdirContext)
-> io::Result<()> {
let dir_readonly = ctx.readonly;
for child in try!(readdir(path)) {
let child = try!(child);
let child_type = try!(child.file_type());
ctx.readonly = try!(child.metadata()).perm().readonly();
if child_type.is_dir() {
try!(remove_dir_all_recursive(&child.path(), ctx));
} else {
try!(remove_item(&child.path().as_ref(), ctx));
}
}
ctx.readonly = dir_readonly;
remove_item(path, ctx)
}
fn remove_item(path: &Path, ctx: &mut RmdirContext) -> io::Result<()> {
if !ctx.readonly {
let mut opts = OpenOptions::new();
opts.access_mode(DELETE);
opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | // delete directory
FILE_FLAG_OPEN_REPARSE_POINT | // delete symlink
FILE_FLAG_DELETE_ON_CLOSE);
let file = try!(File::open(path, &opts));
move_item(&file, ctx)
} else {
// remove read-only permision
try!(set_perm(&path, FilePermissions::new()));
// move and delete file, similar to !readonly.
// only the access mode is different.
let mut opts = OpenOptions::new();
opts.access_mode(DELETE | FILE_WRITE_ATTRIBUTES);
opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS |
FILE_FLAG_OPEN_REPARSE_POINT |
FILE_FLAG_DELETE_ON_CLOSE);
let file = try!(File::open(path, &opts));
try!(move_item(&file, ctx));
// restore read-only flag just in case there are other hard links
let mut perm = FilePermissions::new();
perm.set_readonly(true);
let _ = file.set_perm(perm); // ignore if this fails
Ok(())
}
}
macro_rules! compat_fn {
($module:ident: $(
fn $symbol:ident($($argname:ident: $argtype:ty),*)
-> $rettype:ty {
$($body:expr);*
}
)*) => ($(
#[allow(unused_variables)]
unsafe fn $symbol($($argname: $argtype),*) -> $rettype {
use std::sync::atomic::{AtomicUsize, Ordering};
use std::mem;
use std::ffi::CString;
type F = unsafe extern "system" fn($($argtype),*) -> $rettype;
lazy_static! { static ref PTR: AtomicUsize = AtomicUsize::new(0);}
fn lookup(module: &str, symbol: &str) -> Option<usize> {
let mut module: Vec<u16> = module.encode_utf16().collect();
module.push(0);
let symbol = CString::new(symbol).unwrap();
unsafe {
let handle = GetModuleHandleW(module.as_ptr());
match GetProcAddress(handle, symbol.as_ptr()) as usize {
0 => None,
n => Some(n),
}
}
}
fn store_func(ptr: &AtomicUsize, module: &str, symbol: &str,
fallback: usize) -> usize {
let value = lookup(module, symbol).unwrap_or(fallback);
ptr.store(value, Ordering::SeqCst);
value
}
fn load() -> usize {
store_func(&PTR, stringify!($module), stringify!($symbol), fallback as usize)
}
unsafe extern "system" fn fallback($($argname: $argtype),*)
-> $rettype {
$($body);*
}
let addr = match PTR.load(Ordering::SeqCst) {
0 => load(),
n => n,
};
mem::transmute::<usize, F>(addr)($($argname),*)
}
)*)
}
compat_fn! {
kernel32:
fn GetFinalPathNameByHandleW(_hFile: HANDLE,
_lpszFilePath: LPCWSTR,
_cchFilePath: DWORD,
_dwFlags: DWORD) -> DWORD {
SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0
}
fn SetFileInformationByHandle(_hFile: HANDLE,
_FileInformationClass: FILE_INFO_BY_HANDLE_CLASS,
_lpFileInformation: LPVOID,
_dwBufferSize: DWORD) -> BOOL {
SetLastError(ERROR_CALL_NOT_IMPLEMENTED as DWORD); 0
}
}
fn cvt(i: i32) -> io::Result<i32> {
if i == 0 {
Err(io::Error::last_os_error())
} else {
Ok(i)
}
}
fn to_u16s<S: AsRef<OsStr>>(s: S) -> io::Result<Vec<u16>> {
fn inner(s: &OsStr) -> io::Result<Vec<u16>> {
let mut maybe_result: Vec<u16> = s.encode_wide().collect();
if maybe_result.iter().any(|&u| u == 0) {
return Err(io::Error::new(io::ErrorKind::InvalidInput,
"strings passed to WinAPI cannot contain NULs"));
}
maybe_result.push(0);
Ok(maybe_result)
}
inner(s.as_ref())
}
fn truncate_utf16_at_nul<'a>(v: &'a [u16]) -> &'a [u16] {
match v.iter().position(|c| *c == 0) {
// don't include the 0
Some(i) => &v[..i],
None => v
}
}
fn fill_utf16_buf<F1, F2, T>(mut f1: F1, f2: F2) -> io::Result<T>
where F1: FnMut(*mut u16, DWORD) -> DWORD,
F2: FnOnce(&[u16]) -> T
{
// Start off with a stack buf but then spill over to the heap if we end up
// needing more space.
let mut stack_buf = [0u16; 512];
let mut heap_buf = Vec::new();
unsafe {
let mut n = stack_buf.len();
loop {
let buf = if n <= stack_buf.len() {
&mut stack_buf[..]
} else {
let extra = n - heap_buf.len();
heap_buf.reserve(extra);
heap_buf.set_len(n);
&mut heap_buf[..]
};
// This function is typically called on windows API functions which
// will return the correct length of the string, but these functions
// also return the `0` on error. In some cases, however, the
// returned "correct length" may actually be 0!
//
// To handle this case we call `SetLastError` to reset it to 0 and
// then check it again if we get the "0 error value". If the "last
// error" is still 0 then we interpret it as a 0 length buffer and
// not an actual error.
SetLastError(0);
let k = match f1(buf.as_mut_ptr(), n as DWORD) {
0 if GetLastError() == 0 => 0,
0 => return Err(io::Error::last_os_error()),
n => n,
} as usize;
if k == n && GetLastError() == ERROR_INSUFFICIENT_BUFFER {
n *= 2;
} else if k >= n {
n = k;
} else {
return Ok(f2(&buf[..k]))
}
}
}
}
#[derive(Clone, PartialEq, Eq, Debug, Default)]
struct FilePermissions { readonly: bool }
impl FilePermissions {
fn new() -> FilePermissions { Default::default() }
fn readonly(&self) -> bool { self.readonly }
fn set_readonly(&mut self, readonly: bool) { self.readonly = readonly }
}
#[derive(Clone)]
struct OpenOptions {
// generic
read: bool,
write: bool,
append: bool,
truncate: bool,
create: bool,
create_new: bool,
// system-specific
custom_flags: u32,
access_mode: Option<DWORD>,
attributes: DWORD,
share_mode: DWORD,
security_qos_flags: DWORD,
security_attributes: usize, // FIXME: should be a reference
}
impl OpenOptions {
fn new() -> OpenOptions {
OpenOptions {
// generic
read: false,
write: false,
append: false,
truncate: false,
create: false,
create_new: false,
// system-specific
custom_flags: 0,
access_mode: None,
share_mode: FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
attributes: 0,
security_qos_flags: 0,
security_attributes: 0,
}
}
fn custom_flags(&mut self, flags: u32) { self.custom_flags = flags; }
fn access_mode(&mut self, access_mode: u32) { self.access_mode = Some(access_mode); }
fn get_access_mode(&self) -> io::Result<DWORD> {
const ERROR_INVALID_PARAMETER: i32 = 87;
match (self.read, self.write, self.append, self.access_mode) {
(_, _, _, Some(mode)) => Ok(mode),
(true, false, false, None) => Ok(GENERIC_READ),
(false, true, false, None) => Ok(GENERIC_WRITE),
(true, true, false, None) => Ok(GENERIC_READ | GENERIC_WRITE),
(false, _, true, None) => Ok(FILE_GENERIC_WRITE & !FILE_WRITE_DATA),
(true, _, true, None) => Ok(GENERIC_READ |
(FILE_GENERIC_WRITE & !FILE_WRITE_DATA)),
(false, false, false, None) => Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER)),
}
}
fn get_creation_mode(&self) -> io::Result<DWORD> {
const ERROR_INVALID_PARAMETER: i32 = 87;
match (self.write, self.append) {
(true, false) => {}
(false, false) =>
if self.truncate || self.create || self.create_new {
return Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER));
},
(_, true) =>
if self.truncate && !self.create_new {
return Err(io::Error::from_raw_os_error(ERROR_INVALID_PARAMETER));
},
}
Ok(match (self.create, self.truncate, self.create_new) {
(false, false, false) => OPEN_EXISTING,
(true, false, false) => OPEN_ALWAYS,
(false, true, false) => TRUNCATE_EXISTING,
(true, true, false) => CREATE_ALWAYS,
(_, _, true) => CREATE_NEW,
})
}
fn get_flags_and_attributes(&self) -> DWORD {
self.custom_flags |
self.attributes |
self.security_qos_flags |
if self.security_qos_flags != 0 { SECURITY_SQOS_PRESENT } else { 0 } |
if self.create_new { FILE_FLAG_OPEN_REPARSE_POINT } else { 0 }
}
}
struct File { handle: Handle }
impl File {
fn open(path: &Path, opts: &OpenOptions) -> io::Result<File> {
let path = try!(to_u16s(path));
let handle = unsafe {
CreateFileW(path.as_ptr(),
try!(opts.get_access_mode()),
opts.share_mode,
opts.security_attributes as *mut _,
try!(opts.get_creation_mode()),
opts.get_flags_and_attributes(),
ptr::null_mut())
};
if handle == INVALID_HANDLE_VALUE {
Err(io::Error::last_os_error())
} else {
Ok(File { handle: Handle::new(handle) })
}
}
fn file_attr(&self) -> io::Result<FileAttr> {
unsafe {
let mut info: BY_HANDLE_FILE_INFORMATION = mem::zeroed();
try!(cvt(GetFileInformationByHandle(self.handle.raw(),
&mut info)));
let mut attr = FileAttr {
attributes: info.dwFileAttributes,
creation_time: info.ftCreationTime,
last_access_time: info.ftLastAccessTime,
last_write_time: info.ftLastWriteTime,
file_size: ((info.nFileSizeHigh as u64) << 32) | (info.nFileSizeLow as u64),
reparse_tag: 0,
};
if attr.is_reparse_point() {
let mut b = [0; MAXIMUM_REPARSE_DATA_BUFFER_SIZE];
if let Ok((_, buf)) = self.reparse_point(&mut b) {
attr.reparse_tag = buf.ReparseTag;
}
}
Ok(attr)
}
}
fn set_attributes(&self, attr: DWORD) -> io::Result<()> {
let mut info = FILE_BASIC_INFO {
CreationTime: 0, // do not change
LastAccessTime: 0, // do not change
LastWriteTime: 0, // do not change
ChangeTime: 0, // do not change
FileAttributes: attr,
};
let size = mem::size_of_val(&info);
try!(cvt(unsafe {
SetFileInformationByHandle(self.handle.raw(),
FileBasicInfo,
&mut info as *mut _ as *mut _,
size as DWORD)
}));
Ok(())
}
fn rename(&self, new: &Path, replace: bool) -> io::Result<()> {
// &self must be opened with DELETE permission
use std::iter;
#[cfg(target_arch = "x86")]
const STRUCT_SIZE: usize = 12;
#[cfg(target_arch = "x86_64")]
const STRUCT_SIZE: usize = 20;
// FIXME: check for internal NULs in 'new'
let mut data: Vec<u16> = iter::repeat(0u16).take(STRUCT_SIZE/2)
.chain(new.as_os_str().encode_wide())
.collect();
data.push(0);
let size = data.len() * 2;
unsafe {
// Thanks to alignment guarantees on Windows this works
// (8 for 32-bit and 16 for 64-bit)
let mut info = data.as_mut_ptr() as *mut FILE_RENAME_INFO;
// The type of ReplaceIfExists is BOOL, but it actually expects a
// BOOLEAN. This means true is -1, not c::TRUE.
(*info).ReplaceIfExists = if replace { -1 } else { FALSE };
(*info).RootDirectory = ptr::null_mut();
(*info).FileNameLength = (size - STRUCT_SIZE) as DWORD;
try!(cvt(SetFileInformationByHandle(self.handle().raw(),
FileRenameInfo,
data.as_mut_ptr() as *mut _ as *mut _,
size as DWORD)));
Ok(())
}
}
fn set_perm(&self, perm: FilePermissions) -> io::Result<()> {
let attr = try!(self.file_attr()).attributes;
if perm.readonly == (attr & FILE_ATTRIBUTE_READONLY != 0) {
Ok(())
} else if perm.readonly {
self.set_attributes(attr | FILE_ATTRIBUTE_READONLY)
} else {
self.set_attributes(attr & !FILE_ATTRIBUTE_READONLY)
}
}
fn handle(&self) -> &Handle { &self.handle }
fn reparse_point<'a>(&self,
space: &'a mut [u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE])
-> io::Result<(DWORD, &'a REPARSE_DATA_BUFFER)> {
unsafe {
let mut bytes = 0;
try!(cvt({
DeviceIoControl(self.handle.raw(),
FSCTL_GET_REPARSE_POINT,
ptr::null_mut(),
0,
space.as_mut_ptr() as *mut _,
space.len() as DWORD,
&mut bytes,
ptr::null_mut())
}));
Ok((bytes, &*(space.as_ptr() as *const REPARSE_DATA_BUFFER)))
}
}
}
#[derive(Copy, Clone, PartialEq, Eq, Hash)]
enum FileType {
Dir, File, SymlinkFile, SymlinkDir, ReparsePoint, MountPoint,
}
impl FileType {
fn new(attrs: DWORD, reparse_tag: DWORD) -> FileType {
match (attrs & FILE_ATTRIBUTE_DIRECTORY != 0,
attrs & FILE_ATTRIBUTE_REPARSE_POINT != 0,
reparse_tag) {
(false, false, _) => FileType::File,
(true, false, _) => FileType::Dir,
(false, true, IO_REPARSE_TAG_SYMLINK) => FileType::SymlinkFile,
(true, true, IO_REPARSE_TAG_SYMLINK) => FileType::SymlinkDir,
(true, true, IO_REPARSE_TAG_MOUNT_POINT) => FileType::MountPoint,
(_, true, _) => FileType::ReparsePoint,
// Note: if a _file_ has a reparse tag of the type IO_REPARSE_TAG_MOUNT_POINT it is
// invalid, as junctions always have to be dirs. We set the filetype to ReparsePoint
// to indicate it is something symlink-like, but not something you can follow.
}
}
fn is_dir(&self) -> bool { *self == FileType::Dir }
fn is_symlink_dir(&self) -> bool {
*self == FileType::SymlinkDir || *self == FileType::MountPoint
}
}
impl DirEntry {
fn new(root: &Arc<PathBuf>, wfd: &WIN32_FIND_DATAW) -> Option<DirEntry> {
let first_bytes = &wfd.cFileName[0..3];
if first_bytes.starts_with(&[46, 0]) || first_bytes.starts_with(&[46, 46, 0]) {
None
} else {
Some(DirEntry {
root: root.clone(),
data: *wfd,
})
}
}
fn path(&self) -> PathBuf {
self.root.join(&self.file_name())
}
fn file_name(&self) -> OsString {
let filename = truncate_utf16_at_nul(&self.data.cFileName);
OsString::from_wide(filename)
}
fn file_type(&self) -> io::Result<FileType> {
Ok(FileType::new(self.data.dwFileAttributes,
/* reparse_tag = */ self.data.dwReserved0))
}
fn metadata(&self) -> io::Result<FileAttr> {
Ok(FileAttr {
attributes: self.data.dwFileAttributes,
creation_time: self.data.ftCreationTime,
last_access_time: self.data.ftLastAccessTime,
last_write_time: self.data.ftLastWriteTime,
file_size: ((self.data.nFileSizeHigh as u64) << 32) | (self.data.nFileSizeLow as u64),
reparse_tag: if self.data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
// reserved unless this is a reparse point
self.data.dwReserved0
} else {
0
},
})
}
}
struct DirEntry {
root: Arc<PathBuf>,
data: WIN32_FIND_DATAW,
}
struct ReadDir {
handle: FindNextFileHandle,
root: Arc<PathBuf>,
first: Option<WIN32_FIND_DATAW>,
}
impl Iterator for ReadDir {
type Item = io::Result<DirEntry>;
fn next(&mut self) -> Option<io::Result<DirEntry>> {
if let Some(first) = self.first.take() {
if let Some(e) = DirEntry::new(&self.root, &first) {
return Some(Ok(e));
}
}
unsafe {
let mut wfd = mem::zeroed();
loop {
if FindNextFileW(self.handle.0, &mut wfd) == 0 {
if GetLastError() == ERROR_NO_MORE_FILES {
return None
} else {
return Some(Err(io::Error::last_os_error()))
}
}
if let Some(e) = DirEntry::new(&self.root, &wfd) {
return Some(Ok(e))
}
}
}
}
}
#[derive(Clone)]
struct FileAttr {
attributes: DWORD,
creation_time: FILETIME,
last_access_time: FILETIME,
last_write_time: FILETIME,
file_size: u64,
reparse_tag: DWORD,
}
impl FileAttr {
fn perm(&self) -> FilePermissions {
FilePermissions {
readonly: self.attributes & FILE_ATTRIBUTE_READONLY != 0
}
}
fn file_type(&self) -> FileType {
FileType::new(self.attributes, self.reparse_tag)
}
fn is_reparse_point(&self) -> bool {
self.attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0
}
}
#[repr(C)]
struct REPARSE_DATA_BUFFER {
ReparseTag: c_uint,
ReparseDataLength: c_ushort,
Reserved: c_ushort,
rest: (),
}
const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: usize = 16 * 1024;
/// An owned container for `HANDLE` object, closing them on Drop.
///
/// All methods are inherited through a `Deref` impl to `RawHandle`
struct Handle(RawHandle);
use std::ops::Deref;
/// A wrapper type for `HANDLE` objects to give them proper Send/Sync inference
/// as well as Rust-y methods.
///
/// This does **not** drop the handle when it goes out of scope, use `Handle`
/// instead for that.
#[derive(Copy, Clone)]
struct RawHandle(HANDLE);
unsafe impl Send for RawHandle {}
unsafe impl Sync for RawHandle {}
impl Handle {
fn new(handle: HANDLE) -> Handle {
Handle(RawHandle::new(handle))
}
}
impl Deref for Handle {
type Target = RawHandle;
fn deref(&self) -> &RawHandle { &self.0 }
}
impl Drop for Handle {
fn drop(&mut self) {
unsafe { let _ = CloseHandle(self.raw()); }
}
}
impl RawHandle {
fn new(handle: HANDLE) -> RawHandle {
RawHandle(handle)
}
fn raw(&self) -> HANDLE { self.0 }
}
struct FindNextFileHandle(HANDLE);
fn get_path(f: &File) -> io::Result<PathBuf> {
fill_utf16_buf(|buf, sz| unsafe {
GetFinalPathNameByHandleW(f.handle.raw(), buf, sz,
VOLUME_NAME_DOS)
}, |buf| {
PathBuf::from(OsString::from_wide(buf))
})
}
fn move_item(file: &File, ctx: &mut RmdirContext) -> io::Result<()> {
let mut tmpname = ctx.base_dir.join(format!{"rm-{}", ctx.counter});
ctx.counter += 1;
// Try to rename the file. If it already exists, just retry with an other
// filename.
while let Err(err) = file.rename(tmpname.as_ref(), false) {
if err.kind() != io::ErrorKind::AlreadyExists { return Err(err) };
tmpname = ctx.base_dir.join(format!("rm-{}", ctx.counter));
ctx.counter += 1;
}
Ok(())
}
fn set_perm(path: &Path, perm: FilePermissions) -> io::Result<()> {
let mut opts = OpenOptions::new();
opts.access_mode(FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES);
opts.custom_flags(FILE_FLAG_BACKUP_SEMANTICS);
let file = try!(File::open(path, &opts));
file.set_perm(perm)
}
const VOLUME_NAME_DOS: DWORD = 0x0;
}

View File

@ -0,0 +1,67 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::io::Write;
use errors::*;
use util::*;
const TEMPLATE: &'static str = include_str!("../install-template.sh");
actor!{
#[derive(Debug)]
pub struct Scripter {
/// The name of the product, for display
product_name: String = "Product",
/// The directory under lib/ where the manifest lives
rel_manifest_dir: String = "manifestlib",
/// The string to print after successful installation
success_message: String = "Installed.",
/// Places to look for legacy manifests to uninstall
legacy_manifest_dirs: String = "",
/// The name of the output script
output_script: String = "install.sh",
}
}
impl Scripter {
/// Generate the actual installer script
pub fn run(self) -> Result<()> {
// Replace dashes in the success message with spaces (our arg handling botches spaces)
// (TODO: still needed? kept for compatibility for now...)
let product_name = self.product_name.replace('-', " ");
// Replace dashes in the success message with spaces (our arg handling botches spaces)
// (TODO: still needed? kept for compatibility for now...)
let success_message = self.success_message.replace('-', " ");
let script = TEMPLATE
.replace("%%TEMPLATE_PRODUCT_NAME%%", &sh_quote(&product_name))
.replace("%%TEMPLATE_REL_MANIFEST_DIR%%", &self.rel_manifest_dir)
.replace("%%TEMPLATE_SUCCESS_MESSAGE%%", &sh_quote(&success_message))
.replace("%%TEMPLATE_LEGACY_MANIFEST_DIRS%%", &sh_quote(&self.legacy_manifest_dirs))
.replace("%%TEMPLATE_RUST_INSTALLER_VERSION%%", &sh_quote(&::RUST_INSTALLER_VERSION));
create_new_executable(&self.output_script)?
.write_all(script.as_ref())
.chain_err(|| format!("failed to write output script '{}'", self.output_script))
}
}
fn sh_quote<T: ToString>(s: &T) -> String {
// We'll single-quote the whole thing, so first replace single-quotes with
// '"'"' (leave quoting, double-quote one `'`, re-enter single-quoting)
format!("'{}'", s.to_string().replace('\'', r#"'"'"'"#))
}

View File

@ -0,0 +1,145 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::fs::File;
use std::io::{self, Write};
use std::path::Path;
use flate2;
use flate2::write::GzEncoder;
use tar::{Builder, Header};
use walkdir::WalkDir;
use xz2::write::XzEncoder;
use errors::*;
use util::*;
actor!{
#[derive(Debug)]
pub struct Tarballer {
/// The input folder to be compressed
input: String = "package",
/// The prefix of the tarballs
output: String = "./dist",
/// The folder in which the input is to be found
work_dir: String = "./workdir",
}
}
impl Tarballer {
/// Generate the actual tarballs
pub fn run(self) -> Result<()> {
let tar_gz = self.output.clone() + ".tar.gz";
let tar_xz = self.output.clone() + ".tar.xz";
// Remove any existing files
for file in &[&tar_gz, &tar_xz] {
if Path::new(file).exists() {
remove_file(file)?;
}
}
// Sort files by their suffix, to group files with the same name from
// different locations (likely identical) and files with the same
// extension (likely containing similar data).
let (dirs, mut files) = get_recursive_paths(&self.work_dir, &self.input)
.chain_err(|| "failed to collect file paths")?;
files.sort_by(|a, b| a.bytes().rev().cmp(b.bytes().rev()));
// Prepare the .tar.gz file
let gz = GzEncoder::new(create_new_file(tar_gz)?, flate2::Compression::Best);
// Prepare the .tar.xz file
let xz = XzEncoder::new(create_new_file(tar_xz)?, 9);
// Write the tar into both encoded files. We write all directories
// first, so files may be directly created. (see rustup.rs#1092)
let mut builder = Builder::new(Tee(gz, xz));
for path in dirs {
let src = Path::new(&self.work_dir).join(&path);
builder.append_dir(&path, &src)
.chain_err(|| format!("failed to tar dir '{}'", src.display()))?;
}
for path in files {
let src = Path::new(&self.work_dir).join(&path);
let file = open_file(&src)?;
builder.append_data(&mut header(&src, &file)?, &path, &file)
.chain_err(|| format!("failed to tar file '{}'", src.display()))?;
}
let Tee(gz, xz) = builder.into_inner()
.chain_err(|| "failed to finish writing .tar stream")?;
// Finish both encoded files
gz.finish().chain_err(|| "failed to finish .tar.gz file")?;
xz.finish().chain_err(|| "failed to finish .tar.xz file")?;
Ok(())
}
}
fn header(src: &Path, file: &File) -> Result<Header> {
let mut header = Header::new_gnu();
header.set_metadata(&file.metadata()?);
if cfg!(windows) {
// Windows doesn't really have a mode, so `tar` never marks files executable.
// Use an extension whitelist to update files that usually should be so.
const EXECUTABLES: [&'static str; 4] = ["exe", "dll", "py", "sh"];
if let Some(ext) = src.extension().and_then(|s| s.to_str()) {
if EXECUTABLES.contains(&ext) {
let mode = header.mode()?;
header.set_mode(mode | 0o111);
}
}
}
Ok(header)
}
/// Returns all `(directories, files)` under the source path
fn get_recursive_paths<P, Q>(root: P, name: Q) -> Result<(Vec<String>, Vec<String>)>
where P: AsRef<Path>, Q: AsRef<Path>
{
let root = root.as_ref();
let name = name.as_ref();
if !name.is_relative() && !name.starts_with(root) {
bail!("input '{}' is not in work dir '{}'", name.display(), root.display());
}
let mut dirs = vec![];
let mut files = vec![];
for entry in WalkDir::new(root.join(name)) {
let entry = entry?;
let path = entry.path().strip_prefix(root)?;
let path = path_to_str(&path)?;
if entry.file_type().is_dir() {
dirs.push(path.to_owned());
} else {
files.push(path.to_owned());
}
}
Ok((dirs, files))
}
struct Tee<A, B>(A, B);
impl<A: Write, B: Write> Write for Tee<A, B> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.0.write_all(buf)
.and(self.1.write_all(buf))
.and(Ok(buf.len()))
}
fn flush(&mut self) -> io::Result<()> {
self.0.flush().and(self.1.flush())
}
}

View File

@ -0,0 +1,136 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::fs;
use std::path::Path;
use walkdir::WalkDir;
// Needed to set the script mode to executable.
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
// FIXME: what about Windows? Are default ACLs executable?
use errors::*;
/// Convert a `&Path` to a UTF-8 `&str`
pub fn path_to_str(path: &Path) -> Result<&str> {
path.to_str().ok_or_else(|| {
ErrorKind::Msg(format!("path is not valid UTF-8 '{}'", path.display())).into()
})
}
/// Wrap `fs::copy` with a nicer error message
pub fn copy<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> Result<u64> {
fs::copy(&from, &to)
.chain_err(|| format!("failed to copy '{}' to '{}'",
from.as_ref().display(), to.as_ref().display()))
}
/// Wrap `fs::create_dir` with a nicer error message
pub fn create_dir<P: AsRef<Path>>(path: P) -> Result<()> {
fs::create_dir(&path)
.chain_err(|| format!("failed to create dir '{}'", path.as_ref().display()))
}
/// Wrap `fs::create_dir_all` with a nicer error message
pub fn create_dir_all<P: AsRef<Path>>(path: P) -> Result<()> {
fs::create_dir_all(&path)
.chain_err(|| format!("failed to create dir '{}'", path.as_ref().display()))
}
/// Wrap `fs::OpenOptions::create_new().open()` as executable, with a nicer error message
pub fn create_new_executable<P: AsRef<Path>>(path: P) -> Result<fs::File> {
let mut options = fs::OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)] options.mode(0o755);
options.open(&path)
.chain_err(|| format!("failed to create file '{}'", path.as_ref().display()))
}
/// Wrap `fs::OpenOptions::create_new().open()`, with a nicer error message
pub fn create_new_file<P: AsRef<Path>>(path: P) -> Result<fs::File> {
fs::OpenOptions::new().write(true).create_new(true).open(&path)
.chain_err(|| format!("failed to create file '{}'", path.as_ref().display()))
}
/// Wrap `fs::File::open()` with a nicer error message
pub fn open_file<P: AsRef<Path>>(path: P) -> Result<fs::File> {
fs::File::open(&path)
.chain_err(|| format!("failed to open file '{}'", path.as_ref().display()))
}
/// Wrap `remove_dir_all` with a nicer error message
pub fn remove_dir_all<P: AsRef<Path>>(path: P) -> Result<()> {
::remove_dir_all::remove_dir_all(path.as_ref())
.chain_err(|| format!("failed to remove dir '{}'", path.as_ref().display()))
}
/// Wrap `fs::remove_file` with a nicer error message
pub fn remove_file<P: AsRef<Path>>(path: P) -> Result<()> {
fs::remove_file(path.as_ref())
.chain_err(|| format!("failed to remove file '{}'", path.as_ref().display()))
}
/// Copies the `src` directory recursively to `dst`. Both are assumed to exist
/// when this function is called.
pub fn copy_recursive(src: &Path, dst: &Path) -> Result<()> {
copy_with_callback(src, dst, |_, _| Ok(()))
}
/// Copies the `src` directory recursively to `dst`. Both are assumed to exist
/// when this function is called. Invokes a callback for each path visited.
pub fn copy_with_callback<F>(src: &Path, dst: &Path, mut callback: F) -> Result<()>
where F: FnMut(&Path, fs::FileType) -> Result<()>
{
for entry in WalkDir::new(src).min_depth(1) {
let entry = entry?;
let file_type = entry.file_type();
let path = entry.path().strip_prefix(src)?;
let dst = dst.join(path);
if file_type.is_dir() {
create_dir(&dst)?;
} else {
copy(entry.path(), dst)?;
}
callback(&path, file_type)?;
}
Ok(())
}
/// Create an "actor" with default values and setters for all fields.
macro_rules! actor {
($( #[ $attr:meta ] )+ pub struct $name:ident {
$( $( #[ $field_attr:meta ] )+ $field:ident : $type:ty = $default:expr, )*
}) => {
$( #[ $attr ] )+
pub struct $name {
$( $( #[ $field_attr ] )+ $field : $type, )*
}
impl Default for $name {
fn default() -> Self {
$name {
$( $field : $default.into(), )*
}
}
}
impl $name {
$( $( #[ $field_attr ] )+
pub fn $field<T: Into<$type>>(&mut self, value: T) -> &mut Self {
self.$field = value.into();
self
})+
}
}
}

1458
src/tools/rust-installer/test.sh Executable file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1 @@
rust

View File

@ -0,0 +1 @@
cargo

View File

@ -0,0 +1 @@
#!/bin/bogus

View File

@ -0,0 +1 @@
#!/bin/sh

View File

@ -0,0 +1 @@
#!/bin/sh

View File

@ -0,0 +1 @@
#!/bin/sh

View File

@ -0,0 +1 @@
#!/bin/sh

View File

@ -1 +0,0 @@
{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","Cargo.toml":"18f9cd8e2d5af6eac64f3cb6388220369b8d3b35db5c14b37954e9f05d978d2f","README.md":"3fd53d4441c0b1da34d13313cca70d188c97565c8a7aee9246acb013a50f5819","build.rs":"e063024318a8d117756b5a58dfb3a21d872ab9ba3c8762906f773ddc53eae45a","src/lib.rs":"7c9a4143b6f64df32570cc8a99a89c9b1c83a3e52b9b962fbe5fca2075490424"},"package":"e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a"}

View File

@ -1,17 +0,0 @@
[package]
name = "advapi32-sys"
version = "0.2.0"
authors = ["Peter Atashian <retep998@gmail.com>"]
description = "Contains function definitions for the Windows API library advapi32. See winapi for types and constants."
documentation = "https://retep998.github.io/doc/advapi32/"
repository = "https://github.com/retep998/winapi-rs"
readme = "README.md"
keywords = ["windows", "ffi", "win32"]
license = "MIT"
build = "build.rs"
[lib]
name = "advapi32"
[dependencies]
winapi = { version = "0.2.5", path = "../.." }
[build-dependencies]
winapi-build = { version = "0.1.1", path = "../../build" }

View File

@ -1,13 +0,0 @@
# advapi32 #
Contains function definitions for the Windows API library advapi32. See winapi for types and constants.
```toml
[dependencies]
advapi32-sys = "0.1.2"
```
```rust
extern crate advapi32;
```
[Documentation](https://retep998.github.io/doc/advapi32/)

View File

@ -1,6 +0,0 @@
// Copyright © 2015, Peter Atashian
// Licensed under the MIT License <LICENSE.md>
extern crate build;
fn main() {
build::link("advapi32", false)
}

File diff suppressed because it is too large Load Diff

View File

@ -1 +0,0 @@
{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".gitignore":"d471402ab06e94fb67bda462107845d5b20d9813b6f759fa4ac7f79448f3665c",".travis.yml":"ced7672565c96096f95c76aa9cd1b8e11bd7d933da39850cf6329f8873d4ec18","COPYING":"01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f","Cargo.toml":"b8e416e5dc607e42bf37fd81cac8978f0ff7ffaa23e64b953f388b0319734ffb","LICENSE-MIT":"0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f","Makefile":"a45a128685a2ae7d4fa39d310786674417ee113055ef290a11f88002285865fc","README.md":"9bc60d2cec222b50f87c85cf9475349bb228a36f89796c5d6481c52560ddde3a","UNLICENSE":"7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c","benches/bench.rs":"acf4844efadeafc7bc396c2b16f2a184e140b6c17d1084dbaf454196de2090cd","benches/random.txt":"9386fb3efedc7ffbd09fb49088347f1056bc2d90a861009fa2f804cdb714efcb","benches/sherlock.txt":"242ec73a70f0a03dcbe007e32038e7deeaee004aaec9a09a07fa322743440fa8","ctags.rust":"3d128d3cc59f702e68953ba2fe6c3f46bc6991fc575308db060482d5da0c79f3","examples/dict-search.rs":"30eb44b1a0b599507db4c23a90f74199faabc64a8ae1d603ecdf3bba7428eb1e","session.vim":"95cb1d7caf0ff7fbe76ec911988d908ddd883381c925ba64b537695bc9f021c4","src/autiter.rs":"dc8817af24825c356842c814d771868fb07b6965addf4780e8b9dea9718344a0","src/full.rs":"b83a9c8ff3ef611c316b68650915df2d7f361a49b59dab103dc2c5476f2d8303","src/lib.rs":"68bf2ed02d58bebee6f7f7579038f1e4b60a2c4acc334263cb837bcbe15ffe94","src/main.rs":"fc867cb5f0b02d0f49ecab06b72c05a247cbcf3bf9228c235de8e787bda7bef5"},"package":"ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66"}

View File

@ -1,9 +0,0 @@
.*.swp
doc
tags
examples/ss10pusa.csv
build
target
Cargo.lock
scratch*
bench_large/huge

View File

@ -1,12 +0,0 @@
language: rust
rust:
- stable
- beta
- nightly
script:
- cargo build --verbose
- cargo test --verbose
- cargo doc
- if [ "$TRAVIS_RUST_VERSION" = "nightly" ]; then
cargo bench --verbose;
fi

View File

@ -1,3 +0,0 @@
This project is dual-licensed under the Unlicense and MIT licenses.
You may use this code under the terms of either license.

View File

@ -1,46 +0,0 @@
[package]
name = "aho-corasick"
version = "0.5.3" #:version
authors = ["Andrew Gallant <jamslam@gmail.com>"]
description = "Fast multiple substring searching with finite state machines."
documentation = "http://burntsushi.net/rustdoc/aho_corasick/"
homepage = "https://github.com/BurntSushi/aho-corasick"
repository = "https://github.com/BurntSushi/aho-corasick"
readme = "README.md"
keywords = ["string", "search", "text", "aho", "corasick"]
license = "Unlicense/MIT"
[lib]
name = "aho_corasick"
[[bin]]
name = "aho-corasick-dot"
test = false
doc = false
bench = false
[dependencies]
memchr = "0.1.9"
[dev-dependencies]
csv = "0.14"
docopt = "0.6"
memmap = "0.2"
quickcheck = "0.2"
rand = "0.3"
rustc-serialize = "0.3"
[[bench]]
name = "bench"
path = "benches/bench.rs"
test = false
bench = true
[profile.test]
debug = true
[profile.bench]
debug = true
[profile.release]
debug = true

View File

@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2015 Andrew Gallant
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -1,14 +0,0 @@
all:
echo Nothing to do...
ctags:
ctags --recurse --options=ctags.rust --languages=Rust
docs:
cargo doc
in-dir ./target/doc fix-perms
rscp ./target/doc/* gopher:~/www/burntsushi.net/rustdoc/
push:
git push origin master
git push github master

View File

@ -1,55 +0,0 @@
This crate provides an implementation of the
[Aho-Corasick](http://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_string_matching_algorithm)
algorithm. Its intended use case is for fast substring matching, particularly
when matching multiple substrings in a search text. This is achieved by
compiling the substrings into a finite state machine.
This implementation provides optimal algorithmic time complexity. Construction
of the finite state machine is `O(p)` where `p` is the length of the substrings
concatenated. Matching against search text is `O(n + p + m)`, where `n` is
the length of the search text and `m` is the number of matches.
[![Build status](https://api.travis-ci.org/BurntSushi/aho-corasick.png)](https://travis-ci.org/BurntSushi/aho-corasick)
[![](http://meritbadge.herokuapp.com/aho-corasick)](https://crates.io/crates/aho-corasick)
Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).
### Documentation
[http://burntsushi.net/rustdoc/aho_corasick/](http://burntsushi.net/rustdoc/aho_corasick/).
### Example
The documentation contains several examples, and there is a more complete
example as a full program in `examples/dict-search.rs`.
Here is a quick example showing simple substring matching:
```rust
use aho_corasick::{Automaton, AcAutomaton, Match};
let aut = AcAutomaton::new(vec!["apple", "maple"]);
let mut it = aut.find("I like maple apples.");
assert_eq!(it.next(), Some(Match {
pati: 1,
start: 7,
end: 12,
}));
assert_eq!(it.next(), Some(Match {
pati: 0,
start: 13,
end: 18,
}));
assert_eq!(it.next(), None);
```
### Alternatives
Aho-Corasick is useful for matching multiple substrings against many long
strings. If your long string is fixed, then you might consider building a
[suffix array](https://github.com/BurntSushi/suffix)
of the search text (which takes `O(n)` time). Matches can then be found in
`O(plogn)` time.

View File

@ -1,24 +0,0 @@
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <http://unlicense.org/>

View File

@ -1,339 +0,0 @@
#![feature(test)]
extern crate aho_corasick;
extern crate test;
use std::iter;
use aho_corasick::{Automaton, AcAutomaton, Transitions};
use test::Bencher;
const HAYSTACK_RANDOM: &'static str = include_str!("random.txt");
const HAYSTACK_SHERLOCK: &'static str = include_str!("sherlock.txt");
fn bench_aut_no_match<P: AsRef<[u8]>, T: Transitions>(
b: &mut Bencher,
aut: AcAutomaton<P, T>,
haystack: &str,
) {
b.bytes = haystack.len() as u64;
b.iter(|| assert!(aut.find(haystack).next().is_none()));
}
fn bench_box_aut_no_match<P: AsRef<[u8]>, T: Transitions>(
b: &mut Bencher,
aut: AcAutomaton<P, T>,
haystack: &str,
) {
b.bytes = haystack.len() as u64;
let aut: &Automaton<P> = &aut;
b.iter(|| assert!(Automaton::find(&aut, haystack).next().is_none()));
}
fn bench_full_aut_no_match<P: AsRef<[u8]>, T: Transitions>(
b: &mut Bencher,
aut: AcAutomaton<P, T>,
haystack: &str,
) {
let aut = aut.into_full();
b.bytes = haystack.len() as u64;
b.iter(|| assert!(aut.find(haystack).next().is_none()));
}
fn bench_full_aut_overlapping_no_match<P: AsRef<[u8]>, T: Transitions>(
b: &mut Bencher,
aut: AcAutomaton<P, T>,
haystack: &str,
) {
let aut = aut.into_full();
b.bytes = haystack.len() as u64;
b.iter(|| assert!(aut.find_overlapping(haystack).count() == 0));
}
fn bench_naive_no_match<S>(b: &mut Bencher, needles: Vec<S>, haystack: &str)
where S: Into<String> {
b.bytes = haystack.len() as u64;
let needles: Vec<String> = needles.into_iter().map(Into::into).collect();
b.iter(|| assert!(!naive_find(&needles, haystack)));
}
fn haystack_same(letter: char) -> String {
iter::repeat(letter).take(10000).collect()
}
macro_rules! aut_benches {
($prefix:ident, $aut:expr, $bench:expr) => {
mod $prefix {
#![allow(unused_imports)]
use aho_corasick::{Automaton, AcAutomaton, Sparse};
use test::Bencher;
use super::{
HAYSTACK_RANDOM, haystack_same,
bench_aut_no_match, bench_box_aut_no_match,
bench_full_aut_no_match, bench_full_aut_overlapping_no_match,
};
#[bench]
fn ac_one_byte(b: &mut Bencher) {
let aut = $aut(vec!["a"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_one_prefix_byte_no_match(b: &mut Bencher) {
let aut = $aut(vec!["zbc"]);
$bench(b, aut, &haystack_same('y'));
}
#[bench]
fn ac_one_prefix_byte_every_match(b: &mut Bencher) {
// We lose the benefit of `memchr` because the first byte matches
// in every position in the haystack.
let aut = $aut(vec!["zbc"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_one_prefix_byte_random(b: &mut Bencher) {
let aut = $aut(vec!["zbc\x00"]);
$bench(b, aut, HAYSTACK_RANDOM);
}
#[bench]
fn ac_two_bytes(b: &mut Bencher) {
let aut = $aut(vec!["a", "b"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_two_diff_prefix(b: &mut Bencher) {
let aut = $aut(vec!["abcdef", "bmnopq"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_two_one_prefix_byte_every_match(b: &mut Bencher) {
let aut = $aut(vec!["zbcdef", "zmnopq"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_two_one_prefix_byte_no_match(b: &mut Bencher) {
let aut = $aut(vec!["zbcdef", "zmnopq"]);
$bench(b, aut, &haystack_same('y'));
}
#[bench]
fn ac_two_one_prefix_byte_random(b: &mut Bencher) {
let aut = $aut(vec!["zbcdef\x00", "zmnopq\x00"]);
$bench(b, aut, HAYSTACK_RANDOM);
}
#[bench]
fn ac_ten_bytes(b: &mut Bencher) {
let aut = $aut(vec!["a", "b", "c", "d", "e",
"f", "g", "h", "i", "j"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_ten_diff_prefix(b: &mut Bencher) {
let aut = $aut(vec!["abcdef", "bbcdef", "cbcdef", "dbcdef",
"ebcdef", "fbcdef", "gbcdef", "hbcdef",
"ibcdef", "jbcdef"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_ten_one_prefix_byte_every_match(b: &mut Bencher) {
let aut = $aut(vec!["zacdef", "zbcdef", "zccdef", "zdcdef",
"zecdef", "zfcdef", "zgcdef", "zhcdef",
"zicdef", "zjcdef"]);
$bench(b, aut, &haystack_same('z'));
}
#[bench]
fn ac_ten_one_prefix_byte_no_match(b: &mut Bencher) {
let aut = $aut(vec!["zacdef", "zbcdef", "zccdef", "zdcdef",
"zecdef", "zfcdef", "zgcdef", "zhcdef",
"zicdef", "zjcdef"]);
$bench(b, aut, &haystack_same('y'));
}
#[bench]
fn ac_ten_one_prefix_byte_random(b: &mut Bencher) {
let aut = $aut(vec!["zacdef\x00", "zbcdef\x00", "zccdef\x00",
"zdcdef\x00", "zecdef\x00", "zfcdef\x00",
"zgcdef\x00", "zhcdef\x00", "zicdef\x00",
"zjcdef\x00"]);
$bench(b, aut, HAYSTACK_RANDOM);
}
}
}
}
aut_benches!(dense, AcAutomaton::new, bench_aut_no_match);
aut_benches!(dense_boxed, AcAutomaton::new, bench_box_aut_no_match);
aut_benches!(sparse, AcAutomaton::<&str, Sparse>::with_transitions,
bench_aut_no_match);
aut_benches!(full, AcAutomaton::new, bench_full_aut_no_match);
aut_benches!(full_overlap, AcAutomaton::new, bench_full_aut_overlapping_no_match);
// A naive multi-pattern search.
// We use this to benchmark *throughput*, so it should never match anything.
fn naive_find(needles: &[String], haystack: &str) -> bool {
for hi in 0..haystack.len() {
let rest = &haystack.as_bytes()[hi..];
for needle in needles {
let needle = needle.as_bytes();
if needle.len() > rest.len() {
continue;
}
if needle == &rest[..needle.len()] {
// should never happen in throughput benchmarks.
return true;
}
}
}
false
}
#[bench]
fn naive_one_byte(b: &mut Bencher) {
bench_naive_no_match(b, vec!["a"], &haystack_same('z'));
}
#[bench]
fn naive_one_prefix_byte_no_match(b: &mut Bencher) {
bench_naive_no_match(b, vec!["zbc"], &haystack_same('y'));
}
#[bench]
fn naive_one_prefix_byte_every_match(b: &mut Bencher) {
bench_naive_no_match(b, vec!["zbc"], &haystack_same('z'));
}
#[bench]
fn naive_one_prefix_byte_random(b: &mut Bencher) {
bench_naive_no_match(b, vec!["zbc\x00"], HAYSTACK_RANDOM);
}
#[bench]
fn naive_two_bytes(b: &mut Bencher) {
bench_naive_no_match(b, vec!["a", "b"], &haystack_same('z'));
}
#[bench]
fn naive_two_diff_prefix(b: &mut Bencher) {
bench_naive_no_match(b, vec!["abcdef", "bmnopq"], &haystack_same('z'));
}
#[bench]
fn naive_two_one_prefix_byte_every_match(b: &mut Bencher) {
bench_naive_no_match(b, vec!["zbcdef", "zmnopq"], &haystack_same('z'));
}
#[bench]
fn naive_two_one_prefix_byte_no_match(b: &mut Bencher) {
bench_naive_no_match(b, vec!["zbcdef", "zmnopq"], &haystack_same('y'));
}
#[bench]
fn naive_two_one_prefix_byte_random(b: &mut Bencher) {
bench_naive_no_match(b, vec!["zbcdef\x00", "zmnopq\x00"], HAYSTACK_RANDOM);
}
#[bench]
fn naive_ten_bytes(b: &mut Bencher) {
let needles = vec!["a", "b", "c", "d", "e",
"f", "g", "h", "i", "j"];
bench_naive_no_match(b, needles, &haystack_same('z'));
}
#[bench]
fn naive_ten_diff_prefix(b: &mut Bencher) {
let needles = vec!["abcdef", "bbcdef", "cbcdef", "dbcdef",
"ebcdef", "fbcdef", "gbcdef", "hbcdef",
"ibcdef", "jbcdef"];
bench_naive_no_match(b, needles, &haystack_same('z'));
}
#[bench]
fn naive_ten_one_prefix_byte_every_match(b: &mut Bencher) {
let needles = vec!["zacdef", "zbcdef", "zccdef", "zdcdef",
"zecdef", "zfcdef", "zgcdef", "zhcdef",
"zicdef", "zjcdef"];
bench_naive_no_match(b, needles, &haystack_same('z'));
}
#[bench]
fn naive_ten_one_prefix_byte_no_match(b: &mut Bencher) {
let needles = vec!["zacdef", "zbcdef", "zccdef", "zdcdef",
"zecdef", "zfcdef", "zgcdef", "zhcdef",
"zicdef", "zjcdef"];
bench_naive_no_match(b, needles, &haystack_same('y'));
}
#[bench]
fn naive_ten_one_prefix_byte_random(b: &mut Bencher) {
let needles = vec!["zacdef\x00", "zbcdef\x00", "zccdef\x00",
"zdcdef\x00", "zecdef\x00", "zfcdef\x00",
"zgcdef\x00", "zhcdef\x00", "zicdef\x00",
"zjcdef\x00"];
bench_naive_no_match(b, needles, HAYSTACK_RANDOM);
}
// The organization above is just awful. Let's start over...
mod sherlock {
use aho_corasick::{Automaton, AcAutomaton};
use test::Bencher;
use super::HAYSTACK_SHERLOCK;
macro_rules! sherlock {
($name:ident, $count:expr, $pats:expr) => {
#[bench]
fn $name(b: &mut Bencher) {
let haystack = HAYSTACK_SHERLOCK;
let aut = AcAutomaton::new($pats).into_full();
b.bytes = haystack.len() as u64;
b.iter(|| assert_eq!($count, aut.find(haystack).count()));
}
}
}
sherlock!(name_alt1, 158, vec!["Sherlock", "Street"]);
sherlock!(name_alt2, 558, vec!["Sherlock", "Holmes"]);
sherlock!(name_alt3, 740, vec![
"Sherlock", "Holmes", "Watson", "Irene", "Adler", "John", "Baker",
]);
sherlock!(name_alt3_nocase, 1764, vec![
"ADL", "ADl", "AdL", "Adl", "BAK", "BAk", "BA", "BaK", "Bak", "Ba",
"HOL", "HOl", "HoL", "Hol", "IRE", "IRe", "IrE", "Ire", "JOH", "JOh",
"JoH", "Joh", "SHE", "SHe", "ShE", "She", "WAT", "WAt", "WaT", "Wat",
"aDL", "aDl", "adL", "adl", "bAK", "bAk", "bA", "baK", "bak", "ba",
"hOL", "hOl", "hoL", "hol", "iRE", "iRe", "irE", "ire", "jOH", "jOh",
"joH", "joh", "sHE", "sHe", "shE", "she", "wAT", "wAt", "waT", "wat",
"ſHE", "ſHe", "ſhE", "ſhe",
]);
sherlock!(name_alt4, 582, vec!["Sher", "Hol"]);
sherlock!(name_alt4_nocase, 1307, vec![
"HOL", "HOl", "HoL", "Hol", "SHE", "SHe", "ShE", "She", "hOL", "hOl",
"hoL", "hol", "sHE", "sHe", "shE", "she", "ſHE", "ſHe", "ſhE", "ſhe",
]);
sherlock!(name_alt5, 639, vec!["Sherlock", "Holmes", "Watson"]);
sherlock!(name_alt5_nocase, 1442, vec![
"HOL", "HOl", "HoL", "Hol", "SHE", "SHe", "ShE", "She", "WAT", "WAt",
"WaT", "Wat", "hOL", "hOl", "hoL", "hol", "sHE", "sHe", "shE", "she",
"wAT", "wAt", "waT", "wat", "ſHE", "ſHe", "ſhE", "ſhe",
]);
}

View File

@ -1,513 +0,0 @@
mnxnsynfvuugtbxsxbfxwreuspglnplefzwsp
tacfqcwnmodnmgnyiuvqoco
z
qjuozfkexn
zoaxzncje
sldhqtmgxzyurfyzwazmmu
bbeuv
mzsrihycwcb
xzfqozfmlnpmrzpxxxytqs
xrg
mcplby
nmslhfgjowhzfxsvyddydnsyehdskbydbjksqtpet
indvfw
bvjvvw
pddufodyqtyixbndtumndyz
xjjhtuvmsxhuwqulqtjhqrdqrmtbcphvyuqllocrnkpfv
zemshhz
wss
xewlrxfmgxnwgphcgefa
mbgsgbzrtthxweimcqzcaaheurdmd
osqefupespvh
z
tvvlakwzwjbrgjzfgubsmmonav
pjdskxcfgapsm
zqktqgkrcdrlskx
zwwfebhguskho
zlvvw
czwm
gojnpmboehlsazbexjjnuscqftrfufngygjdxcydib
d
afigycivicnknfxl
ljuwuopctiftfwctxecwipjnljyef
jonwbkodomzhqvlf
jdkizhognqsdogunwedjsmsdzho
zxvni
oynfjf
muvokjuqz
azuwrwtuxzfopwrcex
ixrjinlvxjmn
blaegnmbhsgsbmebwazaeguugtkowexgnqtbfkldadddv
tzabyoftyov
ctbtqbzscxzviuvcigwuwusrdro
ljynr
gnnnyyxslrhsbj
hhzlw
hijalf
rxlfqk
mhaofforwznvmcgplinludpgkucpa
gvvxsqqfmu
xxqhoyosixjfhjuxpv
faadjpvamjekreepizurntvwdynozfawsfawyms
lcbutr
aqyxvpozkjrecrkl
lfmochahrr
ptqyomjlwo
vcmslulznx
lmlsskcihrmxauztuarydlp
beiqsrfnmvmlmybmwpektjbikvpggthpabqsgmjhnthvysuhwbigillugjsp
dfsuegseffwcsnvsrqedytblbpzbfeyfsq
kypvqctrkuds
ylqeduokzgdqaxelhftxnxbidu
bprzyayfopxdsmfhhfqowa
ymiutdtlfaaxpbtaeslv
ggago
owpbicekdeykzfgcbgzobdvvrtetvcv
xsrlgingstiez
gyncqvq
xasohmeiwyscpehctmzmsnjklg
xsudghakxlw
dzqlfptjogzpkvwuticcyugnyopypuqqc
wlxshxbhdvuherumoppcc
znyaptivzncvkpeyeipynqefjxjjcsgfqbnezeebtowdrbjaqjlbxwvyikrmxjwoxngqgvfpbniftnmszuxg
umwpwwyvufy
pallkjtnrmtauqxauewgygwkjjwebbkabhtxticxmxfujpxlrpzlrozfslkzfdsswlmmsbdgjwmjnummk
dhsxylejzityahtqqzmohrpzjprrsraztpnuagtyzfjdekthvdogfidksrdppr
ybc
fyukknoqfnkllkwflwempjijxgo
dltvlau
rhvrvlwsribfctuzodfqkdczfzxnetqqzflnhiyl
goxmcasmq
wljbhwkpahdotqhhrbhqzijv
lszewkgdmkezvgmbmllhpksdkoiwgkvqjmurshrptlctqsosuurndcuzjfwherotv
dudxxihygxblhgchbgzyzffb
eht
fvwxvqoltdcsd
rkuig
e
axhsacsmnicugul
rubtdlhjqndxdzzwfnkuzy
swxteuyxxsktkjgv
hzwwodlqaq
vxgecev
qnwla
vdxjuzpyoqhpmuunyffptopmeauhycs
dkzo
awrfzatzohslgvqlaezepmli
qgxatixvpkkhvkumbwmwcagtgyfljdok
amdnzstpvcqj
xsrvwvhjirzfgkessve
qezwbfltfbikbmoasvoflozsjhrljnszqiciuqmflrlqowwkoevuumh
babskcvavmtvsxqsewirucwzajjcfcqwsydydqo
ywfurpsl
edacsjjkjjewkxfoh
dcgkfpcjezurnuhiatrczcp
xsatnimwbcciu
grzmbrsvvcyigcbmcqfwiiknrohveubhyijxeyzfm
kqyewccgcqrrrznwxmoztlyseagbpyho
najju
nis
awgzdvfjkzlrsjcqfeacx
oisuflfigrjaex
desbdulyuwqxuxianyypybxwlql
ekmqgspvqpftpwswayh
egbyj
fznzprhvnnwcxgcc
wfdsueieosmugirxbymbpmfrspvrktjzguxm
qkjrufshwnfwwpbhukdjlaqvljlgubmqmhnha
hwqpudgnblhlxppbrmbznotteivuzguuwlhtkytky
w
yofkyzbpg
cenolnfnllkvhikrpttcxgqxmufvorekjruyjxmr
hyexmpjijgzumawp
cdbevdilgopbzlo
fivelagckslkugdxprjxkylizewcptwxfhomzuituujixchadmnjoktnqa
csojvlinzmmkkfzqueamnuwkanzdzsavgohposbuoamoevehqrmcxdsuyelvvctoejzoertqormhaaxwofvjzekwt
sbkghhnhutrvwtyjaxndzyjamrhx
jjyqy
majwbnrhveuhrsbbbjrwpwuplifeseylqh
wyvutpxnkrnkuxxetjkkifpqb
dyzucmbcvgnjeecm
hz
uhnuipthxrzkqluosvk
lwqqzsdwiwvwaqfwlvubadlyizlo
jbd
oyzjeu
kydjkbsqxnbfiuesc
smeubjqrcxdvhsabzceyglqjzbfmoacmwvwjbhhxbr
uabipgecujfdfxpmdzrscdyvefizabgspqjrrkmgjt
xgvdgzryz
lw
uimob
ifhn
bqph
ole
g
wt
k
yslzrkwkundxfdibwqvucemepqxlmlpyngabbeciuzhptpjdetyngrtxrdtzmvq
ccwapidp
bwvrgvmtshevrophy
ni
fdkplu
mdykey
i
rhsrenoetdggpjb
djmkplpeabsholx
judxtub
fooakqwvocvpcrvxqhvtmpvhkrecy
uuxscjillynilbkrgt
evtinrmilniguarqritpeipwochmdw
sxaqzjybydyvnmmjtdcgkjnqfcklbfpkdfyewgcukqoiegyfp
kg
ovrwieqhy
jcxqtkerzjwhs
xeonglszbgypafhmqcaseimzjgebkvigbqwsayrnrprtuvhsxyitfqygohgorcdnufbcyvevvgzmjrgjqqquwkszplogx
zdketqqv
yebckucwayckeezfvtnavglpjh
zorkfrwk
pad
xqaquxudybwtgixbfktinctfirjfdayh
rieknj
ebk
qzbcfywfdmhsdruhopovemafijbscagllkmhmof
asbsnbddlobwoqatfhkbhhsymzqxjuixwreheugvngmgcuqpkjhhfwpbarqaxrwgwnjbanljlds
etevdvlc
lqyjrnmenhn
k
tsf
zczgeavcexh
jlpuxywtsrvnvluruqhecjca
ir
rikrgkmhwaosodkxgcnrexfmdrszhnmutpvwztg
bffjqovvkemctnsgeh
weysbhzixiipfithjfsk
usyzvaiyuhmksfluoirfbnsu
o
cgawpdakaszeafdtbdkqtlzkrpnoqomqvuaqcfmzgvfegovtfaonelpv
izmrcjlk
xmzemniyrzy
knqexaafsdlimdamcrprlshq
qkmqw
dntgjwsibclvposdwjuklvtejjjdjibgpyynqpgprvvaetshhmvfkcpb
otvazkrkklrxfotpopyjte
fghkcnpi
rulyaihsowvcgbzeiblhuhhfbmncqsuuqcxvseorn
exirzfmojnxcoqom
zsgpgtokun
zvamxfocorganbtlafifwdqmqtsnktbwwtewborq
cxlnaspjqvsitjyzyriqsuorjsrvzqenisprttudxntsbqrpjtdkxnwcwgjyxmgtqljcrmrbrmyvosojzlumcmjcgfjsdehec
mvx
mt
mckr
teulvroifk
laaicc
koufy
bexmwsvyarnznebdfy
ripvviosbqijsxnjilwddaqaqemzsdarnxmfooxghoypizwtbueo
ljycycuqwfnzbambibqdixmkkvwtubepla
cis
kcg
vmbbiuuoamenzepuagpfujevfstqtndjxjchdvycfrrrowochtjdmkklgnhf
pmorrwguxkvdxpluatagaziin
uwvzbmkmykjkmknzppklx
pnzxuvsrjunqxercsnvayhykcazdeclomdsasgkpqpiufyfqsxhj
yceizkddwojgweegcllaagpvrpo
ek
kuxxgbezqyxvfaxdwnqdgqsmneijunxzlwxkrs
ldldbrxmvtjlqxifngmactzqcygkvuteffcmvphevilabgukatqakamjlridznodcvblvlogulmcixxfimh
iuzjootuywjqklolzzhpeaynydjwtufjavbozxnzckuzdodkvkjfmhinelv
swlfkcufscfcovmghqwcrtxjukwafoeogrkgubbqgwzm
gjcylkwgzroubdssuqeykqjcmguso
fzq
srfvysoxtlylctp
pbfeiuzwoyixews
ocvvunfsjnrtklmuuzjojw
xdjcnrpqhmpmpcwacpcdtmbsczvhllkqapzjuaf
nfnuvjz
fwnuiyqpn
wshxxxpzzxp
hibrxcfeqca
wqhlllarl
bukcbojv
plrytapy
xm
vlgfqoyzdczqbbaxjwbjjevjhxgopuqvqcrj
vpjqfbdnsdxlbuuiqocvrhap
mgumjbvnnzgnrdru
gcgzugazxdcamrhczfzhtmdjj
uislwq
vooai
zjuqfmebuzsqngzekyajujkopvayxtdzvugwwucvlsbrnhitfotmhhmgddlzlvqrkcponictrfweuilfjiuoabkfdvpjiqjrrgi
aptjfhmrnxaq
hbs
w
mwmoxqvucwygunplzvxtxpk
fgmqmtlorfzytjdzffsosfccnfwugrsrynuej
rpmpenrhsxoefnblyumjqwvuyszyppnttuyvazjdug
zdzxraxkroknkmqgvuoqeqdtvclsvvuwmdwzfugcpteohlogxubyoebvrzbqzklvehfcqadtdrkpubfhmokzwyosogepwragcpwxo
ax
dz
de
thvkdmnbdws
ejmubw
umvwkaubzurf
wyxtxeluaoox
wwbioobtgmkebxo
miglgnafmdarzkeblyjctuayzyoeqnfnbtrcbymdzkzg
loavxq
kzhllgsenxlbgdbfzwbg
yxflogzsohlcycbyzegeubfflouvtuatixhjvicjegltjiy
jigqfjppafdiarc
mcnmwtachgearonfcymvjbrnljjxmlzkudvzqsarnfysmxlfrtlvjxwvpdbhvwysnvcdozfcruhjwnucdzakkilmlfgjiolcatpfusm
n
pdjunfcz
dc
edxkkxabsbvmvifiinnoccki
bc
gwtwsvorwzfqpz
exidmexstfflkhi
s
s
c
wtcjfywlayhpbqktcepoybowtkrmnumqsg
ozclkgjdmdk
jmegtbunyexurvfexhqptnqzie
tkoenpagzwqfawlxvzaijsjqhmg
swodqfjpdqcbkc
ujokogocyaygdibgpglecis
shlmdmgonvpuaxlhrymkxtiytmv
brhk
jmsyiuomiywxhegilycjprkyfgojdo
wzdzrgpdiosdsvkcw
odlnmsfnjrcsnflviwvawybpczdkzvdocpwrmavz
p
ubowamlskcqhdxuckrxa
fawhntiwhmdwkddnahmtajqqazpdygttqivhdiodkcpcwv
gmxujmmaufmbipaiulhurzkfdg
eixjhmbaeoybiwk
kumntgrgiofcmujlzbcopuobambsw
mnjkqiyb
iktwnsnv
hfuzcl
tqiyqvagbqgtowpjbedgjot
dfemvamelxadkztogliizdtsddoboafawficudlefo
raecmxiiibljryswntpfed
mbwrtsebkeegw
x
epp
he
vnztrswhiusokqdkmsnpuswucvfhcthjbtam
baxlwidsgbdpzvnlj
tcbjjoadrzo
aiidahyllzzsg
igebuubweicbssgddpmqxunrawavuglmpxrtkqsvjjtscibqiejjfgfnovokodmqcqitlteiakooupvzkwucucrfdzjvjbqbkgutoybmpfvhbutigdxhfiqfplyciz
cnrhbjdnjftwfwlwzrdkwhajgsizsi
qfntnt
okqyfnbresp
asyg
mjqdkdyggdxzwuzglays
h
ifaqcazoy
fol
vvsusbnugduxsceozmsarbp
epjwtorx
bwiuxxiyc
cw
bwogruhctwkfvbexjnwircykxyzjmats
kygiochfwlpsvmxcgmtjrgvfdptd
q
qmpqe
z
jghffhqfoecmszunhxmzmzhlmbrvjabhrkihgjmvckhkfpaygjkg
kfiyfgounmhlvhupswqdgws
ezzdpyqucqoocsdcjtruqpokldfkmjhqzoynirybsifyaxnaxppthjoqy
nwetlgzwrhkhtuubbkbepuhbllxspvagxrqokwnrhkbwdwtp
hlazomrhqogoaxypqaszwfxxmutvbpuuvpdffuqskcbzlwyzcssnflkwiydoveyxjnzllzhyozbsa
hwnitkwbxcyibbqsluuqywbk
ozpfjsdrc
yoepefuy
lvmspzepnetra
genbrcrmuqfvkaouvuymoxhcxotjjhk
pcshyqgbmqdubsdajnyfqvxkqvywffzn
ukhcbyzwslqeq
otfrmcbnhbyffxqregqoufdxucjunwdhlqqeiiawbxlpqeyzzopfungrryqdykgizrhqodirvazm
dhpfhzyq
cloz
eduupqifolfekve
qiec
ishnjukvomntmdthlkajxpiwk
y
axl
tmyskjqkjsvumizlal
wvvolwewsfxhhdieuagdcuhwsgqvswpbkdkpxskloalmr
ryfmhe
z
mmbpgsyrfvzdatbjrjhuipwt
llzwizmmuulgwocowwmugtaoewkhnqxparvtynlffffdfcocdbba
pyczkzbmcgrdnxnmezsx
gsqe
mcocxcolcynhpecstsn
opnpplkccobjuhtbhirpzfxuktmpsiwbvsgiaavvdge
wpaldxzasnrbvtugjwytvtfttrh
zxecurevkjiyxy
wtnovebcmglkktic
fdpwfgvlvovxrwh
bmwgdullzy
uzwhagxinwqifxjbcntqzqoxkmpqxhe
jrfizsnwxwnnhb
inapddlahrp
ndtvkceobe
buskgghihdjmjlwfc
j
rkvffxwtmzoeruhlsurwtnuh
cbvkhfepkdishfpqvijzrpleuy
jzdpxjhcgqnybssfegvrnpgyehdqpgjwudbwrjbavp
xzzvgqdrdwajmdmj
vfatwsxvwfdbdhnijdujoyotwvwjipuuetichcfmvgrsnjpqaaezjtkvc
lbfoqgfshrtwgdqufwnfuitdrjydqctqixlzufkdbp
zgau
qefdpmtkecvtj
kuphldkvnzdtpd
dti
fpd
gfrliyegxsb
i
qsddsrmkyfgzrjeqnitmnypbcakh
vfbvbrpuogzhzrbmklvhji
nkz
xlufbaoblbmeub
alwuzxzmobwdukvwnkiwmuqhuxfhevogdnqtmxjptqznrk
cngpoty
ms
qvenfg
dmeaffm
jycfgnanbmoamhmarkmjcagbp
ysqmbhopgx
jczbzgwedsp
zxzwjrxcwdtleizjlvifjwgxiibezwxhtzywqdi
mtgnlu
xboxirdchurkfnklnpkapnqfxnhrxyseiujrznjm
zm
atddskbghcahlhql
szshwzmmvu
befdtpouamwhiisyybispkchpjhownatawjfbx
ennkzbrlygd
zbt
upphzpdwzmlhhhbqvjsfmbnrar
ddcs
ipbxgzyudjyongtcyygncojdufnufqpdppgvq
gc
isu
foa
wf
jdlvqxgfbowhohhyyngbcs
zjuwjyucdwblatsnywaaoftlcamfbcnw
lzrioesuhoeevczuwrnltmkahfwiu
uicggfbddqltnjyxfltbnaekncnyxsit
zkxsqkqrwrzrxgxbsgxatybfr
ptvmfyxdcglbfipcguqthjygzqnpqssscukzawynidtchjrrxwuxifoe
w
ohu
vg
zagpowezvbniybgold
lhqseqcxteiqtgnpanpvrmvvlltxh
mtfnxn
wyodtg
rawpbgtpbaktqzmmpzxmrlwpvvmdsl
widcfbirvswraukbmkhf
vplrueuxomjkqrtjgyxjdkexttzyozawyq
hrpbahllznvmjudzxpbbv
tlavfrxygjfipkupwnbacltcfepeg
icu
otxcu
aewazy
hl
fmrp
qaacthwzohenzjr
xbyebba
rvkph
mkhhmh
swme
zjmdoypaktglcyzobquunvthcdwegtbywpijxd
jvkuhnxqc
gibhqgjojsxt
bodbktzomiqujtbstqiyquwvqgufphqstenxvddkvtdh
bpusrxkfi
zgp
pmxvgamydyakituvvsucsuidrlznupcsinltmrahulhepxmhoqtfvpjkxzhrrinncuh
jzgkjjhjqykzelaszvcwvvwbnzsxdeaerfnaravk
ynanrqyrxo
zsmuxofullob
brklgrcqefdyoczy
qkpls
snhqumae
iqdtzjadzzvnqvdvjfsaf
nfqfdqiramueblxkaqxbbkxwywzgdbndjjiqk
tc
kp
cpuckbjsxhtxmomfesgxdpz
oseif
ybhxbvyxrpkrexrhjzoaxxohrhsniewsrktjnaztn
ggelspdzhzbchruhbjbjidgjwdlhdycetqaswh
jkgivsngygkbqtlmoj
dwpnanfvitxg
ospxbwxp
wgvmvrnjescemdoiralbkvemalifxnyhrbdgodml
hjtsnkzknkplbzsiwmneefdkihnhsamjsrxggclyjqgpqltizi
sykgbuypwwhweab
nvdkkkskmtiwpoerkon
sx
sbyflwwiqylbskdlxesmylpaz
dnwcjenaluwesyywfaezznwkdwpoesxpu
kie
dslccwfryol
gfhomgfn
zprjtfqvkotktzidmoyrivall
bunvsqkysdelozemnjoeqfolruulpbipm
ullyzfahpkhkja
hwd
kvyqtprpuulgsk
zotbkcadnxmfvqmtlbxalhughceyfcibtzzj
vvpjbgxygl
hpic
mhrqd
dv
thehuzdbaacoidjoljbysnqwrrxxplrdznmgiukkvjqbopb
moszjt
rmtbunktkywqirveeqfa
kse
wbfflnatgzobjrxghjgvcsyxoruenxhyomutbptswjajawqjpqafpdcstkiyjuilimecgejpqmyciolgcmdpcstzdozbmnza

File diff suppressed because it is too large Load Diff

View File

@ -1,11 +0,0 @@
--langdef=Rust
--langmap=Rust:.rs
--regex-Rust=/^[ \t]*(#\[[^\]]\][ \t]*)*(pub[ \t]+)?(extern[ \t]+)?("[^"]+"[ \t]+)?(unsafe[ \t]+)?fn[ \t]+([a-zA-Z0-9_]+)/\6/f,functions,function definitions/
--regex-Rust=/^[ \t]*(pub[ \t]+)?type[ \t]+([a-zA-Z0-9_]+)/\2/T,types,type definitions/
--regex-Rust=/^[ \t]*(pub[ \t]+)?enum[ \t]+([a-zA-Z0-9_]+)/\2/g,enum,enumeration names/
--regex-Rust=/^[ \t]*(pub[ \t]+)?struct[ \t]+([a-zA-Z0-9_]+)/\2/s,structure names/
--regex-Rust=/^[ \t]*(pub[ \t]+)?mod[ \t]+([a-zA-Z0-9_]+)/\2/m,modules,module names/
--regex-Rust=/^[ \t]*(pub[ \t]+)?static[ \t]+([a-zA-Z0-9_]+)/\2/c,consts,static constants/
--regex-Rust=/^[ \t]*(pub[ \t]+)?trait[ \t]+([a-zA-Z0-9_]+)/\2/t,traits,traits/
--regex-Rust=/^[ \t]*(pub[ \t]+)?impl([ \t\n]+<.*>)?[ \t]+([a-zA-Z0-9_]+)/\3/i,impls,trait implementations/
--regex-Rust=/^[ \t]*macro_rules![ \t]+([a-zA-Z0-9_]+)/\1/d,macros,macro definitions/

View File

@ -1,151 +0,0 @@
// This example demonstrates how to use the Aho-Corasick algorithm to rapidly
// scan text for matches in a large dictionary of keywords. This example by
// default reads your system's dictionary (~120,000 words).
extern crate aho_corasick;
extern crate csv;
extern crate docopt;
extern crate memmap;
extern crate rustc_serialize;
use std::error::Error;
use std::fs::File;
use std::io::{self, BufRead, Write};
use std::process;
use aho_corasick::{Automaton, AcAutomaton, Match};
use docopt::Docopt;
use memmap::{Mmap, Protection};
static USAGE: &'static str = "
Usage: dict-search [options] <input>
dict-search --help
Options:
-d <path>, --dict <path> Path to dictionary of keywords to search.
[default: /usr/share/dict/words]
-m <len>, --min-len <len> The minimum length for a keyword in UTF-8
encoded bytes. [default: 5]
--overlapping Report overlapping matches.
-c, --count Show only the numebr of matches.
--memory-usage Show memory usage of automaton.
--full Use fully expanded transition matrix.
Warning: may use lots of memory.
-h, --help Show this usage message.
";
#[derive(Clone, Debug, RustcDecodable)]
struct Args {
arg_input: String,
flag_dict: String,
flag_min_len: usize,
flag_overlapping: bool,
flag_memory_usage: bool,
flag_full: bool,
flag_count: bool,
}
fn main() {
let args: Args = Docopt::new(USAGE)
.and_then(|d| d.decode())
.unwrap_or_else(|e| e.exit());
match run(&args) {
Ok(()) => {}
Err(err) => {
writeln!(&mut io::stderr(), "{}", err).unwrap();
process::exit(1);
}
}
}
fn run(args: &Args) -> Result<(), Box<Error>> {
let aut = try!(build_automaton(&args.flag_dict, args.flag_min_len));
if args.flag_memory_usage {
let (bytes, states) = if args.flag_full {
let aut = aut.into_full();
(aut.heap_bytes(), aut.num_states())
} else {
(aut.heap_bytes(), aut.num_states())
};
println!("{} bytes, {} states", bytes, states);
return Ok(());
}
if args.flag_full {
let aut = aut.into_full();
if args.flag_overlapping {
if args.flag_count {
let mmap = Mmap::open_path(
&args.arg_input, Protection::Read).unwrap();
let text = unsafe { mmap.as_slice() };
println!("{}", aut.find_overlapping(text).count());
} else {
let rdr = try!(File::open(&args.arg_input));
try!(write_matches(&aut, aut.stream_find_overlapping(rdr)));
}
} else {
if args.flag_count {
let mmap = Mmap::open_path(
&args.arg_input, Protection::Read).unwrap();
let text = unsafe { mmap.as_slice() };
println!("{}", aut.find(text).count());
} else {
let rdr = try!(File::open(&args.arg_input));
try!(write_matches(&aut, aut.stream_find(rdr)));
}
}
} else {
if args.flag_overlapping {
if args.flag_count {
let mmap = Mmap::open_path(
&args.arg_input, Protection::Read).unwrap();
let text = unsafe { mmap.as_slice() };
println!("{}", aut.find_overlapping(text).count());
} else {
let rdr = try!(File::open(&args.arg_input));
try!(write_matches(&aut, aut.stream_find_overlapping(rdr)));
}
} else {
if args.flag_count {
let mmap = Mmap::open_path(
&args.arg_input, Protection::Read).unwrap();
let text = unsafe { mmap.as_slice() };
println!("{}", aut.find(text).count());
} else {
let rdr = try!(File::open(&args.arg_input));
try!(write_matches(&aut, aut.stream_find(rdr)));
}
}
}
Ok(())
}
fn write_matches<A, I>(aut: &A, it: I) -> Result<(), Box<Error>>
where A: Automaton<String>, I: Iterator<Item=io::Result<Match>> {
let mut wtr = csv::Writer::from_writer(io::stdout());
try!(wtr.write(["pattern", "start", "end"].iter()));
for m in it {
let m = try!(m);
try!(wtr.write([
aut.pattern(m.pati),
&m.start.to_string(),
&m.end.to_string(),
].iter()));
}
try!(wtr.flush());
Ok(())
}
fn build_automaton(
dict_path: &str,
min_len: usize,
) -> Result<AcAutomaton<String>, Box<Error>> {
let buf = io::BufReader::new(try!(File::open(dict_path)));
let mut lines = Vec::with_capacity(1 << 10);
for line in buf.lines() {
let line = try!(line);
if line.len() >= min_len {
lines.push(line);
}
}
Ok(AcAutomaton::with_transitions(lines))
}

View File

@ -1 +0,0 @@
au BufWritePost *.rs silent!make ctags > /dev/null 2>&1

View File

@ -1,503 +0,0 @@
use std::io::{self, BufRead};
use std::marker::PhantomData;
use memchr::{memchr, memchr2, memchr3};
use super::{ROOT_STATE, StateIdx};
/// An abstraction over automatons and their corresponding iterators.
/// The type parameter `P` is the type of the pattern that was used to
/// construct this Automaton.
pub trait Automaton<P> {
/// Return the next state given the current state and next character.
fn next_state(&self, si: StateIdx, b: u8) -> StateIdx;
/// Return true if and only if the given state and current pattern index
/// indicate a match.
fn has_match(&self, si: StateIdx, outi: usize) -> bool;
/// Build a match given the current state, pattern index and input index.
fn get_match(&self, si: StateIdx, outi: usize, texti: usize) -> Match;
/// Return the set of bytes that have transitions in the root state.
fn start_bytes(&self) -> &[u8];
/// Returns all of the patterns matched by this automaton.
///
/// The order of the patterns is the order in which they were added.
fn patterns(&self) -> &[P];
/// Returns the pattern indexed at `i`.
///
/// The index corresponds to the position at which the pattern was added
/// to the automaton, starting at `0`.
fn pattern(&self, i: usize) -> &P;
/// Return the number of patterns in the automaton.
#[inline]
fn len(&self) -> usize {
self.patterns().len()
}
/// Returns true if the automaton has no patterns.
#[inline]
fn is_empty(&self) -> bool {
self.len() == 0
}
/// Returns an iterator of non-overlapping matches in `s`.
fn find<'a, 's, Q: ?Sized + AsRef<[u8]>>(
&'a self,
s: &'s Q,
) -> Matches<'a, 's, P, Self>
where Self: Sized {
Matches {
aut: self,
text: s.as_ref(),
texti: 0,
si: ROOT_STATE,
_m: PhantomData,
}
}
/// Returns an iterator of overlapping matches in `s`.
fn find_overlapping<'a, 's, Q: ?Sized + AsRef<[u8]>>(
&'a self,
s: &'s Q,
) -> MatchesOverlapping<'a, 's, P, Self>
where Self: Sized {
MatchesOverlapping {
aut: self,
text: s.as_ref(),
texti: 0,
si: ROOT_STATE,
outi: 0,
_m: PhantomData,
}
}
/// Returns an iterator of non-overlapping matches in the given reader.
fn stream_find<'a, R: io::Read>(
&'a self,
rdr: R,
) -> StreamMatches<'a, R, P, Self>
where Self: Sized {
StreamMatches {
aut: self,
buf: io::BufReader::new(rdr),
texti: 0,
si: ROOT_STATE,
_m: PhantomData,
}
}
/// Returns an iterator of overlapping matches in the given reader.
fn stream_find_overlapping<'a, R: io::Read>(
&'a self,
rdr: R,
) -> StreamMatchesOverlapping<'a, R, P, Self>
where Self: Sized {
StreamMatchesOverlapping {
aut: self,
buf: io::BufReader::new(rdr),
texti: 0,
si: ROOT_STATE,
outi: 0,
_m: PhantomData,
}
}
}
impl<'a, P: AsRef<[u8]>, A: 'a + Automaton<P> + ?Sized>
Automaton<P> for &'a A {
fn next_state(&self, si: StateIdx, b: u8) -> StateIdx {
(**self).next_state(si, b)
}
fn has_match(&self, si: StateIdx, outi: usize) -> bool {
(**self).has_match(si, outi)
}
fn start_bytes(&self) -> &[u8] {
(**self).start_bytes()
}
fn patterns(&self) -> &[P] {
(**self).patterns()
}
fn pattern(&self, i: usize) -> &P {
(**self).pattern(i)
}
fn get_match(&self, si: StateIdx, outi: usize, texti: usize) -> Match {
(**self).get_match(si, outi, texti)
}
}
/// Records a match in the search text.
#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
pub struct Match {
/// The pattern index.
///
/// This corresponds to the ordering in which the matched pattern was
/// added to the automaton, starting at `0`.
pub pati: usize,
/// The starting byte offset of the match in the search text.
pub start: usize,
/// The ending byte offset of the match in the search text.
///
/// (This can be re-captiulated with `pati` and adding the pattern's
/// length to `start`, but it is convenient to have it here.)
pub end: usize,
}
/// An iterator of non-overlapping matches for in-memory text.
///
/// This iterator yields `Match` values.
///
/// `'a` is the lifetime of the automaton, `'s` is the lifetime of the
/// search text, and `P` is the type of the Automaton's pattern.
#[derive(Debug)]
pub struct Matches<'a, 's, P, A: 'a + Automaton<P> + ?Sized> {
aut: &'a A,
text: &'s [u8],
texti: usize,
si: StateIdx,
_m: PhantomData<P>,
}
// When there's an initial lone start byte, it is usually worth it
// to use `memchr` to skip along the input. The problem is that
// the skipping function is called in the inner match loop, which
// can be quite costly if the skipping condition is never met.
// Therefore, we lift the case analysis outside of the inner loop at
// the cost of repeating code.
//
// `step_to_match` is the version of the inner loop without skipping,
// and `skip_to_match` is the version with skipping.
#[inline(never)]
fn step_to_match<P, A: Automaton<P> + ?Sized>(
aut: &A,
text: &[u8],
mut texti: usize,
mut si: StateIdx
) -> Option<(usize, StateIdx)> {
while texti < text.len() {
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
texti += 1;
if texti + 4 < text.len() {
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
texti += 1;
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
texti += 1;
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
texti += 1;
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
texti += 1;
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
texti += 1;
}
}
None
}
fn skip_to_match<P, A: Automaton<P> + ?Sized, F: Fn(&A, &[u8], usize) -> usize>(
aut: &A,
text: &[u8],
mut texti: usize,
mut si: StateIdx,
skip: F,
) -> Option<(usize, StateIdx)> {
if si == ROOT_STATE {
texti = skip(aut, text, texti);
}
while texti < text.len() {
si = aut.next_state(si, text[texti]);
if aut.has_match(si, 0) {
return Some((texti, si));
}
if si == ROOT_STATE {
texti = skip(aut, text, texti + 1);
} else {
texti += 1;
}
}
None
}
#[inline]
fn skip1<P, A: Automaton<P> + ?Sized>(
aut: &A,
text: &[u8],
at: usize,
) -> usize {
debug_assert!(aut.start_bytes().len() == 1);
let b = aut.start_bytes()[0];
match memchr(b, &text[at..]) {
None => text.len(),
Some(i) => at + i,
}
}
#[inline]
fn skip2<P, A: Automaton<P> + ?Sized>(
aut: &A,
text: &[u8],
at: usize,
) -> usize {
debug_assert!(aut.start_bytes().len() == 2);
let (b1, b2) = (aut.start_bytes()[0], aut.start_bytes()[1]);
match memchr2(b1, b2, &text[at..]) {
None => text.len(),
Some(i) => at + i,
}
}
#[inline]
fn skip3<P, A: Automaton<P> + ?Sized>(
aut: &A,
text: &[u8],
at: usize,
) -> usize {
debug_assert!(aut.start_bytes().len() == 3);
let (b1, b2, b3) = (
aut.start_bytes()[0], aut.start_bytes()[1], aut.start_bytes()[2],
);
match memchr3(b1, b2, b3, &text[at..]) {
None => text.len(),
Some(i) => at + i,
}
}
impl<'a, 's, P, A: Automaton<P> + ?Sized> Iterator for Matches<'a, 's, P, A> {
type Item = Match;
fn next(&mut self) -> Option<Match> {
if self.aut.start_bytes().len() == 1 {
let skip = skip_to_match(
self.aut, self.text, self.texti, self.si, skip1);
if let Some((texti, si)) = skip {
self.texti = texti + 1;
self.si = ROOT_STATE;
return Some(self.aut.get_match(si, 0, texti));
}
} else if self.aut.start_bytes().len() == 2 {
let skip = skip_to_match(
self.aut, self.text, self.texti, self.si, skip2);
if let Some((texti, si)) = skip {
self.texti = texti + 1;
self.si = ROOT_STATE;
return Some(self.aut.get_match(si, 0, texti));
}
} else if self.aut.start_bytes().len() == 3 {
let skip = skip_to_match(
self.aut, self.text, self.texti, self.si, skip3);
if let Some((texti, si)) = skip {
self.texti = texti + 1;
self.si = ROOT_STATE;
return Some(self.aut.get_match(si, 0, texti));
}
} else {
let step = step_to_match(self.aut, self.text, self.texti, self.si);
if let Some((texti, si)) = step {
self.texti = texti + 1;
self.si = ROOT_STATE;
return Some(self.aut.get_match(si, 0, texti));
}
}
None
}
}
/// An iterator of non-overlapping matches for streaming text.
///
/// This iterator yields `io::Result<Match>` values.
///
/// `'a` is the lifetime of the automaton, `R` is the type of the underlying
/// `io::Read`er, and P is the type of the Automaton's pattern.
#[derive(Debug)]
pub struct StreamMatches<'a, R, P, A: 'a + Automaton<P> + ?Sized> {
aut: &'a A,
buf: io::BufReader<R>,
texti: usize,
si: StateIdx,
_m: PhantomData<P>,
}
impl<'a, R: io::Read, P, A: Automaton<P>>
Iterator for StreamMatches<'a, R, P, A> {
type Item = io::Result<Match>;
fn next(&mut self) -> Option<io::Result<Match>> {
let mut m = None;
let mut consumed = 0;
'LOOP: loop {
self.buf.consume(consumed);
let bs = match self.buf.fill_buf() {
Err(err) => return Some(Err(err)),
Ok(bs) if bs.len() == 0 => break,
Ok(bs) => bs,
};
consumed = bs.len(); // is shortened if we find a match
for (i, &b) in bs.iter().enumerate() {
self.si = self.aut.next_state(self.si, b);
if self.aut.has_match(self.si, 0) {
m = Some(Ok(self.aut.get_match(self.si, 0, self.texti)));
consumed = i + 1;
self.texti += 1;
self.si = ROOT_STATE;
break 'LOOP;
}
self.texti += 1;
}
}
self.buf.consume(consumed);
m
}
}
/// An iterator of overlapping matches for in-memory text.
///
/// This iterator yields `Match` values.
///
/// `'a` is the lifetime of the automaton, `'s` is the lifetime of the
/// search text, and `P` is the type of the Automaton's pattern.
#[derive(Debug)]
pub struct MatchesOverlapping<'a, 's, P, A: 'a + Automaton<P> + ?Sized> {
aut: &'a A,
text: &'s [u8],
texti: usize,
si: StateIdx,
outi: usize,
_m: PhantomData<P>,
}
impl<'a, 's, P, A: Automaton<P> + ?Sized>
Iterator for MatchesOverlapping<'a, 's, P, A> {
type Item = Match;
fn next(&mut self) -> Option<Match> {
if self.aut.has_match(self.si, self.outi) {
let m = self.aut.get_match(self.si, self.outi, self.texti);
self.outi += 1;
if !self.aut.has_match(self.si, self.outi) {
self.texti += 1;
}
return Some(m);
}
self.outi = 0;
if self.aut.start_bytes().len() == 1 {
let skip = skip_to_match(
self.aut, self.text, self.texti, self.si, skip1);
if let Some((texti, si)) = skip {
self.texti = texti;
self.si = si;
return self.next();
}
} else if self.aut.start_bytes().len() == 2 {
let skip = skip_to_match(
self.aut, self.text, self.texti, self.si, skip2);
if let Some((texti, si)) = skip {
self.texti = texti;
self.si = si;
return self.next();
}
} else if self.aut.start_bytes().len() == 3 {
let skip = skip_to_match(
self.aut, self.text, self.texti, self.si, skip3);
if let Some((texti, si)) = skip {
self.texti = texti;
self.si = si;
return self.next();
}
} else {
let step = step_to_match(self.aut, self.text, self.texti, self.si);
if let Some((texti, si)) = step {
self.texti = texti;
self.si = si;
return self.next();
}
}
None
}
}
/// An iterator of overlapping matches for streaming text.
///
/// This iterator yields `io::Result<Match>` values.
///
/// `'a` is the lifetime of the automaton, `R` is the type of the underlying
/// `io::Read`er, and P is the type of the Automaton's pattern.
#[derive(Debug)]
pub struct StreamMatchesOverlapping<'a, R, P, A: 'a + Automaton<P> + ?Sized> {
aut: &'a A,
buf: io::BufReader<R>,
texti: usize,
si: StateIdx,
outi: usize,
_m: PhantomData<P>,
}
impl<'a, R: io::Read, P, A: Automaton<P> + ?Sized>
Iterator for StreamMatchesOverlapping<'a, R, P, A> {
type Item = io::Result<Match>;
fn next(&mut self) -> Option<io::Result<Match>> {
if self.aut.has_match(self.si, self.outi) {
let m = self.aut.get_match(self.si, self.outi, self.texti);
self.outi += 1;
if !self.aut.has_match(self.si, self.outi) {
self.texti += 1;
}
return Some(Ok(m));
}
let mut m = None;
let mut consumed = 0;
self.outi = 0;
'LOOP: loop {
self.buf.consume(consumed);
let bs = match self.buf.fill_buf() {
Err(err) => return Some(Err(err)),
Ok(bs) if bs.len() == 0 => break,
Ok(bs) => bs,
};
consumed = bs.len(); // is shortened if we find a match
for (i, &b) in bs.iter().enumerate() {
self.si = self.aut.next_state(self.si, b);
if self.aut.has_match(self.si, self.outi) {
m = Some(Ok(self.aut.get_match(
self.si, self.outi, self.texti)));
consumed = i + 1;
self.outi += 1;
if !self.aut.has_match(self.si, self.outi) {
self.texti += 1;
}
break 'LOOP;
}
self.texti += 1;
}
}
self.buf.consume(consumed);
m
}
}

View File

@ -1,136 +0,0 @@
use std::fmt;
use std::mem;
use super::{
FAIL_STATE,
StateIdx, AcAutomaton, Transitions, Match,
usize_bytes, vec_bytes,
};
use super::autiter::Automaton;
/// A complete Aho-Corasick automaton.
///
/// This uses a single transition matrix that permits each input character
/// to move to the next state with a single lookup in the matrix.
///
/// This is as fast as it gets, but it is guaranteed to use a lot of memory.
/// Namely, it will use at least `4 * 256 * #states`, where the number of
/// states is capped at length of all patterns concatenated.
#[derive(Clone)]
pub struct FullAcAutomaton<P> {
pats: Vec<P>,
trans: Vec<StateIdx>, // row-major, where states are rows
out: Vec<Vec<usize>>, // indexed by StateIdx
start_bytes: Vec<u8>,
}
impl<P: AsRef<[u8]>> FullAcAutomaton<P> {
/// Build a new expanded Aho-Corasick automaton from an existing
/// Aho-Corasick automaton.
pub fn new<T: Transitions>(ac: AcAutomaton<P, T>) -> FullAcAutomaton<P> {
let mut fac = FullAcAutomaton {
pats: vec![],
trans: vec![FAIL_STATE; 256 * ac.states.len()],
out: vec![vec![]; ac.states.len()],
start_bytes: vec![],
};
fac.build_matrix(&ac);
fac.pats = ac.pats;
fac.start_bytes = ac.start_bytes;
fac
}
#[doc(hidden)]
pub fn memory_usage(&self) -> usize {
self.pats.iter()
.map(|p| vec_bytes() + p.as_ref().len())
.fold(0, |a, b| a + b)
+ (4 * self.trans.len())
+ self.out.iter()
.map(|v| vec_bytes() + (usize_bytes() * v.len()))
.fold(0, |a, b| a + b)
+ self.start_bytes.len()
}
#[doc(hidden)]
pub fn heap_bytes(&self) -> usize {
self.pats.iter()
.map(|p| mem::size_of::<P>() + p.as_ref().len())
.fold(0, |a, b| a + b)
+ (4 * self.trans.len())
+ self.out.iter()
.map(|v| vec_bytes() + (usize_bytes() * v.len()))
.fold(0, |a, b| a + b)
+ self.start_bytes.len()
}
fn set(&mut self, si: StateIdx, i: u8, goto: StateIdx) {
let ns = self.num_states();
self.trans[i as usize * ns + si as usize] = goto;
}
#[doc(hidden)]
#[inline]
pub fn num_states(&self) -> usize {
self.out.len()
}
}
impl<P: AsRef<[u8]>> Automaton<P> for FullAcAutomaton<P> {
#[inline]
fn next_state(&self, si: StateIdx, i: u8) -> StateIdx {
let at = i as usize * self.num_states() + si as usize;
unsafe { *self.trans.get_unchecked(at) }
}
#[inline]
fn get_match(&self, si: StateIdx, outi: usize, texti: usize) -> Match {
let pati = self.out[si as usize][outi];
let patlen = self.pats[pati].as_ref().len();
let start = texti + 1 - patlen;
Match {
pati: pati,
start: start,
end: start + patlen,
}
}
#[inline]
fn has_match(&self, si: StateIdx, outi: usize) -> bool {
unsafe { outi < self.out.get_unchecked(si as usize).len() }
}
#[inline]
fn start_bytes(&self) -> &[u8] {
&self.start_bytes
}
#[inline]
fn patterns(&self) -> &[P] {
&self.pats
}
#[inline]
fn pattern(&self, i: usize) -> &P {
&self.pats[i]
}
}
impl<P: AsRef<[u8]>> FullAcAutomaton<P> {
fn build_matrix<T: Transitions>(&mut self, ac: &AcAutomaton<P, T>) {
for (si, s) in ac.states.iter().enumerate().skip(1) {
for b in (0..256).map(|b| b as u8) {
self.set(si as StateIdx, b, ac.next_state(si as StateIdx, b));
}
for &pati in &s.out {
self.out[si].push(pati);
}
}
}
}
impl<P: AsRef<[u8]> + fmt::Debug> fmt::Debug for FullAcAutomaton<P> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "FullAcAutomaton({:?})", self.pats)
}
}

View File

@ -1,925 +0,0 @@
/*!
An implementation of the
[Aho-Corasick string search algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_string_matching_algorithm).
The Aho-Corasick algorithm is principally useful when you need to search many
large texts for a fixed (possibly large) set of keywords. In particular, the
Aho-Corasick algorithm preprocesses the set of keywords by constructing a
finite state machine. The search phase is then a quick linear scan through the
text. Each character in the search text causes a state transition in the
automaton. Matches are reported when the automaton enters a match state.
# Examples
The main type exposed by this crate is `AcAutomaton`, which can be constructed
from an iterator of pattern strings:
```rust
use aho_corasick::{Automaton, AcAutomaton};
let aut = AcAutomaton::new(vec!["apple", "maple"]);
// AcAutomaton also implements `FromIterator`:
let aut: AcAutomaton<&str> = ["apple", "maple"].iter().cloned().collect();
```
Finding matches can be done with `find`:
```rust
use aho_corasick::{Automaton, AcAutomaton, Match};
let aut = AcAutomaton::new(vec!["apple", "maple"]);
let mut it = aut.find("I like maple apples.");
assert_eq!(it.next(), Some(Match {
pati: 1,
start: 7,
end: 12,
}));
assert_eq!(it.next(), Some(Match {
pati: 0,
start: 13,
end: 18,
}));
assert_eq!(it.next(), None);
```
Use `find_overlapping` if you want to report all matches, even if they
overlap with each other.
```rust
use aho_corasick::{Automaton, AcAutomaton, Match};
let aut = AcAutomaton::new(vec!["abc", "a"]);
let matches: Vec<_> = aut.find_overlapping("abc").collect();
assert_eq!(matches, vec![
Match { pati: 1, start: 0, end: 1}, Match { pati: 0, start: 0, end: 3 },
]);
// Regular `find` will report only one match:
let matches: Vec<_> = aut.find("abc").collect();
assert_eq!(matches, vec![Match { pati: 1, start: 0, end: 1}]);
```
Finally, there are also methods for finding matches on *streams*. Namely, the
search text does not have to live in memory. It's useful to run this on files
that can't fit into memory:
```no_run
use std::fs::File;
use aho_corasick::{Automaton, AcAutomaton};
let aut = AcAutomaton::new(vec!["foo", "bar", "baz"]);
let rdr = File::open("search.txt").unwrap();
for m in aut.stream_find(rdr) {
let m = m.unwrap(); // could be an IO error
println!("Pattern '{}' matched at: ({}, {})",
aut.pattern(m.pati), m.start, m.end);
}
```
There is also `stream_find_overlapping`, which is just like `find_overlapping`,
but it operates on streams.
Please see `dict-search.rs` in this crate's `examples` directory for a more
complete example. It creates a large automaton from a dictionary and can do a
streaming match over arbitrarily large data.
# Memory usage
A key aspect of an Aho-Corasick implementation is how the state transitions
are represented. The easiest way to make the automaton fast is to store a
sparse 256-slot map in each state. It maps an input byte to a state index.
This makes the matching loop extremely fast, since it translates to a simple
pointer read.
The problem is that as the automaton accumulates more states, you end up paying
a `256 * 4` (`4` is for the `u32` state index) byte penalty for every state
regardless of how many transitions it has.
To solve this, only states near the root of the automaton have this sparse
map representation. States near the leaves of the automaton use a dense mapping
that requires a linear scan.
(The specific limit currently set is `3`, so that states with a depth less than
or equal to `3` are less memory efficient. The result is that the memory usage
of the automaton stops growing rapidly past ~60MB, even for automatons with
thousands of patterns.)
If you'd like to opt for the less-memory-efficient-but-faster version, then
you can construct an `AcAutomaton` with a `Sparse` transition strategy:
```rust
use aho_corasick::{Automaton, AcAutomaton, Match, Sparse};
let aut = AcAutomaton::<&str, Sparse>::with_transitions(vec!["abc", "a"]);
let matches: Vec<_> = aut.find("abc").collect();
assert_eq!(matches, vec![Match { pati: 1, start: 0, end: 1}]);
```
*/
#![deny(missing_docs)]
extern crate memchr;
#[cfg(test)] extern crate quickcheck;
#[cfg(test)] extern crate rand;
use std::collections::VecDeque;
use std::fmt;
use std::iter::FromIterator;
use std::mem;
pub use self::autiter::{
Automaton, Match,
Matches, MatchesOverlapping, StreamMatches, StreamMatchesOverlapping,
};
pub use self::full::FullAcAutomaton;
// We're specifying paths explicitly so that we can use
// these modules simultaneously from `main.rs`.
// Should probably make just make `main.rs` a separate crate.
#[path = "autiter.rs"]
mod autiter;
#[path = "full.rs"]
mod full;
/// The integer type used for the state index.
///
/// Limiting this to 32 bit integers can have a big impact on memory usage
/// when using the `Sparse` transition representation.
pub type StateIdx = u32;
// Constants for special state indexes.
const FAIL_STATE: u32 = 0;
const ROOT_STATE: u32 = 1;
// Limit the depth at which we use a sparse alphabet map. Once the limit is
// reached, a dense set is used (and lookup becomes O(n)).
//
// This does have a performance hit, but the (straight forward) alternative
// is to have a `256 * 4` byte overhead for every state.
// Given that Aho-Corasick is typically used for dictionary searching, this
// can lead to dramatic memory bloat.
//
// This limit should only be increased at your peril. Namely, in the worst
// case, `256^DENSE_DEPTH_THRESHOLD * 4` corresponds to the memory usage in
// bytes. A value of `1` gives us a good balance. This is also a happy point
// in the benchmarks. A value of `0` gives considerably worse times on certain
// benchmarks (e.g., `ac_ten_one_prefix_byte_every_match`) than even a value
// of `1`. A value of `2` is slightly better than `1` and it looks like gains
// level off at that point with not much observable difference when set to
// `3`.
//
// Why not make this user configurable? Well, it doesn't make much sense
// because we pay for it with case analysis in the matching loop. Increasing it
// doesn't have much impact on performance (outside of pathological cases?).
//
// N.B. Someone else seems to have discovered an alternative, but I haven't
// grokked it yet: https://github.com/mischasan/aho-corasick
const DENSE_DEPTH_THRESHOLD: u32 = 1;
/// An Aho-Corasick finite automaton.
///
/// The type parameter `P` is the type of the pattern that was used to
/// construct this AcAutomaton.
#[derive(Clone)]
pub struct AcAutomaton<P, T=Dense> {
pats: Vec<P>,
states: Vec<State<T>>,
start_bytes: Vec<u8>,
}
#[derive(Clone)]
struct State<T> {
out: Vec<usize>,
fail: StateIdx,
goto: T,
depth: u32,
}
impl<P: AsRef<[u8]>> AcAutomaton<P> {
/// Create a new automaton from an iterator of patterns.
///
/// The patterns must be convertible to bytes (`&[u8]`) via the `AsRef`
/// trait.
pub fn new<I>(pats: I) -> AcAutomaton<P, Dense>
where I: IntoIterator<Item=P> {
AcAutomaton::with_transitions(pats)
}
}
impl<P: AsRef<[u8]>, T: Transitions> AcAutomaton<P, T> {
/// Create a new automaton from an iterator of patterns.
///
/// This constructor allows one to choose the transition representation.
///
/// The patterns must be convertible to bytes (`&[u8]`) via the `AsRef`
/// trait.
pub fn with_transitions<I>(pats: I) -> AcAutomaton<P, T>
where I: IntoIterator<Item=P> {
AcAutomaton {
pats: vec![], // filled in later, avoid wrath of borrow checker
states: vec![State::new(0), State::new(0)], // empty and root
start_bytes: vec![], // also filled in later
}.build(pats.into_iter().collect())
}
/// Build out the entire automaton into a single matrix.
///
/// This will make searching as fast as possible at the expense of using
/// at least `4 * 256 * #states` bytes of memory.
pub fn into_full(self) -> FullAcAutomaton<P> {
FullAcAutomaton::new(self)
}
#[doc(hidden)]
pub fn num_states(&self) -> usize {
self.states.len()
}
#[doc(hidden)]
pub fn heap_bytes(&self) -> usize {
self.pats.iter()
.map(|p| mem::size_of::<P>() + p.as_ref().len())
.fold(0, |a, b| a + b)
+ self.states.iter()
.map(|s| mem::size_of::<State<T>>() + s.heap_bytes())
.fold(0, |a, b| a + b)
+ self.start_bytes.len()
}
}
impl<P: AsRef<[u8]>, T: Transitions> Automaton<P> for AcAutomaton<P, T> {
#[inline]
fn next_state(&self, mut si: StateIdx, b: u8) -> StateIdx {
loop {
let maybe_si = self.states[si as usize].goto(b);
if maybe_si != FAIL_STATE {
si = maybe_si;
break;
} else {
si = self.states[si as usize].fail;
}
}
si
}
#[inline]
fn get_match(&self, si: StateIdx, outi: usize, texti: usize) -> Match {
let pati = self.states[si as usize].out[outi];
let patlen = self.pats[pati].as_ref().len();
let start = texti + 1 - patlen;
Match {
pati: pati,
start: start,
end: start + patlen,
}
}
#[inline]
fn has_match(&self, si: StateIdx, outi: usize) -> bool {
outi < self.states[si as usize].out.len()
}
#[inline]
fn start_bytes(&self) -> &[u8] {
&self.start_bytes
}
#[inline]
fn patterns(&self) -> &[P] {
&self.pats
}
#[inline]
fn pattern(&self, i: usize) -> &P {
&self.pats[i]
}
}
// Below contains code for *building* the automaton. It's a reasonably faithful
// translation of the description/psuedo-code from:
// http://www.cs.uku.fi/~kilpelai/BSA05/lectures/slides04.pdf
impl<P: AsRef<[u8]>, T: Transitions> AcAutomaton<P, T> {
// This is the first phase and builds the initial keyword tree.
fn build(mut self, pats: Vec<P>) -> AcAutomaton<P, T> {
for (pati, pat) in pats.iter().enumerate() {
if pat.as_ref().is_empty() {
continue;
}
let mut previ = ROOT_STATE;
for &b in pat.as_ref() {
if self.states[previ as usize].goto(b) != FAIL_STATE {
previ = self.states[previ as usize].goto(b);
} else {
let depth = self.states[previ as usize].depth + 1;
let nexti = self.add_state(State::new(depth));
self.states[previ as usize].set_goto(b, nexti);
previ = nexti;
}
}
self.states[previ as usize].out.push(pati);
}
for c in (0..256).into_iter().map(|c| c as u8) {
if self.states[ROOT_STATE as usize].goto(c) == FAIL_STATE {
self.states[ROOT_STATE as usize].set_goto(c, ROOT_STATE);
} else {
self.start_bytes.push(c);
}
}
// If any of the start bytes are non-ASCII, then remove them all,
// because we don't want to be calling memchr on non-ASCII bytes.
// (Well, we could, but it requires being more clever. Simply using
// the prefix byte isn't good enough.)
if self.start_bytes.iter().any(|&b| b > 0x7F) {
self.start_bytes.clear();
}
self.pats = pats;
self.fill()
}
// The second phase that fills in the back links.
fn fill(mut self) -> AcAutomaton<P, T> {
// Fill up the queue with all non-root transitions out of the root
// node. Then proceed by breadth first traversal.
let mut q = VecDeque::new();
for c in (0..256).into_iter().map(|c| c as u8) {
let si = self.states[ROOT_STATE as usize].goto(c);
if si != ROOT_STATE {
q.push_front(si);
}
}
while let Some(si) = q.pop_back() {
for c in (0..256).into_iter().map(|c| c as u8) {
let u = self.states[si as usize].goto(c);
if u != FAIL_STATE {
q.push_front(u);
let mut v = self.states[si as usize].fail;
while self.states[v as usize].goto(c) == FAIL_STATE {
v = self.states[v as usize].fail;
}
let ufail = self.states[v as usize].goto(c);
self.states[u as usize].fail = ufail;
let ufail_out = self.states[ufail as usize].out.clone();
self.states[u as usize].out.extend(ufail_out);
}
}
}
self
}
fn add_state(&mut self, state: State<T>) -> StateIdx {
let i = self.states.len();
self.states.push(state);
i as StateIdx
}
}
impl<T: Transitions> State<T> {
fn new(depth: u32) -> State<T> {
State {
out: vec![],
fail: 1,
goto: Transitions::new(depth),
depth: depth,
}
}
fn goto(&self, b: u8) -> StateIdx {
self.goto.goto(b)
}
fn set_goto(&mut self, b: u8, si: StateIdx) {
self.goto.set_goto(b, si);
}
fn heap_bytes(&self) -> usize {
(self.out.len() * usize_bytes())
+ self.goto.heap_bytes()
}
}
/// An abstraction over state transition strategies.
///
/// This is an attempt to let the caller choose the space/time trade offs
/// used for state transitions.
///
/// (It's possible that this interface is merely good enough for just the two
/// implementations in this crate.)
pub trait Transitions {
/// Return a new state at the given depth.
fn new(depth: u32) -> Self;
/// Return the next state index given the next character.
fn goto(&self, alpha: u8) -> StateIdx;
/// Set the next state index for the character given.
fn set_goto(&mut self, alpha: u8, si: StateIdx);
/// The memory use in bytes (on the heap) of this set of transitions.
fn heap_bytes(&self) -> usize;
}
/// State transitions that can be stored either sparsely or densely.
///
/// This uses less space but at the expense of slower matching.
#[derive(Clone, Debug)]
pub struct Dense(DenseChoice);
#[derive(Clone, Debug)]
enum DenseChoice {
Sparse(Vec<StateIdx>), // indexed by alphabet
Dense(Vec<(u8, StateIdx)>),
}
impl Transitions for Dense {
fn new(depth: u32) -> Dense {
if depth <= DENSE_DEPTH_THRESHOLD {
Dense(DenseChoice::Sparse(vec![0; 256]))
} else {
Dense(DenseChoice::Dense(vec![]))
}
}
fn goto(&self, b1: u8) -> StateIdx {
match self.0 {
DenseChoice::Sparse(ref m) => m[b1 as usize],
DenseChoice::Dense(ref m) => {
for &(b2, si) in m {
if b1 == b2 {
return si;
}
}
FAIL_STATE
}
}
}
fn set_goto(&mut self, b: u8, si: StateIdx) {
match self.0 {
DenseChoice::Sparse(ref mut m) => m[b as usize] = si,
DenseChoice::Dense(ref mut m) => m.push((b, si)),
}
}
fn heap_bytes(&self) -> usize {
match self.0 {
DenseChoice::Sparse(ref m) => m.len() * 4,
DenseChoice::Dense(ref m) => m.len() * (1 + 4),
}
}
}
/// State transitions that are always sparse.
///
/// This can use enormous amounts of memory when there are many patterns,
/// but matching is very fast.
#[derive(Clone, Debug)]
pub struct Sparse(Vec<StateIdx>);
impl Transitions for Sparse {
fn new(_: u32) -> Sparse {
Sparse(vec![0; 256])
}
#[inline]
fn goto(&self, b: u8) -> StateIdx {
self.0[b as usize]
}
fn set_goto(&mut self, b: u8, si: StateIdx) {
self.0[b as usize] = si;
}
fn heap_bytes(&self) -> usize {
self.0.len() * 4
}
}
impl<S: AsRef<[u8]>> FromIterator<S> for AcAutomaton<S> {
/// Create an automaton from an iterator of strings.
fn from_iter<T>(it: T) -> AcAutomaton<S> where T: IntoIterator<Item=S> {
AcAutomaton::new(it)
}
}
// Provide some question debug impls for viewing automatons.
// The custom impls mostly exist for special showing of sparse maps.
impl<P: AsRef<[u8]> + fmt::Debug, T: Transitions>
fmt::Debug for AcAutomaton<P, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use std::iter::repeat;
try!(writeln!(f, "{}", repeat('-').take(79).collect::<String>()));
try!(writeln!(f, "Patterns: {:?}", self.pats));
for (i, state) in self.states.iter().enumerate().skip(1) {
try!(writeln!(f, "{:3}: {}", i, state.debug(i == 1)));
}
write!(f, "{}", repeat('-').take(79).collect::<String>())
}
}
impl<T: Transitions> State<T> {
fn debug(&self, root: bool) -> String {
format!("State {{ depth: {:?}, out: {:?}, fail: {:?}, goto: {{{}}} }}",
self.depth, self.out, self.fail, self.goto_string(root))
}
fn goto_string(&self, root: bool) -> String {
use std::char::from_u32;
let mut goto = vec![];
for b in (0..256).map(|b| b as u8) {
let si = self.goto(b);
if (!root && si == FAIL_STATE) || (root && si == ROOT_STATE) {
continue;
}
goto.push(format!("{} => {}", from_u32(b as u32).unwrap(), si));
}
goto.join(", ")
}
}
impl<T: Transitions> fmt::Debug for State<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self.debug(false))
}
}
impl<T: Transitions> AcAutomaton<String, T> {
#[doc(hidden)]
pub fn dot(&self) -> String {
use std::fmt::Write;
let mut out = String::new();
macro_rules! w {
($w:expr, $($tt:tt)*) => { {write!($w, $($tt)*)}.unwrap() }
}
w!(out, r#"
digraph automaton {{
label=<<FONT POINT-SIZE="20">{}</FONT>>;
labelloc="l";
labeljust="l";
rankdir="LR";
"#, self.pats.join(", "));
for (i, s) in self.states.iter().enumerate().skip(1) {
let i = i as u32;
if s.out.len() == 0 {
w!(out, " {};\n", i);
} else {
w!(out, " {} [peripheries=2];\n", i);
}
w!(out, " {} -> {} [style=dashed];\n", i, s.fail);
for b in (0..256).map(|b| b as u8) {
let si = s.goto(b);
if si == FAIL_STATE || (i == ROOT_STATE && si == ROOT_STATE) {
continue;
}
w!(out, " {} -> {} [label={}];\n", i, si, b as char);
}
}
w!(out, "}}");
out
}
}
fn vec_bytes() -> usize {
usize_bytes() * 3
}
fn usize_bytes() -> usize {
let bits = usize::max_value().count_ones() as usize;
bits / 8
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use std::io;
use quickcheck::{Arbitrary, Gen, quickcheck};
use super::{Automaton, AcAutomaton, Match};
fn aut_find<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
AcAutomaton::new(xs.to_vec()).find(&haystack).collect()
}
fn aut_finds<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
let cur = io::Cursor::new(haystack.as_bytes());
AcAutomaton::new(xs.to_vec())
.stream_find(cur).map(|r| r.unwrap()).collect()
}
fn aut_findf<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
AcAutomaton::new(xs.to_vec()).into_full().find(haystack).collect()
}
fn aut_findfs<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
let cur = io::Cursor::new(haystack.as_bytes());
AcAutomaton::new(xs.to_vec())
.into_full()
.stream_find(cur).map(|r| r.unwrap()).collect()
}
fn aut_findo<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
AcAutomaton::new(xs.to_vec()).find_overlapping(haystack).collect()
}
fn aut_findos<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
let cur = io::Cursor::new(haystack.as_bytes());
AcAutomaton::new(xs.to_vec())
.stream_find_overlapping(cur).map(|r| r.unwrap()).collect()
}
fn aut_findfo<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
AcAutomaton::new(xs.to_vec())
.into_full().find_overlapping(haystack).collect()
}
fn aut_findfos<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + AsRef<[u8]> {
let cur = io::Cursor::new(haystack.as_bytes());
AcAutomaton::new(xs.to_vec())
.into_full()
.stream_find_overlapping(cur).map(|r| r.unwrap()).collect()
}
#[test]
fn one_pattern_one_match() {
let ns = vec!["a"];
let hay = "za";
let matches = vec![
Match { pati: 0, start: 1, end: 2 },
];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn one_pattern_many_match() {
let ns = vec!["a"];
let hay = "zazazzzza";
let matches = vec![
Match { pati: 0, start: 1, end: 2 },
Match { pati: 0, start: 3, end: 4 },
Match { pati: 0, start: 8, end: 9 },
];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn one_longer_pattern_one_match() {
let ns = vec!["abc"];
let hay = "zazabcz";
let matches = vec![ Match { pati: 0, start: 3, end: 6 } ];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn one_longer_pattern_many_match() {
let ns = vec!["abc"];
let hay = "zazabczzzzazzzabc";
let matches = vec![
Match { pati: 0, start: 3, end: 6 },
Match { pati: 0, start: 14, end: 17 },
];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn many_pattern_one_match() {
let ns = vec!["a", "b"];
let hay = "zb";
let matches = vec![ Match { pati: 1, start: 1, end: 2 } ];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn many_pattern_many_match() {
let ns = vec!["a", "b"];
let hay = "zbzazzzzb";
let matches = vec![
Match { pati: 1, start: 1, end: 2 },
Match { pati: 0, start: 3, end: 4 },
Match { pati: 1, start: 8, end: 9 },
];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn many_longer_pattern_one_match() {
let ns = vec!["abc", "xyz"];
let hay = "zazxyzz";
let matches = vec![ Match { pati: 1, start: 3, end: 6 } ];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn many_longer_pattern_many_match() {
let ns = vec!["abc", "xyz"];
let hay = "zazxyzzzzzazzzabcxyz";
let matches = vec![
Match { pati: 1, start: 3, end: 6 },
Match { pati: 0, start: 14, end: 17 },
Match { pati: 1, start: 17, end: 20 },
];
assert_eq!(&aut_find(&ns, hay), &matches);
assert_eq!(&aut_finds(&ns, hay), &matches);
assert_eq!(&aut_findf(&ns, hay), &matches);
assert_eq!(&aut_findfs(&ns, hay), &matches);
}
#[test]
fn many_longer_pattern_overlap_one_match() {
let ns = vec!["abc", "bc"];
let hay = "zazabcz";
let matches = vec![
Match { pati: 0, start: 3, end: 6 },
Match { pati: 1, start: 4, end: 6 },
];
assert_eq!(&aut_findo(&ns, hay), &matches);
assert_eq!(&aut_findos(&ns, hay), &matches);
assert_eq!(&aut_findfo(&ns, hay), &matches);
assert_eq!(&aut_findfos(&ns, hay), &matches);
}
#[test]
fn many_longer_pattern_overlap_one_match_reverse() {
let ns = vec!["abc", "bc"];
let hay = "xbc";
let matches = vec![ Match { pati: 1, start: 1, end: 3 } ];
assert_eq!(&aut_findo(&ns, hay), &matches);
assert_eq!(&aut_findos(&ns, hay), &matches);
assert_eq!(&aut_findfo(&ns, hay), &matches);
assert_eq!(&aut_findfos(&ns, hay), &matches);
}
#[test]
fn many_longer_pattern_overlap_many_match() {
let ns = vec!["abc", "bc", "c"];
let hay = "zzzabczzzbczzzc";
let matches = vec![
Match { pati: 0, start: 3, end: 6 },
Match { pati: 1, start: 4, end: 6 },
Match { pati: 2, start: 5, end: 6 },
Match { pati: 1, start: 9, end: 11 },
Match { pati: 2, start: 10, end: 11 },
Match { pati: 2, start: 14, end: 15 },
];
assert_eq!(&aut_findo(&ns, hay), &matches);
assert_eq!(&aut_findos(&ns, hay), &matches);
assert_eq!(&aut_findfo(&ns, hay), &matches);
assert_eq!(&aut_findfos(&ns, hay), &matches);
}
#[test]
fn many_longer_pattern_overlap_many_match_reverse() {
let ns = vec!["abc", "bc", "c"];
let hay = "zzzczzzbczzzabc";
let matches = vec![
Match { pati: 2, start: 3, end: 4 },
Match { pati: 1, start: 7, end: 9 },
Match { pati: 2, start: 8, end: 9 },
Match { pati: 0, start: 12, end: 15 },
Match { pati: 1, start: 13, end: 15 },
Match { pati: 2, start: 14, end: 15 },
];
assert_eq!(&aut_findo(&ns, hay), &matches);
assert_eq!(&aut_findos(&ns, hay), &matches);
assert_eq!(&aut_findfo(&ns, hay), &matches);
assert_eq!(&aut_findfos(&ns, hay), &matches);
}
#[test]
fn pattern_returns_original_type() {
let aut = AcAutomaton::new(vec!["apple", "maple"]);
// Explicitly given this type to assert that the thing returned
// from the function is our original type.
let pat: &str = aut.pattern(0);
assert_eq!(pat, "apple");
// Also check the return type of the `patterns` function.
let pats: &[&str] = aut.patterns();
assert_eq!(pats, &["apple", "maple"]);
}
// Quickcheck time.
// This generates very small ascii strings, which makes them more likely
// to interact in interesting ways with larger haystack strings.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct SmallAscii(String);
impl Arbitrary for SmallAscii {
fn arbitrary<G: Gen>(g: &mut G) -> SmallAscii {
use std::char::from_u32;
SmallAscii((0..2)
.map(|_| from_u32(g.gen_range(97, 123)).unwrap())
.collect())
}
fn shrink(&self) -> Box<Iterator<Item=SmallAscii>> {
Box::new(self.0.shrink().map(SmallAscii))
}
}
impl From<SmallAscii> for String {
fn from(s: SmallAscii) -> String { s.0 }
}
impl AsRef<[u8]> for SmallAscii {
fn as_ref(&self) -> &[u8] { self.0.as_ref() }
}
// This is the same arbitrary impl as `String`, except it has a bias toward
// ASCII characters.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct BiasAscii(String);
impl Arbitrary for BiasAscii {
fn arbitrary<G: Gen>(g: &mut G) -> BiasAscii {
use std::char::from_u32;
let size = { let s = g.size(); g.gen_range(0, s) };
let mut s = String::with_capacity(size);
for _ in 0..size {
if g.gen_weighted_bool(3) {
s.push(char::arbitrary(g));
} else {
for _ in 0..5 {
s.push(from_u32(g.gen_range(97, 123)).unwrap());
}
}
}
BiasAscii(s)
}
fn shrink(&self) -> Box<Iterator<Item=BiasAscii>> {
Box::new(self.0.shrink().map(BiasAscii))
}
}
fn naive_find<S>(xs: &[S], haystack: &str) -> Vec<Match>
where S: Clone + Into<String> {
let needles: Vec<String> =
xs.to_vec().into_iter().map(Into::into).collect();
let mut matches = vec![];
for hi in 0..haystack.len() {
for (pati, needle) in needles.iter().enumerate() {
let needle = needle.as_bytes();
if needle.len() == 0 || needle.len() > haystack.len() - hi {
continue;
}
if needle == &haystack.as_bytes()[hi..hi+needle.len()] {
matches.push(Match {
pati: pati,
start: hi,
end: hi + needle.len(),
});
}
}
}
matches
}
#[test]
fn qc_ac_equals_naive() {
fn prop(needles: Vec<SmallAscii>, haystack: BiasAscii) -> bool {
let aut_matches = aut_findo(&needles, &haystack.0);
let naive_matches = naive_find(&needles, &haystack.0);
// Ordering isn't always the same. I don't think we care, so do
// an unordered comparison.
let aset: HashSet<Match> = aut_matches.iter().cloned().collect();
let nset: HashSet<Match> = naive_matches.iter().cloned().collect();
aset == nset
}
quickcheck(prop as fn(Vec<SmallAscii>, BiasAscii) -> bool);
}
}

View File

@ -1,13 +0,0 @@
extern crate memchr;
use std::env;
use lib::AcAutomaton;
#[allow(dead_code)]
mod lib;
fn main() {
let aut = AcAutomaton::new(env::args().skip(1));
println!("{}", aut.dot().trim());
}

View File

@ -1 +0,0 @@
{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".gitignore":"c1e953ee360e77de57f7b02f1b7880bd6a3dc22d1a69e953c2ac2c52cc52d247",".travis.yml":"e7a77c1800f9852e4c9a2acb9df041773ecd0bc005bd1b0657ae0512c67100ac","Cargo.toml":"f35826eec96c765ae8aee4f8a66c6b3cb0d918b49935baf05bae79b6df8e1077","Cargo.toml.orig":"46baf2141cf0a39944cd90ff114df4e42570b781e704589da2a6abf4e8ba723f","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb","README.md":"602c63819e332e93c85dc8426db4855f18fe0fabbd642c5b2303ed83f1ba926f","src/example_generated.rs":"161b69d92cf6e5fa4b5dc30f06031f3a0fb590b44be2bcf0f31cb8be4fab36fa","src/lib.rs":"56e86a16356d9322fa6b4e9b910041e2e7558c08b52ffbdacc647eba36b37abc","tests/conflicting_trait_impls.rs":"79993ea67ef09a5f99fddd69d8b73b1c137e41d0e8f8535f03865d6766dcc498","tests/external.rs":"15f7901698e286197666ccd309ad1debd3c35eaff680ca090368494e8b06ccf2","tests/external_no_std.rs":"c3556fd19dd91d1b093eb6a65d09a9d0985544f0377ba3d30c0e265c956f7237","tests/i128_bitflags.rs":"c955ef2c9fd385848195bb416e660e946ccbe59acc87862ef2646eb082d82e3f"},"package":"4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"}

View File

@ -1,2 +0,0 @@
/target
/Cargo.lock

View File

@ -1,29 +0,0 @@
os:
- linux
- osx
language: rust
rust:
- stable
- beta
- nightly
sudo: false
before_script:
- pip install -v 'travis-cargo<0.2' --user && export PATH=$HOME/.local/bin:$PATH
- if [[ -e ~/Library/Python/2.7/bin ]]; then export PATH=~/Library/Python/2.7/bin:$PATH; fi
script:
- cargo build --verbose
- cargo test --verbose
- travis-cargo --only nightly test
- cargo doc --no-deps
after_success:
- travis-cargo --only nightly doc-upload
env:
global:
- TRAVIS_CARGO_NIGHTLY_FEATURE=unstable_testing
- secure: "DoZ8g8iPs+X3xEEucke0Ae02JbkQ1qd1SSv/L2aQqxULmREtRcbzRauhiT+ToQO5Ft1Lul8uck14nPfs4gMr/O3jFFBhEBVpSlbkJx7eNL3kwUdp95UNroA8I43xPN/nccJaHDN6TMTD3+uajTQTje2SyzOQP+1gvdKg17kguvE="
notifications:
email:
on_success: never

View File

@ -1,31 +0,0 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g. crates.io) dependencies
#
# If you believe there's an error in this file please file an
# issue against the rust-lang/cargo repository. If you're
# editing this file be aware that the upstream Cargo.toml
# will likely look very different (and much more reasonable)
[package]
name = "bitflags"
version = "0.9.1"
authors = ["The Rust Project Developers"]
description = "A macro to generate structures which behave like bitflags.\n"
homepage = "https://github.com/rust-lang-nursery/bitflags"
documentation = "https://docs.rs/bitflags"
readme = "README.md"
keywords = ["bit", "bitmask", "bitflags"]
categories = ["no-std"]
license = "MIT/Apache-2.0"
repository = "https://github.com/rust-lang-nursery/bitflags"
[features]
example_generated = []
unstable_testing = []
default = ["example_generated"]
[badges.travis-ci]
repository = "rust-lang-nursery/bitflags"

View File

@ -1,26 +0,0 @@
[package]
name = "bitflags"
# NB: When modifying, also modify:
# 1. html_root_url in lib.rs
# 2. number in readme (for breaking changes)
version = "0.9.1"
authors = ["The Rust Project Developers"]
license = "MIT/Apache-2.0"
keywords = ["bit", "bitmask", "bitflags"]
readme = "README.md"
repository = "https://github.com/rust-lang-nursery/bitflags"
homepage = "https://github.com/rust-lang-nursery/bitflags"
documentation = "https://docs.rs/bitflags"
categories = ["no-std"]
description = """
A macro to generate structures which behave like bitflags.
"""
[badges]
travis-ci = { repository = "rust-lang-nursery/bitflags" }
[features]
default = ["example_generated"]
unstable_testing = []
example_generated = []

View File

@ -1,25 +0,0 @@
Copyright (c) 2014 The Rust Project Developers
Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the
Software without restriction, including without
limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice
shall be included in all copies or substantial portions
of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

View File

@ -1,24 +0,0 @@
bitflags
========
A Rust macro to generate structures which behave like a set of bitflags
[![Build Status](https://travis-ci.org/rust-lang-nursery/bitflags.svg?branch=master)](https://travis-ci.org/rust-lang-nursery/bitflags)
[Documentation](https://docs.rs/bitflags)
## Usage
Add this to your `Cargo.toml`:
```toml
[dependencies]
bitflags = "0.9"
```
and this to your crate root:
```rust
#[macro_use]
extern crate bitflags;
```

View File

@ -1,16 +0,0 @@
//! This module shows an example of code generated by the macro. **IT MUST NOT BE USED OUTSIDE THIS
//! CRATE**.
bitflags! {
/// This is the same `Flags` struct defined in the [crate level example](../index.html#example).
/// Note that this struct is just for documentation purposes only, it must not be used outside
/// this crate.
pub struct Flags: u32 {
const FLAG_A = 0b00000001;
const FLAG_B = 0b00000010;
const FLAG_C = 0b00000100;
const FLAG_ABC = FLAG_A.bits
| FLAG_B.bits
| FLAG_C.bits;
}
}

View File

@ -1,990 +0,0 @@
// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! A typesafe bitmask flag generator useful for sets of C-style bitmask flags.
//! It can be used for creating typesafe wrappers around C APIs.
//!
//! The `bitflags!` macro generates a `struct` that manages a set of flags. The
//! flags should only be defined for integer types, otherwise unexpected type
//! errors may occur at compile time.
//!
//! # Example
//!
//! ```
//! #[macro_use]
//! extern crate bitflags;
//!
//! bitflags! {
//! struct Flags: u32 {
//! const FLAG_A = 0b00000001;
//! const FLAG_B = 0b00000010;
//! const FLAG_C = 0b00000100;
//! const FLAG_ABC = FLAG_A.bits
//! | FLAG_B.bits
//! | FLAG_C.bits;
//! }
//! }
//!
//! fn main() {
//! let e1 = FLAG_A | FLAG_C;
//! let e2 = FLAG_B | FLAG_C;
//! assert_eq!((e1 | e2), FLAG_ABC); // union
//! assert_eq!((e1 & e2), FLAG_C); // intersection
//! assert_eq!((e1 - e2), FLAG_A); // set difference
//! assert_eq!(!e2, FLAG_A); // set complement
//! }
//! ```
//!
//! See [`example_generated::Flags`](./example_generated/struct.Flags.html) for documentation of code
//! generated by the above `bitflags!` expansion.
//!
//! The generated `struct`s can also be extended with type and trait
//! implementations:
//!
//! ```
//! #[macro_use]
//! extern crate bitflags;
//!
//! use std::fmt;
//!
//! bitflags! {
//! struct Flags: u32 {
//! const FLAG_A = 0b00000001;
//! const FLAG_B = 0b00000010;
//! }
//! }
//!
//! impl Flags {
//! pub fn clear(&mut self) {
//! self.bits = 0; // The `bits` field can be accessed from within the
//! // same module where the `bitflags!` macro was invoked.
//! }
//! }
//!
//! impl fmt::Display for Flags {
//! fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
//! write!(f, "hi!")
//! }
//! }
//!
//! fn main() {
//! let mut flags = FLAG_A | FLAG_B;
//! flags.clear();
//! assert!(flags.is_empty());
//! assert_eq!(format!("{}", flags), "hi!");
//! assert_eq!(format!("{:?}", FLAG_A | FLAG_B), "FLAG_A | FLAG_B");
//! assert_eq!(format!("{:?}", FLAG_B), "FLAG_B");
//! }
//! ```
//!
//! # Visibility
//!
//! The generated struct and its associated flag constants are not exported
//! out of the current module by default. A definition can be exported out of
//! the current module by adding `pub` before `flags`:
//!
//! ```ignore
//! #[macro_use]
//! extern crate bitflags;
//!
//! mod example {
//! bitflags! {
//! pub struct Flags1: u32 {
//! const FLAG_A = 0b00000001;
//! }
//! }
//! bitflags! {
//! struct Flags2: u32 {
//! const FLAG_B = 0b00000010;
//! }
//! }
//! }
//!
//! fn main() {
//! let flag1 = example::FLAG_A;
//! let flag2 = example::FLAG_B; // error: const `FLAG_B` is private
//! }
//! ```
//!
//! # Attributes
//!
//! Attributes can be attached to the generated `struct` by placing them
//! before the `flags` keyword.
//!
//! # Trait implementations
//!
//! The `Copy`, `Clone`, `PartialEq`, `Eq`, `PartialOrd`, `Ord` and `Hash`
//! traits automatically derived for the `struct` using the `derive` attribute.
//! Additional traits can be derived by providing an explicit `derive`
//! attribute on `flags`.
//!
//! The `Extend` and `FromIterator` traits are implemented for the `struct`,
//! too: `Extend` adds the union of the instances of the `struct` iterated over,
//! while `FromIterator` calculates the union.
//!
//! The `Binary`, `Debug`, `LowerExp`, `Octal` and `UpperExp` trait is also
//! implemented by displaying the bits value of the internal struct.
//!
//! ## Operators
//!
//! The following operator traits are implemented for the generated `struct`:
//!
//! - `BitOr` and `BitOrAssign`: union
//! - `BitAnd` and `BitAndAssign`: intersection
//! - `BitXor` and `BitXorAssign`: toggle
//! - `Sub` and `SubAssign`: set difference
//! - `Not`: set complement
//!
//! # Methods
//!
//! The following methods are defined for the generated `struct`:
//!
//! - `empty`: an empty set of flags
//! - `all`: the set of all flags
//! - `bits`: the raw value of the flags currently stored
//! - `from_bits`: convert from underlying bit representation, unless that
//! representation contains bits that do not correspond to a flag
//! - `from_bits_truncate`: convert from underlying bit representation, dropping
//! any bits that do not correspond to flags
//! - `is_empty`: `true` if no flags are currently stored
//! - `is_all`: `true` if all flags are currently set
//! - `intersects`: `true` if there are flags common to both `self` and `other`
//! - `contains`: `true` all of the flags in `other` are contained within `self`
//! - `insert`: inserts the specified flags in-place
//! - `remove`: removes the specified flags in-place
//! - `toggle`: the specified flags will be inserted if not present, and removed
//! if they are.
//!
//! ## Default
//!
//! The `Default` trait is not automatically implemented for the generated struct.
//!
//! If your default value is equal to `0` (which is the same value as calling `empty()`
//! on the generated struct), you can simply derive `Default`:
//!
//! ```
//! #[macro_use]
//! extern crate bitflags;
//!
//! bitflags! {
//! // Results in default value with bits: 0
//! #[derive(Default)]
//! struct Flags: u32 {
//! const FLAG_A = 0b00000001;
//! const FLAG_B = 0b00000010;
//! const FLAG_C = 0b00000100;
//! }
//! }
//!
//! fn main() {
//! let derived_default: Flags = Default::default();
//! assert_eq!(derived_default.bits(), 0);
//! }
//! ```
//!
//! If your default value is not equal to `0` you need to implement `Default` yourself:
//!
//! ```
//! #[macro_use]
//! extern crate bitflags;
//!
//! bitflags! {
//! struct Flags: u32 {
//! const FLAG_A = 0b00000001;
//! const FLAG_B = 0b00000010;
//! const FLAG_C = 0b00000100;
//! }
//! }
//!
//! // explicit `Default` implementation
//! impl Default for Flags {
//! fn default() -> Flags {
//! FLAG_A | FLAG_C
//! }
//! }
//!
//! fn main() {
//! let implemented_default: Flags = Default::default();
//! assert_eq!(implemented_default, (FLAG_A | FLAG_C));
//! }
//! ```
#![no_std]
#![doc(html_root_url = "https://docs.rs/bitflags/0.9.1")]
// When compiled for the rustc compiler itself we want to make sure that this is
// an unstable crate.
#![cfg_attr(rustbuild, feature(staged_api))]
#![cfg_attr(rustbuild, unstable(feature = "rustc_private", issue = "27812"))]
#[cfg(test)]
#[macro_use]
extern crate std;
// Re-export libstd/libcore using an alias so that the macros can work in no_std
// crates while remaining compatible with normal crates.
#[doc(hidden)]
pub extern crate core as _core;
/// The macro used to generate the flag structure.
///
/// See the [crate level docs](../bitflags/index.html) for complete documentation.
///
/// # Example
///
/// ```
/// #[macro_use]
/// extern crate bitflags;
///
/// bitflags! {
/// struct Flags: u32 {
/// const FLAG_A = 0b00000001;
/// const FLAG_B = 0b00000010;
/// const FLAG_C = 0b00000100;
/// const FLAG_ABC = FLAG_A.bits
/// | FLAG_B.bits
/// | FLAG_C.bits;
/// }
/// }
///
/// fn main() {
/// let e1 = FLAG_A | FLAG_C;
/// let e2 = FLAG_B | FLAG_C;
/// assert_eq!((e1 | e2), FLAG_ABC); // union
/// assert_eq!((e1 & e2), FLAG_C); // intersection
/// assert_eq!((e1 - e2), FLAG_A); // set difference
/// assert_eq!(!e2, FLAG_A); // set complement
/// }
/// ```
///
/// The generated `struct`s can also be extended with type and trait
/// implementations:
///
/// ```
/// #[macro_use]
/// extern crate bitflags;
///
/// use std::fmt;
///
/// bitflags! {
/// struct Flags: u32 {
/// const FLAG_A = 0b00000001;
/// const FLAG_B = 0b00000010;
/// }
/// }
///
/// impl Flags {
/// pub fn clear(&mut self) {
/// self.bits = 0; // The `bits` field can be accessed from within the
/// // same module where the `bitflags!` macro was invoked.
/// }
/// }
///
/// impl fmt::Display for Flags {
/// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
/// write!(f, "hi!")
/// }
/// }
///
/// fn main() {
/// let mut flags = FLAG_A | FLAG_B;
/// flags.clear();
/// assert!(flags.is_empty());
/// assert_eq!(format!("{}", flags), "hi!");
/// assert_eq!(format!("{:?}", FLAG_A | FLAG_B), "FLAG_A | FLAG_B");
/// assert_eq!(format!("{:?}", FLAG_B), "FLAG_B");
/// }
/// ```
#[macro_export]
macro_rules! bitflags {
($(#[$attr:meta])* pub struct $BitFlags:ident: $T:ty {
$($(#[$Flag_attr:meta])* const $Flag:ident = $value:expr;)+
}) => {
#[derive(Copy, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
$(#[$attr])*
pub struct $BitFlags {
bits: $T,
}
$($(#[$Flag_attr])* pub const $Flag: $BitFlags = $BitFlags { bits: $value };)+
__impl_bitflags! {
struct $BitFlags: $T {
$($(#[$Flag_attr])* const $Flag = $value;)+
}
}
};
($(#[$attr:meta])* struct $BitFlags:ident: $T:ty {
$($(#[$Flag_attr:meta])* const $Flag:ident = $value:expr;)+
}) => {
#[derive(Copy, PartialEq, Eq, Clone, PartialOrd, Ord, Hash)]
$(#[$attr])*
struct $BitFlags {
bits: $T,
}
$($(#[$Flag_attr])* const $Flag: $BitFlags = $BitFlags { bits: $value };)+
__impl_bitflags! {
struct $BitFlags: $T {
$($(#[$Flag_attr])* const $Flag = $value;)+
}
}
};
}
#[macro_export]
#[doc(hidden)]
macro_rules! __impl_bitflags {
(struct $BitFlags:ident: $T:ty {
$($(#[$Flag_attr:meta])* const $Flag:ident = $value:expr;)+
}) => {
impl $crate::_core::fmt::Debug for $BitFlags {
fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {
// This convoluted approach is to handle #[cfg]-based flag
// omission correctly. For example it needs to support:
//
// #[cfg(unix)] const A: Flag = /* ... */;
// #[cfg(windows)] const B: Flag = /* ... */;
// Unconditionally define a check for every flag, even disabled
// ones.
#[allow(non_snake_case)]
trait __BitFlags {
$(
fn $Flag(&self) -> bool { false }
)+
}
// Conditionally override the check for just those flags that
// are not #[cfg]ed away.
impl __BitFlags for $BitFlags {
$(
$(#[$Flag_attr])*
fn $Flag(&self) -> bool {
self.bits & $Flag.bits == $Flag.bits
}
)+
}
let mut first = true;
$(
if <$BitFlags as __BitFlags>::$Flag(self) {
if !first {
try!(f.write_str(" | "));
}
first = false;
try!(f.write_str(stringify!($Flag)));
}
)+
if first {
try!(f.write_str("(empty)"));
}
Ok(())
}
}
impl $crate::_core::fmt::Binary for $BitFlags {
fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {
$crate::_core::fmt::Binary::fmt(&self.bits, f)
}
}
impl $crate::_core::fmt::Octal for $BitFlags {
fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {
$crate::_core::fmt::Octal::fmt(&self.bits, f)
}
}
impl $crate::_core::fmt::LowerHex for $BitFlags {
fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {
$crate::_core::fmt::LowerHex::fmt(&self.bits, f)
}
}
impl $crate::_core::fmt::UpperHex for $BitFlags {
fn fmt(&self, f: &mut $crate::_core::fmt::Formatter) -> $crate::_core::fmt::Result {
$crate::_core::fmt::UpperHex::fmt(&self.bits, f)
}
}
#[allow(dead_code)]
impl $BitFlags {
/// Returns an empty set of flags.
#[inline]
pub fn empty() -> $BitFlags {
$BitFlags { bits: 0 }
}
/// Returns the set containing all flags.
#[inline]
pub fn all() -> $BitFlags {
// See `Debug::fmt` for why this approach is taken.
#[allow(non_snake_case)]
trait __BitFlags {
$(
fn $Flag() -> $T { 0 }
)+
}
impl __BitFlags for $BitFlags {
$(
$(#[$Flag_attr])*
fn $Flag() -> $T { $Flag.bits }
)+
}
$BitFlags { bits: $(<$BitFlags as __BitFlags>::$Flag())|+ }
}
/// Returns the raw value of the flags currently stored.
#[inline]
pub fn bits(&self) -> $T {
self.bits
}
/// Convert from underlying bit representation, unless that
/// representation contains bits that do not correspond to a flag.
#[inline]
pub fn from_bits(bits: $T) -> $crate::_core::option::Option<$BitFlags> {
if (bits & !$BitFlags::all().bits()) == 0 {
$crate::_core::option::Option::Some($BitFlags { bits: bits })
} else {
$crate::_core::option::Option::None
}
}
/// Convert from underlying bit representation, dropping any bits
/// that do not correspond to flags.
#[inline]
pub fn from_bits_truncate(bits: $T) -> $BitFlags {
$BitFlags { bits: bits } & $BitFlags::all()
}
/// Returns `true` if no flags are currently stored.
#[inline]
pub fn is_empty(&self) -> bool {
*self == $BitFlags::empty()
}
/// Returns `true` if all flags are currently set.
#[inline]
pub fn is_all(&self) -> bool {
*self == $BitFlags::all()
}
/// Returns `true` if there are flags common to both `self` and `other`.
#[inline]
pub fn intersects(&self, other: $BitFlags) -> bool {
!(*self & other).is_empty()
}
/// Returns `true` all of the flags in `other` are contained within `self`.
#[inline]
pub fn contains(&self, other: $BitFlags) -> bool {
(*self & other) == other
}
/// Inserts the specified flags in-place.
#[inline]
pub fn insert(&mut self, other: $BitFlags) {
self.bits |= other.bits;
}
/// Removes the specified flags in-place.
#[inline]
pub fn remove(&mut self, other: $BitFlags) {
self.bits &= !other.bits;
}
/// Toggles the specified flags in-place.
#[inline]
pub fn toggle(&mut self, other: $BitFlags) {
self.bits ^= other.bits;
}
/// Inserts or removes the specified flags depending on the passed value.
#[inline]
pub fn set(&mut self, other: $BitFlags, value: bool) {
if value {
self.insert(other);
} else {
self.remove(other);
}
}
}
impl $crate::_core::ops::BitOr for $BitFlags {
type Output = $BitFlags;
/// Returns the union of the two sets of flags.
#[inline]
fn bitor(self, other: $BitFlags) -> $BitFlags {
$BitFlags { bits: self.bits | other.bits }
}
}
impl $crate::_core::ops::BitOrAssign for $BitFlags {
/// Adds the set of flags.
#[inline]
fn bitor_assign(&mut self, other: $BitFlags) {
self.bits |= other.bits;
}
}
impl $crate::_core::ops::BitXor for $BitFlags {
type Output = $BitFlags;
/// Returns the left flags, but with all the right flags toggled.
#[inline]
fn bitxor(self, other: $BitFlags) -> $BitFlags {
$BitFlags { bits: self.bits ^ other.bits }
}
}
impl $crate::_core::ops::BitXorAssign for $BitFlags {
/// Toggles the set of flags.
#[inline]
fn bitxor_assign(&mut self, other: $BitFlags) {
self.bits ^= other.bits;
}
}
impl $crate::_core::ops::BitAnd for $BitFlags {
type Output = $BitFlags;
/// Returns the intersection between the two sets of flags.
#[inline]
fn bitand(self, other: $BitFlags) -> $BitFlags {
$BitFlags { bits: self.bits & other.bits }
}
}
impl $crate::_core::ops::BitAndAssign for $BitFlags {
/// Disables all flags disabled in the set.
#[inline]
fn bitand_assign(&mut self, other: $BitFlags) {
self.bits &= other.bits;
}
}
impl $crate::_core::ops::Sub for $BitFlags {
type Output = $BitFlags;
/// Returns the set difference of the two sets of flags.
#[inline]
fn sub(self, other: $BitFlags) -> $BitFlags {
$BitFlags { bits: self.bits & !other.bits }
}
}
impl $crate::_core::ops::SubAssign for $BitFlags {
/// Disables all flags enabled in the set.
#[inline]
fn sub_assign(&mut self, other: $BitFlags) {
self.bits &= !other.bits;
}
}
impl $crate::_core::ops::Not for $BitFlags {
type Output = $BitFlags;
/// Returns the complement of this set of flags.
#[inline]
fn not(self) -> $BitFlags {
$BitFlags { bits: !self.bits } & $BitFlags::all()
}
}
impl $crate::_core::iter::Extend<$BitFlags> for $BitFlags {
fn extend<T: $crate::_core::iter::IntoIterator<Item=$BitFlags>>(&mut self, iterator: T) {
for item in iterator {
self.insert(item)
}
}
}
impl $crate::_core::iter::FromIterator<$BitFlags> for $BitFlags {
fn from_iter<T: $crate::_core::iter::IntoIterator<Item=$BitFlags>>(iterator: T) -> $BitFlags {
let mut result = Self::empty();
result.extend(iterator);
result
}
}
};
}
#[cfg(feature = "example_generated")]
pub mod example_generated;
#[cfg(test)]
#[allow(non_upper_case_globals, dead_code)]
mod tests {
use std::hash::{Hash, Hasher};
use std::collections::hash_map::DefaultHasher;
bitflags! {
#[doc = "> The first principle is that you must not fool yourself — and"]
#[doc = "> you are the easiest person to fool."]
#[doc = "> "]
#[doc = "> - Richard Feynman"]
struct Flags: u32 {
const FlagA = 0b00000001;
#[doc = "<pcwalton> macros are way better at generating code than trans is"]
const FlagB = 0b00000010;
const FlagC = 0b00000100;
#[doc = "* cmr bed"]
#[doc = "* strcat table"]
#[doc = "<strcat> wait what?"]
const FlagABC = FlagA.bits
| FlagB.bits
| FlagC.bits;
}
}
bitflags! {
struct _CfgFlags: u32 {
#[cfg(windows)]
const _CfgA = 0b01;
#[cfg(unix)]
const _CfgB = 0b01;
#[cfg(windows)]
const _CfgC = _CfgA.bits | 0b10;
}
}
bitflags! {
struct AnotherSetOfFlags: i8 {
const AnotherFlag = -1_i8;
}
}
bitflags! {
struct LongFlags: u32 {
const LongFlagA = 0b1111111111111111;
}
}
#[test]
fn test_bits(){
assert_eq!(Flags::empty().bits(), 0b00000000);
assert_eq!(FlagA.bits(), 0b00000001);
assert_eq!(FlagABC.bits(), 0b00000111);
assert_eq!(AnotherSetOfFlags::empty().bits(), 0b00);
assert_eq!(AnotherFlag.bits(), !0_i8);
}
#[test]
fn test_from_bits() {
assert_eq!(Flags::from_bits(0), Some(Flags::empty()));
assert_eq!(Flags::from_bits(0b1), Some(FlagA));
assert_eq!(Flags::from_bits(0b10), Some(FlagB));
assert_eq!(Flags::from_bits(0b11), Some(FlagA | FlagB));
assert_eq!(Flags::from_bits(0b1000), None);
assert_eq!(AnotherSetOfFlags::from_bits(!0_i8), Some(AnotherFlag));
}
#[test]
fn test_from_bits_truncate() {
assert_eq!(Flags::from_bits_truncate(0), Flags::empty());
assert_eq!(Flags::from_bits_truncate(0b1), FlagA);
assert_eq!(Flags::from_bits_truncate(0b10), FlagB);
assert_eq!(Flags::from_bits_truncate(0b11), (FlagA | FlagB));
assert_eq!(Flags::from_bits_truncate(0b1000), Flags::empty());
assert_eq!(Flags::from_bits_truncate(0b1001), FlagA);
assert_eq!(AnotherSetOfFlags::from_bits_truncate(0_i8), AnotherSetOfFlags::empty());
}
#[test]
fn test_is_empty(){
assert!(Flags::empty().is_empty());
assert!(!FlagA.is_empty());
assert!(!FlagABC.is_empty());
assert!(!AnotherFlag.is_empty());
}
#[test]
fn test_is_all() {
assert!(Flags::all().is_all());
assert!(!FlagA.is_all());
assert!(FlagABC.is_all());
assert!(AnotherFlag.is_all());
}
#[test]
fn test_two_empties_do_not_intersect() {
let e1 = Flags::empty();
let e2 = Flags::empty();
assert!(!e1.intersects(e2));
assert!(AnotherFlag.intersects(AnotherFlag));
}
#[test]
fn test_empty_does_not_intersect_with_full() {
let e1 = Flags::empty();
let e2 = FlagABC;
assert!(!e1.intersects(e2));
}
#[test]
fn test_disjoint_intersects() {
let e1 = FlagA;
let e2 = FlagB;
assert!(!e1.intersects(e2));
}
#[test]
fn test_overlapping_intersects() {
let e1 = FlagA;
let e2 = FlagA | FlagB;
assert!(e1.intersects(e2));
}
#[test]
fn test_contains() {
let e1 = FlagA;
let e2 = FlagA | FlagB;
assert!(!e1.contains(e2));
assert!(e2.contains(e1));
assert!(FlagABC.contains(e2));
assert!(AnotherFlag.contains(AnotherFlag));
}
#[test]
fn test_insert(){
let mut e1 = FlagA;
let e2 = FlagA | FlagB;
e1.insert(e2);
assert_eq!(e1, e2);
let mut e3 = AnotherSetOfFlags::empty();
e3.insert(AnotherFlag);
assert_eq!(e3, AnotherFlag);
}
#[test]
fn test_remove(){
let mut e1 = FlagA | FlagB;
let e2 = FlagA | FlagC;
e1.remove(e2);
assert_eq!(e1, FlagB);
let mut e3 = AnotherFlag;
e3.remove(AnotherFlag);
assert_eq!(e3, AnotherSetOfFlags::empty());
}
#[test]
fn test_operators() {
let e1 = FlagA | FlagC;
let e2 = FlagB | FlagC;
assert_eq!((e1 | e2), FlagABC); // union
assert_eq!((e1 & e2), FlagC); // intersection
assert_eq!((e1 - e2), FlagA); // set difference
assert_eq!(!e2, FlagA); // set complement
assert_eq!(e1 ^ e2, FlagA | FlagB); // toggle
let mut e3 = e1;
e3.toggle(e2);
assert_eq!(e3, FlagA | FlagB);
let mut m4 = AnotherSetOfFlags::empty();
m4.toggle(AnotherSetOfFlags::empty());
assert_eq!(m4, AnotherSetOfFlags::empty());
}
#[test]
fn test_set() {
let mut e1 = FlagA | FlagC;
e1.set(FlagB, true);
e1.set(FlagC, false);
assert_eq!(e1, FlagA | FlagB);
}
#[test]
fn test_assignment_operators() {
let mut m1 = Flags::empty();
let e1 = FlagA | FlagC;
// union
m1 |= FlagA;
assert_eq!(m1, FlagA);
// intersection
m1 &= e1;
assert_eq!(m1, FlagA);
// set difference
m1 -= m1;
assert_eq!(m1, Flags::empty());
// toggle
m1 ^= e1;
assert_eq!(m1, e1);
}
#[test]
fn test_extend() {
let mut flags;
flags = Flags::empty();
flags.extend([].iter().cloned());
assert_eq!(flags, Flags::empty());
flags = Flags::empty();
flags.extend([FlagA, FlagB].iter().cloned());
assert_eq!(flags, FlagA | FlagB);
flags = FlagA;
flags.extend([FlagA, FlagB].iter().cloned());
assert_eq!(flags, FlagA | FlagB);
flags = FlagB;
flags.extend([FlagA, FlagABC].iter().cloned());
assert_eq!(flags, FlagABC);
}
#[test]
fn test_from_iterator() {
assert_eq!([].iter().cloned().collect::<Flags>(), Flags::empty());
assert_eq!([FlagA, FlagB].iter().cloned().collect::<Flags>(), FlagA | FlagB);
assert_eq!([FlagA, FlagABC].iter().cloned().collect::<Flags>(), FlagABC);
}
#[test]
fn test_lt() {
let mut a = Flags::empty();
let mut b = Flags::empty();
assert!(!(a < b) && !(b < a));
b = FlagB;
assert!(a < b);
a = FlagC;
assert!(!(a < b) && b < a);
b = FlagC | FlagB;
assert!(a < b);
}
#[test]
fn test_ord() {
let mut a = Flags::empty();
let mut b = Flags::empty();
assert!(a <= b && a >= b);
a = FlagA;
assert!(a > b && a >= b);
assert!(b < a && b <= a);
b = FlagB;
assert!(b > a && b >= a);
assert!(a < b && a <= b);
}
fn hash<T: Hash>(t: &T) -> u64 {
let mut s = DefaultHasher::new();
t.hash(&mut s);
s.finish()
}
#[test]
fn test_hash() {
let mut x = Flags::empty();
let mut y = Flags::empty();
assert_eq!(hash(&x), hash(&y));
x = Flags::all();
y = FlagABC;
assert_eq!(hash(&x), hash(&y));
}
#[test]
fn test_debug() {
assert_eq!(format!("{:?}", FlagA | FlagB), "FlagA | FlagB");
assert_eq!(format!("{:?}", Flags::empty()), "(empty)");
assert_eq!(format!("{:?}", FlagABC), "FlagA | FlagB | FlagC | FlagABC");
}
#[test]
fn test_binary() {
assert_eq!(format!("{:b}", FlagABC), "111");
assert_eq!(format!("{:#b}", FlagABC), "0b111");
}
#[test]
fn test_octal() {
assert_eq!(format!("{:o}", LongFlagA), "177777");
assert_eq!(format!("{:#o}", LongFlagA), "0o177777");
}
#[test]
fn test_lowerhex() {
assert_eq!(format!("{:x}", LongFlagA), "ffff");
assert_eq!(format!("{:#x}", LongFlagA), "0xffff");
}
#[test]
fn test_upperhex() {
assert_eq!(format!("{:X}", LongFlagA), "FFFF");
assert_eq!(format!("{:#X}", LongFlagA), "0xFFFF");
}
mod submodule {
bitflags! {
pub struct PublicFlags: i8 {
const FlagX = 0;
}
}
bitflags! {
struct PrivateFlags: i8 {
const FlagY = 0;
}
}
#[test]
fn test_private() {
let _ = FlagY;
}
}
#[test]
fn test_public() {
let _ = submodule::FlagX;
}
mod t1 {
mod foo {
pub type Bar = i32;
}
bitflags! {
/// baz
struct Flags: foo::Bar {
const A = 0b00000001;
#[cfg(foo)]
const B = 0b00000010;
#[cfg(foo)]
const C = 0b00000010;
}
}
}
#[test]
fn test_in_function() {
bitflags! {
struct Flags: u8 {
const A = 1;
#[cfg(any())] // false
const B = 2;
}
}
assert_eq!(Flags::all(), A);
assert_eq!(format!("{:?}", A), "A");
}
}

View File

@ -1,20 +0,0 @@
#![allow(dead_code)]
#![no_std]
#[macro_use]
extern crate bitflags;
#[allow(unused_imports)]
use core::fmt::Display;
bitflags! {
/// baz
struct Flags: u32 {
const A = 0b00000001;
}
}
#[test]
fn main() {
}

View File

@ -1,21 +0,0 @@
#![allow(dead_code)]
#[macro_use]
extern crate bitflags;
bitflags! {
/// baz
struct Flags: u32 {
const A = 0b00000001;
#[doc = "bar"]
const B = 0b00000010;
const C = 0b00000100;
#[doc = "foo"]
const ABC = A.bits | B.bits | C.bits;
}
}
#[test]
fn smoke() {
assert_eq!(ABC, A | B | C);
}

View File

@ -1,22 +0,0 @@
#![allow(dead_code)]
#![no_std]
#[macro_use]
extern crate bitflags;
bitflags! {
/// baz
struct Flags: u32 {
const A = 0b00000001;
#[doc = "bar"]
const B = 0b00000010;
const C = 0b00000100;
#[doc = "foo"]
const ABC = A.bits | B.bits | C.bits;
}
}
#[test]
fn smoke() {
assert_eq!(ABC, A | B | C);
}

View File

@ -1,30 +0,0 @@
#![cfg(feature = "unstable_testing")]
#![allow(dead_code, unused_imports)]
#![feature(i128_type)]
#[macro_use]
extern crate bitflags;
bitflags! {
/// baz
struct Flags128: u128 {
const A = 0x0000_0000_0000_0000_0000_0000_0000_0001;
const B = 0x0000_0000_0000_1000_0000_0000_0000_0000;
const C = 0x8000_0000_0000_0000_0000_0000_0000_0000;
const ABC = A.bits | B.bits | C.bits;
}
}
#[test]
fn test_i128_bitflags() {
assert_eq!(ABC, A | B | C);
assert_eq!(A.bits, 0x0000_0000_0000_0000_0000_0000_0000_0001);
assert_eq!(B.bits, 0x0000_0000_0000_1000_0000_0000_0000_0000);
assert_eq!(C.bits, 0x8000_0000_0000_0000_0000_0000_0000_0000);
assert_eq!(ABC.bits, 0x8000_0000_0000_1000_0000_0000_0000_0001);
assert_eq!(format!("{:?}", A), "A");
assert_eq!(format!("{:?}", B), "B");
assert_eq!(format!("{:?}", C), "C");
assert_eq!(format!("{:?}", ABC), "A | B | C | ABC");
}

View File

@ -1 +0,0 @@
{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".gitignore":"f9b1ca6ae27d1c18215265024629a8960c31379f206d9ed20f64e0b2dcf79805",".travis.yml":"987b79076a914cb1279b1a75ac8cf5f802db080cc986332e01d1d65ef824c598","Cargo.toml":"616eaec02a25406f9f5ed0b3b7b44e9f38ce44dba665fe0d7254edad3aee89df","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb","README.md":"52151641bdec86e4341038ee8abb64154840845c1bf718c56d450b09835d220b","src/lib.rs":"b9829ea348eaba4f3cb0b8cdfee92fec2d4e0d6d2c9a2da0d0778ee857879d80"},"package":"f2f382711e76b9de6c744cc00d0497baba02fb00a787f088c879f01d09468e32"}

View File

@ -1,2 +0,0 @@
target
Cargo.lock

View File

@ -1,24 +0,0 @@
language: rust
rust:
- stable
- beta
- nightly
sudo: false
before_script:
- pip install 'travis-cargo<0.2' --user && export PATH=$HOME/.local/bin:$PATH
script:
- cargo build --verbose
- cargo test --verbose
- cargo test --verbose --features tokio
- cargo doc --no-deps
after_success:
- travis-cargo --only nightly doc-upload
env:
global:
secure: "G857gdm63OJ2wcwdEBBeY+53D/zRSSmPfAp/H+8vRy5nnB+4GBq5Xqugq9n4BrvJMoPLMMMAmB46Chk7HHkkk/e4WTQ2orX61c4nNF3b4rdik6fzwVKk4Gy06FlM63cEa9/1iN2BiOg6NA81cmrUrK1ezIdg+8YECsiVu9+7m2g="
notifications:
email:
on_success: never

View File

@ -1,23 +0,0 @@
[package]
name = "bufstream"
version = "0.1.3"
authors = ["The Rust Project Developers"]
license = "MIT/Apache-2.0"
repository = "https://github.com/alexcrichton/bufstream"
homepage = "https://github.com/alexcrichton/bufstream"
documentation = "http://alexcrichton.com/bufstream"
description = """
Buffered I/O for streams where each read/write half is separately buffered
"""
[dependencies.futures]
optional = true
version = "0.1.13"
[dependencies.tokio-io]
optional = true
version = "0.1.1"
[features]
default = []
tokio = ["futures", "tokio-io"]

View File

@ -1,201 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,25 +0,0 @@
Copyright (c) 2014 The Rust Project Developers
Permission is hereby granted, free of charge, to any
person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the
Software without restriction, including without
limitation the rights to use, copy, modify, merge,
publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software
is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice
shall be included in all copies or substantial portions
of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

View File

@ -1,22 +0,0 @@
bufstream
=========
Buffered I/O streams for reading/writing
[![Build Status](https://travis-ci.org/alexcrichton/bufstream.svg?branch=master)](https://travis-ci.org/alexcrichton/bufstream)
[Documentation](http://alexcrichton.com/bufstream)
## Usage
```toml
[dependencies]
bufstream = "0.1"
```
## Tokio
There is support for tokio's `AsyncRead` + `AsyncWrite` traits through the `tokio`
feature. When using this crate with asynchronous IO, make sure to properly flush
the stream before dropping it since IO during drop may cause panics. For the same
reason you should stay away from `BufStream::into_inner`.

View File

@ -1,262 +0,0 @@
// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! A crate for separately buffered streams.
//!
//! This crate provides a `BufStream` type which provides buffering of both the
//! reading and writing halves of a `Read + Write` type. Each half is completely
//! independently buffered of the other, which may not always be desired. For
//! example `BufStream<File>` may have surprising semantics.
//!
//! # Usage
//!
//! ```toml
//! [dependencies]
//! bufstream = "0.1"
//! ```
//!
//! ```no_run
//! use std::io::prelude::*;
//! use std::net::TcpStream;
//! use bufstream::BufStream;
//!
//!
//! let stream = TcpStream::connect("localhost:4000").unwrap();
//! let mut buf = BufStream::new(stream);
//! buf.read(&mut [0; 1024]).unwrap();
//! buf.write(&[0; 1024]).unwrap();
//! ```
//!
//! # Async I/O
//!
//! This crate optionally can support async I/O streams with the [Tokio stack] via
//! the `tokio` feature of this crate:
//!
//! [Tokio stack]: https://tokio.rs/
//!
//! ```toml
//! bufstream = { version = "0.2", features = ["tokio"] }
//! ```
//!
//! All methods are internally capable of working with streams that may return
//! [`ErrorKind::WouldBlock`] when they're not ready to perform the particular
//! operation.
//!
//! [`ErrorKind::WouldBlock`]: https://doc.rust-lang.org/std/io/enum.ErrorKind.html
//!
//! Note that care needs to be taken when using these objects, however. The
//! Tokio runtime, in particular, requires that data is fully flushed before
//! dropping streams. For compatibility with blocking streams all streams are
//! flushed/written when they are dropped, and this is not always a suitable
//! time to perform I/O. If I/O streams are flushed before drop, however, then
//! these operations will be a noop.
#[cfg(feature = "tokio")] extern crate futures;
#[cfg(feature = "tokio")] #[macro_use] extern crate tokio_io;
use std::fmt;
use std::io::prelude::*;
use std::io::{self, BufReader, BufWriter};
use std::error;
#[cfg(feature = "tokio")] use futures::Poll;
#[cfg(feature = "tokio")] use tokio_io::{AsyncRead, AsyncWrite};
const DEFAULT_BUF_SIZE: usize = 8 * 1024;
/// Wraps a Stream and buffers input and output to and from it.
///
/// It can be excessively inefficient to work directly with a `Read+Write`. For
/// example, every call to `read` or `write` on `TcpStream` results in a system
/// call. A `BufStream` keeps in memory buffers of data, making large,
/// infrequent calls to `read` and `write` on the underlying `Read+Write`.
///
/// The output buffer will be written out when this stream is dropped.
#[derive(Debug)]
pub struct BufStream<S: Write> {
inner: BufReader<InternalBufWriter<S>>
}
/// An error returned by `into_inner` which combines an error that
/// happened while writing out the buffer, and the buffered writer object
/// which may be used to recover from the condition.
#[derive(Debug)]
pub struct IntoInnerError<W>(W, io::Error);
impl<W> IntoInnerError<W> {
/// Returns the error which caused the call to `into_inner()` to fail.
///
/// This error was returned when attempting to write the internal buffer.
pub fn error(&self) -> &io::Error { &self.1 }
/// Returns the buffered writer instance which generated the error.
///
/// The returned object can be used for error recovery, such as
/// re-inspecting the buffer.
pub fn into_inner(self) -> W { self.0 }
}
impl<W> From<IntoInnerError<W>> for io::Error {
fn from(iie: IntoInnerError<W>) -> io::Error { iie.1 }
}
impl<W: fmt::Debug> error::Error for IntoInnerError<W> {
fn description(&self) -> &str {
error::Error::description(self.error())
}
}
impl<W> fmt::Display for IntoInnerError<W> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.error().fmt(f)
}
}
struct InternalBufWriter<W: Write>(Option<BufWriter<W>>);
impl<W: Write> InternalBufWriter<W> {
fn get_ref(&self) -> &BufWriter<W> {
self.0.as_ref().unwrap()
}
fn get_mut(&mut self) -> &mut BufWriter<W> {
self.0.as_mut().unwrap()
}
}
impl<W: Read + Write> Read for InternalBufWriter<W> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.get_mut().get_mut().read(buf)
}
}
impl<W: Write + fmt::Debug> fmt::Debug for InternalBufWriter<W> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.get_ref().fmt(f)
}
}
impl<S: Read + Write> BufStream<S> {
/// Creates a new buffered stream with explicitly listed capacities for the
/// reader/writer buffer.
pub fn with_capacities(reader_cap: usize, writer_cap: usize, inner: S)
-> BufStream<S> {
let writer = BufWriter::with_capacity(writer_cap, inner);
let internal_writer = InternalBufWriter(Some(writer));
let reader = BufReader::with_capacity(reader_cap, internal_writer);
BufStream { inner: reader }
}
/// Creates a new buffered stream with the default reader/writer buffer
/// capacities.
pub fn new(inner: S) -> BufStream<S> {
BufStream::with_capacities(DEFAULT_BUF_SIZE, DEFAULT_BUF_SIZE, inner)
}
/// Gets a reference to the underlying stream.
pub fn get_ref(&self) -> &S {
self.inner.get_ref().get_ref().get_ref()
}
/// Gets a mutable reference to the underlying stream.
///
/// # Warning
///
/// It is inadvisable to read directly from or write directly to the
/// underlying stream.
pub fn get_mut(&mut self) -> &mut S {
self.inner.get_mut().get_mut().get_mut()
}
/// Unwraps this `BufStream`, returning the underlying stream.
///
/// The internal write buffer is written out before returning the stream.
/// Any leftover data in the read buffer is lost.
pub fn into_inner(mut self) -> Result<S, IntoInnerError<BufStream<S>>> {
let e = {
let InternalBufWriter(ref mut w) = *self.inner.get_mut();
let (e, w2) = match w.take().unwrap().into_inner() {
Ok(s) => return Ok(s),
Err(err) => {
(io::Error::new(err.error().kind(), err.error().to_string()),
err.into_inner())
}
};
*w = Some(w2);
e
};
Err(IntoInnerError(self, e))
}
}
impl<S: Read + Write> BufRead for BufStream<S> {
fn fill_buf(&mut self) -> io::Result<&[u8]> { self.inner.fill_buf() }
fn consume(&mut self, amt: usize) { self.inner.consume(amt) }
fn read_until(&mut self, byte: u8, buf: &mut Vec<u8>) -> io::Result<usize> {
self.inner.read_until(byte, buf)
}
fn read_line(&mut self, string: &mut String) -> io::Result<usize> {
self.inner.read_line(string)
}
}
impl<S: Read + Write> Read for BufStream<S> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.inner.read(buf)
}
}
impl<S: Read + Write> Write for BufStream<S> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.inner.get_mut().0.as_mut().unwrap().write(buf)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.get_mut().0.as_mut().unwrap().flush()
}
}
#[cfg(feature = "tokio")]
impl<S: AsyncRead + AsyncWrite> AsyncRead for BufStream<S> {}
#[cfg(feature = "tokio")]
impl<S: AsyncRead + AsyncWrite> AsyncWrite for BufStream<S> {
fn shutdown(&mut self) -> Poll<(), io::Error> {
try_nb!(self.flush());
let mut inner = self.inner.get_mut().0.as_mut().unwrap();
inner.shutdown()
}
}
#[cfg(test)]
mod tests {
use std::io::prelude::*;
use std::io;
use super::BufStream;
// This is just here to make sure that we don't infinite loop in the
// newtype struct autoderef weirdness
#[test]
fn test_buffered_stream() {
struct S;
impl Write for S {
fn write(&mut self, b: &[u8]) -> io::Result<usize> { Ok(b.len()) }
fn flush(&mut self) -> io::Result<()> { Ok(()) }
}
impl Read for S {
fn read(&mut self, _: &mut [u8]) -> io::Result<usize> { Ok(0) }
}
let mut stream = BufStream::new(S);
assert_eq!(stream.read(&mut [0; 10]).unwrap(), 0);
stream.write(&[0; 10]).unwrap();
stream.flush().unwrap();
}
}

View File

@ -1 +0,0 @@
{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".gitignore":"b65a8d0e5eb4bb7a0b8dca8be9102ece8810a5e4591dd289da70f28b8e829dfb",".travis.yml":"5c1a66c2f3f70d7a6fd7ee40b87863d95b78294244cc037453220d64fc431362","CHANGELOG.md":"729d4632f518b0c699d1b947e5d8ddd3fc6a8878bd7796d7b96b2f58772f0478","Cargo.toml":"778cf5a227b5f6a0200d9385d2a8adc59a559a6822ab4d1941348f3eee92d791","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"7b63ecd5f1902af1b63729947373683c32745c16a10e8e6292e2e2dcd7e90ae0","README.md":"ea4153526db8a7b7c33fba324fd87f9e43ecac4c973ec1af242c325768c1a4b3","scala-bench/bench.scala":"8f203f98f2c938115247a8ed5ef9d0848fe0f8dca49fc43e2dba88bdd61029e3","src/bin/bench.rs":"7826ed26fcce96f0e2fe3d0b46c36bbfde70168546124018394b1aae8fbf1ec3","src/bin/extra_impls/mod.rs":"b72e67187cbfc1672faacd7d906604f3adc0c7540c781db88c6f4f1726229db9","src/bin/extra_impls/mpsc_queue.rs":"94551cbe0075c40472bae74cb500070c5c621e3690e15c1374b56c5cb7dd6639","src/bin/stress-msq.rs":"f49ba9adb0308013dbd7ed748572659c8b25d045e20609a21ed29ce35784f8f7","src/lib.rs":"b84c7a07bcae8342ce791e42566d103bfc7ac072994a2fe4d6f0b14c8d905f2d","src/mem/cache_padded.rs":"710de7fc76c04bda2e9eafa9f2e9a038fd381330f0e2ac657b56552a9bd9223c","src/mem/epoch/atomic.rs":"ec73c5e271b6b16bc489d3a6b47a48c5ef21cd1d8320d5ed01a6cac271afb42a","src/mem/epoch/garbage.rs":"dd3a3270481da756cf2e8aece5518642059d72492ffbdb107ad9c92d608ed3d3","src/mem/epoch/global.rs":"901df28fdb255cf2466962fa1fcfe3f3006325b94d80a5edb71e3251cecd592e","src/mem/epoch/guard.rs":"75c2a771d88e859f1f53a79a9466ed1e62a6854a1ddca99c6dcaa3d4ca3520e1","src/mem/epoch/local.rs":"d451c2c05fe50e80bdc92313d074b7db13f54d6caa3cf6df6ba5b717566932ae","src/mem/epoch/mod.rs":"a57570492cc2b23b5d6164e0738af5b8d7d65c4b2de08fdfaa2283a47481fce0","src/mem/epoch/participant.rs":"d16e9a81d34f8368340126e9e420bdeec9e661c94aec7a26057be26cad0982df","src/mem/epoch/participants.rs":"c7f4edd7e632130cd149afad8abbeb21888df55b7e4db4206a8840a218bf764d","src/mem/mod.rs":"c60aaeee01ce6abe2418f6f2a3cdd38564a6a46d3c47285d9730a358f52fa6bf","src/scoped.rs":"9ef97832dea5dbdebc88f6c1c8dee5ac5e801f302b70ba17b667214fc3fe57ed","src/sync/arc_cell.rs":"d12dcaca3d59cb0a7c34470dff60c11cb8e25ecde87baf3940bd0747bb107672","src/sync/atomic_option.rs":"dcdfd1080c35d782f041edc7d6c52c1c8fc05f4fe75a9dad261a8982f954ae97","src/sync/chase_lev.rs":"9679cb37bf777466c714e3b8aca7c583638e4534cbb298449130cfa7a07c1d78","src/sync/mod.rs":"35e5f793530e198e891e6ef619da161bbcd31a1de1419dc5b9e9a954d3542c02","src/sync/ms_queue.rs":"cf735b32c12d3227364b2a2abf75a99e5f36f2980b58f34821462cdaf1aac209","src/sync/seg_queue.rs":"e9178f259a0fec71aeb4fb9d5c2bd668eb3dc5f3a3808f167eaae263d76f6646","src/sync/treiber_stack.rs":"60e7f82a42379fbcc2b418b9d50cef98ebc743d9747bf646eb1f3d723189bea4"},"package":"0c5ea215664ca264da8a9d9c3be80d2eaf30923c259d03e870388eb927508f97"}

View File

@ -1,13 +0,0 @@
# Compiled files
*.o
*.so
*.rlib
*.dll
*.class
# Executables
*.exe
# Generated by Cargo
/target/
Cargo.lock

View File

@ -1,36 +0,0 @@
language: rust
# necessary for `travis-cargo coveralls --no-sudo`
addons:
apt:
packages:
- libcurl4-openssl-dev
- libelf-dev
- libdw-dev
# run builds for all the trains (and more)
rust:
- nightly
- beta
- stable
# load travis-cargo
before_script:
- |
pip install 'travis-cargo<0.2' --user &&
export PATH=$HOME/.local/bin:$PATH
# the main build
script:
- |
travis-cargo build &&
travis-cargo test &&
travis-cargo test -- --release &&
travis-cargo run -- --bin bench --release &&
travis-cargo --only stable doc
env:
global:
# override the default `--features unstable` used for the nightly branch (optional)
- TRAVIS_CARGO_NIGHTLY_FEATURE=nightly
notifications:
email:
on_success: never

View File

@ -1,11 +0,0 @@
# Version 0.2
- Changed existing non-blocking `pop` methods to `try_pop`
- Added blocking `pop` support to Michael-Scott queue
- Added Chase-Lev work-stealing deque
# Version 0.1
- Added [epoch-based memory management](http://aturon.github.io/blog/2015/08/27/epoch/)
- Added Michael-Scott queue
- Added Segmented array queue

View File

@ -1,15 +0,0 @@
[package]
name = "crossbeam"
version = "0.2.10"
authors = ["Aaron Turon <aturon@mozilla.com>"]
description = "Support for lock-free data structures, synchronizers, and parallel programming"
repository = "https://github.com/aturon/crossbeam"
documentation = "http://aturon.github.io/crossbeam-doc/crossbeam/"
readme = "README.md"
license = "Apache-2.0/MIT"
[features]
nightly = []
[dev-dependencies]
rand = "0.3"

View File

@ -1,201 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,38 +0,0 @@
# Crossbeam: support for concurrent and parallel programming
[![Build Status](https://travis-ci.org/aturon/crossbeam.svg?branch=master)](https://travis-ci.org/aturon/crossbeam)
This crate is an early work in progress. The focus for the moment is
concurrency:
- **Non-blocking data structures**. These data structures allow for high
performance, highly-concurrent access, much superior to wrapping with a
`Mutex`. Ultimately the goal is to include stacks, queues, deques, bags, sets
and maps.
- **Memory management**. Because non-blocking data structures avoid global
synchronization, it is not easy to tell when internal data can be safely
freed. The `mem` module provides generic, easy to use, and high-performance APIs
for managing memory in these cases.
- **Synchronization**. The standard library provides a few synchronization
primitives (locks, semaphores, barriers, etc) but this crate seeks to expand
that set to include more advanced/niche primitives, as well as userspace
alternatives.
- **Scoped thread API**. Finally, the crate provides a "scoped" thread API,
making it possible to spawn threads that share stack data with their parents.
# Usage
To use Crossbeam, add this to your `Cargo.toml`:
```toml
[dependencies]
crossbeam = "0.2"
```
For examples of what Crossbeam is capable of, see the
[documentation][docs].
[docs]: http://aturon.github.io/crossbeam-doc/crossbeam/

View File

@ -1,195 +0,0 @@
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent._
import scala.concurrent.duration._
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic._
import java.util.Stack
import scala.annotation.tailrec
final class MSQueue[A](a: A) {
private abstract class Q
private final case class Node(data: A, next: AtomicReference[Q] = new AtomicReference(Emp)) extends Q
private final case object Emp extends Q
private val head = new AtomicReference(Node(a))
private val tail = new AtomicReference(head.get())
def enq(a: A) {
val newNode = new Node(a)
while (true) {
val curTail = tail.get()
curTail.next.get()match {
case n@Node(_,_) => tail.compareAndSet(curTail, n)
case Emp => {
if (curTail.next.compareAndSet(Emp, newNode)) {
tail.compareAndSet(curTail, newNode)
return
}
}
}
}
}
def deq(): Option[A] = {
while (true) {
val cur_head = head.get()
cur_head.next.get() match {
case Emp => return None
case n@Node(data, _) => {
if (head.compareAndSet(cur_head, n)) {
return Some(data)
}
}
}
}
None
}
}
abstract class MyBool
final case class MyTrue() extends MyBool
final case class MyFalse() extends MyBool
object Bench {
def time(block: => Unit): Long = {
val t0 = System.nanoTime()
val result = block // call-by-name
val t1 = System.nanoTime()
t1 - t0
}
def do_linked(threads: Int, count: Int) {
val q: ConcurrentLinkedQueue[MyBool] = new ConcurrentLinkedQueue();
val t = time {
var s = new Stack[Future[Unit]]
for (i <- 1 to threads) {
s.push(Future {
for (i <- 1 to count+1) {
//if (i % 100000 == 0) { println(q.size()) }
q.offer(MyTrue())
}
})
}
var rn = 0
while (rn < count) {
if (q.poll() != null) {
rn += 1
}
}
while (!s.empty()) {
Await.ready(s.pop(), Duration.Inf)
}
}
println("Linked: " + t / (count * threads))
}
def do_linked_mpmc(threads: Int, count: Int) {
val q = new ConcurrentLinkedQueue[MyBool];
val prod = new AtomicInteger();
val t = time {
var s = new Stack[Future[Unit]]
for (i <- 1 to threads) {
s.push(Future {
for (i <- 1 to count+1) {
q.offer(MyTrue())
//if (i % 100000 == 0) { println(q.size()) }
}
if (prod.incrementAndGet() == threads) {
for (i <- 0 to threads) { q.offer(MyFalse()) }
}
})
s.push(Future {
var done = false;
while (!done) {
q.poll() match {
case MyFalse() => done = true
case _ => {}
}
}
})
}
while (!s.empty()) {
Await.ready(s.pop(), Duration.Inf)
}
}
println("Linked mpmc: " + t / (count * threads) + " (" + t / 1000000000 + ")")
}
def do_msq(threads: Int, count: Int) {
val q = new MSQueue[Int](0);
val t = time {
var s = new Stack[Future[Unit]]
for (i <- 1 to threads) {
s.push(Future { for (i <- 1 to count+1) { q.enq(i) } })
}
var rn = 0
while (rn < count) {
if (q.deq() != None) {
rn += 1
}
}
while (!s.empty()) {
Await.ready(s.pop(), Duration.Inf)
}
}
println("MSQ: " + t / (count * threads) + " (" + t / 1000000000 + ")")
}
def do_msq_mpmc(threads: Int, count: Int) {
val q = new MSQueue[Boolean](true);
val prod = new AtomicInteger();
val t = time {
var s = new Stack[Future[Unit]]
for (i <- 1 to threads) {
s.push(Future {
for (i <- 1 to count+1) { q.enq(true) }
if (prod.incrementAndGet() == threads) {
for (i <- 1 to threads) { q.enq(false) }
}
})
s.push(Future {
var done = false;
while (!done) {
q.deq() match {
case Some(false) => done = true
case _ => {}
}
}
})
}
while (!s.empty()) {
Await.ready(s.pop(), Duration.Inf)
}
}
println("MSQ mpmc: " + t / (count * threads) + " (" + t / 1000000000 + ")")
}
def main(args: Array[String]) {
do_linked(2, 1000000)
do_linked(2, 10000000)
do_msq(2, 1000000)
do_msq(2, 10000000)
do_linked_mpmc(2, 1000000)
do_linked_mpmc(2, 10000000)
do_msq_mpmc(2, 1000000)
do_msq_mpmc(2, 10000000)
}
}

View File

@ -1,165 +0,0 @@
extern crate crossbeam;
use std::collections::VecDeque;
use std::sync::Mutex;
use std::sync::mpsc::channel;
use std::time::Duration;
use crossbeam::scope;
use crossbeam::sync::MsQueue;
use crossbeam::sync::SegQueue;
use extra_impls::mpsc_queue::Queue as MpscQueue;
mod extra_impls;
const COUNT: u64 = 10000000;
const THREADS: u64 = 2;
#[cfg(feature = "nightly")]
fn time<F: FnOnce()>(f: F) -> Duration {
let start = ::std::time::Instant::now();
f();
start.elapsed()
}
#[cfg(not(feature = "nightly"))]
fn time<F: FnOnce()>(_f: F) -> Duration {
Duration::new(0, 0)
}
fn nanos(d: Duration) -> f64 {
d.as_secs() as f64 * 1000000000f64 + (d.subsec_nanos() as f64)
}
trait Queue<T> {
fn push(&self, T);
fn try_pop(&self) -> Option<T>;
}
impl<T> Queue<T> for MsQueue<T> {
fn push(&self, t: T) { self.push(t) }
fn try_pop(&self) -> Option<T> { self.try_pop() }
}
impl<T> Queue<T> for SegQueue<T> {
fn push(&self, t: T) { self.push(t) }
fn try_pop(&self) -> Option<T> { self.try_pop() }
}
impl<T> Queue<T> for MpscQueue<T> {
fn push(&self, t: T) { self.push(t) }
fn try_pop(&self) -> Option<T> {
use extra_impls::mpsc_queue::*;
loop {
match self.pop() {
Data(t) => return Some(t),
Empty => return None,
Inconsistent => (),
}
}
}
}
impl<T> Queue<T> for Mutex<VecDeque<T>> {
fn push(&self, t: T) { self.lock().unwrap().push_back(t) }
fn try_pop(&self) -> Option<T> { self.lock().unwrap().pop_front() }
}
fn bench_queue_mpsc<Q: Queue<u64> + Sync>(q: Q) -> f64 {
let d = time(|| {
scope(|scope| {
for _i in 0..THREADS {
let qr = &q;
scope.spawn(move || {
for x in 0..COUNT {
let _ = qr.push(x);
}
});
}
let mut count = 0;
while count < COUNT*THREADS {
if q.try_pop().is_some() {
count += 1;
}
}
});
});
nanos(d) / ((COUNT * THREADS) as f64)
}
fn bench_queue_mpmc<Q: Queue<bool> + Sync>(q: Q) -> f64 {
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering::Relaxed;
let prod_count = AtomicUsize::new(0);
let d = time(|| {
scope(|scope| {
for _i in 0..THREADS {
let qr = &q;
let pcr = &prod_count;
scope.spawn(move || {
for _x in 0..COUNT {
qr.push(true);
}
if pcr.fetch_add(1, Relaxed) == (THREADS as usize) - 1 {
for _x in 0..THREADS {
qr.push(false)
}
}
});
scope.spawn(move || {
loop {
if let Some(false) = qr.try_pop() { break }
}
});
}
});
});
nanos(d) / ((COUNT * THREADS) as f64)
}
fn bench_chan_mpsc() -> f64 {
let (tx, rx) = channel();
let d = time(|| {
scope(|scope| {
for _i in 0..THREADS {
let my_tx = tx.clone();
scope.spawn(move || {
for x in 0..COUNT {
let _ = my_tx.send(x);
}
});
}
for _i in 0..COUNT*THREADS {
let _ = rx.recv().unwrap();
}
});
});
nanos(d) / ((COUNT * THREADS) as f64)
}
fn main() {
println!("MSQ mpsc: {}", bench_queue_mpsc(MsQueue::new()));
println!("chan mpsc: {}", bench_chan_mpsc());
println!("mpsc mpsc: {}", bench_queue_mpsc(MpscQueue::new()));
println!("Seg mpsc: {}", bench_queue_mpsc(SegQueue::new()));
println!("MSQ mpmc: {}", bench_queue_mpmc(MsQueue::new()));
println!("Seg mpmc: {}", bench_queue_mpmc(SegQueue::new()));
// println!("queue_mpsc: {}", bench_queue_mpsc());
// println!("queue_mpmc: {}", bench_queue_mpmc());
// println!("mutex_mpmc: {}", bench_mutex_mpmc());
}

View File

@ -1 +0,0 @@
pub mod mpsc_queue;

View File

@ -1,155 +0,0 @@
/* Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
* SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are
* those of the authors and should not be interpreted as representing official
* policies, either expressed or implied, of Dmitry Vyukov.
*/
//! A mostly lock-free multi-producer, single consumer queue.
//!
//! This module contains an implementation of a concurrent MPSC queue. This
//! queue can be used to share data between threads, and is also used as the
//! building block of channels in rust.
//!
//! Note that the current implementation of this queue has a caveat of the `pop`
//! method, and see the method for more information about it. Due to this
//! caveat, this queue may not be appropriate for all use-cases.
// http://www.1024cores.net/home/lock-free-algorithms
// /queues/non-intrusive-mpsc-node-based-queue
pub use self::PopResult::*;
use std::fmt;
use std::ptr;
use std::cell::UnsafeCell;
use std::sync::atomic::{AtomicPtr, Ordering};
/// A result of the `pop` function.
#[derive(Debug)]
pub enum PopResult<T> {
/// Some data has been popped
Data(T),
/// The queue is empty
Empty,
/// The queue is in an inconsistent state. Popping data should succeed, but
/// some pushers have yet to make enough progress in order allow a pop to
/// succeed. It is recommended that a pop() occur "in the near future" in
/// order to see if the sender has made progress or not
Inconsistent,
}
#[derive(Debug)]
struct Node<T> {
next: AtomicPtr<Node<T>>,
value: Option<T>,
}
/// The multi-producer single-consumer structure. This is not cloneable, but it
/// may be safely shared so long as it is guaranteed that there is only one
/// popper at a time (many pushers are allowed).
pub struct Queue<T> {
head: AtomicPtr<Node<T>>,
tail: UnsafeCell<*mut Node<T>>,
}
impl<T> fmt::Debug for Queue<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Queue {{ ... }}")
}
}
unsafe impl<T: Send> Send for Queue<T> { }
unsafe impl<T: Send> Sync for Queue<T> { }
impl<T> Node<T> {
unsafe fn new(v: Option<T>) -> *mut Node<T> {
Box::into_raw(Box::new(Node {
next: AtomicPtr::new(ptr::null_mut()),
value: v,
}))
}
}
impl<T> Queue<T> {
/// Creates a new queue that is safe to share among multiple producers and
/// one consumer.
pub fn new() -> Queue<T> {
let stub = unsafe { Node::new(None) };
Queue {
head: AtomicPtr::new(stub),
tail: UnsafeCell::new(stub),
}
}
/// Pushes a new value onto this queue.
pub fn push(&self, t: T) {
unsafe {
let n = Node::new(Some(t));
let prev = self.head.swap(n, Ordering::AcqRel);
(*prev).next.store(n, Ordering::Release);
}
}
/// Pops some data from this queue.
///
/// Note that the current implementation means that this function cannot
/// return `Option<T>`. It is possible for this queue to be in an
/// inconsistent state where many pushes have succeeded and completely
/// finished, but pops cannot return `Some(t)`. This inconsistent state
/// happens when a pusher is pre-empted at an inopportune moment.
///
/// This inconsistent state means that this queue does indeed have data, but
/// it does not currently have access to it at this time.
pub fn pop(&self) -> PopResult<T> {
unsafe {
let tail = *self.tail.get();
let next = (*tail).next.load(Ordering::Acquire);
if !next.is_null() {
*self.tail.get() = next;
assert!((*tail).value.is_none());
assert!((*next).value.is_some());
let ret = (*next).value.take().unwrap();
let _ = Box::from_raw(tail);
return Data(ret);
}
if self.head.load(Ordering::Acquire) == tail {Empty} else {Inconsistent}
}
}
}
impl<T> Drop for Queue<T> {
fn drop(&mut self) {
unsafe {
let mut cur = *self.tail.get();
while !cur.is_null() {
let next = (*cur).next.load(Ordering::Relaxed);
let _ = Box::from_raw(cur);
cur = next;
}
}
}
}

View File

@ -1,36 +0,0 @@
extern crate crossbeam;
use crossbeam::sync::MsQueue;
use crossbeam::scope;
use std::sync::Arc;
const DUP: usize = 4;
const THREADS: u32 = 2;
const COUNT: u64 = 100000;
fn main() {
scope(|s| {
for _i in 0..DUP {
let q = Arc::new(MsQueue::new());
let qs = q.clone();
s.spawn(move || {
for i in 1..COUNT { qs.push(i) }
});
for _i in 0..THREADS {
let qr = q.clone();
s.spawn(move || {
let mut cur: u64 = 0;
for _j in 0..COUNT {
if let Some(new) = qr.try_pop() {
assert!(new > cur);
cur = new;
}
}
});
}
}
});
}

View File

@ -1,54 +0,0 @@
//! Support for concurrent and parallel programming.
//!
//! This crate is an early work in progress. The focus for the moment is
//! concurrency:
//!
//! - **Non-blocking data structures**. These data structures allow for high
//! performance, highly-concurrent access, much superior to wrapping with a
//! `Mutex`. Ultimately the goal is to include stacks, queues, deques, bags,
//! sets and maps. These live in the `sync` module.
//!
//! - **Memory management**. Because non-blocking data structures avoid global
//! synchronization, it is not easy to tell when internal data can be safely
//! freed. The `mem` module provides generic, easy to use, and high-performance
//! APIs for managing memory in these cases. These live in the `mem` module.
//!
//! - **Synchronization**. The standard library provides a few synchronization
//! primitives (locks, semaphores, barriers, etc) but this crate seeks to expand
//! that set to include more advanced/niche primitives, as well as userspace
//! alternatives. These live in the `sync` module.
//!
//! - **Scoped thread API**. Finally, the crate provides a "scoped" thread API,
//! making it possible to spawn threads that share stack data with their
//! parents. This functionality is exported at the top-level.
//#![deny(missing_docs)]
#![cfg_attr(feature = "nightly",
feature(const_fn, repr_simd, optin_builtin_traits))]
use std::thread;
pub use scoped::{scope, Scope, ScopedJoinHandle};
pub mod mem;
pub mod sync;
mod scoped;
#[doc(hidden)]
trait FnBox {
fn call_box(self: Box<Self>);
}
impl<F: FnOnce()> FnBox for F {
fn call_box(self: Box<Self>) { (*self)() }
}
/// Like `std::thread::spawn`, but without the closure bounds.
pub unsafe fn spawn_unsafe<'a, F>(f: F) -> thread::JoinHandle<()> where F: FnOnce() + Send + 'a {
use std::mem;
let closure: Box<FnBox + 'a> = Box::new(f);
let closure: Box<FnBox + Send> = mem::transmute(closure);
thread::spawn(move || closure.call_box())
}

View File

@ -1,149 +0,0 @@
use std::marker;
use std::cell::UnsafeCell;
use std::fmt;
use std::mem;
use std::ptr;
use std::ops::{Deref, DerefMut};
// For now, treat this as an arch-independent constant.
const CACHE_LINE: usize = 32;
#[cfg_attr(feature = "nightly",
repr(simd))]
#[derive(Debug)]
struct Padding(u64, u64, u64, u64);
/// Pad `T` to the length of a cacheline.
///
/// Sometimes concurrent programming requires a piece of data to be padded out
/// to the size of a cacheline to avoid "false sharing": cachelines being
/// invalidated due to unrelated concurrent activity. Use the `CachePadded` type
/// when you want to *avoid* cache locality.
///
/// At the moment, cache lines are assumed to be 32 * sizeof(usize) on all
/// architectures.
///
/// **Warning**: the wrapped data is never dropped; move out using `ptr::read`
/// if you need to run dtors.
pub struct CachePadded<T> {
data: UnsafeCell<[usize; CACHE_LINE]>,
_marker: ([Padding; 0], marker::PhantomData<T>),
}
impl<T> fmt::Debug for CachePadded<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "CachePadded {{ ... }}")
}
}
unsafe impl<T: Send> Send for CachePadded<T> {}
unsafe impl<T: Sync> Sync for CachePadded<T> {}
/// Types for which mem::zeroed() is safe.
///
/// If a type `T: ZerosValid`, then a sequence of zeros the size of `T` must be
/// a valid member of the type `T`.
pub unsafe trait ZerosValid {}
#[cfg(feature = "nightly")]
unsafe impl ZerosValid for .. {}
macro_rules! zeros_valid { ($( $T:ty )*) => ($(
unsafe impl ZerosValid for $T {}
)*)}
zeros_valid!(u8 u16 u32 u64 usize);
zeros_valid!(i8 i16 i32 i64 isize);
unsafe impl ZerosValid for ::std::sync::atomic::AtomicUsize {}
unsafe impl<T> ZerosValid for ::std::sync::atomic::AtomicPtr<T> {}
impl<T: ZerosValid> CachePadded<T> {
/// A const fn equivalent to mem::zeroed().
#[cfg(not(feature = "nightly"))]
pub fn zeroed() -> CachePadded<T> {
CachePadded {
data: UnsafeCell::new(([0; CACHE_LINE])),
_marker: ([], marker::PhantomData),
}
}
/// A const fn equivalent to mem::zeroed().
#[cfg(feature = "nightly")]
pub const fn zeroed() -> CachePadded<T> {
CachePadded {
data: UnsafeCell::new(([0; CACHE_LINE])),
_marker: ([], marker::PhantomData),
}
}
}
#[inline]
/// Assert that the size and alignment of `T` are consistent with `CachePadded<T>`.
fn assert_valid<T>() {
assert!(mem::size_of::<T>() <= mem::size_of::<CachePadded<T>>());
assert!(mem::align_of::<T>() <= mem::align_of::<CachePadded<T>>());
}
impl<T> CachePadded<T> {
/// Wrap `t` with cacheline padding.
///
/// **Warning**: the wrapped data is never dropped; move out using
/// `ptr:read` if you need to run dtors.
pub fn new(t: T) -> CachePadded<T> {
assert_valid::<T>();
let ret = CachePadded {
data: UnsafeCell::new(([0; CACHE_LINE])),
_marker: ([], marker::PhantomData),
};
unsafe {
let p: *mut T = mem::transmute(&ret.data);
ptr::write(p, t);
}
ret
}
}
impl<T> Deref for CachePadded<T> {
type Target = T;
fn deref(&self) -> &T {
assert_valid::<T>();
unsafe { mem::transmute(&self.data) }
}
}
impl<T> DerefMut for CachePadded<T> {
fn deref_mut(&mut self) -> &mut T {
assert_valid::<T>();
unsafe { mem::transmute(&mut self.data) }
}
}
// FIXME: support Drop by pulling out a version usable for statics
/*
impl<T> Drop for CachePadded<T> {
fn drop(&mut self) {
assert_valid::<T>();
let p: *mut T = mem::transmute(&self.data);
mem::drop(ptr::read(p));
}
}
*/
#[cfg(test)]
mod test {
use super::*;
#[test]
fn cache_padded_store_u64() {
let x: CachePadded<u64> = CachePadded::new(17);
assert_eq!(*x, 17);
}
#[test]
fn cache_padded_store_pair() {
let x: CachePadded<(u64, u64)> = CachePadded::new((17, 37));
assert_eq!(x.0, 17);
assert_eq!(x.1, 37);
}
}

View File

@ -1,181 +0,0 @@
use std::marker::PhantomData;
use std::mem;
use std::ptr;
use std::sync::atomic::{self, Ordering};
use super::{Owned, Shared, Guard};
/// Like `std::sync::atomic::AtomicPtr`.
///
/// Provides atomic access to a (nullable) pointer of type `T`, interfacing with
/// the `Owned` and `Shared` types.
#[derive(Debug)]
pub struct Atomic<T> {
ptr: atomic::AtomicPtr<T>,
_marker: PhantomData<*const ()>,
}
unsafe impl<T: Sync> Send for Atomic<T> {}
unsafe impl<T: Sync> Sync for Atomic<T> {}
fn opt_shared_into_raw<T>(val: Option<Shared<T>>) -> *mut T {
val.map(|p| p.as_raw()).unwrap_or(ptr::null_mut())
}
fn opt_owned_as_raw<T>(val: &Option<Owned<T>>) -> *mut T {
val.as_ref().map(Owned::as_raw).unwrap_or(ptr::null_mut())
}
fn opt_owned_into_raw<T>(val: Option<Owned<T>>) -> *mut T {
let ptr = val.as_ref().map(Owned::as_raw).unwrap_or(ptr::null_mut());
mem::forget(val);
ptr
}
impl<T> Atomic<T> {
/// Create a new, null atomic pointer.
#[cfg(feature = "nightly")]
pub const fn null() -> Atomic<T> {
Atomic {
ptr: atomic::AtomicPtr::new(0 as *mut _),
_marker: PhantomData
}
}
#[cfg(not(feature = "nightly"))]
pub fn null() -> Atomic<T> {
Atomic {
ptr: atomic::AtomicPtr::new(0 as *mut _),
_marker: PhantomData
}
}
/// Create a new atomic pointer
pub fn new(data: T) -> Atomic<T> {
Atomic {
ptr: atomic::AtomicPtr::new(Box::into_raw(Box::new(data))),
_marker: PhantomData
}
}
/// Do an atomic load with the given memory ordering.
///
/// In order to perform the load, we must pass in a borrow of a
/// `Guard`. This is a way of guaranteeing that the thread has pinned the
/// epoch for the entire lifetime `'a`. In return, you get an optional
/// `Shared` pointer back (`None` if the `Atomic` is currently null), with
/// lifetime tied to the guard.
///
/// # Panics
///
/// Panics if `ord` is `Release` or `AcqRel`.
pub fn load<'a>(&self, ord: Ordering, _: &'a Guard) -> Option<Shared<'a, T>> {
unsafe { Shared::from_raw(self.ptr.load(ord)) }
}
/// Do an atomic store with the given memory ordering.
///
/// Transfers ownership of the given `Owned` pointer, if any. Since no
/// lifetime information is acquired, no `Guard` value is needed.
///
/// # Panics
///
/// Panics if `ord` is `Acquire` or `AcqRel`.
pub fn store(&self, val: Option<Owned<T>>, ord: Ordering) {
self.ptr.store(opt_owned_into_raw(val), ord)
}
/// Do an atomic store with the given memory ordering, immediately yielding
/// a shared reference to the pointer that was stored.
///
/// Transfers ownership of the given `Owned` pointer, yielding a `Shared`
/// reference to it. Since the reference is valid only for the curent epoch,
/// it's lifetime is tied to a `Guard` value.
///
/// # Panics
///
/// Panics if `ord` is `Acquire` or `AcqRel`.
pub fn store_and_ref<'a>(&self, val: Owned<T>, ord: Ordering, _: &'a Guard)
-> Shared<'a, T>
{
unsafe {
let shared = Shared::from_owned(val);
self.store_shared(Some(shared), ord);
shared
}
}
/// Do an atomic store of a `Shared` pointer with the given memory ordering.
///
/// This operation does not require a guard, because it does not yield any
/// new information about the lifetime of a pointer.
///
/// # Panics
///
/// Panics if `ord` is `Acquire` or `AcqRel`.
pub fn store_shared(&self, val: Option<Shared<T>>, ord: Ordering) {
self.ptr.store(opt_shared_into_raw(val), ord)
}
/// Do a compare-and-set from a `Shared` to an `Owned` pointer with the
/// given memory ordering.
///
/// As with `store`, this operation does not require a guard; it produces no new
/// lifetime information. The `Result` indicates whether the CAS succeeded; if
/// not, ownership of the `new` pointer is returned to the caller.
pub fn cas(&self, old: Option<Shared<T>>, new: Option<Owned<T>>, ord: Ordering)
-> Result<(), Option<Owned<T>>>
{
if self.ptr.compare_and_swap(opt_shared_into_raw(old),
opt_owned_as_raw(&new),
ord) == opt_shared_into_raw(old)
{
mem::forget(new);
Ok(())
} else {
Err(new)
}
}
/// Do a compare-and-set from a `Shared` to an `Owned` pointer with the
/// given memory ordering, immediatley acquiring a new `Shared` reference to
/// the previously-owned pointer if successful.
///
/// This operation is analogous to `store_and_ref`.
pub fn cas_and_ref<'a>(&self, old: Option<Shared<T>>, new: Owned<T>,
ord: Ordering, _: &'a Guard)
-> Result<Shared<'a, T>, Owned<T>>
{
if self.ptr.compare_and_swap(opt_shared_into_raw(old), new.as_raw(), ord)
== opt_shared_into_raw(old)
{
Ok(unsafe { Shared::from_owned(new) })
} else {
Err(new)
}
}
/// Do a compare-and-set from a `Shared` to another `Shared` pointer with
/// the given memory ordering.
///
/// The boolean return value is `true` when the CAS is successful.
pub fn cas_shared(&self, old: Option<Shared<T>>, new: Option<Shared<T>>, ord: Ordering)
-> bool
{
self.ptr.compare_and_swap(opt_shared_into_raw(old),
opt_shared_into_raw(new),
ord) == opt_shared_into_raw(old)
}
/// Do an atomic swap with an `Owned` pointer with the given memory ordering.
pub fn swap<'a>(&self, new: Option<Owned<T>>, ord: Ordering, _: &'a Guard)
-> Option<Shared<'a, T>> {
unsafe { Shared::from_raw(self.ptr.swap(opt_owned_into_raw(new), ord)) }
}
/// Do an atomic swap with a `Shared` pointer with the given memory ordering.
pub fn swap_shared<'a>(&self, new: Option<Shared<T>>, ord: Ordering, _: &'a Guard)
-> Option<Shared<'a, T>> {
unsafe { Shared::from_raw(self.ptr.swap(opt_shared_into_raw(new), ord)) }
}
}

View File

@ -1,144 +0,0 @@
// Data structures for storing garbage to be freed later (once the
// epochs have sufficiently advanced).
//
// In general, we try to manage the garbage thread locally whenever
// possible. Each thread keep track of three bags of garbage. But if a
// thread is exiting, these bags must be moved into the global garbage
// bags.
use std::ptr;
use std::mem;
use std::sync::atomic::AtomicPtr;
use std::sync::atomic::Ordering::{Relaxed, Release, Acquire};
use mem::ZerosValid;
/// One item of garbage.
///
/// Stores enough information to do a deallocation.
#[derive(Debug)]
struct Item {
ptr: *mut u8,
free: unsafe fn(*mut u8),
}
/// A single, thread-local bag of garbage.
#[derive(Debug)]
pub struct Bag(Vec<Item>);
impl Bag {
fn new() -> Bag {
Bag(vec![])
}
fn insert<T>(&mut self, elem: *mut T) {
let size = mem::size_of::<T>();
if size > 0 {
self.0.push(Item {
ptr: elem as *mut u8,
free: free::<T>,
})
}
unsafe fn free<T>(t: *mut u8) {
drop(Vec::from_raw_parts(t as *mut T, 0, 1));
}
}
fn len(&self) -> usize {
self.0.len()
}
/// Deallocate all garbage in the bag
pub unsafe fn collect(&mut self) {
let mut data = mem::replace(&mut self.0, Vec::new());
for item in data.iter() {
(item.free)(item.ptr);
}
data.truncate(0);
self.0 = data;
}
}
// needed because the bags store raw pointers.
unsafe impl Send for Bag {}
unsafe impl Sync for Bag {}
/// A thread-local set of garbage bags.
#[derive(Debug)]
pub struct Local {
/// Garbage added at least one epoch behind the current local epoch
pub old: Bag,
/// Garbage added in the current local epoch or earlier
pub cur: Bag,
/// Garbage added in the current *global* epoch
pub new: Bag,
}
impl Local {
pub fn new() -> Local {
Local {
old: Bag::new(),
cur: Bag::new(),
new: Bag::new(),
}
}
pub fn insert<T>(&mut self, elem: *mut T) {
self.new.insert(elem)
}
/// Collect one epoch of garbage, rotating the local garbage bags.
pub unsafe fn collect(&mut self) {
let ret = self.old.collect();
mem::swap(&mut self.old, &mut self.cur);
mem::swap(&mut self.cur, &mut self.new);
ret
}
pub fn size(&self) -> usize {
self.old.len() + self.cur.len() + self.new.len()
}
}
/// A concurrent garbage bag, currently based on Treiber's stack.
///
/// The elements are themselves owned `Bag`s.
#[derive(Debug)]
pub struct ConcBag {
head: AtomicPtr<Node>,
}
unsafe impl ZerosValid for ConcBag {}
#[derive(Debug)]
struct Node {
data: Bag,
next: AtomicPtr<Node>,
}
impl ConcBag {
pub fn insert(&self, t: Bag){
let n = Box::into_raw(Box::new(
Node { data: t, next: AtomicPtr::new(ptr::null_mut()) }));
loop {
let head = self.head.load(Acquire);
unsafe { (*n).next.store(head, Relaxed) };
if self.head.compare_and_swap(head, n, Release) == head { break }
}
}
pub unsafe fn collect(&self) {
// check to avoid xchg instruction
// when no garbage exists
let mut head = self.head.load(Relaxed);
if head != ptr::null_mut() {
head = self.head.swap(ptr::null_mut(), Acquire);
while head != ptr::null_mut() {
let mut n = Box::from_raw(head);
n.data.collect();
head = n.next.load(Relaxed);
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More