200217.36 · fdd8510fdda6109c562e479c718d42c8ecc26263
repo: move all source code in crates directory
The top-level listing was just getting a bit too long for my taste. So put all of the code in one directory and shrink the large top-level mess to a small top-level mess.
NOTE: This commit only contains renames. The subsequent commit will actually make ripgrep build again. We do it this way with the naive hope that this will make it easier for git history to track the renames. Sigh.
reading declarations
@@ -0,0 +1,26 @@+[package]+name = "grep-cli"+version = "0.1.3" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+Utilities for search oriented command line applications.+"""+documentation = "https://docs.rs/grep-cli"+homepage = "https://github.com/BurntSushi/ripgrep"+repository = "https://github.com/BurntSushi/ripgrep"+readme = "README.md"+keywords = ["regex", "grep", "cli", "utility", "util"]+license = "Unlicense/MIT"++[dependencies]+atty = "0.2.11"+bstr = "0.2.0"+globset = { version = "0.4.3", path = "../globset" }+lazy_static = "1.1.0"+log = "0.4.5"+regex = "1.1"+same-file = "1.0.4"+termcolor = "1.0.4"++[target.'cfg(windows)'.dependencies.winapi-util]+version = "0.1.1"
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,38 @@+grep-cli+--------+A utility library that provides common routines desired in search oriented+command line applications. This includes, but is not limited to, parsing hex+escapes, detecting whether stdin is readable and more. To the extent possible,+this crate strives for compatibility across Windows, macOS and Linux.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/grep-cli)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).+++### Documentation++[https://docs.rs/grep-cli](https://docs.rs/grep-cli)++**NOTE:** You probably don't want to use this crate directly. Instead, you+should prefer the facade defined in the+[`grep`](https://docs.rs/grep)+crate.+++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+grep-cli = "0.1"+```++and this to your crate root:++```rust+extern crate grep_cli;+```
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
@@ -0,0 +1,376 @@+use std::ffi::{OsStr, OsString};+use std::fs::File;+use std::io;+use std::path::Path;+use std::process::Command;++use globset::{Glob, GlobSet, GlobSetBuilder};++use process::{CommandError, CommandReader, CommandReaderBuilder};++/// A builder for a matcher that determines which files get decompressed.+#[derive(Clone, Debug)]+pub struct DecompressionMatcherBuilder {+ /// The commands for each matching glob.+ commands: Vec<DecompressionCommand>,+ /// Whether to include the default matching rules.+ defaults: bool,+}++/// A representation of a single command for decompressing data+/// out-of-proccess.+#[derive(Clone, Debug)]+struct DecompressionCommand {+ /// The glob that matches this command.+ glob: String,+ /// The command or binary name.+ bin: OsString,+ /// The arguments to invoke with the command.+ args: Vec<OsString>,+}++impl Default for DecompressionMatcherBuilder {+ fn default() -> DecompressionMatcherBuilder {+ DecompressionMatcherBuilder::new()+ }+}++impl DecompressionMatcherBuilder {+ /// Create a new builder for configuring a decompression matcher.+ pub fn new() -> DecompressionMatcherBuilder {+ DecompressionMatcherBuilder { commands: vec![], defaults: true }+ }++ /// Build a matcher for determining how to decompress files.+ ///+ /// If there was a problem compiling the matcher, then an error is+ /// returned.+ pub fn build(&self) -> Result<DecompressionMatcher, CommandError> {+ let defaults = if !self.defaults {+ vec![]+ } else {+ default_decompression_commands()+ };+ let mut glob_builder = GlobSetBuilder::new();+ let mut commands = vec![];+ for decomp_cmd in defaults.iter().chain(&self.commands) {+ let glob = Glob::new(&decomp_cmd.glob).map_err(|err| {+ CommandError::io(io::Error::new(io::ErrorKind::Other, err))+ })?;+ glob_builder.add(glob);+ commands.push(decomp_cmd.clone());+ }+ let globs = glob_builder.build().map_err(|err| {+ CommandError::io(io::Error::new(io::ErrorKind::Other, err))+ })?;+ Ok(DecompressionMatcher { globs, commands })+ }++ /// When enabled, the default matching rules will be compiled into this+ /// matcher before any other associations. When disabled, only the+ /// rules explicitly given to this builder will be used.+ ///+ /// This is enabled by default.+ pub fn defaults(&mut self, yes: bool) -> &mut DecompressionMatcherBuilder {+ self.defaults = yes;+ self+ }++ /// Associates a glob with a command to decompress files matching the glob.+ ///+ /// If multiple globs match the same file, then the most recently added+ /// glob takes precedence.+ ///+ /// The syntax for the glob is documented in the+ /// [`globset` crate](https://docs.rs/globset/#syntax).+ pub fn associate<P, I, A>(+ &mut self,+ glob: &str,+ program: P,+ args: I,+ ) -> &mut DecompressionMatcherBuilder+ where+ P: AsRef<OsStr>,+ I: IntoIterator<Item = A>,+ A: AsRef<OsStr>,+ {+ let glob = glob.to_string();+ let bin = program.as_ref().to_os_string();+ let args =+ args.into_iter().map(|a| a.as_ref().to_os_string()).collect();+ self.commands.push(DecompressionCommand { glob, bin, args });+ self+ }+}++/// A matcher for determining how to decompress files.+#[derive(Clone, Debug)]+pub struct DecompressionMatcher {+ /// The set of globs to match. Each glob has a corresponding entry in+ /// `commands`. When a glob matches, the corresponding command should be+ /// used to perform out-of-process decompression.+ globs: GlobSet,+ /// The commands for each matching glob.+ commands: Vec<DecompressionCommand>,+}++impl Default for DecompressionMatcher {+ fn default() -> DecompressionMatcher {+ DecompressionMatcher::new()+ }+}++impl DecompressionMatcher {+ /// Create a new matcher with default rules.+ ///+ /// To add more matching rules, build a matcher with+ /// [`DecompressionMatcherBuilder`](struct.DecompressionMatcherBuilder.html).+ pub fn new() -> DecompressionMatcher {+ DecompressionMatcherBuilder::new()+ .build()+ .expect("built-in matching rules should always compile")+ }++ /// Return a pre-built command based on the given file path that can+ /// decompress its contents. If no such decompressor is known, then this+ /// returns `None`.+ ///+ /// If there are multiple possible commands matching the given path, then+ /// the command added last takes precedence.+ pub fn command<P: AsRef<Path>>(&self, path: P) -> Option<Command> {+ for i in self.globs.matches(path).into_iter().rev() {+ let decomp_cmd = &self.commands[i];+ let mut cmd = Command::new(&decomp_cmd.bin);+ cmd.args(&decomp_cmd.args);+ return Some(cmd);+ }+ None+ }++ /// Returns true if and only if the given file path has at least one+ /// matching command to perform decompression on.+ pub fn has_command<P: AsRef<Path>>(&self, path: P) -> bool {+ self.globs.is_match(path)+ }+}++/// Configures and builds a streaming reader for decompressing data.+#[derive(Clone, Debug, Default)]+pub struct DecompressionReaderBuilder {+ matcher: DecompressionMatcher,+ command_builder: CommandReaderBuilder,+}++impl DecompressionReaderBuilder {+ /// Create a new builder with the default configuration.+ pub fn new() -> DecompressionReaderBuilder {+ DecompressionReaderBuilder::default()+ }++ /// Build a new streaming reader for decompressing data.+ ///+ /// If decompression is done out-of-process and if there was a problem+ /// spawning the process, then its error is logged at the debug level and a+ /// passthru reader is returned that does no decompression. This behavior+ /// typically occurs when the given file path matches a decompression+ /// command, but is executing in an environment where the decompression+ /// command is not available.+ ///+ /// If the given file path could not be matched with a decompression+ /// strategy, then a passthru reader is returned that does no+ /// decompression.+ pub fn build<P: AsRef<Path>>(+ &self,+ path: P,+ ) -> Result<DecompressionReader, CommandError> {+ let path = path.as_ref();+ let mut cmd = match self.matcher.command(path) {+ None => return DecompressionReader::new_passthru(path),+ Some(cmd) => cmd,+ };+ cmd.arg(path);++ match self.command_builder.build(&mut cmd) {+ Ok(cmd_reader) => Ok(DecompressionReader { rdr: Ok(cmd_reader) }),+ Err(err) => {+ debug!(+ "{}: error spawning command '{:?}': {} \+ (falling back to uncompressed reader)",+ path.display(),+ cmd,+ err,+ );+ DecompressionReader::new_passthru(path)+ }+ }+ }++ /// Set the matcher to use to look up the decompression command for each+ /// file path.+ ///+ /// A set of sensible rules is enabled by default. Setting this will+ /// completely replace the current rules.+ pub fn matcher(+ &mut self,+ matcher: DecompressionMatcher,+ ) -> &mut DecompressionReaderBuilder {+ self.matcher = matcher;+ self+ }++ /// Get the underlying matcher currently used by this builder.+ pub fn get_matcher(&self) -> &DecompressionMatcher {+ &self.matcher+ }++ /// When enabled, the reader will asynchronously read the contents of the+ /// command's stderr output. When disabled, stderr is only read after the+ /// stdout stream has been exhausted (or if the process quits with an error+ /// code).+ ///+ /// Note that when enabled, this may require launching an additional+ /// thread in order to read stderr. This is done so that the process being+ /// executed is never blocked from writing to stdout or stderr. If this is+ /// disabled, then it is possible for the process to fill up the stderr+ /// buffer and deadlock.+ ///+ /// This is enabled by default.+ pub fn async_stderr(+ &mut self,+ yes: bool,+ ) -> &mut DecompressionReaderBuilder {+ self.command_builder.async_stderr(yes);+ self+ }+}++/// A streaming reader for decompressing the contents of a file.+///+/// The purpose of this reader is to provide a seamless way to decompress the+/// contents of file using existing tools in the current environment. This is+/// meant to be an alternative to using decompression libraries in favor of the+/// simplicity and portability of using external commands such as `gzip` and+/// `xz`. This does impose the overhead of spawning a process, so other means+/// for performing decompression should be sought if this overhead isn't+/// acceptable.+///+/// A decompression reader comes with a default set of matching rules that are+/// meant to associate file paths with the corresponding command to use to+/// decompress them. For example, a glob like `*.gz` matches gzip compressed+/// files with the command `gzip -d -c`. If a file path does not match any+/// existing rules, or if it matches a rule whose command does not exist in the+/// current environment, then the decompression reader passes through the+/// contents of the underlying file without doing any decompression.+///+/// The default matching rules are probably good enough for most cases, and if+/// they require revision, pull requests are welcome. In cases where they must+/// be changed or extended, they can be customized through the use of+/// [`DecompressionMatcherBuilder`](struct.DecompressionMatcherBuilder.html)+/// and+/// [`DecompressionReaderBuilder`](struct.DecompressionReaderBuilder.html).+///+/// By default, this reader will asynchronously read the processes' stderr.+/// This prevents subtle deadlocking bugs for noisy processes that write a lot+/// to stderr. Currently, the entire contents of stderr is read on to the heap.+///+/// # Example+///+/// This example shows how to read the decompressed contents of a file without+/// needing to explicitly choose the decompression command to run.+///+/// Note that if you need to decompress multiple files, it is better to use+/// `DecompressionReaderBuilder`, which will amortize the cost of compiling the+/// matcher.+///+/// ```no_run+/// use std::io::Read;+/// use std::process::Command;+/// use grep_cli::DecompressionReader;+///+/// # fn example() -> Result<(), Box<::std::error::Error>> {+/// let mut rdr = DecompressionReader::new("/usr/share/man/man1/ls.1.gz")?;+/// let mut contents = vec![];+/// rdr.read_to_end(&mut contents)?;+/// # Ok(()) }+/// ```+#[derive(Debug)]+pub struct DecompressionReader {+ rdr: Result<CommandReader, File>,+}++impl DecompressionReader {+ /// Build a new streaming reader for decompressing data.+ ///+ /// If decompression is done out-of-process and if there was a problem+ /// spawning the process, then its error is returned.+ ///+ /// If the given file path could not be matched with a decompression+ /// strategy, then a passthru reader is returned that does no+ /// decompression.+ ///+ /// This uses the default matching rules for determining how to decompress+ /// the given file. To change those matching rules, use+ /// [`DecompressionReaderBuilder`](struct.DecompressionReaderBuilder.html)+ /// and+ /// [`DecompressionMatcherBuilder`](struct.DecompressionMatcherBuilder.html).+ ///+ /// When creating readers for many paths. it is better to use the builder+ /// since it will amortize the cost of constructing the matcher.+ pub fn new<P: AsRef<Path>>(+ path: P,+ ) -> Result<DecompressionReader, CommandError> {+ DecompressionReaderBuilder::new().build(path)+ }++ /// Creates a new "passthru" decompression reader that reads from the file+ /// corresponding to the given path without doing decompression and without+ /// executing another process.+ fn new_passthru(path: &Path) -> Result<DecompressionReader, CommandError> {+ let file = File::open(path)?;+ Ok(DecompressionReader { rdr: Err(file) })+ }+}++impl io::Read for DecompressionReader {+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {+ match self.rdr {+ Ok(ref mut rdr) => rdr.read(buf),+ Err(ref mut rdr) => rdr.read(buf),+ }+ }+}++fn default_decompression_commands() -> Vec<DecompressionCommand> {+ const ARGS_GZIP: &[&str] = &["gzip", "-d", "-c"];+ const ARGS_BZIP: &[&str] = &["bzip2", "-d", "-c"];+ const ARGS_XZ: &[&str] = &["xz", "-d", "-c"];+ const ARGS_LZ4: &[&str] = &["lz4", "-d", "-c"];+ const ARGS_LZMA: &[&str] = &["xz", "--format=lzma", "-d", "-c"];+ const ARGS_BROTLI: &[&str] = &["brotli", "-d", "-c"];+ const ARGS_ZSTD: &[&str] = &["zstd", "-q", "-d", "-c"];++ fn cmd(glob: &str, args: &[&str]) -> DecompressionCommand {+ DecompressionCommand {+ glob: glob.to_string(),+ bin: OsStr::new(&args[0]).to_os_string(),+ args: args+ .iter()+ .skip(1)+ .map(|s| OsStr::new(s).to_os_string())+ .collect(),+ }+ }+ vec![+ cmd("*.gz", ARGS_GZIP),+ cmd("*.tgz", ARGS_GZIP),+ cmd("*.bz2", ARGS_BZIP),+ cmd("*.tbz2", ARGS_BZIP),+ cmd("*.xz", ARGS_XZ),+ cmd("*.txz", ARGS_XZ),+ cmd("*.lz4", ARGS_LZ4),+ cmd("*.lzma", ARGS_LZMA),+ cmd("*.br", ARGS_BROTLI),+ cmd("*.zst", ARGS_ZSTD),+ cmd("*.zstd", ARGS_ZSTD),+ ]+}
@@ -0,0 +1,272 @@+use std::ffi::OsStr;+use std::str;++use bstr::{ByteSlice, ByteVec};++/// A single state in the state machine used by `unescape`.+#[derive(Clone, Copy, Eq, PartialEq)]+enum State {+ /// The state after seeing a `\`.+ Escape,+ /// The state after seeing a `\x`.+ HexFirst,+ /// The state after seeing a `\x[0-9A-Fa-f]`.+ HexSecond(char),+ /// Default state.+ Literal,+}++/// Escapes arbitrary bytes into a human readable string.+///+/// This converts `\t`, `\r` and `\n` into their escaped forms. It also+/// converts the non-printable subset of ASCII in addition to invalid UTF-8+/// bytes to hexadecimal escape sequences. Everything else is left as is.+///+/// The dual of this routine is [`unescape`](fn.unescape.html).+///+/// # Example+///+/// This example shows how to convert a byte string that contains a `\n` and+/// invalid UTF-8 bytes into a `String`.+///+/// Pay special attention to the use of raw strings. That is, `r"\n"` is+/// equivalent to `"\\n"`.+///+/// ```+/// use grep_cli::escape;+///+/// assert_eq!(r"foo\nbar\xFFbaz", escape(b"foo\nbar\xFFbaz"));+/// ```+pub fn escape(bytes: &[u8]) -> String {+ let mut escaped = String::new();+ for (s, e, ch) in bytes.char_indices() {+ if ch == '\u{FFFD}' {+ for b in bytes[s..e].bytes() {+ escape_byte(b, &mut escaped);+ }+ } else {+ escape_char(ch, &mut escaped);+ }+ }+ escaped+}++/// Escapes an OS string into a human readable string.+///+/// This is like [`escape`](fn.escape.html), but accepts an OS string.+pub fn escape_os(string: &OsStr) -> String {+ escape(Vec::from_os_str_lossy(string).as_bytes())+}++/// Unescapes a string.+///+/// It supports a limited set of escape sequences:+///+/// * `\t`, `\r` and `\n` are mapped to their corresponding ASCII bytes.+/// * `\xZZ` hexadecimal escapes are mapped to their byte.+///+/// Everything else is left as is, including non-hexadecimal escapes like+/// `\xGG`.+///+/// This is useful when it is desirable for a command line argument to be+/// capable of specifying arbitrary bytes or otherwise make it easier to+/// specify non-printable characters.+///+/// The dual of this routine is [`escape`](fn.escape.html).+///+/// # Example+///+/// This example shows how to convert an escaped string (which is valid UTF-8)+/// into a corresponding sequence of bytes. Each escape sequence is mapped to+/// its bytes, which may include invalid UTF-8.+///+/// Pay special attention to the use of raw strings. That is, `r"\n"` is+/// equivalent to `"\\n"`.+///+/// ```+/// use grep_cli::unescape;+///+/// assert_eq!(&b"foo\nbar\xFFbaz"[..], &*unescape(r"foo\nbar\xFFbaz"));+/// ```+pub fn unescape(s: &str) -> Vec<u8> {+ use self::State::*;++ let mut bytes = vec![];+ let mut state = Literal;+ for c in s.chars() {+ match state {+ Escape => match c {+ '\\' => {+ bytes.push(b'\\');+ state = Literal;+ }+ 'n' => {+ bytes.push(b'\n');+ state = Literal;+ }+ 'r' => {+ bytes.push(b'\r');+ state = Literal;+ }+ 't' => {+ bytes.push(b'\t');+ state = Literal;+ }+ 'x' => {+ state = HexFirst;+ }+ c => {+ bytes.extend(format!(r"\{}", c).into_bytes());+ state = Literal;+ }+ },+ HexFirst => match c {+ '0'..='9' | 'A'..='F' | 'a'..='f' => {+ state = HexSecond(c);+ }+ c => {+ bytes.extend(format!(r"\x{}", c).into_bytes());+ state = Literal;+ }+ },+ HexSecond(first) => match c {+ '0'..='9' | 'A'..='F' | 'a'..='f' => {+ let ordinal = format!("{}{}", first, c);+ let byte = u8::from_str_radix(&ordinal, 16).unwrap();+ bytes.push(byte);+ state = Literal;+ }+ c => {+ let original = format!(r"\x{}{}", first, c);+ bytes.extend(original.into_bytes());+ state = Literal;+ }+ },+ Literal => match c {+ '\\' => {+ state = Escape;+ }+ c => {+ bytes.extend(c.to_string().as_bytes());+ }+ },+ }+ }+ match state {+ Escape => bytes.push(b'\\'),+ HexFirst => bytes.extend(b"\\x"),+ HexSecond(c) => bytes.extend(format!("\\x{}", c).into_bytes()),+ Literal => {}+ }+ bytes+}++/// Unescapes an OS string.+///+/// This is like [`unescape`](fn.unescape.html), but accepts an OS string.+///+/// Note that this first lossily decodes the given OS string as UTF-8. That+/// is, an escaped string (the thing given) should be valid UTF-8.+pub fn unescape_os(string: &OsStr) -> Vec<u8> {+ unescape(&string.to_string_lossy())+}++/// Adds the given codepoint to the given string, escaping it if necessary.+fn escape_char(cp: char, into: &mut String) {+ if cp.is_ascii() {+ escape_byte(cp as u8, into);+ } else {+ into.push(cp);+ }+}++/// Adds the given byte to the given string, escaping it if necessary.+fn escape_byte(byte: u8, into: &mut String) {+ match byte {+ 0x21..=0x5B | 0x5D..=0x7D => into.push(byte as char),+ b'\n' => into.push_str(r"\n"),+ b'\r' => into.push_str(r"\r"),+ b'\t' => into.push_str(r"\t"),+ b'\\' => into.push_str(r"\\"),+ _ => into.push_str(&format!(r"\x{:02X}", byte)),+ }+}++#[cfg(test)]+mod tests {+ use super::{escape, unescape};++ fn b(bytes: &'static [u8]) -> Vec<u8> {+ bytes.to_vec()+ }++ #[test]+ fn empty() {+ assert_eq!(b(b""), unescape(r""));+ assert_eq!(r"", escape(b""));+ }++ #[test]+ fn backslash() {+ assert_eq!(b(b"\\"), unescape(r"\\"));+ assert_eq!(r"\\", escape(b"\\"));+ }++ #[test]+ fn nul() {+ assert_eq!(b(b"\x00"), unescape(r"\x00"));+ assert_eq!(r"\x00", escape(b"\x00"));+ }++ #[test]+ fn nl() {+ assert_eq!(b(b"\n"), unescape(r"\n"));+ assert_eq!(r"\n", escape(b"\n"));+ }++ #[test]+ fn tab() {+ assert_eq!(b(b"\t"), unescape(r"\t"));+ assert_eq!(r"\t", escape(b"\t"));+ }++ #[test]+ fn carriage() {+ assert_eq!(b(b"\r"), unescape(r"\r"));+ assert_eq!(r"\r", escape(b"\r"));+ }++ #[test]+ fn nothing_simple() {+ assert_eq!(b(b"\\a"), unescape(r"\a"));+ assert_eq!(b(b"\\a"), unescape(r"\\a"));+ assert_eq!(r"\\a", escape(b"\\a"));+ }++ #[test]+ fn nothing_hex0() {+ assert_eq!(b(b"\\x"), unescape(r"\x"));+ assert_eq!(b(b"\\x"), unescape(r"\\x"));+ assert_eq!(r"\\x", escape(b"\\x"));+ }++ #[test]+ fn nothing_hex1() {+ assert_eq!(b(b"\\xz"), unescape(r"\xz"));+ assert_eq!(b(b"\\xz"), unescape(r"\\xz"));+ assert_eq!(r"\\xz", escape(b"\\xz"));+ }++ #[test]+ fn nothing_hex2() {+ assert_eq!(b(b"\\xzz"), unescape(r"\xzz"));+ assert_eq!(b(b"\\xzz"), unescape(r"\\xzz"));+ assert_eq!(r"\\xzz", escape(b"\\xzz"));+ }++ #[test]+ fn invalid_utf8() {+ assert_eq!(r"\xFF", escape(b"\xFF"));+ assert_eq!(r"a\xFFb", escape(b"a\xFFb"));+ }+}
@@ -0,0 +1,165 @@+use std::error;+use std::fmt;+use std::io;+use std::num::ParseIntError;++use regex::Regex;++/// An error that occurs when parsing a human readable size description.+///+/// This error provides a end user friendly message describing why the+/// description coudln't be parsed and what the expected format is.+#[derive(Clone, Debug, Eq, PartialEq)]+pub struct ParseSizeError {+ original: String,+ kind: ParseSizeErrorKind,+}++#[derive(Clone, Debug, Eq, PartialEq)]+enum ParseSizeErrorKind {+ InvalidFormat,+ InvalidInt(ParseIntError),+ Overflow,+}++impl ParseSizeError {+ fn format(original: &str) -> ParseSizeError {+ ParseSizeError {+ original: original.to_string(),+ kind: ParseSizeErrorKind::InvalidFormat,+ }+ }++ fn int(original: &str, err: ParseIntError) -> ParseSizeError {+ ParseSizeError {+ original: original.to_string(),+ kind: ParseSizeErrorKind::InvalidInt(err),+ }+ }++ fn overflow(original: &str) -> ParseSizeError {+ ParseSizeError {+ original: original.to_string(),+ kind: ParseSizeErrorKind::Overflow,+ }+ }+}++impl error::Error for ParseSizeError {+ fn description(&self) -> &str {+ "invalid size"+ }+}++impl fmt::Display for ParseSizeError {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ use self::ParseSizeErrorKind::*;++ match self.kind {+ InvalidFormat => write!(+ f,+ "invalid format for size '{}', which should be a sequence \+ of digits followed by an optional 'K', 'M' or 'G' \+ suffix",+ self.original+ ),+ InvalidInt(ref err) => write!(+ f,+ "invalid integer found in size '{}': {}",+ self.original, err+ ),+ Overflow => write!(f, "size too big in '{}'", self.original),+ }+ }+}++impl From<ParseSizeError> for io::Error {+ fn from(size_err: ParseSizeError) -> io::Error {+ io::Error::new(io::ErrorKind::Other, size_err)+ }+}++/// Parse a human readable size like `2M` into a corresponding number of bytes.+///+/// Supported size suffixes are `K` (for kilobyte), `M` (for megabyte) and `G`+/// (for gigabyte). If a size suffix is missing, then the size is interpreted+/// as bytes. If the size is too big to fit into a `u64`, then this returns an+/// error.+///+/// Additional suffixes may be added over time.+pub fn parse_human_readable_size(size: &str) -> Result<u64, ParseSizeError> {+ lazy_static! {+ // Normally I'd just parse something this simple by hand to avoid the+ // regex dep, but we bring regex in any way for glob matching, so might+ // as well use it.+ static ref RE: Regex = Regex::new(r"^([0-9]+)([KMG])?$").unwrap();+ }++ let caps = match RE.captures(size) {+ Some(caps) => caps,+ None => return Err(ParseSizeError::format(size)),+ };+ let value: u64 =+ caps[1].parse().map_err(|err| ParseSizeError::int(size, err))?;+ let suffix = match caps.get(2) {+ None => return Ok(value),+ Some(cap) => cap.as_str(),+ };+ let bytes = match suffix {+ "K" => value.checked_mul(1 << 10),+ "M" => value.checked_mul(1 << 20),+ "G" => value.checked_mul(1 << 30),+ // Because if the regex matches this group, it must be [KMG].+ _ => unreachable!(),+ };+ bytes.ok_or_else(|| ParseSizeError::overflow(size))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn suffix_none() {+ let x = parse_human_readable_size("123").unwrap();+ assert_eq!(123, x);+ }++ #[test]+ fn suffix_k() {+ let x = parse_human_readable_size("123K").unwrap();+ assert_eq!(123 * (1 << 10), x);+ }++ #[test]+ fn suffix_m() {+ let x = parse_human_readable_size("123M").unwrap();+ assert_eq!(123 * (1 << 20), x);+ }++ #[test]+ fn suffix_g() {+ let x = parse_human_readable_size("123G").unwrap();+ assert_eq!(123 * (1 << 30), x);+ }++ #[test]+ fn invalid_empty() {+ assert!(parse_human_readable_size("").is_err());+ }++ #[test]+ fn invalid_non_digit() {+ assert!(parse_human_readable_size("a").is_err());+ }++ #[test]+ fn invalid_overflow() {+ assert!(parse_human_readable_size("9999999999999999G").is_err());+ }++ #[test]+ fn invalid_suffix() {+ assert!(parse_human_readable_size("123T").is_err());+ }+}
@@ -0,0 +1,250 @@+/*!+This crate provides common routines used in command line applications, with a+focus on routines useful for search oriented applications. As a utility+library, there is no central type or function. However, a key focus of this+crate is to improve failure modes and provide user friendly error messages+when things go wrong.++To the best extent possible, everything in this crate works on Windows, macOS+and Linux.+++# Standard I/O++The+[`is_readable_stdin`](fn.is_readable_stdin.html),+[`is_tty_stderr`](fn.is_tty_stderr.html),+[`is_tty_stdin`](fn.is_tty_stdin.html)+and+[`is_tty_stdout`](fn.is_tty_stdout.html)+routines query aspects of standard I/O. `is_readable_stdin` determines whether+stdin can be usefully read from, while the `tty` methods determine whether a+tty is attached to stdin/stdout/stderr.++`is_readable_stdin` is useful when writing an application that changes behavior+based on whether the application was invoked with data on stdin. For example,+`rg foo` might recursively search the current working directory for+occurrences of `foo`, but `rg foo < file` might only search the contents of+`file`.++The `tty` methods are useful for similar reasons. Namely, commands like `ls`+will change their output depending on whether they are printing to a terminal+or not. For example, `ls` shows a file on each line when stdout is redirected+to a file or a pipe, but condenses the output to show possibly many files on+each line when stdout is connected to a tty.+++# Coloring and buffering++The+[`stdout`](fn.stdout.html),+[`stdout_buffered_block`](fn.stdout_buffered_block.html)+and+[`stdout_buffered_line`](fn.stdout_buffered_line.html)+routines are alternative constructors for+[`StandardStream`](struct.StandardStream.html).+A `StandardStream` implements `termcolor::WriteColor`, which provides a way+to emit colors to terminals. Its key use is the encapsulation of buffering+style. Namely, `stdout` will return a line buffered `StandardStream` if and+only if stdout is connected to a tty, and will otherwise return a block+buffered `StandardStream`. Line buffering is important for use with a tty+because it typically decreases the latency at which the end user sees output.+Block buffering is used otherwise because it is faster, and redirecting stdout+to a file typically doesn't benefit from the decreased latency that line+buffering provides.++The `stdout_buffered_block` and `stdout_buffered_line` can be used to+explicitly set the buffering strategy regardless of whether stdout is connected+to a tty or not.+++# Escaping++The+[`escape`](fn.escape.html),+[`escape_os`](fn.escape_os.html),+[`unescape`](fn.unescape.html)+and+[`unescape_os`](fn.unescape_os.html)+routines provide a user friendly way of dealing with UTF-8 encoded strings that+can express arbitrary bytes. For example, you might want to accept a string+containing arbitrary bytes as a command line argument, but most interactive+shells make such strings difficult to type. Instead, we can ask users to use+escape sequences.++For example, `a\xFFz` is itself a valid UTF-8 string corresponding to the+following bytes:++```ignore+[b'a', b'\\', b'x', b'F', b'F', b'z']+```++However, we can+interpret `\xFF` as an escape sequence with the `unescape`/`unescape_os`+routines, which will yield++```ignore+[b'a', b'\xFF', b'z']+```++instead. For example:++```+use grep_cli::unescape;++// Note the use of a raw string!+assert_eq!(vec![b'a', b'\xFF', b'z'], unescape(r"a\xFFz"));+```++The `escape`/`escape_os` routines provide the reverse transformation, which+makes it easy to show user friendly error messages involving arbitrary bytes.+++# Building patterns++Typically, regular expression patterns must be valid UTF-8. However, command+line arguments aren't guaranteed to be valid UTF-8. Unfortunately, the+standard library's UTF-8 conversion functions from `OsStr`s do not provide+good error messages. However, the+[`pattern_from_bytes`](fn.pattern_from_bytes.html)+and+[`pattern_from_os`](fn.pattern_from_os.html)+do, including reporting exactly where the first invalid UTF-8 byte is seen.++Additionally, it can be useful to read patterns from a file while reporting+good error messages that include line numbers. The+[`patterns_from_path`](fn.patterns_from_path.html),+[`patterns_from_reader`](fn.patterns_from_reader.html)+and+[`patterns_from_stdin`](fn.patterns_from_stdin.html)+routines do just that. If any pattern is found that is invalid UTF-8, then the+error includes the file path (if available) along with the line number and the+byte offset at which the first invalid UTF-8 byte was observed.+++# Read process output++Sometimes a command line application needs to execute other processes and read+its stdout in a streaming fashion. The+[`CommandReader`](struct.CommandReader.html)+provides this functionality with an explicit goal of improving failure modes.+In particular, if the process exits with an error code, then stderr is read+and converted into a normal Rust error to show to end users. This makes the+underlying failure modes explicit and gives more information to end users for+debugging the problem.++As a special case,+[`DecompressionReader`](struct.DecompressionReader.html)+provides a way to decompress arbitrary files by matching their file extensions+up with corresponding decompression programs (such as `gzip` and `xz`). This+is useful as a means of performing simplistic decompression in a portable+manner without binding to specific compression libraries. This does come with+some overhead though, so if you need to decompress lots of small files, this+may not be an appropriate convenience to use.++Each reader has a corresponding builder for additional configuration, such as+whether to read stderr asynchronously in order to avoid deadlock (which is+enabled by default).+++# Miscellaneous parsing++The+[`parse_human_readable_size`](fn.parse_human_readable_size.html)+routine parses strings like `2M` and converts them to the corresponding number+of bytes (`2 * 1<<20` in this case). If an invalid size is found, then a good+error message is crafted that typically tells the user how to fix the problem.+*/++#![deny(missing_docs)]++extern crate atty;+extern crate bstr;+extern crate globset;+#[macro_use]+extern crate lazy_static;+#[macro_use]+extern crate log;+extern crate regex;+extern crate same_file;+extern crate termcolor;+#[cfg(windows)]+extern crate winapi_util;++mod decompress;+mod escape;+mod human;+mod pattern;+mod process;+mod wtr;++pub use decompress::{+ DecompressionMatcher, DecompressionMatcherBuilder, DecompressionReader,+ DecompressionReaderBuilder,+};+pub use escape::{escape, escape_os, unescape, unescape_os};+pub use human::{parse_human_readable_size, ParseSizeError};+pub use pattern::{+ pattern_from_bytes, pattern_from_os, patterns_from_path,+ patterns_from_reader, patterns_from_stdin, InvalidPatternError,+};+pub use process::{CommandError, CommandReader, CommandReaderBuilder};+pub use wtr::{+ stdout, stdout_buffered_block, stdout_buffered_line, StandardStream,+};++/// Returns true if and only if stdin is believed to be readable.+///+/// When stdin is readable, command line programs may choose to behave+/// differently than when stdin is not readable. For example, `command foo`+/// might search the current directory for occurrences of `foo` where as+/// `command foo < some-file` or `cat some-file | command foo` might instead+/// only search stdin for occurrences of `foo`.+pub fn is_readable_stdin() -> bool {+ #[cfg(unix)]+ fn imp() -> bool {+ use same_file::Handle;+ use std::os::unix::fs::FileTypeExt;++ let ft = match Handle::stdin().and_then(|h| h.as_file().metadata()) {+ Err(_) => return false,+ Ok(md) => md.file_type(),+ };+ ft.is_file() || ft.is_fifo()+ }++ #[cfg(windows)]+ fn imp() -> bool {+ use winapi_util as winutil;++ winutil::file::typ(winutil::HandleRef::stdin())+ .map(|t| t.is_disk() || t.is_pipe())+ .unwrap_or(false)+ }++ !is_tty_stdin() && imp()+}++/// Returns true if and only if stdin is believed to be connectted to a tty+/// or a console.+pub fn is_tty_stdin() -> bool {+ atty::is(atty::Stream::Stdin)+}++/// Returns true if and only if stdout is believed to be connectted to a tty+/// or a console.+///+/// This is useful for when you want your command line program to produce+/// different output depending on whether it's printing directly to a user's+/// terminal or whether it's being redirected somewhere else. For example,+/// implementations of `ls` will often show one item per line when stdout is+/// redirected, but will condensed output when printing to a tty.+pub fn is_tty_stdout() -> bool {+ atty::is(atty::Stream::Stdout)+}++/// Returns true if and only if stderr is believed to be connectted to a tty+/// or a console.+pub fn is_tty_stderr() -> bool {+ atty::is(atty::Stream::Stderr)+}
@@ -0,0 +1,195 @@+use std::error;+use std::ffi::OsStr;+use std::fmt;+use std::fs::File;+use std::io;+use std::path::Path;+use std::str;++use bstr::io::BufReadExt;++use escape::{escape, escape_os};++/// An error that occurs when a pattern could not be converted to valid UTF-8.+///+/// The purpose of this error is to give a more targeted failure mode for+/// patterns written by end users that are not valid UTF-8.+#[derive(Clone, Debug, Eq, PartialEq)]+pub struct InvalidPatternError {+ original: String,+ valid_up_to: usize,+}++impl InvalidPatternError {+ /// Returns the index in the given string up to which valid UTF-8 was+ /// verified.+ pub fn valid_up_to(&self) -> usize {+ self.valid_up_to+ }+}++impl error::Error for InvalidPatternError {+ fn description(&self) -> &str {+ "invalid pattern"+ }+}++impl fmt::Display for InvalidPatternError {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ write!(+ f,+ "found invalid UTF-8 in pattern at byte offset {} \+ (use hex escape sequences to match arbitrary bytes \+ in a pattern, e.g., \\xFF): '{}'",+ self.valid_up_to, self.original,+ )+ }+}++impl From<InvalidPatternError> for io::Error {+ fn from(paterr: InvalidPatternError) -> io::Error {+ io::Error::new(io::ErrorKind::Other, paterr)+ }+}++/// Convert an OS string into a regular expression pattern.+///+/// This conversion fails if the given pattern is not valid UTF-8, in which+/// case, a targeted error with more information about where the invalid UTF-8+/// occurs is given. The error also suggests the use of hex escape sequences,+/// which are supported by many regex engines.+pub fn pattern_from_os(pattern: &OsStr) -> Result<&str, InvalidPatternError> {+ pattern.to_str().ok_or_else(|| {+ let valid_up_to = pattern+ .to_string_lossy()+ .find('\u{FFFD}')+ .expect("a Unicode replacement codepoint for invalid UTF-8");+ InvalidPatternError {+ original: escape_os(pattern),+ valid_up_to: valid_up_to,+ }+ })+}++/// Convert arbitrary bytes into a regular expression pattern.+///+/// This conversion fails if the given pattern is not valid UTF-8, in which+/// case, a targeted error with more information about where the invalid UTF-8+/// occurs is given. The error also suggests the use of hex escape sequences,+/// which are supported by many regex engines.+pub fn pattern_from_bytes(+ pattern: &[u8],+) -> Result<&str, InvalidPatternError> {+ str::from_utf8(pattern).map_err(|err| InvalidPatternError {+ original: escape(pattern),+ valid_up_to: err.valid_up_to(),+ })+}++/// Read patterns from a file path, one per line.+///+/// If there was a problem reading or if any of the patterns contain invalid+/// UTF-8, then an error is returned. If there was a problem with a specific+/// pattern, then the error message will include the line number and the file+/// path.+pub fn patterns_from_path<P: AsRef<Path>>(path: P) -> io::Result<Vec<String>> {+ let path = path.as_ref();+ let file = File::open(path).map_err(|err| {+ io::Error::new(+ io::ErrorKind::Other,+ format!("{}: {}", path.display(), err),+ )+ })?;+ patterns_from_reader(file).map_err(|err| {+ io::Error::new(+ io::ErrorKind::Other,+ format!("{}:{}", path.display(), err),+ )+ })+}++/// Read patterns from stdin, one per line.+///+/// If there was a problem reading or if any of the patterns contain invalid+/// UTF-8, then an error is returned. If there was a problem with a specific+/// pattern, then the error message will include the line number and the fact+/// that it came from stdin.+pub fn patterns_from_stdin() -> io::Result<Vec<String>> {+ let stdin = io::stdin();+ let locked = stdin.lock();+ patterns_from_reader(locked).map_err(|err| {+ io::Error::new(io::ErrorKind::Other, format!("<stdin>:{}", err))+ })+}++/// Read patterns from any reader, one per line.+///+/// If there was a problem reading or if any of the patterns contain invalid+/// UTF-8, then an error is returned. If there was a problem with a specific+/// pattern, then the error message will include the line number.+///+/// Note that this routine uses its own internal buffer, so the caller should+/// not provide their own buffered reader if possible.+///+/// # Example+///+/// This shows how to parse patterns, one per line.+///+/// ```+/// use grep_cli::patterns_from_reader;+///+/// # fn example() -> Result<(), Box<::std::error::Error>> {+/// let patterns = "\+/// foo+/// bar\\s+foo+/// [a-z]{3}+/// ";+///+/// assert_eq!(patterns_from_reader(patterns.as_bytes())?, vec![+/// r"foo",+/// r"bar\s+foo",+/// r"[a-z]{3}",+/// ]);+/// # Ok(()) }+/// ```+pub fn patterns_from_reader<R: io::Read>(rdr: R) -> io::Result<Vec<String>> {+ let mut patterns = vec![];+ let mut line_number = 0;+ io::BufReader::new(rdr).for_byte_line(|line| {+ line_number += 1;+ match pattern_from_bytes(line) {+ Ok(pattern) => {+ patterns.push(pattern.to_string());+ Ok(true)+ }+ Err(err) => Err(io::Error::new(+ io::ErrorKind::Other,+ format!("{}: {}", line_number, err),+ )),+ }+ })?;+ Ok(patterns)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn bytes() {+ let pat = b"abc\xFFxyz";+ let err = pattern_from_bytes(pat).unwrap_err();+ assert_eq!(3, err.valid_up_to());+ }++ #[test]+ #[cfg(unix)]+ fn os() {+ use std::ffi::OsStr;+ use std::os::unix::ffi::OsStrExt;++ let pat = OsStr::from_bytes(b"abc\xFFxyz");+ let err = pattern_from_os(pat).unwrap_err();+ assert_eq!(3, err.valid_up_to());+ }+}
@@ -0,0 +1,270 @@+use std::error;+use std::fmt;+use std::io::{self, Read};+use std::iter;+use std::process;+use std::thread::{self, JoinHandle};++/// An error that can occur while running a command and reading its output.+///+/// This error can be seamlessly converted to an `io::Error` via a `From`+/// implementation.+#[derive(Debug)]+pub struct CommandError {+ kind: CommandErrorKind,+}++#[derive(Debug)]+enum CommandErrorKind {+ Io(io::Error),+ Stderr(Vec<u8>),+}++impl CommandError {+ /// Create an error from an I/O error.+ pub(crate) fn io(ioerr: io::Error) -> CommandError {+ CommandError { kind: CommandErrorKind::Io(ioerr) }+ }++ /// Create an error from the contents of stderr (which may be empty).+ pub(crate) fn stderr(bytes: Vec<u8>) -> CommandError {+ CommandError { kind: CommandErrorKind::Stderr(bytes) }+ }+}++impl error::Error for CommandError {+ fn description(&self) -> &str {+ "command error"+ }+}++impl fmt::Display for CommandError {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match self.kind {+ CommandErrorKind::Io(ref e) => e.fmt(f),+ CommandErrorKind::Stderr(ref bytes) => {+ let msg = String::from_utf8_lossy(bytes);+ if msg.trim().is_empty() {+ write!(f, "<stderr is empty>")+ } else {+ let div = iter::repeat('-').take(79).collect::<String>();+ write!(+ f,+ "\n{div}\n{msg}\n{div}",+ div = div,+ msg = msg.trim()+ )+ }+ }+ }+ }+}++impl From<io::Error> for CommandError {+ fn from(ioerr: io::Error) -> CommandError {+ CommandError { kind: CommandErrorKind::Io(ioerr) }+ }+}++impl From<CommandError> for io::Error {+ fn from(cmderr: CommandError) -> io::Error {+ match cmderr.kind {+ CommandErrorKind::Io(ioerr) => ioerr,+ CommandErrorKind::Stderr(_) => {+ io::Error::new(io::ErrorKind::Other, cmderr)+ }+ }+ }+}++/// Configures and builds a streaming reader for process output.+#[derive(Clone, Debug, Default)]+pub struct CommandReaderBuilder {+ async_stderr: bool,+}++impl CommandReaderBuilder {+ /// Create a new builder with the default configuration.+ pub fn new() -> CommandReaderBuilder {+ CommandReaderBuilder::default()+ }++ /// Build a new streaming reader for the given command's output.+ ///+ /// The caller should set everything that's required on the given command+ /// before building a reader, such as its arguments, environment and+ /// current working directory. Settings such as the stdout and stderr (but+ /// not stdin) pipes will be overridden so that they can be controlled by+ /// the reader.+ ///+ /// If there was a problem spawning the given command, then its error is+ /// returned.+ pub fn build(+ &self,+ command: &mut process::Command,+ ) -> Result<CommandReader, CommandError> {+ let mut child = command+ .stdout(process::Stdio::piped())+ .stderr(process::Stdio::piped())+ .spawn()?;+ let stdout = child.stdout.take().unwrap();+ let stderr = if self.async_stderr {+ StderrReader::async(child.stderr.take().unwrap())+ } else {+ StderrReader::sync(child.stderr.take().unwrap())+ };+ Ok(CommandReader {+ child: child,+ stdout: stdout,+ stderr: stderr,+ done: false,+ })+ }++ /// When enabled, the reader will asynchronously read the contents of the+ /// command's stderr output. When disabled, stderr is only read after the+ /// stdout stream has been exhausted (or if the process quits with an error+ /// code).+ ///+ /// Note that when enabled, this may require launching an additional+ /// thread in order to read stderr. This is done so that the process being+ /// executed is never blocked from writing to stdout or stderr. If this is+ /// disabled, then it is possible for the process to fill up the stderr+ /// buffer and deadlock.+ ///+ /// This is enabled by default.+ pub fn async_stderr(&mut self, yes: bool) -> &mut CommandReaderBuilder {+ self.async_stderr = yes;+ self+ }+}++/// A streaming reader for a command's output.+///+/// The purpose of this reader is to provide an easy way to execute processes+/// whose stdout is read in a streaming way while also making the processes'+/// stderr available when the process fails with an exit code. This makes it+/// possible to execute processes while surfacing the underlying failure mode+/// in the case of an error.+///+/// Moreover, by default, this reader will asynchronously read the processes'+/// stderr. This prevents subtle deadlocking bugs for noisy processes that+/// write a lot to stderr. Currently, the entire contents of stderr is read+/// on to the heap.+///+/// # Example+///+/// This example shows how to invoke `gzip` to decompress the contents of a+/// file. If the `gzip` command reports a failing exit status, then its stderr+/// is returned as an error.+///+/// ```no_run+/// use std::io::Read;+/// use std::process::Command;+/// use grep_cli::CommandReader;+///+/// # fn example() -> Result<(), Box<::std::error::Error>> {+/// let mut cmd = Command::new("gzip");+/// cmd.arg("-d").arg("-c").arg("/usr/share/man/man1/ls.1.gz");+///+/// let mut rdr = CommandReader::new(&mut cmd)?;+/// let mut contents = vec![];+/// rdr.read_to_end(&mut contents)?;+/// # Ok(()) }+/// ```+#[derive(Debug)]+pub struct CommandReader {+ child: process::Child,+ stdout: process::ChildStdout,+ stderr: StderrReader,+ done: bool,+}++impl CommandReader {+ /// Create a new streaming reader for the given command using the default+ /// configuration.+ ///+ /// The caller should set everything that's required on the given command+ /// before building a reader, such as its arguments, environment and+ /// current working directory. Settings such as the stdout and stderr (but+ /// not stdin) pipes will be overridden so that they can be controlled by+ /// the reader.+ ///+ /// If there was a problem spawning the given command, then its error is+ /// returned.+ ///+ /// If the caller requires additional configuration for the reader+ /// returned, then use+ /// [`CommandReaderBuilder`](struct.CommandReaderBuilder.html).+ pub fn new(+ cmd: &mut process::Command,+ ) -> Result<CommandReader, CommandError> {+ CommandReaderBuilder::new().build(cmd)+ }+}++impl io::Read for CommandReader {+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {+ if self.done {+ return Ok(0);+ }+ let nread = self.stdout.read(buf)?;+ if nread == 0 {+ self.done = true;+ // Reap the child now that we're done reading. If the command+ // failed, report stderr as an error.+ if !self.child.wait()?.success() {+ return Err(io::Error::from(self.stderr.read_to_end()));+ }+ }+ Ok(nread)+ }+}++/// A reader that encapsulates the asynchronous or synchronous reading of+/// stderr.+#[derive(Debug)]+enum StderrReader {+ Async(Option<JoinHandle<CommandError>>),+ Sync(process::ChildStderr),+}++impl StderrReader {+ /// Create a reader for stderr that reads contents asynchronously.+ fn async(mut stderr: process::ChildStderr) -> StderrReader {+ let handle =+ thread::spawn(move || stderr_to_command_error(&mut stderr));+ StderrReader::Async(Some(handle))+ }++ /// Create a reader for stderr that reads contents synchronously.+ fn sync(stderr: process::ChildStderr) -> StderrReader {+ StderrReader::Sync(stderr)+ }++ /// Consumes all of stderr on to the heap and returns it as an error.+ ///+ /// If there was a problem reading stderr itself, then this returns an I/O+ /// command error.+ fn read_to_end(&mut self) -> CommandError {+ match *self {+ StderrReader::Async(ref mut handle) => {+ let handle = handle+ .take()+ .expect("read_to_end cannot be called more than once");+ handle.join().expect("stderr reading thread does not panic")+ }+ StderrReader::Sync(ref mut stderr) => {+ stderr_to_command_error(stderr)+ }+ }+ }+}++fn stderr_to_command_error(stderr: &mut process::ChildStderr) -> CommandError {+ let mut bytes = vec![];+ match stderr.read_to_end(&mut bytes) {+ Ok(_) => CommandError::stderr(bytes),+ Err(err) => CommandError::io(err),+ }+}
@@ -0,0 +1,133 @@+use std::io;++use termcolor;++use is_tty_stdout;++/// A writer that supports coloring with either line or block buffering.+pub struct StandardStream(StandardStreamKind);++/// Returns a possibly buffered writer to stdout for the given color choice.+///+/// The writer returned is either line buffered or block buffered. The decision+/// between these two is made automatically based on whether a tty is attached+/// to stdout or not. If a tty is attached, then line buffering is used.+/// Otherwise, block buffering is used. In general, block buffering is more+/// efficient, but may increase the time it takes for the end user to see the+/// first bits of output.+///+/// If you need more fine grained control over the buffering mode, then use one+/// of `stdout_buffered_line` or `stdout_buffered_block`.+///+/// The color choice given is passed along to the underlying writer. To+/// completely disable colors in all cases, use `ColorChoice::Never`.+pub fn stdout(color_choice: termcolor::ColorChoice) -> StandardStream {+ if is_tty_stdout() {+ stdout_buffered_line(color_choice)+ } else {+ stdout_buffered_block(color_choice)+ }+}++/// Returns a line buffered writer to stdout for the given color choice.+///+/// This writer is useful when printing results directly to a tty such that+/// users see output as soon as it's written. The downside of this approach+/// is that it can be slower, especially when there is a lot of output.+///+/// You might consider using+/// [`stdout`](fn.stdout.html)+/// instead, which chooses the buffering strategy automatically based on+/// whether stdout is connected to a tty.+pub fn stdout_buffered_line(+ color_choice: termcolor::ColorChoice,+) -> StandardStream {+ let out = termcolor::StandardStream::stdout(color_choice);+ StandardStream(StandardStreamKind::LineBuffered(out))+}++/// Returns a block buffered writer to stdout for the given color choice.+///+/// This writer is useful when printing results to a file since it amortizes+/// the cost of writing data. The downside of this approach is that it can+/// increase the latency of display output when writing to a tty.+///+/// You might consider using+/// [`stdout`](fn.stdout.html)+/// instead, which chooses the buffering strategy automatically based on+/// whether stdout is connected to a tty.+pub fn stdout_buffered_block(+ color_choice: termcolor::ColorChoice,+) -> StandardStream {+ let out = termcolor::BufferedStandardStream::stdout(color_choice);+ StandardStream(StandardStreamKind::BlockBuffered(out))+}++enum StandardStreamKind {+ LineBuffered(termcolor::StandardStream),+ BlockBuffered(termcolor::BufferedStandardStream),+}++impl io::Write for StandardStream {+ #[inline]+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {+ use self::StandardStreamKind::*;++ match self.0 {+ LineBuffered(ref mut w) => w.write(buf),+ BlockBuffered(ref mut w) => w.write(buf),+ }+ }++ #[inline]+ fn flush(&mut self) -> io::Result<()> {+ use self::StandardStreamKind::*;++ match self.0 {+ LineBuffered(ref mut w) => w.flush(),+ BlockBuffered(ref mut w) => w.flush(),+ }+ }+}++impl termcolor::WriteColor for StandardStream {+ #[inline]+ fn supports_color(&self) -> bool {+ use self::StandardStreamKind::*;++ match self.0 {+ LineBuffered(ref w) => w.supports_color(),+ BlockBuffered(ref w) => w.supports_color(),+ }+ }++ #[inline]+ fn set_color(&mut self, spec: &termcolor::ColorSpec) -> io::Result<()> {+ use self::StandardStreamKind::*;++ match self.0 {+ LineBuffered(ref mut w) => w.set_color(spec),+ BlockBuffered(ref mut w) => w.set_color(spec),+ }+ }++ #[inline]+ fn reset(&mut self) -> io::Result<()> {+ use self::StandardStreamKind::*;++ match self.0 {+ LineBuffered(ref mut w) => w.reset(),+ BlockBuffered(ref mut w) => w.reset(),+ }+ }++ #[inline]+ fn is_synchronous(&self) -> bool {+ use self::StandardStreamKind::*;++ match self.0 {+ LineBuffered(ref w) => w.is_synchronous(),+ BlockBuffered(ref w) => w.is_synchronous(),+ }+ }+}
crates/core/app.rs
@@ -0,0 +1,2955 @@+// This module defines the set of command line arguments that ripgrep supports,+// including some light validation.+//+// This module is purposely written in a bare-bones way, since it is included+// in ripgrep's build.rs file as a way to generate a man page and completion+// files for common shells.+//+// The only other place that ripgrep deals with clap is in src/args.rs, which+// is where we read clap's configuration from the end user's arguments and turn+// it into a ripgrep-specific configuration type that is not coupled with clap.++use clap::{self, crate_authors, crate_version, App, AppSettings};+use lazy_static::lazy_static;++const ABOUT: &str = "+ripgrep (rg) recursively searches your current directory for a regex pattern.+By default, ripgrep will respect your .gitignore and automatically skip hidden+files/directories and binary files.++ripgrep's default regex engine uses finite automata and guarantees linear+time searching. Because of this, features like backreferences and arbitrary+look-around are not supported. However, if ripgrep is built with PCRE2, then+the --pcre2 flag can be used to enable backreferences and look-around.++ripgrep supports configuration files. Set RIPGREP_CONFIG_PATH to a+configuration file. The file can specify one shell argument per line. Lines+starting with '#' are ignored. For more details, see the man page or the+README.++Tip: to disable all smart filtering and make ripgrep behave a bit more like+classical grep, use 'rg -uuu'.++Project home page: https://github.com/BurntSushi/ripgrep++Use -h for short descriptions and --help for more details.";++const USAGE: &str = "+ rg [OPTIONS] PATTERN [PATH ...]+ rg [OPTIONS] [-e PATTERN ...] [-f PATTERNFILE ...] [PATH ...]+ rg [OPTIONS] --files [PATH ...]+ rg [OPTIONS] --type-list+ command | rg [OPTIONS] PATTERN";++const TEMPLATE: &str = "\+{bin} {version}+{author}+{about}++USAGE:{usage}++ARGS:+{positionals}++OPTIONS:+{unified}";++/// Build a clap application parameterized by usage strings.+pub fn app() -> App<'static, 'static> {+ // We need to specify our version in a static because we've painted clap+ // into a corner. We've told it that every string we give it will be+ // 'static, but we need to build the version string dynamically. We can+ // fake the 'static lifetime with lazy_static.+ lazy_static! {+ static ref LONG_VERSION: String = long_version(None);+ }++ let mut app = App::new("ripgrep")+ .author(crate_authors!())+ .version(crate_version!())+ .long_version(LONG_VERSION.as_str())+ .about(ABOUT)+ .max_term_width(100)+ .setting(AppSettings::UnifiedHelpMessage)+ .setting(AppSettings::AllArgsOverrideSelf)+ .usage(USAGE)+ .template(TEMPLATE)+ .help_message("Prints help information. Use --help for more details.");+ for arg in all_args_and_flags() {+ app = app.arg(arg.claparg);+ }+ app+}++/// Return the "long" format of ripgrep's version string.+///+/// If a revision hash is given, then it is used. If one isn't given, then+/// the RIPGREP_BUILD_GIT_HASH env var is inspected for it. If that isn't set,+/// then a revision hash is not included in the version string returned.+pub fn long_version(revision_hash: Option<&str>) -> String {+ // Do we have a git hash?+ // (Yes, if ripgrep was built on a machine with `git` installed.)+ let hash = match revision_hash.or(option_env!("RIPGREP_BUILD_GIT_HASH")) {+ None => String::new(),+ Some(githash) => format!(" (rev {})", githash),+ };+ // Put everything together.+ let runtime = runtime_cpu_features();+ if runtime.is_empty() {+ format!(+ "{}{}\n{} (compiled)",+ crate_version!(),+ hash,+ compile_cpu_features().join(" ")+ )+ } else {+ format!(+ "{}{}\n{} (compiled)\n{} (runtime)",+ crate_version!(),+ hash,+ compile_cpu_features().join(" "),+ runtime.join(" ")+ )+ }+}++/// Returns the relevant CPU features enabled at compile time.+fn compile_cpu_features() -> Vec<&'static str> {+ let mut features = vec![];+ if cfg!(feature = "simd-accel") {+ features.push("+SIMD");+ } else {+ features.push("-SIMD");+ }+ if cfg!(feature = "avx-accel") {+ features.push("+AVX");+ } else {+ features.push("-AVX");+ }+ features+}++/// Returns the relevant CPU features enabled at runtime.+#[cfg(target_arch = "x86_64")]+fn runtime_cpu_features() -> Vec<&'static str> {+ // This is kind of a dirty violation of abstraction, since it assumes+ // knowledge about what specific SIMD features are being used.++ let mut features = vec![];+ if is_x86_feature_detected!("ssse3") {+ features.push("+SIMD");+ } else {+ features.push("-SIMD");+ }+ if is_x86_feature_detected!("avx2") {+ features.push("+AVX");+ } else {+ features.push("-AVX");+ }+ features+}++/// Returns the relevant CPU features enabled at runtime.+#[cfg(not(target_arch = "x86_64"))]+fn runtime_cpu_features() -> Vec<&'static str> {+ vec![]+}++/// Arg is a light alias for a clap::Arg that is specialized to compile time+/// string literals.+type Arg = clap::Arg<'static, 'static>;++/// RGArg is a light wrapper around a clap::Arg and also contains some metadata+/// about the underlying Arg so that it can be inspected for other purposes+/// (e.g., hopefully generating a man page).+///+/// Note that this type is purposely overly constrained to ripgrep's particular+/// use of clap.+#[allow(dead_code)]+#[derive(Clone)]+pub struct RGArg {+ /// The underlying clap argument.+ claparg: Arg,+ /// The name of this argument. This is always present and is the name+ /// used in the code to find the value of an argument at runtime.+ pub name: &'static str,+ /// A short documentation string describing this argument. This string+ /// should fit on a single line and be a complete sentence.+ ///+ /// This is shown in the `-h` output.+ pub doc_short: &'static str,+ /// A longer documentation string describing this argument. This usually+ /// starts with the contents of `doc_short`. This is also usually many+ /// lines, potentially paragraphs, and may contain examples and additional+ /// prose.+ ///+ /// This is shown in the `--help` output.+ pub doc_long: &'static str,+ /// Whether this flag is hidden or not.+ ///+ /// This is typically used for uncommon flags that only serve to override+ /// other flags. For example, --no-ignore is a prominent flag that disables+ /// ripgrep's gitignore functionality, but --ignore re-enables it. Since+ /// gitignore support is enabled by default, use of the --ignore flag is+ /// somewhat niche and relegated to special cases when users make use of+ /// configuration files to set defaults.+ ///+ /// Generally, these flags should be documented in the documentation for+ /// the flag they override.+ pub hidden: bool,+ /// The type of this argument.+ pub kind: RGArgKind,+}++/// The kind of a ripgrep argument.+///+/// This can be one of three possibilities: a positional argument, a boolean+/// switch flag or a flag that accepts exactly one argument. Each variant+/// stores argument type specific data.+///+/// Note that clap supports more types of arguments than this, but we don't+/// (and probably shouldn't) use them in ripgrep.+///+/// Finally, note that we don't capture *all* state about an argument in this+/// type. Some state is only known to clap. There isn't any particular reason+/// why; the state we do capture is motivated by use cases (like generating+/// documentation).+#[derive(Clone)]+pub enum RGArgKind {+ /// A positional argument.+ Positional {+ /// The name of the value used in the `-h/--help` output. By+ /// convention, this is an all-uppercase string. e.g., `PATH` or+ /// `PATTERN`.+ value_name: &'static str,+ /// Whether an argument can be repeated multiple times or not.+ ///+ /// The only argument this applies to is PATH, where an end user can+ /// specify multiple paths for ripgrep to search.+ ///+ /// If this is disabled, then an argument can only be provided once.+ /// For example, PATTERN is one such argument. (Note that the+ /// -e/--regexp flag is distinct from the positional PATTERN argument,+ /// and it can be provided multiple times.)+ multiple: bool,+ },+ /// A boolean switch.+ Switch {+ /// The long name of a flag. This is always non-empty.+ long: &'static str,+ /// The short name of a flag. This is empty if a flag only has a long+ /// name.+ short: Option<&'static str>,+ /// Whether this switch can be provided multiple times where meaning+ /// is attached to the number of times this flag is given.+ ///+ /// Note that every switch can be provided multiple times. This+ /// particular state indicates whether all instances of a switch are+ /// relevant or not.+ ///+ /// For example, the -u/--unrestricted flag can be provided multiple+ /// times where each repeated use of it indicates more relaxing of+ /// ripgrep's filtering. Conversely, the -i/--ignore-case flag can+ /// also be provided multiple times, but it is simply considered either+ /// present or not. In these cases, -u/--unrestricted has `multiple`+ /// set to `true` while -i/--ignore-case has `multiple` set to `false`.+ multiple: bool,+ },+ /// A flag the accepts a single value.+ Flag {+ /// The long name of a flag. This is always non-empty.+ long: &'static str,+ /// The short name of a flag. This is empty if a flag only has a long+ /// name.+ short: Option<&'static str>,+ /// The name of the value used in the `-h/--help` output. By+ /// convention, this is an all-uppercase string. e.g., `PATH` or+ /// `PATTERN`.+ value_name: &'static str,+ /// Whether this flag can be provided multiple times with multiple+ /// distinct values.+ ///+ /// Note that every flag can be provided multiple times. This+ /// particular state indicates whether all instances of a flag are+ /// relevant or not.+ ///+ /// For example, the -g/--glob flag can be provided multiple times and+ /// all of its values should be interpreted by ripgrep. Conversely,+ /// while the -C/--context flag can also be provided multiple times,+ /// only its last instance is used while all previous instances are+ /// ignored. In these cases, -g/--glob has `multiple` set to `true`+ /// while -C/--context has `multiple` set to `false`.+ multiple: bool,+ /// A set of possible values for this flag. If an end user provides+ /// any value other than what's in this set, then clap will report an+ /// error.+ possible_values: Vec<&'static str>,+ },+}++impl RGArg {+ /// Create a positional argument.+ ///+ /// The `long_name` parameter is the name of the argument, e.g., `pattern`.+ /// The `value_name` parameter is a name that describes the type of+ /// argument this flag accepts. It should be in uppercase, e.g., PATH or+ /// PATTERN.+ fn positional(name: &'static str, value_name: &'static str) -> RGArg {+ RGArg {+ claparg: Arg::with_name(name).value_name(value_name),+ name: name,+ doc_short: "",+ doc_long: "",+ hidden: false,+ kind: RGArgKind::Positional {+ value_name: value_name,+ multiple: false,+ },+ }+ }++ /// Create a boolean switch.+ ///+ /// The `long_name` parameter is the name of the flag, e.g., `--long-name`.+ ///+ /// All switches may be repeated an arbitrary number of times. If a switch+ /// is truly boolean, that consumers of clap's configuration should only+ /// check whether the flag is present or not. Otherwise, consumers may+ /// inspect the number of times the switch is used.+ fn switch(long_name: &'static str) -> RGArg {+ let claparg = Arg::with_name(long_name).long(long_name);+ RGArg {+ claparg: claparg,+ name: long_name,+ doc_short: "",+ doc_long: "",+ hidden: false,+ kind: RGArgKind::Switch {+ long: long_name,+ short: None,+ multiple: false,+ },+ }+ }++ /// Create a flag. A flag always accepts exactly one argument.+ ///+ /// The `long_name` parameter is the name of the flag, e.g., `--long-name`.+ /// The `value_name` parameter is a name that describes the type of+ /// argument this flag accepts. It should be in uppercase, e.g., PATH or+ /// PATTERN.+ ///+ /// All flags may be repeated an arbitrary number of times. If a flag has+ /// only one logical value, that consumers of clap's configuration should+ /// only use the last value.+ fn flag(long_name: &'static str, value_name: &'static str) -> RGArg {+ let claparg = Arg::with_name(long_name)+ .long(long_name)+ .value_name(value_name)+ .takes_value(true)+ .number_of_values(1);+ RGArg {+ claparg: claparg,+ name: long_name,+ doc_short: "",+ doc_long: "",+ hidden: false,+ kind: RGArgKind::Flag {+ long: long_name,+ short: None,+ value_name: value_name,+ multiple: false,+ possible_values: vec![],+ },+ }+ }++ /// Set the short flag name.+ ///+ /// This panics if this arg isn't a switch or a flag.+ fn short(mut self, name: &'static str) -> RGArg {+ match self.kind {+ RGArgKind::Positional { .. } => panic!("expected switch or flag"),+ RGArgKind::Switch { ref mut short, .. } => {+ *short = Some(name);+ }+ RGArgKind::Flag { ref mut short, .. } => {+ *short = Some(name);+ }+ }+ self.claparg = self.claparg.short(name);+ self+ }++ /// Set the "short" help text.+ ///+ /// This should be a single line. It is shown in the `-h` output.+ fn help(mut self, text: &'static str) -> RGArg {+ self.doc_short = text;+ self.claparg = self.claparg.help(text);+ self+ }++ /// Set the "long" help text.+ ///+ /// This should be at least a single line, usually longer. It is shown in+ /// the `--help` output.+ fn long_help(mut self, text: &'static str) -> RGArg {+ self.doc_long = text;+ self.claparg = self.claparg.long_help(text);+ self+ }++ /// Enable this argument to accept multiple values.+ ///+ /// Note that while switches and flags can always be repeated an arbitrary+ /// number of times, this particular method enables the flag to be+ /// logically repeated where each occurrence of the flag may have+ /// significance. That is, when this is disabled, then a switch is either+ /// present or not and a flag has exactly one value (the last one given).+ /// When this is enabled, then a switch has a count corresponding to the+ /// number of times it is used and a flag's value is a list of all values+ /// given.+ ///+ /// For the most part, this distinction is resolved by consumers of clap's+ /// configuration.+ fn multiple(mut self) -> RGArg {+ // Why not put `multiple` on RGArg proper? Because it's useful to+ // document it distinct for each different kind. See RGArgKind docs.+ match self.kind {+ RGArgKind::Positional { ref mut multiple, .. } => {+ *multiple = true;+ }+ RGArgKind::Switch { ref mut multiple, .. } => {+ *multiple = true;+ }+ RGArgKind::Flag { ref mut multiple, .. } => {+ *multiple = true;+ }+ }+ self.claparg = self.claparg.multiple(true);+ self+ }++ /// Hide this flag from all documentation.+ fn hidden(mut self) -> RGArg {+ self.hidden = true;+ self.claparg = self.claparg.hidden(true);+ self+ }++ /// Set the possible values for this argument. If this argument is not+ /// a flag, then this panics.+ ///+ /// If the end user provides any value other than what is given here, then+ /// clap will report an error to the user.+ ///+ /// Note that this will suppress clap's automatic output of possible values+ /// when using -h/--help, so users of this method should provide+ /// appropriate documentation for the choices in the "long" help text.+ fn possible_values(mut self, values: &[&'static str]) -> RGArg {+ match self.kind {+ RGArgKind::Positional { .. } => panic!("expected flag"),+ RGArgKind::Switch { .. } => panic!("expected flag"),+ RGArgKind::Flag { ref mut possible_values, .. } => {+ *possible_values = values.to_vec();+ self.claparg = self+ .claparg+ .possible_values(values)+ .hide_possible_values(true);+ }+ }+ self+ }++ /// Add an alias to this argument.+ ///+ /// Aliases are not show in the output of -h/--help.+ fn alias(mut self, name: &'static str) -> RGArg {+ self.claparg = self.claparg.alias(name);+ self+ }++ /// Permit this flag to have values that begin with a hypen.+ ///+ /// This panics if this arg is not a flag.+ fn allow_leading_hyphen(mut self) -> RGArg {+ match self.kind {+ RGArgKind::Positional { .. } => panic!("expected flag"),+ RGArgKind::Switch { .. } => panic!("expected flag"),+ RGArgKind::Flag { .. } => {+ self.claparg = self.claparg.allow_hyphen_values(true);+ }+ }+ self+ }++ /// Sets this argument to a required argument, unless one of the given+ /// arguments is provided.+ fn required_unless(mut self, names: &[&'static str]) -> RGArg {+ self.claparg = self.claparg.required_unless_one(names);+ self+ }++ /// Sets conflicting arguments. That is, if this argument is used whenever+ /// any of the other arguments given here are used, then clap will report+ /// an error.+ fn conflicts(mut self, names: &[&'static str]) -> RGArg {+ self.claparg = self.claparg.conflicts_with_all(names);+ self+ }++ /// Sets an overriding argument. That is, if this argument and the given+ /// argument are both provided by an end user, then the "last" one will+ /// win. ripgrep will behave as if any previous instantiations did not+ /// happen.+ fn overrides(mut self, name: &'static str) -> RGArg {+ self.claparg = self.claparg.overrides_with(name);+ self+ }++ /// Sets the default value of this argument if and only if the argument+ /// given is present.+ fn default_value_if(+ mut self,+ value: &'static str,+ arg_name: &'static str,+ ) -> RGArg {+ self.claparg = self.claparg.default_value_if(arg_name, None, value);+ self+ }++ /// Indicate that any value given to this argument should be a number. If+ /// it's not a number, then clap will report an error to the end user.+ fn number(mut self) -> RGArg {+ self.claparg = self.claparg.validator(|val| {+ val.parse::<usize>().map(|_| ()).map_err(|err| err.to_string())+ });+ self+ }+}++// We add an extra space to long descriptions so that a blank line is inserted+// between flag descriptions in --help output.+macro_rules! long {+ ($lit:expr) => {+ concat!($lit, " ")+ };+}++/// Generate a sequence of all positional and flag arguments.+pub fn all_args_and_flags() -> Vec<RGArg> {+ let mut args = vec![];+ // The positional arguments must be defined first and in order.+ arg_pattern(&mut args);+ arg_path(&mut args);+ // Flags can be defined in any order, but we do it alphabetically. Note+ // that each function may define multiple flags. For example,+ // `flag_encoding` defines `--encoding` and `--no-encoding`. Most `--no`+ // flags are hidden and merely mentioned in the docs of the corresponding+ // "positive" flag.+ flag_after_context(&mut args);+ flag_auto_hybrid_regex(&mut args);+ flag_before_context(&mut args);+ flag_binary(&mut args);+ flag_block_buffered(&mut args);+ flag_byte_offset(&mut args);+ flag_case_sensitive(&mut args);+ flag_color(&mut args);+ flag_colors(&mut args);+ flag_column(&mut args);+ flag_context(&mut args);+ flag_context_separator(&mut args);+ flag_count(&mut args);+ flag_count_matches(&mut args);+ flag_crlf(&mut args);+ flag_debug(&mut args);+ flag_dfa_size_limit(&mut args);+ flag_encoding(&mut args);+ flag_file(&mut args);+ flag_files(&mut args);+ flag_files_with_matches(&mut args);+ flag_files_without_match(&mut args);+ flag_fixed_strings(&mut args);+ flag_follow(&mut args);+ flag_glob(&mut args);+ flag_glob_case_insensitive(&mut args);+ flag_heading(&mut args);+ flag_hidden(&mut args);+ flag_iglob(&mut args);+ flag_ignore_case(&mut args);+ flag_ignore_file(&mut args);+ flag_ignore_file_case_insensitive(&mut args);+ flag_include_zero(&mut args);+ flag_invert_match(&mut args);+ flag_json(&mut args);+ flag_line_buffered(&mut args);+ flag_line_number(&mut args);+ flag_line_regexp(&mut args);+ flag_max_columns(&mut args);+ flag_max_columns_preview(&mut args);+ flag_max_count(&mut args);+ flag_max_depth(&mut args);+ flag_max_filesize(&mut args);+ flag_mmap(&mut args);+ flag_multiline(&mut args);+ flag_multiline_dotall(&mut args);+ flag_no_config(&mut args);+ flag_no_ignore(&mut args);+ flag_no_ignore_dot(&mut args);+ flag_no_ignore_exclude(&mut args);+ flag_no_ignore_global(&mut args);+ flag_no_ignore_messages(&mut args);+ flag_no_ignore_parent(&mut args);+ flag_no_ignore_vcs(&mut args);+ flag_no_messages(&mut args);+ flag_no_pcre2_unicode(&mut args);+ flag_no_require_git(&mut args);+ flag_no_unicode(&mut args);+ flag_null(&mut args);+ flag_null_data(&mut args);+ flag_one_file_system(&mut args);+ flag_only_matching(&mut args);+ flag_path_separator(&mut args);+ flag_passthru(&mut args);+ flag_pcre2(&mut args);+ flag_pcre2_version(&mut args);+ flag_pre(&mut args);+ flag_pre_glob(&mut args);+ flag_pretty(&mut args);+ flag_quiet(&mut args);+ flag_regex_size_limit(&mut args);+ flag_regexp(&mut args);+ flag_replace(&mut args);+ flag_search_zip(&mut args);+ flag_smart_case(&mut args);+ flag_sort_files(&mut args);+ flag_sort(&mut args);+ flag_sortr(&mut args);+ flag_stats(&mut args);+ flag_text(&mut args);+ flag_threads(&mut args);+ flag_trim(&mut args);+ flag_type(&mut args);+ flag_type_add(&mut args);+ flag_type_clear(&mut args);+ flag_type_list(&mut args);+ flag_type_not(&mut args);+ flag_unrestricted(&mut args);+ flag_vimgrep(&mut args);+ flag_with_filename(&mut args);+ flag_word_regexp(&mut args);+ args+}++fn arg_pattern(args: &mut Vec<RGArg>) {+ const SHORT: &str = "A regular expression used for searching.";+ const LONG: &str = long!(+ "\+A regular expression used for searching. To match a pattern beginning with a+dash, use the -e/--regexp flag.++For example, to search for the literal '-foo', you can use this flag:++ rg -e -foo++You can also use the special '--' delimiter to indicate that no more flags+will be provided. Namely, the following is equivalent to the above:++ rg -- -foo+"+ );+ let arg = RGArg::positional("pattern", "PATTERN")+ .help(SHORT)+ .long_help(LONG)+ .required_unless(&[+ "file",+ "files",+ "regexp",+ "type-list",+ "pcre2-version",+ ]);+ args.push(arg);+}++fn arg_path(args: &mut Vec<RGArg>) {+ const SHORT: &str = "A file or directory to search.";+ const LONG: &str = long!(+ "\+A file or directory to search. Directories are searched recursively. Paths \+specified on the command line override glob and ignore rules. \+"+ );+ let arg = RGArg::positional("path", "PATH")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_after_context(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show NUM lines after each match.";+ const LONG: &str = long!(+ "\+Show NUM lines after each match.++This overrides the --context flag.+"+ );+ let arg = RGArg::flag("after-context", "NUM")+ .short("A")+ .help(SHORT)+ .long_help(LONG)+ .number()+ .overrides("context");+ args.push(arg);+}++fn flag_auto_hybrid_regex(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Dynamically use PCRE2 if necessary.";+ const LONG: &str = long!(+ "\+When this flag is used, ripgrep will dynamically choose between supported regex+engines depending on the features used in a pattern. When ripgrep chooses a+regex engine, it applies that choice for every regex provided to ripgrep (e.g.,+via multiple -e/--regexp or -f/--file flags).++As an example of how this flag might behave, ripgrep will attempt to use+its default finite automata based regex engine whenever the pattern can be+successfully compiled with that regex engine. If PCRE2 is enabled and if the+pattern given could not be compiled with the default regex engine, then PCRE2+will be automatically used for searching. If PCRE2 isn't available, then this+flag has no effect because there is only one regex engine to choose from.++In the future, ripgrep may adjust its heuristics for how it decides which+regex engine to use. In general, the heuristics will be limited to a static+analysis of the patterns, and not to any specific runtime behavior observed+while searching files.++The primary downside of using this flag is that it may not always be obvious+which regex engine ripgrep uses, and thus, the match semantics or performance+profile of ripgrep may subtly and unexpectedly change. However, in many cases,+all regex engines will agree on what constitutes a match and it can be nice+to transparently support more advanced regex features like look-around and+backreferences without explicitly needing to enable them.++This flag can be disabled with --no-auto-hybrid-regex.+"+ );+ let arg = RGArg::switch("auto-hybrid-regex")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-auto-hybrid-regex")+ .overrides("pcre2")+ .overrides("no-pcre2");+ args.push(arg);++ let arg = RGArg::switch("no-auto-hybrid-regex")+ .hidden()+ .overrides("auto-hybrid-regex")+ .overrides("pcre2")+ .overrides("no-pcre2");+ args.push(arg);+}++fn flag_before_context(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show NUM lines before each match.";+ const LONG: &str = long!(+ "\+Show NUM lines before each match.++This overrides the --context flag.+"+ );+ let arg = RGArg::flag("before-context", "NUM")+ .short("B")+ .help(SHORT)+ .long_help(LONG)+ .number()+ .overrides("context");+ args.push(arg);+}++fn flag_binary(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search binary files.";+ const LONG: &str = long!(+ "\+Enabling this flag will cause ripgrep to search binary files. By default,+ripgrep attempts to automatically skip binary files in order to improve the+relevance of results and make the search faster.++Binary files are heuristically detected based on whether they contain a NUL+byte or not. By default (without this flag set), once a NUL byte is seen,+ripgrep will stop searching the file. Usually, NUL bytes occur in the beginning+of most binary files. If a NUL byte occurs after a match, then ripgrep will+still stop searching the rest of the file, but a warning will be printed.++In contrast, when this flag is provided, ripgrep will continue searching a file+even if a NUL byte is found. In particular, if a NUL byte is found then ripgrep+will continue searching until either a match is found or the end of the file is+reached, whichever comes sooner. If a match is found, then ripgrep will stop+and print a warning saying that the search stopped prematurely.++If you want ripgrep to search a file without any special NUL byte handling at+all (and potentially print binary data to stdout), then you should use the+'-a/--text' flag.++The '--binary' flag is a flag for controlling ripgrep's automatic filtering+mechanism. As such, it does not need to be used when searching a file+explicitly or when searching stdin. That is, it is only applicable when+recursively searching a directory.++Note that when the '-u/--unrestricted' flag is provided for a third time, then+this flag is automatically enabled.++This flag can be disabled with '--no-binary'. It overrides the '-a/--text'+flag.+"+ );+ let arg = RGArg::switch("binary")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-binary")+ .overrides("text")+ .overrides("no-text");+ args.push(arg);++ let arg = RGArg::switch("no-binary")+ .hidden()+ .overrides("binary")+ .overrides("text")+ .overrides("no-text");+ args.push(arg);+}++fn flag_block_buffered(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Force block buffering.";+ const LONG: &str = long!(+ "\+When enabled, ripgrep will use block buffering. That is, whenever a matching+line is found, it will be written to an in-memory buffer and will not be+written to stdout until the buffer reaches a certain size. This is the default+when ripgrep's stdout is redirected to a pipeline or a file. When ripgrep's+stdout is connected to a terminal, line buffering will be used. Forcing block+buffering can be useful when dumping a large amount of contents to a terminal.++Forceful block buffering can be disabled with --no-block-buffered. Note that+using --no-block-buffered causes ripgrep to revert to its default behavior of+automatically detecting the buffering strategy. To force line buffering, use+the --line-buffered flag.+"+ );+ let arg = RGArg::switch("block-buffered")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-block-buffered")+ .overrides("line-buffered")+ .overrides("no-line-buffered");+ args.push(arg);++ let arg = RGArg::switch("no-block-buffered")+ .hidden()+ .overrides("block-buffered")+ .overrides("line-buffered")+ .overrides("no-line-buffered");+ args.push(arg);+}++fn flag_byte_offset(args: &mut Vec<RGArg>) {+ const SHORT: &str =+ "Print the 0-based byte offset for each matching line.";+ const LONG: &str = long!(+ "\+Print the 0-based byte offset within the input file before each line of output.+If -o (--only-matching) is specified, print the offset of the matching part+itself.++If ripgrep does transcoding, then the byte offset is in terms of the the result+of transcoding and not the original data. This applies similarly to another+transformation on the source, such as decompression or a --pre filter. Note+that when the PCRE2 regex engine is used, then UTF-8 transcoding is done by+default.+"+ );+ let arg =+ RGArg::switch("byte-offset").short("b").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_case_sensitive(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search case sensitively (default).";+ const LONG: &str = long!(+ "\+Search case sensitively.++This overrides the -i/--ignore-case and -S/--smart-case flags.+"+ );+ let arg = RGArg::switch("case-sensitive")+ .short("s")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-case")+ .overrides("smart-case");+ args.push(arg);+}++fn flag_color(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Controls when to use color.";+ const LONG: &str = long!(+ "\+This flag controls when to use colors. The default setting is 'auto', which+means ripgrep will try to guess when to use colors. For example, if ripgrep is+printing to a terminal, then it will use colors, but if it is redirected to a+file or a pipe, then it will suppress color output. ripgrep will suppress color+output in some other circumstances as well. For example, if the TERM+environment variable is not set or set to 'dumb', then ripgrep will not use+colors.++The possible values for this flag are:++ never Colors will never be used.+ auto The default. ripgrep tries to be smart.+ always Colors will always be used regardless of where output is sent.+ ansi Like 'always', but emits ANSI escapes (even in a Windows console).++When the --vimgrep flag is given to ripgrep, then the default value for the+--color flag changes to 'never'.+"+ );+ let arg = RGArg::flag("color", "WHEN")+ .help(SHORT)+ .long_help(LONG)+ .possible_values(&["never", "auto", "always", "ansi"])+ .default_value_if("never", "vimgrep");+ args.push(arg);+}++fn flag_colors(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Configure color settings and styles.";+ const LONG: &str = long!(+ "\+This flag specifies color settings for use in the output. This flag may be+provided multiple times. Settings are applied iteratively. Colors are limited+to one of eight choices: red, blue, green, cyan, magenta, yellow, white and+black. Styles are limited to nobold, bold, nointense, intense, nounderline+or underline.++The format of the flag is '{type}:{attribute}:{value}'. '{type}' should be+one of path, line, column or match. '{attribute}' can be fg, bg or style.+'{value}' is either a color (for fg and bg) or a text style. A special format,+'{type}:none', will clear all color settings for '{type}'.++For example, the following command will change the match color to magenta and+the background color for line numbers to yellow:++ rg --colors 'match:fg:magenta' --colors 'line:bg:yellow' foo.++Extended colors can be used for '{value}' when the terminal supports ANSI color+sequences. These are specified as either 'x' (256-color) or 'x,x,x' (24-bit+truecolor) where x is a number between 0 and 255 inclusive. x may be given as+a normal decimal number or a hexadecimal number, which is prefixed by `0x`.++For example, the following command will change the match background color to+that represented by the rgb value (0,128,255):++ rg --colors 'match:bg:0,128,255'++or, equivalently,++ rg --colors 'match:bg:0x0,0x80,0xFF'++Note that the the intense and nointense style flags will have no effect when+used alongside these extended color codes.+"+ );+ let arg = RGArg::flag("colors", "COLOR_SPEC")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_column(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show column numbers.";+ const LONG: &str = long!(+ "\+Show column numbers (1-based). This only shows the column numbers for the first+match on each line. This does not try to account for Unicode. One byte is equal+to one column. This implies --line-number.++This flag can be disabled with --no-column.+"+ );+ let arg = RGArg::switch("column")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-column");+ args.push(arg);++ let arg = RGArg::switch("no-column").hidden().overrides("column");+ args.push(arg);+}++fn flag_context(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show NUM lines before and after each match.";+ const LONG: &str = long!(+ "\+Show NUM lines before and after each match. This is equivalent to providing+both the -B/--before-context and -A/--after-context flags with the same value.++This overrides both the -B/--before-context and -A/--after-context flags.+"+ );+ let arg = RGArg::flag("context", "NUM")+ .short("C")+ .help(SHORT)+ .long_help(LONG)+ .number()+ .overrides("before-context")+ .overrides("after-context");+ args.push(arg);+}++fn flag_context_separator(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Set the context separator string.";+ const LONG: &str = long!(+ "\+The string used to separate non-contiguous context lines in the output. This+is only used when one of the context flags is used (-A, -B or -C). Escape+sequences like \\x7F or \\t may be used. The default value is --.++When the context separator is set to an empty string, then a line break+is still inserted. To completely disable context separators, use the+--no-context-separator flag.+"+ );++ let arg = RGArg::flag("context-separator", "SEPARATOR")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-context-separator");+ args.push(arg);++ let arg = RGArg::switch("no-context-separator")+ .hidden()+ .overrides("context-separator");+ args.push(arg);+}++fn flag_count(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Only show the count of matching lines for each file.";+ const LONG: &str = long!(+ "\+This flag suppresses normal output and shows the number of lines that match+the given patterns for each file searched. Each file containing a match has its+path and count printed on each line. Note that this reports the number of lines+that match and not the total number of matches.++If only one file is given to ripgrep, then only the count is printed if there+is a match. The --with-filename flag can be used to force printing the file+path in this case.++This overrides the --count-matches flag. Note that when --count is combined+with --only-matching, then ripgrep behaves as if --count-matches was given.+"+ );+ let arg = RGArg::switch("count")+ .short("c")+ .help(SHORT)+ .long_help(LONG)+ .overrides("count-matches");+ args.push(arg);+}++fn flag_count_matches(args: &mut Vec<RGArg>) {+ const SHORT: &str =+ "Only show the count of individual matches for each file.";+ const LONG: &str = long!(+ "\+This flag suppresses normal output and shows the number of individual+matches of the given patterns for each file searched. Each file+containing matches has its path and match count printed on each line.+Note that this reports the total number of individual matches and not+the number of lines that match.++If only one file is given to ripgrep, then only the count is printed if there+is a match. The --with-filename flag can be used to force printing the file+path in this case.++This overrides the --count flag. Note that when --count is combined with+--only-matching, then ripgrep behaves as if --count-matches was given.+"+ );+ let arg = RGArg::switch("count-matches")+ .help(SHORT)+ .long_help(LONG)+ .overrides("count");+ args.push(arg);+}++fn flag_crlf(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Support CRLF line terminators (useful on Windows).";+ const LONG: &str = long!(+ "\+When enabled, ripgrep will treat CRLF ('\\r\\n') as a line terminator instead+of just '\\n'.++Principally, this permits '$' in regex patterns to match just before CRLF+instead of just before LF. The underlying regex engine may not support this+natively, so ripgrep will translate all instances of '$' to '(?:\\r??$)'. This+may produce slightly different than desired match offsets. It is intended as a+work-around until the regex engine supports this natively.++CRLF support can be disabled with --no-crlf.+"+ );+ let arg = RGArg::switch("crlf")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-crlf")+ .overrides("null-data");+ args.push(arg);++ let arg = RGArg::switch("no-crlf").hidden().overrides("crlf");+ args.push(arg);+}++fn flag_debug(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show debug messages.";+ const LONG: &str = long!(+ "\+Show debug messages. Please use this when filing a bug report.++The --debug flag is generally useful for figuring out why ripgrep skipped+searching a particular file. The debug messages should mention all files+skipped and why they were skipped.++To get even more debug output, use the --trace flag, which implies --debug+along with additional trace data. With --trace, the output could be quite+large and is generally more useful for development.+"+ );+ let arg = RGArg::switch("debug").help(SHORT).long_help(LONG);+ args.push(arg);++ let arg = RGArg::switch("trace").hidden().overrides("debug");+ args.push(arg);+}++fn flag_dfa_size_limit(args: &mut Vec<RGArg>) {+ const SHORT: &str = "The upper size limit of the regex DFA.";+ const LONG: &str = long!(+ "\+The upper size limit of the regex DFA. The default limit is 10M. This should+only be changed on very large regex inputs where the (slower) fallback regex+engine may otherwise be used if the limit is reached.++The argument accepts the same size suffixes as allowed in with the+--max-filesize flag.+"+ );+ let arg = RGArg::flag("dfa-size-limit", "NUM+SUFFIX?")+ .help(SHORT)+ .long_help(LONG);+ args.push(arg);+}++fn flag_encoding(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Specify the text encoding of files to search.";+ const LONG: &str = long!(+ "\+Specify the text encoding that ripgrep will use on all files searched. The+default value is 'auto', which will cause ripgrep to do a best effort automatic+detection of encoding on a per-file basis. Automatic detection in this case+only applies to files that begin with a UTF-8 or UTF-16 byte-order mark (BOM).+No other automatic detection is performed. One can also specify 'none' which+will then completely disable BOM sniffing and always result in searching the+raw bytes, including a BOM if it's present, regardless of its encoding.++Other supported values can be found in the list of labels here:+https://encoding.spec.whatwg.org/#concept-encoding-get++For more details on encoding and how ripgrep deals with it, see GUIDE.md.++This flag can be disabled with --no-encoding.+"+ );+ let arg = RGArg::flag("encoding", "ENCODING")+ .short("E")+ .help(SHORT)+ .long_help(LONG);+ args.push(arg);++ let arg = RGArg::switch("no-encoding").hidden().overrides("encoding");+ args.push(arg);+}++fn flag_file(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search for patterns from the given file.";+ const LONG: &str = long!(+ "\+Search for patterns from the given file, with one pattern per line. When this+flag is used multiple times or in combination with the -e/--regexp flag,+then all patterns provided are searched. Empty pattern lines will match all+input lines, and the newline is not counted as part of the pattern.++A line is printed if and only if it matches at least one of the patterns.+"+ );+ let arg = RGArg::flag("file", "PATTERNFILE")+ .short("f")+ .help(SHORT)+ .long_help(LONG)+ .multiple()+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_files(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print each file that would be searched.";+ const LONG: &str = long!(+ "\+Print each file that would be searched without actually performing the search.+This is useful to determine whether a particular file is being searched or not.+"+ );+ let arg = RGArg::switch("files")+ .help(SHORT)+ .long_help(LONG)+ // This also technically conflicts with pattern, but the first file+ // path will actually be in pattern.+ .conflicts(&["file", "regexp", "type-list"]);+ args.push(arg);+}++fn flag_files_with_matches(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Only print the paths with at least one match.";+ const LONG: &str = long!(+ "\+Only print the paths with at least one match.++This overrides --files-without-match.+"+ );+ let arg = RGArg::switch("files-with-matches")+ .short("l")+ .help(SHORT)+ .long_help(LONG)+ .overrides("files-without-match");+ args.push(arg);+}++fn flag_files_without_match(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Only print the paths that contain zero matches.";+ const LONG: &str = long!(+ "\+Only print the paths that contain zero matches. This inverts/negates the+--files-with-matches flag.++This overrides --files-with-matches.+"+ );+ let arg = RGArg::switch("files-without-match")+ .help(SHORT)+ .long_help(LONG)+ .overrides("files-with-matches");+ args.push(arg);+}++fn flag_fixed_strings(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Treat the pattern as a literal string.";+ const LONG: &str = long!(+ "\+Treat the pattern as a literal string instead of a regular expression. When+this flag is used, special regular expression meta characters such as .(){}*++do not need to be escaped.++This flag can be disabled with --no-fixed-strings.+"+ );+ let arg = RGArg::switch("fixed-strings")+ .short("F")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-fixed-strings");+ args.push(arg);++ let arg =+ RGArg::switch("no-fixed-strings").hidden().overrides("fixed-strings");+ args.push(arg);+}++fn flag_follow(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Follow symbolic links.";+ const LONG: &str = long!(+ "\+When this flag is enabled, ripgrep will follow symbolic links while traversing+directories. This is disabled by default. Note that ripgrep will check for+symbolic link loops and report errors if it finds one.++This flag can be disabled with --no-follow.+"+ );+ let arg = RGArg::switch("follow")+ .short("L")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-follow");+ args.push(arg);++ let arg = RGArg::switch("no-follow").hidden().overrides("follow");+ args.push(arg);+}++fn flag_glob(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Include or exclude files.";+ const LONG: &str = long!(+ "\+Include or exclude files and directories for searching that match the given+glob. This always overrides any other ignore logic. Multiple glob flags may be+used. Globbing rules match .gitignore globs. Precede a glob with a ! to exclude+it.+"+ );+ let arg = RGArg::flag("glob", "GLOB")+ .short("g")+ .help(SHORT)+ .long_help(LONG)+ .multiple()+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_glob_case_insensitive(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Process all glob patterns case insensitively.";+ const LONG: &str = long!(+ "\+Process glob patterns given with the -g/--glob flag case insensitively. This+effectively treats --glob as --iglob.++This flag can be disabled with the --no-glob-case-insensitive flag.+"+ );+ let arg = RGArg::switch("glob-case-insensitive")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-glob-case-insensitive");+ args.push(arg);++ let arg = RGArg::switch("no-glob-case-insensitive")+ .hidden()+ .overrides("glob-case-insensitive");+ args.push(arg);+}++fn flag_heading(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print matches grouped by each file.";+ const LONG: &str = long!(+ "\+This flag prints the file path above clusters of matches from each file instead+of printing the file path as a prefix for each matched line. This is the+default mode when printing to a terminal.++This overrides the --no-heading flag.+"+ );+ let arg = RGArg::switch("heading")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-heading");+ args.push(arg);++ const NO_SHORT: &str = "Don't group matches by each file.";+ const NO_LONG: &str = long!(+ "\+Don't group matches by each file. If --no-heading is provided in addition to+the -H/--with-filename flag, then file paths will be printed as a prefix for+every matched line. This is the default mode when not printing to a terminal.++This overrides the --heading flag.+"+ );+ let arg = RGArg::switch("no-heading")+ .help(NO_SHORT)+ .long_help(NO_LONG)+ .overrides("heading");+ args.push(arg);+}++fn flag_hidden(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search hidden files and directories.";+ const LONG: &str = long!(+ "\+Search hidden files and directories. By default, hidden files and directories+are skipped. Note that if a hidden file or a directory is whitelisted in an+ignore file, then it will be searched even if this flag isn't provided.++This flag can be disabled with --no-hidden.+"+ );+ let arg = RGArg::switch("hidden")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-hidden");+ args.push(arg);++ let arg = RGArg::switch("no-hidden").hidden().overrides("hidden");+ args.push(arg);+}++fn flag_iglob(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Include or exclude files case insensitively.";+ const LONG: &str = long!(+ "\+Include or exclude files and directories for searching that match the given+glob. This always overrides any other ignore logic. Multiple glob flags may be+used. Globbing rules match .gitignore globs. Precede a glob with a ! to exclude+it. Globs are matched case insensitively.+"+ );+ let arg = RGArg::flag("iglob", "GLOB")+ .help(SHORT)+ .long_help(LONG)+ .multiple()+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_ignore_case(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Case insensitive search.";+ const LONG: &str = long!(+ "\+When this flag is provided, the given patterns will be searched case+insensitively. The case insensitivity rules used by ripgrep conform to+Unicode's \"simple\" case folding rules.++This flag overrides -s/--case-sensitive and -S/--smart-case.+"+ );+ let arg = RGArg::switch("ignore-case")+ .short("i")+ .help(SHORT)+ .long_help(LONG)+ .overrides("case-sensitive")+ .overrides("smart-case");+ args.push(arg);+}++fn flag_ignore_file(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Specify additional ignore files.";+ const LONG: &str = long!(+ "\+Specifies a path to one or more .gitignore format rules files. These patterns+are applied after the patterns found in .gitignore and .ignore are applied+and are matched relative to the current working directory. Multiple additional+ignore files can be specified by using the --ignore-file flag several times.+When specifying multiple ignore files, earlier files have lower precedence+than later files.++If you are looking for a way to include or exclude files and directories+directly on the command line, then used -g instead.+"+ );+ let arg = RGArg::flag("ignore-file", "PATH")+ .help(SHORT)+ .long_help(LONG)+ .multiple()+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_ignore_file_case_insensitive(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Process ignore files case insensitively.";+ const LONG: &str = long!(+ "\+Process ignore files (.gitignore, .ignore, etc.) case insensitively. Note that+this comes with a performance penalty and is most useful on case insensitive+file systems (such as Windows).++This flag can be disabled with the --no-ignore-file-case-insensitive flag.+"+ );+ let arg = RGArg::switch("ignore-file-case-insensitive")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-ignore-file-case-insensitive");+ args.push(arg);++ let arg = RGArg::switch("no-ignore-file-case-insensitive")+ .hidden()+ .overrides("ignore-file-case-insensitive");+ args.push(arg);+}++fn flag_include_zero(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Include files with zero matches in summary";+ const LONG: &str = long!(+ "\+When used with --count or --count-matches, print the number of matches for+each file even if there were zero matches. This is disabled by default but can+be enabled to make ripgrep behave more like grep.+"+ );+ let arg = RGArg::switch("include-zero").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_invert_match(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Invert matching.";+ const LONG: &str = long!(+ "\+Invert matching. Show lines that do not match the given patterns.+"+ );+ let arg =+ RGArg::switch("invert-match").short("v").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_json(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show search results in a JSON Lines format.";+ const LONG: &str = long!(+ "\+Enable printing results in a JSON Lines format.++When this flag is provided, ripgrep will emit a sequence of messages, each+encoded as a JSON object, where there are five different message types:++**begin** - A message that indicates a file is being searched and contains at+least one match.++**end** - A message the indicates a file is done being searched. This message+also include summary statistics about the search for a particular file.++**match** - A message that indicates a match was found. This includes the text+and offsets of the match.++**context** - A message that indicates a contextual line was found. This+includes the text of the line, along with any match information if the search+was inverted.++**summary** - The final message emitted by ripgrep that contains summary+statistics about the search across all files.++Since file paths or the contents of files are not guaranteed to be valid UTF-8+and JSON itself must be representable by a Unicode encoding, ripgrep will emit+all data elements as objects with one of two keys: 'text' or 'bytes'. 'text' is+a normal JSON string when the data is valid UTF-8 while 'bytes' is the base64+encoded contents of the data.++The JSON Lines format is only supported for showing search results. It cannot+be used with other flags that emit other types of output, such as --files,+--files-with-matches, --files-without-match, --count or --count-matches.+ripgrep will report an error if any of the aforementioned flags are used in+concert with --json.++Other flags that control aspects of the standard output such as+--only-matching, --heading, --replace, --max-columns, etc., have no effect+when --json is set.++A more complete description of the JSON format used can be found here:+https://docs.rs/grep-printer/*/grep_printer/struct.JSON.html++The JSON Lines format can be disabled with --no-json.+"+ );+ let arg = RGArg::switch("json")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-json")+ .conflicts(&[+ "count",+ "count-matches",+ "files",+ "files-with-matches",+ "files-without-match",+ ]);+ args.push(arg);++ let arg = RGArg::switch("no-json").hidden().overrides("json");+ args.push(arg);+}++fn flag_line_buffered(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Force line buffering.";+ const LONG: &str = long!(+ "\+When enabled, ripgrep will use line buffering. That is, whenever a matching+line is found, it will be flushed to stdout immediately. This is the default+when ripgrep's stdout is connected to a terminal, but otherwise, ripgrep will+use block buffering, which is typically faster. This flag forces ripgrep to+use line buffering even if it would otherwise use block buffering. This is+typically useful in shell pipelines, e.g.,+'tail -f something.log | rg foo --line-buffered | rg bar'.++Forceful line buffering can be disabled with --no-line-buffered. Note that+using --no-line-buffered causes ripgrep to revert to its default behavior of+automatically detecting the buffering strategy. To force block buffering, use+the --block-buffered flag.+"+ );+ let arg = RGArg::switch("line-buffered")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-line-buffered")+ .overrides("block-buffered")+ .overrides("no-block-buffered");+ args.push(arg);++ let arg = RGArg::switch("no-line-buffered")+ .hidden()+ .overrides("line-buffered")+ .overrides("block-buffered")+ .overrides("no-block-buffered");+ args.push(arg);+}++fn flag_line_number(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show line numbers.";+ const LONG: &str = long!(+ "\+Show line numbers (1-based). This is enabled by default when searching in a+terminal.+"+ );+ let arg = RGArg::switch("line-number")+ .short("n")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-line-number");+ args.push(arg);++ const NO_SHORT: &str = "Suppress line numbers.";+ const NO_LONG: &str = long!(+ "\+Suppress line numbers. This is enabled by default when not searching in a+terminal.+"+ );+ let arg = RGArg::switch("no-line-number")+ .short("N")+ .help(NO_SHORT)+ .long_help(NO_LONG)+ .overrides("line-number");+ args.push(arg);+}++fn flag_line_regexp(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Only show matches surrounded by line boundaries.";+ const LONG: &str = long!(+ "\+Only show matches surrounded by line boundaries. This is equivalent to putting+^...$ around all of the search patterns. In other words, this only prints lines+where the entire line participates in a match.++This overrides the --word-regexp flag.+"+ );+ let arg = RGArg::switch("line-regexp")+ .short("x")+ .help(SHORT)+ .long_help(LONG)+ .overrides("word-regexp");+ args.push(arg);+}++fn flag_max_columns(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't print lines longer than this limit.";+ const LONG: &str = long!(+ "\+Don't print lines longer than this limit in bytes. Longer lines are omitted,+and only the number of matches in that line is printed.++When this flag is omitted or is set to 0, then it has no effect.+"+ );+ let arg = RGArg::flag("max-columns", "NUM")+ .short("M")+ .help(SHORT)+ .long_help(LONG)+ .number();+ args.push(arg);+}++fn flag_max_columns_preview(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print a preview for lines exceeding the limit.";+ const LONG: &str = long!(+ "\+When the '--max-columns' flag is used, ripgrep will by default completely+replace any line that is too long with a message indicating that a matching+line was removed. When this flag is combined with '--max-columns', a preview+of the line (corresponding to the limit size) is shown instead, where the part+of the line exceeding the limit is not shown.++If the '--max-columns' flag is not set, then this has no effect.++This flag can be disabled with '--no-max-columns-preview'.+"+ );+ let arg = RGArg::switch("max-columns-preview")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-max-columns-preview");+ args.push(arg);++ let arg = RGArg::switch("no-max-columns-preview")+ .hidden()+ .overrides("max-columns-preview");+ args.push(arg);+}++fn flag_max_count(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Limit the number of matches.";+ const LONG: &str = long!(+ "\+Limit the number of matching lines per file searched to NUM.+"+ );+ let arg = RGArg::flag("max-count", "NUM")+ .short("m")+ .help(SHORT)+ .long_help(LONG)+ .number();+ args.push(arg);+}++fn flag_max_depth(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Descend at most NUM directories.";+ const LONG: &str = long!(+ "\+Limit the depth of directory traversal to NUM levels beyond the paths given. A+value of zero only searches the explicitly given paths themselves.++For example, 'rg --max-depth 0 dir/' is a no-op because dir/ will not be+descended into. 'rg --max-depth 1 dir/' will search only the direct children of+'dir'.+"+ );+ let arg = RGArg::flag("max-depth", "NUM")+ .help(SHORT)+ .long_help(LONG)+ .alias("maxdepth")+ .number();+ args.push(arg);+}++fn flag_max_filesize(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Ignore files larger than NUM in size.";+ const LONG: &str = long!(+ "\+Ignore files larger than NUM in size. This does not apply to directories.++The input format accepts suffixes of K, M or G which correspond to kilobytes,+megabytes and gigabytes, respectively. If no suffix is provided the input is+treated as bytes.++Examples: --max-filesize 50K or --max-filesize 80M+"+ );+ let arg =+ RGArg::flag("max-filesize", "NUM+SUFFIX?").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_mmap(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search using memory maps when possible.";+ const LONG: &str = long!(+ "\+Search using memory maps when possible. This is enabled by default when ripgrep+thinks it will be faster.++Memory map searching doesn't currently support all options, so if an+incompatible option (e.g., --context) is given with --mmap, then memory maps+will not be used.++Note that ripgrep may abort unexpectedly when --mmap if it searches a file that+is simultaneously truncated.++This flag overrides --no-mmap.+"+ );+ let arg =+ RGArg::switch("mmap").help(SHORT).long_help(LONG).overrides("no-mmap");+ args.push(arg);++ const NO_SHORT: &str = "Never use memory maps.";+ const NO_LONG: &str = long!(+ "\+Never use memory maps, even when they might be faster.++This flag overrides --mmap.+"+ );+ let arg = RGArg::switch("no-mmap")+ .help(NO_SHORT)+ .long_help(NO_LONG)+ .overrides("mmap");+ args.push(arg);+}++fn flag_multiline(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Enable matching across multiple lines.";+ const LONG: &str = long!(+ "\+Enable matching across multiple lines.++When multiline mode is enabled, ripgrep will lift the restriction that a match+cannot include a line terminator. For example, when multiline mode is not+enabled (the default), then the regex '\\p{any}' will match any Unicode+codepoint other than '\\n'. Similarly, the regex '\\n' is explicitly forbidden,+and if you try to use it, ripgrep will return an error. However, when multiline+mode is enabled, '\\p{any}' will match any Unicode codepoint, including '\\n',+and regexes like '\\n' are permitted.++An important caveat is that multiline mode does not change the match semantics+of '.'. Namely, in most regex matchers, a '.' will by default match any+character other than '\\n', and this is true in ripgrep as well. In order to+make '.' match '\\n', you must enable the \"dot all\" flag inside the regex.+For example, both '(?s).' and '(?s:.)' have the same semantics, where '.' will+match any character, including '\\n'. Alternatively, the '--multiline-dotall'+flag may be passed to make the \"dot all\" behavior the default. This flag only+applies when multiline search is enabled.++There is no limit on the number of the lines that a single match can span.++**WARNING**: Because of how the underlying regex engine works, multiline+searches may be slower than normal line-oriented searches, and they may also+use more memory. In particular, when multiline mode is enabled, ripgrep+requires that each file it searches is laid out contiguously in memory+(either by reading it onto the heap or by memory-mapping it). Things that+cannot be memory-mapped (such as stdin) will be consumed until EOF before+searching can begin. In general, ripgrep will only do these things when+necessary. Specifically, if the --multiline flag is provided but the regex+does not contain patterns that would match '\\n' characters, then ripgrep+will automatically avoid reading each file into memory before searching it.+Nevertheless, if you only care about matches spanning at most one line, then it+is always better to disable multiline mode.++This flag can be disabled with --no-multiline.+"+ );+ let arg = RGArg::switch("multiline")+ .short("U")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-multiline");+ args.push(arg);++ let arg = RGArg::switch("no-multiline").hidden().overrides("multiline");+ args.push(arg);+}++fn flag_multiline_dotall(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Make '.' match new lines when multiline is enabled.";+ const LONG: &str = long!(+ "\+This flag enables \"dot all\" in your regex pattern, which causes '.' to match+newlines when multiline searching is enabled. This flag has no effect if+multiline searching isn't enabled with the --multiline flag.++Normally, a '.' will match any character except newlines. While this behavior+typically isn't relevant for line-oriented matching (since matches can span at+most one line), this can be useful when searching with the -U/--multiline flag.+By default, the multiline mode runs without this flag.++This flag is generally intended to be used in an alias or your ripgrep config+file if you prefer \"dot all\" semantics by default. Note that regardless of+whether this flag is used, \"dot all\" semantics can still be controlled via+inline flags in the regex pattern itself, e.g., '(?s:.)' always enables \"dot+all\" whereas '(?-s:.)' always disables \"dot all\".++This flag can be disabled with --no-multiline-dotall.+"+ );+ let arg = RGArg::switch("multiline-dotall")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-multiline-dotall");+ args.push(arg);++ let arg = RGArg::switch("no-multiline-dotall")+ .hidden()+ .overrides("multiline-dotall");+ args.push(arg);+}++fn flag_no_config(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Never read configuration files.";+ const LONG: &str = long!(+ "\+Never read configuration files. When this flag is present, ripgrep will not+respect the RIPGREP_CONFIG_PATH environment variable.++If ripgrep ever grows a feature to automatically read configuration files in+pre-defined locations, then this flag will also disable that behavior as well.+"+ );+ let arg = RGArg::switch("no-config").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_no_ignore(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't respect ignore files.";+ const LONG: &str = long!(+ "\+Don't respect ignore files (.gitignore, .ignore, etc.). This implies+--no-ignore-parent, --no-ignore-dot and --no-ignore-vcs.++This flag can be disabled with the --ignore flag.+"+ );+ let arg = RGArg::switch("no-ignore")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore");+ args.push(arg);++ let arg = RGArg::switch("ignore").hidden().overrides("no-ignore");+ args.push(arg);+}++fn flag_no_ignore_dot(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't respect .ignore files.";+ const LONG: &str = long!(+ "\+Don't respect .ignore files.++This flag can be disabled with the --ignore-dot flag.+"+ );+ let arg = RGArg::switch("no-ignore-dot")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-dot");+ args.push(arg);++ let arg = RGArg::switch("ignore-dot").hidden().overrides("no-ignore-dot");+ args.push(arg);+}++fn flag_no_ignore_exclude(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't respect local exclusion files.";+ const LONG: &str = long!(+ "\+Don't respect ignore files that are manually configured for the repository+such as git's '.git/info/exclude'.++This flag can be disabled with the --ignore-exclude flag.+"+ );+ let arg = RGArg::switch("no-ignore-exclude")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-exclude");+ args.push(arg);++ let arg = RGArg::switch("ignore-exclude")+ .hidden()+ .overrides("no-ignore-exclude");+ args.push(arg);+}++fn flag_no_ignore_global(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't respect global ignore files.";+ const LONG: &str = long!(+ "\+Don't respect ignore files that come from \"global\" sources such as git's+`core.excludesFile` configuration option (which defaults to+`$HOME/.config/git/ignore`).++This flag can be disabled with the --ignore-global flag.+"+ );+ let arg = RGArg::switch("no-ignore-global")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-global");+ args.push(arg);++ let arg =+ RGArg::switch("ignore-global").hidden().overrides("no-ignore-global");+ args.push(arg);+}++fn flag_no_ignore_messages(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Suppress gitignore parse error messages.";+ const LONG: &str = long!(+ "\+Suppresses all error messages related to parsing ignore files such as .ignore+or .gitignore.++This flag can be disabled with the --ignore-messages flag.+"+ );+ let arg = RGArg::switch("no-ignore-messages")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-messages");+ args.push(arg);++ let arg = RGArg::switch("ignore-messages")+ .hidden()+ .overrides("no-ignore-messages");+ args.push(arg);+}++fn flag_no_ignore_parent(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't respect ignore files in parent directories.";+ const LONG: &str = long!(+ "\+Don't respect ignore files (.gitignore, .ignore, etc.) in parent directories.++This flag can be disabled with the --ignore-parent flag.+"+ );+ let arg = RGArg::switch("no-ignore-parent")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-parent");+ args.push(arg);++ let arg =+ RGArg::switch("ignore-parent").hidden().overrides("no-ignore-parent");+ args.push(arg);+}++fn flag_no_ignore_vcs(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Don't respect VCS ignore files.";+ const LONG: &str = long!(+ "\+Don't respect version control ignore files (.gitignore, etc.). This implies+--no-ignore-parent for VCS files. Note that .ignore files will continue to be+respected.++This flag can be disabled with the --ignore-vcs flag.+"+ );+ let arg = RGArg::switch("no-ignore-vcs")+ .help(SHORT)+ .long_help(LONG)+ .overrides("ignore-vcs");+ args.push(arg);++ let arg = RGArg::switch("ignore-vcs").hidden().overrides("no-ignore-vcs");+ args.push(arg);+}++fn flag_no_messages(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Suppress some error messages.";+ const LONG: &str = long!(+ "\+Suppress all error messages related to opening and reading files. Error+messages related to the syntax of the pattern given are still shown.++This flag can be disabled with the --messages flag.+"+ );+ let arg = RGArg::switch("no-messages")+ .help(SHORT)+ .long_help(LONG)+ .overrides("messages");+ args.push(arg);++ let arg = RGArg::switch("messages").hidden().overrides("no-messages");+ args.push(arg);+}++fn flag_no_pcre2_unicode(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Disable Unicode mode for PCRE2 matching.";+ const LONG: &str = long!(+ "\+DEPRECATED. Use --no-unicode instead.++This flag is now an alias for --no-unicode. And --pcre2-unicode is an alias+for --unicode.+"+ );+ let arg = RGArg::switch("no-pcre2-unicode")+ .help(SHORT)+ .long_help(LONG)+ .overrides("pcre2-unicode")+ .overrides("unicode");+ args.push(arg);++ let arg = RGArg::switch("pcre2-unicode")+ .hidden()+ .overrides("no-pcre2-unicode")+ .overrides("no-unicode");+ args.push(arg);+}++fn flag_no_require_git(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Do not require a git repository to use gitignores.";+ const LONG: &str = long!(+ "\+By default, ripgrep will only respect global gitignore rules, .gitignore rules+and local exclude rules if ripgrep detects that you are searching inside a+git repository. This flag allows you to relax this restriction such that+ripgrep will respect all git related ignore rules regardless of whether you're+searching in a git repository or not.++This flag can be disabled with --require-git.+"+ );+ let arg = RGArg::switch("no-require-git")+ .help(SHORT)+ .long_help(LONG)+ .overrides("require-git");+ args.push(arg);++ let arg =+ RGArg::switch("require-git").hidden().overrides("no-require-git");+ args.push(arg);+}++fn flag_no_unicode(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Disable Unicode mode.";+ const LONG: &str = long!(+ "\+By default, ripgrep will enable \"Unicode mode\" in all of its regexes. This+has a number of consequences:++* '.' will only match valid UTF-8 encoded scalar values.+* Classes like '\\w', '\\s', '\\d' are all Unicode aware and much bigger+ than their ASCII only versions.+* Case insensitive matching will use Unicode case folding.+* A large array of classes like '\\p{Emoji}' are available.+* Word boundaries ('\\b' and '\\B') use the Unicode definition of a word+ character.++In some cases it can be desirable to turn these things off. The --no-unicode+flag will do exactly that.++For PCRE2 specifically, Unicode mode represents a critical trade off in the+user experience of ripgrep. In particular, unlike the default regex engine,+PCRE2 does not support the ability to search possibly invalid UTF-8 with+Unicode features enabled. Instead, PCRE2 *requires* that everything it searches+when Unicode mode is enabled is valid UTF-8. (Or valid UTF-16/UTF-32, but for+the purposes of ripgrep, we only discuss UTF-8.) This means that if you have+PCRE2's Unicode mode enabled and you attempt to search invalid UTF-8, then+the search for that file will halt and print an error. For this reason, when+PCRE2's Unicode mode is enabled, ripgrep will automatically \"fix\" invalid+UTF-8 sequences by replacing them with the Unicode replacement codepoint. This+penalty does not occur when using the default regex engine.++If you would rather see the encoding errors surfaced by PCRE2 when Unicode mode+is enabled, then pass the --no-encoding flag to disable all transcoding.++The --no-unicode flag can be disabled with --unicode. Note that+--no-pcre2-unicode and --pcre2-unicode are aliases for --no-unicode and+--unicode, respectively.+"+ );+ let arg = RGArg::switch("no-unicode")+ .help(SHORT)+ .long_help(LONG)+ .overrides("unicode")+ .overrides("pcre2-unicode");+ args.push(arg);++ let arg = RGArg::switch("unicode")+ .hidden()+ .overrides("no-unicode")+ .overrides("no-pcre2-unicode");+ args.push(arg);+}++fn flag_null(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print a NUL byte after file paths.";+ const LONG: &str = long!(+ "\+Whenever a file path is printed, follow it with a NUL byte. This includes+printing file paths before matches, and when printing a list of matching files+such as with --count, --files-with-matches and --files. This option is useful+for use with xargs.+"+ );+ let arg = RGArg::switch("null").short("0").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_null_data(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Use NUL as a line terminator instead of \\n.";+ const LONG: &str = long!(+ "\+Enabling this option causes ripgrep to use NUL as a line terminator instead of+the default of '\\n'.++This is useful when searching large binary files that would otherwise have very+long lines if '\\n' were used as the line terminator. In particular, ripgrep+requires that, at a minimum, each line must fit into memory. Using NUL instead+can be a useful stopgap to keep memory requirements low and avoid OOM (out of+memory) conditions.++This is also useful for processing NUL delimited data, such as that emitted+when using ripgrep's -0/--null flag or find's --print0 flag.++Using this flag implies -a/--text.+"+ );+ let arg = RGArg::switch("null-data")+ .help(SHORT)+ .long_help(LONG)+ .overrides("crlf");+ args.push(arg);+}++fn flag_one_file_system(args: &mut Vec<RGArg>) {+ const SHORT: &str =+ "Do not descend into directories on other file systems.";+ const LONG: &str = long!(+ "\+When enabled, ripgrep will not cross file system boundaries relative to where+the search started from.++Note that this applies to each path argument given to ripgrep. For example, in+the command 'rg --one-file-system /foo/bar /quux/baz', ripgrep will search both+'/foo/bar' and '/quux/baz' even if they are on different file systems, but will+not cross a file system boundary when traversing each path's directory tree.++This is similar to find's '-xdev' or '-mount' flag.++This flag can be disabled with --no-one-file-system.+"+ );+ let arg = RGArg::switch("one-file-system")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-one-file-system");+ args.push(arg);++ let arg = RGArg::switch("no-one-file-system")+ .hidden()+ .overrides("one-file-system");+ args.push(arg);+}++fn flag_only_matching(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print only matches parts of a line.";+ const LONG: &str = long!(+ "\+Print only the matched (non-empty) parts of a matching line, with each such+part on a separate output line.+"+ );+ let arg =+ RGArg::switch("only-matching").short("o").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_path_separator(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Set the path separator.";+ const LONG: &str = long!(+ "\+Set the path separator to use when printing file paths. This defaults to your+platform's path separator, which is / on Unix and \\ on Windows. This flag is+intended for overriding the default when the environment demands it (e.g.,+cygwin). A path separator is limited to a single byte.+"+ );+ let arg =+ RGArg::flag("path-separator", "SEPARATOR").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_passthru(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print both matching and non-matching lines.";+ const LONG: &str = long!(+ "\+Print both matching and non-matching lines.++Another way to achieve a similar effect is by modifying your pattern to match+the empty string. For example, if you are searching using 'rg foo' then using+'rg \"^|foo\"' instead will emit every line in every file searched, but only+occurrences of 'foo' will be highlighted. This flag enables the same behavior+without needing to modify the pattern.+"+ );+ let arg = RGArg::switch("passthru")+ .help(SHORT)+ .long_help(LONG)+ .alias("passthrough");+ args.push(arg);+}++fn flag_pcre2(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Enable PCRE2 matching.";+ const LONG: &str = long!(+ "\+When this flag is present, ripgrep will use the PCRE2 regex engine instead of+its default regex engine.++This is generally useful when you want to use features such as look-around+or backreferences.++Note that PCRE2 is an optional ripgrep feature. If PCRE2 wasn't included in+your build of ripgrep, then using this flag will result in ripgrep printing+an error message and exiting. PCRE2 may also have worse user experience in+some cases, since it has fewer introspection APIs than ripgrep's default regex+engine. For example, if you use a '\n' in a PCRE2 regex without the+'-U/--multiline' flag, then ripgrep will silently fail to match anything+instead of reporting an error immediately (like it does with the default+regex engine).++Related flags: --no-pcre2-unicode++This flag can be disabled with --no-pcre2.+"+ );+ let arg = RGArg::switch("pcre2")+ .short("P")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-pcre2")+ .overrides("auto-hybrid-regex")+ .overrides("no-auto-hybrid-regex");+ args.push(arg);++ let arg = RGArg::switch("no-pcre2")+ .hidden()+ .overrides("pcre2")+ .overrides("auto-hybrid-regex")+ .overrides("no-auto-hybrid-regex");+ args.push(arg);+}++fn flag_pcre2_version(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print the version of PCRE2 that ripgrep uses.";+ const LONG: &str = long!(+ "\+When this flag is present, ripgrep will print the version of PCRE2 in use,+along with other information, and then exit. If PCRE2 is not available, then+ripgrep will print an error message and exit with an error code.+"+ );+ let arg = RGArg::switch("pcre2-version").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_pre(args: &mut Vec<RGArg>) {+ const SHORT: &str = "search outputs of COMMAND FILE for each FILE";+ const LONG: &str = long!(+ "\+For each input FILE, search the standard output of COMMAND FILE rather than the+contents of FILE. This option expects the COMMAND program to either be an+absolute path or to be available in your PATH. Either an empty string COMMAND+or the '--no-pre' flag will disable this behavior.++ WARNING: When this flag is set, ripgrep will unconditionally spawn a+ process for every file that is searched. Therefore, this can incur an+ unnecessarily large performance penalty if you don't otherwise need the+ flexibility offered by this flag. One possible mitigation to this is to use+ the '--pre-glob' flag to limit which files a preprocessor is run with.++A preprocessor is not run when ripgrep is searching stdin.++When searching over sets of files that may require one of several decoders+as preprocessors, COMMAND should be a wrapper program or script which first+classifies FILE based on magic numbers/content or based on the FILE name and+then dispatches to an appropriate preprocessor. Each COMMAND also has its+standard input connected to FILE for convenience.++For example, a shell script for COMMAND might look like:++ case \"$1\" in+ *.pdf)+ exec pdftotext \"$1\" -+ ;;+ *)+ case $(file \"$1\") in+ *Zstandard*)+ exec pzstd -cdq+ ;;+ *)+ exec cat+ ;;+ esac+ ;;+ esac++The above script uses `pdftotext` to convert a PDF file to plain text. For+all other files, the script uses the `file` utility to sniff the type of the+file based on its contents. If it is a compressed file in the Zstandard format,+then `pzstd` is used to decompress the contents to stdout.++This overrides the -z/--search-zip flag.+"+ );+ let arg = RGArg::flag("pre", "COMMAND")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-pre")+ .overrides("search-zip");+ args.push(arg);++ let arg = RGArg::switch("no-pre").hidden().overrides("pre");+ args.push(arg);+}++fn flag_pre_glob(args: &mut Vec<RGArg>) {+ const SHORT: &str =+ "Include or exclude files from a preprocessing command.";+ const LONG: &str = long!(+ "\+This flag works in conjunction with the --pre flag. Namely, when one or more+--pre-glob flags are given, then only files that match the given set of globs+will be handed to the command specified by the --pre flag. Any non-matching+files will be searched without using the preprocessor command.++This flag is useful when searching many files with the --pre flag. Namely,+it permits the ability to avoid process overhead for files that don't need+preprocessing. For example, given the following shell script, 'pre-pdftotext':++ #!/bin/sh++ pdftotext \"$1\" -++then it is possible to use '--pre pre-pdftotext --pre-glob \'*.pdf\'' to make+it so ripgrep only executes the 'pre-pdftotext' command on files with a '.pdf'+extension.++Multiple --pre-glob flags may be used. Globbing rules match .gitignore globs.+Precede a glob with a ! to exclude it.++This flag has no effect if the --pre flag is not used.+"+ );+ let arg = RGArg::flag("pre-glob", "GLOB")+ .help(SHORT)+ .long_help(LONG)+ .multiple()+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_pretty(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Alias for --color always --heading --line-number.";+ const LONG: &str = long!(+ "\+This is a convenience alias for '--color always --heading --line-number'. This+flag is useful when you still want pretty output even if you're piping ripgrep+to another program or file. For example: 'rg -p foo | less -R'.+"+ );+ let arg = RGArg::switch("pretty").short("p").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_quiet(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Do not print anything to stdout.";+ const LONG: &str = long!(+ "\+Do not print anything to stdout. If a match is found in a file, then ripgrep+will stop searching. This is useful when ripgrep is used only for its exit+code (which will be an error if no matches are found).++When --files is used, then ripgrep will stop finding files after finding the+first file that matches all ignore rules.+"+ );+ let arg = RGArg::switch("quiet").short("q").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_regex_size_limit(args: &mut Vec<RGArg>) {+ const SHORT: &str = "The upper size limit of the compiled regex.";+ const LONG: &str = long!(+ "\+The upper size limit of the compiled regex. The default limit is 10M.++The argument accepts the same size suffixes as allowed in the --max-filesize+flag.+"+ );+ let arg = RGArg::flag("regex-size-limit", "NUM+SUFFIX?")+ .help(SHORT)+ .long_help(LONG);+ args.push(arg);+}++fn flag_regexp(args: &mut Vec<RGArg>) {+ const SHORT: &str = "A pattern to search for.";+ const LONG: &str = long!(+ "\+A pattern to search for. This option can be provided multiple times, where+all patterns given are searched. Lines matching at least one of the provided+patterns are printed. This flag can also be used when searching for patterns+that start with a dash.++For example, to search for the literal '-foo', you can use this flag:++ rg -e -foo++You can also use the special '--' delimiter to indicate that no more flags+will be provided. Namely, the following is equivalent to the above:++ rg -- -foo+"+ );+ let arg = RGArg::flag("regexp", "PATTERN")+ .short("e")+ .help(SHORT)+ .long_help(LONG)+ .multiple()+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_replace(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Replace matches with the given text.";+ const LONG: &str = long!(+ "\+Replace every match with the text given when printing results. Neither this+flag nor any other ripgrep flag will modify your files.++Capture group indices (e.g., $5) and names (e.g., $foo) are supported in the+replacement string. In shells such as Bash and zsh, you should wrap the+pattern in single quotes instead of double quotes. Otherwise, capture group+indices will be replaced by expanded shell variables which will most likely+be empty.++Note that the replacement by default replaces each match, and NOT the entire+line. To replace the entire line, you should match the entire line.++This flag can be used with the -o/--only-matching flag.+"+ );+ let arg = RGArg::flag("replace", "REPLACEMENT_TEXT")+ .short("r")+ .help(SHORT)+ .long_help(LONG)+ .allow_leading_hyphen();+ args.push(arg);+}++fn flag_search_zip(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search in compressed files.";+ const LONG: &str = long!(+ "\+Search in compressed files. Currently gzip, bzip2, xz, LZ4, LZMA, Brotli and+Zstd files are supported. This option expects the decompression binaries to be+available in your PATH.++This flag can be disabled with --no-search-zip.+"+ );+ let arg = RGArg::switch("search-zip")+ .short("z")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-search-zip")+ .overrides("pre");+ args.push(arg);++ let arg = RGArg::switch("no-search-zip").hidden().overrides("search-zip");+ args.push(arg);+}++fn flag_smart_case(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Smart case search.";+ const LONG: &str = long!(+ "\+Searches case insensitively if the pattern is all lowercase. Search case+sensitively otherwise.++This overrides the -s/--case-sensitive and -i/--ignore-case flags.+"+ );+ let arg = RGArg::switch("smart-case")+ .short("S")+ .help(SHORT)+ .long_help(LONG)+ .overrides("case-sensitive")+ .overrides("ignore-case");+ args.push(arg);+}++fn flag_sort_files(args: &mut Vec<RGArg>) {+ const SHORT: &str = "DEPRECATED";+ const LONG: &str = long!(+ "\+DEPRECATED: Use --sort or --sortr instead.++Sort results by file path. Note that this currently disables all parallelism+and runs search in a single thread.++This flag can be disabled with --no-sort-files.+"+ );+ let arg = RGArg::switch("sort-files")+ .help(SHORT)+ .long_help(LONG)+ .hidden()+ .overrides("no-sort-files")+ .overrides("sort")+ .overrides("sortr");+ args.push(arg);++ let arg = RGArg::switch("no-sort-files")+ .hidden()+ .overrides("sort-files")+ .overrides("sort")+ .overrides("sortr");+ args.push(arg);+}++fn flag_sort(args: &mut Vec<RGArg>) {+ const SHORT: &str =+ "Sort results in ascending order. Implies --threads=1.";+ const LONG: &str = long!(+ "\+This flag enables sorting of results in ascending order. The possible values+for this flag are:++ none (Default) Do not sort results. Fastest. Can be multi-threaded.+ path Sort by file path. Always single-threaded.+ modified Sort by the last modified time on a file. Always single-threaded.+ accessed Sort by the last accessed time on a file. Always single-threaded.+ created Sort by the creation time on a file. Always single-threaded.++If the chosen (manually or by-default) sorting criteria isn't available on your+system (for example, creation time is not available on ext4 file systems), then+ripgrep will attempt to detect this, print an error and exit without searching.++To sort results in reverse or descending order, use the --sortr flag. Also,+this flag overrides --sortr.++Note that sorting results currently always forces ripgrep to abandon+parallelism and run in a single thread.+"+ );+ let arg = RGArg::flag("sort", "SORTBY")+ .help(SHORT)+ .long_help(LONG)+ .possible_values(&["path", "modified", "accessed", "created", "none"])+ .overrides("sortr")+ .overrides("sort-files")+ .overrides("no-sort-files");+ args.push(arg);+}++fn flag_sortr(args: &mut Vec<RGArg>) {+ const SHORT: &str =+ "Sort results in descending order. Implies --threads=1.";+ const LONG: &str = long!(+ "\+This flag enables sorting of results in descending order. The possible values+for this flag are:++ none (Default) Do not sort results. Fastest. Can be multi-threaded.+ path Sort by file path. Always single-threaded.+ modified Sort by the last modified time on a file. Always single-threaded.+ accessed Sort by the last accessed time on a file. Always single-threaded.+ created Sort by the creation time on a file. Always single-threaded.++If the chosen (manually or by-default) sorting criteria isn't available on your+system (for example, creation time is not available on ext4 file systems), then+ripgrep will attempt to detect this, print an error and exit without searching.++To sort results in ascending order, use the --sort flag. Also, this flag+overrides --sort.++Note that sorting results currently always forces ripgrep to abandon+parallelism and run in a single thread.+"+ );+ let arg = RGArg::flag("sortr", "SORTBY")+ .help(SHORT)+ .long_help(LONG)+ .possible_values(&["path", "modified", "accessed", "created", "none"])+ .overrides("sort")+ .overrides("sort-files")+ .overrides("no-sort-files");+ args.push(arg);+}++fn flag_stats(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print statistics about this ripgrep search.";+ const LONG: &str = long!(+ "\+Print aggregate statistics about this ripgrep search. When this flag is+present, ripgrep will print the following stats to stdout at the end of the+search: number of matched lines, number of files with matches, number of files+searched, and the time taken for the entire search to complete.++This set of aggregate statistics may expand over time.++Note that this flag has no effect if --files, --files-with-matches or+--files-without-match is passed.++This flag can be disabled with --no-stats.+"+ );+ let arg = RGArg::switch("stats")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-stats");+ args.push(arg);++ let arg = RGArg::switch("no-stats").hidden().overrides("stats");+ args.push(arg);+}++fn flag_text(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Search binary files as if they were text.";+ const LONG: &str = long!(+ "\+Search binary files as if they were text. When this flag is present, ripgrep's+binary file detection is disabled. This means that when a binary file is+searched, its contents may be printed if there is a match. This may cause+escape codes to be printed that alter the behavior of your terminal.++When binary file detection is enabled it is imperfect. In general, it uses+a simple heuristic. If a NUL byte is seen during search, then the file is+considered binary and search stops (unless this flag is present).+Alternatively, if the '--binary' flag is used, then ripgrep will only quit+when it sees a NUL byte after it sees a match (or searches the entire file).++This flag can be disabled with '--no-text'. It overrides the '--binary' flag.+"+ );+ let arg = RGArg::switch("text")+ .short("a")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-text")+ .overrides("binary")+ .overrides("no-binary");+ args.push(arg);++ let arg = RGArg::switch("no-text")+ .hidden()+ .overrides("text")+ .overrides("binary")+ .overrides("no-binary");+ args.push(arg);+}++fn flag_threads(args: &mut Vec<RGArg>) {+ const SHORT: &str = "The approximate number of threads to use.";+ const LONG: &str = long!(+ "\+The approximate number of threads to use. A value of 0 (which is the default)+causes ripgrep to choose the thread count using heuristics.+"+ );+ let arg =+ RGArg::flag("threads", "NUM").short("j").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_trim(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Trim prefixed whitespace from matches.";+ const LONG: &str = long!(+ "\+When set, all ASCII whitespace at the beginning of each line printed will be+trimmed.++This flag can be disabled with --no-trim.+"+ );+ let arg =+ RGArg::switch("trim").help(SHORT).long_help(LONG).overrides("no-trim");+ args.push(arg);++ let arg = RGArg::switch("no-trim").hidden().overrides("trim");+ args.push(arg);+}++fn flag_type(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Only search files matching TYPE.";+ const LONG: &str = long!(+ "\+Only search files matching TYPE. Multiple type flags may be provided. Use the+--type-list flag to list all available types.++This flag supports the special value 'all', which will behave as if --type+was provided for every file type supported by ripgrep (including any custom+file types). The end result is that '--type all' causes ripgrep to search in+\"whitelist\" mode, where it will only search files it recognizes via its type+definitions.+"+ );+ let arg = RGArg::flag("type", "TYPE")+ .short("t")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_type_add(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Add a new glob for a file type.";+ const LONG: &str = long!(+ "\+Add a new glob for a particular file type. Only one glob can be added at a+time. Multiple --type-add flags can be provided. Unless --type-clear is used,+globs are added to any existing globs defined inside of ripgrep.++Note that this MUST be passed to every invocation of ripgrep. Type settings are+NOT persisted.++Example:++ rg --type-add 'foo:*.foo' -tfoo PATTERN.++--type-add can also be used to include rules from other types with the special+include directive. The include directive permits specifying one or more other+type names (separated by a comma) that have been defined and its rules will+automatically be imported into the type specified. For example, to create a+type called src that matches C++, Python and Markdown files, one can use:++ --type-add 'src:include:cpp,py,md'++Additional glob rules can still be added to the src type by using the+--type-add flag again:++ --type-add 'src:include:cpp,py,md' --type-add 'src:*.foo'++Note that type names must consist only of Unicode letters or numbers.+Punctuation characters are not allowed.+"+ );+ let arg = RGArg::flag("type-add", "TYPE_SPEC")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_type_clear(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Clear globs for a file type.";+ const LONG: &str = long!(+ "\+Clear the file type globs previously defined for TYPE. This only clears the+default type definitions that are found inside of ripgrep.++Note that this MUST be passed to every invocation of ripgrep. Type settings are+NOT persisted.+"+ );+ let arg = RGArg::flag("type-clear", "TYPE")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_type_not(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Do not search files matching TYPE.";+ const LONG: &str = long!(+ "\+Do not search files matching TYPE. Multiple type-not flags may be provided. Use+the --type-list flag to list all available types.+"+ );+ let arg = RGArg::flag("type-not", "TYPE")+ .short("T")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_type_list(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show all supported file types.";+ const LONG: &str = long!(+ "\+Show all supported file types and their corresponding globs.+"+ );+ let arg = RGArg::switch("type-list")+ .help(SHORT)+ .long_help(LONG)+ // This also technically conflicts with PATTERN, but the first file+ // path will actually be in PATTERN.+ .conflicts(&["file", "files", "pattern", "regexp"]);+ args.push(arg);+}++fn flag_unrestricted(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Reduce the level of \"smart\" searching.";+ const LONG: &str = long!(+ "\+Reduce the level of \"smart\" searching. A single -u won't respect .gitignore+(etc.) files. Two -u flags will additionally search hidden files and+directories. Three -u flags will additionally search binary files.++'rg -uuu' is roughly equivalent to 'grep -r'.+"+ );+ let arg = RGArg::switch("unrestricted")+ .short("u")+ .help(SHORT)+ .long_help(LONG)+ .multiple();+ args.push(arg);+}++fn flag_vimgrep(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Show results in vim compatible format.";+ const LONG: &str = long!(+ "\+Show results with every match on its own line, including line numbers and+column numbers. With this option, a line with more than one match will be+printed more than once.+"+ );+ let arg = RGArg::switch("vimgrep").help(SHORT).long_help(LONG);+ args.push(arg);+}++fn flag_with_filename(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Print the file path with the matched lines.";+ const LONG: &str = long!(+ "\+Display the file path for matches. This is the default when more than one+file is searched. If --heading is enabled (the default when printing to a+terminal), the file path will be shown above clusters of matches from each+file; otherwise, the file name will be shown as a prefix for each matched line.++This flag overrides --no-filename.+"+ );+ let arg = RGArg::switch("with-filename")+ .short("H")+ .help(SHORT)+ .long_help(LONG)+ .overrides("no-filename");+ args.push(arg);++ const NO_SHORT: &str = "Never print the file path with the matched lines.";+ const NO_LONG: &str = long!(+ "\+Never print the file path with the matched lines. This is the default when+ripgrep is explicitly instructed to search one file or stdin.++This flag overrides --with-filename.+"+ );+ let arg = RGArg::switch("no-filename")+ .short("I")+ .help(NO_SHORT)+ .long_help(NO_LONG)+ .overrides("with-filename");+ args.push(arg);+}++fn flag_word_regexp(args: &mut Vec<RGArg>) {+ const SHORT: &str = "Only show matches surrounded by word boundaries.";+ const LONG: &str = long!(+ "\+Only show matches surrounded by word boundaries. This is roughly equivalent to+putting \\b before and after all of the search patterns.++This overrides the --line-regexp flag.+"+ );+ let arg = RGArg::switch("word-regexp")+ .short("w")+ .help(SHORT)+ .long_help(LONG)+ .overrides("line-regexp");+ args.push(arg);+}
crates/core/args.rs
@@ -0,0 +1,1805 @@+use std::cmp;+use std::env;+use std::ffi::{OsStr, OsString};+use std::fs;+use std::io::{self, Write};+use std::path::{Path, PathBuf};+use std::process;+use std::sync::Arc;+use std::time::SystemTime;++use clap;+use grep::cli;+use grep::matcher::LineTerminator;+#[cfg(feature = "pcre2")]+use grep::pcre2::{+ RegexMatcher as PCRE2RegexMatcher,+ RegexMatcherBuilder as PCRE2RegexMatcherBuilder,+};+use grep::printer::{+ default_color_specs, ColorSpecs, JSONBuilder, Standard, StandardBuilder,+ Stats, Summary, SummaryBuilder, SummaryKind, JSON,+};+use grep::regex::{+ RegexMatcher as RustRegexMatcher,+ RegexMatcherBuilder as RustRegexMatcherBuilder,+};+use grep::searcher::{+ BinaryDetection, Encoding, MmapChoice, Searcher, SearcherBuilder,+};+use ignore::overrides::{Override, OverrideBuilder};+use ignore::types::{FileTypeDef, Types, TypesBuilder};+use ignore::{Walk, WalkBuilder, WalkParallel};+use log;+use num_cpus;+use regex;+use termcolor::{BufferWriter, ColorChoice, WriteColor};++use crate::app;+use crate::config;+use crate::logger::Logger;+use crate::messages::{set_ignore_messages, set_messages};+use crate::path_printer::{PathPrinter, PathPrinterBuilder};+use crate::search::{+ PatternMatcher, Printer, SearchWorker, SearchWorkerBuilder,+};+use crate::subject::SubjectBuilder;+use crate::Result;++/// The command that ripgrep should execute based on the command line+/// configuration.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum Command {+ /// Search using exactly one thread.+ Search,+ /// Search using possibly many threads.+ SearchParallel,+ /// The command line parameters suggest that a search should occur, but+ /// ripgrep knows that a match can never be found (e.g., no given patterns+ /// or --max-count=0).+ SearchNever,+ /// Show the files that would be searched, but don't actually search them,+ /// and use exactly one thread.+ Files,+ /// Show the files that would be searched, but don't actually search them,+ /// and perform directory traversal using possibly many threads.+ FilesParallel,+ /// List all file type definitions configured, including the default file+ /// types and any additional file types added to the command line.+ Types,+ /// Print the version of PCRE2 in use.+ PCRE2Version,+}++impl Command {+ /// Returns true if and only if this command requires executing a search.+ fn is_search(&self) -> bool {+ use self::Command::*;++ match *self {+ Search | SearchParallel => true,+ SearchNever | Files | FilesParallel | Types | PCRE2Version => {+ false+ }+ }+ }+}++/// The primary configuration object used throughout ripgrep. It provides a+/// high-level convenient interface to the provided command line arguments.+///+/// An `Args` object is cheap to clone and can be used from multiple threads+/// simultaneously.+#[derive(Clone, Debug)]+pub struct Args(Arc<ArgsImp>);++#[derive(Clone, Debug)]+struct ArgsImp {+ /// Mid-to-low level routines for extracting CLI arguments.+ matches: ArgMatches,+ /// The patterns provided at the command line and/or via the -f/--file+ /// flag. This may be empty.+ patterns: Vec<String>,+ /// A matcher built from the patterns.+ ///+ /// It's important that this is only built once, since building this goes+ /// through regex compilation and various types of analyses. That is, if+ /// you need many of theses (one per thread, for example), it is better to+ /// build it once and then clone it.+ matcher: PatternMatcher,+ /// The paths provided at the command line. This is guaranteed to be+ /// non-empty. (If no paths are provided, then a default path is created.)+ paths: Vec<PathBuf>,+ /// Returns true if and only if `paths` had to be populated with a single+ /// default path.+ using_default_path: bool,+}++impl Args {+ /// Parse the command line arguments for this process.+ ///+ /// If a CLI usage error occurred, then exit the process and print a usage+ /// or error message. Similarly, if the user requested the version of+ /// ripgrep, then print the version and exit.+ ///+ /// Also, initialize a global logger.+ pub fn parse() -> Result<Args> {+ // We parse the args given on CLI. This does not include args from+ // the config. We use the CLI args as an initial configuration while+ // trying to parse config files. If a config file exists and has+ // arguments, then we re-parse argv, otherwise we just use the matches+ // we have here.+ let early_matches = ArgMatches::new(clap_matches(env::args_os())?);+ set_messages(!early_matches.is_present("no-messages"));+ set_ignore_messages(!early_matches.is_present("no-ignore-messages"));++ if let Err(err) = Logger::init() {+ return Err(format!("failed to initialize logger: {}", err).into());+ }+ if early_matches.is_present("trace") {+ log::set_max_level(log::LevelFilter::Trace);+ } else if early_matches.is_present("debug") {+ log::set_max_level(log::LevelFilter::Debug);+ } else {+ log::set_max_level(log::LevelFilter::Warn);+ }++ let matches = early_matches.reconfigure()?;+ // The logging level may have changed if we brought in additional+ // arguments from a configuration file, so recheck it and set the log+ // level as appropriate.+ if matches.is_present("trace") {+ log::set_max_level(log::LevelFilter::Trace);+ } else if matches.is_present("debug") {+ log::set_max_level(log::LevelFilter::Debug);+ } else {+ log::set_max_level(log::LevelFilter::Warn);+ }+ set_messages(!matches.is_present("no-messages"));+ set_ignore_messages(!matches.is_present("no-ignore-messages"));+ matches.to_args()+ }++ /// Return direct access to command line arguments.+ fn matches(&self) -> &ArgMatches {+ &self.0.matches+ }++ /// Return the patterns found in the command line arguments. This includes+ /// patterns read via the -f/--file flags.+ fn patterns(&self) -> &[String] {+ &self.0.patterns+ }++ /// Return the matcher builder from the patterns.+ fn matcher(&self) -> &PatternMatcher {+ &self.0.matcher+ }++ /// Return the paths found in the command line arguments. This is+ /// guaranteed to be non-empty. In the case where no explicit arguments are+ /// provided, a single default path is provided automatically.+ fn paths(&self) -> &[PathBuf] {+ &self.0.paths+ }++ /// Returns true if and only if `paths` had to be populated with a default+ /// path, which occurs only when no paths were given as command line+ /// arguments.+ fn using_default_path(&self) -> bool {+ self.0.using_default_path+ }++ /// Return the printer that should be used for formatting the output of+ /// search results.+ ///+ /// The returned printer will write results to the given writer.+ fn printer<W: WriteColor>(&self, wtr: W) -> Result<Printer<W>> {+ match self.matches().output_kind() {+ OutputKind::Standard => {+ let separator_search = self.command()? == Command::Search;+ self.matches()+ .printer_standard(self.paths(), wtr, separator_search)+ .map(Printer::Standard)+ }+ OutputKind::Summary => self+ .matches()+ .printer_summary(self.paths(), wtr)+ .map(Printer::Summary),+ OutputKind::JSON => {+ self.matches().printer_json(wtr).map(Printer::JSON)+ }+ }+ }+}++/// High level public routines for building data structures used by ripgrep+/// from command line arguments.+impl Args {+ /// Create a new buffer writer for multi-threaded printing with color+ /// support.+ pub fn buffer_writer(&self) -> Result<BufferWriter> {+ let mut wtr = BufferWriter::stdout(self.matches().color_choice());+ wtr.separator(self.matches().file_separator()?);+ Ok(wtr)+ }++ /// Return the high-level command that ripgrep should run.+ pub fn command(&self) -> Result<Command> {+ let is_one_search = self.matches().is_one_search(self.paths());+ let threads = self.matches().threads()?;+ let one_thread = is_one_search || threads == 1;++ Ok(if self.matches().is_present("pcre2-version") {+ Command::PCRE2Version+ } else if self.matches().is_present("type-list") {+ Command::Types+ } else if self.matches().is_present("files") {+ if one_thread {+ Command::Files+ } else {+ Command::FilesParallel+ }+ } else if self.matches().can_never_match(self.patterns()) {+ Command::SearchNever+ } else if one_thread {+ Command::Search+ } else {+ Command::SearchParallel+ })+ }++ /// Builder a path printer that can be used for printing just file paths,+ /// with optional color support.+ ///+ /// The printer will print paths to the given writer.+ pub fn path_printer<W: WriteColor>(+ &self,+ wtr: W,+ ) -> Result<PathPrinter<W>> {+ let mut builder = PathPrinterBuilder::new();+ builder+ .color_specs(self.matches().color_specs()?)+ .separator(self.matches().path_separator()?)+ .terminator(self.matches().path_terminator().unwrap_or(b'\n'));+ Ok(builder.build(wtr))+ }++ /// Returns true if and only if ripgrep should be "quiet."+ pub fn quiet(&self) -> bool {+ self.matches().is_present("quiet")+ }++ /// Returns true if and only if the search should quit after finding the+ /// first match.+ pub fn quit_after_match(&self) -> Result<bool> {+ Ok(self.matches().is_present("quiet") && self.stats()?.is_none())+ }++ /// Build a worker for executing searches.+ ///+ /// Search results are written to the given writer.+ pub fn search_worker<W: WriteColor>(+ &self,+ wtr: W,+ ) -> Result<SearchWorker<W>> {+ let matches = self.matches();+ let matcher = self.matcher().clone();+ let printer = self.printer(wtr)?;+ let searcher = matches.searcher(self.paths())?;+ let mut builder = SearchWorkerBuilder::new();+ builder+ .json_stats(matches.is_present("json"))+ .preprocessor(matches.preprocessor())+ .preprocessor_globs(matches.preprocessor_globs()?)+ .search_zip(matches.is_present("search-zip"))+ .binary_detection_implicit(matches.binary_detection_implicit())+ .binary_detection_explicit(matches.binary_detection_explicit());+ Ok(builder.build(matcher, searcher, printer))+ }++ /// Returns a zero value for tracking statistics if and only if it has been+ /// requested.+ ///+ /// When this returns a `Stats` value, then it is guaranteed that the+ /// search worker will be configured to track statistics as well.+ pub fn stats(&self) -> Result<Option<Stats>> {+ Ok(if self.command()?.is_search() && self.matches().stats() {+ Some(Stats::new())+ } else {+ None+ })+ }++ /// Return a builder for constructing subjects. A subject represents a+ /// single unit of something to search. Typically, this corresponds to a+ /// file or a stream such as stdin.+ pub fn subject_builder(&self) -> SubjectBuilder {+ let mut builder = SubjectBuilder::new();+ builder.strip_dot_prefix(self.using_default_path());+ builder+ }++ /// Execute the given function with a writer to stdout that enables color+ /// support based on the command line configuration.+ pub fn stdout(&self) -> cli::StandardStream {+ let color = self.matches().color_choice();+ if self.matches().is_present("line-buffered") {+ cli::stdout_buffered_line(color)+ } else if self.matches().is_present("block-buffered") {+ cli::stdout_buffered_block(color)+ } else {+ cli::stdout(color)+ }+ }++ /// Return the type definitions compiled into ripgrep.+ ///+ /// If there was a problem reading and parsing the type definitions, then+ /// this returns an error.+ pub fn type_defs(&self) -> Result<Vec<FileTypeDef>> {+ Ok(self.matches().types()?.definitions().to_vec())+ }++ /// Return a walker that never uses additional threads.+ pub fn walker(&self) -> Result<Walk> {+ Ok(self.matches().walker_builder(self.paths())?.build())+ }++ /// Return a walker that never uses additional threads.+ pub fn walker_parallel(&self) -> Result<WalkParallel> {+ Ok(self.matches().walker_builder(self.paths())?.build_parallel())+ }+}++/// `ArgMatches` wraps `clap::ArgMatches` and provides semantic meaning to+/// the parsed arguments.+#[derive(Clone, Debug)]+struct ArgMatches(clap::ArgMatches<'static>);++/// The output format. Generally, this corresponds to the printer that ripgrep+/// uses to show search results.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum OutputKind {+ /// Classic grep-like or ack-like format.+ Standard,+ /// Show matching files and possibly the number of matches in each file.+ Summary,+ /// Emit match information in the JSON Lines format.+ JSON,+}++/// The sort criteria, if present.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+struct SortBy {+ /// Whether to reverse the sort criteria (i.e., descending order).+ reverse: bool,+ /// The actual sorting criteria.+ kind: SortByKind,+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum SortByKind {+ /// No sorting at all.+ None,+ /// Sort by path.+ Path,+ /// Sort by last modified time.+ LastModified,+ /// Sort by last accessed time.+ LastAccessed,+ /// Sort by creation time.+ Created,+}++impl SortBy {+ fn asc(kind: SortByKind) -> SortBy {+ SortBy { reverse: false, kind: kind }+ }++ fn desc(kind: SortByKind) -> SortBy {+ SortBy { reverse: true, kind: kind }+ }++ fn none() -> SortBy {+ SortBy::asc(SortByKind::None)+ }++ /// Try to check that the sorting criteria selected is actually supported.+ /// If it isn't, then an error is returned.+ fn check(&self) -> Result<()> {+ match self.kind {+ SortByKind::None | SortByKind::Path => {}+ SortByKind::LastModified => {+ env::current_exe()?.metadata()?.modified()?;+ }+ SortByKind::LastAccessed => {+ env::current_exe()?.metadata()?.accessed()?;+ }+ SortByKind::Created => {+ env::current_exe()?.metadata()?.created()?;+ }+ }+ Ok(())+ }++ fn configure_walk_builder(self, builder: &mut WalkBuilder) {+ // This isn't entirely optimal. In particular, we will wind up issuing+ // a stat for many files redundantly. Aside from having potentially+ // inconsistent results with respect to sorting, this is also slow.+ // We could fix this here at the expense of memory by caching stat+ // calls. A better fix would be to find a way to push this down into+ // directory traversal itself, but that's a somewhat nasty change.+ match self.kind {+ SortByKind::None => {}+ SortByKind::Path => {+ if self.reverse {+ builder.sort_by_file_name(|a, b| a.cmp(b).reverse());+ } else {+ builder.sort_by_file_name(|a, b| a.cmp(b));+ }+ }+ SortByKind::LastModified => {+ builder.sort_by_file_path(move |a, b| {+ sort_by_metadata_time(a, b, self.reverse, |md| {+ md.modified()+ })+ });+ }+ SortByKind::LastAccessed => {+ builder.sort_by_file_path(move |a, b| {+ sort_by_metadata_time(a, b, self.reverse, |md| {+ md.accessed()+ })+ });+ }+ SortByKind::Created => {+ builder.sort_by_file_path(move |a, b| {+ sort_by_metadata_time(a, b, self.reverse, |md| {+ md.created()+ })+ });+ }+ }+ }+}++impl SortByKind {+ fn new(kind: &str) -> SortByKind {+ match kind {+ "none" => SortByKind::None,+ "path" => SortByKind::Path,+ "modified" => SortByKind::LastModified,+ "accessed" => SortByKind::LastAccessed,+ "created" => SortByKind::Created,+ _ => SortByKind::None,+ }+ }+}++/// Encoding mode the searcher will use.+#[derive(Clone, Debug)]+enum EncodingMode {+ /// Use an explicit encoding forcefully, but let BOM sniffing override it.+ Some(Encoding),+ /// Use only BOM sniffing to auto-detect an encoding.+ Auto,+ /// Use no explicit encoding and disable all BOM sniffing. This will+ /// always result in searching the raw bytes, regardless of their+ /// true encoding.+ Disabled,+}++impl EncodingMode {+ /// Checks if an explicit encoding has been set. Returns false for+ /// automatic BOM sniffing and no sniffing.+ ///+ /// This is only used to determine whether PCRE2 needs to have its own+ /// UTF-8 checking enabled. If we have an explicit encoding set, then+ /// we're always guaranteed to get UTF-8, so we can disable PCRE2's check.+ /// Otherwise, we have no such guarantee, and must enable PCRE2' UTF-8+ /// check.+ #[cfg(feature = "pcre2")]+ fn has_explicit_encoding(&self) -> bool {+ match self {+ EncodingMode::Some(_) => true,+ _ => false,+ }+ }+}++impl ArgMatches {+ /// Create an ArgMatches from clap's parse result.+ fn new(clap_matches: clap::ArgMatches<'static>) -> ArgMatches {+ ArgMatches(clap_matches)+ }++ /// Run clap and return the matches using a config file if present. If clap+ /// determines a problem with the user provided arguments (or if --help or+ /// --version are given), then an error/usage/version will be printed and+ /// the process will exit.+ ///+ /// If there are no additional arguments from the environment (e.g., a+ /// config file), then the given matches are returned as is.+ fn reconfigure(self) -> Result<ArgMatches> {+ // If the end user says no config, then respect it.+ if self.is_present("no-config") {+ log::debug!(+ "not reading config files because --no-config is present"+ );+ return Ok(self);+ }+ // If the user wants ripgrep to use a config file, then parse args+ // from that first.+ let mut args = config::args();+ if args.is_empty() {+ return Ok(self);+ }+ let mut cliargs = env::args_os();+ if let Some(bin) = cliargs.next() {+ args.insert(0, bin);+ }+ args.extend(cliargs);+ log::debug!("final argv: {:?}", args);+ Ok(ArgMatches(clap_matches(args)?))+ }++ /// Convert the result of parsing CLI arguments into ripgrep's higher level+ /// configuration structure.+ fn to_args(self) -> Result<Args> {+ // We compute these once since they could be large.+ let patterns = self.patterns()?;+ let matcher = self.matcher(&patterns)?;+ let mut paths = self.paths();+ let using_default_path = if paths.is_empty() {+ paths.push(self.path_default());+ true+ } else {+ false+ };+ Ok(Args(Arc::new(ArgsImp {+ matches: self,+ patterns: patterns,+ matcher: matcher,+ paths: paths,+ using_default_path: using_default_path,+ })))+ }+}++/// High level routines for converting command line arguments into various+/// data structures used by ripgrep.+///+/// Methods are sorted alphabetically.+impl ArgMatches {+ /// Return the matcher that should be used for searching.+ ///+ /// If there was a problem building the matcher (e.g., a syntax error),+ /// then this returns an error.+ #[cfg(feature = "pcre2")]+ fn matcher(&self, patterns: &[String]) -> Result<PatternMatcher> {+ if self.is_present("pcre2") {+ let matcher = self.matcher_pcre2(patterns)?;+ Ok(PatternMatcher::PCRE2(matcher))+ } else if self.is_present("auto-hybrid-regex") {+ let rust_err = match self.matcher_rust(patterns) {+ Ok(matcher) => return Ok(PatternMatcher::RustRegex(matcher)),+ Err(err) => err,+ };+ log::debug!(+ "error building Rust regex in hybrid mode:\n{}",+ rust_err,+ );+ let pcre_err = match self.matcher_pcre2(patterns) {+ Ok(matcher) => return Ok(PatternMatcher::PCRE2(matcher)),+ Err(err) => err,+ };+ Err(From::from(format!(+ "regex could not be compiled with either the default regex \+ engine or with PCRE2.\n\n\+ default regex engine error:\n{}\n{}\n{}\n\n\+ PCRE2 regex engine error:\n{}",+ "~".repeat(79),+ rust_err,+ "~".repeat(79),+ pcre_err,+ )))+ } else {+ let matcher = match self.matcher_rust(patterns) {+ Ok(matcher) => matcher,+ Err(err) => {+ return Err(From::from(suggest_pcre2(err.to_string())));+ }+ };+ Ok(PatternMatcher::RustRegex(matcher))+ }+ }++ /// Return the matcher that should be used for searching.+ ///+ /// If there was a problem building the matcher (e.g., a syntax error),+ /// then this returns an error.+ #[cfg(not(feature = "pcre2"))]+ fn matcher(&self, patterns: &[String]) -> Result<PatternMatcher> {+ if self.is_present("pcre2") {+ return Err(From::from(+ "PCRE2 is not available in this build of ripgrep",+ ));+ }+ let matcher = self.matcher_rust(patterns)?;+ Ok(PatternMatcher::RustRegex(matcher))+ }++ /// Build a matcher using Rust's regex engine.+ ///+ /// If there was a problem building the matcher (such as a regex syntax+ /// error), then an error is returned.+ fn matcher_rust(&self, patterns: &[String]) -> Result<RustRegexMatcher> {+ let mut builder = RustRegexMatcherBuilder::new();+ builder+ .case_smart(self.case_smart())+ .case_insensitive(self.case_insensitive())+ .multi_line(true)+ .unicode(self.unicode())+ .octal(false)+ .word(self.is_present("word-regexp"));+ if self.is_present("multiline") {+ builder.dot_matches_new_line(self.is_present("multiline-dotall"));+ if self.is_present("crlf") {+ builder.crlf(true).line_terminator(None);+ }+ } else {+ builder.line_terminator(Some(b'\n')).dot_matches_new_line(false);+ if self.is_present("crlf") {+ builder.crlf(true);+ }+ // We don't need to set this in multiline mode since mulitline+ // matchers don't use optimizations related to line terminators.+ // Moreover, a mulitline regex used with --null-data should+ // be allowed to match NUL bytes explicitly, which this would+ // otherwise forbid.+ if self.is_present("null-data") {+ builder.line_terminator(Some(b'\x00'));+ }+ }+ if let Some(limit) = self.regex_size_limit()? {+ builder.size_limit(limit);+ }+ if let Some(limit) = self.dfa_size_limit()? {+ builder.dfa_size_limit(limit);+ }+ let res = if self.is_present("fixed-strings") {+ builder.build_literals(patterns)+ } else {+ builder.build(&patterns.join("|"))+ };+ match res {+ Ok(m) => Ok(m),+ Err(err) => Err(From::from(suggest_multiline(err.to_string()))),+ }+ }++ /// Build a matcher using PCRE2.+ ///+ /// If there was a problem building the matcher (such as a regex syntax+ /// error), then an error is returned.+ #[cfg(feature = "pcre2")]+ fn matcher_pcre2(&self, patterns: &[String]) -> Result<PCRE2RegexMatcher> {+ let mut builder = PCRE2RegexMatcherBuilder::new();+ builder+ .case_smart(self.case_smart())+ .caseless(self.case_insensitive())+ .multi_line(true)+ .word(self.is_present("word-regexp"));+ // For whatever reason, the JIT craps out during regex compilation with+ // a "no more memory" error on 32 bit systems. So don't use it there.+ if cfg!(target_pointer_width = "64") {+ builder+ .jit_if_available(true)+ // The PCRE2 docs say that 32KB is the default, and that 1MB+ // should be big enough for anything. But let's crank it to+ // 10MB.+ .max_jit_stack_size(Some(10 * (1 << 20)));+ }+ if self.unicode() {+ builder.utf(true).ucp(true);+ if self.encoding()?.has_explicit_encoding() {+ // SAFETY: If an encoding was specified, then we're guaranteed+ // to get valid UTF-8, so we can disable PCRE2's UTF checking.+ // (Feeding invalid UTF-8 to PCRE2 is undefined behavior.)+ unsafe {+ builder.disable_utf_check();+ }+ }+ }+ if self.is_present("multiline") {+ builder.dotall(self.is_present("multiline-dotall"));+ }+ if self.is_present("crlf") {+ builder.crlf(true);+ }+ Ok(builder.build(&patterns.join("|"))?)+ }++ /// Build a JSON printer that writes results to the given writer.+ fn printer_json<W: io::Write>(&self, wtr: W) -> Result<JSON<W>> {+ let mut builder = JSONBuilder::new();+ builder+ .pretty(false)+ .max_matches(self.max_count()?)+ .always_begin_end(false);+ Ok(builder.build(wtr))+ }++ /// Build a Standard printer that writes results to the given writer.+ ///+ /// The given paths are used to configure aspects of the printer.+ ///+ /// If `separator_search` is true, then the returned printer will assume+ /// the responsibility of printing a separator between each set of+ /// search results, when appropriate (e.g., when contexts are enabled).+ /// When it's set to false, the caller is responsible for handling+ /// separators.+ ///+ /// In practice, we want the printer to handle it in the single threaded+ /// case but not in the multi-threaded case.+ fn printer_standard<W: WriteColor>(+ &self,+ paths: &[PathBuf],+ wtr: W,+ separator_search: bool,+ ) -> Result<Standard<W>> {+ let mut builder = StandardBuilder::new();+ builder+ .color_specs(self.color_specs()?)+ .stats(self.stats())+ .heading(self.heading())+ .path(self.with_filename(paths))+ .only_matching(self.is_present("only-matching"))+ .per_match(self.is_present("vimgrep"))+ .replacement(self.replacement())+ .max_columns(self.max_columns()?)+ .max_columns_preview(self.max_columns_preview())+ .max_matches(self.max_count()?)+ .column(self.column())+ .byte_offset(self.is_present("byte-offset"))+ .trim_ascii(self.is_present("trim"))+ .separator_search(None)+ .separator_context(self.context_separator())+ .separator_field_match(b":".to_vec())+ .separator_field_context(b"-".to_vec())+ .separator_path(self.path_separator()?)+ .path_terminator(self.path_terminator());+ if separator_search {+ builder.separator_search(self.file_separator()?);+ }+ Ok(builder.build(wtr))+ }++ /// Build a Summary printer that writes results to the given writer.+ ///+ /// The given paths are used to configure aspects of the printer.+ ///+ /// This panics if the output format is not `OutputKind::Summary`.+ fn printer_summary<W: WriteColor>(+ &self,+ paths: &[PathBuf],+ wtr: W,+ ) -> Result<Summary<W>> {+ let mut builder = SummaryBuilder::new();+ builder+ .kind(self.summary_kind().expect("summary format"))+ .color_specs(self.color_specs()?)+ .stats(self.stats())+ .path(self.with_filename(paths))+ .max_matches(self.max_count()?)+ .exclude_zero(!self.is_present("include-zero"))+ .separator_field(b":".to_vec())+ .separator_path(self.path_separator()?)+ .path_terminator(self.path_terminator());+ Ok(builder.build(wtr))+ }++ /// Build a searcher from the command line parameters.+ fn searcher(&self, paths: &[PathBuf]) -> Result<Searcher> {+ let (ctx_before, ctx_after) = self.contexts()?;+ let line_term = if self.is_present("crlf") {+ LineTerminator::crlf()+ } else if self.is_present("null-data") {+ LineTerminator::byte(b'\x00')+ } else {+ LineTerminator::byte(b'\n')+ };+ let mut builder = SearcherBuilder::new();+ builder+ .line_terminator(line_term)+ .invert_match(self.is_present("invert-match"))+ .line_number(self.line_number(paths))+ .multi_line(self.is_present("multiline"))+ .before_context(ctx_before)+ .after_context(ctx_after)+ .passthru(self.is_present("passthru"))+ .memory_map(self.mmap_choice(paths));+ match self.encoding()? {+ EncodingMode::Some(enc) => {+ builder.encoding(Some(enc));+ }+ EncodingMode::Auto => {} // default for the searcher+ EncodingMode::Disabled => {+ builder.bom_sniffing(false);+ }+ }+ Ok(builder.build())+ }++ /// Return a builder for recursively traversing a directory while+ /// respecting ignore rules.+ ///+ /// If there was a problem parsing the CLI arguments necessary for+ /// constructing the builder, then this returns an error.+ fn walker_builder(&self, paths: &[PathBuf]) -> Result<WalkBuilder> {+ let mut builder = WalkBuilder::new(&paths[0]);+ for path in &paths[1..] {+ builder.add(path);+ }+ for path in self.ignore_paths() {+ if let Some(err) = builder.add_ignore(path) {+ ignore_message!("{}", err);+ }+ }+ builder+ .max_depth(self.usize_of("max-depth")?)+ .follow_links(self.is_present("follow"))+ .max_filesize(self.max_file_size()?)+ .threads(self.threads()?)+ .same_file_system(self.is_present("one-file-system"))+ .skip_stdout(!self.is_present("files"))+ .overrides(self.overrides()?)+ .types(self.types()?)+ .hidden(!self.hidden())+ .parents(!self.no_ignore_parent())+ .ignore(!self.no_ignore_dot())+ .git_global(!self.no_ignore_vcs() && !self.no_ignore_global())+ .git_ignore(!self.no_ignore_vcs())+ .git_exclude(!self.no_ignore_vcs() && !self.no_ignore_exclude())+ .require_git(!self.is_present("no-require-git"))+ .ignore_case_insensitive(self.ignore_file_case_insensitive());+ if !self.no_ignore() {+ builder.add_custom_ignore_filename(".rgignore");+ }+ let sortby = self.sort_by()?;+ sortby.check()?;+ sortby.configure_walk_builder(&mut builder);+ Ok(builder)+ }+}++/// Mid level routines for converting command line arguments into various types+/// of data structures.+///+/// Methods are sorted alphabetically.+impl ArgMatches {+ /// Returns the form of binary detection to perform on files that are+ /// implicitly searched via recursive directory traversal.+ fn binary_detection_implicit(&self) -> BinaryDetection {+ let none = self.is_present("text") || self.is_present("null-data");+ let convert =+ self.is_present("binary") || self.unrestricted_count() >= 3;+ if none {+ BinaryDetection::none()+ } else if convert {+ BinaryDetection::convert(b'\x00')+ } else {+ BinaryDetection::quit(b'\x00')+ }+ }++ /// Returns the form of binary detection to perform on files that are+ /// explicitly searched via the user invoking ripgrep on a particular+ /// file or files or stdin.+ ///+ /// In general, this should never be BinaryDetection::quit, since that acts+ /// as a filter (but quitting immediately once a NUL byte is seen), and we+ /// should never filter out files that the user wants to explicitly search.+ fn binary_detection_explicit(&self) -> BinaryDetection {+ let none = self.is_present("text") || self.is_present("null-data");+ if none {+ BinaryDetection::none()+ } else {+ BinaryDetection::convert(b'\x00')+ }+ }++ /// Returns true if the command line configuration implies that a match+ /// can never be shown.+ fn can_never_match(&self, patterns: &[String]) -> bool {+ patterns.is_empty() || self.max_count().ok() == Some(Some(0))+ }++ /// Returns true if and only if case should be ignore.+ ///+ /// If --case-sensitive is present, then case is never ignored, even if+ /// --ignore-case is present.+ fn case_insensitive(&self) -> bool {+ self.is_present("ignore-case") && !self.is_present("case-sensitive")+ }++ /// Returns true if and only if smart case has been enabled.+ ///+ /// If either --ignore-case of --case-sensitive are present, then smart+ /// case is disabled.+ fn case_smart(&self) -> bool {+ self.is_present("smart-case")+ && !self.is_present("ignore-case")+ && !self.is_present("case-sensitive")+ }++ /// Returns the user's color choice based on command line parameters and+ /// environment.+ fn color_choice(&self) -> ColorChoice {+ let preference = match self.value_of_lossy("color") {+ None => "auto".to_string(),+ Some(v) => v,+ };+ if preference == "always" {+ ColorChoice::Always+ } else if preference == "ansi" {+ ColorChoice::AlwaysAnsi+ } else if preference == "auto" {+ if cli::is_tty_stdout() || self.is_present("pretty") {+ ColorChoice::Auto+ } else {+ ColorChoice::Never+ }+ } else {+ ColorChoice::Never+ }+ }++ /// Returns the color specifications given by the user on the CLI.+ ///+ /// If the was a problem parsing any of the provided specs, then an error+ /// is returned.+ fn color_specs(&self) -> Result<ColorSpecs> {+ // Start with a default set of color specs.+ let mut specs = default_color_specs();+ for spec_str in self.values_of_lossy_vec("colors") {+ specs.push(spec_str.parse()?);+ }+ Ok(ColorSpecs::new(&specs))+ }++ /// Returns true if and only if column numbers should be shown.+ fn column(&self) -> bool {+ if self.is_present("no-column") {+ return false;+ }+ self.is_present("column") || self.is_present("vimgrep")+ }++ /// Returns the before and after contexts from the command line.+ ///+ /// If a context setting was absent, then `0` is returned.+ ///+ /// If there was a problem parsing the values from the user as an integer,+ /// then an error is returned.+ fn contexts(&self) -> Result<(usize, usize)> {+ let after = self.usize_of("after-context")?.unwrap_or(0);+ let before = self.usize_of("before-context")?.unwrap_or(0);+ let both = self.usize_of("context")?.unwrap_or(0);+ Ok(if both > 0 { (both, both) } else { (before, after) })+ }++ /// Returns the unescaped context separator in UTF-8 bytes.+ ///+ /// If one was not provided, the default `--` is returned.+ /// If --no-context-separator is passed, None is returned.+ fn context_separator(&self) -> Option<Vec<u8>> {+ let nosep = self.is_present("no-context-separator");+ let sep = self.value_of_os("context-separator");+ match (nosep, sep) {+ (true, _) => None,+ (false, None) => Some(b"--".to_vec()),+ (false, Some(sep)) => Some(cli::unescape_os(&sep)),+ }+ }++ /// Returns whether the -c/--count or the --count-matches flags were+ /// passed from the command line.+ ///+ /// If --count-matches and --invert-match were passed in, behave+ /// as if --count and --invert-match were passed in (i.e. rg will+ /// count inverted matches as per existing behavior).+ fn counts(&self) -> (bool, bool) {+ let count = self.is_present("count");+ let count_matches = self.is_present("count-matches");+ let invert_matches = self.is_present("invert-match");+ let only_matching = self.is_present("only-matching");+ if count_matches && invert_matches {+ // Treat `-v --count-matches` as `-v -c`.+ (true, false)+ } else if count && only_matching {+ // Treat `-c --only-matching` as `--count-matches`.+ (false, true)+ } else {+ (count, count_matches)+ }+ }++ /// Parse the dfa-size-limit argument option into a byte count.+ fn dfa_size_limit(&self) -> Result<Option<usize>> {+ let r = self.parse_human_readable_size("dfa-size-limit")?;+ u64_to_usize("dfa-size-limit", r)+ }++ /// Returns the encoding mode to use.+ ///+ /// This only returns an encoding if one is explicitly specified. Otherwise+ /// if set to automatic, the Searcher will do BOM sniffing for UTF-16+ /// and transcode seamlessly. If disabled, no BOM sniffing nor transcoding+ /// will occur.+ fn encoding(&self) -> Result<EncodingMode> {+ if self.is_present("no-encoding") {+ return Ok(EncodingMode::Auto);+ }++ let label = match self.value_of_lossy("encoding") {+ None if self.pcre2_unicode() => "utf-8".to_string(),+ None => return Ok(EncodingMode::Auto),+ Some(label) => label,+ };++ if label == "auto" {+ return Ok(EncodingMode::Auto);+ } else if label == "none" {+ return Ok(EncodingMode::Disabled);+ }++ Ok(EncodingMode::Some(Encoding::new(&label)?))+ }++ /// Return the file separator to use based on the CLI configuration.+ fn file_separator(&self) -> Result<Option<Vec<u8>>> {+ // File separators are only used for the standard grep-line format.+ if self.output_kind() != OutputKind::Standard {+ return Ok(None);+ }++ let (ctx_before, ctx_after) = self.contexts()?;+ Ok(if self.heading() {+ Some(b"".to_vec())+ } else if ctx_before > 0 || ctx_after > 0 {+ self.context_separator()+ } else {+ None+ })+ }++ /// Returns true if and only if matches should be grouped with file name+ /// headings.+ fn heading(&self) -> bool {+ if self.is_present("no-heading") || self.is_present("vimgrep") {+ false+ } else {+ cli::is_tty_stdout()+ || self.is_present("heading")+ || self.is_present("pretty")+ }+ }++ /// Returns true if and only if hidden files/directories should be+ /// searched.+ fn hidden(&self) -> bool {+ self.is_present("hidden") || self.unrestricted_count() >= 2+ }++ /// Returns true if ignore files should be processed case insensitively.+ fn ignore_file_case_insensitive(&self) -> bool {+ self.is_present("ignore-file-case-insensitive")+ }++ /// Return all of the ignore file paths given on the command line.+ fn ignore_paths(&self) -> Vec<PathBuf> {+ let paths = match self.values_of_os("ignore-file") {+ None => return vec![],+ Some(paths) => paths,+ };+ paths.map(|p| Path::new(p).to_path_buf()).collect()+ }++ /// Returns true if and only if ripgrep is invoked in a way where it knows+ /// it search exactly one thing.+ fn is_one_search(&self, paths: &[PathBuf]) -> bool {+ if paths.len() != 1 {+ return false;+ }+ self.is_only_stdin(paths) || paths[0].is_file()+ }++ /// Returns true if and only if we're only searching a single thing and+ /// that thing is stdin.+ fn is_only_stdin(&self, paths: &[PathBuf]) -> bool {+ paths == [Path::new("-")]+ }++ /// Returns true if and only if we should show line numbers.+ fn line_number(&self, paths: &[PathBuf]) -> bool {+ if self.output_kind() == OutputKind::Summary {+ return false;+ }+ if self.is_present("no-line-number") {+ return false;+ }+ if self.output_kind() == OutputKind::JSON {+ return true;+ }++ // A few things can imply counting line numbers. In particular, we+ // generally want to show line numbers by default when printing to a+ // tty for human consumption, except for one interesting case: when+ // we're only searching stdin. This makes pipelines work as expected.+ (cli::is_tty_stdout() && !self.is_only_stdin(paths))+ || self.is_present("line-number")+ || self.is_present("column")+ || self.is_present("pretty")+ || self.is_present("vimgrep")+ }++ /// The maximum number of columns allowed on each line.+ ///+ /// If `0` is provided, then this returns `None`.+ fn max_columns(&self) -> Result<Option<u64>> {+ Ok(self.usize_of_nonzero("max-columns")?.map(|n| n as u64))+ }++ /// Returns true if and only if a preview should be shown for lines that+ /// exceed the maximum column limit.+ fn max_columns_preview(&self) -> bool {+ self.is_present("max-columns-preview")+ }++ /// The maximum number of matches permitted.+ fn max_count(&self) -> Result<Option<u64>> {+ Ok(self.usize_of("max-count")?.map(|n| n as u64))+ }++ /// Parses the max-filesize argument option into a byte count.+ fn max_file_size(&self) -> Result<Option<u64>> {+ self.parse_human_readable_size("max-filesize")+ }++ /// Returns whether we should attempt to use memory maps or not.+ fn mmap_choice(&self, paths: &[PathBuf]) -> MmapChoice {+ // SAFETY: Memory maps are difficult to impossible to encapsulate+ // safely in a portable way that doesn't simultaneously negate some of+ // the benfits of using memory maps. For ripgrep's use, we never mutate+ // a memory map and generally never store the contents of memory map+ // in a data structure that depends on immutability. Generally+ // speaking, the worst thing that can happen is a SIGBUS (if the+ // underlying file is truncated while reading it), which will cause+ // ripgrep to abort. This reasoning should be treated as suspect.+ let maybe = unsafe { MmapChoice::auto() };+ let never = MmapChoice::never();+ if self.is_present("no-mmap") {+ never+ } else if self.is_present("mmap") {+ maybe+ } else if paths.len() <= 10 && paths.iter().all(|p| p.is_file()) {+ // If we're only searching a few paths and all of them are+ // files, then memory maps are probably faster.+ maybe+ } else {+ never+ }+ }++ /// Returns true if ignore files should be ignored.+ fn no_ignore(&self) -> bool {+ self.is_present("no-ignore") || self.unrestricted_count() >= 1+ }++ /// Returns true if .ignore files should be ignored.+ fn no_ignore_dot(&self) -> bool {+ self.is_present("no-ignore-dot") || self.no_ignore()+ }++ /// Returns true if local exclude (ignore) files should be ignored.+ fn no_ignore_exclude(&self) -> bool {+ self.is_present("no-ignore-exclude") || self.no_ignore()+ }++ /// Returns true if global ignore files should be ignored.+ fn no_ignore_global(&self) -> bool {+ self.is_present("no-ignore-global") || self.no_ignore()+ }++ /// Returns true if parent ignore files should be ignored.+ fn no_ignore_parent(&self) -> bool {+ self.is_present("no-ignore-parent") || self.no_ignore()+ }++ /// Returns true if VCS ignore files should be ignored.+ fn no_ignore_vcs(&self) -> bool {+ self.is_present("no-ignore-vcs") || self.no_ignore()+ }++ /// Determine the type of output we should produce.+ fn output_kind(&self) -> OutputKind {+ if self.is_present("quiet") {+ // While we don't technically print results (or aggregate results)+ // in quiet mode, we still support the --stats flag, and those+ // stats are computed by the Summary printer for now.+ return OutputKind::Summary;+ } else if self.is_present("json") {+ return OutputKind::JSON;+ }++ let (count, count_matches) = self.counts();+ let summary = count+ || count_matches+ || self.is_present("files-with-matches")+ || self.is_present("files-without-match");+ if summary {+ OutputKind::Summary+ } else {+ OutputKind::Standard+ }+ }++ /// Builds the set of glob overrides from the command line flags.+ fn overrides(&self) -> Result<Override> {+ let globs = self.values_of_lossy_vec("glob");+ let iglobs = self.values_of_lossy_vec("iglob");+ if globs.is_empty() && iglobs.is_empty() {+ return Ok(Override::empty());+ }++ let mut builder = OverrideBuilder::new(current_dir()?);+ // Make all globs case insensitive with --glob-case-insensitive.+ if self.is_present("glob-case-insensitive") {+ builder.case_insensitive(true).unwrap();+ }+ for glob in globs {+ builder.add(&glob)?;+ }+ // This only enables case insensitivity for subsequent globs.+ builder.case_insensitive(true).unwrap();+ for glob in iglobs {+ builder.add(&glob)?;+ }+ Ok(builder.build()?)+ }++ /// Return all file paths that ripgrep should search.+ ///+ /// If no paths were given, then this returns an empty list.+ fn paths(&self) -> Vec<PathBuf> {+ let mut paths: Vec<PathBuf> = match self.values_of_os("path") {+ None => vec![],+ Some(paths) => paths.map(|p| Path::new(p).to_path_buf()).collect(),+ };+ // If --file, --files or --regexp is given, then the first path is+ // always in `pattern`.+ if self.is_present("file")+ || self.is_present("files")+ || self.is_present("regexp")+ {+ if let Some(path) = self.value_of_os("pattern") {+ paths.insert(0, Path::new(path).to_path_buf());+ }+ }+ paths+ }++ /// Return the default path that ripgrep should search. This should only+ /// be used when ripgrep is not otherwise given at least one file path+ /// as a positional argument.+ fn path_default(&self) -> PathBuf {+ let file_is_stdin = self+ .values_of_os("file")+ .map_or(false, |mut files| files.any(|f| f == "-"));+ let search_cwd = !cli::is_readable_stdin()+ || (self.is_present("file") && file_is_stdin)+ || self.is_present("files")+ || self.is_present("type-list")+ || self.is_present("pcre2-version");+ if search_cwd {+ Path::new("./").to_path_buf()+ } else {+ Path::new("-").to_path_buf()+ }+ }++ /// Returns the unescaped path separator as a single byte, if one exists.+ ///+ /// If the provided path separator is more than a single byte, then an+ /// error is returned.+ fn path_separator(&self) -> Result<Option<u8>> {+ let sep = match self.value_of_os("path-separator") {+ None => return Ok(None),+ Some(sep) => cli::unescape_os(&sep),+ };+ if sep.is_empty() {+ Ok(None)+ } else if sep.len() > 1 {+ Err(From::from(format!(+ "A path separator must be exactly one byte, but \+ the given separator is {} bytes: {}\n\+ In some shells on Windows '/' is automatically \+ expanded. Use '//' instead.",+ sep.len(),+ cli::escape(&sep),+ )))+ } else {+ Ok(Some(sep[0]))+ }+ }++ /// Returns the byte that should be used to terminate paths.+ ///+ /// Typically, this is only set to `\x00` when the --null flag is provided,+ /// and `None` otherwise.+ fn path_terminator(&self) -> Option<u8> {+ if self.is_present("null") {+ Some(b'\x00')+ } else {+ None+ }+ }++ /// Get a sequence of all available patterns from the command line.+ /// This includes reading the -e/--regexp and -f/--file flags.+ ///+ /// Note that if -F/--fixed-strings is set, then all patterns will be+ /// escaped. If -x/--line-regexp is set, then all patterns are surrounded+ /// by `^...$`. Other things, such as --word-regexp, are handled by the+ /// regex matcher itself.+ ///+ /// If any pattern is invalid UTF-8, then an error is returned.+ fn patterns(&self) -> Result<Vec<String>> {+ if self.is_present("files") || self.is_present("type-list") {+ return Ok(vec![]);+ }+ let mut pats = vec![];+ match self.values_of_os("regexp") {+ None => {+ if self.values_of_os("file").is_none() {+ if let Some(os_pat) = self.value_of_os("pattern") {+ pats.push(self.pattern_from_os_str(os_pat)?);+ }+ }+ }+ Some(os_pats) => {+ for os_pat in os_pats {+ pats.push(self.pattern_from_os_str(os_pat)?);+ }+ }+ }+ if let Some(paths) = self.values_of_os("file") {+ for path in paths {+ if path == "-" {+ pats.extend(+ cli::patterns_from_stdin()?+ .into_iter()+ .map(|p| self.pattern_from_string(p)),+ );+ } else {+ pats.extend(+ cli::patterns_from_path(path)?+ .into_iter()+ .map(|p| self.pattern_from_string(p)),+ );+ }+ }+ }+ Ok(pats)+ }++ /// Returns a pattern that is guaranteed to produce an empty regular+ /// expression that is valid in any position.+ fn pattern_empty(&self) -> String {+ // This would normally just be an empty string, which works on its+ // own, but if the patterns are joined in a set of alternations, then+ // you wind up with `foo|`, which is currently invalid in Rust's regex+ // engine.+ "(?:z{0})*".to_string()+ }++ /// Converts an OsStr pattern to a String pattern. The pattern is escaped+ /// if -F/--fixed-strings is set.+ ///+ /// If the pattern is not valid UTF-8, then an error is returned.+ fn pattern_from_os_str(&self, pat: &OsStr) -> Result<String> {+ let s = cli::pattern_from_os(pat)?;+ Ok(self.pattern_from_str(s))+ }++ /// Converts a &str pattern to a String pattern. The pattern is escaped+ /// if -F/--fixed-strings is set.+ fn pattern_from_str(&self, pat: &str) -> String {+ self.pattern_from_string(pat.to_string())+ }++ /// Applies additional processing on the given pattern if necessary+ /// (such as escaping meta characters or turning it into a line regex).+ fn pattern_from_string(&self, pat: String) -> String {+ let pat = self.pattern_line(self.pattern_literal(pat));+ if pat.is_empty() {+ self.pattern_empty()+ } else {+ pat+ }+ }++ /// Returns the given pattern as a line pattern if the -x/--line-regexp+ /// flag is set. Otherwise, the pattern is returned unchanged.+ fn pattern_line(&self, pat: String) -> String {+ if self.is_present("line-regexp") {+ format!(r"^(?:{})$", pat)+ } else {+ pat+ }+ }++ /// Returns the given pattern as a literal pattern if the+ /// -F/--fixed-strings flag is set. Otherwise, the pattern is returned+ /// unchanged.+ fn pattern_literal(&self, pat: String) -> String {+ if self.is_present("fixed-strings") {+ regex::escape(&pat)+ } else {+ pat+ }+ }++ /// Returns the preprocessor command if one was specified.+ fn preprocessor(&self) -> Option<PathBuf> {+ let path = match self.value_of_os("pre") {+ None => return None,+ Some(path) => path,+ };+ if path.is_empty() {+ return None;+ }+ Some(Path::new(path).to_path_buf())+ }++ /// Builds the set of globs for filtering files to apply to the --pre+ /// flag. If no --pre-globs are available, then this always returns an+ /// empty set of globs.+ fn preprocessor_globs(&self) -> Result<Override> {+ let globs = self.values_of_lossy_vec("pre-glob");+ if globs.is_empty() {+ return Ok(Override::empty());+ }+ let mut builder = OverrideBuilder::new(current_dir()?);+ for glob in globs {+ builder.add(&glob)?;+ }+ Ok(builder.build()?)+ }++ /// Parse the regex-size-limit argument option into a byte count.+ fn regex_size_limit(&self) -> Result<Option<usize>> {+ let r = self.parse_human_readable_size("regex-size-limit")?;+ u64_to_usize("regex-size-limit", r)+ }++ /// Returns the replacement string as UTF-8 bytes if it exists.+ fn replacement(&self) -> Option<Vec<u8>> {+ self.value_of_lossy("replace").map(|s| s.into_bytes())+ }++ /// Returns the sorting criteria based on command line parameters.+ fn sort_by(&self) -> Result<SortBy> {+ // For backcompat, continue supporting deprecated --sort-files flag.+ if self.is_present("sort-files") {+ return Ok(SortBy::asc(SortByKind::Path));+ }+ let sortby = match self.value_of_lossy("sort") {+ None => match self.value_of_lossy("sortr") {+ None => return Ok(SortBy::none()),+ Some(choice) => SortBy::desc(SortByKind::new(&choice)),+ },+ Some(choice) => SortBy::asc(SortByKind::new(&choice)),+ };+ Ok(sortby)+ }++ /// Returns true if and only if aggregate statistics for a search should+ /// be tracked.+ ///+ /// Generally, this is only enabled when explicitly requested by in the+ /// command line arguments via the --stats flag, but this can also be+ /// enabled implicity via the output format, e.g., for JSON Lines.+ fn stats(&self) -> bool {+ self.output_kind() == OutputKind::JSON || self.is_present("stats")+ }++ /// When the output format is `Summary`, this returns the type of summary+ /// output to show.+ ///+ /// This returns `None` if the output format is not `Summary`.+ fn summary_kind(&self) -> Option<SummaryKind> {+ let (count, count_matches) = self.counts();+ if self.is_present("quiet") {+ Some(SummaryKind::Quiet)+ } else if count_matches {+ Some(SummaryKind::CountMatches)+ } else if count {+ Some(SummaryKind::Count)+ } else if self.is_present("files-with-matches") {+ Some(SummaryKind::PathWithMatch)+ } else if self.is_present("files-without-match") {+ Some(SummaryKind::PathWithoutMatch)+ } else {+ None+ }+ }++ /// Return the number of threads that should be used for parallelism.+ fn threads(&self) -> Result<usize> {+ if self.sort_by()?.kind != SortByKind::None {+ return Ok(1);+ }+ let threads = self.usize_of("threads")?.unwrap_or(0);+ Ok(if threads == 0 { cmp::min(12, num_cpus::get()) } else { threads })+ }++ /// Builds a file type matcher from the command line flags.+ fn types(&self) -> Result<Types> {+ let mut builder = TypesBuilder::new();+ builder.add_defaults();+ for ty in self.values_of_lossy_vec("type-clear") {+ builder.clear(&ty);+ }+ for def in self.values_of_lossy_vec("type-add") {+ builder.add_def(&def)?;+ }+ for ty in self.values_of_lossy_vec("type") {+ builder.select(&ty);+ }+ for ty in self.values_of_lossy_vec("type-not") {+ builder.negate(&ty);+ }+ builder.build().map_err(From::from)+ }++ /// Returns the number of times the `unrestricted` flag is provided.+ fn unrestricted_count(&self) -> u64 {+ self.occurrences_of("unrestricted")+ }++ /// Returns true if and only if Unicode mode should be enabled.+ fn unicode(&self) -> bool {+ // Unicode mode is enabled by default, so only disable it when+ // --no-unicode is given explicitly.+ !(self.is_present("no-unicode") || self.is_present("no-pcre2-unicode"))+ }++ /// Returns true if and only if PCRE2 is enabled and its Unicode mode is+ /// enabled.+ fn pcre2_unicode(&self) -> bool {+ self.is_present("pcre2") && self.unicode()+ }++ /// Returns true if and only if file names containing each match should+ /// be emitted.+ fn with_filename(&self, paths: &[PathBuf]) -> bool {+ if self.is_present("no-filename") {+ false+ } else {+ let path_stdin = Path::new("-");+ self.is_present("with-filename")+ || self.is_present("vimgrep")+ || paths.len() > 1+ || paths+ .get(0)+ .map_or(false, |p| p != path_stdin && p.is_dir())+ }+ }+}++/// Lower level generic helper methods for teasing values out of clap.+impl ArgMatches {+ /// Like values_of_lossy, but returns an empty vec if the flag is not+ /// present.+ fn values_of_lossy_vec(&self, name: &str) -> Vec<String> {+ self.values_of_lossy(name).unwrap_or_else(Vec::new)+ }++ /// Safely reads an arg value with the given name, and if it's present,+ /// tries to parse it as a usize value.+ ///+ /// If the number is zero, then it is considered absent and `None` is+ /// returned.+ fn usize_of_nonzero(&self, name: &str) -> Result<Option<usize>> {+ let n = match self.usize_of(name)? {+ None => return Ok(None),+ Some(n) => n,+ };+ Ok(if n == 0 { None } else { Some(n) })+ }++ /// Safely reads an arg value with the given name, and if it's present,+ /// tries to parse it as a usize value.+ fn usize_of(&self, name: &str) -> Result<Option<usize>> {+ match self.value_of_lossy(name) {+ None => Ok(None),+ Some(v) => v.parse().map(Some).map_err(From::from),+ }+ }++ /// Parses an argument of the form `[0-9]+(KMG)?`.+ ///+ /// If the aforementioned format is not recognized, then this returns an+ /// error.+ fn parse_human_readable_size(+ &self,+ arg_name: &str,+ ) -> Result<Option<u64>> {+ let size = match self.value_of_lossy(arg_name) {+ None => return Ok(None),+ Some(size) => size,+ };+ Ok(Some(cli::parse_human_readable_size(&size)?))+ }+}++/// The following methods mostly dispatch to the underlying clap methods+/// directly. Methods that would otherwise get a single value will fetch all+/// values and return the last one. (Clap returns the first one.) We only+/// define the ones we need.+impl ArgMatches {+ fn is_present(&self, name: &str) -> bool {+ self.0.is_present(name)+ }++ fn occurrences_of(&self, name: &str) -> u64 {+ self.0.occurrences_of(name)+ }++ fn value_of_lossy(&self, name: &str) -> Option<String> {+ self.0.value_of_lossy(name).map(|s| s.into_owned())+ }++ fn values_of_lossy(&self, name: &str) -> Option<Vec<String>> {+ self.0.values_of_lossy(name)+ }++ fn value_of_os(&self, name: &str) -> Option<&OsStr> {+ self.0.value_of_os(name)+ }++ fn values_of_os(&self, name: &str) -> Option<clap::OsValues> {+ self.0.values_of_os(name)+ }+}++/// Inspect an error resulting from building a Rust regex matcher, and if it's+/// believed to correspond to a syntax error that PCRE2 could handle, then+/// add a message to suggest the use of -P/--pcre2.+#[cfg(feature = "pcre2")]+fn suggest_pcre2(msg: String) -> String {+ if !msg.contains("backreferences") && !msg.contains("look-around") {+ msg+ } else {+ format!(+ "{}++Consider enabling PCRE2 with the --pcre2 flag, which can handle backreferences+and look-around.",+ msg+ )+ }+}++fn suggest_multiline(msg: String) -> String {+ if msg.contains("the literal") && msg.contains("not allowed") {+ format!(+ "{}++Consider enabling multiline mode with the --multiline flag (or -U for short).+When multiline mode is enabled, new line characters can be matched.",+ msg+ )+ } else {+ msg+ }+}++/// Convert the result of parsing a human readable file size to a `usize`,+/// failing if the type does not fit.+fn u64_to_usize(arg_name: &str, value: Option<u64>) -> Result<Option<usize>> {+ use std::usize;++ let value = match value {+ None => return Ok(None),+ Some(value) => value,+ };+ if value <= usize::MAX as u64 {+ Ok(Some(value as usize))+ } else {+ Err(From::from(format!("number too large for {}", arg_name)))+ }+}++/// Builds a comparator for sorting two files according to a system time+/// extracted from the file's metadata.+///+/// If there was a problem extracting the metadata or if the time is not+/// available, then both entries compare equal.+fn sort_by_metadata_time<G>(+ p1: &Path,+ p2: &Path,+ reverse: bool,+ get_time: G,+) -> cmp::Ordering+where+ G: Fn(&fs::Metadata) -> io::Result<SystemTime>,+{+ let t1 = match p1.metadata().and_then(|md| get_time(&md)) {+ Ok(t) => t,+ Err(_) => return cmp::Ordering::Equal,+ };+ let t2 = match p2.metadata().and_then(|md| get_time(&md)) {+ Ok(t) => t,+ Err(_) => return cmp::Ordering::Equal,+ };+ if reverse {+ t1.cmp(&t2).reverse()+ } else {+ t1.cmp(&t2)+ }+}++/// Returns a clap matches object if the given arguments parse successfully.+///+/// Otherwise, if an error occurred, then it is returned unless the error+/// corresponds to a `--help` or `--version` request. In which case, the+/// corresponding output is printed and the current process is exited+/// successfully.+fn clap_matches<I, T>(args: I) -> Result<clap::ArgMatches<'static>>+where+ I: IntoIterator<Item = T>,+ T: Into<OsString> + Clone,+{+ let err = match app::app().get_matches_from_safe(args) {+ Ok(matches) => return Ok(matches),+ Err(err) => err,+ };+ if err.use_stderr() {+ return Err(err.into());+ }+ // Explicitly ignore any error returned by write!. The most likely error+ // at this point is a broken pipe error, in which case, we want to ignore+ // it and exit quietly.+ //+ // (This is the point of this helper function. clap's functionality for+ // doing this will panic on a broken pipe error.)+ let _ = write!(io::stdout(), "{}", err);+ process::exit(0);+}++/// Attempts to discover the current working directory. This mostly just defers+/// to the standard library, however, such things will fail if ripgrep is in+/// a directory that no longer exists. We attempt some fallback mechanisms,+/// such as querying the PWD environment variable, but otherwise return an+/// error.+fn current_dir() -> Result<PathBuf> {+ let err = match env::current_dir() {+ Err(err) => err,+ Ok(cwd) => return Ok(cwd),+ };+ if let Some(cwd) = env::var_os("PWD") {+ if !cwd.is_empty() {+ return Ok(PathBuf::from(cwd));+ }+ }+ Err(format!(+ "failed to get current working directory: {} \+ --- did your CWD get deleted?",+ err,+ )+ .into())+}
crates/core/config.rs
@@ -0,0 +1,169 @@+// This module provides routines for reading ripgrep config "rc" files. The+// primary output of these routines is a sequence of arguments, where each+// argument corresponds precisely to one shell argument.++use std::env;+use std::error::Error;+use std::ffi::OsString;+use std::fs::File;+use std::io;+use std::path::{Path, PathBuf};++use bstr::{io::BufReadExt, ByteSlice};+use log;++use crate::Result;++/// Return a sequence of arguments derived from ripgrep rc configuration files.+pub fn args() -> Vec<OsString> {+ let config_path = match env::var_os("RIPGREP_CONFIG_PATH") {+ None => return vec![],+ Some(config_path) => {+ if config_path.is_empty() {+ return vec![];+ }+ PathBuf::from(config_path)+ }+ };+ let (args, errs) = match parse(&config_path) {+ Ok((args, errs)) => (args, errs),+ Err(err) => {+ message!("{}", err);+ return vec![];+ }+ };+ if !errs.is_empty() {+ for err in errs {+ message!("{}:{}", config_path.display(), err);+ }+ }+ log::debug!(+ "{}: arguments loaded from config file: {:?}",+ config_path.display(),+ args+ );+ args+}++/// Parse a single ripgrep rc file from the given path.+///+/// On success, this returns a set of shell arguments, in order, that should+/// be pre-pended to the arguments given to ripgrep at the command line.+///+/// If the file could not be read, then an error is returned. If there was+/// a problem parsing one or more lines in the file, then errors are returned+/// for each line in addition to successfully parsed arguments.+fn parse<P: AsRef<Path>>(+ path: P,+) -> Result<(Vec<OsString>, Vec<Box<dyn Error>>)> {+ let path = path.as_ref();+ match File::open(&path) {+ Ok(file) => parse_reader(file),+ Err(err) => Err(From::from(format!("{}: {}", path.display(), err))),+ }+}++/// Parse a single ripgrep rc file from the given reader.+///+/// Callers should not provided a buffered reader, as this routine will use its+/// own buffer internally.+///+/// On success, this returns a set of shell arguments, in order, that should+/// be pre-pended to the arguments given to ripgrep at the command line.+///+/// If the reader could not be read, then an error is returned. If there was a+/// problem parsing one or more lines, then errors are returned for each line+/// in addition to successfully parsed arguments.+fn parse_reader<R: io::Read>(+ rdr: R,+) -> Result<(Vec<OsString>, Vec<Box<dyn Error>>)> {+ let bufrdr = io::BufReader::new(rdr);+ let (mut args, mut errs) = (vec![], vec![]);+ let mut line_number = 0;+ bufrdr.for_byte_line_with_terminator(|line| {+ line_number += 1;++ let line = line.trim();+ if line.is_empty() || line[0] == b'#' {+ return Ok(true);+ }+ match line.to_os_str() {+ Ok(osstr) => {+ args.push(osstr.to_os_string());+ }+ Err(err) => {+ errs.push(format!("{}: {}", line_number, err).into());+ }+ }+ Ok(true)+ })?;+ Ok((args, errs))+}++#[cfg(test)]+mod tests {+ use super::parse_reader;+ use std::ffi::OsString;++ #[test]+ fn basic() {+ let (args, errs) = parse_reader(+ &b"\+# Test+--context=0+ --smart-case+-u+++ # --bar+--foo+"[..],+ )+ .unwrap();+ assert!(errs.is_empty());+ let args: Vec<String> =+ args.into_iter().map(|s| s.into_string().unwrap()).collect();+ assert_eq!(args, vec!["--context=0", "--smart-case", "-u", "--foo",]);+ }++ // We test that we can handle invalid UTF-8 on Unix-like systems.+ #[test]+ #[cfg(unix)]+ fn error() {+ use std::os::unix::ffi::OsStringExt;++ let (args, errs) = parse_reader(+ &b"\+quux+foo\xFFbar+baz+"[..],+ )+ .unwrap();+ assert!(errs.is_empty());+ assert_eq!(+ args,+ vec![+ OsString::from("quux"),+ OsString::from_vec(b"foo\xFFbar".to_vec()),+ OsString::from("baz"),+ ]+ );+ }++ // ... but test that invalid UTF-8 fails on Windows.+ #[test]+ #[cfg(not(unix))]+ fn error() {+ let (args, errs) = parse_reader(+ &b"\+quux+foo\xFFbar+baz+"[..],+ )+ .unwrap();+ assert_eq!(errs.len(), 1);+ assert_eq!(args, vec![OsString::from("quux"), OsString::from("baz"),]);+ }+}
@@ -0,0 +1,68 @@+// This module defines a super simple logger that works with the `log` crate.+// We don't need anything fancy; just basic log levels and the ability to+// print to stderr. We therefore avoid bringing in extra dependencies just+// for this functionality.++use log::{self, Log};++/// The simplest possible logger that logs to stderr.+///+/// This logger does no filtering. Instead, it relies on the `log` crates+/// filtering via its global max_level setting.+#[derive(Debug)]+pub struct Logger(());++const LOGGER: &'static Logger = &Logger(());++impl Logger {+ /// Create a new logger that logs to stderr and initialize it as the+ /// global logger. If there was a problem setting the logger, then an+ /// error is returned.+ pub fn init() -> Result<(), log::SetLoggerError> {+ log::set_logger(LOGGER)+ }+}++impl Log for Logger {+ fn enabled(&self, _: &log::Metadata) -> bool {+ // We set the log level via log::set_max_level, so we don't need to+ // implement filtering here.+ true+ }++ fn log(&self, record: &log::Record) {+ match (record.file(), record.line()) {+ (Some(file), Some(line)) => {+ eprintln!(+ "{}|{}|{}:{}: {}",+ record.level(),+ record.target(),+ file,+ line,+ record.args()+ );+ }+ (Some(file), None) => {+ eprintln!(+ "{}|{}|{}: {}",+ record.level(),+ record.target(),+ file,+ record.args()+ );+ }+ _ => {+ eprintln!(+ "{}|{}: {}",+ record.level(),+ record.target(),+ record.args()+ );+ }+ }+ }++ fn flush(&self) {+ // We use eprintln! which is flushed on every call.+ }+}
@@ -0,0 +1,325 @@+use std::error;+use std::io::{self, Write};+use std::process;+use std::sync::Mutex;+use std::time::Instant;++use ignore::WalkState;++use args::Args;+use subject::Subject;++#[macro_use]+mod messages;++mod app;+mod args;+mod config;+mod logger;+mod path_printer;+mod search;+mod subject;++// Since Rust no longer uses jemalloc by default, ripgrep will, by default,+// use the system allocator. On Linux, this would normally be glibc's+// allocator, which is pretty good. In particular, ripgrep does not have a+// particularly allocation heavy workload, so there really isn't much+// difference (for ripgrep's purposes) between glibc's allocator and jemalloc.+//+// However, when ripgrep is built with musl, this means ripgrep will use musl's+// allocator, which appears to be substantially worse. (musl's goal is not to+// have the fastest version of everything. Its goal is to be small and amenable+// to static compilation.) Even though ripgrep isn't particularly allocation+// heavy, musl's allocator appears to slow down ripgrep quite a bit. Therefore,+// when building with musl, we use jemalloc.+//+// We don't unconditionally use jemalloc because it can be nice to use the+// system's default allocator by default. Moreover, jemalloc seems to increase+// compilation times by a bit.+//+// Moreover, we only do this on 64-bit systems since jemalloc doesn't support+// i686.+#[cfg(all(target_env = "musl", target_pointer_width = "64"))]+#[global_allocator]+static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;++type Result<T> = ::std::result::Result<T, Box<dyn error::Error>>;++fn main() {+ if let Err(err) = Args::parse().and_then(try_main) {+ eprintln!("{}", err);+ process::exit(2);+ }+}++fn try_main(args: Args) -> Result<()> {+ use args::Command::*;++ let matched = match args.command()? {+ Search => search(&args),+ SearchParallel => search_parallel(&args),+ SearchNever => Ok(false),+ Files => files(&args),+ FilesParallel => files_parallel(&args),+ Types => types(&args),+ PCRE2Version => pcre2_version(&args),+ }?;+ if matched && (args.quiet() || !messages::errored()) {+ process::exit(0)+ } else if messages::errored() {+ process::exit(2)+ } else {+ process::exit(1)+ }+}++/// The top-level entry point for single-threaded search. This recursively+/// steps through the file list (current directory by default) and searches+/// each file sequentially.+fn search(args: &Args) -> Result<bool> {+ let started_at = Instant::now();+ let quit_after_match = args.quit_after_match()?;+ let subject_builder = args.subject_builder();+ let mut stats = args.stats()?;+ let mut searcher = args.search_worker(args.stdout())?;+ let mut matched = false;++ for result in args.walker()? {+ let subject = match subject_builder.build_from_result(result) {+ Some(subject) => subject,+ None => continue,+ };+ let search_result = match searcher.search(&subject) {+ Ok(search_result) => search_result,+ Err(err) => {+ // A broken pipe means graceful termination.+ if err.kind() == io::ErrorKind::BrokenPipe {+ break;+ }+ err_message!("{}: {}", subject.path().display(), err);+ continue;+ }+ };+ matched = matched || search_result.has_match();+ if let Some(ref mut stats) = stats {+ *stats += search_result.stats().unwrap();+ }+ if matched && quit_after_match {+ break;+ }+ }+ if let Some(ref stats) = stats {+ let elapsed = Instant::now().duration_since(started_at);+ // We don't care if we couldn't print this successfully.+ let _ = searcher.print_stats(elapsed, stats);+ }+ Ok(matched)+}++/// The top-level entry point for multi-threaded search. The parallelism is+/// itself achieved by the recursive directory traversal. All we need to do is+/// feed it a worker for performing a search on each file.+fn search_parallel(args: &Args) -> Result<bool> {+ use std::sync::atomic::AtomicBool;+ use std::sync::atomic::Ordering::SeqCst;++ let quit_after_match = args.quit_after_match()?;+ let started_at = Instant::now();+ let subject_builder = args.subject_builder();+ let bufwtr = args.buffer_writer()?;+ let stats = args.stats()?.map(Mutex::new);+ let matched = AtomicBool::new(false);+ let mut searcher_err = None;+ args.walker_parallel()?.run(|| {+ let bufwtr = &bufwtr;+ let stats = &stats;+ let matched = &matched;+ let subject_builder = &subject_builder;+ let mut searcher = match args.search_worker(bufwtr.buffer()) {+ Ok(searcher) => searcher,+ Err(err) => {+ searcher_err = Some(err);+ return Box::new(move |_| WalkState::Quit);+ }+ };++ Box::new(move |result| {+ let subject = match subject_builder.build_from_result(result) {+ Some(subject) => subject,+ None => return WalkState::Continue,+ };+ searcher.printer().get_mut().clear();+ let search_result = match searcher.search(&subject) {+ Ok(search_result) => search_result,+ Err(err) => {+ err_message!("{}: {}", subject.path().display(), err);+ return WalkState::Continue;+ }+ };+ if search_result.has_match() {+ matched.store(true, SeqCst);+ }+ if let Some(ref locked_stats) = *stats {+ let mut stats = locked_stats.lock().unwrap();+ *stats += search_result.stats().unwrap();+ }+ if let Err(err) = bufwtr.print(searcher.printer().get_mut()) {+ // A broken pipe means graceful termination.+ if err.kind() == io::ErrorKind::BrokenPipe {+ return WalkState::Quit;+ }+ // Otherwise, we continue on our merry way.+ err_message!("{}: {}", subject.path().display(), err);+ }+ if matched.load(SeqCst) && quit_after_match {+ WalkState::Quit+ } else {+ WalkState::Continue+ }+ })+ });+ if let Some(err) = searcher_err.take() {+ return Err(err);+ }+ if let Some(ref locked_stats) = stats {+ let elapsed = Instant::now().duration_since(started_at);+ let stats = locked_stats.lock().unwrap();+ let mut searcher = args.search_worker(args.stdout())?;+ // We don't care if we couldn't print this successfully.+ let _ = searcher.print_stats(elapsed, &stats);+ }+ Ok(matched.load(SeqCst))+}++/// The top-level entry point for listing files without searching them. This+/// recursively steps through the file list (current directory by default) and+/// prints each path sequentially using a single thread.+fn files(args: &Args) -> Result<bool> {+ let quit_after_match = args.quit_after_match()?;+ let subject_builder = args.subject_builder();+ let mut matched = false;+ let mut path_printer = args.path_printer(args.stdout())?;+ for result in args.walker()? {+ let subject = match subject_builder.build_from_result(result) {+ Some(subject) => subject,+ None => continue,+ };+ matched = true;+ if quit_after_match {+ break;+ }+ if let Err(err) = path_printer.write_path(subject.path()) {+ // A broken pipe means graceful termination.+ if err.kind() == io::ErrorKind::BrokenPipe {+ break;+ }+ // Otherwise, we have some other error that's preventing us from+ // writing to stdout, so we should bubble it up.+ return Err(err.into());+ }+ }+ Ok(matched)+}++/// The top-level entry point for listing files without searching them. This+/// recursively steps through the file list (current directory by default) and+/// prints each path sequentially using multiple threads.+fn files_parallel(args: &Args) -> Result<bool> {+ use std::sync::atomic::AtomicBool;+ use std::sync::atomic::Ordering::SeqCst;+ use std::sync::mpsc;+ use std::thread;++ let quit_after_match = args.quit_after_match()?;+ let subject_builder = args.subject_builder();+ let mut path_printer = args.path_printer(args.stdout())?;+ let matched = AtomicBool::new(false);+ let (tx, rx) = mpsc::channel::<Subject>();++ let print_thread = thread::spawn(move || -> io::Result<()> {+ for subject in rx.iter() {+ path_printer.write_path(subject.path())?;+ }+ Ok(())+ });+ args.walker_parallel()?.run(|| {+ let subject_builder = &subject_builder;+ let matched = &matched;+ let tx = tx.clone();++ Box::new(move |result| {+ let subject = match subject_builder.build_from_result(result) {+ Some(subject) => subject,+ None => return WalkState::Continue,+ };+ matched.store(true, SeqCst);+ if quit_after_match {+ WalkState::Quit+ } else {+ match tx.send(subject) {+ Ok(_) => WalkState::Continue,+ Err(_) => WalkState::Quit,+ }+ }+ })+ });+ drop(tx);+ if let Err(err) = print_thread.join().unwrap() {+ // A broken pipe means graceful termination, so fall through.+ // Otherwise, something bad happened while writing to stdout, so bubble+ // it up.+ if err.kind() != io::ErrorKind::BrokenPipe {+ return Err(err.into());+ }+ }+ Ok(matched.load(SeqCst))+}++/// The top-level entry point for --type-list.+fn types(args: &Args) -> Result<bool> {+ let mut count = 0;+ let mut stdout = args.stdout();+ for def in args.type_defs()? {+ count += 1;+ stdout.write_all(def.name().as_bytes())?;+ stdout.write_all(b": ")?;++ let mut first = true;+ for glob in def.globs() {+ if !first {+ stdout.write_all(b", ")?;+ }+ stdout.write_all(glob.as_bytes())?;+ first = false;+ }+ stdout.write_all(b"\n")?;+ }+ Ok(count > 0)+}++/// The top-level entry point for --pcre2-version.+fn pcre2_version(args: &Args) -> Result<bool> {+ #[cfg(feature = "pcre2")]+ fn imp(args: &Args) -> Result<bool> {+ use grep::pcre2;++ let mut stdout = args.stdout();++ let (major, minor) = pcre2::version();+ writeln!(stdout, "PCRE2 {}.{} is available", major, minor)?;++ if cfg!(target_pointer_width = "64") && pcre2::is_jit_available() {+ writeln!(stdout, "JIT is available")?;+ }+ Ok(true)+ }++ #[cfg(not(feature = "pcre2"))]+ fn imp(args: &Args) -> Result<bool> {+ let mut stdout = args.stdout();+ writeln!(stdout, "PCRE2 is not available in this build of ripgrep.")?;+ Ok(false)+ }++ imp(args)+}
@@ -0,0 +1,74 @@+use std::sync::atomic::{AtomicBool, Ordering};++static MESSAGES: AtomicBool = AtomicBool::new(false);+static IGNORE_MESSAGES: AtomicBool = AtomicBool::new(false);+static ERRORED: AtomicBool = AtomicBool::new(false);++/// Emit a non-fatal error message, unless messages were disabled.+#[macro_export]+macro_rules! message {+ ($($tt:tt)*) => {+ if crate::messages::messages() {+ eprintln!($($tt)*);+ }+ }+}++/// Like message, but sets ripgrep's "errored" flag, which controls the exit+/// status.+#[macro_export]+macro_rules! err_message {+ ($($tt:tt)*) => {+ crate::messages::set_errored();+ message!($($tt)*);+ }+}++/// Emit a non-fatal ignore-related error message (like a parse error), unless+/// ignore-messages were disabled.+#[macro_export]+macro_rules! ignore_message {+ ($($tt:tt)*) => {+ if crate::messages::messages() && crate::messages::ignore_messages() {+ eprintln!($($tt)*);+ }+ }+}++/// Returns true if and only if messages should be shown.+pub fn messages() -> bool {+ MESSAGES.load(Ordering::SeqCst)+}++/// Set whether messages should be shown or not.+///+/// By default, they are not shown.+pub fn set_messages(yes: bool) {+ MESSAGES.store(yes, Ordering::SeqCst)+}++/// Returns true if and only if "ignore" related messages should be shown.+pub fn ignore_messages() -> bool {+ IGNORE_MESSAGES.load(Ordering::SeqCst)+}++/// Set whether "ignore" related messages should be shown or not.+///+/// By default, they are not shown.+///+/// Note that this is overridden if `messages` is disabled. Namely, if+/// `messages` is disabled, then "ignore" messages are never shown, regardless+/// of this setting.+pub fn set_ignore_messages(yes: bool) {+ IGNORE_MESSAGES.store(yes, Ordering::SeqCst)+}++/// Returns true if and only if ripgrep came across a non-fatal error.+pub fn errored() -> bool {+ ERRORED.load(Ordering::SeqCst)+}++/// Indicate that ripgrep has come across a non-fatal error.+pub fn set_errored() {+ ERRORED.store(true, Ordering::SeqCst);+}
crates/core/path_printer.rs
@@ -0,0 +1,98 @@+use std::io;+use std::path::Path;++use grep::printer::{ColorSpecs, PrinterPath};+use termcolor::WriteColor;++/// A configuration for describing how paths should be written.+#[derive(Clone, Debug)]+struct Config {+ colors: ColorSpecs,+ separator: Option<u8>,+ terminator: u8,+}++impl Default for Config {+ fn default() -> Config {+ Config {+ colors: ColorSpecs::default(),+ separator: None,+ terminator: b'\n',+ }+ }+}++/// A builder for constructing things to search over.+#[derive(Clone, Debug)]+pub struct PathPrinterBuilder {+ config: Config,+}++impl PathPrinterBuilder {+ /// Return a new subject builder with a default configuration.+ pub fn new() -> PathPrinterBuilder {+ PathPrinterBuilder { config: Config::default() }+ }++ /// Create a new path printer with the current configuration that writes+ /// paths to the given writer.+ pub fn build<W: WriteColor>(&self, wtr: W) -> PathPrinter<W> {+ PathPrinter { config: self.config.clone(), wtr: wtr }+ }++ /// Set the color specification for this printer.+ ///+ /// Currently, only the `path` component of the given specification is+ /// used.+ pub fn color_specs(+ &mut self,+ specs: ColorSpecs,+ ) -> &mut PathPrinterBuilder {+ self.config.colors = specs;+ self+ }++ /// A path separator.+ ///+ /// When provided, the path's default separator will be replaced with+ /// the given separator.+ ///+ /// This is not set by default, and the system's default path separator+ /// will be used.+ pub fn separator(&mut self, sep: Option<u8>) -> &mut PathPrinterBuilder {+ self.config.separator = sep;+ self+ }++ /// A path terminator.+ ///+ /// When printing a path, it will be by terminated by the given byte.+ ///+ /// This is set to `\n` by default.+ pub fn terminator(&mut self, terminator: u8) -> &mut PathPrinterBuilder {+ self.config.terminator = terminator;+ self+ }+}++/// A printer for emitting paths to a writer, with optional color support.+#[derive(Debug)]+pub struct PathPrinter<W> {+ config: Config,+ wtr: W,+}++impl<W: WriteColor> PathPrinter<W> {+ /// Write the given path to the underlying writer.+ pub fn write_path(&mut self, path: &Path) -> io::Result<()> {+ let ppath = PrinterPath::with_separator(path, self.config.separator);+ if !self.wtr.supports_color() {+ self.wtr.write_all(ppath.as_bytes())?;+ } else {+ self.wtr.set_color(self.config.colors.path())?;+ self.wtr.write_all(ppath.as_bytes())?;+ self.wtr.reset()?;+ }+ self.wtr.write_all(&[self.config.terminator])+ }+}
@@ -0,0 +1,534 @@+use std::fs::File;+use std::io;+use std::path::{Path, PathBuf};+use std::process::{Command, Stdio};+use std::time::Duration;++use grep::cli;+use grep::matcher::Matcher;+#[cfg(feature = "pcre2")]+use grep::pcre2::RegexMatcher as PCRE2RegexMatcher;+use grep::printer::{Standard, Stats, Summary, JSON};+use grep::regex::RegexMatcher as RustRegexMatcher;+use grep::searcher::{BinaryDetection, Searcher};+use ignore::overrides::Override;+use serde_json as json;+use serde_json::json;+use termcolor::WriteColor;++use crate::subject::Subject;++/// The configuration for the search worker. Among a few other things, the+/// configuration primarily controls the way we show search results to users+/// at a very high level.+#[derive(Clone, Debug)]+struct Config {+ json_stats: bool,+ preprocessor: Option<PathBuf>,+ preprocessor_globs: Override,+ search_zip: bool,+ binary_implicit: BinaryDetection,+ binary_explicit: BinaryDetection,+}++impl Default for Config {+ fn default() -> Config {+ Config {+ json_stats: false,+ preprocessor: None,+ preprocessor_globs: Override::empty(),+ search_zip: false,+ binary_implicit: BinaryDetection::none(),+ binary_explicit: BinaryDetection::none(),+ }+ }+}++/// A builder for configuring and constructing a search worker.+#[derive(Clone, Debug)]+pub struct SearchWorkerBuilder {+ config: Config,+ command_builder: cli::CommandReaderBuilder,+ decomp_builder: cli::DecompressionReaderBuilder,+}++impl Default for SearchWorkerBuilder {+ fn default() -> SearchWorkerBuilder {+ SearchWorkerBuilder::new()+ }+}++impl SearchWorkerBuilder {+ /// Create a new builder for configuring and constructing a search worker.+ pub fn new() -> SearchWorkerBuilder {+ let mut cmd_builder = cli::CommandReaderBuilder::new();+ cmd_builder.async_stderr(true);++ let mut decomp_builder = cli::DecompressionReaderBuilder::new();+ decomp_builder.async_stderr(true);++ SearchWorkerBuilder {+ config: Config::default(),+ command_builder: cmd_builder,+ decomp_builder: decomp_builder,+ }+ }++ /// Create a new search worker using the given searcher, matcher and+ /// printer.+ pub fn build<W: WriteColor>(+ &self,+ matcher: PatternMatcher,+ searcher: Searcher,+ printer: Printer<W>,+ ) -> SearchWorker<W> {+ let config = self.config.clone();+ let command_builder = self.command_builder.clone();+ let decomp_builder = self.decomp_builder.clone();+ SearchWorker {+ config,+ command_builder,+ decomp_builder,+ matcher,+ searcher,+ printer,+ }+ }++ /// Forcefully use JSON to emit statistics, even if the underlying printer+ /// is not the JSON printer.+ ///+ /// This is useful for implementing flag combinations like+ /// `--json --quiet`, which uses the summary printer for implementing+ /// `--quiet` but still wants to emit summary statistics, which should+ /// be JSON formatted because of the `--json` flag.+ pub fn json_stats(&mut self, yes: bool) -> &mut SearchWorkerBuilder {+ self.config.json_stats = yes;+ self+ }++ /// Set the path to a preprocessor command.+ ///+ /// When this is set, instead of searching files directly, the given+ /// command will be run with the file path as the first argument, and the+ /// output of that command will be searched instead.+ pub fn preprocessor(+ &mut self,+ cmd: Option<PathBuf>,+ ) -> &mut SearchWorkerBuilder {+ self.config.preprocessor = cmd;+ self+ }++ /// Set the globs for determining which files should be run through the+ /// preprocessor. By default, with no globs and a preprocessor specified,+ /// every file is run through the preprocessor.+ pub fn preprocessor_globs(+ &mut self,+ globs: Override,+ ) -> &mut SearchWorkerBuilder {+ self.config.preprocessor_globs = globs;+ self+ }++ /// Enable the decompression and searching of common compressed files.+ ///+ /// When enabled, if a particular file path is recognized as a compressed+ /// file, then it is decompressed before searching.+ ///+ /// Note that if a preprocessor command is set, then it overrides this+ /// setting.+ pub fn search_zip(&mut self, yes: bool) -> &mut SearchWorkerBuilder {+ self.config.search_zip = yes;+ self+ }++ /// Set the binary detection that should be used when searching files+ /// found via a recursive directory search.+ ///+ /// Generally, this binary detection may be `BinaryDetection::quit` if+ /// we want to skip binary files completely.+ ///+ /// By default, no binary detection is performed.+ pub fn binary_detection_implicit(+ &mut self,+ detection: BinaryDetection,+ ) -> &mut SearchWorkerBuilder {+ self.config.binary_implicit = detection;+ self+ }++ /// Set the binary detection that should be used when searching files+ /// explicitly supplied by an end user.+ ///+ /// Generally, this binary detection should NOT be `BinaryDetection::quit`,+ /// since we never want to automatically filter files supplied by the end+ /// user.+ ///+ /// By default, no binary detection is performed.+ pub fn binary_detection_explicit(+ &mut self,+ detection: BinaryDetection,+ ) -> &mut SearchWorkerBuilder {+ self.config.binary_explicit = detection;+ self+ }+}++/// The result of executing a search.+///+/// Generally speaking, the "result" of a search is sent to a printer, which+/// writes results to an underlying writer such as stdout or a file. However,+/// every search also has some aggregate statistics or meta data that may be+/// useful to higher level routines.+#[derive(Clone, Debug, Default)]+pub struct SearchResult {+ has_match: bool,+ stats: Option<Stats>,+}++impl SearchResult {+ /// Whether the search found a match or not.+ pub fn has_match(&self) -> bool {+ self.has_match+ }++ /// Return aggregate search statistics for a single search, if available.+ ///+ /// It can be expensive to compute statistics, so these are only present+ /// if explicitly enabled in the printer provided by the caller.+ pub fn stats(&self) -> Option<&Stats> {+ self.stats.as_ref()+ }+}++/// The pattern matcher used by a search worker.+#[derive(Clone, Debug)]+pub enum PatternMatcher {+ RustRegex(RustRegexMatcher),+ #[cfg(feature = "pcre2")]+ PCRE2(PCRE2RegexMatcher),+}++/// The printer used by a search worker.+///+/// The `W` type parameter refers to the type of the underlying writer.+#[derive(Debug)]+pub enum Printer<W> {+ /// Use the standard printer, which supports the classic grep-like format.+ Standard(Standard<W>),+ /// Use the summary printer, which supports aggregate displays of search+ /// results.+ Summary(Summary<W>),+ /// A JSON printer, which emits results in the JSON Lines format.+ JSON(JSON<W>),+}++impl<W: WriteColor> Printer<W> {+ fn print_stats(+ &mut self,+ total_duration: Duration,+ stats: &Stats,+ ) -> io::Result<()> {+ match *self {+ Printer::JSON(_) => self.print_stats_json(total_duration, stats),+ Printer::Standard(_) | Printer::Summary(_) => {+ self.print_stats_human(total_duration, stats)+ }+ }+ }++ fn print_stats_human(+ &mut self,+ total_duration: Duration,+ stats: &Stats,+ ) -> io::Result<()> {+ write!(+ self.get_mut(),+ "+{matches} matches+{lines} matched lines+{searches_with_match} files contained matches+{searches} files searched+{bytes_printed} bytes printed+{bytes_searched} bytes searched+{search_time:0.6} seconds spent searching+{process_time:0.6} seconds+",+ matches = stats.matches(),+ lines = stats.matched_lines(),+ searches_with_match = stats.searches_with_match(),+ searches = stats.searches(),+ bytes_printed = stats.bytes_printed(),+ bytes_searched = stats.bytes_searched(),+ search_time = fractional_seconds(stats.elapsed()),+ process_time = fractional_seconds(total_duration)+ )+ }++ fn print_stats_json(+ &mut self,+ total_duration: Duration,+ stats: &Stats,+ ) -> io::Result<()> {+ // We specifically match the format laid out by the JSON printer in+ // the grep-printer crate. We simply "extend" it with the 'summary'+ // message type.+ let fractional = fractional_seconds(total_duration);+ json::to_writer(+ self.get_mut(),+ &json!({+ "type": "summary",+ "data": {+ "stats": stats,+ "elapsed_total": {+ "secs": total_duration.as_secs(),+ "nanos": total_duration.subsec_nanos(),+ "human": format!("{:0.6}s", fractional),+ },+ }+ }),+ )?;+ write!(self.get_mut(), "\n")+ }++ /// Return a mutable reference to the underlying printer's writer.+ pub fn get_mut(&mut self) -> &mut W {+ match *self {+ Printer::Standard(ref mut p) => p.get_mut(),+ Printer::Summary(ref mut p) => p.get_mut(),+ Printer::JSON(ref mut p) => p.get_mut(),+ }+ }+}++/// A worker for executing searches.+///+/// It is intended for a single worker to execute many searches, and is+/// generally intended to be used from a single thread. When searching using+/// multiple threads, it is better to create a new worker for each thread.+#[derive(Debug)]+pub struct SearchWorker<W> {+ config: Config,+ command_builder: cli::CommandReaderBuilder,+ decomp_builder: cli::DecompressionReaderBuilder,+ matcher: PatternMatcher,+ searcher: Searcher,+ printer: Printer<W>,+}++impl<W: WriteColor> SearchWorker<W> {+ /// Execute a search over the given subject.+ pub fn search(&mut self, subject: &Subject) -> io::Result<SearchResult> {+ let bin = if subject.is_explicit() {+ self.config.binary_explicit.clone()+ } else {+ self.config.binary_implicit.clone()+ };+ self.searcher.set_binary_detection(bin);++ let path = subject.path();+ if subject.is_stdin() {+ self.search_reader(path, io::stdin().lock())+ } else if self.should_preprocess(path) {+ self.search_preprocessor(path)+ } else if self.should_decompress(path) {+ self.search_decompress(path)+ } else {+ self.search_path(path)+ }+ }++ /// Return a mutable reference to the underlying printer.+ pub fn printer(&mut self) -> &mut Printer<W> {+ &mut self.printer+ }++ /// Print the given statistics to the underlying writer in a way that is+ /// consistent with this searcher's printer's format.+ ///+ /// While `Stats` contains a duration itself, this only corresponds to the+ /// time spent searching, where as `total_duration` should roughly+ /// approximate the lifespan of the ripgrep process itself.+ pub fn print_stats(+ &mut self,+ total_duration: Duration,+ stats: &Stats,+ ) -> io::Result<()> {+ if self.config.json_stats {+ self.printer().print_stats_json(total_duration, stats)+ } else {+ self.printer().print_stats(total_duration, stats)+ }+ }++ /// Returns true if and only if the given file path should be+ /// decompressed before searching.+ fn should_decompress(&self, path: &Path) -> bool {+ if !self.config.search_zip {+ return false;+ }+ self.decomp_builder.get_matcher().has_command(path)+ }++ /// Returns true if and only if the given file path should be run through+ /// the preprocessor.+ fn should_preprocess(&self, path: &Path) -> bool {+ if !self.config.preprocessor.is_some() {+ return false;+ }+ if self.config.preprocessor_globs.is_empty() {+ return true;+ }+ !self.config.preprocessor_globs.matched(path, false).is_ignore()+ }++ /// Search the given file path by first asking the preprocessor for the+ /// data to search instead of opening the path directly.+ fn search_preprocessor(+ &mut self,+ path: &Path,+ ) -> io::Result<SearchResult> {+ let bin = self.config.preprocessor.as_ref().unwrap();+ let mut cmd = Command::new(bin);+ cmd.arg(path).stdin(Stdio::from(File::open(path)?));++ let rdr = self.command_builder.build(&mut cmd).map_err(|err| {+ io::Error::new(+ io::ErrorKind::Other,+ format!(+ "preprocessor command could not start: '{:?}': {}",+ cmd, err,+ ),+ )+ })?;+ self.search_reader(path, rdr).map_err(|err| {+ io::Error::new(+ io::ErrorKind::Other,+ format!("preprocessor command failed: '{:?}': {}", cmd, err),+ )+ })+ }++ /// Attempt to decompress the data at the given file path and search the+ /// result. If the given file path isn't recognized as a compressed file,+ /// then search it without doing any decompression.+ fn search_decompress(&mut self, path: &Path) -> io::Result<SearchResult> {+ let rdr = self.decomp_builder.build(path)?;+ self.search_reader(path, rdr)+ }++ /// Search the contents of the given file path.+ fn search_path(&mut self, path: &Path) -> io::Result<SearchResult> {+ use self::PatternMatcher::*;++ let (searcher, printer) = (&mut self.searcher, &mut self.printer);+ match self.matcher {+ RustRegex(ref m) => search_path(m, searcher, printer, path),+ #[cfg(feature = "pcre2")]+ PCRE2(ref m) => search_path(m, searcher, printer, path),+ }+ }++ /// Executes a search on the given reader, which may or may not correspond+ /// directly to the contents of the given file path. Instead, the reader+ /// may actually cause something else to be searched (for example, when+ /// a preprocessor is set or when decompression is enabled). In those+ /// cases, the file path is used for visual purposes only.+ ///+ /// Generally speaking, this method should only be used when there is no+ /// other choice. Searching via `search_path` provides more opportunities+ /// for optimizations (such as memory maps).+ fn search_reader<R: io::Read>(+ &mut self,+ path: &Path,+ rdr: R,+ ) -> io::Result<SearchResult> {+ use self::PatternMatcher::*;++ let (searcher, printer) = (&mut self.searcher, &mut self.printer);+ match self.matcher {+ RustRegex(ref m) => search_reader(m, searcher, printer, path, rdr),+ #[cfg(feature = "pcre2")]+ PCRE2(ref m) => search_reader(m, searcher, printer, path, rdr),+ }+ }+}++/// Search the contents of the given file path using the given matcher,+/// searcher and printer.+fn search_path<M: Matcher, W: WriteColor>(+ matcher: M,+ searcher: &mut Searcher,+ printer: &mut Printer<W>,+ path: &Path,+) -> io::Result<SearchResult> {+ match *printer {+ Printer::Standard(ref mut p) => {+ let mut sink = p.sink_with_path(&matcher, path);+ searcher.search_path(&matcher, path, &mut sink)?;+ Ok(SearchResult {+ has_match: sink.has_match(),+ stats: sink.stats().map(|s| s.clone()),+ })+ }+ Printer::Summary(ref mut p) => {+ let mut sink = p.sink_with_path(&matcher, path);+ searcher.search_path(&matcher, path, &mut sink)?;+ Ok(SearchResult {+ has_match: sink.has_match(),+ stats: sink.stats().map(|s| s.clone()),+ })+ }+ Printer::JSON(ref mut p) => {+ let mut sink = p.sink_with_path(&matcher, path);+ searcher.search_path(&matcher, path, &mut sink)?;+ Ok(SearchResult {+ has_match: sink.has_match(),+ stats: Some(sink.stats().clone()),+ })+ }+ }+}++/// Search the contents of the given reader using the given matcher, searcher+/// and printer.+fn search_reader<M: Matcher, R: io::Read, W: WriteColor>(+ matcher: M,+ searcher: &mut Searcher,+ printer: &mut Printer<W>,+ path: &Path,+ rdr: R,+) -> io::Result<SearchResult> {+ match *printer {+ Printer::Standard(ref mut p) => {+ let mut sink = p.sink_with_path(&matcher, path);+ searcher.search_reader(&matcher, rdr, &mut sink)?;+ Ok(SearchResult {+ has_match: sink.has_match(),+ stats: sink.stats().map(|s| s.clone()),+ })+ }+ Printer::Summary(ref mut p) => {+ let mut sink = p.sink_with_path(&matcher, path);+ searcher.search_reader(&matcher, rdr, &mut sink)?;+ Ok(SearchResult {+ has_match: sink.has_match(),+ stats: sink.stats().map(|s| s.clone()),+ })+ }+ Printer::JSON(ref mut p) => {+ let mut sink = p.sink_with_path(&matcher, path);+ searcher.search_reader(&matcher, rdr, &mut sink)?;+ Ok(SearchResult {+ has_match: sink.has_match(),+ stats: Some(sink.stats().clone()),+ })+ }+ }+}++/// Return the given duration as fractional seconds.+fn fractional_seconds(duration: Duration) -> f64 {+ (duration.as_secs() as f64) + (duration.subsec_nanos() as f64 * 1e-9)+}
crates/core/subject.rs
@@ -0,0 +1,162 @@+use std::path::Path;++use ignore::{self, DirEntry};+use log;++/// A configuration for describing how subjects should be built.+#[derive(Clone, Debug)]+struct Config {+ strip_dot_prefix: bool,+}++impl Default for Config {+ fn default() -> Config {+ Config { strip_dot_prefix: false }+ }+}++/// A builder for constructing things to search over.+#[derive(Clone, Debug)]+pub struct SubjectBuilder {+ config: Config,+}++impl SubjectBuilder {+ /// Return a new subject builder with a default configuration.+ pub fn new() -> SubjectBuilder {+ SubjectBuilder { config: Config::default() }+ }++ /// Create a new subject from a possibly missing directory entry.+ ///+ /// If the directory entry isn't present, then the corresponding error is+ /// logged if messages have been configured. Otherwise, if the subject is+ /// deemed searchable, then it is returned.+ pub fn build_from_result(+ &self,+ result: Result<DirEntry, ignore::Error>,+ ) -> Option<Subject> {+ match result {+ Ok(dent) => self.build(dent),+ Err(err) => {+ err_message!("{}", err);+ None+ }+ }+ }++ /// Create a new subject using this builder's configuration.+ ///+ /// If a subject could not be created or should otherwise not be searched,+ /// then this returns `None` after emitting any relevant log messages.+ pub fn build(&self, dent: DirEntry) -> Option<Subject> {+ let subj = Subject {+ dent: dent,+ strip_dot_prefix: self.config.strip_dot_prefix,+ };+ if let Some(ignore_err) = subj.dent.error() {+ ignore_message!("{}", ignore_err);+ }+ // If this entry was explicitly provided by an end user, then we always+ // want to search it.+ if subj.is_explicit() {+ return Some(subj);+ }+ // At this point, we only want to search something if it's explicitly a+ // file. This omits symlinks. (If ripgrep was configured to follow+ // symlinks, then they have already been followed by the directory+ // traversal.)+ if subj.is_file() {+ return Some(subj);+ }+ // We got nothin. Emit a debug message, but only if this isn't a+ // directory. Otherwise, emitting messages for directories is just+ // noisy.+ if !subj.is_dir() {+ log::debug!(+ "ignoring {}: failed to pass subject filter: \+ file type: {:?}, metadata: {:?}",+ subj.dent.path().display(),+ subj.dent.file_type(),+ subj.dent.metadata()+ );+ }+ None+ }++ /// When enabled, if the subject's file path starts with `./` then it is+ /// stripped.+ ///+ /// This is useful when implicitly searching the current working directory.+ pub fn strip_dot_prefix(&mut self, yes: bool) -> &mut SubjectBuilder {+ self.config.strip_dot_prefix = yes;+ self+ }+}++/// A subject is a thing we want to search. Generally, a subject is either a+/// file or stdin.+#[derive(Clone, Debug)]+pub struct Subject {+ dent: DirEntry,+ strip_dot_prefix: bool,+}++impl Subject {+ /// Return the file path corresponding to this subject.+ ///+ /// If this subject corresponds to stdin, then a special `<stdin>` path+ /// is returned instead.+ pub fn path(&self) -> &Path {+ if self.strip_dot_prefix && self.dent.path().starts_with("./") {+ self.dent.path().strip_prefix("./").unwrap()+ } else {+ self.dent.path()+ }+ }++ /// Returns true if and only if this entry corresponds to stdin.+ pub fn is_stdin(&self) -> bool {+ self.dent.is_stdin()+ }++ /// Returns true if and only if this entry corresponds to a subject to+ /// search that was explicitly supplied by an end user.+ ///+ /// Generally, this corresponds to either stdin or an explicit file path+ /// argument. e.g., in `rg foo some-file ./some-dir/`, `some-file` is+ /// an explicit subject, but, e.g., `./some-dir/some-other-file` is not.+ ///+ /// However, note that ripgrep does not see through shell globbing. e.g.,+ /// in `rg foo ./some-dir/*`, `./some-dir/some-other-file` will be treated+ /// as an explicit subject.+ pub fn is_explicit(&self) -> bool {+ // stdin is obvious. When an entry has a depth of 0, that means it+ // was explicitly provided to our directory iterator, which means it+ // was in turn explicitly provided by the end user. The !is_dir check+ // means that we want to search files even if their symlinks, again,+ // because they were explicitly provided. (And we never want to try+ // to search a directory.)+ self.is_stdin() || (self.dent.depth() == 0 && !self.is_dir())+ }++ /// Returns true if and only if this subject points to a directory after+ /// following symbolic links.+ fn is_dir(&self) -> bool {+ let ft = match self.dent.file_type() {+ None => return false,+ Some(ft) => ft,+ };+ if ft.is_dir() {+ return true;+ }+ // If this is a symlink, then we want to follow it to determine+ // whether it's a directory or not.+ self.dent.path_is_symlink() && self.dent.path().is_dir()+ }++ /// Returns true if and only if this subject points to a file.+ fn is_file(&self) -> bool {+ self.dent.file_type().map_or(false, |ft| ft.is_file())+ }+}
@@ -0,0 +1,3 @@+This project is dual-licensed under the Unlicense and MIT licenses.++You may use this code under the terms of either license.
@@ -0,0 +1,33 @@+[package]+name = "globset"+version = "0.4.4" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+Cross platform single glob and glob set matching. Glob set matching is the+process of matching one or more glob patterns against a single candidate path+simultaneously, and returning all of the globs that matched.+"""+documentation = "https://docs.rs/globset"+homepage = "https://github.com/BurntSushi/ripgrep/tree/master/globset"+repository = "https://github.com/BurntSushi/ripgrep/tree/master/globset"+readme = "README.md"+keywords = ["regex", "glob", "multiple", "set", "pattern"]+license = "Unlicense/MIT"++[lib]+name = "globset"+bench = false++[dependencies]+aho-corasick = "0.7.3"+bstr = { version = "0.2.0", default-features = false, features = ["std"] }+fnv = "1.0.6"+log = "0.4.5"+regex = "1.1.5"++[dev-dependencies]+glob = "0.3.0"+lazy_static = "1"++[features]+simd-accel = []
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,122 @@+globset+=======+Cross platform single glob and glob set matching. Glob set matching is the+process of matching one or more glob patterns against a single candidate path+simultaneously, and returning all of the globs that matched.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/globset)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).++### Documentation++[https://docs.rs/globset](https://docs.rs/globset)++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+globset = "0.3"+```++and this to your crate root:++```rust+extern crate globset;+```++### Example: one glob++This example shows how to match a single glob against a single file path.++```rust+use globset::Glob;++let glob = Glob::new("*.rs")?.compile_matcher();++assert!(glob.is_match("foo.rs"));+assert!(glob.is_match("foo/bar.rs"));+assert!(!glob.is_match("Cargo.toml"));+```++### Example: configuring a glob matcher++This example shows how to use a `GlobBuilder` to configure aspects of match+semantics. In this example, we prevent wildcards from matching path separators.++```rust+use globset::GlobBuilder;++let glob = GlobBuilder::new("*.rs")+ .literal_separator(true).build()?.compile_matcher();++assert!(glob.is_match("foo.rs"));+assert!(!glob.is_match("foo/bar.rs")); // no longer matches+assert!(!glob.is_match("Cargo.toml"));+```++### Example: match multiple globs at once++This example shows how to match multiple glob patterns at once.++```rust+use globset::{Glob, GlobSetBuilder};++let mut builder = GlobSetBuilder::new();+// A GlobBuilder can be used to configure each glob's match semantics+// independently.+builder.add(Glob::new("*.rs")?);+builder.add(Glob::new("src/lib.rs")?);+builder.add(Glob::new("src/**/foo.rs")?);+let set = builder.build()?;++assert_eq!(set.matches("src/bar/baz/foo.rs"), vec![0, 2]);+```++### Performance++This crate implements globs by converting them to regular expressions, and+executing them with the+[`regex`](https://github.com/rust-lang-nursery/regex)+crate.++For single glob matching, performance of this crate should be roughly on par+with the performance of the+[`glob`](https://github.com/rust-lang-nursery/glob)+crate. (`*_regex` correspond to benchmarks for this library while `*_glob`+correspond to benchmarks for the `glob` library.)+Optimizations in the `regex` crate may propel this library past `glob`,+particularly when matching longer paths.++```+test ext_glob ... bench: 425 ns/iter (+/- 21)+test ext_regex ... bench: 175 ns/iter (+/- 10)+test long_glob ... bench: 182 ns/iter (+/- 11)+test long_regex ... bench: 173 ns/iter (+/- 10)+test short_glob ... bench: 69 ns/iter (+/- 4)+test short_regex ... bench: 83 ns/iter (+/- 2)+```++The primary performance advantage of this crate is when matching multiple+globs against a single path. With the `glob` crate, one must match each glob+synchronously, one after the other. In this crate, many can be matched+simultaneously. For example:++```+test many_short_glob ... bench: 1,063 ns/iter (+/- 47)+test many_short_regex_set ... bench: 186 ns/iter (+/- 11)+```++### Comparison with the [`glob`](https://github.com/rust-lang-nursery/glob) crate++* Supports alternate "or" globs, e.g., `*.{foo,bar}`.+* Can match non-UTF-8 file paths correctly.+* Supports matching multiple globs at once.+* Doesn't provide a recursive directory iterator of matching file paths,+ although I believe this crate should grow one eventually.+* Supports case insensitive and require-literal-separator match options, but+ **doesn't** support the require-literal-leading-dot option.
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
crates/globset/benches/bench.rs
@@ -0,0 +1,116 @@+/*!+This module benchmarks the glob implementation. For benchmarks on the ripgrep+tool itself, see the benchsuite directory.+*/+#![feature(test)]++extern crate glob;+extern crate globset;+extern crate regex;+extern crate test;++use globset::{Candidate, Glob, GlobMatcher, GlobSet, GlobSetBuilder};++const EXT: &'static str = "some/a/bigger/path/to/the/crazy/needle.txt";+const EXT_PAT: &'static str = "*.txt";++const SHORT: &'static str = "some/needle.txt";+const SHORT_PAT: &'static str = "some/**/needle.txt";++const LONG: &'static str = "some/a/bigger/path/to/the/crazy/needle.txt";+const LONG_PAT: &'static str = "some/**/needle.txt";++fn new_glob(pat: &str) -> glob::Pattern {+ glob::Pattern::new(pat).unwrap()+}++fn new_reglob(pat: &str) -> GlobMatcher {+ Glob::new(pat).unwrap().compile_matcher()+}++fn new_reglob_many(pats: &[&str]) -> GlobSet {+ let mut builder = GlobSetBuilder::new();+ for pat in pats {+ builder.add(Glob::new(pat).unwrap());+ }+ builder.build().unwrap()+}++#[bench]+fn ext_glob(b: &mut test::Bencher) {+ let pat = new_glob(EXT_PAT);+ b.iter(|| assert!(pat.matches(EXT)));+}++#[bench]+fn ext_regex(b: &mut test::Bencher) {+ let set = new_reglob(EXT_PAT);+ let cand = Candidate::new(EXT);+ b.iter(|| assert!(set.is_match_candidate(&cand)));+}++#[bench]+fn short_glob(b: &mut test::Bencher) {+ let pat = new_glob(SHORT_PAT);+ b.iter(|| assert!(pat.matches(SHORT)));+}++#[bench]+fn short_regex(b: &mut test::Bencher) {+ let set = new_reglob(SHORT_PAT);+ let cand = Candidate::new(SHORT);+ b.iter(|| assert!(set.is_match_candidate(&cand)));+}++#[bench]+fn long_glob(b: &mut test::Bencher) {+ let pat = new_glob(LONG_PAT);+ b.iter(|| assert!(pat.matches(LONG)));+}++#[bench]+fn long_regex(b: &mut test::Bencher) {+ let set = new_reglob(LONG_PAT);+ let cand = Candidate::new(LONG);+ b.iter(|| assert!(set.is_match_candidate(&cand)));+}++const MANY_SHORT_GLOBS: &'static [&'static str] = &[+ // Taken from a random .gitignore on my system.+ ".*.swp",+ "tags",+ "target",+ "*.lock",+ "tmp",+ "*.csv",+ "*.fst",+ "*-got",+ "*.csv.idx",+ "words",+ "98m*",+ "dict",+ "test",+ "months",+];++const MANY_SHORT_SEARCH: &'static str = "98m-blah.csv.idx";++#[bench]+fn many_short_glob(b: &mut test::Bencher) {+ let pats: Vec<_> = MANY_SHORT_GLOBS.iter().map(|&s| new_glob(s)).collect();+ b.iter(|| {+ let mut count = 0;+ for pat in &pats {+ if pat.matches(MANY_SHORT_SEARCH) {+ count += 1;+ }+ }+ assert_eq!(2, count);+ })+}++#[bench]+fn many_short_regex_set(b: &mut test::Bencher) {+ let set = new_reglob_many(MANY_SHORT_GLOBS);+ b.iter(|| assert_eq!(2, set.matches(MANY_SHORT_SEARCH).iter().count()));+}
@@ -0,0 +1,1521 @@+use std::fmt;+use std::hash;+use std::iter;+use std::ops::{Deref, DerefMut};+use std::path::{is_separator, Path};+use std::str;++use regex;+use regex::bytes::Regex;++use {new_regex, Candidate, Error, ErrorKind};++/// Describes a matching strategy for a particular pattern.+///+/// This provides a way to more quickly determine whether a pattern matches+/// a particular file path in a way that scales with a large number of+/// patterns. For example, if many patterns are of the form `*.ext`, then it's+/// possible to test whether any of those patterns matches by looking up a+/// file path's extension in a hash table.+#[derive(Clone, Debug, Eq, PartialEq)]+pub enum MatchStrategy {+ /// A pattern matches if and only if the entire file path matches this+ /// literal string.+ Literal(String),+ /// A pattern matches if and only if the file path's basename matches this+ /// literal string.+ BasenameLiteral(String),+ /// A pattern matches if and only if the file path's extension matches this+ /// literal string.+ Extension(String),+ /// A pattern matches if and only if this prefix literal is a prefix of the+ /// candidate file path.+ Prefix(String),+ /// A pattern matches if and only if this prefix literal is a prefix of the+ /// candidate file path.+ ///+ /// An exception: if `component` is true, then `suffix` must appear at the+ /// beginning of a file path or immediately following a `/`.+ Suffix {+ /// The actual suffix.+ suffix: String,+ /// Whether this must start at the beginning of a path component.+ component: bool,+ },+ /// A pattern matches only if the given extension matches the file path's+ /// extension. Note that this is a necessary but NOT sufficient criterion.+ /// Namely, if the extension matches, then a full regex search is still+ /// required.+ RequiredExtension(String),+ /// A regex needs to be used for matching.+ Regex,+}++impl MatchStrategy {+ /// Returns a matching strategy for the given pattern.+ pub fn new(pat: &Glob) -> MatchStrategy {+ if let Some(lit) = pat.basename_literal() {+ MatchStrategy::BasenameLiteral(lit)+ } else if let Some(lit) = pat.literal() {+ MatchStrategy::Literal(lit)+ } else if let Some(ext) = pat.ext() {+ MatchStrategy::Extension(ext)+ } else if let Some(prefix) = pat.prefix() {+ MatchStrategy::Prefix(prefix)+ } else if let Some((suffix, component)) = pat.suffix() {+ MatchStrategy::Suffix { suffix: suffix, component: component }+ } else if let Some(ext) = pat.required_ext() {+ MatchStrategy::RequiredExtension(ext)+ } else {+ MatchStrategy::Regex+ }+ }+}++/// Glob represents a successfully parsed shell glob pattern.+///+/// It cannot be used directly to match file paths, but it can be converted+/// to a regular expression string or a matcher.+#[derive(Clone, Debug, Eq)]+pub struct Glob {+ glob: String,+ re: String,+ opts: GlobOptions,+ tokens: Tokens,+}++impl PartialEq for Glob {+ fn eq(&self, other: &Glob) -> bool {+ self.glob == other.glob && self.opts == other.opts+ }+}++impl hash::Hash for Glob {+ fn hash<H: hash::Hasher>(&self, state: &mut H) {+ self.glob.hash(state);+ self.opts.hash(state);+ }+}++impl fmt::Display for Glob {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ self.glob.fmt(f)+ }+}++impl str::FromStr for Glob {+ type Err = Error;++ fn from_str(glob: &str) -> Result<Self, Self::Err> {+ Self::new(glob)+ }+}++/// A matcher for a single pattern.+#[derive(Clone, Debug)]+pub struct GlobMatcher {+ /// The underlying pattern.+ pat: Glob,+ /// The pattern, as a compiled regex.+ re: Regex,+}++impl GlobMatcher {+ /// Tests whether the given path matches this pattern or not.+ pub fn is_match<P: AsRef<Path>>(&self, path: P) -> bool {+ self.is_match_candidate(&Candidate::new(path.as_ref()))+ }++ /// Tests whether the given path matches this pattern or not.+ pub fn is_match_candidate(&self, path: &Candidate) -> bool {+ self.re.is_match(&path.path)+ }++ /// Returns the `Glob` used to compile this matcher.+ pub fn glob(&self) -> &Glob {+ &self.pat+ }+}++/// A strategic matcher for a single pattern.+#[cfg(test)]+#[derive(Clone, Debug)]+struct GlobStrategic {+ /// The match strategy to use.+ strategy: MatchStrategy,+ /// The underlying pattern.+ pat: Glob,+ /// The pattern, as a compiled regex.+ re: Regex,+}++#[cfg(test)]+impl GlobStrategic {+ /// Tests whether the given path matches this pattern or not.+ fn is_match<P: AsRef<Path>>(&self, path: P) -> bool {+ self.is_match_candidate(&Candidate::new(path.as_ref()))+ }++ /// Tests whether the given path matches this pattern or not.+ fn is_match_candidate(&self, candidate: &Candidate) -> bool {+ let byte_path = &*candidate.path;++ match self.strategy {+ MatchStrategy::Literal(ref lit) => lit.as_bytes() == byte_path,+ MatchStrategy::BasenameLiteral(ref lit) => {+ lit.as_bytes() == &*candidate.basename+ }+ MatchStrategy::Extension(ref ext) => {+ ext.as_bytes() == &*candidate.ext+ }+ MatchStrategy::Prefix(ref pre) => {+ starts_with(pre.as_bytes(), byte_path)+ }+ MatchStrategy::Suffix { ref suffix, component } => {+ if component && byte_path == &suffix.as_bytes()[1..] {+ return true;+ }+ ends_with(suffix.as_bytes(), byte_path)+ }+ MatchStrategy::RequiredExtension(ref ext) => {+ let ext = ext.as_bytes();+ &*candidate.ext == ext && self.re.is_match(byte_path)+ }+ MatchStrategy::Regex => self.re.is_match(byte_path),+ }+ }+}++/// A builder for a pattern.+///+/// This builder enables configuring the match semantics of a pattern. For+/// example, one can make matching case insensitive.+///+/// The lifetime `'a` refers to the lifetime of the pattern string.+#[derive(Clone, Debug)]+pub struct GlobBuilder<'a> {+ /// The glob pattern to compile.+ glob: &'a str,+ /// Options for the pattern.+ opts: GlobOptions,+}++#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]+struct GlobOptions {+ /// Whether to match case insensitively.+ case_insensitive: bool,+ /// Whether to require a literal separator to match a separator in a file+ /// path. e.g., when enabled, `*` won't match `/`.+ literal_separator: bool,+ /// Whether or not to use `\` to escape special characters.+ /// e.g., when enabled, `\*` will match a literal `*`.+ backslash_escape: bool,+}++impl GlobOptions {+ fn default() -> GlobOptions {+ GlobOptions {+ case_insensitive: false,+ literal_separator: false,+ backslash_escape: !is_separator('\\'),+ }+ }+}++#[derive(Clone, Debug, Default, Eq, PartialEq)]+struct Tokens(Vec<Token>);++impl Deref for Tokens {+ type Target = Vec<Token>;+ fn deref(&self) -> &Vec<Token> {+ &self.0+ }+}++impl DerefMut for Tokens {+ fn deref_mut(&mut self) -> &mut Vec<Token> {+ &mut self.0+ }+}++#[derive(Clone, Debug, Eq, PartialEq)]+enum Token {+ Literal(char),+ Any,+ ZeroOrMore,+ RecursivePrefix,+ RecursiveSuffix,+ RecursiveZeroOrMore,+ Class { negated: bool, ranges: Vec<(char, char)> },+ Alternates(Vec<Tokens>),+}++impl Glob {+ /// Builds a new pattern with default options.+ pub fn new(glob: &str) -> Result<Glob, Error> {+ GlobBuilder::new(glob).build()+ }++ /// Returns a matcher for this pattern.+ pub fn compile_matcher(&self) -> GlobMatcher {+ let re =+ new_regex(&self.re).expect("regex compilation shouldn't fail");+ GlobMatcher { pat: self.clone(), re: re }+ }++ /// Returns a strategic matcher.+ ///+ /// This isn't exposed because it's not clear whether it's actually+ /// faster than just running a regex for a *single* pattern. If it+ /// is faster, then GlobMatcher should do it automatically.+ #[cfg(test)]+ fn compile_strategic_matcher(&self) -> GlobStrategic {+ let strategy = MatchStrategy::new(self);+ let re =+ new_regex(&self.re).expect("regex compilation shouldn't fail");+ GlobStrategic { strategy: strategy, pat: self.clone(), re: re }+ }++ /// Returns the original glob pattern used to build this pattern.+ pub fn glob(&self) -> &str {+ &self.glob+ }++ /// Returns the regular expression string for this glob.+ ///+ /// Note that regular expressions for globs are intended to be matched on+ /// arbitrary bytes (`&[u8]`) instead of Unicode strings (`&str`). In+ /// particular, globs are frequently used on file paths, where there is no+ /// general guarantee that file paths are themselves valid UTF-8. As a+ /// result, callers will need to ensure that they are using a regex API+ /// that can match on arbitrary bytes. For example, the+ /// [`regex`](https://crates.io/regex)+ /// crate's+ /// [`Regex`](https://docs.rs/regex/*/regex/struct.Regex.html)+ /// API is not suitable for this since it matches on `&str`, but its+ /// [`bytes::Regex`](https://docs.rs/regex/*/regex/bytes/struct.Regex.html)+ /// API is suitable for this.+ pub fn regex(&self) -> &str {+ &self.re+ }++ /// Returns the pattern as a literal if and only if the pattern must match+ /// an entire path exactly.+ ///+ /// The basic format of these patterns is `{literal}`.+ fn literal(&self) -> Option<String> {+ if self.opts.case_insensitive {+ return None;+ }+ let mut lit = String::new();+ for t in &*self.tokens {+ match *t {+ Token::Literal(c) => lit.push(c),+ _ => return None,+ }+ }+ if lit.is_empty() {+ None+ } else {+ Some(lit)+ }+ }++ /// Returns an extension if this pattern matches a file path if and only+ /// if the file path has the extension returned.+ ///+ /// Note that this extension returned differs from the extension that+ /// std::path::Path::extension returns. Namely, this extension includes+ /// the '.'. Also, paths like `.rs` are considered to have an extension+ /// of `.rs`.+ fn ext(&self) -> Option<String> {+ if self.opts.case_insensitive {+ return None;+ }+ let start = match self.tokens.get(0) {+ Some(&Token::RecursivePrefix) => 1,+ Some(_) => 0,+ _ => return None,+ };+ match self.tokens.get(start) {+ Some(&Token::ZeroOrMore) => {+ // If there was no recursive prefix, then we only permit+ // `*` if `*` can match a `/`. For example, if `*` can't+ // match `/`, then `*.c` doesn't match `foo/bar.c`.+ if start == 0 && self.opts.literal_separator {+ return None;+ }+ }+ _ => return None,+ }+ match self.tokens.get(start + 1) {+ Some(&Token::Literal('.')) => {}+ _ => return None,+ }+ let mut lit = ".".to_string();+ for t in self.tokens[start + 2..].iter() {+ match *t {+ Token::Literal('.') | Token::Literal('/') => return None,+ Token::Literal(c) => lit.push(c),+ _ => return None,+ }+ }+ if lit.is_empty() {+ None+ } else {+ Some(lit)+ }+ }++ /// This is like `ext`, but returns an extension even if it isn't sufficent+ /// to imply a match. Namely, if an extension is returned, then it is+ /// necessary but not sufficient for a match.+ fn required_ext(&self) -> Option<String> {+ if self.opts.case_insensitive {+ return None;+ }+ // We don't care at all about the beginning of this pattern. All we+ // need to check for is if it ends with a literal of the form `.ext`.+ let mut ext: Vec<char> = vec![]; // built in reverse+ for t in self.tokens.iter().rev() {+ match *t {+ Token::Literal('/') => return None,+ Token::Literal(c) => {+ ext.push(c);+ if c == '.' {+ break;+ }+ }+ _ => return None,+ }+ }+ if ext.last() != Some(&'.') {+ None+ } else {+ ext.reverse();+ Some(ext.into_iter().collect())+ }+ }++ /// Returns a literal prefix of this pattern if the entire pattern matches+ /// if the literal prefix matches.+ fn prefix(&self) -> Option<String> {+ if self.opts.case_insensitive {+ return None;+ }+ let end = match self.tokens.last() {+ Some(&Token::ZeroOrMore) => {+ if self.opts.literal_separator {+ // If a trailing `*` can't match a `/`, then we can't+ // assume a match of the prefix corresponds to a match+ // of the overall pattern. e.g., `foo/*` with+ // `literal_separator` enabled matches `foo/bar` but not+ // `foo/bar/baz`, even though `foo/bar/baz` has a `foo/`+ // literal prefix.+ return None;+ }+ self.tokens.len() - 1+ }+ _ => self.tokens.len(),+ };+ let mut lit = String::new();+ for t in &self.tokens[0..end] {+ match *t {+ Token::Literal(c) => lit.push(c),+ _ => return None,+ }+ }+ if lit.is_empty() {+ None+ } else {+ Some(lit)+ }+ }++ /// Returns a literal suffix of this pattern if the entire pattern matches+ /// if the literal suffix matches.+ ///+ /// If a literal suffix is returned and it must match either the entire+ /// file path or be preceded by a `/`, then also return true. This happens+ /// with a pattern like `**/foo/bar`. Namely, this pattern matches+ /// `foo/bar` and `baz/foo/bar`, but not `foofoo/bar`. In this case, the+ /// suffix returned is `/foo/bar` (but should match the entire path+ /// `foo/bar`).+ ///+ /// When this returns true, the suffix literal is guaranteed to start with+ /// a `/`.+ fn suffix(&self) -> Option<(String, bool)> {+ if self.opts.case_insensitive {+ return None;+ }+ let mut lit = String::new();+ let (start, entire) = match self.tokens.get(0) {+ Some(&Token::RecursivePrefix) => {+ // We only care if this follows a path component if the next+ // token is a literal.+ if let Some(&Token::Literal(_)) = self.tokens.get(1) {+ lit.push('/');+ (1, true)+ } else {+ (1, false)+ }+ }+ _ => (0, false),+ };+ let start = match self.tokens.get(start) {+ Some(&Token::ZeroOrMore) => {+ // If literal_separator is enabled, then a `*` can't+ // necessarily match everything, so reporting a suffix match+ // as a match of the pattern would be a false positive.+ if self.opts.literal_separator {+ return None;+ }+ start + 1+ }+ _ => start,+ };+ for t in &self.tokens[start..] {+ match *t {+ Token::Literal(c) => lit.push(c),+ _ => return None,+ }+ }+ if lit.is_empty() || lit == "/" {+ None+ } else {+ Some((lit, entire))+ }+ }++ /// If this pattern only needs to inspect the basename of a file path,+ /// then the tokens corresponding to only the basename match are returned.+ ///+ /// For example, given a pattern of `**/*.foo`, only the tokens+ /// corresponding to `*.foo` are returned.+ ///+ /// Note that this will return None if any match of the basename tokens+ /// doesn't correspond to a match of the entire pattern. For example, the+ /// glob `foo` only matches when a file path has a basename of `foo`, but+ /// doesn't *always* match when a file path has a basename of `foo`. e.g.,+ /// `foo` doesn't match `abc/foo`.+ fn basename_tokens(&self) -> Option<&[Token]> {+ if self.opts.case_insensitive {+ return None;+ }+ let start = match self.tokens.get(0) {+ Some(&Token::RecursivePrefix) => 1,+ _ => {+ // With nothing to gobble up the parent portion of a path,+ // we can't assume that matching on only the basename is+ // correct.+ return None;+ }+ };+ if self.tokens[start..].is_empty() {+ return None;+ }+ for t in &self.tokens[start..] {+ match *t {+ Token::Literal('/') => return None,+ Token::Literal(_) => {} // OK+ Token::Any | Token::ZeroOrMore => {+ if !self.opts.literal_separator {+ // In this case, `*` and `?` can match a path+ // separator, which means this could reach outside+ // the basename.+ return None;+ }+ }+ Token::RecursivePrefix+ | Token::RecursiveSuffix+ | Token::RecursiveZeroOrMore => {+ return None;+ }+ Token::Class { .. } | Token::Alternates(..) => {+ // We *could* be a little smarter here, but either one+ // of these is going to prevent our literal optimizations+ // anyway, so give up.+ return None;+ }+ }+ }+ Some(&self.tokens[start..])+ }++ /// Returns the pattern as a literal if and only if the pattern exclusively+ /// matches the basename of a file path *and* is a literal.+ ///+ /// The basic format of these patterns is `**/{literal}`, where `{literal}`+ /// does not contain a path separator.+ fn basename_literal(&self) -> Option<String> {+ let tokens = match self.basename_tokens() {+ None => return None,+ Some(tokens) => tokens,+ };+ let mut lit = String::new();+ for t in tokens {+ match *t {+ Token::Literal(c) => lit.push(c),+ _ => return None,+ }+ }+ Some(lit)+ }+}++impl<'a> GlobBuilder<'a> {+ /// Create a new builder for the pattern given.+ ///+ /// The pattern is not compiled until `build` is called.+ pub fn new(glob: &'a str) -> GlobBuilder<'a> {+ GlobBuilder { glob: glob, opts: GlobOptions::default() }+ }++ /// Parses and builds the pattern.+ pub fn build(&self) -> Result<Glob, Error> {+ let mut p = Parser {+ glob: &self.glob,+ stack: vec![Tokens::default()],+ chars: self.glob.chars().peekable(),+ prev: None,+ cur: None,+ opts: &self.opts,+ };+ p.parse()?;+ if p.stack.is_empty() {+ Err(Error {+ glob: Some(self.glob.to_string()),+ kind: ErrorKind::UnopenedAlternates,+ })+ } else if p.stack.len() > 1 {+ Err(Error {+ glob: Some(self.glob.to_string()),+ kind: ErrorKind::UnclosedAlternates,+ })+ } else {+ let tokens = p.stack.pop().unwrap();+ Ok(Glob {+ glob: self.glob.to_string(),+ re: tokens.to_regex_with(&self.opts),+ opts: self.opts,+ tokens: tokens,+ })+ }+ }++ /// Toggle whether the pattern matches case insensitively or not.+ ///+ /// This is disabled by default.+ pub fn case_insensitive(&mut self, yes: bool) -> &mut GlobBuilder<'a> {+ self.opts.case_insensitive = yes;+ self+ }++ /// Toggle whether a literal `/` is required to match a path separator.+ pub fn literal_separator(&mut self, yes: bool) -> &mut GlobBuilder<'a> {+ self.opts.literal_separator = yes;+ self+ }++ /// When enabled, a back slash (`\`) may be used to escape+ /// special characters in a glob pattern. Additionally, this will+ /// prevent `\` from being interpreted as a path separator on all+ /// platforms.+ ///+ /// This is enabled by default on platforms where `\` is not a+ /// path separator and disabled by default on platforms where `\`+ /// is a path separator.+ pub fn backslash_escape(&mut self, yes: bool) -> &mut GlobBuilder<'a> {+ self.opts.backslash_escape = yes;+ self+ }+}++impl Tokens {+ /// Convert this pattern to a string that is guaranteed to be a valid+ /// regular expression and will represent the matching semantics of this+ /// glob pattern and the options given.+ fn to_regex_with(&self, options: &GlobOptions) -> String {+ let mut re = String::new();+ re.push_str("(?-u)");+ if options.case_insensitive {+ re.push_str("(?i)");+ }+ re.push('^');+ // Special case. If the entire glob is just `**`, then it should match+ // everything.+ if self.len() == 1 && self[0] == Token::RecursivePrefix {+ re.push_str(".*");+ re.push('$');+ return re;+ }+ self.tokens_to_regex(options, &self, &mut re);+ re.push('$');+ re+ }++ fn tokens_to_regex(+ &self,+ options: &GlobOptions,+ tokens: &[Token],+ re: &mut String,+ ) {+ for tok in tokens {+ match *tok {+ Token::Literal(c) => {+ re.push_str(&char_to_escaped_literal(c));+ }+ Token::Any => {+ if options.literal_separator {+ re.push_str("[^/]");+ } else {+ re.push_str(".");+ }+ }+ Token::ZeroOrMore => {+ if options.literal_separator {+ re.push_str("[^/]*");+ } else {+ re.push_str(".*");+ }+ }+ Token::RecursivePrefix => {+ re.push_str("(?:/?|.*/)");+ }+ Token::RecursiveSuffix => {+ re.push_str("(?:/?|/.*)");+ }+ Token::RecursiveZeroOrMore => {+ re.push_str("(?:/|/.*/)");+ }+ Token::Class { negated, ref ranges } => {+ re.push('[');+ if negated {+ re.push('^');+ }+ for r in ranges {+ if r.0 == r.1 {+ // Not strictly necessary, but nicer to look at.+ re.push_str(&char_to_escaped_literal(r.0));+ } else {+ re.push_str(&char_to_escaped_literal(r.0));+ re.push('-');+ re.push_str(&char_to_escaped_literal(r.1));+ }+ }+ re.push(']');+ }+ Token::Alternates(ref patterns) => {+ let mut parts = vec![];+ for pat in patterns {+ let mut altre = String::new();+ self.tokens_to_regex(options, &pat, &mut altre);+ if !altre.is_empty() {+ parts.push(altre);+ }+ }++ // It is possible to have an empty set in which case the+ // resulting alternation '()' would be an error.+ if !parts.is_empty() {+ re.push('(');+ re.push_str(&parts.join("|"));+ re.push(')');+ }+ }+ }+ }+ }+}++/// Convert a Unicode scalar value to an escaped string suitable for use as+/// a literal in a non-Unicode regex.+fn char_to_escaped_literal(c: char) -> String {+ bytes_to_escaped_literal(&c.to_string().into_bytes())+}++/// Converts an arbitrary sequence of bytes to a UTF-8 string. All non-ASCII+/// code units are converted to their escaped form.+fn bytes_to_escaped_literal(bs: &[u8]) -> String {+ let mut s = String::with_capacity(bs.len());+ for &b in bs {+ if b <= 0x7F {+ s.push_str(®ex::escape(&(b as char).to_string()));+ } else {+ s.push_str(&format!("\\x{:02x}", b));+ }+ }+ s+}++struct Parser<'a> {+ glob: &'a str,+ stack: Vec<Tokens>,+ chars: iter::Peekable<str::Chars<'a>>,+ prev: Option<char>,+ cur: Option<char>,+ opts: &'a GlobOptions,+}++impl<'a> Parser<'a> {+ fn error(&self, kind: ErrorKind) -> Error {+ Error { glob: Some(self.glob.to_string()), kind: kind }+ }++ fn parse(&mut self) -> Result<(), Error> {+ while let Some(c) = self.bump() {+ match c {+ '?' => self.push_token(Token::Any)?,+ '*' => self.parse_star()?,+ '[' => self.parse_class()?,+ '{' => self.push_alternate()?,+ '}' => self.pop_alternate()?,+ ',' => self.parse_comma()?,+ '\\' => self.parse_backslash()?,+ c => self.push_token(Token::Literal(c))?,+ }+ }+ Ok(())+ }++ fn push_alternate(&mut self) -> Result<(), Error> {+ if self.stack.len() > 1 {+ return Err(self.error(ErrorKind::NestedAlternates));+ }+ Ok(self.stack.push(Tokens::default()))+ }++ fn pop_alternate(&mut self) -> Result<(), Error> {+ let mut alts = vec![];+ while self.stack.len() >= 2 {+ alts.push(self.stack.pop().unwrap());+ }+ self.push_token(Token::Alternates(alts))+ }++ fn push_token(&mut self, tok: Token) -> Result<(), Error> {+ if let Some(ref mut pat) = self.stack.last_mut() {+ return Ok(pat.push(tok));+ }+ Err(self.error(ErrorKind::UnopenedAlternates))+ }++ fn pop_token(&mut self) -> Result<Token, Error> {+ if let Some(ref mut pat) = self.stack.last_mut() {+ return Ok(pat.pop().unwrap());+ }+ Err(self.error(ErrorKind::UnopenedAlternates))+ }++ fn have_tokens(&self) -> Result<bool, Error> {+ match self.stack.last() {+ None => Err(self.error(ErrorKind::UnopenedAlternates)),+ Some(ref pat) => Ok(!pat.is_empty()),+ }+ }++ fn parse_comma(&mut self) -> Result<(), Error> {+ // If we aren't inside a group alternation, then don't+ // treat commas specially. Otherwise, we need to start+ // a new alternate.+ if self.stack.len() <= 1 {+ self.push_token(Token::Literal(','))+ } else {+ Ok(self.stack.push(Tokens::default()))+ }+ }++ fn parse_backslash(&mut self) -> Result<(), Error> {+ if self.opts.backslash_escape {+ match self.bump() {+ None => Err(self.error(ErrorKind::DanglingEscape)),+ Some(c) => self.push_token(Token::Literal(c)),+ }+ } else if is_separator('\\') {+ // Normalize all patterns to use / as a separator.+ self.push_token(Token::Literal('/'))+ } else {+ self.push_token(Token::Literal('\\'))+ }+ }++ fn parse_star(&mut self) -> Result<(), Error> {+ let prev = self.prev;+ if self.peek() != Some('*') {+ self.push_token(Token::ZeroOrMore)?;+ return Ok(());+ }+ assert!(self.bump() == Some('*'));+ if !self.have_tokens()? {+ if !self.peek().map_or(true, is_separator) {+ self.push_token(Token::ZeroOrMore)?;+ self.push_token(Token::ZeroOrMore)?;+ } else {+ self.push_token(Token::RecursivePrefix)?;+ assert!(self.bump().map_or(true, is_separator));+ }+ return Ok(());+ }++ if !prev.map(is_separator).unwrap_or(false) {+ if self.stack.len() <= 1+ || (prev != Some(',') && prev != Some('{'))+ {+ self.push_token(Token::ZeroOrMore)?;+ self.push_token(Token::ZeroOrMore)?;+ return Ok(());+ }+ }+ let is_suffix = match self.peek() {+ None => {+ assert!(self.bump().is_none());+ true+ }+ Some(',') | Some('}') if self.stack.len() >= 2 => true,+ Some(c) if is_separator(c) => {+ assert!(self.bump().map(is_separator).unwrap_or(false));+ false+ }+ _ => {+ self.push_token(Token::ZeroOrMore)?;+ self.push_token(Token::ZeroOrMore)?;+ return Ok(());+ }+ };+ match self.pop_token()? {+ Token::RecursivePrefix => {+ self.push_token(Token::RecursivePrefix)?;+ }+ Token::RecursiveSuffix => {+ self.push_token(Token::RecursiveSuffix)?;+ }+ _ => {+ if is_suffix {+ self.push_token(Token::RecursiveSuffix)?;+ } else {+ self.push_token(Token::RecursiveZeroOrMore)?;+ }+ }+ }+ Ok(())+ }++ fn parse_class(&mut self) -> Result<(), Error> {+ fn add_to_last_range(+ glob: &str,+ r: &mut (char, char),+ add: char,+ ) -> Result<(), Error> {+ r.1 = add;+ if r.1 < r.0 {+ Err(Error {+ glob: Some(glob.to_string()),+ kind: ErrorKind::InvalidRange(r.0, r.1),+ })+ } else {+ Ok(())+ }+ }+ let mut ranges = vec![];+ let negated = match self.chars.peek() {+ Some(&'!') | Some(&'^') => {+ let bump = self.bump();+ assert!(bump == Some('!') || bump == Some('^'));+ true+ }+ _ => false,+ };+ let mut first = true;+ let mut in_range = false;+ loop {+ let c = match self.bump() {+ Some(c) => c,+ // The only way to successfully break this loop is to observe+ // a ']'.+ None => return Err(self.error(ErrorKind::UnclosedClass)),+ };+ match c {+ ']' => {+ if first {+ ranges.push((']', ']'));+ } else {+ break;+ }+ }+ '-' => {+ if first {+ ranges.push(('-', '-'));+ } else if in_range {+ // invariant: in_range is only set when there is+ // already at least one character seen.+ let r = ranges.last_mut().unwrap();+ add_to_last_range(&self.glob, r, '-')?;+ in_range = false;+ } else {+ assert!(!ranges.is_empty());+ in_range = true;+ }+ }+ c => {+ if in_range {+ // invariant: in_range is only set when there is+ // already at least one character seen.+ add_to_last_range(+ &self.glob,+ ranges.last_mut().unwrap(),+ c,+ )?;+ } else {+ ranges.push((c, c));+ }+ in_range = false;+ }+ }+ first = false;+ }+ if in_range {+ // Means that the last character in the class was a '-', so add+ // it as a literal.+ ranges.push(('-', '-'));+ }+ self.push_token(Token::Class { negated: negated, ranges: ranges })+ }++ fn bump(&mut self) -> Option<char> {+ self.prev = self.cur;+ self.cur = self.chars.next();+ self.cur+ }++ fn peek(&mut self) -> Option<char> {+ self.chars.peek().map(|&ch| ch)+ }+}++#[cfg(test)]+fn starts_with(needle: &[u8], haystack: &[u8]) -> bool {+ needle.len() <= haystack.len() && needle == &haystack[..needle.len()]+}++#[cfg(test)]+fn ends_with(needle: &[u8], haystack: &[u8]) -> bool {+ if needle.len() > haystack.len() {+ return false;+ }+ needle == &haystack[haystack.len() - needle.len()..]+}++#[cfg(test)]+mod tests {+ use super::Token::*;+ use super::{Glob, GlobBuilder, Token};+ use {ErrorKind, GlobSetBuilder};++ #[derive(Clone, Copy, Debug, Default)]+ struct Options {+ casei: Option<bool>,+ litsep: Option<bool>,+ bsesc: Option<bool>,+ }++ macro_rules! syntax {+ ($name:ident, $pat:expr, $tokens:expr) => {+ #[test]+ fn $name() {+ let pat = Glob::new($pat).unwrap();+ assert_eq!($tokens, pat.tokens.0);+ }+ };+ }++ macro_rules! syntaxerr {+ ($name:ident, $pat:expr, $err:expr) => {+ #[test]+ fn $name() {+ let err = Glob::new($pat).unwrap_err();+ assert_eq!(&$err, err.kind());+ }+ };+ }++ macro_rules! toregex {+ ($name:ident, $pat:expr, $re:expr) => {+ toregex!($name, $pat, $re, Options::default());+ };+ ($name:ident, $pat:expr, $re:expr, $options:expr) => {+ #[test]+ fn $name() {+ let mut builder = GlobBuilder::new($pat);+ if let Some(casei) = $options.casei {+ builder.case_insensitive(casei);+ }+ if let Some(litsep) = $options.litsep {+ builder.literal_separator(litsep);+ }+ if let Some(bsesc) = $options.bsesc {+ builder.backslash_escape(bsesc);+ }+ let pat = builder.build().unwrap();+ assert_eq!(format!("(?-u){}", $re), pat.regex());+ }+ };+ }++ macro_rules! matches {+ ($name:ident, $pat:expr, $path:expr) => {+ matches!($name, $pat, $path, Options::default());+ };+ ($name:ident, $pat:expr, $path:expr, $options:expr) => {+ #[test]+ fn $name() {+ let mut builder = GlobBuilder::new($pat);+ if let Some(casei) = $options.casei {+ builder.case_insensitive(casei);+ }+ if let Some(litsep) = $options.litsep {+ builder.literal_separator(litsep);+ }+ if let Some(bsesc) = $options.bsesc {+ builder.backslash_escape(bsesc);+ }+ let pat = builder.build().unwrap();+ let matcher = pat.compile_matcher();+ let strategic = pat.compile_strategic_matcher();+ let set = GlobSetBuilder::new().add(pat).build().unwrap();+ assert!(matcher.is_match($path));+ assert!(strategic.is_match($path));+ assert!(set.is_match($path));+ }+ };+ }++ macro_rules! nmatches {+ ($name:ident, $pat:expr, $path:expr) => {+ nmatches!($name, $pat, $path, Options::default());+ };+ ($name:ident, $pat:expr, $path:expr, $options:expr) => {+ #[test]+ fn $name() {+ let mut builder = GlobBuilder::new($pat);+ if let Some(casei) = $options.casei {+ builder.case_insensitive(casei);+ }+ if let Some(litsep) = $options.litsep {+ builder.literal_separator(litsep);+ }+ if let Some(bsesc) = $options.bsesc {+ builder.backslash_escape(bsesc);+ }+ let pat = builder.build().unwrap();+ let matcher = pat.compile_matcher();+ let strategic = pat.compile_strategic_matcher();+ let set = GlobSetBuilder::new().add(pat).build().unwrap();+ assert!(!matcher.is_match($path));+ assert!(!strategic.is_match($path));+ assert!(!set.is_match($path));+ }+ };+ }++ fn s(string: &str) -> String {+ string.to_string()+ }++ fn class(s: char, e: char) -> Token {+ Class { negated: false, ranges: vec![(s, e)] }+ }++ fn classn(s: char, e: char) -> Token {+ Class { negated: true, ranges: vec![(s, e)] }+ }++ fn rclass(ranges: &[(char, char)]) -> Token {+ Class { negated: false, ranges: ranges.to_vec() }+ }++ fn rclassn(ranges: &[(char, char)]) -> Token {+ Class { negated: true, ranges: ranges.to_vec() }+ }++ syntax!(literal1, "a", vec![Literal('a')]);+ syntax!(literal2, "ab", vec![Literal('a'), Literal('b')]);+ syntax!(any1, "?", vec![Any]);+ syntax!(any2, "a?b", vec![Literal('a'), Any, Literal('b')]);+ syntax!(seq1, "*", vec![ZeroOrMore]);+ syntax!(seq2, "a*b", vec![Literal('a'), ZeroOrMore, Literal('b')]);+ syntax!(+ seq3,+ "*a*b*",+ vec![ZeroOrMore, Literal('a'), ZeroOrMore, Literal('b'), ZeroOrMore,]+ );+ syntax!(rseq1, "**", vec![RecursivePrefix]);+ syntax!(rseq2, "**/", vec![RecursivePrefix]);+ syntax!(rseq3, "/**", vec![RecursiveSuffix]);+ syntax!(rseq4, "/**/", vec![RecursiveZeroOrMore]);+ syntax!(+ rseq5,+ "a/**/b",+ vec![Literal('a'), RecursiveZeroOrMore, Literal('b'),]+ );+ syntax!(cls1, "[a]", vec![class('a', 'a')]);+ syntax!(cls2, "[!a]", vec![classn('a', 'a')]);+ syntax!(cls3, "[a-z]", vec![class('a', 'z')]);+ syntax!(cls4, "[!a-z]", vec![classn('a', 'z')]);+ syntax!(cls5, "[-]", vec![class('-', '-')]);+ syntax!(cls6, "[]]", vec![class(']', ']')]);+ syntax!(cls7, "[*]", vec![class('*', '*')]);+ syntax!(cls8, "[!!]", vec![classn('!', '!')]);+ syntax!(cls9, "[a-]", vec![rclass(&[('a', 'a'), ('-', '-')])]);+ syntax!(cls10, "[-a-z]", vec![rclass(&[('-', '-'), ('a', 'z')])]);+ syntax!(cls11, "[a-z-]", vec![rclass(&[('a', 'z'), ('-', '-')])]);+ syntax!(+ cls12,+ "[-a-z-]",+ vec![rclass(&[('-', '-'), ('a', 'z'), ('-', '-')]),]+ );+ syntax!(cls13, "[]-z]", vec![class(']', 'z')]);+ syntax!(cls14, "[--z]", vec![class('-', 'z')]);+ syntax!(cls15, "[ --]", vec![class(' ', '-')]);+ syntax!(cls16, "[0-9a-z]", vec![rclass(&[('0', '9'), ('a', 'z')])]);+ syntax!(cls17, "[a-z0-9]", vec![rclass(&[('a', 'z'), ('0', '9')])]);+ syntax!(cls18, "[!0-9a-z]", vec![rclassn(&[('0', '9'), ('a', 'z')])]);+ syntax!(cls19, "[!a-z0-9]", vec![rclassn(&[('a', 'z'), ('0', '9')])]);+ syntax!(cls20, "[^a]", vec![classn('a', 'a')]);+ syntax!(cls21, "[^a-z]", vec![classn('a', 'z')]);++ syntaxerr!(err_unclosed1, "[", ErrorKind::UnclosedClass);+ syntaxerr!(err_unclosed2, "[]", ErrorKind::UnclosedClass);+ syntaxerr!(err_unclosed3, "[!", ErrorKind::UnclosedClass);+ syntaxerr!(err_unclosed4, "[!]", ErrorKind::UnclosedClass);+ syntaxerr!(err_range1, "[z-a]", ErrorKind::InvalidRange('z', 'a'));+ syntaxerr!(err_range2, "[z--]", ErrorKind::InvalidRange('z', '-'));++ const CASEI: Options =+ Options { casei: Some(true), litsep: None, bsesc: None };+ const SLASHLIT: Options =+ Options { casei: None, litsep: Some(true), bsesc: None };+ const NOBSESC: Options =+ Options { casei: None, litsep: None, bsesc: Some(false) };+ const BSESC: Options =+ Options { casei: None, litsep: None, bsesc: Some(true) };++ toregex!(re_casei, "a", "(?i)^a$", &CASEI);++ toregex!(re_slash1, "?", r"^[^/]$", SLASHLIT);+ toregex!(re_slash2, "*", r"^[^/]*$", SLASHLIT);++ toregex!(re1, "a", "^a$");+ toregex!(re2, "?", "^.$");+ toregex!(re3, "*", "^.*$");+ toregex!(re4, "a?", "^a.$");+ toregex!(re5, "?a", "^.a$");+ toregex!(re6, "a*", "^a.*$");+ toregex!(re7, "*a", "^.*a$");+ toregex!(re8, "[*]", r"^[\*]$");+ toregex!(re9, "[+]", r"^[\+]$");+ toregex!(re10, "+", r"^\+$");+ toregex!(re11, "☃", r"^\xe2\x98\x83$");+ toregex!(re12, "**", r"^.*$");+ toregex!(re13, "**/", r"^.*$");+ toregex!(re14, "**/*", r"^(?:/?|.*/).*$");+ toregex!(re15, "**/**", r"^.*$");+ toregex!(re16, "**/**/*", r"^(?:/?|.*/).*$");+ toregex!(re17, "**/**/**", r"^.*$");+ toregex!(re18, "**/**/**/*", r"^(?:/?|.*/).*$");+ toregex!(re19, "a/**", r"^a(?:/?|/.*)$");+ toregex!(re20, "a/**/**", r"^a(?:/?|/.*)$");+ toregex!(re21, "a/**/**/**", r"^a(?:/?|/.*)$");+ toregex!(re22, "a/**/b", r"^a(?:/|/.*/)b$");+ toregex!(re23, "a/**/**/b", r"^a(?:/|/.*/)b$");+ toregex!(re24, "a/**/**/**/b", r"^a(?:/|/.*/)b$");+ toregex!(re25, "**/b", r"^(?:/?|.*/)b$");+ toregex!(re26, "**/**/b", r"^(?:/?|.*/)b$");+ toregex!(re27, "**/**/**/b", r"^(?:/?|.*/)b$");+ toregex!(re28, "a**", r"^a.*.*$");+ toregex!(re29, "**a", r"^.*.*a$");+ toregex!(re30, "a**b", r"^a.*.*b$");+ toregex!(re31, "***", r"^.*.*.*$");+ toregex!(re32, "/a**", r"^/a.*.*$");+ toregex!(re33, "/**a", r"^/.*.*a$");+ toregex!(re34, "/a**b", r"^/a.*.*b$");++ matches!(match1, "a", "a");+ matches!(match2, "a*b", "a_b");+ matches!(match3, "a*b*c", "abc");+ matches!(match4, "a*b*c", "a_b_c");+ matches!(match5, "a*b*c", "a___b___c");+ matches!(match6, "abc*abc*abc", "abcabcabcabcabcabcabc");+ matches!(match7, "a*a*a*a*a*a*a*a*a", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");+ matches!(match8, "a*b[xyz]c*d", "abxcdbxcddd");+ matches!(match9, "*.rs", ".rs");+ matches!(match10, "☃", "☃");++ matches!(matchrec1, "some/**/needle.txt", "some/needle.txt");+ matches!(matchrec2, "some/**/needle.txt", "some/one/needle.txt");+ matches!(matchrec3, "some/**/needle.txt", "some/one/two/needle.txt");+ matches!(matchrec4, "some/**/needle.txt", "some/other/needle.txt");+ matches!(matchrec5, "**", "abcde");+ matches!(matchrec6, "**", "");+ matches!(matchrec7, "**", ".asdf");+ matches!(matchrec8, "**", "/x/.asdf");+ matches!(matchrec9, "some/**/**/needle.txt", "some/needle.txt");+ matches!(matchrec10, "some/**/**/needle.txt", "some/one/needle.txt");+ matches!(matchrec11, "some/**/**/needle.txt", "some/one/two/needle.txt");+ matches!(matchrec12, "some/**/**/needle.txt", "some/other/needle.txt");+ matches!(matchrec13, "**/test", "one/two/test");+ matches!(matchrec14, "**/test", "one/test");+ matches!(matchrec15, "**/test", "test");+ matches!(matchrec16, "/**/test", "/one/two/test");+ matches!(matchrec17, "/**/test", "/one/test");+ matches!(matchrec18, "/**/test", "/test");+ matches!(matchrec19, "**/.*", ".abc");+ matches!(matchrec20, "**/.*", "abc/.abc");+ matches!(matchrec21, ".*/**", ".abc");+ matches!(matchrec22, ".*/**", ".abc/abc");+ matches!(matchrec23, "foo/**", "foo");+ matches!(matchrec24, "**/foo/bar", "foo/bar");+ matches!(matchrec25, "some/*/needle.txt", "some/one/needle.txt");++ matches!(matchrange1, "a[0-9]b", "a0b");+ matches!(matchrange2, "a[0-9]b", "a9b");+ matches!(matchrange3, "a[!0-9]b", "a_b");+ matches!(matchrange4, "[a-z123]", "1");+ matches!(matchrange5, "[1a-z23]", "1");+ matches!(matchrange6, "[123a-z]", "1");+ matches!(matchrange7, "[abc-]", "-");+ matches!(matchrange8, "[-abc]", "-");+ matches!(matchrange9, "[-a-c]", "b");+ matches!(matchrange10, "[a-c-]", "b");+ matches!(matchrange11, "[-]", "-");+ matches!(matchrange12, "a[^0-9]b", "a_b");++ matches!(matchpat1, "*hello.txt", "hello.txt");+ matches!(matchpat2, "*hello.txt", "gareth_says_hello.txt");+ matches!(matchpat3, "*hello.txt", "some/path/to/hello.txt");+ matches!(matchpat4, "*hello.txt", "some\\path\\to\\hello.txt");+ matches!(matchpat5, "*hello.txt", "/an/absolute/path/to/hello.txt");+ matches!(matchpat6, "*some/path/to/hello.txt", "some/path/to/hello.txt");+ matches!(+ matchpat7,+ "*some/path/to/hello.txt",+ "a/bigger/some/path/to/hello.txt"+ );++ matches!(matchescape, "_[[]_[]]_[?]_[*]_!_", "_[_]_?_*_!_");++ matches!(matchcasei1, "aBcDeFg", "aBcDeFg", CASEI);+ matches!(matchcasei2, "aBcDeFg", "abcdefg", CASEI);+ matches!(matchcasei3, "aBcDeFg", "ABCDEFG", CASEI);+ matches!(matchcasei4, "aBcDeFg", "AbCdEfG", CASEI);++ matches!(matchalt1, "a,b", "a,b");+ matches!(matchalt2, ",", ",");+ matches!(matchalt3, "{a,b}", "a");+ matches!(matchalt4, "{a,b}", "b");+ matches!(matchalt5, "{**/src/**,foo}", "abc/src/bar");+ matches!(matchalt6, "{**/src/**,foo}", "foo");+ matches!(matchalt7, "{[}],foo}", "}");+ matches!(matchalt8, "{foo}", "foo");+ matches!(matchalt9, "{}", "");+ matches!(matchalt10, "{,}", "");+ matches!(matchalt11, "{*.foo,*.bar,*.wat}", "test.foo");+ matches!(matchalt12, "{*.foo,*.bar,*.wat}", "test.bar");+ matches!(matchalt13, "{*.foo,*.bar,*.wat}", "test.wat");++ matches!(matchslash1, "abc/def", "abc/def", SLASHLIT);+ #[cfg(unix)]+ nmatches!(matchslash2, "abc?def", "abc/def", SLASHLIT);+ #[cfg(not(unix))]+ nmatches!(matchslash2, "abc?def", "abc\\def", SLASHLIT);+ nmatches!(matchslash3, "abc*def", "abc/def", SLASHLIT);+ matches!(matchslash4, "abc[/]def", "abc/def", SLASHLIT); // differs+ #[cfg(unix)]+ nmatches!(matchslash5, "abc\\def", "abc/def", SLASHLIT);+ #[cfg(not(unix))]+ matches!(matchslash5, "abc\\def", "abc/def", SLASHLIT);++ matches!(matchbackslash1, "\\[", "[", BSESC);+ matches!(matchbackslash2, "\\?", "?", BSESC);+ matches!(matchbackslash3, "\\*", "*", BSESC);+ matches!(matchbackslash4, "\\[a-z]", "\\a", NOBSESC);+ matches!(matchbackslash5, "\\?", "\\a", NOBSESC);+ matches!(matchbackslash6, "\\*", "\\\\", NOBSESC);+ #[cfg(unix)]+ matches!(matchbackslash7, "\\a", "a");+ #[cfg(not(unix))]+ matches!(matchbackslash8, "\\a", "/a");++ nmatches!(matchnot1, "a*b*c", "abcd");+ nmatches!(matchnot2, "abc*abc*abc", "abcabcabcabcabcabcabca");+ nmatches!(matchnot3, "some/**/needle.txt", "some/other/notthis.txt");+ nmatches!(matchnot4, "some/**/**/needle.txt", "some/other/notthis.txt");+ nmatches!(matchnot5, "/**/test", "test");+ nmatches!(matchnot6, "/**/test", "/one/notthis");+ nmatches!(matchnot7, "/**/test", "/notthis");+ nmatches!(matchnot8, "**/.*", "ab.c");+ nmatches!(matchnot9, "**/.*", "abc/ab.c");+ nmatches!(matchnot10, ".*/**", "a.bc");+ nmatches!(matchnot11, ".*/**", "abc/a.bc");+ nmatches!(matchnot12, "a[0-9]b", "a_b");+ nmatches!(matchnot13, "a[!0-9]b", "a0b");+ nmatches!(matchnot14, "a[!0-9]b", "a9b");+ nmatches!(matchnot15, "[!-]", "-");+ nmatches!(matchnot16, "*hello.txt", "hello.txt-and-then-some");+ nmatches!(matchnot17, "*hello.txt", "goodbye.txt");+ nmatches!(+ matchnot18,+ "*some/path/to/hello.txt",+ "some/path/to/hello.txt-and-then-some"+ );+ nmatches!(+ matchnot19,+ "*some/path/to/hello.txt",+ "some/other/path/to/hello.txt"+ );+ nmatches!(matchnot20, "a", "foo/a");+ nmatches!(matchnot21, "./foo", "foo");+ nmatches!(matchnot22, "**/foo", "foofoo");+ nmatches!(matchnot23, "**/foo/bar", "foofoo/bar");+ nmatches!(matchnot24, "/*.c", "mozilla-sha1/sha1.c");+ nmatches!(matchnot25, "*.c", "mozilla-sha1/sha1.c", SLASHLIT);+ nmatches!(+ matchnot26,+ "**/m4/ltoptions.m4",+ "csharp/src/packages/repositories.config",+ SLASHLIT+ );+ nmatches!(matchnot27, "a[^0-9]b", "a0b");+ nmatches!(matchnot28, "a[^0-9]b", "a9b");+ nmatches!(matchnot29, "[^-]", "-");+ nmatches!(matchnot30, "some/*/needle.txt", "some/needle.txt");+ nmatches!(+ matchrec31,+ "some/*/needle.txt",+ "some/one/two/needle.txt",+ SLASHLIT+ );+ nmatches!(+ matchrec32,+ "some/*/needle.txt",+ "some/one/two/three/needle.txt",+ SLASHLIT+ );++ macro_rules! extract {+ ($which:ident, $name:ident, $pat:expr, $expect:expr) => {+ extract!($which, $name, $pat, $expect, Options::default());+ };+ ($which:ident, $name:ident, $pat:expr, $expect:expr, $options:expr) => {+ #[test]+ fn $name() {+ let mut builder = GlobBuilder::new($pat);+ if let Some(casei) = $options.casei {+ builder.case_insensitive(casei);+ }+ if let Some(litsep) = $options.litsep {+ builder.literal_separator(litsep);+ }+ if let Some(bsesc) = $options.bsesc {+ builder.backslash_escape(bsesc);+ }+ let pat = builder.build().unwrap();+ assert_eq!($expect, pat.$which());+ }+ };+ }++ macro_rules! literal {+ ($($tt:tt)*) => { extract!(literal, $($tt)*); }+ }++ macro_rules! basetokens {+ ($($tt:tt)*) => { extract!(basename_tokens, $($tt)*); }+ }++ macro_rules! ext {+ ($($tt:tt)*) => { extract!(ext, $($tt)*); }+ }++ macro_rules! required_ext {+ ($($tt:tt)*) => { extract!(required_ext, $($tt)*); }+ }++ macro_rules! prefix {+ ($($tt:tt)*) => { extract!(prefix, $($tt)*); }+ }++ macro_rules! suffix {+ ($($tt:tt)*) => { extract!(suffix, $($tt)*); }+ }++ macro_rules! baseliteral {+ ($($tt:tt)*) => { extract!(basename_literal, $($tt)*); }+ }++ literal!(extract_lit1, "foo", Some(s("foo")));+ literal!(extract_lit2, "foo", None, CASEI);+ literal!(extract_lit3, "/foo", Some(s("/foo")));+ literal!(extract_lit4, "/foo/", Some(s("/foo/")));+ literal!(extract_lit5, "/foo/bar", Some(s("/foo/bar")));+ literal!(extract_lit6, "*.foo", None);+ literal!(extract_lit7, "foo/bar", Some(s("foo/bar")));+ literal!(extract_lit8, "**/foo/bar", None);++ basetokens!(+ extract_basetoks1,+ "**/foo",+ Some(&*vec![Literal('f'), Literal('o'), Literal('o'),])+ );+ basetokens!(extract_basetoks2, "**/foo", None, CASEI);+ basetokens!(+ extract_basetoks3,+ "**/foo",+ Some(&*vec![Literal('f'), Literal('o'), Literal('o'),]),+ SLASHLIT+ );+ basetokens!(extract_basetoks4, "*foo", None, SLASHLIT);+ basetokens!(extract_basetoks5, "*foo", None);+ basetokens!(extract_basetoks6, "**/fo*o", None);+ basetokens!(+ extract_basetoks7,+ "**/fo*o",+ Some(&*vec![Literal('f'), Literal('o'), ZeroOrMore, Literal('o'),]),+ SLASHLIT+ );++ ext!(extract_ext1, "**/*.rs", Some(s(".rs")));+ ext!(extract_ext2, "**/*.rs.bak", None);+ ext!(extract_ext3, "*.rs", Some(s(".rs")));+ ext!(extract_ext4, "a*.rs", None);+ ext!(extract_ext5, "/*.c", None);+ ext!(extract_ext6, "*.c", None, SLASHLIT);+ ext!(extract_ext7, "*.c", Some(s(".c")));++ required_ext!(extract_req_ext1, "*.rs", Some(s(".rs")));+ required_ext!(extract_req_ext2, "/foo/bar/*.rs", Some(s(".rs")));+ required_ext!(extract_req_ext3, "/foo/bar/*.rs", Some(s(".rs")));+ required_ext!(extract_req_ext4, "/foo/bar/.rs", Some(s(".rs")));+ required_ext!(extract_req_ext5, ".rs", Some(s(".rs")));+ required_ext!(extract_req_ext6, "./rs", None);+ required_ext!(extract_req_ext7, "foo", None);+ required_ext!(extract_req_ext8, ".foo/", None);+ required_ext!(extract_req_ext9, "foo/", None);++ prefix!(extract_prefix1, "/foo", Some(s("/foo")));+ prefix!(extract_prefix2, "/foo/*", Some(s("/foo/")));+ prefix!(extract_prefix3, "**/foo", None);+ prefix!(extract_prefix4, "foo/**", None);++ suffix!(extract_suffix1, "**/foo/bar", Some((s("/foo/bar"), true)));+ suffix!(extract_suffix2, "*/foo/bar", Some((s("/foo/bar"), false)));+ suffix!(extract_suffix3, "*/foo/bar", None, SLASHLIT);+ suffix!(extract_suffix4, "foo/bar", Some((s("foo/bar"), false)));+ suffix!(extract_suffix5, "*.foo", Some((s(".foo"), false)));+ suffix!(extract_suffix6, "*.foo", None, SLASHLIT);+ suffix!(extract_suffix7, "**/*_test", Some((s("_test"), false)));++ baseliteral!(extract_baselit1, "**/foo", Some(s("foo")));+ baseliteral!(extract_baselit2, "foo", None);+ baseliteral!(extract_baselit3, "*foo", None);+ baseliteral!(extract_baselit4, "*/foo", None);+}
@@ -0,0 +1,860 @@+/*!+The globset crate provides cross platform single glob and glob set matching.++Glob set matching is the process of matching one or more glob patterns against+a single candidate path simultaneously, and returning all of the globs that+matched. For example, given this set of globs:++```ignore+*.rs+src/lib.rs+src/**/foo.rs+```++and a path `src/bar/baz/foo.rs`, then the set would report the first and third+globs as matching.++# Example: one glob++This example shows how to match a single glob against a single file path.++```+# fn example() -> Result<(), globset::Error> {+use globset::Glob;++let glob = Glob::new("*.rs")?.compile_matcher();++assert!(glob.is_match("foo.rs"));+assert!(glob.is_match("foo/bar.rs"));+assert!(!glob.is_match("Cargo.toml"));+# Ok(()) } example().unwrap();+```++# Example: configuring a glob matcher++This example shows how to use a `GlobBuilder` to configure aspects of match+semantics. In this example, we prevent wildcards from matching path separators.++```+# fn example() -> Result<(), globset::Error> {+use globset::GlobBuilder;++let glob = GlobBuilder::new("*.rs")+ .literal_separator(true).build()?.compile_matcher();++assert!(glob.is_match("foo.rs"));+assert!(!glob.is_match("foo/bar.rs")); // no longer matches+assert!(!glob.is_match("Cargo.toml"));+# Ok(()) } example().unwrap();+```++# Example: match multiple globs at once++This example shows how to match multiple glob patterns at once.++```+# fn example() -> Result<(), globset::Error> {+use globset::{Glob, GlobSetBuilder};++let mut builder = GlobSetBuilder::new();+// A GlobBuilder can be used to configure each glob's match semantics+// independently.+builder.add(Glob::new("*.rs")?);+builder.add(Glob::new("src/lib.rs")?);+builder.add(Glob::new("src/**/foo.rs")?);+let set = builder.build()?;++assert_eq!(set.matches("src/bar/baz/foo.rs"), vec![0, 2]);+# Ok(()) } example().unwrap();+```++# Syntax++Standard Unix-style glob syntax is supported:++* `?` matches any single character. (If the `literal_separator` option is+ enabled, then `?` can never match a path separator.)+* `*` matches zero or more characters. (If the `literal_separator` option is+ enabled, then `*` can never match a path separator.)+* `**` recursively matches directories but are only legal in three situations.+ First, if the glob starts with <code>\*\*/</code>, then it matches+ all directories. For example, <code>\*\*/foo</code> matches `foo`+ and `bar/foo` but not `foo/bar`. Secondly, if the glob ends with+ <code>/\*\*</code>, then it matches all sub-entries. For example,+ <code>foo/\*\*</code> matches `foo/a` and `foo/a/b`, but not `foo`.+ Thirdly, if the glob contains <code>/\*\*/</code> anywhere within+ the pattern, then it matches zero or more directories. Using `**` anywhere+ else is illegal (N.B. the glob `**` is allowed and means "match everything").+* `{a,b}` matches `a` or `b` where `a` and `b` are arbitrary glob patterns.+ (N.B. Nesting `{...}` is not currently allowed.)+* `[ab]` matches `a` or `b` where `a` and `b` are characters. Use+ `[!ab]` to match any character except for `a` and `b`.+* Metacharacters such as `*` and `?` can be escaped with character class+ notation. e.g., `[*]` matches `*`.+* When backslash escapes are enabled, a backslash (`\`) will escape all meta+ characters in a glob. If it precedes a non-meta character, then the slash is+ ignored. A `\\` will match a literal `\\`. Note that this mode is only+ enabled on Unix platforms by default, but can be enabled on any platform+ via the `backslash_escape` setting on `Glob`.++A `GlobBuilder` can be used to prevent wildcards from matching path separators,+or to enable case insensitive matching.+*/++#![deny(missing_docs)]++extern crate aho_corasick;+extern crate bstr;+extern crate fnv;+#[macro_use]+extern crate log;+extern crate regex;++use std::borrow::Cow;+use std::collections::{BTreeMap, HashMap};+use std::error::Error as StdError;+use std::fmt;+use std::hash;+use std::path::Path;+use std::str;++use aho_corasick::AhoCorasick;+use bstr::{ByteSlice, ByteVec, B};+use regex::bytes::{Regex, RegexBuilder, RegexSet};++use glob::MatchStrategy;+pub use glob::{Glob, GlobBuilder, GlobMatcher};+use pathutil::{file_name, file_name_ext, normalize_path};++mod glob;+mod pathutil;++/// Represents an error that can occur when parsing a glob pattern.+#[derive(Clone, Debug, Eq, PartialEq)]+pub struct Error {+ /// The original glob provided by the caller.+ glob: Option<String>,+ /// The kind of error.+ kind: ErrorKind,+}++/// The kind of error that can occur when parsing a glob pattern.+#[derive(Clone, Debug, Eq, PartialEq)]+pub enum ErrorKind {+ /// **DEPRECATED**.+ ///+ /// This error used to occur for consistency with git's glob specification,+ /// but the specification now accepts all uses of `**`. When `**` does not+ /// appear adjacent to a path separator or at the beginning/end of a glob,+ /// it is now treated as two consecutive `*` patterns. As such, this error+ /// is no longer used.+ InvalidRecursive,+ /// Occurs when a character class (e.g., `[abc]`) is not closed.+ UnclosedClass,+ /// Occurs when a range in a character (e.g., `[a-z]`) is invalid. For+ /// example, if the range starts with a lexicographically larger character+ /// than it ends with.+ InvalidRange(char, char),+ /// Occurs when a `}` is found without a matching `{`.+ UnopenedAlternates,+ /// Occurs when a `{` is found without a matching `}`.+ UnclosedAlternates,+ /// Occurs when an alternating group is nested inside another alternating+ /// group, e.g., `{{a,b},{c,d}}`.+ NestedAlternates,+ /// Occurs when an unescaped '\' is found at the end of a glob.+ DanglingEscape,+ /// An error associated with parsing or compiling a regex.+ Regex(String),+ /// Hints that destructuring should not be exhaustive.+ ///+ /// This enum may grow additional variants, so this makes sure clients+ /// don't count on exhaustive matching. (Otherwise, adding a new variant+ /// could break existing code.)+ #[doc(hidden)]+ __Nonexhaustive,+}++impl StdError for Error {+ fn description(&self) -> &str {+ self.kind.description()+ }+}++impl Error {+ /// Return the glob that caused this error, if one exists.+ pub fn glob(&self) -> Option<&str> {+ self.glob.as_ref().map(|s| &**s)+ }++ /// Return the kind of this error.+ pub fn kind(&self) -> &ErrorKind {+ &self.kind+ }+}++impl ErrorKind {+ fn description(&self) -> &str {+ match *self {+ ErrorKind::InvalidRecursive => {+ "invalid use of **; must be one path component"+ }+ ErrorKind::UnclosedClass => {+ "unclosed character class; missing ']'"+ }+ ErrorKind::InvalidRange(_, _) => "invalid character range",+ ErrorKind::UnopenedAlternates => {+ "unopened alternate group; missing '{' \+ (maybe escape '}' with '[}]'?)"+ }+ ErrorKind::UnclosedAlternates => {+ "unclosed alternate group; missing '}' \+ (maybe escape '{' with '[{]'?)"+ }+ ErrorKind::NestedAlternates => {+ "nested alternate groups are not allowed"+ }+ ErrorKind::DanglingEscape => "dangling '\\'",+ ErrorKind::Regex(ref err) => err,+ ErrorKind::__Nonexhaustive => unreachable!(),+ }+ }+}++impl fmt::Display for Error {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match self.glob {+ None => self.kind.fmt(f),+ Some(ref glob) => {+ write!(f, "error parsing glob '{}': {}", glob, self.kind)+ }+ }+ }+}++impl fmt::Display for ErrorKind {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match *self {+ ErrorKind::InvalidRecursive+ | ErrorKind::UnclosedClass+ | ErrorKind::UnopenedAlternates+ | ErrorKind::UnclosedAlternates+ | ErrorKind::NestedAlternates+ | ErrorKind::DanglingEscape+ | ErrorKind::Regex(_) => write!(f, "{}", self.description()),+ ErrorKind::InvalidRange(s, e) => {+ write!(f, "invalid range; '{}' > '{}'", s, e)+ }+ ErrorKind::__Nonexhaustive => unreachable!(),+ }+ }+}++fn new_regex(pat: &str) -> Result<Regex, Error> {+ RegexBuilder::new(pat)+ .dot_matches_new_line(true)+ .size_limit(10 * (1 << 20))+ .dfa_size_limit(10 * (1 << 20))+ .build()+ .map_err(|err| Error {+ glob: Some(pat.to_string()),+ kind: ErrorKind::Regex(err.to_string()),+ })+}++fn new_regex_set<I, S>(pats: I) -> Result<RegexSet, Error>+where+ S: AsRef<str>,+ I: IntoIterator<Item = S>,+{+ RegexSet::new(pats).map_err(|err| Error {+ glob: None,+ kind: ErrorKind::Regex(err.to_string()),+ })+}++type Fnv = hash::BuildHasherDefault<fnv::FnvHasher>;++/// GlobSet represents a group of globs that can be matched together in a+/// single pass.+#[derive(Clone, Debug)]+pub struct GlobSet {+ len: usize,+ strats: Vec<GlobSetMatchStrategy>,+}++impl GlobSet {+ /// Create an empty `GlobSet`. An empty set matches nothing.+ #[inline]+ pub fn empty() -> GlobSet {+ GlobSet { len: 0, strats: vec![] }+ }++ /// Returns true if this set is empty, and therefore matches nothing.+ #[inline]+ pub fn is_empty(&self) -> bool {+ self.len == 0+ }++ /// Returns the number of globs in this set.+ #[inline]+ pub fn len(&self) -> usize {+ self.len+ }++ /// Returns true if any glob in this set matches the path given.+ pub fn is_match<P: AsRef<Path>>(&self, path: P) -> bool {+ self.is_match_candidate(&Candidate::new(path.as_ref()))+ }++ /// Returns true if any glob in this set matches the path given.+ ///+ /// This takes a Candidate as input, which can be used to amortize the+ /// cost of preparing a path for matching.+ pub fn is_match_candidate(&self, path: &Candidate) -> bool {+ if self.is_empty() {+ return false;+ }+ for strat in &self.strats {+ if strat.is_match(path) {+ return true;+ }+ }+ false+ }++ /// Returns the sequence number of every glob pattern that matches the+ /// given path.+ pub fn matches<P: AsRef<Path>>(&self, path: P) -> Vec<usize> {+ self.matches_candidate(&Candidate::new(path.as_ref()))+ }++ /// Returns the sequence number of every glob pattern that matches the+ /// given path.+ ///+ /// This takes a Candidate as input, which can be used to amortize the+ /// cost of preparing a path for matching.+ pub fn matches_candidate(&self, path: &Candidate) -> Vec<usize> {+ let mut into = vec![];+ if self.is_empty() {+ return into;+ }+ self.matches_candidate_into(path, &mut into);+ into+ }++ /// Adds the sequence number of every glob pattern that matches the given+ /// path to the vec given.+ ///+ /// `into` is is cleared before matching begins, and contains the set of+ /// sequence numbers (in ascending order) after matching ends. If no globs+ /// were matched, then `into` will be empty.+ pub fn matches_into<P: AsRef<Path>>(+ &self,+ path: P,+ into: &mut Vec<usize>,+ ) {+ self.matches_candidate_into(&Candidate::new(path.as_ref()), into);+ }++ /// Adds the sequence number of every glob pattern that matches the given+ /// path to the vec given.+ ///+ /// `into` is is cleared before matching begins, and contains the set of+ /// sequence numbers (in ascending order) after matching ends. If no globs+ /// were matched, then `into` will be empty.+ ///+ /// This takes a Candidate as input, which can be used to amortize the+ /// cost of preparing a path for matching.+ pub fn matches_candidate_into(+ &self,+ path: &Candidate,+ into: &mut Vec<usize>,+ ) {+ into.clear();+ if self.is_empty() {+ return;+ }+ for strat in &self.strats {+ strat.matches_into(path, into);+ }+ into.sort();+ into.dedup();+ }++ fn new(pats: &[Glob]) -> Result<GlobSet, Error> {+ if pats.is_empty() {+ return Ok(GlobSet { len: 0, strats: vec![] });+ }+ let mut lits = LiteralStrategy::new();+ let mut base_lits = BasenameLiteralStrategy::new();+ let mut exts = ExtensionStrategy::new();+ let mut prefixes = MultiStrategyBuilder::new();+ let mut suffixes = MultiStrategyBuilder::new();+ let mut required_exts = RequiredExtensionStrategyBuilder::new();+ let mut regexes = MultiStrategyBuilder::new();+ for (i, p) in pats.iter().enumerate() {+ match MatchStrategy::new(p) {+ MatchStrategy::Literal(lit) => {+ lits.add(i, lit);+ }+ MatchStrategy::BasenameLiteral(lit) => {+ base_lits.add(i, lit);+ }+ MatchStrategy::Extension(ext) => {+ exts.add(i, ext);+ }+ MatchStrategy::Prefix(prefix) => {+ prefixes.add(i, prefix);+ }+ MatchStrategy::Suffix { suffix, component } => {+ if component {+ lits.add(i, suffix[1..].to_string());+ }+ suffixes.add(i, suffix);+ }+ MatchStrategy::RequiredExtension(ext) => {+ required_exts.add(i, ext, p.regex().to_owned());+ }+ MatchStrategy::Regex => {+ debug!("glob converted to regex: {:?}", p);+ regexes.add(i, p.regex().to_owned());+ }+ }+ }+ debug!(+ "built glob set; {} literals, {} basenames, {} extensions, \+ {} prefixes, {} suffixes, {} required extensions, {} regexes",+ lits.0.len(),+ base_lits.0.len(),+ exts.0.len(),+ prefixes.literals.len(),+ suffixes.literals.len(),+ required_exts.0.len(),+ regexes.literals.len()+ );+ Ok(GlobSet {+ len: pats.len(),+ strats: vec![+ GlobSetMatchStrategy::Extension(exts),+ GlobSetMatchStrategy::BasenameLiteral(base_lits),+ GlobSetMatchStrategy::Literal(lits),+ GlobSetMatchStrategy::Suffix(suffixes.suffix()),+ GlobSetMatchStrategy::Prefix(prefixes.prefix()),+ GlobSetMatchStrategy::RequiredExtension(+ required_exts.build()?,+ ),+ GlobSetMatchStrategy::Regex(regexes.regex_set()?),+ ],+ })+ }+}++/// GlobSetBuilder builds a group of patterns that can be used to+/// simultaneously match a file path.+#[derive(Clone, Debug)]+pub struct GlobSetBuilder {+ pats: Vec<Glob>,+}++impl GlobSetBuilder {+ /// Create a new GlobSetBuilder. A GlobSetBuilder can be used to add new+ /// patterns. Once all patterns have been added, `build` should be called+ /// to produce a `GlobSet`, which can then be used for matching.+ pub fn new() -> GlobSetBuilder {+ GlobSetBuilder { pats: vec![] }+ }++ /// Builds a new matcher from all of the glob patterns added so far.+ ///+ /// Once a matcher is built, no new patterns can be added to it.+ pub fn build(&self) -> Result<GlobSet, Error> {+ GlobSet::new(&self.pats)+ }++ /// Add a new pattern to this set.+ pub fn add(&mut self, pat: Glob) -> &mut GlobSetBuilder {+ self.pats.push(pat);+ self+ }+}++/// A candidate path for matching.+///+/// All glob matching in this crate operates on `Candidate` values.+/// Constructing candidates has a very small cost associated with it, so+/// callers may find it beneficial to amortize that cost when matching a single+/// path against multiple globs or sets of globs.+#[derive(Clone, Debug)]+pub struct Candidate<'a> {+ path: Cow<'a, [u8]>,+ basename: Cow<'a, [u8]>,+ ext: Cow<'a, [u8]>,+}++impl<'a> Candidate<'a> {+ /// Create a new candidate for matching from the given path.+ pub fn new<P: AsRef<Path> + ?Sized>(path: &'a P) -> Candidate<'a> {+ let path = normalize_path(Vec::from_path_lossy(path.as_ref()));+ let basename = file_name(&path).unwrap_or(Cow::Borrowed(B("")));+ let ext = file_name_ext(&basename).unwrap_or(Cow::Borrowed(B("")));+ Candidate { path: path, basename: basename, ext: ext }+ }++ fn path_prefix(&self, max: usize) -> &[u8] {+ if self.path.len() <= max {+ &*self.path+ } else {+ &self.path[..max]+ }+ }++ fn path_suffix(&self, max: usize) -> &[u8] {+ if self.path.len() <= max {+ &*self.path+ } else {+ &self.path[self.path.len() - max..]+ }+ }+}++#[derive(Clone, Debug)]+enum GlobSetMatchStrategy {+ Literal(LiteralStrategy),+ BasenameLiteral(BasenameLiteralStrategy),+ Extension(ExtensionStrategy),+ Prefix(PrefixStrategy),+ Suffix(SuffixStrategy),+ RequiredExtension(RequiredExtensionStrategy),+ Regex(RegexSetStrategy),+}++impl GlobSetMatchStrategy {+ fn is_match(&self, candidate: &Candidate) -> bool {+ use self::GlobSetMatchStrategy::*;+ match *self {+ Literal(ref s) => s.is_match(candidate),+ BasenameLiteral(ref s) => s.is_match(candidate),+ Extension(ref s) => s.is_match(candidate),+ Prefix(ref s) => s.is_match(candidate),+ Suffix(ref s) => s.is_match(candidate),+ RequiredExtension(ref s) => s.is_match(candidate),+ Regex(ref s) => s.is_match(candidate),+ }+ }++ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ use self::GlobSetMatchStrategy::*;+ match *self {+ Literal(ref s) => s.matches_into(candidate, matches),+ BasenameLiteral(ref s) => s.matches_into(candidate, matches),+ Extension(ref s) => s.matches_into(candidate, matches),+ Prefix(ref s) => s.matches_into(candidate, matches),+ Suffix(ref s) => s.matches_into(candidate, matches),+ RequiredExtension(ref s) => s.matches_into(candidate, matches),+ Regex(ref s) => s.matches_into(candidate, matches),+ }+ }+}++#[derive(Clone, Debug)]+struct LiteralStrategy(BTreeMap<Vec<u8>, Vec<usize>>);++impl LiteralStrategy {+ fn new() -> LiteralStrategy {+ LiteralStrategy(BTreeMap::new())+ }++ fn add(&mut self, global_index: usize, lit: String) {+ self.0.entry(lit.into_bytes()).or_insert(vec![]).push(global_index);+ }++ fn is_match(&self, candidate: &Candidate) -> bool {+ self.0.contains_key(candidate.path.as_bytes())+ }++ #[inline(never)]+ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ if let Some(hits) = self.0.get(candidate.path.as_bytes()) {+ matches.extend(hits);+ }+ }+}++#[derive(Clone, Debug)]+struct BasenameLiteralStrategy(BTreeMap<Vec<u8>, Vec<usize>>);++impl BasenameLiteralStrategy {+ fn new() -> BasenameLiteralStrategy {+ BasenameLiteralStrategy(BTreeMap::new())+ }++ fn add(&mut self, global_index: usize, lit: String) {+ self.0.entry(lit.into_bytes()).or_insert(vec![]).push(global_index);+ }++ fn is_match(&self, candidate: &Candidate) -> bool {+ if candidate.basename.is_empty() {+ return false;+ }+ self.0.contains_key(candidate.basename.as_bytes())+ }++ #[inline(never)]+ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ if candidate.basename.is_empty() {+ return;+ }+ if let Some(hits) = self.0.get(candidate.basename.as_bytes()) {+ matches.extend(hits);+ }+ }+}++#[derive(Clone, Debug)]+struct ExtensionStrategy(HashMap<Vec<u8>, Vec<usize>, Fnv>);++impl ExtensionStrategy {+ fn new() -> ExtensionStrategy {+ ExtensionStrategy(HashMap::with_hasher(Fnv::default()))+ }++ fn add(&mut self, global_index: usize, ext: String) {+ self.0.entry(ext.into_bytes()).or_insert(vec![]).push(global_index);+ }++ fn is_match(&self, candidate: &Candidate) -> bool {+ if candidate.ext.is_empty() {+ return false;+ }+ self.0.contains_key(candidate.ext.as_bytes())+ }++ #[inline(never)]+ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ if candidate.ext.is_empty() {+ return;+ }+ if let Some(hits) = self.0.get(candidate.ext.as_bytes()) {+ matches.extend(hits);+ }+ }+}++#[derive(Clone, Debug)]+struct PrefixStrategy {+ matcher: AhoCorasick,+ map: Vec<usize>,+ longest: usize,+}++impl PrefixStrategy {+ fn is_match(&self, candidate: &Candidate) -> bool {+ let path = candidate.path_prefix(self.longest);+ for m in self.matcher.find_overlapping_iter(path) {+ if m.start() == 0 {+ return true;+ }+ }+ false+ }++ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ let path = candidate.path_prefix(self.longest);+ for m in self.matcher.find_overlapping_iter(path) {+ if m.start() == 0 {+ matches.push(self.map[m.pattern()]);+ }+ }+ }+}++#[derive(Clone, Debug)]+struct SuffixStrategy {+ matcher: AhoCorasick,+ map: Vec<usize>,+ longest: usize,+}++impl SuffixStrategy {+ fn is_match(&self, candidate: &Candidate) -> bool {+ let path = candidate.path_suffix(self.longest);+ for m in self.matcher.find_overlapping_iter(path) {+ if m.end() == path.len() {+ return true;+ }+ }+ false+ }++ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ let path = candidate.path_suffix(self.longest);+ for m in self.matcher.find_overlapping_iter(path) {+ if m.end() == path.len() {+ matches.push(self.map[m.pattern()]);+ }+ }+ }+}++#[derive(Clone, Debug)]+struct RequiredExtensionStrategy(HashMap<Vec<u8>, Vec<(usize, Regex)>, Fnv>);++impl RequiredExtensionStrategy {+ fn is_match(&self, candidate: &Candidate) -> bool {+ if candidate.ext.is_empty() {+ return false;+ }+ match self.0.get(candidate.ext.as_bytes()) {+ None => false,+ Some(regexes) => {+ for &(_, ref re) in regexes {+ if re.is_match(candidate.path.as_bytes()) {+ return true;+ }+ }+ false+ }+ }+ }++ #[inline(never)]+ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ if candidate.ext.is_empty() {+ return;+ }+ if let Some(regexes) = self.0.get(candidate.ext.as_bytes()) {+ for &(global_index, ref re) in regexes {+ if re.is_match(candidate.path.as_bytes()) {+ matches.push(global_index);+ }+ }+ }+ }+}++#[derive(Clone, Debug)]+struct RegexSetStrategy {+ matcher: RegexSet,+ map: Vec<usize>,+}++impl RegexSetStrategy {+ fn is_match(&self, candidate: &Candidate) -> bool {+ self.matcher.is_match(candidate.path.as_bytes())+ }++ fn matches_into(&self, candidate: &Candidate, matches: &mut Vec<usize>) {+ for i in self.matcher.matches(candidate.path.as_bytes()) {+ matches.push(self.map[i]);+ }+ }+}++#[derive(Clone, Debug)]+struct MultiStrategyBuilder {+ literals: Vec<String>,+ map: Vec<usize>,+ longest: usize,+}++impl MultiStrategyBuilder {+ fn new() -> MultiStrategyBuilder {+ MultiStrategyBuilder { literals: vec![], map: vec![], longest: 0 }+ }++ fn add(&mut self, global_index: usize, literal: String) {+ if literal.len() > self.longest {+ self.longest = literal.len();+ }+ self.map.push(global_index);+ self.literals.push(literal);+ }++ fn prefix(self) -> PrefixStrategy {+ PrefixStrategy {+ matcher: AhoCorasick::new_auto_configured(&self.literals),+ map: self.map,+ longest: self.longest,+ }+ }++ fn suffix(self) -> SuffixStrategy {+ SuffixStrategy {+ matcher: AhoCorasick::new_auto_configured(&self.literals),+ map: self.map,+ longest: self.longest,+ }+ }++ fn regex_set(self) -> Result<RegexSetStrategy, Error> {+ Ok(RegexSetStrategy {+ matcher: new_regex_set(self.literals)?,+ map: self.map,+ })+ }+}++#[derive(Clone, Debug)]+struct RequiredExtensionStrategyBuilder(+ HashMap<Vec<u8>, Vec<(usize, String)>>,+);++impl RequiredExtensionStrategyBuilder {+ fn new() -> RequiredExtensionStrategyBuilder {+ RequiredExtensionStrategyBuilder(HashMap::new())+ }++ fn add(&mut self, global_index: usize, ext: String, regex: String) {+ self.0+ .entry(ext.into_bytes())+ .or_insert(vec![])+ .push((global_index, regex));+ }++ fn build(self) -> Result<RequiredExtensionStrategy, Error> {+ let mut exts = HashMap::with_hasher(Fnv::default());+ for (ext, regexes) in self.0.into_iter() {+ exts.insert(ext.clone(), vec![]);+ for (global_index, regex) in regexes {+ let compiled = new_regex(®ex)?;+ exts.get_mut(&ext).unwrap().push((global_index, compiled));+ }+ }+ Ok(RequiredExtensionStrategy(exts))+ }+}++#[cfg(test)]+mod tests {+ use super::GlobSetBuilder;+ use glob::Glob;++ #[test]+ fn set_works() {+ let mut builder = GlobSetBuilder::new();+ builder.add(Glob::new("src/**/*.rs").unwrap());+ builder.add(Glob::new("*.c").unwrap());+ builder.add(Glob::new("src/lib.rs").unwrap());+ let set = builder.build().unwrap();++ assert!(set.is_match("foo.c"));+ assert!(set.is_match("src/foo.c"));+ assert!(!set.is_match("foo.rs"));+ assert!(!set.is_match("tests/foo.rs"));+ assert!(set.is_match("src/foo.rs"));+ assert!(set.is_match("src/grep/src/main.rs"));++ let matches = set.matches("src/lib.rs");+ assert_eq!(2, matches.len());+ assert_eq!(0, matches[0]);+ assert_eq!(2, matches[1]);+ }++ #[test]+ fn empty_set_works() {+ let set = GlobSetBuilder::new().build().unwrap();+ assert!(!set.is_match(""));+ assert!(!set.is_match("a"));+ }+}
crates/globset/src/pathutil.rs
@@ -0,0 +1,129 @@+use std::borrow::Cow;++use bstr::{ByteSlice, ByteVec};++/// The final component of the path, if it is a normal file.+///+/// If the path terminates in ., .., or consists solely of a root of prefix,+/// file_name will return None.+pub fn file_name<'a>(path: &Cow<'a, [u8]>) -> Option<Cow<'a, [u8]>> {+ if path.is_empty() {+ return None;+ } else if path.last_byte() == Some(b'.') {+ return None;+ }+ let last_slash = path.rfind_byte(b'/').map(|i| i + 1).unwrap_or(0);+ Some(match *path {+ Cow::Borrowed(path) => Cow::Borrowed(&path[last_slash..]),+ Cow::Owned(ref path) => {+ let mut path = path.clone();+ path.drain_bytes(..last_slash);+ Cow::Owned(path)+ }+ })+}++/// Return a file extension given a path's file name.+///+/// Note that this does NOT match the semantics of std::path::Path::extension.+/// Namely, the extension includes the `.` and matching is otherwise more+/// liberal. Specifically, the extenion is:+///+/// * None, if the file name given is empty;+/// * None, if there is no embedded `.`;+/// * Otherwise, the portion of the file name starting with the final `.`.+///+/// e.g., A file name of `.rs` has an extension `.rs`.+///+/// N.B. This is done to make certain glob match optimizations easier. Namely,+/// a pattern like `*.rs` is obviously trying to match files with a `rs`+/// extension, but it also matches files like `.rs`, which doesn't have an+/// extension according to std::path::Path::extension.+pub fn file_name_ext<'a>(name: &Cow<'a, [u8]>) -> Option<Cow<'a, [u8]>> {+ if name.is_empty() {+ return None;+ }+ let last_dot_at = match name.rfind_byte(b'.') {+ None => return None,+ Some(i) => i,+ };+ Some(match *name {+ Cow::Borrowed(name) => Cow::Borrowed(&name[last_dot_at..]),+ Cow::Owned(ref name) => {+ let mut name = name.clone();+ name.drain_bytes(..last_dot_at);+ Cow::Owned(name)+ }+ })+}++/// Normalizes a path to use `/` as a separator everywhere, even on platforms+/// that recognize other characters as separators.+#[cfg(unix)]+pub fn normalize_path(path: Cow<[u8]>) -> Cow<[u8]> {+ // UNIX only uses /, so we're good.+ path+}++/// Normalizes a path to use `/` as a separator everywhere, even on platforms+/// that recognize other characters as separators.+#[cfg(not(unix))]+pub fn normalize_path(mut path: Cow<[u8]>) -> Cow<[u8]> {+ use std::path::is_separator;++ for i in 0..path.len() {+ if path[i] == b'/' || !is_separator(path[i] as char) {+ continue;+ }+ path.to_mut()[i] = b'/';+ }+ path+}++#[cfg(test)]+mod tests {+ use std::borrow::Cow;++ use bstr::{ByteVec, B};++ use super::{file_name_ext, normalize_path};++ macro_rules! ext {+ ($name:ident, $file_name:expr, $ext:expr) => {+ #[test]+ fn $name() {+ let bs = Vec::from($file_name);+ let got = file_name_ext(&Cow::Owned(bs));+ assert_eq!($ext.map(|s| Cow::Borrowed(B(s))), got);+ }+ };+ }++ ext!(ext1, "foo.rs", Some(".rs"));+ ext!(ext2, ".rs", Some(".rs"));+ ext!(ext3, "..rs", Some(".rs"));+ ext!(ext4, "", None::<&str>);+ ext!(ext5, "foo", None::<&str>);++ macro_rules! normalize {+ ($name:ident, $path:expr, $expected:expr) => {+ #[test]+ fn $name() {+ let bs = Vec::from_slice($path);+ let got = normalize_path(Cow::Owned(bs));+ assert_eq!($expected.to_vec(), got.into_owned());+ }+ };+ }++ normalize!(normal1, b"foo", b"foo");+ normalize!(normal2, b"foo/bar", b"foo/bar");+ #[cfg(unix)]+ normalize!(normal3, b"foo\\bar", b"foo\\bar");+ #[cfg(not(unix))]+ normalize!(normal3, b"foo\\bar", b"foo/bar");+ #[cfg(unix)]+ normalize!(normal4, b"foo\\bar/baz", b"foo\\bar/baz");+ #[cfg(not(unix))]+ normalize!(normal4, b"foo\\bar/baz", b"foo/bar/baz");+}
@@ -0,0 +1,3 @@+This project is dual-licensed under the Unlicense and MIT licenses.++You may use this code under the terms of either license.
@@ -0,0 +1,32 @@+[package]+name = "grep"+version = "0.2.4" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+Fast line oriented regex searching as a library.+"""+documentation = "http://burntsushi.net/rustdoc/grep/"+homepage = "https://github.com/BurntSushi/ripgrep"+repository = "https://github.com/BurntSushi/ripgrep"+readme = "README.md"+keywords = ["regex", "grep", "egrep", "search", "pattern"]+license = "Unlicense/MIT"++[dependencies]+grep-cli = { version = "0.1.2", path = "../grep-cli" }+grep-matcher = { version = "0.1.2", path = "../grep-matcher" }+grep-pcre2 = { version = "0.1.3", path = "../grep-pcre2", optional = true }+grep-printer = { version = "0.1.2", path = "../grep-printer" }+grep-regex = { version = "0.1.3", path = "../grep-regex" }+grep-searcher = { version = "0.1.4", path = "../grep-searcher" }++[dev-dependencies]+termcolor = "1.0.4"+walkdir = "2.2.7"++[features]+simd-accel = ["grep-searcher/simd-accel"]+pcre2 = ["grep-pcre2"]++# This feature is DEPRECATED. Runtime dispatch is used for SIMD now.+avx-accel = []
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,41 @@+grep+----+ripgrep, as a library.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/grep)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).+++### Documentation++[https://docs.rs/grep](https://docs.rs/grep)++NOTE: This crate isn't ready for wide use yet. Ambitious individuals can+probably piece together the parts, but there is no high level documentation+describing how all of the pieces fit together.+++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+grep = "0.2"+```++and this to your crate root:++```rust+extern crate grep;+```+++### Features++This crate provides a `pcre2` feature (disabled by default) which, when+enabled, re-exports the `grep-pcre2` crate as an alternative `Matcher`+implementation to the standard `grep-regex` implementation.
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
crates/grep/examples/simplegrep.rs
@@ -0,0 +1,72 @@+extern crate grep;+extern crate termcolor;+extern crate walkdir;++use std::env;+use std::error::Error;+use std::ffi::OsString;+use std::process;++use grep::cli;+use grep::printer::{ColorSpecs, StandardBuilder};+use grep::regex::RegexMatcher;+use grep::searcher::{BinaryDetection, SearcherBuilder};+use termcolor::ColorChoice;+use walkdir::WalkDir;++fn main() {+ if let Err(err) = try_main() {+ eprintln!("{}", err);+ process::exit(1);+ }+}++fn try_main() -> Result<(), Box<dyn Error>> {+ let mut args: Vec<OsString> = env::args_os().collect();+ if args.len() < 2 {+ return Err("Usage: simplegrep <pattern> [<path> ...]".into());+ }+ if args.len() == 2 {+ args.push(OsString::from("./"));+ }+ search(cli::pattern_from_os(&args[1])?, &args[2..])+}++fn search(pattern: &str, paths: &[OsString]) -> Result<(), Box<dyn Error>> {+ let matcher = RegexMatcher::new_line_matcher(&pattern)?;+ let mut searcher = SearcherBuilder::new()+ .binary_detection(BinaryDetection::quit(b'\x00'))+ .line_number(false)+ .build();+ let mut printer = StandardBuilder::new()+ .color_specs(ColorSpecs::default_with_color())+ .build(cli::stdout(if cli::is_tty_stdout() {+ ColorChoice::Auto+ } else {+ ColorChoice::Never+ }));++ for path in paths {+ for result in WalkDir::new(path) {+ let dent = match result {+ Ok(dent) => dent,+ Err(err) => {+ eprintln!("{}", err);+ continue;+ }+ };+ if !dent.file_type().is_file() {+ continue;+ }+ let result = searcher.search_path(+ &matcher,+ dent.path(),+ printer.sink_with_path(&matcher, dent.path()),+ );+ if let Err(err) = result {+ eprintln!("{}: {}", dent.path().display(), err);+ }+ }+ }+ Ok(())+}
@@ -0,0 +1,23 @@+/*!+ripgrep, as a library.++This library is intended to provide a high level facade to the crates that+make up ripgrep's core searching routines. However, there is no high level+documentation available yet guiding users on how to fit all of the pieces+together.++Every public API item in the constituent crates is documented, but examples+are sparse.++A cookbook and a guide are planned.+*/++#![deny(missing_docs)]++pub extern crate grep_cli as cli;+pub extern crate grep_matcher as matcher;+#[cfg(feature = "pcre2")]+pub extern crate grep_pcre2 as pcre2;+pub extern crate grep_printer as printer;+pub extern crate grep_regex as regex;+pub extern crate grep_searcher as searcher;
@@ -0,0 +1,3 @@+This project is dual-licensed under the Unlicense and MIT licenses.++You may use this code under the terms of either license.
@@ -0,0 +1,36 @@+[package]+name = "ignore"+version = "0.4.11" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+A fast library for efficiently matching ignore files such as `.gitignore`+against file paths.+"""+documentation = "https://docs.rs/ignore"+homepage = "https://github.com/BurntSushi/ripgrep/tree/master/ignore"+repository = "https://github.com/BurntSushi/ripgrep/tree/master/ignore"+readme = "README.md"+keywords = ["glob", "ignore", "gitignore", "pattern", "file"]+license = "Unlicense/MIT"++[lib]+name = "ignore"+bench = false++[dependencies]+crossbeam-channel = "0.4.0"+crossbeam-utils = "0.7.0"+globset = { version = "0.4.3", path = "../globset" }+lazy_static = "1.1"+log = "0.4.5"+memchr = "2.1"+regex = "1.1"+same-file = "1.0.4"+thread_local = "1"+walkdir = "2.2.7"++[target.'cfg(windows)'.dependencies.winapi-util]+version = "0.1.2"++[features]+simd-accel = ["globset/simd-accel"]
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,66 @@+ignore+======+The ignore crate provides a fast recursive directory iterator that respects+various filters such as globs, file types and `.gitignore` files. This crate+also provides lower level direct access to gitignore and file type matchers.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/ignore)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).++### Documentation++[https://docs.rs/ignore](https://docs.rs/ignore)++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+ignore = "0.4"+```++and this to your crate root:++```rust+extern crate ignore;+```++### Example++This example shows the most basic usage of this crate. This code will+recursively traverse the current directory while automatically filtering out+files and directories according to ignore globs found in files like+`.ignore` and `.gitignore`:+++```rust,no_run+use ignore::Walk;++for result in Walk::new("./") {+ // Each item yielded by the iterator is either a directory entry or an+ // error, so either print the path or the error.+ match result {+ Ok(entry) => println!("{}", entry.path().display()),+ Err(err) => println!("ERROR: {}", err),+ }+}+```++### Example: advanced++By default, the recursive directory iterator will ignore hidden files and+directories. This can be disabled by building the iterator with `WalkBuilder`:++```rust,no_run+use ignore::WalkBuilder;++for result in WalkBuilder::new("./").hidden(false).build() {+ println!("{:?}", result);+}+```++See the documentation for `WalkBuilder` for many other options.
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
crates/ignore/examples/walk.rs
@@ -0,0 +1,84 @@+extern crate crossbeam_channel as channel;+extern crate ignore;+extern crate walkdir;++use std::env;+use std::io::{self, Write};+use std::path::Path;+use std::thread;++use ignore::WalkBuilder;+use walkdir::WalkDir;++fn main() {+ let mut path = env::args().nth(1).unwrap();+ let mut parallel = false;+ let mut simple = false;+ let (tx, rx) = channel::bounded::<DirEntry>(100);+ if path == "parallel" {+ path = env::args().nth(2).unwrap();+ parallel = true;+ } else if path == "walkdir" {+ path = env::args().nth(2).unwrap();+ simple = true;+ }++ let stdout_thread = thread::spawn(move || {+ let mut stdout = io::BufWriter::new(io::stdout());+ for dent in rx {+ write_path(&mut stdout, dent.path());+ }+ });++ if parallel {+ let walker = WalkBuilder::new(path).threads(6).build_parallel();+ walker.run(|| {+ let tx = tx.clone();+ Box::new(move |result| {+ use ignore::WalkState::*;++ tx.send(DirEntry::Y(result.unwrap())).unwrap();+ Continue+ })+ });+ } else if simple {+ let walker = WalkDir::new(path);+ for result in walker {+ tx.send(DirEntry::X(result.unwrap())).unwrap();+ }+ } else {+ let walker = WalkBuilder::new(path).build();+ for result in walker {+ tx.send(DirEntry::Y(result.unwrap())).unwrap();+ }+ }+ drop(tx);+ stdout_thread.join().unwrap();+}++enum DirEntry {+ X(walkdir::DirEntry),+ Y(ignore::DirEntry),+}++impl DirEntry {+ fn path(&self) -> &Path {+ match *self {+ DirEntry::X(ref x) => x.path(),+ DirEntry::Y(ref y) => y.path(),+ }+ }+}++#[cfg(unix)]+fn write_path<W: Write>(mut wtr: W, path: &Path) {+ use std::os::unix::ffi::OsStrExt;+ wtr.write(path.as_os_str().as_bytes()).unwrap();+ wtr.write(b"\n").unwrap();+}++#[cfg(not(unix))]+fn write_path<W: Write>(mut wtr: W, path: &Path) {+ wtr.write(path.to_string_lossy().as_bytes()).unwrap();+ wtr.write(b"\n").unwrap();+}
crates/ignore/src/default_types.rs
@@ -0,0 +1,244 @@+/// This list represents the default file types that ripgrep ships with. In+/// general, any file format is fair game, although it should generally be+/// limited to reasonably popular open formats. For other cases, you can add+/// types to each invocation of ripgrep with the '--type-add' flag.+///+/// If you would like to add or improve this list, please file a PR:+/// https://github.com/BurntSushi/ripgrep+///+/// Please try to keep this list sorted lexicographically and wrapped to 79+/// columns (inclusive).+#[rustfmt::skip]+pub const DEFAULT_TYPES: &[(&str, &[&str])] = &[+ ("agda", &["*.agda", "*.lagda"]),+ ("aidl", &["*.aidl"]),+ ("amake", &["*.mk", "*.bp"]),+ ("asciidoc", &["*.adoc", "*.asc", "*.asciidoc"]),+ ("asm", &["*.asm", "*.s", "*.S"]),+ ("asp", &[+ "*.aspx", "*.aspx.cs", "*.aspx.cs", "*.ascx", "*.ascx.cs", "*.ascx.vb",+ ]),+ ("ats", &["*.ats", "*.dats", "*.sats", "*.hats"]),+ ("avro", &["*.avdl", "*.avpr", "*.avsc"]),+ ("awk", &["*.awk"]),+ ("bazel", &["*.bzl", "WORKSPACE", "BUILD", "BUILD.bazel"]),+ ("bitbake", &["*.bb", "*.bbappend", "*.bbclass", "*.conf", "*.inc"]),+ ("brotli", &["*.br"]),+ ("buildstream", &["*.bst"]),+ ("bzip2", &["*.bz2", "*.tbz2"]),+ ("c", &["*.[chH]", "*.[chH].in", "*.cats"]),+ ("cabal", &["*.cabal"]),+ ("cbor", &["*.cbor"]),+ ("ceylon", &["*.ceylon"]),+ ("clojure", &["*.clj", "*.cljc", "*.cljs", "*.cljx"]),+ ("cmake", &["*.cmake", "CMakeLists.txt"]),+ ("coffeescript", &["*.coffee"]),+ ("config", &["*.cfg", "*.conf", "*.config", "*.ini"]),+ ("cpp", &[+ "*.[ChH]", "*.cc", "*.[ch]pp", "*.[ch]xx", "*.hh", "*.inl",+ "*.[ChH].in", "*.cc.in", "*.[ch]pp.in", "*.[ch]xx.in", "*.hh.in",+ ]),+ ("creole", &["*.creole"]),+ ("crystal", &["Projectfile", "*.cr"]),+ ("cs", &["*.cs"]),+ ("csharp", &["*.cs"]),+ ("cshtml", &["*.cshtml"]),+ ("css", &["*.css", "*.scss"]),+ ("csv", &["*.csv"]),+ ("cython", &["*.pyx", "*.pxi", "*.pxd"]),+ ("d", &["*.d"]),+ ("dart", &["*.dart"]),+ ("dhall", &["*.dhall"]),+ ("diff", &["*.patch", "*.diff"]),+ ("docker", &["*Dockerfile*"]),+ ("edn", &["*.edn"]),+ ("elisp", &["*.el"]),+ ("elixir", &["*.ex", "*.eex", "*.exs"]),+ ("elm", &["*.elm"]),+ ("erb", &["*.erb"]),+ ("erlang", &["*.erl", "*.hrl"]),+ ("fidl", &["*.fidl"]),+ ("fish", &["*.fish"]),+ ("fortran", &[+ "*.f", "*.F", "*.f77", "*.F77", "*.pfo",+ "*.f90", "*.F90", "*.f95", "*.F95",+ ]),+ ("fsharp", &["*.fs", "*.fsx", "*.fsi"]),+ ("gap", &["*.g", "*.gap", "*.gi", "*.gd", "*.tst"]),+ ("gn", &["*.gn", "*.gni"]),+ ("go", &["*.go"]),+ ("gradle", &["*.gradle"]),+ ("groovy", &["*.groovy", "*.gradle"]),+ ("gzip", &["*.gz", "*.tgz"]),+ ("h", &["*.h", "*.hpp"]),+ ("haml", &["*.haml"]),+ ("haskell", &["*.hs", "*.lhs", "*.cpphs", "*.c2hs", "*.hsc"]),+ ("hbs", &["*.hbs"]),+ ("hs", &["*.hs", "*.lhs"]),+ ("html", &["*.htm", "*.html", "*.ejs"]),+ ("idris", &["*.idr", "*.lidr"]),+ ("java", &["*.java", "*.jsp", "*.jspx", "*.properties"]),+ ("jinja", &["*.j2", "*.jinja", "*.jinja2"]),+ ("jl", &["*.jl"]),+ ("js", &["*.js", "*.jsx", "*.vue"]),+ ("json", &["*.json", "composer.lock"]),+ ("jsonl", &["*.jsonl"]),+ ("julia", &["*.jl"]),+ ("jupyter", &["*.ipynb", "*.jpynb"]),+ ("kotlin", &["*.kt", "*.kts"]),+ ("less", &["*.less"]),+ ("license", &[+ // General+ "COPYING", "COPYING[.-]*",+ "COPYRIGHT", "COPYRIGHT[.-]*",+ "EULA", "EULA[.-]*",+ "licen[cs]e", "licen[cs]e.*",+ "LICEN[CS]E", "LICEN[CS]E[.-]*", "*[.-]LICEN[CS]E*",+ "NOTICE", "NOTICE[.-]*",+ "PATENTS", "PATENTS[.-]*",+ "UNLICEN[CS]E", "UNLICEN[CS]E[.-]*",+ // GPL (gpl.txt, etc.)+ "agpl[.-]*",+ "gpl[.-]*",+ "lgpl[.-]*",+ // Other license-specific (APACHE-2.0.txt, etc.)+ "AGPL-*[0-9]*",+ "APACHE-*[0-9]*",+ "BSD-*[0-9]*",+ "CC-BY-*",+ "GFDL-*[0-9]*",+ "GNU-*[0-9]*",+ "GPL-*[0-9]*",+ "LGPL-*[0-9]*",+ "MIT-*[0-9]*",+ "MPL-*[0-9]*",+ "OFL-*[0-9]*",+ ]),+ ("lisp", &["*.el", "*.jl", "*.lisp", "*.lsp", "*.sc", "*.scm"]),+ ("lock", &["*.lock", "package-lock.json"]),+ ("log", &["*.log"]),+ ("lua", &["*.lua"]),+ ("lz4", &["*.lz4"]),+ ("lzma", &["*.lzma"]),+ ("m4", &["*.ac", "*.m4"]),+ ("make", &[+ "[Gg][Nn][Uu]makefile", "[Mm]akefile",+ "[Gg][Nn][Uu]makefile.am", "[Mm]akefile.am",+ "[Gg][Nn][Uu]makefile.in", "[Mm]akefile.in",+ "*.mk", "*.mak"+ ]),+ ("mako", &["*.mako", "*.mao"]),+ ("man", &["*.[0-9lnpx]", "*.[0-9][cEFMmpSx]"]),+ ("markdown", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),+ ("matlab", &["*.m"]),+ ("md", &["*.markdown", "*.md", "*.mdown", "*.mkdn"]),+ ("mk", &["mkfile"]),+ ("ml", &["*.ml"]),+ ("msbuild", &[+ "*.csproj", "*.fsproj", "*.vcxproj", "*.proj", "*.props", "*.targets",+ ]),+ ("nim", &["*.nim", "*.nimf", "*.nimble", "*.nims"]),+ ("nix", &["*.nix"]),+ ("objc", &["*.h", "*.m"]),+ ("objcpp", &["*.h", "*.mm"]),+ ("ocaml", &["*.ml", "*.mli", "*.mll", "*.mly"]),+ ("org", &["*.org", "*.org_archive"]),+ ("pascal", &["*.pas", "*.dpr", "*.lpr", "*.pp", "*.inc"]),+ ("pdf", &["*.pdf"]),+ ("perl", &["*.perl", "*.pl", "*.PL", "*.plh", "*.plx", "*.pm", "*.t"]),+ ("php", &["*.php", "*.php3", "*.php4", "*.php5", "*.phtml"]),+ ("pod", &["*.pod"]),+ ("postscript", &["*.eps", "*.ps"]),+ ("protobuf", &["*.proto"]),+ ("ps", &["*.cdxml", "*.ps1", "*.ps1xml", "*.psd1", "*.psm1"]),+ ("puppet", &["*.erb", "*.pp", "*.rb"]),+ ("purs", &["*.purs"]),+ ("py", &["*.py"]),+ ("qmake", &["*.pro", "*.pri", "*.prf"]),+ ("qml", &["*.qml"]),+ ("r", &["*.R", "*.r", "*.Rmd", "*.Rnw"]),+ ("rdoc", &["*.rdoc"]),+ ("readme", &["README*", "*README"]),+ ("robot", &["*.robot"]),+ ("rst", &["*.rst"]),+ ("ruby", &["Gemfile", "*.gemspec", ".irbrc", "Rakefile", "*.rb"]),+ ("rust", &["*.rs"]),+ ("sass", &["*.sass", "*.scss"]),+ ("scala", &["*.scala", "*.sbt"]),+ ("sh", &[+ // Portable/misc. init files+ ".login", ".logout", ".profile", "profile",+ // bash-specific init files+ ".bash_login", "bash_login",+ ".bash_logout", "bash_logout",+ ".bash_profile", "bash_profile",+ ".bashrc", "bashrc", "*.bashrc",+ // csh-specific init files+ ".cshrc", "*.cshrc",+ // ksh-specific init files+ ".kshrc", "*.kshrc",+ // tcsh-specific init files+ ".tcshrc",+ // zsh-specific init files+ ".zshenv", "zshenv",+ ".zlogin", "zlogin",+ ".zlogout", "zlogout",+ ".zprofile", "zprofile",+ ".zshrc", "zshrc",+ // Extensions+ "*.bash", "*.csh", "*.ksh", "*.sh", "*.tcsh", "*.zsh",+ ]),+ ("slim", &["*.skim", "*.slim", "*.slime"]),+ ("smarty", &["*.tpl"]),+ ("sml", &["*.sml", "*.sig"]),+ ("soy", &["*.soy"]),+ ("spark", &["*.spark"]),+ ("spec", &["*.spec"]),+ ("sql", &["*.sql", "*.psql"]),+ ("stylus", &["*.styl"]),+ ("sv", &["*.v", "*.vg", "*.sv", "*.svh", "*.h"]),+ ("svg", &["*.svg"]),+ ("swift", &["*.swift"]),+ ("swig", &["*.def", "*.i"]),+ ("systemd", &[+ "*.automount", "*.conf", "*.device", "*.link", "*.mount", "*.path",+ "*.scope", "*.service", "*.slice", "*.socket", "*.swap", "*.target",+ "*.timer",+ ]),+ ("taskpaper", &["*.taskpaper"]),+ ("tcl", &["*.tcl"]),+ ("tex", &["*.tex", "*.ltx", "*.cls", "*.sty", "*.bib", "*.dtx", "*.ins"]),+ ("textile", &["*.textile"]),+ ("tf", &["*.tf"]),+ ("thrift", &["*.thrift"]),+ ("toml", &["*.toml", "Cargo.lock"]),+ ("ts", &["*.ts", "*.tsx"]),+ ("twig", &["*.twig"]),+ ("txt", &["*.txt"]),+ ("typoscript", &["*.typoscript", "*.ts"]),+ ("vala", &["*.vala"]),+ ("vb", &["*.vb"]),+ ("verilog", &["*.v", "*.vh", "*.sv", "*.svh"]),+ ("vhdl", &["*.vhd", "*.vhdl"]),+ ("vim", &["*.vim"]),+ ("vimscript", &["*.vim"]),+ ("webidl", &["*.idl", "*.webidl", "*.widl"]),+ ("wiki", &["*.mediawiki", "*.wiki"]),+ ("xml", &[+ "*.xml", "*.xml.dist", "*.dtd", "*.xsl", "*.xslt", "*.xsd", "*.xjb",+ "*.rng", "*.sch", "*.xhtml",+ ]),+ ("xz", &["*.xz", "*.txz"]),+ ("yacc", &["*.y"]),+ ("yaml", &["*.yaml", "*.yml"]),+ ("zig", &["*.zig"]),+ ("zsh", &[+ ".zshenv", "zshenv",+ ".zlogin", "zlogin",+ ".zlogout", "zlogout",+ ".zprofile", "zprofile",+ ".zshrc", "zshrc",+ "*.zsh",+ ]),+ ("zstd", &["*.zst", "*.zstd"]),+];
@@ -0,0 +1,1200 @@+// This module provides a data structure, `Ignore`, that connects "directory+// traversal" with "ignore matchers." Specifically, it knows about gitignore+// semantics and precedence, and is organized based on directory hierarchy.+// Namely, every matcher logically corresponds to ignore rules from a single+// directory, and points to the matcher for its corresponding parent directory.+// In this sense, `Ignore` is a *persistent* data structure.+//+// This design was specifically chosen to make it possible to use this data+// structure in a parallel directory iterator.+//+// My initial intention was to expose this module as part of this crate's+// public API, but I think the data structure's public API is too complicated+// with non-obvious failure modes. Alas, such things haven't been documented+// well.++use std::collections::HashMap;+use std::ffi::{OsStr, OsString};+use std::fs::{File, FileType};+use std::io::{self, BufRead};+use std::path::{Path, PathBuf};+use std::sync::{Arc, RwLock};++use gitignore::{self, Gitignore, GitignoreBuilder};+use overrides::{self, Override};+use pathutil::{is_hidden, strip_prefix};+use types::{self, Types};+use walk::DirEntry;+use {Error, Match, PartialErrorBuilder};++/// IgnoreMatch represents information about where a match came from when using+/// the `Ignore` matcher.+#[derive(Clone, Debug)]+pub struct IgnoreMatch<'a>(IgnoreMatchInner<'a>);++/// IgnoreMatchInner describes precisely where the match information came from.+/// This is private to allow expansion to more matchers in the future.+#[derive(Clone, Debug)]+enum IgnoreMatchInner<'a> {+ Override(overrides::Glob<'a>),+ Gitignore(&'a gitignore::Glob),+ Types(types::Glob<'a>),+ Hidden,+}++impl<'a> IgnoreMatch<'a> {+ fn overrides(x: overrides::Glob<'a>) -> IgnoreMatch<'a> {+ IgnoreMatch(IgnoreMatchInner::Override(x))+ }++ fn gitignore(x: &'a gitignore::Glob) -> IgnoreMatch<'a> {+ IgnoreMatch(IgnoreMatchInner::Gitignore(x))+ }++ fn types(x: types::Glob<'a>) -> IgnoreMatch<'a> {+ IgnoreMatch(IgnoreMatchInner::Types(x))+ }++ fn hidden() -> IgnoreMatch<'static> {+ IgnoreMatch(IgnoreMatchInner::Hidden)+ }+}++/// Options for the ignore matcher, shared between the matcher itself and the+/// builder.+#[derive(Clone, Copy, Debug)]+struct IgnoreOptions {+ /// Whether to ignore hidden file paths or not.+ hidden: bool,+ /// Whether to read .ignore files.+ ignore: bool,+ /// Whether to respect any ignore files in parent directories.+ parents: bool,+ /// Whether to read git's global gitignore file.+ git_global: bool,+ /// Whether to read .gitignore files.+ git_ignore: bool,+ /// Whether to read .git/info/exclude files.+ git_exclude: bool,+ /// Whether to ignore files case insensitively+ ignore_case_insensitive: bool,+ /// Whether a git repository must be present in order to apply any+ /// git-related ignore rules.+ require_git: bool,+}++/// Ignore is a matcher useful for recursively walking one or more directories.+#[derive(Clone, Debug)]+pub struct Ignore(Arc<IgnoreInner>);++#[derive(Clone, Debug)]+struct IgnoreInner {+ /// A map of all existing directories that have already been+ /// compiled into matchers.+ ///+ /// Note that this is never used during matching, only when adding new+ /// parent directory matchers. This avoids needing to rebuild glob sets for+ /// parent directories if many paths are being searched.+ compiled: Arc<RwLock<HashMap<OsString, Ignore>>>,+ /// The path to the directory that this matcher was built from.+ dir: PathBuf,+ /// An override matcher (default is empty).+ overrides: Arc<Override>,+ /// A file type matcher.+ types: Arc<Types>,+ /// The parent directory to match next.+ ///+ /// If this is the root directory or there are otherwise no more+ /// directories to match, then `parent` is `None`.+ parent: Option<Ignore>,+ /// Whether this is an absolute parent matcher, as added by add_parent.+ is_absolute_parent: bool,+ /// The absolute base path of this matcher. Populated only if parent+ /// directories are added.+ absolute_base: Option<Arc<PathBuf>>,+ /// Explicit global ignore matchers specified by the caller.+ explicit_ignores: Arc<Vec<Gitignore>>,+ /// Ignore files used in addition to `.ignore`+ custom_ignore_filenames: Arc<Vec<OsString>>,+ /// The matcher for custom ignore files+ custom_ignore_matcher: Gitignore,+ /// The matcher for .ignore files.+ ignore_matcher: Gitignore,+ /// A global gitignore matcher, usually from $XDG_CONFIG_HOME/git/ignore.+ git_global_matcher: Arc<Gitignore>,+ /// The matcher for .gitignore files.+ git_ignore_matcher: Gitignore,+ /// Special matcher for `.git/info/exclude` files.+ git_exclude_matcher: Gitignore,+ /// Whether this directory contains a .git sub-directory.+ has_git: bool,+ /// Ignore config.+ opts: IgnoreOptions,+}++impl Ignore {+ /// Return the directory path of this matcher.+ pub fn path(&self) -> &Path {+ &self.0.dir+ }++ /// Return true if this matcher has no parent.+ pub fn is_root(&self) -> bool {+ self.0.parent.is_none()+ }++ /// Returns true if this matcher was added via the `add_parents` method.+ pub fn is_absolute_parent(&self) -> bool {+ self.0.is_absolute_parent+ }++ /// Return this matcher's parent, if one exists.+ pub fn parent(&self) -> Option<Ignore> {+ self.0.parent.clone()+ }++ /// Create a new `Ignore` matcher with the parent directories of `dir`.+ ///+ /// Note that this can only be called on an `Ignore` matcher with no+ /// parents (i.e., `is_root` returns `true`). This will panic otherwise.+ pub fn add_parents<P: AsRef<Path>>(+ &self,+ path: P,+ ) -> (Ignore, Option<Error>) {+ if !self.0.opts.parents+ && !self.0.opts.git_ignore+ && !self.0.opts.git_exclude+ && !self.0.opts.git_global+ {+ // If we never need info from parent directories, then don't do+ // anything.+ return (self.clone(), None);+ }+ if !self.is_root() {+ panic!("Ignore::add_parents called on non-root matcher");+ }+ let absolute_base = match path.as_ref().canonicalize() {+ Ok(path) => Arc::new(path),+ Err(_) => {+ // There's not much we can do here, so just return our+ // existing matcher. We drop the error to be consistent+ // with our general pattern of ignoring I/O errors when+ // processing ignore files.+ return (self.clone(), None);+ }+ };+ // List of parents, from child to root.+ let mut parents = vec![];+ let mut path = &**absolute_base;+ while let Some(parent) = path.parent() {+ parents.push(parent);+ path = parent;+ }+ let mut errs = PartialErrorBuilder::default();+ let mut ig = self.clone();+ for parent in parents.into_iter().rev() {+ let mut compiled = self.0.compiled.write().unwrap();+ if let Some(prebuilt) = compiled.get(parent.as_os_str()) {+ ig = prebuilt.clone();+ continue;+ }+ let (mut igtmp, err) = ig.add_child_path(parent);+ errs.maybe_push(err);+ igtmp.is_absolute_parent = true;+ igtmp.absolute_base = Some(absolute_base.clone());+ igtmp.has_git = if self.0.opts.git_ignore {+ parent.join(".git").exists()+ } else {+ false+ };+ ig = Ignore(Arc::new(igtmp));+ compiled.insert(parent.as_os_str().to_os_string(), ig.clone());+ }+ (ig, errs.into_error_option())+ }++ /// Create a new `Ignore` matcher for the given child directory.+ ///+ /// Since building the matcher may require reading from multiple+ /// files, it's possible that this method partially succeeds. Therefore,+ /// a matcher is always returned (which may match nothing) and an error is+ /// returned if it exists.+ ///+ /// Note that all I/O errors are completely ignored.+ pub fn add_child<P: AsRef<Path>>(+ &self,+ dir: P,+ ) -> (Ignore, Option<Error>) {+ let (ig, err) = self.add_child_path(dir.as_ref());+ (Ignore(Arc::new(ig)), err)+ }++ /// Like add_child, but takes a full path and returns an IgnoreInner.+ fn add_child_path(&self, dir: &Path) -> (IgnoreInner, Option<Error>) {+ let git_type = if self.0.opts.git_ignore || self.0.opts.git_exclude {+ dir.join(".git").metadata().ok().map(|md| md.file_type())+ } else {+ None+ };+ let has_git = git_type.map(|_| true).unwrap_or(false);++ let mut errs = PartialErrorBuilder::default();+ let custom_ig_matcher = if self.0.custom_ignore_filenames.is_empty() {+ Gitignore::empty()+ } else {+ let (m, err) = create_gitignore(+ &dir,+ &dir,+ &self.0.custom_ignore_filenames,+ self.0.opts.ignore_case_insensitive,+ );+ errs.maybe_push(err);+ m+ };+ let ig_matcher = if !self.0.opts.ignore {+ Gitignore::empty()+ } else {+ let (m, err) = create_gitignore(+ &dir,+ &dir,+ &[".ignore"],+ self.0.opts.ignore_case_insensitive,+ );+ errs.maybe_push(err);+ m+ };+ let gi_matcher = if !self.0.opts.git_ignore {+ Gitignore::empty()+ } else {+ let (m, err) = create_gitignore(+ &dir,+ &dir,+ &[".gitignore"],+ self.0.opts.ignore_case_insensitive,+ );+ errs.maybe_push(err);+ m+ };+ let gi_exclude_matcher = if !self.0.opts.git_exclude {+ Gitignore::empty()+ } else {+ match resolve_git_commondir(dir, git_type) {+ Ok(git_dir) => {+ let (m, err) = create_gitignore(+ &dir,+ &git_dir,+ &["info/exclude"],+ self.0.opts.ignore_case_insensitive,+ );+ errs.maybe_push(err);+ m+ }+ Err(err) => {+ errs.maybe_push(err);+ Gitignore::empty()+ }+ }+ };+ let ig = IgnoreInner {+ compiled: self.0.compiled.clone(),+ dir: dir.to_path_buf(),+ overrides: self.0.overrides.clone(),+ types: self.0.types.clone(),+ parent: Some(self.clone()),+ is_absolute_parent: false,+ absolute_base: self.0.absolute_base.clone(),+ explicit_ignores: self.0.explicit_ignores.clone(),+ custom_ignore_filenames: self.0.custom_ignore_filenames.clone(),+ custom_ignore_matcher: custom_ig_matcher,+ ignore_matcher: ig_matcher,+ git_global_matcher: self.0.git_global_matcher.clone(),+ git_ignore_matcher: gi_matcher,+ git_exclude_matcher: gi_exclude_matcher,+ has_git: has_git,+ opts: self.0.opts,+ };+ (ig, errs.into_error_option())+ }++ /// Returns true if at least one type of ignore rule should be matched.+ fn has_any_ignore_rules(&self) -> bool {+ let opts = self.0.opts;+ let has_custom_ignore_files =+ !self.0.custom_ignore_filenames.is_empty();+ let has_explicit_ignores = !self.0.explicit_ignores.is_empty();++ opts.ignore+ || opts.git_global+ || opts.git_ignore+ || opts.git_exclude+ || has_custom_ignore_files+ || has_explicit_ignores+ }++ /// Like `matched`, but works with a directory entry instead.+ pub fn matched_dir_entry<'a>(+ &'a self,+ dent: &DirEntry,+ ) -> Match<IgnoreMatch<'a>> {+ let m = self.matched(dent.path(), dent.is_dir());+ if m.is_none() && self.0.opts.hidden && is_hidden(dent) {+ return Match::Ignore(IgnoreMatch::hidden());+ }+ m+ }++ /// Returns a match indicating whether the given file path should be+ /// ignored or not.+ ///+ /// The match contains information about its origin.+ fn matched<'a, P: AsRef<Path>>(+ &'a self,+ path: P,+ is_dir: bool,+ ) -> Match<IgnoreMatch<'a>> {+ // We need to be careful with our path. If it has a leading ./, then+ // strip it because it causes nothing but trouble.+ let mut path = path.as_ref();+ if let Some(p) = strip_prefix("./", path) {+ path = p;+ }+ // Match against the override patterns. If an override matches+ // regardless of whether it's whitelist/ignore, then we quit and+ // return that result immediately. Overrides have the highest+ // precedence.+ if !self.0.overrides.is_empty() {+ let mat = self+ .0+ .overrides+ .matched(path, is_dir)+ .map(IgnoreMatch::overrides);+ if !mat.is_none() {+ return mat;+ }+ }+ let mut whitelisted = Match::None;+ if self.has_any_ignore_rules() {+ let mat = self.matched_ignore(path, is_dir);+ if mat.is_ignore() {+ return mat;+ } else if mat.is_whitelist() {+ whitelisted = mat;+ }+ }+ if !self.0.types.is_empty() {+ let mat =+ self.0.types.matched(path, is_dir).map(IgnoreMatch::types);+ if mat.is_ignore() {+ return mat;+ } else if mat.is_whitelist() {+ whitelisted = mat;+ }+ }+ whitelisted+ }++ /// Performs matching only on the ignore files for this directory and+ /// all parent directories.+ fn matched_ignore<'a>(+ &'a self,+ path: &Path,+ is_dir: bool,+ ) -> Match<IgnoreMatch<'a>> {+ let (+ mut m_custom_ignore,+ mut m_ignore,+ mut m_gi,+ mut m_gi_exclude,+ mut m_explicit,+ ) = (Match::None, Match::None, Match::None, Match::None, Match::None);+ let any_git =+ !self.0.opts.require_git || self.parents().any(|ig| ig.0.has_git);+ let mut saw_git = false;+ for ig in self.parents().take_while(|ig| !ig.0.is_absolute_parent) {+ if m_custom_ignore.is_none() {+ m_custom_ignore =+ ig.0.custom_ignore_matcher+ .matched(path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ if m_ignore.is_none() {+ m_ignore =+ ig.0.ignore_matcher+ .matched(path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ if any_git && !saw_git && m_gi.is_none() {+ m_gi =+ ig.0.git_ignore_matcher+ .matched(path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ if any_git && !saw_git && m_gi_exclude.is_none() {+ m_gi_exclude =+ ig.0.git_exclude_matcher+ .matched(path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ saw_git = saw_git || ig.0.has_git;+ }+ if self.0.opts.parents {+ if let Some(abs_parent_path) = self.absolute_base() {+ let path = abs_parent_path.join(path);+ for ig in+ self.parents().skip_while(|ig| !ig.0.is_absolute_parent)+ {+ if m_custom_ignore.is_none() {+ m_custom_ignore =+ ig.0.custom_ignore_matcher+ .matched(&path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ if m_ignore.is_none() {+ m_ignore =+ ig.0.ignore_matcher+ .matched(&path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ if any_git && !saw_git && m_gi.is_none() {+ m_gi =+ ig.0.git_ignore_matcher+ .matched(&path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ if any_git && !saw_git && m_gi_exclude.is_none() {+ m_gi_exclude =+ ig.0.git_exclude_matcher+ .matched(&path, is_dir)+ .map(IgnoreMatch::gitignore);+ }+ saw_git = saw_git || ig.0.has_git;+ }+ }+ }+ for gi in self.0.explicit_ignores.iter().rev() {+ if !m_explicit.is_none() {+ break;+ }+ m_explicit = gi.matched(&path, is_dir).map(IgnoreMatch::gitignore);+ }+ let m_global = if any_git {+ self.0+ .git_global_matcher+ .matched(&path, is_dir)+ .map(IgnoreMatch::gitignore)+ } else {+ Match::None+ };++ m_custom_ignore+ .or(m_ignore)+ .or(m_gi)+ .or(m_gi_exclude)+ .or(m_global)+ .or(m_explicit)+ }++ /// Returns an iterator over parent ignore matchers, including this one.+ pub fn parents(&self) -> Parents {+ Parents(Some(self))+ }++ /// Returns the first absolute path of the first absolute parent, if+ /// one exists.+ fn absolute_base(&self) -> Option<&Path> {+ self.0.absolute_base.as_ref().map(|p| &***p)+ }+}++/// An iterator over all parents of an ignore matcher, including itself.+///+/// The lifetime `'a` refers to the lifetime of the initial `Ignore` matcher.+pub struct Parents<'a>(Option<&'a Ignore>);++impl<'a> Iterator for Parents<'a> {+ type Item = &'a Ignore;++ fn next(&mut self) -> Option<&'a Ignore> {+ match self.0.take() {+ None => None,+ Some(ig) => {+ self.0 = ig.0.parent.as_ref();+ Some(ig)+ }+ }+ }+}++/// A builder for creating an Ignore matcher.+#[derive(Clone, Debug)]+pub struct IgnoreBuilder {+ /// The root directory path for this ignore matcher.+ dir: PathBuf,+ /// An override matcher (default is empty).+ overrides: Arc<Override>,+ /// A type matcher (default is empty).+ types: Arc<Types>,+ /// Explicit global ignore matchers.+ explicit_ignores: Vec<Gitignore>,+ /// Ignore files in addition to .ignore.+ custom_ignore_filenames: Vec<OsString>,+ /// Ignore config.+ opts: IgnoreOptions,+}++impl IgnoreBuilder {+ /// Create a new builder for an `Ignore` matcher.+ ///+ /// All relative file paths are resolved with respect to the current+ /// working directory.+ pub fn new() -> IgnoreBuilder {+ IgnoreBuilder {+ dir: Path::new("").to_path_buf(),+ overrides: Arc::new(Override::empty()),+ types: Arc::new(Types::empty()),+ explicit_ignores: vec![],+ custom_ignore_filenames: vec![],+ opts: IgnoreOptions {+ hidden: true,+ ignore: true,+ parents: true,+ git_global: true,+ git_ignore: true,+ git_exclude: true,+ ignore_case_insensitive: false,+ require_git: true,+ },+ }+ }++ /// Builds a new `Ignore` matcher.+ ///+ /// The matcher returned won't match anything until ignore rules from+ /// directories are added to it.+ pub fn build(&self) -> Ignore {+ let git_global_matcher = if !self.opts.git_global {+ Gitignore::empty()+ } else {+ let mut builder = GitignoreBuilder::new("");+ builder+ .case_insensitive(self.opts.ignore_case_insensitive)+ .unwrap();+ let (gi, err) = builder.build_global();+ if let Some(err) = err {+ debug!("{}", err);+ }+ gi+ };++ Ignore(Arc::new(IgnoreInner {+ compiled: Arc::new(RwLock::new(HashMap::new())),+ dir: self.dir.clone(),+ overrides: self.overrides.clone(),+ types: self.types.clone(),+ parent: None,+ is_absolute_parent: true,+ absolute_base: None,+ explicit_ignores: Arc::new(self.explicit_ignores.clone()),+ custom_ignore_filenames: Arc::new(+ self.custom_ignore_filenames.clone(),+ ),+ custom_ignore_matcher: Gitignore::empty(),+ ignore_matcher: Gitignore::empty(),+ git_global_matcher: Arc::new(git_global_matcher),+ git_ignore_matcher: Gitignore::empty(),+ git_exclude_matcher: Gitignore::empty(),+ has_git: false,+ opts: self.opts,+ }))+ }++ /// Add an override matcher.+ ///+ /// By default, no override matcher is used.+ ///+ /// This overrides any previous setting.+ pub fn overrides(&mut self, overrides: Override) -> &mut IgnoreBuilder {+ self.overrides = Arc::new(overrides);+ self+ }++ /// Add a file type matcher.+ ///+ /// By default, no file type matcher is used.+ ///+ /// This overrides any previous setting.+ pub fn types(&mut self, types: Types) -> &mut IgnoreBuilder {+ self.types = Arc::new(types);+ self+ }++ /// Adds a new global ignore matcher from the ignore file path given.+ pub fn add_ignore(&mut self, ig: Gitignore) -> &mut IgnoreBuilder {+ self.explicit_ignores.push(ig);+ self+ }++ /// Add a custom ignore file name+ ///+ /// These ignore files have higher precedence than all other ignore files.+ ///+ /// When specifying multiple names, earlier names have lower precedence than+ /// later names.+ pub fn add_custom_ignore_filename<S: AsRef<OsStr>>(+ &mut self,+ file_name: S,+ ) -> &mut IgnoreBuilder {+ self.custom_ignore_filenames.push(file_name.as_ref().to_os_string());+ self+ }++ /// Enables ignoring hidden files.+ ///+ /// This is enabled by default.+ pub fn hidden(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.hidden = yes;+ self+ }++ /// Enables reading `.ignore` files.+ ///+ /// `.ignore` files have the same semantics as `gitignore` files and are+ /// supported by search tools such as ripgrep and The Silver Searcher.+ ///+ /// This is enabled by default.+ pub fn ignore(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.ignore = yes;+ self+ }++ /// Enables reading ignore files from parent directories.+ ///+ /// If this is enabled, then .gitignore files in parent directories of each+ /// file path given are respected. Otherwise, they are ignored.+ ///+ /// This is enabled by default.+ pub fn parents(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.parents = yes;+ self+ }++ /// Add a global gitignore matcher.+ ///+ /// Its precedence is lower than both normal `.gitignore` files and+ /// `.git/info/exclude` files.+ ///+ /// This overwrites any previous global gitignore setting.+ ///+ /// This is enabled by default.+ pub fn git_global(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.git_global = yes;+ self+ }++ /// Enables reading `.gitignore` files.+ ///+ /// `.gitignore` files have match semantics as described in the `gitignore`+ /// man page.+ ///+ /// This is enabled by default.+ pub fn git_ignore(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.git_ignore = yes;+ self+ }++ /// Enables reading `.git/info/exclude` files.+ ///+ /// `.git/info/exclude` files have match semantics as described in the+ /// `gitignore` man page.+ ///+ /// This is enabled by default.+ pub fn git_exclude(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.git_exclude = yes;+ self+ }++ /// Whether a git repository is required to apply git-related ignore+ /// rules (global rules, .gitignore and local exclude rules).+ ///+ /// When disabled, git-related ignore rules are applied even when searching+ /// outside a git repository.+ pub fn require_git(&mut self, yes: bool) -> &mut IgnoreBuilder {+ self.opts.require_git = yes;+ self+ }++ /// Process ignore files case insensitively+ ///+ /// This is disabled by default.+ pub fn ignore_case_insensitive(+ &mut self,+ yes: bool,+ ) -> &mut IgnoreBuilder {+ self.opts.ignore_case_insensitive = yes;+ self+ }+}++/// Creates a new gitignore matcher for the directory given.+///+/// The matcher is meant to match files below `dir`.+/// Ignore globs are extracted from each of the file names relative to+/// `dir_for_ignorefile` in the order given (earlier names have lower+/// precedence than later names).+///+/// I/O errors are ignored.+pub fn create_gitignore<T: AsRef<OsStr>>(+ dir: &Path,+ dir_for_ignorefile: &Path,+ names: &[T],+ case_insensitive: bool,+) -> (Gitignore, Option<Error>) {+ let mut builder = GitignoreBuilder::new(dir);+ let mut errs = PartialErrorBuilder::default();+ builder.case_insensitive(case_insensitive).unwrap();+ for name in names {+ let gipath = dir_for_ignorefile.join(name.as_ref());+ // This check is not necessary, but is added for performance. Namely,+ // a simple stat call checking for existence can often be just a bit+ // quicker than actually trying to open a file. Since the number of+ // directories without ignore files likely greatly exceeds the number+ // with ignore files, this check generally makes sense.+ //+ // However, until demonstrated otherwise, we speculatively do not do+ // this on Windows since Windows is notorious for having slow file+ // system operations. Namely, it's not clear whether this analysis+ // makes sense on Windows.+ //+ // For more details: https://github.com/BurntSushi/ripgrep/pull/1381+ if cfg!(windows) || gipath.exists() {+ errs.maybe_push_ignore_io(builder.add(gipath));+ }+ }+ let gi = match builder.build() {+ Ok(gi) => gi,+ Err(err) => {+ errs.push(err);+ GitignoreBuilder::new(dir).build().unwrap()+ }+ };+ (gi, errs.into_error_option())+}++/// Find the GIT_COMMON_DIR for the given git worktree.+///+/// This is the directory that may contain a private ignore file+/// "info/exclude". Unlike git, this function does *not* read environment+/// variables GIT_DIR and GIT_COMMON_DIR, because it is not clear how to use+/// them when multiple repositories are searched.+///+/// Some I/O errors are ignored.+fn resolve_git_commondir(+ dir: &Path,+ git_type: Option<FileType>,+) -> Result<PathBuf, Option<Error>> {+ let git_dir_path = || dir.join(".git");+ let git_dir = git_dir_path();+ if !git_type.map_or(false, |ft| ft.is_file()) {+ return Ok(git_dir);+ }+ let file = match File::open(git_dir) {+ Ok(file) => io::BufReader::new(file),+ Err(err) => {+ return Err(Some(Error::Io(err).with_path(git_dir_path())));+ }+ };+ let dot_git_line = match file.lines().next() {+ Some(Ok(line)) => line,+ Some(Err(err)) => {+ return Err(Some(Error::Io(err).with_path(git_dir_path())));+ }+ None => return Err(None),+ };+ if !dot_git_line.starts_with("gitdir: ") {+ return Err(None);+ }+ let real_git_dir = PathBuf::from(&dot_git_line["gitdir: ".len()..]);+ let git_commondir_file = || real_git_dir.join("commondir");+ let file = match File::open(git_commondir_file()) {+ Ok(file) => io::BufReader::new(file),+ Err(err) => {+ return Err(Some(Error::Io(err).with_path(git_commondir_file())));+ }+ };+ let commondir_line = match file.lines().next() {+ Some(Ok(line)) => line,+ Some(Err(err)) => {+ return Err(Some(Error::Io(err).with_path(git_commondir_file())));+ }+ None => return Err(None),+ };+ let commondir_abs = if commondir_line.starts_with(".") {+ real_git_dir.join(commondir_line) // relative commondir+ } else {+ PathBuf::from(commondir_line)+ };+ Ok(commondir_abs)+}++#[cfg(test)]+mod tests {+ use std::fs::{self, File};+ use std::io::{self, Write};+ use std::path::Path;++ use dir::IgnoreBuilder;+ use gitignore::Gitignore;+ use tests::TempDir;+ use Error;++ fn wfile<P: AsRef<Path>>(path: P, contents: &str) {+ let mut file = File::create(path).unwrap();+ file.write_all(contents.as_bytes()).unwrap();+ }++ fn mkdirp<P: AsRef<Path>>(path: P) {+ fs::create_dir_all(path).unwrap();+ }++ fn partial(err: Error) -> Vec<Error> {+ match err {+ Error::Partial(errs) => errs,+ _ => panic!("expected partial error but got {:?}", err),+ }+ }++ fn tmpdir() -> TempDir {+ TempDir::new().unwrap()+ }++ #[test]+ fn explicit_ignore() {+ let td = tmpdir();+ wfile(td.path().join("not-an-ignore"), "foo\n!bar");++ let (gi, err) = Gitignore::new(td.path().join("not-an-ignore"));+ assert!(err.is_none());+ let (ig, err) =+ IgnoreBuilder::new().add_ignore(gi).build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_ignore());+ assert!(ig.matched("bar", false).is_whitelist());+ assert!(ig.matched("baz", false).is_none());+ }++ #[test]+ fn git_exclude() {+ let td = tmpdir();+ mkdirp(td.path().join(".git/info"));+ wfile(td.path().join(".git/info/exclude"), "foo\n!bar");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_ignore());+ assert!(ig.matched("bar", false).is_whitelist());+ assert!(ig.matched("baz", false).is_none());+ }++ #[test]+ fn gitignore() {+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ wfile(td.path().join(".gitignore"), "foo\n!bar");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_ignore());+ assert!(ig.matched("bar", false).is_whitelist());+ assert!(ig.matched("baz", false).is_none());+ }++ #[test]+ fn gitignore_no_git() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "foo\n!bar");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_none());+ assert!(ig.matched("bar", false).is_none());+ assert!(ig.matched("baz", false).is_none());+ }++ #[test]+ fn gitignore_allowed_no_git() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "foo\n!bar");++ let (ig, err) = IgnoreBuilder::new()+ .require_git(false)+ .build()+ .add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_ignore());+ assert!(ig.matched("bar", false).is_whitelist());+ assert!(ig.matched("baz", false).is_none());+ }++ #[test]+ fn ignore() {+ let td = tmpdir();+ wfile(td.path().join(".ignore"), "foo\n!bar");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_ignore());+ assert!(ig.matched("bar", false).is_whitelist());+ assert!(ig.matched("baz", false).is_none());+ }++ #[test]+ fn custom_ignore() {+ let td = tmpdir();+ let custom_ignore = ".customignore";+ wfile(td.path().join(custom_ignore), "foo\n!bar");++ let (ig, err) = IgnoreBuilder::new()+ .add_custom_ignore_filename(custom_ignore)+ .build()+ .add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_ignore());+ assert!(ig.matched("bar", false).is_whitelist());+ assert!(ig.matched("baz", false).is_none());+ }++ // Tests that a custom ignore file will override an .ignore.+ #[test]+ fn custom_ignore_over_ignore() {+ let td = tmpdir();+ let custom_ignore = ".customignore";+ wfile(td.path().join(".ignore"), "foo");+ wfile(td.path().join(custom_ignore), "!foo");++ let (ig, err) = IgnoreBuilder::new()+ .add_custom_ignore_filename(custom_ignore)+ .build()+ .add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_whitelist());+ }++ // Tests that earlier custom ignore files have lower precedence than later.+ #[test]+ fn custom_ignore_precedence() {+ let td = tmpdir();+ let custom_ignore1 = ".customignore1";+ let custom_ignore2 = ".customignore2";+ wfile(td.path().join(custom_ignore1), "foo");+ wfile(td.path().join(custom_ignore2), "!foo");++ let (ig, err) = IgnoreBuilder::new()+ .add_custom_ignore_filename(custom_ignore1)+ .add_custom_ignore_filename(custom_ignore2)+ .build()+ .add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_whitelist());+ }++ // Tests that an .ignore will override a .gitignore.+ #[test]+ fn ignore_over_gitignore() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "foo");+ wfile(td.path().join(".ignore"), "!foo");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("foo", false).is_whitelist());+ }++ // Tests that exclude has lower precedent than both .ignore and .gitignore.+ #[test]+ fn exclude_lowest() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "!foo");+ wfile(td.path().join(".ignore"), "!bar");+ mkdirp(td.path().join(".git/info"));+ wfile(td.path().join(".git/info/exclude"), "foo\nbar\nbaz");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ assert!(ig.matched("baz", false).is_ignore());+ assert!(ig.matched("foo", false).is_whitelist());+ assert!(ig.matched("bar", false).is_whitelist());+ }++ #[test]+ fn errored() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "{foo");++ let (_, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_some());+ }++ #[test]+ fn errored_both() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "{foo");+ wfile(td.path().join(".ignore"), "{bar");++ let (_, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert_eq!(2, partial(err.expect("an error")).len());+ }++ #[test]+ fn errored_partial() {+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ wfile(td.path().join(".gitignore"), "{foo\nbar");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_some());+ assert!(ig.matched("bar", false).is_ignore());+ }++ #[test]+ fn errored_partial_and_ignore() {+ let td = tmpdir();+ wfile(td.path().join(".gitignore"), "{foo\nbar");+ wfile(td.path().join(".ignore"), "!bar");++ let (ig, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_some());+ assert!(ig.matched("bar", false).is_whitelist());+ }++ #[test]+ fn not_present_empty() {+ let td = tmpdir();++ let (_, err) = IgnoreBuilder::new().build().add_child(td.path());+ assert!(err.is_none());+ }++ #[test]+ fn stops_at_git_dir() {+ // This tests that .gitignore files beyond a .git barrier aren't+ // matched, but .ignore files are.+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ mkdirp(td.path().join("foo/.git"));+ wfile(td.path().join(".gitignore"), "foo");+ wfile(td.path().join(".ignore"), "bar");++ let ig0 = IgnoreBuilder::new().build();+ let (ig1, err) = ig0.add_child(td.path());+ assert!(err.is_none());+ let (ig2, err) = ig1.add_child(ig1.path().join("foo"));+ assert!(err.is_none());++ assert!(ig1.matched("foo", false).is_ignore());+ assert!(ig2.matched("foo", false).is_none());++ assert!(ig1.matched("bar", false).is_ignore());+ assert!(ig2.matched("bar", false).is_ignore());+ }++ #[test]+ fn absolute_parent() {+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ mkdirp(td.path().join("foo"));+ wfile(td.path().join(".gitignore"), "bar");++ // First, check that the parent gitignore file isn't detected if the+ // parent isn't added. This establishes a baseline.+ let ig0 = IgnoreBuilder::new().build();+ let (ig1, err) = ig0.add_child(td.path().join("foo"));+ assert!(err.is_none());+ assert!(ig1.matched("bar", false).is_none());++ // Second, check that adding a parent directory actually works.+ let ig0 = IgnoreBuilder::new().build();+ let (ig1, err) = ig0.add_parents(td.path().join("foo"));+ assert!(err.is_none());+ let (ig2, err) = ig1.add_child(td.path().join("foo"));+ assert!(err.is_none());+ assert!(ig2.matched("bar", false).is_ignore());+ }++ #[test]+ fn absolute_parent_anchored() {+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ mkdirp(td.path().join("src/llvm"));+ wfile(td.path().join(".gitignore"), "/llvm/\nfoo");++ let ig0 = IgnoreBuilder::new().build();+ let (ig1, err) = ig0.add_parents(td.path().join("src"));+ assert!(err.is_none());+ let (ig2, err) = ig1.add_child("src");+ assert!(err.is_none());++ assert!(ig1.matched("llvm", true).is_none());+ assert!(ig2.matched("llvm", true).is_none());+ assert!(ig2.matched("src/llvm", true).is_none());+ assert!(ig2.matched("foo", false).is_ignore());+ assert!(ig2.matched("src/foo", false).is_ignore());+ }++ #[test]+ fn git_info_exclude_in_linked_worktree() {+ let td = tmpdir();+ let git_dir = td.path().join(".git");+ mkdirp(git_dir.join("info"));+ wfile(git_dir.join("info/exclude"), "ignore_me");+ mkdirp(git_dir.join("worktrees/linked-worktree"));+ let commondir_path =+ || git_dir.join("worktrees/linked-worktree/commondir");+ mkdirp(td.path().join("linked-worktree"));+ let worktree_git_dir_abs = format!(+ "gitdir: {}",+ git_dir.join("worktrees/linked-worktree").to_str().unwrap(),+ );+ wfile(td.path().join("linked-worktree/.git"), &worktree_git_dir_abs);++ // relative commondir+ wfile(commondir_path(), "../..");+ let ib = IgnoreBuilder::new().build();+ let (ignore, err) = ib.add_child(td.path().join("linked-worktree"));+ assert!(err.is_none());+ assert!(ignore.matched("ignore_me", false).is_ignore());++ // absolute commondir+ wfile(commondir_path(), git_dir.to_str().unwrap());+ let (ignore, err) = ib.add_child(td.path().join("linked-worktree"));+ assert!(err.is_none());+ assert!(ignore.matched("ignore_me", false).is_ignore());++ // missing commondir file+ assert!(fs::remove_file(commondir_path()).is_ok());+ let (_, err) = ib.add_child(td.path().join("linked-worktree"));+ assert!(err.is_some());+ assert!(match err {+ Some(Error::WithPath { path, err }) => {+ if path != commondir_path() {+ false+ } else {+ match *err {+ Error::Io(ioerr) => {+ ioerr.kind() == io::ErrorKind::NotFound+ }+ _ => false,+ }+ }+ }+ _ => false,+ });++ wfile(td.path().join("linked-worktree/.git"), "garbage");+ let (_, err) = ib.add_child(td.path().join("linked-worktree"));+ assert!(err.is_none());++ wfile(td.path().join("linked-worktree/.git"), "gitdir: garbage");+ let (_, err) = ib.add_child(td.path().join("linked-worktree"));+ assert!(err.is_some());+ }+}
crates/ignore/src/gitignore.rs
@@ -0,0 +1,786 @@+/*!+The gitignore module provides a way to match globs from a gitignore file+against file paths.++Note that this module implements the specification as described in the+`gitignore` man page from scratch. That is, this module does *not* shell out to+the `git` command line tool.+*/++use std::cell::RefCell;+use std::env;+use std::fs::File;+use std::io::{self, BufRead, Read};+use std::path::{Path, PathBuf};+use std::str;+use std::sync::Arc;++use globset::{Candidate, GlobBuilder, GlobSet, GlobSetBuilder};+use regex::bytes::Regex;+use thread_local::ThreadLocal;++use pathutil::{is_file_name, strip_prefix};+use {Error, Match, PartialErrorBuilder};++/// Glob represents a single glob in a gitignore file.+///+/// This is used to report information about the highest precedent glob that+/// matched in one or more gitignore files.+#[derive(Clone, Debug)]+pub struct Glob {+ /// The file path that this glob was extracted from.+ from: Option<PathBuf>,+ /// The original glob string.+ original: String,+ /// The actual glob string used to convert to a regex.+ actual: String,+ /// Whether this is a whitelisted glob or not.+ is_whitelist: bool,+ /// Whether this glob should only match directories or not.+ is_only_dir: bool,+}++impl Glob {+ /// Returns the file path that defined this glob.+ pub fn from(&self) -> Option<&Path> {+ self.from.as_ref().map(|p| &**p)+ }++ /// The original glob as it was defined in a gitignore file.+ pub fn original(&self) -> &str {+ &self.original+ }++ /// The actual glob that was compiled to respect gitignore+ /// semantics.+ pub fn actual(&self) -> &str {+ &self.actual+ }++ /// Whether this was a whitelisted glob or not.+ pub fn is_whitelist(&self) -> bool {+ self.is_whitelist+ }++ /// Whether this glob must match a directory or not.+ pub fn is_only_dir(&self) -> bool {+ self.is_only_dir+ }++ /// Returns true if and only if this glob has a `**/` prefix.+ fn has_doublestar_prefix(&self) -> bool {+ self.actual.starts_with("**/") || self.actual == "**"+ }+}++/// Gitignore is a matcher for the globs in one or more gitignore files+/// in the same directory.+#[derive(Clone, Debug)]+pub struct Gitignore {+ set: GlobSet,+ root: PathBuf,+ globs: Vec<Glob>,+ num_ignores: u64,+ num_whitelists: u64,+ matches: Option<Arc<ThreadLocal<RefCell<Vec<usize>>>>>,+}++impl Gitignore {+ /// Creates a new gitignore matcher from the gitignore file path given.+ ///+ /// If it's desirable to include multiple gitignore files in a single+ /// matcher, or read gitignore globs from a different source, then+ /// use `GitignoreBuilder`.+ ///+ /// This always returns a valid matcher, even if it's empty. In particular,+ /// a Gitignore file can be partially valid, e.g., when one glob is invalid+ /// but the rest aren't.+ ///+ /// Note that I/O errors are ignored. For more granular control over+ /// errors, use `GitignoreBuilder`.+ pub fn new<P: AsRef<Path>>(+ gitignore_path: P,+ ) -> (Gitignore, Option<Error>) {+ let path = gitignore_path.as_ref();+ let parent = path.parent().unwrap_or(Path::new("/"));+ let mut builder = GitignoreBuilder::new(parent);+ let mut errs = PartialErrorBuilder::default();+ errs.maybe_push_ignore_io(builder.add(path));+ match builder.build() {+ Ok(gi) => (gi, errs.into_error_option()),+ Err(err) => {+ errs.push(err);+ (Gitignore::empty(), errs.into_error_option())+ }+ }+ }++ /// Creates a new gitignore matcher from the global ignore file, if one+ /// exists.+ ///+ /// The global config file path is specified by git's `core.excludesFile`+ /// config option.+ ///+ /// Git's config file location is `$HOME/.gitconfig`. If `$HOME/.gitconfig`+ /// does not exist or does not specify `core.excludesFile`, then+ /// `$XDG_CONFIG_HOME/git/ignore` is read. If `$XDG_CONFIG_HOME` is not+ /// set or is empty, then `$HOME/.config/git/ignore` is used instead.+ pub fn global() -> (Gitignore, Option<Error>) {+ GitignoreBuilder::new("").build_global()+ }++ /// Creates a new empty gitignore matcher that never matches anything.+ ///+ /// Its path is empty.+ pub fn empty() -> Gitignore {+ Gitignore {+ set: GlobSet::empty(),+ root: PathBuf::from(""),+ globs: vec![],+ num_ignores: 0,+ num_whitelists: 0,+ matches: None,+ }+ }++ /// Returns the directory containing this gitignore matcher.+ ///+ /// All matches are done relative to this path.+ pub fn path(&self) -> &Path {+ &*self.root+ }++ /// Returns true if and only if this gitignore has zero globs, and+ /// therefore never matches any file path.+ pub fn is_empty(&self) -> bool {+ self.set.is_empty()+ }++ /// Returns the total number of globs, which should be equivalent to+ /// `num_ignores + num_whitelists`.+ pub fn len(&self) -> usize {+ self.set.len()+ }++ /// Returns the total number of ignore globs.+ pub fn num_ignores(&self) -> u64 {+ self.num_ignores+ }++ /// Returns the total number of whitelisted globs.+ pub fn num_whitelists(&self) -> u64 {+ self.num_whitelists+ }++ /// Returns whether the given path (file or directory) matched a pattern in+ /// this gitignore matcher.+ ///+ /// `is_dir` should be true if the path refers to a directory and false+ /// otherwise.+ ///+ /// The given path is matched relative to the path given when building+ /// the matcher. Specifically, before matching `path`, its prefix (as+ /// determined by a common suffix of the directory containing this+ /// gitignore) is stripped. If there is no common suffix/prefix overlap,+ /// then `path` is assumed to be relative to this matcher.+ pub fn matched<P: AsRef<Path>>(+ &self,+ path: P,+ is_dir: bool,+ ) -> Match<&Glob> {+ if self.is_empty() {+ return Match::None;+ }+ self.matched_stripped(self.strip(path.as_ref()), is_dir)+ }++ /// Returns whether the given path (file or directory, and expected to be+ /// under the root) or any of its parent directories (up to the root)+ /// matched a pattern in this gitignore matcher.+ ///+ /// NOTE: This method is more expensive than walking the directory hierarchy+ /// top-to-bottom and matching the entries. But, is easier to use in cases+ /// when a list of paths are available without a hierarchy.+ ///+ /// `is_dir` should be true if the path refers to a directory and false+ /// otherwise.+ ///+ /// The given path is matched relative to the path given when building+ /// the matcher. Specifically, before matching `path`, its prefix (as+ /// determined by a common suffix of the directory containing this+ /// gitignore) is stripped. If there is no common suffix/prefix overlap,+ /// then `path` is assumed to be relative to this matcher.+ ///+ /// # Panics+ ///+ /// This method panics if the given file path is not under the root path+ /// of this matcher.+ pub fn matched_path_or_any_parents<P: AsRef<Path>>(+ &self,+ path: P,+ is_dir: bool,+ ) -> Match<&Glob> {+ if self.is_empty() {+ return Match::None;+ }+ let mut path = self.strip(path.as_ref());+ assert!(!path.has_root(), "path is expected to be under the root");++ match self.matched_stripped(path, is_dir) {+ Match::None => (), // walk up+ a_match => return a_match,+ }+ while let Some(parent) = path.parent() {+ match self.matched_stripped(parent, /* is_dir */ true) {+ Match::None => path = parent, // walk up+ a_match => return a_match,+ }+ }+ Match::None+ }++ /// Like matched, but takes a path that has already been stripped.+ fn matched_stripped<P: AsRef<Path>>(+ &self,+ path: P,+ is_dir: bool,+ ) -> Match<&Glob> {+ if self.is_empty() {+ return Match::None;+ }+ let path = path.as_ref();+ let _matches = self.matches.as_ref().unwrap().get_or_default();+ let mut matches = _matches.borrow_mut();+ let candidate = Candidate::new(path);+ self.set.matches_candidate_into(&candidate, &mut *matches);+ for &i in matches.iter().rev() {+ let glob = &self.globs[i];+ if !glob.is_only_dir() || is_dir {+ return if glob.is_whitelist() {+ Match::Whitelist(glob)+ } else {+ Match::Ignore(glob)+ };+ }+ }+ Match::None+ }++ /// Strips the given path such that it's suitable for matching with this+ /// gitignore matcher.+ fn strip<'a, P: 'a + AsRef<Path> + ?Sized>(+ &'a self,+ path: &'a P,+ ) -> &'a Path {+ let mut path = path.as_ref();+ // A leading ./ is completely superfluous. We also strip it from+ // our gitignore root path, so we need to strip it from our candidate+ // path too.+ if let Some(p) = strip_prefix("./", path) {+ path = p;+ }+ // Strip any common prefix between the candidate path and the root+ // of the gitignore, to make sure we get relative matching right.+ // BUT, a file name might not have any directory components to it,+ // in which case, we don't want to accidentally strip any part of the+ // file name.+ //+ // As an additional special case, if the root is just `.`, then we+ // shouldn't try to strip anything, e.g., when path begins with a `.`.+ if self.root != Path::new(".") && !is_file_name(path) {+ if let Some(p) = strip_prefix(&self.root, path) {+ path = p;+ // If we're left with a leading slash, get rid of it.+ if let Some(p) = strip_prefix("/", path) {+ path = p;+ }+ }+ }+ path+ }+}++/// Builds a matcher for a single set of globs from a .gitignore file.+#[derive(Clone, Debug)]+pub struct GitignoreBuilder {+ builder: GlobSetBuilder,+ root: PathBuf,+ globs: Vec<Glob>,+ case_insensitive: bool,+}++impl GitignoreBuilder {+ /// Create a new builder for a gitignore file.+ ///+ /// The path given should be the path at which the globs for this gitignore+ /// file should be matched. Note that paths are always matched relative+ /// to the root path given here. Generally, the root path should correspond+ /// to the *directory* containing a `.gitignore` file.+ pub fn new<P: AsRef<Path>>(root: P) -> GitignoreBuilder {+ let root = root.as_ref();+ GitignoreBuilder {+ builder: GlobSetBuilder::new(),+ root: strip_prefix("./", root).unwrap_or(root).to_path_buf(),+ globs: vec![],+ case_insensitive: false,+ }+ }++ /// Builds a new matcher from the globs added so far.+ ///+ /// Once a matcher is built, no new globs can be added to it.+ pub fn build(&self) -> Result<Gitignore, Error> {+ let nignore = self.globs.iter().filter(|g| !g.is_whitelist()).count();+ let nwhite = self.globs.iter().filter(|g| g.is_whitelist()).count();+ let set = self+ .builder+ .build()+ .map_err(|err| Error::Glob { glob: None, err: err.to_string() })?;+ Ok(Gitignore {+ set: set,+ root: self.root.clone(),+ globs: self.globs.clone(),+ num_ignores: nignore as u64,+ num_whitelists: nwhite as u64,+ matches: Some(Arc::new(ThreadLocal::default())),+ })+ }++ /// Build a global gitignore matcher using the configuration in this+ /// builder.+ ///+ /// This consumes ownership of the builder unlike `build` because it+ /// must mutate the builder to add the global gitignore globs.+ ///+ /// Note that this ignores the path given to this builder's constructor+ /// and instead derives the path automatically from git's global+ /// configuration.+ pub fn build_global(mut self) -> (Gitignore, Option<Error>) {+ match gitconfig_excludes_path() {+ None => (Gitignore::empty(), None),+ Some(path) => {+ if !path.is_file() {+ (Gitignore::empty(), None)+ } else {+ let mut errs = PartialErrorBuilder::default();+ errs.maybe_push_ignore_io(self.add(path));+ match self.build() {+ Ok(gi) => (gi, errs.into_error_option()),+ Err(err) => {+ errs.push(err);+ (Gitignore::empty(), errs.into_error_option())+ }+ }+ }+ }+ }+ }++ /// Add each glob from the file path given.+ ///+ /// The file given should be formatted as a `gitignore` file.+ ///+ /// Note that partial errors can be returned. For example, if there was+ /// a problem adding one glob, an error for that will be returned, but+ /// all other valid globs will still be added.+ pub fn add<P: AsRef<Path>>(&mut self, path: P) -> Option<Error> {+ let path = path.as_ref();+ let file = match File::open(path) {+ Err(err) => return Some(Error::Io(err).with_path(path)),+ Ok(file) => file,+ };+ let rdr = io::BufReader::new(file);+ let mut errs = PartialErrorBuilder::default();+ for (i, line) in rdr.lines().enumerate() {+ let lineno = (i + 1) as u64;+ let line = match line {+ Ok(line) => line,+ Err(err) => {+ errs.push(Error::Io(err).tagged(path, lineno));+ break;+ }+ };+ if let Err(err) = self.add_line(Some(path.to_path_buf()), &line) {+ errs.push(err.tagged(path, lineno));+ }+ }+ errs.into_error_option()+ }++ /// Add each glob line from the string given.+ ///+ /// If this string came from a particular `gitignore` file, then its path+ /// should be provided here.+ ///+ /// The string given should be formatted as a `gitignore` file.+ #[cfg(test)]+ fn add_str(+ &mut self,+ from: Option<PathBuf>,+ gitignore: &str,+ ) -> Result<&mut GitignoreBuilder, Error> {+ for line in gitignore.lines() {+ self.add_line(from.clone(), line)?;+ }+ Ok(self)+ }++ /// Add a line from a gitignore file to this builder.+ ///+ /// If this line came from a particular `gitignore` file, then its path+ /// should be provided here.+ ///+ /// If the line could not be parsed as a glob, then an error is returned.+ pub fn add_line(+ &mut self,+ from: Option<PathBuf>,+ mut line: &str,+ ) -> Result<&mut GitignoreBuilder, Error> {+ #![allow(deprecated)]++ if line.starts_with("#") {+ return Ok(self);+ }+ if !line.ends_with("\\ ") {+ line = line.trim_right();+ }+ if line.is_empty() {+ return Ok(self);+ }+ let mut glob = Glob {+ from: from,+ original: line.to_string(),+ actual: String::new(),+ is_whitelist: false,+ is_only_dir: false,+ };+ let mut is_absolute = false;+ if line.starts_with("\\!") || line.starts_with("\\#") {+ line = &line[1..];+ is_absolute = line.chars().nth(0) == Some('/');+ } else {+ if line.starts_with("!") {+ glob.is_whitelist = true;+ line = &line[1..];+ }+ if line.starts_with("/") {+ // `man gitignore` says that if a glob starts with a slash,+ // then the glob can only match the beginning of a path+ // (relative to the location of gitignore). We achieve this by+ // simply banning wildcards from matching /.+ line = &line[1..];+ is_absolute = true;+ }+ }+ // If it ends with a slash, then this should only match directories,+ // but the slash should otherwise not be used while globbing.+ if let Some((i, c)) = line.char_indices().rev().nth(0) {+ if c == '/' {+ glob.is_only_dir = true;+ line = &line[..i];+ }+ }+ glob.actual = line.to_string();+ // If there is a literal slash, then this is a glob that must match the+ // entire path name. Otherwise, we should let it match anywhere, so use+ // a **/ prefix.+ if !is_absolute && !line.chars().any(|c| c == '/') {+ // ... but only if we don't already have a **/ prefix.+ if !glob.has_doublestar_prefix() {+ glob.actual = format!("**/{}", glob.actual);+ }+ }+ // If the glob ends with `/**`, then we should only match everything+ // inside a directory, but not the directory itself. Standard globs+ // will match the directory. So we add `/*` to force the issue.+ if glob.actual.ends_with("/**") {+ glob.actual = format!("{}/*", glob.actual);+ }+ let parsed = GlobBuilder::new(&glob.actual)+ .literal_separator(true)+ .case_insensitive(self.case_insensitive)+ .backslash_escape(true)+ .build()+ .map_err(|err| Error::Glob {+ glob: Some(glob.original.clone()),+ err: err.kind().to_string(),+ })?;+ self.builder.add(parsed);+ self.globs.push(glob);+ Ok(self)+ }++ /// Toggle whether the globs should be matched case insensitively or not.+ ///+ /// When this option is changed, only globs added after the change will be+ /// affected.+ ///+ /// This is disabled by default.+ pub fn case_insensitive(+ &mut self,+ yes: bool,+ ) -> Result<&mut GitignoreBuilder, Error> {+ // TODO: This should not return a `Result`. Fix this in the next semver+ // release.+ self.case_insensitive = yes;+ Ok(self)+ }+}++/// Return the file path of the current environment's global gitignore file.+///+/// Note that the file path returned may not exist.+fn gitconfig_excludes_path() -> Option<PathBuf> {+ // git supports $HOME/.gitconfig and $XDG_CONFIG_HOME/git/config. Notably,+ // both can be active at the same time, where $HOME/.gitconfig takes+ // precedent. So if $HOME/.gitconfig defines a `core.excludesFile`, then+ // we're done.+ match gitconfig_home_contents().and_then(|x| parse_excludes_file(&x)) {+ Some(path) => return Some(path),+ None => {}+ }+ match gitconfig_xdg_contents().and_then(|x| parse_excludes_file(&x)) {+ Some(path) => return Some(path),+ None => {}+ }+ excludes_file_default()+}++/// Returns the file contents of git's global config file, if one exists, in+/// the user's home directory.+fn gitconfig_home_contents() -> Option<Vec<u8>> {+ let home = match home_dir() {+ None => return None,+ Some(home) => home,+ };+ let mut file = match File::open(home.join(".gitconfig")) {+ Err(_) => return None,+ Ok(file) => io::BufReader::new(file),+ };+ let mut contents = vec![];+ file.read_to_end(&mut contents).ok().map(|_| contents)+}++/// Returns the file contents of git's global config file, if one exists, in+/// the user's XDG_CONFIG_HOME directory.+fn gitconfig_xdg_contents() -> Option<Vec<u8>> {+ let path = env::var_os("XDG_CONFIG_HOME")+ .and_then(|x| if x.is_empty() { None } else { Some(PathBuf::from(x)) })+ .or_else(|| home_dir().map(|p| p.join(".config")))+ .map(|x| x.join("git/config"));+ let mut file = match path.and_then(|p| File::open(p).ok()) {+ None => return None,+ Some(file) => io::BufReader::new(file),+ };+ let mut contents = vec![];+ file.read_to_end(&mut contents).ok().map(|_| contents)+}++/// Returns the default file path for a global .gitignore file.+///+/// Specifically, this respects XDG_CONFIG_HOME.+fn excludes_file_default() -> Option<PathBuf> {+ env::var_os("XDG_CONFIG_HOME")+ .and_then(|x| if x.is_empty() { None } else { Some(PathBuf::from(x)) })+ .or_else(|| home_dir().map(|p| p.join(".config")))+ .map(|x| x.join("git/ignore"))+}++/// Extract git's `core.excludesfile` config setting from the raw file contents+/// given.+fn parse_excludes_file(data: &[u8]) -> Option<PathBuf> {+ // N.B. This is the lazy approach, and isn't technically correct, but+ // probably works in more circumstances. I guess we would ideally have+ // a full INI parser. Yuck.+ lazy_static! {+ static ref RE: Regex =+ Regex::new(r"(?im)^\s*excludesfile\s*=\s*(.+)\s*$").unwrap();+ };+ let caps = match RE.captures(data) {+ None => return None,+ Some(caps) => caps,+ };+ str::from_utf8(&caps[1]).ok().map(|s| PathBuf::from(expand_tilde(s)))+}++/// Expands ~ in file paths to the value of $HOME.+fn expand_tilde(path: &str) -> String {+ let home = match home_dir() {+ None => return path.to_string(),+ Some(home) => home.to_string_lossy().into_owned(),+ };+ path.replace("~", &home)+}++/// Returns the location of the user's home directory.+fn home_dir() -> Option<PathBuf> {+ // We're fine with using env::home_dir for now. Its bugs are, IMO, pretty+ // minor corner cases. We should still probably eventually migrate to+ // the `dirs` crate to get a proper implementation.+ #![allow(deprecated)]+ env::home_dir()+}++#[cfg(test)]+mod tests {+ use super::{Gitignore, GitignoreBuilder};+ use std::path::Path;++ fn gi_from_str<P: AsRef<Path>>(root: P, s: &str) -> Gitignore {+ let mut builder = GitignoreBuilder::new(root);+ builder.add_str(None, s).unwrap();+ builder.build().unwrap()+ }++ macro_rules! ignored {+ ($name:ident, $root:expr, $gi:expr, $path:expr) => {+ ignored!($name, $root, $gi, $path, false);+ };+ ($name:ident, $root:expr, $gi:expr, $path:expr, $is_dir:expr) => {+ #[test]+ fn $name() {+ let gi = gi_from_str($root, $gi);+ assert!(gi.matched($path, $is_dir).is_ignore());+ }+ };+ }++ macro_rules! not_ignored {+ ($name:ident, $root:expr, $gi:expr, $path:expr) => {+ not_ignored!($name, $root, $gi, $path, false);+ };+ ($name:ident, $root:expr, $gi:expr, $path:expr, $is_dir:expr) => {+ #[test]+ fn $name() {+ let gi = gi_from_str($root, $gi);+ assert!(!gi.matched($path, $is_dir).is_ignore());+ }+ };+ }++ const ROOT: &'static str = "/home/foobar/rust/rg";++ ignored!(ig1, ROOT, "months", "months");+ ignored!(ig2, ROOT, "*.lock", "Cargo.lock");+ ignored!(ig3, ROOT, "*.rs", "src/main.rs");+ ignored!(ig4, ROOT, "src/*.rs", "src/main.rs");+ ignored!(ig5, ROOT, "/*.c", "cat-file.c");+ ignored!(ig6, ROOT, "/src/*.rs", "src/main.rs");+ ignored!(ig7, ROOT, "!src/main.rs\n*.rs", "src/main.rs");+ ignored!(ig8, ROOT, "foo/", "foo", true);+ ignored!(ig9, ROOT, "**/foo", "foo");+ ignored!(ig10, ROOT, "**/foo", "src/foo");+ ignored!(ig11, ROOT, "**/foo/**", "src/foo/bar");+ ignored!(ig12, ROOT, "**/foo/**", "wat/src/foo/bar/baz");+ ignored!(ig13, ROOT, "**/foo/bar", "foo/bar");+ ignored!(ig14, ROOT, "**/foo/bar", "src/foo/bar");+ ignored!(ig15, ROOT, "abc/**", "abc/x");+ ignored!(ig16, ROOT, "abc/**", "abc/x/y");+ ignored!(ig17, ROOT, "abc/**", "abc/x/y/z");+ ignored!(ig18, ROOT, "a/**/b", "a/b");+ ignored!(ig19, ROOT, "a/**/b", "a/x/b");+ ignored!(ig20, ROOT, "a/**/b", "a/x/y/b");+ ignored!(ig21, ROOT, r"\!xy", "!xy");+ ignored!(ig22, ROOT, r"\#foo", "#foo");+ ignored!(ig23, ROOT, "foo", "./foo");+ ignored!(ig24, ROOT, "target", "grep/target");+ ignored!(ig25, ROOT, "Cargo.lock", "./tabwriter-bin/Cargo.lock");+ ignored!(ig26, ROOT, "/foo/bar/baz", "./foo/bar/baz");+ ignored!(ig27, ROOT, "foo/", "xyz/foo", true);+ ignored!(ig28, "./src", "/llvm/", "./src/llvm", true);+ ignored!(ig29, ROOT, "node_modules/ ", "node_modules", true);+ ignored!(ig30, ROOT, "**/", "foo/bar", true);+ ignored!(ig31, ROOT, "path1/*", "path1/foo");+ ignored!(ig32, ROOT, ".a/b", ".a/b");+ ignored!(ig33, "./", ".a/b", ".a/b");+ ignored!(ig34, ".", ".a/b", ".a/b");+ ignored!(ig35, "./.", ".a/b", ".a/b");+ ignored!(ig36, "././", ".a/b", ".a/b");+ ignored!(ig37, "././.", ".a/b", ".a/b");+ ignored!(ig38, ROOT, "\\[", "[");+ ignored!(ig39, ROOT, "\\?", "?");+ ignored!(ig40, ROOT, "\\*", "*");+ ignored!(ig41, ROOT, "\\a", "a");+ ignored!(ig42, ROOT, "s*.rs", "sfoo.rs");+ ignored!(ig43, ROOT, "**", "foo.rs");+ ignored!(ig44, ROOT, "**/**/*", "a/foo.rs");++ not_ignored!(ignot1, ROOT, "amonths", "months");+ not_ignored!(ignot2, ROOT, "monthsa", "months");+ not_ignored!(ignot3, ROOT, "/src/*.rs", "src/grep/src/main.rs");+ not_ignored!(ignot4, ROOT, "/*.c", "mozilla-sha1/sha1.c");+ not_ignored!(ignot5, ROOT, "/src/*.rs", "src/grep/src/main.rs");+ not_ignored!(ignot6, ROOT, "*.rs\n!src/main.rs", "src/main.rs");+ not_ignored!(ignot7, ROOT, "foo/", "foo", false);+ not_ignored!(ignot8, ROOT, "**/foo/**", "wat/src/afoo/bar/baz");+ not_ignored!(ignot9, ROOT, "**/foo/**", "wat/src/fooa/bar/baz");+ not_ignored!(ignot10, ROOT, "**/foo/bar", "foo/src/bar");+ not_ignored!(ignot11, ROOT, "#foo", "#foo");+ not_ignored!(ignot12, ROOT, "\n\n\n", "foo");+ not_ignored!(ignot13, ROOT, "foo/**", "foo", true);+ not_ignored!(+ ignot14,+ "./third_party/protobuf",+ "m4/ltoptions.m4",+ "./third_party/protobuf/csharp/src/packages/repositories.config"+ );+ not_ignored!(ignot15, ROOT, "!/bar", "foo/bar");+ not_ignored!(ignot16, ROOT, "*\n!**/", "foo", true);+ not_ignored!(ignot17, ROOT, "src/*.rs", "src/grep/src/main.rs");+ not_ignored!(ignot18, ROOT, "path1/*", "path2/path1/foo");+ not_ignored!(ignot19, ROOT, "s*.rs", "src/foo.rs");++ fn bytes(s: &str) -> Vec<u8> {+ s.to_string().into_bytes()+ }++ fn path_string<P: AsRef<Path>>(path: P) -> String {+ path.as_ref().to_str().unwrap().to_string()+ }++ #[test]+ fn parse_excludes_file1() {+ let data = bytes("[core]\nexcludesFile = /foo/bar");+ let got = super::parse_excludes_file(&data).unwrap();+ assert_eq!(path_string(got), "/foo/bar");+ }++ #[test]+ fn parse_excludes_file2() {+ let data = bytes("[core]\nexcludesFile = ~/foo/bar");+ let got = super::parse_excludes_file(&data).unwrap();+ assert_eq!(path_string(got), super::expand_tilde("~/foo/bar"));+ }++ #[test]+ fn parse_excludes_file3() {+ let data = bytes("[core]\nexcludeFile = /foo/bar");+ assert!(super::parse_excludes_file(&data).is_none());+ }++ // See: https://github.com/BurntSushi/ripgrep/issues/106+ #[test]+ fn regression_106() {+ gi_from_str("/", " ");+ }++ #[test]+ fn case_insensitive() {+ let gi = GitignoreBuilder::new(ROOT)+ .case_insensitive(true)+ .unwrap()+ .add_str(None, "*.html")+ .unwrap()+ .build()+ .unwrap();+ assert!(gi.matched("foo.html", false).is_ignore());+ assert!(gi.matched("foo.HTML", false).is_ignore());+ assert!(!gi.matched("foo.htm", false).is_ignore());+ assert!(!gi.matched("foo.HTM", false).is_ignore());+ }++ ignored!(cs1, ROOT, "*.html", "foo.html");+ not_ignored!(cs2, ROOT, "*.html", "foo.HTML");+ not_ignored!(cs3, ROOT, "*.html", "foo.htm");+ not_ignored!(cs4, ROOT, "*.html", "foo.HTM");+}
@@ -0,0 +1,499 @@+/*!+The ignore crate provides a fast recursive directory iterator that respects+various filters such as globs, file types and `.gitignore` files. The precise+matching rules and precedence is explained in the documentation for+`WalkBuilder`.++Secondarily, this crate exposes gitignore and file type matchers for use cases+that demand more fine-grained control.++# Example++This example shows the most basic usage of this crate. This code will+recursively traverse the current directory while automatically filtering out+files and directories according to ignore globs found in files like+`.ignore` and `.gitignore`:+++```rust,no_run+use ignore::Walk;++for result in Walk::new("./") {+ // Each item yielded by the iterator is either a directory entry or an+ // error, so either print the path or the error.+ match result {+ Ok(entry) => println!("{}", entry.path().display()),+ Err(err) => println!("ERROR: {}", err),+ }+}+```++# Example: advanced++By default, the recursive directory iterator will ignore hidden files and+directories. This can be disabled by building the iterator with `WalkBuilder`:++```rust,no_run+use ignore::WalkBuilder;++for result in WalkBuilder::new("./").hidden(false).build() {+ println!("{:?}", result);+}+```++See the documentation for `WalkBuilder` for many other options.+*/++#![deny(missing_docs)]++extern crate crossbeam_channel as channel;+extern crate globset;+#[macro_use]+extern crate lazy_static;+#[macro_use]+extern crate log;+extern crate memchr;+extern crate regex;+extern crate same_file;+extern crate thread_local;+extern crate walkdir;+#[cfg(windows)]+extern crate winapi_util;++use std::error;+use std::fmt;+use std::io;+use std::path::{Path, PathBuf};++pub use walk::{+ DirEntry, ParallelVisitor, ParallelVisitorBuilder, Walk, WalkBuilder,+ WalkParallel, WalkState,+};++mod default_types;+mod dir;+pub mod gitignore;+pub mod overrides;+mod pathutil;+pub mod types;+mod walk;++/// Represents an error that can occur when parsing a gitignore file.+#[derive(Debug)]+pub enum Error {+ /// A collection of "soft" errors. These occur when adding an ignore+ /// file partially succeeded.+ Partial(Vec<Error>),+ /// An error associated with a specific line number.+ WithLineNumber {+ /// The line number.+ line: u64,+ /// The underlying error.+ err: Box<Error>,+ },+ /// An error associated with a particular file path.+ WithPath {+ /// The file path.+ path: PathBuf,+ /// The underlying error.+ err: Box<Error>,+ },+ /// An error associated with a particular directory depth when recursively+ /// walking a directory.+ WithDepth {+ /// The directory depth.+ depth: usize,+ /// The underlying error.+ err: Box<Error>,+ },+ /// An error that occurs when a file loop is detected when traversing+ /// symbolic links.+ Loop {+ /// The ancestor file path in the loop.+ ancestor: PathBuf,+ /// The child file path in the loop.+ child: PathBuf,+ },+ /// An error that occurs when doing I/O, such as reading an ignore file.+ Io(io::Error),+ /// An error that occurs when trying to parse a glob.+ Glob {+ /// The original glob that caused this error. This glob, when+ /// available, always corresponds to the glob provided by an end user.+ /// e.g., It is the glob as written in a `.gitignore` file.+ ///+ /// (This glob may be distinct from the glob that is actually+ /// compiled, after accounting for `gitignore` semantics.)+ glob: Option<String>,+ /// The underlying glob error as a string.+ err: String,+ },+ /// A type selection for a file type that is not defined.+ UnrecognizedFileType(String),+ /// A user specified file type definition could not be parsed.+ InvalidDefinition,+}++impl Clone for Error {+ fn clone(&self) -> Error {+ match *self {+ Error::Partial(ref errs) => Error::Partial(errs.clone()),+ Error::WithLineNumber { line, ref err } => {+ Error::WithLineNumber { line: line, err: err.clone() }+ }+ Error::WithPath { ref path, ref err } => {+ Error::WithPath { path: path.clone(), err: err.clone() }+ }+ Error::WithDepth { depth, ref err } => {+ Error::WithDepth { depth: depth, err: err.clone() }+ }+ Error::Loop { ref ancestor, ref child } => Error::Loop {+ ancestor: ancestor.clone(),+ child: child.clone(),+ },+ Error::Io(ref err) => match err.raw_os_error() {+ Some(e) => Error::Io(io::Error::from_raw_os_error(e)),+ None => Error::Io(io::Error::new(err.kind(), err.to_string())),+ },+ Error::Glob { ref glob, ref err } => {+ Error::Glob { glob: glob.clone(), err: err.clone() }+ }+ Error::UnrecognizedFileType(ref err) => {+ Error::UnrecognizedFileType(err.clone())+ }+ Error::InvalidDefinition => Error::InvalidDefinition,+ }+ }+}++impl Error {+ /// Returns true if this is a partial error.+ ///+ /// A partial error occurs when only some operations failed while others+ /// may have succeeded. For example, an ignore file may contain an invalid+ /// glob among otherwise valid globs.+ pub fn is_partial(&self) -> bool {+ match *self {+ Error::Partial(_) => true,+ Error::WithLineNumber { ref err, .. } => err.is_partial(),+ Error::WithPath { ref err, .. } => err.is_partial(),+ Error::WithDepth { ref err, .. } => err.is_partial(),+ _ => false,+ }+ }++ /// Returns true if this error is exclusively an I/O error.+ pub fn is_io(&self) -> bool {+ match *self {+ Error::Partial(ref errs) => errs.len() == 1 && errs[0].is_io(),+ Error::WithLineNumber { ref err, .. } => err.is_io(),+ Error::WithPath { ref err, .. } => err.is_io(),+ Error::WithDepth { ref err, .. } => err.is_io(),+ Error::Loop { .. } => false,+ Error::Io(_) => true,+ Error::Glob { .. } => false,+ Error::UnrecognizedFileType(_) => false,+ Error::InvalidDefinition => false,+ }+ }++ /// Returns a depth associated with recursively walking a directory (if+ /// this error was generated from a recursive directory iterator).+ pub fn depth(&self) -> Option<usize> {+ match *self {+ Error::WithPath { ref err, .. } => err.depth(),+ Error::WithDepth { depth, .. } => Some(depth),+ _ => None,+ }+ }++ /// Turn an error into a tagged error with the given file path.+ fn with_path<P: AsRef<Path>>(self, path: P) -> Error {+ Error::WithPath {+ path: path.as_ref().to_path_buf(),+ err: Box::new(self),+ }+ }++ /// Turn an error into a tagged error with the given depth.+ fn with_depth(self, depth: usize) -> Error {+ Error::WithDepth { depth: depth, err: Box::new(self) }+ }++ /// Turn an error into a tagged error with the given file path and line+ /// number. If path is empty, then it is omitted from the error.+ fn tagged<P: AsRef<Path>>(self, path: P, lineno: u64) -> Error {+ let errline =+ Error::WithLineNumber { line: lineno, err: Box::new(self) };+ if path.as_ref().as_os_str().is_empty() {+ return errline;+ }+ errline.with_path(path)+ }++ /// Build an error from a walkdir error.+ fn from_walkdir(err: walkdir::Error) -> Error {+ let depth = err.depth();+ if let (Some(anc), Some(child)) = (err.loop_ancestor(), err.path()) {+ return Error::WithDepth {+ depth: depth,+ err: Box::new(Error::Loop {+ ancestor: anc.to_path_buf(),+ child: child.to_path_buf(),+ }),+ };+ }+ let path = err.path().map(|p| p.to_path_buf());+ let mut ig_err = Error::Io(io::Error::from(err));+ if let Some(path) = path {+ ig_err = Error::WithPath { path: path, err: Box::new(ig_err) };+ }+ ig_err+ }+}++impl error::Error for Error {+ #[allow(deprecated)]+ fn description(&self) -> &str {+ match *self {+ Error::Partial(_) => "partial error",+ Error::WithLineNumber { ref err, .. } => err.description(),+ Error::WithPath { ref err, .. } => err.description(),+ Error::WithDepth { ref err, .. } => err.description(),+ Error::Loop { .. } => "file system loop found",+ Error::Io(ref err) => err.description(),+ Error::Glob { ref err, .. } => err,+ Error::UnrecognizedFileType(_) => "unrecognized file type",+ Error::InvalidDefinition => "invalid definition",+ }+ }+}++impl fmt::Display for Error {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match *self {+ Error::Partial(ref errs) => {+ let msgs: Vec<String> =+ errs.iter().map(|err| err.to_string()).collect();+ write!(f, "{}", msgs.join("\n"))+ }+ Error::WithLineNumber { line, ref err } => {+ write!(f, "line {}: {}", line, err)+ }+ Error::WithPath { ref path, ref err } => {+ write!(f, "{}: {}", path.display(), err)+ }+ Error::WithDepth { ref err, .. } => err.fmt(f),+ Error::Loop { ref ancestor, ref child } => write!(+ f,+ "File system loop found: \+ {} points to an ancestor {}",+ child.display(),+ ancestor.display()+ ),+ Error::Io(ref err) => err.fmt(f),+ Error::Glob { glob: None, ref err } => write!(f, "{}", err),+ Error::Glob { glob: Some(ref glob), ref err } => {+ write!(f, "error parsing glob '{}': {}", glob, err)+ }+ Error::UnrecognizedFileType(ref ty) => {+ write!(f, "unrecognized file type: {}", ty)+ }+ Error::InvalidDefinition => write!(+ f,+ "invalid definition (format is type:glob, e.g., \+ html:*.html)"+ ),+ }+ }+}++impl From<io::Error> for Error {+ fn from(err: io::Error) -> Error {+ Error::Io(err)+ }+}++#[derive(Debug, Default)]+struct PartialErrorBuilder(Vec<Error>);++impl PartialErrorBuilder {+ fn push(&mut self, err: Error) {+ self.0.push(err);+ }++ fn push_ignore_io(&mut self, err: Error) {+ if !err.is_io() {+ self.push(err);+ }+ }++ fn maybe_push(&mut self, err: Option<Error>) {+ if let Some(err) = err {+ self.push(err);+ }+ }++ fn maybe_push_ignore_io(&mut self, err: Option<Error>) {+ if let Some(err) = err {+ self.push_ignore_io(err);+ }+ }++ fn into_error_option(mut self) -> Option<Error> {+ if self.0.is_empty() {+ None+ } else if self.0.len() == 1 {+ Some(self.0.pop().unwrap())+ } else {+ Some(Error::Partial(self.0))+ }+ }+}++/// The result of a glob match.+///+/// The type parameter `T` typically refers to a type that provides more+/// information about a particular match. For example, it might identify+/// the specific gitignore file and the specific glob pattern that caused+/// the match.+#[derive(Clone, Debug)]+pub enum Match<T> {+ /// The path didn't match any glob.+ None,+ /// The highest precedent glob matched indicates the path should be+ /// ignored.+ Ignore(T),+ /// The highest precedent glob matched indicates the path should be+ /// whitelisted.+ Whitelist(T),+}++impl<T> Match<T> {+ /// Returns true if the match result didn't match any globs.+ pub fn is_none(&self) -> bool {+ match *self {+ Match::None => true,+ Match::Ignore(_) | Match::Whitelist(_) => false,+ }+ }++ /// Returns true if the match result implies the path should be ignored.+ pub fn is_ignore(&self) -> bool {+ match *self {+ Match::Ignore(_) => true,+ Match::None | Match::Whitelist(_) => false,+ }+ }++ /// Returns true if the match result implies the path should be+ /// whitelisted.+ pub fn is_whitelist(&self) -> bool {+ match *self {+ Match::Whitelist(_) => true,+ Match::None | Match::Ignore(_) => false,+ }+ }++ /// Inverts the match so that `Ignore` becomes `Whitelist` and+ /// `Whitelist` becomes `Ignore`. A non-match remains the same.+ pub fn invert(self) -> Match<T> {+ match self {+ Match::None => Match::None,+ Match::Ignore(t) => Match::Whitelist(t),+ Match::Whitelist(t) => Match::Ignore(t),+ }+ }++ /// Return the value inside this match if it exists.+ pub fn inner(&self) -> Option<&T> {+ match *self {+ Match::None => None,+ Match::Ignore(ref t) => Some(t),+ Match::Whitelist(ref t) => Some(t),+ }+ }++ /// Apply the given function to the value inside this match.+ ///+ /// If the match has no value, then return the match unchanged.+ pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Match<U> {+ match self {+ Match::None => Match::None,+ Match::Ignore(t) => Match::Ignore(f(t)),+ Match::Whitelist(t) => Match::Whitelist(f(t)),+ }+ }++ /// Return the match if it is not none. Otherwise, return other.+ pub fn or(self, other: Self) -> Self {+ if self.is_none() {+ other+ } else {+ self+ }+ }+}++#[cfg(test)]+mod tests {+ use std::env;+ use std::error;+ use std::fs;+ use std::path::{Path, PathBuf};+ use std::result;++ /// A convenient result type alias.+ pub type Result<T> =+ result::Result<T, Box<dyn error::Error + Send + Sync>>;++ macro_rules! err {+ ($($tt:tt)*) => {+ Box::<dyn error::Error + Send + Sync>::from(format!($($tt)*))+ }+ }++ /// A simple wrapper for creating a temporary directory that is+ /// automatically deleted when it's dropped.+ ///+ /// We use this in lieu of tempfile because tempfile brings in too many+ /// dependencies.+ #[derive(Debug)]+ pub struct TempDir(PathBuf);++ impl Drop for TempDir {+ fn drop(&mut self) {+ fs::remove_dir_all(&self.0).unwrap();+ }+ }++ impl TempDir {+ /// Create a new empty temporary directory under the system's configured+ /// temporary directory.+ pub fn new() -> Result<TempDir> {+ use std::sync::atomic::{AtomicUsize, Ordering};++ static TRIES: usize = 100;+ static COUNTER: AtomicUsize = AtomicUsize::new(0);++ let tmpdir = env::temp_dir();+ for _ in 0..TRIES {+ let count = COUNTER.fetch_add(1, Ordering::SeqCst);+ let path = tmpdir.join("rust-ignore").join(count.to_string());+ if path.is_dir() {+ continue;+ }+ fs::create_dir_all(&path).map_err(|e| {+ err!("failed to create {}: {}", path.display(), e)+ })?;+ return Ok(TempDir(path));+ }+ Err(err!("failed to create temp dir after {} tries", TRIES))+ }++ /// Return the underlying path to this temporary directory.+ pub fn path(&self) -> &Path {+ &self.0+ }+ }+}
crates/ignore/src/overrides.rs
@@ -0,0 +1,262 @@+/*!+The overrides module provides a way to specify a set of override globs.+This provides functionality similar to `--include` or `--exclude` in command+line tools.+*/++use std::path::Path;++use gitignore::{self, Gitignore, GitignoreBuilder};+use {Error, Match};++/// Glob represents a single glob in an override matcher.+///+/// This is used to report information about the highest precedent glob+/// that matched.+///+/// Note that not all matches necessarily correspond to a specific glob. For+/// example, if there are one or more whitelist globs and a file path doesn't+/// match any glob in the set, then the file path is considered to be ignored.+///+/// The lifetime `'a` refers to the lifetime of the matcher that produced+/// this glob.+#[derive(Clone, Debug)]+pub struct Glob<'a>(GlobInner<'a>);++#[derive(Clone, Debug)]+enum GlobInner<'a> {+ /// No glob matched, but the file path should still be ignored.+ UnmatchedIgnore,+ /// A glob matched.+ Matched(&'a gitignore::Glob),+}++impl<'a> Glob<'a> {+ fn unmatched() -> Glob<'a> {+ Glob(GlobInner::UnmatchedIgnore)+ }+}++/// Manages a set of overrides provided explicitly by the end user.+#[derive(Clone, Debug)]+pub struct Override(Gitignore);++impl Override {+ /// Returns an empty matcher that never matches any file path.+ pub fn empty() -> Override {+ Override(Gitignore::empty())+ }++ /// Returns the directory of this override set.+ ///+ /// All matches are done relative to this path.+ pub fn path(&self) -> &Path {+ self.0.path()+ }++ /// Returns true if and only if this matcher is empty.+ ///+ /// When a matcher is empty, it will never match any file path.+ pub fn is_empty(&self) -> bool {+ self.0.is_empty()+ }++ /// Returns the total number of ignore globs.+ pub fn num_ignores(&self) -> u64 {+ self.0.num_whitelists()+ }++ /// Returns the total number of whitelisted globs.+ pub fn num_whitelists(&self) -> u64 {+ self.0.num_ignores()+ }++ /// Returns whether the given file path matched a pattern in this override+ /// matcher.+ ///+ /// `is_dir` should be true if the path refers to a directory and false+ /// otherwise.+ ///+ /// If there are no overrides, then this always returns `Match::None`.+ ///+ /// If there is at least one whitelist override and `is_dir` is false, then+ /// this never returns `Match::None`, since non-matches are interpreted as+ /// ignored.+ ///+ /// The given path is matched to the globs relative to the path given+ /// when building the override matcher. Specifically, before matching+ /// `path`, its prefix (as determined by a common suffix of the directory+ /// given) is stripped. If there is no common suffix/prefix overlap, then+ /// `path` is assumed to reside in the same directory as the root path for+ /// this set of overrides.+ pub fn matched<'a, P: AsRef<Path>>(+ &'a self,+ path: P,+ is_dir: bool,+ ) -> Match<Glob<'a>> {+ if self.is_empty() {+ return Match::None;+ }+ let mat = self.0.matched(path, is_dir).invert();+ if mat.is_none() && self.num_whitelists() > 0 && !is_dir {+ return Match::Ignore(Glob::unmatched());+ }+ mat.map(move |giglob| Glob(GlobInner::Matched(giglob)))+ }+}++/// Builds a matcher for a set of glob overrides.+pub struct OverrideBuilder {+ builder: GitignoreBuilder,+}++impl OverrideBuilder {+ /// Create a new override builder.+ ///+ /// Matching is done relative to the directory path provided.+ pub fn new<P: AsRef<Path>>(path: P) -> OverrideBuilder {+ OverrideBuilder { builder: GitignoreBuilder::new(path) }+ }++ /// Builds a new override matcher from the globs added so far.+ ///+ /// Once a matcher is built, no new globs can be added to it.+ pub fn build(&self) -> Result<Override, Error> {+ Ok(Override(self.builder.build()?))+ }++ /// Add a glob to the set of overrides.+ ///+ /// Globs provided here have precisely the same semantics as a single+ /// line in a `gitignore` file, where the meaning of `!` is inverted:+ /// namely, `!` at the beginning of a glob will ignore a file. Without `!`,+ /// all matches of the glob provided are treated as whitelist matches.+ pub fn add(&mut self, glob: &str) -> Result<&mut OverrideBuilder, Error> {+ self.builder.add_line(None, glob)?;+ Ok(self)+ }++ /// Toggle whether the globs should be matched case insensitively or not.+ ///+ /// When this option is changed, only globs added after the change will be affected.+ ///+ /// This is disabled by default.+ pub fn case_insensitive(+ &mut self,+ yes: bool,+ ) -> Result<&mut OverrideBuilder, Error> {+ // TODO: This should not return a `Result`. Fix this in the next semver+ // release.+ self.builder.case_insensitive(yes)?;+ Ok(self)+ }+}++#[cfg(test)]+mod tests {+ use super::{Override, OverrideBuilder};++ const ROOT: &'static str = "/home/andrew/foo";++ fn ov(globs: &[&str]) -> Override {+ let mut builder = OverrideBuilder::new(ROOT);+ for glob in globs {+ builder.add(glob).unwrap();+ }+ builder.build().unwrap()+ }++ #[test]+ fn empty() {+ let ov = ov(&[]);+ assert!(ov.matched("a.foo", false).is_none());+ assert!(ov.matched("a", false).is_none());+ assert!(ov.matched("", false).is_none());+ }++ #[test]+ fn simple() {+ let ov = ov(&["*.foo", "!*.bar"]);+ assert!(ov.matched("a.foo", false).is_whitelist());+ assert!(ov.matched("a.foo", true).is_whitelist());+ assert!(ov.matched("a.rs", false).is_ignore());+ assert!(ov.matched("a.rs", true).is_none());+ assert!(ov.matched("a.bar", false).is_ignore());+ assert!(ov.matched("a.bar", true).is_ignore());+ }++ #[test]+ fn only_ignores() {+ let ov = ov(&["!*.bar"]);+ assert!(ov.matched("a.rs", false).is_none());+ assert!(ov.matched("a.rs", true).is_none());+ assert!(ov.matched("a.bar", false).is_ignore());+ assert!(ov.matched("a.bar", true).is_ignore());+ }++ #[test]+ fn precedence() {+ let ov = ov(&["*.foo", "!*.bar.foo"]);+ assert!(ov.matched("a.foo", false).is_whitelist());+ assert!(ov.matched("a.baz", false).is_ignore());+ assert!(ov.matched("a.bar.foo", false).is_ignore());+ }++ #[test]+ fn gitignore() {+ let ov = ov(&["/foo", "bar/*.rs", "baz/**"]);+ assert!(ov.matched("bar/lib.rs", false).is_whitelist());+ assert!(ov.matched("bar/wat/lib.rs", false).is_ignore());+ assert!(ov.matched("wat/bar/lib.rs", false).is_ignore());+ assert!(ov.matched("foo", false).is_whitelist());+ assert!(ov.matched("wat/foo", false).is_ignore());+ assert!(ov.matched("baz", false).is_ignore());+ assert!(ov.matched("baz/a", false).is_whitelist());+ assert!(ov.matched("baz/a/b", false).is_whitelist());+ }++ #[test]+ fn allow_directories() {+ // This tests that directories are NOT ignored when they are unmatched.+ let ov = ov(&["*.rs"]);+ assert!(ov.matched("foo.rs", false).is_whitelist());+ assert!(ov.matched("foo.c", false).is_ignore());+ assert!(ov.matched("foo", false).is_ignore());+ assert!(ov.matched("foo", true).is_none());+ assert!(ov.matched("src/foo.rs", false).is_whitelist());+ assert!(ov.matched("src/foo.c", false).is_ignore());+ assert!(ov.matched("src/foo", false).is_ignore());+ assert!(ov.matched("src/foo", true).is_none());+ }++ #[test]+ fn absolute_path() {+ let ov = ov(&["!/bar"]);+ assert!(ov.matched("./foo/bar", false).is_none());+ }++ #[test]+ fn case_insensitive() {+ let ov = OverrideBuilder::new(ROOT)+ .case_insensitive(true)+ .unwrap()+ .add("*.html")+ .unwrap()+ .build()+ .unwrap();+ assert!(ov.matched("foo.html", false).is_whitelist());+ assert!(ov.matched("foo.HTML", false).is_whitelist());+ assert!(ov.matched("foo.htm", false).is_ignore());+ assert!(ov.matched("foo.HTM", false).is_ignore());+ }++ #[test]+ fn default_case_sensitive() {+ let ov =+ OverrideBuilder::new(ROOT).add("*.html").unwrap().build().unwrap();+ assert!(ov.matched("foo.html", false).is_whitelist());+ assert!(ov.matched("foo.HTML", false).is_ignore());+ assert!(ov.matched("foo.htm", false).is_ignore());+ assert!(ov.matched("foo.HTM", false).is_ignore());+ }+}
@@ -0,0 +1,142 @@+use std::ffi::OsStr;+use std::path::Path;++use walk::DirEntry;++/// Returns true if and only if this entry is considered to be hidden.+///+/// This only returns true if the base name of the path starts with a `.`.+///+/// On Unix, this implements a more optimized check.+#[cfg(unix)]+pub fn is_hidden(dent: &DirEntry) -> bool {+ use std::os::unix::ffi::OsStrExt;++ if let Some(name) = file_name(dent.path()) {+ name.as_bytes().get(0) == Some(&b'.')+ } else {+ false+ }+}++/// Returns true if and only if this entry is considered to be hidden.+///+/// On Windows, this returns true if one of the following is true:+///+/// * The base name of the path starts with a `.`.+/// * The file attributes have the `HIDDEN` property set.+#[cfg(windows)]+pub fn is_hidden(dent: &DirEntry) -> bool {+ use std::os::windows::fs::MetadataExt;+ use winapi_util::file;++ // This looks like we're doing an extra stat call, but on Windows, the+ // directory traverser reuses the metadata retrieved from each directory+ // entry and stores it on the DirEntry itself. So this is "free."+ if let Ok(md) = dent.metadata() {+ if file::is_hidden(md.file_attributes() as u64) {+ return true;+ }+ }+ if let Some(name) = file_name(dent.path()) {+ name.to_str().map(|s| s.starts_with(".")).unwrap_or(false)+ } else {+ false+ }+}++/// Returns true if and only if this entry is considered to be hidden.+///+/// This only returns true if the base name of the path starts with a `.`.+#[cfg(not(any(unix, windows)))]+pub fn is_hidden(dent: &DirEntry) -> bool {+ if let Some(name) = file_name(dent.path()) {+ name.to_str().map(|s| s.starts_with(".")).unwrap_or(false)+ } else {+ false+ }+}++/// Strip `prefix` from the `path` and return the remainder.+///+/// If `path` doesn't have a prefix `prefix`, then return `None`.+#[cfg(unix)]+pub fn strip_prefix<'a, P: AsRef<Path> + ?Sized>(+ prefix: &'a P,+ path: &'a Path,+) -> Option<&'a Path> {+ use std::os::unix::ffi::OsStrExt;++ let prefix = prefix.as_ref().as_os_str().as_bytes();+ let path = path.as_os_str().as_bytes();+ if prefix.len() > path.len() || prefix != &path[0..prefix.len()] {+ None+ } else {+ Some(&Path::new(OsStr::from_bytes(&path[prefix.len()..])))+ }+}++/// Strip `prefix` from the `path` and return the remainder.+///+/// If `path` doesn't have a prefix `prefix`, then return `None`.+#[cfg(not(unix))]+pub fn strip_prefix<'a, P: AsRef<Path> + ?Sized>(+ prefix: &'a P,+ path: &'a Path,+) -> Option<&'a Path> {+ path.strip_prefix(prefix).ok()+}++/// Returns true if this file path is just a file name. i.e., Its parent is+/// the empty string.+#[cfg(unix)]+pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool {+ use memchr::memchr;+ use std::os::unix::ffi::OsStrExt;++ let path = path.as_ref().as_os_str().as_bytes();+ memchr(b'/', path).is_none()+}++/// Returns true if this file path is just a file name. i.e., Its parent is+/// the empty string.+#[cfg(not(unix))]+pub fn is_file_name<P: AsRef<Path>>(path: P) -> bool {+ path.as_ref().parent().map(|p| p.as_os_str().is_empty()).unwrap_or(false)+}++/// The final component of the path, if it is a normal file.+///+/// If the path terminates in ., .., or consists solely of a root of prefix,+/// file_name will return None.+#[cfg(unix)]+pub fn file_name<'a, P: AsRef<Path> + ?Sized>(+ path: &'a P,+) -> Option<&'a OsStr> {+ use memchr::memrchr;+ use std::os::unix::ffi::OsStrExt;++ let path = path.as_ref().as_os_str().as_bytes();+ if path.is_empty() {+ return None;+ } else if path.len() == 1 && path[0] == b'.' {+ return None;+ } else if path.last() == Some(&b'.') {+ return None;+ } else if path.len() >= 2 && &path[path.len() - 2..] == &b".."[..] {+ return None;+ }+ let last_slash = memrchr(b'/', path).map(|i| i + 1).unwrap_or(0);+ Some(OsStr::from_bytes(&path[last_slash..]))+}++/// The final component of the path, if it is a normal file.+///+/// If the path terminates in ., .., or consists solely of a root of prefix,+/// file_name will return None.+#[cfg(not(unix))]+pub fn file_name<'a, P: AsRef<Path> + ?Sized>(+ path: &'a P,+) -> Option<&'a OsStr> {+ path.as_ref().file_name()+}
@@ -0,0 +1,591 @@+/*!+The types module provides a way of associating globs on file names to file+types.++This can be used to match specific types of files. For example, among+the default file types provided, the Rust file type is defined to be `*.rs`+with name `rust`. Similarly, the C file type is defined to be `*.{c,h}` with+name `c`.++Note that the set of default types may change over time.++# Example++This shows how to create and use a simple file type matcher using the default+file types defined in this crate.++```+use ignore::types::TypesBuilder;++let mut builder = TypesBuilder::new();+builder.add_defaults();+builder.select("rust");+let matcher = builder.build().unwrap();++assert!(matcher.matched("foo.rs", false).is_whitelist());+assert!(matcher.matched("foo.c", false).is_ignore());+```++# Example: negation++This is like the previous example, but shows how negating a file type works.+That is, this will let us match file paths that *don't* correspond to a+particular file type.++```+use ignore::types::TypesBuilder;++let mut builder = TypesBuilder::new();+builder.add_defaults();+builder.negate("c");+let matcher = builder.build().unwrap();++assert!(matcher.matched("foo.rs", false).is_none());+assert!(matcher.matched("foo.c", false).is_ignore());+```++# Example: custom file type definitions++This shows how to extend this library default file type definitions with+your own.++```+use ignore::types::TypesBuilder;++let mut builder = TypesBuilder::new();+builder.add_defaults();+builder.add("foo", "*.foo");+// Another way of adding a file type definition.+// This is useful when accepting input from an end user.+builder.add_def("bar:*.bar");+// Note: we only select `foo`, not `bar`.+builder.select("foo");+let matcher = builder.build().unwrap();++assert!(matcher.matched("x.foo", false).is_whitelist());+// This is ignored because we only selected the `foo` file type.+assert!(matcher.matched("x.bar", false).is_ignore());+```++We can also add file type definitions based on other definitions.++```+use ignore::types::TypesBuilder;++let mut builder = TypesBuilder::new();+builder.add_defaults();+builder.add("foo", "*.foo");+builder.add_def("bar:include:foo,cpp");+builder.select("bar");+let matcher = builder.build().unwrap();++assert!(matcher.matched("x.foo", false).is_whitelist());+assert!(matcher.matched("y.cpp", false).is_whitelist());+```+*/++use std::cell::RefCell;+use std::collections::HashMap;+use std::path::Path;+use std::sync::Arc;++use globset::{GlobBuilder, GlobSet, GlobSetBuilder};+use regex::Regex;+use thread_local::ThreadLocal;++use default_types::DEFAULT_TYPES;+use pathutil::file_name;+use {Error, Match};++/// Glob represents a single glob in a set of file type definitions.+///+/// There may be more than one glob for a particular file type.+///+/// This is used to report information about the highest precedent glob+/// that matched.+///+/// Note that not all matches necessarily correspond to a specific glob.+/// For example, if there are one or more selections and a file path doesn't+/// match any of those selections, then the file path is considered to be+/// ignored.+///+/// The lifetime `'a` refers to the lifetime of the underlying file type+/// definition, which corresponds to the lifetime of the file type matcher.+#[derive(Clone, Debug)]+pub struct Glob<'a>(GlobInner<'a>);++#[derive(Clone, Debug)]+enum GlobInner<'a> {+ /// No glob matched, but the file path should still be ignored.+ UnmatchedIgnore,+ /// A glob matched.+ Matched {+ /// The file type definition which provided the glob.+ def: &'a FileTypeDef,+ /// The index of the glob that matched inside the file type definition.+ which: usize,+ /// Whether the selection was negated or not.+ negated: bool,+ },+}++impl<'a> Glob<'a> {+ fn unmatched() -> Glob<'a> {+ Glob(GlobInner::UnmatchedIgnore)+ }++ /// Return the file type defintion that matched, if one exists. A file type+ /// definition always exists when a specific definition matches a file+ /// path.+ pub fn file_type_def(&self) -> Option<&FileTypeDef> {+ match self {+ Glob(GlobInner::UnmatchedIgnore) => None,+ Glob(GlobInner::Matched { def, .. }) => Some(def),+ }+ }+}++/// A single file type definition.+///+/// File type definitions can be retrieved in aggregate from a file type+/// matcher. File type definitions are also reported when its responsible+/// for a match.+#[derive(Clone, Debug, Eq, PartialEq)]+pub struct FileTypeDef {+ name: String,+ globs: Vec<String>,+}++impl FileTypeDef {+ /// Return the name of this file type.+ pub fn name(&self) -> &str {+ &self.name+ }++ /// Return the globs used to recognize this file type.+ pub fn globs(&self) -> &[String] {+ &self.globs+ }+}++/// Types is a file type matcher.+#[derive(Clone, Debug)]+pub struct Types {+ /// All of the file type definitions, sorted lexicographically by name.+ defs: Vec<FileTypeDef>,+ /// All of the selections made by the user.+ selections: Vec<Selection<FileTypeDef>>,+ /// Whether there is at least one Selection::Select in our selections.+ /// When this is true, a Match::None is converted to Match::Ignore.+ has_selected: bool,+ /// A mapping from glob index in the set to two indices. The first is an+ /// index into `selections` and the second is an index into the+ /// corresponding file type definition's list of globs.+ glob_to_selection: Vec<(usize, usize)>,+ /// The set of all glob selections, used for actual matching.+ set: GlobSet,+ /// Temporary storage for globs that match.+ matches: Arc<ThreadLocal<RefCell<Vec<usize>>>>,+}++/// Indicates the type of a selection for a particular file type.+#[derive(Clone, Debug)]+enum Selection<T> {+ Select(String, T),+ Negate(String, T),+}++impl<T> Selection<T> {+ fn is_negated(&self) -> bool {+ match *self {+ Selection::Select(..) => false,+ Selection::Negate(..) => true,+ }+ }++ fn name(&self) -> &str {+ match *self {+ Selection::Select(ref name, _) => name,+ Selection::Negate(ref name, _) => name,+ }+ }++ fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Selection<U> {+ match self {+ Selection::Select(name, inner) => {+ Selection::Select(name, f(inner))+ }+ Selection::Negate(name, inner) => {+ Selection::Negate(name, f(inner))+ }+ }+ }++ fn inner(&self) -> &T {+ match *self {+ Selection::Select(_, ref inner) => inner,+ Selection::Negate(_, ref inner) => inner,+ }+ }+}++impl Types {+ /// Creates a new file type matcher that never matches any path and+ /// contains no file type definitions.+ pub fn empty() -> Types {+ Types {+ defs: vec![],+ selections: vec![],+ has_selected: false,+ glob_to_selection: vec![],+ set: GlobSetBuilder::new().build().unwrap(),+ matches: Arc::new(ThreadLocal::default()),+ }+ }++ /// Returns true if and only if this matcher has zero selections.+ pub fn is_empty(&self) -> bool {+ self.selections.is_empty()+ }++ /// Returns the number of selections used in this matcher.+ pub fn len(&self) -> usize {+ self.selections.len()+ }++ /// Return the set of current file type definitions.+ ///+ /// Definitions and globs are sorted.+ pub fn definitions(&self) -> &[FileTypeDef] {+ &self.defs+ }++ /// Returns a match for the given path against this file type matcher.+ ///+ /// The path is considered whitelisted if it matches a selected file type.+ /// The path is considered ignored if it matches a negated file type.+ /// If at least one file type is selected and `path` doesn't match, then+ /// the path is also considered ignored.+ pub fn matched<'a, P: AsRef<Path>>(+ &'a self,+ path: P,+ is_dir: bool,+ ) -> Match<Glob<'a>> {+ // File types don't apply to directories, and we can't do anything+ // if our glob set is empty.+ if is_dir || self.set.is_empty() {+ return Match::None;+ }+ // We only want to match against the file name, so extract it.+ // If one doesn't exist, then we can't match it.+ let name = match file_name(path.as_ref()) {+ Some(name) => name,+ None if self.has_selected => {+ return Match::Ignore(Glob::unmatched());+ }+ None => {+ return Match::None;+ }+ };+ let mut matches = self.matches.get_or_default().borrow_mut();+ self.set.matches_into(name, &mut *matches);+ // The highest precedent match is the last one.+ if let Some(&i) = matches.last() {+ let (isel, iglob) = self.glob_to_selection[i];+ let sel = &self.selections[isel];+ let glob = Glob(GlobInner::Matched {+ def: sel.inner(),+ which: iglob,+ negated: sel.is_negated(),+ });+ return if sel.is_negated() {+ Match::Ignore(glob)+ } else {+ Match::Whitelist(glob)+ };+ }+ if self.has_selected {+ Match::Ignore(Glob::unmatched())+ } else {+ Match::None+ }+ }+}++/// TypesBuilder builds a type matcher from a set of file type definitions and+/// a set of file type selections.+pub struct TypesBuilder {+ types: HashMap<String, FileTypeDef>,+ selections: Vec<Selection<()>>,+}++impl TypesBuilder {+ /// Create a new builder for a file type matcher.+ ///+ /// The builder contains *no* type definitions to start with. A set+ /// of default type definitions can be added with `add_defaults`, and+ /// additional type definitions can be added with `select` and `negate`.+ pub fn new() -> TypesBuilder {+ TypesBuilder { types: HashMap::new(), selections: vec![] }+ }++ /// Build the current set of file type definitions *and* selections into+ /// a file type matcher.+ pub fn build(&self) -> Result<Types, Error> {+ let defs = self.definitions();+ let has_selected = self.selections.iter().any(|s| !s.is_negated());++ let mut selections = vec![];+ let mut glob_to_selection = vec![];+ let mut build_set = GlobSetBuilder::new();+ for (isel, selection) in self.selections.iter().enumerate() {+ let def = match self.types.get(selection.name()) {+ Some(def) => def.clone(),+ None => {+ let name = selection.name().to_string();+ return Err(Error::UnrecognizedFileType(name));+ }+ };+ for (iglob, glob) in def.globs.iter().enumerate() {+ build_set.add(+ GlobBuilder::new(glob)+ .literal_separator(true)+ .build()+ .map_err(|err| Error::Glob {+ glob: Some(glob.to_string()),+ err: err.kind().to_string(),+ })?,+ );+ glob_to_selection.push((isel, iglob));+ }+ selections.push(selection.clone().map(move |_| def));+ }+ let set = build_set+ .build()+ .map_err(|err| Error::Glob { glob: None, err: err.to_string() })?;+ Ok(Types {+ defs: defs,+ selections: selections,+ has_selected: has_selected,+ glob_to_selection: glob_to_selection,+ set: set,+ matches: Arc::new(ThreadLocal::default()),+ })+ }++ /// Return the set of current file type definitions.+ ///+ /// Definitions and globs are sorted.+ pub fn definitions(&self) -> Vec<FileTypeDef> {+ let mut defs = vec![];+ for def in self.types.values() {+ let mut def = def.clone();+ def.globs.sort();+ defs.push(def);+ }+ defs.sort_by(|def1, def2| def1.name().cmp(def2.name()));+ defs+ }++ /// Select the file type given by `name`.+ ///+ /// If `name` is `all`, then all file types currently defined are selected.+ pub fn select(&mut self, name: &str) -> &mut TypesBuilder {+ if name == "all" {+ for name in self.types.keys() {+ self.selections.push(Selection::Select(name.to_string(), ()));+ }+ } else {+ self.selections.push(Selection::Select(name.to_string(), ()));+ }+ self+ }++ /// Ignore the file type given by `name`.+ ///+ /// If `name` is `all`, then all file types currently defined are negated.+ pub fn negate(&mut self, name: &str) -> &mut TypesBuilder {+ if name == "all" {+ for name in self.types.keys() {+ self.selections.push(Selection::Negate(name.to_string(), ()));+ }+ } else {+ self.selections.push(Selection::Negate(name.to_string(), ()));+ }+ self+ }++ /// Clear any file type definitions for the type name given.+ pub fn clear(&mut self, name: &str) -> &mut TypesBuilder {+ self.types.remove(name);+ self+ }++ /// Add a new file type definition. `name` can be arbitrary and `pat`+ /// should be a glob recognizing file paths belonging to the `name` type.+ ///+ /// If `name` is `all` or otherwise contains any character that is not a+ /// Unicode letter or number, then an error is returned.+ pub fn add(&mut self, name: &str, glob: &str) -> Result<(), Error> {+ lazy_static! {+ static ref RE: Regex = Regex::new(r"^[\pL\pN]+$").unwrap();+ };+ if name == "all" || !RE.is_match(name) {+ return Err(Error::InvalidDefinition);+ }+ let (key, glob) = (name.to_string(), glob.to_string());+ self.types+ .entry(key)+ .or_insert_with(|| FileTypeDef {+ name: name.to_string(),+ globs: vec![],+ })+ .globs+ .push(glob);+ Ok(())+ }++ /// Add a new file type definition specified in string form. There are two+ /// valid formats:+ /// 1. `{name}:{glob}`. This defines a 'root' definition that associates the+ /// given name with the given glob.+ /// 2. `{name}:include:{comma-separated list of already defined names}.+ /// This defines an 'include' definition that associates the given name+ /// with the definitions of the given existing types.+ /// Names may not include any characters that are not+ /// Unicode letters or numbers.+ pub fn add_def(&mut self, def: &str) -> Result<(), Error> {+ let parts: Vec<&str> = def.split(':').collect();+ match parts.len() {+ 2 => {+ let name = parts[0];+ let glob = parts[1];+ if name.is_empty() || glob.is_empty() {+ return Err(Error::InvalidDefinition);+ }+ self.add(name, glob)+ }+ 3 => {+ let name = parts[0];+ let types_string = parts[2];+ if name.is_empty()+ || parts[1] != "include"+ || types_string.is_empty()+ {+ return Err(Error::InvalidDefinition);+ }+ let types = types_string.split(',');+ // Check ahead of time to ensure that all types specified are+ // present and fail fast if not.+ if types.clone().any(|t| !self.types.contains_key(t)) {+ return Err(Error::InvalidDefinition);+ }+ for type_name in types {+ let globs =+ self.types.get(type_name).unwrap().globs.clone();+ for glob in globs {+ self.add(name, &glob)?;+ }+ }+ Ok(())+ }+ _ => Err(Error::InvalidDefinition),+ }+ }++ /// Add a set of default file type definitions.+ pub fn add_defaults(&mut self) -> &mut TypesBuilder {+ static MSG: &'static str = "adding a default type should never fail";+ for &(name, exts) in DEFAULT_TYPES {+ for ext in exts {+ self.add(name, ext).expect(MSG);+ }+ }+ self+ }+}++#[cfg(test)]+mod tests {+ use super::TypesBuilder;++ macro_rules! matched {+ ($name:ident, $types:expr, $sel:expr, $selnot:expr,+ $path:expr) => {+ matched!($name, $types, $sel, $selnot, $path, true);+ };+ (not, $name:ident, $types:expr, $sel:expr, $selnot:expr,+ $path:expr) => {+ matched!($name, $types, $sel, $selnot, $path, false);+ };+ ($name:ident, $types:expr, $sel:expr, $selnot:expr,+ $path:expr, $matched:expr) => {+ #[test]+ fn $name() {+ let mut btypes = TypesBuilder::new();+ for tydef in $types {+ btypes.add_def(tydef).unwrap();+ }+ for sel in $sel {+ btypes.select(sel);+ }+ for selnot in $selnot {+ btypes.negate(selnot);+ }+ let types = btypes.build().unwrap();+ let mat = types.matched($path, false);+ assert_eq!($matched, !mat.is_ignore());+ }+ };+ }++ fn types() -> Vec<&'static str> {+ vec![+ "html:*.html",+ "html:*.htm",+ "rust:*.rs",+ "js:*.js",+ "foo:*.{rs,foo}",+ "combo:include:html,rust",+ ]+ }++ matched!(match1, types(), vec!["rust"], vec![], "lib.rs");+ matched!(match2, types(), vec!["html"], vec![], "index.html");+ matched!(match3, types(), vec!["html"], vec![], "index.htm");+ matched!(match4, types(), vec!["html", "rust"], vec![], "main.rs");+ matched!(match5, types(), vec![], vec![], "index.html");+ matched!(match6, types(), vec![], vec!["rust"], "index.html");+ matched!(match7, types(), vec!["foo"], vec!["rust"], "main.foo");+ matched!(match8, types(), vec!["combo"], vec![], "index.html");+ matched!(match9, types(), vec!["combo"], vec![], "lib.rs");++ matched!(not, matchnot1, types(), vec!["rust"], vec![], "index.html");+ matched!(not, matchnot2, types(), vec![], vec!["rust"], "main.rs");+ matched!(not, matchnot3, types(), vec!["foo"], vec!["rust"], "main.rs");+ matched!(not, matchnot4, types(), vec!["rust"], vec!["foo"], "main.rs");+ matched!(not, matchnot5, types(), vec!["rust"], vec!["foo"], "main.foo");+ matched!(not, matchnot6, types(), vec!["combo"], vec![], "leftpad.js");++ #[test]+ fn test_invalid_defs() {+ let mut btypes = TypesBuilder::new();+ for tydef in types() {+ btypes.add_def(tydef).unwrap();+ }+ // Preserve the original definitions for later comparison.+ let original_defs = btypes.definitions();+ let bad_defs = vec![+ // Reference to type that does not exist+ "combo:include:html,python",+ // Bad format+ "combo:foobar:html,rust",+ "",+ ];+ for def in bad_defs {+ assert!(btypes.add_def(def).is_err());+ // Ensure that nothing changed, even if some of the includes were valid.+ assert_eq!(btypes.definitions(), original_defs);+ }+ }+}
@@ -0,0 +1,2162 @@+use std::cmp;+use std::ffi::OsStr;+use std::fmt;+use std::fs::{self, FileType, Metadata};+use std::io;+use std::path::{Path, PathBuf};+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::Arc;+use std::vec;++use channel::{self, TryRecvError};+use same_file::Handle;+use walkdir::{self, WalkDir};++use dir::{Ignore, IgnoreBuilder};+use gitignore::GitignoreBuilder;+use overrides::Override;+use types::Types;+use {Error, PartialErrorBuilder};++/// A directory entry with a possible error attached.+///+/// The error typically refers to a problem parsing ignore files in a+/// particular directory.+#[derive(Clone, Debug)]+pub struct DirEntry {+ dent: DirEntryInner,+ err: Option<Error>,+}++impl DirEntry {+ /// The full path that this entry represents.+ pub fn path(&self) -> &Path {+ self.dent.path()+ }++ /// The full path that this entry represents.+ /// Analogous to [`path`], but moves ownership of the path.+ ///+ /// [`path`]: struct.DirEntry.html#method.path+ pub fn into_path(self) -> PathBuf {+ self.dent.into_path()+ }++ /// Whether this entry corresponds to a symbolic link or not.+ pub fn path_is_symlink(&self) -> bool {+ self.dent.path_is_symlink()+ }++ /// Returns true if and only if this entry corresponds to stdin.+ ///+ /// i.e., The entry has depth 0 and its file name is `-`.+ pub fn is_stdin(&self) -> bool {+ self.dent.is_stdin()+ }++ /// Return the metadata for the file that this entry points to.+ pub fn metadata(&self) -> Result<Metadata, Error> {+ self.dent.metadata()+ }++ /// Return the file type for the file that this entry points to.+ ///+ /// This entry doesn't have a file type if it corresponds to stdin.+ pub fn file_type(&self) -> Option<FileType> {+ self.dent.file_type()+ }++ /// Return the file name of this entry.+ ///+ /// If this entry has no file name (e.g., `/`), then the full path is+ /// returned.+ pub fn file_name(&self) -> &OsStr {+ self.dent.file_name()+ }++ /// Returns the depth at which this entry was created relative to the root.+ pub fn depth(&self) -> usize {+ self.dent.depth()+ }++ /// Returns the underlying inode number if one exists.+ ///+ /// If this entry doesn't have an inode number, then `None` is returned.+ #[cfg(unix)]+ pub fn ino(&self) -> Option<u64> {+ self.dent.ino()+ }++ /// Returns an error, if one exists, associated with processing this entry.+ ///+ /// An example of an error is one that occurred while parsing an ignore+ /// file. Errors related to traversing a directory tree itself are reported+ /// as part of yielding the directory entry, and not with this method.+ pub fn error(&self) -> Option<&Error> {+ self.err.as_ref()+ }++ /// Returns true if and only if this entry points to a directory.+ pub(crate) fn is_dir(&self) -> bool {+ self.dent.is_dir()+ }++ fn new_stdin() -> DirEntry {+ DirEntry { dent: DirEntryInner::Stdin, err: None }+ }++ fn new_walkdir(dent: walkdir::DirEntry, err: Option<Error>) -> DirEntry {+ DirEntry { dent: DirEntryInner::Walkdir(dent), err: err }+ }++ fn new_raw(dent: DirEntryRaw, err: Option<Error>) -> DirEntry {+ DirEntry { dent: DirEntryInner::Raw(dent), err: err }+ }+}++/// DirEntryInner is the implementation of DirEntry.+///+/// It specifically represents three distinct sources of directory entries:+///+/// 1. From the walkdir crate.+/// 2. Special entries that represent things like stdin.+/// 3. From a path.+///+/// Specifically, (3) has to essentially re-create the DirEntry implementation+/// from WalkDir.+#[derive(Clone, Debug)]+enum DirEntryInner {+ Stdin,+ Walkdir(walkdir::DirEntry),+ Raw(DirEntryRaw),+}++impl DirEntryInner {+ fn path(&self) -> &Path {+ use self::DirEntryInner::*;+ match *self {+ Stdin => Path::new("<stdin>"),+ Walkdir(ref x) => x.path(),+ Raw(ref x) => x.path(),+ }+ }++ fn into_path(self) -> PathBuf {+ use self::DirEntryInner::*;+ match self {+ Stdin => PathBuf::from("<stdin>"),+ Walkdir(x) => x.into_path(),+ Raw(x) => x.into_path(),+ }+ }++ fn path_is_symlink(&self) -> bool {+ use self::DirEntryInner::*;+ match *self {+ Stdin => false,+ Walkdir(ref x) => x.path_is_symlink(),+ Raw(ref x) => x.path_is_symlink(),+ }+ }++ fn is_stdin(&self) -> bool {+ match *self {+ DirEntryInner::Stdin => true,+ _ => false,+ }+ }++ fn metadata(&self) -> Result<Metadata, Error> {+ use self::DirEntryInner::*;+ match *self {+ Stdin => {+ let err = Error::Io(io::Error::new(+ io::ErrorKind::Other,+ "<stdin> has no metadata",+ ));+ Err(err.with_path("<stdin>"))+ }+ Walkdir(ref x) => x.metadata().map_err(|err| {+ Error::Io(io::Error::from(err)).with_path(x.path())+ }),+ Raw(ref x) => x.metadata(),+ }+ }++ fn file_type(&self) -> Option<FileType> {+ use self::DirEntryInner::*;+ match *self {+ Stdin => None,+ Walkdir(ref x) => Some(x.file_type()),+ Raw(ref x) => Some(x.file_type()),+ }+ }++ fn file_name(&self) -> &OsStr {+ use self::DirEntryInner::*;+ match *self {+ Stdin => OsStr::new("<stdin>"),+ Walkdir(ref x) => x.file_name(),+ Raw(ref x) => x.file_name(),+ }+ }++ fn depth(&self) -> usize {+ use self::DirEntryInner::*;+ match *self {+ Stdin => 0,+ Walkdir(ref x) => x.depth(),+ Raw(ref x) => x.depth(),+ }+ }++ #[cfg(unix)]+ fn ino(&self) -> Option<u64> {+ use self::DirEntryInner::*;+ use walkdir::DirEntryExt;+ match *self {+ Stdin => None,+ Walkdir(ref x) => Some(x.ino()),+ Raw(ref x) => Some(x.ino()),+ }+ }++ /// Returns true if and only if this entry points to a directory.+ fn is_dir(&self) -> bool {+ self.file_type().map(|ft| ft.is_dir()).unwrap_or(false)+ }+}++/// DirEntryRaw is essentially copied from the walkdir crate so that we can+/// build `DirEntry`s from whole cloth in the parallel iterator.+#[derive(Clone)]+struct DirEntryRaw {+ /// The path as reported by the `fs::ReadDir` iterator (even if it's a+ /// symbolic link).+ path: PathBuf,+ /// The file type. Necessary for recursive iteration, so store it.+ ty: FileType,+ /// Is set when this entry was created from a symbolic link and the user+ /// expects the iterator to follow symbolic links.+ follow_link: bool,+ /// The depth at which this entry was generated relative to the root.+ depth: usize,+ /// The underlying inode number (Unix only).+ #[cfg(unix)]+ ino: u64,+ /// The underlying metadata (Windows only). We store this on Windows+ /// because this comes for free while reading a directory.+ #[cfg(windows)]+ metadata: fs::Metadata,+}++impl fmt::Debug for DirEntryRaw {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ // Leaving out FileType because it doesn't have a debug impl+ // in Rust 1.9. We could add it if we really wanted to by manually+ // querying each possibly file type. Meh. ---AG+ f.debug_struct("DirEntryRaw")+ .field("path", &self.path)+ .field("follow_link", &self.follow_link)+ .field("depth", &self.depth)+ .finish()+ }+}++impl DirEntryRaw {+ fn path(&self) -> &Path {+ &self.path+ }++ fn into_path(self) -> PathBuf {+ self.path+ }++ fn path_is_symlink(&self) -> bool {+ self.ty.is_symlink() || self.follow_link+ }++ fn metadata(&self) -> Result<Metadata, Error> {+ self.metadata_internal()+ }++ #[cfg(windows)]+ fn metadata_internal(&self) -> Result<fs::Metadata, Error> {+ if self.follow_link {+ fs::metadata(&self.path)+ } else {+ Ok(self.metadata.clone())+ }+ .map_err(|err| Error::Io(io::Error::from(err)).with_path(&self.path))+ }++ #[cfg(not(windows))]+ fn metadata_internal(&self) -> Result<fs::Metadata, Error> {+ if self.follow_link {+ fs::metadata(&self.path)+ } else {+ fs::symlink_metadata(&self.path)+ }+ .map_err(|err| Error::Io(io::Error::from(err)).with_path(&self.path))+ }++ fn file_type(&self) -> FileType {+ self.ty+ }++ fn file_name(&self) -> &OsStr {+ self.path.file_name().unwrap_or_else(|| self.path.as_os_str())+ }++ fn depth(&self) -> usize {+ self.depth+ }++ #[cfg(unix)]+ fn ino(&self) -> u64 {+ self.ino+ }++ fn from_entry(+ depth: usize,+ ent: &fs::DirEntry,+ ) -> Result<DirEntryRaw, Error> {+ let ty = ent.file_type().map_err(|err| {+ let err = Error::Io(io::Error::from(err)).with_path(ent.path());+ Error::WithDepth { depth: depth, err: Box::new(err) }+ })?;+ DirEntryRaw::from_entry_os(depth, ent, ty)+ }++ #[cfg(windows)]+ fn from_entry_os(+ depth: usize,+ ent: &fs::DirEntry,+ ty: fs::FileType,+ ) -> Result<DirEntryRaw, Error> {+ let md = ent.metadata().map_err(|err| {+ let err = Error::Io(io::Error::from(err)).with_path(ent.path());+ Error::WithDepth { depth: depth, err: Box::new(err) }+ })?;+ Ok(DirEntryRaw {+ path: ent.path(),+ ty: ty,+ follow_link: false,+ depth: depth,+ metadata: md,+ })+ }++ #[cfg(unix)]+ fn from_entry_os(+ depth: usize,+ ent: &fs::DirEntry,+ ty: fs::FileType,+ ) -> Result<DirEntryRaw, Error> {+ use std::os::unix::fs::DirEntryExt;++ Ok(DirEntryRaw {+ path: ent.path(),+ ty: ty,+ follow_link: false,+ depth: depth,+ ino: ent.ino(),+ })+ }++ // Placeholder implementation to allow compiling on non-standard platforms (e.g. wasm32).+ #[cfg(not(any(windows, unix)))]+ fn from_entry_os(+ depth: usize,+ ent: &fs::DirEntry,+ ty: fs::FileType,+ ) -> Result<DirEntryRaw, Error> {+ Err(Error::Io(io::Error::new(+ io::ErrorKind::Other,+ "unsupported platform",+ )))+ }++ #[cfg(windows)]+ fn from_path(+ depth: usize,+ pb: PathBuf,+ link: bool,+ ) -> Result<DirEntryRaw, Error> {+ let md =+ fs::metadata(&pb).map_err(|err| Error::Io(err).with_path(&pb))?;+ Ok(DirEntryRaw {+ path: pb,+ ty: md.file_type(),+ follow_link: link,+ depth: depth,+ metadata: md,+ })+ }++ #[cfg(unix)]+ fn from_path(+ depth: usize,+ pb: PathBuf,+ link: bool,+ ) -> Result<DirEntryRaw, Error> {+ use std::os::unix::fs::MetadataExt;++ let md =+ fs::metadata(&pb).map_err(|err| Error::Io(err).with_path(&pb))?;+ Ok(DirEntryRaw {+ path: pb,+ ty: md.file_type(),+ follow_link: link,+ depth: depth,+ ino: md.ino(),+ })+ }++ // Placeholder implementation to allow compiling on non-standard platforms (e.g. wasm32).+ #[cfg(not(any(windows, unix)))]+ fn from_path(+ depth: usize,+ pb: PathBuf,+ link: bool,+ ) -> Result<DirEntryRaw, Error> {+ Err(Error::Io(io::Error::new(+ io::ErrorKind::Other,+ "unsupported platform",+ )))+ }+}++/// WalkBuilder builds a recursive directory iterator.+///+/// The builder supports a large number of configurable options. This includes+/// specific glob overrides, file type matching, toggling whether hidden+/// files are ignored or not, and of course, support for respecting gitignore+/// files.+///+/// By default, all ignore files found are respected. This includes `.ignore`,+/// `.gitignore`, `.git/info/exclude` and even your global gitignore+/// globs, usually found in `$XDG_CONFIG_HOME/git/ignore`.+///+/// Some standard recursive directory options are also supported, such as+/// limiting the recursive depth or whether to follow symbolic links (disabled+/// by default).+///+/// # Ignore rules+///+/// There are many rules that influence whether a particular file or directory+/// is skipped by this iterator. Those rules are documented here. Note that+/// the rules assume a default configuration.+///+/// * First, glob overrides are checked. If a path matches a glob override,+/// then matching stops. The path is then only skipped if the glob that matched+/// the path is an ignore glob. (An override glob is a whitelist glob unless it+/// starts with a `!`, in which case it is an ignore glob.)+/// * Second, ignore files are checked. Ignore files currently only come from+/// git ignore files (`.gitignore`, `.git/info/exclude` and the configured+/// global gitignore file), plain `.ignore` files, which have the same format+/// as gitignore files, or explicitly added ignore files. The precedence order+/// is: `.ignore`, `.gitignore`, `.git/info/exclude`, global gitignore and+/// finally explicitly added ignore files. Note that precedence between+/// different types of ignore files is not impacted by the directory hierarchy;+/// any `.ignore` file overrides all `.gitignore` files. Within each precedence+/// level, more nested ignore files have a higher precedence than less nested+/// ignore files.+/// * Third, if the previous step yields an ignore match, then all matching+/// is stopped and the path is skipped. If it yields a whitelist match, then+/// matching continues. A whitelist match can be overridden by a later matcher.+/// * Fourth, unless the path is a directory, the file type matcher is run on+/// the path. As above, if it yields an ignore match, then all matching is+/// stopped and the path is skipped. If it yields a whitelist match, then+/// matching continues.+/// * Fifth, if the path hasn't been whitelisted and it is hidden, then the+/// path is skipped.+/// * Sixth, unless the path is a directory, the size of the file is compared+/// against the max filesize limit. If it exceeds the limit, it is skipped.+/// * Seventh, if the path has made it this far then it is yielded in the+/// iterator.+#[derive(Clone)]+pub struct WalkBuilder {+ paths: Vec<PathBuf>,+ ig_builder: IgnoreBuilder,+ max_depth: Option<usize>,+ max_filesize: Option<u64>,+ follow_links: bool,+ same_file_system: bool,+ sorter: Option<Sorter>,+ threads: usize,+ skip: Option<Arc<Handle>>,+}++#[derive(Clone)]+enum Sorter {+ ByName(+ Arc<dyn Fn(&OsStr, &OsStr) -> cmp::Ordering + Send + Sync + 'static>,+ ),+ ByPath(Arc<dyn Fn(&Path, &Path) -> cmp::Ordering + Send + Sync + 'static>),+}++impl fmt::Debug for WalkBuilder {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ f.debug_struct("WalkBuilder")+ .field("paths", &self.paths)+ .field("ig_builder", &self.ig_builder)+ .field("max_depth", &self.max_depth)+ .field("max_filesize", &self.max_filesize)+ .field("follow_links", &self.follow_links)+ .field("threads", &self.threads)+ .field("skip", &self.skip)+ .finish()+ }+}++impl WalkBuilder {+ /// Create a new builder for a recursive directory iterator for the+ /// directory given.+ ///+ /// Note that if you want to traverse multiple different directories, it+ /// is better to call `add` on this builder than to create multiple+ /// `Walk` values.+ pub fn new<P: AsRef<Path>>(path: P) -> WalkBuilder {+ WalkBuilder {+ paths: vec![path.as_ref().to_path_buf()],+ ig_builder: IgnoreBuilder::new(),+ max_depth: None,+ max_filesize: None,+ follow_links: false,+ same_file_system: false,+ sorter: None,+ threads: 0,+ skip: None,+ }+ }++ /// Build a new `Walk` iterator.+ pub fn build(&self) -> Walk {+ let follow_links = self.follow_links;+ let max_depth = self.max_depth;+ let sorter = self.sorter.clone();+ let its = self+ .paths+ .iter()+ .map(move |p| {+ if p == Path::new("-") {+ (p.to_path_buf(), None)+ } else {+ let mut wd = WalkDir::new(p);+ wd = wd.follow_links(follow_links || p.is_file());+ wd = wd.same_file_system(self.same_file_system);+ if let Some(max_depth) = max_depth {+ wd = wd.max_depth(max_depth);+ }+ if let Some(ref sorter) = sorter {+ match sorter.clone() {+ Sorter::ByName(cmp) => {+ wd = wd.sort_by(move |a, b| {+ cmp(a.file_name(), b.file_name())+ });+ }+ Sorter::ByPath(cmp) => {+ wd = wd.sort_by(move |a, b| {+ cmp(a.path(), b.path())+ });+ }+ }+ }+ (p.to_path_buf(), Some(WalkEventIter::from(wd)))+ }+ })+ .collect::<Vec<_>>()+ .into_iter();+ let ig_root = self.ig_builder.build();+ Walk {+ its: its,+ it: None,+ ig_root: ig_root.clone(),+ ig: ig_root.clone(),+ max_filesize: self.max_filesize,+ skip: self.skip.clone(),+ }+ }++ /// Build a new `WalkParallel` iterator.+ ///+ /// Note that this *doesn't* return something that implements `Iterator`.+ /// Instead, the returned value must be run with a closure. e.g.,+ /// `builder.build_parallel().run(|| |path| println!("{:?}", path))`.+ pub fn build_parallel(&self) -> WalkParallel {+ WalkParallel {+ paths: self.paths.clone().into_iter(),+ ig_root: self.ig_builder.build(),+ max_depth: self.max_depth,+ max_filesize: self.max_filesize,+ follow_links: self.follow_links,+ same_file_system: self.same_file_system,+ threads: self.threads,+ skip: self.skip.clone(),+ }+ }++ /// Add a file path to the iterator.+ ///+ /// Each additional file path added is traversed recursively. This should+ /// be preferred over building multiple `Walk` iterators since this+ /// enables reusing resources across iteration.+ pub fn add<P: AsRef<Path>>(&mut self, path: P) -> &mut WalkBuilder {+ self.paths.push(path.as_ref().to_path_buf());+ self+ }++ /// The maximum depth to recurse.+ ///+ /// The default, `None`, imposes no depth restriction.+ pub fn max_depth(&mut self, depth: Option<usize>) -> &mut WalkBuilder {+ self.max_depth = depth;+ self+ }++ /// Whether to follow symbolic links or not.+ pub fn follow_links(&mut self, yes: bool) -> &mut WalkBuilder {+ self.follow_links = yes;+ self+ }++ /// Whether to ignore files above the specified limit.+ pub fn max_filesize(&mut self, filesize: Option<u64>) -> &mut WalkBuilder {+ self.max_filesize = filesize;+ self+ }++ /// The number of threads to use for traversal.+ ///+ /// Note that this only has an effect when using `build_parallel`.+ ///+ /// The default setting is `0`, which chooses the number of threads+ /// automatically using heuristics.+ pub fn threads(&mut self, n: usize) -> &mut WalkBuilder {+ self.threads = n;+ self+ }++ /// Add a global ignore file to the matcher.+ ///+ /// This has lower precedence than all other sources of ignore rules.+ ///+ /// If there was a problem adding the ignore file, then an error is+ /// returned. Note that the error may indicate *partial* failure. For+ /// example, if an ignore file contains an invalid glob, all other globs+ /// are still applied.+ pub fn add_ignore<P: AsRef<Path>>(&mut self, path: P) -> Option<Error> {+ let mut builder = GitignoreBuilder::new("");+ let mut errs = PartialErrorBuilder::default();+ errs.maybe_push(builder.add(path));+ match builder.build() {+ Ok(gi) => {+ self.ig_builder.add_ignore(gi);+ }+ Err(err) => {+ errs.push(err);+ }+ }+ errs.into_error_option()+ }++ /// Add a custom ignore file name+ ///+ /// These ignore files have higher precedence than all other ignore files.+ ///+ /// When specifying multiple names, earlier names have lower precedence than+ /// later names.+ pub fn add_custom_ignore_filename<S: AsRef<OsStr>>(+ &mut self,+ file_name: S,+ ) -> &mut WalkBuilder {+ self.ig_builder.add_custom_ignore_filename(file_name);+ self+ }++ /// Add an override matcher.+ ///+ /// By default, no override matcher is used.+ ///+ /// This overrides any previous setting.+ pub fn overrides(&mut self, overrides: Override) -> &mut WalkBuilder {+ self.ig_builder.overrides(overrides);+ self+ }++ /// Add a file type matcher.+ ///+ /// By default, no file type matcher is used.+ ///+ /// This overrides any previous setting.+ pub fn types(&mut self, types: Types) -> &mut WalkBuilder {+ self.ig_builder.types(types);+ self+ }++ /// Enables all the standard ignore filters.+ ///+ /// This toggles, as a group, all the filters that are enabled by default:+ ///+ /// - [hidden()](#method.hidden)+ /// - [parents()](#method.parents)+ /// - [ignore()](#method.ignore)+ /// - [git_ignore()](#method.git_ignore)+ /// - [git_global()](#method.git_global)+ /// - [git_exclude()](#method.git_exclude)+ ///+ /// They may still be toggled individually after calling this function.+ ///+ /// This is (by definition) enabled by default.+ pub fn standard_filters(&mut self, yes: bool) -> &mut WalkBuilder {+ self.hidden(yes)+ .parents(yes)+ .ignore(yes)+ .git_ignore(yes)+ .git_global(yes)+ .git_exclude(yes)+ }++ /// Enables ignoring hidden files.+ ///+ /// This is enabled by default.+ pub fn hidden(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.hidden(yes);+ self+ }++ /// Enables reading ignore files from parent directories.+ ///+ /// If this is enabled, then .gitignore files in parent directories of each+ /// file path given are respected. Otherwise, they are ignored.+ ///+ /// This is enabled by default.+ pub fn parents(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.parents(yes);+ self+ }++ /// Enables reading `.ignore` files.+ ///+ /// `.ignore` files have the same semantics as `gitignore` files and are+ /// supported by search tools such as ripgrep and The Silver Searcher.+ ///+ /// This is enabled by default.+ pub fn ignore(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.ignore(yes);+ self+ }++ /// Enables reading a global gitignore file, whose path is specified in+ /// git's `core.excludesFile` config option.+ ///+ /// Git's config file location is `$HOME/.gitconfig`. If `$HOME/.gitconfig`+ /// does not exist or does not specify `core.excludesFile`, then+ /// `$XDG_CONFIG_HOME/git/ignore` is read. If `$XDG_CONFIG_HOME` is not+ /// set or is empty, then `$HOME/.config/git/ignore` is used instead.+ ///+ /// This is enabled by default.+ pub fn git_global(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.git_global(yes);+ self+ }++ /// Enables reading `.gitignore` files.+ ///+ /// `.gitignore` files have match semantics as described in the `gitignore`+ /// man page.+ ///+ /// This is enabled by default.+ pub fn git_ignore(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.git_ignore(yes);+ self+ }++ /// Enables reading `.git/info/exclude` files.+ ///+ /// `.git/info/exclude` files have match semantics as described in the+ /// `gitignore` man page.+ ///+ /// This is enabled by default.+ pub fn git_exclude(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.git_exclude(yes);+ self+ }++ /// Whether a git repository is required to apply git-related ignore+ /// rules (global rules, .gitignore and local exclude rules).+ ///+ /// When disabled, git-related ignore rules are applied even when searching+ /// outside a git repository.+ pub fn require_git(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.require_git(yes);+ self+ }++ /// Process ignore files case insensitively+ ///+ /// This is disabled by default.+ pub fn ignore_case_insensitive(&mut self, yes: bool) -> &mut WalkBuilder {+ self.ig_builder.ignore_case_insensitive(yes);+ self+ }++ /// Set a function for sorting directory entries by their path.+ ///+ /// If a compare function is set, the resulting iterator will return all+ /// paths in sorted order. The compare function will be called to compare+ /// entries from the same directory.+ ///+ /// This is like `sort_by_file_name`, except the comparator accepts+ /// a `&Path` instead of the base file name, which permits it to sort by+ /// more criteria.+ ///+ /// This method will override any previous sorter set by this method or+ /// by `sort_by_file_name`.+ ///+ /// Note that this is not used in the parallel iterator.+ pub fn sort_by_file_path<F>(&mut self, cmp: F) -> &mut WalkBuilder+ where+ F: Fn(&Path, &Path) -> cmp::Ordering + Send + Sync + 'static,+ {+ self.sorter = Some(Sorter::ByPath(Arc::new(cmp)));+ self+ }++ /// Set a function for sorting directory entries by file name.+ ///+ /// If a compare function is set, the resulting iterator will return all+ /// paths in sorted order. The compare function will be called to compare+ /// names from entries from the same directory using only the name of the+ /// entry.+ ///+ /// This method will override any previous sorter set by this method or+ /// by `sort_by_file_path`.+ ///+ /// Note that this is not used in the parallel iterator.+ pub fn sort_by_file_name<F>(&mut self, cmp: F) -> &mut WalkBuilder+ where+ F: Fn(&OsStr, &OsStr) -> cmp::Ordering + Send + Sync + 'static,+ {+ self.sorter = Some(Sorter::ByName(Arc::new(cmp)));+ self+ }++ /// Do not cross file system boundaries.+ ///+ /// When this option is enabled, directory traversal will not descend into+ /// directories that are on a different file system from the root path.+ ///+ /// Currently, this option is only supported on Unix and Windows. If this+ /// option is used on an unsupported platform, then directory traversal+ /// will immediately return an error and will not yield any entries.+ pub fn same_file_system(&mut self, yes: bool) -> &mut WalkBuilder {+ self.same_file_system = yes;+ self+ }++ /// Do not yield directory entries that are believed to correspond to+ /// stdout.+ ///+ /// This is useful when a command is invoked via shell redirection to a+ /// file that is also being read. For example, `grep -r foo ./ > results`+ /// might end up trying to search `results` even though it is also writing+ /// to it, which could cause an unbounded feedback loop. Setting this+ /// option prevents this from happening by skipping over the `results`+ /// file.+ ///+ /// This is disabled by default.+ pub fn skip_stdout(&mut self, yes: bool) -> &mut WalkBuilder {+ if yes {+ self.skip = stdout_handle().map(Arc::new);+ } else {+ self.skip = None;+ }+ self+ }+}++/// Walk is a recursive directory iterator over file paths in one or more+/// directories.+///+/// Only file and directory paths matching the rules are returned. By default,+/// ignore files like `.gitignore` are respected. The precise matching rules+/// and precedence is explained in the documentation for `WalkBuilder`.+pub struct Walk {+ its: vec::IntoIter<(PathBuf, Option<WalkEventIter>)>,+ it: Option<WalkEventIter>,+ ig_root: Ignore,+ ig: Ignore,+ max_filesize: Option<u64>,+ skip: Option<Arc<Handle>>,+}++impl Walk {+ /// Creates a new recursive directory iterator for the file path given.+ ///+ /// Note that this uses default settings, which include respecting+ /// `.gitignore` files. To configure the iterator, use `WalkBuilder`+ /// instead.+ pub fn new<P: AsRef<Path>>(path: P) -> Walk {+ WalkBuilder::new(path).build()+ }++ fn skip_entry(&self, ent: &DirEntry) -> Result<bool, Error> {+ if ent.depth() == 0 {+ return Ok(false);+ }++ if let Some(ref stdout) = self.skip {+ if path_equals(ent, stdout)? {+ return Ok(true);+ }+ }+ if should_skip_entry(&self.ig, ent) {+ return Ok(true);+ }+ if self.max_filesize.is_some() && !ent.is_dir() {+ return Ok(skip_filesize(+ self.max_filesize.unwrap(),+ ent.path(),+ &ent.metadata().ok(),+ ));+ }+ Ok(false)+ }+}++impl Iterator for Walk {+ type Item = Result<DirEntry, Error>;++ #[inline(always)]+ fn next(&mut self) -> Option<Result<DirEntry, Error>> {+ loop {+ let ev = match self.it.as_mut().and_then(|it| it.next()) {+ Some(ev) => ev,+ None => {+ match self.its.next() {+ None => return None,+ Some((_, None)) => {+ return Some(Ok(DirEntry::new_stdin()));+ }+ Some((path, Some(it))) => {+ self.it = Some(it);+ if path.is_dir() {+ let (ig, err) = self.ig_root.add_parents(path);+ self.ig = ig;+ if let Some(err) = err {+ return Some(Err(err));+ }+ } else {+ self.ig = self.ig_root.clone();+ }+ }+ }+ continue;+ }+ };+ match ev {+ Err(err) => {+ return Some(Err(Error::from_walkdir(err)));+ }+ Ok(WalkEvent::Exit) => {+ self.ig = self.ig.parent().unwrap();+ }+ Ok(WalkEvent::Dir(ent)) => {+ let mut ent = DirEntry::new_walkdir(ent, None);+ let should_skip = match self.skip_entry(&ent) {+ Err(err) => return Some(Err(err)),+ Ok(should_skip) => should_skip,+ };+ if should_skip {+ self.it.as_mut().unwrap().it.skip_current_dir();+ // Still need to push this on the stack because+ // we'll get a WalkEvent::Exit event for this dir.+ // We don't care if it errors though.+ let (igtmp, _) = self.ig.add_child(ent.path());+ self.ig = igtmp;+ continue;+ }+ let (igtmp, err) = self.ig.add_child(ent.path());+ self.ig = igtmp;+ ent.err = err;+ return Some(Ok(ent));+ }+ Ok(WalkEvent::File(ent)) => {+ let ent = DirEntry::new_walkdir(ent, None);+ let should_skip = match self.skip_entry(&ent) {+ Err(err) => return Some(Err(err)),+ Ok(should_skip) => should_skip,+ };+ if should_skip {+ continue;+ }+ return Some(Ok(ent));+ }+ }+ }+ }+}++/// WalkEventIter transforms a WalkDir iterator into an iterator that more+/// accurately describes the directory tree. Namely, it emits events that are+/// one of three types: directory, file or "exit." An "exit" event means that+/// the entire contents of a directory have been enumerated.+struct WalkEventIter {+ depth: usize,+ it: walkdir::IntoIter,+ next: Option<Result<walkdir::DirEntry, walkdir::Error>>,+}++#[derive(Debug)]+enum WalkEvent {+ Dir(walkdir::DirEntry),+ File(walkdir::DirEntry),+ Exit,+}++impl From<WalkDir> for WalkEventIter {+ fn from(it: WalkDir) -> WalkEventIter {+ WalkEventIter { depth: 0, it: it.into_iter(), next: None }+ }+}++impl Iterator for WalkEventIter {+ type Item = walkdir::Result<WalkEvent>;++ #[inline(always)]+ fn next(&mut self) -> Option<walkdir::Result<WalkEvent>> {+ let dent = self.next.take().or_else(|| self.it.next());+ let depth = match dent {+ None => 0,+ Some(Ok(ref dent)) => dent.depth(),+ Some(Err(ref err)) => err.depth(),+ };+ if depth < self.depth {+ self.depth -= 1;+ self.next = dent;+ return Some(Ok(WalkEvent::Exit));+ }+ self.depth = depth;+ match dent {+ None => None,+ Some(Err(err)) => Some(Err(err)),+ Some(Ok(dent)) => {+ if walkdir_is_dir(&dent) {+ self.depth += 1;+ Some(Ok(WalkEvent::Dir(dent)))+ } else {+ Some(Ok(WalkEvent::File(dent)))+ }+ }+ }+ }+}++/// WalkState is used in the parallel recursive directory iterator to indicate+/// whether walking should continue as normal, skip descending into a+/// particular directory or quit the walk entirely.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum WalkState {+ /// Continue walking as normal.+ Continue,+ /// If the directory entry given is a directory, don't descend into it.+ /// In all other cases, this has no effect.+ Skip,+ /// Quit the entire iterator as soon as possible.+ ///+ /// Note that this is an inherently asynchronous action. It is possible+ /// for more entries to be yielded even after instructing the iterator+ /// to quit.+ Quit,+}++impl WalkState {+ fn is_quit(&self) -> bool {+ *self == WalkState::Quit+ }+}++/// A builder for constructing a visitor when using+/// [`WalkParallel::visit`](struct.WalkParallel.html#method.visit). The builder+/// will be called for each thread started by `WalkParallel`. The visitor+/// returned from each builder is then called for every directory entry.+pub trait ParallelVisitorBuilder<'s> {+ /// Create per-thread `ParallelVisitor`s for `WalkParallel`.+ fn build(&mut self) -> Box<dyn ParallelVisitor + 's>;+}++impl<'a, 's, P: ParallelVisitorBuilder<'s>> ParallelVisitorBuilder<'s>+ for &'a mut P+{+ fn build(&mut self) -> Box<dyn ParallelVisitor + 's> {+ (**self).build()+ }+}++/// Receives files and directories for the current thread.+///+/// Setup for the traversal can be implemented as part of+/// [`ParallelVisitorBuilder::build`](trait.ParallelVisitorBuilder.html#tymethod.build).+/// Teardown when traversal finishes can be implemented by implementing the+/// `Drop` trait on your traversal type.+pub trait ParallelVisitor: Send {+ /// Receives files and directories for the current thread. This is called+ /// once for every directory entry visited by traversal.+ fn visit(&mut self, entry: Result<DirEntry, Error>) -> WalkState;+}++struct FnBuilder<F> {+ builder: F,+}++impl<'s, F: FnMut() -> FnVisitor<'s>> ParallelVisitorBuilder<'s>+ for FnBuilder<F>+{+ fn build(&mut self) -> Box<dyn ParallelVisitor + 's> {+ let visitor = (self.builder)();+ Box::new(FnVisitorImp { visitor })+ }+}++type FnVisitor<'s> =+ Box<dyn FnMut(Result<DirEntry, Error>) -> WalkState + Send + 's>;++struct FnVisitorImp<'s> {+ visitor: FnVisitor<'s>,+}++impl<'s> ParallelVisitor for FnVisitorImp<'s> {+ fn visit(&mut self, entry: Result<DirEntry, Error>) -> WalkState {+ (self.visitor)(entry)+ }+}++/// WalkParallel is a parallel recursive directory iterator over files paths+/// in one or more directories.+///+/// Only file and directory paths matching the rules are returned. By default,+/// ignore files like `.gitignore` are respected. The precise matching rules+/// and precedence is explained in the documentation for `WalkBuilder`.+///+/// Unlike `Walk`, this uses multiple threads for traversing a directory.+pub struct WalkParallel {+ paths: vec::IntoIter<PathBuf>,+ ig_root: Ignore,+ max_filesize: Option<u64>,+ max_depth: Option<usize>,+ follow_links: bool,+ same_file_system: bool,+ threads: usize,+ skip: Option<Arc<Handle>>,+}++impl WalkParallel {+ /// Execute the parallel recursive directory iterator. `mkf` is called+ /// for each thread used for iteration. The function produced by `mkf`+ /// is then in turn called for each visited file path.+ pub fn run<'s, F>(self, mkf: F)+ where+ F: FnMut() -> FnVisitor<'s>,+ {+ self.visit(&mut FnBuilder { builder: mkf })+ }++ /// Execute the parallel recursive directory iterator using a custom+ /// visitor.+ ///+ /// The builder given is used to construct a visitor for every thread+ /// used by this traversal. The visitor returned from each builder is then+ /// called for every directory entry seen by that thread.+ ///+ /// Typically, creating a custom visitor is useful if you need to perform+ /// some kind of cleanup once traversal is finished. This can be achieved+ /// by implementing `Drop` for your builder (or for your visitor, if you+ /// want to execute cleanup for every thread that is launched).+ ///+ /// For example, each visitor might build up a data structure of results+ /// corresponding to the directory entries seen for each thread. Since each+ /// visitor runs on only one thread, this build-up can be done without+ /// synchronization. Then, once traversal is complete, all of the results+ /// can be merged together into a single data structure.+ pub fn visit(mut self, builder: &mut dyn ParallelVisitorBuilder) {+ let threads = self.threads();+ // TODO: Figure out how to use a bounded channel here. With an+ // unbounded channel, the workers can run away and fill up memory+ // with all of the file paths. But a bounded channel doesn't work since+ // our producers are also are consumers, so they end up getting stuck.+ //+ // We probably need to rethink parallel traversal completely to fix+ // this. The best case scenario would be finding a way to use rayon+ // to do this.+ let (tx, rx) = channel::unbounded();+ {+ let mut visitor = builder.build();+ let mut any_work = false;+ let mut paths = Vec::new().into_iter();+ std::mem::swap(&mut paths, &mut self.paths);+ // Send the initial set of root paths to the pool of workers. Note+ // that we only send directories. For files, we send to them the+ // callback directly.+ for path in paths {+ let (dent, root_device) = if path == Path::new("-") {+ (DirEntry::new_stdin(), None)+ } else {+ let root_device = if !self.same_file_system {+ None+ } else {+ match device_num(&path) {+ Ok(root_device) => Some(root_device),+ Err(err) => {+ let err = Error::Io(err).with_path(path);+ if visitor.visit(Err(err)).is_quit() {+ return;+ }+ continue;+ }+ }+ };+ match DirEntryRaw::from_path(0, path, false) {+ Ok(dent) => {+ (DirEntry::new_raw(dent, None), root_device)+ }+ Err(err) => {+ if visitor.visit(Err(err)).is_quit() {+ return;+ }+ continue;+ }+ }+ };+ tx.send(Message::Work(Work {+ dent: dent,+ ignore: self.ig_root.clone(),+ root_device: root_device,+ }))+ .unwrap();+ any_work = true;+ }+ // ... but there's no need to start workers if we don't need them.+ if !any_work {+ return;+ }+ }+ // Create the workers and then wait for them to finish.+ let num_running = Arc::new(AtomicUsize::new(threads));+ let quit_now = Arc::new(AtomicBool::new(false));+ crossbeam_utils::thread::scope(|s| {+ let mut handles = vec![];+ for _ in 0..threads {+ let worker = Worker {+ visitor: builder.build(),+ tx: tx.clone(),+ rx: rx.clone(),+ quit_now: quit_now.clone(),+ num_running: num_running.clone(),+ max_depth: self.max_depth,+ max_filesize: self.max_filesize,+ follow_links: self.follow_links,+ skip: self.skip.clone(),+ };+ handles.push(s.spawn(|_| worker.run()));+ }+ drop(tx);+ drop(rx);+ for handle in handles {+ handle.join().unwrap();+ }+ })+ .unwrap(); // Pass along panics from threads+ }++ fn threads(&self) -> usize {+ if self.threads == 0 {+ 2+ } else {+ self.threads+ }+ }+}++/// Message is the set of instructions that a worker knows how to process.+enum Message {+ /// A work item corresponds to a directory that should be descended into.+ /// Work items for entries that should be skipped or ignored should not+ /// be produced.+ Work(Work),+ /// This instruction indicates that the worker should quit.+ Quit,+}++/// A unit of work for each worker to process.+///+/// Each unit of work corresponds to a directory that should be descended+/// into.+struct Work {+ /// The directory entry.+ dent: DirEntry,+ /// Any ignore matchers that have been built for this directory's parents.+ ignore: Ignore,+ /// The root device number. When present, only files with the same device+ /// number should be considered.+ root_device: Option<u64>,+}++impl Work {+ /// Returns true if and only if this work item is a directory.+ fn is_dir(&self) -> bool {+ self.dent.is_dir()+ }++ /// Returns true if and only if this work item is a symlink.+ fn is_symlink(&self) -> bool {+ self.dent.file_type().map_or(false, |ft| ft.is_symlink())+ }++ /// Adds ignore rules for parent directories.+ ///+ /// Note that this only applies to entries at depth 0. On all other+ /// entries, this is a no-op.+ fn add_parents(&mut self) -> Option<Error> {+ if self.dent.depth() > 0 {+ return None;+ }+ // At depth 0, the path of this entry is a root path, so we can+ // use it directly to add parent ignore rules.+ let (ig, err) = self.ignore.add_parents(self.dent.path());+ self.ignore = ig;+ err+ }++ /// Reads the directory contents of this work item and adds ignore+ /// rules for this directory.+ ///+ /// If there was a problem with reading the directory contents, then+ /// an error is returned. If there was a problem reading the ignore+ /// rules for this directory, then the error is attached to this+ /// work item's directory entry.+ fn read_dir(&mut self) -> Result<fs::ReadDir, Error> {+ let readdir = match fs::read_dir(self.dent.path()) {+ Ok(readdir) => readdir,+ Err(err) => {+ let err = Error::from(err)+ .with_path(self.dent.path())+ .with_depth(self.dent.depth());+ return Err(err);+ }+ };+ let (ig, err) = self.ignore.add_child(self.dent.path());+ self.ignore = ig;+ self.dent.err = err;+ Ok(readdir)+ }+}++/// A worker is responsible for descending into directories, updating the+/// ignore matchers, producing new work and invoking the caller's callback.+///+/// Note that a worker is *both* a producer and a consumer.+struct Worker<'s> {+ /// The caller's callback.+ visitor: Box<dyn ParallelVisitor + 's>,+ /// The push side of our mpmc queue.+ tx: channel::Sender<Message>,+ /// The receive side of our mpmc queue.+ rx: channel::Receiver<Message>,+ /// Whether all workers should terminate at the next opportunity. Note+ /// that we need this because we don't want other `Work` to be done after+ /// we quit. We wouldn't need this if have a priority channel.+ quit_now: Arc<AtomicBool>,+ /// The number of workers waiting for more work.+ num_running: Arc<AtomicUsize>,+ /// The maximum depth of directories to descend. A value of `0` means no+ /// descension at all.+ max_depth: Option<usize>,+ /// The maximum size a searched file can be (in bytes). If a file exceeds+ /// this size it will be skipped.+ max_filesize: Option<u64>,+ /// Whether to follow symbolic links or not. When this is enabled, loop+ /// detection is performed.+ follow_links: bool,+ /// A file handle to skip, currently is either `None` or stdout, if it's+ /// a file and it has been requested to skip files identical to stdout.+ skip: Option<Arc<Handle>>,+}++impl<'s> Worker<'s> {+ /// Runs this worker until there is no more work left to do.+ ///+ /// The worker will call the caller's callback for all entries that aren't+ /// skipped by the ignore matcher.+ fn run(mut self) {+ 'get_work: while let Some(mut work) = self.get_work() {+ // If the work is not a directory, then we can just execute the+ // caller's callback immediately and move on.+ if work.is_symlink() || !work.is_dir() {+ if self.visitor.visit(Ok(work.dent)).is_quit() {+ self.quit_now();+ }+ continue;+ }+ if let Some(err) = work.add_parents() {+ if self.visitor.visit(Err(err)).is_quit() {+ self.quit_now();+ continue;+ }+ }++ let descend = if let Some(root_device) = work.root_device {+ match is_same_file_system(root_device, work.dent.path()) {+ Ok(true) => true,+ Ok(false) => false,+ Err(err) => {+ if self.visitor.visit(Err(err)).is_quit() {+ self.quit_now();+ continue;+ }+ false+ }+ }+ } else {+ true+ };++ // Try to read the directory first before we transfer ownership+ // to the provided closure. Do not unwrap it immediately, though,+ // as we may receive an `Err` value e.g. in the case when we do not+ // have sufficient read permissions to list the directory.+ // In that case we still want to provide the closure with a valid+ // entry before passing the error value.+ let readdir = work.read_dir();+ let depth = work.dent.depth();+ match self.visitor.visit(Ok(work.dent)) {+ WalkState::Continue => {}+ WalkState::Skip => continue,+ WalkState::Quit => {+ self.quit_now();+ continue;+ }+ }+ if !descend {+ continue;+ }++ let readdir = match readdir {+ Ok(readdir) => readdir,+ Err(err) => {+ if self.visitor.visit(Err(err)).is_quit() {+ self.quit_now();+ }+ continue;+ }+ };++ if self.max_depth.map_or(false, |max| depth >= max) {+ continue;+ }+ for result in readdir {+ let state = self.run_one(+ &work.ignore,+ depth + 1,+ work.root_device,+ result,+ );+ if state.is_quit() {+ self.quit_now();+ continue 'get_work;+ }+ }+ }+ }++ /// Runs the worker on a single entry from a directory iterator.+ ///+ /// If the entry is a path that should be ignored, then this is a no-op.+ /// Otherwise, the entry is pushed on to the queue. (The actual execution+ /// of the callback happens in `run`.)+ ///+ /// If an error occurs while reading the entry, then it is sent to the+ /// caller's callback.+ ///+ /// `ig` is the `Ignore` matcher for the parent directory. `depth` should+ /// be the depth of this entry. `result` should be the item yielded by+ /// a directory iterator.+ fn run_one(+ &mut self,+ ig: &Ignore,+ depth: usize,+ root_device: Option<u64>,+ result: Result<fs::DirEntry, io::Error>,+ ) -> WalkState {+ let fs_dent = match result {+ Ok(fs_dent) => fs_dent,+ Err(err) => {+ return self+ .visitor+ .visit(Err(Error::from(err).with_depth(depth)));+ }+ };+ let mut dent = match DirEntryRaw::from_entry(depth, &fs_dent) {+ Ok(dent) => DirEntry::new_raw(dent, None),+ Err(err) => {+ return self.visitor.visit(Err(err));+ }+ };+ let is_symlink = dent.file_type().map_or(false, |ft| ft.is_symlink());+ if self.follow_links && is_symlink {+ let path = dent.path().to_path_buf();+ dent = match DirEntryRaw::from_path(depth, path, true) {+ Ok(dent) => DirEntry::new_raw(dent, None),+ Err(err) => {+ return self.visitor.visit(Err(err));+ }+ };+ if dent.is_dir() {+ if let Err(err) = check_symlink_loop(ig, dent.path(), depth) {+ return self.visitor.visit(Err(err));+ }+ }+ }+ if let Some(ref stdout) = self.skip {+ let is_stdout = match path_equals(&dent, stdout) {+ Ok(is_stdout) => is_stdout,+ Err(err) => return self.visitor.visit(Err(err)),+ };+ if is_stdout {+ return WalkState::Continue;+ }+ }+ let should_skip_path = should_skip_entry(ig, &dent);+ let should_skip_filesize =+ if self.max_filesize.is_some() && !dent.is_dir() {+ skip_filesize(+ self.max_filesize.unwrap(),+ dent.path(),+ &dent.metadata().ok(),+ )+ } else {+ false+ };++ if !should_skip_path && !should_skip_filesize {+ self.tx+ .send(Message::Work(Work {+ dent: dent,+ ignore: ig.clone(),+ root_device: root_device,+ }))+ .unwrap();+ }+ WalkState::Continue+ }++ /// Returns the next directory to descend into.+ ///+ /// If all work has been exhausted, then this returns None. The worker+ /// should then subsequently quit.+ fn get_work(&mut self) -> Option<Work> {+ let mut value = self.rx.try_recv();+ loop {+ // Simulate a priority channel: If quit_now flag is set, we can+ // receive only quit messages.+ if self.is_quit_now() {+ value = Ok(Message::Quit)+ }+ match value {+ Ok(Message::Work(work)) => {+ return Some(work);+ }+ Ok(Message::Quit) => {+ // Repeat quit message to wake up sleeping threads, if+ // any. The domino effect will ensure that every thread+ // will quit.+ self.waiting();+ self.tx.send(Message::Quit).unwrap();+ return None;+ }+ Err(TryRecvError::Empty) => {+ // If it was the last running thread, then no more work can+ // arrive, thus we can safely start quitting. Otherwise, a+ // thread may spawn new work to be done.+ if self.waiting() == 1 {+ // Every other thread is blocked at the next recv().+ // Send the initial quit message and quit.+ self.tx.send(Message::Quit).unwrap();+ return None;+ }+ // Wait for next `Work` or `Quit` message.+ value = Ok(self+ .rx+ .recv()+ .expect("channel disconnected while worker is alive"));+ self.resume();+ }+ Err(TryRecvError::Disconnected) => {+ unreachable!("channel disconnected while worker is alive");+ }+ }+ }+ }++ /// Indicates that all workers should quit immediately.+ fn quit_now(&self) {+ self.quit_now.store(true, Ordering::SeqCst);+ }++ /// Returns true if this worker should quit immediately.+ fn is_quit_now(&self) -> bool {+ self.quit_now.load(Ordering::SeqCst)+ }++ /// Sets this worker's "running" state to false. Returns the previous+ /// number of running workers.+ fn waiting(&self) -> usize {+ self.num_running.fetch_sub(1, Ordering::SeqCst)+ }++ /// Sets this worker's "running" state to true.+ fn resume(&self) {+ self.num_running.fetch_add(1, Ordering::SeqCst);+ }+}++fn check_symlink_loop(+ ig_parent: &Ignore,+ child_path: &Path,+ child_depth: usize,+) -> Result<(), Error> {+ let hchild = Handle::from_path(child_path).map_err(|err| {+ Error::from(err).with_path(child_path).with_depth(child_depth)+ })?;+ for ig in ig_parent.parents().take_while(|ig| !ig.is_absolute_parent()) {+ let h = Handle::from_path(ig.path()).map_err(|err| {+ Error::from(err).with_path(child_path).with_depth(child_depth)+ })?;+ if hchild == h {+ return Err(Error::Loop {+ ancestor: ig.path().to_path_buf(),+ child: child_path.to_path_buf(),+ }+ .with_depth(child_depth));+ }+ }+ Ok(())+}++// Before calling this function, make sure that you ensure that is really+// necessary as the arguments imply a file stat.+fn skip_filesize(+ max_filesize: u64,+ path: &Path,+ ent: &Option<Metadata>,+) -> bool {+ let filesize = match *ent {+ Some(ref md) => Some(md.len()),+ None => None,+ };++ if let Some(fs) = filesize {+ if fs > max_filesize {+ debug!("ignoring {}: {} bytes", path.display(), fs);+ true+ } else {+ false+ }+ } else {+ false+ }+}++fn should_skip_entry(ig: &Ignore, dent: &DirEntry) -> bool {+ let m = ig.matched_dir_entry(dent);+ if m.is_ignore() {+ debug!("ignoring {}: {:?}", dent.path().display(), m);+ true+ } else if m.is_whitelist() {+ debug!("whitelisting {}: {:?}", dent.path().display(), m);+ false+ } else {+ false+ }+}++/// Returns a handle to stdout for filtering search.+///+/// A handle is returned if and only if stdout is being redirected to a file.+/// The handle returned corresponds to that file.+///+/// This can be used to ensure that we do not attempt to search a file that we+/// may also be writing to.+fn stdout_handle() -> Option<Handle> {+ let h = match Handle::stdout() {+ Err(_) => return None,+ Ok(h) => h,+ };+ let md = match h.as_file().metadata() {+ Err(_) => return None,+ Ok(md) => md,+ };+ if !md.is_file() {+ return None;+ }+ Some(h)+}++/// Returns true if and only if the given directory entry is believed to be+/// equivalent to the given handle. If there was a problem querying the path+/// for information to determine equality, then that error is returned.+fn path_equals(dent: &DirEntry, handle: &Handle) -> Result<bool, Error> {+ #[cfg(unix)]+ fn never_equal(dent: &DirEntry, handle: &Handle) -> bool {+ dent.ino() != Some(handle.ino())+ }++ #[cfg(not(unix))]+ fn never_equal(_: &DirEntry, _: &Handle) -> bool {+ false+ }++ // If we know for sure that these two things aren't equal, then avoid+ // the costly extra stat call to determine equality.+ if dent.is_stdin() || never_equal(dent, handle) {+ return Ok(false);+ }+ Handle::from_path(dent.path())+ .map(|h| &h == handle)+ .map_err(|err| Error::Io(err).with_path(dent.path()))+}++/// Returns true if the given walkdir entry corresponds to a directory.+///+/// This is normally just `dent.file_type().is_dir()`, but when we aren't+/// following symlinks, the root directory entry may be a symlink to a+/// directory that we *do* follow---by virtue of it being specified by the user+/// explicitly. In that case, we need to follow the symlink and query whether+/// it's a directory or not. But we only do this for root entries to avoid an+/// additional stat check in most cases.+fn walkdir_is_dir(dent: &walkdir::DirEntry) -> bool {+ if dent.file_type().is_dir() {+ return true;+ }+ if !dent.file_type().is_symlink() || dent.depth() > 0 {+ return false;+ }+ dent.path().metadata().ok().map_or(false, |md| md.file_type().is_dir())+}++/// Returns true if and only if the given path is on the same device as the+/// given root device.+fn is_same_file_system(root_device: u64, path: &Path) -> Result<bool, Error> {+ let dent_device =+ device_num(path).map_err(|err| Error::Io(err).with_path(path))?;+ Ok(root_device == dent_device)+}++#[cfg(unix)]+fn device_num<P: AsRef<Path>>(path: P) -> io::Result<u64> {+ use std::os::unix::fs::MetadataExt;++ path.as_ref().metadata().map(|md| md.dev())+}++#[cfg(windows)]+fn device_num<P: AsRef<Path>>(path: P) -> io::Result<u64> {+ use winapi_util::{file, Handle};++ let h = Handle::from_path_any(path)?;+ file::information(h).map(|info| info.volume_serial_number())+}++#[cfg(not(any(unix, windows)))]+fn device_num<P: AsRef<Path>>(_: P) -> io::Result<u64> {+ Err(io::Error::new(+ io::ErrorKind::Other,+ "walkdir: same_file_system option not supported on this platform",+ ))+}++#[cfg(test)]+mod tests {+ use std::fs::{self, File};+ use std::io::Write;+ use std::path::Path;+ use std::sync::{Arc, Mutex};++ use super::{DirEntry, WalkBuilder, WalkState};+ use tests::TempDir;++ fn wfile<P: AsRef<Path>>(path: P, contents: &str) {+ let mut file = File::create(path).unwrap();+ file.write_all(contents.as_bytes()).unwrap();+ }++ fn wfile_size<P: AsRef<Path>>(path: P, size: u64) {+ let file = File::create(path).unwrap();+ file.set_len(size).unwrap();+ }++ #[cfg(unix)]+ fn symlink<P: AsRef<Path>, Q: AsRef<Path>>(src: P, dst: Q) {+ use std::os::unix::fs::symlink;+ symlink(src, dst).unwrap();+ }++ fn mkdirp<P: AsRef<Path>>(path: P) {+ fs::create_dir_all(path).unwrap();+ }++ fn normal_path(unix: &str) -> String {+ if cfg!(windows) {+ unix.replace("\\", "/")+ } else {+ unix.to_string()+ }+ }++ fn walk_collect(prefix: &Path, builder: &WalkBuilder) -> Vec<String> {+ let mut paths = vec![];+ for result in builder.build() {+ let dent = match result {+ Err(_) => continue,+ Ok(dent) => dent,+ };+ let path = dent.path().strip_prefix(prefix).unwrap();+ if path.as_os_str().is_empty() {+ continue;+ }+ paths.push(normal_path(path.to_str().unwrap()));+ }+ paths.sort();+ paths+ }++ fn walk_collect_parallel(+ prefix: &Path,+ builder: &WalkBuilder,+ ) -> Vec<String> {+ let mut paths = vec![];+ for dent in walk_collect_entries_parallel(builder) {+ let path = dent.path().strip_prefix(prefix).unwrap();+ if path.as_os_str().is_empty() {+ continue;+ }+ paths.push(normal_path(path.to_str().unwrap()));+ }+ paths.sort();+ paths+ }++ fn walk_collect_entries_parallel(builder: &WalkBuilder) -> Vec<DirEntry> {+ let dents = Arc::new(Mutex::new(vec![]));+ builder.build_parallel().run(|| {+ let dents = dents.clone();+ Box::new(move |result| {+ if let Ok(dent) = result {+ dents.lock().unwrap().push(dent);+ }+ WalkState::Continue+ })+ });++ let dents = dents.lock().unwrap();+ dents.to_vec()+ }++ fn mkpaths(paths: &[&str]) -> Vec<String> {+ let mut paths: Vec<_> = paths.iter().map(|s| s.to_string()).collect();+ paths.sort();+ paths+ }++ fn tmpdir() -> TempDir {+ TempDir::new().unwrap()+ }++ fn assert_paths(prefix: &Path, builder: &WalkBuilder, expected: &[&str]) {+ let got = walk_collect(prefix, builder);+ assert_eq!(got, mkpaths(expected), "single threaded");+ let got = walk_collect_parallel(prefix, builder);+ assert_eq!(got, mkpaths(expected), "parallel");+ }++ #[test]+ fn no_ignores() {+ let td = tmpdir();+ mkdirp(td.path().join("a/b/c"));+ mkdirp(td.path().join("x/y"));+ wfile(td.path().join("a/b/foo"), "");+ wfile(td.path().join("x/y/foo"), "");++ assert_paths(+ td.path(),+ &WalkBuilder::new(td.path()),+ &["x", "x/y", "x/y/foo", "a", "a/b", "a/b/foo", "a/b/c"],+ );+ }++ #[test]+ fn custom_ignore() {+ let td = tmpdir();+ let custom_ignore = ".customignore";+ mkdirp(td.path().join("a"));+ wfile(td.path().join(custom_ignore), "foo");+ wfile(td.path().join("foo"), "");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("bar"), "");+ wfile(td.path().join("a/bar"), "");++ let mut builder = WalkBuilder::new(td.path());+ builder.add_custom_ignore_filename(&custom_ignore);+ assert_paths(td.path(), &builder, &["bar", "a", "a/bar"]);+ }++ #[test]+ fn custom_ignore_exclusive_use() {+ let td = tmpdir();+ let custom_ignore = ".customignore";+ mkdirp(td.path().join("a"));+ wfile(td.path().join(custom_ignore), "foo");+ wfile(td.path().join("foo"), "");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("bar"), "");+ wfile(td.path().join("a/bar"), "");++ let mut builder = WalkBuilder::new(td.path());+ builder.ignore(false);+ builder.git_ignore(false);+ builder.git_global(false);+ builder.git_exclude(false);+ builder.add_custom_ignore_filename(&custom_ignore);+ assert_paths(td.path(), &builder, &["bar", "a", "a/bar"]);+ }++ #[test]+ fn gitignore() {+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ mkdirp(td.path().join("a"));+ wfile(td.path().join(".gitignore"), "foo");+ wfile(td.path().join("foo"), "");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("bar"), "");+ wfile(td.path().join("a/bar"), "");++ assert_paths(+ td.path(),+ &WalkBuilder::new(td.path()),+ &["bar", "a", "a/bar"],+ );+ }++ #[test]+ fn explicit_ignore() {+ let td = tmpdir();+ let igpath = td.path().join(".not-an-ignore");+ mkdirp(td.path().join("a"));+ wfile(&igpath, "foo");+ wfile(td.path().join("foo"), "");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("bar"), "");+ wfile(td.path().join("a/bar"), "");++ let mut builder = WalkBuilder::new(td.path());+ assert!(builder.add_ignore(&igpath).is_none());+ assert_paths(td.path(), &builder, &["bar", "a", "a/bar"]);+ }++ #[test]+ fn explicit_ignore_exclusive_use() {+ let td = tmpdir();+ let igpath = td.path().join(".not-an-ignore");+ mkdirp(td.path().join("a"));+ wfile(&igpath, "foo");+ wfile(td.path().join("foo"), "");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("bar"), "");+ wfile(td.path().join("a/bar"), "");++ let mut builder = WalkBuilder::new(td.path());+ builder.standard_filters(false);+ assert!(builder.add_ignore(&igpath).is_none());+ assert_paths(+ td.path(),+ &builder,+ &[".not-an-ignore", "bar", "a", "a/bar"],+ );+ }++ #[test]+ fn gitignore_parent() {+ let td = tmpdir();+ mkdirp(td.path().join(".git"));+ mkdirp(td.path().join("a"));+ wfile(td.path().join(".gitignore"), "foo");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("a/bar"), "");++ let root = td.path().join("a");+ assert_paths(&root, &WalkBuilder::new(&root), &["bar"]);+ }++ #[test]+ fn max_depth() {+ let td = tmpdir();+ mkdirp(td.path().join("a/b/c"));+ wfile(td.path().join("foo"), "");+ wfile(td.path().join("a/foo"), "");+ wfile(td.path().join("a/b/foo"), "");+ wfile(td.path().join("a/b/c/foo"), "");++ let mut builder = WalkBuilder::new(td.path());+ assert_paths(+ td.path(),+ &builder,+ &["a", "a/b", "a/b/c", "foo", "a/foo", "a/b/foo", "a/b/c/foo"],+ );+ assert_paths(td.path(), builder.max_depth(Some(0)), &[]);+ assert_paths(td.path(), builder.max_depth(Some(1)), &["a", "foo"]);+ assert_paths(+ td.path(),+ builder.max_depth(Some(2)),+ &["a", "a/b", "foo", "a/foo"],+ );+ }++ #[test]+ fn max_filesize() {+ let td = tmpdir();+ mkdirp(td.path().join("a/b"));+ wfile_size(td.path().join("foo"), 0);+ wfile_size(td.path().join("bar"), 400);+ wfile_size(td.path().join("baz"), 600);+ wfile_size(td.path().join("a/foo"), 600);+ wfile_size(td.path().join("a/bar"), 500);+ wfile_size(td.path().join("a/baz"), 200);++ let mut builder = WalkBuilder::new(td.path());+ assert_paths(+ td.path(),+ &builder,+ &["a", "a/b", "foo", "bar", "baz", "a/foo", "a/bar", "a/baz"],+ );+ assert_paths(+ td.path(),+ builder.max_filesize(Some(0)),+ &["a", "a/b", "foo"],+ );+ assert_paths(+ td.path(),+ builder.max_filesize(Some(500)),+ &["a", "a/b", "foo", "bar", "a/bar", "a/baz"],+ );+ assert_paths(+ td.path(),+ builder.max_filesize(Some(50000)),+ &["a", "a/b", "foo", "bar", "baz", "a/foo", "a/bar", "a/baz"],+ );+ }++ #[cfg(unix)] // because symlinks on windows are weird+ #[test]+ fn symlinks() {+ let td = tmpdir();+ mkdirp(td.path().join("a/b"));+ symlink(td.path().join("a/b"), td.path().join("z"));+ wfile(td.path().join("a/b/foo"), "");++ let mut builder = WalkBuilder::new(td.path());+ assert_paths(td.path(), &builder, &["a", "a/b", "a/b/foo", "z"]);+ assert_paths(+ td.path(),+ &builder.follow_links(true),+ &["a", "a/b", "a/b/foo", "z", "z/foo"],+ );+ }++ #[cfg(unix)] // because symlinks on windows are weird+ #[test]+ fn first_path_not_symlink() {+ let td = tmpdir();+ mkdirp(td.path().join("foo"));++ let dents = WalkBuilder::new(td.path().join("foo"))+ .build()+ .into_iter()+ .collect::<Result<Vec<_>, _>>()+ .unwrap();+ assert_eq!(1, dents.len());+ assert!(!dents[0].path_is_symlink());++ let dents = walk_collect_entries_parallel(&WalkBuilder::new(+ td.path().join("foo"),+ ));+ assert_eq!(1, dents.len());+ assert!(!dents[0].path_is_symlink());+ }++ #[cfg(unix)] // because symlinks on windows are weird+ #[test]+ fn symlink_loop() {+ let td = tmpdir();+ mkdirp(td.path().join("a/b"));+ symlink(td.path().join("a"), td.path().join("a/b/c"));++ let mut builder = WalkBuilder::new(td.path());+ assert_paths(td.path(), &builder, &["a", "a/b", "a/b/c"]);+ assert_paths(td.path(), &builder.follow_links(true), &["a", "a/b"]);+ }++ // It's a little tricky to test the 'same_file_system' option since+ // we need an environment with more than one file system. We adopt a+ // heuristic where /sys is typically a distinct volume on Linux and roll+ // with that.+ #[test]+ #[cfg(target_os = "linux")]+ fn same_file_system() {+ use super::device_num;++ // If for some reason /sys doesn't exist or isn't a directory, just+ // skip this test.+ if !Path::new("/sys").is_dir() {+ return;+ }++ // If our test directory actually isn't a different volume from /sys,+ // then this test is meaningless and we shouldn't run it.+ let td = tmpdir();+ if device_num(td.path()).unwrap() == device_num("/sys").unwrap() {+ return;+ }++ mkdirp(td.path().join("same_file"));+ symlink("/sys", td.path().join("same_file").join("alink"));++ // Create a symlink to sys and enable following symlinks. If the+ // same_file_system option doesn't work, then this probably will hit a+ // permission error. Otherwise, it should just skip over the symlink+ // completely.+ let mut builder = WalkBuilder::new(td.path());+ builder.follow_links(true).same_file_system(true);+ assert_paths(td.path(), &builder, &["same_file", "same_file/alink"]);+ }++ #[cfg(target_os = "linux")]+ #[test]+ fn no_read_permissions() {+ let dir_path = Path::new("/root");++ // There's no /etc/sudoers.d, skip the test.+ if !dir_path.is_dir() {+ return;+ }+ // We're the root, so the test won't check what we want it to.+ if fs::read_dir(&dir_path).is_ok() {+ return;+ }++ // Check that we can't descend but get an entry for the parent dir.+ let builder = WalkBuilder::new(&dir_path);+ assert_paths(dir_path.parent().unwrap(), &builder, &["root"]);+ }+}
crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.gitignore
@@ -0,0 +1,216 @@+# Based on https://github.com/behnam/gitignore-test/blob/master/.gitignore++### file in root++# MATCH /file_root_1+file_root_00++# NO_MATCH+file_root_01/++# NO_MATCH+file_root_02/*++# NO_MATCH+file_root_03/**+++# MATCH /file_root_10+/file_root_10++# NO_MATCH+/file_root_11/++# NO_MATCH+/file_root_12/*++# NO_MATCH+/file_root_13/**+++# NO_MATCH+*/file_root_20++# NO_MATCH+*/file_root_21/++# NO_MATCH+*/file_root_22/*++# NO_MATCH+*/file_root_23/**+++# MATCH /file_root_30+**/file_root_30++# NO_MATCH+**/file_root_31/++# NO_MATCH+**/file_root_32/*++# NO_MATCH+**/file_root_33/**+++### file in sub-dir++# MATCH /parent_dir/file_deep_1+file_deep_00++# NO_MATCH+file_deep_01/++# NO_MATCH+file_deep_02/*++# NO_MATCH+file_deep_03/**+++# NO_MATCH+/file_deep_10++# NO_MATCH+/file_deep_11/++# NO_MATCH+/file_deep_12/*++# NO_MATCH+/file_deep_13/**+++# MATCH /parent_dir/file_deep_20+*/file_deep_20++# NO_MATCH+*/file_deep_21/++# NO_MATCH+*/file_deep_22/*++# NO_MATCH+*/file_deep_23/**+++# MATCH /parent_dir/file_deep_30+**/file_deep_30++# NO_MATCH+**/file_deep_31/++# NO_MATCH+**/file_deep_32/*++# NO_MATCH+**/file_deep_33/**+++### dir in root++# MATCH /dir_root_00+dir_root_00++# MATCH /dir_root_01+dir_root_01/++# MATCH /dir_root_02+dir_root_02/*++# MATCH /dir_root_03+dir_root_03/**+++# MATCH /dir_root_10+/dir_root_10++# MATCH /dir_root_11+/dir_root_11/++# MATCH /dir_root_12+/dir_root_12/*++# MATCH /dir_root_13+/dir_root_13/**+++# NO_MATCH+*/dir_root_20++# NO_MATCH+*/dir_root_21/++# NO_MATCH+*/dir_root_22/*++# NO_MATCH+*/dir_root_23/**+++# MATCH /dir_root_30+**/dir_root_30++# MATCH /dir_root_31+**/dir_root_31/++# MATCH /dir_root_32+**/dir_root_32/*++# MATCH /dir_root_33+**/dir_root_33/**+++### dir in sub-dir++# MATCH /parent_dir/dir_deep_00+dir_deep_00++# MATCH /parent_dir/dir_deep_01+dir_deep_01/++# NO_MATCH+dir_deep_02/*++# NO_MATCH+dir_deep_03/**+++# NO_MATCH+/dir_deep_10++# NO_MATCH+/dir_deep_11/++# NO_MATCH+/dir_deep_12/*++# NO_MATCH+/dir_deep_13/**+++# MATCH /parent_dir/dir_deep_20+*/dir_deep_20++# MATCH /parent_dir/dir_deep_21+*/dir_deep_21/++# MATCH /parent_dir/dir_deep_22+*/dir_deep_22/*++# MATCH /parent_dir/dir_deep_23+*/dir_deep_23/**+++# MATCH /parent_dir/dir_deep_30+**/dir_deep_30++# MATCH /parent_dir/dir_deep_31+**/dir_deep_31/++# MATCH /parent_dir/dir_deep_32+**/dir_deep_32/*++# MATCH /parent_dir/dir_deep_33+**/dir_deep_33/**
crates/ignore/tests/gitignore_matched_path_or_any_parents_tests.rs
@@ -0,0 +1,300 @@+extern crate ignore;++use std::path::Path;++use ignore::gitignore::{Gitignore, GitignoreBuilder};++const IGNORE_FILE: &'static str =+ "tests/gitignore_matched_path_or_any_parents_tests.gitignore";++fn get_gitignore() -> Gitignore {+ let mut builder = GitignoreBuilder::new("ROOT");+ let error = builder.add(IGNORE_FILE);+ assert!(error.is_none(), "failed to open gitignore file");+ builder.build().unwrap()+}++#[test]+#[should_panic(expected = "path is expected to be under the root")]+fn test_path_should_be_under_root() {+ let gitignore = get_gitignore();+ let path = "/tmp/some_file";+ gitignore.matched_path_or_any_parents(Path::new(path), false);+ assert!(false);+}++#[test]+fn test_files_in_root() {+ let gitignore = get_gitignore();+ let m = |path: &str| {+ gitignore.matched_path_or_any_parents(Path::new(path), false)+ };++ // 0x+ assert!(m("ROOT/file_root_00").is_ignore());+ assert!(m("ROOT/file_root_01").is_none());+ assert!(m("ROOT/file_root_02").is_none());+ assert!(m("ROOT/file_root_03").is_none());++ // 1x+ assert!(m("ROOT/file_root_10").is_ignore());+ assert!(m("ROOT/file_root_11").is_none());+ assert!(m("ROOT/file_root_12").is_none());+ assert!(m("ROOT/file_root_13").is_none());++ // 2x+ assert!(m("ROOT/file_root_20").is_none());+ assert!(m("ROOT/file_root_21").is_none());+ assert!(m("ROOT/file_root_22").is_none());+ assert!(m("ROOT/file_root_23").is_none());++ // 3x+ assert!(m("ROOT/file_root_30").is_ignore());+ assert!(m("ROOT/file_root_31").is_none());+ assert!(m("ROOT/file_root_32").is_none());+ assert!(m("ROOT/file_root_33").is_none());+}++#[test]+fn test_files_in_deep() {+ let gitignore = get_gitignore();+ let m = |path: &str| {+ gitignore.matched_path_or_any_parents(Path::new(path), false)+ };++ // 0x+ assert!(m("ROOT/parent_dir/file_deep_00").is_ignore());+ assert!(m("ROOT/parent_dir/file_deep_01").is_none());+ assert!(m("ROOT/parent_dir/file_deep_02").is_none());+ assert!(m("ROOT/parent_dir/file_deep_03").is_none());++ // 1x+ assert!(m("ROOT/parent_dir/file_deep_10").is_none());+ assert!(m("ROOT/parent_dir/file_deep_11").is_none());+ assert!(m("ROOT/parent_dir/file_deep_12").is_none());+ assert!(m("ROOT/parent_dir/file_deep_13").is_none());++ // 2x+ assert!(m("ROOT/parent_dir/file_deep_20").is_ignore());+ assert!(m("ROOT/parent_dir/file_deep_21").is_none());+ assert!(m("ROOT/parent_dir/file_deep_22").is_none());+ assert!(m("ROOT/parent_dir/file_deep_23").is_none());++ // 3x+ assert!(m("ROOT/parent_dir/file_deep_30").is_ignore());+ assert!(m("ROOT/parent_dir/file_deep_31").is_none());+ assert!(m("ROOT/parent_dir/file_deep_32").is_none());+ assert!(m("ROOT/parent_dir/file_deep_33").is_none());+}++#[test]+fn test_dirs_in_root() {+ let gitignore = get_gitignore();+ let m = |path: &str, is_dir: bool| {+ gitignore.matched_path_or_any_parents(Path::new(path), is_dir)+ };++ // 00+ assert!(m("ROOT/dir_root_00", true).is_ignore());+ assert!(m("ROOT/dir_root_00/file", false).is_ignore());+ assert!(m("ROOT/dir_root_00/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_00/child_dir/file", false).is_ignore());++ // 01+ assert!(m("ROOT/dir_root_01", true).is_ignore());+ assert!(m("ROOT/dir_root_01/file", false).is_ignore());+ assert!(m("ROOT/dir_root_01/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_01/child_dir/file", false).is_ignore());++ // 02+ assert!(m("ROOT/dir_root_02", true).is_none()); // dir itself doesn't match+ assert!(m("ROOT/dir_root_02/file", false).is_ignore());+ assert!(m("ROOT/dir_root_02/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_02/child_dir/file", false).is_ignore());++ // 03+ assert!(m("ROOT/dir_root_03", true).is_none()); // dir itself doesn't match+ assert!(m("ROOT/dir_root_03/file", false).is_ignore());+ assert!(m("ROOT/dir_root_03/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_03/child_dir/file", false).is_ignore());++ // 10+ assert!(m("ROOT/dir_root_10", true).is_ignore());+ assert!(m("ROOT/dir_root_10/file", false).is_ignore());+ assert!(m("ROOT/dir_root_10/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_10/child_dir/file", false).is_ignore());++ // 11+ assert!(m("ROOT/dir_root_11", true).is_ignore());+ assert!(m("ROOT/dir_root_11/file", false).is_ignore());+ assert!(m("ROOT/dir_root_11/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_11/child_dir/file", false).is_ignore());++ // 12+ assert!(m("ROOT/dir_root_12", true).is_none()); // dir itself doesn't match+ assert!(m("ROOT/dir_root_12/file", false).is_ignore());+ assert!(m("ROOT/dir_root_12/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_12/child_dir/file", false).is_ignore());++ // 13+ assert!(m("ROOT/dir_root_13", true).is_none());+ assert!(m("ROOT/dir_root_13/file", false).is_ignore());+ assert!(m("ROOT/dir_root_13/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_13/child_dir/file", false).is_ignore());++ // 20+ assert!(m("ROOT/dir_root_20", true).is_none());+ assert!(m("ROOT/dir_root_20/file", false).is_none());+ assert!(m("ROOT/dir_root_20/child_dir", true).is_none());+ assert!(m("ROOT/dir_root_20/child_dir/file", false).is_none());++ // 21+ assert!(m("ROOT/dir_root_21", true).is_none());+ assert!(m("ROOT/dir_root_21/file", false).is_none());+ assert!(m("ROOT/dir_root_21/child_dir", true).is_none());+ assert!(m("ROOT/dir_root_21/child_dir/file", false).is_none());++ // 22+ assert!(m("ROOT/dir_root_22", true).is_none());+ assert!(m("ROOT/dir_root_22/file", false).is_none());+ assert!(m("ROOT/dir_root_22/child_dir", true).is_none());+ assert!(m("ROOT/dir_root_22/child_dir/file", false).is_none());++ // 23+ assert!(m("ROOT/dir_root_23", true).is_none());+ assert!(m("ROOT/dir_root_23/file", false).is_none());+ assert!(m("ROOT/dir_root_23/child_dir", true).is_none());+ assert!(m("ROOT/dir_root_23/child_dir/file", false).is_none());++ // 30+ assert!(m("ROOT/dir_root_30", true).is_ignore());+ assert!(m("ROOT/dir_root_30/file", false).is_ignore());+ assert!(m("ROOT/dir_root_30/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_30/child_dir/file", false).is_ignore());++ // 31+ assert!(m("ROOT/dir_root_31", true).is_ignore());+ assert!(m("ROOT/dir_root_31/file", false).is_ignore());+ assert!(m("ROOT/dir_root_31/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_31/child_dir/file", false).is_ignore());++ // 32+ assert!(m("ROOT/dir_root_32", true).is_none()); // dir itself doesn't match+ assert!(m("ROOT/dir_root_32/file", false).is_ignore());+ assert!(m("ROOT/dir_root_32/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_32/child_dir/file", false).is_ignore());++ // 33+ assert!(m("ROOT/dir_root_33", true).is_none()); // dir itself doesn't match+ assert!(m("ROOT/dir_root_33/file", false).is_ignore());+ assert!(m("ROOT/dir_root_33/child_dir", true).is_ignore());+ assert!(m("ROOT/dir_root_33/child_dir/file", false).is_ignore());+}++#[test]+fn test_dirs_in_deep() {+ let gitignore = get_gitignore();+ let m = |path: &str, is_dir: bool| {+ gitignore.matched_path_or_any_parents(Path::new(path), is_dir)+ };++ // 00+ assert!(m("ROOT/parent_dir/dir_deep_00", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_00/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_00/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_00/child_dir/file", false).is_ignore());++ // 01+ assert!(m("ROOT/parent_dir/dir_deep_01", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_01/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_01/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_01/child_dir/file", false).is_ignore());++ // 02+ assert!(m("ROOT/parent_dir/dir_deep_02", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_02/file", false).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_02/child_dir", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_02/child_dir/file", false).is_none());++ // 03+ assert!(m("ROOT/parent_dir/dir_deep_03", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_03/file", false).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_03/child_dir", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_03/child_dir/file", false).is_none());++ // 10+ assert!(m("ROOT/parent_dir/dir_deep_10", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_10/file", false).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_10/child_dir", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_10/child_dir/file", false).is_none());++ // 11+ assert!(m("ROOT/parent_dir/dir_deep_11", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_11/file", false).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_11/child_dir", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_11/child_dir/file", false).is_none());++ // 12+ assert!(m("ROOT/parent_dir/dir_deep_12", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_12/file", false).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_12/child_dir", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_12/child_dir/file", false).is_none());++ // 13+ assert!(m("ROOT/parent_dir/dir_deep_13", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_13/file", false).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_13/child_dir", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_13/child_dir/file", false).is_none());++ // 20+ assert!(m("ROOT/parent_dir/dir_deep_20", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_20/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_20/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_20/child_dir/file", false).is_ignore());++ // 21+ assert!(m("ROOT/parent_dir/dir_deep_21", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_21/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_21/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_21/child_dir/file", false).is_ignore());++ // 22+ // dir itself doesn't match+ assert!(m("ROOT/parent_dir/dir_deep_22", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_22/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_22/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_22/child_dir/file", false).is_ignore());++ // 23+ // dir itself doesn't match+ assert!(m("ROOT/parent_dir/dir_deep_23", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_23/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_23/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_23/child_dir/file", false).is_ignore());++ // 30+ assert!(m("ROOT/parent_dir/dir_deep_30", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_30/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_30/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_30/child_dir/file", false).is_ignore());++ // 31+ assert!(m("ROOT/parent_dir/dir_deep_31", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_31/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_31/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_31/child_dir/file", false).is_ignore());++ // 32+ // dir itself doesn't match+ assert!(m("ROOT/parent_dir/dir_deep_32", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_32/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_32/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_32/child_dir/file", false).is_ignore());++ // 33+ // dir itself doesn't match+ assert!(m("ROOT/parent_dir/dir_deep_33", true).is_none());+ assert!(m("ROOT/parent_dir/dir_deep_33/file", false).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_33/child_dir", true).is_ignore());+ assert!(m("ROOT/parent_dir/dir_deep_33/child_dir/file", false).is_ignore());+}
@@ -0,0 +1,24 @@+[package]+name = "grep-matcher"+version = "0.1.3" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+A trait for regular expressions, with a focus on line oriented search.+"""+documentation = "https://docs.rs/grep-matcher"+homepage = "https://github.com/BurntSushi/ripgrep"+repository = "https://github.com/BurntSushi/ripgrep"+readme = "README.md"+keywords = ["regex", "pattern", "trait"]+license = "Unlicense/MIT"+autotests = false++[dependencies]+memchr = "2.1"++[dev-dependencies]+regex = "1.1"++[[test]]+name = "integration"+path = "tests/tests.rs"
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,36 @@+grep-matcher+------------+This crate provides a low level interface for describing regular expression+matchers. The `grep` crate uses this interface in order to make the regex+engine it uses pluggable.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/grep-matcher)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).++### Documentation++[https://docs.rs/grep-matcher](https://docs.rs/grep-matcher)++**NOTE:** You probably don't want to use this crate directly. Instead, you+should prefer the facade defined in the+[`grep`](https://docs.rs/grep)+crate.+++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+grep-matcher = "0.1"+```++and this to your crate root:++```rust+extern crate grep_matcher;+```
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
crates/matcher/src/interpolate.rs
@@ -0,0 +1,328 @@+use std::str;++use memchr::memchr;++/// Interpolate capture references in `replacement` and write the interpolation+/// result to `dst`. References in `replacement` take the form of $N or $name,+/// where `N` is a capture group index and `name` is a capture group name. The+/// function provided, `name_to_index`, maps capture group names to indices.+///+/// The `append` function given is responsible for writing the replacement+/// to the `dst` buffer. That is, it is called with the capture group index+/// of a capture group reference and is expected to resolve the index to its+/// corresponding matched text. If no such match exists, then `append` should+/// not write anything to its given buffer.+pub fn interpolate<A, N>(+ mut replacement: &[u8],+ mut append: A,+ mut name_to_index: N,+ dst: &mut Vec<u8>,+) where+ A: FnMut(usize, &mut Vec<u8>),+ N: FnMut(&str) -> Option<usize>,+{+ while !replacement.is_empty() {+ match memchr(b'$', replacement) {+ None => break,+ Some(i) => {+ dst.extend(&replacement[..i]);+ replacement = &replacement[i..];+ }+ }+ if replacement.get(1).map_or(false, |&b| b == b'$') {+ dst.push(b'$');+ replacement = &replacement[2..];+ continue;+ }+ debug_assert!(!replacement.is_empty());+ let cap_ref = match find_cap_ref(replacement) {+ Some(cap_ref) => cap_ref,+ None => {+ dst.push(b'$');+ replacement = &replacement[1..];+ continue;+ }+ };+ replacement = &replacement[cap_ref.end..];+ match cap_ref.cap {+ Ref::Number(i) => append(i, dst),+ Ref::Named(name) => {+ if let Some(i) = name_to_index(name) {+ append(i, dst);+ }+ }+ }+ }+ dst.extend(replacement);+}++/// `CaptureRef` represents a reference to a capture group inside some text.+/// The reference is either a capture group name or a number.+///+/// It is also tagged with the position in the text immediately proceding the+/// capture reference.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+struct CaptureRef<'a> {+ cap: Ref<'a>,+ end: usize,+}++/// A reference to a capture group in some text.+///+/// e.g., `$2`, `$foo`, `${foo}`.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+enum Ref<'a> {+ Named(&'a str),+ Number(usize),+}++impl<'a> From<&'a str> for Ref<'a> {+ fn from(x: &'a str) -> Ref<'a> {+ Ref::Named(x)+ }+}++impl From<usize> for Ref<'static> {+ fn from(x: usize) -> Ref<'static> {+ Ref::Number(x)+ }+}++/// Parses a possible reference to a capture group name in the given text,+/// starting at the beginning of `replacement`.+///+/// If no such valid reference could be found, None is returned.+fn find_cap_ref(replacement: &[u8]) -> Option<CaptureRef> {+ let mut i = 0;+ if replacement.len() <= 1 || replacement[0] != b'$' {+ return None;+ }+ let mut brace = false;+ i += 1;+ if replacement[i] == b'{' {+ brace = true;+ i += 1;+ }+ let mut cap_end = i;+ while replacement.get(cap_end).map_or(false, is_valid_cap_letter) {+ cap_end += 1;+ }+ if cap_end == i {+ return None;+ }+ // We just verified that the range 0..cap_end is valid ASCII, so it must+ // therefore be valid UTF-8. If we really cared, we could avoid this UTF-8+ // check with an unchecked conversion or by parsing the number straight+ // from &[u8].+ let cap = str::from_utf8(&replacement[i..cap_end])+ .expect("valid UTF-8 capture name");+ if brace {+ if !replacement.get(cap_end).map_or(false, |&b| b == b'}') {+ return None;+ }+ cap_end += 1;+ }+ Some(CaptureRef {+ cap: match cap.parse::<u32>() {+ Ok(i) => Ref::Number(i as usize),+ Err(_) => Ref::Named(cap),+ },+ end: cap_end,+ })+}++/// Returns true if and only if the given byte is allowed in a capture name.+fn is_valid_cap_letter(b: &u8) -> bool {+ match *b {+ b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'_' => true,+ _ => false,+ }+}++#[cfg(test)]+mod tests {+ use super::{find_cap_ref, interpolate, CaptureRef};++ macro_rules! find {+ ($name:ident, $text:expr) => {+ #[test]+ fn $name() {+ assert_eq!(None, find_cap_ref($text.as_bytes()));+ }+ };+ ($name:ident, $text:expr, $capref:expr) => {+ #[test]+ fn $name() {+ assert_eq!(Some($capref), find_cap_ref($text.as_bytes()));+ }+ };+ }++ macro_rules! c {+ ($name_or_number:expr, $pos:expr) => {+ CaptureRef { cap: $name_or_number.into(), end: $pos }+ };+ }++ find!(find_cap_ref1, "$foo", c!("foo", 4));+ find!(find_cap_ref2, "${foo}", c!("foo", 6));+ find!(find_cap_ref3, "$0", c!(0, 2));+ find!(find_cap_ref4, "$5", c!(5, 2));+ find!(find_cap_ref5, "$10", c!(10, 3));+ find!(find_cap_ref6, "$42a", c!("42a", 4));+ find!(find_cap_ref7, "${42}a", c!(42, 5));+ find!(find_cap_ref8, "${42");+ find!(find_cap_ref9, "${42 ");+ find!(find_cap_ref10, " $0 ");+ find!(find_cap_ref11, "$");+ find!(find_cap_ref12, " ");+ find!(find_cap_ref13, "");++ // A convenience routine for using interpolate's unwieldy but flexible API.+ fn interpolate_string(+ mut name_to_index: Vec<(&'static str, usize)>,+ caps: Vec<&'static str>,+ replacement: &str,+ ) -> String {+ name_to_index.sort_by_key(|x| x.0);++ let mut dst = vec![];+ interpolate(+ replacement.as_bytes(),+ |i, dst| {+ if let Some(&s) = caps.get(i) {+ dst.extend(s.as_bytes());+ }+ },+ |name| -> Option<usize> {+ name_to_index+ .binary_search_by_key(&name, |x| x.0)+ .ok()+ .map(|i| name_to_index[i].1)+ },+ &mut dst,+ );+ String::from_utf8(dst).unwrap()+ }++ macro_rules! interp {+ ($name:ident, $map:expr, $caps:expr, $hay:expr, $expected:expr $(,)*) => {+ #[test]+ fn $name() {+ assert_eq!($expected, interpolate_string($map, $caps, $hay));+ }+ };+ }++ interp!(+ interp1,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test $foo test",+ "test xxx test",+ );++ interp!(+ interp2,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test$footest",+ "test",+ );++ interp!(+ interp3,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test${foo}test",+ "testxxxtest",+ );++ interp!(+ interp4,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test$2test",+ "test",+ );++ interp!(+ interp5,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test${2}test",+ "testxxxtest",+ );++ interp!(+ interp6,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test $$foo test",+ "test $foo test",+ );++ interp!(+ interp7,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "test $foo",+ "test xxx",+ );++ interp!(+ interp8,+ vec![("foo", 2)],+ vec!["", "", "xxx"],+ "$foo test",+ "xxx test",+ );++ interp!(+ interp9,+ vec![("bar", 1), ("foo", 2)],+ vec!["", "yyy", "xxx"],+ "test $bar$foo",+ "test yyyxxx",+ );++ interp!(+ interp10,+ vec![("bar", 1), ("foo", 2)],+ vec!["", "yyy", "xxx"],+ "test $ test",+ "test $ test",+ );++ interp!(+ interp11,+ vec![("bar", 1), ("foo", 2)],+ vec!["", "yyy", "xxx"],+ "test ${} test",+ "test ${} test",+ );++ interp!(+ interp12,+ vec![("bar", 1), ("foo", 2)],+ vec!["", "yyy", "xxx"],+ "test ${ } test",+ "test ${ } test",+ );++ interp!(+ interp13,+ vec![("bar", 1), ("foo", 2)],+ vec!["", "yyy", "xxx"],+ "test ${a b} test",+ "test ${a b} test",+ );++ interp!(+ interp14,+ vec![("bar", 1), ("foo", 2)],+ vec!["", "yyy", "xxx"],+ "test ${a} test",+ "test test",+ );+}
@@ -0,0 +1,1151 @@+/*!+This crate provides an interface for regular expressions, with a focus on line+oriented search. The purpose of this crate is to provide a low level matching+interface that permits any kind of substring or regex implementation to power+the search routines provided by the+[`grep-searcher`](https://docs.rs/grep-searcher)+crate.++The primary thing provided by this crate is the+[`Matcher`](trait.Matcher.html)+trait. The trait defines an abstract interface for text search. It is robust+enough to support everything from basic substring search all the way to+arbitrarily complex regular expression implementations without sacrificing+performance.++A key design decision made in this crate is the use of *internal iteration*,+or otherwise known as the "push" model of searching. In this paradigm,+implementations of the `Matcher` trait will drive search and execute callbacks+provided by the caller when a match is found. This is in contrast to the+usual style of *external iteration* (the "pull" model) found throughout the+Rust ecosystem. There are two primary reasons why internal iteration was+chosen:++* Some search implementations may themselves require internal iteration.+ Converting an internal iterator to an external iterator can be non-trivial+ and sometimes even practically impossible.+* Rust's type system isn't quite expressive enough to write a generic interface+ using external iteration without giving something else up (namely, ease of+ use and/or performance).++In other words, internal iteration was chosen because it is the lowest common+denominator and because it is probably the least bad way of expressing the+interface in today's Rust. As a result, this trait isn't specifically intended+for everyday use, although, you might find it to be a happy price to pay if you+want to write code that is generic over multiple different regex+implementations.+*/++#![deny(missing_docs)]++extern crate memchr;++use std::fmt;+use std::io;+use std::ops;+use std::u64;++use interpolate::interpolate;++mod interpolate;++/// The type of a match.+///+/// The type of a match is a possibly empty range pointing to a contiguous+/// block of addressable memory.+///+/// Every `Match` is guaranteed to satisfy the invariant that `start <= end`.+///+/// # Indexing+///+/// This type is structurally identical to `std::ops::Range<usize>`, but+/// is a bit more ergonomic for dealing with match indices. In particular,+/// this type implements `Copy` and provides methods for building new `Match`+/// values based on old `Match` values. Finally, the invariant that `start`+/// is always less than or equal to `end` is enforced.+///+/// A `Match` can be used to slice a `&[u8]`, `&mut [u8]` or `&str` using+/// range notation. e.g.,+///+/// ```+/// use grep_matcher::Match;+///+/// let m = Match::new(2, 5);+/// let bytes = b"abcdefghi";+/// assert_eq!(b"cde", &bytes[m]);+/// ```+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]+pub struct Match {+ start: usize,+ end: usize,+}++impl Match {+ /// Create a new match.+ ///+ /// # Panics+ ///+ /// This function panics if `start > end`.+ #[inline]+ pub fn new(start: usize, end: usize) -> Match {+ assert!(start <= end);+ Match { start, end }+ }++ /// Creates a zero width match at the given offset.+ #[inline]+ pub fn zero(offset: usize) -> Match {+ Match { start: offset, end: offset }+ }++ /// Return the start offset of this match.+ #[inline]+ pub fn start(&self) -> usize {+ self.start+ }++ /// Return the end offset of this match.+ #[inline]+ pub fn end(&self) -> usize {+ self.end+ }++ /// Return a new match with the start offset replaced with the given+ /// value.+ ///+ /// # Panics+ ///+ /// This method panics if `start > self.end`.+ #[inline]+ pub fn with_start(&self, start: usize) -> Match {+ assert!(start <= self.end);+ Match { start, ..*self }+ }++ /// Return a new match with the end offset replaced with the given+ /// value.+ ///+ /// # Panics+ ///+ /// This method panics if `self.start > end`.+ #[inline]+ pub fn with_end(&self, end: usize) -> Match {+ assert!(self.start <= end);+ Match { end, ..*self }+ }++ /// Offset this match by the given amount and return a new match.+ ///+ /// This adds the given offset to the start and end of this match, and+ /// returns the resulting match.+ ///+ /// # Panics+ ///+ /// This panics if adding the given amount to either the start or end+ /// offset would result in an overflow.+ #[inline]+ pub fn offset(&self, amount: usize) -> Match {+ Match {+ start: self.start.checked_add(amount).unwrap(),+ end: self.end.checked_add(amount).unwrap(),+ }+ }++ /// Returns the number of bytes in this match.+ #[inline]+ pub fn len(&self) -> usize {+ self.end - self.start+ }++ /// Returns true if and only if this match is empty.+ #[inline]+ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }+}++impl ops::Index<Match> for [u8] {+ type Output = [u8];++ #[inline]+ fn index(&self, index: Match) -> &[u8] {+ &self[index.start..index.end]+ }+}++impl ops::IndexMut<Match> for [u8] {+ #[inline]+ fn index_mut(&mut self, index: Match) -> &mut [u8] {+ &mut self[index.start..index.end]+ }+}++impl ops::Index<Match> for str {+ type Output = str;++ #[inline]+ fn index(&self, index: Match) -> &str {+ &self[index.start..index.end]+ }+}++/// A line terminator.+///+/// A line terminator represents the end of a line. Generally, every line is+/// either "terminated" by the end of a stream or a specific byte (or sequence+/// of bytes).+///+/// Generally, a line terminator is a single byte, specifically, `\n`, on+/// Unix-like systems. On Windows, a line terminator is `\r\n` (referred to+/// as `CRLF` for `Carriage Return; Line Feed`).+///+/// The default line terminator is `\n` on all platforms.+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]+pub struct LineTerminator(LineTerminatorImp);++#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]+enum LineTerminatorImp {+ /// Any single byte representing a line terminator.+ ///+ /// We represent this as an array so we can safely convert it to a slice+ /// for convenient access. At some point, we can use `std::slice::from_ref`+ /// instead.+ Byte([u8; 1]),+ /// A line terminator represented by `\r\n`.+ ///+ /// When this option is used, consumers may generally treat a lone `\n` as+ /// a line terminator in addition to `\r\n`.+ CRLF,+}++impl LineTerminator {+ /// Return a new single-byte line terminator. Any byte is valid.+ #[inline]+ pub fn byte(byte: u8) -> LineTerminator {+ LineTerminator(LineTerminatorImp::Byte([byte]))+ }++ /// Return a new line terminator represented by `\r\n`.+ ///+ /// When this option is used, consumers may generally treat a lone `\n` as+ /// a line terminator in addition to `\r\n`.+ #[inline]+ pub fn crlf() -> LineTerminator {+ LineTerminator(LineTerminatorImp::CRLF)+ }++ /// Returns true if and only if this line terminator is CRLF.+ #[inline]+ pub fn is_crlf(&self) -> bool {+ self.0 == LineTerminatorImp::CRLF+ }++ /// Returns this line terminator as a single byte.+ ///+ /// If the line terminator is CRLF, then this returns `\n`. This is+ /// useful for routines that, for example, find line boundaries by treating+ /// `\n` as a line terminator even when it isn't preceded by `\r`.+ #[inline]+ pub fn as_byte(&self) -> u8 {+ match self.0 {+ LineTerminatorImp::Byte(array) => array[0],+ LineTerminatorImp::CRLF => b'\n',+ }+ }++ /// Returns this line terminator as a sequence of bytes.+ ///+ /// This returns a singleton sequence for all line terminators except for+ /// `CRLF`, in which case, it returns `\r\n`.+ ///+ /// The slice returned is guaranteed to have length at least `1`.+ #[inline]+ pub fn as_bytes(&self) -> &[u8] {+ match self.0 {+ LineTerminatorImp::Byte(ref array) => array,+ LineTerminatorImp::CRLF => &[b'\r', b'\n'],+ }+ }++ /// Returns true if and only if the given slice ends with this line+ /// terminator.+ ///+ /// If this line terminator is `CRLF`, then this only checks whether the+ /// last byte is `\n`.+ #[inline]+ pub fn is_suffix(&self, slice: &[u8]) -> bool {+ slice.last().map_or(false, |&b| b == self.as_byte())+ }+}++impl Default for LineTerminator {+ #[inline]+ fn default() -> LineTerminator {+ LineTerminator::byte(b'\n')+ }+}++/// A set of bytes.+///+/// In this crate, byte sets are used to express bytes that can never appear+/// anywhere in a match for a particular implementation of the `Matcher` trait.+/// Specifically, if such a set can be determined, then it's possible for+/// callers to perform additional operations on the basis that certain bytes+/// may never match.+///+/// For example, if a search is configured to possibly produce results that+/// span multiple lines but a caller provided pattern can never match across+/// multiple lines, then it may make sense to divert to more optimized line+/// oriented routines that don't need to handle the multi-line match case.+#[derive(Clone, Debug)]+pub struct ByteSet(BitSet);++#[derive(Clone, Copy)]+struct BitSet([u64; 4]);++impl fmt::Debug for BitSet {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ let mut fmtd = f.debug_set();+ for b in (0..256).map(|b| b as u8) {+ if ByteSet(*self).contains(b) {+ fmtd.entry(&b);+ }+ }+ fmtd.finish()+ }+}++impl ByteSet {+ /// Create an empty set of bytes.+ pub fn empty() -> ByteSet {+ ByteSet(BitSet([0; 4]))+ }++ /// Create a full set of bytes such that every possible byte is in the set+ /// returned.+ pub fn full() -> ByteSet {+ ByteSet(BitSet([u64::MAX; 4]))+ }++ /// Add a byte to this set.+ ///+ /// If the given byte already belongs to this set, then this is a no-op.+ pub fn add(&mut self, byte: u8) {+ let bucket = byte / 64;+ let bit = byte % 64;+ (self.0).0[bucket as usize] |= 1 << bit;+ }++ /// Add an inclusive range of bytes.+ pub fn add_all(&mut self, start: u8, end: u8) {+ for b in (start as u64..end as u64 + 1).map(|b| b as u8) {+ self.add(b);+ }+ }++ /// Remove a byte from this set.+ ///+ /// If the given byte is not in this set, then this is a no-op.+ pub fn remove(&mut self, byte: u8) {+ let bucket = byte / 64;+ let bit = byte % 64;+ (self.0).0[bucket as usize] &= !(1 << bit);+ }++ /// Remove an inclusive range of bytes.+ pub fn remove_all(&mut self, start: u8, end: u8) {+ for b in (start as u64..end as u64 + 1).map(|b| b as u8) {+ self.remove(b);+ }+ }++ /// Return true if and only if the given byte is in this set.+ pub fn contains(&self, byte: u8) -> bool {+ let bucket = byte / 64;+ let bit = byte % 64;+ (self.0).0[bucket as usize] & (1 << bit) > 0+ }+}++/// A trait that describes implementations of capturing groups.+///+/// When a matcher supports capturing group extraction, then it is the+/// matcher's responsibility to provide an implementation of this trait.+///+/// Principally, this trait provides a way to access capturing groups+/// in a uniform way that does not require any specific representation.+/// Namely, different matcher implementations may require different in-memory+/// representations of capturing groups. This trait permits matchers to+/// maintain their specific in-memory representation.+///+/// Note that this trait explicitly does not provide a way to construct a new+/// capture value. Instead, it is the responsibility of a `Matcher` to build+/// one, which might require knowledge of the matcher's internal implementation+/// details.+pub trait Captures {+ /// Return the total number of capturing groups. This includes capturing+ /// groups that have not matched anything.+ fn len(&self) -> usize;++ /// Return the capturing group match at the given index. If no match of+ /// that capturing group exists, then this returns `None`.+ ///+ /// When a matcher reports a match with capturing groups, then the first+ /// capturing group (at index `0`) must always correspond to the offsets+ /// for the overall match.+ fn get(&self, i: usize) -> Option<Match>;++ /// Returns true if and only if these captures are empty. This occurs+ /// when `len` is `0`.+ ///+ /// Note that capturing groups that have non-zero length but otherwise+ /// contain no matching groups are *not* empty.+ fn is_empty(&self) -> bool {+ self.len() == 0+ }++ /// Expands all instances of `$name` in `replacement` to the corresponding+ /// capture group `name`, and writes them to the `dst` buffer given.+ ///+ /// (Note: If you're looking for a convenient way to perform replacements+ /// with interpolation, then you'll want to use the `replace_with_captures`+ /// method on the `Matcher` trait.)+ ///+ /// `name` may be an integer corresponding to the index of the+ /// capture group (counted by order of opening parenthesis where `0` is the+ /// entire match) or it can be a name (consisting of letters, digits or+ /// underscores) corresponding to a named capture group.+ ///+ /// A `name` is translated to a capture group index via the given+ /// `name_to_index` function. If `name` isn't a valid capture group+ /// (whether the name doesn't exist or isn't a valid index), then it is+ /// replaced with the empty string.+ ///+ /// The longest possible name is used. e.g., `$1a` looks up the capture+ /// group named `1a` and not the capture group at index `1`. To exert+ /// more precise control over the name, use braces, e.g., `${1}a`. In all+ /// cases, capture group names are limited to ASCII letters, numbers and+ /// underscores.+ ///+ /// To write a literal `$` use `$$`.+ ///+ /// Note that the capture group match indices are resolved by slicing+ /// the given `haystack`. Generally, this means that `haystack` should be+ /// the same slice that was searched to get the current capture group+ /// matches.+ fn interpolate<F>(+ &self,+ name_to_index: F,+ haystack: &[u8],+ replacement: &[u8],+ dst: &mut Vec<u8>,+ ) where+ F: FnMut(&str) -> Option<usize>,+ {+ interpolate(+ replacement,+ |i, dst| {+ if let Some(range) = self.get(i) {+ dst.extend(&haystack[range]);+ }+ },+ name_to_index,+ dst,+ )+ }+}++/// NoCaptures provides an always-empty implementation of the `Captures` trait.+///+/// This type is useful for implementations of `Matcher` that don't support+/// capturing groups.+#[derive(Clone, Debug)]+pub struct NoCaptures(());++impl NoCaptures {+ /// Create an empty set of capturing groups.+ pub fn new() -> NoCaptures {+ NoCaptures(())+ }+}++impl Captures for NoCaptures {+ fn len(&self) -> usize {+ 0+ }+ fn get(&self, _: usize) -> Option<Match> {+ None+ }+}++/// NoError provides an error type for matchers that never produce errors.+///+/// This error type implements the `std::error::Error` and `fmt::Display`+/// traits for use in matcher implementations that can never produce errors.+///+/// The `fmt::Debug` and `fmt::Display` impls for this type panics.+#[derive(Debug, Eq, PartialEq)]+pub struct NoError(());++impl ::std::error::Error for NoError {+ fn description(&self) -> &str {+ "no error"+ }+}++impl fmt::Display for NoError {+ fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {+ panic!("BUG for NoError: an impossible error occurred")+ }+}++impl From<NoError> for io::Error {+ fn from(_: NoError) -> io::Error {+ panic!("BUG for NoError: an impossible error occurred")+ }+}++/// The type of match for a line oriented matcher.+#[derive(Clone, Copy, Debug)]+pub enum LineMatchKind {+ /// A position inside a line that is known to contain a match.+ ///+ /// This position can be anywhere in the line. It does not need to point+ /// at the location of the match.+ Confirmed(usize),+ /// A position inside a line that may contain a match, and must be searched+ /// for verification.+ ///+ /// This position can be anywhere in the line. It does not need to point+ /// at the location of the match.+ Candidate(usize),+}++/// A matcher defines an interface for regular expression implementations.+///+/// While this trait is large, there are only two required methods that+/// implementors must provide: `find_at` and `new_captures`. If captures+/// aren't supported by your implementation, then `new_captures` can be+/// implemented with+/// [`NoCaptures`](struct.NoCaptures.html). If your implementation does support+/// capture groups, then you should also implement the other capture related+/// methods, as dictated by the documentation. Crucially, this includes+/// `captures_at`.+///+/// The rest of the methods on this trait provide default implementations on+/// top of `find_at` and `new_captures`. It is not uncommon for implementations+/// to be able to provide faster variants of some methods; in those cases,+/// simply override the default implementation.+pub trait Matcher {+ /// The concrete type of capturing groups used for this matcher.+ ///+ /// If this implementation does not support capturing groups, then set+ /// this to `NoCaptures`.+ type Captures: Captures;++ /// The error type used by this matcher.+ ///+ /// For matchers in which an error is not possible, they are encouraged to+ /// use the `NoError` type in this crate. In the future, when the "never"+ /// (spelled `!`) type is stabilized, then it should probably be used+ /// instead.+ type Error: fmt::Display;++ /// Returns the start and end byte range of the first match in `haystack`+ /// after `at`, where the byte offsets are relative to that start of+ /// `haystack` (and not `at`). If no match exists, then `None` is returned.+ ///+ /// The text encoding of `haystack` is not strictly specified. Matchers are+ /// advised to assume UTF-8, or at worst, some ASCII compatible encoding.+ ///+ /// The significance of the starting point is that it takes the surrounding+ /// context into consideration. For example, the `\A` anchor can only+ /// match when `at == 0`.+ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, Self::Error>;++ /// Creates an empty group of captures suitable for use with the capturing+ /// APIs of this trait.+ ///+ /// Implementations that don't support capturing groups should use+ /// the `NoCaptures` type and implement this method by calling+ /// `NoCaptures::new()`.+ fn new_captures(&self) -> Result<Self::Captures, Self::Error>;++ /// Returns the total number of capturing groups in this matcher.+ ///+ /// If a matcher supports capturing groups, then this value must always be+ /// at least 1, where the first capturing group always corresponds to the+ /// overall match.+ ///+ /// If a matcher does not support capturing groups, then this should+ /// always return 0.+ ///+ /// By default, capturing groups are not supported, so this always+ /// returns 0.+ fn capture_count(&self) -> usize {+ 0+ }++ /// Maps the given capture group name to its corresponding capture group+ /// index, if one exists. If one does not exist, then `None` is returned.+ ///+ /// If the given capture group name maps to multiple indices, then it is+ /// not specified which one is returned. However, it is guaranteed that+ /// one of them is returned.+ ///+ /// By default, capturing groups are not supported, so this always returns+ /// `None`.+ fn capture_index(&self, _name: &str) -> Option<usize> {+ None+ }++ /// Returns the start and end byte range of the first match in `haystack`.+ /// If no match exists, then `None` is returned.+ ///+ /// The text encoding of `haystack` is not strictly specified. Matchers are+ /// advised to assume UTF-8, or at worst, some ASCII compatible encoding.+ fn find(&self, haystack: &[u8]) -> Result<Option<Match>, Self::Error> {+ self.find_at(haystack, 0)+ }++ /// Executes the given function over successive non-overlapping matches+ /// in `haystack`. If no match exists, then the given function is never+ /// called. If the function returns `false`, then iteration stops.+ fn find_iter<F>(+ &self,+ haystack: &[u8],+ mut matched: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(Match) -> bool,+ {+ self.try_find_iter(haystack, |m| Ok(matched(m)))+ .map(|r: Result<(), ()>| r.unwrap())+ }++ /// Executes the given function over successive non-overlapping matches+ /// in `haystack`. If no match exists, then the given function is never+ /// called. If the function returns `false`, then iteration stops.+ /// Similarly, if the function returns an error then iteration stops and+ /// the error is yielded. If an error occurs while executing the search,+ /// then it is converted to+ /// `E`.+ fn try_find_iter<F, E>(+ &self,+ haystack: &[u8],+ mut matched: F,+ ) -> Result<Result<(), E>, Self::Error>+ where+ F: FnMut(Match) -> Result<bool, E>,+ {+ let mut last_end = 0;+ let mut last_match = None;++ loop {+ if last_end > haystack.len() {+ return Ok(Ok(()));+ }+ let m = match self.find_at(haystack, last_end)? {+ None => return Ok(Ok(())),+ Some(m) => m,+ };+ if m.start == m.end {+ // This is an empty match. To ensure we make progress, start+ // the next search at the smallest possible starting position+ // of the next match following this one.+ last_end = m.end + 1;+ // Don't accept empty matches immediately following a match.+ // Just move on to the next match.+ if Some(m.end) == last_match {+ continue;+ }+ } else {+ last_end = m.end;+ }+ last_match = Some(m.end);+ match matched(m) {+ Ok(true) => continue,+ Ok(false) => return Ok(Ok(())),+ Err(err) => return Ok(Err(err)),+ }+ }+ }++ /// Populates the first set of capture group matches from `haystack` into+ /// `caps`. If no match exists, then `false` is returned.+ ///+ /// The text encoding of `haystack` is not strictly specified. Matchers are+ /// advised to assume UTF-8, or at worst, some ASCII compatible encoding.+ fn captures(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ ) -> Result<bool, Self::Error> {+ self.captures_at(haystack, 0, caps)+ }++ /// Executes the given function over successive non-overlapping matches+ /// in `haystack` with capture groups extracted from each match. If no+ /// match exists, then the given function is never called. If the function+ /// returns `false`, then iteration stops.+ fn captures_iter<F>(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ mut matched: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(&Self::Captures) -> bool,+ {+ self.try_captures_iter(haystack, caps, |caps| Ok(matched(caps)))+ .map(|r: Result<(), ()>| r.unwrap())+ }++ /// Executes the given function over successive non-overlapping matches+ /// in `haystack` with capture groups extracted from each match. If no+ /// match exists, then the given function is never called. If the function+ /// returns `false`, then iteration stops. Similarly, if the function+ /// returns an error then iteration stops and the error is yielded. If+ /// an error occurs while executing the search, then it is converted to+ /// `E`.+ fn try_captures_iter<F, E>(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ mut matched: F,+ ) -> Result<Result<(), E>, Self::Error>+ where+ F: FnMut(&Self::Captures) -> Result<bool, E>,+ {+ let mut last_end = 0;+ let mut last_match = None;++ loop {+ if last_end > haystack.len() {+ return Ok(Ok(()));+ }+ if !self.captures_at(haystack, last_end, caps)? {+ return Ok(Ok(()));+ }+ let m = caps.get(0).unwrap();+ if m.start == m.end {+ // This is an empty match. To ensure we make progress, start+ // the next search at the smallest possible starting position+ // of the next match following this one.+ last_end = m.end + 1;+ // Don't accept empty matches immediately following a match.+ // Just move on to the next match.+ if Some(m.end) == last_match {+ continue;+ }+ } else {+ last_end = m.end;+ }+ last_match = Some(m.end);+ match matched(caps) {+ Ok(true) => continue,+ Ok(false) => return Ok(Ok(())),+ Err(err) => return Ok(Err(err)),+ }+ }+ }++ /// Populates the first set of capture group matches from `haystack`+ /// into `matches` after `at`, where the byte offsets in each capturing+ /// group are relative to the start of `haystack` (and not `at`). If no+ /// match exists, then `false` is returned and the contents of the given+ /// capturing groups are unspecified.+ ///+ /// The text encoding of `haystack` is not strictly specified. Matchers are+ /// advised to assume UTF-8, or at worst, some ASCII compatible encoding.+ ///+ /// The significance of the starting point is that it takes the surrounding+ /// context into consideration. For example, the `\A` anchor can only+ /// match when `at == 0`.+ ///+ /// By default, capturing groups aren't supported, and this implementation+ /// will always behave as if a match were impossible.+ ///+ /// Implementors that provide support for capturing groups must guarantee+ /// that when a match occurs, the first capture match (at index `0`) is+ /// always set to the overall match offsets.+ ///+ /// Note that if implementors seek to support capturing groups, then they+ /// should implement this method. Other methods that match based on+ /// captures will then work automatically.+ fn captures_at(+ &self,+ _haystack: &[u8],+ _at: usize,+ _caps: &mut Self::Captures,+ ) -> Result<bool, Self::Error> {+ Ok(false)+ }++ /// Replaces every match in the given haystack with the result of calling+ /// `append`. `append` is given the start and end of a match, along with+ /// a handle to the `dst` buffer provided.+ ///+ /// If the given `append` function returns `false`, then replacement stops.+ fn replace<F>(+ &self,+ haystack: &[u8],+ dst: &mut Vec<u8>,+ mut append: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(Match, &mut Vec<u8>) -> bool,+ {+ let mut last_match = 0;+ self.find_iter(haystack, |m| {+ dst.extend(&haystack[last_match..m.start]);+ last_match = m.end;+ append(m, dst)+ })?;+ dst.extend(&haystack[last_match..]);+ Ok(())+ }++ /// Replaces every match in the given haystack with the result of calling+ /// `append` with the matching capture groups.+ ///+ /// If the given `append` function returns `false`, then replacement stops.+ fn replace_with_captures<F>(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ dst: &mut Vec<u8>,+ mut append: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,+ {+ let mut last_match = 0;+ self.captures_iter(haystack, caps, |caps| {+ let m = caps.get(0).unwrap();+ dst.extend(&haystack[last_match..m.start]);+ last_match = m.end;+ append(caps, dst)+ })?;+ dst.extend(&haystack[last_match..]);+ Ok(())+ }++ /// Returns true if and only if the matcher matches the given haystack.+ ///+ /// By default, this method is implemented by calling `shortest_match`.+ fn is_match(&self, haystack: &[u8]) -> Result<bool, Self::Error> {+ self.is_match_at(haystack, 0)+ }++ /// Returns true if and only if the matcher matches the given haystack+ /// starting at the given position.+ ///+ /// By default, this method is implemented by calling `shortest_match_at`.+ ///+ /// The significance of the starting point is that it takes the surrounding+ /// context into consideration. For example, the `\A` anchor can only+ /// match when `at == 0`.+ fn is_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<bool, Self::Error> {+ Ok(self.shortest_match_at(haystack, at)?.is_some())+ }++ /// Returns an end location of the first match in `haystack`. If no match+ /// exists, then `None` is returned.+ ///+ /// Note that the end location reported by this method may be less than the+ /// same end location reported by `find`. For example, running `find` with+ /// the pattern `a+` on the haystack `aaa` should report a range of `[0,+ /// 3)`, but `shortest_match` may report `1` as the ending location since+ /// that is the place at which a match is guaranteed to occur.+ ///+ /// This method should never report false positives or false negatives. The+ /// point of this method is that some implementors may be able to provide+ /// a faster implementation of this than what `find` does.+ ///+ /// By default, this method is implemented by calling `find`.+ fn shortest_match(+ &self,+ haystack: &[u8],+ ) -> Result<Option<usize>, Self::Error> {+ self.shortest_match_at(haystack, 0)+ }++ /// Returns an end location of the first match in `haystack` starting at+ /// the given position. If no match exists, then `None` is returned.+ ///+ /// Note that the end location reported by this method may be less than the+ /// same end location reported by `find`. For example, running `find` with+ /// the pattern `a+` on the haystack `aaa` should report a range of `[0,+ /// 3)`, but `shortest_match` may report `1` as the ending location since+ /// that is the place at which a match is guaranteed to occur.+ ///+ /// This method should never report false positives or false negatives. The+ /// point of this method is that some implementors may be able to provide+ /// a faster implementation of this than what `find` does.+ ///+ /// By default, this method is implemented by calling `find_at`.+ ///+ /// The significance of the starting point is that it takes the surrounding+ /// context into consideration. For example, the `\A` anchor can only+ /// match when `at == 0`.+ fn shortest_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<usize>, Self::Error> {+ Ok(self.find_at(haystack, at)?.map(|m| m.end))+ }++ /// If available, return a set of bytes that will never appear in a match+ /// produced by an implementation.+ ///+ /// Specifically, if such a set can be determined, then it's possible for+ /// callers to perform additional operations on the basis that certain+ /// bytes may never match.+ ///+ /// For example, if a search is configured to possibly produce results+ /// that span multiple lines but a caller provided pattern can never+ /// match across multiple lines, then it may make sense to divert to+ /// more optimized line oriented routines that don't need to handle the+ /// multi-line match case.+ ///+ /// Implementations that produce this set must never report false+ /// positives, but may produce false negatives. That is, is a byte is in+ /// this set then it must be guaranteed that it is never in a match. But,+ /// if a byte is not in this set, then callers cannot assume that a match+ /// exists with that byte.+ ///+ /// By default, this returns `None`.+ fn non_matching_bytes(&self) -> Option<&ByteSet> {+ None+ }++ /// If this matcher was compiled as a line oriented matcher, then this+ /// method returns the line terminator if and only if the line terminator+ /// never appears in any match produced by this matcher. If this wasn't+ /// compiled as a line oriented matcher, or if the aforementioned guarantee+ /// cannot be made, then this must return `None`, which is the default.+ /// It is **never wrong** to return `None`, but returning a line terminator+ /// when it can appear in a match results in unspecified behavior.+ ///+ /// The line terminator is typically `b'\n'`, but can be any single byte or+ /// `CRLF`.+ ///+ /// By default, this returns `None`.+ fn line_terminator(&self) -> Option<LineTerminator> {+ None+ }++ /// Return one of the following: a confirmed line match, a candidate line+ /// match (which may be a false positive) or no match at all (which **must+ /// not** be a false negative). When reporting a confirmed or candidate+ /// match, the position returned can be any position in the line.+ ///+ /// By default, this never returns a candidate match, and always either+ /// returns a confirmed match or no match at all.+ ///+ /// When a matcher can match spans over multiple lines, then the behavior+ /// of this method is unspecified. Namely, use of this method only+ /// makes sense in a context where the caller is looking for the next+ /// matching line. That is, callers should only use this method when+ /// `line_terminator` does not return `None`.+ ///+ /// # Design rationale+ ///+ /// A line matcher is, fundamentally, a normal matcher with the addition+ /// of one optional method: finding a line. By default, this routine+ /// is implemented via the matcher's `shortest_match` method, which+ /// always yields either no match or a `LineMatchKind::Confirmed`. However,+ /// implementors may provide a routine for this that can return candidate+ /// lines that need subsequent verification to be confirmed as a match.+ /// This can be useful in cases where it may be quicker to find candidate+ /// lines via some other means instead of relying on the more general+ /// implementations for `find` and `shortest_match`.+ ///+ /// For example, consider the regex `\w+foo\s+`. Both `find` and+ /// `shortest_match` must consider the entire regex, including the `\w+`+ /// and `\s+`, while searching. However, this method could look for lines+ /// containing `foo` and return them as candidates. Finding `foo` might+ /// be implemented as a highly optimized substring search routine (like+ /// `memmem`), which is likely to be faster than whatever more generalized+ /// routine is required for resolving `\w+foo\s+`. The caller is then+ /// responsible for confirming whether a match exists or not.+ ///+ /// Note that while this method may report false positives, it must never+ /// report false negatives. That is, it can never skip over lines that+ /// contain a match.+ fn find_candidate_line(+ &self,+ haystack: &[u8],+ ) -> Result<Option<LineMatchKind>, Self::Error> {+ Ok(self.shortest_match(haystack)?.map(LineMatchKind::Confirmed))+ }+}++impl<'a, M: Matcher> Matcher for &'a M {+ type Captures = M::Captures;+ type Error = M::Error;++ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, Self::Error> {+ (*self).find_at(haystack, at)+ }++ fn new_captures(&self) -> Result<Self::Captures, Self::Error> {+ (*self).new_captures()+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut Self::Captures,+ ) -> Result<bool, Self::Error> {+ (*self).captures_at(haystack, at, caps)+ }++ fn capture_index(&self, name: &str) -> Option<usize> {+ (*self).capture_index(name)+ }++ fn capture_count(&self) -> usize {+ (*self).capture_count()+ }++ fn find(&self, haystack: &[u8]) -> Result<Option<Match>, Self::Error> {+ (*self).find(haystack)+ }++ fn find_iter<F>(+ &self,+ haystack: &[u8],+ matched: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(Match) -> bool,+ {+ (*self).find_iter(haystack, matched)+ }++ fn try_find_iter<F, E>(+ &self,+ haystack: &[u8],+ matched: F,+ ) -> Result<Result<(), E>, Self::Error>+ where+ F: FnMut(Match) -> Result<bool, E>,+ {+ (*self).try_find_iter(haystack, matched)+ }++ fn captures(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ ) -> Result<bool, Self::Error> {+ (*self).captures(haystack, caps)+ }++ fn captures_iter<F>(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ matched: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(&Self::Captures) -> bool,+ {+ (*self).captures_iter(haystack, caps, matched)+ }++ fn try_captures_iter<F, E>(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ matched: F,+ ) -> Result<Result<(), E>, Self::Error>+ where+ F: FnMut(&Self::Captures) -> Result<bool, E>,+ {+ (*self).try_captures_iter(haystack, caps, matched)+ }++ fn replace<F>(+ &self,+ haystack: &[u8],+ dst: &mut Vec<u8>,+ append: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(Match, &mut Vec<u8>) -> bool,+ {+ (*self).replace(haystack, dst, append)+ }++ fn replace_with_captures<F>(+ &self,+ haystack: &[u8],+ caps: &mut Self::Captures,+ dst: &mut Vec<u8>,+ append: F,+ ) -> Result<(), Self::Error>+ where+ F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,+ {+ (*self).replace_with_captures(haystack, caps, dst, append)+ }++ fn is_match(&self, haystack: &[u8]) -> Result<bool, Self::Error> {+ (*self).is_match(haystack)+ }++ fn is_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<bool, Self::Error> {+ (*self).is_match_at(haystack, at)+ }++ fn shortest_match(+ &self,+ haystack: &[u8],+ ) -> Result<Option<usize>, Self::Error> {+ (*self).shortest_match(haystack)+ }++ fn shortest_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<usize>, Self::Error> {+ (*self).shortest_match_at(haystack, at)+ }++ fn non_matching_bytes(&self) -> Option<&ByteSet> {+ (*self).non_matching_bytes()+ }++ fn line_terminator(&self) -> Option<LineTerminator> {+ (*self).line_terminator()+ }++ fn find_candidate_line(+ &self,+ haystack: &[u8],+ ) -> Result<Option<LineMatchKind>, Self::Error> {+ (*self).find_candidate_line(haystack)+ }+}
crates/matcher/tests/test_matcher.rs
@@ -0,0 +1,230 @@+use grep_matcher::{Captures, Match, Matcher};+use regex::bytes::Regex;++use util::{RegexMatcher, RegexMatcherNoCaps};++fn matcher(pattern: &str) -> RegexMatcher {+ RegexMatcher::new(Regex::new(pattern).unwrap())+}++fn matcher_no_caps(pattern: &str) -> RegexMatcherNoCaps {+ RegexMatcherNoCaps(Regex::new(pattern).unwrap())+}++fn m(start: usize, end: usize) -> Match {+ Match::new(start, end)+}++#[test]+fn find() {+ let matcher = matcher(r"(\w+)\s+(\w+)");+ assert_eq!(matcher.find(b" homer simpson ").unwrap(), Some(m(1, 14)));+}++#[test]+fn find_iter() {+ let matcher = matcher(r"(\w+)\s+(\w+)");+ let mut matches = vec![];+ matcher+ .find_iter(b"aa bb cc dd", |m| {+ matches.push(m);+ true+ })+ .unwrap();+ assert_eq!(matches, vec![m(0, 5), m(6, 11)]);++ // Test that find_iter respects short circuiting.+ matches.clear();+ matcher+ .find_iter(b"aa bb cc dd", |m| {+ matches.push(m);+ false+ })+ .unwrap();+ assert_eq!(matches, vec![m(0, 5)]);+}++#[test]+fn try_find_iter() {+ #[derive(Clone, Debug, Eq, PartialEq)]+ struct MyError;++ let matcher = matcher(r"(\w+)\s+(\w+)");+ let mut matches = vec![];+ let err = matcher+ .try_find_iter(b"aa bb cc dd", |m| {+ if matches.is_empty() {+ matches.push(m);+ Ok(true)+ } else {+ Err(MyError)+ }+ })+ .unwrap()+ .unwrap_err();+ assert_eq!(matches, vec![m(0, 5)]);+ assert_eq!(err, MyError);+}++#[test]+fn shortest_match() {+ let matcher = matcher(r"a+");+ // This tests that the default impl isn't doing anything smart, and simply+ // defers to `find`.+ assert_eq!(matcher.shortest_match(b"aaa").unwrap(), Some(3));+ // The actual underlying regex is smarter.+ assert_eq!(matcher.re.shortest_match(b"aaa"), Some(1));+}++#[test]+fn captures() {+ let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");+ assert_eq!(matcher.capture_count(), 3);+ assert_eq!(matcher.capture_index("a"), Some(1));+ assert_eq!(matcher.capture_index("b"), Some(2));+ assert_eq!(matcher.capture_index("nada"), None);++ let mut caps = matcher.new_captures().unwrap();+ assert!(matcher.captures(b" homer simpson ", &mut caps).unwrap());+ assert_eq!(caps.get(0), Some(m(1, 14)));+ assert_eq!(caps.get(1), Some(m(1, 6)));+ assert_eq!(caps.get(2), Some(m(7, 14)));+}++#[test]+fn captures_iter() {+ let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");+ let mut caps = matcher.new_captures().unwrap();+ let mut matches = vec![];+ matcher+ .captures_iter(b"aa bb cc dd", &mut caps, |caps| {+ matches.push(caps.get(0).unwrap());+ matches.push(caps.get(1).unwrap());+ matches.push(caps.get(2).unwrap());+ true+ })+ .unwrap();+ assert_eq!(+ matches,+ vec![m(0, 5), m(0, 2), m(3, 5), m(6, 11), m(6, 8), m(9, 11),]+ );++ // Test that captures_iter respects short circuiting.+ matches.clear();+ matcher+ .captures_iter(b"aa bb cc dd", &mut caps, |caps| {+ matches.push(caps.get(0).unwrap());+ matches.push(caps.get(1).unwrap());+ matches.push(caps.get(2).unwrap());+ false+ })+ .unwrap();+ assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5),]);+}++#[test]+fn try_captures_iter() {+ #[derive(Clone, Debug, Eq, PartialEq)]+ struct MyError;++ let matcher = matcher(r"(?P<a>\w+)\s+(?P<b>\w+)");+ let mut caps = matcher.new_captures().unwrap();+ let mut matches = vec![];+ let err = matcher+ .try_captures_iter(b"aa bb cc dd", &mut caps, |caps| {+ if matches.is_empty() {+ matches.push(caps.get(0).unwrap());+ matches.push(caps.get(1).unwrap());+ matches.push(caps.get(2).unwrap());+ Ok(true)+ } else {+ Err(MyError)+ }+ })+ .unwrap()+ .unwrap_err();+ assert_eq!(matches, vec![m(0, 5), m(0, 2), m(3, 5)]);+ assert_eq!(err, MyError);+}++// Test that our default impls for capturing are correct. Namely, when+// capturing isn't supported by the underlying matcher, then all of the+// various capturing related APIs fail fast.+#[test]+fn no_captures() {+ let matcher = matcher_no_caps(r"(?P<a>\w+)\s+(?P<b>\w+)");+ assert_eq!(matcher.capture_count(), 0);+ assert_eq!(matcher.capture_index("a"), None);+ assert_eq!(matcher.capture_index("b"), None);+ assert_eq!(matcher.capture_index("nada"), None);++ let mut caps = matcher.new_captures().unwrap();+ assert!(!matcher.captures(b"homer simpson", &mut caps).unwrap());++ let mut called = false;+ matcher+ .captures_iter(b"homer simpson", &mut caps, |_| {+ called = true;+ true+ })+ .unwrap();+ assert!(!called);+}++#[test]+fn replace() {+ let matcher = matcher(r"(\w+)\s+(\w+)");+ let mut dst = vec![];+ matcher+ .replace(b"aa bb cc dd", &mut dst, |_, dst| {+ dst.push(b'z');+ true+ })+ .unwrap();+ assert_eq!(dst, b"z z");++ // Test that replacements respect short circuiting.+ dst.clear();+ matcher+ .replace(b"aa bb cc dd", &mut dst, |_, dst| {+ dst.push(b'z');+ false+ })+ .unwrap();+ assert_eq!(dst, b"z cc dd");+}++#[test]+fn replace_with_captures() {+ let matcher = matcher(r"(\w+)\s+(\w+)");+ let haystack = b"aa bb cc dd";+ let mut caps = matcher.new_captures().unwrap();+ let mut dst = vec![];+ matcher+ .replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {+ caps.interpolate(+ |name| matcher.capture_index(name),+ haystack,+ b"$2 $1",+ dst,+ );+ true+ })+ .unwrap();+ assert_eq!(dst, b"bb aa dd cc");++ // Test that replacements respect short circuiting.+ dst.clear();+ matcher+ .replace_with_captures(haystack, &mut caps, &mut dst, |caps, dst| {+ caps.interpolate(+ |name| matcher.capture_index(name),+ haystack,+ b"$2 $1",+ dst,+ );+ false+ })+ .unwrap();+ assert_eq!(dst, b"bb aa cc dd");+}
@@ -0,0 +1,6 @@+extern crate grep_matcher;+extern crate regex;++mod util;++mod test_matcher;
@@ -0,0 +1,95 @@+use std::collections::HashMap;+use std::result;++use grep_matcher::{Captures, Match, Matcher, NoCaptures, NoError};+use regex::bytes::{CaptureLocations, Regex};++#[derive(Debug)]+pub struct RegexMatcher {+ pub re: Regex,+ pub names: HashMap<String, usize>,+}++impl RegexMatcher {+ pub fn new(re: Regex) -> RegexMatcher {+ let mut names = HashMap::new();+ for (i, optional_name) in re.capture_names().enumerate() {+ if let Some(name) = optional_name {+ names.insert(name.to_string(), i);+ }+ }+ RegexMatcher { re: re, names: names }+ }+}++type Result<T> = result::Result<T, NoError>;++impl Matcher for RegexMatcher {+ type Captures = RegexCaptures;+ type Error = NoError;++ fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>> {+ Ok(self+ .re+ .find_at(haystack, at)+ .map(|m| Match::new(m.start(), m.end())))+ }++ fn new_captures(&self) -> Result<RegexCaptures> {+ Ok(RegexCaptures(self.re.capture_locations()))+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut RegexCaptures,+ ) -> Result<bool> {+ Ok(self.re.captures_read_at(&mut caps.0, haystack, at).is_some())+ }++ fn capture_count(&self) -> usize {+ self.re.captures_len()+ }++ fn capture_index(&self, name: &str) -> Option<usize> {+ self.names.get(name).map(|i| *i)+ }++ // We purposely don't implement any other methods, so that we test the+ // default impls. The "real" Regex impl for Matcher provides a few more+ // impls. e.g., Its `find_iter` impl is faster than what we can do here,+ // since the regex crate avoids synchronization overhead.+}++#[derive(Debug)]+pub struct RegexMatcherNoCaps(pub Regex);++impl Matcher for RegexMatcherNoCaps {+ type Captures = NoCaptures;+ type Error = NoError;++ fn find_at(&self, haystack: &[u8], at: usize) -> Result<Option<Match>> {+ Ok(self+ .0+ .find_at(haystack, at)+ .map(|m| Match::new(m.start(), m.end())))+ }++ fn new_captures(&self) -> Result<NoCaptures> {+ Ok(NoCaptures::new())+ }+}++#[derive(Clone, Debug)]+pub struct RegexCaptures(CaptureLocations);++impl Captures for RegexCaptures {+ fn len(&self) -> usize {+ self.0.len()+ }++ fn get(&self, i: usize) -> Option<Match> {+ self.0.pos(i).map(|(s, e)| Match::new(s, e))+ }+}
@@ -0,0 +1,17 @@+[package]+name = "grep-pcre2"+version = "0.1.3" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+Use PCRE2 with the 'grep' crate.+"""+documentation = "https://docs.rs/grep-pcre2"+homepage = "https://github.com/BurntSushi/ripgrep"+repository = "https://github.com/BurntSushi/ripgrep"+readme = "README.md"+keywords = ["regex", "grep", "pcre", "backreference", "look"]+license = "Unlicense/MIT"++[dependencies]+grep-matcher = { version = "0.1.2", path = "../grep-matcher" }+pcre2 = "0.2.0"
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,39 @@+grep-pcre2+----------+The `grep-pcre2` crate provides an implementation of the `Matcher` trait from+the `grep-matcher` crate. This implementation permits PCRE2 to be used in the+`grep` crate for fast line oriented searching.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/grep-pcre2)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).++### Documentation++[https://docs.rs/grep-pcre2](https://docs.rs/grep-pcre2)++**NOTE:** You probably don't want to use this crate directly. Instead, you+should prefer the facade defined in the+[`grep`](https://docs.rs/grep)+crate.++If you're looking to just use PCRE2 from Rust, then you probably want the+[`pcre2`](https://docs.rs/pcre2)+crate, which provide high level safe bindings to PCRE2.++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+grep-pcre2 = "0.1"+```++and this to your crate root:++```rust+extern crate grep_pcre2;+```
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
@@ -0,0 +1,59 @@+use std::error;+use std::fmt;++/// An error that can occur in this crate.+///+/// Generally, this error corresponds to problems building a regular+/// expression, whether it's in parsing, compilation or a problem with+/// guaranteeing a configured optimization.+#[derive(Clone, Debug)]+pub struct Error {+ kind: ErrorKind,+}++impl Error {+ pub(crate) fn regex<E: error::Error>(err: E) -> Error {+ Error { kind: ErrorKind::Regex(err.to_string()) }+ }++ /// Return the kind of this error.+ pub fn kind(&self) -> &ErrorKind {+ &self.kind+ }+}++/// The kind of an error that can occur.+#[derive(Clone, Debug)]+pub enum ErrorKind {+ /// An error that occurred as a result of parsing a regular expression.+ /// This can be a syntax error or an error that results from attempting to+ /// compile a regular expression that is too big.+ ///+ /// The string here is the underlying error converted to a string.+ Regex(String),+ /// Hints that destructuring should not be exhaustive.+ ///+ /// This enum may grow additional variants, so this makes sure clients+ /// don't count on exhaustive matching. (Otherwise, adding a new variant+ /// could break existing code.)+ #[doc(hidden)]+ __Nonexhaustive,+}++impl error::Error for Error {+ fn description(&self) -> &str {+ match self.kind {+ ErrorKind::Regex(_) => "regex error",+ ErrorKind::__Nonexhaustive => unreachable!(),+ }+ }+}++impl fmt::Display for Error {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match self.kind {+ ErrorKind::Regex(ref s) => write!(f, "{}", s),+ ErrorKind::__Nonexhaustive => unreachable!(),+ }+ }+}
@@ -0,0 +1,16 @@+/*!+An implementation of `grep-matcher`'s `Matcher` trait for+[PCRE2](https://www.pcre.org/).+*/++#![deny(missing_docs)]++extern crate grep_matcher;+extern crate pcre2;++pub use error::{Error, ErrorKind};+pub use matcher::{RegexCaptures, RegexMatcher, RegexMatcherBuilder};+pub use pcre2::{is_jit_available, version};++mod error;+mod matcher;
@@ -0,0 +1,456 @@+use std::collections::HashMap;++use grep_matcher::{Captures, Match, Matcher};+use pcre2::bytes::{CaptureLocations, Regex, RegexBuilder};++use error::Error;++/// A builder for configuring the compilation of a PCRE2 regex.+#[derive(Clone, Debug)]+pub struct RegexMatcherBuilder {+ builder: RegexBuilder,+ case_smart: bool,+ word: bool,+}++impl RegexMatcherBuilder {+ /// Create a new matcher builder with a default configuration.+ pub fn new() -> RegexMatcherBuilder {+ RegexMatcherBuilder {+ builder: RegexBuilder::new(),+ case_smart: false,+ word: false,+ }+ }++ /// Compile the given pattern into a PCRE matcher using the current+ /// configuration.+ ///+ /// If there was a problem compiling the pattern, then an error is+ /// returned.+ pub fn build(&self, pattern: &str) -> Result<RegexMatcher, Error> {+ let mut builder = self.builder.clone();+ if self.case_smart && !has_uppercase_literal(pattern) {+ builder.caseless(true);+ }+ let res = if self.word {+ let pattern = format!(r"(?<!\w)(?:{})(?!\w)", pattern);+ builder.build(&pattern)+ } else {+ builder.build(pattern)+ };+ res.map_err(Error::regex).map(|regex| {+ let mut names = HashMap::new();+ for (i, name) in regex.capture_names().iter().enumerate() {+ if let Some(ref name) = *name {+ names.insert(name.to_string(), i);+ }+ }+ RegexMatcher { regex, names }+ })+ }++ /// Enables case insensitive matching.+ ///+ /// If the `utf` option is also set, then Unicode case folding is used+ /// to determine case insensitivity. When the `utf` option is not set,+ /// then only standard ASCII case insensitivity is considered.+ ///+ /// This option corresponds to the `i` flag.+ pub fn caseless(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.caseless(yes);+ self+ }++ /// Whether to enable "smart case" or not.+ ///+ /// When smart case is enabled, the builder will automatically enable+ /// case insensitive matching based on how the pattern is written. Namely,+ /// case insensitive mode is enabled when both of the following things+ /// are believed to be true:+ ///+ /// 1. The pattern contains at least one literal character. For example,+ /// `a\w` contains a literal (`a`) but `\w` does not.+ /// 2. Of the literals in the pattern, none of them are considered to be+ /// uppercase according to Unicode. For example, `foo\pL` has no+ /// uppercase literals but `Foo\pL` does.+ ///+ /// Note that the implementation of this is not perfect. Namely, `\p{Ll}`+ /// will prevent case insensitive matching even though it is part of a meta+ /// sequence. This bug will probably never be fixed.+ pub fn case_smart(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.case_smart = yes;+ self+ }++ /// Enables "dot all" matching.+ ///+ /// When enabled, the `.` metacharacter in the pattern matches any+ /// character, include `\n`. When disabled (the default), `.` will match+ /// any character except for `\n`.+ ///+ /// This option corresponds to the `s` flag.+ pub fn dotall(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.dotall(yes);+ self+ }++ /// Enable "extended" mode in the pattern, where whitespace is ignored.+ ///+ /// This option corresponds to the `x` flag.+ pub fn extended(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.extended(yes);+ self+ }++ /// Enable multiline matching mode.+ ///+ /// When enabled, the `^` and `$` anchors will match both at the beginning+ /// and end of a subject string, in addition to matching at the start of+ /// a line and the end of a line. When disabled, the `^` and `$` anchors+ /// will only match at the beginning and end of a subject string.+ ///+ /// This option corresponds to the `m` flag.+ pub fn multi_line(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.multi_line(yes);+ self+ }++ /// Enable matching of CRLF as a line terminator.+ ///+ /// When enabled, anchors such as `^` and `$` will match any of the+ /// following as a line terminator: `\r`, `\n` or `\r\n`.+ ///+ /// This is disabled by default, in which case, only `\n` is recognized as+ /// a line terminator.+ pub fn crlf(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.crlf(yes);+ self+ }++ /// Require that all matches occur on word boundaries.+ ///+ /// Enabling this option is subtly different than putting `\b` assertions+ /// on both sides of your pattern. In particular, a `\b` assertion requires+ /// that one side of it match a word character while the other match a+ /// non-word character. This option, in contrast, merely requires that+ /// one side match a non-word character.+ ///+ /// For example, `\b-2\b` will not match `foo -2 bar` since `-` is not a+ /// word character. However, `-2` with this `word` option enabled will+ /// match the `-2` in `foo -2 bar`.+ pub fn word(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.word = yes;+ self+ }++ /// Enable Unicode matching mode.+ ///+ /// When enabled, the following patterns become Unicode aware: `\b`, `\B`,+ /// `\d`, `\D`, `\s`, `\S`, `\w`, `\W`.+ ///+ /// When set, this implies UTF matching mode. It is not possible to enable+ /// Unicode matching mode without enabling UTF matching mode.+ ///+ /// This is disabled by default.+ pub fn ucp(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.ucp(yes);+ self+ }++ /// Enable UTF matching mode.+ ///+ /// When enabled, characters are treated as sequences of code units that+ /// make up a single codepoint instead of as single bytes. For example,+ /// this will cause `.` to match any single UTF-8 encoded codepoint, where+ /// as when this is disabled, `.` will any single byte (except for `\n` in+ /// both cases, unless "dot all" mode is enabled).+ ///+ /// Note that when UTF matching mode is enabled, every search performed+ /// will do a UTF-8 validation check, which can impact performance. The+ /// UTF-8 check can be disabled via the `disable_utf_check` option, but it+ /// is undefined behavior to enable UTF matching mode and search invalid+ /// UTF-8.+ ///+ /// This is disabled by default.+ pub fn utf(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.utf(yes);+ self+ }++ /// When UTF matching mode is enabled, this will disable the UTF checking+ /// that PCRE2 will normally perform automatically. If UTF matching mode+ /// is not enabled, then this has no effect.+ ///+ /// UTF checking is enabled by default when UTF matching mode is enabled.+ /// If UTF matching mode is enabled and UTF checking is enabled, then PCRE2+ /// will return an error if you attempt to search a subject string that is+ /// not valid UTF-8.+ ///+ /// # Safety+ ///+ /// It is undefined behavior to disable the UTF check in UTF matching mode+ /// and search a subject string that is not valid UTF-8. When the UTF check+ /// is disabled, callers must guarantee that the subject string is valid+ /// UTF-8.+ pub unsafe fn disable_utf_check(&mut self) -> &mut RegexMatcherBuilder {+ self.builder.disable_utf_check();+ self+ }++ /// Enable PCRE2's JIT and return an error if it's not available.+ ///+ /// This generally speeds up matching quite a bit. The downside is that it+ /// can increase the time it takes to compile a pattern.+ ///+ /// If the JIT isn't available or if JIT compilation returns an error, then+ /// regex compilation will fail with the corresponding error.+ ///+ /// This is disabled by default, and always overrides `jit_if_available`.+ pub fn jit(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.jit(yes);+ self+ }++ /// Enable PCRE2's JIT if it's available.+ ///+ /// This generally speeds up matching quite a bit. The downside is that it+ /// can increase the time it takes to compile a pattern.+ ///+ /// If the JIT isn't available or if JIT compilation returns an error,+ /// then a debug message with the error will be emitted and the regex will+ /// otherwise silently fall back to non-JIT matching.+ ///+ /// This is disabled by default, and always overrides `jit`.+ pub fn jit_if_available(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.builder.jit_if_available(yes);+ self+ }++ /// Set the maximum size of PCRE2's JIT stack, in bytes. If the JIT is+ /// not enabled, then this has no effect.+ ///+ /// When `None` is given, no custom JIT stack will be created, and instead,+ /// the default JIT stack is used. When the default is used, its maximum+ /// size is 32 KB.+ ///+ /// When this is set, then a new JIT stack will be created with the given+ /// maximum size as its limit.+ ///+ /// Increasing the stack size can be useful for larger regular expressions.+ ///+ /// By default, this is set to `None`.+ pub fn max_jit_stack_size(+ &mut self,+ bytes: Option<usize>,+ ) -> &mut RegexMatcherBuilder {+ self.builder.max_jit_stack_size(bytes);+ self+ }+}++/// An implementation of the `Matcher` trait using PCRE2.+#[derive(Clone, Debug)]+pub struct RegexMatcher {+ regex: Regex,+ names: HashMap<String, usize>,+}++impl RegexMatcher {+ /// Create a new matcher from the given pattern using the default+ /// configuration.+ pub fn new(pattern: &str) -> Result<RegexMatcher, Error> {+ RegexMatcherBuilder::new().build(pattern)+ }+}++impl Matcher for RegexMatcher {+ type Captures = RegexCaptures;+ type Error = Error;++ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, Error> {+ Ok(self+ .regex+ .find_at(haystack, at)+ .map_err(Error::regex)?+ .map(|m| Match::new(m.start(), m.end())))+ }++ fn new_captures(&self) -> Result<RegexCaptures, Error> {+ Ok(RegexCaptures::new(self.regex.capture_locations()))+ }++ fn capture_count(&self) -> usize {+ self.regex.captures_len()+ }++ fn capture_index(&self, name: &str) -> Option<usize> {+ self.names.get(name).map(|i| *i)+ }++ fn try_find_iter<F, E>(+ &self,+ haystack: &[u8],+ mut matched: F,+ ) -> Result<Result<(), E>, Error>+ where+ F: FnMut(Match) -> Result<bool, E>,+ {+ for result in self.regex.find_iter(haystack) {+ let m = result.map_err(Error::regex)?;+ match matched(Match::new(m.start(), m.end())) {+ Ok(true) => continue,+ Ok(false) => return Ok(Ok(())),+ Err(err) => return Ok(Err(err)),+ }+ }+ Ok(Ok(()))+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut RegexCaptures,+ ) -> Result<bool, Error> {+ Ok(self+ .regex+ .captures_read_at(&mut caps.locs, haystack, at)+ .map_err(Error::regex)?+ .is_some())+ }+}++/// Represents the match offsets of each capturing group in a match.+///+/// The first, or `0`th capture group, always corresponds to the entire match+/// and is guaranteed to be present when a match occurs. The next capture+/// group, at index `1`, corresponds to the first capturing group in the regex,+/// ordered by the position at which the left opening parenthesis occurs.+///+/// Note that not all capturing groups are guaranteed to be present in a match.+/// For example, in the regex, `(?P<foo>\w)|(?P<bar>\W)`, only one of `foo`+/// or `bar` will ever be set in any given match.+///+/// In order to access a capture group by name, you'll need to first find the+/// index of the group using the corresponding matcher's `capture_index`+/// method, and then use that index with `RegexCaptures::get`.+#[derive(Clone, Debug)]+pub struct RegexCaptures {+ /// Where the locations are stored.+ locs: CaptureLocations,+}++impl Captures for RegexCaptures {+ fn len(&self) -> usize {+ self.locs.len()+ }++ fn get(&self, i: usize) -> Option<Match> {+ self.locs.get(i).map(|(s, e)| Match::new(s, e))+ }+}++impl RegexCaptures {+ pub(crate) fn new(locs: CaptureLocations) -> RegexCaptures {+ RegexCaptures { locs }+ }+}++/// Determine whether the pattern contains an uppercase character which should+/// negate the effect of the smart-case option.+///+/// Ideally we would be able to check the AST in order to correctly handle+/// things like '\p{Ll}' and '\p{Lu}' (which should be treated as explicitly+/// cased), but PCRE doesn't expose enough details for that kind of analysis.+/// For now, our 'good enough' solution is to simply perform a semi-naïve+/// scan of the input pattern and ignore all characters following a '\'. The+/// This at least lets us support the most common cases, like 'foo\w' and+/// 'foo\S', in an intuitive manner.+fn has_uppercase_literal(pattern: &str) -> bool {+ let mut chars = pattern.chars();+ while let Some(c) = chars.next() {+ if c == '\\' {+ chars.next();+ } else if c.is_uppercase() {+ return true;+ }+ }+ false+}++#[cfg(test)]+mod tests {+ use super::*;+ use grep_matcher::{LineMatchKind, Matcher};++ // Test that enabling word matches does the right thing and demonstrate+ // the difference between it and surrounding the regex in `\b`.+ #[test]+ fn word() {+ let matcher =+ RegexMatcherBuilder::new().word(true).build(r"-2").unwrap();+ assert!(matcher.is_match(b"abc -2 foo").unwrap());++ let matcher =+ RegexMatcherBuilder::new().word(false).build(r"\b-2\b").unwrap();+ assert!(!matcher.is_match(b"abc -2 foo").unwrap());+ }++ // Test that enabling CRLF permits `$` to match at the end of a line.+ #[test]+ fn line_terminator_crlf() {+ // Test normal use of `$` with a `\n` line terminator.+ let matcher = RegexMatcherBuilder::new()+ .multi_line(true)+ .build(r"abc$")+ .unwrap();+ assert!(matcher.is_match(b"abc\n").unwrap());++ // Test that `$` doesn't match at `\r\n` boundary normally.+ let matcher = RegexMatcherBuilder::new()+ .multi_line(true)+ .build(r"abc$")+ .unwrap();+ assert!(!matcher.is_match(b"abc\r\n").unwrap());++ // Now check the CRLF handling.+ let matcher = RegexMatcherBuilder::new()+ .multi_line(true)+ .crlf(true)+ .build(r"abc$")+ .unwrap();+ assert!(matcher.is_match(b"abc\r\n").unwrap());+ }++ // Test that smart case works.+ #[test]+ fn case_smart() {+ let matcher =+ RegexMatcherBuilder::new().case_smart(true).build(r"abc").unwrap();+ assert!(matcher.is_match(b"ABC").unwrap());++ let matcher =+ RegexMatcherBuilder::new().case_smart(true).build(r"aBc").unwrap();+ assert!(!matcher.is_match(b"ABC").unwrap());+ }++ // Test that finding candidate lines works as expected.+ #[test]+ fn candidate_lines() {+ fn is_confirmed(m: LineMatchKind) -> bool {+ match m {+ LineMatchKind::Confirmed(_) => true,+ _ => false,+ }+ }++ let matcher = RegexMatcherBuilder::new().build(r"\wfoo\s").unwrap();+ let m = matcher.find_candidate_line(b"afoo ").unwrap().unwrap();+ assert!(is_confirmed(m));+ }+}
@@ -0,0 +1,31 @@+[package]+name = "grep-printer"+version = "0.1.3" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+An implementation of the grep crate's Sink trait that provides standard+printing of search results, similar to grep itself.+"""+documentation = "https://docs.rs/grep-printer"+homepage = "https://github.com/BurntSushi/ripgrep"+repository = "https://github.com/BurntSushi/ripgrep"+readme = "README.md"+keywords = ["grep", "pattern", "print", "printer", "sink"]+license = "Unlicense/MIT"++[features]+default = ["serde1"]+serde1 = ["base64", "serde", "serde_derive", "serde_json"]++[dependencies]+base64 = { version = "0.11.0", optional = true }+bstr = "0.2.0"+grep-matcher = { version = "0.1.2", path = "../grep-matcher" }+grep-searcher = { version = "0.1.4", path = "../grep-searcher" }+termcolor = "1.0.4"+serde = { version = "1.0.77", optional = true }+serde_derive = { version = "1.0.77", optional = true }+serde_json = { version = "1.0.27", optional = true }++[dev-dependencies]+grep-regex = { version = "0.1.3", path = "../grep-regex" }
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,35 @@+grep-printer+------------+Print results from line oriented searching in a human readable, aggregate or+JSON Lines format.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/grep-printer)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).++### Documentation++[https://docs.rs/grep-printer](https://docs.rs/grep-printer)++**NOTE:** You probably don't want to use this crate directly. Instead, you+should prefer the facade defined in the+[`grep`](https://docs.rs/grep)+crate.+++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+grep-printer = "0.1"+```++and this to your crate root:++```rust+extern crate grep_printer;+```
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
@@ -0,0 +1,393 @@+use std::error;+use std::fmt;+use std::str::FromStr;++use termcolor::{Color, ColorSpec, ParseColorError};++/// Returns a default set of color specifications.+///+/// This may change over time, but the color choices are meant to be fairly+/// conservative that work across terminal themes.+///+/// Additional color specifications can be added to the list returned. More+/// recently added specifications override previously added specifications.+pub fn default_color_specs() -> Vec<UserColorSpec> {+ vec![+ #[cfg(unix)]+ "path:fg:magenta".parse().unwrap(),+ #[cfg(windows)]+ "path:fg:cyan".parse().unwrap(),+ "line:fg:green".parse().unwrap(),+ "match:fg:red".parse().unwrap(),+ "match:style:bold".parse().unwrap(),+ ]+}++/// An error that can occur when parsing color specifications.+#[derive(Clone, Debug, Eq, PartialEq)]+pub enum ColorError {+ /// This occurs when an unrecognized output type is used.+ UnrecognizedOutType(String),+ /// This occurs when an unrecognized spec type is used.+ UnrecognizedSpecType(String),+ /// This occurs when an unrecognized color name is used.+ UnrecognizedColor(String, String),+ /// This occurs when an unrecognized style attribute is used.+ UnrecognizedStyle(String),+ /// This occurs when the format of a color specification is invalid.+ InvalidFormat(String),+}++impl error::Error for ColorError {+ fn description(&self) -> &str {+ match *self {+ ColorError::UnrecognizedOutType(_) => "unrecognized output type",+ ColorError::UnrecognizedSpecType(_) => "unrecognized spec type",+ ColorError::UnrecognizedColor(_, _) => "unrecognized color name",+ ColorError::UnrecognizedStyle(_) => "unrecognized style attribute",+ ColorError::InvalidFormat(_) => "invalid color spec",+ }+ }+}++impl ColorError {+ fn from_parse_error(err: ParseColorError) -> ColorError {+ ColorError::UnrecognizedColor(+ err.invalid().to_string(),+ err.to_string(),+ )+ }+}++impl fmt::Display for ColorError {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match *self {+ ColorError::UnrecognizedOutType(ref name) => write!(+ f,+ "unrecognized output type '{}'. Choose from: \+ path, line, column, match.",+ name,+ ),+ ColorError::UnrecognizedSpecType(ref name) => write!(+ f,+ "unrecognized spec type '{}'. Choose from: \+ fg, bg, style, none.",+ name,+ ),+ ColorError::UnrecognizedColor(_, ref msg) => write!(f, "{}", msg),+ ColorError::UnrecognizedStyle(ref name) => write!(+ f,+ "unrecognized style attribute '{}'. Choose from: \+ nobold, bold, nointense, intense, nounderline, \+ underline.",+ name,+ ),+ ColorError::InvalidFormat(ref original) => write!(+ f,+ "invalid color spec format: '{}'. Valid format \+ is '(path|line|column|match):(fg|bg|style):(value)'.",+ original,+ ),+ }+ }+}++/// A merged set of color specifications.+///+/// This set of color specifications represents the various color types that+/// are supported by the printers in this crate. A set of color specifications+/// can be created from a sequence of+/// [`UserColorSpec`s](struct.UserColorSpec.html).+#[derive(Clone, Debug, Default, Eq, PartialEq)]+pub struct ColorSpecs {+ path: ColorSpec,+ line: ColorSpec,+ column: ColorSpec,+ matched: ColorSpec,+}++/// A single color specification provided by the user.+///+/// ## Format+///+/// The format of a `Spec` is a triple: `{type}:{attribute}:{value}`. Each+/// component is defined as follows:+///+/// * `{type}` can be one of `path`, `line`, `column` or `match`.+/// * `{attribute}` can be one of `fg`, `bg` or `style`. `{attribute}` may also+/// be the special value `none`, in which case, `{value}` can be omitted.+/// * `{value}` is either a color name (for `fg`/`bg`) or a style instruction.+///+/// `{type}` controls which part of the output should be styled.+///+/// When `{attribute}` is `none`, then this should cause any existing style+/// settings to be cleared for the specified `type`.+///+/// `{value}` should be a color when `{attribute}` is `fg` or `bg`, or it+/// should be a style instruction when `{attribute}` is `style`. When+/// `{attribute}` is `none`, `{value}` must be omitted.+///+/// Valid colors are `black`, `blue`, `green`, `red`, `cyan`, `magenta`,+/// `yellow`, `white`. Extended colors can also be specified, and are formatted+/// as `x` (for 256-bit colors) or `x,x,x` (for 24-bit true color), where+/// `x` is a number between 0 and 255 inclusive. `x` may be given as a normal+/// decimal number of a hexadecimal number, where the latter is prefixed by+/// `0x`.+///+/// Valid style instructions are `nobold`, `bold`, `intense`, `nointense`,+/// `underline`, `nounderline`.+///+/// ## Example+///+/// The standard way to build a `UserColorSpec` is to parse it from a string.+/// Once multiple `UserColorSpec`s have been constructed, they can be provided+/// to the standard printer where they will automatically be applied to the+/// output.+///+/// A `UserColorSpec` can also be converted to a `termcolor::ColorSpec`:+///+/// ```rust+/// extern crate grep_printer;+/// extern crate termcolor;+///+/// # fn main() {+/// use termcolor::{Color, ColorSpec};+/// use grep_printer::UserColorSpec;+///+/// let user_spec1: UserColorSpec = "path:fg:blue".parse().unwrap();+/// let user_spec2: UserColorSpec = "match:bg:0xff,0x7f,0x00".parse().unwrap();+///+/// let spec1 = user_spec1.to_color_spec();+/// let spec2 = user_spec2.to_color_spec();+///+/// assert_eq!(spec1.fg(), Some(&Color::Blue));+/// assert_eq!(spec2.bg(), Some(&Color::Rgb(0xFF, 0x7F, 0x00)));+/// # }+/// ```+#[derive(Clone, Debug, Eq, PartialEq)]+pub struct UserColorSpec {+ ty: OutType,+ value: SpecValue,+}++impl UserColorSpec {+ /// Convert this user provided color specification to a specification that+ /// can be used with `termcolor`. This drops the type of this specification+ /// (where the type indicates where the color is applied in the standard+ /// printer, e.g., to the file path or the line numbers, etc.).+ pub fn to_color_spec(&self) -> ColorSpec {+ let mut spec = ColorSpec::default();+ self.value.merge_into(&mut spec);+ spec+ }+}++/// The actual value given by the specification.+#[derive(Clone, Debug, Eq, PartialEq)]+enum SpecValue {+ None,+ Fg(Color),+ Bg(Color),+ Style(Style),+}++/// The set of configurable portions of ripgrep's output.+#[derive(Clone, Debug, Eq, PartialEq)]+enum OutType {+ Path,+ Line,+ Column,+ Match,+}++/// The specification type.+#[derive(Clone, Debug, Eq, PartialEq)]+enum SpecType {+ Fg,+ Bg,+ Style,+ None,+}++/// The set of available styles for use in the terminal.+#[derive(Clone, Debug, Eq, PartialEq)]+enum Style {+ Bold,+ NoBold,+ Intense,+ NoIntense,+ Underline,+ NoUnderline,+}++impl ColorSpecs {+ /// Create color specifications from a list of user supplied+ /// specifications.+ pub fn new(specs: &[UserColorSpec]) -> ColorSpecs {+ let mut merged = ColorSpecs::default();+ for spec in specs {+ match spec.ty {+ OutType::Path => spec.merge_into(&mut merged.path),+ OutType::Line => spec.merge_into(&mut merged.line),+ OutType::Column => spec.merge_into(&mut merged.column),+ OutType::Match => spec.merge_into(&mut merged.matched),+ }+ }+ merged+ }++ /// Create a default set of specifications that have color.+ ///+ /// This is distinct from `ColorSpecs`'s `Default` implementation in that+ /// this provides a set of default color choices, where as the `Default`+ /// implementation provides no color choices.+ pub fn default_with_color() -> ColorSpecs {+ ColorSpecs::new(&default_color_specs())+ }++ /// Return the color specification for coloring file paths.+ pub fn path(&self) -> &ColorSpec {+ &self.path+ }++ /// Return the color specification for coloring line numbers.+ pub fn line(&self) -> &ColorSpec {+ &self.line+ }++ /// Return the color specification for coloring column numbers.+ pub fn column(&self) -> &ColorSpec {+ &self.column+ }++ /// Return the color specification for coloring matched text.+ pub fn matched(&self) -> &ColorSpec {+ &self.matched+ }+}++impl UserColorSpec {+ /// Merge this spec into the given color specification.+ fn merge_into(&self, cspec: &mut ColorSpec) {+ self.value.merge_into(cspec);+ }+}++impl SpecValue {+ /// Merge this spec value into the given color specification.+ fn merge_into(&self, cspec: &mut ColorSpec) {+ match *self {+ SpecValue::None => cspec.clear(),+ SpecValue::Fg(ref color) => {+ cspec.set_fg(Some(color.clone()));+ }+ SpecValue::Bg(ref color) => {+ cspec.set_bg(Some(color.clone()));+ }+ SpecValue::Style(ref style) => match *style {+ Style::Bold => {+ cspec.set_bold(true);+ }+ Style::NoBold => {+ cspec.set_bold(false);+ }+ Style::Intense => {+ cspec.set_intense(true);+ }+ Style::NoIntense => {+ cspec.set_intense(false);+ }+ Style::Underline => {+ cspec.set_underline(true);+ }+ Style::NoUnderline => {+ cspec.set_underline(false);+ }+ },+ }+ }+}++impl FromStr for UserColorSpec {+ type Err = ColorError;++ fn from_str(s: &str) -> Result<UserColorSpec, ColorError> {+ let pieces: Vec<&str> = s.split(':').collect();+ if pieces.len() <= 1 || pieces.len() > 3 {+ return Err(ColorError::InvalidFormat(s.to_string()));+ }+ let otype: OutType = pieces[0].parse()?;+ match pieces[1].parse()? {+ SpecType::None => {+ Ok(UserColorSpec { ty: otype, value: SpecValue::None })+ }+ SpecType::Style => {+ if pieces.len() < 3 {+ return Err(ColorError::InvalidFormat(s.to_string()));+ }+ let style: Style = pieces[2].parse()?;+ Ok(UserColorSpec { ty: otype, value: SpecValue::Style(style) })+ }+ SpecType::Fg => {+ if pieces.len() < 3 {+ return Err(ColorError::InvalidFormat(s.to_string()));+ }+ let color: Color =+ pieces[2].parse().map_err(ColorError::from_parse_error)?;+ Ok(UserColorSpec { ty: otype, value: SpecValue::Fg(color) })+ }+ SpecType::Bg => {+ if pieces.len() < 3 {+ return Err(ColorError::InvalidFormat(s.to_string()));+ }+ let color: Color =+ pieces[2].parse().map_err(ColorError::from_parse_error)?;+ Ok(UserColorSpec { ty: otype, value: SpecValue::Bg(color) })+ }+ }+ }+}++impl FromStr for OutType {+ type Err = ColorError;++ fn from_str(s: &str) -> Result<OutType, ColorError> {+ match &*s.to_lowercase() {+ "path" => Ok(OutType::Path),+ "line" => Ok(OutType::Line),+ "column" => Ok(OutType::Column),+ "match" => Ok(OutType::Match),+ _ => Err(ColorError::UnrecognizedOutType(s.to_string())),+ }+ }+}++impl FromStr for SpecType {+ type Err = ColorError;++ fn from_str(s: &str) -> Result<SpecType, ColorError> {+ match &*s.to_lowercase() {+ "fg" => Ok(SpecType::Fg),+ "bg" => Ok(SpecType::Bg),+ "style" => Ok(SpecType::Style),+ "none" => Ok(SpecType::None),+ _ => Err(ColorError::UnrecognizedSpecType(s.to_string())),+ }+ }+}++impl FromStr for Style {+ type Err = ColorError;++ fn from_str(s: &str) -> Result<Style, ColorError> {+ match &*s.to_lowercase() {+ "bold" => Ok(Style::Bold),+ "nobold" => Ok(Style::NoBold),+ "intense" => Ok(Style::Intense),+ "nointense" => Ok(Style::NoIntense),+ "underline" => Ok(Style::Underline),+ "nounderline" => Ok(Style::NoUnderline),+ _ => Err(ColorError::UnrecognizedStyle(s.to_string())),+ }+ }+}
@@ -0,0 +1,90 @@+use std::io::{self, Write};++use termcolor::{ColorSpec, WriteColor};++/// A writer that counts the number of bytes that have been successfully+/// written.+#[derive(Clone, Debug)]+pub struct CounterWriter<W> {+ wtr: W,+ count: u64,+ total_count: u64,+}++impl<W: Write> CounterWriter<W> {+ pub fn new(wtr: W) -> CounterWriter<W> {+ CounterWriter { wtr: wtr, count: 0, total_count: 0 }+ }+}++impl<W> CounterWriter<W> {+ /// Returns the total number of bytes written since construction or the+ /// last time `reset` was called.+ pub fn count(&self) -> u64 {+ self.count+ }++ /// Returns the total number of bytes written since construction.+ pub fn total_count(&self) -> u64 {+ self.total_count + self.count+ }++ /// Resets the number of bytes written to `0`.+ pub fn reset_count(&mut self) {+ self.total_count += self.count;+ self.count = 0;+ }++ /// Clear resets all counting related state for this writer.+ ///+ /// After this call, the total count of bytes written to the underlying+ /// writer is erased and reset.+ #[allow(dead_code)]+ pub fn clear(&mut self) {+ self.count = 0;+ self.total_count = 0;+ }++ #[allow(dead_code)]+ pub fn get_ref(&self) -> &W {+ &self.wtr+ }++ pub fn get_mut(&mut self) -> &mut W {+ &mut self.wtr+ }++ pub fn into_inner(self) -> W {+ self.wtr+ }+}++impl<W: Write> Write for CounterWriter<W> {+ fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {+ let n = self.wtr.write(buf)?;+ self.count += n as u64;+ Ok(n)+ }++ fn flush(&mut self) -> Result<(), io::Error> {+ self.wtr.flush()+ }+}++impl<W: WriteColor> WriteColor for CounterWriter<W> {+ fn supports_color(&self) -> bool {+ self.wtr.supports_color()+ }++ fn set_color(&mut self, spec: &ColorSpec) -> io::Result<()> {+ self.wtr.set_color(spec)+ }++ fn reset(&mut self) -> io::Result<()> {+ self.wtr.reset()+ }++ fn is_synchronous(&self) -> bool {+ self.wtr.is_synchronous()+ }+}
@@ -0,0 +1,936 @@+use std::io::{self, Write};+use std::path::Path;+use std::time::Instant;++use grep_matcher::{Match, Matcher};+use grep_searcher::{+ Searcher, Sink, SinkContext, SinkContextKind, SinkError, SinkFinish,+ SinkMatch,+};+use serde_json as json;++use counter::CounterWriter;+use jsont;+use stats::Stats;++/// The configuration for the JSON printer.+///+/// This is manipulated by the JSONBuilder and then referenced by the actual+/// implementation. Once a printer is build, the configuration is frozen and+/// cannot changed.+#[derive(Debug, Clone)]+struct Config {+ pretty: bool,+ max_matches: Option<u64>,+ always_begin_end: bool,+}++impl Default for Config {+ fn default() -> Config {+ Config { pretty: false, max_matches: None, always_begin_end: false }+ }+}++/// A builder for a JSON lines printer.+///+/// The builder permits configuring how the printer behaves. The JSON printer+/// has fewer configuration options than the standard printer because it is+/// a structured format, and the printer always attempts to find the most+/// information possible.+///+/// Some configuration options, such as whether line numbers are included or+/// whether contextual lines are shown, are drawn directly from the+/// `grep_searcher::Searcher`'s configuration.+///+/// Once a `JSON` printer is built, its configuration cannot be changed.+#[derive(Clone, Debug)]+pub struct JSONBuilder {+ config: Config,+}++impl JSONBuilder {+ /// Return a new builder for configuring the JSON printer.+ pub fn new() -> JSONBuilder {+ JSONBuilder { config: Config::default() }+ }++ /// Create a JSON printer that writes results to the given writer.+ pub fn build<W: io::Write>(&self, wtr: W) -> JSON<W> {+ JSON {+ config: self.config.clone(),+ wtr: CounterWriter::new(wtr),+ matches: vec![],+ }+ }++ /// Print JSON in a pretty printed format.+ ///+ /// Enabling this will no longer produce a "JSON lines" format, in that+ /// each JSON object printed may span multiple lines.+ ///+ /// This is disabled by default.+ pub fn pretty(&mut self, yes: bool) -> &mut JSONBuilder {+ self.config.pretty = yes;+ self+ }++ /// Set the maximum amount of matches that are printed.+ ///+ /// If multi line search is enabled and a match spans multiple lines, then+ /// that match is counted exactly once for the purposes of enforcing this+ /// limit, regardless of how many lines it spans.+ pub fn max_matches(&mut self, limit: Option<u64>) -> &mut JSONBuilder {+ self.config.max_matches = limit;+ self+ }++ /// When enabled, the `begin` and `end` messages are always emitted, even+ /// when no match is found.+ ///+ /// When disabled, the `begin` and `end` messages are only shown if there+ /// is at least one `match` or `context` message.+ ///+ /// This is disabled by default.+ pub fn always_begin_end(&mut self, yes: bool) -> &mut JSONBuilder {+ self.config.always_begin_end = yes;+ self+ }+}++/// The JSON printer, which emits results in a JSON lines format.+///+/// This type is generic over `W`, which represents any implementation of+/// the standard library `io::Write` trait.+///+/// # Format+///+/// This section describes the JSON format used by this printer.+///+/// To skip the rigamarole, take a look at the+/// [example](#example)+/// at the end.+///+/// ## Overview+///+/// The format of this printer is the [JSON Lines](http://jsonlines.org/)+/// format. Specifically, this printer emits a sequence of messages, where+/// each message is encoded as a single JSON value on a single line. There are+/// four different types of messages (and this number may expand over time):+///+/// * **begin** - A message that indicates a file is being searched.+/// * **end** - A message the indicates a file is done being searched. This+/// message also include summary statistics about the search.+/// * **match** - A message that indicates a match was found. This includes+/// the text and offsets of the match.+/// * **context** - A message that indicates a contextual line was found.+/// This includes the text of the line, along with any match information if+/// the search was inverted.+///+/// Every message is encoded in the same envelope format, which includes a tag+/// indicating the message type along with an object for the payload:+///+/// ```json+/// {+/// "type": "{begin|end|match|context}",+/// "data": { ... }+/// }+/// ```+///+/// The message itself is encoded in the envelope's `data` key.+///+/// ## Text encoding+///+/// Before describing each message format, we first must briefly discuss text+/// encoding, since it factors into every type of message. In particular, JSON+/// may only be encoded in UTF-8, UTF-16 or UTF-32. For the purposes of this+/// printer, we need only worry about UTF-8. The problem here is that searching+/// is not limited to UTF-8 exclusively, which in turn implies that matches+/// may be reported that contain invalid UTF-8. Moreover, this printer may+/// also print file paths, and the encoding of file paths is itself not+/// guarnateed to be valid UTF-8. Therefore, this printer must deal with the+/// presence of invalid UTF-8 somehow. The printer could silently ignore such+/// things completely, or even lossily transcode invalid UTF-8 to valid UTF-8+/// by replacing all invalid sequences with the Unicode replacement character.+/// However, this would prevent consumers of this format from accessing the+/// original data in a non-lossy way.+///+/// Therefore, this printer will emit valid UTF-8 encoded bytes as normal+/// JSON strings and otherwise base64 encode data that isn't valid UTF-8. To+/// communicate whether this process occurs or not, strings are keyed by the+/// name `text` where as arbitrary bytes are keyed by `bytes`.+///+/// For example, when a path is included in a message, it is formatted like so,+/// if and only if the path is valid UTF-8:+///+/// ```json+/// {+/// "path": {+/// "text": "/home/ubuntu/lib.rs"+/// }+/// }+/// ```+///+/// If instead our path was `/home/ubuntu/lib\xFF.rs`, where the `\xFF` byte+/// makes it invalid UTF-8, the path would instead be encoded like so:+///+/// ```json+/// {+/// "path": {+/// "bytes": "L2hvbWUvdWJ1bnR1L2xpYv8ucnM="+/// }+/// }+/// ```+///+/// This same representation is used for reporting matches as well.+///+/// The printer guarantees that the `text` field is used whenever the+/// underlying bytes are valid UTF-8.+///+/// ## Wire format+///+/// This section documents the wire format emitted by this printer, starting+/// with the four types of messages.+///+/// Each message has its own format, and is contained inside an envelope that+/// indicates the type of message. The envelope has these fields:+///+/// * **type** - A string indicating the type of this message. It may be one+/// of four possible strings: `begin`, `end`, `match` or `context`. This+/// list may expand over time.+/// * **data** - The actual message data. The format of this field depends on+/// the value of `type`. The possible message formats are+/// [`begin`](#message-begin),+/// [`end`](#message-end),+/// [`match`](#message-match),+/// [`context`](#message-context).+///+/// #### Message: **begin**+///+/// This message indicates that a search has begun. It has these fields:+///+/// * **path** - An+/// [arbitrary data object](#object-arbitrary-data)+/// representing the file path corresponding to the search, if one is+/// present. If no file path is available, then this field is `null`.+///+/// #### Message: **end**+///+/// This message indicates that a search has finished. It has these fields:+///+/// * **path** - An+/// [arbitrary data object](#object-arbitrary-data)+/// representing the file path corresponding to the search, if one is+/// present. If no file path is available, then this field is `null`.+/// * **binary_offset** - The absolute offset in the data searched+/// corresponding to the place at which binary data was detected. If no+/// binary data was detected (or if binary detection was disabled), then this+/// field is `null`.+/// * **stats** - A [`stats` object](#object-stats) that contains summary+/// statistics for the previous search.+///+/// #### Message: **match**+///+/// This message indicates that a match has been found. A match generally+/// corresponds to a single line of text, although it may correspond to+/// multiple lines if the search can emit matches over multiple lines. It+/// has these fields:+///+/// * **path** - An+/// [arbitrary data object](#object-arbitrary-data)+/// representing the file path corresponding to the search, if one is+/// present. If no file path is available, then this field is `null`.+/// * **lines** - An+/// [arbitrary data object](#object-arbitrary-data)+/// representing one or more lines contained in this match.+/// * **line_number** - If the searcher has been configured to report line+/// numbers, then this corresponds to the line number of the first line+/// in `lines`. If no line numbers are available, then this is `null`.+/// * **absolute_offset** - The absolute byte offset corresponding to the start+/// of `lines` in the data being searched.+/// * **submatches** - An array of [`submatch` objects](#object-submatch)+/// corresponding to matches in `lines`. The offsets included in each+/// `submatch` correspond to byte offsets into `lines`. (If `lines` is base64+/// encoded, then the byte offsets correspond to the data after base64+/// decoding.) The `submatch` objects are guaranteed to be sorted by their+/// starting offsets. Note that it is possible for this array to be empty,+/// for example, when searching reports inverted matches.+///+/// #### Message: **context**+///+/// This message indicates that a contextual line has been found. A contextual+/// line is a line that doesn't contain a match, but is generally adjacent to+/// a line that does contain a match. The precise way in which contextual lines+/// are reported is determined by the searcher. It has these fields, which are+/// exactly the same fields found in a [`match`](#message-match):+///+/// * **path** - An+/// [arbitrary data object](#object-arbitrary-data)+/// representing the file path corresponding to the search, if one is+/// present. If no file path is available, then this field is `null`.+/// * **lines** - An+/// [arbitrary data object](#object-arbitrary-data)+/// representing one or more lines contained in this context. This includes+/// line terminators, if they're present.+/// * **line_number** - If the searcher has been configured to report line+/// numbers, then this corresponds to the line number of the first line+/// in `lines`. If no line numbers are available, then this is `null`.+/// * **absolute_offset** - The absolute byte offset corresponding to the start+/// of `lines` in the data being searched.+/// * **submatches** - An array of [`submatch` objects](#object-submatch)+/// corresponding to matches in `lines`. The offsets included in each+/// `submatch` correspond to byte offsets into `lines`. (If `lines` is base64+/// encoded, then the byte offsets correspond to the data after base64+/// decoding.) The `submatch` objects are guaranteed to be sorted by+/// their starting offsets. Note that it is possible for this array to be+/// non-empty, for example, when searching reports inverted matches such that+/// the original matcher could match things in the contextual lines.+///+/// #### Object: **submatch**+///+/// This object describes submatches found within `match` or `context`+/// messages. The `start` and `end` fields indicate the half-open interval on+/// which the match occurs (`start` is included, but `end` is not). It is+/// guaranteed that `start <= end`. It has these fields:+///+/// * **match** - An+/// [arbitrary data object](#object-arbitrary-data)+/// corresponding to the text in this submatch.+/// * **start** - A byte offset indicating the start of this match. This offset+/// is generally reported in terms of the parent object's data. For example,+/// the `lines` field in the+/// [`match`](#message-match) or [`context`](#message-context)+/// messages.+/// * **end** - A byte offset indicating the end of this match. This offset+/// is generally reported in terms of the parent object's data. For example,+/// the `lines` field in the+/// [`match`](#message-match) or [`context`](#message-context)+/// messages.+///+/// #### Object: **stats**+///+/// This object is included in messages and contains summary statistics about+/// a search. It has these fields:+///+/// * **elapsed** - A [`duration` object](#object-duration) describing the+/// length of time that elapsed while performing the search.+/// * **searches** - The number of searches that have run. For this printer,+/// this value is always `1`. (Implementations may emit additional message+/// types that use this same `stats` object that represents summary+/// statistics over multiple searches.)+/// * **searches_with_match** - The number of searches that have run that have+/// found at least one match. This is never more than `searches`.+/// * **bytes_searched** - The total number of bytes that have been searched.+/// * **bytes_printed** - The total number of bytes that have been printed.+/// This includes everything emitted by this printer.+/// * **matched_lines** - The total number of lines that participated in a+/// match. When matches may contain multiple lines, then this includes every+/// line that is part of every match.+/// * **matches** - The total number of matches. There may be multiple matches+/// per line. When matches may contain multiple lines, each match is counted+/// only once, regardless of how many lines it spans.+///+/// #### Object: **duration**+///+/// This object includes a few fields for describing a duration. Two of its+/// fields, `secs` and `nanos`, can be combined to give nanosecond precision+/// on systems that support it. It has these fields:+///+/// * **secs** - A whole number of seconds indicating the length of this+/// duration.+/// * **nanos** - A fractional part of this duration represent by nanoseconds.+/// If nanosecond precision isn't supported, then this is typically rounded+/// up to the nearest number of nanoseconds.+/// * **human** - A human readable string describing the length of the+/// duration. The format of the string is itself unspecified.+///+/// #### Object: **arbitrary data**+///+/// This object is used whenever arbitrary data needs to be represented as a+/// JSON value. This object contains two fields, where generally only one of+/// the fields is present:+///+/// * **text** - A normal JSON string that is UTF-8 encoded. This field is+/// populated if and only if the underlying data is valid UTF-8.+/// * **bytes** - A normal JSON string that is a base64 encoding of the+/// underlying bytes.+///+/// More information on the motivation for this representation can be seen in+/// the section [text encoding](#text-encoding) above.+///+/// ## Example+///+/// This section shows a small example that includes all message types.+///+/// Here's the file we want to search, located at `/home/andrew/sherlock`:+///+/// ```text+/// For the Doctor Watsons of this world, as opposed to the Sherlock+/// Holmeses, success in the province of detective work must always+/// be, to a very large extent, the result of luck. Sherlock Holmes+/// can extract a clew from a wisp of straw or a flake of cigar ash;+/// but Doctor Watson has to have it taken out for him and dusted,+/// and exhibited clearly, with a label attached.+/// ```+///+/// Searching for `Watson` with a `before_context` of `1` with line numbers+/// enabled shows something like this using the standard printer:+///+/// ```text+/// sherlock:1:For the Doctor Watsons of this world, as opposed to the Sherlock+/// --+/// sherlock-4-can extract a clew from a wisp of straw or a flake of cigar ash;+/// sherlock:5:but Doctor Watson has to have it taken out for him and dusted,+/// ```+///+/// Here's what the same search looks like using the JSON wire format described+/// above, where in we show semi-prettified JSON (instead of a strict JSON+/// Lines format), for illustrative purposes:+///+/// ```json+/// {+/// "type": "begin",+/// "data": {+/// "path": {"text": "/home/andrew/sherlock"}}+/// }+/// }+/// {+/// "type": "match",+/// "data": {+/// "path": {"text": "/home/andrew/sherlock"},+/// "lines": {"text": "For the Doctor Watsons of this world, as opposed to the Sherlock\n"},+/// "line_number": 1,+/// "absolute_offset": 0,+/// "submatches": [+/// {"match": {"text": "Watson"}, "start": 15, "end": 21}+/// ]+/// }+/// }+/// {+/// "type": "context",+/// "data": {+/// "path": {"text": "/home/andrew/sherlock"},+/// "lines": {"text": "can extract a clew from a wisp of straw or a flake of cigar ash;\n"},+/// "line_number": 4,+/// "absolute_offset": 193,+/// "submatches": []+/// }+/// }+/// {+/// "type": "match",+/// "data": {+/// "path": {"text": "/home/andrew/sherlock"},+/// "lines": {"text": "but Doctor Watson has to have it taken out for him and dusted,\n"},+/// "line_number": 5,+/// "absolute_offset": 258,+/// "submatches": [+/// {"match": {"text": "Watson"}, "start": 11, "end": 17}+/// ]+/// }+/// }+/// {+/// "type": "end",+/// "data": {+/// "path": {"text": "/home/andrew/sherlock"},+/// "binary_offset": null,+/// "stats": {+/// "elapsed": {"secs": 0, "nanos": 36296, "human": "0.0000s"},+/// "searches": 1,+/// "searches_with_match": 1,+/// "bytes_searched": 367,+/// "bytes_printed": 1151,+/// "matched_lines": 2,+/// "matches": 2+/// }+/// }+/// }+/// ```+#[derive(Debug)]+pub struct JSON<W> {+ config: Config,+ wtr: CounterWriter<W>,+ matches: Vec<Match>,+}++impl<W: io::Write> JSON<W> {+ /// Return a JSON lines printer with a default configuration that writes+ /// matches to the given writer.+ pub fn new(wtr: W) -> JSON<W> {+ JSONBuilder::new().build(wtr)+ }++ /// Return an implementation of `Sink` for the JSON printer.+ ///+ /// This does not associate the printer with a file path, which means this+ /// implementation will never print a file path along with the matches.+ pub fn sink<'s, M: Matcher>(+ &'s mut self,+ matcher: M,+ ) -> JSONSink<'static, 's, M, W> {+ JSONSink {+ matcher: matcher,+ json: self,+ path: None,+ start_time: Instant::now(),+ match_count: 0,+ after_context_remaining: 0,+ binary_byte_offset: None,+ begin_printed: false,+ stats: Stats::new(),+ }+ }++ /// Return an implementation of `Sink` associated with a file path.+ ///+ /// When the printer is associated with a path, then it may, depending on+ /// its configuration, print the path along with the matches found.+ pub fn sink_with_path<'p, 's, M, P>(+ &'s mut self,+ matcher: M,+ path: &'p P,+ ) -> JSONSink<'p, 's, M, W>+ where+ M: Matcher,+ P: ?Sized + AsRef<Path>,+ {+ JSONSink {+ matcher: matcher,+ json: self,+ path: Some(path.as_ref()),+ start_time: Instant::now(),+ match_count: 0,+ after_context_remaining: 0,+ binary_byte_offset: None,+ begin_printed: false,+ stats: Stats::new(),+ }+ }++ /// Write the given message followed by a new line. The new line is+ /// determined from the configuration of the given searcher.+ fn write_message(&mut self, message: &jsont::Message) -> io::Result<()> {+ if self.config.pretty {+ json::to_writer_pretty(&mut self.wtr, message)?;+ } else {+ json::to_writer(&mut self.wtr, message)?;+ }+ self.wtr.write(&[b'\n'])?;+ Ok(())+ }+}++impl<W> JSON<W> {+ /// Returns true if and only if this printer has written at least one byte+ /// to the underlying writer during any of the previous searches.+ pub fn has_written(&self) -> bool {+ self.wtr.total_count() > 0+ }++ /// Return a mutable reference to the underlying writer.+ pub fn get_mut(&mut self) -> &mut W {+ self.wtr.get_mut()+ }++ /// Consume this printer and return back ownership of the underlying+ /// writer.+ pub fn into_inner(self) -> W {+ self.wtr.into_inner()+ }+}++/// An implementation of `Sink` associated with a matcher and an optional file+/// path for the JSON printer.+///+/// This type is generic over a few type parameters:+///+/// * `'p` refers to the lifetime of the file path, if one is provided. When+/// no file path is given, then this is `'static`.+/// * `'s` refers to the lifetime of the+/// [`JSON`](struct.JSON.html)+/// printer that this type borrows.+/// * `M` refers to the type of matcher used by+/// `grep_searcher::Searcher` that is reporting results to this sink.+/// * `W` refers to the underlying writer that this printer is writing its+/// output to.+#[derive(Debug)]+pub struct JSONSink<'p, 's, M: Matcher, W: 's> {+ matcher: M,+ json: &'s mut JSON<W>,+ path: Option<&'p Path>,+ start_time: Instant,+ match_count: u64,+ after_context_remaining: u64,+ binary_byte_offset: Option<u64>,+ begin_printed: bool,+ stats: Stats,+}++impl<'p, 's, M: Matcher, W: io::Write> JSONSink<'p, 's, M, W> {+ /// Returns true if and only if this printer received a match in the+ /// previous search.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search.+ pub fn has_match(&self) -> bool {+ self.match_count > 0+ }++ /// Return the total number of matches reported to this sink.+ ///+ /// This corresponds to the number of times `Sink::matched` is called.+ pub fn match_count(&self) -> u64 {+ self.match_count+ }++ /// If binary data was found in the previous search, this returns the+ /// offset at which the binary data was first detected.+ ///+ /// The offset returned is an absolute offset relative to the entire+ /// set of bytes searched.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search. e.g., If the search prior to the previous search found binary+ /// data but the previous search found no binary data, then this will+ /// return `None`.+ pub fn binary_byte_offset(&self) -> Option<u64> {+ self.binary_byte_offset+ }++ /// Return a reference to the stats produced by the printer for all+ /// searches executed on this sink.+ pub fn stats(&self) -> &Stats {+ &self.stats+ }++ /// Execute the matcher over the given bytes and record the match+ /// locations if the current configuration demands match granularity.+ fn record_matches(&mut self, bytes: &[u8]) -> io::Result<()> {+ self.json.matches.clear();+ // If printing requires knowing the location of each individual match,+ // then compute and stored those right now for use later. While this+ // adds an extra copy for storing the matches, we do amortize the+ // allocation for it and this greatly simplifies the printing logic to+ // the extent that it's easy to ensure that we never do more than+ // one search to find the matches.+ let matches = &mut self.json.matches;+ self.matcher+ .find_iter(bytes, |m| {+ matches.push(m);+ true+ })+ .map_err(io::Error::error_message)?;+ // Don't report empty matches appearing at the end of the bytes.+ if !matches.is_empty()+ && matches.last().unwrap().is_empty()+ && matches.last().unwrap().start() >= bytes.len()+ {+ matches.pop().unwrap();+ }+ Ok(())+ }++ /// Returns true if this printer should quit.+ ///+ /// This implements the logic for handling quitting after seeing a certain+ /// amount of matches. In most cases, the logic is simple, but we must+ /// permit all "after" contextual lines to print after reaching the limit.+ fn should_quit(&self) -> bool {+ let limit = match self.json.config.max_matches {+ None => return false,+ Some(limit) => limit,+ };+ if self.match_count < limit {+ return false;+ }+ self.after_context_remaining == 0+ }++ /// Write the "begin" message.+ fn write_begin_message(&mut self) -> io::Result<()> {+ if self.begin_printed {+ return Ok(());+ }+ let msg = jsont::Message::Begin(jsont::Begin { path: self.path });+ self.json.write_message(&msg)?;+ self.begin_printed = true;+ Ok(())+ }+}++impl<'p, 's, M: Matcher, W: io::Write> Sink for JSONSink<'p, 's, M, W> {+ type Error = io::Error;++ fn matched(+ &mut self,+ searcher: &Searcher,+ mat: &SinkMatch,+ ) -> Result<bool, io::Error> {+ self.write_begin_message()?;++ self.match_count += 1;+ self.after_context_remaining = searcher.after_context() as u64;+ self.record_matches(mat.bytes())?;+ self.stats.add_matches(self.json.matches.len() as u64);+ self.stats.add_matched_lines(mat.lines().count() as u64);++ let submatches = SubMatches::new(mat.bytes(), &self.json.matches);+ let msg = jsont::Message::Match(jsont::Match {+ path: self.path,+ lines: mat.bytes(),+ line_number: mat.line_number(),+ absolute_offset: mat.absolute_byte_offset(),+ submatches: submatches.as_slice(),+ });+ self.json.write_message(&msg)?;+ Ok(!self.should_quit())+ }++ fn context(+ &mut self,+ searcher: &Searcher,+ ctx: &SinkContext,+ ) -> Result<bool, io::Error> {+ self.write_begin_message()?;+ self.json.matches.clear();++ if ctx.kind() == &SinkContextKind::After {+ self.after_context_remaining =+ self.after_context_remaining.saturating_sub(1);+ }+ let submatches = if searcher.invert_match() {+ self.record_matches(ctx.bytes())?;+ SubMatches::new(ctx.bytes(), &self.json.matches)+ } else {+ SubMatches::empty()+ };+ let msg = jsont::Message::Context(jsont::Context {+ path: self.path,+ lines: ctx.bytes(),+ line_number: ctx.line_number(),+ absolute_offset: ctx.absolute_byte_offset(),+ submatches: submatches.as_slice(),+ });+ self.json.write_message(&msg)?;+ Ok(!self.should_quit())+ }++ fn begin(&mut self, _searcher: &Searcher) -> Result<bool, io::Error> {+ self.json.wtr.reset_count();+ self.start_time = Instant::now();+ self.match_count = 0;+ self.after_context_remaining = 0;+ self.binary_byte_offset = None;+ if self.json.config.max_matches == Some(0) {+ return Ok(false);+ }++ if !self.json.config.always_begin_end {+ return Ok(true);+ }+ self.write_begin_message()?;+ Ok(true)+ }++ fn finish(+ &mut self,+ _searcher: &Searcher,+ finish: &SinkFinish,+ ) -> Result<(), io::Error> {+ if !self.begin_printed {+ return Ok(());+ }++ self.binary_byte_offset = finish.binary_byte_offset();+ self.stats.add_elapsed(self.start_time.elapsed());+ self.stats.add_searches(1);+ if self.match_count > 0 {+ self.stats.add_searches_with_match(1);+ }+ self.stats.add_bytes_searched(finish.byte_count());+ self.stats.add_bytes_printed(self.json.wtr.count());++ let msg = jsont::Message::End(jsont::End {+ path: self.path,+ binary_offset: finish.binary_byte_offset(),+ stats: self.stats.clone(),+ });+ self.json.write_message(&msg)?;+ Ok(())+ }+}++/// SubMatches represents a set of matches in a contiguous range of bytes.+///+/// A simpler representation for this would just simply be `Vec<SubMatch>`,+/// but the common case is exactly one match per range of bytes, which we+/// specialize here using a fixed size array without any allocation.+enum SubMatches<'a> {+ Empty,+ Small([jsont::SubMatch<'a>; 1]),+ Big(Vec<jsont::SubMatch<'a>>),+}++impl<'a> SubMatches<'a> {+ /// Create a new set of match ranges from a set of matches and the+ /// corresponding bytes that those matches apply to.+ fn new(bytes: &'a [u8], matches: &[Match]) -> SubMatches<'a> {+ if matches.len() == 1 {+ let mat = matches[0];+ SubMatches::Small([jsont::SubMatch {+ m: &bytes[mat],+ start: mat.start(),+ end: mat.end(),+ }])+ } else {+ let mut match_ranges = vec![];+ for &mat in matches {+ match_ranges.push(jsont::SubMatch {+ m: &bytes[mat],+ start: mat.start(),+ end: mat.end(),+ });+ }+ SubMatches::Big(match_ranges)+ }+ }++ /// Create an empty set of match ranges.+ fn empty() -> SubMatches<'static> {+ SubMatches::Empty+ }++ /// Return this set of match ranges as a slice.+ fn as_slice(&self) -> &[jsont::SubMatch] {+ match *self {+ SubMatches::Empty => &[],+ SubMatches::Small(ref x) => x,+ SubMatches::Big(ref x) => x,+ }+ }+}++#[cfg(test)]+mod tests {+ use grep_matcher::LineTerminator;+ use grep_regex::{RegexMatcher, RegexMatcherBuilder};+ use grep_searcher::SearcherBuilder;++ use super::{JSONBuilder, JSON};++ const SHERLOCK: &'static [u8] = b"\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";++ fn printer_contents(printer: &mut JSON<Vec<u8>>) -> String {+ String::from_utf8(printer.get_mut().to_owned()).unwrap()+ }++ #[test]+ fn binary_detection() {+ use grep_searcher::BinaryDetection;++ const BINARY: &'static [u8] = b"\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew \x00 from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.\+";++ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = JSONBuilder::new().build(vec![]);+ SearcherBuilder::new()+ .binary_detection(BinaryDetection::quit(b'\x00'))+ .heap_limit(Some(80))+ .build()+ .search_reader(&matcher, BINARY, printer.sink(&matcher))+ .unwrap();+ let got = printer_contents(&mut printer);++ assert_eq!(got.lines().count(), 3);+ let last = got.lines().last().unwrap();+ assert!(last.contains(r#""binary_offset":212,"#));+ }++ #[test]+ fn max_matches() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer =+ JSONBuilder::new().max_matches(Some(1)).build(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, printer.sink(&matcher))+ .unwrap();+ let got = printer_contents(&mut printer);++ assert_eq!(got.lines().count(), 3);+ }++ #[test]+ fn no_match() {+ let matcher = RegexMatcher::new(r"DOES NOT MATCH").unwrap();+ let mut printer = JSONBuilder::new().build(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, printer.sink(&matcher))+ .unwrap();+ let got = printer_contents(&mut printer);++ assert!(got.is_empty());+ }++ #[test]+ fn always_begin_end_no_match() {+ let matcher = RegexMatcher::new(r"DOES NOT MATCH").unwrap();+ let mut printer =+ JSONBuilder::new().always_begin_end(true).build(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, printer.sink(&matcher))+ .unwrap();+ let got = printer_contents(&mut printer);++ assert_eq!(got.lines().count(), 2);+ assert!(got.contains("begin") && got.contains("end"));+ }++ #[test]+ fn missing_crlf() {+ let haystack = "test\r\n".as_bytes();++ let matcher = RegexMatcherBuilder::new().build("test").unwrap();+ let mut printer = JSONBuilder::new().build(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, haystack, printer.sink(&matcher))+ .unwrap();+ let got = printer_contents(&mut printer);+ assert_eq!(got.lines().count(), 3);+ assert!(+ got.lines().nth(1).unwrap().contains(r"test\r\n"),+ r"missing 'test\r\n' in '{}'",+ got.lines().nth(1).unwrap(),+ );++ let matcher =+ RegexMatcherBuilder::new().crlf(true).build("test").unwrap();+ let mut printer = JSONBuilder::new().build(vec![]);+ SearcherBuilder::new()+ .line_terminator(LineTerminator::crlf())+ .build()+ .search_reader(&matcher, haystack, printer.sink(&matcher))+ .unwrap();+ let got = printer_contents(&mut printer);+ assert_eq!(got.lines().count(), 3);+ assert!(+ got.lines().nth(1).unwrap().contains(r"test\r\n"),+ r"missing 'test\r\n' in '{}'",+ got.lines().nth(1).unwrap(),+ );+ }+}
@@ -0,0 +1,143 @@+// This module defines the types we use for JSON serialization. We specifically+// omit deserialization, partially because there isn't a clear use case for+// them at this time, but also because deserialization will complicate things.+// Namely, the types below are designed in a way that permits JSON+// serialization with little or no allocation. Allocation is often quite+// convenient for deserialization however, so these types would become a bit+// more complex.++use std::borrow::Cow;+use std::path::Path;+use std::str;++use base64;+use serde::{Serialize, Serializer};++use stats::Stats;++#[derive(Serialize)]+#[serde(tag = "type", content = "data")]+#[serde(rename_all = "snake_case")]+pub enum Message<'a> {+ Begin(Begin<'a>),+ End(End<'a>),+ Match(Match<'a>),+ Context(Context<'a>),+}++#[derive(Serialize)]+pub struct Begin<'a> {+ #[serde(serialize_with = "ser_path")]+ pub path: Option<&'a Path>,+}++#[derive(Serialize)]+pub struct End<'a> {+ #[serde(serialize_with = "ser_path")]+ pub path: Option<&'a Path>,+ pub binary_offset: Option<u64>,+ pub stats: Stats,+}++#[derive(Serialize)]+pub struct Match<'a> {+ #[serde(serialize_with = "ser_path")]+ pub path: Option<&'a Path>,+ #[serde(serialize_with = "ser_bytes")]+ pub lines: &'a [u8],+ pub line_number: Option<u64>,+ pub absolute_offset: u64,+ pub submatches: &'a [SubMatch<'a>],+}++#[derive(Serialize)]+pub struct Context<'a> {+ #[serde(serialize_with = "ser_path")]+ pub path: Option<&'a Path>,+ #[serde(serialize_with = "ser_bytes")]+ pub lines: &'a [u8],+ pub line_number: Option<u64>,+ pub absolute_offset: u64,+ pub submatches: &'a [SubMatch<'a>],+}++#[derive(Serialize)]+pub struct SubMatch<'a> {+ #[serde(rename = "match")]+ #[serde(serialize_with = "ser_bytes")]+ pub m: &'a [u8],+ pub start: usize,+ pub end: usize,+}++/// Data represents things that look like strings, but may actually not be+/// valid UTF-8. To handle this, `Data` is serialized as an object with one+/// of two keys: `text` (for valid UTF-8) or `bytes` (for invalid UTF-8).+///+/// The happy path is valid UTF-8, which streams right through as-is, since+/// it is natively supported by JSON. When invalid UTF-8 is found, then it is+/// represented as arbitrary bytes and base64 encoded.+#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize)]+#[serde(untagged)]+enum Data<'a> {+ Text {+ text: Cow<'a, str>,+ },+ Bytes {+ #[serde(serialize_with = "to_base64")]+ bytes: &'a [u8],+ },+}++impl<'a> Data<'a> {+ fn from_bytes(bytes: &[u8]) -> Data {+ match str::from_utf8(bytes) {+ Ok(text) => Data::Text { text: Cow::Borrowed(text) },+ Err(_) => Data::Bytes { bytes },+ }+ }++ #[cfg(unix)]+ fn from_path(path: &Path) -> Data {+ use std::os::unix::ffi::OsStrExt;++ match path.to_str() {+ Some(text) => Data::Text { text: Cow::Borrowed(text) },+ None => Data::Bytes { bytes: path.as_os_str().as_bytes() },+ }+ }++ #[cfg(not(unix))]+ fn from_path(path: &Path) -> Data {+ // Using lossy conversion means some paths won't round trip precisely,+ // but it's not clear what we should actually do. Serde rejects+ // non-UTF-8 paths, and OsStr's are serialized as a sequence of UTF-16+ // code units on Windows. Neither seem appropriate for this use case,+ // so we do the easy thing for now.+ Data::Text { text: path.to_string_lossy() }+ }+}++fn to_base64<T, S>(bytes: T, ser: S) -> Result<S::Ok, S::Error>+where+ T: AsRef<[u8]>,+ S: Serializer,+{+ ser.serialize_str(&base64::encode(&bytes))+}++fn ser_bytes<T, S>(bytes: T, ser: S) -> Result<S::Ok, S::Error>+where+ T: AsRef<[u8]>,+ S: Serializer,+{+ Data::from_bytes(bytes.as_ref()).serialize(ser)+}++fn ser_path<P, S>(path: &Option<P>, ser: S) -> Result<S::Ok, S::Error>+where+ P: AsRef<Path>,+ S: Serializer,+{+ path.as_ref().map(|p| Data::from_path(p.as_ref())).serialize(ser)+}
@@ -0,0 +1,107 @@+/*!+This crate provides featureful and fast printers that interoperate with the+[`grep-searcher`](https://docs.rs/grep-searcher)+crate.++# Brief overview++The [`Standard`](struct.Standard.html) printer shows results in a human+readable format, and is modeled after the formats used by standard grep-like+tools. Features include, but are not limited to, cross platform terminal+coloring, search & replace, multi-line result handling and reporting summary+statistics.++The [`JSON`](struct.JSON.html) printer shows results in a machine readable+format. To facilitate a stream of search results, the format uses+[JSON Lines](http://jsonlines.org/)+by emitting a series of messages as search results are found.++The [`Summary`](struct.Summary.html) printer shows *aggregate* results for a+single search in a human readable format, and is modeled after similar formats+found in standard grep-like tools. This printer is useful for showing the total+number of matches and/or printing file paths that either contain or don't+contain matches.++# Example++This example shows how to create a "standard" printer and execute a search.++```+extern crate grep_regex;+extern crate grep_printer;+extern crate grep_searcher;++use std::error::Error;++use grep_regex::RegexMatcher;+use grep_printer::Standard;+use grep_searcher::Searcher;++const SHERLOCK: &'static [u8] = b"\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";++# fn main() { example().unwrap(); }+fn example() -> Result<(), Box<Error>> {+ let matcher = RegexMatcher::new(r"Sherlock")?;+ let mut printer = Standard::new_no_color(vec![]);+ Searcher::new().search_slice(&matcher, SHERLOCK, printer.sink(&matcher))?;++ // into_inner gives us back the underlying writer we provided to+ // new_no_color, which is wrapped in a termcolor::NoColor. Thus, a second+ // into_inner gives us back the actual buffer.+ let output = String::from_utf8(printer.into_inner().into_inner())?;+ let expected = "\+1:For the Doctor Watsons of this world, as opposed to the Sherlock+3:be, to a very large extent, the result of luck. Sherlock Holmes+";+ assert_eq!(output, expected);+ Ok(())+}+```+*/++#![deny(missing_docs)]++#[cfg(feature = "serde1")]+extern crate base64;+extern crate bstr;+extern crate grep_matcher;+#[cfg(test)]+extern crate grep_regex;+extern crate grep_searcher;+#[cfg(feature = "serde1")]+extern crate serde;+#[cfg(feature = "serde1")]+#[macro_use]+extern crate serde_derive;+#[cfg(feature = "serde1")]+extern crate serde_json;+extern crate termcolor;++pub use color::{default_color_specs, ColorError, ColorSpecs, UserColorSpec};+#[cfg(feature = "serde1")]+pub use json::{JSONBuilder, JSONSink, JSON};+pub use standard::{Standard, StandardBuilder, StandardSink};+pub use stats::Stats;+pub use summary::{Summary, SummaryBuilder, SummaryKind, SummarySink};+pub use util::PrinterPath;++#[macro_use]+mod macros;++mod color;+mod counter;+#[cfg(feature = "serde1")]+mod json;+#[cfg(feature = "serde1")]+mod jsont;+mod standard;+mod stats;+mod summary;+mod util;
@@ -0,0 +1,24 @@+/// Like assert_eq, but nicer output for long strings.+#[cfg(test)]+#[macro_export]+macro_rules! assert_eq_printed {+ ($expected:expr, $got:expr) => {+ let expected = &*$expected;+ let got = &*$got;+ if expected != got {+ panic!("+printed outputs differ!++expected:+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+{}+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++got:+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+{}+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+", expected, got);+ }+ }+}
crates/printer/src/standard.rs
@@ -0,0 +1,3390 @@+use std::cell::{Cell, RefCell};+use std::cmp;+use std::io::{self, Write};+use std::path::Path;+use std::sync::Arc;+use std::time::Instant;++use bstr::ByteSlice;+use grep_matcher::{Match, Matcher};+use grep_searcher::{+ LineStep, Searcher, Sink, SinkContext, SinkContextKind, SinkError,+ SinkFinish, SinkMatch,+};+use termcolor::{ColorSpec, NoColor, WriteColor};++use color::ColorSpecs;+use counter::CounterWriter;+use stats::Stats;+use util::{trim_ascii_prefix, PrinterPath, Replacer, Sunk};++/// The configuration for the standard printer.+///+/// This is manipulated by the StandardBuilder and then referenced by the+/// actual implementation. Once a printer is build, the configuration is frozen+/// and cannot changed.+#[derive(Debug, Clone)]+struct Config {+ colors: ColorSpecs,+ stats: bool,+ heading: bool,+ path: bool,+ only_matching: bool,+ per_match: bool,+ replacement: Arc<Option<Vec<u8>>>,+ max_columns: Option<u64>,+ max_columns_preview: bool,+ max_matches: Option<u64>,+ column: bool,+ byte_offset: bool,+ trim_ascii: bool,+ separator_search: Arc<Option<Vec<u8>>>,+ separator_context: Arc<Option<Vec<u8>>>,+ separator_field_match: Arc<Vec<u8>>,+ separator_field_context: Arc<Vec<u8>>,+ separator_path: Option<u8>,+ path_terminator: Option<u8>,+}++impl Default for Config {+ fn default() -> Config {+ Config {+ colors: ColorSpecs::default(),+ stats: false,+ heading: false,+ path: true,+ only_matching: false,+ per_match: false,+ replacement: Arc::new(None),+ max_columns: None,+ max_columns_preview: false,+ max_matches: None,+ column: false,+ byte_offset: false,+ trim_ascii: false,+ separator_search: Arc::new(None),+ separator_context: Arc::new(Some(b"--".to_vec())),+ separator_field_match: Arc::new(b":".to_vec()),+ separator_field_context: Arc::new(b"-".to_vec()),+ separator_path: None,+ path_terminator: None,+ }+ }+}++/// A builder for the "standard" grep-like printer.+///+/// The builder permits configuring how the printer behaves. Configurable+/// behavior includes, but is not limited to, limiting the number of matches,+/// tweaking separators, executing pattern replacements, recording statistics+/// and setting colors.+///+/// Some configuration options, such as the display of line numbers or+/// contextual lines, are drawn directly from the+/// `grep_searcher::Searcher`'s configuration.+///+/// Once a `Standard` printer is built, its configuration cannot be changed.+#[derive(Clone, Debug)]+pub struct StandardBuilder {+ config: Config,+}++impl StandardBuilder {+ /// Return a new builder for configuring the standard printer.+ pub fn new() -> StandardBuilder {+ StandardBuilder { config: Config::default() }+ }++ /// Build a printer using any implementation of `termcolor::WriteColor`.+ ///+ /// The implementation of `WriteColor` used here controls whether colors+ /// are used or not when colors have been configured using the+ /// `color_specs` method.+ ///+ /// For maximum portability, callers should generally use either+ /// `termcolor::StandardStream` or `termcolor::BufferedStandardStream`+ /// where appropriate, which will automatically enable colors on Windows+ /// when possible.+ ///+ /// However, callers may also provide an arbitrary writer using the+ /// `termcolor::Ansi` or `termcolor::NoColor` wrappers, which always enable+ /// colors via ANSI escapes or always disable colors, respectively.+ ///+ /// As a convenience, callers may use `build_no_color` to automatically+ /// select the `termcolor::NoColor` wrapper to avoid needing to import+ /// from `termcolor` explicitly.+ pub fn build<W: WriteColor>(&self, wtr: W) -> Standard<W> {+ Standard {+ config: self.config.clone(),+ wtr: RefCell::new(CounterWriter::new(wtr)),+ matches: vec![],+ }+ }++ /// Build a printer from any implementation of `io::Write` and never emit+ /// any colors, regardless of the user color specification settings.+ ///+ /// This is a convenience routine for+ /// `StandardBuilder::build(termcolor::NoColor::new(wtr))`.+ pub fn build_no_color<W: io::Write>(+ &self,+ wtr: W,+ ) -> Standard<NoColor<W>> {+ self.build(NoColor::new(wtr))+ }++ /// Set the user color specifications to use for coloring in this printer.+ ///+ /// A [`UserColorSpec`](struct.UserColorSpec.html) can be constructed from+ /// a string in accordance with the color specification format. See the+ /// `UserColorSpec` type documentation for more details on the format.+ /// A [`ColorSpecs`](struct.ColorSpecs.html) can then be generated from+ /// zero or more `UserColorSpec`s.+ ///+ /// Regardless of the color specifications provided here, whether color+ /// is actually used or not is determined by the implementation of+ /// `WriteColor` provided to `build`. For example, if `termcolor::NoColor`+ /// is provided to `build`, then no color will ever be printed regardless+ /// of the color specifications provided here.+ ///+ /// This completely overrides any previous color specifications. This does+ /// not add to any previously provided color specifications on this+ /// builder.+ pub fn color_specs(&mut self, specs: ColorSpecs) -> &mut StandardBuilder {+ self.config.colors = specs;+ self+ }++ /// Enable the gathering of various aggregate statistics.+ ///+ /// When this is enabled (it's disabled by default), statistics will be+ /// gathered for all uses of `Standard` printer returned by `build`,+ /// including but not limited to, the total number of matches, the total+ /// number of bytes searched and the total number of bytes printed.+ ///+ /// Aggregate statistics can be accessed via the sink's+ /// [`StandardSink::stats`](struct.StandardSink.html#method.stats)+ /// method.+ ///+ /// When this is enabled, this printer may need to do extra work in order+ /// to compute certain statistics, which could cause the search to take+ /// longer.+ ///+ /// For a complete description of available statistics, see+ /// [`Stats`](struct.Stats.html).+ pub fn stats(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.stats = yes;+ self+ }++ /// Enable the use of "headings" in the printer.+ ///+ /// When this is enabled, and if a file path has been given to the printer,+ /// then the file path will be printed once on its own line before showing+ /// any matches. If the heading is not the first thing emitted by the+ /// printer, then a line terminator is printed before the heading.+ ///+ /// By default, this option is disabled. When disabled, the printer will+ /// not show any heading and will instead print the file path (if one is+ /// given) on the same line as each matching (or context) line.+ pub fn heading(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.heading = yes;+ self+ }++ /// When enabled, if a path was given to the printer, then it is shown in+ /// the output (either as a heading or as a prefix to each matching line).+ /// When disabled, then no paths are ever included in the output even when+ /// a path is provided to the printer.+ ///+ /// This is enabled by default.+ pub fn path(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.path = yes;+ self+ }++ /// Only print the specific matches instead of the entire line containing+ /// each match. Each match is printed on its own line. When multi line+ /// search is enabled, then matches spanning multiple lines are printed+ /// such that only the matching portions of each line are shown.+ pub fn only_matching(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.only_matching = yes;+ self+ }++ /// Print at least one line for every match.+ ///+ /// This is similar to the `only_matching` option, except the entire line+ /// is printed for each match. This is typically useful in conjunction with+ /// the `column` option, which will show the starting column number for+ /// every match on every line.+ ///+ /// When multi-line mode is enabled, each match and its accompanying lines+ /// are printed. As with single line matches, if a line contains multiple+ /// matches (even if only partially), then that line is printed once for+ /// each match it participates in.+ pub fn per_match(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.per_match = yes;+ self+ }++ /// Set the bytes that will be used to replace each occurrence of a match+ /// found.+ ///+ /// The replacement bytes given may include references to capturing groups,+ /// which may either be in index form (e.g., `$2`) or can reference named+ /// capturing groups if present in the original pattern (e.g., `$foo`).+ ///+ /// For documentation on the full format, please see the `Capture` trait's+ /// `interpolate` method in the+ /// [grep-printer](https://docs.rs/grep-printer) crate.+ pub fn replacement(+ &mut self,+ replacement: Option<Vec<u8>>,+ ) -> &mut StandardBuilder {+ self.config.replacement = Arc::new(replacement);+ self+ }++ /// Set the maximum number of columns allowed for each line printed. A+ /// single column is heuristically defined as a single byte.+ ///+ /// If a line is found which exceeds this maximum, then it is replaced+ /// with a message indicating that the line has been omitted.+ ///+ /// The default is to not specify a limit, in which each matching or+ /// contextual line is printed regardless of how long it is.+ pub fn max_columns(&mut self, limit: Option<u64>) -> &mut StandardBuilder {+ self.config.max_columns = limit;+ self+ }++ /// When enabled, if a line is found to be over the configured maximum+ /// column limit (measured in terms of bytes), then a preview of the long+ /// line will be printed instead.+ ///+ /// The preview will correspond to the first `N` *grapheme clusters* of+ /// the line, where `N` is the limit configured by `max_columns`.+ ///+ /// If no limit is set, then enabling this has no effect.+ ///+ /// This is disabled by default.+ pub fn max_columns_preview(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.max_columns_preview = yes;+ self+ }++ /// Set the maximum amount of matching lines that are printed.+ ///+ /// If multi line search is enabled and a match spans multiple lines, then+ /// that match is counted exactly once for the purposes of enforcing this+ /// limit, regardless of how many lines it spans.+ pub fn max_matches(&mut self, limit: Option<u64>) -> &mut StandardBuilder {+ self.config.max_matches = limit;+ self+ }++ /// Print the column number of the first match in a line.+ ///+ /// This option is convenient for use with `per_match` which will print a+ /// line for every match along with the starting offset for that match.+ ///+ /// Column numbers are computed in terms of bytes from the start of the+ /// line being printed.+ ///+ /// For matches that span multiple lines, the column number for each+ /// matching line is in terms of the first matching line.+ ///+ /// This is disabled by default.+ pub fn column(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.column = yes;+ self+ }++ /// Print the absolute byte offset of the beginning of each line printed.+ ///+ /// The absolute byte offset starts from the beginning of each search and+ /// is zero based.+ ///+ /// If the `only_matching` option is set, then this will print the absolute+ /// byte offset of the beginning of each match.+ pub fn byte_offset(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.byte_offset = yes;+ self+ }++ /// When enabled, all lines will have prefix ASCII whitespace trimmed+ /// before being written.+ ///+ /// This is disabled by default.+ pub fn trim_ascii(&mut self, yes: bool) -> &mut StandardBuilder {+ self.config.trim_ascii = yes;+ self+ }++ /// Set the separator used between sets of search results.+ ///+ /// When this is set, then it will be printed on its own line immediately+ /// before the results for a single search if and only if a previous search+ /// had already printed results. In effect, this permits showing a divider+ /// between sets of search results that does not appear at the beginning+ /// or end of all search results.+ ///+ /// To reproduce the classic grep format, this is typically set to `--`+ /// (the same as the context separator) if and only if contextual lines+ /// have been requested, but disabled otherwise.+ ///+ /// By default, this is disabled.+ pub fn separator_search(+ &mut self,+ sep: Option<Vec<u8>>,+ ) -> &mut StandardBuilder {+ self.config.separator_search = Arc::new(sep);+ self+ }++ /// Set the separator used between discontiguous runs of search context,+ /// but only when the searcher is configured to report contextual lines.+ ///+ /// The separator is always printed on its own line, even if it's empty.+ ///+ /// If no separator is set, then nothing is printed when a context break+ /// occurs.+ ///+ /// By default, this is set to `--`.+ pub fn separator_context(+ &mut self,+ sep: Option<Vec<u8>>,+ ) -> &mut StandardBuilder {+ self.config.separator_context = Arc::new(sep);+ self+ }++ /// Set the separator used between fields emitted for matching lines.+ ///+ /// For example, when the searcher has line numbers enabled, this printer+ /// will print the line number before each matching line. The bytes given+ /// here will be written after the line number but before the matching+ /// line.+ ///+ /// By default, this is set to `:`.+ pub fn separator_field_match(+ &mut self,+ sep: Vec<u8>,+ ) -> &mut StandardBuilder {+ self.config.separator_field_match = Arc::new(sep);+ self+ }++ /// Set the separator used between fields emitted for context lines.+ ///+ /// For example, when the searcher has line numbers enabled, this printer+ /// will print the line number before each context line. The bytes given+ /// here will be written after the line number but before the context+ /// line.+ ///+ /// By default, this is set to `-`.+ pub fn separator_field_context(+ &mut self,+ sep: Vec<u8>,+ ) -> &mut StandardBuilder {+ self.config.separator_field_context = Arc::new(sep);+ self+ }++ /// Set the path separator used when printing file paths.+ ///+ /// When a printer is configured with a file path, and when a match is+ /// found, that file path will be printed (either as a heading or as a+ /// prefix to each matching or contextual line, depending on other+ /// configuration settings). Typically, printing is done by emitting the+ /// file path as is. However, this setting provides the ability to use a+ /// different path separator from what the current environment has+ /// configured.+ ///+ /// A typical use for this option is to permit cygwin users on Windows to+ /// set the path separator to `/` instead of using the system default of+ /// `\`.+ pub fn separator_path(&mut self, sep: Option<u8>) -> &mut StandardBuilder {+ self.config.separator_path = sep;+ self+ }++ /// Set the path terminator used.+ ///+ /// The path terminator is a byte that is printed after every file path+ /// emitted by this printer.+ ///+ /// If no path terminator is set (the default), then paths are terminated+ /// by either new lines (for when `heading` is enabled) or the match or+ /// context field separators (e.g., `:` or `-`).+ pub fn path_terminator(+ &mut self,+ terminator: Option<u8>,+ ) -> &mut StandardBuilder {+ self.config.path_terminator = terminator;+ self+ }+}++/// The standard printer, which implements grep-like formatting, including+/// color support.+///+/// A default printer can be created with either of the `Standard::new` or+/// `Standard::new_no_color` constructors. However, there are a considerable+/// number of options that configure this printer's output. Those options can+/// be configured using [`StandardBuilder`](struct.StandardBuilder.html).+///+/// This type is generic over `W`, which represents any implementation+/// of the `termcolor::WriteColor` trait. If colors are not desired,+/// then the `new_no_color` constructor can be used, or, alternatively,+/// the `termcolor::NoColor` adapter can be used to wrap any `io::Write`+/// implementation without enabling any colors.+#[derive(Debug)]+pub struct Standard<W> {+ config: Config,+ wtr: RefCell<CounterWriter<W>>,+ matches: Vec<Match>,+}++impl<W: WriteColor> Standard<W> {+ /// Return a standard printer with a default configuration that writes+ /// matches to the given writer.+ ///+ /// The writer should be an implementation of `termcolor::WriteColor`+ /// and not just a bare implementation of `io::Write`. To use a normal+ /// `io::Write` implementation (simultaneously sacrificing colors), use+ /// the `new_no_color` constructor.+ pub fn new(wtr: W) -> Standard<W> {+ StandardBuilder::new().build(wtr)+ }+}++impl<W: io::Write> Standard<NoColor<W>> {+ /// Return a standard printer with a default configuration that writes+ /// matches to the given writer.+ ///+ /// The writer can be any implementation of `io::Write`. With this+ /// constructor, the printer will never emit colors.+ pub fn new_no_color(wtr: W) -> Standard<NoColor<W>> {+ StandardBuilder::new().build_no_color(wtr)+ }+}++impl<W: WriteColor> Standard<W> {+ /// Return an implementation of `Sink` for the standard printer.+ ///+ /// This does not associate the printer with a file path, which means this+ /// implementation will never print a file path along with the matches.+ pub fn sink<'s, M: Matcher>(+ &'s mut self,+ matcher: M,+ ) -> StandardSink<'static, 's, M, W> {+ let stats = if self.config.stats { Some(Stats::new()) } else { None };+ let needs_match_granularity = self.needs_match_granularity();+ StandardSink {+ matcher: matcher,+ standard: self,+ replacer: Replacer::new(),+ path: None,+ start_time: Instant::now(),+ match_count: 0,+ after_context_remaining: 0,+ binary_byte_offset: None,+ stats: stats,+ needs_match_granularity: needs_match_granularity,+ }+ }++ /// Return an implementation of `Sink` associated with a file path.+ ///+ /// When the printer is associated with a path, then it may, depending on+ /// its configuration, print the path along with the matches found.+ pub fn sink_with_path<'p, 's, M, P>(+ &'s mut self,+ matcher: M,+ path: &'p P,+ ) -> StandardSink<'p, 's, M, W>+ where+ M: Matcher,+ P: ?Sized + AsRef<Path>,+ {+ if !self.config.path {+ return self.sink(matcher);+ }+ let stats = if self.config.stats { Some(Stats::new()) } else { None };+ let ppath = PrinterPath::with_separator(+ path.as_ref(),+ self.config.separator_path,+ );+ let needs_match_granularity = self.needs_match_granularity();+ StandardSink {+ matcher: matcher,+ standard: self,+ replacer: Replacer::new(),+ path: Some(ppath),+ start_time: Instant::now(),+ match_count: 0,+ after_context_remaining: 0,+ binary_byte_offset: None,+ stats: stats,+ needs_match_granularity: needs_match_granularity,+ }+ }++ /// Returns true if and only if the configuration of the printer requires+ /// us to find each individual match in the lines reported by the searcher.+ ///+ /// We care about this distinction because finding each individual match+ /// costs more, so we only do it when we need to.+ fn needs_match_granularity(&self) -> bool {+ let supports_color = self.wtr.borrow().supports_color();+ let match_colored = !self.config.colors.matched().is_none();++ // Coloring requires identifying each individual match.+ (supports_color && match_colored)+ // The column feature requires finding the position of the first match.+ || self.config.column+ // Requires finding each match for performing replacement.+ || self.config.replacement.is_some()+ // Emitting a line for each match requires finding each match.+ || self.config.per_match+ // Emitting only the match requires finding each match.+ || self.config.only_matching+ // Computing certain statistics requires finding each match.+ || self.config.stats+ }+}++impl<W> Standard<W> {+ /// Returns true if and only if this printer has written at least one byte+ /// to the underlying writer during any of the previous searches.+ pub fn has_written(&self) -> bool {+ self.wtr.borrow().total_count() > 0+ }++ /// Return a mutable reference to the underlying writer.+ pub fn get_mut(&mut self) -> &mut W {+ self.wtr.get_mut().get_mut()+ }++ /// Consume this printer and return back ownership of the underlying+ /// writer.+ pub fn into_inner(self) -> W {+ self.wtr.into_inner().into_inner()+ }+}++/// An implementation of `Sink` associated with a matcher and an optional file+/// path for the standard printer.+///+/// A `Sink` can be created via the+/// [`Standard::sink`](struct.Standard.html#method.sink)+/// or+/// [`Standard::sink_with_path`](struct.Standard.html#method.sink_with_path)+/// methods, depending on whether you want to include a file path in the+/// printer's output.+///+/// Building a `StandardSink` is cheap, and callers should create a new one+/// for each thing that is searched. After a search has completed, callers may+/// query this sink for information such as whether a match occurred or whether+/// binary data was found (and if so, the offset at which it occurred).+///+/// This type is generic over a few type parameters:+///+/// * `'p` refers to the lifetime of the file path, if one is provided. When+/// no file path is given, then this is `'static`.+/// * `'s` refers to the lifetime of the+/// [`Standard`](struct.Standard.html)+/// printer that this type borrows.+/// * `M` refers to the type of matcher used by+/// `grep_searcher::Searcher` that is reporting results to this sink.+/// * `W` refers to the underlying writer that this printer is writing its+/// output to.+#[derive(Debug)]+pub struct StandardSink<'p, 's, M: Matcher, W: 's> {+ matcher: M,+ standard: &'s mut Standard<W>,+ replacer: Replacer<M>,+ path: Option<PrinterPath<'p>>,+ start_time: Instant,+ match_count: u64,+ after_context_remaining: u64,+ binary_byte_offset: Option<u64>,+ stats: Option<Stats>,+ needs_match_granularity: bool,+}++impl<'p, 's, M: Matcher, W: WriteColor> StandardSink<'p, 's, M, W> {+ /// Returns true if and only if this printer received a match in the+ /// previous search.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search on this sink.+ pub fn has_match(&self) -> bool {+ self.match_count > 0+ }++ /// Return the total number of matches reported to this sink.+ ///+ /// This corresponds to the number of times `Sink::matched` is called+ /// on the previous search.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search on this sink.+ pub fn match_count(&self) -> u64 {+ self.match_count+ }++ /// If binary data was found in the previous search, this returns the+ /// offset at which the binary data was first detected.+ ///+ /// The offset returned is an absolute offset relative to the entire+ /// set of bytes searched.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search. e.g., If the search prior to the previous search found binary+ /// data but the previous search found no binary data, then this will+ /// return `None`.+ pub fn binary_byte_offset(&self) -> Option<u64> {+ self.binary_byte_offset+ }++ /// Return a reference to the stats produced by the printer for all+ /// searches executed on this sink.+ ///+ /// This only returns stats if they were requested via the+ /// [`StandardBuilder`](struct.StandardBuilder.html)+ /// configuration.+ pub fn stats(&self) -> Option<&Stats> {+ self.stats.as_ref()+ }++ /// Execute the matcher over the given bytes and record the match+ /// locations if the current configuration demands match granularity.+ fn record_matches(&mut self, bytes: &[u8]) -> io::Result<()> {+ self.standard.matches.clear();+ if !self.needs_match_granularity {+ return Ok(());+ }+ // If printing requires knowing the location of each individual match,+ // then compute and stored those right now for use later. While this+ // adds an extra copy for storing the matches, we do amortize the+ // allocation for it and this greatly simplifies the printing logic to+ // the extent that it's easy to ensure that we never do more than+ // one search to find the matches (well, for replacements, we do one+ // additional search to perform the actual replacement).+ let matches = &mut self.standard.matches;+ self.matcher+ .find_iter(bytes, |m| {+ matches.push(m);+ true+ })+ .map_err(io::Error::error_message)?;+ // Don't report empty matches appearing at the end of the bytes.+ if !matches.is_empty()+ && matches.last().unwrap().is_empty()+ && matches.last().unwrap().start() >= bytes.len()+ {+ matches.pop().unwrap();+ }+ Ok(())+ }++ /// If the configuration specifies a replacement, then this executes the+ /// replacement, lazily allocating memory if necessary.+ ///+ /// To access the result of a replacement, use `replacer.replacement()`.+ fn replace(&mut self, bytes: &[u8]) -> io::Result<()> {+ self.replacer.clear();+ if self.standard.config.replacement.is_some() {+ let replacement = (*self.standard.config.replacement)+ .as_ref()+ .map(|r| &*r)+ .unwrap();+ self.replacer.replace_all(&self.matcher, bytes, replacement)?;+ }+ Ok(())+ }++ /// Returns true if this printer should quit.+ ///+ /// This implements the logic for handling quitting after seeing a certain+ /// amount of matches. In most cases, the logic is simple, but we must+ /// permit all "after" contextual lines to print after reaching the limit.+ fn should_quit(&self) -> bool {+ let limit = match self.standard.config.max_matches {+ None => return false,+ Some(limit) => limit,+ };+ if self.match_count < limit {+ return false;+ }+ self.after_context_remaining == 0+ }+}++impl<'p, 's, M: Matcher, W: WriteColor> Sink for StandardSink<'p, 's, M, W> {+ type Error = io::Error;++ fn matched(+ &mut self,+ searcher: &Searcher,+ mat: &SinkMatch,+ ) -> Result<bool, io::Error> {+ self.match_count += 1;+ self.after_context_remaining = searcher.after_context() as u64;++ self.record_matches(mat.bytes())?;+ self.replace(mat.bytes())?;++ if let Some(ref mut stats) = self.stats {+ stats.add_matches(self.standard.matches.len() as u64);+ stats.add_matched_lines(mat.lines().count() as u64);+ }+ if searcher.binary_detection().convert_byte().is_some() {+ if self.binary_byte_offset.is_some() {+ return Ok(false);+ }+ }++ StandardImpl::from_match(searcher, self, mat).sink()?;+ Ok(!self.should_quit())+ }++ fn context(+ &mut self,+ searcher: &Searcher,+ ctx: &SinkContext,+ ) -> Result<bool, io::Error> {+ self.standard.matches.clear();+ self.replacer.clear();++ if ctx.kind() == &SinkContextKind::After {+ self.after_context_remaining =+ self.after_context_remaining.saturating_sub(1);+ }+ if searcher.invert_match() {+ self.record_matches(ctx.bytes())?;+ self.replace(ctx.bytes())?;+ }+ if searcher.binary_detection().convert_byte().is_some() {+ if self.binary_byte_offset.is_some() {+ return Ok(false);+ }+ }++ StandardImpl::from_context(searcher, self, ctx).sink()?;+ Ok(!self.should_quit())+ }++ fn context_break(+ &mut self,+ searcher: &Searcher,+ ) -> Result<bool, io::Error> {+ StandardImpl::new(searcher, self).write_context_separator()?;+ Ok(true)+ }++ fn binary_data(+ &mut self,+ _searcher: &Searcher,+ binary_byte_offset: u64,+ ) -> Result<bool, io::Error> {+ self.binary_byte_offset = Some(binary_byte_offset);+ Ok(true)+ }++ fn begin(&mut self, _searcher: &Searcher) -> Result<bool, io::Error> {+ self.standard.wtr.borrow_mut().reset_count();+ self.start_time = Instant::now();+ self.match_count = 0;+ self.after_context_remaining = 0;+ self.binary_byte_offset = None;+ if self.standard.config.max_matches == Some(0) {+ return Ok(false);+ }+ Ok(true)+ }++ fn finish(+ &mut self,+ searcher: &Searcher,+ finish: &SinkFinish,+ ) -> Result<(), io::Error> {+ if let Some(offset) = self.binary_byte_offset {+ StandardImpl::new(searcher, self).write_binary_message(offset)?;+ }+ if let Some(stats) = self.stats.as_mut() {+ stats.add_elapsed(self.start_time.elapsed());+ stats.add_searches(1);+ if self.match_count > 0 {+ stats.add_searches_with_match(1);+ }+ stats.add_bytes_searched(finish.byte_count());+ stats.add_bytes_printed(self.standard.wtr.borrow().count());+ }+ Ok(())+ }+}++/// The actual implementation of the standard printer. This couples together+/// the searcher, the sink implementation and information about the match.+///+/// A StandardImpl is initialized every time a match or a contextual line is+/// reported.+#[derive(Debug)]+struct StandardImpl<'a, M: 'a + Matcher, W: 'a> {+ searcher: &'a Searcher,+ sink: &'a StandardSink<'a, 'a, M, W>,+ sunk: Sunk<'a>,+ /// Set to true if and only if we are writing a match with color.+ in_color_match: Cell<bool>,+}++impl<'a, M: Matcher, W: WriteColor> StandardImpl<'a, M, W> {+ /// Bundle self with a searcher and return the core implementation of Sink.+ fn new(+ searcher: &'a Searcher,+ sink: &'a StandardSink<M, W>,+ ) -> StandardImpl<'a, M, W> {+ StandardImpl {+ searcher: searcher,+ sink: sink,+ sunk: Sunk::empty(),+ in_color_match: Cell::new(false),+ }+ }++ /// Bundle self with a searcher and return the core implementation of Sink+ /// for use with handling matching lines.+ fn from_match(+ searcher: &'a Searcher,+ sink: &'a StandardSink<M, W>,+ mat: &'a SinkMatch<'a>,+ ) -> StandardImpl<'a, M, W> {+ let sunk = Sunk::from_sink_match(+ mat,+ &sink.standard.matches,+ sink.replacer.replacement(),+ );+ StandardImpl { sunk: sunk, ..StandardImpl::new(searcher, sink) }+ }++ /// Bundle self with a searcher and return the core implementation of Sink+ /// for use with handling contextual lines.+ fn from_context(+ searcher: &'a Searcher,+ sink: &'a StandardSink<M, W>,+ ctx: &'a SinkContext<'a>,+ ) -> StandardImpl<'a, M, W> {+ let sunk = Sunk::from_sink_context(+ ctx,+ &sink.standard.matches,+ sink.replacer.replacement(),+ );+ StandardImpl { sunk: sunk, ..StandardImpl::new(searcher, sink) }+ }++ fn sink(&self) -> io::Result<()> {+ self.write_search_prelude()?;+ if self.sunk.matches().is_empty() {+ if self.multi_line() && !self.is_context() {+ self.sink_fast_multi_line()+ } else {+ self.sink_fast()+ }+ } else {+ if self.multi_line() && !self.is_context() {+ self.sink_slow_multi_line()+ } else {+ self.sink_slow()+ }+ }+ }++ /// Print matches (limited to one line) quickly by avoiding the detection+ /// of each individual match in the lines reported in the given+ /// `SinkMatch`.+ ///+ /// This should only be used when the configuration does not demand match+ /// granularity and the searcher is not in multi line mode.+ fn sink_fast(&self) -> io::Result<()> {+ debug_assert!(self.sunk.matches().is_empty());+ debug_assert!(!self.multi_line() || self.is_context());++ self.write_prelude(+ self.sunk.absolute_byte_offset(),+ self.sunk.line_number(),+ None,+ )?;+ self.write_line(self.sunk.bytes())+ }++ /// Print matches (possibly spanning more than one line) quickly by+ /// avoiding the detection of each individual match in the lines reported+ /// in the given `SinkMatch`.+ ///+ /// This should only be used when the configuration does not demand match+ /// granularity. This may be used when the searcher is in multi line mode.+ fn sink_fast_multi_line(&self) -> io::Result<()> {+ debug_assert!(self.sunk.matches().is_empty());+ // This isn't actually a required invariant for using this method,+ // but if we wind up here and multi line mode is disabled, then we+ // should still treat it as a bug since we should be using matched_fast+ // instead.+ debug_assert!(self.multi_line());++ let line_term = self.searcher.line_terminator().as_byte();+ let mut absolute_byte_offset = self.sunk.absolute_byte_offset();+ for (i, line) in self.sunk.lines(line_term).enumerate() {+ self.write_prelude(+ absolute_byte_offset,+ self.sunk.line_number().map(|n| n + i as u64),+ None,+ )?;+ absolute_byte_offset += line.len() as u64;++ self.write_line(line)?;+ }+ Ok(())+ }++ /// Print a matching line where the configuration of the printer requires+ /// finding each individual match (e.g., for coloring).+ fn sink_slow(&self) -> io::Result<()> {+ debug_assert!(!self.sunk.matches().is_empty());+ debug_assert!(!self.multi_line() || self.is_context());++ if self.config().only_matching {+ for &m in self.sunk.matches() {+ self.write_prelude(+ self.sunk.absolute_byte_offset() + m.start() as u64,+ self.sunk.line_number(),+ Some(m.start() as u64 + 1),+ )?;++ let buf = &self.sunk.bytes()[m];+ self.write_colored_line(&[Match::new(0, buf.len())], buf)?;+ }+ } else if self.config().per_match {+ for &m in self.sunk.matches() {+ self.write_prelude(+ self.sunk.absolute_byte_offset() + m.start() as u64,+ self.sunk.line_number(),+ Some(m.start() as u64 + 1),+ )?;+ self.write_colored_line(&[m], self.sunk.bytes())?;+ }+ } else {+ self.write_prelude(+ self.sunk.absolute_byte_offset(),+ self.sunk.line_number(),+ Some(self.sunk.matches()[0].start() as u64 + 1),+ )?;+ self.write_colored_line(self.sunk.matches(), self.sunk.bytes())?;+ }+ Ok(())+ }++ fn sink_slow_multi_line(&self) -> io::Result<()> {+ debug_assert!(!self.sunk.matches().is_empty());+ debug_assert!(self.multi_line());++ if self.config().only_matching {+ return self.sink_slow_multi_line_only_matching();+ } else if self.config().per_match {+ return self.sink_slow_multi_per_match();+ }++ let line_term = self.searcher.line_terminator().as_byte();+ let bytes = self.sunk.bytes();+ let matches = self.sunk.matches();+ let mut midx = 0;+ let mut count = 0;+ let mut stepper = LineStep::new(line_term, 0, bytes.len());+ while let Some((start, end)) = stepper.next(bytes) {+ let line = Match::new(start, end);+ self.write_prelude(+ self.sunk.absolute_byte_offset() + line.start() as u64,+ self.sunk.line_number().map(|n| n + count),+ Some(matches[0].start() as u64 + 1),+ )?;+ count += 1;+ if self.exceeds_max_columns(&bytes[line]) {+ self.write_exceeded_line(bytes, line, matches, &mut midx)?;+ } else {+ self.write_colored_matches(bytes, line, matches, &mut midx)?;+ self.write_line_term()?;+ }+ }+ Ok(())+ }++ fn sink_slow_multi_line_only_matching(&self) -> io::Result<()> {+ let line_term = self.searcher.line_terminator().as_byte();+ let spec = self.config().colors.matched();+ let bytes = self.sunk.bytes();+ let matches = self.sunk.matches();+ let mut midx = 0;+ let mut count = 0;+ let mut stepper = LineStep::new(line_term, 0, bytes.len());+ while let Some((start, end)) = stepper.next(bytes) {+ let mut line = Match::new(start, end);+ self.trim_line_terminator(bytes, &mut line);+ self.trim_ascii_prefix(bytes, &mut line);+ while !line.is_empty() {+ if matches[midx].end() <= line.start() {+ if midx + 1 < matches.len() {+ midx += 1;+ continue;+ } else {+ break;+ }+ }+ let m = matches[midx];++ if line.start() < m.start() {+ let upto = cmp::min(line.end(), m.start());+ line = line.with_start(upto);+ } else {+ let upto = cmp::min(line.end(), m.end());+ self.write_prelude(+ self.sunk.absolute_byte_offset() + m.start() as u64,+ self.sunk.line_number().map(|n| n + count),+ Some(m.start() as u64 + 1),+ )?;++ let this_line = line.with_end(upto);+ line = line.with_start(upto);+ if self.exceeds_max_columns(&bytes[this_line]) {+ self.write_exceeded_line(+ bytes, this_line, matches, &mut midx,+ )?;+ } else {+ self.write_spec(spec, &bytes[this_line])?;+ self.write_line_term()?;+ }+ }+ }+ count += 1;+ }+ Ok(())+ }++ fn sink_slow_multi_per_match(&self) -> io::Result<()> {+ let line_term = self.searcher.line_terminator().as_byte();+ let spec = self.config().colors.matched();+ let bytes = self.sunk.bytes();+ for &m in self.sunk.matches() {+ let mut count = 0;+ let mut stepper = LineStep::new(line_term, 0, bytes.len());+ while let Some((start, end)) = stepper.next(bytes) {+ let mut line = Match::new(start, end);+ if line.start() >= m.end() {+ break;+ } else if line.end() <= m.start() {+ count += 1;+ continue;+ }+ self.write_prelude(+ self.sunk.absolute_byte_offset() + line.start() as u64,+ self.sunk.line_number().map(|n| n + count),+ Some(m.start() as u64 + 1),+ )?;+ count += 1;+ if self.exceeds_max_columns(&bytes[line]) {+ self.write_exceeded_line(bytes, line, &[m], &mut 0)?;+ continue;+ }+ self.trim_line_terminator(bytes, &mut line);+ self.trim_ascii_prefix(bytes, &mut line);++ while !line.is_empty() {+ if m.end() <= line.start() {+ self.write(&bytes[line])?;+ line = line.with_start(line.end());+ } else if line.start() < m.start() {+ let upto = cmp::min(line.end(), m.start());+ self.write(&bytes[line.with_end(upto)])?;+ line = line.with_start(upto);+ } else {+ let upto = cmp::min(line.end(), m.end());+ self.write_spec(spec, &bytes[line.with_end(upto)])?;+ line = line.with_start(upto);+ }+ }+ self.write_line_term()?;+ }+ }+ Ok(())+ }++ /// Write the beginning part of a matching line. This (may) include things+ /// like the file path, line number among others, depending on the+ /// configuration and the parameters given.+ #[inline(always)]+ fn write_prelude(+ &self,+ absolute_byte_offset: u64,+ line_number: Option<u64>,+ column: Option<u64>,+ ) -> io::Result<()> {+ let sep = self.separator_field();++ if !self.config().heading {+ self.write_path_field(sep)?;+ }+ if let Some(n) = line_number {+ self.write_line_number(n, sep)?;+ }+ if let Some(n) = column {+ if self.config().column {+ self.write_column_number(n, sep)?;+ }+ }+ if self.config().byte_offset {+ self.write_byte_offset(absolute_byte_offset, sep)?;+ }+ Ok(())+ }++ #[inline(always)]+ fn write_line(&self, line: &[u8]) -> io::Result<()> {+ if self.exceeds_max_columns(line) {+ let range = Match::new(0, line.len());+ self.write_exceeded_line(+ line,+ range,+ self.sunk.matches(),+ &mut 0,+ )?;+ } else {+ self.write_trim(line)?;+ if !self.has_line_terminator(line) {+ self.write_line_term()?;+ }+ }+ Ok(())+ }++ fn write_colored_line(+ &self,+ matches: &[Match],+ bytes: &[u8],+ ) -> io::Result<()> {+ // If we know we aren't going to emit color, then we can go faster.+ let spec = self.config().colors.matched();+ if !self.wtr().borrow().supports_color() || spec.is_none() {+ return self.write_line(bytes);+ }++ let line = Match::new(0, bytes.len());+ if self.exceeds_max_columns(bytes) {+ self.write_exceeded_line(bytes, line, matches, &mut 0)+ } else {+ self.write_colored_matches(bytes, line, matches, &mut 0)?;+ self.write_line_term()?;+ Ok(())+ }+ }++ /// Write the `line` portion of `bytes`, with appropriate coloring for+ /// each `match`, starting at `match_index`.+ ///+ /// This accounts for trimming any whitespace prefix and will *never* print+ /// a line terminator. If a match exceeds the range specified by `line`,+ /// then only the part of the match within `line` (if any) is printed.+ fn write_colored_matches(+ &self,+ bytes: &[u8],+ mut line: Match,+ matches: &[Match],+ match_index: &mut usize,+ ) -> io::Result<()> {+ self.trim_line_terminator(bytes, &mut line);+ self.trim_ascii_prefix(bytes, &mut line);+ if matches.is_empty() {+ self.write(&bytes[line])?;+ return Ok(());+ }+ while !line.is_empty() {+ if matches[*match_index].end() <= line.start() {+ if *match_index + 1 < matches.len() {+ *match_index += 1;+ continue;+ } else {+ self.end_color_match()?;+ self.write(&bytes[line])?;+ break;+ }+ }++ let m = matches[*match_index];+ if line.start() < m.start() {+ let upto = cmp::min(line.end(), m.start());+ self.end_color_match()?;+ self.write(&bytes[line.with_end(upto)])?;+ line = line.with_start(upto);+ } else {+ let upto = cmp::min(line.end(), m.end());+ self.start_color_match()?;+ self.write(&bytes[line.with_end(upto)])?;+ line = line.with_start(upto);+ }+ }+ self.end_color_match()?;+ Ok(())+ }++ fn write_exceeded_line(+ &self,+ bytes: &[u8],+ mut line: Match,+ matches: &[Match],+ match_index: &mut usize,+ ) -> io::Result<()> {+ if self.config().max_columns_preview {+ let original = line;+ let end = bytes[line]+ .grapheme_indices()+ .map(|(_, end, _)| end)+ .take(self.config().max_columns.unwrap_or(0) as usize)+ .last()+ .unwrap_or(0)+ + line.start();+ line = line.with_end(end);+ self.write_colored_matches(bytes, line, matches, match_index)?;++ if matches.is_empty() {+ self.write(b" [... omitted end of long line]")?;+ } else {+ let remaining = matches+ .iter()+ .filter(|m| {+ m.start() >= line.end() && m.start() < original.end()+ })+ .count();+ let tense = if remaining == 1 { "match" } else { "matches" };+ write!(+ self.wtr().borrow_mut(),+ " [... {} more {}]",+ remaining,+ tense,+ )?;+ }+ self.write_line_term()?;+ return Ok(());+ }+ if self.sunk.original_matches().is_empty() {+ if self.is_context() {+ self.write(b"[Omitted long context line]")?;+ } else {+ self.write(b"[Omitted long matching line]")?;+ }+ } else {+ if self.config().only_matching {+ if self.is_context() {+ self.write(b"[Omitted long context line]")?;+ } else {+ self.write(b"[Omitted long matching line]")?;+ }+ } else {+ write!(+ self.wtr().borrow_mut(),+ "[Omitted long line with {} matches]",+ self.sunk.original_matches().len(),+ )?;+ }+ }+ self.write_line_term()?;+ Ok(())+ }++ /// If this printer has a file path associated with it, then this will+ /// write that path to the underlying writer followed by a line terminator.+ /// (If a path terminator is set, then that is used instead of the line+ /// terminator.)+ fn write_path_line(&self) -> io::Result<()> {+ if let Some(path) = self.path() {+ self.write_spec(self.config().colors.path(), path.as_bytes())?;+ if let Some(term) = self.config().path_terminator {+ self.write(&[term])?;+ } else {+ self.write_line_term()?;+ }+ }+ Ok(())+ }++ /// If this printer has a file path associated with it, then this will+ /// write that path to the underlying writer followed by the given field+ /// separator. (If a path terminator is set, then that is used instead of+ /// the field separator.)+ fn write_path_field(&self, field_separator: &[u8]) -> io::Result<()> {+ if let Some(path) = self.path() {+ self.write_spec(self.config().colors.path(), path.as_bytes())?;+ if let Some(term) = self.config().path_terminator {+ self.write(&[term])?;+ } else {+ self.write(field_separator)?;+ }+ }+ Ok(())+ }++ fn write_search_prelude(&self) -> io::Result<()> {+ let this_search_written = self.wtr().borrow().count() > 0;+ if this_search_written {+ return Ok(());+ }+ if let Some(ref sep) = *self.config().separator_search {+ let ever_written = self.wtr().borrow().total_count() > 0;+ if ever_written {+ self.write(sep)?;+ self.write_line_term()?;+ }+ }+ if self.config().heading {+ self.write_path_line()?;+ }+ Ok(())+ }++ fn write_binary_message(&self, offset: u64) -> io::Result<()> {+ if self.sink.match_count == 0 {+ return Ok(());+ }++ let bin = self.searcher.binary_detection();+ if let Some(byte) = bin.quit_byte() {+ self.write(b"WARNING: stopped searching binary file ")?;+ if let Some(path) = self.path() {+ self.write_spec(self.config().colors.path(), path.as_bytes())?;+ self.write(b" ")?;+ }+ let remainder = format!(+ "after match (found {:?} byte around offset {})\n",+ [byte].as_bstr(),+ offset,+ );+ self.write(remainder.as_bytes())?;+ } else if let Some(byte) = bin.convert_byte() {+ self.write(b"Binary file ")?;+ if let Some(path) = self.path() {+ self.write_spec(self.config().colors.path(), path.as_bytes())?;+ self.write(b" ")?;+ }+ let remainder = format!(+ "matches (found {:?} byte around offset {})\n",+ [byte].as_bstr(),+ offset,+ );+ self.write(remainder.as_bytes())?;+ }+ Ok(())+ }++ fn write_context_separator(&self) -> io::Result<()> {+ if let Some(ref sep) = *self.config().separator_context {+ self.write(sep)?;+ self.write_line_term()?;+ }+ Ok(())+ }++ fn write_line_number(+ &self,+ line_number: u64,+ field_separator: &[u8],+ ) -> io::Result<()> {+ let n = line_number.to_string();+ self.write_spec(self.config().colors.line(), n.as_bytes())?;+ self.write(field_separator)?;+ Ok(())+ }++ fn write_column_number(+ &self,+ column_number: u64,+ field_separator: &[u8],+ ) -> io::Result<()> {+ let n = column_number.to_string();+ self.write_spec(self.config().colors.column(), n.as_bytes())?;+ self.write(field_separator)?;+ Ok(())+ }++ fn write_byte_offset(+ &self,+ offset: u64,+ field_separator: &[u8],+ ) -> io::Result<()> {+ let n = offset.to_string();+ self.write_spec(self.config().colors.column(), n.as_bytes())?;+ self.write(field_separator)?;+ Ok(())+ }++ fn write_line_term(&self) -> io::Result<()> {+ self.write(self.searcher.line_terminator().as_bytes())+ }++ fn write_spec(&self, spec: &ColorSpec, buf: &[u8]) -> io::Result<()> {+ let mut wtr = self.wtr().borrow_mut();+ wtr.set_color(spec)?;+ wtr.write_all(buf)?;+ wtr.reset()?;+ Ok(())+ }++ fn start_color_match(&self) -> io::Result<()> {+ if self.in_color_match.get() {+ return Ok(());+ }+ self.wtr().borrow_mut().set_color(self.config().colors.matched())?;+ self.in_color_match.set(true);+ Ok(())+ }++ fn end_color_match(&self) -> io::Result<()> {+ if !self.in_color_match.get() {+ return Ok(());+ }+ self.wtr().borrow_mut().reset()?;+ self.in_color_match.set(false);+ Ok(())+ }++ fn write_trim(&self, buf: &[u8]) -> io::Result<()> {+ if !self.config().trim_ascii {+ return self.write(buf);+ }+ let mut range = Match::new(0, buf.len());+ self.trim_ascii_prefix(buf, &mut range);+ self.write(&buf[range])+ }++ fn write(&self, buf: &[u8]) -> io::Result<()> {+ self.wtr().borrow_mut().write_all(buf)+ }++ fn trim_line_terminator(&self, buf: &[u8], line: &mut Match) {+ let lineterm = self.searcher.line_terminator();+ if lineterm.is_suffix(&buf[*line]) {+ let mut end = line.end() - 1;+ if lineterm.is_crlf() && buf[end - 1] == b'\r' {+ end -= 1;+ }+ *line = line.with_end(end);+ }+ }++ fn has_line_terminator(&self, buf: &[u8]) -> bool {+ self.searcher.line_terminator().is_suffix(buf)+ }++ fn is_context(&self) -> bool {+ self.sunk.context_kind().is_some()+ }++ /// Return the underlying configuration for this printer.+ fn config(&self) -> &'a Config {+ &self.sink.standard.config+ }++ /// Return the underlying writer that we are printing to.+ fn wtr(&self) -> &'a RefCell<CounterWriter<W>> {+ &self.sink.standard.wtr+ }++ /// Return the path associated with this printer, if one exists.+ fn path(&self) -> Option<&'a PrinterPath<'a>> {+ self.sink.path.as_ref()+ }++ /// Return the appropriate field separator based on whether we are emitting+ /// matching or contextual lines.+ fn separator_field(&self) -> &[u8] {+ if self.is_context() {+ &self.config().separator_field_context+ } else {+ &self.config().separator_field_match+ }+ }++ /// Returns true if and only if the given line exceeds the maximum number+ /// of columns set. If no maximum is set, then this always returns false.+ fn exceeds_max_columns(&self, line: &[u8]) -> bool {+ self.config().max_columns.map_or(false, |m| line.len() as u64 > m)+ }++ /// Returns true if and only if the searcher may report matches over+ /// multiple lines.+ ///+ /// Note that this doesn't just return whether the searcher is in multi+ /// line mode, but also checks if the mater can match over multiple lines.+ /// If it can't, then we don't need multi line handling, even if the+ /// searcher has multi line mode enabled.+ fn multi_line(&self) -> bool {+ self.searcher.multi_line_with_matcher(&self.sink.matcher)+ }++ /// Trim prefix ASCII spaces from the given slice and return the+ /// corresponding range.+ ///+ /// This stops trimming a prefix as soon as it sees non-whitespace or a+ /// line terminator.+ fn trim_ascii_prefix(&self, slice: &[u8], range: &mut Match) {+ if !self.config().trim_ascii {+ return;+ }+ let lineterm = self.searcher.line_terminator();+ *range = trim_ascii_prefix(lineterm, slice, *range)+ }+}++#[cfg(test)]+mod tests {+ use grep_regex::RegexMatcher;+ use grep_searcher::SearcherBuilder;+ use termcolor::NoColor;++ use super::{Standard, StandardBuilder};++ const SHERLOCK: &'static str = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.\+";++ #[allow(dead_code)]+ const SHERLOCK_CRLF: &'static str = "\+For the Doctor Watsons of this world, as opposed to the Sherlock\r+Holmeses, success in the province of detective work must always\r+be, to a very large extent, the result of luck. Sherlock Holmes\r+can extract a clew from a wisp of straw or a flake of cigar ash;\r+but Doctor Watson has to have it taken out for him and dusted,\r+and exhibited clearly, with a label attached.\+";++ fn printer_contents(printer: &mut Standard<NoColor<Vec<u8>>>) -> String {+ String::from_utf8(printer.get_mut().get_ref().to_owned()).unwrap()+ }++ #[test]+ fn reports_match() {+ let matcher = RegexMatcher::new("Sherlock").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ let mut sink = printer.sink(&matcher);+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, SHERLOCK.as_bytes(), &mut sink)+ .unwrap();+ assert!(sink.has_match());++ let matcher = RegexMatcher::new("zzzzz").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ let mut sink = printer.sink(&matcher);+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, SHERLOCK.as_bytes(), &mut sink)+ .unwrap();+ assert!(!sink.has_match());+ }++ #[test]+ fn reports_binary() {+ use grep_searcher::BinaryDetection;++ let matcher = RegexMatcher::new("Sherlock").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ let mut sink = printer.sink(&matcher);+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, SHERLOCK.as_bytes(), &mut sink)+ .unwrap();+ assert!(sink.binary_byte_offset().is_none());++ let matcher = RegexMatcher::new(".+").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ let mut sink = printer.sink(&matcher);+ SearcherBuilder::new()+ .line_number(false)+ .binary_detection(BinaryDetection::quit(b'\x00'))+ .build()+ .search_reader(&matcher, &b"abc\x00"[..], &mut sink)+ .unwrap();+ assert_eq!(sink.binary_byte_offset(), Some(3));+ }++ #[test]+ fn reports_stats() {+ use std::time::Duration;++ let matcher = RegexMatcher::new("Sherlock|opposed").unwrap();+ let mut printer =+ StandardBuilder::new().stats(true).build(NoColor::new(vec![]));+ let stats = {+ let mut sink = printer.sink(&matcher);+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, SHERLOCK.as_bytes(), &mut sink)+ .unwrap();+ sink.stats().unwrap().clone()+ };+ let buf = printer_contents(&mut printer);++ assert!(stats.elapsed() > Duration::default());+ assert_eq!(stats.searches(), 1);+ assert_eq!(stats.searches_with_match(), 1);+ assert_eq!(stats.bytes_searched(), SHERLOCK.len() as u64);+ assert_eq!(stats.bytes_printed(), buf.len() as u64);+ assert_eq!(stats.matched_lines(), 2);+ assert_eq!(stats.matches(), 3);+ }++ #[test]+ fn reports_stats_multiple() {+ use std::time::Duration;++ let matcher = RegexMatcher::new("Sherlock|opposed").unwrap();+ let mut printer =+ StandardBuilder::new().stats(true).build(NoColor::new(vec![]));+ let stats = {+ let mut sink = printer.sink(&matcher);+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, SHERLOCK.as_bytes(), &mut sink)+ .unwrap();+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, &b"zzzzzzzzzz"[..], &mut sink)+ .unwrap();+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(&matcher, SHERLOCK.as_bytes(), &mut sink)+ .unwrap();+ sink.stats().unwrap().clone()+ };+ let buf = printer_contents(&mut printer);++ assert!(stats.elapsed() > Duration::default());+ assert_eq!(stats.searches(), 3);+ assert_eq!(stats.searches_with_match(), 2);+ assert_eq!(stats.bytes_searched(), 10 + 2 * SHERLOCK.len() as u64);+ assert_eq!(stats.bytes_printed(), buf.len() as u64);+ assert_eq!(stats.matched_lines(), 4);+ assert_eq!(stats.matches(), 6);+ }++ #[test]+ fn context_break() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .separator_context(Some(b"--abc--".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+--abc--+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn context_break_multiple_no_heading() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .separator_search(Some(b"--xyz--".to_vec()))+ .separator_context(Some(b"--abc--".to_vec()))+ .build(NoColor::new(vec![]));++ SearcherBuilder::new()+ .line_number(false)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();+ SearcherBuilder::new()+ .line_number(false)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+--abc--+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+--xyz--+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+--abc--+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn context_break_multiple_heading() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .heading(true)+ .separator_search(Some(b"--xyz--".to_vec()))+ .separator_context(Some(b"--abc--".to_vec()))+ .build(NoColor::new(vec![]));++ SearcherBuilder::new()+ .line_number(false)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();+ SearcherBuilder::new()+ .line_number(false)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+--abc--+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+--xyz--+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+--abc--+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn path() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer =+ StandardBuilder::new().path(false).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the Doctor Watsons of this world, as opposed to the Sherlock+5:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn separator_field() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .separator_field_match(b"!!".to_vec())+ .separator_field_context(b"^^".to_vec())+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+sherlock!!For the Doctor Watsons of this world, as opposed to the Sherlock+sherlock^^Holmeses, success in the province of detective work must always+--+sherlock^^can extract a clew from a wisp of straw or a flake of cigar ash;+sherlock!!but Doctor Watson has to have it taken out for him and dusted,+sherlock^^and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn separator_path() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .separator_path(Some(b'Z'))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "books/sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+booksZsherlock:For the Doctor Watsons of this world, as opposed to the Sherlock+booksZsherlock:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn path_terminator() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .path_terminator(Some(b'Z'))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "books/sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+books/sherlockZFor the Doctor Watsons of this world, as opposed to the Sherlock+books/sherlockZbut Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn heading() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer =+ StandardBuilder::new().heading(true).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+sherlock+For the Doctor Watsons of this world, as opposed to the Sherlock+but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn no_heading() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer =+ StandardBuilder::new().heading(false).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock+sherlock:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn no_heading_multiple() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer =+ StandardBuilder::new().heading(false).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let matcher = RegexMatcher::new("Sherlock").unwrap();+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock+sherlock:but Doctor Watson has to have it taken out for him and dusted,+sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock+sherlock:be, to a very large extent, the result of luck. Sherlock Holmes+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn heading_multiple() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer =+ StandardBuilder::new().heading(true).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let matcher = RegexMatcher::new("Sherlock").unwrap();+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+sherlock+For the Doctor Watsons of this world, as opposed to the Sherlock+but Doctor Watson has to have it taken out for him and dusted,+sherlock+For the Doctor Watsons of this world, as opposed to the Sherlock+be, to a very large extent, the result of luck. Sherlock Holmes+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn trim_ascii() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .trim_ascii(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ " Watson".as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+Watson+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn trim_ascii_multi_line() {+ let matcher = RegexMatcher::new("(?s:.{0})Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .trim_ascii(true)+ .stats(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ " Watson".as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+Watson+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn trim_ascii_with_line_term() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .trim_ascii(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .before_context(1)+ .build()+ .search_reader(+ &matcher,+ "\n Watson".as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1-+2:Watson+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn line_number() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the Doctor Watsons of this world, as opposed to the Sherlock+5:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn line_number_multi_line() {+ let matcher = RegexMatcher::new("(?s)Watson.+Watson").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the Doctor Watsons of this world, as opposed to the Sherlock+2:Holmeses, success in the province of detective work must always+3:be, to a very large extent, the result of luck. Sherlock Holmes+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn column_number() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer =+ StandardBuilder::new().column(true).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+16:For the Doctor Watsons of this world, as opposed to the Sherlock+12:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn column_number_multi_line() {+ let matcher = RegexMatcher::new("(?s)Watson.+Watson").unwrap();+ let mut printer =+ StandardBuilder::new().column(true).build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+16:For the Doctor Watsons of this world, as opposed to the Sherlock+16:Holmeses, success in the province of detective work must always+16:be, to a very large extent, the result of luck. Sherlock Holmes+16:can extract a clew from a wisp of straw or a flake of cigar ash;+16:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn byte_offset() {+ let matcher = RegexMatcher::new("Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .byte_offset(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+0:For the Doctor Watsons of this world, as opposed to the Sherlock+258:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn byte_offset_multi_line() {+ let matcher = RegexMatcher::new("(?s)Watson.+Watson").unwrap();+ let mut printer = StandardBuilder::new()+ .byte_offset(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+0:For the Doctor Watsons of this world, as opposed to the Sherlock+65:Holmeses, success in the province of detective work must always+129:be, to a very large extent, the result of luck. Sherlock Holmes+193:can extract a clew from a wisp of straw or a flake of cigar ash;+258:but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns() {+ let matcher = RegexMatcher::new("ash|dusted").unwrap();+ let mut printer = StandardBuilder::new()+ .max_columns(Some(63))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+[Omitted long matching line]+but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_preview() {+ let matcher = RegexMatcher::new("exhibited|dusted").unwrap();+ let mut printer = StandardBuilder::new()+ .max_columns(Some(46))+ .max_columns_preview(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+but Doctor Watson has to have it taken out for [... omitted end of long line]+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_with_count() {+ let matcher = RegexMatcher::new("cigar|ash|dusted").unwrap();+ let mut printer = StandardBuilder::new()+ .stats(true)+ .max_columns(Some(63))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+[Omitted long line with 2 matches]+but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_with_count_preview_no_match() {+ let matcher = RegexMatcher::new("exhibited|has to have it").unwrap();+ let mut printer = StandardBuilder::new()+ .stats(true)+ .max_columns(Some(46))+ .max_columns_preview(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+but Doctor Watson has to have it taken out for [... 0 more matches]+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_with_count_preview_one_match() {+ let matcher = RegexMatcher::new("exhibited|dusted").unwrap();+ let mut printer = StandardBuilder::new()+ .stats(true)+ .max_columns(Some(46))+ .max_columns_preview(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+but Doctor Watson has to have it taken out for [... 1 more match]+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_with_count_preview_two_matches() {+ let matcher =+ RegexMatcher::new("exhibited|dusted|has to have it").unwrap();+ let mut printer = StandardBuilder::new()+ .stats(true)+ .max_columns(Some(46))+ .max_columns_preview(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+but Doctor Watson has to have it taken out for [... 1 more match]+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_multi_line() {+ let matcher = RegexMatcher::new("(?s)ash.+dusted").unwrap();+ let mut printer = StandardBuilder::new()+ .max_columns(Some(63))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+[Omitted long matching line]+but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_columns_multi_line_preview() {+ let matcher =+ RegexMatcher::new("(?s)clew|cigar ash.+have it|exhibited")+ .unwrap();+ let mut printer = StandardBuilder::new()+ .stats(true)+ .max_columns(Some(46))+ .max_columns_preview(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+can extract a clew from a wisp of straw or a f [... 1 more match]+but Doctor Watson has to have it taken out for [... 0 more matches]+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_matches() {+ let matcher = RegexMatcher::new("Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .max_matches(Some(1))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_matches_context() {+ // after context: 1+ let matcher = RegexMatcher::new("Doctor Watsons").unwrap();+ let mut printer = StandardBuilder::new()+ .max_matches(Some(1))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+";+ assert_eq_printed!(expected, got);++ // after context: 4+ let mut printer = StandardBuilder::new()+ .max_matches(Some(1))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .after_context(4)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);++ // after context: 1, max matches: 2+ let matcher = RegexMatcher::new("Doctor Watsons|but Doctor").unwrap();+ let mut printer = StandardBuilder::new()+ .max_matches(Some(2))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+--+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);++ // after context: 4, max matches: 2+ let mut printer = StandardBuilder::new()+ .max_matches(Some(2))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .after_context(4)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_matches_multi_line1() {+ let matcher = RegexMatcher::new("(?s:.{0})Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .max_matches(Some(1))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn max_matches_multi_line2() {+ let matcher =+ RegexMatcher::new(r"(?s)Watson.+?(Holmeses|clearly)").unwrap();+ let mut printer = StandardBuilder::new()+ .max_matches(Some(1))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .multi_line(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching() {+ let matcher = RegexMatcher::new("Doctor Watsons|Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:Doctor Watsons+1:57:Sherlock+3:49:Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_multi_line1() {+ let matcher =+ RegexMatcher::new(r"(?s:.{0})(Doctor Watsons|Sherlock)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:Doctor Watsons+1:57:Sherlock+3:49:Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_multi_line2() {+ let matcher =+ RegexMatcher::new(r"(?s)Watson.+?(Holmeses|clearly)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:16:Watsons of this world, as opposed to the Sherlock+2:16:Holmeses+5:12:Watson has to have it taken out for him and dusted,+6:12:and exhibited clearly+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_max_columns() {+ let matcher = RegexMatcher::new("Doctor Watsons|Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .max_columns(Some(10))+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:[Omitted long matching line]+1:57:Sherlock+3:49:Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_max_columns_preview() {+ let matcher = RegexMatcher::new("Doctor Watsons|Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .max_columns(Some(10))+ .max_columns_preview(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:Doctor Wat [... 0 more matches]+1:57:Sherlock+3:49:Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_max_columns_multi_line1() {+ // The `(?s:.{0})` trick fools the matcher into thinking that it+ // can match across multiple lines without actually doing so. This is+ // so we can test multi-line handling in the case of a match on only+ // one line.+ let matcher =+ RegexMatcher::new(r"(?s:.{0})(Doctor Watsons|Sherlock)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .max_columns(Some(10))+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:[Omitted long matching line]+1:57:Sherlock+3:49:Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_max_columns_preview_multi_line1() {+ // The `(?s:.{0})` trick fools the matcher into thinking that it+ // can match across multiple lines without actually doing so. This is+ // so we can test multi-line handling in the case of a match on only+ // one line.+ let matcher =+ RegexMatcher::new(r"(?s:.{0})(Doctor Watsons|Sherlock)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .max_columns(Some(10))+ .max_columns_preview(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:Doctor Wat [... 0 more matches]+1:57:Sherlock+3:49:Sherlock+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_max_columns_multi_line2() {+ let matcher =+ RegexMatcher::new(r"(?s)Watson.+?(Holmeses|clearly)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .max_columns(Some(50))+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:16:Watsons of this world, as opposed to the Sherlock+2:16:Holmeses+5:12:[Omitted long matching line]+6:12:and exhibited clearly+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn only_matching_max_columns_preview_multi_line2() {+ let matcher =+ RegexMatcher::new(r"(?s)Watson.+?(Holmeses|clearly)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .max_columns(Some(50))+ .max_columns_preview(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:16:Watsons of this world, as opposed to the Sherlock+2:16:Holmeses+5:12:Watson has to have it taken out for him and dusted [... 0 more matches]+6:12:and exhibited clearly+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn per_match() {+ let matcher = RegexMatcher::new("Doctor Watsons|Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .per_match(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:For the Doctor Watsons of this world, as opposed to the Sherlock+1:57:For the Doctor Watsons of this world, as opposed to the Sherlock+3:49:be, to a very large extent, the result of luck. Sherlock Holmes+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn per_match_multi_line1() {+ let matcher =+ RegexMatcher::new(r"(?s:.{0})(Doctor Watsons|Sherlock)").unwrap();+ let mut printer = StandardBuilder::new()+ .per_match(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:9:For the Doctor Watsons of this world, as opposed to the Sherlock+1:57:For the Doctor Watsons of this world, as opposed to the Sherlock+3:49:be, to a very large extent, the result of luck. Sherlock Holmes+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn per_match_multi_line2() {+ let matcher =+ RegexMatcher::new(r"(?s)Watson.+?(Holmeses|clearly)").unwrap();+ let mut printer = StandardBuilder::new()+ .per_match(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:16:For the Doctor Watsons of this world, as opposed to the Sherlock+2:16:Holmeses, success in the province of detective work must always+5:12:but Doctor Watson has to have it taken out for him and dusted,+6:12:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn per_match_multi_line3() {+ let matcher =+ RegexMatcher::new(r"(?s)Watson.+?Holmeses|always.+?be").unwrap();+ let mut printer = StandardBuilder::new()+ .per_match(true)+ .column(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:16:For the Doctor Watsons of this world, as opposed to the Sherlock+2:16:Holmeses, success in the province of detective work must always+2:123:Holmeses, success in the province of detective work must always+3:123:be, to a very large extent, the result of luck. Sherlock Holmes+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement_passthru() {+ let matcher = RegexMatcher::new(r"Sherlock|Doctor (\w+)").unwrap();+ let mut printer = StandardBuilder::new()+ .replacement(Some(b"doctah $1 MD".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .passthru(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the doctah Watsons MD of this world, as opposed to the doctah MD+2-Holmeses, success in the province of detective work must always+3:be, to a very large extent, the result of luck. doctah MD Holmes+4-can extract a clew from a wisp of straw or a flake of cigar ash;+5:but doctah Watson MD has to have it taken out for him and dusted,+6-and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement() {+ let matcher = RegexMatcher::new(r"Sherlock|Doctor (\w+)").unwrap();+ let mut printer = StandardBuilder::new()+ .replacement(Some(b"doctah $1 MD".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the doctah Watsons MD of this world, as opposed to the doctah MD+3:be, to a very large extent, the result of luck. doctah MD Holmes+5:but doctah Watson MD has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement_max_columns() {+ let matcher = RegexMatcher::new(r"Sherlock|Doctor (\w+)").unwrap();+ let mut printer = StandardBuilder::new()+ .max_columns(Some(67))+ .replacement(Some(b"doctah $1 MD".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:[Omitted long line with 2 matches]+3:be, to a very large extent, the result of luck. doctah MD Holmes+5:but doctah Watson MD has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement_max_columns_preview1() {+ let matcher = RegexMatcher::new(r"Sherlock|Doctor (\w+)").unwrap();+ let mut printer = StandardBuilder::new()+ .max_columns(Some(67))+ .max_columns_preview(true)+ .replacement(Some(b"doctah $1 MD".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the doctah Watsons MD of this world, as opposed to the doctah [... 0 more matches]+3:be, to a very large extent, the result of luck. doctah MD Holmes+5:but doctah Watson MD has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement_max_columns_preview2() {+ let matcher =+ RegexMatcher::new("exhibited|dusted|has to have it").unwrap();+ let mut printer = StandardBuilder::new()+ .max_columns(Some(43))+ .max_columns_preview(true)+ .replacement(Some(b"xxx".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(false)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+but Doctor Watson xxx taken out for him and [... 1 more match]+and xxx clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement_only_matching() {+ let matcher = RegexMatcher::new(r"Sherlock|Doctor (\w+)").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .replacement(Some(b"doctah $1 MD".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:doctah Watsons MD+1:doctah MD+3:doctah MD+5:doctah Watson MD+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn replacement_per_match() {+ let matcher = RegexMatcher::new(r"Sherlock|Doctor (\w+)").unwrap();+ let mut printer = StandardBuilder::new()+ .per_match(true)+ .replacement(Some(b"doctah $1 MD".to_vec()))+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1:For the doctah Watsons MD of this world, as opposed to the doctah MD+1:For the doctah Watsons MD of this world, as opposed to the doctah MD+3:be, to a very large extent, the result of luck. doctah MD Holmes+5:but doctah Watson MD has to have it taken out for him and dusted,+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn invert() {+ let matcher = RegexMatcher::new(r"Sherlock").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .invert_match(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+2:Holmeses, success in the province of detective work must always+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+6:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn invert_multi_line() {+ let matcher = RegexMatcher::new(r"(?s:.{0})Sherlock").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .invert_match(true)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+2:Holmeses, success in the province of detective work must always+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+6:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn invert_context() {+ let matcher = RegexMatcher::new(r"Sherlock").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .invert_match(true)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1-For the Doctor Watsons of this world, as opposed to the Sherlock+2:Holmeses, success in the province of detective work must always+3-be, to a very large extent, the result of luck. Sherlock Holmes+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+6:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn invert_context_multi_line() {+ let matcher = RegexMatcher::new(r"(?s:.{0})Sherlock").unwrap();+ let mut printer = StandardBuilder::new().build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .invert_match(true)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1-For the Doctor Watsons of this world, as opposed to the Sherlock+2:Holmeses, success in the province of detective work must always+3-be, to a very large extent, the result of luck. Sherlock Holmes+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+6:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn invert_context_only_matching() {+ let matcher = RegexMatcher::new(r"Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .line_number(true)+ .invert_match(true)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1-Sherlock+2:Holmeses, success in the province of detective work must always+3-Sherlock+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+6:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }++ #[test]+ fn invert_context_only_matching_multi_line() {+ let matcher = RegexMatcher::new(r"(?s:.{0})Sherlock").unwrap();+ let mut printer = StandardBuilder::new()+ .only_matching(true)+ .build(NoColor::new(vec![]));+ SearcherBuilder::new()+ .multi_line(true)+ .line_number(true)+ .invert_match(true)+ .before_context(1)+ .after_context(1)+ .build()+ .search_reader(+ &matcher,+ SHERLOCK.as_bytes(),+ printer.sink(&matcher),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ let expected = "\+1-Sherlock+2:Holmeses, success in the province of detective work must always+3-Sherlock+4:can extract a clew from a wisp of straw or a flake of cigar ash;+5:but Doctor Watson has to have it taken out for him and dusted,+6:and exhibited clearly, with a label attached.+";+ assert_eq_printed!(expected, got);+ }+}
@@ -0,0 +1,147 @@+use std::ops::{Add, AddAssign};+use std::time::Duration;++use util::NiceDuration;++/// Summary statistics produced at the end of a search.+///+/// When statistics are reported by a printer, they correspond to all searches+/// executed with that printer.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+#[cfg_attr(feature = "serde1", derive(Serialize))]+pub struct Stats {+ elapsed: NiceDuration,+ searches: u64,+ searches_with_match: u64,+ bytes_searched: u64,+ bytes_printed: u64,+ matched_lines: u64,+ matches: u64,+}++impl Add for Stats {+ type Output = Stats;++ fn add(self, rhs: Stats) -> Stats {+ self + &rhs+ }+}++impl<'a> Add<&'a Stats> for Stats {+ type Output = Stats;++ fn add(self, rhs: &'a Stats) -> Stats {+ Stats {+ elapsed: NiceDuration(self.elapsed.0 + rhs.elapsed.0),+ searches: self.searches + rhs.searches,+ searches_with_match: self.searches_with_match+ + rhs.searches_with_match,+ bytes_searched: self.bytes_searched + rhs.bytes_searched,+ bytes_printed: self.bytes_printed + rhs.bytes_printed,+ matched_lines: self.matched_lines + rhs.matched_lines,+ matches: self.matches + rhs.matches,+ }+ }+}++impl AddAssign for Stats {+ fn add_assign(&mut self, rhs: Stats) {+ *self += &rhs;+ }+}++impl<'a> AddAssign<&'a Stats> for Stats {+ fn add_assign(&mut self, rhs: &'a Stats) {+ self.elapsed.0 += rhs.elapsed.0;+ self.searches += rhs.searches;+ self.searches_with_match += rhs.searches_with_match;+ self.bytes_searched += rhs.bytes_searched;+ self.bytes_printed += rhs.bytes_printed;+ self.matched_lines += rhs.matched_lines;+ self.matches += rhs.matches;+ }+}++impl Stats {+ /// Return a new value for tracking aggregate statistics across searches.+ ///+ /// All statistics are set to `0`.+ pub fn new() -> Stats {+ Stats::default()+ }++ /// Return the total amount of time elapsed.+ pub fn elapsed(&self) -> Duration {+ self.elapsed.0+ }++ /// Return the total number of searches executed.+ pub fn searches(&self) -> u64 {+ self.searches+ }++ /// Return the total number of searches that found at least one match.+ pub fn searches_with_match(&self) -> u64 {+ self.searches_with_match+ }++ /// Return the total number of bytes searched.+ pub fn bytes_searched(&self) -> u64 {+ self.bytes_searched+ }++ /// Return the total number of bytes printed.+ pub fn bytes_printed(&self) -> u64 {+ self.bytes_printed+ }++ /// Return the total number of lines that participated in a match.+ ///+ /// When matches may contain multiple lines then this includes every line+ /// that is part of every match.+ pub fn matched_lines(&self) -> u64 {+ self.matched_lines+ }++ /// Return the total number of matches.+ ///+ /// There may be multiple matches per line.+ pub fn matches(&self) -> u64 {+ self.matches+ }++ /// Add to the elapsed time.+ pub fn add_elapsed(&mut self, duration: Duration) {+ self.elapsed.0 += duration;+ }++ /// Add to the number of searches executed.+ pub fn add_searches(&mut self, n: u64) {+ self.searches += n;+ }++ /// Add to the number of searches that found at least one match.+ pub fn add_searches_with_match(&mut self, n: u64) {+ self.searches_with_match += n;+ }++ /// Add to the total number of bytes searched.+ pub fn add_bytes_searched(&mut self, n: u64) {+ self.bytes_searched += n;+ }++ /// Add to the total number of bytes printed.+ pub fn add_bytes_printed(&mut self, n: u64) {+ self.bytes_printed += n;+ }++ /// Add to the total number of lines that participated in a match.+ pub fn add_matched_lines(&mut self, n: u64) {+ self.matched_lines += n;+ }++ /// Add to the total number of matches.+ pub fn add_matches(&mut self, n: u64) {+ self.matches += n;+ }+}
@@ -0,0 +1,1060 @@+use std::cell::RefCell;+use std::io::{self, Write};+use std::path::Path;+use std::sync::Arc;+use std::time::Instant;++use grep_matcher::Matcher;+use grep_searcher::{Searcher, Sink, SinkError, SinkFinish, SinkMatch};+use termcolor::{ColorSpec, NoColor, WriteColor};++use color::ColorSpecs;+use counter::CounterWriter;+use stats::Stats;+use util::PrinterPath;++/// The configuration for the summary printer.+///+/// This is manipulated by the SummaryBuilder and then referenced by the actual+/// implementation. Once a printer is build, the configuration is frozen and+/// cannot changed.+#[derive(Debug, Clone)]+struct Config {+ kind: SummaryKind,+ colors: ColorSpecs,+ stats: bool,+ path: bool,+ max_matches: Option<u64>,+ exclude_zero: bool,+ separator_field: Arc<Vec<u8>>,+ separator_path: Option<u8>,+ path_terminator: Option<u8>,+}++impl Default for Config {+ fn default() -> Config {+ Config {+ kind: SummaryKind::Count,+ colors: ColorSpecs::default(),+ stats: false,+ path: true,+ max_matches: None,+ exclude_zero: true,+ separator_field: Arc::new(b":".to_vec()),+ separator_path: None,+ path_terminator: None,+ }+ }+}++/// The type of summary output (if any) to print.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum SummaryKind {+ /// Show only a count of the total number of matches (counting each line+ /// at most once) found.+ ///+ /// If the `path` setting is enabled, then the count is prefixed by the+ /// corresponding file path.+ Count,+ /// Show only a count of the total number of matches (counting possibly+ /// many matches on each line) found.+ ///+ /// If the `path` setting is enabled, then the count is prefixed by the+ /// corresponding file path.+ CountMatches,+ /// Show only the file path if and only if a match was found.+ ///+ /// This ignores the `path` setting and always shows the file path. If no+ /// file path is provided, then searching will immediately stop and return+ /// an error.+ PathWithMatch,+ /// Show only the file path if and only if a match was found.+ ///+ /// This ignores the `path` setting and always shows the file path. If no+ /// file path is provided, then searching will immediately stop and return+ /// an error.+ PathWithoutMatch,+ /// Don't show any output and the stop the search once a match is found.+ ///+ /// Note that if `stats` is enabled, then searching continues in order to+ /// compute statistics.+ Quiet,+}++impl SummaryKind {+ /// Returns true if and only if this output mode requires a file path.+ ///+ /// When an output mode requires a file path, then the summary printer+ /// will report an error at the start of every search that lacks a file+ /// path.+ fn requires_path(&self) -> bool {+ use self::SummaryKind::*;++ match *self {+ PathWithMatch | PathWithoutMatch => true,+ Count | CountMatches | Quiet => false,+ }+ }++ /// Returns true if and only if this output mode requires computing+ /// statistics, regardless of whether they have been enabled or not.+ fn requires_stats(&self) -> bool {+ use self::SummaryKind::*;++ match *self {+ CountMatches => true,+ Count | PathWithMatch | PathWithoutMatch | Quiet => false,+ }+ }++ /// Returns true if and only if a printer using this output mode can+ /// quit after seeing the first match.+ fn quit_early(&self) -> bool {+ use self::SummaryKind::*;++ match *self {+ PathWithMatch | Quiet => true,+ Count | CountMatches | PathWithoutMatch => false,+ }+ }+}++/// A builder for summary printer.+///+/// The builder permits configuring how the printer behaves. The summary+/// printer has fewer configuration options than the standard printer because+/// it aims to produce aggregate output about a single search (typically just+/// one line) instead of output for each match.+///+/// Once a `Summary` printer is built, its configuration cannot be changed.+#[derive(Clone, Debug)]+pub struct SummaryBuilder {+ config: Config,+}++impl SummaryBuilder {+ /// Return a new builder for configuring the summary printer.+ pub fn new() -> SummaryBuilder {+ SummaryBuilder { config: Config::default() }+ }++ /// Build a printer using any implementation of `termcolor::WriteColor`.+ ///+ /// The implementation of `WriteColor` used here controls whether colors+ /// are used or not when colors have been configured using the+ /// `color_specs` method.+ ///+ /// For maximum portability, callers should generally use either+ /// `termcolor::StandardStream` or `termcolor::BufferedStandardStream`+ /// where appropriate, which will automatically enable colors on Windows+ /// when possible.+ ///+ /// However, callers may also provide an arbitrary writer using the+ /// `termcolor::Ansi` or `termcolor::NoColor` wrappers, which always enable+ /// colors via ANSI escapes or always disable colors, respectively.+ ///+ /// As a convenience, callers may use `build_no_color` to automatically+ /// select the `termcolor::NoColor` wrapper to avoid needing to import+ /// from `termcolor` explicitly.+ pub fn build<W: WriteColor>(&self, wtr: W) -> Summary<W> {+ Summary {+ config: self.config.clone(),+ wtr: RefCell::new(CounterWriter::new(wtr)),+ }+ }++ /// Build a printer from any implementation of `io::Write` and never emit+ /// any colors, regardless of the user color specification settings.+ ///+ /// This is a convenience routine for+ /// `SummaryBuilder::build(termcolor::NoColor::new(wtr))`.+ pub fn build_no_color<W: io::Write>(&self, wtr: W) -> Summary<NoColor<W>> {+ self.build(NoColor::new(wtr))+ }++ /// Set the output mode for this printer.+ ///+ /// The output mode controls how aggregate results of a search are printed.+ ///+ /// By default, this printer uses the `Count` mode.+ pub fn kind(&mut self, kind: SummaryKind) -> &mut SummaryBuilder {+ self.config.kind = kind;+ self+ }++ /// Set the user color specifications to use for coloring in this printer.+ ///+ /// A [`UserColorSpec`](struct.UserColorSpec.html) can be constructed from+ /// a string in accordance with the color specification format. See the+ /// `UserColorSpec` type documentation for more details on the format.+ /// A [`ColorSpecs`](struct.ColorSpecs.html) can then be generated from+ /// zero or more `UserColorSpec`s.+ ///+ /// Regardless of the color specifications provided here, whether color+ /// is actually used or not is determined by the implementation of+ /// `WriteColor` provided to `build`. For example, if `termcolor::NoColor`+ /// is provided to `build`, then no color will ever be printed regardless+ /// of the color specifications provided here.+ ///+ /// This completely overrides any previous color specifications. This does+ /// not add to any previously provided color specifications on this+ /// builder.+ ///+ /// The default color specifications provide no styling.+ pub fn color_specs(&mut self, specs: ColorSpecs) -> &mut SummaryBuilder {+ self.config.colors = specs;+ self+ }++ /// Enable the gathering of various aggregate statistics.+ ///+ /// When this is enabled (it's disabled by default), statistics will be+ /// gathered for all uses of `Summary` printer returned by `build`,+ /// including but not limited to, the total number of matches, the total+ /// number of bytes searched and the total number of bytes printed.+ ///+ /// Aggregate statistics can be accessed via the sink's+ /// [`SummarySink::stats`](struct.SummarySink.html#method.stats)+ /// method.+ ///+ /// When this is enabled, this printer may need to do extra work in order+ /// to compute certain statistics, which could cause the search to take+ /// longer. For example, in `Quiet` mode, a search can quit after finding+ /// the first match, but if `stats` is enabled, then the search will+ /// continue after the first match in order to compute statistics.+ ///+ /// For a complete description of available statistics, see+ /// [`Stats`](struct.Stats.html).+ ///+ /// Note that some output modes, such as `CountMatches`, automatically+ /// enable this option even if it has been explicitly disabled.+ pub fn stats(&mut self, yes: bool) -> &mut SummaryBuilder {+ self.config.stats = yes;+ self+ }++ /// When enabled, if a path was given to the printer, then it is shown in+ /// the output (either as a heading or as a prefix to each matching line).+ /// When disabled, then no paths are ever included in the output even when+ /// a path is provided to the printer.+ ///+ /// This setting has no effect in `PathWithMatch` and `PathWithoutMatch`+ /// modes.+ ///+ /// This is enabled by default.+ pub fn path(&mut self, yes: bool) -> &mut SummaryBuilder {+ self.config.path = yes;+ self+ }++ /// Set the maximum amount of matches that are printed.+ ///+ /// If multi line search is enabled and a match spans multiple lines, then+ /// that match is counted exactly once for the purposes of enforcing this+ /// limit, regardless of how many lines it spans.+ ///+ /// This is disabled by default.+ pub fn max_matches(&mut self, limit: Option<u64>) -> &mut SummaryBuilder {+ self.config.max_matches = limit;+ self+ }++ /// Exclude count-related summary results with no matches.+ ///+ /// When enabled and the mode is either `Count` or `CountMatches`, then+ /// results are not printed if no matches were found. Otherwise, every+ /// search prints a result with a possibly `0` number of matches.+ ///+ /// This is enabled by default.+ pub fn exclude_zero(&mut self, yes: bool) -> &mut SummaryBuilder {+ self.config.exclude_zero = yes;+ self+ }++ /// Set the separator used between fields for the `Count` and+ /// `CountMatches` modes.+ ///+ /// By default, this is set to `:`.+ pub fn separator_field(&mut self, sep: Vec<u8>) -> &mut SummaryBuilder {+ self.config.separator_field = Arc::new(sep);+ self+ }++ /// Set the path separator used when printing file paths.+ ///+ /// Typically, printing is done by emitting the file path as is. However,+ /// this setting provides the ability to use a different path separator+ /// from what the current environment has configured.+ ///+ /// A typical use for this option is to permit cygwin users on Windows to+ /// set the path separator to `/` instead of using the system default of+ /// `\`.+ ///+ /// This is disabled by default.+ pub fn separator_path(&mut self, sep: Option<u8>) -> &mut SummaryBuilder {+ self.config.separator_path = sep;+ self+ }++ /// Set the path terminator used.+ ///+ /// The path terminator is a byte that is printed after every file path+ /// emitted by this printer.+ ///+ /// If no path terminator is set (the default), then paths are terminated+ /// by either new lines or the configured field separator.+ pub fn path_terminator(+ &mut self,+ terminator: Option<u8>,+ ) -> &mut SummaryBuilder {+ self.config.path_terminator = terminator;+ self+ }+}++/// The summary printer, which emits aggregate results from a search.+///+/// Aggregate results generally correspond to file paths and/or the number of+/// matches found.+///+/// A default printer can be created with either of the `Summary::new` or+/// `Summary::new_no_color` constructors. However, there are a number of+/// options that configure this printer's output. Those options can be+/// configured using [`SummaryBuilder`](struct.SummaryBuilder.html).+///+/// This type is generic over `W`, which represents any implementation of+/// the `termcolor::WriteColor` trait.+#[derive(Debug)]+pub struct Summary<W> {+ config: Config,+ wtr: RefCell<CounterWriter<W>>,+}++impl<W: WriteColor> Summary<W> {+ /// Return a summary printer with a default configuration that writes+ /// matches to the given writer.+ ///+ /// The writer should be an implementation of `termcolor::WriteColor`+ /// and not just a bare implementation of `io::Write`. To use a normal+ /// `io::Write` implementation (simultaneously sacrificing colors), use+ /// the `new_no_color` constructor.+ ///+ /// The default configuration uses the `Count` summary mode.+ pub fn new(wtr: W) -> Summary<W> {+ SummaryBuilder::new().build(wtr)+ }+}++impl<W: io::Write> Summary<NoColor<W>> {+ /// Return a summary printer with a default configuration that writes+ /// matches to the given writer.+ ///+ /// The writer can be any implementation of `io::Write`. With this+ /// constructor, the printer will never emit colors.+ ///+ /// The default configuration uses the `Count` summary mode.+ pub fn new_no_color(wtr: W) -> Summary<NoColor<W>> {+ SummaryBuilder::new().build_no_color(wtr)+ }+}++impl<W: WriteColor> Summary<W> {+ /// Return an implementation of `Sink` for the summary printer.+ ///+ /// This does not associate the printer with a file path, which means this+ /// implementation will never print a file path. If the output mode of+ /// this summary printer does not make sense without a file path (such as+ /// `PathWithMatch` or `PathWithoutMatch`), then any searches executed+ /// using this sink will immediately quit with an error.+ pub fn sink<'s, M: Matcher>(+ &'s mut self,+ matcher: M,+ ) -> SummarySink<'static, 's, M, W> {+ let stats = if self.config.stats || self.config.kind.requires_stats() {+ Some(Stats::new())+ } else {+ None+ };+ SummarySink {+ matcher: matcher,+ summary: self,+ path: None,+ start_time: Instant::now(),+ match_count: 0,+ binary_byte_offset: None,+ stats: stats,+ }+ }++ /// Return an implementation of `Sink` associated with a file path.+ ///+ /// When the printer is associated with a path, then it may, depending on+ /// its configuration, print the path.+ pub fn sink_with_path<'p, 's, M, P>(+ &'s mut self,+ matcher: M,+ path: &'p P,+ ) -> SummarySink<'p, 's, M, W>+ where+ M: Matcher,+ P: ?Sized + AsRef<Path>,+ {+ if !self.config.path && !self.config.kind.requires_path() {+ return self.sink(matcher);+ }+ let stats = if self.config.stats || self.config.kind.requires_stats() {+ Some(Stats::new())+ } else {+ None+ };+ let ppath = PrinterPath::with_separator(+ path.as_ref(),+ self.config.separator_path,+ );+ SummarySink {+ matcher: matcher,+ summary: self,+ path: Some(ppath),+ start_time: Instant::now(),+ match_count: 0,+ binary_byte_offset: None,+ stats: stats,+ }+ }+}++impl<W> Summary<W> {+ /// Returns true if and only if this printer has written at least one byte+ /// to the underlying writer during any of the previous searches.+ pub fn has_written(&self) -> bool {+ self.wtr.borrow().total_count() > 0+ }++ /// Return a mutable reference to the underlying writer.+ pub fn get_mut(&mut self) -> &mut W {+ self.wtr.get_mut().get_mut()+ }++ /// Consume this printer and return back ownership of the underlying+ /// writer.+ pub fn into_inner(self) -> W {+ self.wtr.into_inner().into_inner()+ }+}++/// An implementation of `Sink` associated with a matcher and an optional file+/// path for the summary printer.+///+/// This type is generic over a few type parameters:+///+/// * `'p` refers to the lifetime of the file path, if one is provided. When+/// no file path is given, then this is `'static`.+/// * `'s` refers to the lifetime of the+/// [`Summary`](struct.Summary.html)+/// printer that this type borrows.+/// * `M` refers to the type of matcher used by+/// `grep_searcher::Searcher` that is reporting results to this sink.+/// * `W` refers to the underlying writer that this printer is writing its+/// output to.+#[derive(Debug)]+pub struct SummarySink<'p, 's, M: Matcher, W: 's> {+ matcher: M,+ summary: &'s mut Summary<W>,+ path: Option<PrinterPath<'p>>,+ start_time: Instant,+ match_count: u64,+ binary_byte_offset: Option<u64>,+ stats: Option<Stats>,+}++impl<'p, 's, M: Matcher, W: WriteColor> SummarySink<'p, 's, M, W> {+ /// Returns true if and only if this printer received a match in the+ /// previous search.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search.+ pub fn has_match(&self) -> bool {+ match self.summary.config.kind {+ SummaryKind::PathWithoutMatch => self.match_count == 0,+ _ => self.match_count > 0,+ }+ }++ /// If binary data was found in the previous search, this returns the+ /// offset at which the binary data was first detected.+ ///+ /// The offset returned is an absolute offset relative to the entire+ /// set of bytes searched.+ ///+ /// This is unaffected by the result of searches before the previous+ /// search. e.g., If the search prior to the previous search found binary+ /// data but the previous search found no binary data, then this will+ /// return `None`.+ pub fn binary_byte_offset(&self) -> Option<u64> {+ self.binary_byte_offset+ }++ /// Return a reference to the stats produced by the printer for all+ /// searches executed on this sink.+ ///+ /// This only returns stats if they were requested via the+ /// [`SummaryBuilder`](struct.SummaryBuilder.html)+ /// configuration.+ pub fn stats(&self) -> Option<&Stats> {+ self.stats.as_ref()+ }++ /// Returns true if this printer should quit.+ ///+ /// This implements the logic for handling quitting after seeing a certain+ /// amount of matches. In most cases, the logic is simple, but we must+ /// permit all "after" contextual lines to print after reaching the limit.+ fn should_quit(&self) -> bool {+ let limit = match self.summary.config.max_matches {+ None => return false,+ Some(limit) => limit,+ };+ self.match_count >= limit+ }++ /// If this printer has a file path associated with it, then this will+ /// write that path to the underlying writer followed by a line terminator.+ /// (If a path terminator is set, then that is used instead of the line+ /// terminator.)+ fn write_path_line(&self, searcher: &Searcher) -> io::Result<()> {+ if let Some(ref path) = self.path {+ self.write_spec(+ self.summary.config.colors.path(),+ path.as_bytes(),+ )?;+ if let Some(term) = self.summary.config.path_terminator {+ self.write(&[term])?;+ } else {+ self.write_line_term(searcher)?;+ }+ }+ Ok(())+ }++ /// If this printer has a file path associated with it, then this will+ /// write that path to the underlying writer followed by the field+ /// separator. (If a path terminator is set, then that is used instead of+ /// the field separator.)+ fn write_path_field(&self) -> io::Result<()> {+ if let Some(ref path) = self.path {+ self.write_spec(+ self.summary.config.colors.path(),+ path.as_bytes(),+ )?;+ if let Some(term) = self.summary.config.path_terminator {+ self.write(&[term])?;+ } else {+ self.write(&self.summary.config.separator_field)?;+ }+ }+ Ok(())+ }++ /// Write the line terminator configured on the given searcher.+ fn write_line_term(&self, searcher: &Searcher) -> io::Result<()> {+ self.write(searcher.line_terminator().as_bytes())+ }++ /// Write the given bytes using the give style.+ fn write_spec(&self, spec: &ColorSpec, buf: &[u8]) -> io::Result<()> {+ self.summary.wtr.borrow_mut().set_color(spec)?;+ self.write(buf)?;+ self.summary.wtr.borrow_mut().reset()?;+ Ok(())+ }++ /// Write all of the given bytes.+ fn write(&self, buf: &[u8]) -> io::Result<()> {+ self.summary.wtr.borrow_mut().write_all(buf)+ }+}++impl<'p, 's, M: Matcher, W: WriteColor> Sink for SummarySink<'p, 's, M, W> {+ type Error = io::Error;++ fn matched(+ &mut self,+ _searcher: &Searcher,+ mat: &SinkMatch,+ ) -> Result<bool, io::Error> {+ self.match_count += 1;+ if let Some(ref mut stats) = self.stats {+ let mut match_count = 0;+ self.matcher+ .find_iter(mat.bytes(), |_| {+ match_count += 1;+ true+ })+ .map_err(io::Error::error_message)?;+ stats.add_matches(match_count);+ stats.add_matched_lines(mat.lines().count() as u64);+ } else if self.summary.config.kind.quit_early() {+ return Ok(false);+ }+ Ok(!self.should_quit())+ }++ fn begin(&mut self, _searcher: &Searcher) -> Result<bool, io::Error> {+ if self.path.is_none() && self.summary.config.kind.requires_path() {+ return Err(io::Error::error_message(format!(+ "output kind {:?} requires a file path",+ self.summary.config.kind,+ )));+ }+ self.summary.wtr.borrow_mut().reset_count();+ self.start_time = Instant::now();+ self.match_count = 0;+ self.binary_byte_offset = None;+ if self.summary.config.max_matches == Some(0) {+ return Ok(false);+ }++ Ok(true)+ }++ fn finish(+ &mut self,+ searcher: &Searcher,+ finish: &SinkFinish,+ ) -> Result<(), io::Error> {+ self.binary_byte_offset = finish.binary_byte_offset();+ if let Some(ref mut stats) = self.stats {+ stats.add_elapsed(self.start_time.elapsed());+ stats.add_searches(1);+ if self.match_count > 0 {+ stats.add_searches_with_match(1);+ }+ stats.add_bytes_searched(finish.byte_count());+ stats.add_bytes_printed(self.summary.wtr.borrow().count());+ }+ // If our binary detection method says to quit after seeing binary+ // data, then we shouldn't print any results at all, even if we've+ // found a match before detecting binary data. The intent here is to+ // keep BinaryDetection::quit as a form of filter. Otherwise, we can+ // present a matching file with a smaller number of matches than+ // there might be, which can be quite misleading.+ //+ // If our binary detection method is to convert binary data, then we+ // don't quit and therefore search the entire contents of the file.+ //+ // There is an unfortunate inconsistency here. Namely, when using+ // Quiet or PathWithMatch, then the printer can quit after the first+ // match seen, which could be long before seeing binary data. This+ // means that using PathWithMatch can print a path where as using+ // Count might not print it at all because of binary data.+ //+ // It's not possible to fix this without also potentially significantly+ // impacting the performance of Quiet or PathWithMatch, so we accept+ // the bug.+ if self.binary_byte_offset.is_some()+ && searcher.binary_detection().quit_byte().is_some()+ {+ // Squash the match count. The statistics reported will still+ // contain the match count, but the "official" match count should+ // be zero.+ self.match_count = 0;+ return Ok(());+ }++ let show_count =+ !self.summary.config.exclude_zero || self.match_count > 0;+ match self.summary.config.kind {+ SummaryKind::Count => {+ if show_count {+ self.write_path_field()?;+ self.write(self.match_count.to_string().as_bytes())?;+ self.write_line_term(searcher)?;+ }+ }+ SummaryKind::CountMatches => {+ if show_count {+ let stats = self+ .stats+ .as_ref()+ .expect("CountMatches should enable stats tracking");+ self.write_path_field()?;+ self.write(stats.matches().to_string().as_bytes())?;+ self.write_line_term(searcher)?;+ }+ }+ SummaryKind::PathWithMatch => {+ if self.match_count > 0 {+ self.write_path_line(searcher)?;+ }+ }+ SummaryKind::PathWithoutMatch => {+ if self.match_count == 0 {+ self.write_path_line(searcher)?;+ }+ }+ SummaryKind::Quiet => {}+ }+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use grep_regex::RegexMatcher;+ use grep_searcher::SearcherBuilder;+ use termcolor::NoColor;++ use super::{Summary, SummaryBuilder, SummaryKind};++ const SHERLOCK: &'static [u8] = b"\+For the Doctor Watsons of this world, as opposed to the Sherlock+Holmeses, success in the province of detective work must always+be, to a very large extent, the result of luck. Sherlock Holmes+can extract a clew from a wisp of straw or a flake of cigar ash;+but Doctor Watson has to have it taken out for him and dusted,+and exhibited clearly, with a label attached.+";++ fn printer_contents(printer: &mut Summary<NoColor<Vec<u8>>>) -> String {+ String::from_utf8(printer.get_mut().get_ref().to_owned()).unwrap()+ }++ #[test]+ fn path_with_match_error() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::PathWithMatch)+ .build_no_color(vec![]);+ let res = SearcherBuilder::new().build().search_reader(+ &matcher,+ SHERLOCK,+ printer.sink(&matcher),+ );+ assert!(res.is_err());+ }++ #[test]+ fn path_without_match_error() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::PathWithoutMatch)+ .build_no_color(vec![]);+ let res = SearcherBuilder::new().build().search_reader(+ &matcher,+ SHERLOCK,+ printer.sink(&matcher),+ );+ assert!(res.is_err());+ }++ #[test]+ fn count_no_path() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, printer.sink(&matcher))+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("2\n", got);+ }++ #[test]+ fn count_no_path_even_with_path() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .path(false)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("2\n", got);+ }++ #[test]+ fn count_path() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlock:2\n", got);+ }++ #[test]+ fn count_path_with_zero() {+ let matcher = RegexMatcher::new(r"NO MATCH").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .exclude_zero(false)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlock:0\n", got);+ }++ #[test]+ fn count_path_without_zero() {+ let matcher = RegexMatcher::new(r"NO MATCH").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .exclude_zero(true)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("", got);+ }++ #[test]+ fn count_path_field_separator() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .separator_field(b"ZZ".to_vec())+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlockZZ2\n", got);+ }++ #[test]+ fn count_path_terminator() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .path_terminator(Some(b'\x00'))+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlock\x002\n", got);+ }++ #[test]+ fn count_path_separator() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .separator_path(Some(b'\\'))+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "/home/andrew/sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("\\home\\andrew\\sherlock:2\n", got);+ }++ #[test]+ fn count_max_matches() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Count)+ .max_matches(Some(1))+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, printer.sink(&matcher))+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("1\n", got);+ }++ #[test]+ fn count_matches() {+ let matcher = RegexMatcher::new(r"Watson|Sherlock").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::CountMatches)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlock:4\n", got);+ }++ #[test]+ fn path_with_match_found() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::PathWithMatch)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlock\n", got);+ }++ #[test]+ fn path_with_match_not_found() {+ let matcher = RegexMatcher::new(r"ZZZZZZZZ").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::PathWithMatch)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("", got);+ }++ #[test]+ fn path_without_match_found() {+ let matcher = RegexMatcher::new(r"ZZZZZZZZZ").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::PathWithoutMatch)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("sherlock\n", got);+ }++ #[test]+ fn path_without_match_not_found() {+ let matcher = RegexMatcher::new(r"Watson").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::PathWithoutMatch)+ .build_no_color(vec![]);+ SearcherBuilder::new()+ .build()+ .search_reader(+ &matcher,+ SHERLOCK,+ printer.sink_with_path(&matcher, "sherlock"),+ )+ .unwrap();++ let got = printer_contents(&mut printer);+ assert_eq_printed!("", got);+ }++ #[test]+ fn quiet() {+ let matcher = RegexMatcher::new(r"Watson|Sherlock").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Quiet)+ .build_no_color(vec![]);+ let match_count = {+ let mut sink = printer.sink_with_path(&matcher, "sherlock");+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, &mut sink)+ .unwrap();+ sink.match_count+ };++ let got = printer_contents(&mut printer);+ assert_eq_printed!("", got);+ // There is actually more than one match, but Quiet should quit after+ // finding the first one.+ assert_eq!(1, match_count);+ }++ #[test]+ fn quiet_with_stats() {+ let matcher = RegexMatcher::new(r"Watson|Sherlock").unwrap();+ let mut printer = SummaryBuilder::new()+ .kind(SummaryKind::Quiet)+ .stats(true)+ .build_no_color(vec![]);+ let match_count = {+ let mut sink = printer.sink_with_path(&matcher, "sherlock");+ SearcherBuilder::new()+ .build()+ .search_reader(&matcher, SHERLOCK, &mut sink)+ .unwrap();+ sink.match_count+ };++ let got = printer_contents(&mut printer);+ assert_eq_printed!("", got);+ // There is actually more than one match, and Quiet will usually quit+ // after finding the first one, but since we request stats, it will+ // mush on to find all matches.+ assert_eq!(3, match_count);+ }+}
@@ -0,0 +1,359 @@+use std::borrow::Cow;+use std::fmt;+use std::io;+use std::path::Path;+use std::time;++use bstr::{ByteSlice, ByteVec};+use grep_matcher::{Captures, LineTerminator, Match, Matcher};+use grep_searcher::{+ LineIter, SinkContext, SinkContextKind, SinkError, SinkMatch,+};+#[cfg(feature = "serde1")]+use serde::{Serialize, Serializer};++/// A type for handling replacements while amortizing allocation.+pub struct Replacer<M: Matcher> {+ space: Option<Space<M>>,+}++struct Space<M: Matcher> {+ /// The place to store capture locations.+ caps: M::Captures,+ /// The place to write a replacement to.+ dst: Vec<u8>,+ /// The place to store match offsets in terms of `dst`.+ matches: Vec<Match>,+}++impl<M: Matcher> fmt::Debug for Replacer<M> {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ let (dst, matches) = self.replacement().unwrap_or((&[], &[]));+ f.debug_struct("Replacer")+ .field("dst", &dst)+ .field("matches", &matches)+ .finish()+ }+}++impl<M: Matcher> Replacer<M> {+ /// Create a new replacer for use with a particular matcher.+ ///+ /// This constructor does not allocate. Instead, space for dealing with+ /// replacements is allocated lazily only when needed.+ pub fn new() -> Replacer<M> {+ Replacer { space: None }+ }++ /// Executes a replacement on the given subject string by replacing all+ /// matches with the given replacement. To access the result of the+ /// replacement, use the `replacement` method.+ ///+ /// This can fail if the underlying matcher reports an error.+ pub fn replace_all<'a>(+ &'a mut self,+ matcher: &M,+ subject: &[u8],+ replacement: &[u8],+ ) -> io::Result<()> {+ {+ let &mut Space { ref mut dst, ref mut caps, ref mut matches } =+ self.allocate(matcher)?;+ dst.clear();+ matches.clear();++ matcher+ .replace_with_captures(subject, caps, dst, |caps, dst| {+ let start = dst.len();+ caps.interpolate(+ |name| matcher.capture_index(name),+ subject,+ replacement,+ dst,+ );+ let end = dst.len();+ matches.push(Match::new(start, end));+ true+ })+ .map_err(io::Error::error_message)?;+ }+ Ok(())+ }++ /// Return the result of the prior replacement and the match offsets for+ /// all replacement occurrences within the returned replacement buffer.+ ///+ /// If no replacement has occurred then `None` is returned.+ pub fn replacement<'a>(&'a self) -> Option<(&'a [u8], &'a [Match])> {+ match self.space {+ None => None,+ Some(ref space) => {+ if space.matches.is_empty() {+ None+ } else {+ Some((&space.dst, &space.matches))+ }+ }+ }+ }++ /// Clear space used for performing a replacement.+ ///+ /// Subsequent calls to `replacement` after calling `clear` (but before+ /// executing another replacement) will always return `None`.+ pub fn clear(&mut self) {+ if let Some(ref mut space) = self.space {+ space.dst.clear();+ space.matches.clear();+ }+ }++ /// Allocate space for replacements when used with the given matcher and+ /// return a mutable reference to that space.+ ///+ /// This can fail if allocating space for capture locations from the given+ /// matcher fails.+ fn allocate(&mut self, matcher: &M) -> io::Result<&mut Space<M>> {+ if self.space.is_none() {+ let caps =+ matcher.new_captures().map_err(io::Error::error_message)?;+ self.space =+ Some(Space { caps: caps, dst: vec![], matches: vec![] });+ }+ Ok(self.space.as_mut().unwrap())+ }+}++/// A simple layer of abstraction over either a match or a contextual line+/// reported by the searcher.+///+/// In particular, this provides an API that unions the `SinkMatch` and+/// `SinkContext` types while also exposing a list of all individual match+/// locations.+///+/// While this serves as a convenient mechanism to abstract over `SinkMatch`+/// and `SinkContext`, this also provides a way to abstract over replacements.+/// Namely, after a replacement, a `Sunk` value can be constructed using the+/// results of the replacement instead of the bytes reported directly by the+/// searcher.+#[derive(Debug)]+pub struct Sunk<'a> {+ bytes: &'a [u8],+ absolute_byte_offset: u64,+ line_number: Option<u64>,+ context_kind: Option<&'a SinkContextKind>,+ matches: &'a [Match],+ original_matches: &'a [Match],+}++impl<'a> Sunk<'a> {+ #[inline]+ pub fn empty() -> Sunk<'static> {+ Sunk {+ bytes: &[],+ absolute_byte_offset: 0,+ line_number: None,+ context_kind: None,+ matches: &[],+ original_matches: &[],+ }+ }++ #[inline]+ pub fn from_sink_match(+ sunk: &'a SinkMatch<'a>,+ original_matches: &'a [Match],+ replacement: Option<(&'a [u8], &'a [Match])>,+ ) -> Sunk<'a> {+ let (bytes, matches) =+ replacement.unwrap_or_else(|| (sunk.bytes(), original_matches));+ Sunk {+ bytes: bytes,+ absolute_byte_offset: sunk.absolute_byte_offset(),+ line_number: sunk.line_number(),+ context_kind: None,+ matches: matches,+ original_matches: original_matches,+ }+ }++ #[inline]+ pub fn from_sink_context(+ sunk: &'a SinkContext<'a>,+ original_matches: &'a [Match],+ replacement: Option<(&'a [u8], &'a [Match])>,+ ) -> Sunk<'a> {+ let (bytes, matches) =+ replacement.unwrap_or_else(|| (sunk.bytes(), original_matches));+ Sunk {+ bytes: bytes,+ absolute_byte_offset: sunk.absolute_byte_offset(),+ line_number: sunk.line_number(),+ context_kind: Some(sunk.kind()),+ matches: matches,+ original_matches: original_matches,+ }+ }++ #[inline]+ pub fn context_kind(&self) -> Option<&'a SinkContextKind> {+ self.context_kind+ }++ #[inline]+ pub fn bytes(&self) -> &'a [u8] {+ self.bytes+ }++ #[inline]+ pub fn matches(&self) -> &'a [Match] {+ self.matches+ }++ #[inline]+ pub fn original_matches(&self) -> &'a [Match] {+ self.original_matches+ }++ #[inline]+ pub fn lines(&self, line_term: u8) -> LineIter<'a> {+ LineIter::new(line_term, self.bytes())+ }++ #[inline]+ pub fn absolute_byte_offset(&self) -> u64 {+ self.absolute_byte_offset+ }++ #[inline]+ pub fn line_number(&self) -> Option<u64> {+ self.line_number+ }+}++/// A simple encapsulation of a file path used by a printer.+///+/// This represents any transforms that we might want to perform on the path,+/// such as converting it to valid UTF-8 and/or replacing its separator with+/// something else. This allows us to amortize work if we are printing the+/// file path for every match.+///+/// In the common case, no transformation is needed, which lets us avoid the+/// allocation. Typically, only Windows requires a transform, since we can't+/// access the raw bytes of a path directly and first need to lossily convert+/// to UTF-8. Windows is also typically where the path separator replacement+/// is used, e.g., in cygwin environments to use `/` instead of `\`.+///+/// Users of this type are expected to construct it from a normal `Path`+/// found in the standard library. It can then be written to any `io::Write`+/// implementation using the `as_bytes` method. This achieves platform+/// portability with a small cost: on Windows, paths that are not valid UTF-16+/// will not roundtrip correctly.+#[derive(Clone, Debug)]+pub struct PrinterPath<'a>(Cow<'a, [u8]>);++impl<'a> PrinterPath<'a> {+ /// Create a new path suitable for printing.+ pub fn new(path: &'a Path) -> PrinterPath<'a> {+ PrinterPath(Vec::from_path_lossy(path))+ }++ /// Create a new printer path from the given path which can be efficiently+ /// written to a writer without allocation.+ ///+ /// If the given separator is present, then any separators in `path` are+ /// replaced with it.+ pub fn with_separator(path: &'a Path, sep: Option<u8>) -> PrinterPath<'a> {+ let mut ppath = PrinterPath::new(path);+ if let Some(sep) = sep {+ ppath.replace_separator(sep);+ }+ ppath+ }++ /// Replace the path separator in this path with the given separator+ /// and do it in place. On Windows, both `/` and `\` are treated as+ /// path separators that are both replaced by `new_sep`. In all other+ /// environments, only `/` is treated as a path separator.+ fn replace_separator(&mut self, new_sep: u8) {+ let transformed_path: Vec<u8> = self+ .0+ .bytes()+ .map(|b| {+ if b == b'/' || (cfg!(windows) && b == b'\\') {+ new_sep+ } else {+ b+ }+ })+ .collect();+ self.0 = Cow::Owned(transformed_path);+ }++ /// Return the raw bytes for this path.+ pub fn as_bytes(&self) -> &[u8] {+ &self.0+ }+}++/// A type that provides "nicer" Display and Serialize impls for+/// std::time::Duration. The serialization format should actually be compatible+/// with the Deserialize impl for std::time::Duration, since this type only+/// adds new fields.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+pub struct NiceDuration(pub time::Duration);++impl fmt::Display for NiceDuration {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ write!(f, "{:0.6}s", self.fractional_seconds())+ }+}++impl NiceDuration {+ /// Returns the number of seconds in this duration in fraction form.+ /// The number to the left of the decimal point is the number of seconds,+ /// and the number to the right is the number of milliseconds.+ fn fractional_seconds(&self) -> f64 {+ let fractional = (self.0.subsec_nanos() as f64) / 1_000_000_000.0;+ self.0.as_secs() as f64 + fractional+ }+}++#[cfg(feature = "serde1")]+impl Serialize for NiceDuration {+ fn serialize<S: Serializer>(&self, ser: S) -> Result<S::Ok, S::Error> {+ use serde::ser::SerializeStruct;++ let mut state = ser.serialize_struct("Duration", 2)?;+ state.serialize_field("secs", &self.0.as_secs())?;+ state.serialize_field("nanos", &self.0.subsec_nanos())?;+ state.serialize_field("human", &format!("{}", self))?;+ state.end()+ }+}++/// Trim prefix ASCII spaces from the given slice and return the corresponding+/// range.+///+/// This stops trimming a prefix as soon as it sees non-whitespace or a line+/// terminator.+pub fn trim_ascii_prefix(+ line_term: LineTerminator,+ slice: &[u8],+ range: Match,+) -> Match {+ fn is_space(b: u8) -> bool {+ match b {+ b'\t' | b'\n' | b'\x0B' | b'\x0C' | b'\r' | b' ' => true,+ _ => false,+ }+ }++ let count = slice[range]+ .iter()+ .take_while(|&&b| -> bool {+ is_space(b) && !line_term.as_bytes().contains(&b)+ })+ .count();+ range.with_start(range.start() + count)+}
@@ -0,0 +1,22 @@+[package]+name = "grep-regex"+version = "0.1.5" #:version+authors = ["Andrew Gallant <jamslam@gmail.com>"]+description = """+Use Rust's regex library with the 'grep' crate.+"""+documentation = "https://docs.rs/grep-regex"+homepage = "https://github.com/BurntSushi/ripgrep"+repository = "https://github.com/BurntSushi/ripgrep"+readme = "README.md"+keywords = ["regex", "grep", "search", "pattern", "line"]+license = "Unlicense/MIT"++[dependencies]+aho-corasick = "0.7.3"+bstr = "0.2.10"+grep-matcher = { version = "0.1.2", path = "../grep-matcher" }+log = "0.4.5"+regex = "1.1"+regex-syntax = "0.6.5"+thread_local = "1"
@@ -0,0 +1,21 @@+The MIT License (MIT)++Copyright (c) 2015 Andrew Gallant++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.
@@ -0,0 +1,35 @@+grep-regex+----------+The `grep-regex` crate provides an implementation of the `Matcher` trait from+the `grep-matcher` crate. This implementation permits Rust's regex engine to+be used in the `grep` crate for fast line oriented searching.++[](https://travis-ci.org/BurntSushi/ripgrep)+[](https://ci.appveyor.com/project/BurntSushi/ripgrep)+[](https://crates.io/crates/grep-regex)++Dual-licensed under MIT or the [UNLICENSE](http://unlicense.org).++### Documentation++[https://docs.rs/grep-regex](https://docs.rs/grep-regex)++**NOTE:** You probably don't want to use this crate directly. Instead, you+should prefer the facade defined in the+[`grep`](https://docs.rs/grep)+crate.++### Usage++Add this to your `Cargo.toml`:++```toml+[dependencies]+grep-regex = "0.1"+```++and this to your crate root:++```rust+extern crate grep_regex;+```
@@ -0,0 +1,24 @@+This is free and unencumbered software released into the public domain.++Anyone is free to copy, modify, publish, use, compile, sell, or+distribute this software, either in source code form or as a compiled+binary, for any purpose, commercial or non-commercial, and by any+means.++In jurisdictions that recognize copyright laws, the author or authors+of this software dedicate any and all copyright interest in the+software to the public domain. We make this dedication for the benefit+of the public at large and to the detriment of our heirs and+successors. We intend this dedication to be an overt act of+relinquishment in perpetuity of all present and future rights to this+software under copyright law.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR+OTHER DEALINGS IN THE SOFTWARE.++For more information, please refer to <http://unlicense.org/>
@@ -0,0 +1,263 @@+use regex_syntax::ast::parse::Parser;+use regex_syntax::ast::{self, Ast};++/// The results of analyzing AST of a regular expression (e.g., for supporting+/// smart case).+#[derive(Clone, Debug)]+pub struct AstAnalysis {+ /// True if and only if a literal uppercase character occurs in the regex.+ any_uppercase: bool,+ /// True if and only if the regex contains any literal at all.+ any_literal: bool,+ /// True if and only if the regex consists entirely of a literal and no+ /// other special regex characters.+ all_verbatim_literal: bool,+}++impl AstAnalysis {+ /// Returns a `AstAnalysis` value by doing analysis on the AST of `pattern`.+ ///+ /// If `pattern` is not a valid regular expression, then `None` is+ /// returned.+ #[allow(dead_code)]+ pub fn from_pattern(pattern: &str) -> Option<AstAnalysis> {+ Parser::new()+ .parse(pattern)+ .map(|ast| AstAnalysis::from_ast(&ast))+ .ok()+ }++ /// Perform an AST analysis given the AST.+ pub fn from_ast(ast: &Ast) -> AstAnalysis {+ let mut analysis = AstAnalysis::new();+ analysis.from_ast_impl(ast);+ analysis+ }++ /// Returns true if and only if a literal uppercase character occurs in+ /// the pattern.+ ///+ /// For example, a pattern like `\pL` contains no uppercase literals,+ /// even though `L` is uppercase and the `\pL` class contains uppercase+ /// characters.+ pub fn any_uppercase(&self) -> bool {+ self.any_uppercase+ }++ /// Returns true if and only if the regex contains any literal at all.+ ///+ /// For example, a pattern like `\pL` reports `false`, but a pattern like+ /// `\pLfoo` reports `true`.+ pub fn any_literal(&self) -> bool {+ self.any_literal+ }++ /// Returns true if and only if the entire pattern is a verbatim literal+ /// with no special meta characters.+ ///+ /// When this is true, then the pattern satisfies the following law:+ /// `escape(pattern) == pattern`. Notable examples where this returns+ /// `false` include patterns like `a\u0061` even though `\u0061` is just+ /// a literal `a`.+ ///+ /// The purpose of this flag is to determine whether the patterns can be+ /// given to non-regex substring search algorithms as-is.+ #[allow(dead_code)]+ pub fn all_verbatim_literal(&self) -> bool {+ self.all_verbatim_literal+ }++ /// Creates a new `AstAnalysis` value with an initial configuration.+ fn new() -> AstAnalysis {+ AstAnalysis {+ any_uppercase: false,+ any_literal: false,+ all_verbatim_literal: true,+ }+ }++ fn from_ast_impl(&mut self, ast: &Ast) {+ if self.done() {+ return;+ }+ match *ast {+ Ast::Empty(_) => {}+ Ast::Flags(_)+ | Ast::Dot(_)+ | Ast::Assertion(_)+ | Ast::Class(ast::Class::Unicode(_))+ | Ast::Class(ast::Class::Perl(_)) => {+ self.all_verbatim_literal = false;+ }+ Ast::Literal(ref x) => {+ self.from_ast_literal(x);+ }+ Ast::Class(ast::Class::Bracketed(ref x)) => {+ self.all_verbatim_literal = false;+ self.from_ast_class_set(&x.kind);+ }+ Ast::Repetition(ref x) => {+ self.all_verbatim_literal = false;+ self.from_ast_impl(&x.ast);+ }+ Ast::Group(ref x) => {+ self.all_verbatim_literal = false;+ self.from_ast_impl(&x.ast);+ }+ Ast::Alternation(ref alt) => {+ self.all_verbatim_literal = false;+ for x in &alt.asts {+ self.from_ast_impl(x);+ }+ }+ Ast::Concat(ref alt) => {+ for x in &alt.asts {+ self.from_ast_impl(x);+ }+ }+ }+ }++ fn from_ast_class_set(&mut self, ast: &ast::ClassSet) {+ if self.done() {+ return;+ }+ match *ast {+ ast::ClassSet::Item(ref item) => {+ self.from_ast_class_set_item(item);+ }+ ast::ClassSet::BinaryOp(ref x) => {+ self.from_ast_class_set(&x.lhs);+ self.from_ast_class_set(&x.rhs);+ }+ }+ }++ fn from_ast_class_set_item(&mut self, ast: &ast::ClassSetItem) {+ if self.done() {+ return;+ }+ match *ast {+ ast::ClassSetItem::Empty(_)+ | ast::ClassSetItem::Ascii(_)+ | ast::ClassSetItem::Unicode(_)+ | ast::ClassSetItem::Perl(_) => {}+ ast::ClassSetItem::Literal(ref x) => {+ self.from_ast_literal(x);+ }+ ast::ClassSetItem::Range(ref x) => {+ self.from_ast_literal(&x.start);+ self.from_ast_literal(&x.end);+ }+ ast::ClassSetItem::Bracketed(ref x) => {+ self.from_ast_class_set(&x.kind);+ }+ ast::ClassSetItem::Union(ref union) => {+ for x in &union.items {+ self.from_ast_class_set_item(x);+ }+ }+ }+ }++ fn from_ast_literal(&mut self, ast: &ast::Literal) {+ if ast.kind != ast::LiteralKind::Verbatim {+ self.all_verbatim_literal = false;+ }+ self.any_literal = true;+ self.any_uppercase = self.any_uppercase || ast.c.is_uppercase();+ }++ /// Returns true if and only if the attributes can never change no matter+ /// what other AST it might see.+ fn done(&self) -> bool {+ self.any_uppercase && self.any_literal && !self.all_verbatim_literal+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn analysis(pattern: &str) -> AstAnalysis {+ AstAnalysis::from_pattern(pattern).unwrap()+ }++ #[test]+ fn various() {+ let x = analysis("");+ assert!(!x.any_uppercase);+ assert!(!x.any_literal);+ assert!(x.all_verbatim_literal);++ let x = analysis("foo");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(x.all_verbatim_literal);++ let x = analysis("Foo");+ assert!(x.any_uppercase);+ assert!(x.any_literal);+ assert!(x.all_verbatim_literal);++ let x = analysis("foO");+ assert!(x.any_uppercase);+ assert!(x.any_literal);+ assert!(x.all_verbatim_literal);++ let x = analysis(r"foo\\");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo\w");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo\S");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo\p{Ll}");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo[a-z]");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo[A-Z]");+ assert!(x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo[\S\t]");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"foo\\S");+ assert!(x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"\p{Ll}");+ assert!(!x.any_uppercase);+ assert!(!x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"aBc\w");+ assert!(x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);++ let x = analysis(r"a\u0061");+ assert!(!x.any_uppercase);+ assert!(x.any_literal);+ assert!(!x.all_verbatim_literal);+ }+}
@@ -0,0 +1,288 @@+use grep_matcher::{ByteSet, LineTerminator};+use regex::bytes::{Regex, RegexBuilder};+use regex_syntax::ast::{self, Ast};+use regex_syntax::hir::{self, Hir};++use ast::AstAnalysis;+use crlf::crlfify;+use error::Error;+use literal::LiteralSets;+use multi::alternation_literals;+use non_matching::non_matching_bytes;+use strip::strip_from_match;++/// Config represents the configuration of a regex matcher in this crate.+/// The configuration is itself a rough combination of the knobs found in+/// the `regex` crate itself, along with additional `grep-matcher` specific+/// options.+///+/// The configuration can be used to build a "configured" HIR expression. A+/// configured HIR expression is an HIR expression that is aware of the+/// configuration which generated it, and provides transformation on that HIR+/// such that the configuration is preserved.+#[derive(Clone, Debug)]+pub struct Config {+ pub case_insensitive: bool,+ pub case_smart: bool,+ pub multi_line: bool,+ pub dot_matches_new_line: bool,+ pub swap_greed: bool,+ pub ignore_whitespace: bool,+ pub unicode: bool,+ pub octal: bool,+ pub size_limit: usize,+ pub dfa_size_limit: usize,+ pub nest_limit: u32,+ pub line_terminator: Option<LineTerminator>,+ pub crlf: bool,+ pub word: bool,+}++impl Default for Config {+ fn default() -> Config {+ Config {+ case_insensitive: false,+ case_smart: false,+ multi_line: false,+ dot_matches_new_line: false,+ swap_greed: false,+ ignore_whitespace: false,+ unicode: true,+ octal: false,+ // These size limits are much bigger than what's in the regex+ // crate.+ size_limit: 100 * (1 << 20),+ dfa_size_limit: 1000 * (1 << 20),+ nest_limit: 250,+ line_terminator: None,+ crlf: false,+ word: false,+ }+ }+}++impl Config {+ /// Parse the given pattern and returned its HIR expression along with+ /// the current configuration.+ ///+ /// If there was a problem parsing the given expression then an error+ /// is returned.+ pub fn hir(&self, pattern: &str) -> Result<ConfiguredHIR, Error> {+ let ast = self.ast(pattern)?;+ let analysis = self.analysis(&ast)?;+ let expr = hir::translate::TranslatorBuilder::new()+ .allow_invalid_utf8(true)+ .case_insensitive(self.is_case_insensitive(&analysis))+ .multi_line(self.multi_line)+ .dot_matches_new_line(self.dot_matches_new_line)+ .swap_greed(self.swap_greed)+ .unicode(self.unicode)+ .build()+ .translate(pattern, &ast)+ .map_err(Error::regex)?;+ let expr = match self.line_terminator {+ None => expr,+ Some(line_term) => strip_from_match(expr, line_term)?,+ };+ Ok(ConfiguredHIR {+ original: pattern.to_string(),+ config: self.clone(),+ analysis: analysis,+ // If CRLF mode is enabled, replace `$` with `(?:\r?$)`.+ expr: if self.crlf { crlfify(expr) } else { expr },+ })+ }++ /// Accounting for the `smart_case` config knob, return true if and only if+ /// this pattern should be matched case insensitively.+ fn is_case_insensitive(&self, analysis: &AstAnalysis) -> bool {+ if self.case_insensitive {+ return true;+ }+ if !self.case_smart {+ return false;+ }+ analysis.any_literal() && !analysis.any_uppercase()+ }++ /// Returns true if and only if this config is simple enough such that+ /// if the pattern is a simple alternation of literals, then it can be+ /// constructed via a plain Aho-Corasick automaton.+ ///+ /// Note that it is OK to return true even when settings like `multi_line`+ /// are enabled, since if multi-line can impact the match semantics of a+ /// regex, then it is by definition not a simple alternation of literals.+ pub fn can_plain_aho_corasick(&self) -> bool {+ !self.word && !self.case_insensitive && !self.case_smart+ }++ /// Perform analysis on the AST of this pattern.+ ///+ /// This returns an error if the given pattern failed to parse.+ fn analysis(&self, ast: &Ast) -> Result<AstAnalysis, Error> {+ Ok(AstAnalysis::from_ast(ast))+ }++ /// Parse the given pattern into its abstract syntax.+ ///+ /// This returns an error if the given pattern failed to parse.+ fn ast(&self, pattern: &str) -> Result<Ast, Error> {+ ast::parse::ParserBuilder::new()+ .nest_limit(self.nest_limit)+ .octal(self.octal)+ .ignore_whitespace(self.ignore_whitespace)+ .build()+ .parse(pattern)+ .map_err(Error::regex)+ }+}++/// A "configured" HIR expression, which is aware of the configuration which+/// produced this HIR.+///+/// Since the configuration is tracked, values with this type can be+/// transformed into other HIR expressions (or regular expressions) in a way+/// that preserves the configuration. For example, the `fast_line_regex`+/// method will apply literal extraction to the inner HIR and use that to build+/// a new regex that matches the extracted literals in a way that is+/// consistent with the configuration that produced this HIR. For example, the+/// size limits set on the configured HIR will be propagated out to any+/// subsequently constructed HIR or regular expression.+#[derive(Clone, Debug)]+pub struct ConfiguredHIR {+ original: String,+ config: Config,+ analysis: AstAnalysis,+ expr: Hir,+}++impl ConfiguredHIR {+ /// Return the configuration for this HIR expression.+ pub fn config(&self) -> &Config {+ &self.config+ }++ /// Compute the set of non-matching bytes for this HIR expression.+ pub fn non_matching_bytes(&self) -> ByteSet {+ non_matching_bytes(&self.expr)+ }++ /// Returns true if and only if this regex needs to have its match offsets+ /// tweaked because of CRLF support. Specifically, this occurs when the+ /// CRLF hack is enabled and the regex is line anchored at the end. In+ /// this case, matches that end with a `\r` have the `\r` stripped.+ pub fn needs_crlf_stripped(&self) -> bool {+ self.config.crlf && self.expr.is_line_anchored_end()+ }++ /// Builds a regular expression from this HIR expression.+ pub fn regex(&self) -> Result<Regex, Error> {+ self.pattern_to_regex(&self.expr.to_string())+ }++ /// If this HIR corresponds to an alternation of literals with no+ /// capturing groups, then this returns those literals.+ pub fn alternation_literals(&self) -> Option<Vec<Vec<u8>>> {+ if !self.config.can_plain_aho_corasick() {+ return None;+ }+ alternation_literals(&self.expr)+ }++ /// Applies the given function to the concrete syntax of this HIR and then+ /// generates a new HIR based on the result of the function in a way that+ /// preserves the configuration.+ ///+ /// For example, this can be used to wrap a user provided regular+ /// expression with additional semantics. e.g., See the `WordMatcher`.+ pub fn with_pattern<F: FnMut(&str) -> String>(+ &self,+ mut f: F,+ ) -> Result<ConfiguredHIR, Error> {+ self.pattern_to_hir(&f(&self.expr.to_string()))+ }++ /// If the current configuration has a line terminator set and if useful+ /// literals could be extracted, then a regular expression matching those+ /// literals is returned. If no line terminator is set, then `None` is+ /// returned.+ ///+ /// If compiling the resulting regular expression failed, then an error+ /// is returned.+ ///+ /// This method only returns something when a line terminator is set+ /// because matches from this regex are generally candidates that must be+ /// confirmed before reporting a match. When performing a line oriented+ /// search, confirmation is easy: just extend the candidate match to its+ /// respective line boundaries and then re-search that line for a full+ /// match. This only works when the line terminator is set because the line+ /// terminator setting guarantees that the regex itself can never match+ /// through the line terminator byte.+ pub fn fast_line_regex(&self) -> Result<Option<Regex>, Error> {+ if self.config.line_terminator.is_none() {+ return Ok(None);+ }+ match LiteralSets::new(&self.expr).one_regex(self.config.word) {+ None => Ok(None),+ Some(pattern) => self.pattern_to_regex(&pattern).map(Some),+ }+ }++ /// Create a regex from the given pattern using this HIR's configuration.+ fn pattern_to_regex(&self, pattern: &str) -> Result<Regex, Error> {+ // The settings we explicitly set here are intentionally a subset+ // of the settings we have. The key point here is that our HIR+ // expression is computed with the settings in mind, such that setting+ // them here could actually lead to unintended behavior. For example,+ // consider the pattern `(?U)a+`. This will get folded into the HIR+ // as a non-greedy repetition operator which will in turn get printed+ // to the concrete syntax as `a+?`, which is correct. But if we+ // set the `swap_greed` option again, then we'll wind up with `(?U)a+?`+ // which is equal to `a+` which is not the same as what we were given.+ //+ // We also don't need to apply `case_insensitive` since this gets+ // folded into the HIR and would just cause us to do redundant work.+ //+ // Finally, we don't need to set `ignore_whitespace` since the concrete+ // syntax emitted by the HIR printer never needs it.+ //+ // We set the rest of the options. Some of them are important, such as+ // the size limit, and some of them are necessary to preserve the+ // intention of the original pattern. For example, the Unicode flag+ // will impact how the WordMatcher functions, namely, whether its+ // word boundaries are Unicode aware or not.+ RegexBuilder::new(&pattern)+ .nest_limit(self.config.nest_limit)+ .octal(self.config.octal)+ .multi_line(self.config.multi_line)+ .dot_matches_new_line(self.config.dot_matches_new_line)+ .unicode(self.config.unicode)+ .size_limit(self.config.size_limit)+ .dfa_size_limit(self.config.dfa_size_limit)+ .build()+ .map_err(Error::regex)+ }++ /// Create an HIR expression from the given pattern using this HIR's+ /// configuration.+ fn pattern_to_hir(&self, pattern: &str) -> Result<ConfiguredHIR, Error> {+ // See `pattern_to_regex` comment for explanation of why we only set+ // a subset of knobs here. e.g., `swap_greed` is explicitly left out.+ let expr = ::regex_syntax::ParserBuilder::new()+ .nest_limit(self.config.nest_limit)+ .octal(self.config.octal)+ .allow_invalid_utf8(true)+ .multi_line(self.config.multi_line)+ .dot_matches_new_line(self.config.dot_matches_new_line)+ .unicode(self.config.unicode)+ .build()+ .parse(pattern)+ .map_err(Error::regex)?;+ Ok(ConfiguredHIR {+ original: self.original.clone(),+ config: self.config.clone(),+ analysis: self.analysis.clone(),+ expr: expr,+ })+ }+}
crates/regex/src/crlf.rs
@@ -0,0 +1,189 @@+use std::collections::HashMap;++use grep_matcher::{Match, Matcher, NoError};+use regex::bytes::Regex;+use regex_syntax::hir::{self, Hir, HirKind};++use config::ConfiguredHIR;+use error::Error;+use matcher::RegexCaptures;++/// A matcher for implementing "word match" semantics.+#[derive(Clone, Debug)]+pub struct CRLFMatcher {+ /// The regex.+ regex: Regex,+ /// A map from capture group name to capture group index.+ names: HashMap<String, usize>,+}++impl CRLFMatcher {+ /// Create a new matcher from the given pattern that strips `\r` from the+ /// end of every match.+ ///+ /// This panics if the given expression doesn't need its CRLF stripped.+ pub fn new(expr: &ConfiguredHIR) -> Result<CRLFMatcher, Error> {+ assert!(expr.needs_crlf_stripped());++ let regex = expr.regex()?;+ let mut names = HashMap::new();+ for (i, optional_name) in regex.capture_names().enumerate() {+ if let Some(name) = optional_name {+ names.insert(name.to_string(), i.checked_sub(1).unwrap());+ }+ }+ Ok(CRLFMatcher { regex, names })+ }++ /// Return the underlying regex used by this matcher.+ pub fn regex(&self) -> &Regex {+ &self.regex+ }+}++impl Matcher for CRLFMatcher {+ type Captures = RegexCaptures;+ type Error = NoError;++ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, NoError> {+ let m = match self.regex.find_at(haystack, at) {+ None => return Ok(None),+ Some(m) => Match::new(m.start(), m.end()),+ };+ Ok(Some(adjust_match(haystack, m)))+ }++ fn new_captures(&self) -> Result<RegexCaptures, NoError> {+ Ok(RegexCaptures::new(self.regex.capture_locations()))+ }++ fn capture_count(&self) -> usize {+ self.regex.captures_len().checked_sub(1).unwrap()+ }++ fn capture_index(&self, name: &str) -> Option<usize> {+ self.names.get(name).map(|i| *i)+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut RegexCaptures,+ ) -> Result<bool, NoError> {+ caps.strip_crlf(false);+ let r =+ self.regex.captures_read_at(caps.locations_mut(), haystack, at);+ if !r.is_some() {+ return Ok(false);+ }++ // If the end of our match includes a `\r`, then strip it from all+ // capture groups ending at the same location.+ let end = caps.locations().get(0).unwrap().1;+ if end > 0 && haystack.get(end - 1) == Some(&b'\r') {+ caps.strip_crlf(true);+ }+ Ok(true)+ }++ // We specifically do not implement other methods like find_iter or+ // captures_iter. Namely, the iter methods are guaranteed to be correct+ // by virtue of implementing find_at and captures_at above.+}++/// If the given match ends with a `\r`, then return a new match that ends+/// immediately before the `\r`.+pub fn adjust_match(haystack: &[u8], m: Match) -> Match {+ if m.end() > 0 && haystack.get(m.end() - 1) == Some(&b'\r') {+ m.with_end(m.end() - 1)+ } else {+ m+ }+}++/// Substitutes all occurrences of multi-line enabled `$` with `(?:\r?$)`.+///+/// This does not preserve the exact semantics of the given expression,+/// however, it does have the useful property that anything that matched the+/// given expression will also match the returned expression. The difference is+/// that the returned expression can match possibly other things as well.+///+/// The principle reason why we do this is because the underlying regex engine+/// doesn't support CRLF aware `$` look-around. It's planned to fix it at that+/// level, but we perform this kludge in the mean time.+///+/// Note that while the match preserving semantics are nice and neat, the+/// match position semantics are quite a bit messier. Namely, `$` only ever+/// matches the position between characters where as `\r??` can match a+/// character and change the offset. This is regretable, but works out pretty+/// nicely in most cases, especially when a match is limited to a single line.+pub fn crlfify(expr: Hir) -> Hir {+ match expr.into_kind() {+ HirKind::Anchor(hir::Anchor::EndLine) => {+ let concat = Hir::concat(vec![+ Hir::repetition(hir::Repetition {+ kind: hir::RepetitionKind::ZeroOrOne,+ greedy: false,+ hir: Box::new(Hir::literal(hir::Literal::Unicode('\r'))),+ }),+ Hir::anchor(hir::Anchor::EndLine),+ ]);+ Hir::group(hir::Group {+ kind: hir::GroupKind::NonCapturing,+ hir: Box::new(concat),+ })+ }+ HirKind::Empty => Hir::empty(),+ HirKind::Literal(x) => Hir::literal(x),+ HirKind::Class(x) => Hir::class(x),+ HirKind::Anchor(x) => Hir::anchor(x),+ HirKind::WordBoundary(x) => Hir::word_boundary(x),+ HirKind::Repetition(mut x) => {+ x.hir = Box::new(crlfify(*x.hir));+ Hir::repetition(x)+ }+ HirKind::Group(mut x) => {+ x.hir = Box::new(crlfify(*x.hir));+ Hir::group(x)+ }+ HirKind::Concat(xs) => {+ Hir::concat(xs.into_iter().map(crlfify).collect())+ }+ HirKind::Alternation(xs) => {+ Hir::alternation(xs.into_iter().map(crlfify).collect())+ }+ }+}++#[cfg(test)]+mod tests {+ use super::crlfify;+ use regex_syntax::Parser;++ fn roundtrip(pattern: &str) -> String {+ let expr1 = Parser::new().parse(pattern).unwrap();+ let expr2 = crlfify(expr1);+ expr2.to_string()+ }++ #[test]+ fn various() {+ assert_eq!(roundtrip(r"(?m)$"), "(?:\r??(?m:$))");+ assert_eq!(roundtrip(r"(?m)$$"), "(?:\r??(?m:$))(?:\r??(?m:$))");+ assert_eq!(+ roundtrip(r"(?m)(?:foo$|bar$)"),+ "(?:foo(?:\r??(?m:$))|bar(?:\r??(?m:$)))"+ );+ assert_eq!(roundtrip(r"(?m)$a"), "(?:\r??(?m:$))a");++ // Not a multiline `$`, so no crlfifying occurs.+ assert_eq!(roundtrip(r"$"), "\\z");+ // It's a literal, derp.+ assert_eq!(roundtrip(r"\$"), "\\$");+ }+}
@@ -0,0 +1,88 @@+use std::error;+use std::fmt;++use util;++/// An error that can occur in this crate.+///+/// Generally, this error corresponds to problems building a regular+/// expression, whether it's in parsing, compilation or a problem with+/// guaranteeing a configured optimization.+#[derive(Clone, Debug)]+pub struct Error {+ kind: ErrorKind,+}++impl Error {+ pub(crate) fn new(kind: ErrorKind) -> Error {+ Error { kind }+ }++ pub(crate) fn regex<E: error::Error>(err: E) -> Error {+ Error { kind: ErrorKind::Regex(err.to_string()) }+ }++ /// Return the kind of this error.+ pub fn kind(&self) -> &ErrorKind {+ &self.kind+ }+}++/// The kind of an error that can occur.+#[derive(Clone, Debug)]+pub enum ErrorKind {+ /// An error that occurred as a result of parsing a regular expression.+ /// This can be a syntax error or an error that results from attempting to+ /// compile a regular expression that is too big.+ ///+ /// The string here is the underlying error converted to a string.+ Regex(String),+ /// An error that occurs when a building a regex that isn't permitted to+ /// match a line terminator. In general, building the regex will do its+ /// best to make matching a line terminator impossible (e.g., by removing+ /// `\n` from the `\s` character class), but if the regex contains a+ /// `\n` literal, then there is no reasonable choice that can be made and+ /// therefore an error is reported.+ ///+ /// The string is the literal sequence found in the regex that is not+ /// allowed.+ NotAllowed(String),+ /// This error occurs when a non-ASCII line terminator was provided.+ ///+ /// The invalid byte is included in this error.+ InvalidLineTerminator(u8),+ /// Hints that destructuring should not be exhaustive.+ ///+ /// This enum may grow additional variants, so this makes sure clients+ /// don't count on exhaustive matching. (Otherwise, adding a new variant+ /// could break existing code.)+ #[doc(hidden)]+ __Nonexhaustive,+}++impl error::Error for Error {+ fn description(&self) -> &str {+ match self.kind {+ ErrorKind::Regex(_) => "regex error",+ ErrorKind::NotAllowed(_) => "literal not allowed",+ ErrorKind::InvalidLineTerminator(_) => "invalid line terminator",+ ErrorKind::__Nonexhaustive => unreachable!(),+ }+ }+}++impl fmt::Display for Error {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ match self.kind {+ ErrorKind::Regex(ref s) => write!(f, "{}", s),+ ErrorKind::NotAllowed(ref lit) => {+ write!(f, "the literal '{:?}' is not allowed in a regex", lit)+ }+ ErrorKind::InvalidLineTerminator(byte) => {+ let x = util::show_bytes(&[byte]);+ write!(f, "line terminators must be ASCII, but '{}' is not", x)+ }+ ErrorKind::__Nonexhaustive => unreachable!(),+ }+ }+}
@@ -0,0 +1,28 @@+/*!+An implementation of `grep-matcher`'s `Matcher` trait for Rust's regex engine.+*/++#![deny(missing_docs)]++extern crate aho_corasick;+extern crate grep_matcher;+#[macro_use]+extern crate log;+extern crate regex;+extern crate regex_syntax;+extern crate thread_local;++pub use error::{Error, ErrorKind};+pub use matcher::{RegexCaptures, RegexMatcher, RegexMatcherBuilder};++mod ast;+mod config;+mod crlf;+mod error;+mod literal;+mod matcher;+mod multi;+mod non_matching;+mod strip;+mod util;+mod word;
@@ -0,0 +1,391 @@+/*+This module is responsible for extracting *inner* literals out of the AST of a+regular expression. Normally this is the job of the regex engine itself, but+the regex engine doesn't look for inner literals. Since we're doing line based+searching, we can use them, so we need to do it ourselves.+*/++use regex_syntax::hir::literal::{Literal, Literals};+use regex_syntax::hir::{self, Hir, HirKind};++use util;++/// Represents prefix, suffix and inner "required" literals for a regular+/// expression.+///+/// Prefixes and suffixes are detected using regex-syntax. The inner required+/// literals are detected using something custom (but based on the code in+/// regex-syntax).+#[derive(Clone, Debug)]+pub struct LiteralSets {+ /// A set of prefix literals.+ prefixes: Literals,+ /// A set of suffix literals.+ suffixes: Literals,+ /// A set of literals such that at least one of them must appear in every+ /// match. A literal in this set may be neither a prefix nor a suffix.+ required: Literals,+}++impl LiteralSets {+ /// Create a set of literals from the given HIR expression.+ pub fn new(expr: &Hir) -> LiteralSets {+ let mut required = Literals::empty();+ union_required(expr, &mut required);+ LiteralSets {+ prefixes: Literals::prefixes(expr),+ suffixes: Literals::suffixes(expr),+ required: required,+ }+ }++ /// If it is deemed advantageuous to do so (via various suspicious+ /// heuristics), this will return a single regular expression pattern that+ /// matches a subset of the language matched by the regular expression that+ /// generated these literal sets. The idea here is that the pattern+ /// returned by this method is much cheaper to search for. i.e., It is+ /// usually a single literal or an alternation of literals.+ pub fn one_regex(&self, word: bool) -> Option<String> {+ // TODO: The logic in this function is basically inscrutable. It grew+ // organically in the old grep 0.1 crate. Ideally, it would be+ // re-worked. In fact, the entire inner literal extraction should be+ // re-worked. Actually, most of regex-syntax's literal extraction+ // should also be re-worked. Alas... only so much time in the day.++ if !word {+ if self.prefixes.all_complete() && !self.prefixes.is_empty() {+ debug!("literal prefixes detected: {:?}", self.prefixes);+ // When this is true, the regex engine will do a literal scan,+ // so we don't need to return anything. But we only do this+ // if we aren't doing a word regex, since a word regex adds+ // a `(?:\W|^)` to the beginning of the regex, thereby+ // defeating the regex engine's literal detection.+ return None;+ }+ }++ // Out of inner required literals, prefixes and suffixes, which one+ // is the longest? We pick the longest to do fast literal scan under+ // the assumption that a longer literal will have a lower false+ // positive rate.+ let pre_lcp = self.prefixes.longest_common_prefix();+ let pre_lcs = self.prefixes.longest_common_suffix();+ let suf_lcp = self.suffixes.longest_common_prefix();+ let suf_lcs = self.suffixes.longest_common_suffix();++ let req_lits = self.required.literals();+ let req = match req_lits.iter().max_by_key(|lit| lit.len()) {+ None => &[],+ Some(req) => &***req,+ };++ let mut lit = pre_lcp;+ if pre_lcs.len() > lit.len() {+ lit = pre_lcs;+ }+ if suf_lcp.len() > lit.len() {+ lit = suf_lcp;+ }+ if suf_lcs.len() > lit.len() {+ lit = suf_lcs;+ }+ if req_lits.len() == 1 && req.len() > lit.len() {+ lit = req;+ }++ // Special case: if we detected an alternation of inner required+ // literals and its longest literal is bigger than the longest+ // prefix/suffix, then choose the alternation. In practice, this+ // helps with case insensitive matching, which can generate lots of+ // inner required literals.+ let any_empty = req_lits.iter().any(|lit| lit.is_empty());+ if req.len() > lit.len() && req_lits.len() > 1 && !any_empty {+ debug!("required literals found: {:?}", req_lits);+ let alts: Vec<String> = req_lits+ .into_iter()+ .map(|x| util::bytes_to_regex(x))+ .collect();+ // We're matching raw bytes, so disable Unicode mode.+ Some(format!("(?-u:{})", alts.join("|")))+ } else if lit.is_empty() {+ // If we're here, then we have no LCP. No LCS. And no detected+ // inner required literals. In theory this shouldn't happen, but+ // the inner literal detector isn't as nice as we hope and doens't+ // actually support returning a set of alternating required+ // literals. (Instead, it only returns a set where EVERY literal+ // in it is required. It cannot currently express "either P or Q+ // is required.")+ //+ // In this case, it is possible that we still have meaningful+ // prefixes or suffixes to use. So we look for the set of literals+ // with the highest minimum length and use that to build our "fast"+ // regex.+ //+ // This manifest in fairly common scenarios. e.g.,+ //+ // rg -w 'foo|bar|baz|quux'+ //+ // Normally, without the `-w`, the regex engine itself would+ // detect the prefix correctly. Unfortunately, the `-w` option+ // turns the regex into something like this:+ //+ // rg '(^|\W)(foo|bar|baz|quux)($|\W)'+ //+ // Which will defeat all prefix and suffix literal optimizations.+ // (Not in theory---it could be better. But the current+ // implementation isn't good enough.) ... So we make up for it+ // here.+ let p_min_len = self.prefixes.min_len();+ let s_min_len = self.suffixes.min_len();+ let lits = match (p_min_len, s_min_len) {+ (None, None) => return None,+ (Some(_), None) => {+ debug!("prefix literals found");+ self.prefixes.literals()+ }+ (None, Some(_)) => {+ debug!("suffix literals found");+ self.suffixes.literals()+ }+ (Some(p), Some(s)) => {+ if p >= s {+ debug!("prefix literals found");+ self.prefixes.literals()+ } else {+ debug!("suffix literals found");+ self.suffixes.literals()+ }+ }+ };++ debug!("prefix/suffix literals found: {:?}", lits);+ let alts: Vec<String> =+ lits.into_iter().map(|x| util::bytes_to_regex(x)).collect();+ // We're matching raw bytes, so disable Unicode mode.+ Some(format!("(?-u:{})", alts.join("|")))+ } else {+ debug!("required literal found: {:?}", util::show_bytes(lit));+ Some(format!("(?-u:{})", util::bytes_to_regex(&lit)))+ }+ }+}++fn union_required(expr: &Hir, lits: &mut Literals) {+ match *expr.kind() {+ HirKind::Literal(hir::Literal::Unicode(c)) => {+ let mut buf = [0u8; 4];+ lits.cross_add(c.encode_utf8(&mut buf).as_bytes());+ }+ HirKind::Literal(hir::Literal::Byte(b)) => {+ lits.cross_add(&[b]);+ }+ HirKind::Class(hir::Class::Unicode(ref cls)) => {+ if count_unicode_class(cls) >= 5 || !lits.add_char_class(cls) {+ lits.cut();+ }+ }+ HirKind::Class(hir::Class::Bytes(ref cls)) => {+ if count_byte_class(cls) >= 5 || !lits.add_byte_class(cls) {+ lits.cut();+ }+ }+ HirKind::Group(hir::Group { ref hir, .. }) => {+ union_required(&**hir, lits);+ }+ HirKind::Repetition(ref x) => match x.kind {+ hir::RepetitionKind::ZeroOrOne => lits.cut(),+ hir::RepetitionKind::ZeroOrMore => lits.cut(),+ hir::RepetitionKind::OneOrMore => {+ union_required(&x.hir, lits);+ }+ hir::RepetitionKind::Range(ref rng) => {+ let (min, max) = match *rng {+ hir::RepetitionRange::Exactly(m) => (m, Some(m)),+ hir::RepetitionRange::AtLeast(m) => (m, None),+ hir::RepetitionRange::Bounded(m, n) => (m, Some(n)),+ };+ repeat_range_literals(+ &x.hir,+ min,+ max,+ x.greedy,+ lits,+ union_required,+ );+ }+ },+ HirKind::Concat(ref es) if es.is_empty() => {}+ HirKind::Concat(ref es) if es.len() == 1 => {+ union_required(&es[0], lits)+ }+ HirKind::Concat(ref es) => {+ for e in es {+ let mut lits2 = lits.to_empty();+ union_required(e, &mut lits2);+ if lits2.is_empty() {+ lits.cut();+ continue;+ }+ if lits2.contains_empty() || !is_simple(&e) {+ lits.cut();+ }+ if !lits.cross_product(&lits2) || !lits2.any_complete() {+ // If this expression couldn't yield any literal that+ // could be extended, then we need to quit. Since we're+ // short-circuiting, we also need to freeze every member.+ lits.cut();+ break;+ }+ }+ }+ HirKind::Alternation(ref es) => {+ alternate_literals(es, lits, union_required);+ }+ _ => lits.cut(),+ }+}++fn repeat_range_literals<F: FnMut(&Hir, &mut Literals)>(+ e: &Hir,+ min: u32,+ _max: Option<u32>,+ _greedy: bool,+ lits: &mut Literals,+ mut f: F,+) {+ if min == 0 {+ // This is a bit conservative. If `max` is set, then we could+ // treat this as a finite set of alternations. For now, we+ // just treat it as `e*`.+ lits.cut();+ } else {+ // We only extract literals from a single repetition, even though+ // we could do more. e.g., `a{3}` will have `a` extracted instead of+ // `aaa`. The reason is that inner literal extraction can't be unioned+ // across repetitions. e.g., extracting `foofoofoo` from `(\w+foo){3}`+ // is wrong.+ f(e, lits);+ lits.cut();+ }+}++fn alternate_literals<F: FnMut(&Hir, &mut Literals)>(+ es: &[Hir],+ lits: &mut Literals,+ mut f: F,+) {+ let mut lits2 = lits.to_empty();+ for e in es {+ let mut lits3 = lits.to_empty();+ lits3.set_limit_size(lits.limit_size() / 5);+ f(e, &mut lits3);+ if lits3.is_empty() || !lits2.union(lits3) {+ // If we couldn't find suffixes for *any* of the+ // alternates, then the entire alternation has to be thrown+ // away and any existing members must be frozen. Similarly,+ // if the union couldn't complete, stop and freeze.+ lits.cut();+ return;+ }+ }+ // All we do at the moment is look for prefixes and suffixes. If both+ // are empty, then we report nothing. We should be able to do better than+ // this, but we'll need something more expressive than just a "set of+ // literals."+ let lcp = lits2.longest_common_prefix();+ let lcs = lits2.longest_common_suffix();+ if !lcp.is_empty() {+ lits.cross_add(lcp);+ }+ lits.cut();+ if !lcs.is_empty() {+ lits.add(Literal::empty());+ lits.add(Literal::new(lcs.to_vec()));+ }+}++fn is_simple(expr: &Hir) -> bool {+ match *expr.kind() {+ HirKind::Empty+ | HirKind::Literal(_)+ | HirKind::Class(_)+ | HirKind::Repetition(_)+ | HirKind::Concat(_)+ | HirKind::Alternation(_) => true,+ HirKind::Anchor(_) | HirKind::WordBoundary(_) | HirKind::Group(_) => {+ false+ }+ }+}++/// Return the number of characters in the given class.+fn count_unicode_class(cls: &hir::ClassUnicode) -> u32 {+ cls.iter().map(|r| 1 + (r.end() as u32 - r.start() as u32)).sum()+}++/// Return the number of bytes in the given class.+fn count_byte_class(cls: &hir::ClassBytes) -> u32 {+ cls.iter().map(|r| 1 + (r.end() as u32 - r.start() as u32)).sum()+}++#[cfg(test)]+mod tests {+ use super::LiteralSets;+ use regex_syntax::Parser;++ fn sets(pattern: &str) -> LiteralSets {+ let hir = Parser::new().parse(pattern).unwrap();+ LiteralSets::new(&hir)+ }++ fn one_regex(pattern: &str) -> Option<String> {+ sets(pattern).one_regex(false)+ }++ // Put a pattern into the same format as the one returned by `one_regex`.+ fn pat(pattern: &str) -> Option<String> {+ Some(format!("(?-u:{})", pattern))+ }++ #[test]+ fn various() {+ // Obviously no literals.+ assert!(one_regex(r"\w").is_none());+ assert!(one_regex(r"\pL").is_none());++ // Tantalizingly close.+ assert!(one_regex(r"\w|foo").is_none());++ // There's a literal, but it's better if the regex engine handles it+ // internally.+ assert!(one_regex(r"abc").is_none());++ // Core use cases.+ assert_eq!(one_regex(r"\wabc\w"), pat("abc"));+ assert_eq!(one_regex(r"abc\w"), pat("abc"));++ // TODO: Make these pass. We're missing some potentially big wins+ // without these.+ // assert_eq!(one_regex(r"\w(foo|bar|baz)"), pat("foo|bar|baz"));+ // assert_eq!(one_regex(r"\w(foo|bar|baz)\w"), pat("foo|bar|baz"));+ }++ #[test]+ fn regression_1064() {+ // Regression from:+ // https://github.com/BurntSushi/ripgrep/issues/1064+ // assert_eq!(one_regex(r"a.*c"), pat("a"));+ assert_eq!(one_regex(r"a(.*c)"), pat("a"));+ }++ #[test]+ fn regression_1319() {+ // Regression from:+ // https://github.com/BurntSushi/ripgrep/issues/1319+ assert_eq!(+ one_regex(r"TTGAGTCCAGGAG[ATCG]{2}C"),+ pat("TTGAGTCCAGGAGA|TTGAGTCCAGGAGC|\+ TTGAGTCCAGGAGG|TTGAGTCCAGGAGT")+ );+ }+}
@@ -0,0 +1,1066 @@+use std::collections::HashMap;++use grep_matcher::{+ ByteSet, Captures, LineMatchKind, LineTerminator, Match, Matcher, NoError,+};+use regex::bytes::{CaptureLocations, Regex};++use config::{Config, ConfiguredHIR};+use crlf::CRLFMatcher;+use error::Error;+use multi::MultiLiteralMatcher;+use word::WordMatcher;++/// A builder for constructing a `Matcher` using regular expressions.+///+/// This builder re-exports many of the same options found on the regex crate's+/// builder, in addition to a few other options such as smart case, word+/// matching and the ability to set a line terminator which may enable certain+/// types of optimizations.+///+/// The syntax supported is documented as part of the regex crate:+/// https://docs.rs/regex/*/regex/#syntax+#[derive(Clone, Debug)]+pub struct RegexMatcherBuilder {+ config: Config,+}++impl Default for RegexMatcherBuilder {+ fn default() -> RegexMatcherBuilder {+ RegexMatcherBuilder::new()+ }+}++impl RegexMatcherBuilder {+ /// Create a new builder for configuring a regex matcher.+ pub fn new() -> RegexMatcherBuilder {+ RegexMatcherBuilder { config: Config::default() }+ }++ /// Build a new matcher using the current configuration for the provided+ /// pattern.+ ///+ /// The syntax supported is documented as part of the regex crate:+ /// https://docs.rs/regex/*/regex/#syntax+ pub fn build(&self, pattern: &str) -> Result<RegexMatcher, Error> {+ let chir = self.config.hir(pattern)?;+ let fast_line_regex = chir.fast_line_regex()?;+ let non_matching_bytes = chir.non_matching_bytes();+ if let Some(ref re) = fast_line_regex {+ debug!("extracted fast line regex: {:?}", re);+ }++ let matcher = RegexMatcherImpl::new(&chir)?;+ trace!("final regex: {:?}", matcher.regex());+ Ok(RegexMatcher {+ config: self.config.clone(),+ matcher: matcher,+ fast_line_regex: fast_line_regex,+ non_matching_bytes: non_matching_bytes,+ })+ }++ /// Build a new matcher from a plain alternation of literals.+ ///+ /// Depending on the configuration set by the builder, this may be able to+ /// build a matcher substantially faster than by joining the patterns with+ /// a `|` and calling `build`.+ pub fn build_literals<B: AsRef<str>>(+ &self,+ literals: &[B],+ ) -> Result<RegexMatcher, Error> {+ let mut has_escape = false;+ let mut slices = vec![];+ for lit in literals {+ slices.push(lit.as_ref());+ has_escape = has_escape || lit.as_ref().contains('\\');+ }+ // Even when we have a fixed set of literals, we might still want to+ // use the regex engine. Specifically, if any string has an escape+ // in it, then we probably can't feed it to Aho-Corasick without+ // removing the escape. Additionally, if there are any particular+ // special match semantics we need to honor, that Aho-Corasick isn't+ // enough. Finally, the regex engine can do really well with a small+ // number of literals (at time of writing, this is changing soon), so+ // we use it when there's a small set.+ //+ // Yes, this is one giant hack. Ideally, this entirely separate literal+ // matcher that uses Aho-Corasick would be pushed down into the regex+ // engine.+ if has_escape+ || !self.config.can_plain_aho_corasick()+ || literals.len() < 40+ {+ return self.build(&slices.join("|"));+ }++ let matcher = MultiLiteralMatcher::new(&slices)?;+ let imp = RegexMatcherImpl::MultiLiteral(matcher);+ Ok(RegexMatcher {+ config: self.config.clone(),+ matcher: imp,+ fast_line_regex: None,+ non_matching_bytes: ByteSet::empty(),+ })+ }++ /// Set the value for the case insensitive (`i`) flag.+ ///+ /// When enabled, letters in the pattern will match both upper case and+ /// lower case variants.+ pub fn case_insensitive(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.case_insensitive = yes;+ self+ }++ /// Whether to enable "smart case" or not.+ ///+ /// When smart case is enabled, the builder will automatically enable+ /// case insensitive matching based on how the pattern is written. Namely,+ /// case insensitive mode is enabled when both of the following things+ /// are true:+ ///+ /// 1. The pattern contains at least one literal character. For example,+ /// `a\w` contains a literal (`a`) but `\w` does not.+ /// 2. Of the literals in the pattern, none of them are considered to be+ /// uppercase according to Unicode. For example, `foo\pL` has no+ /// uppercase literals but `Foo\pL` does.+ pub fn case_smart(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.case_smart = yes;+ self+ }++ /// Set the value for the multi-line matching (`m`) flag.+ ///+ /// When enabled, `^` matches the beginning of lines and `$` matches the+ /// end of lines.+ ///+ /// By default, they match beginning/end of the input.+ pub fn multi_line(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.multi_line = yes;+ self+ }++ /// Set the value for the any character (`s`) flag, where in `.` matches+ /// anything when `s` is set and matches anything except for new line when+ /// it is not set (the default).+ ///+ /// N.B. "matches anything" means "any byte" when Unicode is disabled and+ /// means "any valid UTF-8 encoding of any Unicode scalar value" when+ /// Unicode is enabled.+ pub fn dot_matches_new_line(+ &mut self,+ yes: bool,+ ) -> &mut RegexMatcherBuilder {+ self.config.dot_matches_new_line = yes;+ self+ }++ /// Set the value for the greedy swap (`U`) flag.+ ///+ /// When enabled, a pattern like `a*` is lazy (tries to find shortest+ /// match) and `a*?` is greedy (tries to find longest match).+ ///+ /// By default, `a*` is greedy and `a*?` is lazy.+ pub fn swap_greed(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.swap_greed = yes;+ self+ }++ /// Set the value for the ignore whitespace (`x`) flag.+ ///+ /// When enabled, whitespace such as new lines and spaces will be ignored+ /// between expressions of the pattern, and `#` can be used to start a+ /// comment until the next new line.+ pub fn ignore_whitespace(+ &mut self,+ yes: bool,+ ) -> &mut RegexMatcherBuilder {+ self.config.ignore_whitespace = yes;+ self+ }++ /// Set the value for the Unicode (`u`) flag.+ ///+ /// Enabled by default. When disabled, character classes such as `\w` only+ /// match ASCII word characters instead of all Unicode word characters.+ pub fn unicode(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.unicode = yes;+ self+ }++ /// Whether to support octal syntax or not.+ ///+ /// Octal syntax is a little-known way of uttering Unicode codepoints in+ /// a regular expression. For example, `a`, `\x61`, `\u0061` and+ /// `\141` are all equivalent regular expressions, where the last example+ /// shows octal syntax.+ ///+ /// While supporting octal syntax isn't in and of itself a problem, it does+ /// make good error messages harder. That is, in PCRE based regex engines,+ /// syntax like `\0` invokes a backreference, which is explicitly+ /// unsupported in Rust's regex engine. However, many users expect it to+ /// be supported. Therefore, when octal support is disabled, the error+ /// message will explicitly mention that backreferences aren't supported.+ ///+ /// Octal syntax is disabled by default.+ pub fn octal(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.octal = yes;+ self+ }++ /// Set the approximate size limit of the compiled regular expression.+ ///+ /// This roughly corresponds to the number of bytes occupied by a single+ /// compiled program. If the program exceeds this number, then a+ /// compilation error is returned.+ pub fn size_limit(&mut self, bytes: usize) -> &mut RegexMatcherBuilder {+ self.config.size_limit = bytes;+ self+ }++ /// Set the approximate size of the cache used by the DFA.+ ///+ /// This roughly corresponds to the number of bytes that the DFA will+ /// use while searching.+ ///+ /// Note that this is a *per thread* limit. There is no way to set a global+ /// limit. In particular, if a regex is used from multiple threads+ /// simultaneously, then each thread may use up to the number of bytes+ /// specified here.+ pub fn dfa_size_limit(+ &mut self,+ bytes: usize,+ ) -> &mut RegexMatcherBuilder {+ self.config.dfa_size_limit = bytes;+ self+ }++ /// Set the nesting limit for this parser.+ ///+ /// The nesting limit controls how deep the abstract syntax tree is allowed+ /// to be. If the AST exceeds the given limit (e.g., with too many nested+ /// groups), then an error is returned by the parser.+ ///+ /// The purpose of this limit is to act as a heuristic to prevent stack+ /// overflow for consumers that do structural induction on an `Ast` using+ /// explicit recursion. While this crate never does this (instead using+ /// constant stack space and moving the call stack to the heap), other+ /// crates may.+ ///+ /// This limit is not checked until the entire Ast is parsed. Therefore,+ /// if callers want to put a limit on the amount of heap space used, then+ /// they should impose a limit on the length, in bytes, of the concrete+ /// pattern string. In particular, this is viable since this parser+ /// implementation will limit itself to heap space proportional to the+ /// lenth of the pattern string.+ ///+ /// Note that a nest limit of `0` will return a nest limit error for most+ /// patterns but not all. For example, a nest limit of `0` permits `a` but+ /// not `ab`, since `ab` requires a concatenation, which results in a nest+ /// depth of `1`. In general, a nest limit is not something that manifests+ /// in an obvious way in the concrete syntax, therefore, it should not be+ /// used in a granular way.+ pub fn nest_limit(&mut self, limit: u32) -> &mut RegexMatcherBuilder {+ self.config.nest_limit = limit;+ self+ }++ /// Set an ASCII line terminator for the matcher.+ ///+ /// The purpose of setting a line terminator is to enable a certain class+ /// of optimizations that can make line oriented searching faster. Namely,+ /// when a line terminator is enabled, then the builder will guarantee that+ /// the resulting matcher will never be capable of producing a match that+ /// contains the line terminator. Because of this guarantee, users of the+ /// resulting matcher do not need to slowly execute a search line by line+ /// for line oriented search.+ ///+ /// If the aforementioned guarantee about not matching a line terminator+ /// cannot be made because of how the pattern was written, then the builder+ /// will return an error when attempting to construct the matcher. For+ /// example, the pattern `a\sb` will be transformed such that it can never+ /// match `a\nb` (when `\n` is the line terminator), but the pattern `a\nb`+ /// will result in an error since the `\n` cannot be easily removed without+ /// changing the fundamental intent of the pattern.+ ///+ /// If the given line terminator isn't an ASCII byte (`<=127`), then the+ /// builder will return an error when constructing the matcher.+ pub fn line_terminator(+ &mut self,+ line_term: Option<u8>,+ ) -> &mut RegexMatcherBuilder {+ self.config.line_terminator = line_term.map(LineTerminator::byte);+ self+ }++ /// Set the line terminator to `\r\n` and enable CRLF matching for `$` in+ /// regex patterns.+ ///+ /// This method sets two distinct settings:+ ///+ /// 1. It causes the line terminator for the matcher to be `\r\n`. Namely,+ /// this prevents the matcher from ever producing a match that contains+ /// a `\r` or `\n`.+ /// 2. It translates all instances of `$` in the pattern to `(?:\r??$)`.+ /// This works around the fact that the regex engine does not support+ /// matching CRLF as a line terminator when using `$`.+ ///+ /// In particular, because of (2), the matches produced by the matcher may+ /// be slightly different than what one would expect given the pattern.+ /// This is the trade off made: in many cases, `$` will "just work" in the+ /// presence of `\r\n` line terminators, but matches may require some+ /// trimming to faithfully represent the intended match.+ ///+ /// Note that if you do not wish to set the line terminator but would still+ /// like `$` to match `\r\n` line terminators, then it is valid to call+ /// `crlf(true)` followed by `line_terminator(None)`. Ordering is+ /// important, since `crlf` and `line_terminator` override each other.+ pub fn crlf(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ if yes {+ self.config.line_terminator = Some(LineTerminator::crlf());+ } else {+ self.config.line_terminator = None;+ }+ self.config.crlf = yes;+ self+ }++ /// Require that all matches occur on word boundaries.+ ///+ /// Enabling this option is subtly different than putting `\b` assertions+ /// on both sides of your pattern. In particular, a `\b` assertion requires+ /// that one side of it match a word character while the other match a+ /// non-word character. This option, in contrast, merely requires that+ /// one side match a non-word character.+ ///+ /// For example, `\b-2\b` will not match `foo -2 bar` since `-` is not a+ /// word character. However, `-2` with this `word` option enabled will+ /// match the `-2` in `foo -2 bar`.+ pub fn word(&mut self, yes: bool) -> &mut RegexMatcherBuilder {+ self.config.word = yes;+ self+ }+}++/// An implementation of the `Matcher` trait using Rust's standard regex+/// library.+#[derive(Clone, Debug)]+pub struct RegexMatcher {+ /// The configuration specified by the caller.+ config: Config,+ /// The underlying matcher implementation.+ matcher: RegexMatcherImpl,+ /// A regex that never reports false negatives but may report false+ /// positives that is believed to be capable of being matched more quickly+ /// than `regex`. Typically, this is a single literal or an alternation+ /// of literals.+ fast_line_regex: Option<Regex>,+ /// A set of bytes that will never appear in a match.+ non_matching_bytes: ByteSet,+}++impl RegexMatcher {+ /// Create a new matcher from the given pattern using the default+ /// configuration.+ pub fn new(pattern: &str) -> Result<RegexMatcher, Error> {+ RegexMatcherBuilder::new().build(pattern)+ }++ /// Create a new matcher from the given pattern using the default+ /// configuration, but matches lines terminated by `\n`.+ ///+ /// This is meant to be a convenience constructor for using a+ /// `RegexMatcherBuilder` and setting its+ /// [`line_terminator`](struct.RegexMatcherBuilder.html#method.line_terminator)+ /// to `\n`. The purpose of using this constructor is to permit special+ /// optimizations that help speed up line oriented search. These types of+ /// optimizations are only appropriate when matches span no more than one+ /// line. For this reason, this constructor will return an error if the+ /// given pattern contains a literal `\n`. Other uses of `\n` (such as in+ /// `\s`) are removed transparently.+ pub fn new_line_matcher(pattern: &str) -> Result<RegexMatcher, Error> {+ RegexMatcherBuilder::new().line_terminator(Some(b'\n')).build(pattern)+ }+}++/// An encapsulation of the type of matcher we use in `RegexMatcher`.+#[derive(Clone, Debug)]+enum RegexMatcherImpl {+ /// The standard matcher used for all regular expressions.+ Standard(StandardMatcher),+ /// A matcher for an alternation of plain literals.+ MultiLiteral(MultiLiteralMatcher),+ /// A matcher that strips `\r` from the end of matches.+ ///+ /// This is only used when the CRLF hack is enabled and the regex is line+ /// anchored at the end.+ CRLF(CRLFMatcher),+ /// A matcher that only matches at word boundaries. This transforms the+ /// regex to `(^|\W)(...)($|\W)` instead of the more intuitive `\b(...)\b`.+ /// Because of this, the WordMatcher provides its own implementation of+ /// `Matcher` to encapsulate its use of capture groups to make them+ /// invisible to the caller.+ Word(WordMatcher),+}++impl RegexMatcherImpl {+ /// Based on the configuration, create a new implementation of the+ /// `Matcher` trait.+ fn new(expr: &ConfiguredHIR) -> Result<RegexMatcherImpl, Error> {+ if expr.config().word {+ Ok(RegexMatcherImpl::Word(WordMatcher::new(expr)?))+ } else if expr.needs_crlf_stripped() {+ Ok(RegexMatcherImpl::CRLF(CRLFMatcher::new(expr)?))+ } else {+ if let Some(lits) = expr.alternation_literals() {+ if lits.len() >= 40 {+ let matcher = MultiLiteralMatcher::new(&lits)?;+ return Ok(RegexMatcherImpl::MultiLiteral(matcher));+ }+ }+ Ok(RegexMatcherImpl::Standard(StandardMatcher::new(expr)?))+ }+ }++ /// Return the underlying regex object used.+ fn regex(&self) -> String {+ match *self {+ RegexMatcherImpl::Word(ref x) => x.regex().to_string(),+ RegexMatcherImpl::CRLF(ref x) => x.regex().to_string(),+ RegexMatcherImpl::MultiLiteral(_) => "<N/A>".to_string(),+ RegexMatcherImpl::Standard(ref x) => x.regex.to_string(),+ }+ }+}++// This implementation just dispatches on the internal matcher impl except+// for the line terminator optimization, which is possibly executed via+// `fast_line_regex`.+impl Matcher for RegexMatcher {+ type Captures = RegexCaptures;+ type Error = NoError;++ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.find_at(haystack, at),+ MultiLiteral(ref m) => m.find_at(haystack, at),+ CRLF(ref m) => m.find_at(haystack, at),+ Word(ref m) => m.find_at(haystack, at),+ }+ }++ fn new_captures(&self) -> Result<RegexCaptures, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.new_captures(),+ MultiLiteral(ref m) => m.new_captures(),+ CRLF(ref m) => m.new_captures(),+ Word(ref m) => m.new_captures(),+ }+ }++ fn capture_count(&self) -> usize {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.capture_count(),+ MultiLiteral(ref m) => m.capture_count(),+ CRLF(ref m) => m.capture_count(),+ Word(ref m) => m.capture_count(),+ }+ }++ fn capture_index(&self, name: &str) -> Option<usize> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.capture_index(name),+ MultiLiteral(ref m) => m.capture_index(name),+ CRLF(ref m) => m.capture_index(name),+ Word(ref m) => m.capture_index(name),+ }+ }++ fn find(&self, haystack: &[u8]) -> Result<Option<Match>, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.find(haystack),+ MultiLiteral(ref m) => m.find(haystack),+ CRLF(ref m) => m.find(haystack),+ Word(ref m) => m.find(haystack),+ }+ }++ fn find_iter<F>(&self, haystack: &[u8], matched: F) -> Result<(), NoError>+ where+ F: FnMut(Match) -> bool,+ {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.find_iter(haystack, matched),+ MultiLiteral(ref m) => m.find_iter(haystack, matched),+ CRLF(ref m) => m.find_iter(haystack, matched),+ Word(ref m) => m.find_iter(haystack, matched),+ }+ }++ fn try_find_iter<F, E>(+ &self,+ haystack: &[u8],+ matched: F,+ ) -> Result<Result<(), E>, NoError>+ where+ F: FnMut(Match) -> Result<bool, E>,+ {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.try_find_iter(haystack, matched),+ MultiLiteral(ref m) => m.try_find_iter(haystack, matched),+ CRLF(ref m) => m.try_find_iter(haystack, matched),+ Word(ref m) => m.try_find_iter(haystack, matched),+ }+ }++ fn captures(+ &self,+ haystack: &[u8],+ caps: &mut RegexCaptures,+ ) -> Result<bool, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.captures(haystack, caps),+ MultiLiteral(ref m) => m.captures(haystack, caps),+ CRLF(ref m) => m.captures(haystack, caps),+ Word(ref m) => m.captures(haystack, caps),+ }+ }++ fn captures_iter<F>(+ &self,+ haystack: &[u8],+ caps: &mut RegexCaptures,+ matched: F,+ ) -> Result<(), NoError>+ where+ F: FnMut(&RegexCaptures) -> bool,+ {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.captures_iter(haystack, caps, matched),+ MultiLiteral(ref m) => m.captures_iter(haystack, caps, matched),+ CRLF(ref m) => m.captures_iter(haystack, caps, matched),+ Word(ref m) => m.captures_iter(haystack, caps, matched),+ }+ }++ fn try_captures_iter<F, E>(+ &self,+ haystack: &[u8],+ caps: &mut RegexCaptures,+ matched: F,+ ) -> Result<Result<(), E>, NoError>+ where+ F: FnMut(&RegexCaptures) -> Result<bool, E>,+ {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.try_captures_iter(haystack, caps, matched),+ MultiLiteral(ref m) => {+ m.try_captures_iter(haystack, caps, matched)+ }+ CRLF(ref m) => m.try_captures_iter(haystack, caps, matched),+ Word(ref m) => m.try_captures_iter(haystack, caps, matched),+ }+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut RegexCaptures,+ ) -> Result<bool, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.captures_at(haystack, at, caps),+ MultiLiteral(ref m) => m.captures_at(haystack, at, caps),+ CRLF(ref m) => m.captures_at(haystack, at, caps),+ Word(ref m) => m.captures_at(haystack, at, caps),+ }+ }++ fn replace<F>(+ &self,+ haystack: &[u8],+ dst: &mut Vec<u8>,+ append: F,+ ) -> Result<(), NoError>+ where+ F: FnMut(Match, &mut Vec<u8>) -> bool,+ {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.replace(haystack, dst, append),+ MultiLiteral(ref m) => m.replace(haystack, dst, append),+ CRLF(ref m) => m.replace(haystack, dst, append),+ Word(ref m) => m.replace(haystack, dst, append),+ }+ }++ fn replace_with_captures<F>(+ &self,+ haystack: &[u8],+ caps: &mut RegexCaptures,+ dst: &mut Vec<u8>,+ append: F,+ ) -> Result<(), NoError>+ where+ F: FnMut(&Self::Captures, &mut Vec<u8>) -> bool,+ {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => {+ m.replace_with_captures(haystack, caps, dst, append)+ }+ MultiLiteral(ref m) => {+ m.replace_with_captures(haystack, caps, dst, append)+ }+ CRLF(ref m) => {+ m.replace_with_captures(haystack, caps, dst, append)+ }+ Word(ref m) => {+ m.replace_with_captures(haystack, caps, dst, append)+ }+ }+ }++ fn is_match(&self, haystack: &[u8]) -> Result<bool, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.is_match(haystack),+ MultiLiteral(ref m) => m.is_match(haystack),+ CRLF(ref m) => m.is_match(haystack),+ Word(ref m) => m.is_match(haystack),+ }+ }++ fn is_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<bool, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.is_match_at(haystack, at),+ MultiLiteral(ref m) => m.is_match_at(haystack, at),+ CRLF(ref m) => m.is_match_at(haystack, at),+ Word(ref m) => m.is_match_at(haystack, at),+ }+ }++ fn shortest_match(+ &self,+ haystack: &[u8],+ ) -> Result<Option<usize>, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.shortest_match(haystack),+ MultiLiteral(ref m) => m.shortest_match(haystack),+ CRLF(ref m) => m.shortest_match(haystack),+ Word(ref m) => m.shortest_match(haystack),+ }+ }++ fn shortest_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<usize>, NoError> {+ use self::RegexMatcherImpl::*;+ match self.matcher {+ Standard(ref m) => m.shortest_match_at(haystack, at),+ MultiLiteral(ref m) => m.shortest_match_at(haystack, at),+ CRLF(ref m) => m.shortest_match_at(haystack, at),+ Word(ref m) => m.shortest_match_at(haystack, at),+ }+ }++ fn non_matching_bytes(&self) -> Option<&ByteSet> {+ Some(&self.non_matching_bytes)+ }++ fn line_terminator(&self) -> Option<LineTerminator> {+ self.config.line_terminator+ }++ fn find_candidate_line(+ &self,+ haystack: &[u8],+ ) -> Result<Option<LineMatchKind>, NoError> {+ Ok(match self.fast_line_regex {+ Some(ref regex) => {+ regex.shortest_match(haystack).map(LineMatchKind::Candidate)+ }+ None => {+ self.shortest_match(haystack)?.map(LineMatchKind::Confirmed)+ }+ })+ }+}++/// The implementation of the standard regex matcher.+#[derive(Clone, Debug)]+struct StandardMatcher {+ /// The regular expression compiled from the pattern provided by the+ /// caller.+ regex: Regex,+ /// A map from capture group name to its corresponding index.+ names: HashMap<String, usize>,+}++impl StandardMatcher {+ fn new(expr: &ConfiguredHIR) -> Result<StandardMatcher, Error> {+ let regex = expr.regex()?;+ let mut names = HashMap::new();+ for (i, optional_name) in regex.capture_names().enumerate() {+ if let Some(name) = optional_name {+ names.insert(name.to_string(), i);+ }+ }+ Ok(StandardMatcher { regex, names })+ }+}++impl Matcher for StandardMatcher {+ type Captures = RegexCaptures;+ type Error = NoError;++ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, NoError> {+ Ok(self+ .regex+ .find_at(haystack, at)+ .map(|m| Match::new(m.start(), m.end())))+ }++ fn new_captures(&self) -> Result<RegexCaptures, NoError> {+ Ok(RegexCaptures::new(self.regex.capture_locations()))+ }++ fn capture_count(&self) -> usize {+ self.regex.captures_len()+ }++ fn capture_index(&self, name: &str) -> Option<usize> {+ self.names.get(name).map(|i| *i)+ }++ fn try_find_iter<F, E>(+ &self,+ haystack: &[u8],+ mut matched: F,+ ) -> Result<Result<(), E>, NoError>+ where+ F: FnMut(Match) -> Result<bool, E>,+ {+ for m in self.regex.find_iter(haystack) {+ match matched(Match::new(m.start(), m.end())) {+ Ok(true) => continue,+ Ok(false) => return Ok(Ok(())),+ Err(err) => return Ok(Err(err)),+ }+ }+ Ok(Ok(()))+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut RegexCaptures,+ ) -> Result<bool, NoError> {+ Ok(self+ .regex+ .captures_read_at(&mut caps.locations_mut(), haystack, at)+ .is_some())+ }++ fn shortest_match_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<usize>, NoError> {+ Ok(self.regex.shortest_match_at(haystack, at))+ }+}++/// Represents the match offsets of each capturing group in a match.+///+/// The first, or `0`th capture group, always corresponds to the entire match+/// and is guaranteed to be present when a match occurs. The next capture+/// group, at index `1`, corresponds to the first capturing group in the regex,+/// ordered by the position at which the left opening parenthesis occurs.+///+/// Note that not all capturing groups are guaranteed to be present in a match.+/// For example, in the regex, `(?P<foo>\w)|(?P<bar>\W)`, only one of `foo`+/// or `bar` will ever be set in any given match.+///+/// In order to access a capture group by name, you'll need to first find the+/// index of the group using the corresponding matcher's `capture_index`+/// method, and then use that index with `RegexCaptures::get`.+#[derive(Clone, Debug)]+pub struct RegexCaptures(RegexCapturesImp);++#[derive(Clone, Debug)]+enum RegexCapturesImp {+ AhoCorasick {+ /// The start and end of the match, corresponding to capture group 0.+ mat: Option<Match>,+ },+ Regex {+ /// Where the locations are stored.+ locs: CaptureLocations,+ /// These captures behave as if the capturing groups begin at the given+ /// offset. When set to `0`, this has no affect and capture groups are+ /// indexed like normal.+ ///+ /// This is useful when building matchers that wrap arbitrary regular+ /// expressions. For example, `WordMatcher` takes an existing regex+ /// `re` and creates `(?:^|\W)(re)(?:$|\W)`, but hides the fact that+ /// the regex has been wrapped from the caller. In order to do this,+ /// the matcher and the capturing groups must behave as if `(re)` is+ /// the `0`th capture group.+ offset: usize,+ /// When enable, the end of a match has `\r` stripped from it, if one+ /// exists.+ strip_crlf: bool,+ },+}++impl Captures for RegexCaptures {+ fn len(&self) -> usize {+ match self.0 {+ RegexCapturesImp::AhoCorasick { .. } => 1,+ RegexCapturesImp::Regex { ref locs, offset, .. } => {+ locs.len().checked_sub(offset).unwrap()+ }+ }+ }++ fn get(&self, i: usize) -> Option<Match> {+ match self.0 {+ RegexCapturesImp::AhoCorasick { mat, .. } => {+ if i == 0 {+ mat+ } else {+ None+ }+ }+ RegexCapturesImp::Regex { ref locs, offset, strip_crlf } => {+ if !strip_crlf {+ let actual = i.checked_add(offset).unwrap();+ return locs.pos(actual).map(|(s, e)| Match::new(s, e));+ }++ // currently don't support capture offsetting with CRLF+ // stripping+ assert_eq!(offset, 0);+ let m = match locs.pos(i).map(|(s, e)| Match::new(s, e)) {+ None => return None,+ Some(m) => m,+ };+ // If the end position of this match corresponds to the end+ // position of the overall match, then we apply our CRLF+ // stripping. Otherwise, we cannot assume stripping is correct.+ if i == 0 || m.end() == locs.pos(0).unwrap().1 {+ Some(m.with_end(m.end() - 1))+ } else {+ Some(m)+ }+ }+ }+ }+}++impl RegexCaptures {+ pub(crate) fn simple() -> RegexCaptures {+ RegexCaptures(RegexCapturesImp::AhoCorasick { mat: None })+ }++ pub(crate) fn new(locs: CaptureLocations) -> RegexCaptures {+ RegexCaptures::with_offset(locs, 0)+ }++ pub(crate) fn with_offset(+ locs: CaptureLocations,+ offset: usize,+ ) -> RegexCaptures {+ RegexCaptures(RegexCapturesImp::Regex {+ locs,+ offset,+ strip_crlf: false,+ })+ }++ pub(crate) fn locations(&self) -> &CaptureLocations {+ match self.0 {+ RegexCapturesImp::AhoCorasick { .. } => {+ panic!("getting locations for simple captures is invalid")+ }+ RegexCapturesImp::Regex { ref locs, .. } => locs,+ }+ }++ pub(crate) fn locations_mut(&mut self) -> &mut CaptureLocations {+ match self.0 {+ RegexCapturesImp::AhoCorasick { .. } => {+ panic!("getting locations for simple captures is invalid")+ }+ RegexCapturesImp::Regex { ref mut locs, .. } => locs,+ }+ }++ pub(crate) fn strip_crlf(&mut self, yes: bool) {+ match self.0 {+ RegexCapturesImp::AhoCorasick { .. } => {+ panic!("setting strip_crlf for simple captures is invalid")+ }+ RegexCapturesImp::Regex { ref mut strip_crlf, .. } => {+ *strip_crlf = yes;+ }+ }+ }++ pub(crate) fn set_simple(&mut self, one: Option<Match>) {+ match self.0 {+ RegexCapturesImp::AhoCorasick { ref mut mat } => {+ *mat = one;+ }+ RegexCapturesImp::Regex { .. } => {+ panic!("setting simple captures for regex is invalid")+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use grep_matcher::{LineMatchKind, Matcher};++ // Test that enabling word matches does the right thing and demonstrate+ // the difference between it and surrounding the regex in `\b`.+ #[test]+ fn word() {+ let matcher =+ RegexMatcherBuilder::new().word(true).build(r"-2").unwrap();+ assert!(matcher.is_match(b"abc -2 foo").unwrap());++ let matcher =+ RegexMatcherBuilder::new().word(false).build(r"\b-2\b").unwrap();+ assert!(!matcher.is_match(b"abc -2 foo").unwrap());+ }++ // Test that enabling a line terminator prevents it from matching through+ // said line terminator.+ #[test]+ fn line_terminator() {+ // This works, because there's no line terminator specified.+ let matcher = RegexMatcherBuilder::new().build(r"abc\sxyz").unwrap();+ assert!(matcher.is_match(b"abc\nxyz").unwrap());++ // This doesn't.+ let matcher = RegexMatcherBuilder::new()+ .line_terminator(Some(b'\n'))+ .build(r"abc\sxyz")+ .unwrap();+ assert!(!matcher.is_match(b"abc\nxyz").unwrap());+ }++ // Ensure that the builder returns an error if a line terminator is set+ // and the regex could not be modified to remove a line terminator.+ #[test]+ fn line_terminator_error() {+ assert!(RegexMatcherBuilder::new()+ .line_terminator(Some(b'\n'))+ .build(r"a\nz")+ .is_err())+ }++ // Test that enabling CRLF permits `$` to match at the end of a line.+ #[test]+ fn line_terminator_crlf() {+ // Test normal use of `$` with a `\n` line terminator.+ let matcher = RegexMatcherBuilder::new()+ .multi_line(true)+ .build(r"abc$")+ .unwrap();+ assert!(matcher.is_match(b"abc\n").unwrap());++ // Test that `$` doesn't match at `\r\n` boundary normally.+ let matcher = RegexMatcherBuilder::new()+ .multi_line(true)+ .build(r"abc$")+ .unwrap();+ assert!(!matcher.is_match(b"abc\r\n").unwrap());++ // Now check the CRLF handling.+ let matcher = RegexMatcherBuilder::new()+ .multi_line(true)+ .crlf(true)+ .build(r"abc$")+ .unwrap();+ assert!(matcher.is_match(b"abc\r\n").unwrap());+ }++ // Test that smart case works.+ #[test]+ fn case_smart() {+ let matcher =+ RegexMatcherBuilder::new().case_smart(true).build(r"abc").unwrap();+ assert!(matcher.is_match(b"ABC").unwrap());++ let matcher =+ RegexMatcherBuilder::new().case_smart(true).build(r"aBc").unwrap();+ assert!(!matcher.is_match(b"ABC").unwrap());+ }++ // Test that finding candidate lines works as expected.+ #[test]+ fn candidate_lines() {+ fn is_confirmed(m: LineMatchKind) -> bool {+ match m {+ LineMatchKind::Confirmed(_) => true,+ _ => false,+ }+ }+ fn is_candidate(m: LineMatchKind) -> bool {+ match m {+ LineMatchKind::Candidate(_) => true,+ _ => false,+ }+ }++ // With no line terminator set, we can't employ any optimizations,+ // so we get a confirmed match.+ let matcher = RegexMatcherBuilder::new().build(r"\wfoo\s").unwrap();+ let m = matcher.find_candidate_line(b"afoo ").unwrap().unwrap();+ assert!(is_confirmed(m));++ // With a line terminator and a regex specially crafted to have an+ // easy-to-detect inner literal, we can apply an optimization that+ // quickly finds candidate matches.+ let matcher = RegexMatcherBuilder::new()+ .line_terminator(Some(b'\n'))+ .build(r"\wfoo\s")+ .unwrap();+ let m = matcher.find_candidate_line(b"afoo ").unwrap().unwrap();+ assert!(is_candidate(m));+ }+}
crates/regex/src/multi.rs
@@ -0,0 +1,125 @@+use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};+use grep_matcher::{Match, Matcher, NoError};+use regex_syntax::hir::Hir;++use error::Error;+use matcher::RegexCaptures;++/// A matcher for an alternation of literals.+///+/// Ideally, this optimization would be pushed down into the regex engine, but+/// making this work correctly there would require quite a bit of refactoring.+/// Moreover, doing it one layer above lets us do thing like, "if we+/// specifically only want to search for literals, then don't bother with+/// regex parsing at all."+#[derive(Clone, Debug)]+pub struct MultiLiteralMatcher {+ /// The Aho-Corasick automaton.+ ac: AhoCorasick,+}++impl MultiLiteralMatcher {+ /// Create a new multi-literal matcher from the given literals.+ pub fn new<B: AsRef<[u8]>>(+ literals: &[B],+ ) -> Result<MultiLiteralMatcher, Error> {+ let ac = AhoCorasickBuilder::new()+ .match_kind(MatchKind::LeftmostFirst)+ .auto_configure(literals)+ .build_with_size::<usize, _, _>(literals)+ .map_err(Error::regex)?;+ Ok(MultiLiteralMatcher { ac })+ }+}++impl Matcher for MultiLiteralMatcher {+ type Captures = RegexCaptures;+ type Error = NoError;++ fn find_at(+ &self,+ haystack: &[u8],+ at: usize,+ ) -> Result<Option<Match>, NoError> {+ match self.ac.find(&haystack[at..]) {+ None => Ok(None),+ Some(m) => Ok(Some(Match::new(at + m.start(), at + m.end()))),+ }+ }++ fn new_captures(&self) -> Result<RegexCaptures, NoError> {+ Ok(RegexCaptures::simple())+ }++ fn capture_count(&self) -> usize {+ 1+ }++ fn capture_index(&self, _: &str) -> Option<usize> {+ None+ }++ fn captures_at(+ &self,+ haystack: &[u8],+ at: usize,+ caps: &mut RegexCaptures,+ ) -> Result<bool, NoError> {+ caps.set_simple(None);+ let mat = self.find_at(haystack, at)?;+ caps.set_simple(mat);+ Ok(mat.is_some())+ }++ // We specifically do not implement other methods like find_iter. Namely,+ // the iter methods are guaranteed to be correct by virtue of implementing+ // find_at above.+}++/// Alternation literals checks if the given HIR is a simple alternation of+/// literals, and if so, returns them. Otherwise, this returns None.+pub fn alternation_literals(expr: &Hir) -> Option<Vec<Vec<u8>>> {+ use regex_syntax::hir::{HirKind, Literal};++ // This is pretty hacky, but basically, if `is_alternation_literal` is+ // true, then we can make several assumptions about the structure of our+ // HIR. This is what justifies the `unreachable!` statements below.++ if !expr.is_alternation_literal() {+ return None;+ }+ let alts = match *expr.kind() {+ HirKind::Alternation(ref alts) => alts,+ _ => return None, // one literal isn't worth it+ };++ let extendlit = |lit: &Literal, dst: &mut Vec<u8>| match *lit {+ Literal::Unicode(c) => {+ let mut buf = [0; 4];+ dst.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());+ }+ Literal::Byte(b) => {+ dst.push(b);+ }+ };+
The diff stops at 1024 KiB. git show fdd8510 shows the rest.