261023.2 · cff2849fd49557c2184a3b2858469a2bfbe20753
index: revert indexing additions
I think that this was probably a failed experiment. I suspect that it is too difficult to retro-fit indexing on top of ripgrep. Among other issues, one of the main problems is knowing what to index. Surely most cases want to use ripgrep's defaults (i.e., its "smart" filtering), but some use cases might want to ignore part or all of ripgrep's filtering. Reconciling this with ripgrep's existing command line structure in a user friendly way seems fraught.
Ref #1497
reading declarations
Changes to crates/core/flags/. All files
@@ -25,9 +25,9 @@ use crate::flags::{ Category, Flag, FlagValue, lowargs::{ BinaryMode, BoundaryMode, BufferMode, CaseMode, ColorChoice,- ContextMode, EncodingMode, EngineChoice, GenerateMode, IndexMode,- LoggingMode, LowArgs, MmapMode, Mode, PatternSource, SearchMode,- SortMode, SortModeKind, SpecialMode, TypeChange,+ ContextMode, EncodingMode, EngineChoice, GenerateMode, LoggingMode,+ LowArgs, MmapMode, Mode, PatternSource, SearchMode, SortMode,+ SortModeKind, SpecialMode, TypeChange, }, }; @@ -84,10 +84,6 @@ pub(super) const FLAGS: &[&dyn Flag] = &[ &IgnoreFile, &IgnoreFileCaseInsensitive, &IncludeZero,- &Index,- &IndexCrud,- &IndexForce,- &IndexPath, &InvertMatch, &JSON, &LineBuffered,@@ -155,79 +151,6 @@ pub(super) const FLAGS: &[&dyn Flag] = &[ &SortFiles, ]; -impl LowArgs {- /// Returns a flag that does not support indexing, if it's enabled.- ///- /// If there aren't any flags enabled that don't support indexing, then- /// `None` is returned.- ///- /// The idea of this routine is to start out very paranoid about what is- /// allowed to be used when indexing is enabled. Ideally, most or all of- /// these would eventually become supported.- pub(super) fn indexing_unsupported_flag(- &self,- ) -> Option<&'static dyn Flag> {- if matches!(self.mode, Mode::Search(SearchMode::FilesWithoutMatch)) {- return Some(&FilesWithoutMatch);- }- if matches!(self.binary, BinaryMode::AsText) {- return Some(&Binary);- }- if !matches!(self.encoding, EncodingMode::Auto) {- return Some(&Encoding);- }- if matches!(self.engine, EngineChoice::PCRE2) {- return Some(&Engine);- }- if self.follow {- return Some(&Follow);- }- if !self.globs.is_empty() {- return Some(&Glob);- }- if self.hidden {- return Some(&Hidden);- }- if !self.iglobs.is_empty() {- return Some(&Glob);- }- if !self.ignore_file.is_empty() {- return Some(&IgnoreFile);- }- if self.no_ignore_dot {- return Some(&NoIgnoreDot);- }- if self.no_ignore_exclude {- return Some(&NoIgnoreExclude);- }- if self.no_ignore_files {- return Some(&NoIgnoreFiles);- }- if self.no_ignore_global {- return Some(&NoIgnoreGlobal);- }- if self.no_ignore_parent {- return Some(&NoIgnoreParent);- }- if self.no_ignore_vcs {- return Some(&NoIgnoreVcs);- }- if self.no_require_git {- return Some(&NoRequireGit);- }- if self.one_file_system {- return Some(&OneFileSystem);- }- if self.pre.is_some() {- return Some(&Pre);- }- if self.search_zip {- return Some(&SearchZip);- }- None- }-}- /// -A/--after-context #[derive(Debug)] struct AfterContext;@@ -3450,304 +3373,6 @@ fn test_include_zero() { assert_eq!(false, args.include_zero); } -/// -X/--index-#[derive(Debug)]-struct Index;--impl Flag for Index {- fn is_switch(&self) -> bool {- true- }- fn name_short(&self) -> Option<u8> {- Some(b'X')- }- fn name_long(&self) -> &'static str {- "index"- }- fn doc_category(&self) -> Category {- Category::Indexing- }- fn doc_short(&self) -> &'static str {- r"Use a search index when one is available."- }- fn doc_long(&self) -> &'static str {- r"-Enable searching with an index. Without this flag, ripgrep never uses an-index.-.sp-ripgrep looks for an index in the following order. The first step that finds-one or more indexes wins:-.sp-.IP 1. 4-When path operands are given on the command line, each operand is interpreted-as an index directory and every index is searched in command line order. An-operand that is not a valid index is an error.-.sp-.IP 2. 4-The index named by the \fBRIPGREP_INDEX_PATH\fP environment variable.-.sp-.IP 3. 4-A valid index in a \fB.ripgrep\fP directory in the current working directory.-.sp-.IP 4. 4-A valid index in a \fB.ripgrep\fP directory in the nearest parent of the current-working directory.-.PP-If no index is found, ripgrep performs an ordinary search.-.sp-Indexed candidates are filtered by explicit glob and file-type selections,-hidden-file and depth settings, and the maximum file size. Ignore files,-including \fB.gitignore\fP, are not read or reapplied during an indexed search.-Options that require transformed contents or every file make the query-ineligible for candidate filtering and therefore trigger the fallback below.-.sp-This flag may be given at most twice. When it is given once and the query-cannot use an index, ripgrep performs an ordinary search. When it is given-twice and an index was found, ripgrep stops instead of performing an ordinary-search if the query cannot use the index.-"- }-- fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> {- check_indexing_allowed()?;- assert!(v.unwrap_switch(), "--index has no negation");- args.index = args.index.saturating_add(1);- anyhow::ensure!(- args.index <= 2,- "-X/--index may be given at most twice"- );- Ok(())- }-}--#[cfg(test)]-#[test]-fn test_index() {- if !cfg!(feature = "unstable-index") {- assert!(parse_low_raw(["-X"]).is_err());- return;- }-- let args = parse_low_raw(None::<&str>).unwrap();- assert_eq!(0, args.index);- assert_eq!(Mode::Search(SearchMode::Standard), args.mode);-- let args = parse_low_raw(["-X"]).unwrap();- assert_eq!(1, args.index);- assert_eq!(Mode::Search(SearchMode::Standard), args.mode);-- let args = parse_low_raw(["--index"]).unwrap();- assert_eq!(1, args.index);-- let args = parse_low_raw(["-XX"]).unwrap();- assert_eq!(2, args.index);-- let args = parse_low_raw(["-X", "--index"]).unwrap();- assert_eq!(2, args.index);-- let result = parse_low_raw(["-XXX"]);- assert!(result.is_err(), "{result:?}");-}--/// --x-crud-#[derive(Debug)]-struct IndexCrud;--impl Flag for IndexCrud {- fn is_switch(&self) -> bool {- true- }- fn name_long(&self) -> &'static str {- "x-crud"- }- fn doc_category(&self) -> Category {- Category::Indexing- }- fn doc_short(&self) -> &'static str {- r"Create or update a search index."- }- fn doc_long(&self) -> &'static str {- r"-Create or update an index for the files and directories given on the command-line. When no path is given, this recursively indexes the current working-directory. The files added to the index are exactly those selected by-ripgrep's normal traversal and filtering options.-.sp-An existing file is re-indexed only when its modification time is newer than-the time at which it was indexed. Use \flag{x-force} to re-index every selected-file, including files on file systems with unreliable or deliberately-preserved modification times. A previously indexed path that is no longer-accessible is removed from the index.-.sp-ripgrep chooses the index location in the following order:-.sp-.IP 1. 4-The path given by \flag{x-path}.-.sp-.IP 2. 4-The path in the \fBRIPGREP_INDEX_PATH\fP environment variable.-.sp-.IP 3. 4-A valid index in a \fB.ripgrep\fP directory in the current working directory.-.sp-.IP 4. 4-A valid index in a \fB.ripgrep\fP directory in the nearest parent of the current-working directory.-.sp-.IP 5. 4-A \fB.ripgrep\fP directory in the current working directory, which is created-when necessary.-.PP-If the final location already exists but does not contain a valid index, then-ripgrep reports an error.-.sp-For example, this creates or incrementally updates an index for the current-directory:-.sp-.EX- rg --x-crud-.EE-.sp-This updates one path in the same index:-.sp-.EX- rg --x-crud path/to/file-.EE-"- }-- fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> {- check_indexing_allowed()?;- assert!(v.unwrap_switch(), "--x-crud has no negation");- args.mode.update(Mode::Index(IndexMode::Crud));- Ok(())- }-}--#[cfg(test)]-#[test]-fn test_index_crud() {- if !cfg!(feature = "unstable-index") {- assert!(parse_low_raw(["--x-crud"]).is_err());- return;- }-- let args = parse_low_raw(None::<&str>).unwrap();- assert_eq!(Mode::Search(SearchMode::Standard), args.mode);-- let args = parse_low_raw(["--x-crud"]).unwrap();- assert_eq!(Mode::Index(IndexMode::Crud), args.mode);-- let args = parse_low_raw(["--files", "--x-crud", "foo"]).unwrap();- assert_eq!(Mode::Index(IndexMode::Crud), args.mode);- assert_eq!(vec![std::ffi::OsString::from("foo")], args.positional);-- let args = parse_low_raw(["--x-crud", "--files"]).unwrap();- assert_eq!(Mode::Files, args.mode);-}--/// --x-force-#[derive(Debug)]-struct IndexForce;--impl Flag for IndexForce {- fn is_switch(&self) -> bool {- true- }- fn name_long(&self) -> &'static str {- "x-force"- }- fn doc_category(&self) -> Category {- Category::Indexing- }- fn doc_short(&self) -> &'static str {- r"Force selected files to be re-indexed."- }- fn doc_long(&self) -> &'static str {- r"-When used with \flag{x-crud}, re-index every selected file even when its-modification time indicates that the index is already up to date.-"- }-- fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> {- check_indexing_allowed()?;- assert!(v.unwrap_switch(), "--x-force has no negation");- args.index_force = true;- Ok(())- }-}--#[cfg(test)]-#[test]-fn test_index_force() {- if !cfg!(feature = "unstable-index") {- assert!(parse_low_raw(["--x-force"]).is_err());- return;- }-- let args = parse_low_raw(None::<&str>).unwrap();- assert_eq!(false, args.index_force);-- let args = parse_low_raw(["--x-force"]).unwrap();- assert_eq!(true, args.index_force);-}--/// --x-path-#[derive(Debug)]-struct IndexPath;--impl Flag for IndexPath {- fn is_switch(&self) -> bool {- false- }- fn name_long(&self) -> &'static str {- "x-path"- }- fn doc_variable(&self) -> Option<&'static str> {- Some("PATH")- }- fn doc_category(&self) -> Category {- Category::Indexing- }- fn doc_short(&self) -> &'static str {- r"Set the path of the index to update."- }- fn doc_long(&self) -> &'static str {- r"-Set the index path used by \flag{x-crud}. This takes precedence over-\fBRIPGREP_INDEX_PATH\fP and automatic discovery of a \fB.ripgrep\fP directory.-"- }- fn completion_type(&self) -> CompletionType {- CompletionType::Filename- }-- fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> {- check_indexing_allowed()?;- args.index_path = Some(PathBuf::from(v.unwrap_value()));- Ok(())- }-}--#[cfg(test)]-#[test]-fn test_index_path() {- if !cfg!(feature = "unstable-index") {- assert!(parse_low_raw(["--x-path"]).is_err());- return;- }-- let args = parse_low_raw(None::<&str>).unwrap();- assert_eq!(None, args.index_path);-- let args = parse_low_raw(["--x-path", "foo"]).unwrap();- assert_eq!(Some(PathBuf::from("foo")), args.index_path);-- let args = parse_low_raw(["--x-path=foo", "--x-path", "bar"]).unwrap();- assert_eq!(Some(PathBuf::from("bar")), args.index_path);-}- /// -v/--invert-match #[derive(Debug)] struct InvertMatch;@@ -7947,13 +7572,6 @@ fn test_word_regexp() { assert_eq!(Some(BoundaryMode::Line), args.boundary); } -fn check_indexing_allowed() -> anyhow::Result<()> {- if cfg!(feature = "unstable-index") {- return Ok(());- }- anyhow::bail!("{}", crate::flags::INDEXING_NOT_SUPPORTED)-}- mod convert { use std::ffi::{OsStr, OsString};
@@ -38,17 +38,8 @@ pub(crate) fn generate_short() -> String { TEMPLATE_SHORT.replace("!!VERSION!!", &version::generate_digits()); for (cat, (col1, col2)) in cats.iter() { let var = format!("!!{name}!!", name = cat.as_str());- if !cfg!(feature = "unstable-index")- && matches!(cat, crate::flags::Category::Indexing)- {- out = out.replace(- &var,- &format!(" {}", crate::flags::INDEXING_NOT_SUPPORTED),- );- } else {- let val = format_short_columns(col1, col2, maxcol1, maxcol2);- out = out.replace(&var, &val);- }+ let val = format_short_columns(col1, col2, maxcol1, maxcol2);+ out = out.replace(&var, &val); } out }@@ -131,16 +122,7 @@ pub(crate) fn generate_long() -> String { TEMPLATE_LONG.replace("!!VERSION!!", &version::generate_digits()); for (cat, value) in cats.iter() { let var = format!("!!{name}!!", name = cat.as_str());- if !cfg!(feature = "unstable-index")- && matches!(cat, crate::flags::Category::Indexing)- {- out = out.replace(- &var,- &format!(" {}", crate::flags::INDEXING_NOT_SUPPORTED),- );- } else {- out = out.replace(&var, value);- }+ out = out.replace(&var, value); } out }
@@ -37,13 +37,7 @@ pub(crate) fn generate() -> String { let mut out = TEMPLATE.replace("!!VERSION!!", &version::generate_digits()); for (cat, value) in cats.iter() { let var = format!("!!{name}!!", name = cat.as_str());- if !cfg!(feature = "unstable-index")- && matches!(cat, crate::flags::Category::Indexing)- {- out = out.replace(&var, crate::flags::INDEXING_NOT_SUPPORTED);- } else {- out = out.replace(&var, value);- }+ out = out.replace(&var, value); } out }
crates/core/flags/doc/template.long.help
@@ -54,9 +54,6 @@ OUTPUT OPTIONS: OUTPUT MODES: !!output-modes!! -INDEXING:-!!indexing!!- LOGGING OPTIONS: !!logging!!
crates/core/flags/doc/template.rg.1
@@ -114,9 +114,6 @@ In all cases, the flag specified last takes precedence. .SS OUTPUT MODES !!output-modes!! .-.SS INDEXING-!!indexing!!-. .SS LOGGING OPTIONS !!logging!! .
crates/core/flags/doc/template.short.help
@@ -31,9 +31,6 @@ OUTPUT OPTIONS: OUTPUT MODES: !!output-modes!! -INDEXING:-!!indexing!!- LOGGING OPTIONS: !!logging!!
@@ -61,7 +61,6 @@ pub(crate) struct HiArgs { ignore_file_case_insensitive: bool, ignore_file: Vec<PathBuf>, include_zero: bool,- index: usize, invert_match: bool, is_terminal_stdout: bool, line_number: bool,@@ -122,15 +121,6 @@ impl HiArgs { if let Some(ref sort) = low.sort { sort.supported()?; }- // We aggressively ban things from indexing at present.- if (low.index > 0 || matches!(low.mode, Mode::Index(_)))- && let Some(flag) = low.indexing_unsupported_flag()- {- anyhow::bail!(- "flag --{} does not support indexing",- flag.name_long()- );- } // We modify the mode in-place on `low` so that subsequent conversions // see the correct mode.@@ -288,7 +278,6 @@ impl HiArgs { ignore_file: low.ignore_file, ignore_file_case_insensitive: low.ignore_file_case_insensitive, include_zero: low.include_zero,- index: low.index, invert_match: low.invert_match, is_terminal_stdout: state.is_terminal_stdout, line_number,@@ -927,31 +916,6 @@ impl HiArgs { } Ok(builder) }-- pub(crate) fn index(&self) -> usize {- self.index- }-- #[cfg(feature = "unstable-index")]- pub(crate) fn index_write(&self) -> anyhow::Result<grep_index::Index> {- let index = grep_index::IndexDiscovery::new()- .create(true)- .discover()?- .ok_or_else(|| {- anyhow::anyhow!(- "failed to find a place to create/update an index"- )- })?;- Ok(index)- }-- #[cfg(feature = "unstable-index")]- pub(crate) fn index_read(&self) -> anyhow::Result<grep_index::Index> {- let index = grep_index::IndexDiscovery::new().discover()?.ok_or_else(- || anyhow::anyhow!("failed to find an index to read"),- )?;- Ok(index)- } } /// State that only needs to be computed once during argument parsing.
@@ -65,9 +65,6 @@ pub(crate) struct LowArgs { pub(crate) ignore_file: Vec<PathBuf>, pub(crate) ignore_file_case_insensitive: bool, pub(crate) include_zero: bool,- pub(crate) index: usize,- pub(crate) index_force: bool,- pub(crate) index_path: Option<PathBuf>, pub(crate) invert_match: bool, pub(crate) line_number: Option<bool>, pub(crate) logging: Option<LoggingMode>,@@ -156,8 +153,6 @@ pub(crate) enum SpecialMode { pub(crate) enum Mode { /// ripgrep will execute a search of some kind. Search(SearchMode),- /// Create or update a search index.- Index(IndexMode), /// Show the files that *would* be searched, but don't actually search /// them. Files,@@ -194,13 +189,6 @@ impl Mode { } } -/// The kind of index operation that ripgrep should perform.-#[derive(Clone, Copy, Debug, Eq, PartialEq)]-pub(crate) enum IndexMode {- /// Add, update or remove files in an index.- Crud,-}- /// The kind of search that ripgrep is going to perform. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum SearchMode {
@@ -47,9 +47,6 @@ mod hiargs; mod lowargs; mod parse; -const INDEXING_NOT_SUPPORTED: &'static str =- "Indexing is not supported in this build of ripgrep.";- /// A trait that encapsulates the definition of an optional flag for ripgrep. /// /// This trait is meant to be used via dynamic dispatch. Namely, the `defs`@@ -208,9 +205,6 @@ enum Category { /// lines, and instead just print the total count of matches for each file /// searched. OutputModes,- /// Flags related to indexing. This section includes flags both for- /// searching an index and updating an index.- Indexing, /// Flags related to logging behavior such as emitting non-fatal error /// messages or printing search statistics. Logging,@@ -232,7 +226,6 @@ impl Category { Category::Filter => "filter", Category::Output => "output", Category::OutputModes => "output-modes",- Category::Indexing => "indexing", Category::Logging => "logging", Category::OtherBehaviors => "other-behaviors", }