@ 260915.5

240731.1 · b8b0e37d1da2f7a29dda00678af3b02c6af424a6

released 3.6

fix #410 fix #411 fix #412

reading declarations

Makefile.am

@@ -13,17 +13,17 @@ CONFIGH = $(shell pwd)/$(top_builddir)/config.h  if ENABLE_BASH_COMPLETION bashcompletiondir = $(BASH_COMPLETION_DIR)-dist_bashcompletion_DATA = $(top_builddir)/completions/bash/ug+dist_bashcompletion_DATA = $(top_builddir)/completions/bash/ug $(top_builddir)/completions/bash/ugrep-indexer endif  if ENABLE_FISH_COMPLETION fishcompletiondir = $(FISH_COMPLETION_DIR)-dist_fishcompletion_DATA = $(top_builddir)/completions/fish/ug.fish+dist_fishcompletion_DATA = $(top_builddir)/completions/fish/ug.fish $(top_builddir)/completions/fish/ugrep-indexer.fish endif  if ENABLE_ZSH_COMPLETION zshcompletiondir = $(ZSH_COMPLETION_DIR)-dist_zshcompletion_DATA = $(top_builddir)/completions/zsh/_ug+dist_zshcompletion_DATA = $(top_builddir)/completions/zsh/_ug $(top_builddir)/completions/zsh/_ugrep-indexer endif  all-local:	cp2bin

Makefile.in

@@ -176,11 +176,14 @@ am__installdirs = "$(DESTDIR)$(man1dir)" \ NROFF = nroff MANS = $(man1_MANS) am__dist_bashcompletion_DATA_DIST =  \-	$(top_builddir)/completions/bash/ug+	$(top_builddir)/completions/bash/ug \+	$(top_builddir)/completions/bash/ugrep-indexer am__dist_fishcompletion_DATA_DIST =  \-	$(top_builddir)/completions/fish/ug.fish+	$(top_builddir)/completions/fish/ug.fish \+	$(top_builddir)/completions/fish/ugrep-indexer.fish am__dist_zshcompletion_DATA_DIST =  \-	$(top_builddir)/completions/zsh/_ug+	$(top_builddir)/completions/zsh/_ug \+	$(top_builddir)/completions/zsh/_ugrep-indexer DATA = $(dist_bashcompletion_DATA) $(dist_fishcompletion_DATA) \ 	$(dist_zshcompletion_DATA) RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive	\@@ -391,11 +394,11 @@ EXTRA_DIST = README.md LICENSE.txt CONTRIBUTING.md CODE_OF_CONDUCT.md UGREP = $(shell pwd)/$(top_builddir)/bin/ugrep$(EXEEXT) CONFIGH = $(shell pwd)/$(top_builddir)/config.h @ENABLE_BASH_COMPLETION_TRUE@bashcompletiondir = $(BASH_COMPLETION_DIR)-@ENABLE_BASH_COMPLETION_TRUE@dist_bashcompletion_DATA = $(top_builddir)/completions/bash/ug+@ENABLE_BASH_COMPLETION_TRUE@dist_bashcompletion_DATA = $(top_builddir)/completions/bash/ug $(top_builddir)/completions/bash/ugrep-indexer @ENABLE_FISH_COMPLETION_TRUE@fishcompletiondir = $(FISH_COMPLETION_DIR)-@ENABLE_FISH_COMPLETION_TRUE@dist_fishcompletion_DATA = $(top_builddir)/completions/fish/ug.fish+@ENABLE_FISH_COMPLETION_TRUE@dist_fishcompletion_DATA = $(top_builddir)/completions/fish/ug.fish $(top_builddir)/completions/fish/ugrep-indexer.fish @ENABLE_ZSH_COMPLETION_TRUE@zshcompletiondir = $(ZSH_COMPLETION_DIR)-@ENABLE_ZSH_COMPLETION_TRUE@dist_zshcompletion_DATA = $(top_builddir)/completions/zsh/_ug+@ENABLE_ZSH_COMPLETION_TRUE@dist_zshcompletion_DATA = $(top_builddir)/completions/zsh/_ug $(top_builddir)/completions/zsh/_ugrep-indexer all: config.h 	$(MAKE) $(AM_MAKEFLAGS) all-recursive 

README.md

@@ -3133,6 +3133,8 @@ To display the line and column numbers of matches in XML with `--xml`:             (invert), `u' (underline).  Parameter `hl' enables file name             hyperlinks.  Parameter `rv' reverses the `sl=' and `cx=' parameters             when option -v is specified.  Selectively overrides GREP_COLORS.+            Legacy grep single parameter codes may be specified, for example+            --colors='7;32' or --colors=ig to set ms (match selected).     --tag[=TAG[,END]]             Disables colors to mark up matches with TAG.  END marks the end of             a match if specified, otherwise TAG.  The default is `___'.@@ -4130,7 +4132,9 @@ in markdown:                   `i' (invert), `u' (underline).  Parameter `hl' enables file name                   hyperlinks.  Parameter `rv' reverses the `sl=' and `cx='                   parameters when option -v is specified.  Selectively overrides-                  GREP_COLORS.+                  GREP_COLORS.  Legacy grep single parameter codes may be specified,+                  for example --colors='7;32' or --colors=ig to set ms (match+                  selected).             --config[=FILE], ---[FILE]                   Use configuration FILE.  The default FILE is `.ugrep'.  The@@ -4435,17 +4439,17 @@ in markdown:                   priority over --include-fs=MOUNTS.  This option may be repeated.             --index-                  Perform index-based recursive search.  This option assumes, but-                  does not require, that files are indexed with ugrep-indexer.  This-                  option accelerates recursive searching by skipping non-matching-                  files, archives and compressed files when indexed.  Significant+                  Perform fast index-based recursive search.  This option assumes,+                  but does not require, that files are indexed with ugrep-indexer.+                  This option also enables option -r or --recursive.  Skips indexed+                  non-matching files, archives and compressed files.  Significant                   acceleration may be achieved on cold (not file-cached) and large                   file systems, or any file system that is slow to search.  Note-                  that the start-up time to search is increased, which may be-                  significant when complex search patterns are specified that-                  contain large Unicode character classes combined with `*' or `+'-                  repeats, which should be avoided.  Option -U (--ascii) improves-                  performance.  Option --stats displays an index search report.+                  that the start-up time to search may be increased when complex+                  search patterns are specified that contain large Unicode character+                  classes combined with `*' or `+' repeats, which should be avoided.+                  Option -U (--ascii) improves performance.  Option --stats displays+                  an index search report.             -J NUM, --jobs=NUM                   Specifies the number of threads spawned to search files.  By@@ -5392,7 +5396,7 @@ in markdown:   -    ugrep 6.2.0                       July 9, 2024                          UGREP(1)+    ugrep 6.3.0                       July 31, 2024                         UGREP(1)  � [Back to table of contents](#toc) 

bin/win32/ug.exe

Binary file changed

bin/win32/ugrep-indexer.exe

Binary file changed

bin/win32/ugrep.exe

Binary file changed

bin/win64/ug.exe

Binary file changed

bin/win64/ugrep-indexer.exe

Binary file changed

bin/win64/ugrep.exe

Binary file changed

completions/bash/ug

@@ -267,7 +267,7 @@ cat <<'END' --include-dir=GLOB Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/. --include-from=FILE Read the globs from FILE and search only files and directories whose name matches one or more globs. --include-fs=MOUNTS Only file systems specified by MOUNTS are included in recursive searches.---index Perform index-based recursive search.+--index Perform fast index-based recursive search. -J NUM, --jobs=NUM Specifies the number of threads spawned to search files. -j, --smart-case Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter. --json Output file matches in JSON.

completions/bash/ugrep-indexer

new file mode 100644@@ -0,0 +1,93 @@+# bash completion for ugrep-indexer+# recommended: add the following lines to ~/.inputrc+# set colored-stats on+# set colored-completion-prefix on++if ! declare -F _init_completion >/dev/null 2>&1; then+    return+fi++_comp_cmd_ugrep_indexer()+{+    local IFS=$' \t\n'+    local cur prev words cword+    _init_completion -s || return++    if [[ $cword -eq 1 && "${words[1]}" == "" ]]; then+        _comp_cmd_ugrep_indexer_usage+        return+    fi++    local i+    for i in "${!words[@]}"; do+        if [ "${words[$i]}" = "--" ]; then+            if [ $cword -gt $i ]; then+                _filedir+                return+            fi+            words=( "${words[@]:0:$i}" )+            break+        fi+    done++    case $cur in+    --*)+        # complete long options by generating them+        COMPREPLY=( $(compgen -W "$(_parse_help _comp_cmd_ugrep_indexer_help)" -- $cur) )+        if [[ ! "${COMPREPLY[@]}" =~ "=" ]]; then+            # add space after long options that do not end with a =+            compopt +o nospace+        fi+        return+        ;;+    -?*)+        # add space after short option(s)+        COMPREPLY=( $cur )+        compopt +o nospace+        return+        ;;+    -)+        _comp_cmd_ugrep_indexer_usage+        return+        ;;+    esac++    _filedir+} &&+    complete -o nospace -F _comp_cmd_ugrep_indexer ugrep-indexer++_comp_cmd_ugrep_indexer_usage()+{+    local -a usage+    local line i=1+    usage[0]="Usage:"+    # generate list of options, concat the first sentence to them+    while read -r line; do+        # truncate to screen width+        usage[$i]=${line:0:$COLUMNS}+        (( ++i ))+    done < <(_comp_cmd_ugrep_indexer_help)+    COMPREPLY=( "${usage[@]}" )+    compopt -o nosort+}++_comp_cmd_ugrep_indexer_help()+{+cat <<'END'+--accuracy=DIGIT, -0, -1, -2, -3, -4, -5, -6, -7, -8, -9 Specifies indexing accuracy.+-., --hidden Index hidden files and directories.+-?, --help Display a help message and exit.+-c, --check Recursively check and report indexes without reindexing files.+-d, --delete Recursively remove index files.+-f, --force Force reindexing of files.+-I, --ignore-binary Do not index binary files.+-q, --quiet, --silent Quiet mode: do not display indexing statistics.+-S, --dereference-files Follow symbolic links to files.+-s, --no-messages Silent mode: nonexistent and unreadable files are ignored, i.e.  their error messages and warnings are suppressed.+-V, --version Display version and exit.+-v, --verbose Produce verbose output.+-X, --ignore-files, --ignore-files=FILE Do not index files and directories matching the globs in FILE encountered during indexing.+-z, --decompress Index the contents of compressed files and archives.+--zmax=NUM When used with option -z (--decompress), indexes the contents of compressed files and archives stored within archives by up to NUM expansion levels deep.+END+}

completions/fish/ug+.fish

@@ -57,7 +57,7 @@ complete -c ug+ -l include -d 'Only search files whose name matches GLOB, same a complete -c ug+ -l include-dir -d 'Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/' complete -c ug+ -l include-from -d 'Read the globs from FILE and search only files and directories whose name matches one or more globs' complete -c ug+ -l include-fs -d 'Only file systems specified by MOUNTS are included in recursive searches'-complete -c ug+ -l index -d 'Perform index-based recursive search'+complete -c ug+ -l index -d 'Perform fast index-based recursive search' complete -c ug+ -s J -r -l jobs -d 'Specifies the number of threads spawned to search files' complete -c ug+ -s j -l smart-case -d 'Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter' complete -c ug+ -l json -d 'Output file matches in JSON'

completions/fish/ug.fish

@@ -57,7 +57,7 @@ complete -c ug -l include -d 'Only search files whose name matches GLOB, same as complete -c ug -l include-dir -d 'Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/' complete -c ug -l include-from -d 'Read the globs from FILE and search only files and directories whose name matches one or more globs' complete -c ug -l include-fs -d 'Only file systems specified by MOUNTS are included in recursive searches'-complete -c ug -l index -d 'Perform index-based recursive search'+complete -c ug -l index -d 'Perform fast index-based recursive search' complete -c ug -s J -r -l jobs -d 'Specifies the number of threads spawned to search files' complete -c ug -s j -l smart-case -d 'Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter' complete -c ug -l json -d 'Output file matches in JSON'

completions/fish/ugrep+.fish

@@ -57,7 +57,7 @@ complete -c ugrep+ -l include -d 'Only search files whose name matches GLOB, sam complete -c ugrep+ -l include-dir -d 'Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/' complete -c ugrep+ -l include-from -d 'Read the globs from FILE and search only files and directories whose name matches one or more globs' complete -c ugrep+ -l include-fs -d 'Only file systems specified by MOUNTS are included in recursive searches'-complete -c ugrep+ -l index -d 'Perform index-based recursive search'+complete -c ugrep+ -l index -d 'Perform fast index-based recursive search' complete -c ugrep+ -s J -r -l jobs -d 'Specifies the number of threads spawned to search files' complete -c ugrep+ -s j -l smart-case -d 'Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter' complete -c ugrep+ -l json -d 'Output file matches in JSON'

completions/fish/ugrep-indexer.fish

new file mode 100644@@ -0,0 +1,16 @@+# Copy this file to ~/.config/fish/completions+complete -c ugrep-indexer -l accuracy -s 0 -s 1 -s 2 -s 3 -s 4 -s 5 -s 6 -s 7 -s 8 -s 9 -d 'Specifies indexing accuracy'+complete -c ugrep-indexer -l hidden -s '.' -d 'Index hidden files and directories'+complete -c ugrep-indexer -l help -s '?' -d 'Display a help message and exit'+complete -c ugrep-indexer -s c -l check -d 'Recursively check and report indexes without reindexing files'+complete -c ugrep-indexer -s d -l delete -d 'Recursively remove index files'+complete -c ugrep-indexer -s f -l force -d 'Force reindexing of files, even those that are already indexed'+complete -c ugrep-indexer -s I -l ignore-binary -d 'Do not index binary files'+complete -c ugrep-indexer -s q -l quiet -l silent -d 'Quiet mode: do not display indexing statistics'+complete -c ugrep-indexer -s S -l dereference-files -d 'Follow symbolic links to files'+complete -c ugrep-indexer -s s -l no-messages -d 'Silent mode: nonexistent and unreadable files are ignored, i.e. their error messages and warnings are suppressed'+complete -c ugrep-indexer -s V -l version -d 'Display version and exit'+complete -c ugrep-indexer -s v -l verbose -d 'Produce verbose output'+complete -c ugrep-indexer -s X -l ignore-files -d 'Do not index files and directories matching the globs in FILE encountered during indexing'+complete -c ugrep-indexer -s z -l decompress -d 'Index the contents of compressed files and archives'+complete -c ugrep-indexer -l zmax -d 'When used with option -z (--decompress), indexes the contents of compressed files and archives stored within archives by up to NUM expansion levels deep'

completions/fish/ugrep.fish

@@ -57,7 +57,7 @@ complete -c ugrep -l include -d 'Only search files whose name matches GLOB, same complete -c ugrep -l include-dir -d 'Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/' complete -c ugrep -l include-from -d 'Read the globs from FILE and search only files and directories whose name matches one or more globs' complete -c ugrep -l include-fs -d 'Only file systems specified by MOUNTS are included in recursive searches'-complete -c ugrep -l index -d 'Perform index-based recursive search'+complete -c ugrep -l index -d 'Perform fast index-based recursive search' complete -c ugrep -s J -r -l jobs -d 'Specifies the number of threads spawned to search files' complete -c ugrep -s j -l smart-case -d 'Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter' complete -c ugrep -l json -d 'Output file matches in JSON'

completions/zsh/_ug

@@ -64,7 +64,7 @@ $matchers{-G,--basic-regexp}'[Interpret patterns as basic regular expressions (B '(1)*'--include-dir=-'[Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/]' '(1)*'--include-from=-'[Read the globs from FILE and search only files and directories whose name matches one or more globs]' '(1)*'--include-fs=-'[Only file systems specified by MOUNTS are included in recursive searches]'---index'[Perform index-based recursive search]'+--index'[Perform fast index-based recursive search]' {-J,--jobs=-}'[Specifies the number of threads spawned to search files]:NUM:( )' {-j,--smart-case}'[Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter]' --json'[Output file matches in JSON]'

completions/zsh/_ug+

@@ -64,7 +64,7 @@ $matchers{-G,--basic-regexp}'[Interpret patterns as basic regular expressions (B '(1)*'--include-dir=-'[Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/]' '(1)*'--include-from=-'[Read the globs from FILE and search only files and directories whose name matches one or more globs]' '(1)*'--include-fs=-'[Only file systems specified by MOUNTS are included in recursive searches]'---index'[Perform index-based recursive search]'+--index'[Perform fast index-based recursive search]' {-J,--jobs=-}'[Specifies the number of threads spawned to search files]:NUM:( )' {-j,--smart-case}'[Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter]' --json'[Output file matches in JSON]'

completions/zsh/_ugrep

@@ -64,7 +64,7 @@ $matchers{-G,--basic-regexp}'[Interpret patterns as basic regular expressions (B '(1)*'--include-dir=-'[Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/]' '(1)*'--include-from=-'[Read the globs from FILE and search only files and directories whose name matches one or more globs]' '(1)*'--include-fs=-'[Only file systems specified by MOUNTS are included in recursive searches]'---index'[Perform index-based recursive search]'+--index'[Perform fast index-based recursive search]' {-J,--jobs=-}'[Specifies the number of threads spawned to search files]:NUM:( )' {-j,--smart-case}'[Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter]' --json'[Output file matches in JSON]'

completions/zsh/_ugrep+

@@ -64,7 +64,7 @@ $matchers{-G,--basic-regexp}'[Interpret patterns as basic regular expressions (B '(1)*'--include-dir=-'[Only directories whose name matches GLOB are included in recursive searches, same as -g GLOB/]' '(1)*'--include-from=-'[Read the globs from FILE and search only files and directories whose name matches one or more globs]' '(1)*'--include-fs=-'[Only file systems specified by MOUNTS are included in recursive searches]'---index'[Perform index-based recursive search]'+--index'[Perform fast index-based recursive search]' {-J,--jobs=-}'[Specifies the number of threads spawned to search files]:NUM:( )' {-j,--smart-case}'[Perform case insensitive matching, unless a pattern is specified with a literal upper case ASCII letter]' --json'[Output file matches in JSON]'

completions/zsh/_ugrep-indexer

new file mode 100644@@ -0,0 +1,22 @@+#compdef ugrep-indexer++# zsh completions for 'ugrep-indexer'++local arguments=(+{--accuracy=-,-0,-1,-2,-3,-4,-5,-6,-7,-8,-9}'[Specifies indexing accuracy, the default is 4]'+{--hidden,-.}'[Index hidden files and directories]'+'(-)'{--help,-?}'[Display a help message and exit]'+{-c,--check}'[Recursively check and report indexes without reindexing files]'+{-d,--delete}'[Recursively remove index files]'+{-f,--force}'[Force reindexing of files]'+{-I,--ignore-binary}'[Do not index binary files]'+{-q,--quiet,--silent}'[Quiet mode: do not display indexing statistics]'+{-S,--dereference-files}'[Follow symbolic links to files]'+{-s,--no-messages}'[Silent mode: nonexistent and unreadable files are ignored, i.e. their error messages and warnings are suppressed]'+{-V,--version}'[Display version with linked libraries and exit]'+{-v,--verbose}'[Produce verbose output]'+'(1)*'{-X,--ignore-files}'[Do not index files and directories matching the globs in FILE encountered during indexing]'+{-z,--decompress}'[Index the contents of compressed files and archives]'+--zmax=-'[When used with option -z (--decompress), indexes the contents of compressed files and archives stored within archives by up to NUM expansion levels deep]'+)+_arguments -s -S $arguments

configure

@@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles.-# Generated by GNU Autoconf 2.72 for ugrep 6.2.0.+# Generated by GNU Autoconf 2.72 for ugrep 6.3.0. # # Report bugs to <https://github.com/Genivia/ugrep/issues>. #@@ -606,8 +606,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='ugrep' PACKAGE_TARNAME='ugrep'-PACKAGE_VERSION='6.2.0'-PACKAGE_STRING='ugrep 6.2.0'+PACKAGE_VERSION='6.3.0'+PACKAGE_STRING='ugrep 6.3.0' PACKAGE_BUGREPORT='https://github.com/Genivia/ugrep/issues' PACKAGE_URL='https://ugrep.com' @@ -1382,7 +1382,7 @@ if test "$ac_init_help" = "long"; then   # Omit some internal or obsolete options to make the list less imposing.   # This message is too long to be a string in the A/UX 3.1 sh.   cat <<_ACEOF-'configure' configures ugrep 6.2.0 to adapt to many kinds of systems.+'configure' configures ugrep 6.3.0 to adapt to many kinds of systems.  Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1453,7 +1453,7 @@ fi  if test -n "$ac_init_help"; then   case $ac_init_help in-     short | recursive ) echo "Configuration of ugrep 6.2.0:";;+     short | recursive ) echo "Configuration of ugrep 6.3.0:";;    esac   cat <<\_ACEOF @@ -1630,7 +1630,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then   cat <<\_ACEOF-ugrep configure 6.2.0+ugrep configure 6.3.0 generated by GNU Autoconf 2.72  Copyright (C) 2023 Free Software Foundation, Inc.@@ -2184,7 +2184,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ugrep $as_me 6.2.0, which was+It was created by ugrep $as_me 6.3.0, which was generated by GNU Autoconf 2.72.  Invocation command line was    $ $0$ac_configure_args_raw@@ -3694,7 +3694,7 @@ fi  # Define the identity of the package.  PACKAGE='ugrep'- VERSION='6.2.0'+ VERSION='6.3.0'   printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h@@ -11790,7 +11790,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log="-This file was extended by ugrep $as_me 6.2.0, which was+This file was extended by ugrep $as_me 6.3.0, which was generated by GNU Autoconf 2.72.  Invocation command line was    CONFIG_FILES    = $CONFIG_FILES@@ -11863,7 +11863,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\-ugrep config.status 6.2.0+ugrep config.status 6.3.0 configured by $0, generated by GNU Autoconf 2.72,   with options \\"\$ac_cs_config\\" 

configure.ac

@@ -1,4 +1,4 @@-AC_INIT([ugrep],[6.2.0],[https://github.com/Genivia/ugrep/issues],[ugrep],[https://ugrep.com])+AC_INIT([ugrep],[6.3.0],[https://github.com/Genivia/ugrep/issues],[ugrep],[https://ugrep.com]) AM_INIT_AUTOMAKE([foreign subdir-objects dist-xz no-dist-gzip]) AC_CONFIG_HEADERS([config.h]) AC_COPYRIGHT([Copyright (C) 2019-2024 Robert van Engelen, Genivia Inc.])

include/reflex/matcher.h

@@ -1489,7 +1489,7 @@ class Matcher : public PatternMatcher<reflex::Pattern> {   std::vector<int>  lap_; ///< lookahead position in input that heads a lookahead match (indexed by lookahead number)   std::stack<Stops> stk_; ///< stack to push/pop stops   FSM               fsm_; ///< local state for FSM code-  bool (Matcher::*  adv_)(size_t loc); ///< advance FIND+  bool (Matcher::*  adv_)(size_t loc); ///< advance FIND function pointer   bool              mrk_; ///< indent \i or dedent \j in pattern found: should check and update indent stops   bool              anc_; ///< match is anchored, advance slowly to retry when searching };

include/reflex/pattern.h

@@ -71,7 +71,7 @@ class Pattern {   friend class FuzzyMatcher; ///< permit access by the reflex::FuzzyMatcher engine  public:   typedef uint8_t  Pred;   ///< predict match bits-  typedef uint16_t Hash;   ///< hash value type, max value is Const::HASH+  typedef uint32_t Hash;   ///< hash value type, max value is Const::HASH   typedef uint32_t Index;  ///< index into opcodes array Pattern::opc_ and subpattern indexing   typedef uint32_t Accept; ///< group capture index   typedef uint32_t Opcode; ///< 32 bit opcode word@@ -358,7 +358,7 @@ class Pattern {   inline bool predict_match(const char *s, size_t n) const   {     Hash h = static_cast<uint8_t>(*s);-    Pred f = pmh_[h] & 1;+    Hash f = pmh_[h] & 1;     h = hash(h, static_cast<uint8_t>(*++s));     f |= pmh_[h] & 2;     h = hash(h, static_cast<uint8_t>(*++s));@@ -366,7 +366,7 @@ class Pattern {     h = hash(h, static_cast<uint8_t>(*++s));     f |= pmh_[h] & 8;     const char *e = s + n - 3;-    Pred m = 16;+    Hash m = 16;     while (f == 0 && ++s < e)     {       h = hash(h, static_cast<uint8_t>(*s));@@ -946,11 +946,14 @@ class Pattern {   void graph_dfa(const DFA::State *start) const;   void export_code() const;   void analyze_dfa(DFA::State *start);-  void gen_predict_match(std::set<DFA::State*> states);-  void gen_predict_match_start(std::set<DFA::State*> states, std::map<DFA::State*,ORanges<Hash> >& hashes);+  void gen_min(std::set<DFA::State*>& states);+  void gen_min_start(std::set<DFA::State*>& states, std::set<DFA::State*>& next);+  void gen_min_transitions(size_t level, std::set<DFA::State*>& next);+  void gen_predict_match(std::set<DFA::State*>& states);+  void gen_predict_match_start(std::set<DFA::State*>& states, std::map<DFA::State*,ORanges<Hash> >& hashes);   void gen_predict_match_transitions(size_t level, DFA::State *state, const ORanges<Hash>& labels, std::map<DFA::State*,ORanges<Hash> >& hashes);-  void gen_match_hfa(DFA::State* start);-  void gen_match_hfa_start(DFA::State* start, HFA::State& index, HFA::StateHashes& hashes);+  void gen_match_hfa(DFA::State *start);+  void gen_match_hfa_start(DFA::State *start, HFA::State& index, HFA::StateHashes& hashes);   bool gen_match_hfa_transitions(size_t level, size_t& max_level, DFA::State *state, const HFA::HashRanges& previous, HFA::State& index, HFA::StateHashes& hashes);  public:   bool has_hfa() const@@ -961,8 +964,8 @@ class Pattern {  private:   bool match_hfa_transitions(size_t level, const HFA::Hashes& hashes, const uint8_t *indexed, size_t size, HFA::VisitSet& visit, HFA::VisitSet& next_visit, bool& accept) const;   void write_predictor(FILE *fd) const;-  void write_namespace_open(FILE* fd) const;-  void write_namespace_close(FILE* fd) const;+  void write_namespace_open(FILE *fd) const;+  void write_namespace_close(FILE *fd) const;   size_t find_at(       Location loc,       char     c) const@@ -1145,7 +1148,7 @@ class Pattern {   /// file indexing hash 0 <= indexhash() < 65536, must be additive: indexhash(x,b+1) = indexhash(x,b)+1 modulo 2^16.   static inline Hash indexhash(Hash h, uint8_t b)   {-    return (h << 6) - h - h - h + b;+    return static_cast<uint16_t>((h << 6) - h - h - h + b);   }   Option                opt_; ///< pattern compiler options   HFA                   hfa_; ///< indexing hash finite state automaton

lib/pattern.cpp

@@ -319,7 +319,7 @@ void Pattern::init(const char *options, const uint8_t *pred)     }     // needle count and frequency thresholds to enable needle-based search     uint16_t pinmax = 8;-    uint8_t freqmax = 251;+    uint16_t freqmax = 251; #if defined(HAVE_AVX512BW) || defined(HAVE_AVX2) || defined(HAVE_SSE2)     if (have_HW_AVX512BW() || have_HW_AVX2())       pinmax = 16;@@ -338,16 +338,19 @@ void Pattern::init(const char *options, const uint8_t *pred)     lcs_ = 0;     uint16_t nlcp = 65535; // max and undefined     uint16_t nlcs = 65535; // max and undefined-    uint8_t freqlcp = 255; // max-    uint8_t freqlcs = 255; // max+    uint8_t freqlcp = 255; // max and undefined+    uint8_t freqlcs = 255; // max and undefined     size_t min = (min_ == 0 ? 1 : min_);-    for (uint16_t k = 0; k < min; ++k)+    uint8_t score[8][3];  // avg freq, unique position k < min, number of pins n <= pinmax+    size_t scores = 0;+    for (uint8_t k = 0; k < min; ++k)     {       Pred mask = 1 << k;-      uint16_t n = 0;-      uint8_t max = 0;-      // at position k count the matching characters and find the max character frequency-      for (uint16_t i = 0; i < 256; ++i)+      uint8_t n = 0;+      uint16_t max = 0;+      uint16_t sum = 0;+      // at position k count the matching characters and find the max and sum character frequency+      for (uint16_t i = 0; i < 256 && n <= pinmax; ++i)       {         if ((bit_[i] & mask) == 0)         {@@ -355,40 +358,63 @@ void Pattern::init(const char *options, const uint8_t *pred)           uint8_t freq = frequency(static_cast<uint8_t>(i));           if (freq > max)             max = freq;+          sum += freq;         }       }-      if (n <= pinmax)+      if (n > 0 && n <= pinmax && max <= freqmax)       {-        // pick the fewest and rarest (less frequently occurring) needles to search-        if (max < freqlcp || (n < nlcp && max == freqlcp))-        {-          lcs_ = lcp_;-          nlcs = nlcp;-          freqlcs = freqlcp;-          lcp_ = static_cast<uint8_t>(k);-          nlcp = n;-          freqlcp = max;-        }-        else if (n < nlcs ||-            (n == nlcs &&-             (max < freqlcs ||-              abs(static_cast<int>(lcp_) - static_cast<int>(lcs_)) < abs(static_cast<int>(lcp_) - static_cast<int>(k)))))+        // score average frequency, twice penalize pins > 8+        uint8_t m = static_cast<uint8_t>(std::min((sum + n - 1) / n * ((n > 8) + 1), 255));+        size_t i;+        for (i = 0; i < scores; ++i)         {-          lcs_ = static_cast<uint8_t>(k);-          nlcs = n;-          freqlcs = max;+          // keep scores sorted by average (mean) frequency+          if (score[i][0] > m || (score[i][0] == m && score[i][2] > n))+          {+            memmove(score[i+1], score[i], static_cast<uint8_t*>(score[scores]) - static_cast<uint8_t*>(score[i]));+            break;+          }         }+        score[i][0] = m;+        score[i][1] = k;+        score[i][2] = n;+        ++scores;       }     }-    // one position to pin: make lcp and lcs equal to 0 (only one position at 0)-    if (min == 1 || nlcs == 65535)+    if (scores == 1)     {-      nlcs = nlcp;-      lcs_ = lcp_;+      freqlcp = freqlcs = score[0][0];+      lcp_ = lcs_ = score[0][1];+      nlcp = nlcs = score[0][2];+    }+    else if (scores >= 2)+    {+      freqlcp = score[0][0];+      lcp_ = score[0][1];+      nlcp = score[0][2];+      freqlcs = score[1][0];+      lcs_ = score[1][1];+      nlcs = score[1][2];+      if (scores > 2)+      {+        if (lcp_ + 1 == lcs_ || lcs_ + 1 == lcp_ || nlcs > 8)+        {+          for (size_t i = 2; i < scores; ++i)+          {+            if (score[i][2] <= 8 && abs(lcp_ - lcs_) < abs(lcp_ - score[i][1]))+            {+              freqlcs = score[i][0];+              lcs_ = score[i][1];+              nlcs = score[i][2];+              break;+            }+          }+        }+      }     }     // number of needles required-    uint16_t n = nlcp > nlcs ? nlcp : nlcs;-    DBGLOG("min=%zu lcp=%hu(%hu) pin=%hu nlcp=%hu(%hu) freq=%hu(%hu) npy=%zu", min, lcp_, lcs_, n, nlcp, nlcs, freqlcp, freqlcs, npy_);+    uint16_t n = std::max(nlcp, nlcs);+    DBGLOG("min=%zu lcp=%hu(%hu) pin=%hu nlcp=%hu(%hu) freq=%hu(%hu) npy=%zu cut=%u", min, lcp_, lcs_, n, nlcp, nlcs, freqlcp, freqlcs, npy_, cut_);     // determine if a needle-based search is worthwhile, below or meeting the thresholds     if (n <= pinmax && freqlcp <= freqmax)     {@@ -405,7 +431,7 @@ void Pattern::init(const char *options, const uint8_t *pred)         if ((bit_[i] & masklcs) == 0)           chr_[k++] = static_cast<uint8_t>(i);       }-      // fill up the rest of the character tables with duplicates if necessary+      // fill up the rest of the character tables with duplicates as necessary       for (; j < n; ++j)         chr_[j] = chr_[j - 1];       for (; k < 2*n; ++k)@@ -3820,16 +3846,9 @@ void Pattern::analyze_dfa(DFA::State *start)         // make a cut?         bool make_cut = false;         if (has_backedge)-        {           make_cut = (max_count > 4 || max_freq > 251);-        }-        else if (cut_span >= 3 && prev_min_count < 0xffff)-        {-          make_cut = (prev_min_count > 2 * min_count ||-              (prev_min_count > 16 && min_count <= 16) ||-              (prev_min_count >  8 && min_count <=  8) ||-              (prev_min_count >  1 && min_count <=  1));-        }+        else+          make_cut = (cut_span >= 3 && prev_min_count < 0xffff && prev_min_count > 8 && prev_min_count >= 2 * min_count);         if (make_cut)         {           // determine if this is a better cut than the last@@ -3915,8 +3934,7 @@ void Pattern::analyze_dfa(DFA::State *start)         if (cut_span > best_cut_span)           better = true;         else if (cut_span >= 2 || cut_span == best_cut_span)-          better = ((best_min_count > 16 && min_count <= 16) ||-                    (best_min_count >  8 && min_count <=  8));+          better = (best_min_count > min_count);       }       if (better)       {@@ -4201,23 +4219,23 @@ void Pattern::analyze_dfa(DFA::State *start)   DBGLOG("END Pattern::analyze_dfa()"); } -void Pattern::gen_predict_match(std::set<DFA::State*> states)+void Pattern::gen_min(std::set<DFA::State*>& states) {+  // find min between 0 and 8   min_ = 8;-  std::map<DFA::State*,ORanges<Hash> > hashes[8];-  gen_predict_match_start(states, hashes[0]);-  for (int level = 1; level < 8; ++level)-    for (std::map<DFA::State*,ORanges<Hash> >::iterator from = hashes[level - 1].begin(); from != hashes[level - 1].end(); ++from)-      gen_predict_match_transitions(level, from->first, from->second, hashes[level]);+  std::set<DFA::State*> next;+  gen_min_start(states, next);+  for (size_t level = 1; level < min_; ++level)+    gen_min_transitions(level, next); } -void Pattern::gen_predict_match_start(std::set<DFA::State*> states, std::map<DFA::State*,ORanges<Hash> >& hashes)+void Pattern::gen_min_start(std::set<DFA::State*>& states, std::set<DFA::State*>& next) {   bool empty = true;   for (std::set<DFA::State*>::iterator it = states.begin(); it != states.end(); ++it)   {     DFA::State *state = *it;-    if (!state->edges.empty())+    if (empty && !state->edges.empty())       empty = false;     for (DFA::State::Edges::iterator edge = state->edges.begin(); edge != state->edges.end(); ++edge)     {@@ -4225,25 +4243,91 @@ void Pattern::gen_predict_match_start(std::set<DFA::State*> states, std::map<DFA       if (is_meta(lo))       {         min_ = 0;-        break;+        return;       }       DFA::State *next_state = edge->second.second;       // ignore edges from a state to a state with breadth-first depth <= cut       if (next_state != NULL && lbk_ > 0)         if (next_state->first > 0 && next_state->first <= cut_)           continue;-      if (next_state != NULL && (next_state->edges.empty() || is_meta(next_state->edges.begin()->first)))+      // don't visit next state if it is accepting or when any edges from it are meta+      if (next_state != NULL && (next_state->accept > 0 || next_state->edges.empty() || is_meta(next_state->edges.rbegin()->first)))         next_state = NULL;+      if (next_state != NULL)+        next.insert(next_state);+      else+        min_ = 1;+    }+  }+  if (empty)+    min_ = 0;+}++void Pattern::gen_min_transitions(size_t level, std::set<DFA::State*>& next)+{+  std::set<DFA::State*> prev;+  prev.swap(next);+  for (std::set<DFA::State*>::iterator from = prev.begin(); from != prev.end(); ++from)+  {+    DFA::State *state = *from;+    for (DFA::State::Edges::iterator edge = state->edges.begin(); edge != state->edges.end(); ++edge)+    {+      Char lo = edge->first;+      if (is_meta(lo))+      {+        min_ = level;+        return;+      }+      DFA::State *next_state = level < 7 ? edge->second.second : NULL;+      // ignore edges from a state to a state with breadth-first depth <= cut+      if (next_state != NULL && lbk_ > 0)+        if (next_state->first > 0 && next_state->first <= cut_)+          continue;+      // don't visit next state if it is accepting or when any edges from it are meta+      if (next_state != NULL && (next_state->accept > 0 || next_state->edges.empty() || is_meta(next_state->edges.rbegin()->first)))+        next_state = NULL;+      if (next_state != NULL)+        next.insert(next_state);+      else if (min_ > level)+        min_ = level + 1;+    }+  }+}++void Pattern::gen_predict_match(std::set<DFA::State*>& states)+{+  // find min between 0 and 8 then populate bitap and PM hashes (bounded by min)+  gen_min(states);+  std::map<DFA::State*,ORanges<Hash> > hashes[8];+  gen_predict_match_start(states, hashes[0]);+  for (size_t level = 1; level < std::max<size_t>(min_, 4); ++level)+    for (std::map<DFA::State*,ORanges<Hash> >::iterator from = hashes[level - 1].begin(); from != hashes[level - 1].end(); ++from)+      gen_predict_match_transitions(level, from->first, from->second, hashes[level]);+}++void Pattern::gen_predict_match_start(std::set<DFA::State*>& states, std::map<DFA::State*,ORanges<Hash> >& hashes)+{+  for (std::set<DFA::State*>::iterator it = states.begin(); it != states.end(); ++it)+  {+    DFA::State *state = *it; // state at level 0+    for (DFA::State::Edges::iterator edge = state->edges.begin(); edge != state->edges.end(); ++edge)+    {+      Char lo = edge->first;+      if (is_meta(lo))+        break;+      // next level state to visit+      DFA::State *next_state = edge->second.second;+      // ignore states before the cut, since we don't use them for bitap and PM hashing+      if (next_state != NULL && lbk_ > 0)+        if (next_state->first > 0 && next_state->first <= cut_)+          continue;       bool accept = next_state == NULL || next_state->accept > 0 || is_meta(next_state->edges.rbegin()->first);       Char hi = edge->second.first;       if (next_state != NULL)         hashes[next_state].insert(lo, hi);       uint8_t mask = ~(1 << 6);       if (accept)-      {         mask &= ~(1 << 7);-        min_ = 1;-      }       for (Char ch = lo; ch <= hi; ++ch)       {         bit_[ch] &= ~1;@@ -4253,43 +4337,39 @@ void Pattern::gen_predict_match_start(std::set<DFA::State*> states, std::map<DFA       DBGLOG("0 bitap %u..%u -> %p", lo, hi, next_state);     }   }-  if (empty)-    min_ = 0; } -void Pattern::gen_predict_match_transitions(size_t level, DFA::State *state, const ORanges<Hash>& previous, std::map<DFA::State*,ORanges<Hash> >& hashes)+void Pattern::gen_predict_match_transitions(size_t level, DFA::State *state, const ORanges<Hash>& previous, std::map<DFA::State*,ORanges<Hash> >& level_hashes) {   for (DFA::State::Edges::iterator edge = state->edges.begin(); edge != state->edges.end(); ++edge)   {     Char lo = edge->first;     if (is_meta(lo))-    {-      if (min_ > level)-        min_ = level;       break;-    }-    DFA::State *next_state = level < 7 ? edge->second.second : NULL;-    // ignore edges from a state to a state with breadth-first depth <= cut+    // next level state to visit+    DFA::State *next_state = edge->second.second;+    // ignore states before the cut, since we don't use them for bitap and PM hashing     if (next_state != NULL && lbk_ > 0)       if (next_state->first > 0 && next_state->first <= cut_)         continue;-    if (next_state != NULL && (next_state->edges.empty() || is_meta(next_state->edges.begin()->first)))-      next_state = NULL;     bool accept = next_state == NULL || next_state->accept > 0 || is_meta(next_state->edges.rbegin()->first);-    ORanges<Hash> *next_hashes = next_state != NULL ? &hashes[next_state] : NULL;-    if (accept && min_ > level)-      min_ = level + 1;+    ORanges<Hash> *next_hashes = next_state != NULL && level + 1 < std::max<size_t>(min_, 4) ? &level_hashes[next_state] : NULL;+    Char hi = edge->second.first;+    if (level < min_)+    {+      // populate bitap+      uint8_t mask = ~(1 << level);+      for (Char ch = lo; ch <= hi; ++ch)+        bit_[ch] &= mask;+      DBGLOG("%zu bitap %p: %u..%u -> %p", level, state, lo, hi, next_state);+    }     if (level < 4)     {+      // populate PM4       uint8_t pmh_mask = ~(1 << level);       uint8_t pma_mask = ~(1 << (6 - 2 * level));       if (level == 3 || accept)         pma_mask &= ~(1 << (7 - 2 * level));-      Char hi = edge->second.first;-      if (level <= min_)-        for (Char ch = lo; ch <= hi; ++ch)-          bit_[ch] &= pmh_mask;-      DBGLOG("%zu bitap %p: %u..%u -> %p", level, state, lo, hi, next_state);       if (next_hashes != NULL)       {         for (ORanges<Hash>::iterator prev_range = previous.begin(); prev_range != previous.end(); ++prev_range)@@ -4326,13 +4406,9 @@ void Pattern::gen_predict_match_transitions(size_t level, DFA::State *state, con         }       }     }-    else if (level <= min_)+    else if (level < min_)     {       uint8_t pmh_mask = ~(1 << level);-      Char hi = edge->second.first;-      for (Char ch = lo; ch <= hi; ++ch)-        bit_[ch] &= pmh_mask;-      DBGLOG("%zu bitap %p: %u..%u -> %p", level, state, lo, hi, next_state);       if (next_hashes != NULL)       {         for (ORanges<Hash>::iterator prev_range = previous.begin(); prev_range != previous.end(); ++prev_range)@@ -4370,7 +4446,7 @@ void Pattern::gen_predict_match_transitions(size_t level, DFA::State *state, con   } } -void Pattern::gen_match_hfa(DFA::State* start)+void Pattern::gen_match_hfa(DFA::State *start) {   size_t max_level = HFA::MAX_DEPTH - 1; // max level from start state(s) is reduced when hashes exponentially increase   HFA::State index = 1; // DFA states are enumarated for breadth-first matching with the state visit set in match_hfa()@@ -4515,7 +4591,7 @@ bool Pattern::match_hfa_transitions(size_t level, const HFA::Hashes& hashes, con     if (level == 0 || visit.test(next->first))     {       bool all = true;-      for (size_t offset = std::max(static_cast<size_t>(7), level) - 7; offset <= level; ++offset)+      for (size_t offset = std::max<size_t>(7, level) - 7; offset <= level; ++offset)       {         uint8_t mask = 1 << (level - offset);         bool flag = false;

man/ug.1

@@ -1,4 +1,4 @@-.TH UGREP "1" "July 09, 2024" "ugrep 6.2.0" "User Commands"+.TH UGREP "1" "July 31, 2024" "ugrep 6.3.0" "User Commands" .SH NAME \fBugrep\fR, \fBug\fR -- file pattern searcher .SH SYNOPSIS@@ -249,6 +249,8 @@ font properties `n' (normal), `f' (faint), `h' (highlight), `i' (invert), `u' (underline).  Parameter `hl' enables file name hyperlinks.  Parameter `rv' reverses the `sl=' and `cx=' parameters when option \fB\-v\fR is specified.  Selectively overrides GREP_COLORS.+Legacy grep single parameter codes may be specified, for example+\fB\-\-colors\fR='7;32' or \fB\-\-colors\fR=ig to set ms (match selected). .TP \fB\-\-config\fR[=\fIFILE\fR], \fB\-\-\-\fR[\fIFILE\fR] Use configuration FILE.  The default FILE is `.ugrep'.  The working@@ -546,17 +548,17 @@ same as \fB\-\-include\-fs\fR=. (dot). Note that \fB\-\-exclude\-fs\fR=\fIMOUNTS priority over \fB\-\-include\-fs\fR=\fIMOUNTS\fR.  This option may be repeated. .TP \fB\-\-index\fR-Perform index\-based recursive search.  This option assumes, but-does not require, that files are indexed with ugrep\-indexer.  This-option accelerates recursive searching by skipping non\-matching-files, archives and compressed files when indexed.  Significant+Perform fast index\-based recursive search.  This option assumes,+but does not require, that files are indexed with ugrep\-indexer.+This option also enables option \fB\-r\fR or \fB\-\-recursive\fR.  Skips indexed+non\-matching files, archives and compressed files.  Significant acceleration may be achieved on cold (not file\-cached) and large file systems, or any file system that is slow to search.  Note that-the start\-up time to search is increased, which may be significant-when complex search patterns are specified that contain large-Unicode character classes combined with `*' or `+' repeats, which-should be avoided.  Option \fB\-U\fR (\fB\-\-ascii\fR) improves performance.-Option \fB\-\-stats\fR displays an index search report.+the start\-up time to search may be increased when complex search+patterns are specified that contain large Unicode character classes+combined with `*' or `+' repeats, which should be avoided.  Option+\fB\-U\fR (\fB\-\-ascii\fR) improves performance.  Option \fB\-\-stats\fR displays an+index search report. .TP \fB\-J\fR \fINUM\fR, \fB\-\-jobs\fR=\fINUM\fR Specifies the number of threads spawned to search files.  By

man/ugrep-indexer.1

@@ -1,4 +1,4 @@-.TH UGREP-INDEXER "1" "July 09, 2024" "ugrep-indexer 6.2.0" "User Commands"+.TH UGREP-INDEXER "1" "July 31, 2024" "ugrep-indexer 6.3.0" "User Commands" .SH NAME \fBugrep-indexer\fR -- file indexer to accelerate recursive searching .SH SYNOPSIS

man/ugrep.1

@@ -1,4 +1,4 @@-.TH UGREP "1" "July 09, 2024" "ugrep 6.2.0" "User Commands"+.TH UGREP "1" "July 31, 2024" "ugrep 6.3.0" "User Commands" .SH NAME \fBugrep\fR, \fBug\fR -- file pattern searcher .SH SYNOPSIS@@ -249,6 +249,8 @@ font properties `n' (normal), `f' (faint), `h' (highlight), `i' (invert), `u' (underline).  Parameter `hl' enables file name hyperlinks.  Parameter `rv' reverses the `sl=' and `cx=' parameters when option \fB\-v\fR is specified.  Selectively overrides GREP_COLORS.+Legacy grep single parameter codes may be specified, for example+\fB\-\-colors\fR='7;32' or \fB\-\-colors\fR=ig to set ms (match selected). .TP \fB\-\-config\fR[=\fIFILE\fR], \fB\-\-\-\fR[\fIFILE\fR] Use configuration FILE.  The default FILE is `.ugrep'.  The working@@ -546,17 +548,17 @@ same as \fB\-\-include\-fs\fR=. (dot). Note that \fB\-\-exclude\-fs\fR=\fIMOUNTS priority over \fB\-\-include\-fs\fR=\fIMOUNTS\fR.  This option may be repeated. .TP \fB\-\-index\fR-Perform index\-based recursive search.  This option assumes, but-does not require, that files are indexed with ugrep\-indexer.  This-option accelerates recursive searching by skipping non\-matching-files, archives and compressed files when indexed.  Significant+Perform fast index\-based recursive search.  This option assumes,+but does not require, that files are indexed with ugrep\-indexer.+This option also enables option \fB\-r\fR or \fB\-\-recursive\fR.  Skips indexed+non\-matching files, archives and compressed files.  Significant acceleration may be achieved on cold (not file\-cached) and large file systems, or any file system that is slow to search.  Note that-the start\-up time to search is increased, which may be significant-when complex search patterns are specified that contain large-Unicode character classes combined with `*' or `+' repeats, which-should be avoided.  Option \fB\-U\fR (\fB\-\-ascii\fR) improves performance.-Option \fB\-\-stats\fR displays an index search report.+the start\-up time to search may be increased when complex search+patterns are specified that contain large Unicode character classes+combined with `*' or `+' repeats, which should be avoided.  Option+\fB\-U\fR (\fB\-\-ascii\fR) improves performance.  Option \fB\-\-stats\fR displays an+index search report. .TP \fB\-J\fR \fINUM\fR, \fB\-\-jobs\fR=\fINUM\fR Specifies the number of threads spawned to search files.  By

src/cnf.cpp

@@ -30,7 +30,7 @@ @file      cnf.cpp @brief     CNF class for normalization of Boolean search queries @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/cnf.hpp

@@ -30,7 +30,7 @@ @file      cnf.hpp @brief     CNF class for normalization of Boolean search queries @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/flag.hpp

@@ -30,7 +30,7 @@ @file      flag.hpp @brief     Tri-state Flag class and global flags set by options @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ @@ -198,6 +198,7 @@ extern const char *flag_tag; extern const char *flag_view; extern std::string              flag_filter; extern std::string              flag_hyperlink_prefix; // internal flag+extern std::string              flag_hyperlink_host; // internal flag extern std::string              flag_hyperlink_path; // internal flag extern std::string              flag_regexp; // internal flag extern std::set<std::string>    flag_config_files; // internal flag

src/mmap.hpp

@@ -30,7 +30,7 @@ @file      mmap.hpp @brief     class to manage memory-mapped files @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/output.cpp

@@ -30,7 +30,7 @@ @file      output.cpp @brief     Output management @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ @@ -236,25 +236,7 @@ void Output::header(const char *pathname, const std::string& partname, bool& hea   bool hyp = pathname != Static::LABEL_STANDARD_INPUT && color_hl != NULL; // include hyperlinks    if (hyp)-  {-    str(color_hl);-    str(flag_hyperlink_prefix);-    str("://", 3);-    uri(flag_hyperlink_path);-    chr('/');-    uri(pathname);-    if (!(heading && flag_heading) && flag_hyperlink_line)-    {-      chr(':');-      num(lineno);-      if (flag_column_number)-      {-        chr(':');-        num(columno);-      }-    }-    str(color_st);-  }+    open_hyperlink(pathname, !(heading && flag_heading) && flag_hyperlink_line, lineno, columno);    // when a separator is needed   bool sep = false;@@ -281,10 +263,7 @@ void Output::header(const char *pathname, const std::string& partname, bool& hea     {       // --hyperlink: close link       if (hyp)-      {-        str(color_hl);-        str(color_st);-      }+        close_hyperlink();        str(color_fn);       str(color_del);@@ -293,25 +272,7 @@ void Output::header(const char *pathname, const std::string& partname, bool& hea        // --hyperlink: open link       if (hyp)-      {-        str(color_hl);-        str(flag_hyperlink_prefix);-        str("://", 3);-        uri(flag_hyperlink_path);-        chr('/');-        uri(pathname);-        if (flag_hyperlink_line)-        {-          chr(':');-          num(lineno);-          if (flag_column_number)-          {-            chr(':');-            num(columno);-          }-        }-        str(color_st);-      }+        open_hyperlink(pathname, flag_hyperlink_line, lineno, columno);        // the next headers should not include this pathname       heading = false;@@ -380,10 +341,7 @@ void Output::header(const char *pathname, const std::string& partname, bool& hea    // --hyperlink: close link   if (hyp)-  {-    str(color_hl);-    str(color_st);-  }+    close_hyperlink();    if (flag_byte_offset)   {@@ -500,15 +458,7 @@ void Output::header(const char *pathname, const std::string& partname)     str(color_fn);      if (hyp)-    {-      str(color_hl);-      str(flag_hyperlink_prefix);-      str("://", 3);-      uri(flag_hyperlink_path);-      chr('/');-      uri(pathname);-      str(color_st);-    }+      open_hyperlink(pathname);      if (nul)       chr('\0');@@ -527,15 +477,7 @@ void Output::header(const char *pathname, const std::string& partname)     str(color_fn);      if (hyp)-    {-      str(color_hl);-      str(flag_hyperlink_prefix);-      str("://", 3);-      uri(flag_hyperlink_path);-      chr('/');-      uri(pathname);-      str(color_st);-    }+      open_hyperlink(pathname);      if (nul)       chr('\0');@@ -546,10 +488,7 @@ void Output::header(const char *pathname, const std::string& partname)       chr('\0');      if (hyp)-    {-      str(color_hl);-      str(color_st);-    }+      close_hyperlink();   }    if (!partname.empty())@@ -573,23 +512,12 @@ void Output::binary_file_matches(const char *pathname, const std::string& partna   str(color_fn);    if (pathname != Static::LABEL_STANDARD_INPUT && color_hl != NULL)-  {-    str(color_hl);-    str(flag_hyperlink_prefix);-    str("://", 3);-    uri(flag_hyperlink_path);-    chr('/');-    uri(pathname);-    str(color_st);-  }+    open_hyperlink(pathname);    str(pathname);    if (pathname != Static::LABEL_STANDARD_INPUT && color_hl != NULL)-  {-    str(color_hl);-    str(color_st);-  }+    close_hyperlink();    if (!partname.empty())   {

src/output.hpp

@@ -30,7 +30,7 @@ @file      output.hpp @brief     Output management @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ @@ -554,6 +554,40 @@ class Output {     chr('0' + (b & 7));   } +  // open a hyperlink+  inline void open_hyperlink(const char *pathname, bool linkpos = false, size_t lineno = 0, size_t columno = 0)+  {+    str(color_hl);+    str(flag_hyperlink_prefix);+    str("://", 3);+    uri(flag_hyperlink_host);+    if (*pathname != '/')+    {+      chr('/');+      uri(flag_hyperlink_path);+      chr('/');+    }+    uri(pathname);+    if (linkpos)+    {+      chr(':');+      num(lineno);+      if (flag_column_number)+      {+        chr(':');+        num(columno);+      }+    }+    str(color_st);+  }++  // close the hyperlink+  inline void close_hyperlink()+  {+    str(color_hl);+    str(color_st);+  }+   // output a newline (platform-specific conditional "\r\n" or "\n"); flush if --line-buffered   inline void nl(bool lf_only = false)   {

src/query.cpp

@@ -30,7 +30,7 @@ @file      query.cpp @brief     Query engine TUI @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/query.hpp

@@ -30,7 +30,7 @@ @file      query.hpp @brief     Query engine and UI @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/screen.cpp

@@ -30,7 +30,7 @@ @file      screen.cpp @brief     ANSI SGR code controlled screen API - static, not thread safe @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/screen.hpp

@@ -30,7 +30,7 @@ @file      screen.hpp @brief     ANSI SGR code controlled screen API - static, not thread safe @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/stats.cpp

@@ -30,7 +30,7 @@ @file      stats.cpp @brief     collect global statistics - static, partially thread-safe @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/stats.hpp

@@ -30,7 +30,7 @@ @file      stats.hpp @brief     collect global statistics - static, partially thread-safe (see below) @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt  Thread-safe functions:

src/ugrep-indexer.cpp

@@ -35,7 +35,7 @@ */  // DO NOT ALTER THIS LINE: updated by makemake.sh and we need it physically here for MSVC++ build from source-#define UGREP_VERSION "6.2.0"+#define UGREP_VERSION "6.3.0"  // use a task-parallel thread to decompress the stream into a pipe to search, also handles nested archives #define WITH_DECOMPRESSION_THREAD@@ -806,10 +806,10 @@ bool is_binary(const char *s, size_t n)   return false; } -// prime 61 file indexing hash function-inline uint16_t indexhash(uint16_t h, uint8_t b)+// prime 61 mod 2^16 file indexing hash function+inline uint32_t indexhash(uint32_t h, uint8_t b) {-  return (h << 6) - h - h - h + b;+  return static_cast<uint16_t>((h << 6) - h - h - h + b); }  // index a file to produce hashes[0..hashes_size-1] table, noise, and archive/binary file detection flags@@ -924,7 +924,7 @@ bool index(Stream& stream, const char *pathname, uint8_t *hashes, size_t& hashes     while (true)     {       // compute 8 staggered Bloom filters, hashing 1-grams to 8-grams for N^2 = 64 Bloom hash functions-      uint16_t h = window[0];+      uint32_t h = window[0];       hashes[h] &= ~0x01;       h = indexhash(h, window[1]);       hashes[h] &= ~0x02;@@ -961,7 +961,7 @@ bool index(Stream& stream, const char *pathname, uint8_t *hashes, size_t& hashes    for (size_t i = 0; i < winlen; ++i)   {-    uint16_t h = window[i];+    uint32_t h = window[i];     hashes[h] &= ~0x01;     for (size_t j = i + 1, k = 0x02; j < winlen; ++j, k <<= 1)     {

src/ugrep.cpp

@@ -30,7 +30,7 @@ @file      ugrep.cpp @brief     a pattern search utility @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt  User manual:@@ -431,6 +431,7 @@ const char *flag_tag               = NULL; const char *flag_view              = ""; std::string              flag_filter; std::string              flag_hyperlink_prefix;+std::string              flag_hyperlink_host; std::string              flag_hyperlink_path; std::string              flag_regexp; std::set<std::string>    flag_config_files;@@ -818,7 +819,7 @@ inline void read_line(reflex::AbstractMatcher *matcher, const std::string& line) // copy color buffers inline void copy_color(char to[COLORLEN], const char from[COLORLEN]) {-  size_t len = std::min(strlen(from), static_cast<size_t>(COLORLEN - 1));+  size_t len = std::min<size_t>(strlen(from), COLORLEN - 1);    memcpy(to, from, len);   to[len] = '\0';@@ -1407,7 +1408,7 @@ struct Zthread {           memmove(buf, buf + BLOCKSIZE, static_cast<size_t>(len));            // check if archived file meets selection criteria-          size_t minlen = static_cast<size_t>(std::min(static_cast<uint64_t>(len), size)); // size_t is OK: len is streamsize but non-negative+          size_t minlen = static_cast<size_t>(std::min<uint64_t>(len, size)); // size_t is OK: len is streamsize but non-negative           is_selected = select_matching(archive.c_str(), path.c_str(), buf, minlen, is_regular);            // if extended headers are present@@ -1470,7 +1471,7 @@ struct Zthread {            while (len > 0 && !stop)           {-            size_t len_out = static_cast<size_t>(std::min(static_cast<uint64_t>(len), size)); // size_t is OK: len is streamsize but non-negative+            size_t len_out = static_cast<size_t>(std::min<uint64_t>(len, size)); // size_t is OK: len is streamsize but non-negative              if (ok)             {@@ -1690,7 +1691,7 @@ struct Zthread {            while (len > 0 && !stop)           {-            size_t n = std::min(static_cast<size_t>(len), size);+            size_t n = std::min<size_t>(len, size);             char *b = reinterpret_cast<char*>(buf);              path.append(b, n);@@ -1742,7 +1743,7 @@ struct Zthread {           }            // check if archived file meets selection criteria-          size_t minlen = std::min(static_cast<size_t>(len), filesize);+          size_t minlen = std::min<size_t>(len, filesize);           is_selected = select_matching(archive.c_str(), path.c_str(), buf, minlen, is_regular);            // if the pipe is closed, then get a new pipe to search the next part in the archive@@ -1785,7 +1786,7 @@ struct Zthread {            while (len > 0 && !stop)           {-            size_t len_out = std::min(static_cast<size_t>(len), size); // size_t is OK: len is streamsize but non-negative+            size_t len_out = std::min<size_t>(len, size); // size_t is OK: len is streamsize but non-negative              if (ok)             {@@ -5596,6 +5597,8 @@ void options(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int a                   flag_ignore_case = false;                 else if (strcmp(arg, "no-ignore-files") == 0)                   flag_ignore_files.clear();+                else if (strcmp(arg, "no-index") == 0)+                  flag_index = NULL;                 else if (strcmp(arg, "no-initial-tab") == 0)                   flag_initial_tab = false;                 else if (strcmp(arg, "no-invert-match") == 0)@@ -5635,7 +5638,7 @@ void options(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int a                 else if (strcmp(arg, "neg-regexp") == 0)                   usage("missing argument for --", arg);                 else-                  usage("invalid option --", arg, "--neg-regexp=, --not, --no-any-line, --no-ascii, --no-binary, --no-bool, --no-break, --no-byte-offset, --no-color, --no-config, --no-confirm, --no-decompress, --no-dereference, --no-dereference-files, --no-dotall, --no-empty, --no-filename, --no-filter, --no-glob-ignore-case, --no-group-separator, --no-heading, --no-hidden, --no-hyperlink, --no-ignore-binary, --no-ignore-case, --no-ignore-files --no-initial-tab, --no-invert-match, --no-line-number, --no-only-line-number, --no-only-matching, --no-messages, --no-mmap, --no-pager, --no-pretty, --no-smart-case, --no-sort, --no-split, --no-stats, --no-tree, --no-ungroup, --no-view or --null");+                  usage("invalid option --", arg, "--neg-regexp=, --not, --no-any-line, --no-ascii, --no-binary, --no-bool, --no-break, --no-byte-offset, --no-color, --no-config, --no-confirm, --no-decompress, --no-dereference, --no-dereference-files, --no-dotall, --no-empty, --no-filename, --no-filter, --no-glob-ignore-case, --no-group-separator, --no-heading, --no-hidden, --no-hyperlink, --no-ignore-binary, --no-ignore-case, --no-ignore-files, --no-index, --no-initial-tab, --no-invert-match, --no-line-number, --no-only-line-number, --no-only-matching, --no-messages, --no-mmap, --no-pager, --no-pretty, --no-smart-case, --no-sort, --no-split, --no-stats, --no-tree, --no-ungroup, --no-view or --null");                 break;                case 'o':@@ -7649,7 +7652,8 @@ void set_terminal_hyperlink()       gethostname(host, sizeof(host)); #endif -      flag_hyperlink_path.assign(host).append("/").append(path);+      flag_hyperlink_host.assign(host);+      flag_hyperlink_path.append(path);        free(cwd); @@ -8268,9 +8272,17 @@ void ugrep()     flag_after_context = flag_before_context = 0;    // --depth: if -R or -r is not specified then enable -r-  if ((flag_min_depth > 0 || flag_max_depth > 0) && flag_directories_action == Action::UNSP)+  if (flag_min_depth > 0 || flag_max_depth > 0)     flag_directories_action = Action::RECURSE; +  // --index: enable -r if -r or -R is not enabled, reject -dskip+  if (flag_index != NULL)+  {+    if (flag_directories_action == Action::SKIP)+      usage("option --index searches recursively, but option -dskip is specified");+    flag_directories_action = Action::RECURSE;+  }+   // -p (--no-dereference) and -S (--dereference): -p takes priority over -S and -R   if (flag_no_dereference)     flag_dereference = flag_dereference_files = false;@@ -8411,10 +8423,6 @@ void ugrep()   if (flag_format_begin != NULL)     Output(Static::output).format(flag_format_begin, 0); -  // --index: index search is not applicable to non-recursive searching-  if (flag_directories_action != Action::RECURSE)-    flag_index = NULL;-   // --index: search is only possible with compatible options   if (flag_index != NULL)   {@@ -10995,7 +11003,7 @@ void Grep::search(const char *pathname, uint16_t cost)              lineno += matcher->lines() - 1;           }-          else if (flag_before_context + flag_after_context > 0)+          else           {             if (restline_size > 0)             {@@ -11024,76 +11032,100 @@ void Grep::search(const char *pathname, uint16_t cost)             }             else             {-              // do not include additional lines in the match output-              const char *to = static_cast<const char*>(memchr(begin, '\n', size));-              if (to != NULL)-                size = to - begin;--              // length of the match is the number of UTF-8-encoded Unicode characters-              size_t length = utf8nlen(begin, size);-              size_t fit_length = length;--              if (fit_length > width)-              {-                if (fit_length > width + 4)-                  fit_length = width;-                width = 0;-              }-              else-              {-                width -= fit_length;-              }--              if (!nl)+              while (true)               {-                const char *bol = matcher->bol();-                size_t border = matcher->border();-                size_t margin = utf8nlen(bol, border);-                size_t before = flag_before_context * fit_length / (flag_before_context + flag_after_context);+                // check for additional lines in the match output+                size_t rest = 0;+                const char *to = static_cast<const char*>(memchr(begin, '\n', size));+                if (to != NULL)+                {+                  rest = size;+                  size = to - begin;+                  rest -= size;+                } -                if (before < flag_before_context)-                  before = flag_before_context - before;-                else-                  before = 0;+                // length of the match is the number of UTF-8-encoded Unicode characters+                size_t length = utf8nlen(begin, size);+                size_t fit_length = length; -                if (margin > before)+                if (fit_length > width)                 {-                  out.str(color_se);-                  out.str("...", 3);-                  out.str(color_off);-                  out.str(color_cx);-                  out.utf8strn(utf8skipn(bol, border, margin - before), border, before);-                  out.str(color_off);-                  width -= before;+                  if (fit_length > width + 4)+                    fit_length = width;+                  width = 0;                 }                 else                 {-                  if (border > 0)+                  width -= fit_length;+                }++                if (!nl)+                {+                  const char *bol = matcher->bol();+                  size_t border = matcher->border();+                  size_t margin = utf8nlen(bol, border);+                  size_t before = flag_before_context * fit_length / (flag_before_context + flag_after_context);++                  if (before < flag_before_context)+                    before = flag_before_context - before;+                  else+                    before = 0;++                  if (margin > before)                   {+                    out.str(color_se);+                    out.str("...", 3);+                    out.str(color_off);                     out.str(color_cx);-                    out.str(bol, border);+                    out.utf8strn(utf8skipn(bol, border, margin - before), border, before);                     out.str(color_off);+                    width -= before;                   }-                  if (margin >= 3)-                    width -= margin - 3;                   else-                    width += 3 - margin;+                  {+                    if (border > 0)+                    {+                      out.str(color_cx);+                      out.str(bol, border);+                      out.str(color_off);+                    }+                    if (margin >= 3)+                      width -= margin - 3;+                    else+                      width += 3 - margin;+                  }                 }-              } -              out.str(match_ms);-              if (fit_length == length)-              {-                out.str(begin, size);-              }-              else-              {-                out.utf8strn(begin, size, fit_length);-                out.str("[+", 2);-                out.num(length - fit_length);-                out.chr(']');+                out.str(match_ms);+                if (fit_length == length)+                {+                  out.str(begin, size);+                }+                else+                {+                  out.utf8strn(begin, size, fit_length);+                  out.str("[+", 2);+                  out.num(length - fit_length);+                  out.chr(']');+                }+                out.str(match_off);++                // no more additional lines to output?+                if (to == NULL)+                  break;++                out.nl();++                // output header and repeat to output a multiline match after previous newline+                if (!flag_no_header)+                  out.header(pathname, partname, heading, ++lineno, matcher, matcher->first() + (to - begin), flag_separator_bar, false);++                begin = to + 1;+                size = rest - 1;+                width = flag_before_context + flag_after_context;++                nl = true;               }-              out.str(match_off);                const char *eol = matcher->eol(); // warning: call eol() before bol() and end()               const char *end = matcher->end();@@ -11105,47 +11137,6 @@ void Grep::search(const char *pathname, uint16_t cost)              nl = true;           }-          else-          {-            if (flag_multiline)-            {-              // echo multi-line matches line-by-line-              const char *from = begin;-              const char *to;--              while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)-              {-                out.str(match_ms);-                out.str(from, to - from);-                out.str(match_off);-                out.chr('\n');--                out.header(pathname, partname, heading, ++lineno, NULL, matcher->first() + (to - begin) + 1, flag_separator_bar, false);--                from = to + 1;-              }--              size -= from - begin;-              begin = from;-            }--            if (size > 0)-            {-              bool lf_only = begin[size - 1] == '\n';-              size -= lf_only;-              if (size > 0)-              {-                out.str(match_ms);-                out.str(begin, size);-                out.str(match_off);-              }-              out.nl(lf_only);-            }-            else-            {-              nl = true;-            }-          }         }          // --min-count: require at least min-count matches@@ -13878,6 +13869,8 @@ void help(std::ostream& out)             (invert), `u' (underline).  Parameter `hl' enables file name\n\             hyperlinks.  Parameter `rv' reverses the `sl=' and `cx=' parameters\n\             when option -v is specified.  Selectively overrides GREP_COLORS.\n\+            Legacy grep single parameter codes may be specified, for example\n\+            --colors='7;32' or --colors=ig to set ms (match selected).\n\     --config[=FILE], ---[FILE]\n\             Use configuration FILE.  The default FILE is `.ugrep'.  The working\n\             directory is checked first for FILE, then the home directory.  The\n\@@ -14159,17 +14152,17 @@ void help(std::ostream& out) #endif             "\     --index\n\-            Perform index-based recursive search.  This option assumes, but\n\-            does not require, that files are indexed with ugrep-indexer.  This\n\-            option accelerates recursive searching by skipping non-matching\n\-            files, archives and compressed files when indexed.  Significant\n\+            Perform fast index-based recursive search.  This option assumes,\n\+            but does not require, that files are indexed with ugrep-indexer.\n\+            This option also enables option -r or --recursive.  Skips indexed\n\+            non-matching files, archives and compressed files.  Significant\n\             acceleration may be achieved on cold (not file-cached) and large\n\             file systems, or any file system that is slow to search.  Note that\n\-            the start-up time to search is increased, which may be significant\n\-            when complex search patterns are specified that contain large\n\-            Unicode character classes combined with `*' or `+' repeats, which\n\-            should be avoided.  Option -U (--ascii) improves performance.\n\-            Option --stats displays an index search report.\n\+            the start-up time to search may be increased when complex search\n\+            patterns are specified that contain large Unicode character classes\n\+            combined with `*' or `+' repeats, which should be avoided.  Option\n\+            -U (--ascii) improves performance.  Option --stats displays an\n\+            index search report.\n\     -J NUM, --jobs=NUM\n\             Specifies the number of threads spawned to search files.  By\n\             default an optimum number of threads is spawned to search files\n\

src/ugrep.hpp

@@ -30,7 +30,7 @@ @file      ugrep.hpp @brief     a pattern search utility @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ @@ -38,7 +38,7 @@ #define UGREP_HPP  // DO NOT ALTER THIS LINE: updated by makemake.sh and we need it physically here for MSVC++ build from source-#define UGREP_VERSION "6.2.0"+#define UGREP_VERSION "6.3.0"  // disable mmap because mmap is almost always slower than the file reading speed improvements since 3.0.0 #define WITH_NO_MMAP

src/vkey.cpp

@@ -30,7 +30,7 @@ @file      vkey.cpp @brief     Virtual terminal keyboard input API - static, not thread safe @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/vkey.hpp

@@ -30,7 +30,7 @@ @file      vkey.hpp @brief     Virtual terminal keyboard input API - static, not thread safe @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2022, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt  Supports Unix/Linux, MacOS, DOS, and Windows.

src/zstream.hpp

@@ -30,7 +30,7 @@ @file      zstream.hpp @brief     file decompression streams - zstreambuf extends std::streambuf @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

src/zthread.hpp

@@ -30,7 +30,7 @@ @file      zthread.hpp @brief     file decompression threads @author    Robert van Engelen - engelen@genivia.com-@copyright (c) 2019-2023, Robert van Engelen, Genivia Inc. All rights reserved.+@copyright (c) 2019,2024, Robert van Engelen, Genivia Inc. All rights reserved. @copyright (c) BSD-3 License - see LICENSE.txt */ 

tests/archive.gz

Binary file changed

tests/archive.tar.zip

Binary file changed

tests/archive.tgz

Binary file changed

tests/archive2.tgz

Binary file changed

tests/archive3.tgz

Binary file changed

tests/out/Hello_SnS-I-onC10.out

@@ -1,4 +1,8 @@-Hello.bat:1:#!/bin/shcho "Hello Wor...-Hello.java:3:...d! greeting public s...+Hello.bat:1:#!/bin/sh+Hello.bat|2|echo "Hello World!"+Hello.java:3:...d! greeting+Hello.java|4|{ public static void... Hello.java:6:  }-Hello.sh:1:#!/bin/shcho "Hello Wor...+Hello.java|7|}+Hello.sh:1:#!/bin/sh+Hello.sh|2|echo "Hello World!"

tests/out/Hello_SnS-W-onC10.out

@@ -1,4 +1,5 @@-Hello.bat:1:#!/bin/shcho "Hello Wor...+Hello.bat:1:#!/bin/sh+Hello.bat|2|echo "Hello World!" Hello.class:1: 00000000  -- -- -- -- -- -- -- --  -- 1d 0a 00 -- -- -- --  |---------]J@----| Hello.class:2:@@ -13,37 +14,60 @@ 00000180  -- -- -- -- -- -- 00 0a  00 -- -- -- -- -- -- --  |------@J@-------| Hello.class:7: 00000180  -- -- -- -- -- -- -- --  -- -- 00 0a 00 -- -- --  |----------@J@---|-Hello.java:3:...d! greeting public s...+Hello.java:3:...d! greeting+Hello.java|4|{ public static void... Hello.java:6:  }+Hello.java|7|} Hello.pdf:1:%PDF-1.3+Hello.pdf|2|% Hello.pdf:2: 00000010  -- -- -- -- c6 0a 34 --  -- -- -- -- -- -- -- --  |----.J4---------|-Hello.pdf:3:Äåòåë§ó ÐÄÆ4 0 obj< ...-Hello.pdf:4:...teDecode >>tream+Hello.pdf:3:Äåòåë§ó ÐÄÆ4 0 obj+Hello.pdf|4|<< /Length 5 0 R /Fi...+Hello.pdf:4:...teDecode >>+Hello.pdf|5|stream Hello.pdf:5:stream+Hello.pdf|6|x Hello.pdf:6: 00000050  -- -- bb 0a 02 -- -- --  -- -- -- -- -- -- -- --  |--.JB-----------| Hello.pdf:7: 00000080  -- -- -- -- -- -- -- --  59 0a 9f -- -- -- -- --  |--------YJ.-----| Hello.pdf:8: 000000c0  -- -- -- -- -- -- -- --  a5 0a 65 -- -- -- -- --  |--------.Je-----|-Hello.pdf:9:EŽ»endstreamndobj-Hello.pdf:10:endobj 0 obj-Hello.pdf:11:5 0 obj23-Hello.pdf:12:123ndobj-Hello.pdf:13:endobj 0 obj-Hello.pdf:14:2 0 obj< /Type /Page /P...-Hello.pdf:15:... 0 612 792]>-Hello.pdf:16:>>ndobj-Hello.pdf:17:endobj 0 obj-Hello.pdf:18:6 0 obj< /ProcSet [ /PD...-Hello.pdf:19:... /TT1 8 0 R> >>-Hello.pdf:20:>> >>ndobj-Hello.pdf:21:endobj 0 obj-Hello.pdf:22:9 0 obj< /Length 10 0 R...-Hello.pdf:23:...teDecode >>tream-Hello.pdf:24:stream…U]hU>»sg´
-´ƒ?...+Hello.pdf:9:EŽ»endstream+Hello.pdf|10|endobj+Hello.pdf:10:endobj+Hello.pdf|11|5 0 obj+Hello.pdf:11:5 0 obj+Hello.pdf|12|123+Hello.pdf:12:123+Hello.pdf|13|endobj+Hello.pdf:13:endobj+Hello.pdf|14|2 0 obj+Hello.pdf:14:2 0 obj+Hello.pdf|15|<< /Type /Page /Pare...+Hello.pdf:15:... 0 612 792]+Hello.pdf|16|>>+Hello.pdf:16:>>+Hello.pdf|17|endobj+Hello.pdf:17:endobj+Hello.pdf|18|6 0 obj+Hello.pdf:18:6 0 obj+Hello.pdf|19|<< /ProcSet [ /PDF /...+Hello.pdf:19:... /TT1 8 0 R+Hello.pdf|20|>> >>+Hello.pdf:20:>> >>+Hello.pdf|21|endobj+Hello.pdf:21:endobj+Hello.pdf|22|9 0 obj+Hello.pdf:22:9 0 obj+Hello.pdf|23|<< /Length 10 0 R /N...+Hello.pdf:23:...teDecode >>+Hello.pdf|24|stream+Hello.pdf:24:stream+Hello.pdf|25|x…U]hU>»sg´
-´ƒ?m	... Hello.pdf:25:...>Æ‚¶*6/ö¥¥Å’J5+Hello.pdf|26| Hello.pdf:26: 000003f0  -- -- -- -- -- -- -- --  -- 9b 0a d0 -- -- -- --  |---------.J.----| Hello.pdf:27:@@ -51,40 +75,70 @@ Hello.pdf:28: 00000550  -- -- -- -- -- -- -- --  -- -- -- -- -- -- -- cc  |---------------.| 00000560  0a 4b -- -- -- -- -- --  -- -- -- -- -- -- -- --  |JK--------------|-Hello.pdf:29:ZŒ (}Rð;3Û...[+1 more]-Hello.pdf:30:...a4üf,£Tê&Ñ·�“4ndstream-Hello.pdf:31:endstreamndobj-Hello.pdf:32:endobj0 0 obj-Hello.pdf:33:10 0 obj116-Hello.pdf:34:1116ndobj-Hello.pdf:35:endobj 0 obj-Hello.pdf:36:7 0 obj /ICCBased 9 0 R ]-Hello.pdf:37:...sed 9 0 R ]ndobj-Hello.pdf:38:endobj 0 obj-Hello.pdf:39:3 0 obj< /Type /Pages /...-Hello.pdf:40:... 2 0 R ] >>ndobj-Hello.pdf:41:endobj1 0 obj-Hello.pdf:42:11 0 obj< /Type /Catalo...-Hello.pdf:43:...es 3 0 R >>ndobj-Hello.pdf:44:endobj 0 obj-Hello.pdf:45:8 0 obj< /Type /Font /S...-Hello.pdf:46:...tDescriptor2 0 R /En...-Hello.pdf:47:...idths [ 602 0 0 0 0 ...-Hello.pdf:48:...0 0 0 0 602 0 602 ] >>-Hello.pdf:49:... 0 602 ] >>ndobj-Hello.pdf:50:endobj2 0 obj-Hello.pdf:51:12 0 obj< /Type /FontDe...-Hello.pdf:52:...3 /FontBBox-558 -375...-Hello.pdf:53:...pHeight 729StemV 99 ...-Hello.pdf:54:...ile2 13 0 R>-Hello.pdf:55:>>ndobj-Hello.pdf:56:endobj3 0 obj-Hello.pdf:57:13 0 obj< /Length 14 0 ...-Hello.pdf:58:...teDecode >>tream+Hello.pdf:29:ZŒ (}Rð;3ÛdvA2Ã�ùî¹çž{Î...[+1 more]+Hello.pdf:30:...a4üf,£Tê&Ñ·�“4+Hello.pdf|31|endstream+Hello.pdf:31:endstream+Hello.pdf|32|endobj+Hello.pdf:32:endobj+Hello.pdf|33|10 0 obj+Hello.pdf:33:10 0 obj+Hello.pdf|34|1116+Hello.pdf:34:1116+Hello.pdf|35|endobj+Hello.pdf:35:endobj+Hello.pdf|36|7 0 obj+Hello.pdf:36:7 0 obj+Hello.pdf|37|[ /ICCBased 9 0 R ]+Hello.pdf:37:...sed 9 0 R ]+Hello.pdf|38|endobj+Hello.pdf:38:endobj+Hello.pdf|39|3 0 obj+Hello.pdf:39:3 0 obj+Hello.pdf|40|<< /Type /Pages /Med...+Hello.pdf:40:... 2 0 R ] >>+Hello.pdf|41|endobj+Hello.pdf:41:endobj+Hello.pdf|42|11 0 obj+Hello.pdf:42:11 0 obj+Hello.pdf|43|<< /Type /Catalog /P...+Hello.pdf:43:...es 3 0 R >>+Hello.pdf|44|endobj+Hello.pdf:44:endobj+Hello.pdf|45|8 0 obj+Hello.pdf:45:8 0 obj+Hello.pdf|46|<< /Type /Font /Subt...+Hello.pdf:46:...tDescriptor+Hello.pdf|47|12 0 R /Encoding /Ma...+Hello.pdf:47:...idths [ 602+Hello.pdf|48|0 0 0 0 0 0 0 0 0 0 ...+Hello.pdf:48:...0 0 0 0 602+Hello.pdf|49|0 0 602 ] >>+Hello.pdf:49:... 0 602 ] >>+Hello.pdf|50|endobj+Hello.pdf:50:endobj+Hello.pdf|51|12 0 obj+Hello.pdf:51:12 0 obj+Hello.pdf|52|<< /Type /FontDescri...+Hello.pdf:52:...3 /FontBBox+Hello.pdf|53|[-558 -375 718 1041]...+Hello.pdf:53:...pHeight 729+Hello.pdf|54|/StemV 99 /XHeight 5...+Hello.pdf:54:...ile2 13 0 R+Hello.pdf|55|>>+Hello.pdf:55:>>+Hello.pdf|56|endobj+Hello.pdf:56:endobj+Hello.pdf|57|13 0 obj+Hello.pdf:57:13 0 obj+Hello.pdf|58|<< /Length 14 0 R /L...+Hello.pdf:58:...teDecode >>+Hello.pdf|59|stream Hello.pdf:59:stream+Hello.pdf|60|x Hello.pdf:60: 000009c0  -- -- -- -- -- -- -- --  -- -- -- -- -- 02 0a f8  |-------------BJ.|-Hello.pdf:61:ÝW{xTÅ?sw6¯Í>’MH+Hello.pdf:61:ÝW{xTÅ?sw6¯Í>’MHH€] Hello.pdf:62: 00000be0  -- -- -- 2f 0a ef -- --  -- -- -- -- -- -- -- --  |---/J.----------| Hello.pdf:63:@@ -119,37 +173,71 @@ 00001420  -- -- -- -- -- -- -- --  -- -- -- -- -- -- -- e4  |---------------.| 00001430  0a 65 -- -- -- -- -- --  -- -- -- -- -- -- -- --  |Je--------------| Hello.pdf:79:...[+4 more]-Hello.pdf:80:endobj4 0 obj-Hello.pdf:81:14 0 obj734-Hello.pdf:82:2734ndobj-Hello.pdf:83:endobj5 0 obj-Hello.pdf:84:15 0 objHello)-Hello.pdf:85:(Hello)ndobj-Hello.pdf:86:endobj6 0 obj-Hello.pdf:87:16 0 objMac OS X 10.12....-Hello.pdf:88:...PDFContext)ndobj-Hello.pdf:89:endobj7 0 obj-Hello.pdf:90:17 0 objTextEdit)-Hello.pdf:91:(TextEdit)ndobj-Hello.pdf:92:endobj8 0 obj-Hello.pdf:93:18 0 objD:2020012401071...-Hello.pdf:94:...711Z00'00')ndobj-Hello.pdf:95:endobj9 0 obj-Hello.pdf:96:19 0 obj)-Hello.pdf:97:()ndobj-Hello.pdf:98:endobj0 0 obj-Hello.pdf:99:20 0 obj ]-Hello.pdf:100:[ ]ndobj-Hello.pdf:101:endobj 0 obj-Hello.pdf:102:1 0 obj< /Title 15 0 R ...-Hello.pdf:103:... R /ModDate8 0 R /Ke...-Hello.pdf:104:...s 20 0 R >>ndobj-Hello.pdf:105:endobjref-Hello.pdf:106:xref 21-Hello.pdf:107:0 21000000000 65535 f -Hello.pdf:129:trailer< /Size 21 /Root...-Hello.pdf:130:...583cf769a3>8db88d333...-Hello.pdf:131:...769a3> ] >>tartxref-Hello.pdf:132:startxref536-Hello.pdf:133:5536%EOF-Hello.sh:1:#!/bin/shcho "Hello Wor...+Hello.pdf:80:endobj+Hello.pdf|81|14 0 obj+Hello.pdf:81:14 0 obj+Hello.pdf|82|2734+Hello.pdf:82:2734+Hello.pdf|83|endobj+Hello.pdf:83:endobj+Hello.pdf|84|15 0 obj+Hello.pdf:84:15 0 obj+Hello.pdf|85|(Hello)+Hello.pdf:85:(Hello)+Hello.pdf|86|endobj+Hello.pdf:86:endobj+Hello.pdf|87|16 0 obj+Hello.pdf:87:16 0 obj+Hello.pdf|88|(Mac OS X 10.12.6 Qu...+Hello.pdf:88:...PDFContext)+Hello.pdf|89|endobj+Hello.pdf:89:endobj+Hello.pdf|90|17 0 obj+Hello.pdf:90:17 0 obj+Hello.pdf|91|(TextEdit)+Hello.pdf:91:(TextEdit)+Hello.pdf|92|endobj+Hello.pdf:92:endobj+Hello.pdf|93|18 0 obj+Hello.pdf:93:18 0 obj+Hello.pdf|94|(D:20200124010711Z00...+Hello.pdf:94:...711Z00'00')+Hello.pdf|95|endobj+Hello.pdf:95:endobj+Hello.pdf|96|19 0 obj+Hello.pdf:96:19 0 obj+Hello.pdf|97|()+Hello.pdf:97:()+Hello.pdf|98|endobj+Hello.pdf:98:endobj+Hello.pdf|99|20 0 obj+Hello.pdf:99:20 0 obj+Hello.pdf|100|[ ]+Hello.pdf:100:[ ]+Hello.pdf|101|endobj+Hello.pdf:101:endobj+Hello.pdf|102|1 0 obj+Hello.pdf:102:1 0 obj+Hello.pdf|103|<< /Title 15 0 R /Pr...+Hello.pdf:103:... R /ModDate+Hello.pdf|104|18 0 R /Keywords 19 ...+Hello.pdf:104:...s 20 0 R >>+Hello.pdf|105|endobj+Hello.pdf:105:endobj+Hello.pdf|106|xref+Hello.pdf:106:xref+Hello.pdf|107|0 21+Hello.pdf:107:0 21+Hello.pdf|108|0000000000 65535 f +Hello.pdf:129:trailer+Hello.pdf|130|<< /Size 21 /Root 11...+Hello.pdf:130:...583cf769a3>+Hello.pdf|131|<8db88d3335070bd9583...+Hello.pdf:131:...769a3> ] >>+Hello.pdf|132|startxref+Hello.pdf:132:startxref+Hello.pdf|133|5536+Hello.pdf:133:5536+Hello.pdf|134|%%EOF+Hello.sh:1:#!/bin/sh+Hello.sh|2|echo "Hello World!"

tests/out/Hello_SnS-onC10.out

@@ -1,6 +1,10 @@-Hello.bat:1:#!/bin/shcho "Hello Wor...+Hello.bat:1:#!/bin/sh+Hello.bat|2|echo "Hello World!" Binary file Hello.class matches-Hello.java:3:...d! greeting public s...+Hello.java:3:...d! greeting+Hello.java|4|{ public static void... Hello.java:6:  }+Hello.java|7|} Binary file Hello.pdf matches-Hello.sh:1:#!/bin/shcho "Hello Wor...+Hello.sh:1:#!/bin/sh+Hello.sh|2|echo "Hello World!"

tests/out/Hello_wnhS-I-onC10.out

@@ -1,13 +1,16 @@-Hello.bat:1:#!/bin/sh "Hello World!"+Hello.bat:1:#!/bin/sh+Hello.bat|2|echo "Hello World!" Hello.bat:2:...ho "Hello World!" Hello.java:1:import java.lang.*; Hello.java:3:public class Hello // p... Hello.java:3:...class Hello // print... Hello.java:3:...lo // prints a Hello... Hello.java:3:...s a Hello World! gre...-Hello.java:3:...orld! greeting publi...+Hello.java:3:...orld! greeting+Hello.java|4|{ public static void... Hello.java:4:{ public static void ma... Hello.java:4:... void main(String[] ... Hello.java:5:...n("Hello World!");-Hello.sh:1:#!/bin/sh "Hello World!"+Hello.sh:1:#!/bin/sh+Hello.sh|2|echo "Hello World!" Hello.sh:2:...ho "Hello World!"

tests/out/Hello_wnhS-W-onC10.out

@@ -1,4 +1,5 @@-Hello.bat:1:#!/bin/sh "Hello World!"+Hello.bat:1:#!/bin/sh+Hello.bat|2|echo "Hello World!" Hello.bat:2:...ho "Hello World!" Hello.class:5: 00000090  -- -- -- -- 48 65 6c 6c  6f 20 57 6f 72 6c 64 21  |----Hello World!|@@ -8,7 +9,8 @@ Hello.java:3:...class Hello // print... Hello.java:3:...lo // prints a Hello... Hello.java:3:...s a Hello World! gre...-Hello.java:3:...orld! greeting publi...+Hello.java:3:...orld! greeting+Hello.java|4|{ public static void... Hello.java:4:{ public static void ma... Hello.java:4:... void main(String[] ... Hello.java:5:...n("Hello World!");@@ -16,7 +18,8 @@ 00000000  -- -- -- -- -- -- -- 33  0a 25 c4 e5 f2 e5 eb a7  |-------3J%......| 00000010  f3 a0 d0 c4 c6 -- -- --  -- -- -- -- -- -- -- --  |.....-----------| Hello.pdf:3:4 0 obj-Hello.pdf:3:4 0 obj /Length 5 0 R /...+Hello.pdf:3:4 0 obj+Hello.pdf|4|<< /Length 5 0 R /Fi... Hello.pdf:4:<< /Length 5 0 R /Filte... Hello.pdf:4:...Length 5 0 R /Filter... Hello.pdf:4:.../Filter /FlateDecode@@ -30,7 +33,9 @@ Hello.pdf:9:...[+1 more] Hello.pdf:11:5 0 obj Hello.pdf:11:5 0 obj-Hello.pdf:13:endobj 0 obj+Hello.pdf|12|123+Hello.pdf:13:endobj+Hello.pdf|14|2 0 obj Hello.pdf:14:2 0 obj Hello.pdf:15:<< /Type /Page /Parent ... Hello.pdf:15:...Page /Parent 3 0 R /...@@ -41,7 +46,8 @@ Hello.pdf:15:...ntents 4 0 R /MediaB... Hello.pdf:15:...0 R /MediaBox [0 0 6... Hello.pdf:15:...aBox [0 0 612 792]-Hello.pdf:17:endobj 0 obj+Hello.pdf:17:endobj+Hello.pdf|18|6 0 obj Hello.pdf:18:6 0 obj Hello.pdf:19:<< /ProcSet [ /PDF /Tex... Hello.pdf:19:...et [ /PDF /Text ] /C...@@ -51,7 +57,8 @@ Hello.pdf:19:... R >> /Font << /TT1 ... Hello.pdf:19:...ont << /TT1 8 0 R Hello.pdf:19:...< /TT1 8 0 R-Hello.pdf:21:endobj 0 obj+Hello.pdf:21:endobj+Hello.pdf|22|9 0 obj Hello.pdf:22:9 0 obj Hello.pdf:23:<< /Length 10 0 R /N 1 ... Hello.pdf:23:...ength 10 0 R /N 1 /A...@@ -70,14 +77,18 @@ 000003e0  -- -- -- -- -- -- -- --  -- -- -- -- -- -- 37 20  |--------------7 | 000003f0  8b 9a 11 f2 c8 c7 58 --  -- -- -- -- -- -- -- --  |..Q...X---------| Hello.pdf:30:...[+1 more]-Hello.pdf:32:endobj 0 obj+Hello.pdf:32:endobj+Hello.pdf|33|10 0 obj Hello.pdf:33:10 0 obj Hello.pdf:34:1116+Hello.pdf|35|endobj Hello.pdf:36:7 0 obj-Hello.pdf:36:7 0 obj /ICCBased 9 0 R ]+Hello.pdf:36:7 0 obj+Hello.pdf|37|[ /ICCBased 9 0 R ] Hello.pdf:37:[ /ICCBased 9 0 R ] Hello.pdf:37:...CBased 9 0 R ]-Hello.pdf:38:endobj 0 obj+Hello.pdf:38:endobj+Hello.pdf|39|3 0 obj Hello.pdf:39:3 0 obj Hello.pdf:40:<< /Type /Pages /MediaB... Hello.pdf:40:...ges /MediaBox [0 0 6...@@ -86,23 +97,27 @@ Hello.pdf:40:...unt 1 /Kids [ 2 0 R ... Hello.pdf:40:... /Kids [ 2 0 R ] >> Hello.pdf:40:...ds [ 2 0 R ] >>-Hello.pdf:41:endobj 0 obj+Hello.pdf:41:endobj+Hello.pdf|42|11 0 obj Hello.pdf:42:11 0 obj Hello.pdf:43:<< /Type /Catalog /Page... Hello.pdf:43:...talog /Pages 3 0 R >> Hello.pdf:43:.../Pages 3 0 R >>-Hello.pdf:44:endobj 0 obj+Hello.pdf:44:endobj+Hello.pdf|45|8 0 obj Hello.pdf:45:8 0 obj Hello.pdf:46:<< /Type /Font /Subtype... Hello.pdf:46:... /Subtype /TrueType ... Hello.pdf:46:...BaseFont /GQTMZF+Men[+10]...-Hello.pdf:46:...r /FontDescriptor 0 ...+Hello.pdf:46:...r /FontDescriptor+Hello.pdf|47|12 0 R /Encoding /Ma... Hello.pdf:47:12 0 R /Encoding /MacRo... Hello.pdf:47:...Encoding /MacRomanEn[+6]... Hello.pdf:47:...ng /FirstChar 72 /La... Hello.pdf:47:...72 /LastChar 111 /Wi... Hello.pdf:47:... 111 /Widths [ 602-Hello.pdf:47:...Widths [ 602 0 0 0 0...+Hello.pdf:47:...Widths [ 602+Hello.pdf|48|0 0 0 0 0 0 0 0 0 0 ... Hello.pdf:48:0 0 0 0 0 0 0 0 0 0 0 0... Hello.pdf:48:0 0 0 0 0 0 0 0 0 0 0 0... Hello.pdf:48:... 0 0 0 0 0 0 0 0 0 0...@@ -120,14 +135,17 @@ Hello.pdf:48:... 0 0 602 0 0 0 0 0 0... Hello.pdf:48:... 602 0 0 0 0 0 0 602 Hello.pdf:48:... 0 0 0 0 0 0 602-Hello.pdf:48:... 0 0 0 0 602 0 602 ] >>+Hello.pdf:48:... 0 0 0 0 602+Hello.pdf|49|0 0 602 ] >> Hello.pdf:49:0 0 602 ] >>-Hello.pdf:50:endobj 0 obj+Hello.pdf:50:endobj+Hello.pdf|51|12 0 obj Hello.pdf:51:12 0 obj Hello.pdf:52:...Type /FontDescriptor... Hello.pdf:52:...FontName /GQTMZF+Men[+10]... Hello.pdf:52:...ular /Flags 33 /Font...-Hello.pdf:52:...s 33 /FontBBox -375 ...+Hello.pdf:52:...s 33 /FontBBox+Hello.pdf|53|[-558 -375 718 1041]... Hello.pdf:53:[-558 -375 718 1041] /I... Hello.pdf:53:...1] /ItalicAngle 0 /A... Hello.pdf:53:...e 0 /Ascent 928 /Des...@@ -140,7 +158,8 @@ Hello.pdf:54:...02 /MaxWidth 602 /Fo... Hello.pdf:54:...02 /FontFile2 13 0 R Hello.pdf:54:...File2 13 0 R-Hello.pdf:56:endobj 0 obj+Hello.pdf:56:endobj+Hello.pdf|57|13 0 obj Hello.pdf:57:13 0 obj Hello.pdf:58:<< /Length 14 0 R /Leng... Hello.pdf:58:...ength 14 0 R /Length...@@ -193,22 +212,30 @@ Hello.pdf:79:...[+1 more] Hello.pdf:81:14 0 obj Hello.pdf:81:14 0 obj-Hello.pdf:83:endobj 0 obj+Hello.pdf|82|2734+Hello.pdf:83:endobj+Hello.pdf|84|15 0 obj Hello.pdf:84:15 0 obj-Hello.pdf:86:endobj 0 obj+Hello.pdf:86:endobj+Hello.pdf|87|16 0 obj Hello.pdf:87:16 0 obj Hello.pdf:88:(Mac OS X 10.12.6 Quart... Hello.pdf:88:...ac OS X 10.12.6 Quar... Hello.pdf:88:... Quartz PDFContext)-Hello.pdf:89:endobj 0 obj+Hello.pdf:89:endobj+Hello.pdf|90|17 0 obj Hello.pdf:90:17 0 obj-Hello.pdf:92:endobj 0 obj+Hello.pdf:92:endobj+Hello.pdf|93|18 0 obj Hello.pdf:93:18 0 obj-Hello.pdf:95:endobj 0 obj+Hello.pdf:95:endobj+Hello.pdf|96|19 0 obj Hello.pdf:96:19 0 obj-Hello.pdf:98:endobj 0 obj+Hello.pdf:98:endobj+Hello.pdf|99|20 0 obj Hello.pdf:99:20 0 obj-Hello.pdf:101:endobj 0 obj+Hello.pdf:101:endobj+Hello.pdf|102|1 0 obj Hello.pdf:102:1 0 obj Hello.pdf:103:<< /Title 15 0 R /Produ... Hello.pdf:103:...Title 15 0 R /Produc...@@ -218,46 +245,59 @@ Hello.pdf:103:...eator 17 0 R /Creati... Hello.pdf:103:...R /CreationDate 18 0... Hello.pdf:103:...nDate 18 0 R /ModDate-Hello.pdf:103:...8 0 R /ModDate 0 R /...+Hello.pdf:103:...8 0 R /ModDate+Hello.pdf|104|18 0 R /Keywords 19 ... Hello.pdf:104:18 0 R /Keywords 19 0 R... Hello.pdf:104:...0 R /Keywords 19 0 R... Hello.pdf:104:...words 19 0 R /AAPL:K... Hello.pdf:104:...AAPL:Keywords 20 0 R >> Hello.pdf:104:...words 20 0 R >> Hello.pdf:105:endobj+Hello.pdf|106|xref Hello.pdf:107:0 21 Hello.pdf:108:0000000000 65535 f -Hello.pdf:108:...00 65535 f  00000 n +Hello.pdf:108:...00 65535 f +Hello.pdf|109|0000005392 00000 n  Hello.pdf:109:...005392 00000 n  Hello.pdf:110:0000000238 00000 n -Hello.pdf:110:...38 00000 n  00000 n +Hello.pdf:110:...38 00000 n +Hello.pdf|111|0000001714 00000 n  Hello.pdf:111:...001714 00000 n  Hello.pdf:112:0000000022 00000 n -Hello.pdf:112:...22 00000 n  00000 n +Hello.pdf:112:...22 00000 n +Hello.pdf|113|0000000219 00000 n  Hello.pdf:113:...000219 00000 n  Hello.pdf:114:0000000342 00000 n -Hello.pdf:114:...42 00000 n  00000 n +Hello.pdf:114:...42 00000 n +Hello.pdf|115|0000001679 00000 n  Hello.pdf:115:...001679 00000 n  Hello.pdf:116:0000001847 00000 n -Hello.pdf:116:...47 00000 n  00000 n +Hello.pdf:116:...47 00000 n +Hello.pdf|117|0000000439 00000 n  Hello.pdf:117:...000439 00000 n  Hello.pdf:118:0000001658 00000 n -Hello.pdf:118:...58 00000 n  00000 n +Hello.pdf:118:...58 00000 n +Hello.pdf|119|0000001797 00000 n  Hello.pdf:119:...001797 00000 n  Hello.pdf:120:0000002110 00000 n -Hello.pdf:120:...10 00000 n  00000 n +Hello.pdf:120:...10 00000 n +Hello.pdf|121|0000002362 00000 n  Hello.pdf:121:...002362 00000 n  Hello.pdf:122:0000005186 00000 n -Hello.pdf:122:...86 00000 n  00000 n +Hello.pdf:122:...86 00000 n +Hello.pdf|123|0000005207 00000 n  Hello.pdf:123:...005207 00000 n  Hello.pdf:124:0000005231 00000 n -Hello.pdf:124:...31 00000 n  00000 n +Hello.pdf:124:...31 00000 n +Hello.pdf|125|0000005284 00000 n  Hello.pdf:125:...005284 00000 n  Hello.pdf:126:0000005311 00000 n -Hello.pdf:126:...11 00000 n  00000 n +Hello.pdf:126:...11 00000 n +Hello.pdf|127|0000005353 00000 n  Hello.pdf:127:...005353 00000 n  Hello.pdf:128:0000005372 00000 n  Hello.pdf:128:...72 00000 n +Hello.pdf|129|trailer Hello.pdf:130:<< /Size 21 /Root 11 0 ... Hello.pdf:130:...ze 21 /Root 11 0 R /... Hello.pdf:130:.../Root 11 0 R /Info 1...@@ -265,5 +305,7 @@ Hello.pdf:130:... /Info 1 0 R /ID [ <... Hello.pdf:130:... 1 0 R /ID [ <8db88d... Hello.pdf:132:startxref-Hello.sh:1:#!/bin/sh "Hello World!"+Hello.pdf|133|5536+Hello.sh:1:#!/bin/sh+Hello.sh|2|echo "Hello World!" Hello.sh:2:...ho "Hello World!"

tests/out/Hello_wnhS-onC10.out

@@ -1,4 +1,5 @@-Hello.bat:1:#!/bin/sh "Hello World!"+Hello.bat:1:#!/bin/sh+Hello.bat|2|echo "Hello World!" Hello.bat:2:...ho "Hello World!" Binary file Hello.class matches Hello.java:1:import java.lang.*;@@ -6,10 +7,12 @@ Hello.java:3:...class Hello // print... Hello.java:3:...lo // prints a Hello... Hello.java:3:...s a Hello World! gre...-Hello.java:3:...orld! greeting publi...+Hello.java:3:...orld! greeting+Hello.java|4|{ public static void... Hello.java:4:{ public static void ma... Hello.java:4:... void main(String[] ... Hello.java:5:...n("Hello World!"); Binary file Hello.pdf matches-Hello.sh:1:#!/bin/sh "Hello World!"+Hello.sh:1:#!/bin/sh+Hello.sh|2|echo "Hello World!" Hello.sh:2:...ho "Hello World!"

tests/out/lorem.utf16-F-iwco.out

@@ -1 +1 @@-19+18

tests/out/lorem.utf16-G-iwco.out

@@ -1 +1 @@-19+18

tests/out/lorem.utf16-iwco.out

@@ -1 +1 @@-19+18

tests/out/lorem.utf32-F-iwco.out

@@ -1 +1 @@-19+18

tests/out/lorem.utf32-G-iwco.out

@@ -1 +1 @@-19+18

tests/out/lorem.utf32-iwco.out

@@ -1 +1 @@-19+18