/******************************************************************************\
* Copyright (c) 2019, Robert van Engelen, Genivia Inc. All rights reserved. *
* *
* Redistribution and use in source and binary forms, with or without *
* modification, are permitted provided that the following conditions are met: *
* *
* (1) Redistributions of source code must retain the above copyright notice, *
* this list of conditions and the following disclaimer. *
* *
* (2) Redistributions in binary form must reproduce the above copyright *
* notice, this list of conditions and the following disclaimer in the *
* documentation and/or other materials provided with the distribution. *
* *
* (3) The name of the author may not be used to endorse or promote products *
* derived from this software without specific prior written permission. *
* *
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED *
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF *
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO *
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, *
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, *
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; *
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, *
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR *
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF *
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *
\******************************************************************************/
/**
@file ugrep.cpp
@brief file pattern searcher
@author Robert van Engelen - engelen@genivia.com
@copyright (c) 2019-2026, Robert van Engelen, Genivia Inc. All rights reserved.
@copyright (c) BSD-3 License - see LICENSE.txt
User manual:
<https://ugrep.com>
Source code repository:
<https://github.com/Genivia/ugrep>
This program uses RE/flex:
<https://github.com/Genivia/RE-flex>
Optional libraries to support options -P and -z:
-P: PCRE2 (or Boost.Regex) recommended
-z: zlib (.gz) recommended
-z: bzip2 (.bz, bz2, .bzip2) recommended
-z: lzma (.lzma, .xz) recommended
-z: lz4 (.lz4) optional
-z: zstd (.zst, .zstd) optional
-z: brotli (.br) optional
-z: bzip3 (.bz3) optional
Build ugrep as follows:
$ ./configure
$ make -j
Git does not preserve time stamps so ./configure may fail, in that case do:
$ autoreconf -fi
$ ./configure
$ make -j
After this, you may want to test ugrep and install it (optional):
$ make test
$ sudo make install
*/
#include "ugrep.hpp"
#include "glob.hpp"
#include "mmap.hpp"
#include "output.hpp"
#include "query.hpp"
#include "stats.hpp"
#include <reflex/matcher.h>
#include <reflex/linematcher.h>
#include <reflex/fuzzymatcher.h>
#include <reflex/unicode.h>
#include <iomanip>
#include <cctype>
#include <limits>
#include <functional>
#include <list>
#include <deque>
#include <thread>
#include <memory>
#include <chrono>
#include <sstream>
#ifdef OS_WIN_OR_MINGW // compiling for a windows OS, but not Cygwin
// optionally enable --color=auto by default
// #define WITH_COLOR
// optionally enable PCRE2 for -P
// #define HAVE_PCRE2
// optionally enable Boost.Regex for -P
// #define HAVE_BOOST_REGEX
// optionally enable zlib for -z
// #define HAVE_LIBZ
// optionally enable libbz2 for -z
// #define HAVE_LIBBZ2
// optionally enable liblzma for -z
// #define HAVE_LIBLZMA
// optionally enable liblz4 for -z
// #define HAVE_LIBLZ4
// optionally enable libzstd for -z
// #define HAVE_LIBZSTD
// optionally enable brotli library for -z
// #define HAVE_LIBBROTLI
// optionally enable libbzip3 for -z
// #define HAVE_LIBBZIP3
#include <stringapiset.h> // internationalization
#include <direct.h> // directory access
#include <winsock.h> // gethostname() for --hyperlink
#include <securitybaseapi.h> // GetTokenInformation()
#include <aclapi.h> // GetSecurityInfo()
#ifdef OS_WIN
#pragma comment(lib, "Ws2_32.lib")
#endif
#else // not compiling for a windows OS or for windows OS
#include <signal.h>
#include <dirent.h>
#endif
// use PCRE2 for option -P
#ifdef HAVE_PCRE2
# include <reflex/pcre2matcher.h>
#else
// use Boost.Regex for option -P
# ifdef HAVE_BOOST_REGEX
# include <reflex/boostmatcher.h>
# endif
#endif
// optional: specify an optimal decompression block size, default is 65536, must be larger than 1024 for tar extraction
// #define Z_BUF_LEN 16384
// #define Z_BUF_LEN 32768
// use zlib, libbz2, liblzma etc for option -z
#ifdef HAVE_LIBZ
# define WITH_MAX_7ZIP_SIZE 1073741824 // 7zip expands archived files in memory, we should limit expansion up to 1GB
# include "zstream.hpp"
#endif
// ugrep exit codes
#define EXIT_OK 0 // One or more lines were selected
#define EXIT_FAIL 1 // No lines were selected
#define EXIT_ERROR 2 // An error occurred
// limit the total number of threads spawn (i.e. limit spawn overhead), because grepping is practically IO bound
#ifndef DEFAULT_MAX_JOBS
# define DEFAULT_MAX_JOBS 12
#endif
// default limit on the job queue size to wait for worker threads to finish more work
#ifndef DEFAULT_MAX_JOB_QUEUE_SIZE
# define DEFAULT_MAX_JOB_QUEUE_SIZE 8192
#endif
// a hard limit on the recursive search depth
#ifndef MAX_DEPTH
# define MAX_DEPTH 100
#endif
// --min-steal default, the minimum co-worker's queue size of pending jobs to steal a job from, smaller values result in higher job stealing rates, should not be less than 3
#ifndef MIN_STEAL
# define MIN_STEAL 3U
#endif
// use dirent d_type when available to improve performance
#ifdef HAVE_STRUCT_DIRENT_D_TYPE
# define DIRENT_TYPE_UNKNOWN DT_UNKNOWN
# define DIRENT_TYPE_LNK DT_LNK
# define DIRENT_TYPE_DIR DT_DIR
# define DIRENT_TYPE_REG DT_REG
#else
# define DIRENT_TYPE_UNKNOWN 0
# define DIRENT_TYPE_LNK 1
# define DIRENT_TYPE_DIR 1
# define DIRENT_TYPE_REG 1
#endif
// ANSI SGR substrings extracted from GREP_COLORS
char color_sl[COLORLEN]; // selected line
char color_cx[COLORLEN]; // context line
char color_mt[COLORLEN]; // matched text in any matched line
char color_ms[COLORLEN]; // matched text in a selected line
char color_mc[COLORLEN]; // matched text in a context line
char color_fn[COLORLEN]; // file name
char color_ln[COLORLEN]; // line number
char color_cn[COLORLEN]; // column number
char color_bn[COLORLEN]; // byte offset
char color_se[COLORLEN]; // separator
char color_qp[COLORLEN]; // TUI prompt
char color_qe[COLORLEN]; // TUI errors
char color_qr[COLORLEN]; // TUI regex highlight
char color_qm[COLORLEN]; // TUI regex meta characters highlight
char color_ql[COLORLEN]; // TUI regex lists and literals highlight
char color_qb[COLORLEN]; // TUI regex braces highlight
char match_ms[COLORLEN]; // --match or --tag: matched text in a selected line
char match_mc[COLORLEN]; // --match or --tag: matched text in a context line
char match_off[COLORLEN]; // --match or --tag: off
const char *color_hl = NULL; // hyperlink
const char *color_st = NULL; // ST
const char *color_del = ""; // erase line after the cursor
const char *color_off = ""; // disable colors
const char *color_high = ""; // stderr highlighted text
const char *color_error = ""; // stderr error text
const char *color_warning = ""; // stderr warning text
const char *color_message = ""; // stderr error or warning message text
#ifndef OS_WIN_OR_MINGW
// output file stat() result is available and S_ISREG (regular file)
bool Static::output_stat_result = false;
bool Static::output_stat_regular = false;
struct stat Static::output_stat;
// container of inodes to detect directory cycles when symlinks are traversed with --dereference
std::set<ino_t> visited;
// use statvfs() for --include-fs and --exclude-fs
#if defined(HAVE_STATVFS)
// containers of file system ids to exclude from recursive searches or include in recursive searches
std::set<uint64_t> exclude_fs_ids, include_fs_ids;
// the type of statvfs buffer to use
typedef struct statvfs stat_fs_t;
// wrapper to call statvfs()
inline int stat_fs(const char *path, stat_fs_t *buf)
{
return statvfs(path, buf);
}
#if defined(_AIX) || defined(__TOS_AIX__)
// NetBSD compatible struct fsid_t with unsigned integers f_fsid.val[2]
inline uint64_t fsid_to_uint64(fsid_t& fsid)
{
return static_cast<uint64_t>(fsid.val[0]) | (static_cast<uint64_t>(fsid.val[1]) << 32);
}
#else
// POSIX compliant unsigned integer f_fsid
template<typename T>
inline uint64_t fsid_to_uint64(T& fsid)
{
return static_cast<uint64_t>(fsid);
}
#endif
// if statvfs() is not available then use statfs() for --include-fs and --exclude-fs
#elif defined(HAVE_STATFS)
// containers of file system ids to exclude from recursive searches or include in recursive searches
std::set<uint64_t> exclude_fs_ids, include_fs_ids;
// the type of statfs buffer to use
typedef struct statfs stat_fs_t;
// wrapper to call statfs()
inline int stat_fs(const char *path, stat_fs_t *buf)
{
return statfs(path, buf);
}
inline uint64_t fsid_to_uint64(fsid_t& fsid)
{
return static_cast<uint64_t>(fsid.val[0]) | (static_cast<uint64_t>(fsid.val[1]) << 32);
}
#endif
#endif
// ugrep command-line options
bool flag_all_threads = false;
bool flag_any_line = false;
bool flag_best_match = false;
bool flag_bool = false;
bool flag_color_term = false;
bool flag_confirm = DEFAULT_CONFIRM;
bool flag_count = false;
bool flag_cpp = false;
bool flag_csv = false;
bool flag_decompress = false;
bool flag_dereference = false;
bool flag_dereference_files = false;
bool flag_files = false;
bool flag_files_with_matches = false;
bool flag_files_without_match = false;
bool flag_glob_ignore_case = false;
bool flag_grep = false;
bool flag_hex = false;
bool flag_hex_star = false;
bool flag_hex_cbr = true;
bool flag_hex_chr = true;
bool flag_hex_hbr = true;
bool flag_hidden = DEFAULT_HIDDEN;
bool flag_hyperlink_line = false;
bool flag_invert_match = false;
bool flag_json = false;
bool flag_line_buffered = false;
bool flag_line_regexp = false;
bool flag_match = false;
bool flag_multiline = false;
bool flag_no_dereference = false;
bool flag_no_filename = false;
bool flag_no_header = false;
bool flag_no_messages = false;
bool flag_not = false;
bool flag_null = false;
bool flag_null_data = false;
bool flag_only_line_number = false;
bool flag_only_matching = false;
bool flag_query = false;
bool flag_quiet = false;
bool flag_sort_rev = false;
bool flag_split = false;
bool flag_stdin = false;
bool flag_tty_term = false;
bool flag_usage_warnings = false;
bool flag_word_regexp = false;
bool flag_xml = false;
bool flag_with_hex = false;
bool flag_with_filename = false;
Flag flag_binary;
Flag flag_binary_without_match;
Flag flag_break;
Flag flag_byte_offset;
Flag flag_column_number;
Flag flag_empty;
Flag flag_dotall;
Flag flag_free_space;
Flag flag_heading;
Flag flag_ignore_case;
Flag flag_initial_tab;
Flag flag_line_number;
Flag flag_smart_case;
Flag flag_text;
Flag flag_tree;
Flag flag_ungroup;
Sort flag_sort_key = Sort::NA;
Action flag_devices_action = Action::UNSP;
Action flag_directories_action = Action::UNSP;
Mode flag_mode = Mode::UNSP;
size_t flag_after_context = 0;
size_t flag_before_context = 0;
size_t flag_delay = DEFAULT_QUERY_DELAY;
size_t flag_exclude_iglob_size = 0;
size_t flag_exclude_iglob_dir_size = 0;
size_t flag_fuzzy = 0;
size_t flag_hex_after = 0;
size_t flag_hex_before = 0;
size_t flag_hex_columns = 16;
size_t flag_include_iglob_size = 0;
size_t flag_include_iglob_dir_size = 0;
size_t flag_jobs = 0;
size_t flag_max_count = 0;
size_t flag_max_depth = 0;
size_t flag_max_files = 0;
size_t flag_max_line = 0;
size_t flag_max_size = 0;
size_t flag_max_mmap = DEFAULT_MAX_MMAP_SIZE;
size_t flag_max_queue = DEFAULT_MAX_JOB_QUEUE_SIZE;
size_t flag_min_count = 0;
size_t flag_min_depth = 0;
size_t flag_min_line = 0;
size_t flag_min_size = 0;
size_t flag_min_magic = 1;
size_t flag_min_steal = MIN_STEAL;
size_t flag_not_magic = 0;
size_t flag_tabs = DEFAULT_TABS;
size_t flag_width = 0;
size_t flag_zmax = 1;
const char *flag_binary_files = "binary";
const char *flag_color = DEFAULT_COLOR;
const char *flag_color_query = NULL;
const char *flag_colors = NULL;
const char *flag_config = NULL;
const char *flag_devices = NULL;
const char *flag_directories = NULL;
const char *flag_encoding = NULL;
const char *flag_format = NULL;
const char *flag_format_begin = NULL;
const char *flag_format_close = NULL;
const char *flag_format_end = NULL;
const char *flag_format_open = NULL;
const char *flag_group_separator = "--";
const char *flag_hexdump = NULL;
const char *flag_hyperlink = NULL;
const char *flag_index = NULL;
const char *flag_label = Static::LABEL_STANDARD_INPUT;
const char *flag_pager = NULL;
const char *flag_pretty = DEFAULT_PRETTY;
const char *flag_replace = NULL;
const char *flag_save_config = NULL;
const char *flag_separator = NULL;
const char *flag_separator_dash = "-";
const char *flag_separator_bar = "|";
const char *flag_sort = NULL;
const char *flag_stats = NULL;
const char *flag_tag = NULL;
const char *flag_view = "";
std::string flag_filter;
std::string flag_hyperlink_prefix;
std::string flag_hyperlink_host;
std::string flag_hyperlink_path;
std::string flag_regexp;
std::set<std::string> flag_config_files;
std::set<std::string> flag_ignore_files;
std::vector<std::string> flag_file;
std::vector<std::string> flag_file_type;
std::vector<std::string> flag_file_extension;
std::vector<std::string> flag_file_magic;
std::vector<std::string> flag_filter_magic_label;
std::vector<std::string> flag_from;
std::vector<std::string> flag_glob;
std::vector<std::string> flag_iglob;
std::vector<std::string> flag_include;
std::vector<std::string> flag_include_dir;
std::vector<std::string> flag_include_from;
std::vector<std::string> flag_include_fs;
std::vector<std::string> flag_exclude;
std::vector<std::string> flag_exclude_dir;
std::vector<std::string> flag_exclude_from;
std::vector<std::string> flag_exclude_fs;
std::vector<std::string> flag_all_include;
std::vector<std::string> flag_all_include_dir;
std::vector<std::string> flag_all_exclude;
std::vector<std::string> flag_all_exclude_dir;
reflex::Input::file_encoding_type flag_encoding_type = reflex::Input::file_encoding::plain;
// store string arguments and the UTF-8 arguments decoded from wargv[] in strings to re-populate argv[] with pointers
std::list<std::string> arg_strings;
// helper function protos
void options(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int argc, const char **argv);
void option_regexp(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg, bool is_neg = false);
void option_and(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int& i, int argc, const char **argv);
void option_and(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg);
void option_andnot(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int& i, int argc, const char **argv);
void option_andnot(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg);
void option_not(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int& i, int argc, const char **argv);
void option_not(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg);
void option_all_files();
void init(int argc, const char **argv);
void set_color(const char *colors, const char *parameter, char color[COLORLEN]);
void trim(std::string& line);
void trim_pathname_arg(const char *arg);
bool is_output(ino_t inode);
const char *getoptarg(int argc, const char **argv, const char *arg, int& i);
const char *getloptarg(int argc, const char **argv, const char *arg, int& i);
const char *strarg(const char *string);
size_t strtonum(const char *string, const char *message);
size_t strtopos(const char *string, const char *message);
void strtopos2(const char *string, size_t& pos1, size_t& pos2, const char *message);
size_t strtosize(const char *string, const char *message);
size_t strtofuzzy(const char *string, const char *message);
void import_globs(FILE *file, std::vector<std::string>& files, std::vector<std::string>& dirs, bool gitignore = false, const char *pathname = NULL);
void usage(const char *message, const char *arg = NULL, const char *valid = NULL);
void help(std::ostream& out);
void help(const char *what = NULL);
void version();
void is_directory(const char *pathname);
void cannot_decompress(const char *pathname, const char *message);
void open_pager();
void close_pager();
#ifdef OS_WIN_OR_MINGW
// CTRL-C handler
BOOL WINAPI sigint(DWORD signal)
{
if (signal == CTRL_C_EVENT || signal == CTRL_BREAK_EVENT)
{
// be nice, reset colors on interrupt when sending output to a color TTY
if (flag_color_term)
flag_color_term = write(1, "\033[m", 3) > 0; // appease -Wunused-result
close_pager();
}
// return FALSE to invoke the next handler (when applicable) or just exit
return FALSE;
}
#else
// SIGINT and SIGTERM handler
static void sigint(int sig)
{
// reset to the default handler
signal(sig, SIG_DFL);
// be nice, reset colors on interrupt when sending output to a color TTY
if (flag_color_term)
flag_color_term = write(1, "\033[m", 3) > 0; // appease -Wunused-result
close_pager();
// signal again
kill(getpid(), sig);
}
#endif
// set this thread's affinity and priority, if supported by the OS, ignore errors to leave scheduling to the OS
static void set_this_thread_affinity_and_priority(size_t cpu)
{
// set affinity
#if defined(OS_WIN_OR_MINGW) || defined(__CYGWIN__)
(void)SetThreadAffinityMask(GetCurrentThread(), DWORD_PTR(1) << cpu);
#elif defined(__APPLE__) && defined(HAVE_PTHREAD_SET_QOS_CLASS_SELF_NP)
(void)pthread_set_qos_class_self_np(QOS_CLASS_USER_INITIATED, 0);
#elif defined(__FreeBSD__) && defined(HAVE_CPUSET_SETAFFINITY)
cpuset_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
(void)cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
#elif defined(__DragonFly__) && defined(HAVE_PTHREAD_SETAFFINITY_NP)
cpuset_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
(void)pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
#elif defined(__NetBSD__) && defined(HAVE_PTHREAD_SETAFFINITY_NP)
cpuset_t *cpuset = cpuset_create();
cpuset_set(cpu, cpuset);
(void)pthread_setaffinity_np(pthread_self(), cpuset_size(cpuset), cpuset);
cpuset_destroy(cpuset);
#elif defined(HAVE_SCHED_SETAFFINITY)
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
(void)sched_setaffinity(0, sizeof(cpuset), &cpuset);
#elif defined(HAVE_CPUSET_SETAFFINITY)
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
(void)cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(cpuset), &cpuset);
#elif defined(HAVE_PTHREAD_SETAFFINITY_NP)
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(cpu, &cpuset);
(void)pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
#endif
// set priority
#if defined(OS_WIN_OR_MINGW) || defined(__CYGWIN__)
(void)SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST);
#elif defined(__APPLE__)
(void)setpriority(PRIO_DARWIN_THREAD, 0, -20);
#elif defined(HAVE_PTHREAD_SETSCHEDPRIO)
(void)pthread_setschedprio(pthread_self(), -20);
#elif defined(HAVE_SETPRIORITY)
(void)setpriority(PRIO_PROCESS, 0, -20);
#endif
(void)cpu;
}
// open a pager if output is to a TTY then page through the results
void open_pager()
{
if (flag_pager != NULL)
{
// --pager without argument uses $PAGER or DEFAULT_PAGER_COMMAND
if (*flag_pager == '\0')
{
const char *pager = getenv("PAGER");
if (pager != NULL && *pager != '\0')
flag_pager = pager;
else
flag_pager = DEFAULT_PAGER_COMMAND;
}
// pager 'less' requires -R
if (strcmp(flag_pager, "less") == 0)
flag_pager = "less -R";
// open a pipe to a forked pager
#ifdef OS_WIN
Static::output = popen(flag_pager, "wb");
#else
Static::output = popen(flag_pager, "w");
#endif
if (Static::output == NULL)
error("cannot open pipe to pager", flag_pager);
Static::errout = Static::output;
// enable --heading if not explicitly disabled (enables --break later)
if (flag_heading.is_undefined())
flag_heading = true;
// enable --line-buffered to flush output to the pager immediately
flag_line_buffered = true;
}
}
// close the pipe to the forked pager
void close_pager()
{
if (flag_pager != NULL && Static::output != NULL && Static::output != stdout)
pclose(Static::output);
}
// open a file where - means stdin (mode "r") or stdout (mode "a" or "w") and an initial ~ expands to home directory
int fopen_smart(FILE **file, const char *filename, const char *mode)
{
*file = NULL;
if (filename == NULL || *filename == '\0')
return errno = ENOENT;
if (strcmp(filename, "-") == 0)
{
*file = strchr(mode, 'r') != NULL ? stdin : stdout;
return 0;
}
if (*filename == '~')
return fopenw_s(file, std::string(Static::home_dir).append(filename + 1).c_str(), mode);
return fopenw_s(file, filename, mode);
}
// read a line from buffered input, returns true when eof
inline bool getline(reflex::BufferedInput& input, std::string& line)
{
int ch;
line.erase();
while ((ch = input.get()) != EOF && ch != '\n' && ch != '\0')
line.push_back(ch);
if (!line.empty() && line.back() == '\r')
line.pop_back();
return ch == EOF && line.empty();
}
// return true if s[0..n-1] contains a \0 (NUL) or a non-displayable invalid UTF-8 encoding
inline bool is_binary(const char *s, size_t n)
{
// not --null-data or --encoding=null-data that permit NUL in the input and non-UTF-8 like GNU grep
if (flag_encoding_type == reflex::Input::file_encoding::null_data)
return false;
// not -a and -U or -W: file is binary if it has a \0 (NUL) or an invalid UTF-8 encoding when --encoding is unset
if (!flag_text && (!flag_binary || flag_with_hex) && flag_encoding_type == reflex::Input::file_encoding::plain)
return !reflex::isutf8(s, s + n);
// otherwise, file is binary if it contains a \0 (NUL) which is what GNU grep checks
return memchr(s, '\0', n) != NULL;
}
// return true if -X or if -W and match s[0..n-1] is binary
inline bool check_binary(const char *s, size_t n)
{
return (flag_hex || (flag_with_hex && is_binary(s, n)));
}
// check if a file's inode is the current output file, to avoid searching the file we write to
inline bool is_output(ino_t inode)
{
#ifdef OS_WIN_OR_MINGW
(void)inode; // appease -Wunused
return false; // TODO check that two FILE* on Windows are the same, is this possible?
#else
return Static::output_stat_regular && inode == Static::output_stat.st_ino;
#endif
}
// specify a line of input for the matcher to read, matcher must not use text() or rest() to keep the line contents unmodified
inline void read_line(reflex::AbstractMatcher *matcher, const char *line, size_t size)
{
// safe cast: buffer() is read-only if no matcher.text() and matcher.rest() are used, size + 1 to include final \0
matcher->buffer(const_cast<char*>(line), size + 1);
}
// specify a line of input for the matcher to read, matcher must not use text() or rest() to keep the line contents unmodified
inline void read_line(reflex::AbstractMatcher *matcher, const std::string& line)
{
// safe cast: buffer() is read-only if no matcher.text() and matcher.rest() are used, size + 1 to include final \0
matcher->buffer(const_cast<char*>(line.c_str()), line.size() + 1);
}
// copy color buffers
inline void copy_color(char to[COLORLEN], const char from[COLORLEN])
{
size_t len = std::min<size_t>(strlen(from), COLORLEN - 1);
memcpy(to, from, len);
to[len] = '\0';
char *comma = strchr(to, ',');
if (comma != NULL)
*comma = '\0';
}
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
// decompression thread state with shared objects (similar code to zthread.hpp, but with ugrep-related mods)
struct Zthread {
Zthread(bool is_chained, std::string& partname) :
ztchain(NULL),
zstream(NULL),
zpipe_in(NULL),
is_chained(is_chained),
quit(false),
stop(false),
is_extracting(false),
is_waiting(false),
is_assigned(false),
partnameref(partname),
findpart(NULL)
{
pipe_fd[0] = -1;
pipe_fd[1] = -1;
}
~Zthread()
{
// recursively join all stages of the decompression thread chain (--zmax>1), delete zstream
join();
// delete the decompression chain (--zmax>1)
if (ztchain != NULL)
{
delete ztchain;
ztchain = NULL;
}
}
// start decompression thread if not running, open new pipe, returns pipe or NULL on failure, this function is called by the main thread
FILE *start(size_t ztstage, const char *pathname, FILE *file_in, const char *find = NULL)
{
// return pipe
FILE *pipe_in = NULL;
// reset pipe descriptors, pipe is closed
pipe_fd[0] = -1;
pipe_fd[1] = -1;
// partnameref is not assigned yet, used only when this decompression thread is chained
is_assigned = false;
// if there is a specific part to search in a (nested) archive, NULL otherwise
findpart = find;
// a colon separator means we should find the part in a nested archive corresponding to a decompressor depth (stage)
if (findpart != NULL)
{
for (size_t strip_prefix = 1; strip_prefix < ztstage; ++strip_prefix)
{
const char *colon = strchr(findpart, ':');
if (colon == NULL)
break;
// this nested archive part to find is after the colon
findpart = colon + 1;
}
}
// open pipe between Grep (or previous decompression) thread and this (new) decompression thread
if (pipe(pipe_fd) == 0 && (pipe_in = fdopen(pipe_fd[0], "rb")) != NULL)
{
// recursively add decompression stages to decompress multi-compressed files
if (ztstage > 1)
{
// create a new decompression chain if not already created
if (ztchain == NULL)
ztchain = new Zthread(true, partname);
// close the input pipe from the next decompression stage in the chain, if still open
if (zpipe_in != NULL)
{
fclose(zpipe_in);
zpipe_in = NULL;
}
// start the next stage in the decompression chain, return NULL if failed
zpipe_in = ztchain->start(ztstage - 1, pathname, file_in, find);
if (zpipe_in == NULL)
return NULL;
// wait for the partname to be assigned by the next decompression thread in the decompression chain
std::unique_lock<std::mutex> lock(ztchain->pipe_mutex);
if (!ztchain->is_assigned)
ztchain->part_ready.wait(lock);
lock.unlock();
// create or open a zstreambuf to (re)start the decompression thread, reading from zpipe_in from the next stage in the chain
if (zstream == NULL)
zstream = new zstreambuf(partname.c_str(), zpipe_in);
else
zstream->open(partname.c_str(), zpipe_in);
}
else
{
// create or open a zstreambuf to (re)start the decompression thread, reading from the source input
if (zstream == NULL)
zstream = new zstreambuf(pathname, file_in);
else
zstream->open(pathname, file_in);
}
if (thread.joinable())
{
// wake decompression thread waiting in close_wait_zstream_open(), there is work to do
pipe_zstrm.notify_one();
}
else
{
// start a new decompression thread
try
{
// reset flags
quit = false;
stop = false;
is_extracting = false;
is_waiting = false;
thread = std::thread(&Zthread::decompress, this);
}
catch (std::system_error&)
{
// thread creation failed
fclose(pipe_in);
close(pipe_fd[1]);
pipe_fd[0] = -1;
pipe_fd[1] = -1;
warning("cannot create thread to decompress", pathname);
return NULL;
}
}
}
else
{
// pipe failed
if (pipe_fd[0] != -1)
{
close(pipe_fd[0]);
close(pipe_fd[1]);
pipe_fd[0] = -1;
pipe_fd[1] = -1;
}
warning("cannot create pipe to decompress", pathname);
return NULL;
}
return pipe_in;
}
// open pipe to the next file or part in the archive or return NULL, this function is called by the main thread or by the previous decompression thread
FILE *open_next(const char *pathname)
{
if (pipe_fd[0] != -1)
{
// our end of the pipe was closed earlier, before open_next() was called
pipe_fd[0] = -1;
// if extracting and the decompression filter thread is not yet waiting, then wait until decompression thread closed its end of the pipe
std::unique_lock<std::mutex> lock(pipe_mutex);
if (!is_waiting)
pipe_close.wait(lock);
lock.unlock();
// partnameref is not assigned yet, used only when this decompression thread is chained
is_assigned = false;
// extract the next file from the archive when applicable, e.g. zip format
if (is_extracting)
{
FILE *pipe_in = NULL;
// open pipe between worker and decompression thread, then start decompression thread
if (pipe(pipe_fd) == 0 && (pipe_in = fdopen(pipe_fd[0], "rb")) != NULL)
{
if (is_chained)
{
// wait for partname ready
lock.lock();
// notify the decompression filter thread of the new pipe
pipe_ready.notify_one();
// wait for the partname to be set by the next decompression thread in the ztchain
if (!is_assigned)
part_ready.wait(lock);
lock.unlock();
}
else
{
// notify the decompression filter thread of the new pipe
pipe_ready.notify_one();
}
return pipe_in;
}
// failed to create a new pipe
warning("cannot create pipe to decompress", is_chained ? NULL : pathname);
if (pipe_fd[0] != -1)
{
close(pipe_fd[0]);
close(pipe_fd[1]);
}
// reset pipe descriptors, pipe was closed
pipe_fd[0] = -1;
pipe_fd[1] = -1;
// notify the decompression thread filter_tar/filter_cpio of the closed pipe
pipe_ready.notify_one();
// when an error occurred, we still need to notify the receiver in case it is waiting on the partname
lock.lock();
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
}
return NULL;
}
// cancel decompression gracefully
void cancel()
{
stop = true;
// recursively cancel decompression threads in the chain
if (ztchain != NULL)
ztchain->cancel();
}
// join this thread, this function is called by the main thread
void join()
{
// --zmax>1: recursively join all stages of the decompression thread chain
if (ztchain != NULL)
ztchain->join();
if (thread.joinable())
{
std::unique_lock<std::mutex> lock(pipe_mutex);
// decompression thread should quit to join
quit = true;
if (!is_waiting)
{
// wait until decompression thread closes the pipe
pipe_close.wait(lock);
}
else
{
// wake decompression thread waiting in close_wait_zstream_open(), there is no more work to do
pipe_zstrm.notify_one();
}
lock.unlock();
// now wait for the decomprssion thread to join
thread.join();
}
// we can release the zstream that is no longer needed
if (zstream != NULL)
{
delete zstream;
zstream = NULL;
}
}
// if the pipe was closed, then wait until the main thread opens a new pipe to search the next file or part in an archive
bool wait_pipe_ready()
{
if (pipe_fd[1] == -1)
{
// signal close and wait until a new zstream pipe is ready
std::unique_lock<std::mutex> lock(pipe_mutex);
pipe_close.notify_one();
is_waiting = true;
pipe_ready.wait(lock);
is_waiting = false;
lock.unlock();
// the receiver did not create a new pipe in close_file()
if (pipe_fd[1] == -1)
return false;
}
return true;
}
// close the pipe and wait until the main thread opens a new zstream and pipe for the next decompression job, unless quitting
void close_wait_zstream_open()
{
if (pipe_fd[1] != -1)
{
// close our end of the pipe
close(pipe_fd[1]);
pipe_fd[1] = -1;
}
// signal close and wait until zstream is open
std::unique_lock<std::mutex> lock(pipe_mutex);
pipe_close.notify_one();
if (!quit)
{
is_waiting = true;
pipe_zstrm.wait(lock);
is_waiting = false;
}
lock.unlock();
}
// decompression thread execution
void decompress()
{
while (!quit)
{
// use the zstreambuf internal buffer to hold decompressed data
unsigned char *buf;
size_t maxlen;
zstream->get_buffer(buf, maxlen);
// reset flags
is_extracting = false;
is_waiting = false;
// extract the parts of a zip file, one by one, if zip file detected
while (!stop)
{
// to hold the path (prefix + name) extracted from a zip file
std::string path;
// a regular file, may be reset when unzipping a directory
bool is_regular = true;
const zstreambuf::ZipInfo *zipinfo = zstream->zipinfo();
if (zipinfo != NULL)
{
// extracting a zip file
is_extracting = true;
if (!zipinfo->name.empty() && zipinfo->name.back() == '/')
{
// skip zip directories
is_regular = false;
}
else
{
// save the zip path (prefix + name), since zipinfo will become invalid
path.assign(zipinfo->name);
// produce headers with zip file pathnames for each archived part (Grep::partname)
if (!flag_no_filename)
flag_no_header = false;
}
}
bool is_selected = false;
// decompress a block of data into the buffer
std::streamsize len = zstream->decompress(buf, maxlen);
if (len >= 0)
{
is_selected = true;
if (!filter_tar(path, buf, maxlen, len, is_selected) &&
!filter_cpio(path, buf, maxlen, len, is_selected))
{
// not a tar/cpio file, decompress the data into pipe, if not unzipping or if zipped file meets selection criteria
is_selected = is_regular && (zipinfo == NULL || select_matching(NULL, path.c_str(), buf, static_cast<size_t>(len), true));
if (is_selected)
{
// if pipe is closed, then wait until receiver reopens it, break if failed
if (!wait_pipe_ready())
{
// close the input pipe from the next decompression chain stage
if (ztchain != NULL && zpipe_in != NULL)
{
fclose(zpipe_in);
zpipe_in = NULL;
}
break;
}
// assign the Grep::partname (synchronized on pipe_mutex and pipe), before sending to the new pipe
if (ztchain == NULL)
partnameref.assign(std::move(path));
else if (path.empty())
partnameref.assign(partname);
else
partnameref.assign(partname).append(":").append(std::move(path));
// notify the receiver of the new partname
if (is_chained)
{
std::unique_lock<std::mutex> lock(pipe_mutex);
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
}
// push decompressed data into pipe
bool drain = false;
while (len > 0 && !stop)
{
// write buffer data to the pipe, if the pipe is broken then the receiver is waiting for this thread to join so we drain the rest of the decompressed data
if (is_selected && !drain && write(pipe_fd[1], buf, static_cast<size_t>(len)) < len)
{
// if no next decompression thread and decompressing a single file (not zip), then stop immediately
if (ztchain == NULL && zipinfo == NULL)
break;
drain = true;
}
// decompress the next block of data into the buffer
len = zstream->decompress(buf, maxlen);
}
}
}
// break if not unzipping or if no more files to unzip
if (zstream->zipinfo() == NULL)
{
// no decompression chain
if (ztchain == NULL)
break;
// close the input pipe from the next decompression chain stage
if (zpipe_in != NULL)
{
fclose(zpipe_in);
zpipe_in = NULL;
}
// open pipe to the next file in an archive if there is a next file to extract
zpipe_in = ztchain->open_next(partname.c_str());
if (zpipe_in == NULL)
break;
// open a zstreambuf to (re)start the decompression thread
zstream->open(partname.c_str(), zpipe_in);
}
// extracting a file
is_extracting = true;
// after extracting files from an archive, close our end of the pipe and loop for the next file
if (is_selected && pipe_fd[1] != -1)
{
close(pipe_fd[1]);
pipe_fd[1] = -1;
}
}
is_extracting = false;
// when an error occurred or nothing was selected, then we still need to notify the receiver in case it is waiting on the partname
if (is_chained)
{
std::unique_lock<std::mutex> lock(pipe_mutex);
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
// close the pipe and wait until zstream pipe is open, unless quitting
close_wait_zstream_open();
}
}
// if tar/pax file, extract regular file contents and push into pipes one by one, return true when done
bool filter_tar(const std::string& archive, unsigned char *buf, size_t maxlen, std::streamsize len, bool& is_selected)
{
const int BLOCKSIZE = 512;
if (len > BLOCKSIZE)
{
// v7 and ustar formats
const char ustar_magic[8] = { 'u', 's', 't', 'a', 'r', 0, '0', '0' };
bool is_ustar = *buf != '\0' && memcmp(buf + 257, ustar_magic, 8) == 0;
// gnu and oldgnu formats
const char gnutar_magic[8] = { 'u', 's', 't', 'a', 'r', ' ', ' ', 0 };
bool is_gnutar = *buf != '\0' && memcmp(buf + 257, gnutar_magic, 8) == 0;
// is this a tar/pax archive?
if (is_ustar || is_gnutar)
{
// produce headers with tar file pathnames for each archived part (Grep::partname)
if (!flag_no_filename)
flag_no_header = false;
// inform the main grep thread we are extracting an archive
is_extracting = true;
// to hold the path (prefix + name) extracted from the header
std::string path;
// to hold long path extracted from the previous header block that is marked with typeflag 'x' or 'L'
std::string long_path;
while (!stop)
{
// tar header fields name, size and prefix and make them \0-terminated by overwriting fields we do not use
buf[100] = '\0';
const char *name = reinterpret_cast<const char*>(buf);
// path prefix is up to 155 bytes (ustar) or up to 131 bytes (gnutar)
buf[345 + (is_ustar ? 155 : 131)] = '\0';
const char *prefix = reinterpret_cast<const char*>(buf + 345);
// check gnutar size with leading byte 0x80 (unsigned positive) or leading byte 0xff (negative)
uint64_t size = 0;
if (buf[124] == 0x80)
{
// 11 byte big-endian size field without the leading 0x80
for (short i = 125; i < 136; ++i)
size = (size << 8) + buf[i];
}
else if (buf[124] == 0xff)
{
// a negative size makes no sense, but let's not ignore it and cast to unsigned
for (short i = 124; i < 136; ++i)
size = (size << 8) + buf[i];
}
else
{
// v7/ustar size stored in 11 octal digits padded with leading zeros or spaces followed by a space or NUL
char *octal = reinterpret_cast<char*>(buf + 124);
buf[135] = '\0';
size = strtoull(octal, &octal, 8);
// we could be festidious about zero size when tar/pax is corrupted,
// but it's not an issue to report it as a "binary file" mess when that happens
// if (size == 0 && *octal != ' ' && *octal != '\0')
// break;
}
// header types
unsigned char typeflag = buf[156];
bool is_regular = typeflag == '0' || typeflag == '\0';
bool is_xhd = typeflag == 'x';
bool is_extended = typeflag == 'L';
// padding size
int padding = (BLOCKSIZE - size % BLOCKSIZE) % BLOCKSIZE;
// assign the (long) tar pathname, name and prefix are now \0-terminated
path.clear();
if (long_path.empty())
{
if (*prefix != '\0')
{
path.assign(prefix);
path.push_back('/');
}
path.append(name);
}
else
{
path.assign(std::move(long_path));
}
// remove header to advance to the body
len -= BLOCKSIZE;
memmove(buf, buf + BLOCKSIZE, static_cast<size_t>(len));
// check if archived file meets selection criteria
size_t minlen = static_cast<size_t>(std::min<uint64_t>(len, size)); // size_t is OK: len is streamsize but non-negative
is_selected = select_matching(archive.c_str(), path.c_str(), buf, minlen, is_regular);
// if extended headers are present
if (is_xhd)
{
// typeflag 'x': extract the long path from the pax extended header block in the body
const char *body = reinterpret_cast<const char*>(buf);
const char *end = body + minlen;
const char *key = "path=";
const char *str = std::search(body, end, key, key + 5);
if (str != NULL)
{
end = static_cast<const char*>(memchr(str, '\n', end - str));
if (end != NULL)
long_path.assign(str + 5, end - str - 5);
}
}
else if (is_extended)
{
// typeflag 'L': get long name from the body
const char *body = reinterpret_cast<const char*>(buf);
long_path.assign(body, strnlen(body, minlen));
}
// if the pipe is closed, then get a new pipe to search the next part in the archive
if (is_selected)
{
// if pipe is closed, then wait until receiver reopens it, break if failed
if (!wait_pipe_ready())
break;
// assign the Grep::partname (synchronized on pipe_mutex and pipe), before sending to the (new) pipe
if (ztchain != NULL)
{
if (!archive.empty())
partnameref.assign(partname).append(":").append(archive).append(":").append(std::move(path));
else
partnameref.assign(partname).append(":").append(std::move(path));
}
else
{
if (!archive.empty())
partnameref.assign(archive).append(":").append(std::move(path));
else
partnameref.assign(std::move(path));
}
// notify the receiver of the new partname after wait_pipe_ready()
if (is_chained)
{
std::unique_lock<std::mutex> lock(pipe_mutex);
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
}
// it is ok to push the body into the pipe for the main thread to search
bool ok = is_selected;
while (len > 0 && !stop)
{
size_t len_out = static_cast<size_t>(std::min<uint64_t>(len, size)); // size_t is OK: len is streamsize but non-negative
if (ok)
{
// write decompressed data to the pipe, if the pipe is broken then stop pushing more data into this pipe
if (write(pipe_fd[1], buf, len_out) < static_cast<ssize_t>(len_out))
ok = false;
}
size -= len_out;
// reached the end of the tar body?
if (size == 0)
{
len -= len_out;
memmove(buf, buf + len_out, static_cast<size_t>(len)); // size_t is OK: len is streamsize but non-negative
break;
}
// decompress the next block of data into the buffer
len = zstream->decompress(buf, maxlen);
}
// error?
if (len < 0 || stop)
break;
// fill the rest of the buffer with decompressed data
while (len < BLOCKSIZE || static_cast<size_t>(len) < maxlen)
{
std::streamsize len_in = zstream->decompress(buf + len, maxlen - static_cast<size_t>(len));
// error or EOF?
if (len_in <= 0)
break;
len += len_in;
}
// skip padding
if (len > padding)
{
len -= padding;
memmove(buf, buf + padding, static_cast<size_t>(len));
}
// rest of the file is too short, something is wrong
if (len <= BLOCKSIZE)
break;
// no more parts to extract?
if (*buf == '\0' || (memcmp(buf + 257, ustar_magic, 8) != 0 && memcmp(buf + 257, gnutar_magic, 8) != 0))
break;
// get a new pipe to search the next part in the archive, if the previous part was a regular file
if (is_selected)
{
// close our end of the pipe
close(pipe_fd[1]);
pipe_fd[1] = -1;
is_selected = false;
}
}
// if we're stopping we still need to notify the receiver in case it is waiting on the partname
if (is_chained)
{
std::unique_lock<std::mutex> lock(pipe_mutex);
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
// done extracting the tar file
return true;
}
}
// not a tar file
return false;
}
// if cpio file, extract regular file contents and push into pipes one by one, return true when done
bool filter_cpio(const std::string& archive, unsigned char *buf, size_t maxlen, std::streamsize len, bool& is_selected)
{
const int HEADERSIZE = 110;
if (len > HEADERSIZE)
{
// cpio odc format
const char odc_magic[6] = { '0', '7', '0', '7', '0', '7' };
// cpio newc format
const char newc_magic[6] = { '0', '7', '0', '7', '0', '1' };
// cpio newc+crc format
const char newc_crc_magic[6] = { '0', '7', '0', '7', '0', '2' };
// is this a cpio archive?
if (memcmp(buf, odc_magic, 6) == 0 || memcmp(buf, newc_magic, 6) == 0 || memcmp(buf, newc_crc_magic, 6) == 0)
{
// produce headers with cpio file pathnames for each archived part (Grep::partname)
if (!flag_no_filename)
flag_no_header = false;
// inform the main grep thread we are extracting an archive
is_extracting = true;
// to hold the path (prefix + name) extracted from the header
std::string path;
// need a new pipe, close current pipe first to create a new pipe
bool in_progress = false;
while (!stop)
{
// true if odc format, false if newc format
bool is_odc = buf[5] == '7';
// odc header length is 76, newc header length is 110
int header_len = is_odc ? 76 : 110;
char tmp[16];
char *rest = tmp;
// get the namesize
size_t namesize;
if (is_odc)
{
memcpy(tmp, buf + 59, 6);
tmp[6] = '\0';
namesize = strtoul(tmp, &rest, 8);
}
else
{
memcpy(tmp, buf + 94, 8);
tmp[8] = '\0';
namesize = strtoul(tmp, &rest, 16);
}
// if not a valid mode value, then something is wrong
if (*rest != '\0')
{
// data was read, stop reading more
if (in_progress)
break;
// assume this is not a cpio file and return false
return false;
}
// pathnames with trailing \0 cannot be empty or too large
if (namesize <= 1 || namesize >= 65536)
break;
// get the filesize
size_t filesize;
if (is_odc)
{
memcpy(tmp, buf + 65, 11);
tmp[11] = '\0';
filesize = strtoul(tmp, &rest, 8);
}
else
{
memcpy(tmp, buf + 54, 8);
tmp[8] = '\0';
filesize = strtoul(tmp, &rest, 16);
}
// if not a valid mode value, then something is wrong
if (*rest != '\0')
{
// data was read, stop reading more
if (in_progress)
break;
// assume this is not a cpio file and return false
return false;
}
// true if this is a regular file when (mode & 0170000) == 0100000
bool is_regular;
if (is_odc)
{
memcpy(tmp, buf + 18, 6);
tmp[6] = '\0';
is_regular = (strtoul(tmp, &rest, 8) & 0170000) == 0100000;
}
else
{
memcpy(tmp, buf + 14, 8);
tmp[8] = '\0';
is_regular = (strtoul(tmp, &rest, 16) & 0170000) == 0100000;
}
// if not a valid mode value, then something is wrong
if (*rest != '\0')
{
// data was read, stop reading more
if (in_progress)
break;
// assume this is not a cpio file and return false
return false;
}
// remove header to advance to the body
len -= header_len;
memmove(buf, buf + header_len, static_cast<size_t>(len));
// assign the cpio pathname
path.clear();
size_t size = namesize;
while (len > 0 && !stop)
{
size_t n = std::min<size_t>(len, size);
char *b = reinterpret_cast<char*>(buf);
path.append(b, n);
size -= n;
if (size == 0)
{
// remove pathname to advance to the body
len -= n;
memmove(buf, buf + n, static_cast<size_t>(len));
break;
}
// decompress the next block of data into the buffer
len = zstream->decompress(buf, maxlen);
}
// error?
if (len < 0 || stop)
break;
// remove trailing \0
if (path.back() == '\0')
path.pop_back();
// reached the end of the cpio archive?
if (path == "TRAILER!!!")
break;
// fill the rest of the buffer with decompressed data
if (static_cast<size_t>(len) < maxlen)
{
std::streamsize len_in = zstream->decompress(buf + len, maxlen - static_cast<size_t>(len));
// error?
if (len_in < 0)
break;
len += len_in;
}
// skip newc format \0 padding after the pathname
if (!is_odc && len > 3)
{
size_t n = 4 - (110 + namesize) % 4;
len -= n;
memmove(buf, buf + n, static_cast<size_t>(len));
}
// check if archived file meets selection criteria
size_t minlen = std::min<size_t>(len, filesize);
is_selected = select_matching(archive.c_str(), path.c_str(), buf, minlen, is_regular);
// if the pipe is closed, then get a new pipe to search the next part in the archive
if (is_selected)
{
// if pipe is closed, then wait until receiver reopens it, break if failed
if (!wait_pipe_ready())
break;
// assign the Grep::partname (synchronized on pipe_mutex and pipe), before sending to the (new) pipe
if (ztchain != NULL)
{
if (!archive.empty())
partnameref.assign(partname).append(":").append(archive).append(":").append(std::move(path));
else
partnameref.assign(partname).append(":").append(std::move(path));
}
else
{
if (!archive.empty())
partnameref.assign(archive).append(":").append(std::move(path));
else
partnameref.assign(std::move(path));
}
// notify the receiver of the new partname
if (is_chained)
{
std::unique_lock<std::mutex> lock(pipe_mutex);
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
}
// it is ok to push the body into the pipe for the main thread to search
bool ok = is_selected;
size = filesize;
while (len > 0 && !stop)
{
size_t len_out = std::min<size_t>(len, size); // size_t is OK: len is streamsize but non-negative
if (ok)
{
// write decompressed data to the pipe, if the pipe is broken then stop pushing more data into this pipe
if (write(pipe_fd[1], buf, len_out) < static_cast<ssize_t>(len_out))
ok = false;
}
size -= len_out;
// reached the end of the cpio body?
if (size == 0)
{
len -= len_out;
memmove(buf, buf + len_out, static_cast<size_t>(len));
break;
}
// decompress the next block of data into the buffer
len = zstream->decompress(buf, maxlen);
}
// error?
if (len < 0 || stop)
break;
if (static_cast<size_t>(len) < maxlen)
{
// fill the rest of the buffer with decompressed data
std::streamsize len_in = zstream->decompress(buf + len, maxlen - static_cast<size_t>(len));
// error?
if (len_in < 0)
break;
len += len_in;
}
// skip newc format \0 padding
if (!is_odc && len > 2)
{
size_t n = (4 - filesize % 4) % 4;
len -= n;
memmove(buf, buf + n, static_cast<size_t>(len));
}
// rest of the file is too short, something is wrong
if (len <= HEADERSIZE)
break;
// quit if this is not valid cpio header magic
if (memcmp(buf, odc_magic, 6) != 0 && memcmp(buf, newc_magic, 6) != 0 && memcmp(buf, newc_crc_magic, 6) != 0)
break;
// get a new pipe to search the next part in the archive, if the previous part was a regular file
if (is_selected)
{
// close our end of the pipe
close(pipe_fd[1]);
pipe_fd[1] = -1;
in_progress = true;
is_selected = false;
}
}
// if we're stopping we still need to notify the receiver in case it is waiting on the partname
if (is_chained)
{
std::unique_lock<std::mutex> lock(pipe_mutex);
is_assigned = true;
part_ready.notify_one();
lock.unlock();
}
// done extracting the cpio file
return true;
}
}
// not a cpio file
return false;
}
// true if path matches search constraints or buf contains matching magic bytes
bool select_matching(const char *archive, const char *path, const unsigned char *buf, size_t len, bool is_regular)
{
bool is_selected = is_regular;
if (is_selected)
{
// select one specific archive part to search, up to a colon when present
if (findpart != NULL)
{
const char *colon = strchr(findpart, ':');
// empty archive name means no nested archive, coding artifact
if (archive != NULL && *archive == '\0')
archive = NULL;
// check if this is a nested archive that is also in findpart up to the first colon
if (archive != NULL)
if (colon == NULL || strncmp(archive, findpart, colon - findpart) != 0 || archive[colon - findpart] != '\0')
return false;
const char *start = findpart;
// if this is not a nested archive, but path is a nested archive, then compare to findpart up to the first colon
if (archive == NULL && colon != NULL)
return strncmp(path, start, colon - start) == 0 && path[colon - start] == '\0';
// path to compare to starts after the colon of the archive in findpart
if (colon != NULL)
{
start = colon + 1;
colon = strchr(start, ':');
// now compare the path between the colons in findpart
if (colon != NULL)
return strncmp(path, start, colon - start) == 0 && path[colon - start] == '\0';
}
return strcmp(path, start) == 0;
}
// extract the basename from the path
const char *basename = strrchr(path, '/');
if (basename == NULL)
basename = path;
else
++basename;
if (*basename == '.' && !flag_hidden)
return false;
// exclude files whose basename matches any one of the --exclude globs
for (const auto& glob : flag_all_exclude)
{
bool ignore_case = &glob < &flag_all_exclude.front() + flag_exclude_iglob_size;
if (glob_match(path, basename, glob.c_str(), ignore_case))
return false;
}
// include files whose basename matches any one of the --include globs
for (const auto& glob : flag_all_include)
{
bool ignore_case = &glob < &flag_all_include.front() + flag_include_iglob_size;
if ((is_selected = glob_match(path, basename, glob.c_str(), ignore_case)))
break;
}
// -M: check magic bytes, requires sufficiently large len of buf[] to match patterns, which is fine when Z_BUF_LEN is large e.g. 64K to contain all magic bytes
if (buf != NULL && !flag_file_magic.empty() && (flag_all_include.empty() || !is_selected))
{
// create a matcher to match the magic pattern, we cannot use magic_matcher because it is not thread safe
reflex::Matcher magic(Static::magic_pattern);
magic.buffer(const_cast<char*>(reinterpret_cast<const char*>(buf)), len + 1);
size_t match = magic.scan();
is_selected = match == flag_not_magic || match >= flag_min_magic;
}
}
return is_selected;
}
Zthread *ztchain; // chain of decompression threads to decompress multi-compressed/archived files
zstreambuf *zstream; // the decompressed stream buffer from compressed input
FILE *zpipe_in; // input pipe from the next ztchain stage, if any
std::thread thread; // decompression thread handle
bool is_chained; // true if decompression thread is chained before another decompression thread
std::atomic_bool quit; // true if decompression thread should terminate to exit the program
std::atomic_bool stop; // true if decompression thread should stop (cancel search)
volatile bool is_extracting; // true if extracting files from TAR or ZIP archive (no concurrent r/w)
volatile bool is_waiting; // true if decompression thread is waiting (no concurrent r/w)
volatile bool is_assigned; // true when partnameref was assigned (no concurrent r/w)
int pipe_fd[2]; // decompressed stream pipe
std::mutex pipe_mutex; // mutex to extract files in thread
std::condition_variable pipe_zstrm; // cv to control new pipe creation
std::condition_variable pipe_ready; // cv to control new pipe creation
std::condition_variable pipe_close; // cv to control new pipe creation
std::condition_variable part_ready; // cv to control new partname creation to pass along decompression chains
std::string partname; // name of the archive part extracted by the next decompressor in the ztchain
std::string& partnameref; // reference to the partname of Grep or of the previous decompressor
const char *findpart; // when non-NULL, select a specific part in an archive to search
};
#endif
#endif
// grep manages output, matcher, input, and decompression
struct Grep {
// exit search exception
struct EXIT_SEARCH : public std::exception { };
// entry type
enum class Type { SKIP, DIRECTORY, OTHER };
// entry data extracted from directory contents, moves pathname to this entry
struct Entry {
static const uint16_t MIN_COST = 0;
static const uint16_t UNDEFINED_COST = 65534;
static const uint16_t MAX_COST = 65535;
Entry(std::string& pathname, ino_t inode, uint64_t info)
:
pathname(std::move(pathname)),
inode(inode),
info(info),
cost(UNDEFINED_COST)
{ }
std::string pathname;
ino_t inode;
uint64_t info;
uint16_t cost;
#ifdef OS_WIN_OR_MINGW
// get modification time (micro seconds) from directory entry
static uint64_t modified_time(const WIN32_FIND_DATAW& ffd)
{
const struct _FILETIME& time = ffd.ftLastWriteTime;
return static_cast<uint64_t>(time.dwLowDateTime) | (static_cast<uint64_t>(time.dwHighDateTime) << 32);
}
// get modification time (micro seconds) from file handle
static int64_t modified_time(const HANDLE& hFile)
{
struct _FILETIME time;
GetFileTime(hFile, NULL, NULL, &time);
return static_cast<uint64_t>(time.dwLowDateTime) | (static_cast<uint64_t>(time.dwHighDateTime) << 32);
}
#else
// get sortable info from stat buf
static uint64_t sort_info(const struct stat& buf)
{
#if defined(HAVE_STAT_ST_ATIM) && defined(HAVE_STAT_ST_MTIM) && defined(HAVE_STAT_ST_CTIM)
// tv_sec may be 64 bit, but value is small enough to multiply by 1000000 to fit in 64 bits
return static_cast<uint64_t>(flag_sort_key == Sort::SIZE ? buf.st_size : flag_sort_key == Sort::USED ? static_cast<uint64_t>(buf.st_atim.tv_sec) * 1000000 + buf.st_atim.tv_nsec / 1000 : flag_sort_key == Sort::CHANGED ? static_cast<uint64_t>(buf.st_mtim.tv_sec) * 1000000 + buf.st_mtim.tv_nsec / 1000 : flag_sort_key == Sort::CREATED ? static_cast<uint64_t>(buf.st_ctim.tv_sec) * 1000000 + buf.st_ctim.tv_nsec / 1000 : 0);
#elif defined(HAVE_STAT_ST_ATIMESPEC) && defined(HAVE_STAT_ST_MTIMESPEC) && defined(HAVE_STAT_ST_CTIMESPEC)
// tv_sec may be 64 bit, but value is small enough to multiply by 1000000 to fit in 64 bits
return static_cast<uint64_t>(flag_sort_key == Sort::SIZE ? buf.st_size : flag_sort_key == Sort::USED ? static_cast<uint64_t>(buf.st_atimespec.tv_sec) * 1000000 + buf.st_atimespec.tv_nsec / 1000 : flag_sort_key == Sort::CHANGED ? static_cast<uint64_t>(buf.st_mtimespec.tv_sec) * 1000000 + buf.st_mtimespec.tv_nsec / 1000 : flag_sort_key == Sort::CREATED ? static_cast<uint64_t>(buf.st_ctimespec.tv_sec) * 1000000 + buf.st_ctimespec.tv_nsec / 1000 : 0);
#else
return static_cast<uint64_t>(flag_sort_key == Sort::SIZE ? buf.st_size : flag_sort_key == Sort::USED ? buf.st_atime : flag_sort_key == Sort::CHANGED ? buf.st_mtime : flag_sort_key == Sort::CREATED ? buf.st_ctime : 0);
#endif
}
// get modification time (micro seconds) from stat buf
static uint64_t modified_time(const struct stat& buf)
{
#if defined(HAVE_STAT_ST_ATIM) && defined(HAVE_STAT_ST_MTIM) && defined(HAVE_STAT_ST_CTIM)
// tv_sec may be 64 bit, but value is small enough to multiply by 1000000 to fit in 64 bits
return static_cast<uint64_t>(static_cast<uint64_t>(buf.st_mtim.tv_sec) * 1000000 + buf.st_mtim.tv_nsec / 1000);
#elif defined(HAVE_STAT_ST_ATIMESPEC) && defined(HAVE_STAT_ST_MTIMESPEC) && defined(HAVE_STAT_ST_CTIMESPEC)
// tv_sec may be 64 bit, but value is small enough to multiply by 1000000 to fit in 64 bits
return static_cast<uint64_t>(static_cast<uint64_t>(buf.st_mtimespec.tv_sec) * 1000000 + buf.st_mtimespec.tv_nsec / 1000);
#else
return static_cast<uint64_t>(buf.st_mtime);
#endif
}
#endif
// compare two entries by pathname
static bool comp_by_path(const Entry& a, const Entry& b)
{
return a.pathname < b.pathname;
}
// compare two entries by size or time (atime, mtime, or ctime), if equal compare by pathname
static bool comp_by_info(const Entry& a, const Entry& b)
{
return a.info < b.info || (a.info == b.info && a.pathname < b.pathname);
}
// compare two entries by edit distance cost
static bool comp_by_best(const Entry& a, const Entry& b)
{
return a.cost < b.cost || (a.cost == b.cost && a.pathname < b.pathname);
}
// reverse compare two entries by pathname
static bool rev_comp_by_path(const Entry& a, const Entry& b)
{
return a.pathname > b.pathname;
}
// reverse compare two entries by size or time (atime, mtime, or ctime), if equal reverse compare by pathname
static bool rev_comp_by_info(const Entry& a, const Entry& b)
{
return a.info > b.info || (a.info == b.info && a.pathname > b.pathname);
}
// reverse compare two entries by edit distance cost
static bool rev_comp_by_best(const Entry& a, const Entry& b)
{
return a.cost > b.cost || (a.cost == b.cost && a.pathname > b.pathname);
}
};
// a job in the job queue
struct Job {
// sentinel job NONE
static const size_t NONE = UNDEFINED_SIZE;
Job()
:
pathname(),
cost(Entry::UNDEFINED_COST),
slot(NONE)
{ }
Job(const char *pathname, uint16_t cost, size_t slot)
:
pathname(pathname != Static::LABEL_STANDARD_INPUT ? pathname : ""), // empty pathname means stdin
cost(cost),
slot(slot)
{ }
bool none()
{
return slot == NONE;
}
std::string pathname;
uint16_t cost;
size_t slot;
};
#ifdef WITH_LOCK_FREE_JOB_QUEUE
// a lock-free job queue for one producer and one consumer with a bounded circular buffer
struct JobQueue {
JobQueue()
:
head(ring),
tail(ring),
todo(0)
{ }
bool empty() const
{
return head.load() == tail.load();
}
// add a sentinel NONE job to the queue
void enqueue()
{
enqueue("", Entry::UNDEFINED_COST, Job::NONE);
}
// add a job to the queue
void enqueue(const char *pathname, uint16_t cost, size_t slot)
{
Job *job = tail.load();
Job *next = job + 1;
if (next == &ring[DEFAULT_MAX_JOB_QUEUE_SIZE])
next = ring;
while (next == head.load())
{
// we must lock and wait until the buffer is not full
std::unique_lock<std::mutex> lock(queue_mutex);
queue_full.wait(lock);
lock.unlock();
}
job->pathname.assign(pathname);
job->cost = cost;
job->slot = slot;
tail.store(next);
++todo;
queue_data.notify_one();
}
// try to add a job to the queue if the queue is not too large
bool try_enqueue(const char *pathname, uint16_t cost, size_t slot)
{
Job *job = tail.load();
Job *next = job + 1;
if (next == &ring[DEFAULT_MAX_JOB_QUEUE_SIZE])
next = ring;
if (next == head.load())
return false;
job->pathname.assign(pathname);
job->cost = cost;
job->slot = slot;
tail.store(next);
++todo;
queue_data.notify_one();
return true;
}
// pop a job
void dequeue(Job& job)
{
while (empty())
{
// we must lock and wait until the buffer is not empty
std::unique_lock<std::mutex> lock(queue_mutex);
queue_data.wait(lock);
lock.unlock();
}
Job *next = head.load() + 1;
if (next == &ring[DEFAULT_MAX_JOB_QUEUE_SIZE])
next = ring;
job = *head.load();
head.store(next);
--todo;
queue_full.notify_one();
}
Job ring[DEFAULT_MAX_JOB_QUEUE_SIZE];
std::atomic<Job*> head;
std::atomic<Job*> tail;
std::mutex queue_mutex; // job queue mutex used when queue is empty or full
std::condition_variable queue_data; // cv to control the job queue
std::condition_variable queue_full; // cv to control the job queue
std::atomic_size_t todo; // number of jobs in the queue
};
#else
// a job queue
struct JobQueue : public std::deque<Job> {
JobQueue()
:
todo(0)
{ }
// add a sentinel NONE job to the queue
void enqueue()
{
std::unique_lock<std::mutex> lock(queue_mutex);
emplace_back();
++todo;
lock.unlock();
queue_work.notify_one();
}
// add a job to the queue
void enqueue(const char *pathname, uint16_t cost, size_t slot)
{
std::unique_lock<std::mutex> lock(queue_mutex);
emplace_back(pathname, cost, slot);
++todo;
lock.unlock();
queue_work.notify_one();
}
// pop a job
void dequeue(Job& job)
{
std::unique_lock<std::mutex> lock(queue_mutex);
while (empty())
queue_work.wait(lock);
job = front();
pop_front();
--todo;
// if we popped a Job::NONE sentinel but the queue has some jobs, then move the sentinel to the back of the queue
if (job.none() && !empty())
{
emplace_back();
job = front();
pop_front();
}
lock.unlock();
}
// steal a job from this worker, if at least --min-steal jobs to do, returns true if successful
bool steal_job(Job& job)
{
std::unique_lock<std::mutex> lock(queue_mutex);
if (empty())
return false;
job = front();
// we cannot steal a Job::NONE sentinel
if (job.none())
return false;
pop_front();
--todo;
lock.unlock();
return true;
}
// move a stolen job to this worker, maintaining job slot order
void move_job(Job& job)
{
bool inserted = false;
std::unique_lock<std::mutex> lock(queue_mutex);
// insert job in the small queue to maintain job order
const auto e = end();
for (auto j = begin(); j != e; ++j)
{
if (j->slot > job.slot)
{
insert(j, std::move(job));
inserted = true;
break;
}
}
if (!inserted)
emplace_back(std::move(job));
++todo;
lock.unlock();
queue_work.notify_one();
}
std::mutex queue_mutex; // job queue mutex
std::condition_variable queue_work; // cv to control the job queue
std::atomic_size_t todo; // number of jobs in the queue, atomic for job stealing
};
#endif
// extend the reflex::Input::Handler to handle detection of binary input like GNU grep (check for NULs)
struct BinaryHandler : public reflex::Input::Handler {
BinaryHandler(Grep& grep)
:
grep(grep)
{ }
Grep& grep;
size_t operator()(FILE*, char *buf, size_t len)
{
// simpler form of binary file detection without Unicode UTF-8 validation, check for NULs like GNU grep
grep.binfile = grep.binfile || (memchr(buf, 0, len) != NULL);
return len;
}
};
#ifndef OS_WIN
// extend the reflex::Input::Handler to handle non-blocking input from a character device (TTY) or from a pipe
struct StdInHandler : public reflex::Input::Handler {
StdInHandler(Grep& grep)
:
grep(grep)
{ }
Grep& grep;
size_t operator()(FILE*, char *buf, size_t len)
{
if (len == 0)
{
// could not read input data, but we aren't at EOF yet (i.e. non-blocking read returned zero)
// invoke buffer shift GrepHandler when set, to output the rest of a matching line and its context
grep.matcher->handle();
// flush output
grep.out.try_flush();
}
else if (!flag_text && !flag_hex && !flag_with_hex && flag_encoding_type != reflex::Input::file_encoding::null_data && !grep.binfile)
{
// simpler form of binary file detection without Unicode UTF-8 validation, check for NULs like GNU grep
grep.binfile = (memchr(buf, 0, len) != NULL);
}
return len;
}
};
#endif
// extend the reflex::AbstractMatcher::Handler with a grep object reference and references to some of the grep::search locals
struct GrepHandler : public reflex::AbstractMatcher::Handler {
GrepHandler(Grep& grep, const char *& pathname)
:
grep(grep),
pathname(pathname)
{ }
Grep& grep; // grep object
const char *& pathname; // grep::search() argument pathname reference (pointer may be changed)
// define a reflex::AbstractMatcher functor to handle buffer shifts
virtual void operator()(reflex::AbstractMatcher&, const char*, size_t, size_t) override
{
grep.output_or_save_restline(false, flag_hex_after > 0);
}
// get the start of the after/before context lines, if present
void begin_before(reflex::AbstractMatcher& matcher, const char *buf, size_t len, size_t num, const char *& ptr, size_t& size, size_t& offset)
{
ptr = NULL;
size = 0;
offset = 0;
// if no gap to shift then return (this also happens when the buffer grows)
if (len == 0)
return;
size_t current = matcher.lineno();
size_t between = current - grep.lineno;
if (between > 1)
{
const char *s = buf + len;
const char *e = s;
if (buf[len - 1] != '\n')
--between;
while (--s >= buf)
{
if (*s == '\n')
{
if (--between == 0)
break;;
e = s + 1;
}
}
ptr = ++s;
size = e - s;
offset = s - buf + num;
++grep.lineno;
}
}
// advance to the next before context line, if present
void next_before(const char *buf, size_t len, size_t num, const char *& ptr, size_t& size, size_t& offset)
{
if (ptr == NULL)
return;
const char *pos = ptr + size;
const char *end = buf + len;
if (pos >= end)
{
ptr = NULL;
}
else
{
const char *eol = static_cast<const char*>(memchr(pos, '\n', end - pos));
if (eol == NULL)
eol = buf + len;
else
++eol;
ptr = pos;
size = eol - pos;
offset = pos - buf + num;
++grep.lineno;
}
}
};
// extend event GrepHandler for --only-matching with context
struct OnlyMatchingContextGrepHandler : public GrepHandler {
OnlyMatchingContextGrepHandler(Grep& grep, const char *& pathname)
:
GrepHandler(grep, pathname)
{ }
// define a reflex::AbstractMatcher functor to handle buffer shifts by saving the restline used for context
virtual void operator()(reflex::AbstractMatcher&, const char*, size_t, size_t) override
{
grep.save_restline();
}
};
// extend event GrepHandler to output invert match lines for -v
struct InvertMatchGrepHandler : public GrepHandler {
InvertMatchGrepHandler(Grep& grep, const char *& pathname)
:
GrepHandler(grep, pathname)
{ }
// functor invoked by the reflex::AbstractMatcher when the buffer contents are shifted out, also called explicitly in grep::search
virtual void operator()(reflex::AbstractMatcher& matcher, const char *buf, size_t len, size_t num) override
{
const char *ptr;
size_t size;
size_t offset;
// if no gap to shift buffer, then let gap span the buffer until the begin of the matching line (bol)
if (len == 0)
len = matcher.before().len;
begin_before(matcher, buf, len, num, ptr, size, offset);
while (ptr != NULL)
{
// --range: max line exceeded?
if (flag_max_line > 0 && grep.lineno > flag_max_line)
break;
// --max-files: max reached?
if (grep.matches == 0 && !Stats::found_part())
{
grep.stop = true;
break;
}
// --max-count: max number of matches reached?
if (flag_max_count > 0 && grep.matches >= flag_max_count)
break;
// output blocked?
if (grep.out.eof)
break;
++grep.matches;
if (flag_with_hex)
grep.binary = false;
grep.binary = grep.binary || check_binary(ptr, size);
if (grep.binfile || (grep.binary && !flag_hex && !flag_with_hex))
break;
if (grep.hex && !grep.binary)
grep.out.dump.done();
if (!flag_no_header)
grep.out.header(pathname, grep.partname, grep.heading, grep.lineno, NULL, offset, flag_separator, grep.binary);
grep.hex = grep.binary;
if (grep.binary)
{
grep.out.dump.hex(Output::Dump::HEX_LINE, offset, ptr, size);
}
else
{
bool lf_only = false;
if (size > 0)
{
lf_only = ptr[size - 1] == '\n';
size_t sizen = size - lf_only;
if (sizen > 0)
{
grep.out.str(color_sl);
grep.out.str(ptr, sizen);
grep.out.str(color_off);
}
}
grep.out.nl(lf_only);
}
next_before(buf, len, num, ptr, size, offset);
}
}
};
// extend event GrepHandler to output formatted invert match lines for --format -v
struct FormatInvertMatchGrepHandler : public GrepHandler {
FormatInvertMatchGrepHandler(Grep& grep, const char *& pathname)
:
GrepHandler(grep, pathname)
{ }
// functor invoked by the reflex::AbstractMatcher when the buffer contents are shifted out, also called explicitly in grep::search
virtual void operator()(reflex::AbstractMatcher& matcher, const char *buf, size_t len, size_t num) override
{
const char *ptr;
size_t size;
size_t offset;
// if no gap to shift buffer, then let gap span the buffer until the begin of the matching line (bol)
if (len == 0)
len = matcher.before().len;
begin_before(matcher, buf, len, num, ptr, size, offset);
while (ptr != NULL)
{
// --range: max line exceeded?
if (flag_max_line > 0 && grep.lineno > flag_max_line)
break;
// output --format-open
if (grep.matches == 0)
{
// --format-open or --format-close: we must out.acquire() lock before Stats::found_part()
if (flag_format_open != NULL || flag_format_close != NULL)
grep.out.acquire();
// --max-files: max reached?
if (!Stats::found_part())
{
grep.stop = true;
break;
}
if (flag_format_open != NULL)
grep.out.format(flag_format_open, pathname, grep.partname, Stats::found_parts(), NULL, &matcher, grep.heading, false, Stats::found_parts() > 1);
}
// --max-count: max number of matches reached?
if (flag_max_count > 0 && grep.matches >= flag_max_count)
break;
// output blocked?
if (grep.out.eof)
break;
++grep.matches;
// output --format
grep.out.format_invert(flag_format, pathname, grep.partname, grep.matches, grep.lineno, offset, ptr, size - (size > 0 && ptr[size - 1] == '\n'), grep.heading, grep.matches > 1);
next_before(buf, len, num, ptr, size, offset);
}
}
};
// extend event GrepHandler to output any context lines for -y
struct AnyLineGrepHandler : public GrepHandler {
AnyLineGrepHandler(Grep& grep, const char *& pathname)
:
GrepHandler(grep, pathname)
{ }
// functor invoked by the reflex::AbstractMatcher when the buffer contents are shifted out, also called explicitly in grep::search
virtual void operator()(reflex::AbstractMatcher& matcher, const char *buf, size_t len, size_t num) override
{
const char *ptr;
size_t size;
size_t offset;
// if no gap to shift buffer, then let gap span the buffer until the begin of the matching line (bol)
if (len == 0)
len = matcher.before().len;
begin_before(matcher, buf, len, num, ptr, size, offset);
grep.output_or_save_restline(flag_invert_match);
// context colors with or without -v
short v_hex_context_line = flag_invert_match ? Output::Dump::HEX_LINE : Output::Dump::HEX_CONTEXT_LINE;
const char *v_color_cx = flag_invert_match ? color_sl : color_cx;
const char *separator = flag_invert_match ? flag_separator : flag_separator_dash;
while (ptr != NULL)
{
// --range: max line exceeded?
if (flag_max_line > 0 && grep.lineno > flag_max_line)
break;
if (grep.matches == 0 && flag_invert_match)
{
// --max-files: max reached?
if (!Stats::found_part())
{
grep.stop = true;
break;
}
}
// --max-count: max number of matches reached?
if (flag_invert_match && flag_max_count > 0 && grep.matches >= flag_max_count)
{
grep.stop = true;
break;
}
// output blocked?
if (grep.out.eof)
break;
if (flag_with_hex)
grep.binary = false;
if (flag_invert_match)
++grep.matches;
grep.binary = grep.binary || check_binary(ptr, size);
if (grep.binfile || (grep.binary && !flag_hex && !flag_with_hex))
break;
if (grep.hex && !grep.binary)
grep.out.dump.done();
if (!flag_no_header)
grep.out.header(pathname, grep.partname, grep.heading, grep.lineno, NULL, offset, separator, grep.binary);
grep.hex = grep.binary;
if (grep.binary)
{
grep.out.dump.hex(v_hex_context_line, offset, ptr, size);
}
else
{
bool lf_only = false;
if (size > 0)
{
lf_only = ptr[size - 1] == '\n';
size_t sizen = size - lf_only;
if (sizen > 0)
{
grep.out.str(v_color_cx);
grep.out.str(ptr, sizen);
grep.out.str(color_off);
}
}
grep.out.nl(lf_only);
}
next_before(buf, len, num, ptr, size, offset);
}
}
};
// extend event AnyLineGrepHandler to output specific context lines for -ABC without -v
struct ContextGrepHandler : public AnyLineGrepHandler {
// context state to track context lines before and after a match
struct ContextState {
struct Line {
Line() :
binary(false),
offset(0),
ptr(NULL),
size(0)
{ }
bool binary; // before context binary line
size_t offset; // before context offset of line
const char* ptr; // before context pointer to line
size_t size; // before context length of the line
std::string line; // before context line data saved from ptr with size bytes
};
typedef std::vector<Line> Lines;
ContextState()
:
before_index(0),
before_length(0),
after_lineno(0),
after_length(flag_after_context)
{
before_lines.resize(flag_before_context);
}
size_t before_index; // before context rotation index
size_t before_length; // accumulated length of the before context
Lines before_lines; // before context line data
size_t after_lineno; // after context line number
size_t after_length; // accumulated length of the after context
};
ContextGrepHandler(Grep& grep, const char *& pathname)
:
AnyLineGrepHandler(grep, pathname)
{ }
// output the before context
void output_before_context()
{
// the group separator indicates lines skipped between contexts, like GNU grep
if (state.after_lineno > 0 && state.after_lineno + state.after_length < grep.matcher->lineno() - state.before_length)
{
if (grep.hex)
grep.out.dump.done();
if (flag_group_separator != NULL)
{
if (flag_query && !flag_text)
{
grep.out.chr('\0');
grep.out.str(color_se);
grep.out.chr('\0');
grep.out.str(flag_group_separator);
grep.out.chr('\0');
}
else
{
grep.out.str(color_se);
grep.out.str(flag_group_separator);
}
grep.out.str(color_off);
grep.out.nl();
}
}
// output the before context
if (state.before_length > 0)
{
// the first line number of the before context
size_t before_lineno = grep.matcher->lineno() - state.before_length;
for (size_t i = 0; i < state.before_length; ++i)
{
size_t j = (state.before_index + i) % state.before_length;
const ContextState::Line& before_line = state.before_lines[j];
if (grep.hex && !before_line.binary)
grep.out.dump.done();
if (!flag_no_header)
grep.out.header(pathname, grep.partname, grep.heading, before_lineno + i, NULL, before_line.offset, flag_separator_dash, before_line.binary);
grep.hex = before_line.binary;
// lines are always saved at the moment, so ptr is NULL, but we may optimize this in the future to not always save
const char *ptr = before_line.ptr != NULL ? before_line.ptr : before_line.line.c_str();
size_t size = before_line.size;
if (grep.hex)
{
grep.out.dump.hex(Output::Dump::HEX_CONTEXT_LINE, before_line.offset, ptr, size);
}
else
{
bool lf_only = false;
if (size > 0)
{
lf_only = ptr[size - 1] == '\n';
size -= lf_only;
if (size > 0)
{
grep.out.str(color_cx);
grep.out.str(ptr, size);
grep.out.str(color_off);
}
}
grep.out.nl(lf_only);
}
}
}
// reset the before context state
state.before_index = 0;
state.before_length = 0;
}
// set the after context to the current line + 1
void set_after_lineno(size_t lineno)
{
// set the after context state with the first after context line number
state.after_length = 0;
state.after_lineno = lineno;
}
// functor invoked by the reflex::AbstractMatcher when the buffer contents are shifted out, also called explicitly in grep::search
virtual void operator()(reflex::AbstractMatcher& matcher, const char *buf, size_t len, size_t num) override
{
const char *ptr;
size_t size;
size_t offset;
// if no gap to shift buffer, then let gap span the buffer until the begin of the matching line (bol)
if (len == 0)
len = matcher.before().len;
// if we only need the before context, then look for it right before the current lineno
if (state.after_length >= flag_after_context)
{
size_t current = matcher.lineno();
if (grep.lineno + flag_before_context + 1 < current)
grep.lineno = current - flag_before_context - 1;
}
begin_before(matcher, buf, len, num, ptr, size, offset);
grep.output_or_save_restline();
while (ptr != NULL)
{
// --range: max line exceeded?
if (flag_max_line > 0 && grep.lineno > flag_max_line)
break;
// output blocked?
if (grep.out.eof)
break;
if (flag_with_hex)
grep.binary = false;
grep.binary = grep.binary || check_binary(ptr, size);
if (grep.binfile || (grep.binary && !flag_hex && !flag_with_hex))
break;
if (state.after_lineno > 0 && state.after_length < flag_after_context)
{
++state.after_length;
if (grep.hex && !grep.binary)
grep.out.dump.done();
if (!flag_no_header)
grep.out.header(pathname, grep.partname, grep.heading, grep.lineno, NULL, offset, flag_separator_dash, grep.binary);
grep.hex = grep.binary;
if (grep.binary)
{
grep.out.dump.hex(Output::Dump::HEX_CONTEXT_LINE, offset, ptr, size);
}
else
{
bool lf_only = false;
if (size > 0)
{
lf_only = ptr[size - 1] == '\n';
size_t sizen = size - lf_only;
if (sizen > 0)
{
grep.out.str(color_cx);
grep.out.str(ptr, sizen);
grep.out.str(color_off);
}
}
grep.out.nl(lf_only);
}
}
else if (flag_before_context > 0)
{
if (state.before_length < flag_before_context)
++state.before_length;
state.before_index %= state.before_length;
state.before_lines[state.before_index].binary = grep.binary;
state.before_lines[state.before_index].offset = offset;
state.before_lines[state.before_index].ptr = ptr;
state.before_lines[state.before_index].size = size;
++state.before_index;
}
else
{
break;
}
next_before(buf, len, num, ptr, size, offset);
}
// save the (new or additional) before lines that become inaccessible after shift or buffer growth
for (size_t i = 0; i < state.before_length; ++i)
{
if (state.before_lines[i].ptr != NULL)
{
state.before_lines[i].line.assign(state.before_lines[i].ptr, state.before_lines[i].size);
state.before_lines[i].ptr = NULL;
}
}
}
ContextState state;
};
// extend event AnyLineGrepHandler to output specific context lines for -ABC with -v
struct InvertContextGrepHandler : public AnyLineGrepHandler {
// context state to track matching lines before non-matching lines
struct InvertContextState {
struct Match {
Match(size_t pos, size_t size, size_t offset)
:
pos(pos),
size(size),
offset(offset)
{ }
size_t pos; // position on the line
size_t size; // size of the match
size_t offset; // byte offset of the match
};
typedef std::vector<Match> Matches;
struct Line {
Line()
:
binary(false),
columno(0),
offset(0)
{ }
bool binary; // before context binary line
size_t columno; // before context column number of first match
size_t offset; // before context offset of first match
std::string line; // before context line data
Matches matches; // before context matches per line
};
typedef std::vector<Line> Lines;
InvertContextState()
:
before_index(0),
before_length(0),
after_lineno(0)
{
before_lines.resize(flag_before_context);
}
size_t before_index; // before context rotation index
size_t before_length; // accumulated length of the before context
Lines before_lines; // before context line data
size_t after_lineno; // the after context line number
};
InvertContextGrepHandler(Grep& grep, const char *& pathname)
:
AnyLineGrepHandler(grep, pathname)
{ }
// output the inverted before context
void output_before_context()
{
// the group separator indicates lines skipped between contexts, like GNU grep
if (state.after_lineno > 0 && state.after_lineno + flag_after_context + flag_before_context < grep.lineno && flag_group_separator != NULL)
{
if (grep.hex)
grep.out.dump.done();
if (flag_query && !flag_text)
{
grep.out.chr('\0');
grep.out.str(color_se);
grep.out.chr('\0');
grep.out.str(flag_group_separator);
grep.out.chr('\0');
}
else
{
grep.out.str(color_se);
grep.out.str(flag_group_separator);
}
grep.out.str(color_off);
grep.out.nl();
}
// output the inverted before context
if (state.before_length > 0)
{
// the first line number of the before context
size_t before_lineno = grep.lineno - state.before_length;
for (size_t i = 0; i < state.before_length; ++i)
{
size_t j = (state.before_index + i) % state.before_length;
const InvertContextState::Line& before_line = state.before_lines[j];
size_t offset = before_line.matches.empty() ? before_line.offset : before_line.matches.front().offset;
if (grep.hex && !before_line.binary)
grep.out.dump.done();
if (!flag_no_header)
grep.out.header(pathname, grep.partname, grep.heading, before_lineno + i, NULL, offset, flag_separator_dash, before_line.binary);
grep.hex = before_line.binary;
const char *ptr = before_line.line.c_str();
size_t size = before_line.line.size();
size_t pos = 0;
for (const auto& match : before_line.matches)
{
if (grep.hex)
{
grep.out.dump.hex(Output::Dump::HEX_CONTEXT_LINE, match.offset - (match.pos - pos), ptr + pos, match.pos - pos);
grep.out.dump.hex(Output::Dump::HEX_CONTEXT_MATCH, match.offset, ptr + match.pos, match.size);
}
else
{
if (match.pos > pos)
{
grep.out.str(color_cx);
grep.out.str(ptr + pos, match.pos - pos);
grep.out.str(color_off);
}
if (match.size > 0)
{
size_t sizen = match.size - (ptr[match.pos + match.size - 1] == '\n');
if (sizen > 0)
{
grep.out.str(match_mc);
grep.out.str(ptr + match.pos, sizen);
grep.out.str(match_off);
}
}
}
pos = match.pos + match.size;
}
if (grep.hex)
{
grep.out.dump.hex(Output::Dump::HEX_CONTEXT_LINE, before_line.offset + pos, ptr + pos, size - pos);
}
else
{
bool lf_only = false;
if (size > pos)
{
lf_only = ptr[size - 1] == '\n';
size -= lf_only;
if (size > pos)
{
grep.out.str(color_cx);
grep.out.str(ptr + pos, size - pos);
grep.out.str(color_off);
}
}
grep.out.nl(lf_only);
}
}
}
// reset the context state
state.before_index = 0;
state.before_length = 0;
state.after_lineno = grep.lineno + 1;
}
// add context line with the first match to the inverted before context
void add_before_context_line(const char *bol, const char *eol, size_t columno, size_t offset)
{
if (state.before_length < flag_before_context)
++state.before_length;
state.before_index %= state.before_length;
state.before_lines[state.before_index].binary = grep.binary;
state.before_lines[state.before_index].columno = columno;
state.before_lines[state.before_index].offset = offset;
state.before_lines[state.before_index].line.assign(bol, eol - bol);
state.before_lines[state.before_index].matches.clear();
++state.before_index;
}
// add match fragment to the inverted before context
void add_before_context_match(size_t pos, size_t size, size_t offset)
{
// only add a match if we have a before line, i.e. not an after line with a multiline match
if (state.before_length > 0)
{
size_t index = (state.before_index + state.before_length - 1) % state.before_length;
state.before_lines[index].matches.emplace_back(pos, size, offset);
}
}
// functor invoked by the reflex::AbstractMatcher when the buffer contents are shifted out, also called explicitly in grep::search
virtual void operator()(reflex::AbstractMatcher& matcher, const char *buf, size_t len, size_t num) override
{
const char *ptr;
size_t size;
size_t offset;
// if no gap to shift buffer, then let gap span the buffer until the begin of the matching line (bol)
if (len == 0)
len = matcher.before().len;
begin_before(matcher, buf, len, num, ptr, size, offset);
grep.output_or_save_restline(true);
if (ptr != NULL)
output_before_context();
while (ptr != NULL)
{
state.after_lineno = grep.lineno + 1;
// --range: max line exceeded?
if (flag_max_line > 0 && grep.lineno > flag_max_line)
break;
if (grep.matches == 0)
{
// --max-files: max reached?
if (!Stats::found_part())
{
grep.stop = true;
break;
}
}
// --max-count: max number of matches reached?
if (flag_max_count > 0 && grep.matches >= flag_max_count)
break;
// output blocked?
if (grep.out.eof)
break;
++grep.matches;
if (flag_with_hex)
grep.binary = false;
grep.binary = grep.binary || check_binary(ptr, size);
if (grep.binfile || (grep.binary && !flag_hex && !flag_with_hex))
break;
if (grep.hex && !grep.binary)
grep.out.dump.done();
if (!flag_no_header)
grep.out.header(pathname, grep.partname, grep.heading, grep.lineno, NULL, offset, flag_separator, grep.binary);
grep.hex = grep.binary;
if (grep.binary)
{
grep.out.dump.hex(Output::Dump::HEX_LINE, offset, ptr, size);
}
else
{
bool lf_only = false;
if (size > 0)
{
lf_only = ptr[size - 1] == '\n';
size_t sizen = size - lf_only;
if (sizen > 0)
{
grep.out.str(color_sl);
grep.out.str(ptr, sizen);
grep.out.str(color_off);
}
}
grep.out.nl(lf_only);
}
next_before(buf, len, num, ptr, size, offset);
}
}
InvertContextState state;
};
Grep(FILE *file, reflex::AbstractMatcher *matcher, Static::Matchers *matchers)
:
filename(NULL),
restline_data(NULL),
restline_size(0),
restline_last(0),
out(file),
matcher(matcher),
matchers(matchers),
file_in(NULL),
bin_handler(*this)
#ifndef OS_WIN_OR_MINGW
, stdin_handler(*this)
#endif
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
, zthread(false, partname)
#else
, zstream(NULL)
, stream(NULL)
#endif
#endif
{ }
virtual ~Grep()
{
#ifdef HAVE_LIBZ
#ifndef WITH_DECOMPRESSION_THREAD
if (stream != NULL)
{
delete stream;
stream = NULL;
}
if (zstream != NULL)
{
delete zstream;
zstream = NULL;
}
#endif
#endif
}
// cancel all active searches
void cancel()
{
// global cancellation is forced by cancelling the shared output
out.cancel();
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
// -z: cancel decompression thread's decompression loop
if (flag_decompress)
zthread.cancel();
#endif
#endif
}
// search the specified files or standard input for pattern matches
virtual void ugrep();
// search file or directory for pattern matches
Type select(size_t level, const char *pathname, const char *basename, int type, uint32_t attr, ino_t& inode, uint64_t& info, bool is_argument = false);
// recurse a directory
virtual void recurse(size_t level, const char *pathname);
// -Z and --sort=best: perform a presearch to determine edit distance cost, return cost of pathname file, MAX_COST when no match is found
uint16_t compute_cost(const char *pathname);
// search a file or archive
virtual void search(const char *pathname, uint16_t cost);
// search input after lineno to populate a string vector with the matching line and lines after up to max lines
void find_text_preview(const char *filename, const char *partname, size_t from_lineno, size_t max, size_t& lineno, size_t& num, std::vector<std::string>& text);
// extract a part from an archive and send to a stream
void extract(const char *filename, const char *partname, FILE *output);
// check CNF AND/OR/NOT conditions are met for the line(s) spanning bol to eol
bool cnf_matching(const char *bol, const char *eol, bool acquire = false)
{
if (flag_files)
{
if (out.holding())
{
size_t k = 0; // iterate over matching[] bitmask
bool all = true; // all terms matched
// for each AND term check if the AND term was matched before or has a match this time
for (const auto& i : *matchers)
{
// an OR term hasn't matched before
if (!matching[k])
{
auto j = i.begin();
auto e = i.end();
if (j != e)
{
// check OR terms
if (*j && (*j)->buffer(const_cast<char*>(bol), eol - bol + 1).find() != 0)
{
matching[k] = true;
++j;
}
else
{
// check OR NOT terms
size_t l = 0; // iterate over notmaching[k] bitmask
bool none = true; // all not-terms matched
while (++j != e)
{
if (*j && !notmatching[k][l])
{
if ((*j)->buffer(const_cast<char*>(bol), eol - bol + 1).find() != 0)
notmatching[k][l] = true;
else
all = none = false;
}
++l;
}
if (none)
{
// when all not-terms matched and we don't have a positive alternative then stop searching this file
if (!*i.begin())
throw EXIT_SEARCH();
all = false;
}
}
}
}
++k;
}
// if all terms matched globally per file then remove the hold to launch output
if (all)
{
if (acquire)
out.acquire();
// --max-files: max reached?
if (!Stats::found_part())
throw EXIT_SEARCH();
out.launch();
}
}
}
else
{
// for each AND term check if the line has a match
for (const auto& i : *matchers)
{
auto j = i.begin();
auto e = i.end();
if (j != e)
{
// check OR terms
if (*j && (*j)->buffer(const_cast<char*>(bol), eol - bol + 1).find() != 0)
continue;
// check OR NOT terms
while (++j != e)
if (*j && (*j)->buffer(const_cast<char*>(bol), eol - bol + 1).find() == 0)
break;
if (j == e)
return false;
}
}
}
return true;
}
// if CNF AND/OR/NOT conditions are met globally then launch output after searching a file with --files
bool cnf_satisfied(bool acquire = false)
{
if (out.holding())
{
size_t k = 0; // iterate over matching[] bitmask
// for each AND term check if the term was matched before
for (const auto& i : *matchers)
{
// an OR term hasn't matched
if (!matching[k])
{
// return if there are no OR NOT terms to check
if (i.size() <= 1)
return false;
auto j = i.begin();
auto e = i.end();
// check if not all of the OR NOT terms matched
if (j != e)
{
size_t l = 0; // iterate over notmaching[k] bitmask
while (++j != e)
{
if (*j && !notmatching[k][l])
break;
++l;
}
// return if all OR NOT terms matched
if (j == e)
return false;
}
}
++k;
}
if (acquire)
out.acquire();
// --max-files: max reached?
if (!Stats::found_part())
throw EXIT_SEARCH();
out.launch();
}
return true;
}
// open a file for (binary) reading and assign input, decompress the file when -z, --decompress specified, may throw bad_alloc
bool open_file(const char *pathname, const char *find = NULL)
{
if (pathname == Static::LABEL_STANDARD_INPUT)
{
if (Static::source == NULL)
return false;
pathname = flag_label;
file_in = Static::source;
#ifdef OS_WIN
(void)_setmode(fileno(Static::source), _O_BINARY);
#endif
}
else if (fopenw_s(&file_in, pathname, "rb") != 0)
{
warning("cannot read", pathname);
return false;
}
// "interactive" means possibly slow input from a TTY or pipe
interactive = false;
// --filter: fork process to filter file, when applicable
if (!filter(file_in, pathname))
return false;
#if defined(__linux__)
if (file_in != stdin && file_in != Static::source)
{
// Linux has system "regular files" that can be set non-blocking to raise EAGAIN e.g. for /proc and /sys files
if (flag_devices_action != Action::READ)
{
int fd = fileno(file_in);
struct stat buf;
if (fstat(fd, &buf) == 0 && S_ISREG(buf.st_mode))
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK);
}
}
#endif
#ifdef HAVE_LIBZ
if (flag_decompress)
{
#ifdef WITH_DECOMPRESSION_THREAD
// start decompression thread if not running, get pipe with decompressed input
FILE *pipe_in = zthread.start(flag_zmax, pathname, file_in, find);
if (pipe_in == NULL)
{
fclose(file_in);
file_in = NULL;
return false;
}
input = reflex::Input(pipe_in, flag_encoding_type);
#else
(void)find; // appease -Wunused
// create or open a new zstreambuf
if (zstream == NULL)
zstream = new zstreambuf(pathname, file_in);
else
zstream->open(pathname, file_in);
if (stream != NULL)
delete stream;
stream = new std::istream(zstream);
input = stream;
#endif
}
else
{
input = reflex::Input(file_in, flag_encoding_type);
}
#else
(void)find; // appease -Wunused
input = reflex::Input(file_in, flag_encoding_type);
#endif
#ifndef OS_WIN_OR_MINGW
if (!flag_decompress && !flag_quiet && !flag_files_with_matches && !flag_count)
{
int fd = fileno(file_in);
struct stat buf;
// if input is a character device (e.g. TTY) or a pipe, then make it non-blocking and register a stdin handler to continue reading and flush results to output, don't check for invalid Unicode input
if (fstat(fd, &buf) == 0 && (S_ISCHR(buf.st_mode) || S_ISFIFO(buf.st_mode) || S_ISSOCK(buf.st_mode)))
interactive = (fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK) != -1);
}
#else
// for Windows we don't check for invalid Unicode input, no stdin handler
if (input == stdin && !flag_quiet && !flag_files_with_matches && !flag_count)
interactive = true;
#endif
return true;
}
// --filter: return true on success, create a pipe to replace file input if filtering files in a forked process
bool filter(FILE*& in, const char *pathname)
{
if (!flag_filter.empty() && in != NULL)
{
const char *basename = strrchr(pathname, PATHSEPCHR);
if (basename == NULL)
basename = pathname;
else
++basename;
// get the basenames's extension suffix
const char *suffix = strrchr(basename, '.');
// don't consider . at the front of basename, otherwise skip .
if (suffix == basename)
suffix = NULL;
else if (suffix != NULL)
++suffix;
// --filter-magic-label: if the file is seekable (to rewind later), then check for a magic pattern match
if (!flag_filter_magic_label.empty() && fseek(in, 0, SEEK_SET) == 0)
{
bool is_plus = false;
// --filter-magic-label: check for overriding +
if (suffix != NULL)
{
for (const auto& i : flag_filter_magic_label)
{
if (i.front() == '+')
{
is_plus = true;
break;
}
}
}
// --filter-magic-label: if the basename has no suffix or a +LABEL + then check magic bytes
if (suffix == NULL || is_plus)
{
// create a matcher to match the magic pattern
size_t match = reflex::Matcher(Static::filter_magic_pattern, in).scan();
// rewind the input after scan
rewind(in);
if (match > 0 && match <= flag_filter_magic_label.size())
{
suffix = flag_filter_magic_label[match - 1].c_str();
if (*suffix == '+')
++suffix;
}
}
}
// basenames without a suffix get "*" as a suffix
if (suffix == NULL || *suffix == '\0')
suffix = "*";
size_t sep = strlen(suffix);
const char *command = flag_filter.c_str();
const char *default_command = NULL;
// find the command corresponding to the first matching suffix specified in the filter
while (true)
{
while (isspace(static_cast<unsigned char>(*command)))
++command;
// wildcard *:command is considered only when no matching suffix was found
if (*command == '*')
default_command = strchr(command, ':');
// match filter filename extension (case insensitive)
bool ext_match = strncasecmp(suffix, command, sep) == 0;
if (ext_match && (command[sep] == ':' || command[sep] == ',' || isspace(static_cast<unsigned char>(command[sep]))))
{
command = strchr(command, ':');
break;
}
command = strchr(command, ',');
if (command == NULL)
break;
++command;
}
// if no matching command, use the wildcard *:command when specified
if (command == NULL)
command = default_command;
// suffix has a command to execute
if (command != NULL)
{
// skip over the ':'
++command;
int fd[2];
#ifdef OS_WIN_OR_MINGW
// Windows CreateProcess requires an "inherited" pipe handle specific to Windows
bool ok = (pipe_inherit(fd) == 0);
#else
bool ok = (pipe(fd) == 0);
#endif
if (ok)
{
#ifdef OS_WIN_OR_MINGW
std::wstring wcommand(utf8_decode(command));
size_t pathname_pos = 0;
// replace all % by the pathname, except when quoted
while (true)
{
size_t size = wcommand.size();
for (; pathname_pos < size && wcommand[pathname_pos] != L'%'; ++pathname_pos)
if (wcommand[pathname_pos] == L'"')
while (++pathname_pos < size && wcommand[pathname_pos] != L'"')
continue;
if (pathname_pos >= size)
break;
std::wstring wpathname(utf8_decode(in == stdin ? "-" : pathname));
wcommand.replace(pathname_pos, 1, wpathname);
pathname_pos += wpathname.size();
}
// set up inherited stdin, stdout and stderr for the child process
STARTUPINFOW si;
memset(&si, 0, sizeof(STARTUPINFOW));
si.cb = sizeof(STARTUPINFOW);
si.dwFlags = STARTF_USESTDHANDLES;
si.hStdInput = reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(in)));
si.hStdOutput = reinterpret_cast<HANDLE>(_get_osfhandle(fd[1]));
if (!flag_quiet && !flag_no_messages)
si.hStdError = GetStdHandle(STD_ERROR_HANDLE);
if (in != stdin)
SetHandleInformation(si.hStdInput, HANDLE_FLAG_INHERIT, 1);
PROCESS_INFORMATION pi;
memset(&pi, 0, sizeof(PROCESS_INFORMATION));
// use buffer to allow CreateProcessW to change the command and arguments to pass them as argc argv
wchar_t *wbuffer = new wchar_t[wcommand.size() + 1];
wcscpy(wbuffer, wcommand.c_str());
if (CreateProcessW(NULL, wbuffer, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi))
{
CloseHandle(pi.hProcess);
CloseHandle(pi.hThread);
}
else
{
const char *end;
for (end = command; *end != '\0' && *end != ','; ++end)
if (*end == '"')
while (*++end != '\0' && *end != '"')
continue;
std::string arg(command, end - command);
errno = GetLastError();
warning("--filter: cannot create process for command", arg.c_str());
}
delete[] wbuffer;
#else
int pid;
// fork to execute the specified --filter utility command on the input to produce filtered output
if ((pid = fork()) == 0)
{
// child process
// close the reading end of the pipe
close(fd[0]);
// dup the input file to stdin unless reading stdin
if (in != stdin)
{
dup2(fileno(in), STDIN_FILENO);
fclose(in);
}
// dup the writing end of the pipe to stdout
dup2(fd[1], STDOUT_FILENO);
close(fd[1]);
// -q or -q: suppress error messages sent to stderr by the filter command by redirecting to /dev/null
if (flag_quiet || flag_no_messages)
{
int dev_null = open("/dev/null", O_WRONLY);
if (dev_null >= 0)
{
dup2(dev_null, STDERR_FILENO);
close(dev_null);
}
}
// populate argv[] with the command and its arguments, destroying flag_filter in the child process
std::vector<const char*> args;
char *arg = const_cast<char*>(command);
while (*arg != '\0' && *arg != ',')
{
while (isspace(static_cast<unsigned char>(*arg)))
++arg;
char *s = arg;
if (*arg == '"')
{
// "quoted argument" separated by space
++s;
while (*s != '\0' &&
(*s != '"' ||
(s[1] != '\0' && s[1] != ',' && !isspace(static_cast<unsigned char>(s[1])))))
++s;
if (*s == '"')
{
++arg;
*s++ = '\0';
}
}
else
{
// space-separated argument
while (*s != '\0' && *s != ',' && !isspace(static_cast<unsigned char>(*s)))
++s;
}
if (s > arg)
{
if (s - arg == 1 && *arg == '%')
args.push_back(in == stdin ? "-" : pathname);
else
args.push_back(arg);
}
if (*s == ',')
*s = '\0';
if (*s == '\0')
break;
*s = '\0';
arg = s + 1;
}
// silently bail out if there is no command
if (args.empty())
exit(EXIT_SUCCESS);
// add sentinel
args.push_back(NULL);
// get argv[] array data
char * const *argv = const_cast<char * const *>(args.data());
// execute
execvp(argv[0], argv);
error("--filter: cannot exec", argv[0]);
}
#endif
// close the writing end of the pipe
close(fd[1]);
// close the file and use the reading end of the pipe
if (in != NULL && in != stdin)
fclose(in);
in = fdopen(fd[0], "r");
}
else
{
if (in != stdin)
fclose(in);
in = NULL;
warning("--filter: cannot create pipe", flag_filter.c_str());
return false;
}
}
}
return true;
}
// close the file and clear input, return true if next file is extracted from an archive to search
bool close_file(const char *pathname)
{
// check if the input has an error condition, but do not check stdin which is handled differently
if (file_in != NULL && file_in != stdin && file_in != Static::source && ferror(file_in))
warning("cannot read", pathname);
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
// -z: open next archived file if any or close the compressed file/archive
if (flag_decompress)
{
// close the input FILE* and its underlying pipe previously created with pipe() and fdopen()
if (input.file() != NULL)
{
// close and unassign input, i.e. input.file() == NULL, also closes pipe_fd[0] per fdopen()
fclose(input.file());
input.clear();
}
// if output is blocked, then cancel decompression threads
if (out.eof)
zthread.cancel();
// open pipe to the next file or part in an archive if there is a next file to extract
FILE *pipe_in = zthread.open_next(pathname);
if (pipe_in != NULL)
{
// assign the next extracted file as input to search
input = reflex::Input(pipe_in, flag_encoding_type);
// loop back in search() to start searching the next file in the archive
return true;
}
}
#else
if (stream != NULL)
{
delete stream;
stream = NULL;
}
#endif
#endif
#if defined(WITH_STDIN_DRAIN) && !defined(OS_WIN_OR_MINGW)
// drain stdin when non-seekable file such as a pipe until eof, unless option -q is used
if (file_in == stdin && !flag_quiet && !flag_files_with_matches && flag_max_count == 0 && flag_max_line == 0 &&
!feof(stdin) && fseek(stdin, 0, SEEK_END) < 0 && errno != EINVAL)
{
char buf[16384];
while (input.get(buf, sizeof(buf)) > 0)
continue;
}
#endif
// close the current input file
if (file_in != NULL && file_in != stdin && file_in != Static::source)
{
fclose(file_in);
file_in = NULL;
}
input.clear();
return false;
}
// specify input to read for matcher, when input is a regular file then try mmap for zero copy overhead
bool init_read()
{
// initialize Grep search and output variables
binfile = false;
lineno = flag_min_line > 0 ? flag_min_line - 1 : 0;
heading = flag_with_filename;
hex = false;
binary = false;
matches = 0;
stop = false;
const char *base;
size_t size;
// attempt to mmap the input file, if mmap is supported and enabled (disabled by default)
if (mmap.file(input, base, size))
{
// matcher reads directly from protected mmap memory (cast is safe: base[0..size] is not modified!)
matcher->buffer(const_cast<char*>(base), size + 1);
}
else
{
// assign input to the matcher to search
matcher->input(input);
#if !defined(HAVE_PCRE2) && defined(HAVE_BOOST_REGEX)
// buffer all input to work around Boost.Regex partial matching bug, but this may throw std::bad_alloc if the file is too large
if (flag_mode == Mode::PERL)
{
matcher->buffer();
}
else
#endif
#ifndef OS_WIN_OR_MINGW
// set interactive search handler e.g. for slow stdin input
if (interactive)
matcher->in.set_handler(&stdin_handler);
#endif
}
// interactive: do not check init_is_binary(), wait until handler detects binary files that have NUL bytes
if (!interactive)
{
// otherwise, check if the file is a binary file
if (flag_binary_without_match)
{
// -I: do not match binary files
if (init_is_binary())
return false;
}
else if (!flag_quiet && !flag_files_with_matches && !flag_count && flag_format == NULL && !flag_text && !flag_hex && !flag_with_hex && flag_encoding_type != reflex::Input::file_encoding::null_data)
{
// not -q, -l, -c, --format, -a, -X, -W, -00: check if initial part of the file is binary
binfile = init_is_binary();
// detect binary input dynamically when more input is read after checking the inital input
matcher->in.set_handler(&bin_handler);
}
}
// --range=[MIN,][MAX]: start searching at line MIN
for (size_t i = flag_min_line; i > 1; --i)
if (!matcher->skip('\n'))
break;
return true;
}
// initial part of a file is binary
bool init_is_binary()
{
// check initial input for binary data, fill up to 131072 bytes to check (similar GNU grep, but a bit more)
size_t avail = matcher->avail(reflex::AbstractMatcher::Const::BUFSZ);
if (avail == 0)
return false;
// do not cut off the last UTF-8 sequence, ignore it, otherwise we risk failing the UTF-8 check
const char *buf = matcher->begin();
if ((buf[avail - 1] & 0x80) == 0x80)
{
size_t n = std::min<size_t>(avail, 4); // note: 1 <= n <= 4 bytes to check
while (n > 0 && (buf[--avail] & 0xc0) == 0x80)
--n;
if ((buf[avail] & 0xc0) != 0xc0)
return true;
}
return is_binary(matcher->begin(), avail);
}
// output the rest of the matching line or save it for later, uses lineno of previous match to check
void output_or_save_restline(bool invert = false, bool hex_context = false)
{
if (restline_data != NULL)
{
if (flag_ungroup || lineno != matcher->lineno())
output_restline(invert, hex_context);
else
save_restline();
}
}
// output the rest of the matching line or save it for later, this is necessary when the buffer shifts
void output_restline(bool invert = false, bool hex_context = false)
{
if (restline_data != NULL)
{
if (binary)
{
// --hexdump with ABC context: if binary then show hex context
if (hex_context)
{
size_t right = flag_hex_after * flag_hex_columns - ((restline_last - 1) % flag_hex_columns) - 1;
if (restline_last + right > matcher->first())
right = matcher->first() - restline_last;
if (right < restline_size)
restline_size = right;
}
out.dump.hex(invert ? Output::Dump::HEX_CONTEXT_LINE : Output::Dump::HEX_LINE, restline_last, restline_data, restline_size);
}
else
{
bool lf_only = false;
if (restline_size > 0)
{
lf_only = restline_data[restline_size - 1] == '\n';
restline_size -= lf_only;
if (restline_size > 0)
{
out.str(invert ? color_cx : color_sl);
out.str(restline_data, restline_size);
out.str(color_off);
}
}
out.nl(lf_only);
}
restline_data = NULL;
}
}
// save the rest of the line to a string buffer to output later, may self-assign, so swap with a temp string
void save_restline()
{
std::string temp(restline_data, restline_size);
temp.swap(restline);
restline_data = restline.c_str();
}
const char *filename; // the name of the file being searched
std::string partname; // the name of an extracted file from an archive
std::string restline; // a buffer to store the rest of a line to search
const char *restline_data; // rest of the line data pointer or NULL
size_t restline_size; // rest of the line size
size_t restline_last; // rest of the line last byte offset
Output out; // asynchronous output
reflex::AbstractMatcher *matcher; // the pattern matcher we're using, never NULL
Static::Matchers *matchers; // the CNF of AND/OR/NOT matchers or NULL
std::vector<bool> matching; // bitmap to keep track of globally matching CNF terms
std::vector<std::vector<bool>> notmatching; // bitmap to keep track of globally matching OR NOT CNF terms
MMap mmap; // mmap state
reflex::Input input; // input to the matcher
FILE *file_in; // the current input file
bool interactive; // the file being searched is a TTY or pipe
bool binfile; // the file being searched is a binary file
size_t lineno; // the line number of the last match
bool heading; // heading on/off, initially set to --with-filename
bool hex; // the match is output in hex
bool binary; // the match is binary as per option -X or -W
size_t matches; // number of matches
bool stop; // stop searching when --max-files max reached
BinaryHandler bin_handler; // a handler to detect binary input like GNU grep
#ifndef OS_WIN_OR_MINGW
StdInHandler stdin_handler; // a handler to handle non-blocking input from a TTY or a slow pipe
#endif
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
Zthread zthread;
#else
zstreambuf *zstream; // the decompressed stream buffer from compressed input
std::istream *stream; // input stream is the decompressed zstream
#endif
#endif
};
struct GrepWorker;
// master submits jobs to workers and implements operations to support job stealing
struct GrepMaster : public Grep {
GrepMaster(FILE *file, reflex::AbstractMatcher *matcher, Static::Matchers *matchers)
:
Grep(file, matcher, matchers),
sync(flag_sort_key == Sort::NA ? Output::Sync::Mode::UNORDERED : Output::Sync::Mode::ORDERED)
{
// master and workers synchronize their output
out.sync_on(&sync);
// set global handle to be able to call cancel_ugrep()
Static::set_grep_handle(this);
if (Static::cores >= 8)
set_this_thread_affinity_and_priority(Static::cores - 1);
start_workers();
iworker = workers.begin();
}
virtual ~GrepMaster()
{
stop_workers();
Static::clear_grep_handle();
}
// clone the pattern matcher - the caller is responsible to deallocate the returned matcher
reflex::AbstractMatcher *matcher_clone() const
{
return matcher->clone();
}
// clone the CNF of AND/OR/NOT matchers - the caller is responsible to deallocate the returned list of matchers if not NULL
Static::Matchers *matchers_clone() const
{
if (matchers == NULL)
return NULL;
Static::Matchers *new_matchers = new Static::Matchers;
for (const auto& i : *matchers)
{
new_matchers->emplace_back();
auto& last = new_matchers->back();
for (const auto& j : i)
{
if (j)
last.emplace_back(j->clone());
else
last.emplace_back();
}
}
return new_matchers;
}
// search a file by submitting it as a job to a worker
void search(const char *pathname, uint16_t cost) override
{
submit(pathname, cost);
}
// start worker threads
void start_workers();
// stop all workers
void stop_workers();
// submit a job with a pathname to a worker, workers are visited round-robin
void submit(const char *pathname, uint16_t cost);
// job stealing on behalf of a worker from a co-worker with at least --min-steal jobs still to do
bool steal(GrepWorker *worker);
std::list<GrepWorker> workers; // workers running threads
std::list<GrepWorker>::iterator iworker; // the next worker to submit a job to
Output::Sync sync; // sync output of workers
};
// worker runs a thread to execute jobs submitted by the master
struct GrepWorker : public Grep {
GrepWorker(FILE *file, size_t id, GrepMaster *master)
:
Grep(file, master->matcher_clone(), master->matchers_clone()),
id(id),
master(master)
{
// all workers synchronize their output on the master's sync object
out.sync_on(&master->sync);
// run worker thread executing jobs assigned to its queue
thread = std::thread(&GrepWorker::execute, this);
}
virtual ~GrepWorker()
{
// delete the cloned matcher
delete matcher;
// delete the cloned matchers, if any
if (matchers != NULL)
delete matchers;
}
// worker thread execution
void execute();
// submit Job::NONE sentinel to this worker
void submit_job()
{
jobs.enqueue();
}
// submit a job to this worker
void submit_job(const char *pathname, uint16_t cost, size_t slot)
{
jobs.enqueue(pathname, cost, slot);
}
// receive a job for this worker, wait until one arrives
void next_job(Job& job)
{
jobs.dequeue(job);
}
// submit Job::NONE sentinel to stop this worker
void stop()
{
submit_job();
}
std::thread thread; // thread of this worker, spawns GrepWorker::execute()
const size_t id; // worker number 0 and up
GrepMaster *master; // the master of this worker
JobQueue jobs; // queue of pending jobs submitted to this worker
};
// start worker threads
void GrepMaster::start_workers()
{
size_t num;
// create worker threads
try
{
for (num = 0; num < Static::threads; ++num)
workers.emplace(workers.end(), out.file, num, this);
}
// if sufficient resources are not available then reduce the number of threads to the number of active workers created
catch (std::system_error& error)
{
if (error.code() != std::errc::resource_unavailable_try_again)
throw;
Static::threads = num;
}
}
// stop all workers
void GrepMaster::stop_workers()
{
// submit Job::NONE sentinel to workers
for (auto& worker : workers)
worker.stop();
// wait for workers to join
for (auto& worker : workers)
worker.thread.join();
}
// submit a job with a pathname to a worker
void GrepMaster::submit(const char *pathname, uint16_t cost)
{
while (true)
{
size_t min_todo = iworker->jobs.todo;
// find a worker with the minimum number of jobs
if (min_todo > 0)
{
auto min_worker = iworker;
for (size_t num = 1; num < Static::threads; ++num)
{
++iworker;
if (iworker == workers.end())
iworker = workers.begin();
if (iworker->jobs.todo < min_todo)
{
min_todo = iworker->jobs.todo;
if (min_todo == 0)
break;
min_worker = iworker;
}
}
iworker = min_worker;
}
#ifdef WITH_LOCK_FREE_JOB_QUEUE
if (iworker->try_submit_job(pathname, cost, sync.next) || out.eof || out.cancelled())
break;
// give the worker threads some slack to make progress, then try again
std::this_thread::sleep_for(std::chrono::milliseconds(1));
#else
// give the worker threads some slack to make progress when their queues reached a soft max size
if (min_todo > flag_max_queue && flag_max_queue > 0)
std::this_thread::sleep_for(std::chrono::milliseconds(1));
// submit job to worker, do not loop
iworker->submit_job(pathname, cost, sync.next);
break;
#endif
}
++sync.next;
// around we go
++iworker;
if (iworker == workers.end())
iworker = workers.begin();
}
#ifndef WITH_LOCK_FREE_JOB_QUEUE
// job stealing on behalf of a worker from a co-worker with at least --min-steal jobs still to do
bool GrepMaster::steal(GrepWorker *worker)
{
// try to steal a job from a co-worker with the most jobs
auto coworker = workers.begin();
auto max_worker = coworker;
size_t max_todo = 0;
for (size_t i = 0; i < Static::threads; ++i)
{
if (&*coworker != worker && coworker->jobs.todo > max_todo)
{
max_todo = coworker->jobs.todo;
max_worker = coworker;
}
// around we go
++coworker;
if (coworker == workers.end())
coworker = workers.begin();
}
// not enough jobs in the co-worker's queue to steal from
if (max_todo < flag_min_steal)
return false;
coworker = max_worker;
Job job;
// steal a job from the co-worker for this worker
if (coworker->jobs.steal_job(job))
{
worker->jobs.move_job(job);
return true;
}
// couldn't steal any job
return false;
}
#endif
// execute worker thread
void GrepWorker::execute()
{
if (Static::cores >= 3)
set_this_thread_affinity_and_priority(id);
Job job;
while (!out.eof && !out.cancelled())
{
// wait for next job
next_job(job);
// worker should stop?
if (job.none())
break;
// start synchronizing output for this job slot in ORDERED mode (--sort)
out.begin(job.slot);
// search the file for this job, an empty pathname means stdin
search(job.pathname.empty() ? Static::LABEL_STANDARD_INPUT : job.pathname.c_str(), job.cost);
// end output in ORDERED mode (--sort) for this job slot
out.end();
#ifndef WITH_LOCK_FREE_JOB_QUEUE
// if only one job is left to do or nothing to do, then try stealing another job from a co-worker
if (jobs.todo <= 1)
master->steal(this);
#endif
}
}
// the CNF of Boolean search queries and patterns
CNF Static::bcnf;
// unique address to identify standard input path
const char *Static::LABEL_STANDARD_INPUT = "(standard input)";
// unique address to identify color and pretty WHEN arguments
const char *Static::NEVER = "never";
const char *Static::ALWAYS = "always";
const char *Static::AUTO = "auto";
// pointer to the --index pattern DFA with HFA constructed before threads start
const reflex::Pattern *Static::index_pattern = NULL; // concurrent access is thread safe
// the -M MAGIC pattern DFA constructed before threads start, read-only afterwards
reflex::Pattern Static::magic_pattern; // concurrent access is thread safe
reflex::Matcher Static::magic_matcher; // concurrent access is not thread safe
// the --filter-magic-label pattern DFA
reflex::Pattern Static::filter_magic_pattern; // concurrent access is thread safe
// ugrep command-line arguments pointing to argv[]
const char *Static::arg_pattern = NULL;
std::vector<const char*> Static::arg_files;
// number of cores
size_t Static::cores;
// number of concurrent threads for workers
size_t Static::threads;
// number of warnings given
std::atomic_size_t Static::warnings;
// redirectable source is standard input by default or a pipe
FILE *Static::source = stdin;
// redirectable output destination is standard output by default or a pipe
FILE *Static::output = stdout;
// redirectable error output destination is standard error by default or a pipe
FILE *Static::errout = stderr;
// full home directory path or NULL to expand ~ in options with path arguments
const char *Static::home_dir = NULL;
// Grep object handle, to cancel the search with cancel_ugrep()
struct Grep *Static::grep_handle = NULL;
std::mutex Static::grep_handle_mutex;
// patterns
reflex::Pattern Static::reflex_pattern;
std::string Static::string_pattern;
std::list<reflex::Pattern> Static::reflex_patterns;
std::list<std::string> Static::string_patterns;
// the LineMatcher, PCRE2Matcher, FuzzyMatcher or Matcher, concurrent access is not thread safe
std::unique_ptr<reflex::AbstractMatcher> Static::matcher;
// the CNF of AND/OR/NOT matchers or NULL, concurrent access is not thread safe
Static::Matchers Static::matchers;
// table of RE/flex file encodings for option --encoding (may be specified in any case)
const Encoding encoding_table[] = {
{ "binary", reflex::Input::file_encoding::plain },
{ "ASCII", reflex::Input::file_encoding::utf8 },
{ "UTF-8", reflex::Input::file_encoding::utf8 },
{ "UTF-16", reflex::Input::file_encoding::utf16be },
{ "UTF-16BE", reflex::Input::file_encoding::utf16be },
{ "UTF-16LE", reflex::Input::file_encoding::utf16le },
{ "UTF-32", reflex::Input::file_encoding::utf32be },
{ "UTF-32BE", reflex::Input::file_encoding::utf32be },
{ "UTF-32LE", reflex::Input::file_encoding::utf32le },
{ "LATIN1", reflex::Input::file_encoding::latin },
{ "ISO-8859-1", reflex::Input::file_encoding::latin },
{ "ISO-8859-2", reflex::Input::file_encoding::iso8859_2 },
{ "ISO-8859-3", reflex::Input::file_encoding::iso8859_3 },
{ "ISO-8859-4", reflex::Input::file_encoding::iso8859_4 },
{ "ISO-8859-5", reflex::Input::file_encoding::iso8859_5 },
{ "ISO-8859-6", reflex::Input::file_encoding::iso8859_6 },
{ "ISO-8859-7", reflex::Input::file_encoding::iso8859_7 },
{ "ISO-8859-8", reflex::Input::file_encoding::iso8859_8 },
{ "ISO-8859-9", reflex::Input::file_encoding::iso8859_9 },
{ "ISO-8859-10", reflex::Input::file_encoding::iso8859_10 },
{ "ISO-8859-11", reflex::Input::file_encoding::iso8859_11 },
{ "ISO-8859-13", reflex::Input::file_encoding::iso8859_13 },
{ "ISO-8859-14", reflex::Input::file_encoding::iso8859_14 },
{ "ISO-8859-15", reflex::Input::file_encoding::iso8859_15 },
{ "ISO-8859-16", reflex::Input::file_encoding::iso8859_16 },
{ "MAC", reflex::Input::file_encoding::macroman },
{ "MACROMAN", reflex::Input::file_encoding::macroman },
{ "EBCDIC", reflex::Input::file_encoding::ebcdic },
{ "CP437", reflex::Input::file_encoding::cp437 },
{ "CP850", reflex::Input::file_encoding::cp850 },
{ "CP858", reflex::Input::file_encoding::cp858 },
{ "CP1250", reflex::Input::file_encoding::cp1250 },
{ "CP1251", reflex::Input::file_encoding::cp1251 },
{ "CP1252", reflex::Input::file_encoding::cp1252 },
{ "CP1253", reflex::Input::file_encoding::cp1253 },
{ "CP1254", reflex::Input::file_encoding::cp1254 },
{ "CP1255", reflex::Input::file_encoding::cp1255 },
{ "CP1256", reflex::Input::file_encoding::cp1256 },
{ "CP1257", reflex::Input::file_encoding::cp1257 },
{ "CP1258", reflex::Input::file_encoding::cp1258 },
{ "KOI8-R", reflex::Input::file_encoding::koi8_r },
{ "KOI8-U", reflex::Input::file_encoding::koi8_u },
{ "KOI8-RU", reflex::Input::file_encoding::koi8_ru },
{ "null-data", reflex::Input::file_encoding::null_data },
{ NULL, 0 }
};
// table of file types for option -t, --file-type
const Type type_table[] = {
{ "actionscript", "as,mxml", NULL, NULL },
{ "ada", "ada,adb,ads", NULL, NULL },
{ "adoc", "adoc", NULL, NULL },
{ "asm", "asm,s,S", NULL, NULL },
{ "asp", "asp", NULL, NULL },
{ "aspx", "master,ascx,asmx,aspx,svc", NULL, NULL },
{ "autoconf", "ac,in", NULL, NULL },
{ "automake", "am,in", NULL, NULL },
{ "awk", "awk", NULL, NULL },
{ "Awk", "awk", NULL, "#!\\h*/.*\\Wg?awk(\\W.*)?\\n" },
{ "basic", "bas,BAS,cls,frm,ctl,vb,resx", NULL, NULL },
{ "batch", "bat,BAT,cmd,CMD", NULL, NULL },
{ "bison", "y,yy,ymm,ypp,yxx", NULL, NULL },
{ "c", "c,h,H,hdl,xs", NULL, NULL },
{ "c++", "cpp,CPP,cc,cxx,CXX,h,hh,H,hpp,hxx,Hxx,HXX", NULL, NULL },
{ "clojure", "clj", NULL, NULL },
{ "cpp", "cpp,CPP,cc,cxx,CXX,h,hh,H,hpp,hxx,Hxx,HXX", NULL, NULL },
{ "csharp", "cs", NULL, NULL },
{ "css", "css", NULL, NULL },
{ "csv", "csv", NULL, NULL },
{ "dart", "dart", NULL, NULL },
{ "Dart", "dart", NULL, "#!\\h*/.*\\Wdart(\\W.*)?\\n" },
{ "delphi", "pas,int,dfm,nfm,dof,dpk,dproj,groupproj,bdsgroup,bdsproj", NULL, NULL },
{ "elisp", "el", NULL, NULL },
{ "elixir", "ex,exs", NULL, NULL },
{ "erlang", "erl,hrl", NULL, NULL },
{ "fortran", "for,ftn,fpp,f,F,f77,F77,f90,F90,f95,F95,f03,F03", NULL, NULL },
{ "gif", "gif", NULL, NULL },
{ "Gif", "gif", NULL, "GIF87a|GIF89a" },
{ "go", "go", NULL, NULL },
{ "groovy", "groovy,gtmpl,gpp,grunit,gradle", NULL, NULL },
{ "gsp", "gsp", NULL, NULL },
{ "haskell", "hs,lhs", NULL, NULL },
{ "html", "htm,html,xhtml", NULL, NULL },
{ "jade", "jade", NULL, NULL },
{ "java", "java,properties", NULL, NULL },
{ "jpeg", "jpg,jpeg", NULL, NULL },
{ "Jpeg", "jpg,jpeg", NULL, "\\xff\\xd8\\xff[\\xdb\\xe0\\xe1\\xee]" },
{ "js", "js", NULL, NULL },
{ "json", "json", NULL, NULL },
{ "jsp", "jsp,jspx,jthm,jhtml", NULL, NULL },
{ "julia", "jl", NULL, NULL },
{ "kotlin", "kt,kts", NULL, NULL },
{ "less", "less", NULL, NULL },
{ "lex", "l,ll,lmm,lpp,lxx", NULL, NULL },
{ "lisp", "lisp,lsp", NULL, NULL },
{ "lua", "lua", NULL, NULL },
{ "m4", "m4", NULL, NULL },
{ "make", "mk,mak", "makefile,Makefile,Makefile.Debug,Makefile.Release", NULL },
{ "markdown", "md", NULL, NULL },
{ "matlab", "m", NULL, NULL },
{ "node", "js", NULL, NULL },
{ "Node", "js", NULL, "#!\\h*/.*\\Wnode(\\W.*)?\\n" },
{ "objc", "m,h", NULL, NULL },
{ "objc++", "mm,h", NULL, NULL },
{ "ocaml", "ml,mli,mll,mly", NULL, NULL },
{ "parrot", "pir,pasm,pmc,ops,pod,pg,tg", NULL, NULL },
{ "pascal", "pas,pp", NULL, NULL },
{ "pdf", "pdf", NULL, NULL },
{ "Pdf", "pdf", NULL, "\\x25\\x50\\x44\\x46\\x2d" },
{ "perl", "pl,PL,pm,pod,t,psgi", NULL, NULL },
{ "Perl", "pl,PL,pm,pod,t,psgi", NULL, "#!\\h*/.*\\Wperl(\\W.*)?\\n" },
{ "php", "php,php3,php4,phtml", NULL, NULL },
{ "Php", "php,php3,php4,phtml", NULL, "#!\\h*/.*\\Wphp(\\W.*)?\\n" },
{ "png", "png", NULL, NULL },
{ "Png", "png", NULL, "\\x89PNG\\x0d\\x0a\\x1a\\x0a" },
{ "prolog", "pl,pro", NULL, NULL },
{ "python", "py", NULL, NULL },
{ "Python", "py", NULL, "#!\\h*/.*\\Wpython[23]?(\\W.*)?\\n" },
{ "r", "R", NULL, NULL },
{ "rpm", "rpm", NULL, NULL },
{ "Rpm", "rpm", NULL, "\\xed\\xab\\xee\\xdb" },
{ "rst", "rst", NULL, NULL },
{ "rtf", "rtf", NULL, NULL },
{ "Rtf", "rtf", NULL, "\\{\\rtf1" },
{ "ruby", "rb,rhtml,rjs,rxml,erb,rake,spec", "Rakefile", NULL },
{ "Ruby", "rb,rhtml,rjs,rxml,erb,rake,spec", "Rakefile", "#!\\h*/.*\\Wruby(\\W.*)?\\n" },
{ "rust", "rs", NULL, NULL },
{ "scala", "scala", NULL, NULL },
{ "scheme", "scm,ss", NULL, NULL },
{ "seed7", "sd7,s7i", NULL, NULL },
{ "shell", "sh,bash,dash,csh,tcsh,ksh,zsh,fish", NULL, NULL },
{ "Shell", "sh,bash,dash,csh,tcsh,ksh,zsh,fish", NULL, "#!\\h*/.*\\W(ba|da|t?c|k|z|fi)?sh(\\W.*)?\\n" },
{ "smalltalk", "st", NULL, NULL },
{ "sql", "sql,ctl", NULL, NULL },
{ "svg", "svg", NULL, NULL },
{ "swift", "swift", NULL, NULL },
{ "tcl", "tcl,itcl,itk", NULL, NULL },
{ "tex", "tex,cls,sty,bib", NULL, NULL },
{ "text", "text,txt,TXT,md,rst,adoc", NULL, NULL },
{ "tiff", "tif,tiff", NULL, NULL },
{ "Tiff", "tif,tiff", NULL, "\\x49\\x49\\x2a\\x00|\\x4d\\x4d\\x00\\x2a" },
{ "tt", "tt,tt2,ttml", NULL, NULL },
{ "typescript", "ts,tsx", NULL, NULL },
{ "verilog", "v,vh,sv", NULL, NULL },
{ "vhdl", "vhd,vhdl", NULL, NULL },
{ "vim", "vim", NULL, NULL },
{ "xml", "xml,xsd,xsl,xslt,wsdl,rss,svg,ent,plist", NULL, NULL },
{ "Xml", "xml,xsd,xsl,xslt,wsdl,rss,svg,ent,plist", NULL, "<\\?xml " },
{ "yacc", "y", NULL, NULL },
{ "yaml", "yaml,yml", NULL, NULL },
{ "zig", "zig,zon", NULL, NULL },
{ NULL, NULL, NULL, NULL }
};
#ifdef OS_WIN_OR_MINGW
// ugrep main() for windows to support wide string arguments and globbing
#ifdef OS_WIN
// ugrep wmain() for VS
int wmain(int argc, const wchar_t **wargv)
#else
// ugrep main() for MinGW, pick up argc and wargv from CommandLineToArgvW()
int main()
#endif
#else
// ugrep main() for non-windows
int main(int argc, const char **argv)
#endif
{
#ifdef OS_WIN_OR_MINGW
#ifdef OS_WIN_MINGW
int argc = 0;
wchar_t **wargv = CommandLineToArgvW(GetCommandLineW(), &argc);
#endif
// store UTF-8 arguments for the duration of main() and convert Unicode command line arguments wargv[] to UTF-8 arguments argv[]
const char **argv = new const char *[argc];
for (int i = 0; i < argc; ++i)
{
arg_strings.emplace_back(utf8_encode(wargv[i]));
argv[i] = arg_strings.back().c_str();
}
// handle CTRL-C
SetConsoleCtrlHandler(&sigint, TRUE);
#else
// ignore SIGPIPE
signal(SIGPIPE, SIG_IGN);
// reset color on SIGINT and SIGTERM
signal(SIGINT, sigint);
signal(SIGTERM, sigint);
#endif
try
{
init(argc, argv);
}
catch (std::exception& error)
{
abort("error: ", error.what());
}
if (flag_query)
{
if (!flag_no_messages && Static::warnings > 0)
abort("option -Q: warnings are present, specify -s to ignore");
// increase worker threads queue
flag_max_queue = 65536;
// -Q: TUI query mode
Query::query();
}
else
{
if (!flag_no_messages && flag_pager != NULL && Static::warnings > 0)
abort("option --pager: warnings are present, specify -s to ignore");
try
{
ugrep();
}
catch (reflex::regex_error& error)
{
abort("error: ", error.what());
}
catch (std::exception& error)
{
abort("error: ", error.what());
}
}
#ifdef OS_WIN_OR_MINGW
delete[] argv;
#endif
return Static::warnings > 0 ? EXIT_ERROR : Stats::found_any_file() ? EXIT_OK : EXIT_FAIL;
}
// set -1,...,-9,-10,... recursion depth flags
static void set_depth(const char *& arg)
{
const char *range = arg;
char *rest = NULL;
if (flag_max_depth > 0)
{
if (flag_min_depth == 0)
flag_min_depth = flag_max_depth;
flag_max_depth = static_cast<size_t>(strtoull(range, &rest, 10));
range = rest;
}
else
{
flag_max_depth = static_cast<size_t>(strtoull(range, &rest, 10));
range = rest;
if (*range == '-' || *range == ',')
{
flag_min_depth = flag_max_depth;
flag_max_depth = static_cast<size_t>(strtoull(range + 1, &rest, 10));
range = rest;
}
}
if (flag_min_depth > flag_max_depth)
usage("invalid argument -", arg);
arg = range - 1;
}
// set --10,... and --depth=10,... recursion depth flags
static void set_depth_long(const char *arg)
{
const char *range = arg;
set_depth(range);
if (range[1] != '\0')
usage("invalid argument --depth=", arg);
}
// load config file specified or the default .ugrep, located in the working directory or home directory
static void load_config(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, bool recurse = false)
{
// the default config file is .ugrep when FILE is not specified
if (flag_config == NULL || *flag_config == '\0')
flag_config = ".ugrep";
// if config file was parsed before, then only try parsing the home dir config file
bool home = flag_config_files.find(flag_config) != flag_config_files.end();
bool wdir = !home;
// open a config file in the working dir or in the home directory
std::string config_file(flag_config);
FILE *file = NULL;
if (wdir)
{
// try opening a config file in the working directory
if (fopen_smart(&file, flag_config, "r") == 0)
{
if (file != NULL && file != stdin)
{
// check if we own this config file located in the working directory
#ifdef OS_WIN_OR_MINGW
HANDLE hFile = reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(file)));
PSID pSidOwner = NULL;
PSECURITY_DESCRIPTOR pSD = NULL;
if (GetSecurityInfo(hFile, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION, &pSidOwner, NULL, NULL, NULL, &pSD) == ERROR_SUCCESS)
{
HANDLE hToken = NULL;
if (OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hToken))
{
DWORD dwBufferSize = 0;
GetTokenInformation(hToken, TokenUser, NULL, 0, &dwBufferSize);
PTOKEN_USER pTokenUser = reinterpret_cast<PTOKEN_USER>(HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, dwBufferSize));
if (pTokenUser != NULL)
{
if (!GetTokenInformation(hToken, TokenUser, pTokenUser, dwBufferSize, &dwBufferSize) ||
!EqualSid(pSidOwner, pTokenUser->User.Sid))
{
fclose(file);
file = NULL;
}
HeapFree(GetProcessHeap(), 0, pTokenUser);
}
CloseHandle(hToken);
}
if (pSD)
LocalFree(pSD);
}
#else
struct stat buf;
if (fstat(fileno(file), &buf) == 0 && buf.st_uid != getuid())
{
fclose(file);
file = NULL;
}
#endif
}
}
else
{
file = NULL;
}
}
if (file == NULL)
{
wdir = false;
// if not in the working directory, then check the home directory
if (Static::home_dir != NULL && *flag_config != '~' && *flag_config != PATHSEPCHR)
{
// check the home directory for the configuration file, parse only if not parsed before
config_file.assign(Static::home_dir).append(PATHSEPSTR).append(flag_config);
if (flag_config_files.find(config_file) != flag_config_files.end() ||
fopen_smart(&file, config_file.c_str(), "r") != 0)
file = NULL;
}
#if defined(WITH_XDG_CONFIG_HOME) && !defined(OS_WIN)
// if the default configuration file is to be used, then check XDG_CONFIG_HOME/ugrep/config first
if (file == NULL && strcmp(flag_config, ".ugrep") == 0)
{
// get XDG_CONFIG_HOME path, defaults to $HOME/.config/ugrep/config when present
const char *xdg_config_home = getenv("XDG_CONFIG_HOME");
if (xdg_config_home == NULL || *xdg_config_home == '\0')
xdg_config_home = ".config";
// check if $XDG_CONFIG_HOME/ugrep/config or $HOME/.config/ugrep/config is present to parse
config_file.assign(xdg_config_home).append(PATHSEPSTR).append("ugrep").append(PATHSEPSTR).append("config");
if (flag_config_files.find(config_file) != flag_config_files.end() ||
fopen_smart(&file, config_file.c_str(), "r") != 0)
file = NULL;
}
#endif
}
// parse config file
if (file != NULL)
{
// add config file path to the set
flag_config_files.insert(config_file);
reflex::BufferedInput input(file);
std::string line;
size_t lineno = 1;
bool errors = false;
while (true)
{
// read the next line
if (getline(input, line))
break;
trim(line);
// parse option or skip empty lines and comments
if (!line.empty() && line.front() != '#')
{
// construct an option argument to parse as argv[]
line.insert(0, "--");
const char *arg = line.c_str();
const char *args[2] = { NULL, arg };
Static::warnings = 0;
// warn about invalid options but do not exit
flag_usage_warnings = true;
options(pattern_args, 2, args);
if (Static::warnings > 0 ||
flag_bool || flag_fuzzy || flag_mode != Mode::UNSP ||
flag_any_line || flag_count || flag_files_with_matches || flag_files_without_match ||
flag_invert_match || flag_line_regexp || flag_only_line_number || flag_only_matching ||
flag_replace || flag_text || flag_word_regexp)
{
// reject invalid options and options that don't make sense for config
std::cerr << "ugrep: error in config file " << config_file << " at line " << lineno << '\n';
errors = true;
}
else if (line.compare(0, 8, "--config") == 0)
{
// parse a config file, but do not recurse more than one level deep
if (recurse)
{
std::cerr << "ugrep: error in config file " << config_file << " recursion at line " << lineno << '\n';
errors = true;
}
else
{
// save flag and pathname
const char *this_config = flag_config;
std::string this_file;
this_file.swap(config_file);
// load config to include in this config
flag_config = line.size() == 8 ? NULL : line.c_str() + 9;
load_config(pattern_args, true);
// restore flag and pathname
config_file.swap(this_file);
flag_config = this_config;
}
}
}
++lineno;
}
// silently ignore some options that don't make sense for config, clashing and confusing to allow
flag_file.clear();
flag_from.clear();
flag_match = false;
flag_max_count = 0;
flag_max_files = 0;
flag_max_line = 0;
flag_max_size = 0;
flag_min_line = 0;
flag_min_size = 0;
flag_pager = NULL;
flag_query = false;
flag_quiet = false;
flag_regexp.clear();
flag_save_config = NULL;
if (wdir)
{
flag_view = NULL;
flag_filter.clear();
flag_filter_magic_label.clear();
}
if (ferror(file))
error("error while reading", config_file.c_str());
if (file != stdin)
fclose(file);
if (errors)
exit(EXIT_ERROR);
}
else if (strcmp(flag_config, ".ugrep") != 0)
{
error("option --config: cannot read", flag_config);
}
flag_usage_warnings = false;
}
// save a configuration file after loading it first when present
static void save_config()
{
bool exists = false;
// rename old config file to .old when present and not standard input
if (strcmp(flag_save_config, "-") != 0)
{
std::string config_old(flag_save_config);
config_old.append(".old");
if (std::rename(flag_save_config, config_old.c_str()) == 0)
{
exists = true;
errno = EEXIST;
warning("saved old configuration file to", config_old.c_str());
}
}
FILE *file = NULL;
// if not saved to standard output ("-"), then inform user
if (!flag_no_messages && strcmp(flag_save_config, "-") != 0)
{
fprintf(Static::errout, "ugrep: %s configuration file %s", exists ? "updating" : "saving", flag_save_config);
if (!flag_config_files.empty())
{
fprintf(Static::errout, " with options based on");
for (const auto& i : flag_config_files)
fprintf(Static::errout, " %s", i.c_str());
}
fprintf(Static::errout, "\n");
}
if (fopen_smart(&file, flag_save_config, "w") != 0)
{
usage("cannot save configuration file ", flag_save_config);
return;
}
if (strcmp(flag_save_config, ".ugrep") == 0)
fprintf(file, "# ugrep configuration used by ug and ugrep --config.\n");
else if (strcmp(flag_save_config, "-") == 0)
fprintf(file, "# ugrep configuration\n");
else
fprintf(file, "# ugrep configuration used with --config=%s or ---%s.\n", flag_save_config, flag_save_config);
fprintf(file, "\
#\n\
# A long option is defined per line with an optional `=' and its argument,\n\
# when applicable. Options that change patterns and search modes are ignored.\n\
# Empty lines and lines starting with a `#' are ignored.\n\
#\n\
# Try `ug --help' or `ug --help WHAT' for help with options.\n\n");
fprintf(file, "### TERMINAL DISPLAY ###\n\n");
fprintf(file, "# Custom color scheme, overrides default GREP_COLORS parameters\n");
if (flag_colors != NULL)
fprintf(file, "colors=%s\n", flag_colors);
else
fprintf(file, "# colors=\n");
fprintf(file, "\
# The argument is a colon-separated list of one or more parameters `sl='\n\
# (selected line), `cx=' (context line), `mt=' (matched text), `ms=' (match\n\
# selected), `mc=' (match context), `fn=' (file name), `ln=' (line number),\n\
# `cn=' (column number), `bn=' (byte offset), `se=' (separator), `qp=' (TUI\n\
# prompt), `qe=' (TUI errors), `qr=' (TUI regex), `qm=' (TUI regex meta\n\
# characters), `ql=' (TUI regex lists and literals), `qb=' (TUI regex braces).\n\
# Parameter values are ANSI SGR color codes or `k' (black), `r' (red), `g'\n\
# (green), `y' (yellow), `b' (blue), `m' (magenta), `c' (cyan), `w' (white), or\n\
# leave empty for no color.\n\
# Upper case specifies background colors.\n\
# A `+' qualifies a color as bright.\n\
# A foreground and a background color may be combined with font properties `n'\n\
# (normal), `f' (faint), `h' (highlight), `i' (invert), `u' (underline).\n\
# Parameter `hl' enables file name hyperlinks (same as --hyperlink).\n\
# Parameter `rv' reverses the `sl=' and `cx=' parameters when option -v is\n\
# used.\n\
#\n\
# The ugrep default color scheme:\n\
# colors=cx=33:mt=1;31:fn=1;35:ln=1;32:cn=1;32:bn=1;32:se=36:qp=1;32:qe=1;37;41:qm=1;32:ql=36:qb=1;35\n\
# The GNU grep and ripgrep default color scheme:\n\
# colors=sl=37:cx=33:mt=1;31:fn=35:ln=32:cn=32:bn=32:se=36:qp=1;32:qe=1;37;41:qm=1;32:ql=36:qb=1;35\n\
# The silver searcher default color scheme:\n\
# colors=mt=30;43:fn=1;32:ln=1;33:cn=1;33:bn=1;33:qp=1;32:qe=1;37;41:qm=1;32:ql=36:qb=1;35\n\
# Underlined bright green matches with shaded background on bright selected lines:\n\
# colors=sl=1:cx=33:ms=1;4;32;100:mc=1;4;32:fn=1;32;100:ln=1;32:cn=1;32:bn=1;32:se=36:qp=1;32:qe=1;37;41:qm=1;32:ql=36:qb=1;35\n\
# Inverted bright yellow matches and TUI regex syntax highlighting with background colors:\n\
# colors=cx=hb:ms=hiy:mc=hic:fn=hi+y+K:ln=hg:cn=hg:bn=hg:se=c:gp=hg:qr=hwB:qm=hwG:ql=hwC:qb=hwM\n\
# Only change the TUI regex syntax highlighting to use background colors:\n\
# colors=gp=hg:qr=hwB:qm=hwG:ql=hwC:qb=hwM\n\n");
fprintf(file, "# Enable color output to the terminal\n%s\n\n", flag_color != NULL ? "color" : "no-color");
if (flag_hyperlink != NULL && *flag_hyperlink == '\0')
fprintf(file, "# Enable hyperlinks in color output\nhyperlink\n\n");
else if (flag_hyperlink != NULL)
fprintf(file, "# Enable hyperlinks in color output\nhyperlink=%s\n\n", flag_hyperlink);
fprintf(file, "# Enable query TUI confirmation prompts, default: confirm\n%sno-confirm\n\n", flag_confirm ? "# " : "");
fprintf(file, "# Split query TUI screen on startup, default: no-split\n%ssplit\n\n", flag_split ? "" : "# ");
fprintf(file, "# Default query TUI response delay in units of 100ms, default: delay=4\n");
if (flag_delay == DEFAULT_QUERY_DELAY)
fprintf(file, "# delay=4\n\n");
else
fprintf(file, "delay=%zu\n\n", flag_delay);
fprintf(file, "# Enable query TUI file viewing command with CTRL-Y or F2, default: view\n");
if (flag_view != NULL && *flag_view == '\0')
fprintf(file, "# view=less\n\n");
else if (flag_view != NULL)
fprintf(file, "view=%s\n\n", flag_view);
else
fprintf(file, "no-view\n\n");
fprintf(file, "# Enable pretty output to the terminal, default: pretty\n%s\n\n", flag_pretty != NULL ? "pretty" : "no-pretty");
fprintf(file, "# Enable directory tree output to the terminal for -l (--files-with-matches) and -c (--count)\n%s\n\n", flag_tree ? "tree" : "no-tree");
if (flag_heading.is_defined() && flag_heading != (flag_pretty != NULL))
fprintf(file, "# Enable headings (enabled with --pretty)\n%s\n\n", flag_heading ? "heading" : "no-heading");
if (flag_break.is_defined() && flag_break != (flag_pretty != NULL))
fprintf(file, "# Enable break after matching files (enabled with --pretty)\n%s\n\n", flag_break ? "break" : "no-break");
if (flag_initial_tab.is_defined() && flag_initial_tab != (flag_pretty != NULL))
fprintf(file, "# Enable initial tab (enabled with --pretty)\n%s\n\n", flag_initial_tab ? "initial-tab" : "no-initial-tab");
if (flag_line_number.is_defined() && flag_line_number != (flag_pretty != NULL))
fprintf(file, "# Enable line numbers (enabled with --pretty)\n%s\n\n", flag_line_number ? "line-number" : "no-line-number");
if (flag_column_number.is_defined())
fprintf(file, "# Enable column numbers\n%s\n\n", flag_column_number ? "column-number" : "no-column-number");
if (flag_byte_offset.is_defined())
fprintf(file, "# Enable byte offsets\n%s\n\n", flag_byte_offset ? "byte-offset" : "no-byte-offset");
if (strcmp(flag_binary_files, "hex") == 0)
fprintf(file, "# Hex output\nhex\n\n");
else if (strcmp(flag_binary_files, "with-hex") == 0)
fprintf(file, "# Output with hex for binary matches\nwith-hex\n\n");
if (flag_hexdump != NULL)
fprintf(file, "# Hex dump (columns, no space breaks, no character column, no hex spacing)\nhexdump=%s\n\n", flag_hexdump);
if (flag_group_separator == NULL)
fprintf(file, "# Disable group separator for contexts\nno-group-separator\n\n");
else if (strcmp(flag_group_separator, "--") != 0)
fprintf(file, "# Group separator for contexts\ngroup-separator=%s\n\n", flag_group_separator);
fprintf(file, "### SEARCH PATTERNS ###\n\n");
fprintf(file, "# Enable case-insensitive search, default: no-ignore-case\n%signore-case\n\n", flag_ignore_case.is_undefined() ? "# " : flag_ignore_case ? "" : "no-");
fprintf(file, "# Enable smart case, default: no-smart-case\n%ssmart-case\n\n", flag_smart_case.is_undefined() ? "# " : flag_smart_case ? "" : "no-");
fprintf(file, "# Enable empty pattern matches, default: no-empty\n%sempty\n\n", flag_empty.is_undefined() ? "# " : flag_empty ? "" : "no-");
fprintf(file, "# Force option -c (--count) to return nonzero matches with --min-count=1, default: --min-count=0\n");
if (flag_min_count == 0)
fprintf(file, "# min-count=1\n\n");
else
fprintf(file, "min-count=%zu\n\n", flag_min_count);
fprintf(file, "### SEARCH TARGETS ###\n\n");
fprintf(file, "# Case-insensitive glob matching, default: no-glob-ignore-case\n%sglob-ignore-case\n\n", flag_glob_ignore_case ? "" : "# ");
fprintf(file, "# Search hidden files and directories, default: no-hidden\n%shidden\n\n", flag_hidden ? "" : "# ");
fprintf(file, "# Ignore binary files, default: no-ignore-binary\n%signore-binary\n\n", strcmp(flag_binary_files, "without-match") == 0 ? "" : "# ");
if (!flag_include_fs.empty())
{
fprintf(file, "# Include specific file systems only\n");
for (const auto& fs : flag_include_fs)
fprintf(file, "include-fs=%s\n", fs.c_str());
fprintf(file, "\n");
}
if (!flag_exclude_fs.empty())
{
fprintf(file, "# Exclude specific file systems\n");
for (const auto& fs : flag_exclude_fs)
fprintf(file, "exclude-fs=%s\n", fs.c_str());
fprintf(file, "\n");
}
if (!flag_include_dir.empty())
{
fprintf(file, "# Include specific directories only\n");
for (const auto& d : flag_include_dir)
fprintf(file, "include-dir=%s\n", d.c_str());
fprintf(file, "\n");
}
if (!flag_exclude_dir.empty())
{
fprintf(file, "# Exclude specific directories\n");
for (const auto& d : flag_exclude_dir)
fprintf(file, "exclude-dir=%s\n", d.c_str());
fprintf(file, "\n");
}
if (!flag_include.empty())
{
fprintf(file, "# Include specific files only\n");
for (const auto& f : flag_include)
fprintf(file, "include=%s\n", f.c_str());
fprintf(file, "\n");
}
if (!flag_exclude.empty())
{
fprintf(file, "# Exclude specific files\n");
for (const auto& f : flag_exclude)
fprintf(file, "exclude-dir=%s\n", f.c_str());
fprintf(file, "\n");
}
fprintf(file, "# Enable decompression and archive search, default: no-decompress\n%sdecompress\n\n", flag_decompress ? "" : "# ");
fprintf(file, "# Maximum decompression and de-archiving nesting levels, default: zmax=1\nzmax=%zu\n\n", flag_zmax);
if (flag_dereference)
fprintf(file, "# Dereference symlinks, default: no-dereference\ndereference\n\n");
else if (flag_dereference_files)
fprintf(file, "# Dereference symlinks to files, not directories, default: no-dereference-files\ndereference-files\n\n");
fprintf(file, "# Search devices, default: devices=skip\n%sdevices=%s\n\n", flag_devices == NULL ? "# " : "", flag_devices == NULL ? "skip" : flag_devices);
if (flag_directories == NULL || strcmp(flag_directories, "read") == 0)
fprintf(file, "# Warn when searching directories specified on the command line (like grep) with directories=read\n%sdirectories=read\n\n", flag_directories == NULL ? "# " : "");
if (flag_max_depth > 0)
fprintf(file, "# Recursively search directories up to %zu levels deep\nmax-depth=%zu\n\n", flag_max_depth, flag_max_depth);
if (flag_ignore_files.empty())
{
fprintf(file, "# Ignore files and directories specified in .gitignore, default: no-ignore-files\n# ignore-files\n\n");
}
else
{
fprintf(file, "# Ignore files and directories specified in .gitignore, default: no-ignore-files\n");
for (const auto& ignore : flag_ignore_files)
fprintf(file, "ignore-files=%s\n", ignore.c_str());
fprintf(file, "\n");
}
if (!flag_filter.empty())
{
fprintf(file, "# Filter search with file format conversion tools\nfilter=%s\n\n", flag_filter.c_str());
if (!flag_filter_magic_label.empty())
{
fprintf(file, "# Filter by file signature magic bytes\n");
for (const auto& label : flag_filter_magic_label)
fprintf(file, "filter-magic-label=%s\n", label.c_str());
fprintf(file, "# Warning: filter-magic-label significantly reduces performance!\n\n");
}
}
fprintf(file, "### OUTPUT ###\n\n");
if (flag_separator != NULL)
fprintf(file, "# Separator, default: none specified to output a `:'\nseparator=%s\n\n", flag_separator);
fprintf(file, "# Sort the list of files and directories searched and matched, default: sort\n");
if (flag_sort != NULL)
fprintf(file, "sort=%s\n\n", flag_sort);
else
fprintf(file, "# sort\n\n");
if (ferror(file))
error("cannot save", flag_save_config);
if (file != stdout)
fclose(file);
if (!flag_file_type.empty() || !flag_file_extension.empty() || !flag_file_magic.empty() || !flag_glob.empty() || !flag_iglob.empty())
warning("options --file-type, --file-extension, --file-magic, --glob and --iglob are not saved to", flag_save_config);
}
// parse the command-line options
void options(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int argc, const char **argv)
{
bool options = true;
for (int i = 1; i < argc; ++i)
{
const char *arg = argv[i];
if ((*arg == '-'
#ifdef OS_WIN
|| *arg == '/'
#endif
) && arg[1] != '\0' && options)
{
bool is_grouped = true;
// parse a ugrep command-line option
while (is_grouped && *++arg != '\0')
{
switch (*arg)
{
case '-':
is_grouped = false;
if (*++arg == '\0')
{
options = false;
continue;
}
switch (*arg)
{
case '-':
break;
case 'a':
if (strcmp(arg, "after-context") == 0) // legacy form --after-context NUM
flag_after_context = strtonum(getloptarg(argc, argv, "", i), "invalid argument --after-context=");
else if (strncmp(arg, "after-context=", 14) == 0)
flag_after_context = strtonum(getloptarg(argc, argv, arg + 14, i), "invalid argument --after-context=");
else if (strcmp(arg, "all") == 0)
option_all_files();
else if (strcmp(arg, "and") == 0)
option_and(pattern_args, i, argc, argv);
else if (strncmp(arg, "and=", 4) == 0)
option_and(pattern_args, getloptarg(argc, argv, arg + 4, i));
else if (strcmp(arg, "andnot") == 0)
option_andnot(pattern_args, i, argc, argv);
else if (strncmp(arg, "andnot=", 7) == 0)
option_andnot(pattern_args, getloptarg(argc, argv, arg + 7, i));
else if (strcmp(arg, "any-line") == 0)
flag_any_line = true;
else if (strcmp(arg, "ascii") == 0)
flag_binary = true;
else
usage("invalid option --", arg, "--after-context=, all, --and, --andnot, --any-line or --ascii");
break;
case 'b':
if (strcmp(arg, "basic-regexp") == 0)
flag_mode = Mode::BRE;
else if (strcmp(arg, "before-context") == 0) // legacy form --before-context NUM
flag_before_context = strtonum(getloptarg(argc, argv, "", i), "invalid argument --before-context=");
else if (strncmp(arg, "before-context=", 15) == 0)
flag_before_context = strtonum(getloptarg(argc, argv, arg + 15, i), "invalid argument --before-context=");
else if (strcmp(arg, "best-match") == 0)
flag_best_match = true;
else if (strcmp(arg, "binary") == 0)
flag_binary = true;
else if (strcmp(arg, "binary-files") == 0) // legacy form --binary-files TYPE
flag_binary_files = strarg(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "binary-files=", 13) == 0)
flag_binary_files = strarg(getloptarg(argc, argv, arg + 13, i));
else if (strcmp(arg, "bool") == 0)
flag_bool = true;
else if (strcmp(arg, "break") == 0)
flag_break = true;
else if (strcmp(arg, "byte-offset") == 0)
flag_byte_offset = true;
else
usage("invalid option --", arg, "--basic-regexp, --before-context=, --binary, --binary-files=, --bool, --break or --byte-offset");
break;
case 'c':
if (strcmp(arg, "color") == 0 || strcmp(arg, "colour") == 0)
flag_color = Static::AUTO;
else if (strncmp(arg, "color=", 6) == 0)
flag_color = strarg(getloptarg(argc, argv, arg + 6, i));
else if (strncmp(arg, "colour=", 7) == 0)
flag_color = strarg(getloptarg(argc, argv, arg + 7, i));
else if (strncmp(arg, "colors=", 7) == 0)
flag_colors = strarg(arg + 7);
else if (strncmp(arg, "colours=", 8) == 0)
flag_colors = strarg(arg + 8);
else if (strcmp(arg, "column-number") == 0)
flag_column_number = true;
else if (strcmp(arg, "config") == 0 || strncmp(arg, "config=", 7) == 0)
; // --config is pre-parsed before other options are parsed
else if (strcmp(arg, "confirm") == 0)
flag_confirm = true;
else if (strcmp(arg, "context") == 0) // legacy form --context NUM
flag_after_context = flag_before_context = strtonum(getloptarg(argc, argv, "", i), "invalid argument --context=");
else if (strncmp(arg, "context=", 8) == 0)
flag_after_context = flag_before_context = strtonum(getloptarg(argc, argv, arg + 8, i), "invalid argument --context=");
else if (strncmp(arg, "context-separator=", 18) == 0)
flag_separator_dash = strarg(arg + 18);
else if (strcmp(arg, "count") == 0)
flag_count = true;
else if (strcmp(arg, "cpp") == 0)
flag_cpp = true;
else if (strcmp(arg, "csv") == 0)
flag_csv = true;
else if (strcmp(arg, "colors") == 0 || strcmp(arg, "colours") == 0 || strcmp(arg, "context-separator") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--color, --colors=, --column-number, --config, --confirm, --context=, context-separator=, --count, --cpp or --csv");
break;
case 'd':
if (strcmp(arg, "decompress") == 0)
flag_decompress = true;
else if (strncmp(arg, "delay=", 6) == 0)
flag_delay = strtonum(getloptarg(argc, argv, arg + 6, i), "invalid argument --delay=");
else if (strcmp(arg, "depth") == 0) // legacy form --depth NUM
set_depth_long(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "depth=", 6) == 0)
set_depth_long(getloptarg(argc, argv, arg + 6, i));
else if (strcmp(arg, "dereference") == 0)
flag_dereference = true;
else if (strcmp(arg, "dereference-files") == 0)
flag_dereference_files = true;
else if (strcmp(arg, "dereference-recursive") == 0)
flag_directories = "dereference-recurse";
else if (strcmp(arg, "devices") == 0) // legacy form --devices ACTION
flag_devices = strarg(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "devices=", 8) == 0)
flag_devices = strarg(getloptarg(argc, argv, arg + 8, i));
else if (strcmp(arg, "directories") == 0) // legacy form --directories ACTION
flag_directories = strarg(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "directories=", 12) == 0)
flag_directories = strarg(getloptarg(argc, argv, arg + 12, i));
else if (strcmp(arg, "dotall") == 0)
flag_dotall = true;
else if (strcmp(arg, "delay") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--decompress, --delay=, --depth=, --dereference, --dereference-files, --dereference-recursive, --devices=, --directories= or --dotall");
break;
case 'e':
if (strcmp(arg, "empty") == 0)
flag_empty = true;
else if (strncmp(arg, "encoding=", 9) == 0)
flag_encoding = strarg(getloptarg(argc, argv, arg + 9, i));
else if (strcmp(arg, "exclude") == 0) // legacy form --exclude GLOB
flag_exclude.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "exclude=", 8) == 0)
flag_exclude.emplace_back(getloptarg(argc, argv, arg + 8, i));
else if (strcmp(arg, "exclude-dir") == 0) // legacy form --exclude-dir GLOB
flag_exclude_dir.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "exclude-dir=", 12) == 0)
flag_exclude_dir.emplace_back(getloptarg(argc, argv, arg + 12, i));
else if (strcmp(arg, "exclude-from") == 0) // legacy form --exclude-from FILE
flag_exclude_from.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "exclude-from=", 13) == 0)
flag_exclude_from.emplace_back(getloptarg(argc, argv, arg + 13, i));
else if (strcmp(arg, "exclude-fs") == 0)
flag_exclude_fs.emplace_back();
else if (strncmp(arg, "exclude-fs=", 11) == 0)
flag_exclude_fs.emplace_back(getloptarg(argc, argv, arg + 11, i));
else if (strcmp(arg, "extended-regexp") == 0)
flag_mode = Mode::ERE;
else if (strcmp(arg, "encoding") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--empty, --encoding=, --exclude=, --exclude-dir=, --exclude-from=, --exclude-fs= or --extended-regexp");
break;
case 'f':
if (strcmp(arg, "file") == 0) // legacy form --file FILE
flag_file.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "file=", 5) == 0)
flag_file.emplace_back(getloptarg(argc, argv, arg + 5, i));
else if (strncmp(arg, "file-extension=", 15) == 0)
flag_file_extension.emplace_back(getloptarg(argc, argv, arg + 15, i));
else if (strncmp(arg, "file-magic=", 11) == 0)
flag_file_magic.emplace_back(getloptarg(argc, argv, arg + 11, i));
else if (strncmp(arg, "file-type=", 10) == 0)
flag_file_type.emplace_back(getloptarg(argc, argv, arg + 10, i));
else if (strcmp(arg, "files") == 0)
flag_files = true;
else if (strcmp(arg, "files-with-matches") == 0)
flag_files_with_matches = true;
else if (strcmp(arg, "files-without-match") == 0)
flag_files_without_match = true;
else if (strcmp(arg, "fixed-strings") == 0)
flag_mode = Mode::FIXED;
else if (strncmp(arg, "filter=", 7) == 0)
flag_filter.append(flag_filter.empty() ? "" : ",").append(getloptarg(argc, argv, arg + 7, i));
else if (strncmp(arg, "filter-magic-label=", 19) == 0)
flag_filter_magic_label.emplace_back(getloptarg(argc, argv, arg + 19, i));
else if (strncmp(arg, "format=", 7) == 0)
flag_format = strarg(getloptarg(argc, argv, arg + 7, i));
else if (strncmp(arg, "format-begin=", 13) == 0)
flag_format_begin = strarg(arg + 13);
else if (strncmp(arg, "format-close=", 13) == 0)
flag_format_close = strarg(arg + 13);
else if (strncmp(arg, "format-end=", 11) == 0)
flag_format_end = strarg(arg + 11);
else if (strncmp(arg, "format-open=", 12) == 0)
flag_format_open = strarg(arg + 12);
else if (strcmp(arg, "from") == 0) // legacy form --from FILE
flag_from.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "from=", 5) == 0)
flag_from.emplace_back(getloptarg(argc, argv, arg + 5, i));
else if (strcmp(arg, "fuzzy") == 0)
flag_fuzzy = 1;
else if (strncmp(arg, "fuzzy=", 6) == 0)
flag_fuzzy = strtofuzzy(getloptarg(argc, argv, arg + 6, i), "invalid argument --fuzzy=");
else if (strcmp(arg, "free-space") == 0)
flag_free_space = true;
else if (strcmp(arg, "file-extension") == 0 ||
strcmp(arg, "file-magic") == 0 ||
strcmp(arg, "file-type") == 0 ||
strcmp(arg, "filter") == 0 ||
strcmp(arg, "filter-magic-label") == 0 ||
strcmp(arg, "format") == 0 ||
strcmp(arg, "format-begin") == 0 ||
strcmp(arg, "format-close") == 0 ||
strcmp(arg, "format-end") == 0 ||
strcmp(arg, "format-open") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--file=, --file-extension=, --file-magic=, --file-type=, --files, --files-with-matches, --files-without-match, --fixed-strings, --filter=, --filter-magic-label=, --format=, --format-begin=, --format-close, --format-end=, --format-open=, --fuzzy or --free-space");
break;
case 'g':
if (strncmp(arg, "glob=", 5) == 0)
flag_glob.emplace_back(getloptarg(argc, argv, arg + 5, i));
else if (strcmp(arg, "glob-ignore-case") == 0)
flag_glob_ignore_case = true;
else if (strcmp(arg, "grep") == 0)
flag_empty = flag_grep = true;
else if (strcmp(arg, "group-separator") == 0)
flag_group_separator = "--";
else if (strncmp(arg, "group-separator=", 16) == 0)
flag_group_separator = strarg(arg + 16);
else if (strcmp(arg, "glob") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--glob=, --glob-ignore-case, --grep or --group-separator");
break;
case 'h':
if (strcmp(arg, "heading") == 0)
flag_heading = true;
else if (strncmp(arg, "help", 4) == 0)
help(arg[4] != '\0' ? arg + 4 : ++i < argc ? argv[i] : NULL);
else if (strcmp(arg, "hex") == 0)
flag_binary_files = "hex";
else if (strcmp(arg, "hexdump") == 0)
flag_hexdump = "2";
else if (strncmp(arg, "hexdump=", 8) == 0)
flag_hexdump = strarg(arg + 8);
else if (strcmp(arg, "hidden") == 0)
flag_hidden = true;
else if (strncmp(arg, "hyperlink=", 10) == 0)
flag_hyperlink = strarg(arg + 10);
else if (strcmp(arg, "hyperlink") == 0)
flag_hyperlink = "";
else
usage("invalid option --", arg, "--heading, --help, --hex, --hexdump, --hidden or --hyperlink");
break;
case 'i':
if (strncmp(arg, "iglob=", 6) == 0)
flag_iglob.emplace_back(getloptarg(argc, argv, arg + 6, i));
else if (strcmp(arg, "ignore-binary") == 0)
flag_binary_files = "without-match";
else if (strcmp(arg, "ignore-case") == 0)
flag_ignore_case = true;
else if (strcmp(arg, "ignore-files") == 0)
flag_ignore_files.insert(DEFAULT_IGNORE_FILE);
else if (strncmp(arg, "ignore-files=", 13) == 0)
flag_ignore_files.insert(getloptarg(argc, argv, arg + 13, i));
else if (strcmp(arg, "include") == 0) // legacy form --include GLOB
flag_include.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "include=", 8) == 0)
flag_include.emplace_back(getloptarg(argc, argv, arg + 8, i));
else if (strcmp(arg, "include-dir") == 0) // legacy form --include-dir GLOB
flag_include_dir.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "include-dir=", 12) == 0)
flag_include_dir.emplace_back(getloptarg(argc, argv, arg + 12, i));
else if (strcmp(arg, "include-from") == 0) // legacy form --include-from FILE
flag_include_from.emplace_back(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "include-from=", 13) == 0)
flag_include_from.emplace_back(getloptarg(argc, argv, arg + 13, i));
else if (strcmp(arg, "include-fs") == 0)
flag_include_fs.emplace_back();
else if (strncmp(arg, "include-fs=", 11) == 0)
flag_include_fs.emplace_back(getloptarg(argc, argv, arg + 11, i));
else if (strcmp(arg, "index") == 0)
flag_index = "safe";
else if (strncmp(arg, "index=", 6) == 0)
flag_index = strarg(getloptarg(argc, argv, arg + 6, i));
else if (strcmp(arg, "initial-tab") == 0)
flag_initial_tab = true;
else if (strcmp(arg, "invert-match") == 0)
flag_invert_match = true;
else
usage("invalid option --", arg, "--iglob=, --ignore-case, --ignore-files, --include=, --include-dir=, --include-from=, --include-fs=, --initial-tab or --invert-match");
break;
case 'j':
if (strncmp(arg, "jobs=", 5) == 0)
flag_jobs = strtonum(getloptarg(argc, argv, arg + 5, i), "invalid argument --jobs=");
else if (strcmp(arg, "json") == 0)
flag_json = true;
else if (strcmp(arg, "jobs") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--jobs= or --json");
break;
case 'l':
if (strcmp(arg, "label") == 0) // legacy form --label LABEL
flag_label = strarg(getloptarg(argc, argv, "", i));
else if (strncmp(arg, "label=", 6) == 0)
flag_label = strarg(arg + 6);
else if (strcmp(arg, "line-buffered") == 0)
flag_line_buffered = true;
else if (strcmp(arg, "line-number") == 0)
flag_line_number = true;
else if (strcmp(arg, "line-regexp") == 0)
flag_line_regexp = true;
else if (strcmp(arg, "lines") == 0)
flag_files = false;
else
usage("invalid option --", arg, "--label=, --line-buffered, --line-number, --line-regexp or --lines");
break;
case 'm':
if (strcmp(arg, "match") == 0)
flag_match = true;
else if (strncmp(arg, "max-count=", 10) == 0)
flag_max_count = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --max-count=");
else if (strncmp(arg, "max-depth=", 10) == 0)
flag_max_depth = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --max-depth=");
else if (strncmp(arg, "max-files=", 10) == 0)
flag_max_files = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --max-files=");
else if (strncmp(arg, "max-line=", 9) == 0)
flag_max_line = strtopos(getloptarg(argc, argv, arg + 9, i), "invalid argument --max-line=");
else if (strncmp(arg, "max-size=", 9) == 0)
flag_max_size = strtosize(getloptarg(argc, argv, arg + 9, i), "invalid argument --max-size=");
else if (strncmp(arg, "max-queue=", 10) == 0)
flag_max_queue = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --max-queue=");
else if (strncmp(arg, "min-count=", 10) == 0)
flag_min_count = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --min-count=");
else if (strncmp(arg, "min-depth=", 10) == 0)
flag_min_depth = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --min-depth=");
else if (strncmp(arg, "min-line=", 9) == 0)
flag_min_line = strtopos(getloptarg(argc, argv, arg + 9, i), "invalid argument --min-line=");
else if (strncmp(arg, "min-size=", 9) == 0)
flag_min_size = strtosize(getloptarg(argc, argv, arg + 9, i), "invalid argument --min-size=");
else if (strncmp(arg, "min-steal=", 10) == 0)
flag_min_steal = strtopos(getloptarg(argc, argv, arg + 10, i), "invalid argument --min-steal=");
else if (strcmp(arg, "mmap") == 0)
flag_max_mmap = MAX_MMAP_SIZE;
else if (strncmp(arg, "mmap=", 5) == 0)
flag_max_mmap = strtopos(getloptarg(argc, argv, arg + 5, i), "invalid argument --mmap=");
else if (strcmp(arg, "messages") == 0)
flag_no_messages = false;
else if (strcmp(arg, "max-count") == 0 ||
strcmp(arg, "max-depth") == 0 ||
strcmp(arg, "max-files") == 0 ||
strcmp(arg, "max-line") == 0 ||
strcmp(arg, "max-size") == 0 ||
strcmp(arg, "min-count") == 0 ||
strcmp(arg, "min-depth") == 0 ||
strcmp(arg, "min-line") == 0 ||
strcmp(arg, "min-size") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--match, --max-count=, --max-depth=, --max-files=, --max-line=, --max-size=--min-count=, --min-depth=, --min-line=, --min-size=, --mmap or --messages");
break;
case 'n':
if (strncmp(arg, "neg-regexp=", 11) == 0)
option_regexp(pattern_args, getloptarg(argc, argv, arg + 11, i), true);
else if (strcmp(arg, "not") == 0)
option_not(pattern_args, i, argc, argv);
else if (strncmp(arg, "not=", 4) == 0)
option_not(pattern_args, getloptarg(argc, argv, arg + 4, i));
else if (strcmp(arg, "no-any-line") == 0)
flag_any_line = false;
else if (strcmp(arg, "no-ascii") == 0)
flag_binary = false;
else if (strcmp(arg, "no-binary") == 0)
flag_binary = false;
else if (strcmp(arg, "no-bool") == 0)
flag_bool = false;
else if (strcmp(arg, "no-break") == 0)
flag_break = false;
else if (strcmp(arg, "no-byte-offset") == 0)
flag_byte_offset = false;
else if (strcmp(arg, "no-color") == 0 || strcmp(arg, "no-colour") == 0)
flag_color = Static::NEVER;
else if (strcmp(arg, "no-column-number") == 0)
flag_column_number = false;
else if (strcmp(arg, "no-config") == 0)
;
else if (strcmp(arg, "no-confirm") == 0)
flag_confirm = false;
else if (strcmp(arg, "no-count") == 0)
flag_count = false;
else if (strcmp(arg, "no-decompress") == 0)
flag_decompress = false;
else if (strcmp(arg, "no-dereference") == 0)
flag_no_dereference = true;
else if (strcmp(arg, "no-dereference-files") == 0)
flag_dereference_files = false;
else if (strcmp(arg, "no-dotall") == 0)
flag_dotall = false;
else if (strcmp(arg, "no-empty") == 0)
flag_empty = false;
else if (strcmp(arg, "no-encoding") == 0)
flag_encoding = NULL;
else if (strcmp(arg, "no-filename") == 0)
flag_no_filename = true;
else if (strcmp(arg, "no-files-with-matches") == 0)
flag_files_with_matches = false;
else if (strcmp(arg, "no-filter") == 0)
flag_filter.clear();
else if (strcmp(arg, "no-glob-ignore-case") == 0)
flag_glob_ignore_case = false;
else if (strcmp(arg, "no-group-separator") == 0)
flag_group_separator = NULL;
else if (strcmp(arg, "no-heading") == 0)
flag_heading = false;
else if (strcmp(arg, "no-hidden") == 0)
flag_hidden = false;
else if (strcmp(arg, "no-hyperlink") == 0)
flag_hyperlink = NULL;
else if (strcmp(arg, "no-ignore-binary") == 0)
flag_binary_files = "binary";
else if (strcmp(arg, "no-ignore-case") == 0)
flag_ignore_case = false;
else if (strcmp(arg, "no-ignore-files") == 0)
flag_ignore_files.clear();
else if (strcmp(arg, "no-index") == 0)
flag_index = NULL;
else if (strcmp(arg, "no-initial-tab") == 0)
flag_initial_tab = false;
else if (strcmp(arg, "no-invert-match") == 0)
flag_invert_match = false;
else if (strcmp(arg, "no-line-number") == 0)
flag_line_number = false;
else if (strcmp(arg, "no-only-line-number") == 0)
flag_only_line_number = false;
else if (strcmp(arg, "no-only-matching") == 0)
flag_only_matching = false;
else if (strcmp(arg, "no-messages") == 0)
flag_no_messages = true;
else if (strcmp(arg, "no-mmap") == 0)
flag_max_mmap = 0;
else if (strcmp(arg, "no-pager") == 0)
flag_pager = NULL;
else if (strcmp(arg, "no-passthru") == 0)
flag_any_line = false;
else if (strcmp(arg, "no-pretty") == 0)
flag_pretty = NULL;
else if (strcmp(arg, "no-smart-case") == 0)
flag_smart_case = false;
else if (strcmp(arg, "no-sort") == 0)
flag_sort = NULL;
else if (strcmp(arg, "no-split") == 0)
flag_split = false;
else if (strcmp(arg, "no-tree") == 0)
flag_tree = false;
else if (strcmp(arg, "no-stats") == 0)
flag_stats = NULL;
else if (strcmp(arg, "no-ungroup") == 0)
flag_ungroup = false;
else if (strcmp(arg, "no-view") == 0)
flag_view = NULL;
else if (strcmp(arg, "null") == 0)
flag_null = true;
else if (strcmp(arg, "null-data") == 0)
flag_null_data = true;
else if (strcmp(arg, "neg-regexp") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--neg-regexp=, --not, --no-any-line, --no-ascii, --no-binary, --no-bool, --no-break, --no-byte-offset, --no-color, --no-config, --no-confirm, --no-count, --no-decompress, --no-dereference, --no-dereference-files, --no-dotall, --no-encoding, --no-empty, --no-filename, --no-files-with-matches, --no-filter, --no-glob-ignore-case, --no-group-separator, --no-heading, --no-hidden, --no-hyperlink, --no-ignore-binary, --no-ignore-case, --no-ignore-files, --no-index, --no-initial-tab, --no-invert-match, --no-line-number, --no-only-line-number, --no-only-matching, --no-messages, --no-mmap, --no-pager, --no-pretty, --no-smart-case, --no-sort, --no-split, --no-stats, --no-tree, --no-ungroup, --no-view, --null or --null-data");
break;
case 'o':
if (strcmp(arg, "only-line-number") == 0)
flag_only_line_number = true;
else if (strcmp(arg, "only-matching") == 0)
flag_only_matching = true;
else
usage("invalid option --", arg, "--only-line-number or --only-matching");
break;
case 'p':
if (strcmp(arg, "pager") == 0)
flag_pager = "";
else if (strncmp(arg, "pager=", 6) == 0)
flag_pager = strarg(getloptarg(argc, argv, arg + 6, i));
else if (strcmp(arg, "passthru") == 0)
flag_any_line = true;
else if (strcmp(arg, "perl-regexp") == 0)
flag_mode = Mode::PERL;
else if (strcmp(arg, "pretty") == 0)
flag_pretty = Static::AUTO;
else if (strncmp(arg, "pretty=", 7) == 0)
flag_pretty = strarg(getloptarg(argc, argv, arg + 7, i));
else
usage("invalid option --", arg, "--pager, --passthru, --perl-regexp= or --pretty");
break;
case 'q':
if (strcmp(arg, "query") == 0)
flag_query = true;
else if (strncmp(arg, "query=", 6) == 0)
flag_query = (flag_delay = strtonum(getloptarg(argc, argv, arg + 6, i), "invalid argument --query="), true);
else if (strcmp(arg, "quiet") == 0)
flag_quiet = flag_no_messages = true;
else
usage("invalid option --", arg, "--query or --quiet");
break;
case 'r':
if (strncmp(arg, "range=", 6) == 0)
strtopos2(getloptarg(argc, argv, arg + 6, i), flag_min_line, flag_max_line, "invalid argument --range=");
else if (strcmp(arg, "recursive") == 0)
flag_directories = "recurse";
else if (strcmp(arg, "regexp") == 0) // legacy form --regexp PATTERN
option_regexp(pattern_args, getloptarg(argc, argv, "", i));
else if (strncmp(arg, "regexp=", 7) == 0)
option_regexp(pattern_args, getloptarg(argc, argv, arg + 7, i));
else if (strncmp(arg, "replace=", 8) == 0)
flag_replace = strarg(getloptarg(argc, argv, arg + 8, i));
else if (strcmp(arg, "range") == 0 ||
strcmp(arg, "replace") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--range=, --recursive, --regexp= or --replace=");
break;
case 's':
if (strcmp(arg, "save-config") == 0)
flag_save_config = ".ugrep";
else if (strncmp(arg, "save-config=", 12) == 0)
flag_save_config = strarg(getloptarg(argc, argv, arg + 12, i));
else if (strcmp(arg, "separator") == 0)
flag_separator = NULL;
else if (strncmp(arg, "separator=", 10) == 0)
flag_separator = strarg(arg + 10);
else if (strcmp(arg, "silent") == 0)
flag_quiet = flag_no_messages = true;
else if (strcmp(arg, "smart-case") == 0)
flag_smart_case = true;
else if (strcmp(arg, "sort") == 0)
flag_sort = "name";
else if (strncmp(arg, "sort=", 5) == 0)
flag_sort = strarg(getloptarg(argc, argv, arg + 5, i));
else if (strcmp(arg, "split") == 0)
flag_split = true;
else if (strcmp(arg, "stats") == 0)
flag_stats = "";
else if (strncmp(arg, "stats=", 6) == 0)
flag_stats = strarg(arg + 6);
else
usage("invalid option --", arg, "--save-config, --separator, --silent, --smart-case, --sort, --split or --stats");
break;
case 't':
if (strcmp(arg, "tabs") == 0)
flag_tabs = DEFAULT_TABS;
else if (strncmp(arg, "tabs=", 5) == 0)
flag_tabs = strtopos(getloptarg(argc, argv, arg + 5, i), "invalid argument --tabs=");
else if (strcmp(arg, "tag") == 0)
flag_tag = DEFAULT_TAG;
else if (strncmp(arg, "tag=", 4) == 0)
flag_tag = strarg(getloptarg(argc, argv, arg + 4, i));
else if (strcmp(arg, "text") == 0)
flag_binary_files = "text";
else if (strcmp(arg, "tree") == 0)
flag_tree = true;
else
usage("invalid option --", arg, "--tabs, --tag, --text or --tree");
break;
case 'u':
if (strcmp(arg, "ungroup") == 0)
flag_ungroup = true;
else
usage("invalid option --", arg, "--ungroup");
break;
case 'v':
if (strcmp(arg, "version") == 0)
version();
else if (strncmp(arg, "view=", 5) == 0)
flag_view = strarg(getloptarg(argc, argv, arg + 5, i));
else if (strcmp(arg, "view") == 0)
flag_view = "";
else
usage("invalid option --", arg, "--view or --version");
break;
case 'w':
if (strcmp(arg, "width") == 0)
flag_width = Screen::getsize();
else if (strncmp(arg, "width=", 6) == 0)
flag_width = strtopos(getloptarg(argc, argv, arg + 6, i), "invalid argument --width=");
else if (strcmp(arg, "with-filename") == 0)
flag_with_filename = true;
else if (strcmp(arg, "with-hex") == 0)
flag_binary_files = "with-hex";
else if (strcmp(arg, "word-regexp") == 0)
flag_word_regexp = true;
else
usage("invalid option --", arg, "--width, --with-filename, --with-hex or --word-regexp");
break;
case 'x':
if (strcmp(arg, "xml") == 0)
flag_xml = true;
else
usage("invalid option --", arg, "--xml");
break;
case 'z':
if (strncmp(arg, "zmax=", 5) == 0)
flag_zmax = strtopos(getloptarg(argc, argv, arg + 5, i), "invalid argument --zmax=");
else if (strcmp(arg, "zmax") == 0)
usage("missing argument for --", arg);
else
usage("invalid option --", arg, "--zmax=");
break;
default:
if (isdigit(static_cast<unsigned char>(*arg)))
set_depth_long(arg);
else
usage("invalid option --", arg);
}
break;
case 'A':
flag_after_context = strtonum(getoptarg(argc, argv, arg, i), "invalid argument -A=");
is_grouped = false;
break;
case 'a':
flag_binary_files = "text";
break;
case 'B':
flag_before_context = strtonum(getoptarg(argc, argv, arg, i), "invalid argument -B=");
is_grouped = false;
break;
case 'b':
flag_byte_offset = true;
break;
case 'C':
flag_after_context = flag_before_context = strtonum(getoptarg(argc, argv, arg, i), "invalid argument -C=");
is_grouped = false;
break;
case 'c':
flag_count = true;
break;
case 'D':
flag_devices = getoptarg(argc, argv, arg, i);
is_grouped = false;
break;
case 'd':
flag_directories = getoptarg(argc, argv, arg, i);
is_grouped = false;
break;
case 'E':
flag_mode = Mode::ERE;
break;
case 'e':
option_regexp(pattern_args, getoptarg(argc, argv, arg, i));
is_grouped = false;
break;
case 'F':
flag_mode = Mode::FIXED;
break;
case 'f':
flag_file.emplace_back(getoptarg(argc, argv, arg, i));
is_grouped = false;
break;
case 'G':
flag_mode = Mode::BRE;
break;
case 'g':
flag_glob.emplace_back(getoptarg(argc, argv, arg, i));
is_grouped = false;
break;
case 'H':
flag_with_filename = true;
break;
case 'h':
flag_no_filename = true;
break;
case 'I':
flag_binary_files = "without-match";
break;
case 'i':
flag_ignore_case = true;
break;
case 'J':
flag_jobs = strtonum(getoptarg(argc, argv, arg, i), "invalid argument -J=");
is_grouped = false;
break;
case 'j':
flag_smart_case = true;
break;
case 'K':
strtopos2(getoptarg(argc, argv, arg, i), flag_min_line, flag_max_line, "invalid argument -K=");
is_grouped = false;
break;
case 'k':
flag_column_number = true;
break;
case 'L':
flag_files_without_match = true;
break;
case 'l':
flag_files_with_matches = true;
break;
case 'M':
flag_file_magic.emplace_back(getoptarg(argc, argv, arg, i));
is_grouped = false;
break;
case 'm':
strtopos2(getoptarg(argc, argv, arg, i), flag_min_count, flag_max_count, "invalid argument -m=");
is_grouped = false;
break;
case 'N':
option_regexp(pattern_args, getoptarg(argc, argv, arg, i), true);
is_grouped = false;
break;
case 'n':
flag_line_number = true;
break;
case 'O':
flag_file_extension.emplace_back(getoptarg(argc, argv, arg, i));
is_grouped = false;
break;
case 'o':
flag_only_matching = true;
break;
case 'P':
flag_mode = Mode::PERL;
break;
case 'p':
flag_no_dereference = true;
break;
case 'Q':
if (arg[1] == '=')
{
++arg;
flag_delay = strtonum(arg + 1, "invalid argument -Q=");
flag_query = true;
is_grouped = false;
}
else
{
flag_query = true;
}
break;
case 'q':
flag_quiet = flag_no_messages = true;
break;
case 'R':
flag_directories = "dereference-recurse";
break;
case 'r':
flag_directories = "recurse";
break;
case 'S':
flag_dereference_files = true;
break;
case 's':
flag_no_messages = true;
break;
case 'T':
flag_initial_tab = true;
break;
case 't':
flag_file_type.emplace_back(getoptarg(argc, argv, arg, i));
is_grouped = false;
break;
case 'U':
flag_binary = true;
break;
case 'u':
flag_ungroup = true;
break;
case 'V':
version();
break;
case 'v':
flag_invert_match = true;
break;
case 'W':
flag_binary_files = "with-hex";
break;
case 'w':
flag_word_regexp = true;
break;
case 'X':
flag_binary_files = "hex";
break;
case 'x':
flag_line_regexp = true;
break;
case 'Y':
flag_empty = true;
break;
case 'y':
flag_any_line = true;
break;
case 'Z':
if (flag_grep)
{
flag_null = true;
}
else
{
++arg;
if (*arg == '=' || strncmp(arg, "best", 4) == 0 || isdigit(static_cast<unsigned char>(*arg)) || strchr("+-~", *arg) != NULL)
{
flag_fuzzy = strtofuzzy(&arg[*arg == '='], "invalid argument -Z=");
is_grouped = false;
}
else
{
flag_fuzzy = 1;
--arg;
}
}
break;
case 'z':
if (flag_grep)
flag_null_data = true;
else
flag_decompress = true;
break;
case '0':
flag_null_data = flag_null;
flag_null = !flag_null;
break;
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
set_depth(arg);
break;
case '?':
help(arg[1] != '\0' ? arg + 1 : ++i < argc ? argv[i] : NULL);
break;
case '%':
if (flag_bool)
flag_files = true;
flag_bool = true;
break;
case '^':
flag_tree = true;
break;
case '+':
flag_heading = true;
break;
case '.':
flag_hidden = true;
break;
case '@':
option_all_files();
break;
default:
usage("invalid option -", arg);
}
if (!is_grouped)
break;
}
}
else if (strcmp(arg, "-") == 0)
{
// read standard input
flag_stdin = true;
}
else if (Static::arg_pattern == NULL && !flag_match && !flag_not && pattern_args.empty() && flag_file.empty())
{
// no regex pattern specified yet, so assume it is PATTERN
Static::arg_pattern = arg;
}
else
{
// otherwise add the file argument to the list of FILE files
Static::arg_files.emplace_back(arg);
}
}
if (flag_not)
usage("missing PATTERN for --not");
}
// parse -e PATTERN and -N PATTERN
void option_regexp(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg, bool is_neg)
{
pattern_args.emplace_back((flag_not ? CNF::PATTERN::NOT : CNF::PATTERN::ALT) | (is_neg ? CNF::PATTERN::NEG : CNF::PATTERN::ALT), arg);
}
// parse --and [PATTERN]
void option_and(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int& i, int argc, const char **argv)
{
if (flag_not)
usage("missing PATTERN for --not");
pattern_args.emplace_back(CNF::PATTERN::AND, "");
if (i + 1 < argc && *argv[i + 1] != '-')
pattern_args.emplace_back((flag_not ? CNF::PATTERN::NOT : CNF::PATTERN::ALT), argv[++i]);
}
// parse --and=PATTERN
void option_and(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg)
{
if (flag_not)
usage("missing PATTERN for --not");
pattern_args.emplace_back(CNF::PATTERN::AND, "");
pattern_args.emplace_back((flag_not ? CNF::PATTERN::NOT : CNF::PATTERN::ALT), arg);
}
// parse --andnot [PATTERN]
void option_andnot(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int& i, int argc, const char **argv)
{
if (flag_not)
usage("missing PATTERN for --not");
pattern_args.emplace_back(CNF::PATTERN::AND, "");
flag_not = true;
if (i + 1 < argc && *argv[i + 1] != '-')
{
pattern_args.emplace_back(CNF::PATTERN::NOT, argv[++i]);
flag_not = false;
}
}
// parse --andnot=PATTERN
void option_andnot(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg)
{
if (flag_not)
usage("missing PATTERN for --not");
pattern_args.emplace_back(CNF::PATTERN::AND, "");
pattern_args.emplace_back(CNF::PATTERN::NOT, arg);
}
// parse --not [PATTERN]
void option_not(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, int& i, int argc, const char **argv)
{
flag_not = !flag_not;
if (i + 1 < argc && *argv[i + 1] != '-')
{
pattern_args.emplace_back((flag_not ? CNF::PATTERN::NOT : CNF::PATTERN::ALT), argv[++i]);
flag_not = false;
}
}
// parse --not=PATTERN
void option_not(std::list<std::pair<CNF::PATTERN,const char*>>& pattern_args, const char *arg)
{
flag_not = !flag_not;
pattern_args.emplace_back((flag_not ? CNF::PATTERN::NOT : CNF::PATTERN::ALT), arg);
flag_not = false;
}
// --all: cancel all previously specified file and directory search restrictions
void option_all_files()
{
flag_glob.clear();
flag_iglob.clear();
flag_exclude.clear();
flag_exclude_dir.clear();
flag_exclude_from.clear();
flag_include.clear();
flag_include_dir.clear();
flag_include_from.clear();
flag_file_type.clear();
flag_file_extension.clear();
flag_file_magic.clear();
flag_ignore_files.clear();
if (strcmp(flag_binary_files, "without-match") == 0)
flag_binary_files = "binary";
}
// parse the command-line options and initialize
void init(int argc, const char **argv)
{
// get home directory path to expand ~ in options with path arguments, using fopen_smart()
#ifdef OS_WIN
Static::home_dir = getenv("USERPROFILE");
#else
Static::home_dir = getenv("HOME");
#endif
// --config=FILE or ---FILE or --no-config: load configuration file first before parsing any other options
bool no_config = false;
for (int i = 1; i < argc; ++i)
{
if (strcmp(argv[i], "--") == 0)
break;
if (strcmp(argv[1], "-e") == 0 ||
strcmp(argv[1], "-g") == 0 ||
strcmp(argv[1], "-N") == 0 ||
strcmp(argv[1], "--regexp") == 0)
{
++i;
}
else if (strncmp(argv[i], "--config", 8) == 0)
{
if (flag_config != NULL)
std::cerr << "ugrep: warning: multiple configurations specified, ignoring extra " << argv[i] << '\n';
else if (argv[i][8] == '\0')
flag_config = "";
else if (argv[i][8] == '=')
flag_config = argv[i] + 9;
}
else if (strncmp(argv[i], "---", 3) == 0)
{
if (flag_config != NULL)
std::cerr << "ugrep: warning: multiple configurations specified, ignoring extra " << argv[i] << '\n';
else
flag_config = argv[i] + 3;
}
else if (strcmp(argv[i], "--no-config") == 0)
{
no_config = true;
}
}
// collect regex pattern arguments -e PATTERN, -N PATTERN, --and PATTERN, --andnot PATTERN
std::list<std::pair<CNF::PATTERN,const char*>> pattern_args;
if (flag_config != NULL)
load_config(pattern_args);
// reset warnings
Static::warnings = 0;
// apply the appropriate options when the program is named grep, egrep, fgrep, zgrep, zegrep, zfgrep
const char *program = strrchr(argv[0], PATHSEPCHR);
if (program == NULL)
program = argv[0];
else
++program;
size_t len = strlen(program);
#ifdef OS_WIN_OR_MINGW
// Windows: compare executable name up to a dot when present, e.g. "ug.exe" compares as "ug"
const char *dot = strchr(program, '.');
if (dot != NULL)
len = dot - program;
#endif
if (strncmp(program, "ug", len) == 0)
{
// the 'ug' command is equivalent to 'ugrep --config --pretty --sort' to load custom configuration files, when no --config=FILE is specified
flag_pretty = Static::AUTO;
flag_sort = "name";
if (!no_config && flag_config == NULL)
load_config(pattern_args);
}
else if (strncmp(program, "grep", len) == 0)
{
// the 'grep' command is equivalent to 'ugrep --grep -G -. --sort'
flag_mode = Mode::BRE;
flag_grep = true;
flag_empty = true;
flag_hidden = true;
flag_sort = "name";
}
else if (strncmp(program, "egrep", len) == 0)
{
// the 'egrep' command is equivalent to 'ugrep --grep -E -. --sort'
flag_grep = true;
flag_empty = true;
flag_hidden = true;
flag_sort = "name";
}
else if (strncmp(program, "fgrep", len) == 0)
{
// the 'fgrep' command is equivalent to 'ugrep --grep -F -. --sort'
flag_mode = Mode::FIXED;
flag_grep = true;
flag_empty = true;
flag_hidden = true;
flag_sort = "name";
}
else if (strncmp(program, "zgrep", len) == 0)
{
// the 'zgrep' command is equivalent to 'ugrep --decompress --grep -G -. --sort'
flag_decompress = true;
flag_mode = Mode::BRE;
flag_grep = true;
flag_empty = true;
flag_hidden = true;
flag_sort = "name";
}
else if (strncmp(program, "zegrep", len) == 0)
{
// the 'zegrep' command is equivalent to 'ugrep --decompress --grep -E -. --sort'
flag_decompress = true;
flag_grep = true;
flag_empty = true;
flag_hidden = true;
flag_sort = "name";
}
else if (strncmp(program, "zfgrep", len) == 0)
{
// the 'zfgrep' command is equivalent to 'ugrep --decompress --grep -F -. --sort'
flag_decompress = true;
flag_mode = Mode::FIXED;
flag_grep = true;
flag_empty = true;
flag_hidden = true;
flag_sort = "name";
}
// parse ugrep command-line options and arguments
options(pattern_args, argc, argv);
if (Static::warnings > 0)
{
std::cerr << "Usage: ugrep [OPTIONS] [PATTERN] [-f FILE] [-e PATTERN] [FILE ...]\n";
std::cerr << "Try `ugrep --help' or `ugrep --help WHAT' for more information\n";
exit(EXIT_ERROR);
}
// -t list: list table of types and exit
if (flag_file_type.size() == 1 && flag_file_type[0] == "list")
{
std::cerr << std::setw(12) << "FILE TYPE" << " -O EXTENSIONS, -g FILENAMES AND FILE SIGNATURE -M 'MAGIC BYTES'\n";
for (int i = 0; type_table[i].type != NULL; ++i)
{
std::cerr << std::setw(12) << type_table[i].type << " = -O " << type_table[i].extensions << '\n';
if (type_table[i].filenames)
std::cerr << std::setw(18) << "-g " << type_table[i].filenames << "\n";
if (type_table[i].magic)
std::cerr << std::setw(19) << "-M '" << type_table[i].magic << "'\n";
}
exit(EXIT_ERROR);
}
#ifndef HAVE_LIBZ
// -z: but we don't have libz
if (flag_decompress)
usage("option -z is not available in this build configuration of ugrep");
#endif
// --zmax: NUM argument exceeds limit?
if (flag_zmax > 99)
usage("option --zmax argument exceeds upper limit");
// -P is incompatible with option -Z and with -N patterns
if (flag_mode == Mode::PERL)
{
#if defined(HAVE_PCRE2) || defined(HAVE_BOOST_REGEX)
if (flag_fuzzy > 0)
usage("options -P and -Z are not compatible");
for (const auto& arg : pattern_args)
if ((arg.first & CNF::PATTERN::NEG) != 0)
usage("options -P and -N are not compatible");
#else
usage("option -P is not available in this build configuration of ugrep");
#endif
}
// -o (or -u) disables -Y to emulate grep behavior when ugrep is aliased to grep/egrep/fgrep
if (flag_only_matching || flag_ungroup)
flag_empty = false;
// -e, -N, --and, --andnot, --not
if (!pattern_args.empty())
{
if (flag_bool || flag_query)
{
// -Q: if --and, --andnot or --not specified then switch to bool mode, -F can't work with -N
for (const auto& arg : pattern_args)
{
if (arg.first == CNF::PATTERN::AND || arg.first == CNF::PATTERN::NOT)
flag_bool = true;
else if (arg.first == CNF::PATTERN::NEG && flag_mode == Mode::FIXED)
usage("option -F with -% or -Q does not support -N PATTERN");
}
// -%: collect -e, --and, --andnot, --not patterns
const char *lparen = flag_bool ? "(" : "";
const char *rparen = flag_bool ? ")" : "";
bool sep = false;
for (const auto& arg : pattern_args)
{
if (sep)
flag_regexp.append(arg.first == CNF::PATTERN::AND ? " " : "\n");
sep = true;
if (arg.first == CNF::PATTERN::ALT)
flag_regexp.append(lparen).append(arg.second).append(rparen);
else if (arg.first == CNF::PATTERN::NEG && *arg.second != '\0')
flag_regexp.append("(?^").append(arg.second).append(")");
else if (arg.first == CNF::PATTERN::NOT && *arg.second != '\0')
flag_regexp.append("-(").append(arg.second).append(")");
else
sep = false;
}
if (!flag_query)
Static::bcnf.new_pattern(CNF::PATTERN::ALT, flag_regexp.c_str());
}
else
{
// populate the CNF with the collected pattern args, each arg points to a persistent command line argv[]
for (const auto& arg : pattern_args)
{
if (arg.first == CNF::PATTERN::AND)
Static::bcnf.new_term();
else
Static::bcnf.new_pattern(arg.first, arg.second);
}
}
}
// --query: override --pager
if (flag_query)
flag_pager = NULL;
// --tree: require sort to produce directory tree
if (flag_tree && flag_sort == NULL)
flag_sort = "name";
// check TTY info and set colors (warnings and errors may occur from here on)
terminal();
// --save-config and --save-config=FILE
if (flag_save_config != NULL)
{
save_config();
exit(EXIT_OK);
}
// --separator: override : and | otherwise use default separators : and |
if (flag_separator == NULL || *flag_separator == '\0')
flag_separator = ":";
else
flag_separator_bar = flag_separator;
#ifdef OS_WIN
// save_config() and help() assume text mode, so switch to
// binary after we're no longer going to call them.
(void)_setmode(fileno(stdout), _O_BINARY);
#endif
// --encoding: parse ENCODING value
if (flag_encoding != NULL)
{
int i, j;
if (strcmp(flag_encoding, "list") == 0)
{
// list the encoding_table[]
for (i = 0; encoding_table[i].format != NULL; ++i)
std::cerr << encoding_table[i].format << '\n';
exit(EXIT_ERROR);
}
// scan the encoding_table[] for a matching encoding, case insensitive
for (i = 0; encoding_table[i].format != NULL; ++i)
{
for (j = 0; flag_encoding[j] != '\0' && encoding_table[i].format[j] != '\0'; ++j)
if (toupper(flag_encoding[j]) != toupper(encoding_table[i].format[j]))
break;
if (flag_encoding[j] == '\0' && encoding_table[i].format[j] == '\0')
break;
}
if (encoding_table[i].format == NULL)
{
std::string msg("invalid argument --encoding=ENCODING, valid arguments are");
for (j = 0; encoding_table[j].format != NULL; ++j)
msg.append(" '").append(encoding_table[j].format).append("',");
msg.pop_back();
usage(msg.c_str());
}
// encoding is the file encoding used by all input files, if no BOM is present
flag_encoding_type = encoding_table[i].encoding;
}
else if (flag_null_data)
{
flag_encoding_type = reflex::Input::file_encoding::null_data;
}
// --binary-files: normalize by assigning flags
if (strcmp(flag_binary_files, "without-match") == 0)
flag_binary_without_match = true;
else if (strcmp(flag_binary_files, "text") == 0)
flag_text = true;
else if (strcmp(flag_binary_files, "hex") == 0)
flag_hex = true;
else if (strcmp(flag_binary_files, "with-hex") == 0)
flag_with_hex = true;
else if (strcmp(flag_binary_files, "binary") != 0)
usage("invalid argument --binary-files=TYPE, valid arguments are 'binary', 'without-match', 'text', 'hex' and 'with-hex'");
// --hexdump: normalize by assigning flags
if (flag_hexdump != NULL)
{
int context = 0;
flag_hex_after = flag_after_context + 1;
flag_hex_before = flag_before_context + 1;
for (const char *s = flag_hexdump; *s != '\0'; ++s)
{
switch (*s)
{
case 'a':
flag_hex_star = true;
context = 0;
break;
case 'b':
flag_hex_hbr = flag_hex_cbr = false;
context = 0;
break;
case 'c':
flag_hex_chr = false;
context = 0;
break;
case 'h':
flag_hex_hbr = false;
context = 0;
break;
case 'A':
flag_hex_after = 2;
context = 1;
break;
case 'B':
flag_hex_before = 2;
context = 2;
break;
case 'C':
flag_hex_after = flag_hex_before = 2;
context = 3;
break;
default:
char *r = NULL;
size_t num = static_cast<size_t>(strtoull(s, &r, 10));
if (r == NULL)
usage("invalid argument --hexdump=[1-8][a][bch][A[NUM]][B[NUM]][C[NUM]]");
s = r - 1;
switch (context)
{
case 0:
flag_hex_columns = 8 * num;
if (flag_hex_columns == 0 || flag_hex_columns > MAX_HEX_COLUMNS)
usage("invalid argument --hexdump=[1-8][a][bch][A[NUM]][B[NUM]][C[NUM]]");
break;
case 1:
flag_hex_after = num + 1;
break;
case 2:
flag_hex_before = num + 1;
break;
case 3:
flag_hex_after = flag_hex_before = num + 1;
break;
}
}
}
// enable option -X if -W is not enabled
if (!flag_with_hex)
flag_hex = true;
// disable -ABC line context to use hex binary context
flag_after_context = flag_before_context = 0;
}
// --hex takes priority over --with-hex takes priority over -I takes priority over -a
if (flag_hex)
flag_with_hex = (flag_binary_without_match = flag_text = false);
else if (flag_with_hex)
flag_binary_without_match = (flag_text = false);
else if (flag_binary_without_match)
flag_text = false;
// --tabs: value should be 1, 2, 4, or 8
if (flag_tabs && flag_tabs != 1 && flag_tabs != 2 && flag_tabs != 4 && flag_tabs != 8)
usage("invalid argument --tabs=NUM, valid arguments are 1, 2, 4, or 8");
// --match: same as specifying an empty "" pattern argument
if (flag_match)
Static::arg_pattern = "";
// if no regex pattern is specified and no -e PATTERN and no -f FILE and not -Q, then exit with usage message
if (Static::arg_pattern == NULL && pattern_args.empty() && flag_file.empty() && !flag_query)
usage("no search PATTERN specified\n\n"
"Examples:\n"
" ugrep \"PATTERN\" # match PATTERN\n"
" ugrep -e \"-PATTERN\" # match -PATTERN (including the leading dash)\n"
#ifdef OS_WIN
" ugrep -e \"/PATTERN\" # match /PATTERN (including the leading slash)\n"
#endif
" ugrep \"\" # match all lines\n"
" ugrep --match # match all lines\n");
// regex PATTERN should be a FILE argument when -Q or -e PATTERN is specified
if (!flag_match && Static::arg_pattern != NULL && (flag_query || !pattern_args.empty()))
{
Static::arg_files.insert(Static::arg_files.begin(), Static::arg_pattern);
Static::arg_pattern = NULL;
}
#ifdef OS_WIN
// Windows shell does not expand wildcards in arguments, do that now (basename part only)
if (!Static::arg_files.empty())
{
std::vector<const char*> expanded_arg_files;
for (const auto& arg_file : Static::arg_files)
{
std::wstring filename(utf8_decode(arg_file));
bool has_wildcard_char = false;
size_t basename_pos;
for (basename_pos = filename.size(); basename_pos > 0; --basename_pos)
{
wchar_t ch = filename[basename_pos - 1];
if (ch == L'*' || ch == L'?')
has_wildcard_char = true;
else if (ch == L'\\' || ch == L'/' || ch == L':')
break;
}
if (!has_wildcard_char)
{
// no wildcard chars, use argument as-is
expanded_arg_files.push_back(arg_file);
continue;
}
WIN32_FIND_DATAW find_data;
HANDLE hFile = FindFirstFileExW(filename.c_str(), FindExInfoBasic, &find_data, FindExSearchNameMatch, NULL, 0);
if (hFile == INVALID_HANDLE_VALUE)
{
// glob pattern didn't match any files, use argument as-is which will trigger a warning later
expanded_arg_files.push_back(arg_file);
continue;
}
bool glob_starts_with_dot = filename[basename_pos] == L'.';
do
{
if (find_data.cFileName[0] == L'.')
{
// don't expand directories "." or ".."
if (find_data.cFileName[1] == 0 ||
(find_data.cFileName[1] == L'.' && find_data.cFileName[2] == 0))
continue;
// don't expand hidden files unless --hidden or the pattern started with '.'
if (!flag_hidden && !glob_starts_with_dot)
continue;
}
// replace glob pattern with matching filename converted to UTF-8, then add to expanded filename list
filename.erase(basename_pos);
filename += find_data.cFileName;
arg_strings.emplace_back(utf8_encode(filename));
expanded_arg_files.push_back(arg_strings.back().c_str());
} while (FindNextFileW(hFile, &find_data));
FindClose(hFile);
}
// replace the original filenames list with the expanded list
Static::arg_files.swap(expanded_arg_files);
}
#endif
if (flag_devices != NULL)
{
// -D: check ACTION value
if (strcmp(flag_devices, "skip") == 0)
flag_devices_action = Action::SKIP;
else if (strcmp(flag_devices, "read") == 0)
flag_devices_action = Action::READ;
else
usage("invalid argument -D ACTION, valid arguments are 'skip' and 'read'");
}
if (flag_directories != NULL)
{
// normalize -R (--dereference-recurse) option
if (strcmp(flag_directories, "dereference-recurse") == 0)
{
flag_directories = "recurse";
flag_dereference = true;
}
// -d: check ACTION value and set flags
if (strcmp(flag_directories, "skip") == 0)
flag_directories_action = Action::SKIP;
else if (strcmp(flag_directories, "read") == 0)
flag_directories_action = Action::READ;
else if (strcmp(flag_directories, "recurse") == 0)
flag_directories_action = Action::RECURSE;
else
usage("invalid argument -d ACTION, valid arguments are 'skip', 'read', 'recurse' and 'dereference-recurse'");
}
if (flag_index != NULL)
if (strcmp(flag_index, "safe") != 0 && strcmp(flag_index, "fast") != 0 && strcmp(flag_index, "log") != 0)
usage("invalid argument --index=MODE, valid arguments are 'safe', 'fast' and 'log'");
// --from=FILE: add file pathnames to search from FILE as if specified on the command line
for (const auto& from : flag_from)
{
if (!from.empty())
{
FILE *file = NULL;
if (fopen_smart(&file, from.c_str(), "r") != 0)
error("option --from: cannot read", from.c_str());
// read globs from the specified file or files
reflex::BufferedInput input(file);
std::string line;
while (true)
{
// read the next line
if (getline(input, line))
break;
if (!line.empty())
{
// push file pathnames imported from the file to the back of the arg files vector, saving std::string first in a list
arg_strings.emplace_back();
arg_strings.back().swap(line);
Static::arg_files.push_back(arg_strings.back().c_str());
}
}
if (file != stdin)
fclose(file);
}
}
// check all FILE arguments, warn about non-existing and non-readable files and directories
auto arg_file = Static::arg_files.begin();
while (arg_file != Static::arg_files.end())
{
#ifdef OS_WIN_OR_MINGW
DWORD attr = GetFileAttributesW(utf8_decode(*arg_file).c_str());
if (attr == INVALID_FILE_ATTRIBUTES)
{
// argument FILE does not exist
errno = ENOENT;
warning(NULL, *arg_file);
arg_file = Static::arg_files.erase(arg_file);
if (Static::arg_files.empty())
exit(EXIT_ERROR);
}
else
{
// use threads to descent into a directory
if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0)
{
if (flag_directories_action == Action::UNSP)
flag_all_threads = true;
// remove trailing path separators, if any (*arg_file points to argv[])
trim_pathname_arg(*arg_file);
}
++arg_file;
}
#else
struct stat buf;
int ret;
if (flag_no_dereference)
ret = lstat(*arg_file, &buf);
else
ret = stat(*arg_file, &buf);
if (ret != 0)
{
// the specified argument FILE does not exist
warning(NULL, *arg_file);
arg_file = Static::arg_files.erase(arg_file);
if (Static::arg_files.empty())
exit(EXIT_ERROR);
}
else
{
if (flag_no_dereference && S_ISLNK(buf.st_mode))
{
// -p: skip symlinks
arg_file = Static::arg_files.erase(arg_file);
if (Static::arg_files.empty())
exit(EXIT_ERROR);
}
#ifdef WITH_WARN_UNREADABLE_FILE_ARG
else if ((buf.st_mode & S_IRUSR) == 0)
{
// the specified argument FILE or directory is not readable, even when executing as su
errno = EACCES;
warning("cannot read", *arg_file);
arg_file = Static::arg_files.erase(arg_file);
if (Static::arg_files.empty())
exit(EXIT_ERROR);
}
#endif
else
{
// use threads to descent into a directory
if (S_ISDIR(buf.st_mode))
{
if (flag_directories_action == Action::UNSP)
flag_all_threads = true;
// remove trailing path separators, if any (*arg_file points to argv[])
trim_pathname_arg(*arg_file);
}
++arg_file;
}
}
#endif
}
if (!flag_stdin && Static::arg_files.empty())
{
if (!flag_from.empty())
{
// all --from=FILE are empty, warn (don't search recursively)
for (const auto& from : flag_from)
warning("--from: empty", from.c_str());
}
else if (isatty(STDIN_FILENO) && (flag_directories_action == Action::UNSP || flag_directories_action == Action::RECURSE))
{
// if no FILE specified when reading standard input from a TTY then enable -r if not already -r or -R
flag_directories_action = Action::RECURSE;
// recursive search with worker threads
flag_all_threads = true;
}
else
{
// if no FILE specified then read standard input
flag_stdin = true;
}
}
// normalize --cpp, --csv, --json, --xml to their corresponding --format
if (flag_cpp)
{
flag_format_begin = "const struct grep {\n const char *file;\n size_t line;\n size_t column;\n size_t offset;\n const char *match;\n} matches[] = {\n";
flag_format_open = " // %f\n";
flag_format = " { %h, %n, %k, %b, %C },\n%u";
flag_format_close = " \n";
flag_format_end = " { NULL, 0, 0, 0, NULL }\n};\n";
}
else if (flag_csv)
{
flag_format_open = "%+";
flag_format = "%[,]$%H%N%K%B%V\n%u";
}
else if (flag_json)
{
flag_format_begin = "[";
flag_format_open = "%,\n {\n %[,\n ]$%[\"file\": ]H\"matches\": [";
flag_format = "%,\n { %[, ]$%[\"line\": ]N%[\"column\": ]K%[\"offset\": ]B\"match\": %J }%u";
flag_format_close = "\n ]\n }";
flag_format_end = "\n]\n";
}
else if (flag_xml)
{
flag_format_begin = "<grep>\n";
flag_format_open = " <file%[\"]$%[ name=\"]I>\n";
flag_format = " <match%[\"]$%[ line=\"]N%[ column=\"]K%[ offset=\"]B>%X</match>\n%u";
flag_format_close = " </file>\n";
flag_format_end = "</grep>\n";
}
else if (flag_only_line_number)
{
flag_format_open = "%[fn]=%+%=";
flag_format = "%[fn]=%F%=%[ln]=%n%=%[se]=%s%=%[cn]=%K%=%[bn]=%B%=\n%u";
flag_format_close = "%R";
}
// --replace clashes with --format
if (flag_replace != NULL && flag_format != NULL)
abort("--format is not permitted with --replace");
// -v with --files is not permitted
if (flag_invert_match && flag_files)
abort("--invert-match is not permitted with --files, invert the Boolean query instead");
// --min-count is not permitted with --files
if (flag_min_count > 0 && flag_files)
abort("--min-count is not permitted with --files");
// --min-count is not permitted with -v when not combined with -q, -l, -L or -c
if (flag_min_count > 0 && flag_invert_match && !flag_quiet && !flag_files_with_matches && !flag_files_without_match && !flag_count)
abort("--min-count is not permitted with --invert-match");
#if defined(HAVE_STATVFS) || defined(HAVE_STATFS)
// --exclude-fs: add file system ids to exclude
for (const auto& mounts : flag_exclude_fs)
{
stat_fs_t buf;
if (mounts.empty())
{
if (Static::arg_files.empty())
{
// --exclude-fs without MOUNT points and no targets for recursive search: only include the working dir FS
if (stat_fs(".", &buf) == 0)
include_fs_ids.insert(fsid_to_uint64(buf.f_fsid));
}
else
{
// --exclude-fs without MOUNT points: only include FS associated with the specified file and directory targets
for (const auto& target : Static::arg_files)
if (stat_fs(target, &buf) == 0)
include_fs_ids.insert(fsid_to_uint64(buf.f_fsid));
}
}
else
{
size_t from = 0;
while (true)
{
size_t to = mounts.find(',', from);
size_t size = (to == std::string::npos ? mounts.size() : to) - from;
if (size > 0)
{
std::string mount(mounts, from, size);
if (stat_fs(mount.c_str(), &buf) == 0)
exclude_fs_ids.insert(fsid_to_uint64(buf.f_fsid));
else
warning("--exclude-fs", mount.c_str());
}
if (to == std::string::npos)
break;
from = to + 1;
}
}
}
// --include-fs: add file system ids to include
for (const auto& mounts : flag_include_fs)
{
stat_fs_t buf;
if (mounts.empty())
{
// --include-fs without MOUNT points includes the working directory
if (stat_fs(".", &buf) == 0)
include_fs_ids.insert(fsid_to_uint64(buf.f_fsid));
}
else
{
size_t from = 0;
while (true)
{
size_t to = mounts.find(',', from);
size_t size = (to == std::string::npos ? mounts.size() : to) - from;
if (size > 0)
{
std::string mount(mounts, from, size);
if (stat_fs(mount.c_str(), &buf) == 0)
include_fs_ids.insert(fsid_to_uint64(buf.f_fsid));
else
warning("--include-fs", mount.c_str());
}
if (to == std::string::npos)
break;
from = to + 1;
}
}
}
#endif
// --ignore-file=FILE when FILE are pathnames: add globs to the exclude and exclude-dir lists with gitignore rules
auto ignore_file = flag_ignore_files.begin();
while (ignore_file != flag_ignore_files.end())
{
if (ignore_file->compare("-") == 0 || ignore_file->find(PATHSEPCHR) != std::string::npos)
{
FILE *file = NULL;
if (fopen_smart(&file, ignore_file->c_str(), "r") != 0)
error("option --ignore-files: cannot read", ignore_file->c_str());
// push globs imported from the file to the back of the vectors with gitignore flag = true
import_globs(file, flag_exclude, flag_exclude_dir, true);
if (file != stdin)
fclose(file);
flag_ignore_files.erase(ignore_file++);
}
else
{
++ignore_file;
}
}
// --exclude-from=FILE: add globs to the exclude and exclude-dir lists
for (const auto& from : flag_exclude_from)
{
if (!from.empty())
{
FILE *file = NULL;
if (fopen_smart(&file, from.c_str(), "r") != 0)
error("option --exclude-from: cannot read", from.c_str());
// push globs imported from the file to the back of the vectors
import_globs(file, flag_exclude, flag_exclude_dir);
if (file != stdin)
fclose(file);
}
}
// --include-from=FILE: add globs to the include and include-dir lists
for (const auto& from : flag_include_from)
{
if (!from.empty())
{
FILE *file = NULL;
if (fopen_smart(&file, from.c_str(), "r") != 0)
error("option --include-from: cannot read", from.c_str());
// push globs imported from the file to the back of the vectors
import_globs(file, flag_include, flag_include_dir);
if (file != stdin)
fclose(file);
}
}
// -t: parse TYPES and access type table to add -O (--file-extension), -g (--glob) and -M (--file-magic) values
for (const auto& types : flag_file_type)
{
size_t from = 0;
while (true)
{
size_t to = types.find(',', from);
size_t size = (to == std::string::npos ? types.size() : to) - from;
if (size > 0)
{
bool negate = size > 1 && (types[from] == '!' || types[from] == '^');
if (negate)
{
++from;
--size;
}
std::string type(types, from, size);
size_t i = 0;
bool found = false;
bool valid = true;
// scan the type_table[] for the specified type, allowing type to be a prefix when unambiguous
for (size_t j = 0; type_table[j].type != NULL; ++j)
{
size_t typelen = strlen(type_table[j].type);
if (size <= typelen && type.compare(0, size, type_table[j].type, size) == 0)
{
// an ambiguous prefix is not a valid type
if (found)
valid = false;
found = true;
i = j;
// if full type match then we found a valid type match
if (size == typelen)
{
valid = true;
break;
}
}
}
// if not found a valid type, then find an unambiguous type with the specified filename suffix
if (!found && valid)
{
for (size_t j = 0; type_table[j].type != NULL; ++j)
{
if (islower(static_cast<unsigned char>(type_table[j].type[0])))
{
const char *s = strstr(type_table[j].extensions, type.c_str());
if (s != NULL && (s == type_table[j].extensions || *--s == ','))
{
if (found)
{
valid = false;
break;
}
found = true;
i = j;
}
}
}
}
if (!found || !valid)
{
std::string msg("invalid argument -t TYPES, valid arguments are");
for (i = 0; type_table[i].type != NULL; ++i)
msg.append(" '").append(type_table[i].type).append("',");
msg.append(" and 'list' to show a detailed list of file types");
usage(msg.c_str());
}
std::string temp(type_table[i].extensions);
if (negate)
{
temp.insert(0, "!");
size_t j = 0;
while ((j = temp.find(',', j)) != std::string::npos)
temp.insert(++j, "!");
}
flag_file_extension.emplace_back(temp);
if (type_table[i].filenames != NULL)
{
temp.assign(type_table[i].filenames);
if (negate)
{
temp.insert(0, "!");
size_t j = 0;
while ((j = temp.find(',', j)) != std::string::npos)
temp.insert(++j, "!");
}
flag_glob.emplace_back(temp);
}
if (type_table[i].magic != NULL)
{
flag_file_magic.emplace_back(type_table[i].magic);
if (negate)
flag_file_magic.back().insert(0, "!");
}
}
if (to == std::string::npos)
break;
from = to + 1;
}
}
// -O: add filename extensions as globs
for (const auto& extensions : flag_file_extension)
{
size_t from = 0;
std::string glob;
while (true)
{
size_t to;
for (to = from; to < extensions.size() && extensions[to] != ','; ++to)
{
if (extensions[to] == '[')
while (++to < extensions.size() && extensions[to] != ']')
to += extensions[to] == '\\';
else if (extensions[to] == '\\')
++to;
}
size_t size = to - from;
if (size > 0)
{
bool negate = size > 1 && (extensions[from] == '!' || extensions[from] == '^');
if (negate)
{
++from;
--size;
}
flag_glob.emplace_back(glob.assign(negate ? "^*." : "*.").append(extensions, from, size));
}
if (to >= extensions.size())
break;
from = to + 1;
}
}
// -M: file "magic bytes" regex string
std::string magic_regex;
// -M !MAGIC: combine to create a regex string
for (const auto& magic : flag_file_magic)
{
if (magic.size() > 1 && (magic.front() == '!' || magic.front() == '^'))
{
if (!magic_regex.empty())
magic_regex.push_back('|');
magic_regex.append(magic.substr(1));
// tally negative MAGIC patterns
++flag_min_magic;
}
}
// -M MAGIC: append to regex string
for (const auto& magic : flag_file_magic)
{
if (magic.size() <= 1 || (magic.front() != '!' && magic.front() != '^'))
{
if (!magic_regex.empty())
magic_regex.push_back('|');
magic_regex.append(magic);
// we have positive MAGIC patterns, so scan() is a match when flag_min_magic or greater
flag_not_magic = flag_min_magic;
}
}
// -M: create a magic matcher for the MAGIC regex to match file with magic.scan()
try
{
// construct magic_pattern DFA for -M !MAGIC and -M MAGIC
if (!magic_regex.empty())
Static::magic_pattern.assign(magic_regex, "r");
Static::magic_matcher.pattern(Static::magic_pattern);
}
catch (reflex::regex_error& error)
{
abort("option -M: ", error.what());
}
// --filter-magic-label: construct Static::filter_magic_pattern and map "magic bytes" to labels
magic_regex = "(";
// --filter-magic-label: append pattern to magic_labels, parenthesized to ensure capture indexing
for (auto& label : flag_filter_magic_label)
{
if (!label.empty())
{
size_t sep = label.find(':');
if (sep != std::string::npos && sep > 0 && sep + 1 < label.size())
{
if (!label.empty() && magic_regex.size() > 1)
magic_regex.append(")|(");
magic_regex.append(label.substr(sep + 1));
// truncate so we end up with a list of labels without patterns
label.resize(sep);
}
else
{
abort("option --filter-magic-label: invalid LABEL:MAGIC argument ", label);
}
}
}
magic_regex.push_back(')');
// --filter-magic-label: create a Static::filter_magic_pattern
try
{
// construct Static::filter_magic_pattern DFA
if (magic_regex.size() > 2)
Static::filter_magic_pattern.assign(magic_regex, "r");
}
catch (reflex::regex_error& error)
{
abort("option --filter-magic-label: ", error.what());
}
#if defined(__CYGWIN__)
// --filter: Cygwin forked process may hang when searching with multiple threads, force one worker thread
if (!flag_filter.empty())
flag_jobs = 1;
#endif
}
// check TTY info and set colors
void terminal()
{
// check if standard output is a TTY
flag_tty_term = isatty(STDOUT_FILENO) != 0;
if (flag_query)
{
// -Q: disable --quiet
flag_quiet = false;
}
else if (!flag_quiet)
{
#ifndef OS_WIN_OR_MINGW
// if not to a TTY, is output sent to a pager or to /dev/null?
if (!flag_tty_term)
{
Static::output_stat_result = fstat(STDOUT_FILENO, &Static::output_stat) == 0;
Static::output_stat_regular = Static::output_stat_result && S_ISREG(Static::output_stat.st_mode);
// if output is sent to /dev/null, then enable -q (i.e. "cheat" like GNU grep!)
struct stat dev_null_stat;
if (Static::output_stat_result &&
S_ISCHR(Static::output_stat.st_mode) &&
stat("/dev/null", &dev_null_stat) == 0 &&
Static::output_stat.st_dev == dev_null_stat.st_dev &&
Static::output_stat.st_ino == dev_null_stat.st_ino)
{
flag_quiet = true;
}
}
#endif
}
// if not to a TTY or --quiet then disable --pager
if (!flag_tty_term || flag_quiet)
flag_pager = NULL;
// --color=WHEN: normalize WHEN argument
if (flag_color != NULL)
{
if (strcmp(flag_color, "never") == 0 || strcmp(flag_color, "no") == 0 || strcmp(flag_color, "none") == 0)
flag_color = Static::NEVER;
else if (strcmp(flag_color, "always") == 0 || strcmp(flag_color, "yes") == 0 || strcmp(flag_color, "force") == 0)
flag_color = Static::ALWAYS;
else if (strcmp(flag_color, "auto") == 0 || strcmp(flag_color, "tty") == 0 || strcmp(flag_color, "if-tty") == 0)
flag_color = Static::AUTO;
else
usage("invalid argument --color=WHEN, valid arguments are 'never', 'always' and 'auto'");
}
// --pretty=WHEN: normalize WHEN argument
if (flag_pretty != NULL)
{
if (strcmp(flag_pretty, "never") == 0 || strcmp(flag_pretty, "no") == 0 || strcmp(flag_pretty, "none") == 0)
flag_pretty = NULL;
else if (strcmp(flag_pretty, "always") == 0 || strcmp(flag_pretty, "yes") == 0 || strcmp(flag_pretty, "force") == 0)
flag_pretty = Static::ALWAYS;
else if (strcmp(flag_pretty, "auto") == 0 || strcmp(flag_pretty, "tty") == 0 || strcmp(flag_pretty, "if-tty") == 0)
flag_pretty = Static::AUTO;
else
usage("invalid argument --pretty=WHEN, valid arguments are 'never', 'always' and 'auto'");
}
// whether to apply colors based on --tag, --query and --pretty
if (flag_tag != NULL)
flag_color = NULL;
if (!flag_quiet)
{
if (flag_tty_term || flag_query || flag_pretty == Static::ALWAYS)
{
if (flag_pretty != NULL)
{
// --pretty: if output is sent to a TTY then enable --color, --heading, -T, -n, --sort and --tree
// enable --color if not set yet and not --tag
if (flag_color == NULL && flag_tag == NULL)
flag_color = Static::ALWAYS;
// enable --heading if not explicitly disabled (enables --break later)
if (flag_heading.is_undefined())
flag_heading = true;
// enable -T if not explicitly disabled (initial tab)
if (flag_initial_tab.is_undefined())
flag_initial_tab = true;
// enable -n if not explicitly disabled
if (flag_line_number.is_undefined())
flag_line_number = true;
// enable --sort=name if no --sort specified
if (flag_sort == NULL)
flag_sort = "name";
// enable --tree
if (flag_tree.is_undefined())
flag_tree = true;
}
if (flag_query)
{
// --query: run the interactive query UI
// enable --heading if not explicitly disabled (enables --break later)
if (flag_heading.is_undefined())
flag_heading = true;
// enable --line-buffered to flush output immediately
flag_line_buffered = true;
}
}
// --tree: check UTF-8 terminal support, this is a guess on LANG or LC_CTYPE or LC_ALL
if (flag_tree && (flag_query || flag_files_with_matches || flag_files_without_match || flag_count))
{
const char *lang = getenv("LANG");
if (lang == NULL || strstr(lang, "UTF-8") == NULL)
{
lang = getenv("LC_CTYPE");
if (lang == NULL || strstr(lang, "UTF-8") == NULL)
{
lang = getenv("LC_ALL");
if (lang == NULL || strstr(lang, "UTF-8") == NULL)
lang = NULL;
}
}
if (lang != NULL)
{
Output::Tree::bar = "│ ";
Output::Tree::ptr = "╰╴";
Output::Tree::end = "▔ ";
}
}
// --color: (re)set flag_color depending on color term and TTY output
if (flag_color != NULL)
{
if (flag_color == Static::NEVER)
{
flag_color = NULL;
}
else
{
#ifdef OS_WIN_OR_MINGW
if (flag_tty_term)
{
#ifdef ENABLE_VIRTUAL_TERMINAL_PROCESSING
// assume we have a color terminal on Windows if isatty() is true
HANDLE hConOut = GetStdHandle(STD_OUTPUT_HANDLE);
if (hConOut != INVALID_HANDLE_VALUE)
{
#ifdef CP_UTF8
// enable UTF-8 output
SetConsoleOutputCP(CP_UTF8);
#endif
// try virtual terminal processing for ANSI SGR codes, enable colors when successful
DWORD outMode;
GetConsoleMode(hConOut, &outMode);
outMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
flag_color_term = SetConsoleMode(hConOut, outMode) != 0;
}
#endif
const char *val;
// no color when NO_COLOR is set or when ConEmuANSI is OFF
if (((val = getenv("NO_COLOR")) != NULL && *val != '\0') ||
((val = getenv("ConEmuANSI")) != NULL && strcmp(val, "OFF") == 0))
{
flag_color_term = false;
}
else if (!flag_color_term)
{
// try ANSICON or COLORTERM or ConEmuANSI is ON
if (getenv("ANSICON") != NULL ||
getenv("COLORTERM") != NULL ||
((val = getenv("ConEmuANSI")) != NULL && strcmp(val, "ON") == 0))
{
flag_color_term = true;
}
}
}
#else
// check whether we have a color terminal
if (flag_tty_term)
{
const char *val;
// if NO_COLOR is unset
if ((val = getenv("NO_COLOR")) == NULL || *val == '\0')
{
// check COLORTERM or if TERM supports color
if (getenv("COLORTERM") != NULL)
flag_color_term = true;
else if ((val = getenv("TERM")) != NULL &&
(strstr(val, "ansi") != NULL ||
strstr(val, "xterm") != NULL ||
strstr(val, "screen") != NULL ||
strstr(val, "color") != NULL))
flag_color_term = true;
}
}
#endif
if (flag_query)
{
// --query: assume that a color terminal is used, save color to use with TUI output
if (flag_color_term || flag_color == Static::ALWAYS)
flag_color_query = flag_color;
flag_color = Static::ALWAYS;
}
else if (flag_color == Static::AUTO)
{
if (flag_pretty == Static::ALWAYS)
flag_color = Static::ALWAYS;
else if (!flag_color_term && flag_save_config == NULL)
flag_color = NULL;
}
if (flag_color != NULL)
{
// get GREP_COLOR and GREP_COLORS, when defined
char *env_grep_color = NULL;
dupenv_s(&env_grep_color, "GREP_COLOR");
char *env_grep_colors = NULL;
dupenv_s(&env_grep_colors, "GREP_COLORS");
const char *grep_colors = env_grep_colors;
std::string deprecated_grep_colors;
// if GREP_COLOR is defined but not GREP_COLORS, use it
if (env_grep_colors == NULL && env_grep_color != NULL)
grep_colors = env_grep_color;
else if (grep_colors == NULL)
grep_colors = DEFAULT_GREP_COLORS;
// parse deprecated GREP_COLOR format with ANSI SGR codes for matched text only
if (grep_colors != NULL && strchr(grep_colors, '=') == NULL)
{
// use default colors then override matched text color with the specified color
deprecated_grep_colors.assign(grep_colors);
grep_colors = DEFAULT_GREP_COLORS;
}
// parse GREP_COLORS
if (grep_colors != NULL)
{
set_color(grep_colors, "sl=", color_sl); // selected line
set_color(grep_colors, "cx=", color_cx); // context line
set_color(grep_colors, "mt=", color_mt); // matched text in any line
set_color(grep_colors, "ms=", color_ms); // matched text in selected line
set_color(grep_colors, "mc=", color_mc); // matched text in a context line
set_color(grep_colors, "fn=", color_fn); // file name
set_color(grep_colors, "ln=", color_ln); // line number
set_color(grep_colors, "cn=", color_cn); // column number
set_color(grep_colors, "bn=", color_bn); // byte offset
set_color(grep_colors, "se=", color_se); // separator
set_color(grep_colors, "qp=", color_qp); // TUI prompt
set_color(grep_colors, "qe=", color_qe); // TUI errors
set_color(grep_colors, "qr=", color_qr); // TUI regex highlight
set_color(grep_colors, "qm=", color_qm); // TUI regex meta characters highlight
set_color(grep_colors, "ql=", color_ql); // TUI regex lists and literals highlight
set_color(grep_colors, "qb=", color_qb); // TUI regex braces highlight
}
// deprecated GREP_COLOR format with ANSI SGR codes for matched text only
if (!deprecated_grep_colors.empty())
set_color(deprecated_grep_colors.c_str(), "", color_ms);
// parse --colors to override GREP_COLORS
if (flag_colors != NULL && strchr(flag_colors, '=') == NULL)
{
// deprecated format with ANSI SGR codes only
set_color(flag_colors, "", color_mt); // matched text in any line
}
else if (flag_colors != NULL)
{
set_color(flag_colors, "sl=", color_sl); // selected line
set_color(flag_colors, "cx=", color_cx); // context line
set_color(flag_colors, "mt=", color_mt); // matched text in any line
set_color(flag_colors, "ms=", color_ms); // matched text in selected line
set_color(flag_colors, "mc=", color_mc); // matched text in a context line
set_color(flag_colors, "fn=", color_fn); // file name
set_color(flag_colors, "ln=", color_ln); // line number
set_color(flag_colors, "cn=", color_cn); // column number
set_color(flag_colors, "bn=", color_bn); // byte offset
set_color(flag_colors, "se=", color_se); // separator
set_color(flag_colors, "qp=", color_qp); // TUI prompt
set_color(flag_colors, "qe=", color_qe); // TUI errors
set_color(flag_colors, "qr=", color_qr); // TUI regex highlight
set_color(flag_colors, "qm=", color_qm); // TUI regex meta characters highlight
set_color(flag_colors, "ql=", color_ql); // TUI regex lists and literals highlight
set_color(flag_colors, "qb=", color_qb); // TUI regex braces highlight
}
// -v: if rv in GREP_COLORS then swap the sl and cx colors (note that rv does not match color letters)
if (flag_invert_match &&
((grep_colors != NULL && strstr(grep_colors, "rv") != NULL) ||
(flag_colors != NULL && strstr(flag_colors, "rv") != NULL)))
{
char color_tmp[COLORLEN];
copy_color(color_tmp, color_sl);
copy_color(color_sl, color_cx);
copy_color(color_cx, color_tmp);
}
// if ms= is not specified, use the mt= value
if (*color_ms == '\0')
copy_color(color_ms, color_mt);
// if mc= is not specified, use the mt= value
if (*color_mc == '\0')
copy_color(color_mc, color_mt);
// if not --hyperlink and hl or hl= is specified, then set --hyperlink
if (flag_hyperlink == NULL)
{
if (grep_colors != NULL && strstr(grep_colors, "hl=") != NULL)
flag_hyperlink = grep_colors + 3;
else if (flag_colors != NULL && strstr(flag_colors, "hl=") != NULL)
flag_hyperlink = flag_colors + 3;
else if (grep_colors != NULL && strstr(grep_colors, "hl") != NULL)
flag_hyperlink = "";
else if (flag_colors != NULL && strstr(flag_colors, "hl") != NULL)
flag_hyperlink = "";
}
// --hyperlink
set_terminal_hyperlink();
// if CSI erase line is OK (note that ne does not match color letters so strstr can be used)
if ((grep_colors == NULL || strstr(grep_colors, "ne") == NULL) && (flag_colors == NULL || strstr(flag_colors, "ne") == NULL))
color_del = "\033[K";
color_off = "\033[m";
copy_color(match_off, color_off);
if (isatty(STDERR_FILENO))
{
color_high = "\033[1m";
color_error = "\033[1;31m";
color_warning = "\033[1;35m";
color_message = "\033[1;36m";
}
if (env_grep_color != NULL)
free(env_grep_color);
if (env_grep_colors != NULL)
free(env_grep_colors);
}
}
}
}
}
// --hyperlink parsing
void set_terminal_hyperlink()
{
// set prefix, host, current working directory path to output hyperlinks with --hyperlink
if (flag_hyperlink != NULL)
{
// get current working directory path in hyperlink
char *cwd = getcwd0();
if (cwd != NULL)
{
char *path = cwd;
if (*path == PATHSEPCHR)
++path;
// get host in hyperlink
char host[80] = "localhost";
#ifdef OS_WIN_OR_MINGW
WSADATA w;
if (WSAStartup(MAKEWORD(1, 1), &w) == 0)
{
gethostname(host, sizeof(host));
WSACleanup();
}
#else
gethostname(host, sizeof(host));
#endif
flag_hyperlink_host.assign(host);
flag_hyperlink_path.append(path);
free(cwd);
// get custom prefix in hyperlink or default file://
const char *s = flag_hyperlink;
while (*s != '\0' && isalnum(static_cast<unsigned char>(*s)))
++s;
if (s == flag_hyperlink)
flag_hyperlink_prefix.assign("file");
else
flag_hyperlink_prefix.assign(flag_hyperlink, s - flag_hyperlink);
flag_hyperlink_line = *s == '+';
color_hl = "\033]8;;";
color_st = "\033\\";
}
}
}
// search the specified files, directories, and/or standard input for pattern matches, may throw an exception
void ugrep()
{
// reset stats
Stats::reset();
// --tree: reset directory tree output
Output::Tree::path.clear();
Output::Tree::depth = 0;
// clear internal combined all-include and all-exclude globs
flag_all_exclude.clear();
flag_all_include.clear();
flag_all_exclude_dir.clear();
flag_all_include_dir.clear();
// --iglob: add case-insensitive globs to all-include-iglob/all-exclude-iglob if not --glob-ignore-case
for (const auto& globs : flag_iglob)
{
size_t from = 0;
while (true)
{
size_t to;
for (to = from; to < globs.size() && globs[to] != ','; ++to)
{
if (globs[to] == '[')
while (++to < globs.size() && globs[to] != ']')
to += globs[to] == '\\';
else if (globs[to] == '\\')
++to;
}
size_t size = to - from;
if (size > 0)
{
bool negate = size > 1 && (globs[from] == '!' || globs[from] == '^');
if (negate)
{
++from;
--size;
if (globs[from + size - 1] == '/')
flag_all_exclude_dir.emplace_back(globs, from, size - 1);
else
flag_all_exclude.emplace_back(globs, from, size);
}
else
{
if (globs[from + size - 1] == '/')
flag_all_include_dir.emplace_back(globs, from, size - 1);
else
flag_all_include.emplace_back(globs, from, size);
}
}
if (to >= globs.size())
break;
from = to + 1;
}
}
// all --iglob globs are case-sensitive
flag_include_iglob_size = flag_all_include.size();
flag_exclude_iglob_size = flag_all_exclude.size();
flag_include_iglob_dir_size = flag_all_include_dir.size();
flag_exclude_iglob_dir_size = flag_all_exclude_dir.size();
// -g, --glob: add globs to all-include/all-exclude
for (const auto& globs : flag_glob)
{
size_t from = 0;
while (true)
{
size_t to;
for (to = from; to < globs.size() && globs[to] != ','; ++to)
{
if (globs[to] == '[')
while (++to < globs.size() && globs[to] != ']')
to += globs[to] == '\\';
else if (globs[to] == '\\')
++to;
}
size_t size = to - from;
if (size > 0)
{
bool negate = size > 1 && (globs[from] == '!' || globs[from] == '^');
if (negate)
{
++from;
--size;
if (globs[from + size - 1] == '/')
flag_all_exclude_dir.emplace_back(globs, from, size - 1);
else
flag_all_exclude.emplace_back(globs, from, size);
}
else
{
if (globs[from + size - 1] == '/')
flag_all_include_dir.emplace_back(globs, from, size - 1);
else
flag_all_include.emplace_back(globs, from, size);
}
}
if (to >= globs.size())
break;
from = to + 1;
}
}
// populate the internal combined all-include and all-exclude globs by appending --include and --exclude
flag_all_exclude.insert(flag_all_exclude.end(), flag_exclude.begin(), flag_exclude.end());
flag_all_include.insert(flag_all_include.end(), flag_include.begin(), flag_include.end());
// same for dirs
flag_all_exclude_dir.insert(flag_all_exclude_dir.end(), flag_exclude_dir.begin(), flag_exclude_dir.end());
flag_all_include_dir.insert(flag_all_include_dir.end(), flag_include_dir.begin(), flag_include_dir.end());
// all excluded files: normalize by moving directory globs (globs ending in a path separator /) to --exclude-dir and remove consecutive duplicates
std::string *last = NULL;
auto it = flag_all_exclude.begin();
while (it != flag_all_exclude.end())
{
// remove empty and duplicate --exclude globs
if (it->empty() || (last != NULL && it->compare(*last) == 0))
{
it = flag_all_exclude.erase(it);
if (static_cast<size_t>(std::distance(flag_all_exclude.begin(), it)) < flag_exclude_iglob_size)
--flag_exclude_iglob_size;
}
else if (it->back() == '/')
{
// move directory glob (ending in a /) from --exclude to --exclude-dir (these are all case sensitive)
flag_all_exclude_dir.emplace_back(*it);
it = flag_all_exclude.erase(it);
}
else
{
last = &*it++;
}
}
// if all --include files have paths, then add them to --include-dir to speed up searching
std::vector<std::string> all_include_dir_from_iglob;
std::vector<std::string> all_include_dir_from_glob;
// all included files: normalize by moving or adding directory globs (globs ending in a path separator /) to --include-dir and remove consecutive duplicates
bool add_wdir = false;
last = NULL;
it = flag_all_include.begin();
while (it != flag_all_include.end())
{
// remove empty and duplicate --include globs
if (it->empty() || (last != NULL && it->compare(*last) == 0))
{
it = flag_all_include.erase(it);
if (static_cast<size_t>(std::distance(flag_all_include.begin(), it)) < flag_include_iglob_size)
--flag_include_iglob_size;
}
else
{
if (it->back() == '/')
{
// move directory glob (ending in a /) from --include to --include-dir (these are all case sensitive)
flag_all_include_dir.emplace_back(*it);
it = flag_all_include.erase(it);
}
else
{
size_t last_sep = it->find_last_of('/');
// if include file has a path, then add the path part to the include-dirs, respect case (in)sensitivity
if (last_sep != std::string::npos)
{
size_t first_sep = it->find_first_of('/');
if (first_sep == last_sep && (first_sep == 0 || (first_sep == 1 && it->front() == '.')))
{
add_wdir = true;
}
else if (static_cast<size_t>(std::distance(flag_all_include.begin(), it)) < flag_include_iglob_size)
{
all_include_dir_from_iglob.emplace_back(*it, 0, last_sep);
if (first_sep == last_sep)
all_include_dir_from_iglob.back().insert(0, "./");
}
else
{
all_include_dir_from_glob.emplace_back(*it, 0, last_sep);
if (first_sep == last_sep)
all_include_dir_from_glob.back().insert(0, "./");
}
}
// if an include file glob starts with a dot, then enable --hidden
if (last_sep != std::string::npos ? it->at(last_sep + 1) == '.' : it->front() == '.')
flag_hidden = true;
last = &*it++;
}
}
}
// update include-dir to search case insensitive directories collected from --iglob
flag_include_iglob_dir_size += all_include_dir_from_iglob.size();
flag_all_include_dir.insert(flag_all_include_dir.begin(), all_include_dir_from_iglob.begin(), all_include_dir_from_iglob.end());
// search the working dir?
if (add_wdir)
flag_all_include_dir.emplace_back("./");
// update include-dir to search case sensitive directories collected from --include, --glob
flag_all_include_dir.insert(flag_all_include_dir.end(), all_include_dir_from_glob.begin(), all_include_dir_from_glob.end());
// all excluded-dir: normalize by removing trailing path separators and removing consecutive duplicates
last = NULL;
it = flag_all_exclude_dir.begin();
while (it != flag_all_exclude_dir.end())
{
while ((it->size() > 2 || (it->size() == 2 && it->front() != '.')) && it->back() == '/')
it->pop_back();
if (last != NULL && it->compare(*last) == 0)
it = flag_all_exclude_dir.erase(it);
else
last = &*it++;
}
// all included-dir: normalize by removing trailing path separators and removing consecutive duplicates
last = NULL;
it = flag_all_include_dir.begin();
while (it != flag_all_include_dir.end())
{
while ((it->size() > 2 || (it->size() == 2 && it->front() != '.')) && it->back() == '/')
it->pop_back();
if (last != NULL && it->compare(*last) == 0)
it = flag_all_include_dir.erase(it);
else
last = &*it++;
}
// if an include (sub)dir glob starts with a dot, then enable --hidden
if (!flag_hidden)
{
for (const auto& dir : flag_all_include_dir)
{
if ((dir.front() == '.' && dir.compare(0, 2, "./") != 0) || dir.find("/.") != std::string::npos)
{
flag_hidden = true;
break;
}
}
}
// if --include-dir is nonempty, and -d unspecified, then we should recurse into directories
if (!flag_all_include_dir.empty() && flag_directories_action == Action::UNSP)
flag_directories_action = Action::RECURSE;
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
// -z with -M or -O/-t/-g/--include: add globs to also search archive contents for matching files
if (flag_decompress && (!flag_file_magic.empty() || !flag_all_include.empty()))
{
flag_all_include.emplace_back("*.cpio");
flag_all_include.emplace_back("*.pax");
flag_all_include.emplace_back("*.tar");
flag_all_include.emplace_back("*.zip");
flag_all_include.emplace_back("*.zipx");
flag_all_include.emplace_back("*.ZIP");
#ifndef WITH_NO_7ZIP
flag_all_include.emplace_back("*.7z");
flag_all_include.emplace_back("*.7Z");
#endif
flag_all_include.emplace_back("*.cpio.gz");
flag_all_include.emplace_back("*.pax.gz");
flag_all_include.emplace_back("*.tar.gz");
flag_all_include.emplace_back("*.taz");
flag_all_include.emplace_back("*.tgz");
flag_all_include.emplace_back("*.tpz");
flag_all_include.emplace_back("*.cpio.Z");
flag_all_include.emplace_back("*.pax.Z");
flag_all_include.emplace_back("*.tar.Z");
flag_all_include.emplace_back("*.cpio.zip");
flag_all_include.emplace_back("*.pax.zip");
flag_all_include.emplace_back("*.tar.zip");
#ifdef HAVE_LIBBZ2
flag_all_include.emplace_back("*.cpio.bz");
flag_all_include.emplace_back("*.pax.bz");
flag_all_include.emplace_back("*.tar.bz");
flag_all_include.emplace_back("*.cpio.bz2");
flag_all_include.emplace_back("*.pax.bz2");
flag_all_include.emplace_back("*.tar.bz2");
flag_all_include.emplace_back("*.cpio.bzip2");
flag_all_include.emplace_back("*.pax.bzip2");
flag_all_include.emplace_back("*.tar.bzip2");
flag_all_include.emplace_back("*.tb2");
flag_all_include.emplace_back("*.tbz");
flag_all_include.emplace_back("*.tbz2");
flag_all_include.emplace_back("*.tz2");
#endif
#ifdef HAVE_LIBLZMA
flag_all_include.emplace_back("*.cpio.lzma");
flag_all_include.emplace_back("*.pax.lzma");
flag_all_include.emplace_back("*.tar.lzma");
flag_all_include.emplace_back("*.cpio.xz");
flag_all_include.emplace_back("*.pax.xz");
flag_all_include.emplace_back("*.tar.xz");
flag_all_include.emplace_back("*.tlz");
flag_all_include.emplace_back("*.txz");
#endif
#ifdef HAVE_LIBLZ4
flag_all_include.emplace_back("*.cpio.lz4");
flag_all_include.emplace_back("*.pax.lz4");
flag_all_include.emplace_back("*.tar.lz4");
#endif
#ifdef HAVE_LIBZSTD
flag_all_include.emplace_back("*.cpio.zst");
flag_all_include.emplace_back("*.pax.zst");
flag_all_include.emplace_back("*.tar.zst");
flag_all_include.emplace_back("*.cpio.zstd");
flag_all_include.emplace_back("*.pax.zstd");
flag_all_include.emplace_back("*.tar.zstd");
flag_all_include.emplace_back("*.tzst");
#endif
#ifdef HAVE_LIBBROTLI
flag_all_include.emplace_back("*.cpio.br");
flag_all_include.emplace_back("*.pax.br");
flag_all_include.emplace_back("*.tar.br");
#endif
#ifdef HAVE_LIBBZIP3
flag_all_include.emplace_back("*.cpio.bz3");
flag_all_include.emplace_back("*.pax.bz3");
flag_all_include.emplace_back("*.tar.bz3");
#endif
}
#endif
#endif
// keep track which globs are case-insensitive when --glob-ignore-case is specified, i.e. not --ignore-file globs
if (flag_glob_ignore_case)
{
flag_exclude_iglob_size = flag_all_exclude.size();
flag_exclude_iglob_dir_size = flag_all_exclude_dir.size();
flag_include_iglob_size = flag_all_include.size();
flag_include_iglob_dir_size = flag_all_include_dir.size();
}
// --sort: check sort KEY and set flags
if (flag_sort != NULL)
{
flag_sort_rev = *flag_sort == 'r' || *flag_sort == '^';
const char *sort_by = flag_sort + flag_sort_rev;
if (strcmp(sort_by, "name") == 0)
flag_sort_key = Sort::NAME;
else if (strcmp(sort_by, "best") == 0)
flag_sort_key = Sort::BEST;
else if (strcmp(sort_by, "size") == 0)
flag_sort_key = Sort::SIZE;
else if (strcmp(sort_by, "used") == 0 || strcmp(sort_by, "atime") == 0)
flag_sort_key = Sort::USED;
else if (strcmp(sort_by, "changed") == 0 || strcmp(sort_by, "mtime") == 0)
flag_sort_key = Sort::CHANGED;
else if (strcmp(sort_by, "created") == 0 || strcmp(sort_by, "ctime") == 0)
flag_sort_key = Sort::CREATED;
else if (strcmp(sort_by, "list") == 0)
flag_sort_key = Sort::LIST;
else
usage("invalid argument --sort=KEY, valid arguments are 'name', 'best', 'size', 'used' ('atime'), 'changed' ('mtime'), 'created' ('ctime'), 'list', 'rname', 'rbest', 'rsize', 'rused' ('ratime'), 'rchanged' ('rmtime'), 'rcreated' ('rctime') and 'rlist'");
}
// add PATTERN to the CNF
if (Static::arg_pattern != NULL)
Static::bcnf.new_pattern(CNF::PATTERN::ALT, Static::arg_pattern);
// the regex compiled from PATTERN, -e PATTERN, -N PATTERN, and -f FILE
std::string regex;
if (Static::bcnf.defined())
{
// prune empty terms from the CNF that match anything
Static::bcnf.prune();
// split the patterns at newlines, standard grep behavior
Static::bcnf.split();
if (flag_file.empty())
{
// the CNF patterns to search, this matches more than necessary to support multiline matching and to highlight all matches in color
regex.assign(Static::bcnf.adjoin());
// an empty pattern specified matches every line, including empty lines
if (regex.empty())
{
if (Static::bcnf.empty())
{
if (!flag_hex)
flag_only_matching = false;
}
else if (!flag_quiet && !flag_files_with_matches && !flag_files_without_match)
{
if (flag_hex)
regex.assign("(?-u)[^\\n]*\\n?"); // include trailing \n of a line when outputting hex per line
else
regex.assign("(?-u)[^\\n]*");
}
// an empty pattern matches every line, including empty lines
flag_empty = true;
flag_dotall = false;
}
// CNF is empty if all patterns are empty, i.e. match anything unless -f FILE specified
if (Static::bcnf.empty())
{
flag_match = true;
flag_dotall = false;
}
}
else
{
// -f FILE is combined with -e, --and, --andnot, --not
if (Static::bcnf.first_empty())
{
if (!flag_quiet && !flag_files_with_matches && !flag_files_without_match)
{
if (flag_hex)
regex.assign("(?-u)[^\\n]*\\n?"); // include trailing \n of a line when outputting hex per line
else
regex.assign("(?-u)[^\\n]*");
}
// an empty pattern specified with -e '' matches every line, including empty lines
flag_empty = true;
}
else
{
// for efficiency, take only the first CNF OR-list terms to search in combination with -f FILE patterns
regex.assign(Static::bcnf.first());
}
}
}
// -f: get patterns from file
if (!flag_file.empty())
{
// -F: make newline-separated lines in regex literal with \Q and \E
bool fixed_strings = flag_mode == Mode::FIXED;
const char *bar = flag_mode == Mode::BRE ? "\\|" : "|";
// PATTERN or -e PATTERN: add an ending '|' (or BRE '\|') to the regex to concatenate sub-expressions
if (!regex.empty())
{
// -F does not apply to patterns in -f FILE when PATTERN or -e PATTERN is specified
fixed_strings = false;
regex.append(bar);
}
// -f: read patterns from the specified file or files
for (const auto& filename : flag_file)
{
FILE *file = NULL;
if (fopen_smart(&file, filename.c_str(), "r") != 0)
{
// could not open, try GREP_PATH environment variable
char *env_grep_path = NULL;
dupenv_s(&env_grep_path, "GREP_PATH");
if (env_grep_path != NULL)
{
if (fopen_smart(&file, std::string(env_grep_path).append(PATHSEPSTR).append(filename).c_str(), "r") != 0)
file = NULL;
free(env_grep_path);
}
}
#ifdef GREP_PATH
if (file == NULL)
{
if (fopen_smart(&file, std::string(GREP_PATH).append(PATHSEPSTR).append(filename).c_str(), "r") != 0)
file = NULL;
}
#endif
if (file == NULL)
throw std::runtime_error(std::string("option -f: cannot read ").append(filename)); // to catch in query UI
reflex::BufferedInput input(file);
std::string line;
while (true)
{
// read the next line
if (getline(input, line))
break;
// add line to the regex if not empty
if (!line.empty())
{
if (fixed_strings)
CNF::quote(line);
regex.append(line).append(bar);
}
}
if (file != stdin)
fclose(file);
}
if (regex.empty())
{
if (Static::bcnf.empty())
{
if (!flag_hex)
flag_only_matching = false;
}
else if (!flag_quiet && !flag_files_with_matches && !flag_files_without_match)
{
if (flag_hex)
regex.assign("(?-u)[^\\n]*\\n?"); // include trailing \n of a line when outputting hex per line
else
regex.assign("(?-u)[^\\n]*");
}
// an empty pattern matches every line, including empty lines
flag_empty = true;
flag_dotall = false;
// CNF is empty if all patterns are empty
if (Static::bcnf.empty())
{
flag_match = true;
flag_word_regexp = false;
}
}
else
{
// pop unused ending '|' (or BRE '\|') from the |-concatenated regexes in the regex string
regex.pop_back();
if (flag_mode == Mode::BRE)
regex.pop_back();
}
// -x or -w: add line or word anchors
CNF::anchor(regex);
}
// patterns ^, $ and ^$ are special cases to optimize for search
if (flag_mode != Mode::FIXED && Static::bcnf.singleton_or_undefined())
{
if (regex == "^$")
{
flag_match = true;
flag_line_regexp = true;
regex.clear();
}
else if (regex == "^" || regex == "$")
{
flag_match = true;
regex.clear();
}
}
// -x or --match: enable -Y and disable --dotall and -w
if (flag_line_regexp || flag_match)
{
flag_empty = true;
flag_dotall = false;
flag_word_regexp = false;
}
// --match: adjust color highlighting to show matches as selected lines without color
if (flag_match)
{
copy_color(match_ms, color_sl);
copy_color(match_mc, color_cx);
copy_color(match_off, color_off);
}
else
{
// --tag: output tagged matches instead of colors
if (flag_tag != NULL)
{
const char *s1 = strchr(flag_tag, ',');
const char *s2 = s1 != NULL ? strchr(s1 + 1, ',') : NULL;
copy_color(match_ms, flag_tag);
if (s1 == NULL)
{
copy_color(match_mc, flag_tag);
copy_color(match_off, flag_tag);
}
else
{
copy_color(match_off, s1 + 1);
if (s2 == NULL)
copy_color(match_mc, match_ms);
else
copy_color(match_mc, s2 + 1);
}
}
else
{
copy_color(match_ms, color_ms);
copy_color(match_mc, color_mc);
copy_color(match_off, color_off);
}
}
// patterns .* and .+ are common easy special cases to optimize for search
if (flag_mode != Mode::FIXED && !flag_dotall && Static::bcnf.singleton_or_undefined())
{
if (regex == ".*")
{
flag_match = true;
flag_empty = true;
regex.clear();
}
else if (regex == ".+")
{
flag_match = true;
regex.clear();
}
}
// -i: disable -j (unconditional overrides conditional)
if (flag_ignore_case)
flag_smart_case = false;
// -j: smart case insensitive search if regex does not contain a Unicode upper case letter
if (flag_smart_case)
{
flag_ignore_case = true;
for (const char *s = regex.c_str(); *s != '\0'; ++s)
{
if (*s == '\\')
{
++s;
}
else if (*s == '{')
{
while (*++s != '\0' && *s != '}')
continue;
}
else
{
int c = reflex::utf8(s, &s);
if (reflex::Unicode::tolower(c) != c)
{
flag_ignore_case = false;
break;
}
--s;
}
}
}
// -v or -y: disable -o and -u
if (flag_invert_match || flag_any_line)
flag_only_matching = flag_ungroup = false;
// --match: when matching everything disable -ABC unless -o or -x
if (flag_match && !flag_only_matching && !flag_line_regexp)
flag_after_context = flag_before_context = 0;
// -y: disable -ABC
if (flag_any_line)
flag_after_context = flag_before_context = 0;
// -o and --replace: disable -ABC
if (flag_only_matching && flag_replace != NULL)
flag_after_context = flag_before_context = 0;
// --depth: if -R or -r is not specified then enable -r
if (flag_min_depth > 0 || flag_max_depth > 0)
flag_directories_action = Action::RECURSE;
// --index: enable -r if -r or -R is not enabled, reject -dskip
if (flag_index != NULL)
{
if (flag_directories_action == Action::SKIP)
usage("option --index searches recursively, but option -dskip is specified");
flag_directories_action = Action::RECURSE;
}
// -p (--no-dereference) and -S (--dereference-files): -p takes priority over -S and -R
if (flag_no_dereference)
flag_dereference = flag_dereference_files = false;
// output file name if more than one input file is specified or options -R, -r, and option -h --no-filename is not specified
if (!flag_no_filename && (flag_all_threads || flag_directories_action == Action::RECURSE || Static::arg_files.size() > 1 || (flag_stdin && !Static::arg_files.empty())))
flag_with_filename = true;
// if no output options -H, -n, -k, -b are set, enable --no-header to suppress headers for speed
if (!flag_with_filename && !flag_line_number && !flag_column_number && !flag_byte_offset)
flag_no_header = true;
// -q: we only need to find one matching file and we're done
if (flag_quiet)
{
flag_max_files = 1;
// -q overrides -l and -L
flag_files_with_matches = false;
flag_files_without_match = false;
// disable --format options
flag_format_begin = NULL;
flag_format_open = NULL;
flag_format = NULL;
flag_format_close = NULL;
flag_format_end = NULL;
// disable sort, unnecessary
flag_sort_key = Sort::NA;
}
// -o and --format: enable -u to ungroup matches
if (flag_only_matching && flag_format != NULL)
flag_ungroup = true;
// -L: enable -l and flip -v i.e. -L=-lv and -l=-Lv
if (flag_files_without_match)
{
flag_files_with_matches = true;
flag_invert_match = !flag_invert_match;
}
// -l or -L: enable -H, disable -c
if (flag_files_with_matches)
{
flag_with_filename = true;
flag_count = false;
}
// --heading: enable --break when filenames are shown
if (flag_heading && flag_with_filename && flag_break.is_undefined())
flag_break = true;
// get number of CPU cores, returns 0 if unknown
#if defined(__APPLE__) && defined(HAVE_NEON)
// apple silicon with 8 or more cores should be reduced to 4 P cores
Static::cores = 4;
#else
Static::cores = std::thread::hardware_concurrency();
#endif
// -J: the default is the number of cores, minus one for >= 8 cores, limited to DEFAULT_MAX_JOBS
if (flag_jobs == 0)
{
flag_jobs = Static::cores - (Static::cores >= 8);
// we want at least 2 threads to be available for workers if needed, but not more than DEFAULT_MAX_JOBS
if (flag_jobs < 2)
flag_jobs = 2;
else if (flag_jobs > DEFAULT_MAX_JOBS)
flag_jobs = DEFAULT_MAX_JOBS;
}
// --sort and --max-filea and not -l or -L or -c: limit number of threads to --max-files to prevent unordered results, this is a special case
if (flag_sort_key != Sort::NA && flag_max_files > 0 && !flag_files_with_matches && !flag_count)
flag_jobs = std::min(flag_jobs, flag_max_files);
// set the number of threads to the number of files or when recursing to the value of -J, --jobs
if (flag_all_threads || flag_directories_action == Action::RECURSE)
Static::threads = flag_jobs;
else
Static::threads = std::min(Static::arg_files.size() + flag_stdin, flag_jobs);
// inverted character classes and \s do not match newlines, e.g. [^x] matches anything except x and \n
reflex::convert_flag_type convert_flags = reflex::convert_flag::notnewline;
// not -U: convert regex to Unicode
if (!flag_binary)
convert_flags |= reflex::convert_flag::unicode;
// -G: convert basic regex (BRE) to extended regex (ERE)
if (flag_mode == Mode::BRE)
convert_flags |= reflex::convert_flag::basic;
// set reflex::Pattern options to enable multiline mode
std::string pattern_options("(?m");
// -i: case insensitive reflex::Pattern option
if (flag_ignore_case)
pattern_options.push_back('i');
// --dotall and not --match (or empty pattern): dot matches newline
if (flag_dotall)
pattern_options.push_back('s');
// --free-space: convert_flags is needed to check free-space conformance by the converter
if (flag_free_space)
{
convert_flags |= reflex::convert_flag::freespace;
pattern_options.push_back('x');
}
// prepend the pattern options (?m...) to the regex
pattern_options.push_back(')');
regex.insert(0, pattern_options);
// reflex::Matcher options
std::string matcher_options;
// grep compatibility mode: literally match closing ) like GNU grep when not paired with ( unless -F or -P
if (flag_grep && flag_mode != Mode::FIXED && flag_mode != Mode::PERL)
convert_flags |= reflex::convert_flag::closing;
// -Y or --empty: permit empty pattern matches
if (flag_empty)
matcher_options.push_back('N');
// -w: match whole words
if (flag_word_regexp)
matcher_options.push_back('W');
// --tabs: set reflex::Matcher option T to NUM (1, 2, 4, or 8) tab size
if (flag_tabs)
matcher_options.append("T=").push_back(static_cast<char>(flag_tabs) + '0');
// --format-begin
if (flag_format_begin != NULL)
Output(Static::output).format(flag_format_begin, 0);
// --index: search is only possible with compatible options
if (flag_index != NULL)
{
if (flag_mode == Mode::PERL)
usage("options --index and -P (--perl-regexp) are not compatible");
if (!flag_filter.empty())
usage("options --index and --filter are not compatible");
if (flag_fuzzy > 0)
usage("options --index and -Z (--fuzzy) are not compatible");
if (flag_invert_match)
usage("options --index and -v (--invert-match) are not compatible");
if (flag_encoding != NULL)
usage("options --index and --encoding are not compatible");
// -c and --index: force --min-count larger than 0, because indexed search skips non-matching files
if (flag_count && flag_min_count == 0)
flag_min_count = 1;
}
if (flag_match)
{
// --match: match lines
if (flag_line_regexp)
{
// -xv and --match: don't match empty lines (remove LineMatcher option N)
if (flag_invert_match)
matcher_options.clear();
else // -x and --match: only match empty lines (set LineMatcher option W)
matcher_options.push_back('X');
// do not invert when searching
flag_invert_match = false;
}
// --match and -v matches nothing, but -y and also -c should still list files when --min-count > 0
if (!flag_invert_match || flag_any_line || (flag_count && flag_min_count == 0))
{
// -X (--hex) and --match: also output terminating \n in hexdumps (set LineMatcher option A)
if (flag_hex)
matcher_options.push_back('A');
// --match: match lines with the RE/flex Line Matcher
Static::matcher = std::unique_ptr<reflex::AbstractMatcher>(new reflex::LineMatcher(reflex::Input(), matcher_options.c_str()));
// --pager
open_pager();
if (Static::threads > 1)
{
GrepMaster grep(Static::output, Static::matcher.get(), NULL);
grep.ugrep();
}
else
{
Grep grep(Static::output, Static::matcher.get(), NULL);
Static::set_grep_handle(&grep);
grep.ugrep();
Static::clear_grep_handle();
}
}
}
else if (flag_mode == Mode::PERL)
{
// -P: Perl matching with PCRE2 or Boost.Regex
#if defined(HAVE_PCRE2)
// construct the PCRE2 JIT-optimized Perl pattern matcher
uint32_t options = flag_binary ? (PCRE2_NEVER_UTF | PCRE2_NEVER_UCP) : (PCRE2_UTF | PCRE2_UCP);
if (!Static::bcnf.singleton_or_undefined())
options |= PCRE2_DUPNAMES;
Static::string_pattern.assign(flag_binary ? reflex::PCRE2Matcher::convert(regex, convert_flags, &flag_multiline) : reflex::PCRE2UTFMatcher::convert(regex, convert_flags, &flag_multiline));
Static::matcher = std::unique_ptr<reflex::AbstractMatcher>(new reflex::PCRE2Matcher(Static::string_pattern, reflex::Input(), matcher_options.c_str(), options));
Static::matchers.clear();
if (!Static::bcnf.singleton_or_undefined())
{
std::string subregex;
for (const auto& i : Static::bcnf.lists())
{
Static::matchers.emplace_back();
auto& submatchers = Static::matchers.back();
for (const auto& j : i)
{
if (j)
{
subregex.assign(pattern_options).append(*j);
submatchers.emplace_back(new reflex::PCRE2Matcher((flag_binary ? reflex::PCRE2Matcher::convert(subregex, convert_flags) : reflex::PCRE2UTFMatcher::convert(subregex, convert_flags)), reflex::Input(), matcher_options.c_str(), options));
}
else
{
submatchers.emplace_back();
}
}
}
}
// --pager
open_pager();
if (Static::threads > 1)
{
GrepMaster grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
grep.ugrep();
}
else
{
Grep grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
Static::set_grep_handle(&grep);
grep.ugrep();
Static::clear_grep_handle();
}
#elif defined(HAVE_BOOST_REGEX)
try
{
// construct the Boost.Regex Perl pattern matcher
Static::string_pattern.assign(reflex::BoostPerlMatcher::convert(regex, convert_flags, &flag_multiline));
Static::matcher = std::unique_ptr<reflex::AbstractMatcher>(new reflex::BoostPerlMatcher(Static::string_pattern, reflex::Input(), matcher_options.c_str()));
Static::matchers.clear();
if (!Static::bcnf.singleton_or_undefined())
{
std::string subregex;
for (const auto& i : Static::bcnf.lists())
{
Static::matchers.emplace_back();
auto& submatchers = Static::matchers.back();
for (const auto& j : i)
{
if (j)
{
subregex.assign(pattern_options).append(*j);
submatchers.emplace_back(new reflex::BoostPerlMatcher(reflex::BoostPerlMatcher::convert(subregex, convert_flags), reflex::Input(), matcher_options.c_str()));
}
else
{
submatchers.emplace_back();
}
}
}
}
// --pager
open_pager();
if (Static::threads > 1)
{
GrepMaster grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
grep.ugrep();
}
else
{
Grep grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
Static::set_grep_handle(&grep);
grep.ugrep();
Static::clear_grep_handle();
}
}
catch (boost::regex_error& error)
{
reflex::regex_error_type code;
switch (error.code())
{
case boost::regex_constants::error_collate:
code = reflex::regex_error::invalid_collating;
break;
case boost::regex_constants::error_ctype:
code = reflex::regex_error::invalid_class;
break;
case boost::regex_constants::error_escape:
code = reflex::regex_error::invalid_escape;
break;
case boost::regex_constants::error_backref:
code = reflex::regex_error::invalid_backreference;
break;
case boost::regex_constants::error_brack:
code = reflex::regex_error::invalid_class;
break;
case boost::regex_constants::error_paren:
code = reflex::regex_error::mismatched_parens;
break;
case boost::regex_constants::error_brace:
code = reflex::regex_error::mismatched_braces;
break;
case boost::regex_constants::error_badbrace:
code = reflex::regex_error::invalid_repeat;
break;
case boost::regex_constants::error_range:
code = reflex::regex_error::invalid_class_range;
break;
case boost::regex_constants::error_space:
code = reflex::regex_error::exceeds_limits;
break;
case boost::regex_constants::error_badrepeat:
code = reflex::regex_error::invalid_repeat;
break;
case boost::regex_constants::error_complexity:
code = reflex::regex_error::exceeds_limits;
break;
case boost::regex_constants::error_stack:
code = reflex::regex_error::exceeds_limits;
break;
default:
code = reflex::regex_error::invalid_syntax;
}
throw reflex::regex_error(code, Static::string_pattern, error.position() + 1);
}
#endif
}
else
{
// construct the RE/flex DFA-based pattern matcher and start matching files
Static::reflex_pattern.assign(reflex::Matcher::convert(regex, convert_flags, &flag_multiline), (flag_index != NULL ? "hr" : "r"));
Static::matchers.clear();
if (flag_fuzzy > 0)
{
// -U or --binary: disable fuzzy Unicode matching, ASCII/binary matching with -Z MAX edit distance
uint16_t max = static_cast<uint16_t>(flag_fuzzy) | (flag_binary ? reflex::FuzzyMatcher::BIN : 0);
Static::matcher = std::unique_ptr<reflex::AbstractMatcher>(new reflex::FuzzyMatcher(Static::reflex_pattern, max, reflex::Input(), matcher_options.c_str()));
if (!Static::bcnf.singleton_or_undefined())
{
std::string subregex;
for (const auto& i : Static::bcnf.lists())
{
Static::matchers.emplace_back();
auto& submatchers = Static::matchers.back();
for (const auto& j : i)
{
if (j)
{
subregex.assign(pattern_options).append(*j);
Static::reflex_patterns.emplace_back(reflex::FuzzyMatcher::convert(subregex, convert_flags), "r");
submatchers.emplace_back(new reflex::FuzzyMatcher(Static::reflex_patterns.back(), reflex::Input(), matcher_options.c_str()));
}
else
{
submatchers.emplace_back();
}
}
}
}
// --pager
open_pager();
if (Static::threads > 1)
{
GrepMaster grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
grep.ugrep();
}
else
{
Grep grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
Static::set_grep_handle(&grep);
grep.ugrep();
Static::clear_grep_handle();
}
}
else
{
Static::matcher = std::unique_ptr<reflex::AbstractMatcher>(new reflex::Matcher(Static::reflex_pattern, reflex::Input(), matcher_options.c_str()));
if (!Static::bcnf.singleton_or_undefined())
{
std::string subregex;
for (const auto& i : Static::bcnf.lists())
{
Static::matchers.emplace_back();
auto& submatchers = Static::matchers.back();
for (const auto& j : i)
{
if (j)
{
subregex.assign(pattern_options).append(*j);
Static::reflex_patterns.emplace_back(reflex::Matcher::convert(subregex, convert_flags), "r");
submatchers.emplace_back(new reflex::Matcher(Static::reflex_patterns.back(), reflex::Input(), matcher_options.c_str()));
}
else
{
submatchers.emplace_back();
}
}
}
}
// --index: perform indexed search using the pattern indexing hash finite state automaton (HFA)
if (flag_index != NULL && Static::reflex_pattern.has_hfa())
Static::index_pattern = &Static::reflex_pattern;
// --pager
open_pager();
if (Static::threads > 1)
{
GrepMaster grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
grep.ugrep();
}
else
{
Grep grep(Static::output, Static::matcher.get(), Static::bcnf.singleton_or_undefined() ? NULL : &Static::matchers);
Static::set_grep_handle(&grep);
grep.ugrep();
Static::clear_grep_handle();
}
// pattern is out of scope and implicitly deleted, invalidating this pointer
Static::index_pattern = NULL;
}
}
// --tree with -l or -c but not --format: finish tree output
if (flag_tree && (flag_files_with_matches || flag_count) && flag_format == NULL)
{
Output out(Static::output);
for (int i = 1; i < Output::Tree::depth; ++i)
out.str(Output::Tree::end);
out.nl();
}
// --format-end
if (flag_format_end != NULL)
Output(Static::output).format(flag_format_end, Stats::found_parts());
// --stats: output stats when we're done
if (flag_stats != NULL)
{
Stats::report(Static::output);
Static::bcnf.report(Static::output);
if (strcmp(flag_stats, "vm") == 0)
{
size_t nodes = Static::reflex_pattern.nodes();
size_t edges = Static::reflex_pattern.edges();
size_t words = Static::reflex_pattern.words();
size_t hashes = Static::reflex_pattern.hashes();
size_t nodes_time = static_cast<size_t>(Static::reflex_pattern.nodes_time());
size_t edges_time = static_cast<size_t>(Static::reflex_pattern.parse_time() + Static::reflex_pattern.edges_time());
size_t words_time = static_cast<size_t>(Static::reflex_pattern.words_time());
size_t study_time = static_cast<size_t>(Static::reflex_pattern.analysis_time());
fprintf(Static::output, "VM: %zu nodes (%zums) %zu edges (%zums) %zu opcode words (%zums)", nodes, nodes_time, edges, edges_time, words, words_time);
if (hashes > 0)
fprintf(Static::output, " %zu hash tables (%zums)", hashes, study_time);
fprintf(Static::output, NEWLINESTR);
}
}
// --pager
close_pager();
}
// perform a limited ugrep search on a single file with optional archive part and store up to num results in a vector, may throw an exception, used by the TUI
void ugrep_find_text_preview(const char *filename, const char *partname, size_t from_lineno, size_t max, size_t& lineno, size_t& num, std::vector<std::string>& text)
{
// we can only return a text preview when the main search engine ugrep() already started and executes, or after it ends
if (!Static::matcher)
return;
// clone matcher, since it is not thread safe to use when the main search engine still executes
reflex::AbstractMatcher *matcher = Static::matcher->clone();
// clone the CNF matchers if any are used
Static::Matchers *matchers = Static::bcnf.singleton_or_undefined() ? NULL : Static::matchers_clone();
Grep grep(NULL, matcher, matchers);
grep.find_text_preview(filename, partname, from_lineno, max, lineno, num, text);
// delete the cloned matchers, if any
if (matchers != NULL)
delete matchers;
// delete the cloned matcher
delete matcher;
}
// extract a part from an archive and send to a stream, used by the TUI
void ugrep_extract(const char *filename, const char *partname, FILE *output)
{
Grep grep(NULL, NULL, NULL);
grep.extract(filename, partname, output);
}
// set the handle to be able to use cancel_ugrep()
void Static::set_grep_handle(Grep *grep)
{
std::unique_lock<std::mutex> lock(Static::grep_handle_mutex);
Static::grep_handle = grep;
lock.unlock();
}
// reset the grep handle
void Static::clear_grep_handle()
{
std::unique_lock<std::mutex> lock(Static::grep_handle_mutex);
Static::grep_handle = NULL;
lock.unlock();
}
// cancel the search
void Static::cancel_ugrep()
{
std::unique_lock<std::mutex> lock(Static::grep_handle_mutex);
if (Static::grep_handle != NULL)
Static::grep_handle->cancel();
lock.unlock();
}
// search the specified files or standard input for pattern matches
void Grep::ugrep()
{
// read each input file to find pattern matches
if (flag_stdin)
{
Stats::score_file();
// search standard input
search(Static::LABEL_STANDARD_INPUT, static_cast<uint16_t>(flag_fuzzy));
}
if (Static::arg_files.empty())
{
if (flag_directories_action == Action::RECURSE)
recurse(1, ".");
}
else
{
#ifdef OS_WIN_OR_MINGW
const DWORD attr = INVALID_FILE_ATTRIBUTES;
#else
const uint32_t attr = 0;
std::pair<std::set<ino_t>::iterator,bool> vino;
#endif
for (const auto pathname : Static::arg_files)
{
// stop after finding max-files matching files
if (flag_max_files > 0 && Stats::found_parts() >= flag_max_files)
break;
// stop when output is blocked or search cancelled
if (out.eof || out.cancelled())
break;
// search file or directory, get the basename from the file argument first
const char *basename = strrchr(pathname, PATHSEPCHR);
if (basename != NULL)
++basename;
else
basename = pathname;
ino_t inode = 0;
uint64_t info;
// search file or recursively search directory based on selection criteria
switch (select(1, pathname, basename, DIRENT_TYPE_UNKNOWN, attr, inode, info, true))
{
case Type::DIRECTORY:
if (flag_directories_action != Action::SKIP)
{
#ifndef OS_WIN_OR_MINGW
if (flag_dereference)
vino = visited.insert(inode);
#endif
recurse(1, pathname);
#ifndef OS_WIN_OR_MINGW
if (flag_dereference)
visited.erase(vino.first);
#endif
}
break;
case Type::OTHER:
search(pathname, Entry::UNDEFINED_COST);
break;
case Type::SKIP:
break;
}
}
}
}
// select file or directory to search for pattern matches, return SKIP, DIRECTORY or OTHER
Grep::Type Grep::select(size_t level, const char *pathname, const char *basename, int type, uint32_t attr, ino_t& inode, uint64_t& info, bool is_argument)
{
if (*basename == '.' && !flag_hidden && !is_argument)
return Type::SKIP;
#ifdef OS_WIN_OR_MINGW
(void)type; (void)inode; (void)info; // appease -Wunused
// use file attributes from FindFirstFileW() WIN32_FIND_DATAW dwFileAttributes otherwise attr == INVALID_FILE_ATTRIBUTES
if (attr == INVALID_FILE_ATTRIBUTES)
attr = GetFileAttributesW(utf8_decode(pathname).c_str());
if (attr == INVALID_FILE_ATTRIBUTES)
{
errno = ENOENT;
warning("cannot read", pathname);
return Type::SKIP;
}
// skip hidden and system files unless explicitly searched with --hidden
if (!flag_hidden && !is_argument && ((attr & FILE_ATTRIBUTE_HIDDEN) || (attr & FILE_ATTRIBUTE_SYSTEM)))
return Type::SKIP;
if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0)
{
if (flag_directories_action == Action::READ)
{
// directories cannot be read actually, so grep produces a warning message (errno is not set)
is_directory(pathname);
return Type::SKIP;
}
if (is_argument || flag_directories_action == Action::RECURSE)
{
// --depth: recursion level exceeds max depth?
if (flag_max_depth > 0 && level > flag_max_depth)
return Type::SKIP;
// hard maximum recursion depth reached?
if (level > MAX_DEPTH)
{
if (!flag_no_messages)
fprintf(Static::errout, "%sugrep: %s%s%s recursion depth hit hard limit of %d\n", color_off, color_high, pathname, color_off, MAX_DEPTH);
return Type::SKIP;
}
// check for --exclude-dir and --include-dir constraints if pathname != "."
if (strcmp(pathname, ".") != 0)
{
if (!flag_all_exclude_dir.empty())
{
// exclude directories whose pathname matches any one of the --exclude-dir globs unless negated with !
bool ok = true;
for (const auto& glob : flag_all_exclude_dir)
{
bool ignore_case = &glob < &flag_all_exclude_dir.front() + flag_exclude_iglob_size;
if (glob.front() == '!')
{
if (!ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = true;
}
else if (ok && glob_match(pathname, basename, glob.c_str(), ignore_case))
{
ok = false;
}
}
if (!ok)
return Type::SKIP;
}
if (!flag_all_include_dir.empty())
{
// include directories whose pathname matches any one of the --include-dir globs unless negated with !
bool ok = false;
for (const auto& glob : flag_all_include_dir)
{
bool ignore_case = &glob < &flag_all_include_dir.front() + flag_include_iglob_dir_size;
if (glob.front() == '!')
{
if (ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = false;
}
else if (!ok && glob_match(pathname, basename, glob.c_str(), ignore_case, true))
{
// the pathname matches the leading --include-dir glob, OK to descent into directory
ok = true;
}
}
if (!ok)
return Type::SKIP;
}
}
return Type::DIRECTORY;
}
}
else if ((attr & FILE_ATTRIBUTE_DEVICE) == 0 || (is_argument && flag_devices_action == Action::UNSP) || flag_devices_action == Action::READ)
{
// --depth: recursion level not deep enough?
if (flag_min_depth > 0 && level <= flag_min_depth)
return Type::SKIP;
bool ok;
if (!flag_all_include_dir.empty())
{
// only include paths to the file to search that match an --include-dir path glob
ok = false;
for (const auto& glob : flag_all_include_dir)
{
bool ignore_case = &glob < &flag_all_include_dir.front() + flag_include_iglob_dir_size;
if (glob.front() != '!' && glob_match(pathname, basename, glob.c_str(), ignore_case, false, true))
{
// pathname matches the --include-dir path glob
ok = true;
break;
}
}
if (!ok)
return Type::SKIP;
}
ok = true;
if (!flag_all_exclude.empty())
{
// exclude files whose pathname matches any one of the --exclude globs unless negated with !
for (const auto& glob : flag_all_exclude)
{
bool ignore_case = &glob < &flag_all_exclude.front() + flag_exclude_iglob_size;
if (glob.front() == '!')
{
if (!ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = true;
}
else if (ok && glob_match(pathname, basename, glob.c_str(), ignore_case))
{
ok = false;
}
}
if (!ok)
return Type::SKIP;
}
if (!flag_all_include.empty())
{
// include files whose pathname matches any one of the --include globs unless negated with !
ok = false;
for (const auto& glob : flag_all_include)
{
bool ignore_case = &glob < &flag_all_include.front() + flag_include_iglob_size;
if (glob.front() == '!')
{
if (ok && glob_match(pathname, basename, glob.c_str() + 1), ignore_case)
ok = false;
}
else if (!ok && glob_match(pathname, basename, glob.c_str(), ignore_case))
{
ok = true;
}
}
if (!ok && flag_file_magic.empty())
return Type::SKIP;
}
// check magic pattern against the file signature, when --file-magic=MAGIC is specified
if (!flag_file_magic.empty() && (flag_all_include.empty() || !ok))
{
FILE *file;
if (fopenw_s(&file, pathname, "rb") != 0)
{
warning("cannot read", pathname);
return Type::SKIP;
}
#ifdef HAVE_LIBZ
if (flag_decompress)
{
zstreambuf streambuf(pathname, file);
std::istream stream(&streambuf);
// file has the magic bytes we're looking for: search the file
size_t match = Static::magic_matcher.input(&stream).scan();
if (match == flag_not_magic || match >= flag_min_magic)
{
fclose(file);
Stats::score_file();
return Type::OTHER;
}
}
else
#endif
{
size_t match = Static::magic_matcher.input(reflex::Input(file, flag_encoding_type)).scan();
if (match == flag_not_magic || match >= flag_min_magic)
{
fclose(file);
Stats::score_file();
return Type::OTHER;
}
}
fclose(file);
return Type::SKIP;
}
Stats::score_file();
return Type::OTHER;
}
#else
(void)attr;
struct stat buf;
// -R or command line argument FILE to search: follow symlinks to directories
bool follow = flag_dereference || is_argument;
// if dir entry is unknown and not following, then use lstat() to check if pathname is a symlink
if (type != DIRENT_TYPE_UNKNOWN || follow || lstat(pathname, &buf) == 0)
{
// is it a symlink? If dir entry unknown and following then set to symlink = true to call stat() below
bool symlink = type != DIRENT_TYPE_UNKNOWN ? type == DIRENT_TYPE_LNK : follow ? true : S_ISLNK(buf.st_mode);
// if we got a symlink, use stat() to check if pathname is a directory or a regular file, we also stat when following and when sorting by stat info such as modification time
if (( ( (type != DIRENT_TYPE_UNKNOWN && type != DIRENT_TYPE_LNK) || // type is known and not symlink
(!follow && !symlink) // or not following and not symlink
) &&
(flag_sort_key == Sort::NA || flag_sort_key == Sort::NAME) // and we're not sorting or by name
) ||
stat(pathname, &buf) == 0) // otherwise, do a stat
{
// check if directory
if (type == DIRENT_TYPE_DIR || ((type == DIRENT_TYPE_UNKNOWN || type == DIRENT_TYPE_LNK) && S_ISDIR(buf.st_mode)))
{
// if symlinked directory, then follow only if -R is specified or if FILE is a command line argument
if (!symlink || follow)
{
if (flag_directories_action == Action::READ)
{
// directories cannot be read actually, so grep produces a warning message (errno is not set)
is_directory(pathname);
return Type::SKIP;
}
if (is_argument || flag_directories_action == Action::RECURSE)
{
// --depth: recursion level exceeds max depth?
if (flag_max_depth > 0 && level > flag_max_depth)
return Type::SKIP;
// hard maximum recursion depth reached?
if (level > MAX_DEPTH)
{
if (!flag_no_messages)
fprintf(Static::errout, "%sugrep: %s%s%s recursion depth hit hard limit of %d\n", color_off, color_high, pathname, color_off, MAX_DEPTH);
return Type::SKIP;
}
// check for --exclude-dir and --include-dir constraints if pathname != "."
if (strcmp(pathname, ".") != 0)
{
if (!flag_all_exclude_dir.empty())
{
// exclude directories whose pathname matches any one of the --exclude-dir globs unless negated with !
bool ok = true;
for (const auto& glob : flag_all_exclude_dir)
{
bool ignore_case = &glob < &flag_all_exclude_dir.front() + flag_exclude_iglob_dir_size;
if (glob.front() == '!')
{
if (!ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = true;
}
else if (ok && glob_match(pathname, basename, glob.c_str(), ignore_case))
{
ok = false;
}
}
if (!ok)
return Type::SKIP;
}
if (!flag_all_include_dir.empty())
{
// include directories whose name matches any one of the --include-dir globs unless negated with !
bool ok = false;
for (const auto& glob : flag_all_include_dir)
{
bool ignore_case = &glob < &flag_all_include_dir.front() + flag_include_iglob_dir_size;
if (glob.front() == '!')
{
if (ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = false;
}
else if (!ok && glob_match(pathname, basename, glob.c_str(), ignore_case, true))
{
// the pathname matches the leading --include-dir glob, OK to descent into directory
ok = true;
}
}
if (!ok)
return Type::SKIP;
}
}
if (type != DIRENT_TYPE_DIR)
inode = buf.st_ino;
info = Entry::sort_info(buf);
return Type::DIRECTORY;
}
}
}
else if (type == DIRENT_TYPE_REG ? !is_output(inode) : (type == DIRENT_TYPE_UNKNOWN || type == DIRENT_TYPE_LNK) && S_ISREG(buf.st_mode) ? !is_output(buf.st_ino) : (is_argument && flag_devices_action == Action::UNSP) || flag_devices_action == Action::READ)
{
// if symlinked files, then follow only if -R or -S is specified or if FILE is a command line argument
if (!symlink || follow || flag_dereference_files)
{
// --depth: recursion level not deep enough?
if (flag_min_depth > 0 && level <= flag_min_depth)
return Type::SKIP;
bool ok;
if (!flag_all_include_dir.empty())
{
// only include paths to the file to search that match an --include-dir path glob
ok = false;
for (const auto& glob : flag_all_include_dir)
{
bool ignore_case = &glob < &flag_all_include_dir.front() + flag_include_iglob_dir_size;
if (glob.front() != '!' && glob_match(pathname, basename, glob.c_str(), ignore_case, false, true))
{
// pathname matches the --include-dir path glob
ok = true;
break;
}
}
if (!ok)
return Type::SKIP;
}
ok = true;
if (!flag_all_exclude.empty())
{
// exclude files whose pathname matches any one of the --exclude globs unless negated with !
for (const auto& glob : flag_all_exclude)
{
bool ignore_case = &glob < &flag_all_exclude.front() + flag_exclude_iglob_size;
if (glob.front() == '!')
{
if (!ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = true;
}
else if (ok && glob_match(pathname, basename, glob.c_str(), ignore_case))
{
ok = false;
}
}
if (!ok)
return Type::SKIP;
}
if (!flag_all_include.empty())
{
// include files whose basename matches any one of the --include globs if not negated with !
ok = false;
for (const auto& glob : flag_all_include)
{
bool ignore_case = &glob < &flag_all_include.front() + flag_include_iglob_size;
if (glob.front() == '!')
{
if (ok && glob_match(pathname, basename, glob.c_str() + 1, ignore_case))
ok = false;
}
else if (!ok && glob_match(pathname, basename, glob.c_str(), ignore_case))
{
ok = true;
}
}
if (!ok && flag_file_magic.empty())
return Type::SKIP;
}
// --min-size, --max-size: skip regular files that are too small or too large
if (flag_min_size != 0 || flag_max_size != 0)
{
// if stat not done then do it to get size
if (type == DIRENT_TYPE_REG && (flag_sort_key == Sort::NA || flag_sort_key == Sort::NAME))
stat(pathname, &buf);
if (buf.st_size < static_cast<off_t>(flag_min_size) || (flag_max_size > 0 && buf.st_size > static_cast<off_t>(flag_max_size)))
return Type::SKIP;
}
// check magic pattern against the file signature, when --file-magic=MAGIC is specified
if (!flag_file_magic.empty() && (flag_all_include.empty() || !ok))
{
FILE *file;
if (fopenw_s(&file, pathname, "rb") != 0)
{
warning("cannot read", pathname);
return Type::SKIP;
}
#ifdef HAVE_LIBZ
if (flag_decompress)
{
zstreambuf streambuf(pathname, file);
std::istream stream(&streambuf);
// file has the magic bytes we're looking for: search the file
size_t match = Static::magic_matcher.input(&stream).scan();
if (match == flag_not_magic || match >= flag_min_magic)
{
fclose(file);
Stats::score_file();
info = Entry::sort_info(buf);
return Type::OTHER;
}
}
else
#endif
{
// if file has the magic bytes we're looking for: search the file
size_t match = Static::magic_matcher.input(reflex::Input(file, flag_encoding_type)).scan();
if (match == flag_not_magic || match >= flag_min_magic)
{
fclose(file);
Stats::score_file();
info = Entry::sort_info(buf);
return Type::OTHER;
}
}
fclose(file);
return Type::SKIP;
}
Stats::score_file();
info = Entry::sort_info(buf);
return Type::OTHER;
}
}
}
}
else
{
warning("lstat", pathname);
}
#endif
return Type::SKIP;
}
// recurse over directory, searching for pattern matches in files and subdirectories
void Grep::recurse(size_t level, const char *pathname)
{
// output is closed or cancelled?
if (out.eof || out.cancelled())
return;
#ifdef OS_WIN_OR_MINGW
WIN32_FIND_DATAW ffd;
std::string glob;
if (strcmp(pathname, ".") != 0)
glob.assign(pathname).append("/*");
else
glob.assign("*");
std::wstring wglob(utf8_decode(glob));
HANDLE hFind = FindFirstFileW(wglob.c_str(), &ffd);
if (hFind == INVALID_HANDLE_VALUE)
{
if (GetLastError() != ERROR_FILE_NOT_FOUND)
warning("cannot open directory", pathname);
return;
}
#else
#if defined(HAVE_STATVFS) || defined(HAVE_STATFS)
if (!exclude_fs_ids.empty() || !include_fs_ids.empty())
{
stat_fs_t buf;
if (stat_fs(pathname, &buf) == 0)
{
if (exclude_fs_ids.find(fsid_to_uint64(buf.f_fsid)) != exclude_fs_ids.end())
return;
if (!include_fs_ids.empty() && include_fs_ids.find(fsid_to_uint64(buf.f_fsid)) == include_fs_ids.end())
return;
}
}
#endif
DIR *dir = opendir(pathname);
if (dir == NULL)
{
warning("cannot open directory", pathname);
return;
}
#endif
// --index: check index file, but only when necessary on demand to store indexed file entries in a map
bool index_demand = Static::index_pattern != NULL;
std::map<std::string,bool> indexed;
// the indexing file stored per indexed directory and index file identifying magic bytes
static const char ugrep_index_filename[] = "._UG#_Store";
static const char ugrep_index_file_magic[5] = "UG#\x03";
// --ignore-files: check if one or more are present to read and extend the file and dir exclusions
size_t saved_all_exclude_size = 0;
size_t saved_all_exclude_dir_size = 0;
bool saved = false;
if (!flag_ignore_files.empty())
{
std::string ignore_filename;
for (const auto& ignore_file : flag_ignore_files)
{
ignore_filename.assign(pathname).append(PATHSEPSTR).append(ignore_file);
FILE *file = NULL;
if (fopenw_s(&file, ignore_filename.c_str(), "r") == 0)
{
// save current exclusions vector depths to restore afterwards
if (!saved)
{
saved_all_exclude_size = flag_all_exclude.size();
saved_all_exclude_dir_size = flag_all_exclude_dir.size();
saved = true;
}
Stats::ignore_file(ignore_filename);
// push globs imported from the ignore file to the back of the vectors
import_globs(file, flag_all_exclude, flag_all_exclude_dir, true, pathname);
fclose(file);
}
}
}
Stats::score_dir();
std::vector<Entry> file_entries;
std::vector<Entry> dir_entries;
std::string entry_pathname;
#ifdef OS_WIN_OR_MINGW
std::string cFileName;
uint64_t list = 0;
uint64_t index_time = 0;
do
{
cFileName.assign(utf8_encode(ffd.cFileName));
// search directory entries that aren't . or .. or hidden
if (cFileName[0] != '.' || (flag_hidden && cFileName[1] != '\0' && cFileName[1] != '.'))
{
// --index: do not search index files themselves, even when --hidden is specified
if (flag_index != NULL && cFileName == ugrep_index_filename)
continue;
// --min-size, --max-size: skip regular files that are too small or too large
if (flag_min_size != 0 || flag_max_size != 0)
{
uint64_t size = static_cast<uint64_t>(ffd.nFileSizeLow) | (static_cast<uint64_t>(ffd.nFileSizeHigh) << 32);
if (size < flag_min_size || (flag_max_size != 0 && size > flag_max_size))
continue;
}
size_t len = strlen(pathname);
if (len == 1 && pathname[0] == '.')
entry_pathname.assign(cFileName);
else if (len > 0 && pathname[len - 1] == PATHSEPCHR)
entry_pathname.assign(pathname).append(cFileName);
else
entry_pathname.assign(pathname).append(PATHSEPSTR).append(cFileName);
uint64_t info = 0;
// --sort: get file info
if (flag_sort_key != Sort::NA && flag_sort_key != Sort::NAME)
{
if (flag_sort_key == Sort::SIZE)
{
info = static_cast<uint64_t>(ffd.nFileSizeLow) | (static_cast<uint64_t>(ffd.nFileSizeHigh) << 32);
}
else if (flag_sort_key == Sort::LIST)
{
info = list++;
}
else
{
struct _FILETIME& time = flag_sort_key == Sort::USED ? ffd.ftLastAccessTime : flag_sort_key == Sort::CHANGED ? ffd.ftLastWriteTime : ffd.ftCreationTime;
info = static_cast<uint64_t>(time.dwLowDateTime) | (static_cast<uint64_t>(time.dwHighDateTime) << 32);
}
}
ino_t inode = 0;
Type type = select(level + 1, entry_pathname.c_str(), cFileName.c_str(), DIRENT_TYPE_UNKNOWN, ffd.dwFileAttributes, inode, info);
// check if we're recursing on a symlink to skip them when -p or -r is specified
if (!flag_dereference && (ffd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 && ffd.dwReserved0 == IO_REPARSE_TAG_SYMLINK)
{
if (type == Type::DIRECTORY)
type = Type::SKIP;
else if (type == Type::OTHER && !flag_dereference_files)
type = Type::SKIP;
}
// --index: search indexed files quickly, but only when necessary on demand
if (type == Type::OTHER && Static::index_pattern != NULL)
{
// if we did not check for an index file yet, then check it now on demand and read it when found
if (index_demand)
{
// check no more than once at most
index_demand = false;
// check if an index file is present and use it
std::string index_filename(pathname);
index_filename.append(PATHSEPSTR).append(ugrep_index_filename);
std::wstring windex_filename(utf8_decode(index_filename));
HANDLE hFile = CreateFileW(windex_filename.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
if (hFile != INVALID_HANDLE_VALUE)
{
char check_magic[sizeof(ugrep_index_file_magic)];
DWORD numread = 0;
// if an index file is present in the directory pathname, then read and stat it
if (ReadFile(hFile, check_magic, sizeof(ugrep_index_file_magic), &numread, NULL) &&
numread == sizeof(ugrep_index_file_magic) &&
memcmp(check_magic, ugrep_index_file_magic, sizeof(ugrep_index_file_magic)) == 0)
{
// time of indexing to check which files were modified after indexing
index_time = Entry::modified_time(hFile);
// allocate a buffer, not on the stack, because we are in a deeply recursive function
char *buffer = new char[65536]; // size must be 65536
uint8_t header[4];
std::map<std::string,bool>::iterator skip = indexed.end();
// populate indexed map
while (true)
{
if (!ReadFile(hFile, header, sizeof(header), &numread, NULL) || numread != sizeof(header))
break;
uint16_t basename_size = header[2] | (header[3] << 8);
if (!ReadFile(hFile, buffer, basename_size, &numread, NULL) || numread != basename_size)
break;
// make basename in buffer 0-terminated
buffer[basename_size] = '\0';
// hashes table size, zero to skip empty files and binary files when -I is specified
uint32_t hashes_size = 0;
uint8_t logsize = header[1] & 0x1f;
if (logsize > 0)
for (hashes_size = 1; logsize > 0; --logsize)
hashes_size <<= 1;
// sanity check
if (hashes_size > 65536)
break;
// archives have multiple entries in the index file, we keep the skip iterator to compare entries
if ((header[1] & 0x40) == 0)
skip = indexed.end();
// we're still looking in the same archive?
bool same_archive = skip != indexed.end() && skip->first == buffer;
// if not and archive or not the same archive entry, then add basename to the indexed map
if (!same_archive)
{
Stats::score_indexed();
skip = indexed.emplace(buffer, true).first;
}
// now read the hashes into the buffer to check for a possible match
if (hashes_size > 0 && (!ReadFile(hFile, buffer, hashes_size, &numread, NULL) || numread != hashes_size))
break;
if ((header[1] & 0x80) != 0 && flag_binary_without_match)
{
// -I: if the indexed file to search is a binary file, then skip it
}
else if ((header[1] & 0x60) != 0 && !flag_decompress)
{
// not -z: if the indexed file to search is an archive or is a compressed file, then skip it
}
else if (hashes_size > 0)
{
// check if the hashed pattern has a potential match with the file's index hash
if (Static::index_pattern->match_hfa(reinterpret_cast<const uint8_t*>(buffer), hashes_size))
{
skip->second = false;
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s" PATHSEPSTR "%s\n", pathname, skip->first.c_str());
}
else
{
// skip indexed file, does not match
}
}
else if ((header[1] & 0x80) == 0)
{
// skip indexed file
}
else
{
// not -I: do not skip non-indexed binary files (marked empty in index) that we need to search
skip->second = false;
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s" PATHSEPSTR "%s (not indexed binary)\n", pathname, skip->first.c_str());
}
}
delete[] buffer;
}
CloseHandle(hFile);
}
}
// check if the file to search was indexed and did not match the specified pattern
const std::map<std::string,bool>::const_iterator skip = indexed.find(cFileName);
if (skip == indexed.end())
{
// a new file that was not indexed
Stats::score_added();
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s (not indexed)\n", entry_pathname.c_str());
}
else if (skip->second)
{
bool is_changed = Entry::modified_time(ffd) > index_time;
if (is_changed)
{
// search the file that was changed after indexing
Stats::score_changed();
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s (changed)\n", entry_pathname.c_str());
}
else
{
// skip this indexed file that does not match the specified pattern
Stats::score_skipped();
type = Type::SKIP;
}
}
}
// search entry_pathname, unless searchable directory into which we should recurse
switch (type)
{
case Type::DIRECTORY:
dir_entries.emplace_back(entry_pathname, 0, info);
break;
case Type::OTHER:
if (flag_sort_key == Sort::NA)
search(entry_pathname.c_str(), Entry::UNDEFINED_COST);
else
file_entries.emplace_back(entry_pathname, 0, info);
break;
case Type::SKIP:
break;
}
// stop after finding max-files matching files
if (flag_max_files > 0 && Stats::found_parts() >= flag_max_files)
break;
// stop when output is blocked or search cancelled
if (out.eof || out.cancelled())
break;
}
} while (FindNextFileW(hFind, &ffd) != 0);
FindClose(hFind);
#else
struct dirent *dirent = NULL;
uint64_t list = 0;
while ((dirent = readdir(dir)) != NULL)
{
// search directory entries that aren't . or .. or hidden
if (dirent->d_name[0] != '.' || (flag_hidden && dirent->d_name[1] != '\0' && dirent->d_name[1] != '.'))
{
// --index: do not search index files themselves, even when --hidden is specified
if (flag_index != NULL && strcmp(dirent->d_name, ugrep_index_filename) == 0)
continue;
size_t len = strlen(pathname);
if (len == 1 && pathname[0] == '.')
entry_pathname.assign(dirent->d_name);
else if (len > 0 && pathname[len - 1] == PATHSEPCHR)
entry_pathname.assign(pathname).append(dirent->d_name);
else
entry_pathname.assign(pathname).append(PATHSEPSTR).append(dirent->d_name);
ino_t inode;
uint64_t info;
Type type;
// search entry_pathname, unless searchable directory into which we should recurse
#if defined(HAVE_STRUCT_DIRENT_D_TYPE) && defined(HAVE_STRUCT_DIRENT_D_INO)
inode = dirent->d_ino;
type = select(level + 1, entry_pathname.c_str(), dirent->d_name, dirent->d_type, 0, inode, info);
#else
inode = 0;
type = select(level + 1, entry_pathname.c_str(), dirent->d_name, DIRENT_TYPE_UNKNOWN, 0, inode, info);
#endif
if (flag_sort_key == Sort::LIST)
info = list++;
// --index: search indexed files quickly, but only when necessary on demand
if (type == Type::OTHER && Static::index_pattern != NULL)
{
// if we did not check for an index file yet, then check it now on demand and read it when found
if (index_demand)
{
// check no more than once at most
index_demand = false;
// check if an index file is present and use it
FILE *index_file = NULL;
std::string index_filename(pathname);
index_filename.append(PATHSEPSTR).append(ugrep_index_filename);
if (fopenw_s(&index_file, index_filename.c_str(), "rb") == 0)
{
char check_magic[sizeof(ugrep_index_file_magic)];
// if an index file is present in the directory pathname, then read and stat it
if (fread(check_magic, sizeof(ugrep_index_file_magic), 1, index_file) != 0 &&
memcmp(check_magic, ugrep_index_file_magic, sizeof(ugrep_index_file_magic)) == 0)
{
struct stat buf;
if (fstat(fileno(index_file), &buf) == 0)
{
// time of indexing to check which files were modified after indexing
uint64_t index_time = Entry::modified_time(buf);
// allocate a buffer, not on the stack, because we are in a deeply recursive function
char *buffer = new char[65536]; // size must be 65536
uint8_t header[4];
std::string index_pathname;
bool is_changed = false;
std::map<std::string,bool>::iterator skip = indexed.end();
// populate indexed map
while (true)
{
if (fread(header, sizeof(header), 1, index_file) == 0)
break;
uint16_t basename_size = header[2] | (header[3] << 8);
if (fread(buffer, 1, basename_size, index_file) < basename_size)
break;
// make basename in buffer 0-terminated
buffer[basename_size] = '\0';
// hashes table size, zero to skip empty files and binary files when -I is specified
uint32_t hashes_size = 0;
uint8_t logsize = header[1] & 0x1f;
if (logsize > 0)
for (hashes_size = 1; logsize > 0; --logsize)
hashes_size <<= 1;
// sanity check
if (hashes_size > 65536)
break;
// archives have multiple entries in the index file, we keep the skip iterator to compare entries
if ((header[1] & 0x40) == 0)
skip = indexed.end();
// we're still looking in the same archive?
bool same_archive = skip != indexed.end() && skip->first == buffer;
// if not and archive or not the same archive entry, then add basename to the indexed map
if (!same_archive)
{
Stats::score_indexed();
skip = indexed.emplace(buffer, true).first;
// not --index=fast: check if the file to search was not modified after indexing
if (*flag_index != 'f')
{
if (len == 1 && pathname[0] == '.')
index_pathname.assign(buffer, basename_size);
else if (len > 0 && pathname[len - 1] == PATHSEPCHR)
index_pathname.assign(pathname).append(buffer, basename_size);
else
index_pathname.assign(pathname).append(PATHSEPSTR).append(buffer, basename_size);
// does the file exist and was changed after indexing?
is_changed = stat(index_pathname.c_str(), &buf) == 0 && Entry::modified_time(buf) > index_time;
if (is_changed)
{
// search the file that was changed after indexing
skip->second = false;
Stats::score_changed();
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s (changed)\n", index_pathname.c_str());
}
}
}
// now read the hashes into the buffer to check for a possible match
if (hashes_size > 0 && fread(buffer, hashes_size, 1, index_file) == 0)
break;
if (!is_changed)
{
if ((header[1] & 0x80) != 0 && flag_binary_without_match)
{
// -I: if the indexed file to search is a binary file, then skip it
}
else if ((header[1] & 0x60) != 0 && !flag_decompress)
{
// not -z: if the indexed file to search is an archive or is a compressed file, then skip it
}
else if (hashes_size > 0)
{
// check if the hashed pattern has a potential match with the file's index hash
if (Static::index_pattern->match_hfa(reinterpret_cast<const uint8_t*>(buffer), hashes_size))
{
skip->second = false;
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s\n", index_pathname.c_str());
}
else
{
// skip indexed file, not changed and does not match
}
}
else if ((header[1] & 0x80) == 0)
{
// skip indexed file, not changed and is empty
}
else
{
// not -I: do not skip non-indexed binary files (marked empty in index) that we need to search
skip->second = false;
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s (not indexed binary)\n", index_pathname.c_str());
}
}
}
delete[] buffer;
}
}
}
if (index_file != NULL)
fclose(index_file);
}
// check if the file to search was indexed and did not match the specified pattern
const std::map<std::string,bool>::const_iterator skip = indexed.find(dirent->d_name);
if (skip == indexed.end())
{
// a new file that was not indexed
Stats::score_added();
if (*flag_index == 'l')
fprintf(Static::errout, "INDEX LOG: %s (not indexed)\n", entry_pathname.c_str());
}
else if (skip->second)
{
// skip this indexed file that does not match the specified pattern
Stats::score_skipped();
type = Type::SKIP;
}
}
switch (type)
{
case Type::DIRECTORY:
dir_entries.emplace_back(entry_pathname, inode, info);
break;
case Type::OTHER:
if (flag_sort_key == Sort::NA)
search(entry_pathname.c_str(), Entry::UNDEFINED_COST);
else
file_entries.emplace_back(entry_pathname, inode, info);
break;
case Type::SKIP:
break;
}
// stop after finding max-files matching files
if (flag_max_files > 0 && Stats::found_parts() >= flag_max_files)
break;
// stop when output is blocked or search cancelled
if (out.eof || out.cancelled())
break;
}
}
closedir(dir);
dir = NULL;
#endif
// -Z and --sort=best without --match: presearch the selected files to determine edit distance cost
if (flag_fuzzy > 0 && flag_sort_key == Sort::BEST && !flag_match)
{
auto entry = file_entries.begin();
while (entry != file_entries.end())
{
entry->cost = compute_cost(entry->pathname.c_str());
// if a file cannot be opened, then remove it
if (entry->cost == Entry::UNDEFINED_COST)
entry = file_entries.erase(entry);
else
++entry;
}
}
// --sort: sort the selected non-directory entries and search them
if (flag_sort_key != Sort::NA)
{
if (flag_sort_key == Sort::NAME)
{
if (flag_sort_rev)
std::sort(file_entries.begin(), file_entries.end(), Entry::rev_comp_by_path);
else
std::sort(file_entries.begin(), file_entries.end(), Entry::comp_by_path);
}
else if (flag_sort_key == Sort::BEST)
{
if (flag_sort_rev)
std::sort(file_entries.begin(), file_entries.end(), Entry::rev_comp_by_best);
else
std::sort(file_entries.begin(), file_entries.end(), Entry::comp_by_best);
}
else
{
if (flag_sort_rev)
std::sort(file_entries.begin(), file_entries.end(), Entry::rev_comp_by_info);
else
std::sort(file_entries.begin(), file_entries.end(), Entry::comp_by_info);
}
// search the select sorted non-directory entries
for (const auto& entry : file_entries)
{
search(entry.pathname.c_str(), entry.cost);
// stop after finding max-files matching files
if (flag_max_files > 0 && Stats::found_parts() >= flag_max_files)
break;
// stop when output is blocked or search cancelled
if (out.eof || out.cancelled())
break;
}
}
// --sort: sort the selected subdirectory entries
if (flag_sort_key != Sort::NA)
{
if (flag_sort_key == Sort::NAME || flag_sort_key == Sort::BEST)
{
if (flag_sort_rev)
std::sort(dir_entries.begin(), dir_entries.end(), Entry::rev_comp_by_path);
else
std::sort(dir_entries.begin(), dir_entries.end(), Entry::comp_by_path);
}
else
{
if (flag_sort_rev)
std::sort(dir_entries.begin(), dir_entries.end(), Entry::rev_comp_by_info);
else
std::sort(dir_entries.begin(), dir_entries.end(), Entry::comp_by_info);
}
}
// recurse into the selected subdirectories
for (const auto& entry : dir_entries)
{
// stop after finding max-files matching files
if (flag_max_files > 0 && Stats::found_parts() >= flag_max_files)
break;
// stop when output is blocked or search cancelled
if (out.eof || out.cancelled())
break;
#ifndef OS_WIN_OR_MINGW
// -R: check if this directory was visited before
std::pair<std::set<ino_t>::iterator,bool> vino;
if (flag_dereference)
{
vino = visited.insert(entry.inode);
// if visited before, then do not recurse on this directory again
if (!vino.second)
continue;
}
#endif
recurse(level + 1, entry.pathname.c_str());
#ifndef OS_WIN_OR_MINGW
if (flag_dereference)
visited.erase(vino.first);
#endif
}
// --ignore-files: restore all exclusions when saved by popping off all additions
if (saved)
{
flag_all_exclude.resize(saved_all_exclude_size);
flag_all_exclude_dir.resize(saved_all_exclude_dir_size);
}
}
// -Z and --sort=best: perform a presearch to determine edit distance cost, returns MAX_COST when no match is found
uint16_t Grep::compute_cost(const char *pathname)
{
// default cost is undefined, which erases pathname from the sorted list
uint16_t cost = Entry::UNDEFINED_COST;
// stop when output is blocked
if (out.eof)
return cost;
try
{
// open (archive or compressed) file (pathname is NULL to read stdin), return on failure
if (!open_file(pathname))
return cost;
}
catch (...)
{
// this should never happen
warning("exception while opening", pathname);
return cost;
}
cost = Entry::MAX_COST;
// -Z: matcher is a FuzzyMatcher for sure
reflex::FuzzyMatcher *fuzzy_matcher = dynamic_cast<reflex::FuzzyMatcher*>(matcher);
fuzzy_matcher->distance(static_cast<uint16_t>(flag_fuzzy));
// search file to compute minimum cost
do
{
try
{
if (init_read())
{
while (fuzzy_matcher->find())
{
if (fuzzy_matcher->edits() < cost)
cost = fuzzy_matcher->edits();
// exact match?
if (cost == 0)
break;
}
}
}
catch (...)
{
// this should never happen
warning("exception while searching", pathname);
}
#ifdef HAVE_LIBZ
#ifdef WITH_DECOMPRESSION_THREAD
if (flag_decompress && cost == 0)
zthread.cancel();
#endif
#endif
// close file or -z: loop over next extracted archive parts, when applicable
} while (close_file(pathname));
return cost;
}
// search input to output the pattern matches
void Grep::search(const char *pathname, uint16_t cost)
{
// -Zbest (or --best-match) without --match: compute cost if not yet computed by --sort=best
if (flag_best_match && flag_fuzzy > 0 && !flag_match && !flag_quiet && (!flag_files_with_matches || flag_format != NULL) && matchers == NULL && pathname != Static::LABEL_STANDARD_INPUT)
{
// -Z: matcher is a FuzzyMatcher for sure
reflex::FuzzyMatcher *fuzzy_matcher = dynamic_cast<reflex::FuzzyMatcher*>(matcher);
fuzzy_matcher->distance(static_cast<uint16_t>(flag_fuzzy));
// compute distance cost if not yet computed by --sort=best
if (cost == Entry::UNDEFINED_COST)
{
cost = compute_cost(pathname);
// if no match, then stop searching this file
if (cost == Entry::UNDEFINED_COST)
return;
}
// no match found?
if (cost == Entry::MAX_COST)
{
if (!flag_invert_match)
return;
// -v: invert match when no match was found, zero cost since we don't expect any matches
cost = 0;
}
// combine max distance cost (lower byte) with INS, DEL, SUB and BIN fuzzy flags (upper byte)
fuzzy_matcher->distance((cost & 0xff) | (flag_fuzzy & 0xff00));
}
// stop when output is blocked
if (out.eof)
return;
try
{
// open (archive or compressed) file (pathname is NULL to read stdin), return on failure
if (!open_file(pathname))
return;
}
catch (...)
{
// this should never happen
warning("exception while opening", pathname != NULL && *pathname != '\0' ? pathname : flag_label);
return;
}
// pathname is NULL when stdin is searched
if (pathname == Static::LABEL_STANDARD_INPUT)
pathname = flag_label;
bool matched = false;
// -z: loop over extracted archive parts, when applicable
do
{
if (!init_read())
goto exit_search;
try
{
// --files: reset the matching[] bitmask used in cnf_matching() for each matcher in matchers
if (flag_files && matchers != NULL)
{
// hold the output
out.hold();
// reset the bit corresponding to each matcher in matchers
size_t n = matchers->size();
matching.resize(0);
matching.resize(n);
// reset the bit corresponding to the OR NOT terms of each matcher in matchers
notmatching.resize(n);
size_t j = 0;
for (auto& i : *matchers)
{
notmatching[j].resize(0);
notmatching[j].resize(i.size() > 0 ? i.size() - 1 : 0);
++j;
}
}
if (flag_quiet || flag_files_with_matches)
{
// option -q, -l, or -L
// --format with open/close or --max-files: we must out.acquire() lock before Stats::found_part()
bool acquire = (flag_format != NULL && (flag_format_open != NULL || flag_format_close != NULL)) || (!flag_quiet && flag_max_files > 0);
// skip accurate line number counting, only count matching lines
if (!flag_multiline && flag_max_line == 0 && flag_stats == NULL)
matcher->lineno_skip(true);
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (!cnf_matching(bol, eol, acquire) || out.holding())
continue;
}
if (flag_min_count == 0 || flag_ungroup || flag_only_matching || lineno != current_lineno)
++matches;
// --min-count: require at least min-count matches, otherwise stop searching
if (flag_min_count == 0 || matches >= flag_min_count)
break;
lineno = current_lineno;
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
matches = 0;
// --files: if we are still holding the output and CNF is finally satisfyable then a match was made
if (flag_files && matchers != NULL)
{
if (!cnf_satisfied(acquire))
goto exit_search;
matches = 1;
}
// -v: invert
if (flag_invert_match)
matches = (matches == 0);
if (matches > 0)
{
// --format with open/close or --max-files: we must out.acquire() lock before Stats::found_part()
if (acquire)
out.acquire();
if (!flag_files || matchers == NULL)
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
// -l or -L
if (flag_files_with_matches)
{
if (flag_format != NULL)
{
if (flag_format_open != NULL)
out.format(flag_format_open, pathname, partname, Stats::found_parts(), NULL, matcher, heading, false, Stats::found_parts() > 1);
out.format(flag_format, pathname, partname, 1, NULL, matcher, heading, false, false);
if (flag_format_close != NULL)
out.format(flag_format_close, pathname, partname, Stats::found_parts(), NULL, matcher, heading, false, Stats::found_parts() > 1);
}
else
{
out.header(pathname, partname);
if (flag_null)
out.chr('\0');
else
out.nl();
}
}
}
}
else if (flag_count)
{
// option -c
// --format with open/close or --max-files: we must out.acquire() lock before Stats::found_part()
bool acquire = (flag_format != NULL && (flag_format_open != NULL || flag_format_close != NULL)) || flag_max_files > 0;
// not --match to match all or not -v
if (!flag_match || !flag_invert_match)
{
if (flag_ungroup || flag_only_matching)
{
// -co or -cu: count the number of patterns matched
// skip accurate line number counting, only count matching lines
if (!flag_multiline && flag_max_line == 0 && flag_stats == NULL)
matcher->lineno_skip(true);
while (matcher->find())
{
// --range: max line exceeded?
if (flag_max_line > 0 && matcher->lineno() > flag_max_line)
break;
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching, with --files acquire lock before Stats::found_part()
if (!cnf_matching(bol, eol, acquire))
continue;
}
++matches;
// -m: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
}
}
else if (!flag_multiline && !flag_invert_match && flag_max_line == 0 && matchers == NULL && flag_stats == NULL)
{
// -c without -o/-u/-v/-K/-%: count the number of matching lines
// skip accurate line number counting, only count matching lines
matcher->lineno_skip(true);
while (matcher->find())
{
++matches;
// -m: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
// if the match does not span more than one line, then skip to end of the line (we count matching lines)
if (!matcher->at_bol())
matcher->skip('\n');
}
}
else
{
// -c without -o/-u: count the number of matching lines
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
if (lineno != current_lineno)
{
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching, with --files acquire lock before Stats::found_part()
if (!cnf_matching(bol, eol, acquire))
continue;
}
++matches;
// -m: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
lineno = current_lineno;
// if the match does not span more than one line, then skip to end of the line (we count matching lines)
if (!flag_multiline && !matcher->at_bol())
matcher->skip('\n');
}
}
// -c with -v: count non-matching lines
if (flag_invert_match)
{
matches = matcher->lineno() - matches;
if (matches > 0)
--matches;
}
}
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
goto exit_search;
// --files: if we are still holding the output and CNF is not satisfyable then no global matches were made
if (flag_files && matchers != NULL)
{
if (!cnf_satisfied(acquire))
goto exit_search; // we cannot report 0 matches and ensure accurate output
}
else
{
// --format with open/close or --max-files: we must out.acquire() lock before Stats::found_part()
if (acquire)
out.acquire();
// --max-files: max reached?
// unfortunately, allowing 'acquire' below produces "x matching + y in archives"
// but without this we cannot produce correct format-open and format-close outputs
if (matches > 0 || acquire)
if (!Stats::found_part())
goto exit_search;
}
if (flag_format != NULL)
{
if (flag_format_open != NULL)
out.format(flag_format_open, pathname, partname, Stats::found_parts(), NULL, matcher, heading, false, Stats::found_parts() > 1);
out.format(flag_format, pathname, partname, matches, NULL, matcher, heading, false, false);
if (flag_format_close != NULL)
out.format(flag_format_close, pathname, partname, Stats::found_parts(), NULL, matcher, heading, false, Stats::found_parts() > 1);
}
else
{
if (flag_with_filename || !partname.empty())
{
out.header(pathname, partname);
if (flag_null)
{
out.chr('\0');
}
else
{
out.str(color_se);
out.str(flag_separator);
out.str(color_off);
}
}
out.num(matches);
out.nl();
}
}
else if (flag_format != NULL)
{
// option --format
// --format with open/close: we must out.acquire() lock before Stats::found_part()
bool acquire = flag_format_open != NULL || flag_format_close != NULL;
if (flag_invert_match)
{
// construct event handler functor with captured *this and some of the locals
FormatInvertMatchGrepHandler invert_match_handler(*this, pathname);
// register the event handler to output non-matching lines, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&invert_match_handler);
// to get the context from the invert_match handler explicitly
reflex::AbstractMatcher::Context context;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
if (lineno != current_lineno)
{
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (!cnf_matching(bol, eol))
continue;
}
// get the lines before the matched line
context = matcher->before();
// output non-matching lines up to this line
if (context.len > 0)
invert_match_handler(*matcher, context.buf, context.len, context.num);
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
goto done_search;
// --max-files: max reached?
if (stop)
goto exit_search;
// -m: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
goto done_search;
// output blocked?
if (out.eof)
goto exit_search;
}
lineno = current_lineno + matcher->lines() - 1;
}
// get the remaining context after the last match to output
context = matcher->after();
if (context.len > 0)
invert_match_handler(*matcher, context.buf, context.len, context.num);
}
else
{
// counter for format field %m matches
size_t format_matching = 0;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (!cnf_matching(bol, eol, acquire))
continue;
}
if (current_lineno != lineno || flag_ungroup)
{
++matches;
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
{
lineno = current_lineno;
continue;
}
// output --format-open=FORMAT for the first (or --min-count) match found
if (matches == flag_min_count + (flag_min_count == 0))
{
if (flag_files && matchers != NULL)
{
// --format-open: we must acquire lock early before Stats::found_part()
if (acquire && out.holding())
{
out.acquire();
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
}
else
{
// --format-open: we must acquire lock early before Stats::found_part()
if (acquire)
out.acquire();
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
if (flag_format_open != NULL)
{
// output --format-open=FORMAT
out.format(flag_format_open, pathname, partname, Stats::found_parts(), NULL, matcher, heading, false, Stats::found_parts() > 1);
// --files: undo files count
if (flag_files && matchers != NULL && out.holding())
Stats::undo_found_part();
}
}
}
// output --format=FORMAT
// if the match does not span more than one line, then skip to end of the line
if (!out.format(flag_format, pathname, partname, matches, &format_matching, matcher, heading, matches > 1 || current_lineno == lineno, matches > 1))
if (!flag_multiline && !matcher->at_bol())
if (!matcher->skip('\n'))
break;
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
out.check_flush();
lineno = current_lineno;
}
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
goto exit_search;
// --files: if we are still holding the output and CNF is not satisfyable then no global matches were made
if (flag_files && matchers != NULL)
if (!cnf_satisfied(true))
goto exit_search;
// output --format-close=FORMAT
if (matches > 0 && flag_format_close != NULL)
out.format(flag_format_close, pathname, partname, Stats::found_parts(), NULL, matcher, heading, false, Stats::found_parts() > 1);
}
else if (flag_only_matching && flag_before_context == 0 && flag_after_context == 0)
{
// option -o without -ABC
// counter for replace field %m matches
size_t replace_matching = 0;
bool nl = false;
// skip accurate line number counting, only count matching lines
if (!flag_line_number && !flag_multiline && flag_max_line == 0 && flag_stats == NULL)
matcher->lineno_skip(true);
while (matcher->find())
{
const char *begin = matcher->begin();
size_t size = matcher->size();
binary = check_binary(begin, size);
if (hex && !binary)
{
out.dump.done();
}
else if (!hex && binary && nl)
{
out.nl();
nl = false;
}
if (nl)
{
out.nl();
nl = false;
}
size_t current_lineno = matcher->lineno();
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
// output blocked?
if (out.eof)
goto exit_search;
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (!cnf_matching(bol, eol))
continue;
}
if (lineno != current_lineno || flag_ungroup)
{
lineno = current_lineno;
++matches;
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches > flag_max_count)
break;
if (matches == flag_min_count + (flag_min_count == 0) && (!flag_files || matchers == NULL))
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
}
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
if (!flag_no_header)
out.header(pathname, partname, heading, current_lineno, matcher, matcher->first(), flag_separator, binary);
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
hex = binary;
if (binary)
{
if (flag_hex || flag_with_hex)
{
out.dump.next(matcher->first());
out.dump.hex(Output::Dump::HEX_MATCH, matcher->first(), begin, size);
}
else
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
lineno += matcher->lines() - 1;
}
else if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
}
else
{
if (flag_multiline)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, ++lineno, NULL, matcher->first() + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
}
size -= from - begin;
begin = from;
}
if (size > 0)
{
bool lf_only = begin[size - 1] == '\n';
size -= lf_only;
if (size > 0)
{
out.str(match_ms);
out.str(begin, size);
out.str(match_off);
}
out.nl(lf_only);
}
else
{
nl = true;
}
}
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
goto exit_search;
if (nl)
out.nl();
if (hex)
out.dump.done();
}
else if (flag_only_matching)
{
// option -o with -ABC specified
size_t count = 0;
size_t width = flag_before_context + flag_after_context;
bool nl = false;
// skip accurate line number counting, only count matching lines
if (!flag_line_number && !flag_multiline && flag_max_line == 0 && flag_stats == NULL)
matcher->lineno_skip(true);
// construct event handler functor with captured *this and some of the locals to save the restline when the buffer shifts
OnlyMatchingContextGrepHandler handler(*this, pathname);
// register the event handler to update restline on buffer shift, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&handler);
// the rest of the matching line
restline_data = NULL;
restline_size = 0;
restline_last = 0;
while (matcher->find())
{
const char *begin = matcher->begin();
size_t size = matcher->size();
binary = check_binary(begin, size);
if (hex && !binary)
{
out.dump.done();
}
else if (!hex && binary && nl)
{
out.nl();
nl = false;
}
if (nl)
{
if (restline_size > 0)
{
out.str(color_cx);
if (utf8nlen(restline_data, restline_size) > width + 3)
{
out.utf8strn(restline_data, restline_size, width);
out.str(color_off);
out.str(color_se);
out.str("...", 3);
}
else
{
out.str(restline_data, restline_size);
}
out.str(color_off);
restline_size = 0;
}
if (count > 0)
{
out.str(color_se);
out.str("...", 3);
out.str(color_off);
out.str(color_cn);
out.str("[+", 2);
out.num(count);
out.str(" more]", 6);
out.str(color_off);
}
out.nl();
count = 0;
width = flag_before_context + flag_after_context;
nl = false;
}
size_t current_lineno = matcher->lineno();
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
// output blocked?
if (out.eof)
goto exit_search;
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (!cnf_matching(bol, eol))
continue;
}
if (lineno != current_lineno || flag_ungroup)
{
lineno = current_lineno;
++matches;
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches > flag_max_count)
break;
if (matches == flag_min_count + (flag_min_count == 0) && (!flag_files || matchers == NULL))
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
}
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
size_t first = matcher->first();
size_t last = matcher->last();
if (!flag_no_header)
out.header(pathname, partname, heading, current_lineno, matcher, first, flag_separator, binary);
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
hex = binary;
if (binary)
{
if (flag_hex || flag_with_hex)
{
out.dump.next(first);
out.dump.hex(Output::Dump::HEX_MATCH, first, begin, size);
}
else
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
lineno += matcher->lines() - 1;
}
else
{
if (restline_size > 0)
{
if (first - restline_last < restline_size)
restline_size = first - restline_last;
size_t after = utf8nlen(restline_data, restline_size);
if (after > width)
after = width;
width -= after;
if (after > 0)
{
out.str(color_cx);
out.utf8strn(restline_data, restline_size, after);
out.str(color_off);
}
restline_size = 0;
}
if (width == 0)
{
++count;
}
else
{
while (true)
{
// check for additional lines in the match output
size_t rest = 0;
const char *to = static_cast<const char*>(memchr(begin, '\n', size));
if (to != NULL)
{
rest = size;
size = to - begin;
rest -= size;
}
// length of the match is the number of UTF-8-encoded Unicode characters
size_t length = utf8nlen(begin, size);
size_t fit_length = length;
if (fit_length > width)
{
if (fit_length > width + 4)
fit_length = width;
width = 0;
}
else
{
width -= fit_length;
}
if (!nl)
{
const char *bol = matcher->bol();
size_t border = matcher->border();
size_t margin = utf8nlen(bol, border);
size_t before = flag_before_context * fit_length / (flag_before_context + flag_after_context);
if (before < flag_before_context)
before = flag_before_context - before;
else
before = 0;
if (margin > before)
{
out.str(color_se);
out.str("...", 3);
out.str(color_off);
out.str(color_cx);
out.utf8strn(utf8skipn(bol, border, margin - before), border, before);
out.str(color_off);
width -= before;
}
else
{
if (border > 0)
{
out.str(color_cx);
out.str(bol, border);
out.str(color_off);
}
if (margin >= 3)
width -= margin - 3;
else
width += 3 - margin;
}
}
out.str(match_ms);
if (fit_length == length)
{
out.str(begin, size);
}
else
{
out.utf8strn(begin, size, fit_length);
out.str("[+", 2);
out.num(length - fit_length);
out.chr(']');
}
out.str(match_off);
// no more additional lines to output?
if (to == NULL)
break;
out.nl();
// output header and repeat to output a multiline match after previous newline
if (!flag_no_header)
out.header(pathname, partname, heading, ++lineno, matcher, first + (to - begin), flag_separator_bar, false);
begin = to + 1;
size = rest - 1;
width = flag_before_context + flag_after_context;
nl = true;
}
const char *eol = matcher->eol(); // warning: call eol() before bol() and end()
const char *end = matcher->end();
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
nl = true;
}
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
goto exit_search;
if (nl)
{
if (restline_size > 0)
{
out.str(color_cx);
if (utf8nlen(restline_data, restline_size) > width + 3)
{
out.utf8strn(restline_data, restline_size, width);
out.str(color_off);
out.str(color_se);
out.str("...", 3);
}
else
{
out.str(restline_data, restline_size);
}
out.str(color_off);
restline_size = 0;
}
if (count > 0)
{
out.str(color_se);
out.str("...", 3);
out.str(color_off);
out.str(color_cn);
out.str("[+", 2);
out.num(count);
out.str(" more]", 6);
out.str(color_off);
}
out.nl();
}
if (hex)
out.dump.done();
}
else if (flag_before_context == 0 && flag_after_context == 0 && !flag_any_line && !flag_invert_match)
{
// options -ABC, -y, -v are not specified, --hexdump context is supported (including with options -ABC)
// counter for replace field %m matches
size_t replace_matching = 0;
bool colorize = flag_color != NULL || flag_replace != NULL || flag_tag != NULL;
// skip accurate line number counting, only count matching lines
if (!flag_line_number && !flag_multiline && flag_max_line == 0 && flag_stats == NULL)
matcher->lineno_skip(true);
// construct event handler functor with captured *this and some of the locals to output or save restline when the buffer shifts
GrepHandler handler(*this, pathname);
// register the event handler to update restline on buffer shift, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&handler);
// --hexdump: keep -B NUM+1 before context times hex column bytes in the buffer when shifting
if (flag_hex_before > 0)
matcher->set_reserve(flag_hex_before * flag_hex_columns);
// the rest of the matching line
restline_data = NULL;
restline_size = 0;
restline_last = 0;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
if (lineno != current_lineno || flag_ungroup)
{
output_restline(false, flag_hex_after > 0);
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
// output blocked?
if (out.eof)
goto exit_search;
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (matchers != NULL && !cnf_matching(bol, eol))
continue;
++matches;
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
{
lineno = current_lineno;
continue;
}
if (matches == flag_min_count + (flag_min_count == 0) && (!flag_files || matchers == NULL))
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
binary = check_binary(bol, eol - bol);
// --hexdump with ABC context: if binary then show hex context
if (binary && (flag_hex_after > 0 || flag_hex_before > 0))
{
const char *aft = matcher->aft((flag_hex_after + flag_hex_before) * flag_hex_columns);
if (aft > eol)
{
eol = aft;
bol = matcher->bol(); // warning: bol() position may have shifted with aft()
}
}
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
size_t border = matcher->border();
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
const char *end = matcher->end();
size_t size = matcher->size();
if (hex && !binary)
out.dump.done();
if (!flag_no_header)
out.header(pathname, partname, heading, current_lineno, matcher, first, flag_separator, binary);
hex = binary;
lineno = current_lineno;
if (binary)
{
// --hexdump with ABC context: if binary then show hex context
if (flag_hex_before > 0)
{
const char *bef = matcher->bef(flag_hex_before * flag_hex_columns);
if (bef < bol)
{
bol = bef;
border = begin - bol;
}
size_t left = 0;
if (restline_last + restline_size < first)
{
left = flag_hex_before * flag_hex_columns + (first % flag_hex_columns) - flag_hex_columns;
if (restline_last + restline_size + left > first)
left = first - (restline_last + restline_size);
}
if (begin > bol + left)
{
bol = begin - left;
border = left;
}
}
out.dump.hex(Output::Dump::HEX_LINE, first - border, bol, border);
out.dump.hex(Output::Dump::HEX_MATCH, first, begin, size);
if (flag_ungroup)
{
// --hexdump with ABC context: if binary then show hex context
if (flag_hex_after > 0)
{
size_t right = flag_hex_after * flag_hex_columns - ((last - 1) % flag_hex_columns) - 1;
if (end + right < eol)
eol = end + right;
}
out.dump.hex(Output::Dump::HEX_LINE, last, end, eol - end);
out.dump.done();
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
lineno += matcher->lines() - 1;
}
else
{
if (border > 0)
{
out.str(color_sl);
out.str(bol, border);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
lineno += matcher->lines() - 1;
}
else
{
if (flag_multiline)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, ++lineno, NULL, first + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
}
size -= from - begin;
begin = from;
}
out.str(match_ms);
out.str(begin, size);
out.str(match_off);
}
if (flag_ungroup)
{
if (eol > end)
{
bool lf_only = end[eol - end - 1] == '\n';
eol -= lf_only;
if (eol > end)
{
out.str(color_sl);
out.str(end, eol - end);
out.str(color_off);
}
out.nl(lf_only);
}
else if (matcher->hit_end())
{
out.nl();
}
else
{
out.check_flush();
}
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
}
// no -u, no --hexdump and no colors: if the match does not span more than one line, then skip to end of the line
if (!flag_ungroup && !colorize && !(binary && (flag_hex_after > 0 || flag_hex_before > 0)))
if (!flag_multiline && !matcher->at_bol())
if (!matcher->skip('\n'))
break;
}
else
{
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
size_t size = matcher->size();
if (size > 0)
{
size_t lines = matcher->lines();
if (lines > 1 || colorize)
{
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
if (binary)
{
if (restline_data != NULL)
{
if (restline_last + restline_size > first)
restline_size = first - restline_last;
// --hexdump with ABC context: if binary then show hex context
if (flag_hex_after > 0)
{
size_t right = flag_hex_after * flag_hex_columns - ((restline_last - 1) % flag_hex_columns) - 1;
if (right < restline_size)
{
out.dump.hex(Output::Dump::HEX_LINE, restline_last, restline_data, right);
restline_data += right;
restline_size -= right;
restline_last += right;
}
else
{
out.dump.hex(Output::Dump::HEX_LINE, restline_last, restline_data, restline_size);
restline_data = NULL;
}
}
if (restline_data != NULL)
{
// --hexdump with ABC context: if binary then show hex context
if (flag_hex_before > 0)
{
size_t left = flag_hex_before * flag_hex_columns + (first % flag_hex_columns) - flag_hex_columns;
if (left < restline_size)
{
if (!flag_no_header)
out.header(pathname, partname, heading, current_lineno, matcher, first, flag_separator_bar, binary);
restline_data += restline_size - left;
restline_last += restline_size - left;
restline_size = left;
}
}
out.dump.hex(Output::Dump::HEX_LINE, restline_last, restline_data, restline_size);
}
restline_data = NULL;
}
out.dump.hex(Output::Dump::HEX_MATCH, first, begin, size);
}
else
{
if (restline_data != NULL)
{
if (restline_last + restline_size > first)
restline_size = first - restline_last;
out.str(color_sl);
out.str(restline_data, restline_size);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
}
else
{
if (lines > 1)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
size_t num = 1;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, lineno + num, NULL, first + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
++num;
}
size -= from - begin;
begin = from;
}
out.str(match_ms);
out.str(begin, size);
out.str(match_off);
}
}
const char *eol = matcher->eol(true); // warning: call eol() before end()
const char *end = matcher->end();
if (lines == 1)
{
// --hexdump with ABC context: if binary then show hex context
if (binary && (flag_hex_after > 0 || flag_hex_before > 0))
{
const char *aft = matcher->aft((flag_hex_after + flag_hex_before) * flag_hex_columns);
if (aft > eol)
{
eol = aft;
end = matcher->end(); // warning: end() position may have shifted with aft()
}
}
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
else
{
binary = check_binary(end, eol - end);
// --hexdump with ABC context: if binary then show hex context
if (binary && (flag_hex_after > 0 || flag_hex_before > 0))
{
const char *aft = matcher->aft((flag_hex_after + flag_hex_before) * flag_hex_columns);
if (aft > eol)
{
eol = aft;
end = matcher->end(); // warning: end() position may have shifted with aft()
}
}
if (hex && !binary)
out.dump.done();
else if (!hex && binary)
out.nl();
if (hex != binary && !flag_no_header)
out.header(pathname, partname, heading, lineno + lines - 1, matcher, last, flag_separator, binary);
hex = binary;
if (flag_ungroup)
{
if (binary)
{
// --hexdump with ABC context: if binary then show hex context
if (flag_hex_after > 0)
{
size_t right = flag_hex_after * flag_hex_columns - ((last - 1) % flag_hex_columns) - 1;
if (end + right < eol)
eol = end + right;
}
out.dump.hex(Output::Dump::HEX_LINE, last, end, eol - end);
out.dump.done();
}
else
{
if (eol > end)
{
bool lf_only = end[eol - end - 1] == '\n';
eol -= lf_only;
if (eol > end)
{
out.str(color_sl);
out.str(end, eol - end);
out.str(color_off);
}
out.nl(lf_only);
}
else if (matcher->hit_end())
{
out.nl();
}
else
{
out.check_flush();
}
}
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
lineno += lines - 1;
}
}
}
}
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
goto exit_search;
output_restline(false, flag_hex_after > 0);
if (binary)
out.dump.done();
}
else if (flag_before_context == 0 && flag_after_context == 0 && !flag_any_line)
{
// option -v without -ABC, -y
// construct event handler functor with captured *this and some of the locals to output or save restline when the buffer shifts
InvertMatchGrepHandler invert_match_handler(*this, pathname);
// register the event handler to output non-matching lines, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&invert_match_handler);
// to get the context from the invert_match handler explicitly
reflex::AbstractMatcher::Context context;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
if (lineno != current_lineno)
{
if (matchers != NULL)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (!cnf_matching(bol, eol))
continue;
}
// get the lines before the matched line
context = matcher->before();
// output non-matching lines up to this line
if (context.len > 0)
invert_match_handler(*matcher, context.buf, context.len, context.num);
if (matches > 0 && (binfile || (binary && !flag_hex && !flag_with_hex)))
break;
if (binary)
out.dump.done();
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
goto done_search;
// --max-files: max reached?
if (stop)
goto exit_search;
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
goto done_search;
// output blocked?
if (out.eof)
goto exit_search;
}
lineno = current_lineno + matcher->lines() - 1;
}
// get the remaining context after the last match to output
context = matcher->after();
if (context.len > 0)
invert_match_handler(*matcher, context.buf, context.len, context.num);
if (matches > 0 && (binfile || (binary && !flag_hex && !flag_with_hex)))
{
if (flag_binary_without_match)
matches = 0;
else
out.binary_file_matches(pathname, partname);
}
if (binary)
out.dump.done();
}
else if (flag_any_line)
{
// option -y
// counter for replace field %m matches
size_t replace_matching = 0;
bool colorize = flag_color != NULL || flag_replace != NULL || flag_tag != NULL;
// to output the rest of the matching line
restline_data = NULL;
restline_size = 0;
restline_last = 0;
// construct event handler functor with captured *this and some of the locals to output or save restline when the buffer shifts
AnyLineGrepHandler any_line_handler(*this, pathname);
// register the event handler functor to output non-matching lines, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&any_line_handler);
// to output colors with or without -v
short v_hex_line = flag_invert_match ? Output::Dump::HEX_CONTEXT_LINE : Output::Dump::HEX_LINE;
short v_hex_match = flag_invert_match ? Output::Dump::HEX_CONTEXT_MATCH : Output::Dump::HEX_MATCH;
const char *v_color_sl = flag_invert_match ? color_cx : color_sl;
const char *v_match_ms = flag_invert_match ? match_mc : match_ms;
const char *separator = flag_invert_match ? flag_separator_dash : flag_separator;
// to get the context from the any_line handler explicitly
reflex::AbstractMatcher::Context context;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
if (lineno != current_lineno) /* || flag_ungroup) // logically OK but dead code because -y */
{
output_restline(flag_invert_match);
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (matchers != NULL && !cnf_matching(bol, eol))
continue;
if (!flag_invert_match)
{
++matches;
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
{
lineno = current_lineno;
continue;
}
if (matches == flag_min_count + (flag_min_count == 0) && (!flag_files || matchers == NULL))
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
}
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
// --max-files: max reached?
if (stop)
goto exit_search;
// get the lines before the matched line
context = matcher->before();
if (context.len > 0)
{
any_line_handler(*matcher, context.buf, context.len, context.num);
if (matches > 0 && (binfile || (binary && !flag_hex && !flag_with_hex)))
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
}
// output blocked?
if (out.eof)
goto exit_search;
binary = check_binary(bol, eol - bol);
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else if (flag_invert_match)
{
lineno = current_lineno + matcher->lines() - 1;
continue;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
size_t border = matcher->border();
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
const char *end = matcher->end();
size_t size = matcher->size();
if (hex && !binary)
out.dump.done();
if (!flag_no_header)
out.header(pathname, partname, heading, current_lineno, matcher, first, separator, binary);
hex = binary;
lineno = current_lineno;
if (binary)
{
out.dump.hex(v_hex_line, first - border, bol, border);
out.dump.hex(v_hex_match, first, begin, size);
if (false) /* flag_ungroup) // logically OK but dead code because -y */
{
out.dump.hex(v_hex_line, last, end, eol - end);
out.dump.done();
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
lineno += matcher->lines() - 1;
}
else
{
if (border > 0)
{
out.str(v_color_sl);
out.str(bol, border);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(v_match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
lineno += matcher->lines() - 1;
}
else
{
if (flag_multiline)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(v_match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, ++lineno, NULL, first + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
}
size -= from - begin;
begin = from;
}
out.str(v_match_ms);
out.str(begin, size);
out.str(match_off);
}
if (false) /* flag_ungroup) // logically OK but dead code because -y */
{
if (eol > end)
{
bool lf_only = end[eol - end - 1] == '\n';
eol -= end[eol - end - 1] == '\n';
if (eol > end)
{
out.str(v_color_sl);
out.str(end, eol - end);
out.str(color_off);
}
out.nl(lf_only);
}
else if (matcher->hit_end())
{
out.nl();
}
else
{
out.check_flush();
}
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
}
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
// no -u and no colors: if the match does not span more than one line, then skip to end of the line
if (!flag_ungroup && !colorize)
if (!flag_multiline && !matcher->at_bol())
if (!matcher->skip('\n'))
break;
}
else if (!binfile && (!binary || flag_hex || flag_with_hex))
{
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
size_t size = matcher->size();
if (size > 0)
{
size_t lines = matcher->lines();
if (lines > 1 || colorize)
{
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
if (binary)
{
if (restline_data != NULL)
{
if (restline_last + restline_size > first)
restline_size = first - restline_last;
out.dump.hex(v_hex_line, restline_last, restline_data, restline_size);
}
out.dump.hex(v_hex_match, first, begin, size);
}
else
{
if (restline_data != NULL)
{
if (restline_last + restline_size > first)
restline_size = first - restline_last;
out.str(v_color_sl);
out.str(restline_data, restline_size);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(v_match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
}
else
{
if (lines > 1)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
size_t num = 1;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(v_match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, lineno + num, NULL, first + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
++num;
}
size -= from - begin;
begin = from;
}
out.str(v_match_ms);
out.str(begin, size);
out.str(match_off);
}
}
const char *eol = matcher->eol(true); // warning: call eol() before end()
const char *end = matcher->end();
if (lines == 1)
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
else
{
binary = check_binary(end, eol - end);
if (hex && !binary)
out.dump.done();
else if (!hex && binary)
out.nl();
if (hex != binary && !flag_no_header)
out.header(pathname, partname, heading, lineno + lines - 1, matcher, last, flag_separator, binary);
hex = binary;
if (false) /* flag_ungroup) // logically OK but dead code because -y */
{
if (binary)
{
out.dump.hex(v_hex_line, last, end, eol - end);
out.dump.done();
}
else
{
if (eol > end)
{
bool lf_only = end[eol - end - 1] == '\n';
eol -= lf_only;
if (eol > end)
{
out.str(v_color_sl);
out.str(end, eol - end);
out.str(color_off);
}
out.nl(lf_only);
}
else if (matcher->hit_end())
{
out.nl();
}
else
{
out.check_flush();
}
}
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
lineno += lines - 1;
}
}
}
}
}
output_restline(flag_invert_match);
if (flag_max_count == 0 || matches < flag_max_count)
{
// get the remaining context after the last match to output
context = matcher->after();
if (context.len > 0)
any_line_handler(*matcher, context.buf, context.len, context.num);
}
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
matches = 0;
if (matches > 0 && (binfile || (binary && !flag_hex && !flag_with_hex)))
{
if (flag_binary_without_match)
matches = 0;
else
out.binary_file_matches(pathname, partname);
}
if (binary)
out.dump.done();
}
else if (!flag_invert_match)
{
// options -ABC without -v
// counter for replace field %m matches
size_t replace_matching = 0;
bool colorize = flag_color != NULL || flag_replace != NULL || flag_tag != NULL;
// to output the rest of the matching line
restline_data = NULL;
restline_size = 0;
restline_last = 0;
// construct event handler functor with captured *this and some of the locals to output or save restline when the buffer shifts
ContextGrepHandler context_handler(*this, pathname);
// register the event handler functor to output non-matching lines, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&context_handler);
// to get the context from the context handler explicitly
reflex::AbstractMatcher::Context context;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
if (lineno != current_lineno || flag_ungroup)
{
output_restline();
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (matchers != NULL && !cnf_matching(bol, eol))
continue;
++matches;
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
{
lineno = current_lineno;
continue;
}
// --range: max line exceeded?
if (flag_max_line > 0 && current_lineno > flag_max_line)
break;
// --max-files: max reached?
if (stop)
goto exit_search;
if (matches == 1 && (!flag_files || matchers == NULL))
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
// output blocked?
if (out.eof)
goto exit_search;
// get the lines before the matched line
context = matcher->before();
if (context.len > 0)
context_handler(*matcher, context.buf, context.len, context.num);
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches > flag_max_count)
break;
context_handler.output_before_context();
binary = check_binary(bol, eol - bol);
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
if (flag_files && matchers != NULL && out.holding())
continue;
goto done_search;
}
size_t border = matcher->border();
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
const char *end = matcher->end();
size_t size = matcher->size();
if (hex && !binary)
out.dump.done();
if (!flag_no_header)
out.header(pathname, partname, heading, current_lineno, matcher, first, flag_separator, binary);
hex = binary;
lineno = current_lineno;
if (binary)
{
out.dump.hex(Output::Dump::HEX_LINE, first - border, bol, border);
out.dump.hex(Output::Dump::HEX_MATCH, first, begin, size);
if (flag_ungroup)
{
out.dump.hex(Output::Dump::HEX_LINE, last, end, eol - end);
out.dump.done();
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
lineno += matcher->lines() - 1;
}
else
{
if (border > 0)
{
out.str(color_sl);
out.str(bol, border);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
lineno += matcher->lines() - 1;
}
else
{
if (flag_multiline)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, ++lineno, NULL, first + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
}
size -= from - begin;
begin = from;
}
out.str(match_ms);
out.str(begin, size);
out.str(match_off);
}
if (flag_ungroup)
{
if (eol > end)
{
bool lf_only = end[eol - end - 1] == '\n';
eol -= lf_only;
if (eol > end)
{
out.str(color_sl);
out.str(end, eol - end);
out.str(color_off);
}
out.nl(lf_only);
}
else if (matcher->hit_end())
{
out.nl();
}
else
{
out.check_flush();
}
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
}
// no -u and no colors: if the match does not span more than one line, then skip to end of the line
if (!flag_ungroup && !colorize)
if (!flag_multiline && !matcher->at_bol())
if (!matcher->skip('\n'))
break;
}
else
{
// --min-count: require at least min-count matches
if (flag_min_count > 0 && matches < flag_min_count)
continue;
size_t size = matcher->size();
if (size > 0)
{
size_t lines = matcher->lines();
if (lines > 1 || colorize)
{
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
if (binary)
{
if (restline_data != NULL)
{
if (restline_last + restline_size > first)
restline_size = first - restline_last;
out.dump.hex(Output::Dump::HEX_LINE, restline_last, restline_data, restline_size);
}
out.dump.hex(Output::Dump::HEX_MATCH, first, begin, size);
}
else
{
if (restline_data != NULL)
{
if (restline_last + restline_size > first)
restline_size = first - restline_last;
out.str(color_sl);
out.str(restline_data, restline_size);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_ms);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
}
else
{
if (lines > 1)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
size_t num = 1;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_ms);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, lineno + num, NULL, first + (to - begin) + 1, flag_separator_bar, false);
from = to + 1;
++num;
}
size -= from - begin;
begin = from;
}
out.str(match_ms);
out.str(begin, size);
out.str(match_off);
}
}
const char *eol = matcher->eol(true); // warning: call eol() before end()
const char *end = matcher->end();
if (lines == 1)
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
else
{
binary = check_binary(end, eol - end);
if (hex && !binary)
out.dump.done();
else if (!hex && binary)
out.nl();
if (hex != binary && !flag_no_header)
out.header(pathname, partname, heading, lineno + lines - 1, matcher, last, flag_separator, binary);
hex = binary;
if (flag_ungroup)
{
if (binary)
{
out.dump.hex(Output::Dump::HEX_LINE, last, end, eol - end);
out.dump.done();
}
else
{
if (eol > end)
{
bool lf_only = end[eol - end - 1] == '\n';
eol -= lf_only;
if (eol > end)
{
out.str(color_sl);
out.str(end, eol - end);
out.str(color_off);
}
out.nl(lf_only);
}
else if (matcher->hit_end())
{
out.nl();
}
else
{
out.check_flush();
}
}
}
else
{
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
lineno += lines - 1;
}
}
}
}
context_handler.set_after_lineno(lineno + 1);
}
output_restline();
// get the remaining context after the last match to output
context = matcher->after();
if (context.len > 0)
context_handler(*matcher, context.buf, context.len, context.num);
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
matches = 0;
else if (matches > 0)
out.binary_file_matches(pathname, partname);
}
if (binary)
out.dump.done();
}
else
{
// options -ABC with -v
size_t after = flag_after_context;
size_t last_lineno = lineno;
size_t next_lineno = lineno + 1;
// counter for replace field %m matches
size_t replace_matching = 0;
bool colorize = flag_color != NULL || flag_replace != NULL || flag_tag != NULL;
// to output the rest of the matching line
restline_data = NULL;
restline_size = 0;
restline_last = 0;
// construct event handler functor with captured *this and some of the locals to output or save restline when the buffer shifts
InvertContextGrepHandler invert_context_handler(*this, pathname);
// register the event handler functor to output non-matching lines, matcher resets to NULL when a new input source is assigned
matcher->set_handler(&invert_context_handler);
// to get the context from the context handler explicitly
reflex::AbstractMatcher::Context context;
while (matcher->find())
{
size_t current_lineno = matcher->lineno();
size_t lines = matcher->lines();
// adjust the number of after lines output so far upward if uninterrupted, or reset to zero
if (last_lineno != current_lineno)
{
if (next_lineno >= current_lineno)
after += lines;
else
after = 0;
}
if (lineno != current_lineno)
{
output_restline(true);
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (matchers != NULL && !cnf_matching(bol, eol))
continue;
// get the lines before the matched line
context = matcher->before();
if (context.len > 0)
{
invert_context_handler(*matcher, context.buf, context.len, context.num);
if (matches > 0 && (binfile || (binary && !flag_hex && !flag_with_hex)))
{
if (flag_binary_without_match)
{
matches = 0;
}
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
goto done_search;
}
}
lineno = current_lineno;
// --range: max line exceeded?
if (flag_max_line > 0 && lineno > flag_max_line)
break;
// --max-files: max reached?
if (stop)
goto exit_search;
/* logically OK but dead code because -v
if (matches == 0 && !flag_invert_match && (!flag_files || matchers == NULL))
{
// --max-files: max reached?
if (!Stats::found_part())
goto exit_search;
}
*/
// --max-count: max number of matches reached?
if (flag_max_count > 0 && matches >= flag_max_count)
break;
// output blocked?
if (out.eof)
goto exit_search;
if (after < flag_after_context)
{
binary = check_binary(bol, eol - bol);
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else // if (flag_invert_match) is true
{
last_lineno = current_lineno;
next_lineno = current_lineno + lines;
lineno = next_lineno - 1;
continue;
}
/* logically OK but dead code because -v
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
*/
goto done_search;
}
size_t border = matcher->border();
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
const char *end = matcher->end();
size_t size = matcher->size();
if (hex && !binary)
out.dump.done();
if (!flag_no_header)
out.header(pathname, partname, heading, lineno, matcher, first, flag_separator_dash, binary);
hex = binary;
if (binary)
{
if (flag_hex || flag_with_hex)
{
out.dump.hex(Output::Dump::HEX_CONTEXT_LINE, first - border, bol, border);
out.dump.hex(Output::Dump::HEX_CONTEXT_MATCH, first, begin, size);
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
}
else
{
if (border > 0)
{
out.str(color_cx);
out.str(bol, border);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_mc);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
}
else
{
if (lines > 1)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
size_t num = 1;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_mc);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, lineno + num, NULL, first + (to - begin) + 1, flag_separator_dash, false);
from = to + 1;
++num;
}
size -= from - begin;
begin = from;
}
out.str(match_mc);
out.str(begin, size);
out.str(match_off);
}
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
}
else if (flag_before_context > 0)
{
binary = check_binary(bol, eol - bol);
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else // if (flag_invert_match) is true
{
last_lineno = current_lineno;
next_lineno = current_lineno + lines;
lineno = next_lineno - 1;
continue;
}
/* logically OK but dead code because -v
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
*/
goto done_search;
}
if (hex && !binary)
out.dump.done();
hex = binary;
const char *begin = matcher->begin();
size_t size = matcher->size();
size_t offset = matcher->first();
if (lines == 1)
{
invert_context_handler.add_before_context_line(bol, eol, matcher->columno(), offset - (begin - bol));
invert_context_handler.add_before_context_match(begin - bol, size, offset);
}
else
{
// add lines to the before context
const char *from = begin;
const char *to;
while ((to = static_cast<const char*>(memchr(from, '\n', eol - from))) != NULL)
{
if (from == begin)
{
invert_context_handler.add_before_context_line(bol, to + 1, matcher->columno(), offset - (begin - bol));
invert_context_handler.add_before_context_match(begin - bol, to - from + 1, offset);
}
else
{
invert_context_handler.add_before_context_line(from, to + 1, 1, offset);
invert_context_handler.add_before_context_match(0, to + 1 < from + size ? to - from + 1 : size, offset);
}
size -= to - from + 1;
offset += to - from + 1;
from = to + 1;
}
}
}
}
else if (after < flag_after_context)
{
size_t size = matcher->size();
if (size > 0)
{
if (lines > 1 || colorize)
{
size_t first = matcher->first();
size_t last = matcher->last();
const char *begin = matcher->begin();
if (binary)
{
if (restline_data != NULL)
out.dump.hex(Output::Dump::HEX_CONTEXT_LINE, restline_last, restline_data, first - restline_last);
out.dump.hex(Output::Dump::HEX_CONTEXT_MATCH, first, begin, size);
}
else
{
if (restline_data != NULL)
{
out.str(color_cx);
out.str(restline_data, first - restline_last);
out.str(color_off);
}
if (flag_replace != NULL)
{
// output --replace=FORMAT
out.str(match_mc);
out.format(flag_replace, pathname, partname, matches, &replace_matching, matcher, heading, matches > 1, matches > 1);
out.str(match_off);
}
else
{
if (lines > 1)
{
// echo multi-line matches line-by-line
const char *from = begin;
const char *to;
size_t num = 1;
while ((to = static_cast<const char*>(memchr(from, '\n', size - (from - begin)))) != NULL)
{
out.str(match_mc);
out.str(from, to - from);
out.str(match_off);
out.chr('\n');
out.header(pathname, partname, heading, lineno + num, NULL, first + (to - begin) + 1, flag_separator_dash, false);
from = to + 1;
++num;
}
size -= from - begin;
begin = from;
}
out.str(match_mc);
out.str(begin, size);
out.str(match_off);
}
}
if (lines == 1)
{
restline_data += last - restline_last;
restline_size -= last - restline_last;
restline_last = last;
}
else
{
const char *eol = matcher->eol(true); // warning: call eol() before end()
const char *end = matcher->end();
binary = check_binary(end, eol - end);
if (hex && !binary)
out.dump.done();
else if (!hex && binary)
out.nl();
if (hex != binary && !flag_no_header)
out.header(pathname, partname, heading, lineno + lines - 1, matcher, last, flag_separator_dash, binary);
hex = binary;
restline_data = end;
restline_size = eol - end;
restline_last = last;
}
}
}
}
else
{
output_restline(true);
if (flag_before_context > 0)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol()
const char *bol = matcher->bol();
const char *begin = matcher->begin();
size_t size = matcher->size();
size_t offset = matcher->first();
if (lines == 1)
{
invert_context_handler.add_before_context_match(begin - bol, size, offset);
}
else
{
// add lines to the before context
const char *end = matcher->end();
binary = check_binary(end, eol - end);
if (binfile || (binary && !flag_hex && !flag_with_hex))
{
if (flag_binary_without_match)
{
matches = 0;
}
else // if (flag_invert_match) is true
{
last_lineno = current_lineno;
next_lineno = current_lineno + lines;
lineno = next_lineno - 1;
continue;
}
/* logically OK but dead code because -v
else
{
out.binary_file_matches(pathname, partname);
matches = 1;
}
*/
goto done_search;
}
if (hex && !binary)
out.dump.done();
hex = binary;
const char *from = begin;
const char *to;
while ((to = static_cast<const char*>(memchr(from, '\n', eol - from))) != NULL)
{
if (from == begin)
{
invert_context_handler.add_before_context_match(begin - bol, to - from + 1, offset);
}
else
{
invert_context_handler.add_before_context_line(from, to + 1, 1, offset);
invert_context_handler.add_before_context_match(0, to + 1 < from + size ? to - from + 1 : size, offset);
}
size -= to - from + 1;
offset += to - from + 1;
from = to + 1;
}
}
}
}
last_lineno = current_lineno;
next_lineno = current_lineno + lines;
lineno = next_lineno - 1;
}
output_restline(true);
// get the remaining context after the last match to output
context = matcher->after();
if (context.len > 0)
invert_context_handler(*matcher, context.buf, context.len, context.num);
if (matches > 0 && (binfile || (binary && !flag_hex && !flag_with_hex)))
{
if (flag_binary_without_match)
matches = 0;
else
out.binary_file_matches(pathname, partname);
}
if (binary)
out.dump.done();
}
done_search:
// --files: check if all CNF conditions are met globally to launch output or reset matches
if (flag_files && matchers != NULL)
if (!cnf_satisfied())
matches = 0;
// any matches in this file or archive?
if (matches > 0)
matched = true;
// --break: add a line break when applicable
if (flag_break && (matches > 0 || flag_any_line) && !flag_quiet && !flag_files_with_matches && !flag_count && flag_format == NULL)
out.nl();
if (flag_stats != NULL)
Stats::score_matches(matches, matcher->lineno() > 0 ? matcher->lineno() - 1 : 0);
}
catch (EXIT_SEARCH&)
{
// --files: cnf_matching() rejected a file, no need to search this file any further
}
catch (...)
{
// this should never happen
warning("exception while searching", pathname);
}
exit_search:
// flush and release output to allow other workers to output results
out.release();
// close file or -z: loop over next extracted archive parts, when applicable
} while (close_file(pathname));
// this file or archive has a match
if (matched)
Stats::found_file();
}
// search input after lineno to populate a string vector with the matching line and lines after up to max lines. used by the TUI
void Grep::find_text_preview(const char *filename, const char *findpart, size_t from_lineno, size_t max, size_t& lineno, size_t& num, std::vector<std::string>& text)
{
// no results yet
lineno = from_lineno;
num = 0;
try
{
// open (archive or compressed) file
if (!open_file(filename, findpart))
return;
}
catch (...)
{
// this should never happen
return;
}
matcher->input(input);
#if !defined(HAVE_PCRE2) && defined(HAVE_BOOST_REGEX)
// buffer all input to work around Boost.Regex partial matching bug, but this may throw std::bad_alloc if the file is too large
if (flag_mode == Mode::PERL)
matcher->buffer();
#endif
bool eof = true;
for (size_t i = from_lineno; i > 1; --i)
if (!matcher->skip('\n'))
break;
if (flag_invert_match)
{
lineno = matcher->lineno();
eof = false;
}
else
{
while (matcher->find())
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
// check CNF AND/OR/NOT matching
if (matchers != NULL && !flag_files && !cnf_matching(bol, eol))
continue;
lineno = matcher->lineno();
if (text.empty())
text.emplace_back(bol, eol - bol);
else
text[0].assign(bol, eol - bol);
num = 1;
eof = !matcher->skip('\n');
break;
}
}
while (!eof && num < max)
{
const char *eol = matcher->eol(true); // warning: call eol() before bol() and end()
const char *bol = matcher->bol();
if (text.size() <= num)
text.emplace_back(bol, eol - bol);
else
text[num].assign(bol, eol - bol);
++num;
eof = !matcher->skip('\n');
}
// close file or -z: loop over next extracted archive parts, when applicable
while (close_file(filename))
continue;
}
// extract a part from an archive and send to a stream, used by the TUI
void Grep::extract(const char *filename, const char *findpart, FILE *output)
{
try
{
// open (archive or compressed) file
if (!open_file(filename, findpart))
return;
}
catch (...)
{
// this should never happen
return;
}
// copy input to output
char buffer[65536];
while (true)
{
size_t len = input.get(buffer, sizeof(buffer));
if (len == 0 || fwrite(buffer, 1, len, output) < len)
break;
}
// close file or -z: loop over next extracted archive parts, when applicable
while (close_file(filename))
continue;
}
// read globs from a file and split them into files or dirs to include or exclude by pushing them onto the vectors, gitignore compatible (true), use local pathname as root dir / in globs (when non-NULL)
void import_globs(FILE *file, std::vector<std::string>& files, std::vector<std::string>& dirs, bool gitignore, const char *pathname)
{
// read globs from the specified file or files
reflex::BufferedInput input(file);
std::string pathname_glob;
std::string line;
while (true)
{
// read the next line
if (getline(input, line))
break;
// trim white space from either end
trim(line);
// add glob to files or dirs using gitignore glob pattern rules
if (!line.empty() && line.front() != '#')
{
if (line.size() > 1 || line.front() != '!') // negated matches start with ! (but not ^)
{
// if pathname is specified, then expand the local pathname to replace the root dir / in globs
if (pathname != NULL && *pathname != '\0')
{
size_t pos = line.front() == '!'; // negated matches start with ! (but not ^)
if (line.size() > pos && line.at(pos) == '/')
{
// create pathname glob on demand by escaping *, ?, [, and \ in the pathname to match literally
if (pathname_glob.empty())
{
pathname_glob.assign(pathname);
for (size_t i = 0; i < pathname_glob.size(); ++i)
{
int ch = pathname_glob.at(i);
if (ch == '*' || ch == '?' || ch == '[' || ch == '\\')
pathname_glob.insert(i++, "\\", 1);
}
}
line.insert(pos, pathname_glob);
}
}
if (line.back() == '/')
{
if (line.size() > 1)
line.pop_back();
dirs.emplace_back(line);
}
else
{
files.emplace_back(line);
if (gitignore)
dirs.emplace_back(line);
}
}
}
}
}
// trim white space from either end of the line
void trim(std::string& line)
{
size_t len = line.length();
size_t pos;
for (pos = 0; pos < len && isspace(static_cast<unsigned char>(line.at(pos))); ++pos)
continue;
if (pos > 0)
line.erase(0, pos);
len -= pos;
for (pos = len; pos > 0 && isspace(static_cast<unsigned char>(line.at(pos - 1))); --pos)
continue;
if (len > pos)
line.erase(pos, len - pos);
}
// trim path separators from an argv[] argument - important: modifies the argv[] string passed as arg
void trim_pathname_arg(const char *arg)
{
// remove trailing path separators after the drive prefix and path, if any - note: this truncates argv[] strings
const char *path = strchr(arg, ':');
if (path != NULL)
++path;
else
path = arg;
size_t len = strlen(path);
while (len > 1 && path[--len] == PATHSEPCHR)
const_cast<char*>(path)[len] = '\0';
}
// convert GREP_COLORS and set the color substring to the ANSI SGR codes
void set_color(const char *colors, const char *parameter, char color[COLORLEN])
{
const char *s = strstr(colors, parameter);
// check if substring parameter is present in colors
if (s != NULL)
{
s += strlen(parameter);
char *t = color + 2;
#ifdef WITH_EASY_GREP_COLORS
// foreground colors: k=black, r=red, g=green, y=yellow b=blue, m=magenta, c=cyan, w=white
// background colors: K=black, R=red, G=green, Y=yellow B=blue, M=magenta, C=cyan, W=white
// bright colors: +k, +r, +g, +y, +b, +m, +c, +w, +K, +R, +G, +Y, +B, +M, +C, +W
// modifiers: h=highlight, u=underline, i=invert, f=faint, n=normal, H=highlight off, U=underline off, I=invert off
// semicolons are not required and abbreviations can be mixed with numeric ANSI SGR codes
uint8_t offset = 30;
bool sep = false;
while (*s != '\0' && *s != ':' && t - color < COLORLEN - 6)
{
if (isdigit(static_cast<unsigned char>(*s)))
{
if (sep)
*t++ = ';';
if (offset == 90)
{
*t++ = '1';
*t++ = ';';
offset = 30;
}
*t++ = *s++;
while (isdigit(static_cast<unsigned char>(*s)) && t - color < COLORLEN - 2)
*t++ = *s++;
sep = true;
continue;
}
if (*s == '+')
{
offset = 90;
}
else if (*s == 'n')
{
if (sep)
*t++ = ';';
*t++ = '0';
sep = true;
}
else if (*s == 'h')
{
if (sep)
*t++ = ';';
*t++ = '1';
sep = true;
}
else if (*s == 'H')
{
if (sep)
*t++ = ';';
*t++ = '2';
*t++ = '1';
offset = 30;
sep = true;
}
else if (*s == 'f')
{
if (sep)
*t++ = ';';
*t++ = '2';
sep = true;
}
else if (*s == 'u')
{
if (sep)
*t++ = ';';
*t++ = '4';
sep = true;
}
else if (*s == 'U')
{
if (sep)
*t++ = ';';
*t++ = '2';
*t++ = '4';
sep = true;
}
else if (*s == 'i')
{
if (sep)
*t++ = ';';
*t++ = '7';
sep = true;
}
else if (*s == 'I')
{
if (sep)
*t++ = ';';
*t++ = '2';
*t++ = '7';
sep = true;
}
else if (*s == ',' || *s == ';' || isspace(static_cast<unsigned char>(*s)))
{
if (sep)
*t++ = ';';
sep = false;
}
else
{
const char *c = "krgybmcw KRGYBMCW";
const char *k = strchr(c, *s);
if (k != NULL)
{
if (sep)
*t++ = ';';
uint8_t n = offset + static_cast<uint8_t>(k - c);
if (n >= 100)
{
*t++ = '1';
n -= 100;
}
*t++ = '0' + n / 10;
*t++ = '0' + n % 10;
offset = 30;
sep = true;
}
}
++s;
}
#else
// traditional grep SGR parameters
while ((*s == ';' || isdigit(static_cast<unsigned char>(*s))) && t - color < COLORLEN - 2)
*t++ = *s++;
#endif
if (t > color + 2)
{
color[0] = '\033';
color[1] = '[';
*t++ = 'm';
*t++ = '\0';
}
else
{
color[0] = '\0';
}
}
}
// get short option argument
const char *getoptarg(int argc, const char **argv, const char *arg, int& i)
{
if (*++arg == '=')
++arg;
if (*arg != '\0')
return arg;
if (++i < argc)
return argv[i];
return "";
}
// get required non-empty long option argument after =
const char *getloptarg(int argc, const char **argv, const char *arg, int& i)
{
if (*arg != '\0')
return arg;
if (++i < argc)
return argv[i];
return "";
}
// save a string argument parsed from the command line or from a config file
const char *strarg(const char *string)
{
arg_strings.emplace_back(string);
return arg_strings.back().c_str();
}
// convert unsigned decimal to non-negative size_t, produce error when conversion fails
size_t strtonum(const char *string, const char *message)
{
char *rest = NULL;
size_t size = static_cast<size_t>(strtoull(string, &rest, 10));
if (rest == NULL || *rest != '\0')
usage(message, string);
return size;
}
// convert unsigned decimal to positive size_t, produce error when conversion fails or when the value is zero
size_t strtopos(const char *string, const char *message)
{
size_t size = strtonum(string, message);
if (size == 0)
usage(message, string);
return size;
}
// convert one or two comma-separated unsigned decimals specifying a range to positive size_t, produce error when conversion fails or when the range is invalid
void strtopos2(const char *string, size_t& min, size_t& max, const char *message)
{
if (*string == '\0')
{
usage(message, string);
return;
}
char *rest = const_cast<char*>(string);
if (*string != ',')
min = static_cast<size_t>(strtoull(string, &rest, 10));
else
min = 0;
if (*rest == ',')
max = static_cast<size_t>(strtoull(rest + 1, &rest, 10));
else
max = min, min = 0;
if (rest == NULL || *rest != '\0' || (max > 0 && min > max))
usage(message, string);
}
// convert unsigned decimal with optional kb, mb, gb, tb suffix to size_t, produce error when conversion fails
size_t strtosize(const char *string, const char *message)
{
char *rest = NULL;
size_t size = static_cast<size_t>(strtoull(string, &rest, 10));
if (rest == NULL || size == 0)
usage(message, string);
switch (*rest)
{
case 'k': case 'K': size *= 1024ULL; ++rest; break;
case 'm': case 'M': size *= 1024ULL*1024ULL; ++rest; break;
case 'g': case 'G': size *= 1024ULL*1024ULL*1024ULL; ++rest; break;
case 't': case 'T': if (sizeof(size_t) >= 8) { size *= 1024ULL*1024ULL*1024ULL*1024ULL; ++rest; } break;
}
if (*rest != 'b' && *rest != 'B' && *rest != '\0')
usage(message, string);
return size;
}
// convert unsigned decimal MAX fuzzy with optional prefix '+', '-', or '~' to positive size_t
size_t strtofuzzy(const char *string, const char *message)
{
char *rest = NULL;
size_t flags = 0;
size_t max = 1;
while (*string != '\0')
{
switch (*string)
{
case 'b':
if (strncmp(string, "best", 4) != 0)
usage(message, string);
flag_best_match = true;
string += 4;
break;
case '+':
flags |= reflex::FuzzyMatcher::INS;
++string;
break;
case '-':
flags |= reflex::FuzzyMatcher::DEL;
++string;
break;
case '~':
flags |= reflex::FuzzyMatcher::SUB;
++string;
break;
default:
max = static_cast<size_t>(strtoull(string, &rest, 10));
if (max == 0 || max > 0xff || rest == NULL || *rest != '\0')
usage(message, string);
string = rest;
}
}
return max | flags;
}
// print a diagnostic message
void usage(const char *message, const char *arg, const char *valid)
{
std::cerr << "ugrep: " << message << (arg != NULL ? arg : "");
if (valid != NULL)
{
std::cerr << ", did you mean " << valid << "?" << std::endl;
std::cerr << "For more help, run `ugrep --help' or `ugrep --help WHAT'" << std::endl;
}
else
{
const char *s = message;
while (*s != '\0' && *s != '-')
++s;
std::cerr << std::endl;
std::cerr << "For more help, run `ugrep --help' or `ugrep --help ";
if (*s == '\0')
{
std::cerr << "WHAT'" << std::endl;
}
else
{
const char *e = s;
while (*e == '-')
++e;
if (*e == '\0')
{
if (arg != NULL)
std::cerr << std::string(s, e - s) << arg << "'" << std::endl;
else
std::cerr << "WHAT'" << std::endl;
}
else
{
while (*e != '\0' && (*e == '-' || isalpha(static_cast<unsigned char>(*e))))
++e;
std::cerr << std::string(s, e - s) << "'" << std::endl;
}
}
}
// do not exit when reading a config file
if (!flag_usage_warnings)
exit(EXIT_ERROR);
++Static::warnings;
}
// print usage/help information and exit
void help(std::ostream& out)
{
out <<
"Usage: ugrep [OPTIONS] [PATTERN] [-f FILE] [-e PATTERN] [FILE ...]\n\n\
-A NUM, --after-context=NUM\n\
Output NUM lines of trailing context after matching lines. Places\n\
a --group-separator between contiguous groups of matches. If -o is\n\
specified, output the match with context to fit NUM columns after\n\
the match or shortens the match. See also options -B, -C and -y.\n\
-a, --text\n\
Process a binary file as if it were text. This is equivalent to\n\
the --binary-files=text option. This option might output binary\n\
garbage to the terminal, which can have problematic consequences if\n\
the terminal driver interprets some of it as terminal commands.\n\
--all, -@\n\
Search all files except hidden: cancel previous file and directory\n\
search restrictions and cancel --ignore-binary and --ignore-files\n\
when specified. Restrictions specified after this option, i.e. to\n\
the right, are still applied. For example, -@I searches all\n\
non-binary files and -@. searches all files including hidden files.\n\
Note that hidden files and directories are never searched, unless\n\
option -. or --hidden is specified.\n\
--and [-e] PATTERN\n\
Specify additional PATTERN that must match. Additional -e PATTERN\n\
following this option is considered an alternative pattern to\n\
match, i.e. each -e is interpreted as an OR pattern enclosed within\n\
the AND. For example, -e A -e B --and -e C -e D matches lines with\n\
(`A' or `B') and (`C' or `D'). Note that multiple -e PATTERN are\n\
alternations that bind more tightly together than --and. Option\n\
--stats displays the search patterns applied. See also options\n\
--not, --andnot, --bool, --files and --lines.\n\
--andnot [-e] PATTERN\n\
Combines --and --not. See also options --and, --not and --bool.\n\
-B NUM, --before-context=NUM\n\
Output NUM lines of leading context before matching lines. Places\n\
a --group-separator between contiguous groups of matches. If -o is\n\
specified, output the match with context to fit NUM columns before\n\
the match or shortens the match. See also options -A, -C and -y.\n\
-b, --byte-offset\n\
The offset in bytes of a pattern match is displayed in front of the\n\
respective matched line. When -u or --ungroup is specified,\n\
displays the offset for each pattern matched on the same line.\n\
Byte offsets are exact for ASCII, UTF-8 and raw binary input.\n\
Otherwise, the offset in the UTF-8-normalized input is displayed.\n\
--binary-files=TYPE\n\
Controls searching and reporting pattern matches in binary files.\n\
TYPE can be `binary', `without-match`, `text`, `hex` and\n\
`with-hex'. The default is `binary' to search binary files and to\n\
report a match without displaying the match. `without-match'\n\
ignores binary matches. `text' treats all binary files as text,\n\
which might output binary garbage to the terminal, which can have\n\
problematic consequences if the terminal driver interprets some of\n\
it as commands. `hex' reports all matches in hexadecimal.\n\
`with-hex' only reports binary matches in hexadecimal, leaving text\n\
matches alone. Files having NUL (zero) bytes are binary and files\n\
with invalid UTF encoding are binary unless option -U or --ascii\n\
or --binary is specified. Short options are -a, -I, -W, -X.\n\
--bool, -%, -%%\n\
Specifies Boolean query patterns. A Boolean query pattern is\n\
composed of `AND', `OR', `NOT' operators and grouping with `(' `)'.\n\
Spacing between subpatterns is the same as `AND', `|' is the same\n\
as `OR' and a `-' is the same as `NOT'. The `OR' operator binds\n\
more tightly than `AND'. For example, --bool 'A|B C|D' matches\n\
lines with (`A' or `B') and (`C' or `D'), --bool 'A -B' matches\n\
lines with `A' and not `B'. Operators `AND', `OR', `NOT' require\n\
proper spacing. For example, --bool 'A OR B AND C OR D' matches\n\
lines with (`A' or `B') and (`C' or `D'), --bool 'A AND NOT B'\n\
matches lines with `A' without `B'. Quoted subpatterns are matched\n\
literally as strings. For example, --bool 'A \"AND\"|\"OR\"' matches\n\
lines with `A' and also either `AND' or `OR'. Parentheses are used\n\
for grouping. For example, --bool '(A B)|C' matches lines with `A'\n\
and `B', or lines with `C'. Note that all subpatterns in a Boolean\n\
query pattern are regular expressions, unless -F is specified.\n\
Options -E, -F, -G, -P and -Z can be combined with --bool to match\n\
subpatterns as strings or regular expressions (-E is the default.)\n\
This option does not apply to -f FILE patterns. The double short\n\
option -%% enables options --bool --files. Option --stats displays\n\
the Boolean search patterns applied. See also options --and,\n\
--andnot, --not, --files and --lines.\n\
--break\n\
Adds a line break between results from different files. This\n\
option is enabled by --heading.\n\
-C NUM, --context=NUM\n\
Output NUM lines of leading and trailing context surrounding each\n\
matching line. Places a --group-separator between contiguous\n\
groups of matches. If -o is specified, output the match with\n\
context to fit NUM columns before and after the match or shortens\n\
the match. See also options -A, -B and -y.\n\
-c, --count\n\
Only a count of selected lines is written to standard output.\n\
When -o or -u is specified, counts the number of patterns matched.\n\
When -v is specified, counts the number of non-matching lines.\n\
When -m1, (with a comma or --min-count=1) is specified, counts only\n\
matching files without outputting zero matches.\n\
--color[=WHEN], --colour[=WHEN]\n\
Mark up the matching text with the colors specified with option\n\
--colors or the GREP_COLOR or GREP_COLORS environment variable.\n\
WHEN can be `never', `always', or `auto', where `auto' marks up\n\
matches only when output on a terminal. The default is `auto'.\n\
--colors=COLORS, --colours=COLORS\n\
Use COLORS to mark up text. COLORS is a colon-separated list of\n\
one or more parameters `sl=' (selected line), `cx=' (context line),\n\
`mt=' (matched text), `ms=' (match selected), `mc=' (match\n\
context), `fn=' (file name), `ln=' (line number), `cn=' (column\n\
number), `bn=' (byte offset), `se=' (separator), `qp=' (TUI\n\
prompt), `qe=' (TUI errors), `qr=' (TUI regex), `qm=' (TUI regex\n\
meta characters), `ql=' (TUI regex lists and literals), `qb=' (TUI\n\
regex braces). Parameter values are ANSI SGR color codes or `k'\n\
(black), `r' (red), `g' (green), `y' (yellow), `b' (blue), `m'\n\
(magenta), `c' (cyan), `w' (white), or leave empty for no color.\n\
Upper case specifies background colors. A `+' qualifies a color as\n\
bright. A foreground and a background color may be combined with\n\
font properties `n' (normal), `f' (faint), `h' (highlight), `i'\n\
(invert), `u' (underline). Parameter `hl' enables file name\n\
hyperlinks. Parameter `rv' reverses the `sl=' and `cx=' parameters\n\
when option -v is specified. Selectively overrides GREP_COLORS.\n\
Legacy grep single parameter codes may be specified, for example\n\
--colors='7;32' or --colors=ig to set ms (match selected).\n\
--config[=FILE], ---[FILE]\n\
Use configuration FILE. The default FILE is `.ugrep'. The working\n\
directory is checked first for FILE, then the home directory. The\n\
options specified in the configuration FILE are parsed first,\n\
followed by the remaining options specified on the command line.\n\
The ug command automatically loads a `.ugrep' configuration file\n\
when present, unless --config=FILE or --no-config is specified.\n\
--no-config\n\
Do not automatically load the default .ugrep configuration file.\n\
--no-confirm\n\
Do not confirm actions in -Q query TUI. The default is confirm.\n\
--cpp\n\
Output file matches in C++. See also options --format and -u.\n\
--csv\n\
Output file matches in CSV. When -H, -n, -k, or -b is specified,\n\
additional values are output. See also options --format and -u.\n\
-D ACTION, --devices=ACTION\n\
If an input file is a device, FIFO or socket, use ACTION to process\n\
it. When ACTION is `skip', devices are ignored. When ACTION is\n\
`read', devices are read just as if they were ordinary files. The\n\
default ACTION is `skip', except for devices specified as arguments\n\
on the command line, which can be explicitly skipped with -Dskip.\n\
-d ACTION, --directories=ACTION\n\
If an input file is a directory, use ACTION to process it. When\n\
ACTION is `read', warn when directories are read as input. When\n\
ACTION is `recurse', read all files under each directory,\n\
recursively, following symbolic links only if they are on the\n\
command line. This is equivalent to the -r option. When ACTION is\n\
`dereference-recurse', read all files under each directory,\n\
recursively, following symbolic links. This is equivalent to the\n\
-R option. When no files or directories are specified on the\n\
command line, ACTION defaults to `recurse` or option -r.\n\
--delay=DELAY\n\
Set the default -Q key response delay. Default is 3 for 300ms.\n\
--depth=[MIN,][MAX], -1, -2, -3, ... -9, -10, -11, ...\n\
Restrict recursive searches from MIN to MAX directory levels deep,\n\
where -1 (--depth=1) searches the specified path without recursing\n\
into subdirectories. The short forms -3 -5, -3-5 and -3,5 search 3\n\
to 5 levels deep. Enables -r if -R or -r is not specified.\n\
--dotall\n\
Dot `.' in regular expressions matches anything, including newline.\n\
Note that `.*' matches all input and should not be used.\n\
-E, --extended-regexp\n\
Interpret patterns as extended regular expressions (EREs). This is\n\
the default.\n\
-e PATTERN, --regexp=PATTERN\n\
Specify a PATTERN to search the input. An input line is selected\n\
if it matches any of the specified patterns. This option is useful\n\
when multiple -e options are used to specify multiple patterns, or\n\
when a pattern starts with a dash (`-'), or to specify a pattern\n\
after option -f or after the FILE arguments.\n\
--encoding=ENCODING\n\
The encoding format of the input. The default ENCODING is binary\n\
or UTF-8 which are treated the same. Therefore, --encoding=binary\n\
has no effect. Contrast this with option -U or --ascii or --binary\n\
that specifies raw binary PATTERN matching. ENCODING can be:\n\
";
size_t k = 10;
for (int i = 0; encoding_table[i].format != NULL; ++i)
{
size_t n = strlen(encoding_table[i].format);
k += n + 4;
out << (i == 0 ? "" : ",");
if (k > 79)
{
out << "\n ";
k = 14 + n;
}
out << " `" << encoding_table[i].format << "'";
}
out << ".\n\
--exclude=GLOB\n\
Exclude files whose name matches GLOB, same as -g ^GLOB. GLOB can\n\
use **, *, ?, and [...] as wildcards and \\ to quote a wildcard or\n\
backslash character literally. When GLOB contains a `/', full\n\
pathnames are matched. Otherwise basenames are matched. When GLOB\n\
ends with a `/', directories are excluded as if --exclude-dir is\n\
specified. Otherwise files are excluded. Note that --exclude\n\
patterns take priority over --include patterns. GLOB should be\n\
quoted to prevent shell globbing. This option may be repeated.\n\
--exclude-dir=GLOB\n\
Exclude directories whose name matches GLOB from recursive\n\
searches, same as -g ^GLOB/. GLOB may use **, *, ?, and [...] as\n\
wildcards and \\ to quote a wildcard or backslash character\n\
literally. When GLOB contains a `/', full pathnames are matched.\n\
Otherwise basenames are matched. Note that --exclude-dir patterns\n\
take priority over --include-dir patterns. GLOB should be quoted\n\
to prevent shell globbing. This option may be repeated.\n\
--exclude-from=FILE\n\
Read the globs from FILE and skip files and directories whose name\n\
matches one or more globs. A glob may use **, *, ?, and [...] as\n\
wildcards and \\ to quote a wildcard or backslash character\n\
literally. When a glob contains a `/', full pathnames are matched.\n\
Otherwise basenames are matched. When a glob ends with a `/',\n\
directories are excluded as if --exclude-dir is specified.\n\
Otherwise files are excluded. A glob starting with a `!' overrides\n\
previously-specified exclusions by including matching files. Lines\n\
starting with a `#' in FILE are ignored. When FILE is a `-',\n\
standard input is read. This option may be repeated. See also\n\
related option --ignore-files for subdirectory-specific exclusions.\n\
--exclude-fs=MOUNTS\n\
Exclude file systems specified by MOUNTS from recursive searches.\n\
MOUNTS is a comma-separated list of mount points or pathnames to\n\
directories. When MOUNTS is not specified, only descends into the\n\
file systems associated with the specified file and directory\n\
search targets, i.e. excludes all other file systems. Note that\n\
--exclude-fs=MOUNTS take priority over --include-fs=MOUNTS. This\n\
option may be repeated.\n"
#if !defined(HAVE_STATVFS) && !defined(HAVE_STATFS)
"\
This option is not available in this build configuration of ugrep.\n"
#endif
"\
-F, --fixed-strings\n\
Interpret pattern as a set of fixed strings, separated by newlines,\n\
any of which is to be matched. This makes ugrep behave as fgrep.\n\
If a PATTERN is specified, or -e PATTERN or -N PATTERN, then this\n\
option has no effect on -f FILE patterns to allow -f FILE patterns\n\
to narrow or widen the scope of the PATTERN search.\n\
-f FILE, --file=FILE\n\
Read newline-separated patterns from FILE. White space in patterns\n\
is significant. Empty lines in FILE are ignored. If FILE does not\n\
exist, the GREP_PATH environment variable is used as path to FILE.\n"
#ifdef GREP_PATH
"\
If that fails, looks for FILE in " GREP_PATH ".\n"
#endif
"\
When FILE is a `-', standard input is read. Empty files contain no\n\
patterns; thus nothing is matched. This option may be repeated.\n\
--files, -%%\n\
Boolean file matching mode, the opposite of --lines. When combined\n\
with option --bool, matches a file if all Boolean conditions are\n\
satisfied. For example, --bool --files 'A B|C -D' matches a file\n\
if some lines match `A', and some lines match either `B' or `C',\n\
and no line matches `D'. See also options --and, --andnot, --not,\n\
--bool and --lines. The double short option -%% enables options\n\
--bool --files.\n\
--filter=COMMANDS\n\
Filter files through the specified COMMANDS first before searching.\n\
COMMANDS is a comma-separated list of `exts:command arguments',\n\
where `exts' is a comma-separated list of filename extensions and\n\
`command' is a filter utility. Files matching one of `exts' are\n\
filtered. A `*' matches any file. The specified `command' may\n\
include arguments separated by spaces. An argument may be quoted\n\
to include spacing, commas or a `%'. A `%' argument expands into\n\
the pathname to search. For example, --filter='pdf:pdftotext % -'\n\
searches PDF files. The `%' expands into a `-' when searching\n\
standard input. When a `%' is not specified, the filter command\n\
should read from standard input and write to standard output.\n\
Option --label=.ext may be used to specify extension `ext' when\n\
searching standard input. Note that `exts` filename extension\n\
matching is case insensitive. This option may be repeated.\n\
--filter-magic-label=[+]LABEL:MAGIC\n\
Associate LABEL with files whose signature \"magic bytes\" match the\n\
MAGIC regex pattern. Only files that have no filename extension\n\
are labeled, unless +LABEL is specified. When LABEL matches an\n\
extension specified in --filter=COMMANDS, the corresponding command\n\
is invoked. This option may be repeated.\n\
--format=FORMAT\n\
Output FORMAT-formatted matches. For example --format='%f:%n:%O%~'\n\
outputs matching lines `%O' with filename `%f` and line number `%n'\n\
followed by a newline `%~'. If -P is specified, FORMAT may include\n\
`%1' to `%9', `%[NUM]#' and `%[NAME]#' to output group captures. A\n\
`%%' outputs `%'. See `ugrep --help format' and `man ugrep'\n\
section FORMAT for details. When option -o is specified, option -u\n\
is also enabled. Context options -A, -B, -C and -y are ignored.\n\
--free-space\n\
Spacing (blanks and tabs) in regular expressions are ignored.\n\
--from=FILE\n\
Read additional pathnames of files to search from FILE. When FILE\n\
is a `-', standard input is read. This option is useful with `find\n\
... -print | ugrep --from=- ...' to search specific files.\n\
-G, --basic-regexp\n\
Interpret patterns as basic regular expressions (BREs).\n\
-g GLOBS, --glob=GLOBS, --iglob=GLOBS\n\
Only search files whose name matches the specified comma-separated\n\
list of GLOBS, same as --include='glob' for each `glob' in GLOBS.\n\
When a `glob' is preceded by a `!' or a `^', skip files whose name\n\
matches `glob', same as --exclude='glob'. When `glob' contains a\n\
`/', full pathnames are matched and option -r is enabled to recurse\n\
into directories if -d, -r or -R is not already specified.\n\
Otherwise basenames are matched. When `glob' ends with a `/',\n\
directories are matched, same as --include-dir='glob' and\n\
--exclude-dir='glob'. A leading `./' or just `/' matches the\n\
working directory. Option --iglob=GLOBS performs case-insensitive\n\
name matching. This option may be repeated and may be combined\n\
with options -M, -O and -t. For more details, see `ugrep --help\n\
globs' and `man ugrep' section GLOBBING for details.\n\
--glob-ignore-case\n\
Perform case-insensitive glob matching in general.\n\
--group-separator[=SEP]\n\
Use SEP as a group separator for context options -A, -B and -C.\n\
The default is a double hyphen (`--').\n\
--no-group-separator\n\
Removes the group separator line from the output for context\n\
options -A, -B and -C.\n\
-H, --with-filename\n\
Always print the filename with output lines. This is the default\n\
when there is more than one file to search.\n\
-h, --no-filename\n\
Never print filenames with output lines. This is the default\n\
when there is only one file (or only standard input) to search.\n\
--heading, -+\n\
Group matches per file. Adds a heading and a line break between\n\
results from different files. This option is enabled by --pretty\n\
when the output is sent to the terminal.\n\
--help [WHAT], -? [WHAT]\n\
Display a help message on options related to WHAT when specified.\n\
In addition, `--help regex' displays an overview of regular\n\
expressions, `--help globs' displays an overview of glob syntax and\n\
conventions, `--help fuzzy' displays details of fuzzy search, and\n\
`--help format' displays a list of option --format=FORMAT fields.\n\
--hexdump[=[1-8][a][bch][A[NUM]][B[NUM]][C[NUM]]]\n\
Output matches in 1 to 8 columns of 8 hexadecimal octets. The\n\
default is 2 columns or 16 octets per line. Argument `a' outputs a\n\
`*' for all hex lines that are identical to the previous hex line,\n\
`b' removes all space breaks, `c' removes the character column, `h'\n\
removes hex spacing, `A' includes up to NUM hex lines after a\n\
match, `B' includes up to NUM hex lines before a match and `C'\n\
includes up to NUM hex lines before and after a match. Arguments\n\
`A', `B' and `C' are the same as options -A, -B and -C when used\n\
with --hexdump. See also options -U, -W and -X.\n\
--hidden, -.\n\
Search "
#ifdef OS_WIN_OR_MINGW
"Windows system and "
#endif
"hidden files and directories\n\
(enabled by default in grep compatibility mode).\n\
--hyperlink[=[PREFIX][+]]\n\
Hyperlinks are enabled for file names when colors are enabled.\n\
Same as --colors=hl. When PREFIX is specified, replaces file://\n\
with PREFIX:// in the hyperlink. A `+' includes the line number in\n\
the hyperlink and when option -k is specified, the column number.\n\
-I, --ignore-binary\n\
Ignore matches in binary files. This option is equivalent to the\n\
--binary-files=without-match option. Files having NUL (zero) bytes\n\
are binary. Also files with invalid UTF encoding are binary unless\n\
option -U or --ascii or --binary is specified.\n\
-i, --ignore-case\n\
Perform case insensitive matching. By default, ugrep is case\n\
sensitive.\n\
--ignore-files[=FILE]\n\
Ignore files and directories matching the globs in each FILE that\n\
is encountered in recursive searches. The default FILE is\n\
`" DEFAULT_IGNORE_FILE "'. Matching files and directories located in the\n\
directory of the FILE and in subdirectories are ignored. When FILE is\n\
a full pathname, FILE is read ahead of the recursive search and\n\
globs are globally applied to ignore files and directories. When\n\
FILE is a `-', standard input is read. Globbing is the same as for\n\
--exclude-from=FILE, except that --ignore-files excludes files and\n\
also directories when globs match. This option may be repeated.\n\
--no-ignore-files\n\
Do not ignore files, i.e. cancel --ignore-files when specified.\n\
--include=GLOB\n\
Only search files whose name matches GLOB, same as -g GLOB. GLOB\n\
may use **, *, ?, and [...] as wildcards and \\ to quote a wildcard\n\
or backslash character literally. When GLOB contains a `/', full\n\
pathnames are matched and option -r is enabled to recurse into\n\
directories if -d, -r or -R is not already specified. Otherwise\n\
basenames are matched. When GLOB ends with a `/', directories are\n\
included as if --include-dir is specified. Otherwise files are\n\
included. Note that --exclude patterns take priority over\n\
--include patterns. GLOB should be quoted to prevent shell\n\
globbing. This option may be repeated.\n\
--include-dir=GLOB\n\
Only directories whose name matches GLOB are included in recursive\n\
searches, same as -g GLOB/. GLOB may use **, *, ?, and [...] as\n\
wildcards and \\ to quote a wildcard or backslash character\n\
literally. When GLOB contains a `/', full pathnames are matched\n\
and option -r is enabled to recurse into directories if -d, -r or\n\
-R is not already specified. Otherwise basenames are matched.\n\
Note that --exclude-dir patterns take priority over --include-dir\n\
patterns. GLOB should be quoted to prevent shell globbing. This\n\
option may be repeated.\n\
--include-from=FILE\n\
Read the globs from FILE and search only files and directories\n\
whose name matches one or more globs. A glob may use **, *, ?, and\n\
[...] as wildcards and \\ to quote a wildcard or backslash\n\
character literally. When a glob contains a `/', full pathnames\n\
are matched. Otherwise basenames are matched. When a glob ends\n\
with a `/', directories are included as if --include-dir is\n\
specified. Otherwise files are included. A glob starting with a\n\
`!' overrides previously-specified inclusions by excluding matching\n\
files. Lines starting with a `#' and empty lines in FILE are\n\
ignored. When FILE is a `-', standard input is read. This option\n\
may be repeated.\n\
--include-fs=MOUNTS\n\
Only file systems specified by MOUNTS are included in recursive\n\
searches. MOUNTS is a comma-separated list of mount points or\n\
pathnames to directories. When MOUNTS is not specified, restricts\n\
recursive searches to the file system of the working directory,\n\
same as --include-fs=. (dot). Note that --exclude-fs=MOUNTS take\n\
priority over --include-fs=MOUNTS. This option may be repeated.\n"
#if !defined(HAVE_STATVFS) && !defined(HAVE_STATFS)
"\
This option is not available in this build configuration of ugrep.\n"
#endif
"\
--index\n\
Perform fast index-based recursive search. This option assumes,\n\
but does not require, that files are indexed with ugrep-indexer.\n\
This option also enables option -r or --recursive. Skips indexed\n\
non-matching files, archives and compressed files. Significant\n\
acceleration may be achieved on cold (not file-cached) and large\n\
file systems, or any file system that is slow to search. Note that\n\
the start-up time to search may be increased when complex search\n\
patterns are specified that contain large Unicode character classes\n\
combined with `*' or `+' repeats, which should be avoided. Option\n\
-U or --ascii or --binary improves performance. Option --stats\n\
displays an index search report.\n\
-J NUM, --jobs=NUM\n\
Specifies the number of threads spawned to search files. By\n\
default an optimum number of threads is spawned to search files\n\
simultaneously. -J1 disables threading: files are searched in the\n\
same order as specified.\n\
-j, --smart-case\n\
Perform case insensitive matching, unless a pattern is specified\n\
with a literal upper case letter.\n\
--json\n\
Output file matches in JSON. When -H, -n, -k, or -b is specified,\n\
additional values are output. See also options --format and -u.\n\
-K [MIN,][MAX], --range=[MIN,][MAX], --min-line=MIN, --max-line=MAX\n\
Start searching at line MIN, stop reading input after line MAX.\n\
-k, --column-number\n\
The column number of a pattern match is displayed in front of the\n\
respective matched line, starting at column 1. Tabs are expanded\n\
in counting columns, see also option --tabs.\n\
-L, --files-without-match\n\
Only the names of files not containing selected lines are written\n\
to standard output. Pathnames are listed once per file searched.\n\
If the standard input is searched, the string ``(standard input)''\n\
is written.\n\
-l, --files-with-matches\n\
Only the names of files containing selected lines are written to\n\
standard output. Pathnames are listed once per file searched. If\n\
the standard input is searched, the string ``(standard input)'' is\n\
written. Stop reading input upon the first match.\n\
--label=LABEL\n\
Displays the LABEL value when input is read from standard input\n\
where a file name would normally be printed in the output.\n\
Associates a filename extension with standard input when LABEL has\n\
a suffix. The default value is `(standard input)'.\n\
--line-buffered\n\
Force output to be line buffered instead of block buffered.\n\
--lines\n\
Boolean line matching mode for option --bool, the default mode.\n\
-M MAGIC, --file-magic=MAGIC\n\
Only search files matching the magic signature pattern MAGIC. The\n\
signature \"magic bytes\" at the start of a file are compared to\n\
the MAGIC regex pattern. When matching, the file will be searched.\n\
When MAGIC is preceded by a `!' or a `^', skip files with matching\n\
MAGIC signatures. This option may be repeated and may be combined\n\
with options -O and -t. Every file on the search path is read,\n\
making recursive searches potentially more expensive.\n\
-m [MIN,][MAX], --min-count=MIN, --max-count=MAX\n\
Require MIN matches, stop reading input upon MAX matches. Output\n\
MIN to MAX matches. For example, -m1 outputs the first match and\n\
-cm1, (with a comma) counts nonzero matches. When -u or --ungroup\n\
is specified, each individual match counts. See also option -K.\n\
--match\n\
Match all lines. Same as specifying an empty pattern to search.\n\
--max-files=NUM\n\
Restrict the number of files matched to NUM. Note that --sort or\n\
-J1 may be specified to produce replicable results. If --sort is\n\
specified, then the number of threads spawned is limited to NUM.\n\
--max-size=MAX\n\
Only search files whose physical size does not exceed MAX bytes.\n\
Suffix MAX with kb, mb, gb, tb for kilo, mega, giga, or terabytes.\n\
--min-size=MIN\n\
Only search files whose physical size equals or exceeds MIN bytes.\n\
Suffix MIN with kb, mb, gb, tb for kilo, mega, giga, or terabytes.\n\
--mmap[=MAX]\n\
Use memory maps to search files. By default, memory maps are used\n\
under certain conditions to improve performance. When MAX is\n\
specified, use up to MAX mmap memory per thread.\n\
-N PATTERN, --neg-regexp=PATTERN\n\
Specify a negative PATTERN to reject specific -e PATTERN matches\n\
with a counter pattern. Note that longer patterns take precedence\n\
over shorter patterns, i.e. a negative pattern must be of the same\n\
length or longer to reject matching patterns. Option -N cannot be\n\
specified with -P. This option may be repeated.\n\
-n, --line-number\n\
Each output line is preceded by its relative line number in the\n\
file, starting at line 1. The line number counter is reset for\n\
each file processed.\n\
--not [-e] PATTERN\n\
Specifies that PATTERN should not match. Note that -e A --not -e B\n\
matches lines with `A' or lines without a `B'. To match lines with\n\
`A' that have no `B', specify -e A --andnot -e B. Option --stats\n\
displays the search patterns applied. See also options --and,\n\
--andnot, --bool, --files and --lines.\n\
--null, -0";
out << (flag_grep ? ", -Z" : "") << "\n\
Output a zero byte after the file name. This option can be used\n\
with commands such as `find -print0' and `xargs -0' to process\n\
arbitrary file names, even those that contain newlines. See also\n\
options -H or --with-filename and --null-data.\n\
--null-data, -00";
out << (flag_grep ? ", -z" : "") << "\n\
Input and output are treated as sequences of lines with each line\n\
terminated by a zero byte instead of a newline; effectively swaps\n\
NUL with LF in the input and the output. When combined with option\n\
--encoding=ENCODING, output each line terminated by a zero byte\n\
without affecting the input specified as per ENCODING. Instead of\n\
option --null-data, option --encoding=null-data treats the input as\n\
a sequence of lines terminated by a zero byte without affecting the\n\
output. Option --null-data is not compatible with UTF-16/32 input.\n\
Since ugrep matches multiple lines when patterns are specified\n\
containing a newline `\\n' or `\\R', this option is not required for\n\
multi-line matching. See also options --encoding and --null.\n\
-O EXTENSIONS, --file-extension=EXTENSIONS\n\
Only search files whose filename extensions match the specified\n\
comma-separated list of EXTENSIONS, same as -g '*.ext' for each\n\
`ext' in EXTENSIONS. When an `ext' is preceded by a `!' or a `^',\n\
skip files whose filename extensions matches `ext', same as\n\
-g '^*.ext'. This option may be repeated and may be combined with\n\
options -g, -M and -t.\n\
-o, --only-matching\n\
Only the matching part of a pattern match is output. When -A, -B\n\
or -C is specified, fits the match and its context on a line within\n\
the specified number of columns.\n\
--only-line-number\n\
Only the line number of a matching line is output. The line number\n\
counter is reset for each file processed.\n\
-P, --perl-regexp\n\
Interpret PATTERN as a Perl regular expression"
#if defined(HAVE_PCRE2)
" using PCRE2.\n"
#elif defined(HAVE_BOOST_REGEX)
" using Boost.Regex.\n"
#else
".\n\
This option is not available in this build configuration of ugrep.\n"
#endif
"\
Note that Perl pattern matching differs from the default grep POSIX\n\
pattern matching.\n\
-p, --no-dereference\n\
If -R or -r is specified, do not follow symbolic links, even when\n\
symbolic links are specified on the command line.\n\
--pager[=COMMAND]\n\
When output is sent to the terminal, uses COMMAND to page through\n\
the output. COMMAND defaults to environment variable PAGER when\n\
defined or `" DEFAULT_PAGER_COMMAND "'. Enables --heading and --line-buffered.\n\
--pretty[=WHEN]\n\
When output is sent to the terminal, enables options --color,\n\
--heading, -n, --sort, --tree and -T when not explicitly disabled.\n\
WHEN can be `never', `always', or `auto'. The default is `auto'.\n\
-Q[=DELAY], --query[=DELAY]\n\
Query mode: start a TUI to perform interactive searches. This mode\n\
requires an ANSI capable terminal. An optional DELAY argument may\n\
be specified to reduce or increase the response time to execute\n\
searches after the last key press, in increments of 100ms, where\n\
the default is 3 (300ms delay). No whitespace may be given between\n\
-Q and its argument DELAY. Initial patterns may be specified with\n\
-e PATTERN, i.e. a PATTERN argument requires option -e. Press F1\n\
or CTRL-Z to view the help screen. Press F2 or CTRL-Y to invoke a\n\
command to view or edit the file shown at the top of the screen.\n\
This command can be specified with option --view and defaults to\n\
environment variable PAGER when defined, or VISUAL or EDITOR.\n\
Press TAB or SHIFT-TAB to navigate directories and to select a file\n\
to search. Press ENTER to select lines to output. Press ALT-l for\n\
option -l to list files, ALT-n for -n, etc. Non-option commands\n\
include ALT-] to increase context and ALT-} to increase fuzzyness.\n\
If ALT or OPTION keys are not available, then press CTRL-O + KEY to\n\
switch option `KEY', or press F1 or CTRL-Z for help and press KEY.\n\
See also options --no-confirm, --delay, --split and --view.\n\
-q, --quiet, --silent\n\
Quiet mode: suppress all output. Only search a file until a match\n\
has been found.\n\
-R, --dereference-recursive\n\
Recursively read all files under each directory, following symbolic\n\
links to files and directories, unlike -r.\n\
-r, --recursive\n\
Recursively read all files under each directory, following symbolic\n\
links only if they are on the command line. Note that when no FILE\n\
arguments are specified and input is read from a terminal,\n\
recursive searches are performed as if -r is specified.\n\
--replace=FORMAT\n\
Replace matching patterns in the output by FORMAT with `%' fields.\n\
If -P is specified, FORMAT may include `%1' to `%9', `%[NUM]#' and\n\
`%[NAME]#' to output group captures. A `%%' outputs `%' and `%~'\n\
outputs a newline. See also option --format, `ugrep --help format'\n\
and `man ugrep' section FORMAT for details.\n\
-S, --dereference-files\n\
When -r is specified, follow symbolic links to files, but not to\n\
directories. The default is not to follow symbolic links.\n\
-s, --no-messages\n\
Silent mode: nonexistent and unreadable files are ignored and\n\
their error messages and warnings are suppressed.\n\
--save-config[=FILE] [OPTIONS]\n\
Save configuration FILE to include OPTIONS. Update FILE when\n\
first loaded with --config=FILE. The default FILE is `.ugrep',\n\
which is automatically loaded by the ug command. When FILE is a\n\
`-', writes the configuration to standard output. Only part of the\n\
OPTIONS are saved that do not cause searches to fail when combined\n\
with other options. Additional options may be specified by editing\n\
the saved configuration file. A configuration file may be modified\n\
by adding one or more config=FILE to include configuration files,\n\
but recursive configuration file inclusion is not permitted.\n\
--separator[=SEP], --context-separator=SEP\n\
Use SEP as field separator between file name, line number, column\n\
number, byte offset and the matched line. The default separator is\n\
a colon (`:') and a bar (`|') for multi-line pattern matches, and a\n\
dash (`-') for context lines. See also option --group-separator.\n\
--split\n\
Split the -Q query TUI screen on startup.\n\
--sort[=KEY]\n\
Displays matching files in the order specified by KEY in recursive\n\
searches. Normally the ug command sorts by name whereas the ugrep\n\
batch command displays matches in no particular order to improve\n\
performance. The sort KEY can be `name' to sort by pathname\n\
(default), `best' to sort by best match with option -Z (sort by\n\
best match requires two passes over files, which is expensive),\n\
`size' to sort by file size, `used' to sort by last access time,\n\
`changed' to sort by last modification time and `created' to sort\n\
by creation time. Sorting is reversed with `rname', `rbest',\n\
`rsize', `rused', `rchanged', or `rcreated'. Archive contents are\n\
not sorted. Subdirectories are sorted and displayed after matching\n\
files. FILE arguments are searched in the same order as specified.\n\
--stats\n\
Output statistics on the number of files and directories searched\n\
and the inclusion and exclusion constraints applied.\n\
-T, --initial-tab\n\
Add a tab space to separate the file name, line number, column\n\
number and byte offset with the matched line.\n\
-t TYPES, --file-type=TYPES\n\
Search only files associated with TYPES, a comma-separated list of\n\
file types. Each file type corresponds to a set of filename\n\
extensions passed to option -O and filenames passed to option -g.\n\
For capitalized file types, the search is expanded to include files\n\
with matching file signature magic bytes, as if passed to option\n\
-M. When a type is preceded by a `!' or a `^', excludes files of\n\
the specified type. Specifying the initial part of a type name\n\
suffices when the choice is unambiguous. This option may be\n\
repeated. The possible file types can be (-tlist displays a list):";
for (int i = 0; type_table[i].type != NULL; ++i)
out << (i == 0 ? "" : ",") << (i % 7 ? " " : "\n ") << "`" << type_table[i].type << "'";
out << ".\n\
--tabs[=NUM]\n\
Set the tab size to NUM to expand tabs for option -k. The value of\n\
NUM may be 1 (no expansion), 2, 4, or 8. The default size is 8.\n\
--tag[=TAG[,END]]\n\
Disables colors to mark up matches with TAG. END marks the end of\n\
a match if specified, otherwise TAG. The default is `___'.\n\
--tree, -^\n\
Output directories with matching files in a tree-like format for\n\
option -c or --count, -l or --files-with-matches, -L or\n\
--files-without-match. This option is enabled by --pretty when the\n\
output is sent to the terminal.\n\
-U, --ascii, --binary\n\
Disables Unicode matching for ASCII and binary matching. PATTERN\n\
matches bytes. For example, -U '\\xa3' matches byte A3 (hex)\n\
instead of the Unicode code point U+00A3 represented by the UTF-8\n\
sequence C2 A3. Input is not flagged as \"binary\" for having\n\
invalid UTF, only for having NUL (zero) bytes.\n\
-u, --ungroup\n\
Do not group multiple pattern matches on the same matched line.\n\
Output the matched line again for each additional pattern match.\n\
-V, --version\n\
Display version with linked libraries and exit.\n\
-v, --invert-match\n\
Selected lines are those not matching any of the specified\n\
patterns.\n\
--view[=[+]COMMAND]\n\
Use COMMAND to view/edit a file in -Q query TUI by pressing CTRL-Y,\n\
+COMMAND waits for a key press after COMMAND terminated.\n\
-W, --with-hex\n\
Output binary matches in hexadecimal, leaving text matches alone.\n\
This option is equivalent to the --binary-files=with-hex option.\n\
To omit the matching line from the hex output, use both options -W\n\
and --hexdump. See also options -U.\n\
-w, --word-regexp\n\
The PATTERN is searched for as a word, such that the matching text\n\
is preceded by a non-word character and is followed by a non-word\n\
character. Word-like characters are Unicode letters, digits and\n\
connector punctuations such as underscore.\n\
--width[=NUM]\n\
Truncate the output to NUM visible characters per line. The width\n\
of the terminal window is used if NUM is not specified. Note that\n\
double-width characters in the output may result in wider lines.\n\
-X, --hex\n\
Output matches and matching lines in hexadecimal. This option is\n\
equivalent to the --binary-files=hex option. To omit the matching\n\
line from the hex output use option --hexdump. See also option -U.\n\
-x, --line-regexp\n\
Select only those matches that exactly match the whole line, as if\n\
the patterns are surrounded by ^ and $.\n\
--xml\n\
Output file matches in XML. When -H, -n, -k, or -b is specified,\n\
additional values are output. See also options --format and -u.\n\
-Y, --empty\n\
Empty-matching patterns match all lines. Normally, empty matches\n\
are not output, unless a pattern starts with `^' or ends with `$'.\n\
With this option, empty-matching patterns, such as x*y?, x*y*z*,\n\
and x*(y?|z) match all lines, not only the expected pattern.\n\
This option is enabled by default in grep compatibility mode;\n\
specify option --no-empty to prevent matching all lines.\n\
-y, --any-line, --passthru\n\
Any line is output (passthru). Non-matching lines are output as\n\
context with a `-' separator. See also options -A, -B and -C.\n\
";
if (!flag_grep)
out << "-Z[best][+-~][MAX], ";
out << "--fuzzy[=[best][+-~][MAX]]\n\
Fuzzy mode: report approximate pattern matches within MAX errors.\n\
The default is -Z1: one deletion, insertion or substitution is\n\
allowed. If `+`, `-' and/or `~' is specified, then `+' allows\n\
insertions, `-' allows deletions and `~' allows substitutions. For\n\
example, -Z+~3 allows up to three insertions or substitutions, but\n\
no deletions. If `best' is specified, then only the best matching\n\
lines are output with the lowest cost per file. Option -Zbest\n\
requires two passes over a file and cannot be used with standard\n\
input or Boolean queries. Option --sort=best orders matching files\n\
by best match. The first character of an approximate match always\n\
matches a character at the beginning of the pattern. To fuzzy\n\
match the first character, replace it with a `.' or `.?'. Option\n\
-U applies fuzzy matching to ASCII and bytes instead of Unicode\n\
text. No whitespace may be given between -Z and its argument.\n\
";
if (!flag_grep)
out << "-z, ";
out << "--decompress\n\
Search compressed files and archives. Archives (.cpio, .pax, .tar)\n\
and compressed archives (e.g. .zip, .7z, .taz, .tgz, .tpz, .tbz,\n\
.tbz2, .tb2, .tz2, .tlz, .txz, .tzst) are searched and matching\n\
pathnames of files in archives are output in braces. When used\n\
with option --zmax=NUM, searches the contents of compressed files\n\
and archives stored within archives up to NUM levels. When -g, -O,\n\
-M, or -t is specified, searches archives for files that match the\n\
specified globs, filename extensions, file signature magic bytes,\n\
or file types, respectively; a side-effect of these options is that\n\
the compressed files and archives searched are only those with\n\
filename extensions that match known compression and archive types.\n"
#ifndef HAVE_LIBZ
"\
This option is not available in this build configuration of ugrep.\n"
#else
"\
Supported compression formats: gzip (.gz), compress (.Z), zip"
#ifndef WITH_NO_7ZIP
", 7z"
#endif
#ifdef HAVE_LIBBZ2
",\n\
bzip2 (.bz, .bz2, .bzip2, .tbz, .tbz2, .tb2, .tz2)"
#endif
#ifdef HAVE_LIBLZMA
",\n\
xz (.xz, .txz) and lzma (requires suffix .lzma, .tlz)"
#endif
#ifdef HAVE_LIBZSTD
",\n\
zstd (.zst, .zstd, .tzst)"
#endif
#ifdef HAVE_LIBLZ4
",\n\
lz4 (requires suffix .lz4)"
#endif
#ifdef HAVE_LIBBROTLI
",\n\
brotli (requires suffix .br)"
#endif
#ifdef HAVE_LIBBZIP3
",\n\
bzip3 (requires suffix .bz3)"
#endif
".\n"
#endif
"\
--zmax=NUM\n\
When used with option -z or --decompress, searches the contents of\n\
compressed files and archives stored within archives by up to NUM\n\
expansion stages. The default --zmax=1 only permits searching\n\
uncompressed files stored in cpio, pax, tar, zip and 7z archives;\n\
compressed files and archives are detected as binary files and are\n\
effectively ignored. Specify --zmax=2 to search compressed files\n\
and archives stored in cpio, pax, tar, zip and 7z archives. NUM\n\
may range from 1 to 99 for up to 99 decompression and de-archiving\n\
steps. Increasing NUM values gradually degrades performance.\n"
#ifndef WITH_DECOMPRESSION_THREAD
"\
This option is not available in this build configuration of ugrep.\n"
#endif
;
if (flag_grep)
out << "\nGrep compatibility mode: -Z and -z reassigned to --null and --null-data.\n";
#ifdef OS_WIN
out << "\nWindows mode: short options may start with `-' or with `/'.\n";
#endif
out << "\n\
Long options may start with `--no-' to disable, when applicable.\n\
\n\
The ugrep utility exits with one of the following values:\n\
0 One or more lines were selected.\n\
1 No lines were selected.\n\
>1 An error occurred.\n\
\n\
If -q or --quiet or --silent is used and a line is selected, the exit\n\
status is 0 even if an error occurred.\n\n";
}
// print a helpful information for WHAT, if specified, and exit
void help(const char *what)
{
// strip = from =WHAT
if (what != NULL && *what == '=')
++what;
// strip --no from --no-WHAT
if (what != NULL && strncmp(what, "--no", 4) == 0)
what += 4;
// strip one dash from --WHAT
if (what != NULL && strncmp(what, "--", 2) == 0)
++what;
if (what == NULL || *what == '\0')
{
help(std::cout);
std::cout << "\n Specify `ugrep --help WHAT' for help on WHAT.\n\n";
}
else
{
std::stringstream text;
help(text);
const std::string& str = text.str();
int found = 0;
for (int pass = 0; pass < 2; ++pass)
{
size_t pos = 0;
while (true)
{
size_t end = str.find("\n -", pos + 1);
if (end == std::string::npos)
end = str.find("\n\n", pos + 1);
if (end == std::string::npos)
break;
size_t nl = str.find('\n', pos + 1);
// roughly find a case-independent match of WHAT
for (size_t i = pos + 5; i < (pass == 0 ? nl : end); ++i)
{
size_t j = 0;
for (j = 0; what[j] != '\0'; ++j)
if (((what[j] ^ str.at(i + j)) & ~0x20) != 0)
break;
if (what[j] == '\0' || what[j] == '=' || isspace(static_cast<unsigned char>(what[j])))
{
if (pass == 0 ? i < nl: i > nl)
{
if (found == 0 && pass == 0)
std::cout << "\nOptions and arguments:\n";
else if (found == 1 && pass == 1)
std::cout << "\n\nRelated options:\n";
else if (found == 0)
std::cout << "\nNo matching option, other relevant options:\n";
std::cout << str.substr(pos, end - pos);
found = pass + 1;
}
break;
}
}
pos = end;
}
}
if (found == 0)
std::cout << "ugrep --help: nothing appropriate for " << what;
else
std::cout << "\n\n Long options may start with `--no-' to disable, when applicable.";
std::cout << "\n\n";
if (strcmp(what, "format") == 0)
{
std::cout <<
"FORMAT fields for --format and --replace:\n\
\n\
field output field output\n\
---------- -------------------------- ---------- --------------------------\n\
%% % %[...]< text ... if %m = 1\n\
%~ newline (LF or CRLF) %[...]> text ... if %m > 1\n\
%a basename of matching file %, , if %m > 1, same as %[,]>\n\
%A byte range in hex of match %: : if %m > 1, same as %[:]>\n\
%b byte offset of a match %; ; if %m > 1, same as %[;]>\n\
%B %[...]B ... + byte offset, if -b %| | if %m > 1, same as %[|]>\n\
%c matching pattern as C/C++ %[...]$ assign ... to separator\n\
%C matching line as C/C++ %$ reset to default separator\n\
%d byte size of a match %[ms]=...%= color of ms ... color off\n\
%e end offset of a match --------------------------------------\n\
%f pathname of matching file \n\
%F %[...]F ... + pathname, if -H \n\
%+ %F as heading/break, if -+ Fields that require -P for captures:\n\
%h quoted \"pathname\" \n\
%H %[...]H ... + \"pathname\", if -H field output\n\
%i pathname as XML ---------- --------------------------\n\
%I %[...]I ... + pathname XML, if -H %1 %2...%9 group capture\n\
%j matching pattern as JSON %[n]# nth group capture\n\
%J matching line as JSON %[n]b nth capture byte offset\n\
%k column number of a match %[n]d nth capture byte size\n\
%K %[...]K ... + column number, if -k %[n]e nth capture end offset\n\
%l last line number of match %[n]j nth capture as JSON\n\
%L number of lines of a match %[n]q nth capture quoted\n\
%m number of matches %[n]v nth capture as CSV\n\
%M number of matching lines %[n]x nth capture as XML\n\
%n line number of a match %[n]y nth capture as hex\n\
%N %[...]N ... + line number, if -n %[name]# named group capture\n\
%o matching pattern, also %0 %[name]b named capture byte offset\n\
%O matching line %[name]d named capture byte size\n\
%p path to matching file %[name]e named capture end offset\n\
%q quoted matching pattern %[name]j named capture as JSON\n\
%Q quoted matching line %[name]q named capture quoted\n\
%R newline, if --break %[name]v named capture as CSV\n\
%s separator (: by default) %[name]x named capture as XML\n\
%S %[...]S ... + separator, if %m > 1 %[name]y named capture as hex\n\
%t tab %[n|...]# capture n,... that matched\n\
%T %[...]T ... + tab, if -T %[n|...]b capture n,... byte offset\n\
%u unique lines, unless -u %[n|...]d capture n,... byte size\n\
%[hhhh]U U+hhhh Unicode code point %[n|...]e capture n,... end offset\n\
%v matching pattern as CSV %[n|...]j capture n,... as JSON\n\
%V matching line as CSV %[n|...]q capture n,... quoted\n\
%w match width in wide chars %[n|...]v capture n,... as CSV\n\
%x matching pattern as XML %[n|...]x capture n,... as XML\n\
%X matching line as XML %[n|...]y capture n,... as hex\n\
%y matching pattern as hex %g capture number or name\n\
%Y matching line as hex %G all capture numbers/names\n\
%z path in archive %[t|...]g text t indexed by capture\n\
%Z edit distance cost, if -Z %[t|...]G all t indexed by captures\n\
-------------------------------------- --------------------------------------\n\
\n\
Options -X and -W change the %o and %O fields to output hex and hex/text.\n\
\n\
Option -o changes the %O and %Q fields to output the match only.\n\
\n\
Options -c, -l and -o change the output of %C, %J, %V, %X and %Y accordingly.\n\
\n\
Numeric fields such as %n are padded with spaces when %{width}n is specified.\n\
\n\
Matching line fields such as %O are cut to width when %{width}O is specified or\n\
when %{-width}O is specified to cut from the end of the line.\n\
\n\
Character context on a matching line before or after a match is output when\n\
%{-width}o or %{+width}o is specified for match fields such as %o, where\n\
%{width}o without a +/- sign cuts the match to the specified width.\n\
\n\
";
}
else if (strcmp(what, "regex") == 0)
{
std::cout <<
"Extended regular expression syntax overview (excludes some advanced patterns):\n\
\n\
pattern matches pattern removes special meaning \n\
---------- -------------------------- ---------- --------------------------\n\
. any character except \\n \\. escapes . to match .\n\
a the character a \\Q...\\E the literal string ...\n\
ab the string ab --------------------------------------\n\
a|b a or b \n\
a* zero or more a's pattern special characters\n\
a+ one or more a's ---------- --------------------------\n\
a? zero or one a \\f form feed\n\
a{3} 3 a's \\n newline\n\
a{3,} 3 or more a's \\r carriage return\n\
a{3,7} 3 to 7 a's \\R any Unicode line break\n\
a*? zero or more a's lazily \\t tab\n\
a+? one or more a's lazily \\v vertical tab\n\
a?? zero or one a lazily \\X any character and \\n\n\
a{3}? 3 a's lazily \\cZ control character ^Z\n\
a{3,}? 3 or more a's lazily \\0 NUL\n\
a{3,7}? 3 to 7 a's lazily \\0ddd octal character code ddd\n\
a(b|cd?) ab or ac or acd \\xhh hex character code hh\n\
-------------------------------------- \\x{hhhh} Unicode code point U+hhhh\n\
\\u{hhhh} Unicode code point U+hhhh\n\
pattern character classes --------------------------------------\n\
---------- -------------------------- \n\
[abc-e] one character a,b,c,d,e pattern anchors and boundaries\n\
[^abc-e] one char not a,b,c,d,e,\\n ---------- --------------------------\n\
[[:name:]] one char in POSIX class: ^ begin of line anchor\n\
alnum a-z,A-Z,0-9 $ end of line anchor\n\
alpha a-z,A-Z \\A begin of file anchor\n\
ascii ASCII char \\x00-\\x7f \\Z end of file anchor\n\
blank space or tab \\b word boundary\n\
cntrl control characters \\B non-word boundary\n\
digit 0-9 \\< start of word boundary\n\
graph visible characters \\> end of word boundary\n\
lower a-z (?=...) lookahead (-P)\n\
print visible chars and space (?!...) negative lookahead (-P)\n\
punct punctuation characters (?<=...) lookbehind (-P)\n\
space space,\\t,\\v,\\f,\\r (?<!...) negative lookbehind (-P)\n\
upper A-Z --------------------------------------\n\
word a-z,A-Z,0-9,_ (-P): pattern requires option -P\n\
xdigit 0-9,a-f,A-F \n\
\\p{class} one character in class pattern grouping\n\
\\P{class} one char not in class ---------- --------------------------\n\
\\d a digit (...) non-capturing group\n\
\\D a non-digit (...) capturing group (-P)\n\
\\h a space or tab (?:...) non-capturing group (-P)\n\
\\H not a space or tab (?<X>...) capturing, named X (-P)\n\
\\s a whitespace except \\n \\1 matches group 1 (-P)\n\
\\S a non-whitespace \\g{10} matches group 10 (-P)\n\
\\w a word character \\g{X} matches group name X (-P)\n\
\\W a non-word character (?#...) comments ... are ignored\n\
-------------------------------------- --------------------------------------\n\
(-P): pattern requires option -P\n\
\n\
Option -P enables Perl regex matching with Unicode patterns, lookarounds and\n\
capturing groups.\n\
\n\
Option -U disables full Unicode pattern matching: non-POSIX Unicode character\n\
classes \\p{class} are disabled, ASCII, LATIN1 and binary regex patterns only.\n\
\n\
To match multiple lines, specify a pattern that matches line breaks, such as\n\
\\n or \\R. Character classes do not match a newline character, except for\n\
\\P, \\D, \\H, \\W and \\X and when a \\n or a \\R is explicitly specified,\n\
such as [\\s\\n].\n\
\n\
Options -% (--bool --lines) and -%% (--bool --files) augments pattern syntax:\n\
\n\
pattern operation pattern operation\n\
---------- -------------------------- ---------- --------------------------\n\
(...) grouping \"...\" the literal string ...\n\
-a logical not a NOT a locical not a\n\
a|b logical a or b a OR b logical a or b\n\
a b logical a and b a AND b locical a and b\n\
-------------------------------------- --------------------------------------\n\
\n\
Listed from the highest level of precedence to the lowest, NOT is performed\n\
before OR and OR is performed before AND. Thus, `-x|y z' is `((-x)|y) z'.\n\
Spacing of the logical AND, OR and NOT operators is required.\n\
\n\
Nested parentheses that are part of a regex sub-pattern do not group logical\n\
operators. For example, `((x y)z)' matches `x yz'. Note that `((x y){3} z)'\n\
matches `x y' three times and a `z', since the outer parentheses group the AND\n\
(a space).\n\
\n\
Option -% matches lines satisfying the Boolean query. The double short option\n\
-%% matches files satisfying the Boolean query.\n\
\n\
See also options --and, --andnot, --not to specify sub-patterns as command-line\n\
arguments with these options as logical operators and option -e as logical OR.\n\
\n\
Option --stats displays the options and patterns applied to the matching files.\n\
\n\
NOTE: always quote regex patterns to prevent the shell from expanding globs.\n\
NOTE: a pattern starting with a `-' requires option -e as in -e '-PATTERN'.\n\
\n\
";
}
else if (strcmp(what, "glob") == 0 || strcmp(what, "globs") == 0 || strcmp(what, "globbing") == 0)
{
std::cout <<
"Glob syntax and conventions:\n\
\n\
Gitignore-style globbing is performed by all glob-related options: -g (--glob),\n\
--iglob, --include, --exclude, --include-dir, --exclude-dir, --include-from,\n\
--exclude-from, and --ignore-files.\n\
\n\
pattern matches\n\
---------- -----------------------------------------------------------\n\
* anything except /\n\
? any one character except /\n\
[abc-e] one character a,b,c,d,e\n\
[^abc-e] one character not a,b,c,d,e,/\n\
[!abc-e] one character not a,b,c,d,e,/\n\
/ when used at the start of a glob, matches working directory\n\
when used at the end of a glob, matches directories only\n\
**/ zero or more directories\n\
/** when at the end of a glob, matches everything after the /\n\
\\? a ? or any other character specified after the backslash\n\
\n\
A glob pattern starting with a ! or a ^ inverts the match. That is, instead of\n\
matching a file or directory name, the file or directory is ignored.\n\
\n\
When a glob pattern contains a /, the full pathname is matched from the working\n\
directory down. Otherwise, the basename of a file or directory is matched in\n\
recursive searches.\n\
\n\
When a glob pattern starts with ./ or just a single /, the glob is matched at\n\
the working directory.\n\
\n\
When a glob pattern ends with a /, only directories are matched, not files.\n\
\n\
Option -g (--glob) performs glob matching with the specified glob pattern or\n\
a set of comma-separated globs. When a glob is preceded with a ! or ^, the\n\
glob match is inverted by excluding matching files or directories.\n\
\n\
Option --iglob performs case-insensitive glob matching with the specified glob\n\
pattern or a set of comma-separated globs. When a glob is preceded with a !\n\
or ^, the glob match is inverted by excluding matching files or directories.\n\
\n\
Option --glob-ignore-case performs case-insensitive glob matching in general,\n\
except with the globs specified in --ignore-files, such as .gitignore.\n\
\n\
Option --ignore-files specifies a file with gitignore-style globs, where the\n\
default file is .gitignore. When ignore files are encountered in recursive\n\
searches, the search is narrowed by excluding file and directory names matching\n\
the globs. The --ignore-files glob matching of both file and directory names\n\
differs from all other glob-related options, such as --exclude-from, that\n\
require globs ending in / to match directory names.\n\
\n\
Option -O (--file-extension) matches filename extensions, or ignores extensions\n\
when preceded with a ! or ^.\n\
\n\
Option -t (--file-type) matches file types, or ignores file types when\n\
preceded with a ! or a ^. Use -tlist to view the list of supported file types\n\
with corresponding glob patterns.\n\
\n\
Option --stats displays the search path globs applied to the matching files.\n\
\n\
NOTE: always quote glob patterns to prevent the shell from expanding globs.\n\
For example, specify -g \"*foo.*\" or -g\"*foo.*\" or \"-g*foo.*\".\n\
\n\
";
}
else if (strcmp(what, "fuzzy") == 0)
{
std::cout <<
"Fuzzy (approximate) search is performed with option -Z:\n\
\n\
-Z fuzzy match\n\
---- ---------------------------------------------------------------------\n\
allow 1 character insertion, deletion or substitution (-Z default)\n\
2 allow 2 character insertions, deletions and/or substitutions\n\
+1 allow 1 character insertion, e.g. pattern 'foo' also matches 'fomo'\n\
-1 allow 1 character deletion, e.g. pattern 'food' also matches 'fod'\n\
~1 allow 1 character substitution, e.g. pattern 'foo' also matches 'for'\n\
+-1 allow 1 character insertion or deletion\n\
+~1 allow 1 character insertion or substitution\n\
-~1 allow 1 character deletion or substitution\n\
best when prefixed to the above, output the best matches only\n\
\n\
No whitespace may be given between -Z and its argument.\n\
\n\
Insertions, deletions and/or substitutions are applied to Unicode characters,\n\
except when option -U is specified for ASCII/binary pattern search.\n\
\n\
The 'best' prefix outputs the best matching lines with the lowest cost (minimal\n\
edit distance) per file. For example, if a file has an exact match anywhere,\n\
then only exact matches are output, no approximate matches. This requires two\n\
passes over an input file to determine the minimal edit distance, which reduces\n\
performance significantly. This option cannot be used with standard input or\n\
with Boolean queries.\n\
\n\
Option --sort=best sorts the directory output by best fuzzy match based on edit\n\
distance, i.e. files with at least one exact match are output first, followed\n\
by files with at least a match that is one edit distance from exact, and so on.\n\
This sort option requires two passes over each input file, which reduces\n\
performance significantly.\n\
\n\
NOTE: Fuzzy search anchors each pattern match at the beginning character of the\n\
specified regex pattern. This significantly improves search performance and\n\
prevents overmatching. For example, the regex pattern `[1-9]buzz' starts with\n\
a nonzero digit at which all matches are anchored. Replace the beginning of a\n\
regex pattern with a wildcard to match any character, for example `.buzz', or\n\
make the first characters of the regex pattern optional, for example\n\
`[1-9]?buzz' or `.?buzz'.\n\
\n\
";
}
}
exit(EXIT_ERROR);
}
// print the version info
void version()
{
#if defined(HAVE_PCRE2)
uint32_t tmp = 0;
#endif
std::cout << "ugrep " UGREP_VERSION;
if (flag_grep)
std::cout << " (" << (flag_mode == Mode::BRE ? "" : flag_mode == Mode::FIXED ? "f" : "e") << "grep compat)";
std::cout << " " PLATFORM <<
#if defined(HAVE_AVX512BW)
(reflex::have_HW_AVX512BW() ? " +avx512" : (reflex::have_HW_AVX2() ? " +avx2" : reflex::have_HW_SSE2() ? " +sse2" : " (no sse2!)")) <<
#elif defined(HAVE_AVX2)
(reflex::have_HW_AVX2() ? " +avx2" : reflex::have_HW_SSE2() ? " +sse2" : " (no sse2!)") <<
#elif defined(HAVE_SSE2)
(reflex::have_HW_SSE2() ? " +sse2" : " (no sse2!)") <<
#elif defined(HAVE_NEON)
" +neon/AArch64" <<
#endif
#if defined(HAVE_PCRE2)
(pcre2_config(PCRE2_CONFIG_JIT, &tmp) >= 0 && tmp != 0 ? "; -P:pcre2jit" : "; -P:pcre2") <<
#elif defined(HAVE_BOOST_REGEX)
"; -P:boost_regex" <<
#endif
#ifdef HAVE_LIBZ
"; -z:zlib" <<
#ifdef HAVE_LIBBZ2
",bzip2" <<
#endif
#ifdef HAVE_LIBLZMA
",lzma" <<
#endif
#ifdef HAVE_LIBLZ4
",lz4" <<
#endif
#ifdef HAVE_LIBZSTD
",zstd" <<
#endif
#ifdef HAVE_LIBBROTLI
",brotli" <<
#endif
#ifdef HAVE_LIBBZIP3
",bzip3" <<
#endif
#ifndef WITH_NO_7ZIP
",7z" <<
#endif
",tar/pax/cpio/zip" <<
#endif
"\n"
"License: BSD-3-Clause; ugrep user manual: <https://ugrep.com>\n"
"Written by Robert van Engelen and others: <https://github.com/Genivia/ugrep>\n"
"Ugrep utilizes the RE/flex regex library: <https://github.com/Genivia/RE-flex>" << std::endl;
exit(EXIT_OK);
}
// print to standard error: ... is a directory if -s and -q not specified
void is_directory(const char *pathname)
{
if (!flag_no_messages)
fprintf(Static::errout, "%sugrep: %s%s%s is a directory\n", color_off, color_high, pathname, color_off);
}
#ifdef HAVE_LIBZ
// print to standard error: cannot decompress message if -s and -q not specified
void cannot_decompress(const char *pathname, const char *message)
{
if (!flag_no_messages)
fprintf(Static::errout, "%sugrep: cannot decompress %s%s%s: %s\n", color_off, color_fn, pathname, color_off, message != NULL ? message : "");
}
#endif
// print to standard error: warning message if -s and -q not specified, display error if errno is set, like perror()
void warning(const char *message, const char *arg)
{
if (!flag_no_messages)
{
const char *errmsg = NULL;
#if defined(__STDC_LIB_EXT1__) || defined(OS_WIN_OR_MINGW)
char errbuf[256];
if (errno)
{
// use safe strerror_s() instead of strerror() when available
strerror_s(errbuf, sizeof(errbuf), errno);
errmsg = errbuf;
}
fprintf(Static::errout, "%sugrep: %swarning:%s %s%s%s%s%c%s %s%s%s\n", color_off, color_warning, color_off, color_high, message != NULL ? message : "", message != NULL ? " " : "", arg != NULL ? arg : "", errmsg != NULL ? ':' : ' ', color_off, color_message, errmsg != NULL ? errmsg : "", color_off);
#else
if (errno)
errmsg = strerror(errno);
fprintf(Static::errout, "%sugrep: %swarning:%s %s%s%s%s%c%s %s%s%s\n", color_off, color_warning, color_off, color_high, message != NULL ? message : "", message != NULL ? " " : "", arg != NULL ? arg : "", errmsg != NULL ? ':' : ' ', color_off, color_message, errmsg != NULL ? errmsg : "", color_off);
#endif
}
++Static::warnings;
}
// print to standard error: error message, assumes errno is set, like perror(), then exit
void error(const char *message, const char *arg)
{
// use safe strerror_s() instead of strerror() when available
#if defined(__STDC_LIB_EXT1__) || defined(OS_WIN_OR_MINGW)
char errmsg[256];
strerror_s(errmsg, sizeof(errmsg), errno);
#else
const char *errmsg = strerror(errno);
#endif
fprintf(Static::errout, "%sugrep: %serror:%s %s%s%s%s:%s %s%s%s\n\n", color_off, color_error, color_off, color_high, message != NULL ? message : "", message != NULL ? " " : "", arg != NULL ? arg : "", color_off, color_message, errmsg, color_off);
exit(EXIT_ERROR);
}
// print to standard error: abort message with exception details, then exit
void abort(const char *message)
{
fprintf(Static::errout, "%sugrep: %s%s%s\n\n", color_off, color_error, message, color_off);
exit(EXIT_ERROR);
}
// print to standard error: abort message with exception details, then exit
void abort(const char *message, const std::string& what)
{
fprintf(Static::errout, "%sugrep: %s%s%s%s%s%s\n\n", color_off, color_error, message != NULL ? message : "", color_off, color_high, what.c_str(), color_off);
exit(EXIT_ERROR);
}