261008.3 · e001deb4cfb735a6b14a802398a41bfa03df5a2a
Strip UTF-8 BOM at the start of each input file (#3657)
The UTF-8 BOM was stripped only at the start of the parser stream, but jq feeds all input files into a single parser, so a BOM at the start of the second and later files reached the parser as content and failed with a confusing "Invalid numeric literal" error. Strip it in the file readers instead, at the start of each input file including stdin, and in jv_load_file.
This also strips the BOM for raw input, so -R and --rawfile no longer yield a leading U+FEFF, and a BOM at the start of a -f program file or an imported module is now accepted rather than reported as a syntax error. In exchange, "\uFEFF1" | fromjson is now a parse error, since a U+FEFF inside a string is content rather than an encoding marker, and the dedicated "Malformed BOM" error is gone.
reading declarations
Changes to src/util.c. All files
@@ -193,6 +193,7 @@ struct jq_util_input_state { size_t buf_valid_len; jv current_filename; size_t current_line;+ int at_file_start; }; static void fprinter(void *data, const char *fname) {@@ -277,6 +278,7 @@ static int jq_util_input_read_more(jq_util_input_state *state) { if (f != NULL) { jv_free(state->current_filename); state->current_line = 0;+ state->at_file_start = 1; if (!strcmp(f, "-")) { state->current_input = stdin; state->current_filename = jv_string("<stdin>");@@ -343,6 +345,14 @@ static int jq_util_input_read_more(jq_util_input_state *state) { state->buf_valid_len = (p - state->buf) + 1; } }+ if (state->at_file_start) {+ // Strip UTF-8 BOM at the start of each file+ if (state->buf_valid_len >= 3 && !memcmp(state->buf, "\xEF\xBB\xBF", 3)) {+ state->buf_valid_len -= 3;+ memmove(state->buf, state->buf + 3, state->buf_valid_len);+ }+ state->at_file_start = 0;+ } } return state->curr_file == state->nfiles && !state->current_input; }