261008.3 · e001deb4cfb735a6b14a802398a41bfa03df5a2a
Strip UTF-8 BOM at the start of each input file (#3657)
The UTF-8 BOM was stripped only at the start of the parser stream, but jq feeds all input files into a single parser, so a BOM at the start of the second and later files reached the parser as content and failed with a confusing "Invalid numeric literal" error. Strip it in the file readers instead, at the start of each input file including stdin, and in jv_load_file.
This also strips the BOM for raw input, so -R and --rawfile no longer yield a leading U+FEFF, and a BOM at the start of a -f program file or an imported module is now accepted rather than reported as a syntax error. In exchange, "\uFEFF1" | fromjson is now a parse error, since a U+FEFF inside a string is content rather than an encoding marker, and the dedicated "Malformed BOM" error is gone.
reading declarations
Changes to src/jv_file.c. All files
@@ -43,6 +43,7 @@ jv jv_load_file(const char* filename, int raw) { // buffer, we need to be able to read the remainder of a sequence and add that // before appending. char buf[4096 + 4];+ int at_file_start = 1; while (!feof(file) && !ferror(file)) { size_t n = fread(buf, 1, sizeof(buf) - 4, file); if (n == 0)@@ -55,12 +56,22 @@ jv jv_load_file(const char* filename, int raw) { n += fread(end, 1, len, file); } + char *p = buf;+ if (at_file_start) {+ // Strip UTF-8 BOM at the start of the file+ if (n >= 3 && !memcmp(buf, "\xEF\xBB\xBF", 3)) {+ p += 3;+ n -= 3;+ }+ at_file_start = 0;+ }+ if (raw) {- data = jv_string_append_buf(data, buf, n);+ data = jv_string_append_buf(data, p, n); if (!jv_is_valid(data)) break; } else {- jv_parser_set_buf(parser, buf, n, !feof(file));+ jv_parser_set_buf(parser, p, n, !feof(file)); jv value; while (jv_is_valid((value = jv_parser_next(parser)))) data = jv_array_append(data, value);