261008.3 · e001deb4cfb735a6b14a802398a41bfa03df5a2a
Strip UTF-8 BOM at the start of each input file (#3657)
The UTF-8 BOM was stripped only at the start of the parser stream, but jq feeds all input files into a single parser, so a BOM at the start of the second and later files reached the parser as content and failed with a confusing "Invalid numeric literal" error. Strip it in the file readers instead, at the start of each input file including stdin, and in jv_load_file.
This also strips the BOM for raw input, so -R and --rawfile no longer yield a leading U+FEFF, and a BOM at the start of a -f program file or an imported module is now accepted rather than reported as a syntax error. In exchange, "\uFEFF1" | fromjson is now a parse error, since a U+FEFF inside a string is content rather than an encoding marker, and the dedicated "Malformed BOM" error is gone.
reading declarations
@@ -43,6 +43,7 @@ jv jv_load_file(const char* filename, int raw) { // buffer, we need to be able to read the remainder of a sequence and add that // before appending. char buf[4096 + 4];+ int at_file_start = 1; while (!feof(file) && !ferror(file)) { size_t n = fread(buf, 1, sizeof(buf) - 4, file); if (n == 0)@@ -55,12 +56,22 @@ jv jv_load_file(const char* filename, int raw) { n += fread(end, 1, len, file); } + char *p = buf;+ if (at_file_start) {+ // Strip UTF-8 BOM at the start of the file+ if (n >= 3 && !memcmp(buf, "\xEF\xBB\xBF", 3)) {+ p += 3;+ n -= 3;+ }+ at_file_start = 0;+ }+ if (raw) {- data = jv_string_append_buf(data, buf, n);+ data = jv_string_append_buf(data, p, n); if (!jv_is_valid(data)) break; } else {- jv_parser_set_buf(parser, buf, n, !feof(file));+ jv_parser_set_buf(parser, p, n, !feof(file)); jv value; while (jv_is_valid((value = jv_parser_next(parser)))) data = jv_array_append(data, value);
@@ -37,7 +37,6 @@ struct jv_parser { int curr_buf_pos; int curr_buf_is_partial; int eof;- unsigned bom_strip_position; int flags; @@ -90,7 +89,6 @@ static void parser_init(struct jv_parser* p, int flags) { p->eof = 0; p->curr_buf = 0; p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;- p->bom_strip_position = 0; p->last_ch_was_ws = 0; p->line = 1; p->column = 0;@@ -729,8 +727,6 @@ void jv_parser_free(struct jv_parser* p) { jv_mem_free(p); } -static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};- int jv_parser_remaining(struct jv_parser* p) { if (p->curr_buf == 0) return 0;@@ -740,22 +736,6 @@ int jv_parser_remaining(struct jv_parser* p) { void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) { assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length) && "previous buffer not exhausted");- while (length > 0 && p->bom_strip_position < sizeof(UTF8_BOM)) {- if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) {- // matched a BOM character- buf++;- length--;- p->bom_strip_position++;- } else {- if (p->bom_strip_position == 0) {- // no BOM in this document- p->bom_strip_position = sizeof(UTF8_BOM);- } else {- // malformed BOM (prefix present, rest missing)- p->bom_strip_position = 0xff;- }- }- } p->curr_buf = buf; p->curr_buf_length = length; p->curr_buf_pos = 0;@@ -779,12 +759,6 @@ jv jv_parser_next(struct jv_parser* p) { return jv_invalid(); if (!p->curr_buf) return jv_invalid(); // Need a buffer- if (p->bom_strip_position == 0xff) {- if (!(p->flags & JV_PARSE_SEQ))- return jv_invalid_with_msg(jv_string("Malformed BOM"));- p->st =JV_PARSER_WAITING_FOR_RS;- parser_reset(p);- } jv value = jv_invalid(); if ((p->flags & JV_PARSE_STREAMING) && stream_check_done(p, &value)) return value;
@@ -193,6 +193,7 @@ struct jq_util_input_state { size_t buf_valid_len; jv current_filename; size_t current_line;+ int at_file_start; }; static void fprinter(void *data, const char *fname) {@@ -277,6 +278,7 @@ static int jq_util_input_read_more(jq_util_input_state *state) { if (f != NULL) { jv_free(state->current_filename); state->current_line = 0;+ state->at_file_start = 1; if (!strcmp(f, "-")) { state->current_input = stdin; state->current_filename = jv_string("<stdin>");@@ -343,6 +345,14 @@ static int jq_util_input_read_more(jq_util_input_state *state) { state->buf_valid_len = (p - state->buf) + 1; } }+ if (state->at_file_start) {+ // Strip UTF-8 BOM at the start of each file+ if (state->buf_valid_len >= 3 && !memcmp(state->buf, "\xEF\xBB\xBF", 3)) {+ state->buf_valid_len -= 3;+ memmove(state->buf, state->buf + 3, state->buf_valid_len);+ }+ state->at_file_start = 0;+ } } return state->curr_file == state->nfiles && !state->current_input; }
@@ -42,13 +42,6 @@ null {"x":-1} {"x":1} -# The input line starts with a 0xFEFF (byte order mark) codepoint-# No, there is no reason to have a byte order mark in UTF8 text.-# But apparently people do, so jq shouldn't break on it.-.-"byte order mark"-"byte order mark"- # We test escapes by matching them against Unicode codepoints # FIXME: more tests needed for weird unicode stuff (e.g. utf16 pairs) "Aa\r\n\t\b\f\u03bc"
@@ -57,6 +57,17 @@ for exp in '1+1' '1-1' '2*3' '9/3' '9%3' '9==3' '9!=3' \ fi done +## Test UTF-8 BOM is stripped at the start of each input file+printf '\357\273\277[{"foo": 1}\n,{"foo": 2}\n]\n' > $d/bom1.json+printf '\357\273\277[{"foo": 3}\n,{"foo": 4}\n]\n' > $d/bom2.json+$VALGRIND $Q $JQ -e '.[0].foo == 1' $d/bom1.json+$VALGRIND $Q $JQ -se '. == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json $d/bom2.json+$VALGRIND $Q $JQ -ne '[inputs] == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json $d/bom2.json+$VALGRIND $Q $JQ -ne '[inputs] == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json - < $d/bom2.json+$VALGRIND $Q $JQ -Rne '[inputs] == ["[{\"foo\": 1}", ",{\"foo\": 2}", "]", "[{\"foo\": 3}", ",{\"foo\": 4}", "]"]' $d/bom1.json $d/bom2.json+$VALGRIND $Q $JQ -ne --slurpfile x $d/bom1.json '$x == [[{"foo":1},{"foo":2}]]'+$VALGRIND $Q $JQ -ne --rawfile x $d/bom1.json '$x == "[{\"foo\": 1}\n,{\"foo\": 2}\n]\n"'+ ## Test JSON sequence support cat > $d/expected <<EOF jq: ignoring parse error: Truncated value at line 2, column 5