261020.1 · 852c4d609611d1177de832010ff2de810ef026c8
Improve performance of UTF-8 validation (#3675)
jvp_utf8_next() had a fast path for ASCII, but it is not inlined into its callers, so every byte of a string cost a function call.
Move the end check and the ASCII case into a static inline function in jv_unicode.h and keep the rest out of line. This speeds up parsing of string-heavy input by up to 1.17x and string functions such as length, explode and @html, with the same output for every input.
reading declarations
@@ -784,9 +784,10 @@ jv jv_parser_next(struct jv_parser* p) { } else if (msg) { jv_free(value); if (ch != '\036' && (p->flags & JV_PARSE_SEQ)) {- // Skip to the next RS value = make_error(p, "%s at line %d, column %d (need RS to resync)", msg, p->line, p->column); parser_reset(p);+ // Skip to the next RS. This must come after parser_reset(), which+ // sets the state back to JV_PARSER_NORMAL. p->st = JV_PARSER_WAITING_FOR_RS; return value; }
@@ -27,19 +27,14 @@ const char* jvp_utf8_backtrack(const char* start, const char* min, int *missing_ return start; } -const char* jvp_utf8_next(const char* in, const char* end, int* codepoint_ret) {- assert(in <= end);- if (in == end) {- return 0;- }+// The rest of jvp_utf8_next(), for a first byte that is not ASCII.+const char* jvp_utf8_next_multibyte(const char* in, const char* end, int* codepoint_ret) {+ assert(in < end); int codepoint = -1; unsigned char first = (unsigned char)in[0];+ assert(first & 0x80); int length = utf8_coding_length[first];- if ((first & 0x80) == 0) {- /* Fast-path for ASCII */- codepoint = first;- length = 1;- } else if (length == 0 || length == UTF8_CONTINUATION_BYTE) {+ if (length == 0 || length == UTF8_CONTINUATION_BYTE) { /* Bad single byte - either an invalid byte or an out-of-place continuation byte */ length = 1; } else if (length > end - in) {
@@ -1,10 +1,27 @@ #ifndef JV_UNICODE_H #define JV_UNICODE_H +#include <assert.h>+ const char* jvp_utf8_backtrack(const char* start, const char* min, int *missing_bytes);-const char* jvp_utf8_next(const char* in, const char* end, int* codepoint);+const char* jvp_utf8_next_multibyte(const char* in, const char* end, int* codepoint); int jvp_utf8_is_valid(const char* in, const char* end); +// Decode one codepoint. ASCII is handled here so that callers looping over+// a string do not make a call per byte; everything else goes to+// jvp_utf8_next_multibyte().+static inline const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {+ if (in >= end) {+ assert(in == end);+ return 0;+ }+ if (!(*in & 0x80)) {+ *codepoint = (unsigned char)*in;+ return in + 1;+ }+ return jvp_utf8_next_multibyte(in, end, codepoint);+}+ int jvp_utf8_decode_length(char startchar); int jvp_utf8_encode_length(int codepoint);
@@ -84,6 +84,11 @@ null "Zm/Ds2Jhcgo=" "foóbar\n" +# Invalid UTF-8 next to ASCII runs is still replaced+map(@base64d)+["YWJjZGVmZ2hpav9rbG1ub3A=", "YWJjZGVmZ2hpasOpa2xtbm9w5pel", "/2FiY2RlZmdoaWo=", "YWJjZGVmZ2hpauaX", "YWJjZGVmZ2hpau2ggHh5eg=="]+["abcdefghij�klmnop","abcdefghijéklmnop日","�abcdefghij","abcdefghij�","abcdefghij�xyz"]+ @uri "\u03bc" "%CE%BC"