Changes to src/core/stringtools.cpp. All files

src/core/stringtools.cpp

@@ -131,6 +131,71 @@ bool endsWith(std::string const & value, std::string const & ending)     return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); } +static void appendUtf8(string& out, unsigned long cp)+{+    if (cp < 0x80) {+        out += (char) cp;+    } else if (cp < 0x800) {+        out += (char) (0xC0 | (cp >> 6));+        out += (char) (0x80 | (cp & 0x3F));+    } else if (cp < 0x10000) {+        out += (char) (0xE0 | (cp >> 12));+        out += (char) (0x80 | ((cp >> 6) & 0x3F));+        out += (char) (0x80 | (cp & 0x3F));+    } else {+        out += (char) (0xF0 | (cp >> 18));+        out += (char) (0x80 | ((cp >> 12) & 0x3F));+        out += (char) (0x80 | ((cp >> 6) & 0x3F));+        out += (char) (0x80 | (cp & 0x3F));+    }+}++string utfWideToUtf8(const string& bytes, unsigned int unitSize, bool bigEndian)+{+    const unsigned long replacement = 0xFFFD;+    string result;+    result.reserve(bytes.size());++    auto readUnit = [&](size_t pos) {+        unsigned long unit = 0;+        for (unsigned int i=0; i<unitSize; i++) {+            unsigned char b = (unsigned char) bytes[pos + (bigEndian ? i : unitSize-1-i)];+            unit = (unit << 8) | b;+        }+        return unit;+    };++    size_t pos = 0;+    while (pos + unitSize <= bytes.size()) {+        unsigned long cp = readUnit(pos);+        pos += unitSize;++        if (unitSize == 2 && cp >= 0xD800 && cp <= 0xDBFF) {+            // high surrogate, needs a following low surrogate+            if (pos + 2 <= bytes.size()) {+                unsigned long low = readUnit(pos);+                if (low >= 0xDC00 && low <= 0xDFFF) {+                    cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00);+                    pos += 2;+                } else {+                    cp = replacement;+                }+            } else {+                cp = replacement;+            }+        } else if ((cp >= 0xD800 && cp <= 0xDFFF) || cp > 0x10FFFF) {+            cp = replacement;+        }+        appendUtf8(result, cp);+    }++    // truncated trailing code unit+    if (pos < bytes.size())+        appendUtf8(result, replacement);++    return result;+}+ int utf8_strlen(const string& str) {     int c,i,ix,q;