ChangeLog.adoc

@@ -7,6 +7,12 @@ :toclevels: 1  +== highlight 4.22++xx.xx.2026++ - added support for UTF-16 and UTF-32 input files with byte order mark (https://gitlab.com/saalen/highlight/-/work_items/266)+ == highlight 4.21  09.08.2026

README.adoc

@@ -491,6 +491,14 @@ Use `--verbose` to display Lua and syntax data. Apply twice to print more inform  Use `--validate-input` to get rid of UTF8 byte order marks. +==== UTF-16 and UTF-32 input++Input files starting with a UTF-16 or UTF-32 byte order mark (little or big endian)+are converted to UTF-8 before parsing, also if read from stdin. The output encoding+is set to utf-8 for these files, unless `--encoding=none` is given.+Files in these encodings without a BOM are not detected; convert them to UTF-8+beforehand (e.g. with `iconv -f UTF-16LE -t UTF-8`).+ ==== Force output to stdout  Use `--stdout` to write output files in batch mode to stdout.

README_DE.adoc

@@ -500,6 +500,15 @@ um mehr Informationen zu erhalten. Geben Sie `--validate-input` an, um das UTF8 Byte Order Mark (Startsequenz) zu entfernen. +==== UTF-16 und UTF-32 Eingabe++Eingabedateien, die mit einem UTF-16 oder UTF-32 Byte Order Mark beginnen (Little+oder Big Endian), werden vor dem Parsen nach UTF-8 konvertiert, auch bei Eingabe+ueber stdin. Die Ausgabekodierung wird fuer diese Dateien auf utf-8 gesetzt, sofern+nicht `--encoding=none` angegeben ist.+Dateien in diesen Kodierungen ohne BOM werden nicht erkannt; konvertieren Sie diese+vorher nach UTF-8 (z.B. mit `iconv -f UTF-16LE -t UTF-8`).+ ==== Ausgabe in stdout erzwingen  Mit `--stdout` wird die Ausgabe auch im Batch-Modus nach stdout ausgegeben.

README_FR.adoc

@@ -492,6 +492,15 @@ fois pour obtenir plus d'informations.  Utilisez `--validate-input` pour supprimer la marque d'ordre des octets UTF8 (séquence de début). +==== Entrée UTF-16 et UTF-32++Les fichiers d'entrée commençant par une marque d'ordre des octets UTF-16 ou UTF-32+(little ou big endian) sont convertis en UTF-8 avant l'analyse, y compris depuis stdin.+L'encodage de sortie de ces fichiers est défini sur utf-8, sauf si `--encoding=none`+est indiqué.+Les fichiers dans ces encodages sans BOM ne sont pas détectés ; convertissez-les+d'abord en UTF-8 (par ex. avec `iconv -f UTF-16LE -t UTF-8`).+ ==== Forcer la sortie vers stdout  Avec `--stdout`, la sortie est dirigée vers stdout même en mode batch.

man/highlight.1

@@ -117,7 +117,8 @@ set highlighting style (theme). Add 'base16/' prefix to use a Base16 theme. The .IP "\fB-t\fR  \fB--replace-tabs\fR=<\fInum\fR>" replace tabs by num spaces .IP "\fB-u\fR, \fB--encoding\fR=<\fIenc\fR>"-set output encoding which matches input file encoding; omit encoding information if set to "NONE"+set output encoding which matches input file encoding; omit encoding information if set to "NONE".+Input starting with a UTF-16 or UTF-32 byte order mark is converted to UTF-8, and the output encoding is set to utf-8 for this file (unless "NONE") .IP "\fB-V\fR, \fB--wrap-simple\fR" wrap lines after 80 (default) characters without indenting function parameters and statements. .IP "\fB-W\fR, \fB--wrap\fR"

src/core/codegenerator.cpp

@@ -1211,6 +1211,77 @@ LoadResult CodeGenerator::loadLanguage ( const string& langDefPath, bool embedde     return result; } +bool CodeGenerator::convertUnicodeInput()+{+    if ( !in || in->fail() ) return false;++    // UTF-32LE has to be tested before UTF-16LE (same first two bytes)+    struct UnicodeBom {+        const char* magic;+        size_t length;+        unsigned int unitSize;+        bool bigEndian;+    };+    const UnicodeBom boms[] = {+        { "\xFF\xFE\x00\x00", 4, 4, false },+        { "\x00\x00\xFE\xFF", 4, 4, true },+        { "\xFF\xFE", 2, 2, false },+        { "\xFE\xFF", 2, 2, true },+    };++    // read the first bytes sequentially, so that this works with stdin too+    string head;+    while ( head.size() < 4 ) {+        int c = in->peek();+        if ( c == EOF ) break;+        // stop as soon as no BOM can match anymore+        bool candidate = false;+        for ( const UnicodeBom& bom : boms ) {+            if ( head.size() < bom.length && head.compare ( 0, head.size(), bom.magic, head.size() ) == 0+                    && (char) c == bom.magic[head.size()] ) {+                candidate = true;+                break;+            }+        }+        if ( !candidate ) break;+        head += (char) in->get();+    }++    const UnicodeBom* found = nullptr;+    for ( const UnicodeBom& bom : boms ) {+        if ( head.size() >= bom.length && head.compare ( 0, bom.length, bom.magic, bom.length ) == 0 ) {+            found = &bom;+            break;+        }+    }++    if ( !found ) {+        // nothing consumed except for partial BOM candidates; restore them+        if ( !head.empty() ) {+            in->clear();+            for ( size_t i=head.size(); i>0; --i ) {+                if ( !in->putback ( head[i-1] ) ) break;+            }+        }+        in->clear();+        return false;+    }++    // a UTF-16LE BOM may have been followed by a NUL character matching the UTF-32LE pattern+    string content = head.substr ( found->length );+    std::ostringstream rest;+    rest << in->rdbuf();+    content += rest.str();++    if ( in != &cin ) delete in;+    in = new istringstream ( StringTools::utfWideToUtf8 ( content, found->unitSize, found->bigEndian ) );++    if ( StringTools::change_case ( encoding ) != "none" )+        setEncoding ( "utf-8" );++    return true;+}+ bool CodeGenerator::validateInputStream() {     if ( !in ) return false;@@ -1370,6 +1441,9 @@ ParseError CodeGenerator::generateFile ( const string &inFileName,      in = ( inFileName.empty() ? &cin :new ifstream ( inFileName.c_str() ) ); +    const string savedEncoding = encoding;+    convertUnicodeInput();+     if ( validateInput )         if ( !validateInputStream() ) error= BAD_INPUT; @@ -1396,10 +1470,14 @@ ParseError CodeGenerator::generateFile ( const string &inFileName,         delete out;         out=nullptr;     }-    if ( !inFileName.empty() ) {+    if ( in != &cin ) {         delete in;-        in=nullptr;     }+    in=nullptr;++    if ( encoding != savedEncoding )+        setEncoding ( savedEncoding );+     return error; } @@ -1474,7 +1552,11 @@ string CodeGenerator::generateStringFromFile ( const string &inFileName )         return "";     } +    const string savedEncoding = encoding;+    convertUnicodeInput();+     if ( validateInput && !validateInputStream() ) {+        setEncoding ( savedEncoding );         return "ERROR: detected binary input";     } @@ -1493,6 +1575,9 @@ string CodeGenerator::generateStringFromFile ( const string &inFileName )     delete in;     in=nullptr; +    if ( encoding != savedEncoding )+        setEncoding ( savedEncoding );+     return result; } 

src/core/stringtools.cpp

@@ -131,6 +131,71 @@ bool endsWith(std::string const & value, std::string const & ending)     return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); } +static void appendUtf8(string& out, unsigned long cp)+{+    if (cp < 0x80) {+        out += (char) cp;+    } else if (cp < 0x800) {+        out += (char) (0xC0 | (cp >> 6));+        out += (char) (0x80 | (cp & 0x3F));+    } else if (cp < 0x10000) {+        out += (char) (0xE0 | (cp >> 12));+        out += (char) (0x80 | ((cp >> 6) & 0x3F));+        out += (char) (0x80 | (cp & 0x3F));+    } else {+        out += (char) (0xF0 | (cp >> 18));+        out += (char) (0x80 | ((cp >> 12) & 0x3F));+        out += (char) (0x80 | ((cp >> 6) & 0x3F));+        out += (char) (0x80 | (cp & 0x3F));+    }+}++string utfWideToUtf8(const string& bytes, unsigned int unitSize, bool bigEndian)+{+    const unsigned long replacement = 0xFFFD;+    string result;+    result.reserve(bytes.size());++    auto readUnit = [&](size_t pos) {+        unsigned long unit = 0;+        for (unsigned int i=0; i<unitSize; i++) {+            unsigned char b = (unsigned char) bytes[pos + (bigEndian ? i : unitSize-1-i)];+            unit = (unit << 8) | b;+        }+        return unit;+    };++    size_t pos = 0;+    while (pos + unitSize <= bytes.size()) {+        unsigned long cp = readUnit(pos);+        pos += unitSize;++        if (unitSize == 2 && cp >= 0xD800 && cp <= 0xDBFF) {+            // high surrogate, needs a following low surrogate+            if (pos + 2 <= bytes.size()) {+                unsigned long low = readUnit(pos);+                if (low >= 0xDC00 && low <= 0xDFFF) {+                    cp = 0x10000 + ((cp - 0xD800) << 10) + (low - 0xDC00);+                    pos += 2;+                } else {+                    cp = replacement;+                }+            } else {+                cp = replacement;+            }+        } else if ((cp >= 0xD800 && cp <= 0xDFFF) || cp > 0x10FFFF) {+            cp = replacement;+        }+        appendUtf8(result, cp);+    }++    // truncated trailing code unit+    if (pos < bytes.size())+        appendUtf8(result, replacement);++    return result;+}+ int utf8_strlen(const string& str) {     int c,i,ix,q;

src/include/codegenerator.h

@@ -987,6 +987,11 @@ private:     /** \return true if input is no binary stream */     bool validateInputStream(); +    /** replace UTF-16 or UTF-32 input (detected by BOM) by a UTF-8 stream+        and switch the output encoding to UTF-8+        \return true if the input was converted */+    bool convertUnicodeInput();+     /** load syntax description of embedded snippet's language      \param embedLangDefPath path to language definition       \return true if successful */

src/include/stringtools.h

@@ -87,6 +87,14 @@ bool endsWith(std::string const & value, std::string const & ending);  int utf8_strlen(const std::string& str); +/** Convert UTF-16 or UTF-32 encoded bytes (without BOM) to UTF-8.+    Invalid code units are replaced by U+FFFD.+    \param bytes input bytes+    \param unitSize 2 for UTF-16, 4 for UTF-32+    \param bigEndian true if code units are big endian+    \return UTF-8 string */+std::string utfWideToUtf8(const std::string& bytes, unsigned int unitSize, bool bigEndian);+ std::string sanitizeForTerminal(std::string_view s); }