Changes to src/core/codegenerator.cpp. All files

src/core/codegenerator.cpp

@@ -1211,6 +1211,77 @@ LoadResult CodeGenerator::loadLanguage ( const string& langDefPath, bool embedde     return result; } +bool CodeGenerator::convertUnicodeInput()+{+    if ( !in || in->fail() ) return false;++    // UTF-32LE has to be tested before UTF-16LE (same first two bytes)+    struct UnicodeBom {+        const char* magic;+        size_t length;+        unsigned int unitSize;+        bool bigEndian;+    };+    const UnicodeBom boms[] = {+        { "\xFF\xFE\x00\x00", 4, 4, false },+        { "\x00\x00\xFE\xFF", 4, 4, true },+        { "\xFF\xFE", 2, 2, false },+        { "\xFE\xFF", 2, 2, true },+    };++    // read the first bytes sequentially, so that this works with stdin too+    string head;+    while ( head.size() < 4 ) {+        int c = in->peek();+        if ( c == EOF ) break;+        // stop as soon as no BOM can match anymore+        bool candidate = false;+        for ( const UnicodeBom& bom : boms ) {+            if ( head.size() < bom.length && head.compare ( 0, head.size(), bom.magic, head.size() ) == 0+                    && (char) c == bom.magic[head.size()] ) {+                candidate = true;+                break;+            }+        }+        if ( !candidate ) break;+        head += (char) in->get();+    }++    const UnicodeBom* found = nullptr;+    for ( const UnicodeBom& bom : boms ) {+        if ( head.size() >= bom.length && head.compare ( 0, bom.length, bom.magic, bom.length ) == 0 ) {+            found = &bom;+            break;+        }+    }++    if ( !found ) {+        // nothing consumed except for partial BOM candidates; restore them+        if ( !head.empty() ) {+            in->clear();+            for ( size_t i=head.size(); i>0; --i ) {+                if ( !in->putback ( head[i-1] ) ) break;+            }+        }+        in->clear();+        return false;+    }++    // a UTF-16LE BOM may have been followed by a NUL character matching the UTF-32LE pattern+    string content = head.substr ( found->length );+    std::ostringstream rest;+    rest << in->rdbuf();+    content += rest.str();++    if ( in != &cin ) delete in;+    in = new istringstream ( StringTools::utfWideToUtf8 ( content, found->unitSize, found->bigEndian ) );++    if ( StringTools::change_case ( encoding ) != "none" )+        setEncoding ( "utf-8" );++    return true;+}+ bool CodeGenerator::validateInputStream() {     if ( !in ) return false;@@ -1370,6 +1441,9 @@ ParseError CodeGenerator::generateFile ( const string &inFileName,      in = ( inFileName.empty() ? &cin :new ifstream ( inFileName.c_str() ) ); +    const string savedEncoding = encoding;+    convertUnicodeInput();+     if ( validateInput )         if ( !validateInputStream() ) error= BAD_INPUT; @@ -1396,10 +1470,14 @@ ParseError CodeGenerator::generateFile ( const string &inFileName,         delete out;         out=nullptr;     }-    if ( !inFileName.empty() ) {+    if ( in != &cin ) {         delete in;-        in=nullptr;     }+    in=nullptr;++    if ( encoding != savedEncoding )+        setEncoding ( savedEncoding );+     return error; } @@ -1474,7 +1552,11 @@ string CodeGenerator::generateStringFromFile ( const string &inFileName )         return "";     } +    const string savedEncoding = encoding;+    convertUnicodeInput();+     if ( validateInput && !validateInputStream() ) {+        setEncoding ( savedEncoding );         return "ERROR: detected binary input";     } @@ -1493,6 +1575,9 @@ string CodeGenerator::generateStringFromFile ( const string &inFileName )     delete in;     in=nullptr; +    if ( encoding != savedEncoding )+        setEncoding ( savedEncoding );+     return result; }