/***************************************************************************
codegenerator.cpp - description
-------------------
begin : Die Jul 9 2002
copyright : (C) 2002-2026 by Andre Simon
email : a.simon@mailbox.org
***************************************************************************/
/*
This file is part of Highlight.
Highlight is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Highlight is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Highlight. If not, see <http://www.gnu.org/licenses/>.
*/
#include <climits>
#include <memory>
#include <chrono>
#include <thread>
#include <boost/xpressive/xpressive_dynamic.hpp>
#include "codegenerator.h"
#include "htmlgenerator.h"
#include "xhtmlgenerator.h"
#include "rtfgenerator.h"
#include "latexgenerator.h"
#include "texgenerator.h"
#include "svggenerator.h"
#include "bbcodegenerator.h"
#include "pangogenerator.h"
#include "odtgenerator.h"
#include "astyle/astyle.h"
//#if !defined (QT)
#include "ansigenerator.h"
#include "xterm256generator.h"
//#endif
using std::cin;
using std::cout;
using std::ifstream;
using std::ios;
using std::istringstream;
using std::iterator;
using std::flush;
using std::make_pair;
using std::map;
using std::ofstream;
using std::ostream;
using std::ostringstream;
using std::right;
using std::setw;
using std::string;
using std::vector;
namespace highlight
{
const unsigned int CodeGenerator::NUMBER_BUILTIN_STATES = highlight::KEYWORD;
// must not start with kw, st, sm prefixes
const string CodeGenerator::STY_NAME_STD="def";
const string CodeGenerator::STY_NAME_STR="sng";
const string CodeGenerator::STY_NAME_NUM="num";
const string CodeGenerator::STY_NAME_SLC="slc";
const string CodeGenerator::STY_NAME_COM="com";
const string CodeGenerator::STY_NAME_ESC="esc";
const string CodeGenerator::STY_NAME_DIR="ppc"; //preprocessor
const string CodeGenerator::STY_NAME_DST="pps"; //preprocessor string
const string CodeGenerator::STY_NAME_LIN="lin";
const string CodeGenerator::STY_NAME_SYM="opt"; //operator
const string CodeGenerator::STY_NAME_IPL="ipl"; //interpolation
const string CodeGenerator::STY_NAME_HVR="hvr";
const string CodeGenerator::STY_NAME_ERR="err";
const string CodeGenerator::STY_NAME_ERM="erm";
const string CodeGenerator::STY_NAME_GUI="gui"; //indentation guide
vector<Diluculum::LuaFunction*> CodeGenerator::pluginChunks;
CodeGenerator * CodeGenerator::getInstance ( OutputType type )
{
CodeGenerator* generator=nullptr;
switch ( type ) {
case HTML:
generator = new HtmlGenerator();
break;
case XHTML:
generator = new XHtmlGenerator();
break;
case TEX:
generator = new TexGenerator ();
break;
case LATEX:
generator = new LatexGenerator();
break;
case RTF:
generator = new RtfGenerator ();
break;
case SVG:
generator = new SVGGenerator();
break;
case BBCODE:
generator = new BBCodeGenerator();
break;
case PANGO:
generator = new PangoGenerator();
break;
case ODTFLAT:
generator = new ODTGenerator();
break;
case ESC_ANSI:
generator = new AnsiGenerator();
break;
case ESC_XTERM256:
case ESC_TRUECOLOR:
generator = new Xterm256Generator();
generator->setESCTrueColor(type==ESC_TRUECOLOR);
break;
default:
break;
}
return generator;
}
CodeGenerator::CodeGenerator ( highlight::OutputType type )
:currentSyntax(nullptr),
in ( nullptr ),
out ( nullptr ),
encoding ( "none" ),
docTitle ( "Source file" ),
isUtf8Encoding ( false ),
maskWs ( false ),
excludeWs ( false ),
fragmentOutput ( false ),
keepInjections( false ),
showLineNumbers ( false ),
lineNumberFillZeroes ( false ),
printNewLines(true),
omitVersionComment(false),
isolateTags(false),
disableStyleCache(false),
baseFontSize("10"),
lineNumber ( 0 ),
lineNumberOffset ( 0 ),
currentState ( _UNKNOWN ),
currentKeywordClass ( 0 ),
includeStyleDef ( false ),
numberCurrentLine ( false ),
lineIndex ( 0 ),
lastLineLength( 0 ),
syntaxChangeIndex(UINT_MAX),
syntaxChangeLineNo(UINT_MAX),
lineNumberWidth ( 5 ),
startLineCnt( 1 ),
startLineCntCurFile( 1 ),
extraEOFChar( 255 ),
maxLineCnt ( UINT_MAX ),
skipLeadingCols ( 0 ),
lineHeadPending ( false ),
inputFilesCnt (0),
processedFilesCnt (0),
kwOffset(0),
noTrailingNewLine(0),
terminatingChar ( '\0' ),
formatter ( nullptr ),
streamIterator ( nullptr ),
formattingEnabled ( false ),
formattingPossible ( false ),
validateInput ( false ),
numberWrappedLines ( true ),
resultOfHook(false),
lineContainedTestCase(false),
lineContainedStmt(false),
applySyntaxTestCase(false),
toggleDynRawString(false),
lsEnableHoverRequests(false),
lsCheckSemanticTokens(false),
lsCheckSyntaxErrors(false),
keywordCase ( StringTools::CASE_UNCHANGED ),
eolDelimiter ('\n'),
outputType ( type )
{
}
CodeGenerator::~CodeGenerator()
{
delete formatter;
delete streamIterator;
resetSyntaxReaders();
for (unsigned int i=0; i<pluginChunks.size(); i++) {
delete pluginChunks[i];
}
pluginChunks.clear();
}
bool CodeGenerator::initTheme ( const string& themePath, bool loadSemanticStyles)
{
this->themePath=themePath;
bool loadOK = docStyle.load ( themePath, outputType, loadSemanticStyles );
initOutputTags();
return loadOK;
}
LSResult CodeGenerator::initLanguageServer ( const string& executable, const vector<string> &options,
const string& workspace, const string& syntax,
int delay, int logLevel, bool legacy )
{
if (LSPClient.isInitialized()) {
return LSResult::INIT_OK;
}
LSPClient.setLogging(logLevel>1);
LSPClient.setExecutable(executable);
LSPClient.setWorkspace(workspace);
LSPClient.setOptions(options);
LSPClient.setSyntax(syntax);
LSPClient.setInitializeDelay(delay);
LSPClient.setLegacyProtocol(legacy);
if (!LSPClient.connect()){
return LSResult::INIT_BAD_PIPE;
}
if (!LSPClient.runInitialize()){
return LSResult::INIT_BAD_REQUEST;
}
for (int i=0; i<docStyle.getSemanticTokenStyleCount();i++) {
currentSyntax->generateNewKWClass(i+1, "st");
}
LSPClient.runInitialized();
updateKeywordClasses();
return LSResult::INIT_OK;
}
bool CodeGenerator::lsOpenDocument(const string& fileName, const string & suffix){
lsDocumentPath = fileName;
return LSPClient.runDidOpen(fileName, suffix);
}
bool CodeGenerator::lsCloseDocument(const string& fileName, const string & suffix){
lsDocumentPath.clear();
return LSPClient.runDidClose(fileName, suffix);
}
bool CodeGenerator::lsAddSemanticInfo(const string& fileName, const string & suffix){
lsCheckSemanticTokens = LSPClient.runSemanticTokensFull(fileName);
return lsCheckSemanticTokens;
}
bool CodeGenerator::isHoverProvider(){
return LSPClient.isHoverProvider();
}
bool CodeGenerator::isSemanticTokensProvider(){
return LSPClient.isSemanticTokensProvider();
}
void CodeGenerator::lsAddHoverInfo(bool hover){
lsEnableHoverRequests = hover;
}
void CodeGenerator::lsAddSyntaxErrorInfo(bool error) {
lsCheckSyntaxErrors = error;
}
void CodeGenerator::exitLanguageServer () {
LSPClient.runShutdown();
LSPClient.runExit();
}
const string& CodeGenerator::getStyleName()
{
return themePath;
}
void CodeGenerator::setLineNumberWidth ( int w )
{
lineNumberWidth=w;
}
int CodeGenerator::getLineNumberWidth()
{
return lineNumberWidth;
}
void CodeGenerator::setPrintLineNumbers ( bool flag, unsigned int startCnt )
{
showLineNumbers=flag;
lineNumberOffset = startCnt-1;
}
bool CodeGenerator::getPrintLineNumbers()
{
return showLineNumbers;
}
void CodeGenerator::setPrintZeroes ( bool flag )
{
lineNumberFillZeroes=flag;
}
bool CodeGenerator::getPrintZeroes()
{
return lineNumberFillZeroes;
}
void CodeGenerator::setIncludeStyle ( bool flag )
{
includeStyleDef = flag;
}
void CodeGenerator::disableTrailingNL ( int flag )
{
noTrailingNewLine = flag;
}
void CodeGenerator::setStyleInputPath ( const string& path )
{
styleInputPath = path;
}
void CodeGenerator::setStyleOutputPath ( const string& path )
{
styleOutputPath = path;
}
void CodeGenerator::setPluginParameter ( const string& param )
{
pluginParameter = param;
}
const string& CodeGenerator::getStyleInputPath()
{
return styleInputPath;
}
const string& CodeGenerator::getStyleOutputPath()
{
return styleOutputPath;
}
void CodeGenerator::setFragmentCode ( bool flag )
{
fragmentOutput=flag;
}
bool CodeGenerator::getFragmentCode()
{
return fragmentOutput;
}
void CodeGenerator::setKeepInjections ( bool flag )
{
keepInjections=flag;
}
bool CodeGenerator::getKeepInjections()
{
return keepInjections;
}
void CodeGenerator::setValidateInput ( bool flag )
{
validateInput=flag;
}
bool CodeGenerator::getValidateInput()
{
return validateInput;
}
void CodeGenerator::setNumberWrappedLines ( bool flag )
{
numberWrappedLines=flag;
}
bool CodeGenerator::getNumberWrappedLines()
{
return numberWrappedLines;
}
void CodeGenerator::setOmitVersionComment ( bool flag )
{
omitVersionComment=flag;
}
bool CodeGenerator::getOmitVersionComment ()
{
return omitVersionComment;
}
void CodeGenerator::setIsolateTags ( bool flag )
{
isolateTags=flag;
}
bool CodeGenerator::getIsolateTags ()
{
return isolateTags;
}
void CodeGenerator::setBaseFont ( const string& fontName )
{
baseFont = fontName;
}
void CodeGenerator::setBaseFontSize ( const string& fontSize)
{
baseFontSize = fontSize;
}
void CodeGenerator::setStyleCaching ( bool flag )
{
disableStyleCache=!flag;
}
const string CodeGenerator::getBaseFont() const
{
if ( !baseFont.empty() ) return baseFont;
switch ( outputType ) {
case HTML:
case XHTML:
case SVG:
return "'Courier New',monospace";
break;
case LATEX:
return "ttfamily";
break;
case TEX:
return "tt";
break;
default:
return "Courier New";
}
}
const string CodeGenerator::getBaseFontSize()
{
return baseFontSize;
}
void CodeGenerator::setTitle ( const string & title )
{
if ( !title.empty() ) docTitle= title;
}
string CodeGenerator::getTitle()
{
return docTitle;
}
void CodeGenerator::setEncoding ( const string& encodingName )
{
encoding = encodingName;
isUtf8Encoding = encoding=="utf-8";
}
bool CodeGenerator::formattingDisabled()
{
return !formattingEnabled;
}
void CodeGenerator::setStartingInputLine ( unsigned int begin )
{
startLineCnt = startLineCntCurFile = begin;
}
void CodeGenerator::setMaxInputLineCnt ( unsigned int cnt )
{
maxLineCnt = cnt;
}
void CodeGenerator::setSkipLeadingCols ( unsigned int cols )
{
skipLeadingCols = cols;
}
void CodeGenerator::setFilesCnt ( unsigned int cnt )
{
inputFilesCnt = cnt;
processedFilesCnt = 0;
}
bool CodeGenerator::formattingIsPossible()
{
return formattingPossible;
}
unsigned char CodeGenerator::getAdditionalEOFChar()
{
return extraEOFChar;
}
void CodeGenerator::setAdditionalEOFChar ( unsigned char eofChar )
{
extraEOFChar = eofChar;
}
void CodeGenerator::setPreformatting ( WrapMode lineWrappingStyle,
unsigned int lineLength,
int numberSpaces )
{
bool enableWrap = lineWrappingStyle!=WRAP_DISABLED;
bool replaceTabs = numberSpaces > 0;
if ( enableWrap || replaceTabs ) {
preFormatter.setWrap ( enableWrap );
preFormatter.setWrapIndentBraces ( lineWrappingStyle==WRAP_DEFAULT );
preFormatter.setWrapLineLength ( lineLength );
preFormatter.setReplaceTabs ( replaceTabs );
preFormatter.setNumberSpaces ( numberSpaces );
}
}
void CodeGenerator::setKeyWordCase ( StringTools::KeywordCase keyCase )
{
keywordCase = keyCase;
}
void CodeGenerator::setEOLDelimiter(char delim)
{
eolDelimiter = delim;
}
void CodeGenerator::reset()
{
lookaheadLines.clear();
lookaheadEof = false;
lineIndex = 0;
lineNumber = 0;
line.clear();
preFormatter.reset();
inFile.clear();
outFile.clear();
embedLangDefPath.clear();
printNewLines=true;
syntaxChangeIndex = syntaxChangeLineNo = UINT_MAX;
startLineCntCurFile = startLineCnt;
applySyntaxTestCase=lineContainedTestCase=false;
if (currentSyntax){
vector<int> overrideStyleAttrs=currentSyntax->getOverrideStyleAttributes();
docStyle.overrideAttributes(overrideStyleAttrs);
if (overrideStyleAttrs.size())
disableStyleCache = true;
}
}
string CodeGenerator::getThemeInitError()
{
return docStyle.getErrorMessage();
}
string CodeGenerator::getPluginScriptError()
{
return userScriptError;
}
string CodeGenerator::getSyntaxRegexError()
{
return (currentSyntax)? currentSyntax->getFailedRegex(): "syntax undef";
}
string CodeGenerator::getSyntaxLuaError()
{
return (currentSyntax)? currentSyntax->getLuaErrorText(): "syntax undef";
}
string CodeGenerator::getSyntaxDescription()
{
return (currentSyntax)? currentSyntax->getDescription(): "syntax undef";
}
string CodeGenerator::getSyntaxEncodingHint()
{
return (currentSyntax)? currentSyntax->getEncodingHint(): "";
}
string CodeGenerator::getThemeDescription()
{
return docStyle.getDescription();
}
string CodeGenerator::getSyntaxCatDescription(){
return (currentSyntax)? currentSyntax->getCategoryDescription(): "";
}
string CodeGenerator::getThemeCatDescription()
{
return docStyle.getCategoryDescription();
}
float CodeGenerator::getThemeContrast()
{
return docStyle.getContrast();
}
unsigned int CodeGenerator::getLineNumber()
{
return lineNumber;
}
bool CodeGenerator::AtEnd(char c) const {
bool instream_eof = in->eof();
if (extraEOFChar == 255)
return instream_eof;
bool c_null = c == extraEOFChar;
bool instream_peek_null = false;
if (instream_eof == false && c_null == false)
instream_peek_null = in->peek() == extraEOFChar;
bool ret = instream_eof || c_null || instream_peek_null;
return ret;
}
bool CodeGenerator::fetchRawLine ( string &newLine )
{
if ( formattingPossible && formattingEnabled ) {
if ( !formatter->hasMoreLines() ) return true;
newLine = formatter->nextLine();
return false;
}
return AtEnd() || ! getline ( *in, newLine, eolDelimiter );
}
bool CodeGenerator::readLookahead ( string &newLine )
{
if ( lookaheadLines.empty() ) {
if ( lookaheadEof ) return true;
string bufferedLine;
if ( ( lookaheadEof = fetchRawLine ( bufferedLine ) ) ) return true;
lookaheadLines.push_back ( bufferedLine );
}
newLine = lookaheadLines.front();
lookaheadLines.pop_front();
return false;
}
unsigned int CodeGenerator::nextCodeIndent()
{
// bounded so that a long run of blank lines cannot buffer the whole input
for ( size_t i=0; i<MAX_LOOKAHEAD_LINES; ++i ) {
while ( i >= lookaheadLines.size() ) {
string bufferedLine;
if ( lookaheadEof || ( lookaheadEof = fetchRawLine ( bufferedLine ) ) ) return 0;
lookaheadLines.push_back ( bufferedLine );
}
const string &l = lookaheadLines[i];
size_t codeStart = l.find_first_not_of ( " \t" );
if ( codeStart == string::npos ) continue;
unsigned int tabWidth = preFormatter.getNumberSpaces(), indent=0;
for ( size_t p=0; p<codeStart; ++p )
indent += ( l[p]=='\t' ) ? tabWidth - ( indent % tabWidth ) : 1;
return indent;
}
return 0;
}
void CodeGenerator::setIndentGuides ( bool b )
{
showIndentGuides = b;
}
void CodeGenerator::initIndentGuides()
{
guidePrefix.clear();
guideSkipWs = 0;
guideColDelta = 0;
// wrapped continuation lines carry a synthetic alignment prefix, not real indentation
if ( !showIndentGuides || guideTagOpen.empty() || !numberCurrentLine ) return;
if ( guideGlyph.empty() )
guideGlyph = ( StringTools::change_case ( encoding ) == "utf-8" ) ? "\xE2\x94\x82" : "|";
size_t codeStart = line.find_first_not_of ( " \t" );
unsigned int indent;
if ( codeStart == string::npos ) {
// blank lines adopt the indentation of the next line which contains code
guideSkipWs = line.length();
indent = nextCodeIndent();
} else {
guideSkipWs = codeStart;
indent = codeStart;
}
unsigned int tabWidth = preFormatter.getNumberSpaces();
unsigned int guideCnt = indent / tabWidth;
if ( !guideCnt ) {
guideSkipWs = ( codeStart == string::npos ) ? line.length() : 0;
guideColDelta = -(int) guideSkipWs;
return;
}
string raw;
for ( unsigned int i=0; i<guideCnt; ++i )
raw += guideGlyph + string ( tabWidth-1, ' ' );
raw.append ( indent % tabWidth, ' ' );
// encode the glyph and spaces through the format's own escaping, so the
// guide does not collapse where literal whitespace would (latex, tex, odt)
string guides;
for ( unsigned char c : raw )
guides += maskCharacter ( c );
guidePrefix = guideTagOpen + guides + guideTagClose;
guideColDelta = (int) indent - (int) guideSkipWs;
}
bool CodeGenerator::readNewLine ( string &newLine )
{
bool eof=false;
if ( lineIndex ) terminatingChar=newLine[lineIndex-1];
while (!eof && startLineCntCurFile>0) {
eof = readLookahead ( newLine );
--startLineCntCurFile;
}
startLineCntCurFile=1;
#ifndef _WIN32
// drop CR of CRLF files
if (!newLine.empty() && newLine[newLine.size() - 1] == '\r')
newLine.erase(newLine.size() - 1);
#endif
// keep the leading columns out of the parser, so that quotes or comment
// delimiters within them cannot open a state which spans several lines
if ( skipLeadingCols ) {
string::size_type headEnd = 0;
for ( unsigned int col=0; col<skipLeadingCols && headEnd<newLine.size(); ++col ) {
// do not split a multi byte character
do {
++headEnd;
} while ( headEnd<newLine.size() && ( newLine[headEnd] & 0xC0 ) == 0x80 );
}
lineHead = newLine.substr ( 0, headEnd );
newLine.erase ( 0, headEnd );
lineHeadPending = true;
}
return eof || ( lineNumber == maxLineCnt );
}
void CodeGenerator::matchRegex ( const string &line, State skipState)
{
regexGroups.clear();
int matchBegin=0;
int groupID=0;
// cycle through all regex, save the start and ending indices of matches to report them later
for ( unsigned int i=0; i<currentSyntax->getRegexElements().size(); i++ ) {
RegexElement *regexElem = currentSyntax->getRegexElements() [i];
if (regexElem->open == skipState) continue;
if (regexElem->constraintLineNum && regexElem->constraintLineNum != lineNumber) {
continue;
}
if (regexElem->constraintFilename.size() && regexElem->constraintFilename != inFile) {
continue;
}
boost::xpressive::sregex_iterator cur( line.begin(), line.end(), regexElem->rex );
boost::xpressive::sregex_iterator end;
for( ; cur != end; ++cur ) {
groupID = ( regexElem->capturingGroup<0 ) ? cur->size()-1 : regexElem->capturingGroup;
matchBegin = cur->position(groupID);
// if another regex already claimed this start position, keep whichever
// match is longer (e.g. a raw string open delimiter like R"( must win
// over a single-character identifier match on the same 'R')
auto insertResult = regexGroups.insert (
make_pair ( matchBegin + 1, RegexToken ( regexElem->open, cur->length(groupID), regexElem->kwClass, regexElem->langName ) ) );
if ( !insertResult.second && insertResult.first->second.length < ( unsigned int ) cur->length ( groupID ) ) {
insertResult.first->second = RegexToken ( regexElem->open, cur->length(groupID), regexElem->kwClass, regexElem->langName );
}
// priority regex (match required)
if (regexElem->priority) {
return;
}
}
}
}
unsigned char CodeGenerator::getInputChar()
{
// end of line?
if ( lineIndex == line.length() ) {
//more testing required:
if (outputType==ESC_TRUECOLOR || outputType==ESC_XTERM256)
lastLineLength=StringTools::utf8_strlen(line + lsSyntaxErrorDesc) + guideColDelta;
bool eof=false, freshSourceLine=true;
if ( preFormatter.isEnabled() ) {
if ( !preFormatter.hasMoreLines() ) {
eof=readNewLine ( line );
preFormatter.setLine ( line );
++lineNumber;
numberCurrentLine = true;
} else {
freshSourceLine = false;
if (numberWrappedLines)
++lineNumber;
numberCurrentLine = numberWrappedLines;
}
line = preFormatter.getNextLine();
} else {
eof=readNewLine ( line );
++lineNumber;
numberCurrentLine = true;
}
lineIndex=0;
if (!lineContainedTestCase && applySyntaxTestCase){
stateTraceTest = stateTraceCurrent;
stateTraceCurrent.clear();
}
lineContainedTestCase=false;
lineContainedStmt=false;
if ( freshSourceLine ) initIndentGuides();
matchRegex ( line );
return ( eof ) ?'\0':'\n';
}
return line[lineIndex++];
}
/** changing this method requires regression testing with nested syntax files (HTML+PHP+JS+CSS,
* Coffeescript with block regex, Pas + ASM)
* especially nested syntax in one line
*/
State CodeGenerator::getCurrentState (State oldState)
{
unsigned char c='\0';
if ( token.length() ==0 ) {
c=getInputChar();
} else {
lineIndex-= ( token.length()-1 );
c=token[0];
}
if ( c=='\n' ) {
return _EOL; // End of line
}
if ( c=='\0' ) {
return _EOF; // End of file
}
if ( c==' ' || c=='\t' ) {
token= c;
return _WS; // White space
}
if ( applySyntaxTestCase && ( c=='^' || c=='<') && (oldState == ML_COMMENT || oldState==SL_COMMENT) ) {
token= c;
return _TESTPOS;
}
// at this position the syntax change takes place
if (lineIndex >= syntaxChangeIndex-1 || syntaxChangeLineNo < lineNumber){
loadEmbeddedLang(embedLangDefPath); // load new syntax
matchRegex(line); // recognize new patterns in the (remaining) line
syntaxChangeIndex = syntaxChangeLineNo = UINT_MAX;
}
SKIP_EMBEDDED:
if (lsCheckSyntaxErrors && LSPClient.errorExists(lineNumber, lineIndex)) {
highlight::SemanticToken errorToken = LSPClient.getError(lineNumber, lineIndex);
token = line.substr ( lineIndex-1, errorToken.length);
lineIndex += errorToken.length-1;
lsSyntaxErrorDesc = errorToken.id;
//std::cerr <<"error num "<<lineNumber<< " idx "<<lineIndex<< " error "<<errorToken.id<< "\n";
return SYNTAX_ERROR;
}
if (lsCheckSemanticTokens && LSPClient.tokenExists(lineNumber, lineIndex)) {
highlight::SemanticToken semToken = LSPClient.getToken(lineNumber, lineIndex);
int semStyleKwId = docStyle.getSemanticStyle(semToken.id);
if (semStyleKwId) {
token = line.substr ( lineIndex-1, semToken.length);
lineIndex += semToken.length-1;
currentKeywordClass = semStyleKwId + kwOffset; // +offset of missing kw groups in the theme
//std::cerr <<"l "<<lineNumber<< "t "<<token<< " semStyleKwId "<< semStyleKwId << " off "<<kwOffset<<" -> " << semToken.id <<"\n";
return KEYWORD;
}
}
// Test if a regular expression was found at the current position
if ( !regexGroups.empty() ) {
if ( regexGroups.count ( lineIndex ) ) {
token = line.substr ( lineIndex-1, regexGroups[lineIndex].length );
unsigned int oldIndex= lineIndex;
if ( regexGroups[oldIndex].length>1 ) lineIndex+= regexGroups[oldIndex].length-1;
if ( regexGroups[oldIndex].state==EMBEDDED_CODE_BEGIN ) {
//do not handle a nested section if the syntax is marked as "sealed"
if (embedLangDefPath.length()==0 || currentSyntax->allowsInnerSection(embedLangDefPath) ) {
embedLangDefPath = currentSyntax->getNewPath(regexGroups[oldIndex].name);
//remember position
syntaxChangeIndex = lineIndex+2;
syntaxChangeLineNo = lineNumber;
}
// repeat parsing of this line without nested state recognition to highlight opening delimiter in the host syntax
matchRegex(line, EMBEDDED_CODE_BEGIN);
lineIndex = oldIndex;
goto SKIP_EMBEDDED; // this is how it should be done
}
if ( regexGroups[oldIndex].state==IDENTIFIER_BEGIN || regexGroups[oldIndex].state==KEYWORD ) {
string reservedWord= ( currentSyntax->isIgnoreCase() ) ? StringTools::change_case ( token ) :token;
currentKeywordClass=currentSyntax->getKeywordListGroup ( reservedWord ); //check in lists (no regex)
if ( !currentKeywordClass && regexGroups[oldIndex].state==KEYWORD ){
currentKeywordClass = regexGroups[oldIndex].kwClass;
}
return validateState(( currentKeywordClass ) ? KEYWORD : STANDARD, oldState );
} else {
return validateState(regexGroups[oldIndex].state, oldState);
}
}
}
// Character not referring to any state
token = c;
return STANDARD;
}
State CodeGenerator::validateState(State newState, State oldState)
{
if (currentSyntax->getValidateStateChangeFct()) {
Diluculum::LuaValueList params;
params.emplace_back(oldState);
params.emplace_back(newState);
params.emplace_back(token);
params.emplace_back(getCurrentKeywordClassId() );
params.emplace_back(lineNumber );
params.emplace_back(lineIndex-(unsigned int)token.length() );
Diluculum::LuaValueList res=
currentSyntax->getLuaState()->call ( *currentSyntax->getValidateStateChangeFct(),
params,"getValidateStateChangeFct call") ;
resultOfHook = res.size()>=1;
if (resultOfHook) {
setOverrideParams();
auto validatedState = (State)res[0].asInteger();
if ( validatedState== _REJECT) {
// proceed using only the first character of the token
if (res.size()==1) {
lineIndex -= (token.length() -1);
token=token.substr(0, 1);
}
//experimental for slim.lang: evaluate second return arg after _REJECT
if (res.size()>=2) {
lineIndex -= (token.length() );
token.clear();
return (State)res[1].asInteger();
}
return oldState;
}
return validatedState;
}
}
resultOfHook = false;
return newState;
}
unsigned int CodeGenerator::getCurrentKeywordClassId(){
unsigned int kwClassId=0;
// this vector contains the defined keyword classes, and currentKeywordClass is its index:
const vector<string>& kwClasses=currentSyntax->getKeywordClasses();
if (currentKeywordClass && currentKeywordClass<=kwClasses.size()) {
string kwClassName=kwClasses[currentKeywordClass-1];
if (kwClassName.size()==3)
kwClassId = kwClassName[2] - 'a' + 1;
}
return kwClassId;
}
//it is faster to pass ostream reference
void CodeGenerator::maskString ( ostream& ss, const string & s )
{
string escHoverText;
if (lsEnableHoverRequests && (currentState==STANDARD || currentState==NUMBER || currentState==KEYWORD)) {
string hoverText = LSPClient.runHover(lsDocumentPath, lineIndex - s.size(), lineNumber-1);
for(const auto &c : hoverText)
{
if (isascii(c))
escHoverText.append(maskCharacter(c));
}
}
if (escHoverText.size()) {
ss << getHoverTagOpen(escHoverText);
}
string maskedText;
maskedText.reserve(s.size());
for (const auto &c : s)
{
maskedText.append(maskCharacter ( c ));
}
ss << maskedText;
if (escHoverText.size()) {
ss << getHoverTagClose();
}
// The test markers position should also be deternmined by calculating the code points
if ( applySyntaxTestCase ) {
PositionState ps(currentState, getCurrentKeywordClassId(), false);
int slen = isUtf8Encoding ? StringTools::utf8_strlen(s) : s.length();
for (int i=0; i< slen; i++ ) {
stateTraceCurrent.push_back(ps);
}
if (stateTraceCurrent.size()>200)
stateTraceCurrent.erase(stateTraceCurrent.begin(), stateTraceCurrent.begin() + 100 );
}
}
void CodeGenerator::printSyntaxError ( ostream& ss ) {
if ( !lsSyntaxErrorDesc.empty()) {
ss << openTags[ highlight::SYNTAX_ERROR_MSG ];
for(const auto &c : lsSyntaxErrorDesc)
{
ss << maskCharacter ( c );
}
ss << closeTags[ highlight::SYNTAX_ERROR_MSG ];
lsSyntaxErrorDesc.clear();
}
}
Diluculum::LuaValueList CodeGenerator::callDecorateFct(const string& token)
{
Diluculum::LuaValueList params;
params.emplace_back(token);
params.emplace_back(currentState);
params.emplace_back(currentKeywordClass);
params.emplace_back(lineContainedStmt);
params.emplace_back(lineNumber );
params.emplace_back(lineIndex-(unsigned int)token.length() );
return currentSyntax->getLuaState()->call ( *currentSyntax->getDecorateFct(),
params,"getDecorateFct call") ;
}
void CodeGenerator::printMaskedToken (bool flushWhiteSpace, StringTools::KeywordCase tcase )
{
if ( flushWhiteSpace )
flushWs(1);
string caseToken = StringTools::change_case ( token, tcase );
if (currentSyntax->getDecorateFct()) {
Diluculum::LuaValueList res=callDecorateFct(caseToken);
if (res.size()==1) {
*out<<res[0].asString();
} else {
maskString ( *out, caseToken );
}
} else {
maskString ( *out, caseToken );
}
// check this *after* the decorate call
if ( currentState == STANDARD || currentState == KEYWORD || currentState == NUMBER
|| currentState == STRING || currentState == IDENTIFIER_BEGIN) {
lineContainedStmt = true;
}
token.clear();
}
bool CodeGenerator::styleFound()
{
return docStyle.found();
}
bool CodeGenerator::printIndexFile ( const vector<string> &fileList, const string &outPath )
{
return true;
}
bool CodeGenerator::initIndentationScheme ( const string &indentScheme )
{
if ( formatter!=nullptr ) {
return true;
}
if ( !indentScheme.size() ) return false;
formatter=new astyle::ASFormatter();
if ( indentScheme=="allman" || indentScheme=="bsd" || indentScheme=="ansi" ) {
formatter->setFormattingStyle ( astyle::STYLE_ALLMAN );
} else if ( indentScheme=="kr"||indentScheme=="k&r"||indentScheme=="k/r" ) {
formatter->setFormattingStyle ( astyle::STYLE_KR );
} else if ( indentScheme=="java" ) {
formatter->setFormattingStyle ( astyle::STYLE_JAVA );
} else if ( indentScheme=="stroustrup" ) {
formatter->setFormattingStyle ( astyle::STYLE_STROUSTRUP );
} else if ( indentScheme=="whitesmith" ) {
formatter->setFormattingStyle ( astyle::STYLE_WHITESMITH );
} else if ( indentScheme=="banner" || indentScheme=="ratliff") {
formatter->setFormattingStyle ( astyle::STYLE_RATLIFF );
} else if ( indentScheme=="gnu" ) {
formatter->setFormattingStyle ( astyle::STYLE_GNU );
} else if ( indentScheme=="linux" ) {
formatter->setFormattingStyle ( astyle::STYLE_LINUX );
} else if ( indentScheme=="horstmann" ) {
formatter->setFormattingStyle ( astyle::STYLE_HORSTMANN );
} else if ( indentScheme=="otbs" || indentScheme=="1tbs") {
formatter->setFormattingStyle ( astyle::STYLE_1TBS );
} else if ( indentScheme=="google") {
formatter->setFormattingStyle ( astyle::STYLE_GOOGLE );
} else if ( indentScheme=="pico" || indentScheme=="a11") {
formatter->setFormattingStyle ( astyle::STYLE_PICO );
} else if ( indentScheme=="lisp" || indentScheme=="python"|| indentScheme=="a12") {
formatter->setFormattingStyle ( astyle::STYLE_LISP );
} else if ( indentScheme=="vtk") {
formatter->setFormattingStyle ( astyle::STYLE_VTK );
} else if ( indentScheme=="mozilla") {
formatter->setFormattingStyle ( astyle::STYLE_MOZILLA );
} else if ( indentScheme=="webkit") {
formatter->setFormattingStyle ( astyle::STYLE_WEBKIT );
} else if ( indentScheme!="user" ){
return false;
}
return formattingEnabled=true;
}
LoadResult CodeGenerator::loadLanguage ( const string& langDefPath, bool embedded )
{
if (!embedded) {
while (!nestedLangs.empty()) {
nestedLangs.pop();
}
}
bool reloadNecessary= currentSyntax ? currentSyntax->needsReload ( langDefPath ): true;
LoadResult result=LOAD_OK;
if ( reloadNecessary ) {
if (syntaxReaders.count(langDefPath)) {
currentSyntax=syntaxReaders[langDefPath];
result=LOAD_OK;
} else {
currentSyntax=new SyntaxReader();
result=currentSyntax->load(langDefPath, pluginParameter, outputType);
syntaxReaders[langDefPath]=currentSyntax;
}
if ( result==LOAD_OK ) {
formattingPossible=currentSyntax->enableReformatting();
updateKeywordClasses();
}
}
kwOffset=currentSyntax->getKeywordCount() - docStyle.getKeywordStyleCount();
return result;
}
bool CodeGenerator::convertUnicodeInput()
{
if ( !in || in->fail() ) return false;
// UTF-32LE has to be tested before UTF-16LE (same first two bytes)
struct UnicodeBom {
const char* magic;
size_t length;
unsigned int unitSize;
bool bigEndian;
};
const UnicodeBom boms[] = {
{ "\xFF\xFE\x00\x00", 4, 4, false },
{ "\x00\x00\xFE\xFF", 4, 4, true },
{ "\xFF\xFE", 2, 2, false },
{ "\xFE\xFF", 2, 2, true },
};
// read the first bytes sequentially, so that this works with stdin too
string head;
while ( head.size() < 4 ) {
int c = in->peek();
if ( c == EOF ) break;
// stop as soon as no BOM can match anymore
bool candidate = false;
for ( const UnicodeBom& bom : boms ) {
if ( head.size() < bom.length && head.compare ( 0, head.size(), bom.magic, head.size() ) == 0
&& (char) c == bom.magic[head.size()] ) {
candidate = true;
break;
}
}
if ( !candidate ) break;
head += (char) in->get();
}
const UnicodeBom* found = nullptr;
for ( const UnicodeBom& bom : boms ) {
if ( head.size() >= bom.length && head.compare ( 0, bom.length, bom.magic, bom.length ) == 0 ) {
found = &bom;
break;
}
}
if ( !found ) {
// nothing consumed except for partial BOM candidates; restore them
if ( !head.empty() ) {
in->clear();
for ( size_t i=head.size(); i>0; --i ) {
if ( !in->putback ( head[i-1] ) ) break;
}
}
in->clear();
return false;
}
// a UTF-16LE BOM may have been followed by a NUL character matching the UTF-32LE pattern
string content = head.substr ( found->length );
std::ostringstream rest;
rest << in->rdbuf();
content += rest.str();
if ( in != &cin ) delete in;
in = new istringstream ( StringTools::utfWideToUtf8 ( content, found->unitSize, found->bigEndian ) );
if ( StringTools::change_case ( encoding ) != "none" )
setEncoding ( "utf-8" );
return true;
}
bool CodeGenerator::validateInputStream()
{
if ( !in ) return false;
// it is not possible to move stream pointer back with stdin
if ( ( int ) in->tellg() == -1 ) // -1 : stdin
return true;
// Sources: http://en.wikipedia.org/wiki/Magic_number_(programming)
// Magic configuration of "file"
// This is intended for web plugins - only check filetypes often found in the net
char magic_gif[] = {'G','I','F','8', 0};
char magic_png[] = {'\x89','P','N','G', 0};
char magic_java[] = {'\xCA','\xFE','\xBA','\xBE', 0};
char magic_jpeg[] = {'\xFF','\xD8','\xFF', 0};
char magic_bmp[] = {'B','M', 0};
char magic_pdf[] = {'%','P','D','F', 0};
char magic_utf8[] = {'\xEF','\xBB','\xBF',0};
char magic_rar[] = {'R','a','r','!', 0};
char magic_zip[] = {'P','K','\x03','\x04', 0};
char magic_ace[] = {'*','*','A','C','E','*','*', 0};
char magic_tgz[] = {'\x8b','\x1f', '\x00', '\x08', 0};
char magic_bzip[] = {'B','Z', 0};
char* magic_table[] = {magic_utf8,
magic_gif, magic_png, magic_jpeg, magic_bmp, magic_pdf,
magic_java,
magic_rar, magic_zip, magic_ace, magic_tgz, magic_bzip,
nullptr
};
char buffer [10]= {0};
in->read ( buffer,8 ); //only read the first 8 bytes of input stream
int magic_index=0;
while ( magic_table[magic_index] ) {
if ( !strncmp ( buffer, magic_table[magic_index], strlen ( magic_table[magic_index] ) ) ) {
break;
}
magic_index++;
}
int streamReadPos=0;
if ( magic_table[magic_index] == magic_utf8 ) {
//setEncoding("utf-8");
streamReadPos=3; // remove UTF-8 magic number from output
}
in -> seekg ( streamReadPos, ios::beg );
in-> clear(); // clear fail bit to continue reading
return !magic_table[magic_index] // points to 0 if no pattern was found
|| magic_table[magic_index] == magic_utf8;
}
void CodeGenerator::applyPluginChunk(const string& fctName, string *result, bool *keepDefault) {
if (currentSyntax && pluginChunks.size()) {
Diluculum::LuaState luaState;
Diluculum::LuaValueList chunkParams;
chunkParams.emplace_back(currentSyntax->getDescription());
for (unsigned int i=0; i<pluginChunks.size(); i++) {
luaState.call(*pluginChunks[i], chunkParams, "format user function");
}
if (luaState.globals().count(fctName)) {
auto* documentFct=new Diluculum::LuaFunction(luaState[fctName].value().asFunction());
luaState["HL_PLUGIN_PARAM"] = pluginParameter;
luaState["HL_OUTPUT"] = outputType;
luaState["HL_FORMAT_HTML"]=HTML;
luaState["HL_FORMAT_XHTML"]=XHTML;
luaState["HL_FORMAT_TEX"]=TEX;
luaState["HL_FORMAT_LATEX"]=LATEX;
luaState["HL_FORMAT_RTF"]=RTF;
luaState["HL_FORMAT_ANSI"]=ESC_ANSI;
luaState["HL_FORMAT_XTERM256"]=ESC_XTERM256;
luaState["HL_FORMAT_TRUECOLOR"]=ESC_TRUECOLOR;
luaState["HL_FORMAT_SVG"]=SVG;
luaState["HL_FORMAT_BBCODE"]=BBCODE;
luaState["HL_FORMAT_PANGO"]=PANGO;
luaState["HL_FORMAT_ODT"]=ODTFLAT;
Diluculum::LuaValueList params;
Diluculum::LuaValueMap options;
options[Diluculum::LuaValue("title")] = Diluculum::LuaValue( docTitle );
options[Diluculum::LuaValue("encoding")] = Diluculum::LuaValue(encoding);
options[Diluculum::LuaValue("fragment")] = Diluculum::LuaValue(fragmentOutput);
options[Diluculum::LuaValue("font")] = Diluculum::LuaValue(getBaseFont());
options[Diluculum::LuaValue("fontsize")] = Diluculum::LuaValue(getBaseFontSize());
params.emplace_back(inputFilesCnt);
params.emplace_back(processedFilesCnt);
params.emplace_back(options);
Diluculum::LuaValueList res=luaState.call ( *documentFct, params, fctName+" call");
if (res.size()>=1) {
*keepDefault=false;
*result = res[0].asString();
if (res.size()==2)
*keepDefault = res[1].asBoolean();
}
delete documentFct;
}
}
}
void CodeGenerator::printHeader()
{
bool keepDefaultHeader=true;
string pluginHeader;
processedFilesCnt++;
applyPluginChunk("DocumentHeader", &pluginHeader, &keepDefaultHeader);
if ( ! fragmentOutput && keepDefaultHeader)
*out << getHeader();
*out << pluginHeader;
if ( !fragmentOutput || keepInjections)
*out << currentSyntax->getHeaderInjection();
}
void CodeGenerator::printFooter()
{
bool keepDefaultFooter=true;
string pluginFooter;
applyPluginChunk("DocumentFooter", &pluginFooter, &keepDefaultFooter);
if ( !fragmentOutput || keepInjections)
*out << currentSyntax->getFooterInjection();
*out << pluginFooter;
if ( ! fragmentOutput && keepDefaultFooter )
*out << getFooter();
}
ParseError CodeGenerator::generateFile ( const string &inFileName,
const string &outFileName )
{
if ( !docStyle.found() ) {
return BAD_STYLE;
}
reset();
ParseError error=PARSE_OK;
inFile=inFileName;
outFile=outFileName;
in = ( inFileName.empty() ? &cin :new ifstream ( inFileName.c_str() ) );
const string savedEncoding = encoding;
convertUnicodeInput();
if ( validateInput )
if ( !validateInputStream() ) error= BAD_INPUT;
if ( !in->fail() && error==PARSE_OK ) {
out = ( outFileName.empty() ? &cout :new ofstream ( outFileName.c_str() ) );
if ( out->fail() ) {
error=BAD_OUTPUT;
}
}
if ( in->fail() ) {
error=BAD_INPUT;
}
if ( error==PARSE_OK ) {
initASStream();
currentSyntax->setInputFileName(inFile);
printHeader();
printBody();
printFooter();
}
if ( !outFileName.empty() ) {
delete out;
out=nullptr;
}
if ( in != &cin ) {
delete in;
}
in=nullptr;
if ( encoding != savedEncoding )
setEncoding ( savedEncoding );
return error;
}
string CodeGenerator::generateString ( const string &input )
{
if ( !docStyle.found() ) {
return "";
}
reset();
in = new istringstream ( input );
out = new ostringstream ();
if ( in->fail() || out->fail() ) {
return "";
}
initASStream();
printHeader();
printBody();
printFooter();
string result = static_cast<ostringstream*> ( out )->str();
delete out;
out=nullptr;
delete in;
in=nullptr;
return result;
}
void CodeGenerator::initASStream() {
if ( formatter != nullptr ) {
if (streamIterator) delete streamIterator;
streamIterator = new astyle::ASStreamIterator ( in, extraEOFChar );
formatter->init ( streamIterator );
string desc = currentSyntax->getDescription();
if (desc=="C#") {
formatter->setSharpStyle();
} else if (desc=="Java") {
formatter->setJavaStyle();
} else if (desc=="Javascript") {
formatter->setJSStyle();
} else if (desc=="Objective C") {
formatter->setObjCStyle();
} else {
formatter->setCStyle();
}
}
}
string CodeGenerator::generateStringFromFile ( const string &inFileName )
{
if ( !docStyle.found() ) {
return "";
}
reset();
inFile = inFileName;
in = new ifstream ( inFileName.c_str() );
out = new ostringstream ();
if ( in->fail() || out->fail() ) {
return "";
}
const string savedEncoding = encoding;
convertUnicodeInput();
if ( validateInput && !validateInputStream() ) {
setEncoding ( savedEncoding );
return "ERROR: detected binary input";
}
initASStream();
currentSyntax->setInputFileName(inFile);
printHeader();
printBody();
printFooter();
string result = static_cast<ostringstream*> ( out )->str();
delete out;
out=nullptr;
delete in;
in=nullptr;
if ( encoding != savedEncoding )
setEncoding ( savedEncoding );
return result;
}
unsigned int CodeGenerator::getStyleID ( State s, unsigned int kwClassID )
{
if ( s==KEYWORD && kwClassID ) {
return NUMBER_BUILTIN_STATES + kwClassID-1;
}
return ( unsigned int ) s ;
}
void CodeGenerator::openTag ( State s )
{
*out << openTags[ ( unsigned int ) s];
currentState=s;
}
void CodeGenerator::closeTag ( State s )
{
*out << closeTags[ ( unsigned int ) s];
flushWs(2);
currentState=_UNKNOWN;
}
void CodeGenerator::openKWTag ( unsigned int kwClassID )
{
*out << openTags.at(getStyleID ( KEYWORD, kwClassID ) );
currentState=KEYWORD;
}
bool CodeGenerator::loadEmbeddedLang(const string&embedLangDefPath)
{
if (nestedLangs.empty()) {
nestedLangs.push(currentSyntax->getCurrentPath() );
}
if (nestedLangs.top() != embedLangDefPath) {
nestedLangs.push(embedLangDefPath);
}
LoadResult res = loadLanguage(embedLangDefPath, true);
//pass end delimiter regex to syntax description
currentSyntax->restoreLangEndDelim(embedLangDefPath);
return res == LOAD_OK;
}
///////////////////////////////////////////////////////////////////////////////
void CodeGenerator::processRootState()
{
bool eof=false,
firstLine=true; // avoid newline before printing the first output line
applySyntaxTestCase = inFile.find("syntax_test_")!=string::npos;
if ( currentSyntax->highlightingDisabled() ) {
string line;
while ( getline ( *in, line ) && lineNumber < maxLineCnt ) {
++lineNumber;
insertLineNumber ( !firstLine );
flushWs(4);
firstLine=false;
if (lineNumber>=startLineCntCurFile && lineNumber <=maxLineCnt)
maskString ( *out, line );
}
*out << flush;
return;
}
State state=STANDARD;
openTag ( STANDARD );
do {
// determine next state
state= getCurrentState(STANDARD);
// handle current state
switch ( state ) {
case KEYWORD:
closeTag ( STANDARD );
eof=processKeywordState ( state );
openTag ( STANDARD );
break;
case NUMBER:
closeTag ( STANDARD );
eof=processNumberState();
openTag ( STANDARD );
break;
case ML_COMMENT:
closeTag ( STANDARD );
eof=processMultiLineCommentState();
openTag ( STANDARD );
break;
case SL_COMMENT:
closeTag ( STANDARD );
eof=processSingleLineCommentState();
openTag ( STANDARD );
break;
case STRING:
closeTag ( STANDARD );
eof=processStringState ( STANDARD );
openTag ( STANDARD );
break;
case DIRECTIVE:
closeTag ( STANDARD );
eof=processDirectiveState();
openTag ( STANDARD );
break;
case ESC_CHAR:
closeTag ( STANDARD );
eof=processEscapeCharState();
openTag ( STANDARD );
break;
case SYMBOL:
closeTag ( STANDARD );
eof=processSymbolState();
openTag ( STANDARD );
break;
case EMBEDDED_CODE_END:
closeTag ( STANDARD );
eof=processSyntaxChangeState(state);
openTag ( STANDARD );
break;
case SYNTAX_ERROR:
closeTag ( STANDARD );
eof=processSyntaxErrorState();
openTag ( STANDARD );
break;
case _EOL:
// XTERM256 fix (issue with less cmd)
if (!firstLine || showLineNumbers || !guidePrefix.empty()) {
closeTag ( STANDARD );
}
{
bool reopen = !firstLine || showLineNumbers || !guidePrefix.empty();
insertLineNumber(!firstLine);
if (reopen) {
flushWs(5);
stateTraceCurrent.clear();
openTag ( STANDARD );
}
}
firstLine=false;
break;
case _EOF:
eof=true;
break;
case _WS:
processWsState();
break;
default:
printMaskedToken();
break;
}
} while ( !eof );
if (token.size() || lineNumber>1 || (outputType!=ESC_TRUECOLOR && outputType!=ESC_XTERM256))
closeTag ( STANDARD );
if (currentSyntax->getDecorateLineEndFct()) {
Diluculum::LuaValueList res=callDecorateLineFct(false);
if (res.size()==1) {
*out << res[0].asString();
}
}
printNewLines = noTrailingNewLine==0 || ( noTrailingNewLine==2 && ( token.size() || lineNumber>1) );
*out << getNewLine();
*out << flush;
}
bool CodeGenerator::processSyntaxChangeState(State myState)
{
State newState=STANDARD;
bool eof=false,
exitState=false;
openTag ( KEYWORD );
do {
if (myState==EMBEDDED_CODE_END) {
if (!nestedLangs.empty()) {
nestedLangs.pop();
}
// load host language syntax
if (!nestedLangs.empty()) {
loadLanguage(nestedLangs.top(), true);
}
matchRegex(line, EMBEDDED_CODE_BEGIN); // match remaining line using the host syntax
}
printMaskedToken ( newState!=_WS );
newState= getCurrentState(myState);
switch ( newState ) {
case _WS:
processWsState();
break;
case _EOL:
insertLineNumber();
exitState=true;
break;
case _EOF:
eof = true;
break;
default:
exitState=true;
break;
}
} while ( !exitState && !eof );
closeTag ( KEYWORD );
return eof;
}
bool CodeGenerator::processKeywordState ( State myState )
{
State newState=STANDARD;
unsigned int myClassID=currentKeywordClass;
bool eof=false,
exitState=false;
openKWTag ( myClassID );
// Capture the closing tag now: a NestedSections delimiter may switch the
// active syntax mid-token (via getCurrentState() -> loadEmbeddedLang()),
// which resizes openTags/closeTags to match the nested language's keyword
// class count. Deriving the closing tag from myClassID again afterwards
// would then index into the wrong (differently sized) vector.
const string myCloseTag = closeTags.at ( getStyleID ( KEYWORD, myClassID ) );
do {
printMaskedToken ( newState!=_WS,
( currentSyntax->isIgnoreCase() ) ? keywordCase : StringTools::CASE_UNCHANGED );
newState= getCurrentState(myState);
switch ( newState ) {
case _WS:
processWsState();
exitState=isolateTags;
break;
case _EOL:
insertLineNumber();
exitState=true;
break;
case _EOF:
eof = true;
break;
case KEYWORD_END:
exitState=true;
break;
default:
exitState= ( myClassID!=currentKeywordClass ) || ( myState!=newState );
break;
}
} while ( !exitState && !eof );
*out << myCloseTag;
flushWs(3);
currentState=_UNKNOWN;
currentKeywordClass=0;
return eof;
}
bool CodeGenerator::processNumberState()
{
State newState=STANDARD;
bool eof=false,
exitState=false;
openTag ( NUMBER );
do {
printMaskedToken ( newState!=_WS );
newState= getCurrentState(NUMBER);
switch ( newState ) {
case _WS:
processWsState();
exitState=isolateTags;
break;
case _EOL:
insertLineNumber();
exitState=true;
break;
case _EOF:
eof = true;
break;
default:
exitState=newState!=NUMBER;
break;
}
} while ( !exitState && !eof );
closeTag ( NUMBER );
return eof;
}
bool CodeGenerator::processMultiLineCommentState()
{
int commentCount=1;
int openDelimID=currentSyntax->getOpenDelimiterID ( token, ML_COMMENT);
State newState=STANDARD;
bool eof=false, exitState=false, containedTestCase=false;
unsigned int startColumn=lineIndex - token.size() ;
openTag ( ML_COMMENT );
do {
printMaskedToken (newState!=_WS );
newState= getCurrentState(ML_COMMENT);
switch ( newState ) {
case _WS:
processWsState();
break;
case _EOL:
wsBuffer += closeTags[ML_COMMENT];
insertLineNumber();
wsBuffer += openTags[ML_COMMENT];
startColumn=0;
break;
case _EOF:
eof = true;
break;
case _TESTPOS:
runSyntaxTestcases(token=="<" ? startColumn : lineIndex - 1 );
printMaskedToken();
containedTestCase=true;
break;
case ML_COMMENT:
if ( currentSyntax->allowNestedMLComments() ) {
++commentCount;
}
// if delimiters are equal, close the comment by continuing to
// ML_COMMENT_END section
if (currentSyntax->delimiterIsDistinct(currentSyntax->getOpenDelimiterID ( token, ML_COMMENT ))) break;
case ML_COMMENT_END:
if (!currentSyntax->matchesOpenDelimiter (token, ML_COMMENT_END, openDelimID)) {
break;
}
commentCount--;
if ( !commentCount ) {
printMaskedToken();
exitState=true;
}
break;
default:
break;
}
} while ( !exitState && !eof );
closeTag ( ML_COMMENT );
if (containedTestCase){
stateTraceCurrent.clear();
}
return eof;
}
bool CodeGenerator::processSingleLineCommentState()
{
State newState=STANDARD;
bool eof=false, exitState=false, containedTestCase=false;
unsigned int startColumn = lineIndex - token.size() ;
openTag ( SL_COMMENT );
do {
printMaskedToken ( newState!=_WS );
newState= getCurrentState(SL_COMMENT);
switch ( newState ) {
case _WS:
processWsState();
break;
case _EOL:
printMaskedToken();
if ( preFormatter.isEnabled() && preFormatter.isWrappedLine ( lineNumber-1 ) ) {
exitState=false;
} else {
exitState=true;
}
if ( !exitState ) wsBuffer += closeTags[SL_COMMENT];
insertLineNumber();
if ( !exitState ) wsBuffer += openTags[SL_COMMENT];
break;
case _EOF:
eof = true;
break;
case _TESTPOS:
runSyntaxTestcases(token=="<" ? startColumn : lineIndex - 1 );
printMaskedToken();
containedTestCase=true;
break;
default:
break;
}
} while ( !exitState && !eof );
closeTag ( SL_COMMENT );
if (containedTestCase) {
stateTraceCurrent.clear();
}
return eof;
}
bool CodeGenerator::processDirectiveState()
{
State newState=STANDARD;
bool eof=false, exitState=false;
openTag ( DIRECTIVE );
do {
printMaskedToken ( newState!=_WS );
newState= getCurrentState(DIRECTIVE);
switch ( newState ) {
case _WS:
processWsState();
break;
case DIRECTIVE_END:
printMaskedToken();
exitState=true;
break;
case _EOL:
printMaskedToken();
if ( preFormatter.isEnabled() && preFormatter.isWrappedLine ( lineNumber-1 ) ) {
exitState=false;
} else {
if (currentSyntax->getContinuationChar()!=0x13){
exitState= ( terminatingChar!=currentSyntax->getContinuationChar() );
}
}
if ( !exitState ) wsBuffer += closeTags[DIRECTIVE];
insertLineNumber();
if ( !exitState ) wsBuffer += openTags[DIRECTIVE];
break;
case ML_COMMENT:
closeTag ( DIRECTIVE );
eof= processMultiLineCommentState();
openTag ( DIRECTIVE );
break;
case SL_COMMENT:
closeTag ( DIRECTIVE );
eof= processSingleLineCommentState();
openTag ( DIRECTIVE );
exitState=true;
break;
case STRING:
closeTag ( DIRECTIVE );
eof=processStringState ( DIRECTIVE );
openTag ( DIRECTIVE );
break;
case _EOF:
eof = true;
break;
default:
break;
}
} while ( !exitState && !eof );
closeTag ( DIRECTIVE );
return eof;
}
bool CodeGenerator::processStringState ( State oldState )
{
State newState=STANDARD;
bool eof=false, exitState=false;
bool returnedFromOtherState=false;
State myState= ( oldState==DIRECTIVE ) ? DIRECTIVE_STRING : STRING;
int openDelimID=currentSyntax->getOpenDelimiterID ( token, myState);
string openDelim=token;
//Raw String by definition:
bool isRawString=currentSyntax->delimiterIsRawString(openDelimID) || toggleDynRawString;
// Test if character before string open delimiter token equals to the
// raw string prefix (Example: r" ", r""" """ in Python)
//Raw String Prefix:
if ( lineIndex>token.length() &&line[lineIndex-token.length()-1]==currentSyntax->getRawStringPrefix() ) {
isRawString=true;
}
openTag ( myState );
do {
// true if last token was an escape char
if ( !returnedFromOtherState ) {
printMaskedToken (newState!=_WS );
}
returnedFromOtherState=false;
newState= getCurrentState(myState);
switch ( newState ) {
case _WS:
processWsState();
break;
case _EOL:
wsBuffer += closeTags[myState];
insertLineNumber();
wsBuffer += openTags[myState];
break;
case STRING_END:
if (resultOfHook || currentSyntax->matchesOpenDelimiter (token, STRING_END, openDelimID)) {
if (currentSyntax->assertDelimEqualLength()) {
exitState= openDelim.length()==token.length();
} else {
exitState= true;
}
printMaskedToken();
}
break;
case STRING:
// if there exist multiple string delimiters, close string if
// current delimiter is equal to the opening delimiter
exitState=currentSyntax->delimiterIsDistinct(currentSyntax->getOpenDelimiterID ( token, STRING )) && token==openDelim;
printMaskedToken();
break;
case ESC_CHAR:
if ( !isRawString ) {
closeTag ( myState );
eof=processEscapeCharState();
openTag ( myState );
returnedFromOtherState=true;
} else {
// FIXME not a fix for Python r"""\"""
exitState=token.size()>1 && token[1] == openDelim[0];
printMaskedToken();
}
break;
case STRING_INTERPOLATION:
closeTag ( myState );
eof=processInterpolationState();
openTag ( myState );
returnedFromOtherState=true;
break;
case _EOF:
eof = true;
break;
default:
printMaskedToken();
break;
}
} while ( !exitState && !eof );
closeTag ( myState );
toggleDynRawString = false;
return eof;
}
bool CodeGenerator::processSymbolState()
{
State newState=STANDARD;
bool eof=false,
exitState=false;
openTag ( SYMBOL );
do {
printMaskedToken ( newState!=_WS );
newState= getCurrentState(SYMBOL);
switch ( newState ) {
case _WS:
processWsState();
exitState=isolateTags;
break;
case _EOL:
insertLineNumber();
exitState=true;
break;
case _EOF:
eof = true;
break;
default:
exitState=newState!=SYMBOL;
break;
}
} while ( !exitState && !eof );
closeTag ( SYMBOL );
return eof;
}
bool CodeGenerator::processSyntaxErrorState()
{
State newState=STANDARD;
bool eof=false,
exitState=false;
openTag ( SYNTAX_ERROR );
do {
printMaskedToken ( newState!=_WS );
newState= getCurrentState(SYNTAX_ERROR);
switch ( newState ) {
case _WS:
processWsState();
exitState=isolateTags;
break;
case _EOL:
insertLineNumber();
exitState=true;
break;
case _EOF:
eof = true;
break;
default:
exitState=newState!=SYMBOL;
break;
}
} while ( !exitState && !eof );
closeTag ( SYNTAX_ERROR );
return eof;
}
bool CodeGenerator::processEscapeCharState()
{
State newState=STANDARD;
bool eof=false, exitState=false;
openTag ( ESC_CHAR );
do {
printMaskedToken (newState!=_WS );
newState= getCurrentState(ESC_CHAR);
switch ( newState ) {
case _EOL:
insertLineNumber();
exitState=true;
break;
case _WS:
processWsState();
exitState=isolateTags;
break;
case _EOF:
eof = true;
break;
default:
exitState=newState!=ESC_CHAR;
break;
}
} while ( !exitState && !eof );
closeTag ( ESC_CHAR );
return eof;
}
bool CodeGenerator::processInterpolationState()
{
State newState=STANDARD;
bool eof=false, exitState=false;
openTag ( STRING_INTERPOLATION );
do {
printMaskedToken (newState!=_WS );
newState= getCurrentState(STRING_INTERPOLATION);
switch ( newState ) {
case _EOL:
insertLineNumber();
exitState=true;
break;
case _WS:
processWsState();
exitState=isolateTags;
break;
case _EOF:
eof = true;
break;
default:
exitState=newState!=STRING_INTERPOLATION;
break;
}
} while ( !exitState && !eof );
closeTag ( STRING_INTERPOLATION );
return eof;
}
void CodeGenerator::processWsState()
{
if ( !maskWs ) {
if ( guideSkipWs ) --guideSkipWs;
else wsBuffer += token;
token.clear();
return;
}
flushWs(6);
int cntWs=0;
lineIndex--;
PositionState ps(currentState, 0, true);
while ( line[lineIndex]==' ' || line[lineIndex]=='\t' ) {
++cntWs;
++lineIndex;
}
if ( guideSkipWs ) {
int skipped = std::min ( (int) guideSkipWs, cntWs );
guideSkipWs -= skipped;
cntWs -= skipped;
}
if ( cntWs>1 ) {
unsigned int styleID=getStyleID ( currentState, currentKeywordClass );
// A NestedSections delimiter may switch the active syntax between the
// keyword's opening and this whitespace, resizing openTags/closeTags
// to the nested language's (possibly smaller) keyword class count.
// styleID may then be stale, so guard the lookup.
bool validStyleID = styleID!=_UNKNOWN && styleID<closeTags.size() && styleID<openTags.size();
if ( excludeWs && validStyleID ) {
*out << closeTags[styleID];
}
*out << maskWsBegin;
for ( int i=0; i<cntWs; i++ ) {
*out << spacer;
if (applySyntaxTestCase){
stateTraceCurrent.push_back(ps);
}
}
*out << maskWsEnd;
if ( excludeWs && validStyleID ) {
*out << openTags[styleID];
}
} else if ( cntWs==1 ) {
*out << spacer; //Bugfix fehlender Space nach Strings
if (applySyntaxTestCase){
stateTraceCurrent.push_back(ps);
}
}
spacer = initialSpacer;
token.clear();
}
void CodeGenerator::flushWs(int arg)
{
PositionState ps(currentState, 0, true);
//workaround condition
for ( size_t i=0; i<wsBuffer.size() && ((arg > 3) || ( (arg<4) && lineIndex>1)) && applySyntaxTestCase ; i++ ) {
stateTraceCurrent.push_back(ps);
//std::cerr <<"\nflush >"<<wsBuffer<<"< arg:"<<arg;
}
//fix canvas whitespace
if (wsBuffer.length() && (outputType==ESC_XTERM256 || outputType==ESC_TRUECOLOR) ){
*out<<maskWsBegin;
}
*out << wsBuffer;
wsBuffer.clear();
}
string CodeGenerator::getTestcaseName(State s, unsigned int kwClass) {
switch (s) {
case STANDARD:
return STY_NAME_STD;
case STRING:
return STY_NAME_STR;
case NUMBER:
return STY_NAME_NUM;
case SL_COMMENT:
return STY_NAME_SLC;
case ML_COMMENT:
return STY_NAME_COM;
case ESC_CHAR:
return STY_NAME_ESC;
case DIRECTIVE:
return STY_NAME_DIR;
case DIRECTIVE_STRING:
return STY_NAME_DST;
case SYMBOL:
return STY_NAME_SYM;
case STRING_INTERPOLATION:
return STY_NAME_IPL;
case SYNTAX_ERROR:
return STY_NAME_ERR;
case _WS:
return "ws";
case KEYWORD: {
if (!kwClass)
return "ws";
char kwName[20] = {0};
snprintf(kwName, sizeof(kwName), "keyword %c", ('a'+kwClass-1));
return string(kwName);
}
default:
return "unknown_test";
}
}
void CodeGenerator::printTrace(const string &s){
std::cout<<"\n curr "<<lineNumber<<" "<<s<<": ";
for (unsigned int i=0; i< stateTraceCurrent.size(); i++) {
std::cout<<" "<<stateTraceCurrent[i].state;
}
std::cout<<"\n test "<<lineNumber<<" "<<s<<": ";
for (unsigned int i=0; i< stateTraceTest.size(); i++) {
std::cout<<" "<<stateTraceTest[i].state;
}
std::cout<<"\n";
}
//column: lineIndex (not a UTF-8 validated string position)
void CodeGenerator::runSyntaxTestcases(unsigned int column){
if (isUtf8Encoding)
column = StringTools::utf8_strlen(line.substr(0, column));
if (column >= stateTraceTest.size()) return;
unsigned int assertGroup=0;
size_t typeDescPos=line.find_first_not_of("\t ^", lineIndex);
State assertState=_UNKNOWN;
bool negation=false;
bool testFailed=false;
ostringstream errMsg;
string prefix;
//printTrace("trace 2");
if (typeDescPos!=string::npos) {
if (line[typeDescPos]=='~') {
negation=true;
prefix="~";
++typeDescPos;
}
if (line.find(STY_NAME_NUM, typeDescPos)==typeDescPos)
assertState=NUMBER;
//TODO temp. fix to allow old and new string classes
else if (line.find(STY_NAME_STR, typeDescPos)==typeDescPos || line.find("str", typeDescPos)==typeDescPos)
assertState=STRING;
else if (line.find(STY_NAME_ESC, typeDescPos)==typeDescPos)
assertState=ESC_CHAR;
else if (line.find(STY_NAME_IPL, typeDescPos)==typeDescPos)
assertState=STRING_INTERPOLATION;
else if (line.find(STY_NAME_SYM, typeDescPos)==typeDescPos)
assertState=SYMBOL;
else if (line.find(STY_NAME_DIR, typeDescPos)==typeDescPos)
assertState=DIRECTIVE;
else if (line.find(STY_NAME_SLC, typeDescPos)==typeDescPos)
assertState=SL_COMMENT;
else if (line.find(STY_NAME_COM, typeDescPos)==typeDescPos)
assertState=ML_COMMENT;
else if (line.find("ws", typeDescPos)==typeDescPos)
assertState=_WS;
//TODO temp. fix to allow old and new default classes
else if (line.find(STY_NAME_STD, typeDescPos)==typeDescPos || line.find("std", typeDescPos)==typeDescPos)
assertState=STANDARD;
else if (line.find(STY_NAME_DST, typeDescPos)==typeDescPos)
assertState=DIRECTIVE_STRING;
else if (line.find("kw", typeDescPos)==typeDescPos || line.find("st", typeDescPos)==typeDescPos) {
assertState=KEYWORD;
if (isalpha(line[typeDescPos+2]))
assertGroup=line[typeDescPos+2] - 'a' +1;
}
if ( (assertState!=_WS && stateTraceTest[column].state != assertState && !stateTraceTest[column].isWhiteSpace )
|| (assertState==_WS && !stateTraceTest[column].isWhiteSpace)
|| assertGroup != stateTraceTest[column].kwClass) {
testFailed=!negation;
} else if (negation ) {
//TODO Fix ~ws
if (assertState!=_WS && !stateTraceTest[column].isWhiteSpace )
testFailed=true;
}
if (testFailed) {
errMsg << inFile << " line " << lineNumber << ", column "<< column
<< ": got " << getTestcaseName(stateTraceTest[column].state, stateTraceTest[column].kwClass)
<< " instead of " << prefix << getTestcaseName(assertState, assertGroup);
failedPosTests.push_back(errMsg.str());
}
}
lineContainedTestCase=true;
}
string CodeGenerator::getNewLine()
{
ostringstream ss;
printSyntaxError(ss);
if (printNewLines)
ss << newLineTag;
return ss.str();
}
Diluculum::LuaValueList CodeGenerator::callDecorateLineFct(bool isLineStart)
{
Diluculum::LuaValueList params;
params.emplace_back(lineNumber);
return currentSyntax->getLuaState()->call ( isLineStart ?
*currentSyntax->getDecorateLineBeginFct(): *currentSyntax->getDecorateLineEndFct(),
params,"getDecorateLineFct call");
}
void CodeGenerator::setOverrideParams() {
if (currentSyntax->requiresParamUpdate()) {
if ( currentSyntax->getOverrideConfigVal("state.string.raw")=="true"){
toggleDynRawString=true; // reset to false in string state fct
}
if ( currentSyntax->getOverrideConfigVal("format.maskws")=="true") {
maskWs=true;
}
if ( currentSyntax->getOverrideConfigVal("format.spacer").size()) {
spacer=currentSyntax->getOverrideConfigVal("format.spacer");
}
}
}
void CodeGenerator::insertLineNumber ( bool insertNewLine )
{
if ( insertNewLine ) {
if (currentSyntax->getDecorateLineEndFct()) {
Diluculum::LuaValueList res=callDecorateLineFct(false);
if (res.size()==1) {
setOverrideParams();
wsBuffer +=res[0].asString();
}
}
wsBuffer += getNewLine();
}
if (currentSyntax->getDecorateLineBeginFct()) {
Diluculum::LuaValueList res=callDecorateLineFct(true);
if (res.size()==1) {
setOverrideParams();
wsBuffer += res[0].asString();
}
}
if ( showLineNumbers ) {
ostringstream os;
ostringstream numberPrefix;
os << setw ( getLineNumberWidth() ) << right;
if( numberCurrentLine ) {
if ( lineNumberFillZeroes ) {
os.fill ( '0' );
}
os << lineNumber+lineNumberOffset;
} else {
os << "";
}
numberPrefix << openTags[LINENUMBER];
maskString ( numberPrefix, os.str() );
//use initialSpacer here, spacer can be overridden by plug-in (format.spacer)
numberPrefix << initialSpacer << closeTags[LINENUMBER];
wsBuffer += numberPrefix.str();
}
insertLineHead();
}
void CodeGenerator::insertLineHead()
{
// wrapped segments continue a line whose head was already printed
if ( !lineHeadPending ) {
wsBuffer += guidePrefix;
guidePrefix.clear();
return;
}
ostringstream headPrefix;
maskString ( headPrefix, lineHead );
wsBuffer += headPrefix.str();
lineHeadPending = false;
wsBuffer += guidePrefix;
guidePrefix.clear();
}
unsigned int CodeGenerator::getLineIndex()
{
return lineIndex;
}
unsigned int CodeGenerator::getLastLineLength()
{
return lastLineLength;
}
bool CodeGenerator::requiresTwoPassParsing() const {
if (!currentSyntax) return false;
return currentSyntax->getPersistentSnippetsNum()>0;
}
bool CodeGenerator::printExternalStyle ( const string &outFile )
{
if ( !includeStyleDef ) {
ostream *cssOutFile = ( outFile.empty() ? &cout :new ofstream ( outFile.c_str() ) );
if ( !cssOutFile->fail() ) {
if (!omitVersionComment) {
*cssOutFile << styleCommentOpen
<<" Style definition file generated by highlight "
<< Info::getVersion() << ", " << Info::getWebsite()
<< " " << styleCommentClose << "\n";
}
*cssOutFile << getStyleDefinition()
<< "\n";
*cssOutFile << readUserStyleDef();
if ( !outFile.empty() ) delete cssOutFile;
} else {
return false;
}
}
return true;
}
bool CodeGenerator::printPersistentState ( const string &outFile )
{
if (!currentSyntax) return false;
ofstream pluginOutFile( outFile.c_str());
if ( !pluginOutFile.fail() ) {
pluginOutFile <<"Description=\"Plugin generated by highlight using the --two-pass option\"\n\n"
<<"Categories = {\"two-pass\" }\n\n"
<<"function syntaxUpdate(desc)\n\n";
pluginOutFile << currentSyntax->getPersistentHookConditions();
for (const auto& snippet: currentSyntax->getPersistentSnippets())
{
pluginOutFile << snippet <<"\n\n";
}
pluginOutFile<<"end\n\n"
<<"Plugins={\n"
<<" { Type=\"lang\", Chunk=syntaxUpdate }\n"
<<"}\n";
} else {
return false;
}
return true;
}
string CodeGenerator::readUserStyleDef()
{
ostringstream ostr;
if ( !styleInputPath.empty() ) {
ifstream userStyleDef ( styleInputPath.c_str() );
if ( userStyleDef ) {
ostr << "\n" << styleCommentOpen
<< " Content of " << styleInputPath
<< ": " <<styleCommentClose << "\n";
string line;
while ( getline ( userStyleDef, line ) ) {
ostr << line << "\n";
}
userStyleDef.close();
} else {
ostr << styleCommentOpen
<< " ERROR: Could not include " << styleInputPath
<< "." << styleCommentClose << "\n";
}
}
string injections=docStyle.getInjections();
if (!injections.empty()) {
ostr << "\n" << styleCommentOpen
<< " Plug-in theme injections: " <<styleCommentClose << "\n";
ostr << injections<<"\n";
}
return ostr.str();
}
bool CodeGenerator::initPluginScript(const string& script)
{
if (script.empty()) return true;
try {
userScriptError="";
Diluculum::LuaState ls;
ls.doFile (script);
int listIdx=1;
while (ls["Plugins"][listIdx].value() !=Diluculum::Nil) {
// Theme plugins
if (ls["Plugins"][listIdx]["Type"].value().asString()=="theme") {
if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
docStyle.addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
}
}
// Syntax plugins
else if (ls["Plugins"][listIdx]["Type"].value().asString()=="lang") {
if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
currentSyntax->addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
}
}
// Format plugins
else if (ls["Plugins"][listIdx]["Type"].value().asString()=="format") {
if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
}
}
listIdx++;
}
} catch (Diluculum::LuaError &err) {
userScriptError=err.what();
return false;
}
return true;
}
void CodeGenerator::resetSyntaxReaders() {
for ( auto it=syntaxReaders.begin(); it!=syntaxReaders.end(); it++ ) {
delete it->second;
}
currentSyntax=nullptr;
syntaxReaders.clear();
}
bool CodeGenerator::syntaxRequiresTwoPassRun() {
if (!currentSyntax) return false;
return currentSyntax->requiresTwoPassRun();
}
void CodeGenerator::clearPersistentSnippets(){
if (currentSyntax) {
currentSyntax->clearPersistentSnippets();
}
}
void CodeGenerator::updateKeywordClasses(){
if (openTags.size()) {
if ( openTags.size() >NUMBER_BUILTIN_STATES ) {
// remove dynamic keyword tag delimiters of the old language definition
auto keyStyleOpenBegin =
openTags.begin() + NUMBER_BUILTIN_STATES;
auto keyStyleCloseBegin =
closeTags.begin() + NUMBER_BUILTIN_STATES;
openTags.erase ( keyStyleOpenBegin, openTags.end() );
closeTags.erase ( keyStyleCloseBegin, closeTags.end() );
}
// add new keyword tag delimiters
for ( unsigned int i=0; i< currentSyntax->getKeywordClasses().size(); i++ ) {
openTags.push_back ( getKeywordOpenTag ( i ) );
closeTags.push_back ( getKeywordCloseTag ( i ) );
}
}
}
}