ChangeLog.adoc

@@ -6,6 +6,14 @@ :toc: left :toclevels: 1 ++== highlight 3.60++01.12.2020++  - fixed single argument of mark_lines.lua plug-in (https://gitlab.com/saalen/highlight/-/issues/164)+  - added lineno and column parameters to Decorate hook+ == highlight 3.59  13.11.2020

README_PLUGINS.adoc

@@ -227,10 +227,10 @@ RemoveKeyword(keyword) === FUNCTION ADDPERSISTENTSTATE  [[AddPersistentState]]-This function enables storage of keywords and keyword ranges in a plug-in file. -If the syntax contains elements which depend on a context, you can highlight +This function enables storage of keywords and keyword ranges in a plug-in file.+If the syntax contains elements which depend on a context, you can highlight them although this context is lost in other input files or code sections.-The invocation of AddPersistentState will cause highlight to save a plugin as +The invocation of AddPersistentState will cause highlight to save a plugin as temporary file and parse input files using this plug-in again if necessary.  ................................................................................@@ -296,9 +296,9 @@ does no longer match.  [IMPORTANT] ================================================================================-If both the syntax file and the plug-in script contain an `OnStateChange` +If both the syntax file and the plug-in script contain an `OnStateChange` function, the plug-in definition will replace the function of the syntax file.-In order to prevent this, the existing function can be assigned to an new +In order to prevent this, the existing function can be assigned to an new variable and called using this name in the plug-in `OnStateChange` code chunk.  See this example in the plug-in `bash_functions.lua`:@@ -365,7 +365,7 @@ identified. It can be used to alter the token or to add additional text in the target output format (e.g. hyperlinks).  ................................................................................-Decorate(token, state, kwGroupID, stateTrace)+Decorate(token, state, kwGroupID, lineHasStmt, lineno, column)    Hook Event: Token identification   Parameters: token:       current token@@ -373,6 +373,8 @@ Decorate(token, state, kwGroupID, stateTrace)               kwGroupID:   if state is HL_KEYWORD, the parameter                            contains the keyword group ID               lineHasStmt: true if current line contains a statement+              lineno:    line number (since 3.60)+              column:    line column (since 3.60)   Returns:    Altered token string or nothing if original token should be               outputted ................................................................................@@ -426,7 +428,7 @@ The return values are not modified or validated by highlight. StoreValue(name, value)    Set or get a value to exchange information across Lua states-  Parameters: name: the parameter name +  Parameters: name: the parameter name               value: the parameter value   Returns:   if value is not set, the previously assigned value of name ................................................................................

makefile

@@ -96,9 +96,9 @@ install: 	${INSTALL_DATA} ./themes/base16/*.theme ${DESTDIR}${data_dir}themes/base16/ 	${INSTALL_DATA} ./plugins/*.lua ${DESTDIR}${data_dir}plugins/ 	${INSTALL_DATA} ./man/highlight.1 ${DESTDIR}${man_dir}man1/-	-${GZIP} ${DESTDIR}${man_dir}man1/highlight.1+	${GZIP} ${DESTDIR}${man_dir}man1/highlight.1 	${INSTALL_DATA} ./man/filetypes.conf.5 ${DESTDIR}${man_dir}man5/-	-${GZIP} ${DESTDIR}${man_dir}man5/filetypes.conf.5+	${GZIP} ${DESTDIR}${man_dir}man5/filetypes.conf.5  	${INSTALL_DATA} ./AUTHORS ${DESTDIR}${doc_dir} 	${INSTALL_DATA} ./README* ${DESTDIR}${doc_dir}

plugins/mark_lines.lua

@@ -24,7 +24,10 @@ function syntaxUpdate(desc)     local t, ll     t={}     ll=0-    if(#p == 1) then return {p} end+    if(#p == 1) then+      t[tonumber(p)] = 1+      return t+    end     while true do       l=string.find(p,d,ll,true) -- find the next d in the string       if l~=nil then -- if "not not" found then..

plugins/outhtml_lsp_clangd.lua

new file mode 100644@@ -0,0 +1,205 @@+--[[++This is a rough test implementation of the Language Server Protocol to get+things started.++This only works with absolute input paths which are part of a workspace+configured properly for the clangd LSP server.++Temp files in /tmp/ are not deleted.++To synchronize I/O using stdin/stdout, the script sleeps after each call.++1.  Copy this Lua JSON parser to your current directory:+    https://github.com/rxi/json.lua+2.  Setup your project for clangd (ie provide a compile_flags.txt file)+3.  Call the plugin with the project direcory and the source file as+    --plug-in-param parameter:+    highlight --plug-in plugins/outhtml_lsp_clangd.lua -I keystore.cpp+            --plug-in-param '/home/andre/Projekte/git/highlight/src/:/home/andre/Projekte/git/highlight/src/core/keystore.cpp'+            > ~/Projekte/test_out/lsp.html+]]++Description="Add tooltips based on clangd LSP output (WIP - very slow - Linux only - see contained info)"++Categories = {"language-server-protocol", "html" }++-- optional parameter: syntax description+function syntaxUpdate(desc)++  if (desc ~= "C and C++") then+      return+  end++  if (HL_OUTPUT ~= HL_FORMAT_HTML and HL_OUTPUT ~= HL_FORMAT_XHTML) then+      return+  end++  tmp_file = "/tmp/lsp_client_"..os.time()++  json = require "json" -- https://github.com/rxi/json.lua+  --inotify = require 'inotify' -- https://github.com/hoelzro/linotify++  function html_escape(s)+    return (string.gsub(s, "[}{\">/<'&]", {+        ["&"] = "&amp;",+        ["<"] = "&lt;",+        [">"] = "&gt;",+        ['"'] = "&quot;",+        ["'"] = "&#39;",+        ["/"] = "&#47;"+    }))+  end++  function fibonacci(n)+    local function inner(m)+      if m < 2 then+        return m+      end+      return inner(m-1) + inner(m-2)+    end+    return inner(n)+  end++  function sleep(s)+    local ntime = os.time() + s+    repeat until os.time() > ntime+  end++  function readResponse(fr)++    j = nil+    x = fr:read("*line")+    if string.find(x, 'Content-Length:', 1, true) then+      numBytes = tonumber(x:sub(17))+      fr:read("*line")+      x = fr:read(numBytes)+      j = json.decode(x)+    end+    return j+  end++  function initServer(fw, root_uri)++      init_request = {}+      params = {}+      params.processId = 0+      params.rootUri = root_uri+      params.capabilities = {}++      init_request.jsonrpc = "2.0"+      init_request.id = 1+      init_request.method = "initialize"+      init_request.params = params++      json_s = json.encode(init_request)+      init_cmd = "Content-Length: ".. #json_s.."\n\n"..json_s+      fw:write(init_cmd)+      fw:flush()+      sleep(1)+  end++  function openDocument(fw, input_file)++      fs = io.open(input_file, "r")++      if fs==nil then return false end++      source_text = fs:read("*a")+      fs:close()++      doc_request = {}+      params = {}+      params.textDocument = {}+      params.textDocument.uri = "file://"..input_file+      params.textDocument.languageId = "cpp"+      params.textDocument.version = 1+      params.textDocument.text = source_text++      doc_request.jsonrpc = "2.0"+      doc_request.method = "textDocument/didOpen"+      doc_request.params = params++      json_s = json.encode(doc_request)+      init_cmd = "Content-Length: ".. #json_s.."\n\n"..json_s+      fw:write(init_cmd)+      fw:flush()+      sleep(5) -- TODO maybe wait for textDocument/publishDiagnostics line in stdin+      return true+  end++  function hover(fw, line, col, input_file)++      hover_request = {}+      params = {}+      params.textDocument = {}+      params.textDocument.uri = "file://"..input_file+      params.position = {}+      params.position.line = line+      params.position.character = col++      hover_request.id = 1+      hover_request.jsonrpc = "2.0"+      hover_request.method = "textDocument/hover"+      hover_request.params = params++      json_s = json.encode(hover_request)++      init_cmd = "Content-Length: ".. #json_s.."\n\n"..json_s+      fw:write(init_cmd)+      fw:flush()+      fibonacci(32)+      return true+  end++  function string:split(sep)+    local sep, fields = sep or ":", {}+    local pattern = string.format("([^%s]+)", sep)+    self:gsub(pattern, function(c) fields[#fields+1] = c end)+    return fields+  end++  args=string.split(HL_PLUGIN_PARAM, ':')++  if #args~=2 then return end++  clangd_cmd= "clangd >"..tmp_file++  fw = io.popen(clangd_cmd, "w")++  initServer(fw, "file://"..args[1])+  fr = io.open(tmp_file, "r+")+  init_res = readResponse(fr)++  --TODO set in Lua state+  input_source = args[2]++  if init_res.result.serverInfo.name=='clangd' then++      if openDocument(fw, input_source) then+          readResponse(fr)+      end+  end++  function Decorate(token, state, kwclass, lcs, line, col)++    if ( state ~= HL_STANDARD and state ~= HL_KEYWORD) then+      return+    end++    hover(fw, line-1, col+1, input_source)+    hover_res = readResponse(fr)++    if hover_res.result then+        tooltip = html_escape(hover_res.result.contents.value)+        return '<span title="'..tooltip..'">'..token .. '</span>'+    end++  end+end++Plugins={++  { Type="lang", Chunk=syntaxUpdate },++}

src/compile_flags.txt

new file mode 100644@@ -0,0 +1,2 @@+-std=c++11+-I./include/

src/core/codegenerator.cpp

@@ -795,6 +795,8 @@ Diluculum::LuaValueList CodeGenerator::callDecorateFct(const string& token)     params.push_back(Diluculum::LuaValue(currentState));     params.push_back(Diluculum::LuaValue(currentKeywordClass));     params.push_back(Diluculum::LuaValue(lineContainedStmt));+    params.push_back(Diluculum::LuaValue(lineNumber) );+    params.push_back(Diluculum::LuaValue(lineIndex-(unsigned int)token.length()) );      return currentSyntax->getLuaState()->call ( *currentSyntax->getDecorateFct(),             params,"getDecorateFct call")  ;@@ -1446,7 +1448,7 @@ void CodeGenerator::applyPluginChunk(const string& fctName, string *result, bool         if (luaState.globals().count(fctName)) {             Diluculum::LuaFunction* documentFct=new Diluculum::LuaFunction(luaState[fctName].value().asFunction()); -            luaState["HL_INPUT_FILE"] = luaState["HL_PLUGIN_PARAM"] = pluginParameter;+            luaState["HL_PLUGIN_PARAM"] = pluginParameter;             luaState["HL_OUTPUT"] = outputType;             luaState["HL_FORMAT_HTML"]=HTML;             luaState["HL_FORMAT_XHTML"]=XHTML;

src/core/syntaxreader.cpp

@@ -101,7 +101,7 @@ void  SyntaxReader::initLuaState(Diluculum::LuaState& ls, const string& langDefP     string::size_type Pos = langDefPath.find_last_of ( Platform::pathSeparator );     ls["HL_LANG_DIR"] =langDefPath.substr ( 0, Pos+1 ); -    ls["HL_INPUT_FILE"] = ls["HL_PLUGIN_PARAM"] = pluginParameter;+    ls["HL_PLUGIN_PARAM"] = pluginParameter;     ls["HL_OUTPUT"] = type;      ls["Identifiers"]=REGEX_IDENTIFIER;@@ -184,12 +184,12 @@ LoadResult SyntaxReader::load ( const string& langDefPath, const string& pluginR         ls.doFile (langDefPath);          langDesc = ls["Description"].value().asString();-        +         if (ls["Categories"].value() !=Diluculum::Nil){              Diluculum::LuaValueMap categoryMap;             categoryMap = ls["Categories"].value().asTable();-                +             for(Diluculum::LuaValueMap::const_iterator it = categoryMap.begin(); it != categoryMap.end(); ++it)             {                 categories.append(it->second.asString());@@ -217,7 +217,7 @@ LoadResult SyntaxReader::load ( const string& langDefPath, const string& pluginR         int kwId=0;         while (ls["Keywords"][idx].value() !=Diluculum::Nil) {             kwId= generateNewKWClass ( ls["Keywords"][idx]["Id"].value().asInteger() );-            +             if (ls["Keywords"][idx]["List"].value()!=Diluculum::Nil) {                 int listIdx=1;                 Diluculum::LuaVariable luaList=ls["Keywords"][idx]["List"];@@ -250,12 +250,12 @@ LoadResult SyntaxReader::load ( const string& langDefPath, const string& pluginR             }             idx++;         }-        +         if (globals.count("KeywordFormatHints")) {             idx=1;             Diluculum::LuaValue lVal = ls["KeywordFormatHints"][idx].value();             while (lVal !=Diluculum::Nil) {-                +                 int kwStyleOverride = lVal["Id"].asInteger();                  if (lVal["Bold"]!=Diluculum::Nil){@@ -267,13 +267,13 @@ LoadResult SyntaxReader::load ( const string& langDefPath, const string& pluginR                 if (lVal["Underline"]!=Diluculum::Nil){                     kwStyleOverride += lVal["Underline"].asBoolean() ? 512 : 4096;                 }-                 +                 overrideStyles.push_back(kwStyleOverride);                 idx++;                 lVal = ls["KeywordFormatHints"][idx].value();             }         }-        +         if (globals.count("Comments")) {              int listIdx=1;@@ -396,16 +396,16 @@ LoadResult SyntaxReader::load ( const string& langDefPath, const string& pluginR                 string lang= ls["NestedSections"][listIdx]["Lang"].value().asString();                 string openDelim=StringTools::trim(ls["NestedSections"][listIdx]["Delimiter"][1].value().asString());                 regex.insert(regex.begin(), 1, new RegexElement(EMBEDDED_CODE_BEGIN, EMBEDDED_CODE_BEGIN, openDelim, 0, -1, lang));-                +                 string closeDelim=StringTools::trim(ls["NestedSections"][listIdx]["Delimiter"][2].value().asString());                 nestedStateEndDelimiters[getNewPath(lang)] = closeDelim;-                +                 bool allowInnerSectionsFlag=true;                 if (ls["NestedSections"][listIdx]["Sealed"].value()!=Diluculum::Nil){                     allowInnerSectionsFlag = !ls["NestedSections"][listIdx]["Sealed"].value().asBoolean();                 }                 allowInnerSections[getNewPath(lang)] = allowInnerSectionsFlag;-                +                 ++listIdx;             }             //allow host syntax@@ -423,7 +423,7 @@ LoadResult SyntaxReader::load ( const string& langDefPath, const string& pluginR         if (globals.count("EncodingHint")) {             encodingHint = ls["EncodingHint"].value().asString();         }-        +         // load hook functions         if (globals.count("OnStateChange")) {             validateStateChangeFct=new Diluculum::LuaFunction(ls["OnStateChange"].value().asFunction());@@ -468,7 +468,7 @@ int SyntaxReader::getKeywordListGroup ( const string &s ) {     }     return 0; }-    + int SyntaxReader::luaAddKeyword (lua_State *L) {     int retVal=0;@@ -605,11 +605,11 @@ bool SyntaxReader::matchesOpenDelimiter ( const string& token, State s, int open void SyntaxReader::addPersistentKeyword(unsigned int groupID, const string& kw){     ostringstream expr;     expr <<"AddKeyword(\""<<kw<<"\", "<<groupID<<")";-             +     persistentSnippets.push_back(expr.str());     persistentSyntaxDescriptions.insert(langDesc); }-    + void SyntaxReader::addPersistentStateRange(unsigned int groupID, unsigned int column,unsigned int length, unsigned int lineNumber, const string& fileName){     ostringstream expr;     expr <<"table.insert(Keywords,\n"@@ -621,7 +621,7 @@ void SyntaxReader::addPersistentStateRange(unsigned int groupID, unsigned int co          <<"    Filename = [=["<<fileName<<"]=],\n"          <<" }\n"          <<"})";-                +     persistentSnippets.push_back(expr.str());     persistentSyntaxDescriptions.insert(langDesc); }@@ -629,17 +629,17 @@ void SyntaxReader::addPersistentStateRange(unsigned int groupID, unsigned int co string SyntaxReader::getPersistentHookConditions() {     ostringstream expr;     expr << "ValidDesc = {";-    +     for (auto desc: persistentSyntaxDescriptions) {         expr << "\"" << desc << "\",";     }-    +     expr << "}\nif ValidDesc[desc] ~= nil then return end\n\n";-    +     return expr.str();-    + }- + int SyntaxReader::luaAddPersistentState (lua_State *L) {     int retVal=0;@@ -653,7 +653,7 @@ int SyntaxReader::luaAddPersistentState (lua_State *L)                 (*a)->addKeyword(kwgroupID, keyword);                 (*a)->addPersistentKeyword(kwgroupID, keyword);             }-            +             retVal=1;         }     }@@ -671,7 +671,7 @@ int SyntaxReader::luaAddPersistentState (lua_State *L)     }     lua_pushboolean(L, retVal);     return 1;-} +}  bool SyntaxReader::requiresTwoPassRun(){     return persistentSyntaxDescriptions.count(langDesc)>0;

src/makefile

@@ -25,7 +25,7 @@ CFLAGS:=-Wall -O2 ${CFLAGS} -DNDEBUG -std=c++11 -D_FILE_OFFSET_BITS=64  CFLAGS_DILU=-fno-strict-aliasing -SO_VERSION=3.59+SO_VERSION=3.60  # Source paths CORE_DIR=./core/