Flexible grammar installer

README.md

@@ -14,21 +14,98 @@ It is work in progress, but functions well.
 | ![](before.png)                                |                                 ![](after.png) |
 
 # Supported Languages
-- [x] [C][tree-sitter-c]
-- [x] [C++][tree-sitter-cpp]
-- [ ] CSS
-- [x] [D][tree-sitter-d]
-- [x] [Diff][tree-sitter-diff]
-- [x] [Go][tree-sitter-go]
-- [x] [go.mod][tree-sitter-go-mod]
-- [ ] HTML
-- [x] [Javascript/JSX][tree-sitter-javascript]
-- [x] [Julia][tree-sitter-julia]
-- [x] [Lua][tree-sitter-lua]
-- [x] [Rust][tree-sitter-rust]
-- [x] [Zig][tree-sitter-zig]
-
-If you want more languages supported, open an issue.
+
+Any language supported by `tree-sitter` should (in theory) be supported as well.
+
+It is possible to use grammar installed locally or directly from their git repository. Additionally 
+it is possible to specify where to find the queries of the language as well as use sub path for projects
+containing more then one language.
+## Default grammars
+
+A few grammars have default configurations, pre-compiled parsers and custom queries:
+
+The supported languages are:
+
+- [C][tree-sitter-c]
+- [C++][tree-sitter-cpp]
+- [D][tree-sitter-d]
+- [Diff][tree-sitter-diff]
+- [Go][tree-sitter-go]
+- [go.mod][tree-sitter-go-mod]
+- [Javascript/JSX][tree-sitter-javascript]
+- [Julia][tree-sitter-julia]
+- [Lua][tree-sitter-lua]
+- [Rust][tree-sitter-rust]
+- [Zig][tree-sitter-zig]
+
+To use any of this grammars simply add this to your conifg 
+
+```lua
+local egconfig = require "plugins.evergreen.config"
+
+egconfig.addGrammar {
+  lang = "LANGUAGE",
+  precompiled = true -- optional: default true, if false use default git repository to install grammar
+}
+```
+
+
+## Locally installed grammars
+
+To add a grammar to lite-xl simply add the following configuration to your `init.lua` script:
+
+```lua
+local egconfig = require "plugins.evergreen.config"
+
+egconfig.addGrammar {
+  path = "YOUR GRAMMAR LOCAL PATH",
+  lang = "LANGUAGE NAME",
+  extensions = "FILE EXTENSIONS", -- optional for extension name based
+  filename = "FILE NAME", -- optional for file name based 
+  queries = "QUERIES SUB PATH", -- optional sub path of grammar queries, default = 'queries'
+}
+```
+
+## From git repository 
+To add a grammar directly from a git repository add this to your `init.lua` scripts:
+
+```lua
+local egconfig = require "plugins.evergreen.config"
+egconfig.addGrammar {
+  git = "GIT REPO",
+  lang = "LANGUAGE NAME",
+  extensions = "FILE EXTENSIONS", -- optional for extension name based
+  filename = "FILE NAME", -- optional for file name based 
+  queries = "QUERIES SUB PATH", -- optional sub path of grammar queries, default = 'queries'
+  subpath = "GRAMMAR SUB PATH", -- optional sub path if grammar is nested, default = nil
+  revision = "SPECIFIC GIT REVISION", -- optional version of the code to clone
+}
+```
+
+## Example 
+
+```lua
+local egconfig = require "plugins.evergreen.config"
+
+egconfig.addGrammar {
+  path = "~/.grammars/tree-sitter-epics/epics-db",
+  lang = "epics_db",
+  extensions = "db,vdb",
+}
+
+egconfig.addGrammar {
+  git = "https://github.com/tree-sitter/tree-sitter-cpp.git",
+  lang = "cpp",
+  extensions = "cpp,hpp,cc",
+}
+
+egconfig.addGrammar {
+  git = "https://github.com/tree-sitter/tree-sitter-c.git",
+  lang = "c",
+  extensions = "c,h",
+}
+
+```
 
 # Requirements
 - Lite XL 2.1+ or [Pragtical](https://github.com/pragtical/pragtical)

config.lua

@@ -1,12 +1,20 @@
 local util = require 'plugins.evergreen.util'
+local installer = require 'plugins.evergreen.installer'
+
 local home = HOME or os.getenv 'HOME'
 
 local M = {
 	dataDir = PLATFORM ~= 'Windows'
-	and ('~/.local/share/evergreen'):gsub('~', home)
-	or string.format('%s\\evergreen', os.getenv 'APPDATA')
+			and ('~/.local/share/evergreen'):gsub('~', home)
+			or string.format('%s\\evergreen', os.getenv 'APPDATA')
 }
 
-M.parserLocation = util.join {M.dataDir, 'parsers'}
+M.parserLocation = util.join { M.dataDir, 'parsers' }
+M.queryLocation = util.join { M.dataDir, 'queries' }
+
+-- add grammar
+function M.addGrammar(options)
+	installer.addGrammar(options, M)
+end
 
 return M

highlights.lua

@@ -1,15 +1,12 @@
 local parser = require 'plugins.evergreen.parser'
 local languages = require 'plugins.evergreen.languages'
+local config = require 'plugins.evergreen.config'
 
 local M = {}
 
-local function localPath()
-	local str = debug.getinfo(2, 'S').source:sub(2)
-	return str:match '(.*[/\\])'
-end
 
 function M.query(ftype)
-	local ff = io.open(string.format('%s/queries/%s/highlights.scm', localPath(), ftype))
+	local ff = io.open(string.format('%s/%s/highlights.scm', config.queryLocation, ftype))
 	if not ff then
 		return ""
 	end
@@ -23,8 +20,8 @@ end
 --- @param doc core.doc
 function M.init(doc)
 	local function getSource(n)
-		local startPt = n:start_point()
-		local endPt   = n:end_point()
+		local startPt            = n:start_point()
+		local endPt              = n:end_point()
 		local startRow, startCol = startPt.row + 1, startPt.column + 1
 		local endRow, endCol     = endPt.row + 1, endPt.column
 
@@ -53,7 +50,7 @@ function M.init(doc)
 			query = p:query(M.query(languages.fromDoc(doc))):with {
 				['any-of?'] = function(t, ...)
 					local src = getSource(t)
-					for _, match in ipairs {...} do
+					for _, match in ipairs { ... } do
 						if src == match then return true end
 					end
 					return false

init.lua

@@ -3,6 +3,9 @@ local core = require 'core'
 local config = require 'plugins.evergreen.config'
 local util = require 'plugins.evergreen.util'
 local home = HOME or os.getenv 'HOME'
+local languages = require 'plugins.evergreen.languages'
+local installer = require 'plugins.evergreen.installer'
+
 local function appendPaths(paths)
 	for _, path in ipairs(paths) do
 		package.cpath = package.cpath .. ';' .. path:gsub('~', home)
@@ -11,11 +14,12 @@ end
 
 system.mkdir(config.dataDir)
 system.mkdir(config.parserLocation)
+system.mkdir(config.queryLocation)
 
 appendPaths {
-	util.join {config.dataDir, '?' .. util.soname},
-	util.join {config.parserLocation, '?', 'libtree-sitter-?' .. util.soname},
-	util.join {config.parserLocation, '?', 'parser' .. util.soname},
+	util.join { config.dataDir, '?' .. util.soname },
+	util.join { config.parserLocation, '?', 'libtree-sitter-?' .. util.soname },
+	util.join { config.parserLocation, '?', 'parser' .. util.soname },
 }
 
 if PLATFORM ~= 'Windows' then
@@ -44,13 +48,17 @@ core.log('%s', ok)
 if not ok then
 	core.add_thread(function()
 		core.log 'Could not require ltreesitter, attempting to install...'
-		local url = string.format('https://github.com/TorchedSammy/evergreen-builds/releases/download/ltreesitter/ltreesitter%s', util.soname)
+		local url = string.format(
+			'https://github.com/TorchedSammy/evergreen-builds/releases/download/ltreesitter/ltreesitter%s', util.soname)
 
 		local out, exitCode
 		if PLATFORM == 'Windows' then
-			out, exitCode = exec({'powershell', '-Command', string.format('Invoke-WebRequest -OutFile ( New-Item -Path "%s" -Force ) -Uri %s', util.join {config.dataDir, 'ltreesitter' .. util.soname}, url)})
+			out, exitCode = exec({ 'powershell', '-Command',
+				string.format('Invoke-WebRequest -OutFile ( New-Item -Path "%s" -Force ) -Uri %s',
+					util.join { config.dataDir, 'ltreesitter' .. util.soname }, url) })
 		else
-			out, exitCode = exec({'curl', '-L', '--create-dirs', '--output-dir', config.dataDir, '--fail', url, '-o', 'ltreesitter' .. util.soname})
+			out, exitCode = exec({ 'curl', '-L', '--create-dirs', '--output-dir', config.dataDir, '--fail', url, '-o',
+				'ltreesitter' .. util.soname })
 		end
 		if exitCode ~= 0 then
 			core.error('An error occured while attempting to download ltreesitter\n%s', out)
@@ -71,7 +79,6 @@ local Highlight = require 'core.doc.highlighter'
 local parser = require 'plugins.evergreen.parser'
 local highlights = require 'plugins.evergreen.highlights'
 require 'plugins.evergreen.style'
-require 'plugins.evergreen.installer'
 
 --- @class core.doc
 --- @field treesit boolean
@@ -86,7 +93,7 @@ end
 local function accumulateLen(tbl, s, e)
 	local len = 0
 
-	for i=s,e do
+	for i = s, e do
 		len = len + tbl[i]:len()
 	end
 
@@ -184,7 +191,7 @@ end
 local oldTokenize = Highlight.tokenize_line
 function Highlight:tokenize_line(idx, state)
 	if not self.doc.treesit then return oldTokenize(self, idx, state) end
-	
+
 	local txt      = self.doc.lines[idx]
 	local row      = idx - 1
 	local toks     = {}
@@ -207,7 +214,7 @@ function Highlight:tokenize_line(idx, state)
 		local startPos = startPt.row < row and 1 or startPt.column + 1
 		local endPos   = endPt.row > row and #txt or endPt.column
 
-		local i = #buf - 1
+		local i        = #buf - 1
 		while i >= 1 and buf[i + 1] < startPos do
 			local e = buf[i + 1]
 			toks[#toks + 1] = buf[i]
@@ -237,7 +244,7 @@ function Highlight:tokenize_line(idx, state)
 
 		i = i - 2
 	end
-	
+
 	return {
 		init_state = state,
 		state      = state,
@@ -255,3 +262,91 @@ command.add('core.docview!', {
 		end
 	end
 })
+
+command.add(nil, {
+	['evergreen:status'] = function()
+		local notInstalled = {}
+		local installed = {}
+		local errorCounter = 0
+		for lang, options in pairs(languages.grammars) do
+			local lib = util.join { config.parserLocation, lang, 'parser.so' }
+			local queries = util.join { config.queryLocation, lang, 'highlights.scm' }
+			if not util.exists(lib) or not util.exists(queries) then
+				errorCounter = errorCounter + 1
+				table.insert(notInstalled, lang)
+			else
+				local str = ' - ' .. lang .. ': '
+				if options.extensions ~= nil then
+					str = str .. options.extensions .. ' '
+				end
+				if options.filename ~= nil then
+					str = str .. options.filename .. ' '
+				end
+				if options.filename == nil and options.extensions == nil then
+					str = str .. lang
+				end
+				table.insert(installed, str)
+			end
+		end
+		core.log('[Evergreen] Installed grammars:\n%s', table.concat(installed, '\n'))
+		if errorCounter > 0 then
+			core.warn('[Evergreen] grammars not operational:\n%s', table.concat(notInstalled, '\n'))
+		end
+	end
+})
+
+
+local exts = {}
+
+for k, _ in pairs(languages.grammars) do
+	table.insert(exts, k)
+end
+
+
+command.add(nil, {
+	['evergreen:update-all'] = function()
+		for lang, options in pairs(languages.grammars) do
+			core.log('[Evergreen] updating grammar for language "%s"', lang)
+			installer.installGrammar(options, config)
+		end
+	end
+})
+
+command.add(nil, {
+	['evergreen:update'] = function()
+		core.command_view:enter('Update Treesitter parser for', {
+			submit = function(lang)
+				if not languages.grammars[lang] then
+					core.error('Unknown parser for language ' .. lang)
+					return
+				end
+				core.log('Installing parser for ' .. lang)
+				installer.installGrammar(languages.grammars[lang], config)
+			end,
+			suggest = function()
+				return exts
+			end
+		})
+	end
+})
+
+command.add(nil, {
+	['evergreen:clean'] = function()
+		core.add_thread(function()
+			for _, parser_dir in pairs(system.list_dir(config.parserLocation)) do
+				local path = util.join { config.parserLocation, parser_dir }
+				if util.isDir(path) and languages.grammars[parser_dir] == nil then
+					core.log('[Evergreen] removing unused parser "%s"', parser_dir)
+					util.rmDir(path)
+				end
+			end
+			for _, query_dir in pairs(system.list_dir(config.queryLocation)) do
+				local path = util.join { config.queryLocation, query_dir }
+				if util.isDir(path) and languages.grammars[query_dir] == nil then
+					core.log('[Evergreen] removing unused queries "%s"', query_dir)
+					util.rmDir(path)
+				end
+			end
+		end)
+	end
+})