diff --git a/README.md b/README.md index d7ea2b9..4d2d1a9 100644 --- a/README.md +++ b/README.md @@ -60,27 +60,25 @@ Here is the default configuration: { dir_path = vim.fn.stdpath("data") .. "/devdocs", -- installation directory telescope = {}, -- passed to the telescope picker - telescope_alt = { -- when searching globally without preview - layout_config = { - width = 75, - }, - }, float_win = { -- passed to nvim_open_win(), see :h api-floatwin relative = "editor", height = 25, width = 100, border = "rounded", }, - previewer_cmd = nil, -- like glow - cmd_args = {}, -- example using glow { "-s", "dark", "-w", "80" } wrap = false, -- text wrap, only applies to floating window + previewer_cmd = nil, -- for example: "glow" + cmd_args = {}, -- example using glow: { "-s", "dark", "-w", "80" } + cmd_ignore = {}, -- ignore cmd rendering for the listed docs + picker_cmd = false, -- use cmd previewer in picker preview + picker_cmd_args = {}, -- example using glow: { "-p" } ensure_installed = {}, -- get automatically installed } ``` ## Usage -To use the documentations from nvim-devdocs you have to **install** the documentation using `:DevdocsInstall`. +To use the documentations from nvim-devdocs, you need to install it by executing `:DevdocsInstall`. The documentation is indexed and built during the download. Since the building process is done synchronously and may block input, you may want to download larger documents (more than 10MB) in headless mode: `nvim --headless +"DevdocsInstall rust"`. ## Commands @@ -98,10 +96,6 @@ Available commands: Commands support completion, and the Telescope picker will be used when no argument is provided. -> ℹ️ **NOTE**:
-> At the moment, Telescope's Previewer is available only when opening a specific documentation. -> E.g. `:DevdocsOpen javascript` - ## TODO - More search options diff --git a/lua/nvim-devdocs/build.lua b/lua/nvim-devdocs/build.lua new file mode 100644 index 0000000..4394482 --- /dev/null +++ b/lua/nvim-devdocs/build.lua @@ -0,0 +1,69 @@ +local path = require("plenary.path") + +local notify = require("nvim-devdocs.notify") +local plugin_config = require("nvim-devdocs.config").get() +local html_to_md = require("nvim-devdocs.transpiler").html_to_md + +local function build_docs(entry, index, docs) + local alias = entry.slug:gsub("~", "-") + + notify.log("Building " .. alias .. " documentation...") + + local docs_dir = path:new(plugin_config.dir_path, "docs") + local current_doc_dir = path:new(docs_dir, alias) + local index_path = path:new(plugin_config.dir_path, "index.json") + local lock_path = path:new(plugin_config.dir_path, "docs-lock.json") + + if not docs_dir:exists() then docs_dir:mkdir() end + if not current_doc_dir:exists() then current_doc_dir:mkdir() end + if not index_path:exists() then index_path:write("{}", "w") end + if not lock_path:exists() then lock_path:write("{}", "w") end + + local section_map = {} + local path_map = {} + + for _, index_entry in pairs(index.entries) do + local splited = vim.split(index_entry.path, "#") + local main = splited[1] + local id = splited[2] + + if not section_map[main] then section_map[main] = {} end + if id then table.insert(section_map[main], id) end + end + + local count = 1 + + for key, doc in pairs(docs) do + local sections = section_map[key] + + local markdown, md_sections = html_to_md(doc, sections) + + for id, md_path in pairs(md_sections) do + path_map[key .. "#" .. id] = count .. "," .. md_path + end + + path_map[key] = tostring(count) + + local file_path = path:new(current_doc_dir, tostring(count) .. ".md") + + file_path:write(markdown, "w") + count = count + 1 + end + + for i, index_entry in ipairs(index.entries) do + local main = vim.split(index_entry.path, "#")[1] + index.entries[i].path = path_map[index_entry.path] or path_map[main] + end + + local index_parsed = vim.fn.json_decode(index_path:read()) + index_parsed[alias] = index + index_path:write(vim.fn.json_encode(index_parsed), "w") + + local lock_parsed = vim.fn.json_decode(lock_path:read()) + lock_parsed[alias] = entry + lock_path:write(vim.fn.json_encode(lock_parsed), "w") + + notify.log("Build complete!") +end + +return build_docs diff --git a/lua/nvim-devdocs/config.lua b/lua/nvim-devdocs/config.lua index 52f541e..b4cf83a 100644 --- a/lua/nvim-devdocs/config.lua +++ b/lua/nvim-devdocs/config.lua @@ -3,20 +3,18 @@ local M = {} local config = { dir_path = vim.fn.stdpath("data") .. "/devdocs", telescope = {}, - telescope_alt = { - layout_config = { - width = 75, - }, - }, float_win = { relative = "editor", height = 25, width = 100, border = "rounded", }, + wrap = false, previewer_cmd = nil, cmd_args = {}, - wrap = false, + cmd_ignore = {}, + picker_cmd = false, + picker_cmd_args = {}, ensure_installed = {}, } diff --git a/lua/nvim-devdocs/list.lua b/lua/nvim-devdocs/list.lua index 7c9f242..87214ee 100644 --- a/lua/nvim-devdocs/list.lua +++ b/lua/nvim-devdocs/list.lua @@ -1,25 +1,19 @@ local M = {} local path = require("plenary.path") -local scandir = require("plenary.scandir") local notify = require("nvim-devdocs.notify") local plugin_config = require("nvim-devdocs.config").get() -local docs_dir = path:new(plugin_config.dir_path, "docs") -local index_path = path:new(plugin_config.dir_path, "index.json") +local lock_path = path:new(plugin_config.dir_path, "docs-lock.json") local registery_path = path:new(plugin_config.dir_path, "registery.json") M.get_installed_alias = function() - if not docs_dir:exists() then return {} end + if not lock_path:exists() then return {} end - local files = scandir.scan_dir(path.__tostring(docs_dir), { all_dirs = false }) - local installed = vim.tbl_map(function(file_path) - local splited = path._split(path:new(file_path)) - local filename = splited[#splited] - local basename = filename:gsub(".json", "") - return basename - end, files) + local lockfile = lock_path:read() + local lock_parsed = vim.fn.json_decode(lockfile) + local installed = vim.tbl_keys(lock_parsed) return installed end @@ -45,20 +39,20 @@ M.get_installed_entry = function() end M.get_updatable = function() - if not registery_path:exists() then return {} end - if not index_path:exists() then return {} end + if not registery_path:exists() or not lock_path:exists() then return {} end local results = {} - local registery_content = registery_path:read() - local registery_parsed = vim.fn.json_decode(registery_content) - local index_content = index_path:read() - local index_parsed = vim.fn.json_decode(index_content) - - for alias, value in pairs(index_parsed) do - local slug = alias:gsub("-", "~") + local registery = registery_path:read() + local registery_parsed = vim.fn.json_decode(registery) + local lockfile = lock_path:read() + local lock_parsed = vim.fn.json_decode(lockfile) + for alias, value in pairs(lock_parsed) do for _, doc in pairs(registery_parsed) do - if doc.slug == slug and doc.mtime > value.mtime then table.insert(results, alias) end + if doc.slug == value.slug and doc.mtime > value.mtime then + table.insert(results, alias) + break + end end end diff --git a/lua/nvim-devdocs/operations.lua b/lua/nvim-devdocs/operations.lua index 04832bb..a316835 100644 --- a/lua/nvim-devdocs/operations.lua +++ b/lua/nvim-devdocs/operations.lua @@ -6,12 +6,13 @@ local path = require("plenary.path") local list = require("nvim-devdocs.list") local notify = require("nvim-devdocs.notify") -local transpiler = require("nvim-devdocs.transpiler") local plugin_config = require("nvim-devdocs.config").get() +local build_docs = require("nvim-devdocs.build") local devdocs_site_url = "https://devdocs.io" local devdocs_cdn_url = "https://documents.devdocs.io" local docs_dir = path:new(plugin_config.dir_path, "docs") +local lock_path = path:new(plugin_config.dir_path, "docs-lock.json") local registery_path = path:new(plugin_config.dir_path, "registery.json") local index_path = path:new(plugin_config.dir_path, "index.json") @@ -42,41 +43,36 @@ M.install = function(entry, verbose, is_update) end local alias = entry.slug:gsub("~", "-") - local doc_path = path:new(docs_dir, alias .. ".json") + local installed = list.get_installed_alias() + local is_installed = vim.tbl_contains(installed, alias) - if not docs_dir:exists() then docs_dir:mkdir() end - if not index_path:exists() then index_path:write("{}", "w", 438) end - - if not is_update and doc_path:exists() then + if not is_update and is_installed then if verbose then notify.log("Documentation for " .. alias .. " is already installed") end else - local doc_url = string.format("%s/%s/db.json?%s", devdocs_cdn_url, entry.slug, entry.mtime) - - notify.log((is_update and "Updating " or "Installing ") .. alias .. " documentation...") - curl.get(doc_url, { - callback = function(response) - doc_path:write(response.body, "w", 438) - notify.log(alias .. " documentation has been " .. (is_update and "updated" or "installed")) - end, - on_error = function(error) - notify.log_err( - "nvim-devdocs[" .. alias .. "]: Error during download, exit code: " .. error.exit - ) - end, - }) + local callback = function(index) + local doc_url = string.format("%s/%s/db.json?%s", devdocs_cdn_url, entry.slug, entry.mtime) + + notify.log("Downloading " .. alias .. " documentation...") + curl.get(doc_url, { + callback = vim.schedule_wrap(function(response) + local docs = vim.fn.json_decode(response.body) + build_docs(entry, index, docs) + end), + on_error = function(error) + notify.log_err( + "nvim-devdocs[" .. alias .. "]: Error during download, exit code: " .. error.exit + ) + end, + }) + end local index_url = string.format("%s/%s/index.json?%s", devdocs_cdn_url, entry.slug, entry.mtime) + notify.log("Fetching " .. alias .. " documentation entries...") curl.get(index_url, { callback = vim.schedule_wrap(function(response) - local index_content = index_path:read() - local index_parsed = vim.fn.json_decode(index_content) - local response_parsed = vim.fn.json_decode(response.body) - - response_parsed.mtime = entry.mtime - index_parsed[alias] = response_parsed - index_path:write(vim.fn.json_encode(index_parsed), "w", 438) - notify.log(alias .. " documentation has been indexed") + local index = vim.fn.json_decode(response.body) + callback(index) end), on_error = function(error) notify.log_err( @@ -121,65 +117,39 @@ M.install_args = function(args, verbose, is_update) end M.uninstall = function(alias) - local file_path = path:new(docs_dir, alias .. ".json") + local installed = list.get_installed_alias() - if not file_path:exists() then + if not vim.tbl_contains(installed, alias) then notify.log(alias .. " documentation is already uninstalled") else - local index = index_path:read() - local parsed = vim.fn.json_decode(index) + local index = vim.fn.json_decode(index_path:read()) + local lockfile = vim.fn.json_decode(lock_path:read()) + local doc_path = path:new(docs_dir, alias) - parsed[alias] = nil + index[alias] = nil + lockfile[alias] = nil - if vim.tbl_isempty(parsed) then - index_path:write("{}", "w", 438) - else - index_path:write(vim.fn.json_encode(parsed), "w", 438) - end + index_path:write(vim.fn.json_encode(index), "w") + lock_path:write(vim.fn.json_encode(lockfile), "w") + doc_path:rm({ recursive = true }) - file_path:rm() notify.log(alias .. " documentation has been uninstalled") end end -M.get_entry = function(alias, entry_path) - local file_path = path:new(plugin_config.dir_path, "docs", alias .. ".json") - - if index_path:exists() or not file_path:exists() then - local content = file_path:read() - local parsed = vim.fn.json_decode(content) - local main_path = vim.split(entry_path, "#")[1] - local entry = { key = main_path, value = parsed[main_path] } - - return entry - end -end - M.get_entries = function(alias) - local file_path = path:new(plugin_config.dir_path, "docs", alias .. ".json") + local installed = list.get_installed_alias() + local is_installed = vim.tbl_contains(installed, alias) - if not index_path:exists() or not file_path:exists() then return end + if not index_path:exists() or not is_installed then return end - local entries = {} - local index_content = index_path:read() - local index_parsed = vim.fn.json_decode(index_content) - local docs_content = file_path:read() - local docs_decoded = vim.fn.json_decode(docs_content) - - for _, entry in pairs(index_parsed[alias].entries) do - local doc = "" - local entry_path = vim.split(entry.path, "#") - local local_path = entry_path[2] and entry_path[2] or entry_path[1] - - for doc_entry, value in pairs(docs_decoded) do - if string.lower(doc_entry) == string.lower(entry_path[1]) then doc = value end - end + local index_parsed = vim.fn.json_decode(index_path:read()) + local entries = index_parsed[alias].entries - table.insert(entries, { name = entry.name, path = local_path, value = doc }) + for key, _ in ipairs(entries) do + entries[key].alias = alias end - table.insert(entries, { name = "index", path = "index", value = docs_decoded["index"] }) - return entries end @@ -187,8 +157,7 @@ M.get_all_entries = function() if not index_path:exists() then return {} end local entries = {} - local index_content = index_path:read() - local index_parsed = vim.fn.json_decode(index_content) + local index_parsed = vim.fn.json_decode(index_path:read()) for alias, index in pairs(index_parsed) do for _, doc_entry in ipairs(index.entries) do @@ -204,16 +173,11 @@ M.get_all_entries = function() return entries end -M.open = function(entry, float) - local markdown = transpiler.html_to_md(entry.value) - local lines = vim.split(markdown, "\n") - local buf = vim.api.nvim_create_buf(not float, true) - - vim.api.nvim_buf_set_lines(buf, 0, -1, false, lines) - vim.api.nvim_buf_set_option(buf, "modifiable", false) +M.open = function(alias, bufnr, pattern, float) + vim.api.nvim_buf_set_option(bufnr, "modifiable", false) if not float then - vim.api.nvim_set_current_buf(buf) + vim.api.nvim_set_current_buf(bufnr) else local ui = vim.api.nvim_list_uis()[1] local row = (ui.height - plugin_config.float_win.height) * 0.5 @@ -223,7 +187,7 @@ M.open = function(entry, float) if not plugin_config.row then float_opts.row = row end if not plugin_config.col then float_opts.col = col end - local win = vim.api.nvim_open_win(buf, true, float_opts) + local win = vim.api.nvim_open_win(bufnr, true, float_opts) vim.wo[win].wrap = plugin_config.wrap vim.wo[win].linebreak = plugin_config.wrap @@ -231,8 +195,14 @@ M.open = function(entry, float) vim.wo[win].relativenumber = false end - if plugin_config.previewer_cmd then - local chan = vim.api.nvim_open_term(buf, {}) + vim.fn.search(pattern) + + local ignore = vim.tbl_contains(plugin_config.cmd_ignore, alias) + if plugin_config.previewer_cmd and not ignore then + vim.bo[bufnr].ft = "glow" + + local lines = vim.api.nvim_buf_get_lines(bufnr, 0, -1, false) + local chan = vim.api.nvim_open_term(bufnr, {}) local previewer = job:new({ command = plugin_config.previewer_cmd, args = plugin_config.cmd_args, @@ -242,11 +212,18 @@ M.open = function(entry, float) vim.api.nvim_chan_send(chan, line .. "\r\n") end end), - writer = markdown, + writer = table.concat(lines, "\n"), }) previewer:start() + + if pattern then + local formatted_pattern = pattern:gsub("`", "") + + -- TODO: wait for the rendering before jumping + vim.defer_fn(function() vim.fn.search(formatted_pattern) end, 500) + end else - vim.bo[buf].ft = "markdown" + vim.bo[bufnr].ft = "markdown" end end diff --git a/lua/nvim-devdocs/pickers.lua b/lua/nvim-devdocs/pickers.lua index 7d6b5ae..93bc303 100644 --- a/lua/nvim-devdocs/pickers.lua +++ b/lua/nvim-devdocs/pickers.lua @@ -4,6 +4,7 @@ local path = require("plenary.path") local finders = require("telescope.finders") local pickers = require("telescope.pickers") local previewers = require("telescope.previewers") +local state = require("telescope.state") local actions = require("telescope.actions") local action_state = require("telescope.actions.state") local config = require("telescope.config").values @@ -33,7 +34,7 @@ local new_docs_picker = function(prompt, entries, previwer, attach) }) end -local metadata_priewer = previewers.new_buffer_previewer({ +local metadata_previewer = previewers.new_buffer_previewer({ title = "Metadata", define_preview = function(self, entry) local bufnr = self.state.bufnr @@ -45,6 +46,67 @@ local metadata_priewer = previewers.new_buffer_previewer({ end, }) +local buf_doc_previewer = previewers.new_buffer_previewer({ + title = "Preview", + keep_last_buf = true, + define_preview = function(self, entry) + local splited_path = vim.split(entry.value.path, ",") + local file = splited_path[1] + local file_path = path:new(plugin_config.dir_path, "docs", entry.value.alias, file .. ".md") + local bufnr = self.state.bufnr + + local display_lines = function(lines) + vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, lines) + vim.bo[bufnr].ft = "markdown" + -- TODO: highlight in picker + -- vim.api.nvim_buf_call(bufnr, function() + -- vim.fn.search(section) + -- vim.fn.matchadd("Search", pattern) + -- end) + end + + file_path:_read_async(vim.schedule_wrap(function(content) + local lines = vim.split(content, "\n") + display_lines(lines) + end)) + end, +}) + +local term_doc_previewer = previewers.new_termopen_previewer({ + title = "Preview", + get_command = function(entry) + local splited_path = vim.split(entry.value.path, ",") + local file = splited_path[1] + local file_path = path:new(plugin_config.dir_path, "docs", entry.value.alias, file .. ".md") + local args = { plugin_config.previewer_cmd } + + vim.list_extend(args, plugin_config.picker_cmd_args) + table.insert(args, path.__tostring(file_path)) + + return args + end, +}) + +local open_doc = function(float) + local bufnr + local selection = action_state.get_selected_entry() + + if plugin_config.picker_cmd then + bufnr = vim.api.nvim_create_buf(false, true) + local splited_path = vim.split(selection.value.path, ",") + local file = splited_path[1] + local file_path = path:new(plugin_config.dir_path, "docs", selection.value.alias, file .. ".md") + local content = file_path:read() + vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, vim.split(content, "\n")) + else + bufnr = state.get_global_key("last_preview_bufnr") + end + + local splited_path = vim.split(selection.value.path, ",") + + operations.open(selection.value.alias, bufnr, splited_path[2], float) +end + M.installation_picker = function() local registery_path = path:new(plugin_config.dir_path, "registery.json") @@ -55,7 +117,7 @@ M.installation_picker = function() local content = registery_path:read() local parsed = vim.fn.json_decode(content) - local picker = new_docs_picker("Install documentation", parsed, metadata_priewer, function() + local picker = new_docs_picker("Install documentation", parsed, metadata_previewer, function() actions.select_default:replace(function(prompt_bufnr) local selection = action_state.get_selected_entry() @@ -70,23 +132,28 @@ end M.uninstallation_picker = function() local installed = list.get_installed_entry() - local picker = new_docs_picker("Uninstall documentation", installed, metadata_priewer, function() - actions.select_default:replace(function(prompt_bufnr) - local selection = action_state.get_selected_entry() - local alias = selection.value.slug:gsub("~", "-") + local picker = new_docs_picker( + "Uninstall documentation", + installed, + metadata_previewer, + function() + actions.select_default:replace(function(prompt_bufnr) + local selection = action_state.get_selected_entry() + local alias = selection.value.slug:gsub("~", "-") - actions.close(prompt_bufnr) - operations.uninstall(alias) - end) - return true - end) + actions.close(prompt_bufnr) + operations.uninstall(alias) + end) + return true + end + ) picker:find() end M.update_picker = function() local installed = list.get_updatable() - local picker = new_docs_picker("Update documentation", installed, metadata_priewer, function() + local picker = new_docs_picker("Update documentation", installed, metadata_previewer, function() actions.select_default:replace(function(prompt_bufnr) local selection = action_state.get_selected_entry() local alias = selection.value.slug:gsub("~", "-") @@ -108,6 +175,12 @@ M.open_picker = function(alias, float) return end + local previewer = buf_doc_previewer + + if plugin_config.previewer_cmd and plugin_config.previewer_cmd then + previewer = term_doc_previewer + end + local picker = pickers.new(plugin_config.telescope, { prompt_title = "Select an entry", finder = finders.new_table({ @@ -121,23 +194,11 @@ M.open_picker = function(alias, float) end, }), sorter = config.generic_sorter(plugin_config.telescope), - previewer = previewers.new_buffer_previewer({ - title = "Preview", - define_preview = function(self, entry) - local bufnr = self.state.bufnr - local markdown = transpiler.html_to_md(entry.value.value) - local lines = vim.split(markdown, "\n") - - vim.api.nvim_buf_set_lines(bufnr, 0, -1, false, lines) - vim.bo[bufnr].ft = "markdown" - end, - }), + previewer = previewer, attach_mappings = function() actions.select_default:replace(function(prompt_bufnr) - local selection = action_state.get_selected_entry() - actions.close(prompt_bufnr) - operations.open(selection.value, float) + open_doc(float) end) return true end, @@ -148,7 +209,13 @@ end M.global_search_picker = function(float) local entries = operations.get_all_entries() - local picker = pickers.new(plugin_config.telescope_alt, { + local previewer = buf_doc_previewer + + if plugin_config.previewer_cmd and plugin_config.previewer_cmd then + previewer = term_doc_previewer + end + + local picker = pickers.new(plugin_config.telescope, { prompt_title = "Select an entry", finder = finders.new_table({ results = entries, @@ -160,21 +227,12 @@ M.global_search_picker = function(float) } end, }), - sorter = config.generic_sorter(plugin_config.telescope_alt), + sorter = config.generic_sorter(plugin_config.telescope), + previewer = previewer, attach_mappings = function() actions.select_default:replace(function(prompt_bufnr) actions.close(prompt_bufnr) - - local selection = action_state.get_selected_entry() - local entry_path = selection.value.path - local alias = selection.value.alias - local entry = operations.get_entry(alias, entry_path) - - if entry then - operations.open(entry, float) - else - notify.log_err(alias .. " documentation is not installed") - end + open_doc(float) end) return true diff --git a/lua/nvim-devdocs/transpiler.lua b/lua/nvim-devdocs/transpiler.lua index 8ef48cc..600e8a2 100644 --- a/lua/nvim-devdocs/transpiler.lua +++ b/lua/nvim-devdocs/transpiler.lua @@ -1,4 +1,4 @@ -M = {} +local M = {} local normalize_html = function(str) str = str:gsub("<", "<") @@ -70,7 +70,7 @@ local tag_mappings = { mi = {}, br = { right = "\n" }, - hr = { right = "---" }, + hr = { right = "---\n\n" }, } local skipable_tag = { @@ -84,316 +84,337 @@ local skipable_tag = { "tbody", } -M.to_yaml = function(entry) - local lines = {} +local transpiler = {} - for key, value in pairs(entry) do - if key == "attribution" then - value = normalize_html(value) - value = value:gsub("(.*)", "%1") - value = value:gsub("
", "") - value = value:gsub("\n *", " ") - end - if key == "links" then value = vim.fn.json_encode(value) end - table.insert(lines, key .. ": " .. value) - end - - return table.concat(lines, "\n") -end - -M.html_to_md = function(html) - local transpiler = { - parser = vim.treesitter.get_string_parser(html, "html"), - lines = vim.split(html, "\n"), +function transpiler:new(source, section_map) + local new = { + parser = vim.treesitter.get_string_parser(source, "html"), + lines = vim.split(source, "\n"), result = "", + section_map = section_map, + sections = {}, } + new.parser:parse() + self.__index = self - function transpiler:get_text_range(row_start, col_start, row_end, col_end) - local extracted_lines = {} + return setmetatable(new, self) +end - for i = row_start, row_end do - local line = self.lines[i + 1] +function transpiler:get_text_range(row_start, col_start, row_end, col_end) + local extracted_lines = {} - if row_start == row_end then - line = line:sub(col_start + 1, col_end) - elseif i == row_start then - line = line:sub(col_start + 1) - elseif i == row_end then - line = line:sub(1, col_end) - end + for i = row_start, row_end do + local line = self.lines[i + 1] - table.insert(extracted_lines, line) + if row_start == row_end then + line = line:sub(col_start + 1, col_end) + elseif i == row_start then + line = line:sub(col_start + 1) + elseif i == row_end then + line = line:sub(1, col_end) end - return table.concat(extracted_lines, "\n") + table.insert(extracted_lines, line) end - ---@param node TSNode - function transpiler:get_node_text(node) - local row_start, col_start = node:start() - local row_end, col_end = node:end_() - local text = self:get_text_range(row_start, col_start, row_end, col_end) + return table.concat(extracted_lines, "\n") +end - return text - end +---@param node TSNode +function transpiler:get_node_text(node) + if not node then return "" end - ---@param node TSNode - function transpiler:get_node_tag_name(node) - local tag_name = nil - local child = node:named_child() + local row_start, col_start = node:start() + local row_end, col_end = node:end_() + local text = self:get_text_range(row_start, col_start, row_end, col_end) - if child then - local tag_node = child:named_child() - if tag_node then tag_name = self:get_node_text(tag_node) end - end + return text +end + +---@param node TSNode +function transpiler:get_node_tag_name(node) + local tag_name = nil + local child = node:named_child() - return tag_name + if child then + local tag_node = child:named_child() + if tag_node then tag_name = self:get_node_text(tag_node) end end - ---@param node TSNode - function transpiler:get_node_attributes(node) - local attributes = {} - local tag_node = node:named_child() + return tag_name +end - if tag_node == nil then return {} end +---@param node TSNode +function transpiler:get_node_attributes(node) + if not node then return {} end - local tag_children = tag_node:named_children() + local attributes = {} + local tag_node = node:named_child() - for i = 2, #tag_children do - local attribute_node = tag_children[i] - local attribute_name_node = attribute_node:named_child() - local attribute_name = self:get_node_text(attribute_name_node) - local value = "" + if tag_node == nil then return {} end - if attribute_name_node:next_named_sibling() then - local quotetd_value_node = attribute_name_node:next_named_sibling() - local value_node = quotetd_value_node:named_child() - if value_node then value = self:get_node_text(value_node) end - end + local tag_children = tag_node:named_children() - attributes[attribute_name] = value + for i = 2, #tag_children do + local attribute_node = tag_children[i] + local attribute_name_node = attribute_node:named_child() + local attribute_name = self:get_node_text(attribute_name_node) + local value = "" + + if attribute_name_node and attribute_name_node:next_named_sibling() then + local quotetd_value_node = attribute_name_node:next_named_sibling() + local value_node = quotetd_value_node:named_child() + if value_node then value = self:get_node_text(value_node) end end - return attributes + attributes[attribute_name] = value end - ---@param node TSNode - ---@return TSNode[] - function transpiler:filter_tag_children(node) - local children = node:named_children() - local filtered = vim.tbl_filter(function(child) - local type = child:type() - return type ~= "start_tag" and type ~= "end_tag" - end, children) + return attributes +end - return filtered - end +---@param node TSNode +---@return TSNode[] +function transpiler:filter_tag_children(node) + local children = node:named_children() + local filtered = vim.tbl_filter(function(child) + local type = child:type() + return type ~= "start_tag" and type ~= "end_tag" + end, children) - ---@param node TSNode - function transpiler:eval(node) - local result = "" - local node_type = node:type() - local node_text = self:get_node_text(node) - - if node_type == "text" or node_type == "entity" then - result = result .. normalize_html(node_text) - elseif node_type == "element" then - local tag_node = node:named_child() - local tag_type = tag_node:type() - local tag_name = self:get_node_text(tag_node:named_child()) - local attributes = self:get_node_attributes(node) - - if tag_type == "start_tag" then - local children = self:filter_tag_children(node) - - for _, child in ipairs(children) do - result = result .. self:eval_child(child, tag_name) - end + return filtered +end + +---@param node TSNode +function transpiler:eval(node) + local result = "" + local node_type = node:type() + local node_text = self:get_node_text(node) + local attributes = self:get_node_attributes(node) + + if node_type == "text" or node_type == "entity" then + result = result .. normalize_html(node_text) + elseif node_type == "element" then + local tag_node = node:named_child() + local tag_type = tag_node:type() + local tag_name = self:get_node_text(tag_node:named_child()) + + if tag_type == "start_tag" then + local children = self:filter_tag_children(node) + + for _, child in ipairs(children) do + result = result .. self:eval_child(child, tag_name) end + end - if vim.tbl_contains(skipable_tag, tag_name) then return "" end - - if tag_name == "a" then - result = string.format("[%s](%s)", result, attributes.href) - elseif tag_name == "img" then - result = string.format("![%s](%s)", attributes.alt, attributes.src) - elseif tag_name == "pre" and attributes["data-language"] then - result = "\n```" .. attributes["data-language"] .. "\n" .. result .. "\n```\n" - elseif tag_name == "abbr" then - result = string.format("%s(%s)", result, attributes.title) - elseif tag_name == "iframe" then - result = string.format("[%s](%s)\n", attributes.title, attributes.src) - elseif tag_name == "details" then - result = "..." - elseif tag_name == "table" then - result = self:eval_table(node) .. "\n" - elseif tag_name == "li" then - local parent_node = node:parent() - local parent_tag_name = self:get_node_tag_name(parent_node) - - if parent_tag_name == "ul" then result = "- " .. result .. "\n" end - if parent_tag_name == "ol" then - local siblings = self:filter_tag_children(parent_node) - for i, sibling in ipairs(siblings) do - if node:equal(sibling) then result = i .. ". " .. result .. "\n" end - end + if vim.tbl_contains(skipable_tag, tag_name) then return "" end + + if tag_name == "a" then + result = string.format("[%s](%s)", result, attributes.href) + elseif tag_name == "img" then + result = string.format("![%s](%s)", attributes.alt, attributes.src) + elseif tag_name == "pre" and attributes["data-language"] then + result = "\n```" .. attributes["data-language"] .. "\n" .. result .. "\n```\n" + elseif tag_name == "abbr" then + result = string.format("%s(%s)", result, attributes.title) + elseif tag_name == "iframe" then + result = string.format("[%s](%s)\n", attributes.title, attributes.src) + elseif tag_name == "details" then + result = "..." + elseif tag_name == "table" then + result = self:eval_table(node) .. "\n" + elseif tag_name == "li" then + local parent_node = node:parent() + local parent_tag_name = self:get_node_tag_name(parent_node) + + if parent_tag_name == "ul" then result = "- " .. result .. "\n" end + if parent_tag_name == "ol" then + local siblings = self:filter_tag_children(parent_node) + for i, sibling in ipairs(siblings) do + if node:equal(sibling) then result = i .. ". " .. result .. "\n" end end + end + else + local map = tag_mappings[tag_name] + if map then + local left = map.left and map.left or "" + local right = map.right and map.right or "" + result = left .. result .. right else - local map = tag_mappings[tag_name] - if map then - local left = map.left and map.left or "" - local right = map.right and map.right or "" - result = left .. result .. right - else - result = result .. node_text - end + result = result .. node_text end end + end + + -- use the markdown text for indexing docs + local id = attributes.id - return result + if id and self.section_map and vim.tbl_contains(self.section_map, id) then + self.sections[id] = vim.trim(result) end - ---@param node TSNode - function transpiler:eval_child(node, parent_tag) - local result = self:eval(node) - local tag_name = self:get_node_tag_name(node) - local sibling = node:next_named_sibling() - - -- check if there should be additional spaces/characters between two elements - if sibling then - local c_row_end, c_col_end = node:end_() - local s_row_start, s_col_start = sibling:start() - - if parent_tag == "pre" then - local row, col = c_row_end, c_col_end - while row ~= s_row_start or col ~= s_col_start do - local char = self:get_text_range(row, col, row, col + 1) - if char ~= "" then - result = result .. char - col = col + 1 - else - result = result .. "\n" - row, col = row + 1, 0 - end + return result +end + +---@param node TSNode +function transpiler:eval_child(node, parent_tag) + local result = self:eval(node) + local tag_name = self:get_node_tag_name(node) + local sibling = node:next_named_sibling() + + -- check if there should be additional spaces/characters between two elements + if sibling then + local c_row_end, c_col_end = node:end_() + local s_row_start, s_col_start = sibling:start() + + if parent_tag == "pre" then + local row, col = c_row_end, c_col_end + while row ~= s_row_start or col ~= s_col_start do + local char = self:get_text_range(row, col, row, col + 1) + if char ~= "" then + result = result .. char + col = col + 1 + else + result = result .. "\n" + row, col = row + 1, 0 end - else - local is_inline = is_inline_tag(tag_name) or not tag_name -- is text - if is_inline and c_col_end ~= s_col_start then result = result .. " " end end + else + local is_inline = is_inline_tag(tag_name) or not tag_name -- is text + if is_inline and c_col_end ~= s_col_start then result = result .. " " end end - - return result end - ---@param node TSNode - function transpiler:eval_table(node) - local result = "" - local children = self:filter_tag_children(node) - ---@type TSNode[] - local tr_nodes = {} - local first_child_tag = self:get_node_tag_name(children[1]) + return result +end - if first_child_tag == "tr" then - vim.list_extend(tr_nodes, children) - else - -- extracts tr from thead, tbody - for _, child in ipairs(children) do - vim.list_extend(tr_nodes, self:filter_tag_children(child)) - end +---@param node TSNode +function transpiler:eval_table(node) + local result = "" + local children = self:filter_tag_children(node) + ---@type TSNode[] + local tr_nodes = {} + local first_child_tag = self:get_node_tag_name(children[1]) + + if first_child_tag == "tr" then + vim.list_extend(tr_nodes, children) + else + -- extracts tr from thead, tbody + for _, child in ipairs(children) do + vim.list_extend(tr_nodes, self:filter_tag_children(child)) end + end - local max_col_len_map = {} - local result_map = {} - local colspan_map = {} + local max_col_len_map = {} + local result_map = {} + local colspan_map = {} - for i, tr in ipairs(tr_nodes) do - local tr_children = self:filter_tag_children(tr) - result_map[i] = {} - colspan_map[i] = {} + for i, tr in ipairs(tr_nodes) do + local tr_children = self:filter_tag_children(tr) + result_map[i] = {} + colspan_map[i] = {} - for j, tcol_node in ipairs(tr_children) do - local inner_result = "" - local tcol_children = self:filter_tag_children(tcol_node) - local attributes = self:get_node_attributes(tcol_node) + for j, tcol_node in ipairs(tr_children) do + local inner_result = "" + local tcol_children = self:filter_tag_children(tcol_node) + local attributes = self:get_node_attributes(tcol_node) - for _, tcol_child in ipairs(tcol_children) do - inner_result = self:eval(tcol_child) - end + for _, tcol_child in ipairs(tcol_children) do + inner_result = self:eval(tcol_child) + end + + result_map[i][j] = inner_result + colspan_map[i][j] = attributes.colspan and attributes.colspan or 1 - result_map[i][j] = inner_result - colspan_map[i][j] = attributes.colspan and attributes.colspan or 1 + if max_col_len_map[j] == nil then max_col_len_map[j] = 0 end + if max_col_len_map[j] < #inner_result then max_col_len_map[j] = #inner_result end + end + end - if max_col_len_map[j] == nil then max_col_len_map[j] = 0 end - if max_col_len_map[j] < #inner_result then max_col_len_map[j] = #inner_result end + -- draws columns evenly + for i = 1, #tr_nodes do + local current_col = 1 + for j, value in ipairs(result_map[i]) do + local col_len = max_col_len_map[current_col] + local colspan = tonumber(colspan_map[i][j]) + result = result .. "| " .. value .. string.rep(" ", col_len - #value + 1) + current_col = current_col + 1 + + if colspan > 1 then + local len = current_col + colspan - 1 + while current_col < len do + local spacing = max_col_len_map[current_col] + if spacing then result = result .. string.rep(" ", spacing + 3) end + current_col = current_col + 1 + end end end - -- draws columns evenly - for i = 1, #tr_nodes do - local current_col = 1 - for j, value in ipairs(result_map[i]) do + result = result .. "|\n" + + -- generates row separator + if i == 1 then + current_col = 1 + for j = 1, #result_map[i] do local col_len = max_col_len_map[current_col] local colspan = tonumber(colspan_map[i][j]) - result = result .. "| " .. value .. string.rep(" ", col_len - #value + 1) + local line = string.rep("-", col_len) current_col = current_col + 1 if colspan > 1 then local len = current_col + colspan - 1 while current_col < len do local spacing = max_col_len_map[current_col] - if spacing then result = result .. string.rep(" ", spacing + 3) end + if spacing then line = line .. string.rep("-", spacing + 3) end current_col = current_col + 1 end end + result = result .. "| " .. line .. " " end result = result .. "|\n" + end + end - -- generates row separator - if i == 1 then - current_col = 1 - for j = 1, #result_map[i] do - local col_len = max_col_len_map[current_col] - local colspan = tonumber(colspan_map[i][j]) - local line = string.rep("-", col_len) - current_col = current_col + 1 - - if colspan > 1 then - local len = current_col + colspan - 1 - while current_col < len do - local spacing = max_col_len_map[current_col] - if spacing then line = line .. string.rep("-", spacing + 3) end - current_col = current_col + 1 - end - end - result = result .. "| " .. line .. " " - end + return result +end - result = result .. "|\n" +function transpiler:transpile() + self.parser:for_each_tree(function(tree) + local root = tree:root() + if root then + local children = root:named_children() + for _, node in ipairs(children) do + self.result = self.result .. self:eval(node) end end + end) - return result - end + self.result = self.result:gsub("\n\n\n+", "\n\n") - function transpiler:transpile() - self.parser:parse() - self.parser:for_each_tree(function(tree) - local root = tree:root() - if root then - local children = root:named_children() - for _, node in ipairs(children) do - self.result = self.result .. self:eval(node) - end - end - end) + return self.result, self.sections +end - self.result = self.result:gsub("\n\n\n+", "\n\n") +M.to_yaml = function(entry) + local lines = {} - return self.result + for key, value in pairs(entry) do + if key == "attribution" then + value = normalize_html(value) + value = value:gsub("(.*)", "%1") + value = value:gsub("
", "") + value = value:gsub("\n *", " ") + end + if key == "links" then value = vim.fn.json_encode(value) end + table.insert(lines, key .. ": " .. value) end - return transpiler:transpile() + return table.concat(lines, "\n") +end + +M.html_to_md = function(html, section_map) + local t = transpiler:new(html, section_map) + return t:transpile() end return M