diff --git a/lib/ex_doc/autolink.ex b/lib/ex_doc/autolink.ex index 57d809e3b..32f4eb4ab 100644 --- a/lib/ex_doc/autolink.ex +++ b/lib/ex_doc/autolink.ex @@ -99,10 +99,13 @@ defmodule ExDoc.Autolink do if app in config.apps do path <> ext <> suffix else + #  TODO: remove this if/when hexdocs.pm starts including .md files + ext = ".html" + config.deps |> Keyword.get_lazy(app, fn -> base_url <> "#{app}" end) |> String.trim_trailing("/") - |> Kernel.<>("/" <> path <> ".html" <> suffix) + |> Kernel.<>("/" <> path <> ext <> suffix) end else path <> ext <> suffix diff --git a/lib/ex_doc/doc_ast.ex b/lib/ex_doc/doc_ast.ex index 375c670fa..969df04f7 100644 --- a/lib/ex_doc/doc_ast.ex +++ b/lib/ex_doc/doc_ast.ex @@ -27,7 +27,7 @@ defmodule ExDoc.DocAST do meta param source track wbr)a @doc """ - Transform AST into string. + Transform AST into an HTML string. """ def to_string(ast, fun \\ fn _ast, string -> string end) @@ -64,6 +64,70 @@ defmodule ExDoc.DocAST do Enum.map(attrs, fn {key, val} -> " #{key}=\"#{val}\"" end) end + @doc """ + Transform AST into a markdown string. + """ + def to_markdown_string(ast, fun \\ fn _ast, string -> string end) + + def to_markdown_string(binary, _fun) when is_binary(binary) do + ExDoc.Utils.h(binary) + end + + def to_markdown_string(list, fun) when is_list(list) do + result = Enum.map_join(list, "", &to_markdown_string(&1, fun)) + fun.(list, result) + end + + def to_markdown_string({:comment, _attrs, inner, _meta} = ast, fun) do + fun.(ast, "") + end + + def to_markdown_string({:code, _attrs, inner, _meta} = ast, fun) do + result = """ + ``` + #{inner} + ``` + """ + + fun.(ast, result) + end + + def to_markdown_string({:a, attrs, inner, _meta} = ast, fun) do + result = "[#{inner}](#{attrs[:href]})" + fun.(ast, result) + end + + def to_markdown_string({:hr, _attrs, _inner, _meta} = ast, fun) do + result = "\n\n---\n\n" + fun.(ast, result) + end + + def to_markdown_string({tag, _attrs, _inner, _meta} = ast, fun) when tag in [:p, :br] do + result = "\n\n" + fun.(ast, result) + end + + def to_markdown_string({:img, attrs, _inner, _meta} = ast, fun) do + result = "![#{attrs[:alt]}](#{attrs[:src]} \"#{attrs[:title]}\")" + fun.(ast, result) + end + + # ignoring these: area base col command embed input keygen link meta param source track wbr + def to_markdown_string({tag, _attrs, _inner, _meta} = ast, fun) when tag in @void_elements do + result = "" + fun.(ast, result) + end + + def to_markdown_string({_tag, _attrs, inner, %{verbatim: true}} = ast, fun) do + result = Enum.join(inner, "") + fun.(ast, result) + end + + def to_markdown_string({_tag, _attrs, inner, _meta} = ast, fun) do + result = to_string(inner, fun) + fun.(ast, result) + end + ## parse markdown defp parse_markdown(markdown, opts) do diff --git a/lib/ex_doc/formatter.ex b/lib/ex_doc/formatter.ex new file mode 100644 index 000000000..9ffdd70c6 --- /dev/null +++ b/lib/ex_doc/formatter.ex @@ -0,0 +1,247 @@ +defmodule ExDoc.Formatter do + @moduledoc false + + alias ExDoc.{Markdown, GroupMatcher, Utils} + + @doc """ + Autolinks and renders all docs. + """ + def render_all(project_nodes, filtered_modules, ext, config, opts) do + base = [ + apps: config.apps, + deps: config.deps, + ext: ext, + extras: extra_paths(config), + skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on, + skip_code_autolink_to: config.skip_code_autolink_to, + filtered_modules: filtered_modules + ] + + project_nodes + |> Task.async_stream( + fn node -> + language = node.language + + autolink_opts = + [ + current_module: node.module, + file: node.moduledoc_file, + line: node.moduledoc_line, + module_id: node.id, + language: language + ] ++ base + + docs = + for child_node <- node.docs do + id = id(node, child_node) + + autolink_opts = + autolink_opts ++ + [ + id: id, + line: child_node.doc_line, + file: child_node.doc_file, + current_kfa: {child_node.type, child_node.name, child_node.arity} + ] + + specs = Enum.map(child_node.specs, &language.autolink_spec(&1, autolink_opts)) + child_node = %{child_node | specs: specs} + render_doc(child_node, ext, language, autolink_opts, opts) + end + + %{ + render_doc(node, ext, language, [{:id, node.id} | autolink_opts], opts) + | docs: docs + } + end, + timeout: :infinity + ) + |> Enum.map(&elem(&1, 1)) + end + + defp render_doc(%{doc: nil} = node, _ext, _language, _autolink_opts, _opts), + do: node + + defp render_doc(%{doc: doc} = node, ext, language, autolink_opts, opts) do + rendered = autolink_and_render(doc, ext, language, autolink_opts, opts) + %{node | rendered_doc: rendered} + end + + defp id(%{id: mod_id}, %{id: "c:" <> id}) do + "c:" <> mod_id <> "." <> id + end + + defp id(%{id: mod_id}, %{id: "t:" <> id}) do + "t:" <> mod_id <> "." <> id + end + + defp id(%{id: mod_id}, %{id: id}) do + mod_id <> "." <> id + end + + defp autolink_and_render(doc, ".md", language, autolink_opts, _opts) do + doc + |> language.autolink_doc(autolink_opts) + |> ExDoc.DocAST.to_markdown_string() + end + + defp autolink_and_render(doc, _html_ext, language, autolink_opts, opts) do + doc + |> language.autolink_doc(autolink_opts) + |> ExDoc.DocAST.to_string() + |> ExDoc.DocAST.highlight(language, opts) + end + + @doc """ + Builds extra nodes by normalizing the config entries. + """ + def build_extras(config, ext) do + groups = config.groups_for_extras + + language = + case config.proglang do + :erlang -> ExDoc.Language.Erlang + _ -> ExDoc.Language.Elixir + end + + source_url_pattern = config.source_url_pattern + + autolink_opts = [ + apps: config.apps, + deps: config.deps, + ext: ext, + extras: extra_paths(config), + language: language, + skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on, + skip_code_autolink_to: config.skip_code_autolink_to + ] + + extras = + config.extras + |> Task.async_stream( + &build_extra(&1, groups, ext, language, autolink_opts, source_url_pattern), + timeout: :infinity + ) + |> Enum.map(&elem(&1, 1)) + + ids_count = Enum.reduce(extras, %{}, &Map.update(&2, &1.id, 1, fn c -> c + 1 end)) + + extras + |> Enum.map_reduce(1, fn extra, idx -> + if ids_count[extra.id] > 1, do: {disambiguate_id(extra, idx), idx + 1}, else: {extra, idx} + end) + |> elem(0) + |> Enum.sort_by(fn extra -> GroupMatcher.index(groups, extra.group) end) + end + + defp build_extra( + {input, input_options}, + groups, + ext, + language, + autolink_opts, + source_url_pattern + ) do + input = to_string(input) + id = input_options[:filename] || input |> filename_to_title() |> Utils.text_to_id() + source_file = input_options[:source] || input + opts = [file: source_file, line: 1] + + {source, ast} = + case extension_name(input) do + extension when extension in ["", ".txt"] -> + source = File.read!(input) + ast = [{:pre, [], "\n" <> source, %{}}] + {source, ast} + + extension when extension in [".md", ".livemd", ".cheatmd"] -> + source = File.read!(input) + + ast = + source + |> Markdown.to_ast(opts) + |> sectionize(extension) + + {source, ast} + + _ -> + raise ArgumentError, + "file extension not recognized, allowed extension is either .cheatmd, .livemd, .md, .txt or no extension" + end + + {title_ast, ast} = + case ExDoc.DocAST.extract_title(ast) do + {:ok, title_ast, ast} -> {title_ast, ast} + :error -> {nil, ast} + end + + title_text = title_ast && ExDoc.DocAST.text_from_ast(title_ast) + title_html = title_ast && ExDoc.DocAST.to_string(title_ast) + content_html = autolink_and_render(ast, ext, language, [file: input] ++ autolink_opts, opts) + + group = GroupMatcher.match_extra(groups, input) + title = input_options[:title] || title_text || filename_to_title(input) + + source_path = source_file |> Path.relative_to(File.cwd!()) |> String.replace_leading("./", "") + source_url = Utils.source_url_pattern(source_url_pattern, source_path, 1) + + %{ + source: source, + content: content_html, + group: group, + id: id, + source_path: source_path, + source_url: source_url, + title: title, + title_content: title_html || title + } + end + + defp build_extra(input, groups, ext, language, autolink_opts, source_url_pattern) do + build_extra({input, []}, groups, ext, language, autolink_opts, source_url_pattern) + end + + defp extra_paths(config) do + Map.new(config.extras, fn + path when is_binary(path) -> + base = Path.basename(path) + {base, Utils.text_to_id(Path.rootname(base))} + + {path, opts} -> + base = path |> to_string() |> Path.basename() + {base, opts[:filename] || Utils.text_to_id(Path.rootname(base))} + end) + end + + defp disambiguate_id(extra, discriminator) do + Map.put(extra, :id, "#{extra.id}-#{discriminator}") + end + + defp sectionize(ast, ".cheatmd") do + ExDoc.DocAST.sectionize(ast, fn + {:h2, _, _, _} -> true + {:h3, _, _, _} -> true + _ -> false + end) + end + + defp sectionize(ast, _), do: ast + + defp filename_to_title(input) do + input |> Path.basename() |> Path.rootname() + end + + def filter_list(:module, nodes) do + Enum.filter(nodes, &(&1.type != :task)) + end + + def filter_list(type, nodes) do + Enum.filter(nodes, &(&1.type == type)) + end + + def extension_name(input) do + input + |> Path.extname() + |> String.downcase() + end +end diff --git a/lib/ex_doc/formatter/epub.ex b/lib/ex_doc/formatter/epub.ex index 338cc497e..606b046c7 100644 --- a/lib/ex_doc/formatter/epub.ex +++ b/lib/ex_doc/formatter/epub.ex @@ -4,6 +4,7 @@ defmodule ExDoc.Formatter.EPUB do @mimetype "application/epub+zip" @assets_dir "OEBPS/assets" alias __MODULE__.{Assets, Templates} + alias ExDoc.Formatter alias ExDoc.Formatter.HTML alias ExDoc.Utils @@ -17,16 +18,18 @@ defmodule ExDoc.Formatter.EPUB do File.mkdir_p!(Path.join(config.output, "OEBPS")) project_nodes = - HTML.render_all(project_nodes, filtered_modules, ".xhtml", config, highlight_tag: "samp") + Formatter.render_all(project_nodes, filtered_modules, ".xhtml", config, + highlight_tag: "samp" + ) nodes_map = %{ - modules: HTML.filter_list(:module, project_nodes), - tasks: HTML.filter_list(:task, project_nodes) + modules: Formatter.filter_list(:module, project_nodes), + tasks: Formatter.filter_list(:task, project_nodes) } extras = config - |> HTML.build_extras(".xhtml") + |> Formatter.build_extras(".xhtml") |> Enum.chunk_by(& &1.group) |> Enum.map(&{hd(&1).group, &1}) diff --git a/lib/ex_doc/formatter/html.ex b/lib/ex_doc/formatter/html.ex index 1f07edef0..32cb19f05 100644 --- a/lib/ex_doc/formatter/html.ex +++ b/lib/ex_doc/formatter/html.ex @@ -2,7 +2,7 @@ defmodule ExDoc.Formatter.HTML do @moduledoc false alias __MODULE__.{Assets, Templates, SearchData} - alias ExDoc.{Markdown, GroupMatcher, Utils} + alias ExDoc.{Formatter, Utils} @main "api-reference" @assets_dir "assets" @@ -18,8 +18,8 @@ defmodule ExDoc.Formatter.HTML do build = Path.join(config.output, ".build") output_setup(build, config) - project_nodes = render_all(project_nodes, filtered_modules, ".html", config, []) - extras = build_extras(config, ".html") + project_nodes = Formatter.render_all(project_nodes, filtered_modules, ".html", config, []) + extras = Formatter.build_extras(config, ".html") # Generate search early on without api reference in extras static_files = generate_assets(".", default_assets(config), config) @@ -27,8 +27,8 @@ defmodule ExDoc.Formatter.HTML do # TODO: Move this categorization to the language nodes_map = %{ - modules: filter_list(:module, project_nodes), - tasks: filter_list(:task, project_nodes) + modules: Formatter.filter_list(:module, project_nodes), + tasks: Formatter.filter_list(:task, project_nodes) } extras = @@ -63,89 +63,6 @@ defmodule ExDoc.Formatter.HTML do %{config | main: main || @main} end - @doc """ - Autolinks and renders all docs. - """ - def render_all(project_nodes, filtered_modules, ext, config, opts) do - base = [ - apps: config.apps, - deps: config.deps, - ext: ext, - extras: extra_paths(config), - skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on, - skip_code_autolink_to: config.skip_code_autolink_to, - filtered_modules: filtered_modules - ] - - project_nodes - |> Task.async_stream( - fn node -> - language = node.language - - autolink_opts = - [ - current_module: node.module, - file: node.moduledoc_file, - line: node.moduledoc_line, - module_id: node.id, - language: language - ] ++ base - - docs = - for child_node <- node.docs do - id = id(node, child_node) - - autolink_opts = - autolink_opts ++ - [ - id: id, - line: child_node.doc_line, - file: child_node.doc_file, - current_kfa: {child_node.type, child_node.name, child_node.arity} - ] - - specs = Enum.map(child_node.specs, &language.autolink_spec(&1, autolink_opts)) - child_node = %{child_node | specs: specs} - render_doc(child_node, language, autolink_opts, opts) - end - - %{ - render_doc(node, language, [{:id, node.id} | autolink_opts], opts) - | docs: docs - } - end, - timeout: :infinity - ) - |> Enum.map(&elem(&1, 1)) - end - - defp render_doc(%{doc: nil} = node, _language, _autolink_opts, _opts), - do: node - - defp render_doc(%{doc: doc} = node, language, autolink_opts, opts) do - rendered = autolink_and_render(doc, language, autolink_opts, opts) - %{node | rendered_doc: rendered} - end - - defp id(%{id: mod_id}, %{id: "c:" <> id}) do - "c:" <> mod_id <> "." <> id - end - - defp id(%{id: mod_id}, %{id: "t:" <> id}) do - "t:" <> mod_id <> "." <> id - end - - defp id(%{id: mod_id}, %{id: id}) do - mod_id <> "." <> id - end - - defp autolink_and_render(doc, language, autolink_opts, opts) do - doc - |> language.autolink_doc(autolink_opts) - |> ExDoc.DocAST.to_string() - |> ExDoc.DocAST.highlight(language, opts) - end - defp output_setup(build, config) do if File.exists?(build) do build @@ -237,7 +154,7 @@ defmodule ExDoc.Formatter.HTML do defp copy_extras(config, extras) do for %{source_path: source_path, id: id} when source_path != nil <- extras, - ext = extension_name(source_path), + ext = Formatter.extension_name(source_path), ext == ".livemd" do output = "#{config.output}/#{id}#{ext}" @@ -320,48 +237,6 @@ defmodule ExDoc.Formatter.HTML do } end - @doc """ - Builds extra nodes by normalizing the config entries. - """ - def build_extras(config, ext) do - groups = config.groups_for_extras - - language = - case config.proglang do - :erlang -> ExDoc.Language.Erlang - _ -> ExDoc.Language.Elixir - end - - source_url_pattern = config.source_url_pattern - - autolink_opts = [ - apps: config.apps, - deps: config.deps, - ext: ext, - extras: extra_paths(config), - language: language, - skip_undefined_reference_warnings_on: config.skip_undefined_reference_warnings_on, - skip_code_autolink_to: config.skip_code_autolink_to - ] - - extras = - config.extras - |> Task.async_stream( - &build_extra(&1, groups, language, autolink_opts, source_url_pattern), - timeout: :infinity - ) - |> Enum.map(&elem(&1, 1)) - - ids_count = Enum.reduce(extras, %{}, &Map.update(&2, &1.id, 1, fn c -> c + 1 end)) - - extras - |> Enum.map_reduce(1, fn extra, idx -> - if ids_count[extra.id] > 1, do: {disambiguate_id(extra, idx), idx + 1}, else: {extra, idx} - end) - |> elem(0) - |> Enum.sort_by(fn extra -> GroupMatcher.index(groups, extra.group) end) - end - def generate_redirects(config, ext) do config.redirects |> Map.new() @@ -381,90 +256,6 @@ defmodule ExDoc.Formatter.HTML do end) end - defp disambiguate_id(extra, discriminator) do - Map.put(extra, :id, "#{extra.id}-#{discriminator}") - end - - defp build_extra({input, input_options}, groups, language, autolink_opts, source_url_pattern) do - input = to_string(input) - id = input_options[:filename] || input |> filename_to_title() |> Utils.text_to_id() - source_file = input_options[:source] || input - opts = [file: source_file, line: 1] - - {source, ast} = - case extension_name(input) do - extension when extension in ["", ".txt"] -> - source = File.read!(input) - ast = [{:pre, [], "\n" <> source, %{}}] - {source, ast} - - extension when extension in [".md", ".livemd", ".cheatmd"] -> - source = File.read!(input) - - ast = - source - |> Markdown.to_ast(opts) - |> sectionize(extension) - - {source, ast} - - _ -> - raise ArgumentError, - "file extension not recognized, allowed extension is either .cheatmd, .livemd, .md, .txt or no extension" - end - - {title_ast, ast} = - case ExDoc.DocAST.extract_title(ast) do - {:ok, title_ast, ast} -> {title_ast, ast} - :error -> {nil, ast} - end - - title_text = title_ast && ExDoc.DocAST.text_from_ast(title_ast) - title_html = title_ast && ExDoc.DocAST.to_string(title_ast) - content_html = autolink_and_render(ast, language, [file: input] ++ autolink_opts, opts) - - group = GroupMatcher.match_extra(groups, input) - title = input_options[:title] || title_text || filename_to_title(input) - - source_path = source_file |> Path.relative_to(File.cwd!()) |> String.replace_leading("./", "") - source_url = Utils.source_url_pattern(source_url_pattern, source_path, 1) - - %{ - source: source, - content: content_html, - group: group, - id: id, - source_path: source_path, - source_url: source_url, - title: title, - title_content: title_html || title - } - end - - defp build_extra(input, groups, language, autolink_opts, source_url_pattern) do - build_extra({input, []}, groups, language, autolink_opts, source_url_pattern) - end - - defp extension_name(input) do - input - |> Path.extname() - |> String.downcase() - end - - defp sectionize(ast, ".cheatmd") do - ExDoc.DocAST.sectionize(ast, fn - {:h2, _, _, _} -> true - {:h3, _, _, _} -> true - _ -> false - end) - end - - defp sectionize(ast, _), do: ast - - defp filename_to_title(input) do - input |> Path.basename() |> Path.rootname() - end - @doc """ Generates the logo from config into the given directory. """ @@ -522,14 +313,6 @@ defmodule ExDoc.Formatter.HTML do end end - def filter_list(:module, nodes) do - Enum.filter(nodes, &(&1.type != :task)) - end - - def filter_list(type, nodes) do - Enum.filter(nodes, &(&1.type == type)) - end - defp generate_list(nodes, nodes_map, config) do nodes |> Task.async_stream(&generate_module_page(&1, nodes_map, config), timeout: :infinity) @@ -556,16 +339,4 @@ defmodule ExDoc.Formatter.HTML do config end end - - defp extra_paths(config) do - Map.new(config.extras, fn - path when is_binary(path) -> - base = Path.basename(path) - {base, Utils.text_to_id(Path.rootname(base))} - - {path, opts} -> - base = path |> to_string() |> Path.basename() - {base, opts[:filename] || Utils.text_to_id(Path.rootname(base))} - end) - end end diff --git a/lib/ex_doc/formatter/html/templates.ex b/lib/ex_doc/formatter/html/templates.ex index 72407b672..235e441f1 100644 --- a/lib/ex_doc/formatter/html/templates.ex +++ b/lib/ex_doc/formatter/html/templates.ex @@ -64,7 +64,10 @@ defmodule ExDoc.Formatter.HTML.Templates do Regex.replace(~r|(<[^>]*) id="[^"]*"([^>]*>)|, doc, ~S"\1\2", []) end - defp enc(binary), do: URI.encode(binary) + defp presence([]), do: nil + defp presence(other), do: other + + def enc(binary), do: URI.encode(binary) @doc """ Create a JS object which holds all the items displayed in the sidebar area @@ -206,6 +209,21 @@ defmodule ExDoc.Formatter.HTML.Templates do defp relative_asset([h | _], output, _pattern), do: Path.relative_to(h, output) + defp get_hex_url(config, source_path) do + case config.package do + nil -> + nil + + package -> + base_url = "https://preview.hex.pm/preview/#{package}/#{config.version}" + if source_path, do: "#{base_url}/show/#{source_path}", else: base_url + end + end + + defp get_markdown_path(node) do + if node && node.id, do: URI.encode(node.id), else: "index" + end + # TODO: Move link_headings and friends to html.ex or even to autolinking code, # so content is built with it upfront instead of added at the template level. diff --git a/lib/ex_doc/formatter/html/templates/footer_template.eex b/lib/ex_doc/formatter/html/templates/footer_template.eex index 5488a0212..b9feb54ef 100644 --- a/lib/ex_doc/formatter/html/templates/footer_template.eex +++ b/lib/ex_doc/formatter/html/templates/footer_template.eex @@ -1,27 +1,43 @@