feat: allow extensions to manually insert pages into the graph (#96)

fix: bump MDEx and fix breaking change

Author: mhanberg

lib/mix/tasks/tableau.build.ex

@@ -12,19 +12,22 @@ defmodule Mix.Tasks.Tableau.Build do
   def run(argv) do
     Application.ensure_all_started(:telemetry)
     {:ok, config} = Tableau.Config.new(@config)
-    token = %{site: %{config: config}}
+    token = %{site: %{config: config}, graph: Graph.new()}
     Mix.Task.run("app.start", ["--preload-modules"])
 
     {opts, _argv} = OptionParser.parse!(argv, strict: [out: :string])
 
     out = Keyword.get(opts, :out, "_site")
 
-    mods = :code.all_available()
+    mods =
+      :code.all_available()
+      |> Task.async_stream(fn {mod, _, _} -> Module.concat([to_string(mod)]) end)
+      |> Stream.map(fn {:ok, mod} -> mod end)
+      |> Enum.to_list()
 
-    token = mods |> pre_build_extensions() |> run_extensions(token)
+    token = mods |> extensions_for(:pre_build) |> run_extensions(token)
 
-    mods = :code.all_available()
-    graph = Tableau.Graph.new(mods)
+    graph = Tableau.Graph.insert(token.graph, mods)
 
     File.mkdir_p!(out)
 
@@ -37,7 +40,7 @@ defmodule Mix.Tasks.Tableau.Build do
 
     token = put_in(token.site[:pages], just_pages)
 
-    token = mods |> pre_write_extensions() |> run_extensions(token)
+    token = mods |> extensions_for(:pre_write) |> run_extensions(token)
 
     for {mod, page} <- Enum.zip(page_mods, token.site.pages) do
       content = Tableau.Document.render(graph, mod, token, page)
@@ -53,7 +56,7 @@ defmodule Mix.Tasks.Tableau.Build do
       File.cp_r!(config.include_dir, out)
     end
 
-    token = run_extensions(post_write_extensions(mods), token)
+    token = mods |> extensions_for(:post_write) |> run_extensions(token)
 
     token
   end
@@ -66,33 +69,9 @@ defmodule Mix.Tasks.Tableau.Build do
     end
   end
 
-  defp pre_build_extensions(modules) do
+  defp extensions_for(modules, type) do
     extensions =
-      for {mod, _, _} <- modules,
-          mod = Module.concat([to_string(mod)]),
-          match?({:ok, :pre_build}, Tableau.Extension.type(mod)) do
-        mod
-      end
-
-    Enum.sort_by(extensions, & &1.__tableau_extension_priority__())
-  end
-
-  defp pre_write_extensions(modules) do
-    extensions =
-      for {mod, _, _} <- modules,
-          mod = Module.concat([to_string(mod)]),
-          match?({:ok, :pre_write}, Tableau.Extension.type(mod)) do
-        mod
-      end
-
-    Enum.sort_by(extensions, & &1.__tableau_extension_priority__())
-  end
-
-  defp post_write_extensions(modules) do
-    extensions =
-      for {mod, _, _} <- modules,
-          mod = Module.concat([to_string(mod)]),
-          match?({:ok, :post_write}, Tableau.Extension.type(mod)) do
+      for mod <- modules, {:ok, type} == Tableau.Extension.type(mod) do
         mod
       end
 

lib/tableau/extension.ex

@@ -19,22 +19,39 @@ defmodule Tableau.Extension do
   - `:pre_write` - executed after tableau builds your site but before it writes anything to disk.
   - `:post_write` - executed after tableau builds your site and writes everything to disk.
 
+  ## The Graph
+
+  Tableau pages and layuts are a DAG, a Directed Acyclic Graph, and this graph is used to build each page when writing to disk.
+
+  Your extension can create data to to insert into the extension token and can also inserted pages into the graph to be written to disk when the time comes.
+
   ## Example
 
+  In this example, we create a simple post extension that reads markdown files from disk, inserts them into the graph, and inserts them into the token.
+
+  By inserting them into the token, you are able to access them inside your templates, for example, a posts listing page.
+
   ```elixir
   defmodule MySite.PostsExtension do
     use Tableau.Extension, key: :posts, type: :pre_build, priority: 300
 
     def run(token) do
-      posts = Path.wildcard("_posts/**/*.md")
-
-      for post <- post do
-        post
-        |> Markdown.render()
-        |> then(&File.write(Path.join(Path.rootname(post), "index.html"), &1))
-      end
-
-      {:ok, token}
+      posts = 
+        for post <- Path.wildcard("_posts/**/*.md") do
+          %Tableau.Page{
+            parent: MySite.RootLayout,
+            permalink: Path.join("posts", Path.rootname(post)),
+            template: Markdown.render(post),
+            opts: %{}
+          }
+        end
+
+      graph = Tableau.Graph.insert(token.graph, posts)
+
+      {:ok,
+       token
+       |> Map.put(:posts, posts)
+       |> Map.put(:graph, graph)}
     end
   end
   ```

lib/tableau/extensions/page_extension.ex

@@ -48,7 +48,19 @@ defmodule Tableau.PageExtension do
   use Tableau.Extension, key: :pages, type: :pre_build, priority: 100
 
   def run(token) do
-    token = put_in(token.pages, Tableau.PageExtension.Pages.pages())
-    {:ok, token}
+    pages = Tableau.PageExtension.Pages.pages()
+
+    graph =
+      Tableau.Graph.insert(
+        token.graph,
+        Enum.map(pages, fn page ->
+          %Tableau.Page{parent: page.layout, permalink: page.permalink, template: page.body, opts: page}
+        end)
+      )
+
+    {:ok,
+     token
+     |> Map.put(:pages, pages)
+     |> Map.put(:graph, graph)}
   end
 end

lib/tableau/extensions/page_extension/pages.ex

@@ -4,23 +4,7 @@ defmodule Tableau.PageExtension.Pages do
 
   @config Map.new(Application.compile_env(:tableau, Tableau.PageExtension, %{}))
 
-  def __tableau_type__, do: :pages
-
   def pages(opts \\ []) do
-    opts
-    |> pages2()
-    |> Enum.map(fn page ->
-      %{
-        type: :page,
-        parent: page.layout,
-        permalink: page.permalink,
-        template: page.body,
-        opts: page
-      }
-    end)
-  end
-
-  def pages2(opts \\ []) do
     {:ok, config} =
       Tableau.PageExtension.Config.new(@config)
 

lib/tableau/extensions/page_extension/pages/html_converter.ex

@@ -3,6 +3,6 @@ defmodule Tableau.PageExtension.Pages.HTMLConverter do
   def convert(_filepath, body, _attrs, _opts) do
     {:ok, config} = Tableau.Config.new(Map.new(Application.get_env(:tableau, :config, %{})))
 
-    MDEx.to_html(body, config.markdown[:mdex])
+    MDEx.to_html!(body, config.markdown[:mdex])
   end
 end

lib/tableau/extensions/post_extension.ex

@@ -54,7 +54,19 @@ defmodule Tableau.PostExtension do
   use Tableau.Extension, key: :posts, type: :pre_build, priority: 100
 
   def run(token) do
-    token = put_in(token.posts, Tableau.PostExtension.Posts.posts())
-    {:ok, token}
+    posts = Tableau.PostExtension.Posts.posts()
+
+    graph =
+      Tableau.Graph.insert(
+        token.graph,
+        Enum.map(posts, fn post ->
+          %Tableau.Page{parent: post.layout, permalink: post.permalink, template: post.body, opts: post}
+        end)
+      )
+
+    {:ok,
+     token
+     |> Map.put(:posts, posts)
+     |> Map.put(:graph, graph)}
   end
 end

lib/tableau/extensions/post_extension/posts.ex

@@ -4,22 +4,6 @@ defmodule Tableau.PostExtension.Posts do
 
   @config Map.new(Application.compile_env(:tableau, Tableau.PostExtension, %{}))
 
-  def __tableau_type__, do: :pages
-
-  def pages(opts \\ []) do
-    opts
-    |> posts()
-    |> Enum.map(fn post ->
-      %{
-        type: :page,
-        parent: post.layout,
-        permalink: post.permalink,
-        template: post.body,
-        opts: post
-      }
-    end)
-  end
-
   def posts(opts \\ []) do
     {:ok, config} =
       Tableau.PostExtension.Config.new(@config)

lib/tableau/extensions/post_extension/posts/html_converter.ex

@@ -3,6 +3,6 @@ defmodule Tableau.PostExtension.Posts.HTMLConverter do
   def convert(_filepath, body, _attrs, _opts) do
     {:ok, config} = Tableau.Config.new(Map.new(Application.get_env(:tableau, :config, %{})))
 
-    MDEx.to_html(body, config.markdown[:mdex])
+    MDEx.to_html!(body, config.markdown[:mdex])
   end
 end

lib/tableau/extensions/sitemap_extension.ex

@@ -1,9 +1,3 @@
-defmodule Tableau.SitemapExtension.Config do
-  @moduledoc false
-
-  def new(input), do: {:ok, input}
-end
-
 defmodule Tableau.SitemapExtension do
   @moduledoc """
   Generate a sitemap.xml for your site, for improved search-engine indexing.
@@ -52,6 +46,8 @@ defmodule Tableau.SitemapExtension do
 
   use Tableau.Extension, key: :sitemap, type: :post_write, priority: 300
 
+  require Logger
+
   def run(%{site: %{config: %{url: root}, pages: pages}} = token) do
     urls =
       for page <- pages, uniq: true do
@@ -79,6 +75,10 @@ defmodule Tableau.SitemapExtension do
     File.write!("_site/sitemap.xml", XmlBuilder.generate_iodata(xml))
 
     {:ok, token}
+  rescue
+    e ->
+      Logger.error(Exception.format(:error, e, __STACKTRACE__))
+      {:error, :fail}
   end
 
   defp prepend_lastmod(body, %{date: date}) do

lib/tableau/graph.ex

@@ -1,26 +1,12 @@
 defmodule Tableau.Graph do
   @moduledoc false
-  alias Tableau.Graph.Node
+  alias Tableau.Graph.Nodable
 
-  def new(modules) do
-    graph = Graph.new()
-
-    for {mod, _, _} <- modules,
-        mod = Module.concat([to_string(mod)]),
-        Node.type(mod) != :error,
-        reduce: graph do
+  def insert(graph, nodes) do
+    for node <- nodes, Nodable.type(node) != :error, reduce: graph do
       graph ->
-        case Node.type(mod) do
-          {:ok, :pages} ->
-            for page <- mod.pages(), reduce: graph do
-              graph ->
-                Graph.add_edge(graph, page, page.parent)
-            end
-
-          _ ->
-            {:ok, parent} = Node.parent(mod)
-            Graph.add_edge(graph, mod, parent)
-        end
+        {:ok, parent} = Nodable.parent(node)
+        Graph.add_edge(graph, node, parent)
     end
   end
 end

lib/tableau/graph/nodable.ex

@@ -2,20 +2,15 @@ defprotocol Tableau.Graph.Nodable do
   @moduledoc false
   def template(nodable, assigns)
   def type(nodable)
+  def parent(nodable)
   def permalink(nodable)
   def opts(nodable)
 end
 
-defimpl Tableau.Graph.Nodable, for: Map do
-  def template(nodable, _assigns), do: nodable.template
-  def type(nodable), do: {:ok, nodable.type}
-  def opts(nodable), do: nodable.opts
-  def permalink(nodable), do: nodable.permalink
-end
-
 defimpl Tableau.Graph.Nodable, for: Atom do
   def template(nodable, assigns), do: nodable.template(assigns)
   def type(nodable), do: Tableau.Graph.Node.type(nodable)
+  def parent(nodable), do: Tableau.Graph.Node.parent(nodable)
   def opts(nodable), do: nodable.__tableau_opts__()
   def permalink(nodable), do: nodable.__tableau_permalink__()
 end

lib/tableau/page.ex

@@ -26,6 +26,19 @@ defmodule Tableau.Page do
   @type assigns :: map()
   @type template :: any()
 
+  @doc """
+  The page struct allows you to create pages as data structures and manually insert them into the Graph.
+  """
+  defstruct [:parent, :permalink, :template, :opts]
+
+  defimpl Tableau.Graph.Nodable do
+    def template(nodable, _assigns), do: nodable.template
+    def type(_nodable), do: {:ok, :page}
+    def parent(nodable), do: {:ok, nodable.parent}
+    def opts(nodable), do: nodable.opts
+    def permalink(nodable), do: nodable.permalink
+  end
+
   @doc """
   The page template.
   """

mix.exs

@@ -38,7 +38,7 @@ defmodule Tableau.MixProject do
       {:floki, floki_constraint()},
       {:html_entities, "~> 0.5.2"},
       {:libgraph, "~> 0.16.0"},
-      {:mdex, "~> 0.1"},
+      {:mdex, "~> 0.2.0"},
       {:plug_static_index_html, "~> 1.0"},
       {:schematic, "~> 0.3.1"},
       {:tz, "~> 0.28.1"},

mix.lock

@@ -13,13 +13,13 @@
   "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
   "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"},
-  "mdex": {:hex, :mdex, "0.1.11", "83bac0b339811310362c86087c1ea1d37cf3190f41993a7de41fea81ccdbc8a1", [:mix], [{:rustler, "~> 0.29", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.6", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "303510829f3c59295e13b27992ef542356db8276a3a514f1369ae91afb62f60b"},
+  "mdex": {:hex, :mdex, "0.2.0", "af93e03bc964f2628c3940d22ba03435b119e070bd423fd62d31772d428a7e6c", [:mix], [{:rustler, "~> 0.32", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.7", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "d8d21d3d6ecb0b2a10f88b539f3a61df974f9570226bf6bd24404c9e361c8089"},
   "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
   "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"},
   "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
-  "rustler_precompiled": {:hex, :rustler_precompiled, "0.7.0", "5d0834fc06dbc76dd1034482f17b1797df0dba9b491cef8bb045fcaca94bcade", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "fdf43a6835f4e4de5bfbc4c019bfb8c46d124bd4635fefa3e20d9a2bbbec1512"},
+  "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.2", "5f25cbe220a8fac3e7ad62e6f950fcdca5a5a5f8501835d2823e8c74bf4268d5", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "63d1bd5f8e23096d1ff851839923162096364bac8656a4a3c00d1fff8e83ee0a"},
   "schematic": {:hex, :schematic, "0.3.1", "be633c1472959dc0ace22dd0e1f1445b099991fec39f6d6e5273d35ebd217ac4", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "52c419b5c405286e2d0369b9ca472b00b850c59a8b0bdf0dd69172ad4418d5ea"},
   "styler": {:hex, :styler, "0.9.7", "c06ebe565db5d2d8cf749f623d5c1f019da33bf837e6b06e575528b355770ae2", [:mix], [], "hexpm", "5762579d2faa76ad9e1ba92ed5f0905c923a5670af652b8e118aef5d79b9dc58"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},