Add environment variable for reusing Mix.install/2 installation (#13378)

Author: jonatanklosko

lib/iex/lib/iex/helpers.ex

@@ -105,6 +105,14 @@ defmodule IEx.Helpers do
 
       Mix.installed?() ->
         Mix.in_install_project(fn ->
+          # TODO: remove this once Mix requires Hex with the fix from
+          # https://github.com/hexpm/hex/pull/1015
+          # Context: Mix.install/1 starts :hex if necessary and stops
+          # it afterwards. Calling compile here may require hex to be
+          # started and that should happen automatically, but because
+          # of a bug it is not (fixed in the linked PR).
+          _ = Application.ensure_all_started(:hex)
+
           do_recompile(options)
           # Just as with Mix.install/2 we clear all task invocations,
           # so that we can recompile the dependencies again next time

lib/mix/lib/mix.ex

@@ -661,6 +661,13 @@ defmodule Mix do
   This function can only be called outside of a Mix project and only with the
   same dependencies in the given VM.
 
+  The `MIX_INSTALL_RESTORE_PROJECT_DIR` environment variable may be specified.
+  It should point to a previous installation directory, which can be obtained
+  with `Mix.install_project_dir/0` (after calling `Mix.install/2`). Using a
+  restore dir may speed up the installation, since matching dependencies do
+  not need be refetched nor recompiled. This environment variable is ignored
+  if `:force` is enabled.
+
   ## Options
 
     * `:force` - if `true`, runs with empty install cache. This is useful when you want
@@ -845,14 +852,14 @@ defmodule Mix do
         Application.put_all_env(config, persistent: true)
         System.put_env(system_env)
 
-        install_dir = install_dir(id)
+        install_project_dir = install_project_dir(id)
 
         if Keyword.fetch!(opts, :verbose) do
-          Mix.shell().info("Mix.install/2 using #{install_dir}")
+          Mix.shell().info("Mix.install/2 using #{install_project_dir}")
         end
 
         if force? do
-          File.rm_rf!(install_dir)
+          File.rm_rf!(install_project_dir)
         end
 
         dynamic_config = [
@@ -865,21 +872,28 @@ defmodule Mix do
 
         started_apps = Application.started_applications()
         :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile")
-        build_dir = Path.join(install_dir, "_build")
+        build_dir = Path.join(install_project_dir, "_build")
         external_lockfile = expand_path(opts[:lockfile], deps, :lockfile, "mix.lock")
 
         try do
           first_build? = not File.dir?(build_dir)
-          File.mkdir_p!(install_dir)
 
-          File.cd!(install_dir, fn ->
+          restore_dir = System.get_env("MIX_INSTALL_RESTORE_PROJECT_DIR")
+
+          if first_build? and restore_dir != nil and not force? do
+            File.cp_r(restore_dir, install_project_dir)
+          end
+
+          File.mkdir_p!(install_project_dir)
+
+          File.cd!(install_project_dir, fn ->
             if config_path do
               Mix.Task.rerun("loadconfig")
             end
 
             cond do
               external_lockfile ->
-                md5_path = Path.join(install_dir, "merge.lock.md5")
+                md5_path = Path.join(install_project_dir, "merge.lock.md5")
 
                 old_md5 =
                   case File.read(md5_path) do
@@ -890,7 +904,7 @@ defmodule Mix do
                 new_md5 = external_lockfile |> File.read!() |> :erlang.md5()
 
                 if old_md5 != new_md5 do
-                  lockfile = Path.join(install_dir, "mix.lock")
+                  lockfile = Path.join(install_project_dir, "mix.lock")
                   old_lock = Mix.Dep.Lock.read(lockfile)
                   new_lock = Mix.Dep.Lock.read(external_lockfile)
                   Mix.Dep.Lock.write(Map.merge(old_lock, new_lock), file: lockfile)
@@ -928,6 +942,10 @@ defmodule Mix do
             end
           end
 
+          if restore_dir do
+            remove_leftover_deps(install_project_dir)
+          end
+
           Mix.State.put(:installed, {id, dynamic_config})
           :ok
         after
@@ -965,7 +983,29 @@ defmodule Mix do
     Path.join(app_dir, relative_path)
   end
 
-  defp install_dir(cache_id) do
+  defp remove_leftover_deps(install_project_dir) do
+    build_lib_dir = Path.join([install_project_dir, "_build", "dev", "lib"])
+    deps_dir = Path.join(install_project_dir, "deps")
+
+    deps = File.ls!(build_lib_dir)
+
+    loaded_deps =
+      for {app, _description, _version} <- Application.loaded_applications(),
+          into: MapSet.new(),
+          do: Atom.to_string(app)
+
+    # We want to keep :mix_install, but it has no application
+    loaded_deps = MapSet.put(loaded_deps, "mix_install")
+
+    for dep <- deps, not MapSet.member?(loaded_deps, dep) do
+      build_path = Path.join(build_lib_dir, dep)
+      File.rm_rf(build_path)
+      dep_path = Path.join(deps_dir, dep)
+      File.rm_rf(dep_path)
+    end
+  end
+
+  defp install_project_dir(cache_id) do
     install_root =
       System.get_env("MIX_INSTALL_DIR") ||
         Path.join(Mix.Utils.mix_cache(), "installs")
@@ -996,9 +1036,9 @@ defmodule Mix do
       {id, dynamic_config} ->
         config = install_project_config(dynamic_config)
 
-        install_dir = install_dir(id)
+        install_project_dir = install_project_dir(id)
 
-        File.cd!(install_dir, fn ->
+        File.cd!(install_project_dir, fn ->
           :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile")
 
           try do
@@ -1013,6 +1053,18 @@ defmodule Mix do
     end
   end
 
+  @doc """
+  Returns the directory where the current `Mix.install/2` project
+  resides.
+  """
+  @spec install_project_dir() :: Path.t()
+  def install_project_dir() do
+    case Mix.State.get(:installed) do
+      {id, _dynamic_config} -> install_project_dir(id)
+      nil -> nil
+    end
+  end
+
   @doc """
   Returns whether `Mix.install/2` was called in the current node.
   """

lib/mix/test/mix_test.exs

@@ -263,30 +263,26 @@ defmodule MixTest do
         [
           {:git_repo, git: fixture_path("git_repo")}
         ],
-        lockfile: lockfile,
-        verbose: true
+        lockfile: lockfile
       )
 
       assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]}
-      assert_received {:mix_shell, :info, ["Mix.install/2 using " <> install_dir]}
-      assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev
-    after
-      purge([GitRepo, GitRepo.MixProject])
+
+      install_project_dir = Mix.install_project_dir()
+      assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev
     end
 
     test ":lockfile merging", %{tmp_dir: tmp_dir} do
       [rev1, rev2 | _] = get_git_repo_revs("git_repo")
 
-      Mix.install(
-        [
-          {:git_repo, git: fixture_path("git_repo")}
-        ],
-        verbose: true
-      )
+      Mix.install([
+        {:git_repo, git: fixture_path("git_repo")}
+      ])
 
       assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]}
-      assert_received {:mix_shell, :info, ["Mix.install/2 using " <> install_dir]}
-      assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev1
+
+      install_project_dir = Mix.install_project_dir()
+      assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev1
 
       Mix.Project.push(GitApp)
       lockfile = Path.join(tmp_dir, "lock")
@@ -300,9 +296,7 @@ defmodule MixTest do
         lockfile: lockfile
       )
 
-      assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev1
-    after
-      purge([GitRepo, GitRepo.MixProject])
+      assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev1
     end
 
     test ":lockfile with application name", %{tmp_dir: tmp_dir} do
@@ -318,15 +312,12 @@ defmodule MixTest do
           {:install_test, path: Path.join(tmp_dir, "install_test")},
           {:git_repo, git: fixture_path("git_repo")}
         ],
-        lockfile: :install_test,
-        verbose: true
+        lockfile: :install_test
       )
 
       assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]}
-      assert_received {:mix_shell, :info, ["Mix.install/2 using " <> install_dir]}
-      assert File.read!(Path.join(install_dir, "mix.lock")) =~ rev
-    after
-      purge([GitRepo, GitRepo.MixProject])
+      install_project_dir = Mix.install_project_dir()
+      assert File.read!(Path.join(install_project_dir, "mix.lock")) =~ rev
     end
 
     test ":lockfile that does not exist" do
@@ -335,6 +326,73 @@ defmodule MixTest do
       end
     end
 
+    test "restore dir", %{tmp_dir: tmp_dir} do
+      with_cleanup(fn ->
+        Mix.install([
+          {:git_repo, git: fixture_path("git_repo")}
+        ])
+
+        assert_received {:mix_shell, :info, ["* Getting git_repo " <> _]}
+        assert_received {:mix_shell, :info, ["==> git_repo"]}
+        assert_received {:mix_shell, :info, ["Compiling 1 file (.ex)"]}
+        assert_received {:mix_shell, :info, ["Generated git_repo app"]}
+        refute_received _
+
+        install_project_dir = Mix.install_project_dir()
+        build_lib_path = Path.join([install_project_dir, "_build", "dev", "lib"])
+        deps_path = Path.join([install_project_dir, "deps"])
+
+        assert File.ls!(build_lib_path) |> Enum.sort() == ["git_repo", "mix_install"]
+        assert File.ls!(deps_path) == ["git_repo"]
+
+        System.put_env("MIX_INSTALL_RESTORE_PROJECT_DIR", install_project_dir)
+      end)
+
+      # Adding a dependency
+
+      with_cleanup(fn ->
+        Mix.install([
+          {:git_repo, git: fixture_path("git_repo")},
+          {:install_test, path: Path.join(tmp_dir, "install_test")}
+        ])
+
+        assert_received {:mix_shell, :info, ["==> install_test"]}
+        assert_received {:mix_shell, :info, ["Compiling 2 files (.ex)"]}
+        assert_received {:mix_shell, :info, ["Generated install_test app"]}
+        refute_received _
+
+        install_project_dir = Mix.install_project_dir()
+        build_lib_path = Path.join([install_project_dir, "_build", "dev", "lib"])
+        deps_path = Path.join([install_project_dir, "deps"])
+
+        assert File.ls!(build_lib_path) |> Enum.sort() ==
+                 ["git_repo", "install_test", "mix_install"]
+
+        assert File.ls!(deps_path) == ["git_repo"]
+
+        System.put_env("MIX_INSTALL_RESTORE_PROJECT_DIR", install_project_dir)
+      end)
+
+      # Removing a dependency
+
+      with_cleanup(fn ->
+        Mix.install([
+          {:install_test, path: Path.join(tmp_dir, "install_test")}
+        ])
+
+        refute_received _
+
+        install_project_dir = Mix.install_project_dir()
+        build_lib_path = Path.join([install_project_dir, "_build", "dev", "lib"])
+        deps_path = Path.join([install_project_dir, "deps"])
+
+        assert File.ls!(build_lib_path) |> Enum.sort() == ["install_test", "mix_install"]
+        assert File.ls!(deps_path) == []
+      end)
+    after
+      System.delete_env("MIX_INSTALL_RESTORE_PROJECT_DIR")
+    end
+
     test "installed?", %{tmp_dir: tmp_dir} do
       refute Mix.installed?()
 
@@ -380,15 +438,7 @@ defmodule MixTest do
 
       on_exit(fn ->
         :code.set_path(path)
-        purge([InstallTest, InstallTest.MixProject, InstallTest.Protocol])
-
-        ExUnit.CaptureLog.capture_log(fn ->
-          Application.stop(:git_repo)
-          Application.unload(:git_repo)
-
-          Application.stop(:install_test)
-          Application.unload(:install_test)
-        end)
+        cleanup_deps()
       end)
 
       Mix.State.put(:installed, nil)
@@ -424,5 +474,37 @@ defmodule MixTest do
 
       [tmp_dir: tmp_dir]
     end
+
+    defp with_cleanup(fun) do
+      path = :code.get_path()
+
+      try do
+        fun.()
+      after
+        :code.set_path(path)
+        cleanup_deps()
+
+        Mix.State.clear_cache()
+        Mix.State.put(:installed, nil)
+      end
+    end
+
+    defp cleanup_deps() do
+      purge([
+        GitRepo,
+        GitRepo.MixProject,
+        InstallTest,
+        InstallTest.MixProject,
+        InstallTest.Protocol
+      ])
+
+      ExUnit.CaptureLog.capture_log(fn ->
+        Application.stop(:git_repo)
+        Application.unload(:git_repo)
+
+        Application.stop(:install_test)
+        Application.unload(:install_test)
+      end)
+    end
   end
 end