Add MIX_OS_DEPS_COMPILE_PARTITION_COUNT for concurrent deps compilation (elixir-lang#14340)

Author: josevalim

lib/mix/lib/mix.ex

@@ -355,7 +355,11 @@ defmodule Mix do
     * `MIX_OS_CONCURRENCY_LOCK` - when set to `0` or `false`, disables mix compilation locking.
       While not recommended, this may be necessary in cases where hard links or TCP sockets are
       not available. When opting for this behaviour, make sure to not start concurrent compilations
-      of the same project.
+      of the same project
+
+    * `MIX_OS_DEPS_COMPILE_PARTITION_COUNT` - when set to a number greater than 1, it enables
+      compilation of dependencies over multiple operating system processes. See `mix help deps.compile`
+      for more information
 
     * `MIX_PATH` - appends extra code paths
 
@@ -414,7 +418,7 @@ defmodule Mix do
 
   """
 
-  @mix_install_project __MODULE__.InstallProject
+  @mix_install_project Mix.InstallProject
   @mix_install_app :mix_install
   @mix_install_app_string Atom.to_string(@mix_install_app)
 
@@ -900,9 +904,7 @@ defmodule Mix do
 
     case Mix.State.get(:installed) do
       nil ->
-        Application.put_all_env(config, persistent: true)
         System.put_env(system_env)
-
         install_project_dir = install_project_dir(id)
 
         if Keyword.fetch!(opts, :verbose) do
@@ -919,10 +921,14 @@ defmodule Mix do
           config_path: config_path
         ]
 
-        config = install_project_config(dynamic_config)
+        :ok =
+          Mix.ProjectStack.push(
+            @mix_install_project,
+            [compile_config: config] ++ install_project_config(dynamic_config),
+            "nofile"
+          )
 
         started_apps = Application.started_applications()
-        :ok = Mix.ProjectStack.push(@mix_install_project, config, "nofile")
         build_dir = Path.join(install_project_dir, "_build")
         external_lockfile = expand_path(opts[:lockfile], deps, :lockfile, "mix.lock")
 
@@ -939,9 +945,9 @@ defmodule Mix do
           File.mkdir_p!(install_project_dir)
 
           File.cd!(install_project_dir, fn ->
-            if config_path do
-              Mix.Task.rerun("loadconfig")
-            end
+            # This steps need to be mirror in mix deps.partition
+            Application.put_all_env(config, persistent: true)
+            Mix.Task.rerun("loadconfig")
 
             cond do
               external_lockfile ->
@@ -1074,7 +1080,7 @@ defmodule Mix do
 
   defp install_project_config(dynamic_config) do
     [
-      version: "0.1.0",
+      version: "1.0.0",
       build_embedded: false,
       build_per_environment: true,
       build_path: "_build",

lib/mix/lib/mix/tasks/deps.compile.ex

@@ -31,6 +31,24 @@ defmodule Mix.Tasks.Deps.Compile do
   recompiled without propagating those changes upstream. To ensure
   `b` is included in the compilation step, pass `--include-children`.
 
+  ## Compiling dependencies across multiple OS processes
+
+  If you set the environment variable `MIX_OS_DEPS_COMPILE_PARTITION_COUNT`
+  to a number greater than 1, Mix will start multiple operating system
+  processes to compile your dependencies concurrently.
+
+  While Mix and Rebar compile all files within a given project in parallel,
+  enabling this environment variable can still yield useful gains in several
+  cases, such as when compiling dependencies with native code, dependencies
+  that must download assets, or dependencies where the compilation time is not
+  evenly distributed (for example, one file takes much longer to compile than
+  all others).
+
+  While most configuration in Mix is done via command line flags, this particular
+  environment variable exists because the best number will vary per machine
+  (and often per project too). The environment variable also makes it more accessible
+  to enable concurrent compilation in CI and also during `Mix.install/2` commands.
+
   ## Command line options
 
     * `--force` - force compilation of deps
@@ -53,7 +71,6 @@ defmodule Mix.Tasks.Deps.Compile do
     end
 
     Mix.Project.get!()
-
     config = Mix.Project.config()
 
     Mix.Project.with_build_lock(config, fn ->
@@ -71,86 +88,82 @@ defmodule Mix.Tasks.Deps.Compile do
 
   @doc false
   def compile(deps, options \\ []) do
-    shell = Mix.shell()
-    config = Mix.Project.deps_config()
     Mix.Task.run("deps.precompile")
+    force? = Keyword.get(options, :force, false)
 
-    compiled =
+    deps =
       deps
       |> reject_umbrella_children(options)
       |> reject_local_deps(options)
-      |> Enum.map(fn %Mix.Dep{app: app, status: status, opts: opts, scm: scm} = dep ->
-        check_unavailable!(app, scm, status)
-        maybe_clean(dep, options)
 
-        compiled? =
-          cond do
-            not is_nil(opts[:compile]) ->
-              do_compile(dep, config)
+    count = System.get_env("MIX_OS_DEPS_COMPILE_PARTITION_COUNT", "0") |> String.to_integer()
 
-            Mix.Dep.mix?(dep) ->
-              do_mix(dep, config)
+    compiled? =
+      if count > 1 and length(deps) > 1 do
+        Mix.shell().info("mix deps.compile running across #{count} OS processes")
+        Mix.Tasks.Deps.Partition.server(deps, count, force?)
+      else
+        config = Mix.Project.deps_config()
+        true in Enum.map(deps, &compile_single(&1, force?, config))
+      end
 
-            Mix.Dep.make?(dep) ->
-              do_make(dep, config)
+    if compiled?, do: Mix.Task.run("will_recompile"), else: :ok
+  end
 
-            dep.manager == :rebar3 ->
-              do_rebar3(dep, config)
+  @doc false
+  def compile_single(%Mix.Dep{} = dep, force?, config) do
+    %{app: app, status: status, opts: opts, scm: scm} = dep
+    check_unavailable!(app, scm, status)
 
-            true ->
-              shell.error(
-                "Could not compile #{inspect(app)}, no \"mix.exs\", \"rebar.config\" or \"Makefile\" " <>
-                  "(pass :compile as an option to customize compilation, set it to \"false\" to do nothing)"
-              )
+    # If a dependency was marked as fetched or with an out of date lock
+    # or missing the app file, we always compile it from scratch.
+    if force? or Mix.Dep.compilable?(dep) do
+      File.rm_rf!(Path.join([Mix.Project.build_path(), "lib", Atom.to_string(dep.app)]))
+    end
 
-              false
-          end
+    compiled? =
+      cond do
+        not is_nil(opts[:compile]) ->
+          do_compile(dep, config)
 
-        if compiled? do
-          build_path = Mix.Project.build_path(config)
+        Mix.Dep.mix?(dep) ->
+          do_mix(dep, config)
 
-          lazy_message = fn ->
-            info = %{
-              app: dep.app,
-              scm: dep.scm,
-              manager: dep.manager,
-              os_pid: System.pid()
-            }
+        Mix.Dep.make?(dep) ->
+          do_make(dep, config)
 
-            {:dep_compiled, info}
-          end
+        dep.manager == :rebar3 ->
+          do_rebar3(dep, config)
 
-          Mix.Sync.PubSub.broadcast(build_path, lazy_message)
-        end
+        true ->
+          Mix.shell().error(
+            "Could not compile #{inspect(app)}, no \"mix.exs\", \"rebar.config\" or \"Makefile\" " <>
+              "(pass :compile as an option to customize compilation, set it to \"false\" to do nothing)"
+          )
 
-        # We should touch fetchable dependencies even if they
-        # did not compile otherwise they will always be marked
-        # as stale, even when there is nothing to do.
-        fetchable? = touch_fetchable(scm, opts[:build])
+          false
+      end
 
-        compiled? and fetchable?
+    if compiled? do
+      config
+      |> Mix.Project.build_path()
+      |> Mix.Sync.PubSub.broadcast(fn ->
+        info = %{
+          app: dep.app,
+          scm: dep.scm,
+          manager: dep.manager,
+          os_pid: System.pid()
+        }
+
+        {:dep_compiled, info}
       end)
-
-    if true in compiled, do: Mix.Task.run("will_recompile"), else: :ok
-  end
-
-  defp maybe_clean(dep, opts) do
-    # If a dependency was marked as fetched or with an out of date lock
-    # or missing the app file, we always compile it from scratch.
-    if Keyword.get(opts, :force, false) or Mix.Dep.compilable?(dep) do
-      File.rm_rf!(Path.join([Mix.Project.build_path(), "lib", Atom.to_string(dep.app)]))
     end
-  end
 
-  defp touch_fetchable(scm, path) do
-    if scm.fetchable?() do
-      path = Path.join(path, ".mix")
-      File.mkdir_p!(path)
-      File.touch!(Path.join(path, "compile.fetch"))
-      true
-    else
-      false
-    end
+    # We should touch fetchable dependencies even if they
+    # did not compile otherwise they will always be marked
+    # as stale, even when there is nothing to do.
+    fetchable? = touch_fetchable(scm, opts[:build])
+    compiled? and fetchable?
   end
 
   defp check_unavailable!(app, scm, {:unavailable, path}) do
@@ -172,6 +185,17 @@ defmodule Mix.Tasks.Deps.Compile do
     :ok
   end
 
+  defp touch_fetchable(scm, path) do
+    if scm.fetchable?() do
+      path = Path.join(path, ".mix")
+      File.mkdir_p!(path)
+      File.touch!(Path.join(path, "compile.fetch"))
+      true
+    else
+      false
+    end
+  end
+
   defp do_mix(dep, _config) do
     Mix.Dep.in_dependency(dep, fn _ ->
       config = Mix.Project.config()

lib/mix/lib/mix/tasks/deps.loadpaths.ex

@@ -134,11 +134,7 @@ defmodule Mix.Tasks.Deps.Loadpaths do
   defp partition([dep | deps], not_ok, compile) do
     cond do
       Mix.Dep.compilable?(dep) or (Mix.Dep.ok?(dep) and local?(dep)) ->
-        if from_umbrella?(dep) do
-          partition(deps, not_ok, compile)
-        else
-          partition(deps, not_ok, [dep | compile])
-        end
+        partition(deps, not_ok, [dep | compile])
 
       Mix.Dep.ok?(dep) ->
         partition(deps, not_ok, compile)
@@ -158,11 +154,6 @@ defmodule Mix.Tasks.Deps.Loadpaths do
     |> Mix.Dep.filter_by_name(Mix.Dep.load_and_cache())
   end
 
-  # Those are compiled by umbrella.
-  defp from_umbrella?(dep) do
-    dep.opts[:from_umbrella]
-  end
-
   # Every local dependency (i.e. that are not fetchable)
   # are automatically recompiled if they are ok.
   defp local?(dep) do