Bug 1726619 - Replace MOZ_LOW_PARALLELISM_BUILD with automated heuristic parallelism based on memory usage, and use for hazard builds r=firefox-build-system-reviewers,glandium

Differential Revision: https://phabricator.services.mozilla.com/D123147

Author: hotsphink

js/src/devtools/rootAnalysis/mach_commands.py

@@ -253,6 +253,7 @@ def gather_hazard_data(self, command_context, **kwargs):
         buildscript = " ".join(
             [
                 command_context.topsrcdir + "/mach hazards compile",
+                "--job-size=3.0",  # Conservatively estimate 3GB/process
                 "--application=" + application,
                 "--haz-objdir=" + objdir,
             ]

python/mozbuild/mozbuild/base.py

@@ -43,6 +43,11 @@
     memoized_property,
 )
 
+try:
+    import psutil
+except Exception:
+    psutil = None
+
 
 def ancestors(path):
     """Emit the parent directories of a path."""
@@ -731,6 +736,7 @@ def _run_make(
         print_directory=True,
         pass_thru=False,
         num_jobs=0,
+        job_size=0,
         keep_going=False,
     ):
         """Invoke make.
@@ -775,17 +781,24 @@ def _run_make(
                 else:
                     args.append(flag)
 
-        if num_jobs > 0:
-            args.append("-j%d" % num_jobs)
-        elif os.environ.get("MOZ_LOW_PARALLELISM_BUILD"):
+        if num_jobs == 0:
+            if job_size == 0:
+                job_size = 2.0 if self.substs.get("CC_TYPE") == "gcc" else 1.0  # GiB
+
             cpus = multiprocessing.cpu_count()
-            jobs = max(1, int(0.75 * cpus))
-            print(
-                "  Low parallelism requested: using %d jobs for %d cores" % (jobs, cpus)
-            )
-            args.append("-j%d" % jobs)
-        else:
-            args.append("-j%d" % multiprocessing.cpu_count())
+            if not psutil or not job_size:
+                num_jobs = cpus
+            else:
+                mem_gb = psutil.virtual_memory().total / 1024 ** 3
+                from_mem = round(mem_gb / job_size)
+                num_jobs = max(1, min(cpus, from_mem))
+                print(
+                    "  Parallelism determined by memory: using %d jobs for %d cores "
+                    "based on %.1f GiB RAM and estimated job size of %.1f GiB"
+                    % (num_jobs, cpus, mem_gb, job_size)
+                )
+
+        args.append("-j%d" % num_jobs)
 
         if ignore_errors:
             args.append("-k")

python/mozbuild/mozbuild/build_commands.py

@@ -40,7 +40,16 @@ class Build(MachCommandBase):
         default="0",
         metavar="jobs",
         type=int,
-        help="Number of concurrent jobs to run. Default is the number of CPUs.",
+        help="Number of concurrent jobs to run. Default is based on the number of "
+        "CPUs and the estimated size of the jobs (see --job-size).",
+    )
+    @CommandArgument(
+        "--job-size",
+        default="0",
+        metavar="size",
+        type=float,
+        help="Estimated RAM required, in GiB, for each parallel job. Used to "
+        "compute a default number of concurrent jobs.",
     )
     @CommandArgument(
         "-C",
@@ -65,6 +74,7 @@ def build(
         command_context,
         what=None,
         jobs=0,
+        job_size=0,
         directory=None,
         verbose=False,
         keep_going=False,
@@ -115,6 +125,7 @@ def build(
                 command_context.metrics,
                 what=what,
                 jobs=jobs,
+                job_size=job_size,
                 directory=directory,
                 verbose=verbose,
                 keep_going=keep_going,
@@ -156,6 +167,7 @@ def build(
             command_context.metrics,
             what=what,
             jobs=jobs,
+            job_size=job_size,
             directory=directory,
             verbose=verbose,
             keep_going=keep_going,

python/mozbuild/mozbuild/controller/building.py

@@ -1064,6 +1064,7 @@ def build(
         metrics,
         what=None,
         jobs=0,
+        job_size=0,
         directory=None,
         verbose=False,
         keep_going=False,
@@ -1255,6 +1256,7 @@ def get_substs_flag(name):
                         print_directory=False,
                         ensure_exit_code=False,
                         num_jobs=jobs,
+                        job_size=job_size,
                         silent=not verbose,
                         append_env=tgt_env,
                         keep_going=keep_going,
@@ -1269,6 +1271,7 @@ def get_substs_flag(name):
                 status = self._run_client_mk(
                     line_handler=output.on_line,
                     jobs=jobs,
+                    job_size=job_size,
                     verbose=verbose,
                     keep_going=keep_going,
                     append_env=append_env,
@@ -1619,6 +1622,7 @@ def _run_client_mk(
         target=None,
         line_handler=None,
         jobs=0,
+        job_size=0,
         verbose=None,
         keep_going=False,
         append_env=None,
@@ -1701,6 +1705,7 @@ def _run_client_mk(
             line_handler=line_handler,
             log=False,
             num_jobs=jobs,
+            job_size=job_size,
             silent=not verbose,
             keep_going=keep_going,
             append_env=append_env,

taskcluster/ci/build/linux-base-toolchains.yml

@@ -19,7 +19,6 @@ linux64-base-toolchains/opt:
     worker:
         max-run-time: 7200
         env:
-            MOZ_LOW_PARALLELISM_BUILD: '1'
             PERFHERDER_EXTRA_OPTIONS: base-toolchains
             FORCE_GCC: '1'
     run:
@@ -59,7 +58,6 @@ linux64-base-toolchains/debug:
     worker:
         max-run-time: 7200
         env:
-            MOZ_LOW_PARALLELISM_BUILD: '1'
             PERFHERDER_EXTRA_OPTIONS: base-toolchains
             FORCE_GCC: '1'
     run:

taskcluster/ci/build/linux.yml

@@ -183,7 +183,6 @@ linux64-gcc/opt:
     worker:
         max-run-time: 7200
         env:
-            MOZ_LOW_PARALLELISM_BUILD: '1'
             PERFHERDER_EXTRA_OPTIONS: gcc
             FORCE_GCC: '1'
     run: