WIP: Experiment with 2-phase compilation with outline and batch parallel compile

compiler/src/dotty/tools/backend/jvm/CodeGen.scala

@@ -125,17 +125,16 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
 
   // Creates a callback that will be evaluated in PostProcessor after creating a file
   private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: util.SourceFile)(using Context): AbstractFile => Unit = {
-    val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
-      (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
+    val isLocal = atPhase(sbtExtractDependenciesPhase) {
+      claszSymbol.isLocal
     }
     clsFile => {
       val className = cls.name.replace('/', '.')
       if (ctx.compilerCallback != null)
         ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(clsFile), className)
 
-      ctx.withIncCallback: cb =>
-        if (isLocal) cb.generatedLocalClass(sourceFile, clsFile.jpath)
-        else cb.generatedNonLocalClass(sourceFile, clsFile.jpath, className, fullClassName)
+      if isLocal then
+        ctx.withIncCallback(_.generatedLocalClass(sourceFile, clsFile.jpath))
     }
   }
 

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -18,6 +18,8 @@ import StdNames.nme
 import scala.annotation.internal.sharable
 import scala.util.control.NoStackTrace
 import transform.MacroAnnotations
+import dotty.tools.dotc.interfaces.AbstractFile
+import dotty.tools.io.NoAbstractFile
 
 class CompilationUnit protected (val source: SourceFile, val info: CompilationUnitInfo | Null) {
 
@@ -28,20 +30,24 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
   var tpdTree: tpd.Tree = tpd.EmptyTree
 
   /** Is this the compilation unit of a Java file */
-  def isJava: Boolean = source.file.name.endsWith(".java")
+  def isJava: Boolean = source.file.ext.isJava
 
   /** Is this the compilation unit of a Java file, or TASTy derived from a Java file */
-  def typedAsJava = isJava || {
-    val infoNN = info
-    infoNN != null && infoNN.tastyInfo.exists(_.attributes.isJava)
-  }
+  def typedAsJava =
+    val ext = source.file.ext
+    ext.isJavaOrTasty && (ext.isJava || tastyInfo.exists(_.attributes.isJava))
+
+  def tastyInfo: Option[TastyInfo] =
+    val local = info
+    if local == null then None else local.tastyInfo
 
 
   /** The source version for this unit, as determined by a language import */
   var sourceVersion: Option[SourceVersion] = None
 
   /** Pickled TASTY binaries, indexed by class. */
   var pickled: Map[ClassSymbol, () => Array[Byte]] = Map()
+  var outlinePickled: Map[ClassSymbol, () => Array[Byte]] = Map()
 
   /** The fresh name creator for the current unit.
    *  FIXME(#7661): This is not fine-grained enough to enable reproducible builds,
@@ -94,12 +100,17 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
     // when this unit is unsuspended.
     depRecorder.clear()
     if !suspended then
-      if (ctx.settings.XprintSuspension.value)
-        report.echo(i"suspended: $this")
-      suspended = true
-      ctx.run.nn.suspendedUnits += this
-      if ctx.phase == Phases.inliningPhase then
-        suspendedAtInliningPhase = true
+      if ctx.settings.YnoSuspendedUnits.value then
+        report.error(i"Compilation unit suspended $this (-Yno-suspended-units is set)")
+      else
+        if (ctx.settings.XprintSuspension.value)
+          report.echo(i"suspended: $this")
+        suspended = true
+        ctx.run.nn.suspendedUnits += this
+        if ctx.phase == Phases.inliningPhase then
+          suspendedAtInliningPhase = true
+        else if ctx.settings.YearlyTastyOutput.value != NoAbstractFile then
+          report.error(i"Compilation units may not be suspended before inlining with -Ypickle-write")
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -40,20 +40,21 @@ class Compiler {
     List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
+    // List(new sbt.ExtractAPI.Outline) ::   // [runs in outline] Sends a representation of the API of classes to sbt via callbacks
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
-    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new SetRootTree) ::        // Set the `rootTreeOrProvider` on class symbols
     Nil
 
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
-    List(new Pickler) ::            // Generate TASTY info
-    List(new Inlining) ::           // Inline and execute macros
-    List(new PostInlining) ::       // Add mirror support for inlined code
-    List(new CheckUnused.PostInlining) ::  // Check for unused elements
-    List(new Staging) ::            // Check staging levels and heal staged types
-    List(new Splicing) ::           // Replace level 1 splices with holes
-    List(new PickleQuotes) ::       // Turn quoted trees into explicit run-time data structures
+    List(new Pickler) ::                  // Generate TASTY info
+    List(new sbt.ExtractAPI) ::           // [runs when not outline] Sends a representation of the API of classes to sbt via callbacks
+    List(new Inlining) ::                 // Inline and execute macros
+    List(new PostInlining) ::             // Add mirror support for inlined code
+    List(new CheckUnused.PostInlining) :: // Check for unused elements
+    List(new Staging) ::                  // Check staging levels and heal staged types
+    List(new Splicing) ::                 // Replace level 1 splices with holes
+    List(new PickleQuotes) ::             // Turn quoted trees into explicit run-time data structures
     Nil
 
   /** Phases dealing with the transformation from pickled trees to backend trees */

compiler/src/dotty/tools/dotc/Driver.scala

@@ -6,13 +6,35 @@ import core.Comments.{ContextDoc, ContextDocstrings}
 import core.Contexts.*
 import core.{MacroClassLoader, TypeError}
 import dotty.tools.dotc.ast.Positioned
-import dotty.tools.io.AbstractFile
+import dotty.tools.io.{AbstractFile, FileExtension}
 import reporting.*
 import core.Decorators.*
 import config.Feature
 
 import scala.util.control.NonFatal
 import fromtasty.{TASTYCompiler, TastyFileUtil}
+import dotty.tools.io.NoAbstractFile
+import dotty.tools.io.{VirtualFile, VirtualDirectory}
+
+import java.nio.file.Path as JPath
+import scala.concurrent.*
+import scala.annotation.internal.sharable
+import scala.concurrent.duration.*
+import scala.util.{Success, Failure}
+import scala.annotation.targetName
+import dotty.tools.dotc.classpath.FileUtils.hasScalaExtension
+import dotty.tools.dotc.core.Symbols
+
+object Driver {
+  @sharable lazy val executor =
+    // TODO: systemParallelism may change over time - is it possible to update the pool size?
+    val pool = java.util.concurrent.Executors.newFixedThreadPool(systemParallelism()).nn
+    sys.addShutdownHook(pool.shutdown())
+    ExecutionContext.fromExecutor(pool)
+
+  /** 1 less than the system's own processor count (minimum 1) */
+  def systemParallelism() = math.max(1, Runtime.getRuntime().nn.availableProcessors() - 1)
+}
 
 /** Run the Dotty compiler.
  *
@@ -28,6 +50,183 @@ class Driver {
 
   protected def emptyReporter: Reporter = new StoreReporter(null)
 
+  protected def doCompile(files: List[AbstractFile])(using ictx: Context): Reporter =
+    val isOutline = ictx.settings.Youtline.value(using ictx)
+
+    if !isOutline then inContext(ictx):
+      report.echo(s"basic compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx)
+      doCompile(newCompiler, files) // standard compilation
+    else
+      report.echo(s"Outline compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx)
+      val maxParallelism = ictx.settings.YmaxParallelism.valueIn(ictx.settingsState)
+      val absParallelism = math.abs(maxParallelism)
+      val isParallel = maxParallelism >= 0
+      val parallelism =
+        val ceiling = Driver.systemParallelism()
+        if absParallelism > 0 then math.min(absParallelism, ceiling)
+        else ceiling
+
+      // NOTE: sbt will delete this potentially as soon as you call `apiPhaseCompleted`
+      val pickleWriteOutput = ictx.settings.YearlyTastyOutput.valueIn(ictx.settingsState)
+      val profileDestination = ictx.settings.YprofileDestination.valueIn(ictx.settingsState)
+
+      if pickleWriteOutput == NoAbstractFile then
+        report.error("Requested outline compilation with `-Yexperimental-outline` " +
+          "but did not provide output directory for TASTY files (missing `-Yearly-tasty-output` flag).")(using ictx)
+        return ictx.reporter
+
+      val pickleWriteSource =
+        pickleWriteOutput.underlyingSource match
+          case Some(source) =>
+            source.file.asInstanceOf[java.io.File | Null] match
+              case f: java.io.File => Some(source)
+              case null =>
+                report.warning(s"Could not resolve file of ${source} (of class ${source.getClass.getName})")
+                None
+          case None =>
+            if pickleWriteOutput.isInstanceOf[dotty.tools.io.JarArchive] then
+              report.warning(s"Could not resolve underlying source of jar ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})")
+              None
+            else
+              report.warning(s"Could not resolve underlying source of ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})")
+              Some(pickleWriteOutput)
+
+      val outlineOutput = new VirtualDirectory("<outline-classpath>") {
+        override def underlyingSource: Option[AbstractFile] = pickleWriteSource
+      }
+
+      val firstPassCtx = ictx.fresh
+        .setSetting(ictx.settings.YoutlineClasspath, outlineOutput)
+      inContext(firstPassCtx):
+        doCompile(newCompiler, files)
+
+      def secondPassCtx(id: Int, group: List[AbstractFile], promise: scala.concurrent.Promise[Unit]): Context =
+        val profileDestination0 =
+          if profileDestination.nonEmpty then
+            val ext = dotty.tools.io.Path.fileExtension(profileDestination)
+            val filename = dotty.tools.io.Path.fileName(profileDestination)
+            s"$filename-worker-$id${if ext.isEmpty then "" else s".$ext"}"
+          else profileDestination
+
+        val baseCtx = initCtx.fresh
+          .setSettings(ictx.settingsState) // copy over the classpath arguments also
+          .setSetting(ictx.settings.YsecondPass, true)
+          .setSetting(ictx.settings.YoutlineClasspath, outlineOutput)
+          .setCallbacks(ictx.store)
+          .setDepsFinishPromise(promise)
+          .setReporter(if isParallel then new StoreReporter(ictx.reporter) else ictx.reporter)
+
+        if profileDestination0.nonEmpty then
+          baseCtx.setSetting(ictx.settings.YprofileDestination, profileDestination0)
+
+        // if ictx.settings.YoutlineClasspath.valueIn(ictx.settingsState).isEmpty then
+        //   baseCtx.setSetting(baseCtx.settings.YoutlineClasspath, pickleWriteAsClasspath)
+        val fileNames: Array[String] =
+          if sourcesRequired then group.map(_.toString).toArray else Array.empty
+        setup(fileNames, baseCtx) match
+          case Some((_, ctx)) =>
+            assert(ctx.incCallback != null, s"cannot run outline without incremental callback")
+            assert(ctx.depsFinishPromiseOpt.isDefined, s"cannot run outline without dependencies promise")
+            ctx
+          case None => baseCtx
+      end secondPassCtx
+
+      val scalaFiles = files.filter(_.hasScalaExtension)
+
+      // 516 units, 8 cores => maxGroupSize = 65, unitGroups = 8, compilers = 8
+      if !firstPassCtx.reporter.hasErrors && scalaFiles.nonEmpty then
+        val maxGroupSize = Math.ceil(scalaFiles.length.toDouble / parallelism).toInt
+        val fileGroups = scalaFiles.grouped(maxGroupSize).toList
+        val compilers = fileGroups.length
+
+
+
+        def userRequestedSingleGroup = maxParallelism == 1
+
+        // TODO: probably not good to warn here because maybe compile is incremental
+        // if compilers == 1 && !userRequestedSingleGroup then
+        //   val knownParallelism = maxParallelism > 0
+        //   val requestedParallelism = s"Requested parallelism with `-Ymax-parallelism` was ${maxParallelism}"
+        //   val computedAddedum =
+        //     if knownParallelism then "."
+        //     else s""",
+        //       |  therefore operating with computed parallelism of ${parallelism}.""".stripMargin
+        //   val message =
+        //     s"""Outline compilation second pass will run with a single compile group.
+        //       |  ${requestedParallelism}$computedAddedum
+        //       |  With ${scalaUnits.length} units to compile I can only batch them into a single group.
+        //       |  This will increase build times.
+        //       |  Perhaps consider turning off -Youtline for this project.""".stripMargin
+        //   report.warning(message)(using firstPassCtx)
+
+        val promises = fileGroups.map(_ => scala.concurrent.Promise[Unit]())
+
+        locally:
+          import scala.concurrent.ExecutionContext.Implicits.global
+          Future.sequence(promises.map(_.future)).andThen {
+            case Success(_) =>
+              ictx.withIncCallback(_.dependencyPhaseCompleted())
+            case Failure(ex) =>
+              ex.printStackTrace()
+              report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx)
+          }
+
+        report.echo(s"Compiling $compilers groups of files ${if isParallel then "in parallel" else "sequentially"}")(using firstPassCtx)
+
+        def compileEager(
+            id: Int,
+            promise: Promise[Unit],
+            fileGroup: List[AbstractFile]
+        ): Reporter = {
+          if ctx.settings.verbose.value then
+            report.echo("#Compiling: " + fileGroup.take(3).mkString("", ", ", "..."))
+          val secondCtx = secondPassCtx(id, fileGroup, promise)
+          val reporter = inContext(secondCtx):
+            doCompile(newCompiler, fileGroup) // second pass
+          if !secondCtx.reporter.hasErrors then
+            assert(promise.isCompleted, s"promise was not completed")
+          if ctx.settings.verbose.value then
+            report.echo("#Done: " + fileGroup.mkString(" "))
+          reporter
+        }
+
+        def compileFuture(
+            id: Int,
+            promise: Promise[Unit],
+            fileGroup: List[AbstractFile]
+        )(using ExecutionContext): Future[Reporter] =
+          Future {
+            // println("#Compiling: " + fileGroup.mkString(" "))
+            val secondCtx = secondPassCtx(id, fileGroup, promise)
+            val reporter = inContext(secondCtx):
+              doCompile(newCompiler, fileGroup) // second pass
+            // println("#Done: " + fileGroup.mkString(" "))
+            reporter
+          }
+
+        def fileGroupIds = LazyList.iterate(0)(_ + 1).take(compilers)
+        def taggedGroups = fileGroupIds.lazyZip(promises).lazyZip(fileGroups)
+
+        if isParallel then
+          // val executor = java.util.concurrent.Executors.newFixedThreadPool(compilers).nn
+          given ec: ExecutionContext = Driver.executor // ExecutionContext.fromExecutor(executor)
+          val futureReporters = Future.sequence(taggedGroups.map(compileFuture)).andThen {
+            case Success(reporters) =>
+              reporters.foreach(_.flush()(using firstPassCtx))
+            case Failure(ex) =>
+              ex.printStackTrace
+              report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx)
+          }
+          Await.ready(futureReporters, Duration.Inf)
+          // executor.shutdown()
+        else
+          taggedGroups.map(compileEager)
+        firstPassCtx.reporter
+      else
+        ictx.withIncCallback(_.dependencyPhaseCompleted()) // may be just java files compiled
+        firstPassCtx.reporter
+  end doCompile
+
   protected def doCompile(compiler: Compiler, files: List[AbstractFile])(using Context): Reporter =
     if files.nonEmpty then
       var runOrNull = ctx.run
@@ -97,9 +296,9 @@ class Driver {
           if !file.exists then
             report.error(em"File does not exist: ${file.path}")
             None
-          else file.extension match
-            case "jar" => Some(file.path)
-            case "tasty" =>
+          else file.ext match
+            case FileExtension.Jar => Some(file.path)
+            case FileExtension.Tasty =>
               TastyFileUtil.getClassPath(file) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
@@ -193,7 +392,7 @@ class Driver {
   def process(args: Array[String], rootCtx: Context): Reporter = {
     setup(args, rootCtx) match
       case Some((files, compileCtx)) =>
-        doCompile(newCompiler(using compileCtx), files)(using compileCtx)
+        doCompile(files)(using compileCtx)
       case None =>
         rootCtx.reporter
   }

compiler/src/dotty/tools/dotc/Run.scala

@@ -287,9 +287,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       then ActiveProfile(ctx.settings.VprofileDetails.value.max(0).min(1000))
       else NoProfile
 
-    // If testing pickler, make sure to stop after pickling phase:
+    // If testing pickler, of outline first pass, make sure to stop after pickling phase:
     val stopAfter =
       if (ctx.settings.YtestPickler.value) List("pickler")
+      else if (ctx.isOutlineFirstPass) List("sbt-api")
       else ctx.settings.YstopAfter.value
 
     val pluginPlan = ctx.base.addPluginPhases(ctx.base.phasePlan)

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -1381,6 +1381,17 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def runtimeCall(name: TermName, args: List[Tree])(using Context): Tree =
     Ident(defn.ScalaRuntimeModule.requiredMethod(name).termRef).appliedToTermArgs(args)
 
+  object ElidedTree:
+    def from(tree: Tree)(using Context): Tree =
+      Typed(tree, TypeTree(AnnotatedType(tree.tpe, Annotation(defn.ElidedTreeAnnot, tree.span))))
+
+    def isElided(tree: Tree)(using Context): Boolean = unapply(tree)
+
+    def unapply(tree: Tree)(using Context): Boolean = tree match
+      case Typed(_, tpt) => tpt.tpe.hasAnnotation(defn.ElidedTreeAnnot)
+      case _ => false
+
+
   /** An extractor that pulls out type arguments */
   object MaybePoly:
     def unapply(tree: Tree): Option[(Tree, List[Tree])] = tree match