add support for pipeline build of Scala and Java files

- add '-Ypickle-java' and '-Ypickle-write' flags expected by Zinc
- move ExtractAPI phase to after Pickler, this way we can do it in
  parallel with generating TASTy bytes. At the end of this phase we
  write the TASTy to the -Yearly-tasty-output destination.
- test the pipelining with sbt scripted tests, including for inline
  methods and macros with pipelining
- describe semantics with respect to suspensions,
  introduce -Yno-suspended-units flag for greater control by the user.

Author: bishabosha

Diff for: compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -94,12 +94,15 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
     // when this unit is unsuspended.
     depRecorder.clear()
     if !suspended then
-      if (ctx.settings.XprintSuspension.value)
-        report.echo(i"suspended: $this")
-      suspended = true
-      ctx.run.nn.suspendedUnits += this
-      if ctx.phase == Phases.inliningPhase then
-        suspendedAtInliningPhase = true
+      if ctx.settings.YnoSuspendedUnits.value then
+        report.error(i"Compilation unit suspended $this (-Yno-suspended-units is set)")
+      else
+        if (ctx.settings.XprintSuspension.value)
+          report.echo(i"suspended: $this")
+        suspended = true
+        ctx.run.nn.suspendedUnits += this
+        if ctx.phase == Phases.inliningPhase then
+          suspendedAtInliningPhase = true
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

Diff for: compiler/src/dotty/tools/dotc/Compiler.scala

@@ -41,13 +41,13 @@ class Compiler {
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
-    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new SetRootTree) ::        // Set the `rootTreeOrProvider` on class symbols
     Nil
 
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
     List(new Pickler) ::            // Generate TASTY info
+    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new CheckUnused.PostInlining) ::  // Check for unused elements

Diff for: compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -374,6 +374,7 @@ private sealed trait YSettings:
   val YprintPos: Setting[Boolean] = BooleanSetting("-Yprint-pos", "Show tree positions.")
   val YprintPosSyms: Setting[Boolean] = BooleanSetting("-Yprint-pos-syms", "Show symbol definitions positions.")
   val YnoDeepSubtypes: Setting[Boolean] = BooleanSetting("-Yno-deep-subtypes", "Throw an exception on deep subtyping call stacks.")
+  val YnoSuspendedUnits: Setting[Boolean] = BooleanSetting("-Yno-suspended-units", "Do not suspend units, e.g. when calling a macro defined in the same run. This will error instead of suspending.")
   val YnoPatmatOpt: Setting[Boolean] = BooleanSetting("-Yno-patmat-opt", "Disable all pattern matching optimizations.")
   val YplainPrinter: Setting[Boolean] = BooleanSetting("-Yplain-printer", "Pretty-print using a plain printer.")
   val YprintSyms: Setting[Boolean] = BooleanSetting("-Yprint-syms", "When printing trees print info in symbols instead of corresponding info in trees.")
@@ -435,7 +436,7 @@ private sealed trait YSettings:
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
 
   // Pipeline compilation options
-  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
-  val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting("-Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile)
+  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools", aliases = List("-Ypickle-java"))
+  val YearlyTastyOutput: Setting[AbstractFile] = OutputSetting("-Yearly-tasty-output", "directory|jar", "Destination for generated .tasty files containing possibly outline type signatures.", NoAbstractFile, aliases = List("-Ypickle-write"))
   val YallowOutlineFromTasty: Setting[Boolean] = BooleanSetting("-Yallow-outline-from-tasty", "Allow outline TASTy to be loaded with the -from-tasty option.")
 end YSettings

Diff for: compiler/src/dotty/tools/dotc/config/Settings.scala

@@ -308,8 +308,8 @@ object Settings:
     def MultiStringSetting(name: String, helpArg: String, descr: String, default: List[String] = Nil, aliases: List[String] = Nil): Setting[List[String]] =
       publish(Setting(name, descr, default, helpArg, aliases = aliases))
 
-    def OutputSetting(name: String, helpArg: String, descr: String, default: AbstractFile): Setting[AbstractFile] =
-      publish(Setting(name, descr, default, helpArg))
+    def OutputSetting(name: String, helpArg: String, descr: String, default: AbstractFile, aliases: List[String] = Nil): Setting[AbstractFile] =
+      publish(Setting(name, descr, default, helpArg, aliases = aliases))
 
     def PathSetting(name: String, descr: String, default: String, aliases: List[String] = Nil): Setting[String] =
       publish(Setting(name, descr, default, aliases = aliases))

Diff for: compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1062,6 +1062,7 @@ class Definitions {
   @tu lazy val RetainsByNameAnnot: ClassSymbol = requiredClass("scala.annotation.retainsByName")
 
   @tu lazy val JavaRepeatableAnnot: ClassSymbol = requiredClass("java.lang.annotation.Repeatable")
+  @tu lazy val JavaAnnotationAnnot: ClassSymbol = requiredClass("java.lang.annotation.Annotation")
 
   // Initialization annotations
   @tu lazy val InitModule: Symbol = requiredModule("scala.annotation.init")

Diff for: compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -210,6 +210,7 @@ object Phases {
     private var myTyperPhase: Phase = uninitialized
     private var myPostTyperPhase: Phase = uninitialized
     private var mySbtExtractDependenciesPhase: Phase = uninitialized
+    private var mySbtExtractAPIPhase: Phase = uninitialized
     private var myPicklerPhase: Phase = uninitialized
     private var myInliningPhase: Phase = uninitialized
     private var myStagingPhase: Phase = uninitialized
@@ -235,6 +236,7 @@ object Phases {
     final def typerPhase: Phase = myTyperPhase
     final def postTyperPhase: Phase = myPostTyperPhase
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
+    final def sbtExtractAPIPhase: Phase = mySbtExtractAPIPhase
     final def picklerPhase: Phase = myPicklerPhase
     final def inliningPhase: Phase = myInliningPhase
     final def stagingPhase: Phase = myStagingPhase
@@ -263,6 +265,7 @@ object Phases {
       myTyperPhase = phaseOfClass(classOf[TyperPhase])
       myPostTyperPhase = phaseOfClass(classOf[PostTyper])
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
+      mySbtExtractAPIPhase = phaseOfClass(classOf[sbt.ExtractAPI])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
       myInliningPhase = phaseOfClass(classOf[Inlining])
       myStagingPhase = phaseOfClass(classOf[Staging])
@@ -343,7 +346,7 @@ object Phases {
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       val buf = List.newBuilder[CompilationUnit]
       // factor out typedAsJava check when not needed
-      val doSkipJava = ctx.settings.YjavaTasty.value && this <= picklerPhase && skipIfJava
+      val doSkipJava = ctx.settings.YjavaTasty.value && this <= sbtExtractAPIPhase && skipIfJava
       for unit <- units do
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
         if ctx.run.enterUnit(unit) then
@@ -503,6 +506,7 @@ object Phases {
   def typerPhase(using Context): Phase                  = ctx.base.typerPhase
   def postTyperPhase(using Context): Phase              = ctx.base.postTyperPhase
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
+  def sbtExtractAPIPhase(using Context): Phase          = ctx.base.sbtExtractAPIPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
   def stagingPhase(using Context): Phase               = ctx.base.stagingPhase

Diff for: compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -467,7 +467,8 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
       val tastyUUID = unpickler.unpickler.header.uuid
       new ClassfileTastyUUIDParser(classfile)(ctx).checkTastyUUID(tastyUUID)
     else
-      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`
+      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`, or when
+      // tasty file compiled by `-Yearly-tasty-output-write` comes from an early output jar.
       report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
   private def mayLoadTreesFromTasty(using Context): Boolean =

Diff for: compiler/src/dotty/tools/dotc/inlines/Inliner.scala

@@ -1042,6 +1042,9 @@ class Inliner(val call: tpd.Tree)(using Context):
       for sym <- dependencies do
         if ctx.compilationUnit.source.file == sym.associatedFile then
           report.error(em"Cannot call macro $sym defined in the same source file", call.srcPos)
+        else if ctx.settings.YnoSuspendedUnits.value then
+          val addendum = ", suspension prevented by -Yno-suspended-units"
+          report.error(em"Cannot call macro $sym defined in the same compilation run$addendum", call.srcPos)
         if (suspendable && ctx.settings.XprintSuspension.value)
           report.echo(i"suspension triggered by macro call to ${sym.showLocated} in ${sym.associatedFile}", call.srcPos)
       if suspendable then

Diff for: compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -19,7 +19,8 @@ import StdNames.str
 import NameOps.*
 import inlines.Inlines
 import transform.ValueClasses
-import dotty.tools.io.{File, FileExtension, JarArchive}
+import transform.Pickler
+import dotty.tools.io.{File, FileExtension, JarArchive, ClassfileWriterOps}
 import util.{Property, SourceFile}
 import java.io.PrintWriter
 
@@ -65,16 +66,68 @@ class ExtractAPI extends Phase {
   // after `PostTyper` (unlike `ExtractDependencies`, the simplication to trees
   // done by `PostTyper` do not affect this phase because it only cares about
   // definitions, and `PostTyper` does not change definitions).
-  override def runsAfter: Set[String] = Set(transform.PostTyper.name)
+  override def runsAfter: Set[String] = Set(transform.Pickler.name)
 
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
+    val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YearlyTastyOutput.value match
+      case earlyOut if earlyOut.isDirectory && earlyOut.exists =>
+        Some(Pickler.EarlyFileWriter(ClassfileWriterOps(earlyOut)))
+      case _ =>
+        None
     val nonLocalClassSymbols = new mutable.HashSet[Symbol]
     val ctx0 = ctx.withProperty(NonLocalClassSymbolsInCurrentUnits, Some(nonLocalClassSymbols))
     val units0 = super.runOn(units)(using ctx0)
+    sigWriter.foreach(writeSigFiles(units0, _))
     ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _))
-    units0
+    if ctx.settings.YjavaTasty.value then
+      units0.filterNot(_.typedAsJava) // remove java sources, this is the terminal phase when `-Yjava-tasty` is set
+    else
+      units0
   end runOn
 
+  // Why we only write to early output in the first run?
+  // ===================================================
+  // TL;DR the point of pipeline compilation is to start downstream projects early,
+  // so we don't want to wait for suspended units to be compiled.
+  //
+  // But why is it safe to ignore suspended units?
+  // If this project contains a transparent macro that is called in the same project,
+  // the compilation unit of that call will be suspended (if the macro implementation
+  // is also in this project), causing a second run.
+  // However before we do that run, we will have already requested sbt to begin
+  // early downstream compilation. This means that the suspended definitions will not
+  // be visible in *early* downstream compilation.
+  //
+  // However, sbt will by default prevent downstream compilation happening in this scenario,
+  // due to the existence of macro definitions. So we are protected from failure if user tries
+  // to use the suspended definitions.
+  //
+  // Additionally, it is recommended for the user to move macro implementations to another project
+  // if they want to force early output. In this scenario the suspensions will no longer occur, so now
+  // they will become visible in the early-output.
+  //
+  // See `sbt-test/pipelining/pipelining-scala-macro` and `sbt-test/pipelining/pipelining-scala-macro-force`
+  // for examples of this in action.
+  //
+  // Therefore we only need to write to early output in the first run. We also provide the option
+  // to diagnose suspensions with the `-Yno-suspended-units` flag.
+  private def writeSigFiles(units: List[CompilationUnit], writer: Pickler.EarlyFileWriter)(using Context): Unit = {
+    try
+      for
+        unit <- units
+        (cls, pickled) <- unit.pickled
+        if cls.isDefinedInCurrentRun
+      do
+        val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn
+        val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName
+        writer.writeTasty(binaryClassName, pickled())
+    finally
+      writer.close()
+      if ctx.settings.verbose.value then
+        report.echo("[sig files written]")
+    end try
+  }
+
   private def recordNonLocalClasses(nonLocalClassSymbols: mutable.HashSet[Symbol], cb: interfaces.IncrementalCallback)(using Context): Unit =
     for cls <- nonLocalClassSymbols if !cls.isLocal do
       val sourceFile = cls.source

Diff for: compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -45,7 +45,7 @@ class Pickler extends Phase {
 
   // No need to repickle trees coming from TASTY
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && (!ctx.settings.fromTasty.value || ctx.settings.YjavaTasty.value)
+    super.isRunnable && !ctx.settings.fromTasty.value
 
   // when `-Yjava-tasty` is set we actually want to run this phase on Java sources
   override def skipIfJava(using Context): Boolean = false
@@ -82,9 +82,7 @@ class Pickler extends Phase {
 
   private val executor = Executor[Array[Byte]]()
 
-  private def useExecutor(using Context) =
-    Pickler.ParallelPickling && !ctx.settings.YtestPickler.value &&
-    !ctx.settings.YjavaTasty.value // disable parallel pickling when `-Yjava-tasty` is set (internal testing only)
+  private def useExecutor(using Context) = Pickler.ParallelPickling && !ctx.settings.YtestPickler.value
 
   override def run(using Context): Unit = {
     val unit = ctx.compilationUnit
@@ -172,22 +170,13 @@ class Pickler extends Phase {
   }
 
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = {
-    val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YjavaTastyOutput.value match
-      case jar: JarArchive if jar.exists =>
-        Some(Pickler.EarlyFileWriter(ClassfileWriterOps(jar)))
-      case _ =>
-        None
-    val units0 =
-      if ctx.settings.fromTasty.value then
-        // we still run the phase for the side effect of writing the pipeline tasty files
-        units
+    val result =
+      if useExecutor then
+        executor.start()
+        try super.runOn(units)
+        finally executor.close()
       else
-        if useExecutor then
-          executor.start()
-          try super.runOn(units)
-          finally executor.close()
-        else
-          super.runOn(units)
+        super.runOn(units)
     if ctx.settings.YtestPickler.value then
       val ctx2 = ctx.fresh
         .setSetting(ctx.settings.YreadComments, true)
@@ -198,34 +187,9 @@ class Pickler extends Phase {
           .setReporter(new ThrowingReporter(ctx.reporter))
           .addMode(Mode.ReadPositions)
       )
-    val result =
-      if ctx.settings.YjavaTasty.value then
-        sigWriter.foreach(writeJavaSigFiles(units0, _))
-        units0.filterNot(_.typedAsJava) // remove java sources, this is the terminal phase when `-Yjava-tasty` is set
-      else
-        units0
     result
   }
 
-  private def writeJavaSigFiles(units: List[CompilationUnit], writer: Pickler.EarlyFileWriter)(using Context): Unit = {
-    var count = 0
-    try
-      for
-        unit <- units if unit.typedAsJava
-        (cls, pickled) <- unit.pickled
-        if cls.isDefinedInCurrentRun
-      do
-        val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn
-        val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName
-        writer.writeTasty(binaryClassName, pickled())
-        count += 1
-    finally
-      writer.close()
-      if ctx.settings.verbose.value then
-        report.echo(s"[$count java sig files written]")
-    end try
-  }
-
   private def testUnpickler(using Context): Unit =
     pickling.println(i"testing unpickler at run ${ctx.runId}")
     ctx.initialize()