Reduce ammount of allocations, increase ammontof counters tracing and other cleanups

Author: WojciechMazur

Diff for: compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -444,7 +444,7 @@ private sealed trait YSettings:
   val YprofileDestination: Setting[String] = StringSetting(ForkSetting, "Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "", depends = List(YprofileEnabled -> true))
   val YprofileExternalTool: Setting[List[String]] = PhasesSetting(ForkSetting, "Yprofile-external-tool", "Enable profiling for a phase using an external tool hook. Generally only useful for a single phase.", "typer", depends = List(YprofileEnabled -> true))
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting(ForkSetting, "Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_", depends = List(YprofileEnabled -> true))
-  val YprofileTrace: Setting[String]  = StringSetting(ForkSetting, "Yprofile-trace", "file", s"Capture trace of compilation in Chrome Trace format, requires ${YprofileEnabled.name}", "", depends = List(YprofileEnabled -> true))
+  val YprofileTrace: Setting[String]  = StringSetting(ForkSetting, "Yprofile-trace", "file", s"Capture trace of compilation in JSON Chrome Trace format to the specified file. This option requires ${YprofileEnabled.name}. The output file can be visualized using https://ui.perfetto.dev/.", "", depends = List(YprofileEnabled -> true))
 
   val YbestEffort: Setting[Boolean] = BooleanSetting(ForkSetting, "Ybest-effort", "Enable best-effort compilation attempting to produce betasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
   val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Ywith-best-effort-tasty", "Allow to compile using best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")

Diff for: compiler/src/dotty/tools/dotc/config/Settings.scala

@@ -390,8 +390,8 @@ object Settings:
     def BooleanSetting(category: SettingCategory, name: String, descr: String, initialValue: Boolean = false, aliases: List[String] = Nil, preferPrevious: Boolean = false, deprecation: Option[Deprecation] = None, ignoreInvalidArgs: Boolean = false): Setting[Boolean] =
       publish(Setting(category, prependName(name), descr, initialValue, aliases = aliases, preferPrevious = preferPrevious, deprecation = deprecation, ignoreInvalidArgs = ignoreInvalidArgs))
 
-    def StringSetting(category: SettingCategory, name: String, helpArg: String, descr: String, default: String, aliases: List[String] = Nil): Setting[String] =
-      publish(Setting(category, prependName(name), descr, default, helpArg, aliases = aliases))
+    def StringSetting(category: SettingCategory, name: String, helpArg: String, descr: String, default: String, aliases: List[String] = Nil, deprecation: Option[Deprecation] = None, depends: SettingDependencies = Nil): Setting[String] =
+      publish(Setting(category, prependName(name), descr, default, helpArg, aliases = aliases, deprecation = deprecation, depends = depends))
 
     def ChoiceSetting(category: SettingCategory, name: String, helpArg: String, descr: String, choices: List[String], default: String, aliases: List[String] = Nil, legacyArgs: Boolean = false, deprecation: Option[Deprecation] = None): Setting[String] =
       publish(Setting(category, prependName(name), descr, default, helpArg, Some(choices), aliases = aliases, legacyArgs = legacyArgs, deprecation = deprecation))
@@ -417,8 +417,8 @@ object Settings:
     def PathSetting(category: SettingCategory, name: String, descr: String, default: String, aliases: List[String] = Nil, deprecation: Option[Deprecation] = None): Setting[String] =
       publish(Setting(category, prependName(name), descr, default, aliases = aliases, deprecation = deprecation))
 
-    def PhasesSetting(category: SettingCategory, name: String, descr: String, default: String = "", aliases: List[String] = Nil): Setting[List[String]] =
-      publish(Setting(category, prependName(name), descr, if (default.isEmpty) Nil else List(default), aliases = aliases))
+    def PhasesSetting(category: SettingCategory, name: String, descr: String, default: String = "", aliases: List[String] = Nil, deprecation: Option[Deprecation] = None, depends: SettingDependencies = Nil): Setting[List[String]] =
+      publish(Setting(category, prependName(name), descr, if (default.isEmpty) Nil else List(default), aliases = aliases, deprecation = deprecation, depends = depends))
 
     def PrefixSetting(category: SettingCategory, name0: String, descr: String, deprecation: Option[Deprecation] = None): Setting[List[String]] =
       val name = prependName(name0)

Diff for: compiler/src/dotty/tools/dotc/profile/ChromeTrace.scala

@@ -38,8 +38,8 @@ object ChromeTrace {
   }
 }
 
-/** Allows writing a subset of https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview#
-  * for use in Chrome's about://tracing or the tooling in https://www.google.com.au/search?q=catapult+tracing&oq=catapult+tracing+&aqs=chrome..69i57.3974j0j4&sourceid=chrome&ie=UTF-8 */
+/** Allows writing a subset of captrue traces based on https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview#
+  * Can be visualized using https://ui.perfetto.dev/, Chrome's about://tracing (outdated) or the tooling in https://www.google.com.au/search?q=catapult+tracing&oq=catapult+tracing+&aqs=chrome..69i57.3974j0j4&sourceid=chrome&ie=UTF-8 */
 final class ChromeTrace(f: Path) extends Closeable {
   import ChromeTrace.EventType
   private val traceWriter = FileUtils.newAsyncBufferedWriter(f)
@@ -125,11 +125,11 @@ final class ChromeTrace(f: Path) extends Closeable {
 
   private def microTime(): Long = nanosToMicros(System.nanoTime())
 
-  sealed abstract class JsonContext
-  case class ArrayContext(var first: Boolean) extends JsonContext
-  case class ObjectContext(var first: Boolean) extends JsonContext
-  case object ValueContext extends JsonContext
-  case object TopContext extends JsonContext
+  private sealed abstract class JsonContext
+  private case class ArrayContext(var first: Boolean) extends JsonContext
+  private case class ObjectContext(var first: Boolean) extends JsonContext
+  private case object ValueContext extends JsonContext
+  private case object TopContext extends JsonContext
 
   private def str(name: String, value: String): Unit = {
     fld(name)

Diff for: compiler/src/dotty/tools/dotc/profile/FileUtils.scala

@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Promise}
 import scala.util.{Failure, Success}
+import scala.annotation.internal.sharable
 
 object FileUtils {
   def newAsyncBufferedWriter(path: Path, charset: Charset = StandardCharsets.UTF_8.nn, options: Array[OpenOption] = NO_OPTIONS, threadsafe: Boolean = false): LineWriter = {
@@ -72,8 +73,8 @@ object FileUtils {
   }
 
   private object AsyncBufferedWriter {
-    private val Close = CharBuffer.allocate(0)
-    private val Flush = CharBuffer.allocate(0)
+    @sharable private val Close = CharBuffer.allocate(0)
+    @sharable private val Flush = CharBuffer.allocate(0)
   }
   private class AsyncBufferedWriter(val underlying: Writer, bufferSize : Int = 4096) extends LineWriter {
     private var current: CharBuffer = allocate

Diff for: compiler/src/dotty/tools/dotc/profile/Profiler.scala

@@ -92,7 +92,8 @@ sealed trait Profiler {
     val (event, snapshot) = beforePhase(phase)
     try body
     finally afterPhase(event, phase, snapshot)
-  protected def beforePhase(phase: Phase): (TracedEventId, ProfileSnap) = (TracedEventId.Empty, Profiler.emptySnap)
+  protected final val EmptyPhaseEvent = (TracedEventId.Empty, Profiler.emptySnap)
+  protected def beforePhase(phase: Phase): (TracedEventId, ProfileSnap) = EmptyPhaseEvent
   protected def afterPhase(event: TracedEventId, phase: Phase, profileBefore: ProfileSnap): Unit = ()
 
   inline def onUnit[T](phase: Phase, unit: CompilationUnit)(inline body: T): T =
@@ -127,13 +128,14 @@ sealed trait Profiler {
     val (event, completionName) = beforeCompletion(root, associatedFile)
     try body
     finally afterCompletion(event, completionName)
-  protected def beforeCompletion(root: Symbol, associatedFile: => AbstractFile): (TracedEventId, String) = (TracedEventId.Empty, "")
+  protected final val EmptyCompletionEvent = (TracedEventId.Empty, "")
+  protected def beforeCompletion(root: Symbol, associatedFile: => AbstractFile): (TracedEventId, String) = EmptyCompletionEvent
   protected def afterCompletion(event: TracedEventId, completionName: String): Unit = ()
 }
 private [profile] object NoOpProfiler extends Profiler {
-
   override def finished(): Unit = ()
 }
+
 private [profile] object RealProfiler {
   import scala.jdk.CollectionConverters.*
   val runtimeMx: RuntimeMXBean = ManagementFactory.getRuntimeMXBean
@@ -234,10 +236,7 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
     import java.lang.{Integer => jInt}
     val reportNs = System.nanoTime()
     val data = notification.getUserData
-    // val seq = notification.getSequenceNumber
-    // val message = notification.getMessage
     val tpe = notification.getType
-    // val time= notification.getTimeStamp
     data match {
       case cd: CompositeData if tpe == "com.sun.management.gc.notification" =>
         val name = cd.get("gcName").toString
@@ -267,12 +266,13 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
     }
     else initialSnap
     traceDurationEnd(Category.Phase, event)
-
+    traceThreadSnapshotCounters()
     reporter.reportForeground(this, ProfileRange(snapBefore, finalSnap, phase, "", 0, Thread.currentThread))
   }
 
   override def beforePhase(phase: Phase): (TracedEventId, ProfileSnap) = {
     assert(mainThread eq Thread.currentThread())
+    traceThreadSnapshotCounters()
     val eventId = traceDurationStart(Category.Phase, phase.phaseName)
     if (ctx.settings.YprofileRunGcBetweenPhases.value.contains(phase.toString))
       doGC()
@@ -285,26 +285,32 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
 
   override def beforeUnit(phase: Phase, unit: CompilationUnit): TracedEventId = {
     assert(mainThread eq Thread.currentThread())
-    traceDurationStart(Category.File, unit.source.name)
+    if chromeTrace != null then
+      traceThreadSnapshotCounters()
+      traceDurationStart(Category.File, unit.source.name)
+    else TracedEventId.Empty
   }
 
   override def afterUnit(event: TracedEventId): Unit = {
     assert(mainThread eq Thread.currentThread())
     if chromeTrace != null then
-      val now = System.nanoTime()
       traceDurationEnd(Category.File, event)
-      if now > nextAfterUnitSnap then
-        val initialSnap = RealProfiler.snapThread(0)
-        chromeTrace.traceCounterEvent("allocBytes", "allocBytes", initialSnap.allocatedBytes, processWide = false)
-        chromeTrace.traceCounterEvent("heapBytes", "heapBytes", initialSnap.heapBytes, processWide = true)
-        chromeTrace.traceCounterEvent("classesLoaded", "classesLoaded", initialSnap.totalClassesLoaded, processWide = true)
-        chromeTrace.traceCounterEvent("jitCompilationTime", "jitCompilationTime", initialSnap.totalJITCompilationTime, processWide = true)
-        chromeTrace.traceCounterEvent("userTime", "userTime", initialSnap.userTimeNanos, processWide = false)
-        chromeTrace.traceCounterEvent("cpuTime", "cpuTime", initialSnap.cpuTimeNanos, processWide = false)
-        chromeTrace.traceCounterEvent("idleTime", "idleTime", initialSnap.idleTimeNanos, processWide = false)
-        nextAfterUnitSnap = System.nanoTime() + 10 * 1000 * 1000
+      traceThreadSnapshotCounters()
   }
 
+  private def traceThreadSnapshotCounters(initialSnap: => ProfileSnap = RealProfiler.snapThread(0)) =
+    if chromeTrace != null && System.nanoTime() > nextAfterUnitSnap then {
+      val snap = initialSnap
+      chromeTrace.traceCounterEvent("allocBytes", "allocBytes", snap.allocatedBytes, processWide = false)
+      chromeTrace.traceCounterEvent("heapBytes", "heapBytes", snap.heapBytes, processWide = true)
+      chromeTrace.traceCounterEvent("classesLoaded", "classesLoaded", snap.totalClassesLoaded, processWide = true)
+      chromeTrace.traceCounterEvent("jitCompilationTime", "jitCompilationTime", snap.totalJITCompilationTime, processWide = true)
+      chromeTrace.traceCounterEvent("userTime", "userTime", snap.userTimeNanos, processWide = false)
+      chromeTrace.traceCounterEvent("cpuTime", "cpuTime", snap.cpuTimeNanos, processWide = false)
+      chromeTrace.traceCounterEvent("idleTime", "idleTime", snap.idleTimeNanos, processWide = false)
+      nextAfterUnitSnap = System.nanoTime() + 10 * 1000 * 1000
+    }
+
   override def beforeTypedDef(sym: Symbol): TracedEventId = traceDurationStart(Category.TypeCheck, symbolName(sym))
   override def afterTypedDef(event: TracedEventId): Unit = traceDurationEnd(Category.TypeCheck, event)
 
@@ -316,9 +322,9 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
 
   override def beforeCompletion(root: Symbol, associatedFile: => AbstractFile): (TracedEventId, String) =
     if chromeTrace == null
-    then (TracedEventId.Empty, "")
+    then EmptyCompletionEvent
     else
-      val completionName=  this.completionName(root, associatedFile)
+      val completionName = this.completionName(root, associatedFile)
       val event = TracedEventId(associatedFile.name)
       chromeTrace.traceDurationEventStart(Category.Completion.name, "↯", colour = "thread_state_sleeping")
       chromeTrace.traceDurationEventStart(Category.File.name, event)
@@ -332,7 +338,6 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
       chromeTrace.traceDurationEventEnd(Category.File.name, event)
       chromeTrace.traceDurationEventEnd(Category.Completion.name, "↯", colour = "thread_state_sleeping")
 
-
   private inline def traceDurationStart(category: Category, inline eventName: String, colour: String = ""): TracedEventId =
     if chromeTrace == null
     then TracedEventId.Empty
@@ -342,9 +347,8 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
       event
 
   private inline def traceDurationEnd(category: Category, event: TracedEventId, colour: String = ""): Unit =
-    if chromeTrace != null
-    then chromeTrace.traceDurationEventEnd(category.name, event, colour)
-
+    if chromeTrace != null then
+      chromeTrace.traceDurationEventEnd(category.name, event, colour)
 
   private def symbolName(sym: Symbol): String = s"${sym.showKind} ${sym.showName}"
   private def completionName(root: Symbol, associatedFile: AbstractFile): String =
@@ -356,15 +360,13 @@ private [profile] class RealProfiler(reporter : ProfileReporter)(using Context)
       s"${enclosing.javaBinaryName}::${root.name}"
 }
 
-case class EventType(name: String)
-object EventType {
-  //main thread with other tasks
-  val MAIN: EventType = EventType("main")
-  //other task ( background thread)
-  val BACKGROUND: EventType = EventType("background")
-  //total for compile
-  val GC: EventType = EventType("GC")
-}
+enum EventType(name: String):
+  // main thread with other tasks
+  case MAIN extends EventType("main")
+  // other task ( background thread)
+  case BACKGROUND extends EventType("background")
+  // total for compile
+  case GC extends EventType("GC")
 
 sealed trait ProfileReporter {
   def reportBackground(profiler: RealProfiler, threadRange: ProfileRange): Unit
@@ -418,7 +420,6 @@ class StreamProfileReporter(out:PrintWriter) extends ProfileReporter {
     out.println(s"${EventType.GC},$start,${data.reportTimeNs},${data.gcStartMillis}, ${data.gcEndMillis},${data.name},${data.action},${data.cause},${data.threads}")
   }
 
-
   override def close(profiler: RealProfiler): Unit = {
     out.flush()
     out.close()