From ebc6f9de4a63959d7a466401a06db9d66662affe Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Wed, 27 Jun 2018 04:25:20 +0200 Subject: [PATCH 1/4] Output .tasty files in Pickler phase Also simplify the logic to write the .tasty files. --- compiler/src/dotty/tools/backend/jvm/GenBCode.scala | 5 ----- compiler/src/dotty/tools/dotc/transform/Pickler.scala | 11 +++++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala index ffae0c29ff7f..8ba9fa924a2a 100644 --- a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala +++ b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala @@ -218,11 +218,6 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter val store = if (mirrorC ne null) mirrorC else plainC val tasty = if (!ctx.settings.YemitTastyInClass.value) { - val outTastyFile = getFileForClassfile(outF, store.name, ".tasty") - val outstream = new DataOutputStream(outTastyFile.bufferedOutput) - try outstream.write(binary) - finally outstream.close() - val uuid = new TastyHeaderUnpickler(binary).readHeader() val lo = uuid.getMostSignificantBits val hi = uuid.getLeastSignificantBits diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 46e437938cc8..d89cbec616a6 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -13,6 +13,7 @@ import Symbols._ import Flags.Module import reporting.ThrowingReporter import collection.mutable +import NameOps._ object Pickler { val name = "pickler" @@ -67,6 +68,16 @@ class Pickler extends Phase { val pickled = pickler.assembleParts() unit.pickled += (cls -> pickled) + if (!ctx.settings.YemitTastyInClass.value) { + val parts = cls.fullName.stripModuleClassSuffix.mangledString.split('.') + val name = parts.last + val tastyDirectory = parts.init.foldLeft(ctx.settings.outputDir.value)((dir, part) => dir.subdirectoryNamed(part)) + val tastyFile = tastyDirectory.fileNamed(s"${name}.tasty") + val tastyOutput = tastyFile.output + try tastyOutput.write(pickled) + finally tastyOutput.close() + } + def rawBytes = // not needed right now, but useful to print raw format. pickled.iterator.grouped(10).toList.zipWithIndex.map { case (row, i) => s"${i}0: ${row.mkString(" ")}" From 648b5f655427428e1effdc83749071d1773a3c1d Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 17 Aug 2018 13:13:41 +0200 Subject: [PATCH 2/4] Small optimization/cleanup --- compiler/src/dotty/tools/dotc/transform/Pickler.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index d89cbec616a6..98616da2f950 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -78,14 +78,14 @@ class Pickler extends Phase { finally tastyOutput.close() } - def rawBytes = // not needed right now, but useful to print raw format. - pickled.iterator.grouped(10).toList.zipWithIndex.map { - case (row, i) => s"${i}0: ${row.mkString(" ")}" - } +// def rawBytes = // not needed right now, but useful to print raw format. +// pickled.iterator.grouped(10).toList.zipWithIndex.map { +// case (row, i) => s"${i}0: ${row.mkString(" ")}" +// } // println(i"rawBytes = \n$rawBytes%\n%") // DEBUG if (pickling ne noPrinter) { println(i"**** pickled info of $cls") - new TastyPrinter(pickler.assembleParts()).printContents() + new TastyPrinter(pickled).printContents() } } } From b032c9f4fbe5ad9e2ae664af51805a53f1870b5e Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 17 Aug 2018 13:40:58 +0200 Subject: [PATCH 3/4] Do not store tasty bytes when not placed the classfile To allow the bytes to be GCed after pickler --- .../dotty/tools/backend/jvm/GenBCode.scala | 41 +++++++++++-------- .../dotty/tools/dotc/CompilationUnit.scala | 7 +++- .../dotc/core/quoted/PickledQuotes.scala | 2 +- .../tools/dotc/core/tasty/TastyPickler.scala | 9 ++-- .../dotty/tools/dotc/transform/Pickler.scala | 11 +++-- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala index 8ba9fa924a2a..115eeac645c0 100644 --- a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala +++ b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala @@ -213,31 +213,36 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter val outF = if (needsOutFolder) getOutFolder(claszSymbol, pcb.thisName) else null; val plainC = pcb.cnode - if (claszSymbol.isClass) // @DarkDimius is this test needed here? - for (binary <- ctx.compilationUnit.pickled.get(claszSymbol.asClass)) { + if (claszSymbol.isClass) { // @DarkDimius is this test needed here? + def storeTastyTagBytes(getBytes: String => Array[Byte]): Unit = { val store = if (mirrorC ne null) mirrorC else plainC - val tasty = - if (!ctx.settings.YemitTastyInClass.value) { - val uuid = new TastyHeaderUnpickler(binary).readHeader() - val lo = uuid.getMostSignificantBits - val hi = uuid.getLeastSignificantBits - + val bytes = getBytes(store.name) + val dataAttr = new CustomAttr(nme.TASTYATTR.mangledString, bytes) + store.visitAttribute(dataAttr) + } + if (ctx.settings.YemitTastyInClass.value) { + for (binary <- ctx.compilationUnit.pickled.get(claszSymbol.asClass)) { + storeTastyTagBytes { name => + // Create an empty file to signal that a tasty section exist in the corresponding .class + // This is much cheaper and simpler to check than doing classfile parsing + getFileForClassfile(outF, name, ".hasTasty") + binary + } + } + } else { + for (uuid <- ctx.compilationUnit.tastyUUID.get(claszSymbol.asClass)) { + assert(!ctx.settings.YemitTastyInClass.value) + storeTastyTagBytes { name => // TASTY attribute is created but only the UUID bytes are stored in it. // A TASTY attribute has length 16 if and only if the .tasty file exists. val buffer = new TastyBuffer(16) - buffer.writeUncompressedLong(lo) - buffer.writeUncompressedLong(hi) + buffer.writeUncompressedLong(uuid.getLeastSignificantBits) + buffer.writeUncompressedLong(uuid.getMostSignificantBits) buffer.bytes - } else { - // Create an empty file to signal that a tasty section exist in the corresponding .class - // This is much cheaper and simpler to check than doing classfile parsing - getFileForClassfile(outF, store.name, ".hasTasty") - binary } - val dataAttr = new CustomAttr(nme.TASTYATTR.mangledString, tasty) - store.visitAttribute(dataAttr) + } } - + } // ----------- create files diff --git a/compiler/src/dotty/tools/dotc/CompilationUnit.scala b/compiler/src/dotty/tools/dotc/CompilationUnit.scala index 210894277f92..20a03b5456f9 100644 --- a/compiler/src/dotty/tools/dotc/CompilationUnit.scala +++ b/compiler/src/dotty/tools/dotc/CompilationUnit.scala @@ -10,6 +10,8 @@ import dotty.tools.dotc.core.SymDenotations.ClassDenotation import dotty.tools.dotc.core.Symbols._ import dotty.tools.dotc.transform.SymUtils._ +import java.util.UUID + class CompilationUnit(val source: SourceFile) { override def toString = source.toString @@ -18,11 +20,14 @@ class CompilationUnit(val source: SourceFile) { var tpdTree: tpd.Tree = tpd.EmptyTree - def isJava = source.file.name.endsWith(".java") + def isJava: Boolean = source.file.name.endsWith(".java") /** Pickled TASTY binaries, indexed by class. */ var pickled: Map[ClassSymbol, Array[Byte]] = Map() + /** UUID of the pickled TASTY, indexed by class. */ + var tastyUUID: Map[ClassSymbol, UUID] = Map() + /** Will be reset to `true` if `untpdTree` contains `Quote` trees. The information * is used in phase ReifyQuotes in order to avoid traversing a quote-less tree. */ diff --git a/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala b/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala index bedcd8c05724..52b37843fb4a 100644 --- a/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala +++ b/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala @@ -92,7 +92,7 @@ object PickledQuotes { if (pickling ne noPrinter) println(i"**** pickling quote of \n${tree.show}") - val pickled = pickler.assembleParts() + val pickled = pickler.assembleParts()._2 if (pickling ne noPrinter) new TastyPrinter(pickled).printContents() diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala index d7e1ddf0d42f..d7c96deab14a 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala @@ -6,10 +6,12 @@ package tasty import TastyFormat._ import collection.mutable import TastyBuffer._ -import core.Symbols.{Symbol, ClassSymbol} +import core.Symbols.{ClassSymbol, Symbol} import ast.tpd import Decorators._ +import java.util.UUID + class TastyPickler(val rootCls: ClassSymbol) { private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)] @@ -19,7 +21,8 @@ class TastyPickler(val rootCls: ClassSymbol) { def newSection(name: String, buf: TastyBuffer) = sections += ((nameBuffer.nameIndex(name.toTermName), buf)) - def assembleParts(): Array[Byte] = { + /** Returns the UUID and bytes of the tasty file */ + def assembleParts(): (UUID, Array[Byte]) = { def lengthWithLength(buf: TastyBuffer) = buf.length + natSize(buf.length) @@ -60,7 +63,7 @@ class TastyPickler(val rootCls: ClassSymbol) { all.writeBytes(buf.bytes, buf.length) } assert(all.length == totalSize && all.bytes.length == totalSize, s"totalSize = $totalSize, all.length = ${all.length}, all.bytes.length = ${all.bytes.length}") - all.bytes + (new UUID(uuidHi, uuidLow), all.bytes) } /** The address in the TASTY file of a given tree, or None if unknown. diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 98616da2f950..a60c0e3df4a6 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -65,10 +65,12 @@ class Pickler extends Phase { new CommentPickler(pickler, treePkl.buf.addrOfTree).pickleComment(tree) // other pickle sections go here. - val pickled = pickler.assembleParts() - unit.pickled += (cls -> pickled) + val (uuid, pickled) = pickler.assembleParts() - if (!ctx.settings.YemitTastyInClass.value) { + if (ctx.settings.YemitTastyInClass.value) { + unit.pickled += (cls -> pickled) + } else { + unit.tastyUUID += (cls -> uuid) val parts = cls.fullName.stripModuleClassSuffix.mangledString.split('.') val name = parts.last val tastyDirectory = parts.init.foldLeft(ctx.settings.outputDir.value)((dir, part) => dir.subdirectoryNamed(part)) @@ -108,7 +110,8 @@ class Pickler extends Phase { ctx.initialize() val unpicklers = for ((cls, pickler) <- picklers) yield { - val unpickler = new DottyUnpickler(pickler.assembleParts()) + val bytes = pickler.assembleParts()._2 + val unpickler = new DottyUnpickler(bytes) unpickler.enter(roots = Set.empty) cls -> unpickler } From a1d4760cbda339989c6f58fcd9c03144f7249267 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Sun, 19 Aug 2018 17:20:57 +0200 Subject: [PATCH 4/4] WIP Enable scripted tests --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 88e6fe8e4007..b29239cef3c7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -42,9 +42,9 @@ pipeline: commands: - cp -R . /tmp/4/ && cd /tmp/4/ - ./project/scripts/sbt sbt-dotty/scripted - when: - # sbt scripted tests are slow and only run on nightly or deployment - event: [ tag, deployment ] + # when: + # # sbt scripted tests are slow and only run on nightly or deployment + # event: [ tag, deployment ] # DOCUMENTATION: documentation: