Merge pull request scala#9685 from retronym/topic/reproducible-osgi

Configure OSGI bnd tool for reproducible builds

Author: lrytz

build.sbt

@@ -15,7 +15,7 @@ object AutomaticModuleName  {
   def settings(name: String): Seq[Def.Setting[_]] = {
     val pair = ("Automatic-Module-Name" -> name)
     Seq(
-      packageOptions in (Compile, packageBin) += Package.ManifestAttributes(pair),
+      (Compile / packageBin / packageOptions) += Package.ManifestAttributes(pair),
       Osgi.headers += pair
     )
   }

project/AutomaticModuleName.scala

@@ -34,14 +34,14 @@ object JitWatchFilePlugin extends AutoPlugin {
 
       // Transitive sources from the projects that contribute to this classpath.
       val projects: Seq[ProjectRef] = buildDependencies.value.classpathTransitiveRefs(thisProjectRef.value) :+ thisProjectRef.value
-      val projectArtifacts: Map[ProjectRef, Seq[Artifact]] = projects.map(project => (project -> (Keys.artifacts in project get settingsData.value).getOrElse(Nil))).toMap
-      val artifactNameToProject: Map[String, Seq[ProjectRef]] = projects.groupBy(project => (Keys.name in project get settingsData.value).getOrElse(""))
+      val projectArtifacts: Map[ProjectRef, Seq[Artifact]] = projects.map(project => (project -> ((project / Keys.artifacts) get settingsData.value).getOrElse(Nil))).toMap
+      val artifactNameToProject: Map[String, Seq[ProjectRef]] = projects.groupBy(project => ((project / Keys.name) get settingsData.value).getOrElse(""))
       val transitiveSourceDirectories =  projects.flatMap { project =>
-        val projectArtifacts: Seq[Artifact] = (Keys.artifacts in project get settingsData.value).getOrElse(Nil)
+        val projectArtifacts: Seq[Artifact] = ((project / Keys.artifacts) get settingsData.value).getOrElse(Nil)
         val matching = projectArtifacts.filter(artifacts.contains(_))
         val configs = matching.flatMap(artifact => artifact.configurations).distinct
         val sourceDirectories: Seq[File] = configs.flatMap { configRef =>
-          (Keys.sourceDirectories in project in sbt.Configuration.of(configRef.name.capitalize, configRef.name)).get(settingsData.value).toList.flatten
+          (project / sbt.Configuration.of(configRef.name.capitalize, configRef.name) / Keys.sourceDirectories).get(settingsData.value).toList.flatten
         }
         sourceDirectories
       }.distinct
@@ -50,7 +50,7 @@ object JitWatchFilePlugin extends AutoPlugin {
         projects.flatMap { project: ProjectRef =>
           val configs = artifact.configurations
           val sourceDirectories: Seq[File] = configs.toList.flatMap { configRef =>
-            (Keys.sourceDirectories in project in sbt.Configuration.of(configRef.name.capitalize, configRef.name)).get(settingsData.value).toList.flatten
+            (project / sbt.Configuration.of(configRef.name.capitalize, configRef.name) / Keys.sourceDirectories).get(settingsData.value).toList.flatten
           }
           sourceDirectories
         }

project/JitWatch.scala

@@ -10,7 +10,7 @@ object License extends AutoPlugin {
 
   override def projectSettings: Seq[Def.Setting[_]] =
     List(packageSrc, packageBin, packageDoc)
-      .map(pkg => mappings in (Compile, pkg) ++= licenseMapping.value)
+      .map(pkg => (Compile / pkg / mappings) ++= licenseMapping.value)
 
   override def buildSettings: Seq[Def.Setting[_]] = Seq(
     licenseMapping := List("LICENSE", "NOTICE").map(fn => (baseDirectory.value / fn) -> fn)

project/License.scala

@@ -8,11 +8,11 @@ import sbt.Keys._
 import collection.JavaConverters._
 import VersionUtil.versionProperties
 
-/** OSGi packaging for the Scala build, distilled from sbt-osgi. We do not use sbt-osgi because it
-  * depends on a newer version of BND which gives slightly different output (probably OK to upgrade
-  * in the future, now that the Ant build has been removed) and does not allow a crucial bit of
+/** OSGi packaging for the Scala build, distilled from sbt-osgi.
+ *
+ * We don't use sbt-osgi (yet) because it does not allow a crucial bit of
   * configuration that we need: Setting the classpath for BND. In sbt-osgi this is always
-  *  `fullClasspath in Compile` whereas we want `products in Compile in packageBin`. */
+  * `fullClasspath in Compile` whereas we want `products in Compile in packageBin`. */
 object Osgi {
   val bundle = TaskKey[File]("osgiBundle", "Create an OSGi bundle.")
   val bundleName = SettingKey[String]("osgiBundleName", "The Bundle-Name for the manifest.")
@@ -29,23 +29,42 @@ object Osgi {
         "Bundle-Name" -> bundleName.value,
         "Bundle-SymbolicName" -> bundleSymbolicName.value,
         "ver" -> v,
-        "Export-Package" -> "*;version=${ver};-split-package:=merge-first",
+
+        // bnd 3.0 fixes for https://github.com/bndtools/bnd/issues/971. This changes our OSGi
+        // metadata by adding Import-Package automatically for all of our exported packages.
+        // Supposedly this is the right thing to do: https://blog.osgi.org/2007/04/importance-of-exporting-nd-importing.html
+        // but I'm disabling the feature (`-noimport:=true`) to avoid changing this detail of
+        // our little understood OSGi metadata for now.
+        "Export-Package" -> "*;version=${ver};-noimport:=true;-split-package:=merge-first",
+
         "Import-Package" -> "scala.*;version=\"${range;[==,=+);${ver}}\",*",
         "Bundle-Version" -> v,
         "Bundle-RequiredExecutionEnvironment" -> "JavaSE-1.8",
-        "-eclipse" -> "false"
+        "-eclipse" -> "false",
+
+        // Great new feature in modern bnd versions: reproducible builds.
+        // Omits the Bundle-LastModified header and avoids using System.currentTimeMillis
+        // for ZIP metadata.
+        "-reproducible" -> "true",
+
+        // https://github.com/bndtools/bnd/commit/2f1d89428559d21857b87b6d5b465a18a300becc (bndlib 4.2.0)
+        // seems to have fixed a bug in its detection class references in Class.forName("some.Class")
+        // For our build, this adds an import on the package "com.cloudius.util" (referred to by an optional
+        // part of JLine. This directive disables the Class.forName scanning. An alternative fix would be
+        // direct this to be an optional dependency (as we do for jline itself with `"Import-Package" -> ("jline.*;resolution:=optional," + ... )`)
+        "-noclassforname" -> "true" //
       )
     },
     jarlist := false,
     bundle := Def.task {
-      val cp = (products in Compile in packageBin).value
+      val cp = (Compile / packageBin / products).value
       val licenseFiles = License.licenseMapping.value.map(_._1)
       bundleTask(headers.value.toMap, jarlist.value, cp,
-        (artifactPath in (Compile, packageBin)).value, cp ++ licenseFiles, streams.value)
+        (Compile / packageBin / artifactPath).value, cp ++ licenseFiles, streams.value)
     }.value,
-    packagedArtifact in (Compile, packageBin) := (((artifact in (Compile, packageBin)).value, bundle.value)),
+    (Compile / packageBin / packagedArtifact) := (((Compile / packageBin / artifact).value, bundle.value)),
     // Also create OSGi source bundles:
-    packageOptions in (Compile, packageSrc) += Package.ManifestAttributes(
+    (Compile / packageSrc / packageOptions) += Package.ManifestAttributes(
       "Bundle-Name" -> (description.value + " Sources"),
       "Bundle-SymbolicName" -> (bundleSymbolicName.value + ".source"),
       "Bundle-Version" -> versionProperties.value.osgiVersion,

project/Osgi.scala

@@ -15,7 +15,7 @@ object ScaladocSettings {
       s.get(artifact.key).isDefined && s.get(moduleID.key).exists(_.organization == "org.webjars")
     val dest = (resourceManaged.value / "webjars").getAbsoluteFile
     IO.createDirectory(dest)
-    val classpathes = (dependencyClasspath in Compile).value
+    val classpathes = (Compile / dependencyClasspath).value
     val files: Seq[File] = classpathes.filter(isWebjar).flatMap { classpathEntry =>
       val jarFile = classpathEntry.data
       IO.unzip(jarFile, dest)

project/ScaladocSettings.scala

@@ -26,7 +26,7 @@ object ScriptCommands {
    * The optional argument is the Artifactory snapshot repository URL. */
   def setupPublishCoreNonOpt = setup("setupPublishCoreNonOpt") { args =>
     Seq(
-      baseVersionSuffix in Global := "SHA-SNAPSHOT"
+      (Global / baseVersionSuffix) := "SHA-SNAPSHOT"
     ) ++ (args match {
       case Seq(url) => publishTarget(url)
       case Nil => Nil
@@ -37,7 +37,7 @@ object ScriptCommands {
     * The optional argument is the Artifactory snapshot repository URL. */
   def setupPublishCore = setup("setupPublishCore") { args =>
     Seq(
-      baseVersionSuffix in Global := "SHA-SNAPSHOT"
+      (Global / baseVersionSuffix) := "SHA-SNAPSHOT"
     ) ++ (args match {
       case Seq(url) => publishTarget(url)
       case Nil => Nil
@@ -48,9 +48,9 @@ object ScriptCommands {
     * The optional argument is the Artifactory snapshot repository URL. */
   def setupValidateTest = setup("setupValidateTest") { args =>
     Seq(
-      testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
+      LocalProject("test") / IntegrationTest / testOptions ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
     ) ++ (args match {
-      case Seq(url) => Seq(resolvers in Global += "scala-pr" at url)
+      case Seq(url) => Seq((Global / resolvers) += "scala-pr" at url)
       case Nil => Nil
     }) ++ enableOptimizer
   }
@@ -61,8 +61,8 @@ object ScriptCommands {
   def setupBootstrapStarr = setup("setupBootstrapStarr") { case Seq(fileOrUrl, ver) =>
     val url = fileToUrl(fileOrUrl)
     Seq(
-      baseVersion in Global := ver,
-      baseVersionSuffix in Global := "SPLIT"
+      (Global / baseVersion) := ver,
+      (Global / baseVersionSuffix) := "SPLIT"
     ) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
   }
 
@@ -72,9 +72,9 @@ object ScriptCommands {
   def setupBootstrapLocker = setup("setupBootstrapLocker") { case Seq(fileOrUrl, ver) =>
     val url = fileToUrl(fileOrUrl)
     Seq(
-      baseVersion in Global := ver,
-      baseVersionSuffix in Global := "SPLIT",
-      resolvers in Global += "scala-pr" at url
+      (Global / baseVersion) := ver,
+      (Global / baseVersionSuffix) := "SPLIT",
+      (Global / resolvers) += "scala-pr" at url
     ) ++ publishTarget(url) ++ noDocs ++ enableOptimizer
   }
 
@@ -88,10 +88,10 @@ object ScriptCommands {
     val targetUrl = fileToUrl(targetFileOrUrl)
     val resolverUrl = fileToUrl(resolverFileOrUrl)
     Seq(
-      baseVersion in Global := ver,
-      baseVersionSuffix in Global := "SPLIT",
-      resolvers in Global += "scala-pr" at resolverUrl,
-      testOptions in IntegrationTest in LocalProject("test") ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
+      (Global / baseVersion) := ver,
+      (Global / baseVersionSuffix) := "SPLIT",
+      (Global / resolvers) += "scala-pr" at resolverUrl,
+      LocalProject("test") / IntegrationTest / testOptions ++= Seq(Tests.Argument("--show-log"), Tests.Argument("--show-diff"))
     ) ++ publishTarget(targetUrl) ++ enableOptimizer
   }
 
@@ -102,11 +102,11 @@ object ScriptCommands {
   def setupBootstrapPublish = setup("setupBootstrapPublish") { case Seq(fileOrUrl, ver) =>
     val url = fileToUrl(fileOrUrl)
     Seq(
-      baseVersion in Global := ver,
-      baseVersionSuffix in Global := "SPLIT",
-      resolvers in Global += "scala-pr" at url,
-      publishTo in Global := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"),
-      credentials in Global ++= {
+      (Global / baseVersion) := ver,
+      (Global / baseVersionSuffix) := "SPLIT",
+      (Global / resolvers) += "scala-pr" at url,
+      (Global / publishTo) := Some("sonatype-releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2"),
+      (Global / credentials) ++= {
         val user = env("SONA_USER")
         val pass = env("SONA_PASS")
         if (user != "" && pass != "")
@@ -152,20 +152,20 @@ object ScriptCommands {
   }
 
   val enableOptimizer = Seq(
-    scalacOptions in Compile in ThisBuild ++= Seq("-opt:l:inline", "-opt-inline-from:scala/**")
+    ThisBuild / Compile / scalacOptions ++= Seq("-opt:l:inline", "-opt-inline-from:scala/**")
   )
 
   val noDocs = Seq(
-    publishArtifact in (Compile, packageDoc) in ThisBuild := false
+    ThisBuild / Compile / packageDoc / publishArtifact := false
   )
 
   private[this] def publishTarget(url: String) = {
     // Append build.timestamp to Artifactory URL to get consistent build numbers (see https://github.com/sbt/sbt/issues/2088):
     val url2 = if(url.startsWith("file:")) url else url.replaceAll("/$", "") + ";build.timestamp=" + System.currentTimeMillis
 
     Seq(
-      publishTo in Global := Some("scala-pr-publish" at url2),
-      credentials in Global ++= {
+      (Global / publishTo) := Some("scala-pr-publish" at url2),
+      (Global / credentials) ++= {
         val pass = env("PRIVATE_REPO_PASS")
         if (pass != "")
           List(Credentials("Artifactory Realm", "scala-ci.typesafe.com", "scala-ci", env("PRIVATE_REPO_PASS")))

project/ScriptCommands.scala

@@ -2,13 +2,13 @@ package scala.build
 
 import sbt._
 import Keys._
+
 import java.util.{Date, Locale, Properties, TimeZone}
-import java.io.{File, FileInputStream}
+import java.io.{File, FileInputStream, StringWriter}
 import java.text.SimpleDateFormat
 import java.time.Instant
 import java.time.format.DateTimeFormatter
 import java.time.temporal.{TemporalAccessor, TemporalQueries, TemporalQuery}
-
 import scala.collection.JavaConverters._
 import BuildSettings.autoImport._
 
@@ -24,9 +24,9 @@ object VersionUtil {
 
   lazy val globalVersionSettings = Seq[Setting[_]](
     // Set the version properties globally (they are the same for all projects)
-    versionProperties in Global := versionPropertiesImpl.value,
+    (Global / versionProperties) := versionPropertiesImpl.value,
     gitProperties := gitPropertiesImpl.value,
-    version in Global := versionProperties.value.mavenVersion
+    (Global / version) := versionProperties.value.mavenVersion
   )
 
   lazy val generatePropertiesFileSettings = Seq[Setting[_]](
@@ -37,12 +37,12 @@ object VersionUtil {
       |  __\ \/ /__/ __ |/ /__/ __ |
       | /____/\___/_/ |_/____/_/ | |
       |                          |/  %s""".stripMargin.linesIterator.drop(1).map(s => s"${ "%n" }${ s }").mkString,
-    resourceGenerators in Compile += generateVersionPropertiesFile.map(file => Seq(file)).taskValue,
+    (Compile / resourceGenerators) += generateVersionPropertiesFile.map(file => Seq(file)).taskValue,
     generateVersionPropertiesFile := generateVersionPropertiesFileImpl.value
   )
 
   lazy val generateBuildCharacterFileSettings = Seq[Setting[_]](
-    buildCharacterPropertiesFile := ((baseDirectory in ThisBuild).value / "buildcharacter.properties"),
+    buildCharacterPropertiesFile := ((ThisBuild / baseDirectory).value / "buildcharacter.properties"),
     generateBuildCharacterPropertiesFile := generateBuildCharacterPropertiesFileImpl.value
   )
 
@@ -161,7 +161,7 @@ object VersionUtil {
         "copyright.string" -> copyrightString.value,
         "shell.welcome"    -> shellWelcomeString.value
       ),
-      (resourceManaged in Compile).value / s"${thisProject.value.id}.properties")
+      (Compile / resourceManaged).value / s"${thisProject.value.id}.properties")
   }
 
   private lazy val generateBuildCharacterPropertiesFileImpl: Def.Initialize[Task[File]] = Def.task {
@@ -173,13 +173,18 @@ object VersionUtil {
   }
 
   private def writeProps(m: Map[String, String], propFile: File): File = {
-    val props = new Properties
-    m.foreach { case (k, v) => props.put(k, v) }
-    // unfortunately, this will write properties in arbitrary order
-    // this makes it harder to test for stability of generated artifacts
-    // consider using https://github.com/etiennestuder/java-ordered-properties
-    // instead of java.util.Properties
-    IO.write(props, null, propFile)
+    // Like:
+    // IO.write(props, null, propFile)
+    // But with deterministic key ordering and no timestamp
+    val fullWriter = new StringWriter()
+    for (k <- m.keySet.toVector.sorted) {
+      val writer = new StringWriter()
+      val props = new Properties()
+      props.put(k, m(k))
+      props.store(writer, null)
+      writer.toString.linesIterator.drop(1).foreach{line => fullWriter.write(line); fullWriter.write("\n")}
+    }
+    IO.write(propFile, fullWriter.toString)
     propFile
   }
 

project/VersionUtil.scala

@@ -1 +1 @@
-sbt.version=1.3.13
+sbt.version=1.5.4

project/build.properties

@@ -1,10 +1,10 @@
 scalacOptions ++= Seq("-unchecked", "-feature"/*, "-deprecation", "-Xlint" , "-Xfatal-warnings"*/)
 
-libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.3.2"
+libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.12.0"
 
-libraryDependencies += "org.pantsbuild" % "jarjar" % "1.6.5"
+libraryDependencies += "org.pantsbuild" % "jarjar" % "1.7.2"
 
-libraryDependencies += "biz.aQute.bnd" % "biz.aQute.bnd" % "2.4.1"
+libraryDependencies += "biz.aQute.bnd" % "biz.aQute.bndlib" % "5.3.0"
 
 enablePlugins(BuildInfoPlugin)
 
@@ -22,9 +22,9 @@ addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.8.1")
 
 libraryDependencies ++= Seq(
   "org.eclipse.jgit" % "org.eclipse.jgit" % "4.6.0.201612231935-r",
-  "org.slf4j" % "slf4j-nop" % "1.7.23",
+  "org.slf4j" % "slf4j-nop" % "1.7.31",
   "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0"
-)
+  )
 
 concurrentRestrictions in Global := Seq(
   Tags.limitAll(1) // workaround for https://github.com/sbt/sbt/issues/2970

project/plugins.sbt

@@ -15,7 +15,7 @@ rm -rf "$WORKSPACE/resolutionScratch_"
 mkdir -p "$WORKSPACE/resolutionScratch_"
 
 SBT_CMD=${SBT_CMD-sbt}
-SBT_CMD="$SBT_CMD -sbt-version 1.3.13"
+SBT_CMD="$SBT_CMD -sbt-version 1.5.4"
 
 # repo to publish builds
 integrationRepoUrl=${integrationRepoUrl-"https://scala-ci.typesafe.com/artifactory/scala-integration/"}