From 9d41c7b8530be0c8d355ef2eee324a7f1f9ca3c3 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 19 Jun 2024 12:21:16 +0200 Subject: [PATCH 1/3] replace pack command, do not produce lib directory, write classpath to file --- dist/bin/common | 63 ++--------- dist/bin/common.bat | 16 +-- dist/bin/scalac | 0 dist/bin/scalac.bat | 43 ++++---- dist/bin/scaladoc | 57 +--------- dist/bin/scaladoc.bat | 66 +++--------- project/Build.scala | 7 +- project/RepublishPlugin.scala | 193 ++++++++++++++++++++++++++++------ 8 files changed, 217 insertions(+), 228 deletions(-) mode change 100755 => 100644 dist/bin/common mode change 100644 => 100755 dist/bin/scalac diff --git a/dist/bin/common b/dist/bin/common old mode 100755 new mode 100644 index 4a0152fbc4cb..1ff0ca66274c --- a/dist/bin/common +++ b/dist/bin/common @@ -6,62 +6,21 @@ source "$PROG_HOME/bin/common-shared" # * The code below is for Dotty # *-------------------------------------------------*/ -find_lib () { - for lib in "$PROG_HOME"/lib/$1 ; do - if [[ -f "$lib" ]]; then - if [ -n "$CYGPATHCMD" ]; then - "$CYGPATHCMD" -am "$lib" - elif [[ $mingw || $msys ]]; then - echo "$lib" | sed 's|/|\\\\|g' - else - echo "$lib" - fi - return +load_classpath () { + command="$1" + psep_pattern="$2" + __CLASS_PATH="" + while IFS= read -r line; do + if ! [[ ( -n ${conemu-} || -n ${msys-}) && "$line" == "*jna-5*" ]]; then + # jna-5 only appropriate for some combinations + __CLASS_PATH+="$PROG_HOME/maven2/$line$psep_pattern" fi - done + done < "$PROG_HOME/etc/$command.classpath" + echo "$__CLASS_PATH" } -DOTTY_COMP=$(find_lib "*scala3-compiler*") -DOTTY_INTF=$(find_lib "*scala3-interfaces*") -DOTTY_LIB=$(find_lib "*scala3-library*") -DOTTY_STAGING=$(find_lib "*scala3-staging*") -DOTTY_TASTY_INSPECTOR=$(find_lib "*scala3-tasty-inspector*") -TASTY_CORE=$(find_lib "*tasty-core*") -SCALA_ASM=$(find_lib "*scala-asm*") -SCALA_LIB=$(find_lib "*scala-library*") -SBT_INTF=$(find_lib "*compiler-interface*") -JLINE_READER=$(find_lib "*jline-reader-3*") -JLINE_TERMINAL=$(find_lib "*jline-terminal-3*") -JLINE_TERMINAL_JNA=$(find_lib "*jline-terminal-jna-3*") - -# jna-5 only appropriate for some combinations -[[ ${conemu-} && ${msys-} ]] || JNA=$(find_lib "*jna-5*") - compilerJavaClasspathArgs () { - # echo "dotty-compiler: $DOTTY_COMP" - # echo "dotty-interface: $DOTTY_INTF" - # echo "dotty-library: $DOTTY_LIB" - # echo "tasty-core: $TASTY_CORE" - # echo "scala-asm: $SCALA_ASM" - # echo "scala-lib: $SCALA_LIB" - # echo "sbt-intface: $SBT_INTF" - - toolchain="" - toolchain+="$SCALA_LIB$PSEP" - toolchain+="$DOTTY_LIB$PSEP" - toolchain+="$SCALA_ASM$PSEP" - toolchain+="$SBT_INTF$PSEP" - toolchain+="$DOTTY_INTF$PSEP" - toolchain+="$DOTTY_COMP$PSEP" - toolchain+="$TASTY_CORE$PSEP" - toolchain+="$DOTTY_STAGING$PSEP" - toolchain+="$DOTTY_TASTY_INSPECTOR$PSEP" - - # jine - toolchain+="$JLINE_READER$PSEP" - toolchain+="$JLINE_TERMINAL$PSEP" - toolchain+="$JLINE_TERMINAL_JNA$PSEP" - [ -n "${JNA-}" ] && toolchain+="$JNA$PSEP" + toolchain="$(load_classpath "scala" "$PSEP")" if [ -n "${jvm_cp_args-}" ]; then jvm_cp_args="$toolchain$jvm_cp_args" diff --git a/dist/bin/common.bat b/dist/bin/common.bat index 7aef606d5509..f9c35e432b36 100644 --- a/dist/bin/common.bat +++ b/dist/bin/common.bat @@ -38,20 +38,6 @@ if not defined _PROG_HOME ( set _EXITCODE=1 goto :eof ) -set "_LIB_DIR=%_PROG_HOME%\lib" +set "_ETC_DIR=%_PROG_HOME%\etc" set _PSEP=; - -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-compiler*"') do set "_SCALA3_COMP=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-interfaces*"') do set "_SCALA3_INTF=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-library*"') do set "_SCALA3_LIB=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-staging*"') do set "_SCALA3_STAGING=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-tasty-inspector*"') do set "_SCALA3_TASTY_INSPECTOR=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*tasty-core*"') do set "_TASTY_CORE=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala-asm*"') do set "_SCALA_ASM=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala-library*"') do set "_SCALA_LIB=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*compiler-interface*"') do set "_SBT_INTF=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jline-reader-3*"') do set "_JLINE_READER=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jline-terminal-3*"') do set "_JLINE_TERMINAL=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jline-terminal-jna-3*"') do set "_JLINE_TERMINAL_JNA=%_LIB_DIR%\%%f" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jna-5*"') do set "_JNA=%_LIB_DIR%\%%f" diff --git a/dist/bin/scalac b/dist/bin/scalac old mode 100644 new mode 100755 diff --git a/dist/bin/scalac.bat b/dist/bin/scalac.bat index c8cd0babe60b..fe6d7e3fad4d 100644 --- a/dist/bin/scalac.bat +++ b/dist/bin/scalac.bat @@ -88,29 +88,10 @@ goto :eof @rem output parameter: _JVM_CP_ARGS :compilerJavaClasspathArgs -@rem echo scala3-compiler: %_SCALA3_COMP% -@rem echo scala3-interface: %_SCALA3_INTF% -@rem echo scala3-library: %_SCALA3_LIB% -@rem echo tasty-core: %_TASTY_CORE% -@rem echo scala-asm: %_SCALA_ASM% -@rem echo scala-lib: %_SCALA_LIB% -@rem echo sbt-intface: %_SBT_INTF% - -set "__TOOLCHAIN=%_SCALA_LIB%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_LIB%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA_ASM%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SBT_INTF%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_INTF%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_COMP%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_TASTY_CORE%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_STAGING%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_TASTY_INSPECTOR%%_PSEP%" - -@rem # jline -set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_READER%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_TERMINAL%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_TERMINAL_JNA%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_JNA%%_PSEP%" + +call :loadClasspathFromFile + +set "__TOOLCHAIN=%_CLASS_PATH%" if defined _SCALA_CPATH ( set "_JVM_CP_ARGS=%__TOOLCHAIN%%_SCALA_CPATH%" @@ -119,6 +100,22 @@ if defined _SCALA_CPATH ( ) goto :eof +@REM concatentate every line in "%_ETC_DIR%\scala.classpath" with _PSEP +:loadClasspathFromFile +set _CLASS_PATH= +if exist "%_ETC_DIR%\scala.classpath" ( + for /f "usebackq delims=" %%i in ("%_ETC_DIR%\scala.classpath") do ( + set "_LIB=%_PROG_HOME%\maven2\%%i" + set "_LIB=!_LIB:/=\!" + if not defined _CLASS_PATH ( + set "_CLASS_PATH=!_LIB!" + ) else ( + set "_CLASS_PATH=!_CLASS_PATH!%_PSEP%!_LIB!" + ) + ) +) +goto :eof + @rem ######################################################################### @rem ## Cleanups diff --git a/dist/bin/scaladoc b/dist/bin/scaladoc index 8b9ec41a7f8c..15bc0813f93a 100755 --- a/dist/bin/scaladoc +++ b/dist/bin/scaladoc @@ -53,62 +53,7 @@ addScrip() { } classpathArgs () { - CLASS_PATH="" - CLASS_PATH+="$(find_lib "*scaladoc*")$PSEP" - CLASS_PATH+="$(find_lib "*scala3-compiler*")$PSEP" - CLASS_PATH+="$(find_lib "*scala3-interfaces*")$PSEP" - CLASS_PATH+="$(find_lib "*scala3-library*")$PSEP" - CLASS_PATH+="$(find_lib "*tasty-core*")$PSEP" - CLASS_PATH+="$(find_lib "*scala3-tasty-inspector*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-0*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-anchorlink*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-autolink*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-emoji*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-gfm-strikethrough*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-gfm-tasklist*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-wikilink*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-yaml-front-matter*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-tables*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-ins*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ext-superscript*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-ast*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-data*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-dependency*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-misc*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-format*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-sequence*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-builder*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-collection*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-visitor*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-options*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-util-html*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-formatter*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-ast*")$PSEP" - CLASS_PATH+="$(find_lib "*liqp*")$PSEP" - CLASS_PATH+="$(find_lib "*jsoup*")$PSEP" - CLASS_PATH+="$(find_lib "*jackson-dataformat-yaml*")$PSEP" - CLASS_PATH+="$(find_lib "*jackson-datatype-jsr310*")$PSEP" - CLASS_PATH+="$(find_lib "*strftime4j*")$PSEP" - CLASS_PATH+="$(find_lib "*scala-asm*")$PSEP" - CLASS_PATH+="$(find_lib "*compiler-interface*")$PSEP" - CLASS_PATH+="$(find_lib "*jline-reader*")$PSEP" - CLASS_PATH+="$(find_lib "*jline-terminal-3*")$PSEP" - CLASS_PATH+="$(find_lib "*jline-terminal-jna*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-formatter*")$PSEP" - CLASS_PATH+="$(find_lib "*autolink-0.6*")$PSEP" - CLASS_PATH+="$(find_lib "*flexmark-jira-converter*")$PSEP" - CLASS_PATH+="$(find_lib "*antlr4*")$PSEP" - CLASS_PATH+="$(find_lib "*jackson-annotations*")$PSEP" - CLASS_PATH+="$(find_lib "*jackson-core*")$PSEP" - CLASS_PATH+="$(find_lib "*jackson-databind*")$PSEP" - CLASS_PATH+="$(find_lib "*snakeyaml*")$PSEP" - CLASS_PATH+="$(find_lib "*scala-library*")$PSEP" - CLASS_PATH+="$(find_lib "*protobuf-java*")$PSEP" - CLASS_PATH+="$(find_lib "*util-interface*")$PSEP" - CLASS_PATH+="$(find_lib "*jna-5*")$PSEP" - CLASS_PATH+="$(find_lib "*antlr4-runtime*")$PSEP" + CLASS_PATH="$(load_classpath "scaladoc" "$PSEP")" jvm_cp_args="-classpath \"$CLASS_PATH\"" } diff --git a/dist/bin/scaladoc.bat b/dist/bin/scaladoc.bat index c30a4689244c..16433a83f501 100644 --- a/dist/bin/scaladoc.bat +++ b/dist/bin/scaladoc.bat @@ -105,60 +105,24 @@ goto :eof @rem output parameter: _CLASS_PATH :classpathArgs -set "_LIB_DIR=%_PROG_HOME%\lib" -set _CLASS_PATH= +set "_ETC_DIR=%_PROG_HOME%\etc" @rem keep list in sync with bash script `bin\scaladoc` ! -call :updateClasspath "scaladoc" -call :updateClasspath "scala3-compiler" -call :updateClasspath "scala3-interfaces" -call :updateClasspath "scala3-library" -call :updateClasspath "tasty-core" -call :updateClasspath "scala3-tasty-inspector" -call :updateClasspath "flexmark-0" -call :updateClasspath "flexmark-html-parser" -call :updateClasspath "flexmark-ext-anchorlink" -call :updateClasspath "flexmark-ext-autolink" -call :updateClasspath "flexmark-ext-emoji" -call :updateClasspath "flexmark-ext-gfm-strikethrough" -call :updateClasspath "flexmark-ext-gfm-tables" -call :updateClasspath "flexmark-ext-gfm-tasklist" -call :updateClasspath "flexmark-ext-wikilink" -call :updateClasspath "flexmark-ext-yaml-front-matter" -call :updateClasspath "liqp" -call :updateClasspath "jsoup" -call :updateClasspath "jackson-dataformat-yaml" -call :updateClasspath "jackson-datatype-jsr310" -call :updateClasspath "strftime4j" -call :updateClasspath "scala-asm" -call :updateClasspath "compiler-interface" -call :updateClasspath "jline-reader" -call :updateClasspath "jline-terminal-3" -call :updateClasspath "jline-terminal-jna" -call :updateClasspath "flexmark-util" -call :updateClasspath "flexmark-formatter" -call :updateClasspath "autolink-0.6" -call :updateClasspath "flexmark-jira-converter" -call :updateClasspath "antlr4" -call :updateClasspath "jackson-annotations" -call :updateClasspath "jackson-core" -call :updateClasspath "jackson-databind" -call :updateClasspath "snakeyaml" -call :updateClasspath "scala-library" -call :updateClasspath "protobuf-java" -call :updateClasspath "util-interface" -call :updateClasspath "jna-5" -call :updateClasspath "flexmark-ext-tables" -call :updateClasspath "flexmark-ext-ins" -call :updateClasspath "flexmark-ext-superscript" -call :updateClasspath "antlr4-runtime" +call :loadClasspathFromFile goto :eof -@rem input parameter: %1=pattern for library file -@rem output parameter: _CLASS_PATH -:updateClasspath -set "__PATTERN=%~1" -for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*%__PATTERN%*" 2^>NUL') do ( - set "_CLASS_PATH=!_CLASS_PATH!%_LIB_DIR%\%%f%_PSEP%" +@REM concatentate every line in "%_ETC_DIR%\scaladoc.classpath" with _PSEP +:loadClasspathFromFile +set _CLASS_PATH= +if exist "%_ETC_DIR%\scaladoc.classpath" ( + for /f "usebackq delims=" %%i in ("%_ETC_DIR%\scaladoc.classpath") do ( + set "_LIB=%_PROG_HOME%\maven2\%%i" + set "_LIB=!_LIB:/=\!" + if not defined _CLASS_PATH ( + set "_CLASS_PATH=!_LIB!" + ) else ( + set "_CLASS_PATH=!_CLASS_PATH!%_PSEP%!_LIB!" + ) + ) ) goto :eof diff --git a/project/Build.scala b/project/Build.scala index c1a8800421a6..fa798e9c25ae 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2127,7 +2127,12 @@ object Build { republishRepo := target.value / "republish", packResourceDir += (republishRepo.value / "bin" -> "bin"), packResourceDir += (republishRepo.value / "maven2" -> "maven2"), - Compile / pack := (Compile / pack).dependsOn(republish).value, + packResourceDir += (republishRepo.value / "etc" -> "etc"), + republishCommandLibs += + ("scala" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core", "scala3-staging", "scala3-tasty-inspector")), + republishCommandLibs += + ("scaladoc" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core", "scala3-tasty-inspector", "scaladoc")), + Compile / pack := republishPack.value, ) lazy val dist = project.asDist(Bootstrapped) diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index 537c82d62cce..6ce83c2f0abf 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -2,6 +2,7 @@ package dotty.tools.sbtplugin import sbt._ import xerial.sbt.pack.PackPlugin +import xerial.sbt.pack.PackPlugin.autoImport.{packResourceDir, packDir} import sbt.Keys._ import sbt.AutoPlugin import sbt.PublishBinPlugin @@ -66,7 +67,9 @@ object RepublishPlugin extends AutoPlugin { val republishBinDir = settingKey[File]("where to find static files for the bin dir.") val republishCoursierDir = settingKey[File]("where to download the coursier launcher jar.") val republishBinOverrides = settingKey[Seq[File]]("files to override those in bin-dir.") + val republishCommandLibs = settingKey[Seq[(String, List[String])]]("libraries needed for each command.") val republish = taskKey[File]("cache the dependencies and download launchers for the distribution") + val republishPack = taskKey[File]("do the pack command") val republishRepo = settingKey[File]("the location to store the republished artifacts.") val republishLaunchers = settingKey[Seq[(String, String)]]("launchers to download. Sequence of (name, URL).") val republishCoursier = settingKey[Seq[(String, String)]]("coursier launcher to download. Sequence of (name, URL).") @@ -99,7 +102,7 @@ object RepublishPlugin extends AutoPlugin { }.toSet } - private def coursierCmd(jar: File, cache: File, args: Seq[String]): Unit = { + private def coursierCmd(jar: File, cache: File): Seq[String] => List[String] = { val jar0 = jar.getAbsolutePath.toString val javaHome = sys.props.get("java.home").getOrElse { throw new MessageOnlyException("java.home property not set") @@ -108,38 +111,88 @@ object RepublishPlugin extends AutoPlugin { val cmd = if (scala.util.Properties.isWin) "java.exe" else "java" (file(javaHome) / "bin" / cmd).getAbsolutePath } - val env = Map("COURSIER_CACHE" -> cache.getAbsolutePath.toString) - val cmdLine = Seq(javaCmd, "-jar", jar0) ++ args - // invoke cmdLine with env - val p = new ProcessBuilder(cmdLine: _*).inheritIO() - p.environment().putAll(env.asJava) - val proc = p.start() - proc.waitFor() - if (proc.exitValue() != 0) - throw new MessageOnlyException(s"Error running coursier.jar with args ${args.mkString(" ")}") + val env = Map("COURSIER_CACHE" -> cache.getAbsolutePath.toString).asJava + val cmdLine0 = Seq(javaCmd, "-jar", jar0) + args => + val cmdLine = cmdLine0 ++ args + // invoke cmdLine with env, but also capture the output + val p = new ProcessBuilder(cmdLine: _*) + .directory(cache) + .inheritIO() + .redirectOutput(ProcessBuilder.Redirect.PIPE) + p.environment().putAll(env) + + val proc = p.start() + val in = proc.getInputStream + val output = { + try { + val src = scala.io.Source.fromInputStream(in) + try src.getLines().toList + finally src.close() + } finally { + in.close() + } + } + + proc.waitFor() + + if (proc.exitValue() != 0) + throw new MessageOnlyException(s"Error running coursier.jar with args ${args.mkString(" ")}") + + output + } + + private def resolveMaven2(repo: File): Path = { + java.nio.file.Files.walk(repo.toPath) + .filter(_.getFileName.toString == "maven2") + .findFirst() + .orElseThrow(() => new MessageOnlyException(s"Could not find maven2 directory in $repo")) + .toAbsolutePath() } - private def coursierFetch(coursierJar: File, log: Logger, cacheDir: File, localRepo: File, libs: Seq[String]): Unit = { + private def coursierFetch( + coursierJar: File, log: Logger, cacheDir: File, localRepo: File, libs: Seq[String]): Map[String, List[String]] = { + val localRepoPath = localRepo.getAbsolutePath val localRepoArg = { - val path = localRepo.getAbsolutePath - if (scala.util.Properties.isWin) { - val path0 = path.replace('\\', '/') - s"file:///$path0" // extra root slash for Windows paths + val uriPart = { + if (scala.util.Properties.isWin) { + s"/${localRepoPath.replace('\\', '/')}" // extra root slash for Windows paths + } + else { + localRepoPath // no change needed for Unix paths + } } - else - s"file://$path" + s"file://$uriPart" } - IO.createDirectory(cacheDir) - for (lib <- libs) { + val cacheDirPath = cacheDir.getAbsolutePath + lazy val maven2RootLocal = resolveMaven2(localRepo) + lazy val maven2RootCache = resolveMaven2(cacheDir) // lazy because cache dir isn't populated until after fetch + val cmd = coursierCmd(coursierJar, cacheDir) + val resolved = for (lib <- libs) yield { log.info(s"[republish] Fetching $lib with coursier.jar...") - coursierCmd(coursierJar, cacheDir, + val out = cmd( Seq( "fetch", + "--no-default", + "--repository", "central", "--repository", localRepoArg, lib ) ) + lib -> out.collect { + case s if s.startsWith(localRepoPath) => + maven2RootLocal.relativize(java.nio.file.Paths.get(s)).toString().replace('\\', '/') // format as uri + case s if s.startsWith(cacheDirPath) => + maven2RootCache.relativize(java.nio.file.Paths.get(s)).toString().replace('\\', '/') // format as uri + } + } + resolved.toMap + } + + private def fuzzyFind[V](map: Map[String, V], key: String): V = { + map.collectFirst({ case (k, v) if k.contains(key) => v }).getOrElse { + throw new MessageOnlyException(s"Could not find key $key in map $map") } } @@ -148,28 +201,34 @@ object RepublishPlugin extends AutoPlugin { private def resolveLibraryDeps( coursierJar: File, log: Logger, + republishDir: File, csrCacheDir: File, localRepo: File, - resolvedLocal: Seq[ResolvedArtifacts]): Seq[ResolvedArtifacts] = { + resolvedLocal: Seq[ResolvedArtifacts], + commandLibs: Seq[(String, List[String])]): Seq[ResolvedArtifacts] = { // publish the local artifacts to the local repo, so coursier can resolve them republishResolvedArtifacts(resolvedLocal, localRepo, logOpt = None) - coursierFetch(coursierJar, log, csrCacheDir, localRepo, resolvedLocal.map(_.id.toString)) + val classpaths = coursierFetch(coursierJar, log, csrCacheDir, localRepo, resolvedLocal.map(_.id.toString)) - val maven2Root = java.nio.file.Files.walk(csrCacheDir.toPath) - .filter(_.getFileName.toString == "maven2") - .findFirst() - .orElseThrow(() => new MessageOnlyException(s"Could not find maven2 directory in $csrCacheDir")) + if (commandLibs.nonEmpty) { + IO.createDirectory(republishDir / "etc") + for ((command, libs) <- commandLibs) { + val entries = libs.map(fuzzyFind(classpaths, _)).reduce(_ ++ _).distinct + IO.write(republishDir / "etc" / s"$command.classpath", entries.mkString("\n")) + } + } + + val maven2Root = resolveMaven2(csrCacheDir) def pathToArtifact(p: Path): ResolvedArtifacts = { // relative path from maven2Root - val lastAsString = p.getFileName.toString val relP = maven2Root.relativize(p) val parts = relP.iterator().asScala.map(_.toString).toVector - val (orgParts :+ name :+ rev :+ _) = parts + val (orgParts :+ name :+ rev :+ artifact) = parts val id = SimpleModuleId(orgParts.mkString("."), name, rev) - if (lastAsString.endsWith(".jar")) { + if (artifact.endsWith(".jar")) { ResolvedArtifacts(id, Some(p.toFile), None) } else { ResolvedArtifacts(id, None, Some(p.toFile)) @@ -279,6 +338,7 @@ object RepublishPlugin extends AutoPlugin { republishCoursier := Seq.empty, republishBinOverrides := Seq.empty, republishExtraProps := Seq.empty, + republishCommandLibs := Seq.empty, republishLocalResolved / republishProjectRefs := { val proj = thisProjectRef.value val deps = buildDependencies.value @@ -326,13 +386,15 @@ object RepublishPlugin extends AutoPlugin { val s = streams.value val lm = (republishAllResolved / dependencyResolution).value val cacheDir = republishRepo.value + val commandLibs = republishCommandLibs.value val log = s.log val csrCacheDir = s.cacheDirectory / "csr-cache" val localRepo = s.cacheDirectory / "localRepo" / "maven2" // resolve the transitive dependencies of the local artifacts - val resolvedLibs = resolveLibraryDeps(coursierJar, log, csrCacheDir, localRepo, resolvedLocal) + val resolvedLibs = resolveLibraryDeps( + coursierJar, log, cacheDir, csrCacheDir, localRepo, resolvedLocal, commandLibs) // the combination of local artifacts and resolved transitive dependencies val merged = @@ -395,6 +457,77 @@ object RepublishPlugin extends AutoPlugin { val launchers = republishFetchLaunchers.value val extraProps = republishWriteExtraProps.value cacheDir + }, + republishPack := { + val cacheDir = republish.value + val s = streams.value + val log = s.log + val distDir = target.value / packDir.value + val progVersion = version.value + + IO.createDirectory(distDir) + for ((path, dir) <- packResourceDir.value) { + val target = distDir / dir + IO.copyDirectory(path, target) + } + + locally { + // everything in this block is copied from sbt-pack plugin + import scala.util.Try + import java.time.format.DateTimeFormatterBuilder + import java.time.format.SignStyle + import java.time.temporal.ChronoField.* + import java.time.ZoneId + import java.time.Instant + import java.time.ZonedDateTime + import java.time.ZonedDateTime + import java.util.Locale + import java.util.Date + val base: File = new File(".") // Using the working directory as base for readability + + def write(path: String, content: String) { + val p = distDir / path + IO.write(p, content) + } + + val humanReadableTimestampFormatter = new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD) + .appendLiteral('-') + .appendValue(MONTH_OF_YEAR, 2) + .appendLiteral('-') + .appendValue(DAY_OF_MONTH, 2) + .appendLiteral(' ') + .appendValue(HOUR_OF_DAY, 2) + .appendLiteral(':') + .appendValue(MINUTE_OF_HOUR, 2) + .appendLiteral(':') + .appendValue(SECOND_OF_MINUTE, 2) + .appendOffset("+HHMM", "Z") + .toFormatter(Locale.US) + + // Retrieve build time + val systemZone = ZoneId.systemDefault().normalized() + val timestamp = ZonedDateTime.ofInstant(Instant.ofEpochMilli(new Date().getTime), systemZone) + val buildTime = humanReadableTimestampFormatter.format(timestamp) + + // Check the current Git revision + val gitRevision: String = Try { + if ((base / ".git").exists()) { + log.info("[republish] Checking the git revision of the current project") + sys.process.Process("git rev-parse HEAD").!! + } else { + "unknown" + } + }.getOrElse("unknown").trim + + + // Output the version number and Git revision + write("VERSION", s"version:=${progVersion}\nrevision:=${gitRevision}\nbuildTime:=${buildTime}\n") + } + + + distDir } ) } From c004c74aae23f224617667717bafe50c0f5633b0 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Fri, 21 Jun 2024 14:13:57 +0200 Subject: [PATCH 2/3] add back in copy of mapped sequence --- dist/bin-native-overrides/cli-common-platform.bat | 6 +++++- project/Build.scala | 2 +- project/RepublishPlugin.scala | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dist/bin-native-overrides/cli-common-platform.bat b/dist/bin-native-overrides/cli-common-platform.bat index e0cfa40692b5..d1c4f1c4716b 100644 --- a/dist/bin-native-overrides/cli-common-platform.bat +++ b/dist/bin-native-overrides/cli-common-platform.bat @@ -12,7 +12,11 @@ FOR /F "usebackq delims=" %%G IN ("%_PROG_HOME%\EXTRA_PROPERTIES") DO ( ) ) +@REM we didn't find it, so we should fail +echo "ERROR: cli_version not found in EXTRA_PROPERTIES file" +exit /b 1 + :foundCliVersion endlocal & set "SCALA_CLI_VERSION=%_SCALA_CLI_VERSION%" -set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" "--cli-version" "%SCALA_CLI_VERSION%" \ No newline at end of file +set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" "--cli-version" "%SCALA_CLI_VERSION%" diff --git a/project/Build.scala b/project/Build.scala index fa798e9c25ae..cdabcd3471f1 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2172,7 +2172,7 @@ object Build { republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", republishFetchCoursier := (dist / republishFetchCoursier).value, republishExtraProps += ("cli_version" -> scalaCliLauncherVersion), - mappings += (republishRepo.value / "etc" / "EXTRA_PROPERTIES" -> "EXTRA_PROPERTIES"), + mappings += (republishRepo.value / "EXTRA_PROPERTIES" -> "EXTRA_PROPERTIES"), republishLaunchers += ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersionWindows/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") ) diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index 6ce83c2f0abf..a0a8ce7dae74 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -114,7 +114,7 @@ object RepublishPlugin extends AutoPlugin { val env = Map("COURSIER_CACHE" -> cache.getAbsolutePath.toString).asJava val cmdLine0 = Seq(javaCmd, "-jar", jar0) args => - val cmdLine = cmdLine0 ++ args + val cmdLine = cmdLine0 ++ args // invoke cmdLine with env, but also capture the output val p = new ProcessBuilder(cmdLine: _*) .directory(cache) @@ -441,7 +441,7 @@ object RepublishPlugin extends AutoPlugin { } else { val repoDir = republishRepo.value - val propsFile = repoDir / "etc" / "EXTRA_PROPERTIES" + val propsFile = repoDir / "EXTRA_PROPERTIES" log.info(s"[republish] Writing extra properties to $propsFile...") Using.fileWriter()(propsFile) { writer => extraProps.foreach { case (k, v) => @@ -485,6 +485,16 @@ object RepublishPlugin extends AutoPlugin { import java.util.Date val base: File = new File(".") // Using the working directory as base for readability + // Copy explicitly added dependencies + val mapped: Seq[(File, String)] = mappings.value + log.info("[republish] Copying explicit dependencies:") + val explicitDepsJars = for ((file, path) <- mapped) yield { + log.info(file.getPath) + val dest = distDir / path + IO.copyFile(file, dest, true) + dest + } + def write(path: String, content: String) { val p = distDir / path IO.write(p, content) From bf670324891a4db0c2282ac893dfba9b8578b72b Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Fri, 21 Jun 2024 14:58:42 +0200 Subject: [PATCH 3/3] read last line, split-off with-compiler classpath --- dist/bin/common | 16 ++++++++++++---- dist/bin/scalac.bat | 27 ++++++++++++++++++--------- project/Build.scala | 4 +++- project/RepublishPlugin.scala | 8 +++++++- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/dist/bin/common b/dist/bin/common index 1ff0ca66274c..63e598d70d7e 100644 --- a/dist/bin/common +++ b/dist/bin/common @@ -10,10 +10,13 @@ load_classpath () { command="$1" psep_pattern="$2" __CLASS_PATH="" - while IFS= read -r line; do + while IFS= read -r line || [ -n "$line" ]; do + # jna-5 only appropriate for some combinations if ! [[ ( -n ${conemu-} || -n ${msys-}) && "$line" == "*jna-5*" ]]; then - # jna-5 only appropriate for some combinations - __CLASS_PATH+="$PROG_HOME/maven2/$line$psep_pattern" + if [ -n "$__CLASS_PATH" ]; then + __CLASS_PATH+="$psep_pattern" + fi + __CLASS_PATH+="$PROG_HOME/maven2/$line" fi done < "$PROG_HOME/etc/$command.classpath" echo "$__CLASS_PATH" @@ -21,11 +24,16 @@ load_classpath () { compilerJavaClasspathArgs () { toolchain="$(load_classpath "scala" "$PSEP")" + toolchain_extra="$(load_classpath "with_compiler" "$PSEP")" + + if [ -n "$toolchain_extra" ]; then + toolchain+="$PSEP$toolchain_extra" + fi if [ -n "${jvm_cp_args-}" ]; then jvm_cp_args="$toolchain$jvm_cp_args" else - jvm_cp_args="$toolchain$PSEP" + jvm_cp_args="$toolchain" fi } diff --git a/dist/bin/scalac.bat b/dist/bin/scalac.bat index fe6d7e3fad4d..dbcbaf11b8e2 100644 --- a/dist/bin/scalac.bat +++ b/dist/bin/scalac.bat @@ -89,9 +89,16 @@ goto :eof @rem output parameter: _JVM_CP_ARGS :compilerJavaClasspathArgs -call :loadClasspathFromFile +set "CP_FILE=%_ETC_DIR%\scala.classpath" +call :loadClasspathFromFile %CP_FILE% +set "__TOOLCHAIN=%_CLASS_PATH_RESULT%" -set "__TOOLCHAIN=%_CLASS_PATH%" +set "CP_FILE=%_ETC_DIR%\with_compiler.classpath" +call :loadClasspathFromFile %CP_FILE% + +if defined _CLASS_PATH_RESULT ( + set "__TOOLCHAIN=%__TOOLCHAIN%%_PSEP%%_CLASS_PATH_RESULT%" +) if defined _SCALA_CPATH ( set "_JVM_CP_ARGS=%__TOOLCHAIN%%_SCALA_CPATH%" @@ -100,17 +107,19 @@ if defined _SCALA_CPATH ( ) goto :eof -@REM concatentate every line in "%_ETC_DIR%\scala.classpath" with _PSEP +@REM concatentate every line in "%_ARG_FILE%" with _PSEP +@REM arg 1 - file to read :loadClasspathFromFile -set _CLASS_PATH= -if exist "%_ETC_DIR%\scala.classpath" ( - for /f "usebackq delims=" %%i in ("%_ETC_DIR%\scala.classpath") do ( +set _ARG_FILE=%1 +set _CLASS_PATH_RESULT= +if exist "%_ARG_FILE%" ( + for /f "usebackq delims=" %%i in ("%_ARG_FILE%") do ( set "_LIB=%_PROG_HOME%\maven2\%%i" set "_LIB=!_LIB:/=\!" - if not defined _CLASS_PATH ( - set "_CLASS_PATH=!_LIB!" + if not defined _CLASS_PATH_RESULT ( + set "_CLASS_PATH_RESULT=!_LIB!" ) else ( - set "_CLASS_PATH=!_CLASS_PATH!%_PSEP%!_LIB!" + set "_CLASS_PATH_RESULT=!_CLASS_PATH_RESULT!%_PSEP%!_LIB!" ) ) ) diff --git a/project/Build.scala b/project/Build.scala index cdabcd3471f1..bd5c06a512e8 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2129,7 +2129,9 @@ object Build { packResourceDir += (republishRepo.value / "maven2" -> "maven2"), packResourceDir += (republishRepo.value / "etc" -> "etc"), republishCommandLibs += - ("scala" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core", "scala3-staging", "scala3-tasty-inspector")), + ("scala" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core")), + republishCommandLibs += + ("with_compiler" -> List("scala3-staging", "scala3-tasty-inspector", "^!scala3-interfaces", "^!scala3-compiler", "^!scala3-library", "^!tasty-core")), republishCommandLibs += ("scaladoc" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core", "scala3-tasty-inspector", "scaladoc")), Compile / pack := republishPack.value, diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index a0a8ce7dae74..e4bf40545a6b 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -215,7 +215,13 @@ object RepublishPlugin extends AutoPlugin { if (commandLibs.nonEmpty) { IO.createDirectory(republishDir / "etc") for ((command, libs) <- commandLibs) { - val entries = libs.map(fuzzyFind(classpaths, _)).reduce(_ ++ _).distinct + val (negated, actual) = libs.partition(_.startsWith("^!")) + val subtractions = negated.map(_.stripPrefix("^!")) + + def compose(libs: List[String]): List[String] = + libs.map(fuzzyFind(classpaths, _)).reduceOption(_ ++ _).map(_.distinct).getOrElse(Nil) + + val entries = compose(actual).diff(compose(subtractions)) IO.write(republishDir / "etc" / s"$command.classpath", entries.mkString("\n")) } }