Skip to content

Commit

Permalink
Virtualize compile task
Browse files Browse the repository at this point in the history
  • Loading branch information
@eed3si9n
eed3si9n committed Dec 11, 2023
1 parent be5c7a7 commit 2023d3e
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 56 deletions.
167 changes: 112 additions & 55 deletions main/src/main/scala/sbt/Defaults.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ object Defaults extends BuildCommon {
Seq(
internalConfigurationMap :== Configurations.internalMap _,
credentials :== SysProp.sbtCredentialsEnv.toList,
exportJars :== false,
exportJars :== true,
trackInternalDependencies :== TrackLevel.TrackAlways,
exportToInternal :== TrackLevel.TrackAlways,
useCoursier :== SysProp.defaultUseCoursier,
Expand Down Expand Up @@ -946,7 +946,6 @@ object Defaults extends BuildCommon {
},
internalDependencyConfigurations := InternalDependencies.configurations.value,
manipulateBytecode := compileSplit.value,
compileIncremental := compileIncrementalTask.tag(Tags.Compile, Tags.CPU).value,
printWarnings := printWarningsTask.value,
compileAnalysisFilename := {
// Here, if the user wants cross-scala-versioning, we also append it
Expand Down Expand Up @@ -1041,7 +1040,9 @@ object Defaults extends BuildCommon {
// note that we use the same runner and mainClass as plain run
mainBgRunMainTaskForConfig(This),
mainBgRunTaskForConfig(This)
) ++ inTask(run)(runnerSettings ++ newRunnerSettings)
) ++ inTask(run)(runnerSettings ++ newRunnerSettings) ++ compileIncrementalTaskSettings(
compileIncremental
)

private[this] lazy val configGlobal = globalDefaults(
Seq(
Expand Down Expand Up @@ -1937,17 +1938,6 @@ object Defaults extends BuildCommon {
}
}

@deprecated("The configuration(s) should not be decided based on the classifier.", "1.0.0")
def artifactConfigurations(
base: Artifact,
scope: Configuration,
classifier: Option[String]
): Iterable[Configuration] =
classifier match {
case Some(c) => Artifact.classifierConf(c) :: Nil
case None => scope :: Nil
}

def packageTaskSettings(
key: TaskKey[HashedVirtualFileRef],
mappingsTask: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]]
Expand Down Expand Up @@ -2370,20 +2360,22 @@ object Defaults extends BuildCommon {
private[sbt] def compileScalaBackendTask: Initialize[Task[CompileResult]] = Def.task {
val setup: Setup = compileIncSetup.value
val useBinary: Boolean = enableBinaryCompileAnalysis.value
val analysisResult: CompileResult = compileIncremental.value
val _ = compileIncremental.value
val exportP = exportPipelining.value
// Save analysis midway if pipelining is enabled
if (analysisResult.hasModified && exportP) {
val store =
MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary)
val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
store.set(contents)
val store = MixedAnalyzingCompiler.staticCachedStore(setup.cachePath, !useBinary)
val contents = store.unsafeGet()
if (exportP) {
// this stores the eary analysis (again) in case the subproject contains a macro
setup.earlyAnalysisStore.toOption map { earlyStore =>
earlyStore.set(contents)
}
}
analysisResult
CompileResult.of(
contents.getAnalysis(),
contents.getMiniSetup(),
contents.getAnalysis().readCompilations().getAllCompilations().nonEmpty
)
}

/**
Expand All @@ -2407,6 +2399,7 @@ object Defaults extends BuildCommon {
compile.value
}
}

def compileTask: Initialize[Task[CompileAnalysis]] = Def.task {
val setup: Setup = compileIncSetup.value
val useBinary: Boolean = enableBinaryCompileAnalysis.value
Expand All @@ -2427,17 +2420,73 @@ object Defaults extends BuildCommon {
}
analysis
}
def compileIncrementalTask = Def.task {
val s = streams.value
val ci = (compile / compileInputs).value
val ping = earlyOutputPing.value
val reporter = (compile / bspReporter).value
BspCompileTask.compute(bspTargetIdentifier.value, thisProjectRef.value, configuration.value) {
task =>
// TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too?
compileIncrementalTaskImpl(task, s, ci, ping, reporter)
}
}

def compileIncrementalTaskSettings(key: TaskKey[(Boolean, HashedVirtualFileRef)]) =
inTask(key)(
Seq(
(TaskZero / key) := (Def
.cachedTask {
val s = streams.value
val ci = (compile / compileInputs).value
// This is a cacheable version
val ci2 = (compile / compileInputs2).value
val ping = (TaskZero / earlyOutputPing).value
val reporter = (compile / bspReporter).value
val setup: Setup = (TaskZero / compileIncSetup).value
val useBinary: Boolean = enableBinaryCompileAnalysis.value
val c = fileConverter.value
val analysisResult: CompileResult =
BspCompileTask
.compute(bspTargetIdentifier.value, thisProjectRef.value, configuration.value) {
bspTask =>
// TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too?
compileIncrementalTaskImpl(bspTask, s, ci, ping, reporter)
}
val analysisOut = c.toVirtualFile(setup.cachePath())
val store =
MixedAnalyzingCompiler.staticCachedStore(setup.cachePath, !useBinary)
val contents =
AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
store.set(contents)
Def.declareOutput(analysisOut)
val dir = classDirectory.value
if (dir / "META-INF").exists then IO.delete(dir)
// inline mappings
val mappings = Path
.allSubpaths(dir)
.filter(_._1.isFile())
.map { case (p, path) =>
val vf = c.toVirtualFile(p.toPath())
(vf: HashedVirtualFileRef) -> path
}
.toSeq
// inlined to avoid caching mappings
val pkgConfig = Pkg.Configuration(
mappings,
artifactPath.value,
packageOptions.value,
)
val out = Pkg(
pkgConfig,
c,
s.log,
Pkg.timeFromConfiguration(pkgConfig)
)
Def.declareOutput(out)
analysisResult.hasModified() -> (out: HashedVirtualFileRef)
})
.tag(Tags.Compile, Tags.CPU)
.value,
packagedArtifact := {
val (hasModified, out) = key.value
artifact.value -> out
},
artifact := artifactSetting.value,
artifactClassifier := Some("noresources"),
artifactPath := artifactPathSetting(artifact).value,
)
)

private val incCompiler = ZincUtil.defaultIncrementalCompiler
private[sbt] def compileJavaTask: Initialize[Task[CompileResult]] = Def.task {
val s = streams.value
Expand All @@ -2459,6 +2508,7 @@ object Defaults extends BuildCommon {
throw e
}
}

private[this] def compileIncrementalTaskImpl(
task: BspCompileTask,
s: TaskStreams,
Expand All @@ -2467,43 +2517,35 @@ object Defaults extends BuildCommon {
reporter: BuildServerReporter,
): CompileResult = {
lazy val x = s.text(ExportStream)
def onArgs(cs: Compilers) = {
def onArgs(cs: Compilers) =
cs.withScalac(
cs.scalac match {
cs.scalac match
case ac: AnalyzingCompiler => ac.onArgs(exported(x, "scalac"))
case x => x
}
)
}
def onProgress(s: Setup) = {
def onProgress(s: Setup) =
val cp = new BspCompileProgress(task, s.progress.asScala)
s.withProgress(cp)
}
val compilers: Compilers = ci.compilers
val setup: Setup = ci.setup
val i = ci
.withCompilers(onArgs(compilers))
.withSetup(onProgress(setup))
try {
val i = ci.withCompilers(onArgs(compilers)).withSetup(onProgress(setup))
try
val result = incCompiler.compile(i, s.log)
reporter.sendSuccessReport(result.getAnalysis)
result
} catch {
catch
case e: Throwable =>
if (!promise.isCompleted) {
if !promise.isCompleted then
promise.failure(e)
ConcurrentRestrictions.cancelAllSentinels()
}
reporter.sendFailureReport(ci.options.sources)

throw e
} finally {
x.close() // workaround for #937
}
finally x.close() // workaround for #937
}

def compileIncSetupTask = Def.task {
val cp = dependencyPicklePath.value
val lookup = new PerClasspathEntryLookup {
val lookup = new PerClasspathEntryLookup:
private val cachedAnalysisMap: VirtualFile => Option[CompileAnalysis] =
analysisMap(cp)
private val cachedPerEntryDefinesClassLookup: VirtualFile => DefinesClass =
Expand All @@ -2512,12 +2554,12 @@ object Defaults extends BuildCommon {
cachedAnalysisMap(classpathEntry).toOptional
override def definesClass(classpathEntry: VirtualFile): DefinesClass =
cachedPerEntryDefinesClassLookup(classpathEntry)
}
val extra = extraIncOptions.value.map(t2)
val useBinary: Boolean = enableBinaryCompileAnalysis.value
val eapath = earlyCompileAnalysisFile.value.toPath
val eaOpt =
if (exportPipelining.value) Some(MixedAnalyzingCompiler.staticCachedStore(eapath, !useBinary))
if exportPipelining.value then
Some(MixedAnalyzingCompiler.staticCachedStore(eapath, !useBinary))
else None
Setup.of(
lookup,
Expand All @@ -2531,6 +2573,7 @@ object Defaults extends BuildCommon {
extra.toArray,
)
}

def compileInputsSettings: Seq[Setting[_]] =
compileInputsSettings(dependencyPicklePath)
def compileInputsSettings(classpathTask: TaskKey[VirtualClasspath]): Seq[Setting[_]] = {
Expand Down Expand Up @@ -2580,7 +2623,17 @@ object Defaults extends BuildCommon {
setup,
prev
)
}
},
// todo: Zinc's hashing should automatically handle directories
compileInputs2 := {
val c = fileConverter.value
val cp0 = classpathTask.value
val inputs = compileInputs.value
CompileInputs2(
data(cp0).toVector,
inputs.options.sources,
)
},
)
}

Expand Down Expand Up @@ -4147,8 +4200,12 @@ object Classpaths {
val c = fileConverter.value
Def.unit(copyResources.value)
Def.unit(compile.value)

c.toPath(backendOutput.value).toFile :: Nil
val dir = c.toPath(backendOutput.value)
val rawJar = compileIncremental.value._2
val rawJarPath = c.toPath(rawJar)
IO.unzip(rawJarPath.toFile, dir.toFile)
IO.delete(dir.toFile / "META-INF")
dir.toFile :: Nil
}

private[sbt] def makePickleProducts: Initialize[Task[Seq[VirtualFile]]] = Def.task {
Expand Down
18 changes: 17 additions & 1 deletion main/src/main/scala/sbt/Keys.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,12 @@ object Keys {
val buildDependencies = settingKey[BuildDependencies]("Definitive source of inter-project dependencies for compilation and dependency management.\n\tThis is populated by default by the dependencies declared on Project instances, but may be modified.\n\tThe main restriction is that new builds may not be introduced.").withRank(DSetting)
val appConfiguration = settingKey[xsbti.AppConfiguration]("Provides access to the launched sbt configuration, including the ScalaProvider, Launcher, and GlobalLock.").withRank(DSetting)
val thisProject = settingKey[ResolvedProject]("Provides the current project for the referencing scope.").withRank(CSetting)

@cacheOptOut(reason = "contains file path")
val thisProjectRef = settingKey[ProjectRef]("Provides a fully-resolved reference to the current project for the referencing scope.").withRank(CSetting)
val configurationStr = StringAttributeKey("configuration")

@cacheOptOut("")
val configuration = settingKey[Configuration]("Provides the current configuration of the referencing scope.").withRank(CSetting)
val commands = settingKey[Seq[Command]]("Defines commands to be registered when this project or build is the current selected one.").withRank(CSetting)
val initialize = settingKey[Unit]("A convenience setting for performing side-effects during initialization.").withRank(BSetting)
Expand Down Expand Up @@ -166,6 +170,7 @@ object Keys {
val resources = taskKey[Seq[File]]("All resource files, both managed and unmanaged.").withRank(BTask)

// Output paths
@cacheOptOut(reason = "File is machine-specific")
val classDirectory = settingKey[File]("Directory for compiled classes and copied resources.").withRank(AMinusSetting)
val earlyOutput = settingKey[VirtualFile]("JAR file for pickles used for build pipelining")
val backendOutput = settingKey[VirtualFile]("Directory or JAR file for compiled classes and copied resources")
Expand All @@ -191,7 +196,10 @@ object Keys {
val cleanupCommands = settingKey[String]("Commands to execute before the Scala interpreter exits.").withRank(BMinusSetting)
val asciiGraphWidth = settingKey[Int]("Determines maximum width of the settings graph in ASCII mode").withRank(AMinusSetting)
val compileOptions = taskKey[CompileOptions]("Collects basic options to configure compilers").withRank(DTask)

@cacheOptOut(reason = "contains uncachable settings")
val compileInputs = taskKey[Inputs]("Collects all inputs needed for compilation.").withRank(DTask)
val compileInputs2 = taskKey[CompileInputs2]("")
val scalaHome = settingKey[Option[File]]("If Some, defines the local Scala installation to use for compilation, running, and testing.").withRank(ASetting)
val scalaInstance = taskKey[ScalaInstance]("Defines the Scala instance to use for compilation, running, and testing.").withRank(DTask)
val scalaOrganization = settingKey[String]("Organization/group ID of the Scala used in the project. Default value is 'org.scala-lang'. This is an advanced setting used for clones of the Scala Language. It should be disregarded in standard use cases.").withRank(CSetting)
Expand Down Expand Up @@ -229,11 +237,13 @@ object Keys {
val consoleProject = taskKey[Unit]("Starts the Scala interpreter with the sbt and the build definition on the classpath and useful imports.").withRank(AMinusTask)
val compile = taskKey[CompileAnalysis]("Compiles sources.").withRank(APlusTask)
val manipulateBytecode = taskKey[CompileResult]("Manipulates generated bytecode").withRank(BTask)
val compileIncremental = taskKey[CompileResult]("Actually runs the incremental compilation").withRank(DTask)
val compileIncremental = taskKey[(Boolean, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask)
val previousCompile = taskKey[PreviousResult]("Read the incremental compiler analysis from disk").withRank(DTask)
val tastyFiles = taskKey[Seq[File]]("Returns the TASTy files produced by compilation").withRank(DTask)
private[sbt] val compileScalaBackend = taskKey[CompileResult]("Compiles only Scala sources if pipelining is enabled. Compiles both Scala and Java sources otherwise").withRank(Invisible)
private[sbt] val compileEarly = taskKey[CompileAnalysis]("Compiles only Scala sources if pipelining is enabled, and produce an early output (pickle JAR)").withRank(Invisible)

@cacheOptOut("this is just for timing signal, so no need to cache")
private[sbt] val earlyOutputPing = taskKey[PromiseWrap[Boolean]]("When pipelining is enabled, this returns true when early output (pickle JAR) is created; false otherwise").withRank(Invisible)
private[sbt] val compileJava = taskKey[CompileResult]("Compiles only Java sources (called only for pipelining)").withRank(Invisible)
private[sbt] val compileSplit = taskKey[CompileResult]("When pipelining is enabled, compile Scala then Java; otherwise compile both").withRank(Invisible)
Expand All @@ -245,6 +255,8 @@ object Keys {
val earlyCompileAnalysisTargetRoot = settingKey[File]("The output directory to produce Zinc Analysis files").withRank(DSetting)
val compileAnalysisFile = taskKey[File]("Zinc analysis storage.").withRank(DSetting)
val earlyCompileAnalysisFile = taskKey[File]("Zinc analysis storage for early compilation").withRank(DSetting)

@cacheOptOut
val compileIncSetup = taskKey[Setup]("Configures aspects of incremental compilation.").withRank(DTask)
val compilerCache = taskKey[GlobalsCache]("Cache of scala.tools.nsc.Global instances. This should typically be cached so that it isn't recreated every task run.").withRank(DTask)
val stateCompilerCache = AttributeKey[GlobalsCache]("stateCompilerCache", "Internal use: Global cache.")
Expand Down Expand Up @@ -410,6 +422,8 @@ object Keys {
val bspConfig = taskKey[Unit]("Create or update the BSP connection files").withRank(DSetting)
val bspEnabled = SettingKey[Boolean](BasicKeys.bspEnabled)
val bspSbtEnabled = settingKey[Boolean]("Should BSP export meta-targets for the SBT build itself?")

@cacheOptOut(reason = "target identifier includes file path")
val bspTargetIdentifier = settingKey[BuildTargetIdentifier]("Build target identifier of a project and configuration.").withRank(DSetting)
val bspWorkspace = settingKey[Map[BuildTargetIdentifier, Scope]]("Mapping of BSP build targets to sbt scopes").withRank(DSetting)
private[sbt] val bspFullWorkspace = settingKey[BspFullWorkspace]("Mapping of BSP build targets to sbt scopes and meta-targets for the SBT build itself").withRank(DSetting)
Expand Down Expand Up @@ -440,6 +454,8 @@ object Keys {
val bspScalaTestClassesItem = taskKey[Seq[ScalaTestClassesItem]]("").withRank(DTask)
val bspScalaMainClasses = inputKey[Unit]("Corresponds to buildTarget/scalaMainClasses request").withRank(DTask)
val bspScalaMainClassesItem = taskKey[ScalaMainClassesItem]("").withRank(DTask)

@cacheOptOut(reason = "no need to invalidate based on this")
val bspReporter = taskKey[BuildServerReporter]("").withRank(DTask)

val useCoursier = settingKey[Boolean]("Use Coursier for dependency resolution.").withRank(BSetting)
Expand Down
29 changes: 29 additions & 0 deletions main/src/main/scala/sbt/internal/CompileInputs2.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package sbt.internal

import scala.reflect.ClassTag
import sjsonnew.*
import xsbti.HashedVirtualFileRef

// CompileOption has the list of sources etc
case class CompileInputs2(
classpath: Seq[HashedVirtualFileRef],
sources: Seq[HashedVirtualFileRef],
)

object CompileInputs2:
import sbt.util.CacheImplicits.given

given IsoLList.Aux[
CompileInputs2,
Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: LNil
] =
LList.iso(
{ (v: CompileInputs2) =>
("classpath", v.classpath.toVector) :*: ("sources", v.sources.toVector) :*: LNil
},
{ (in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: LNil) =>
CompileInputs2(in.head, in.tail.head)
}
)
given JsonFormat[CompileInputs2] = summon
end CompileInputs2

0 comments on commit 2023d3e

Please sign in to comment.