From de82414dbcae3864dfc4d53f96a4e1ef56f895a5 Mon Sep 17 00:00:00 2001 From: Piotr Chabelski Date: Mon, 30 Sep 2024 12:27:33 +0200 Subject: [PATCH 1/2] Add a global offline config key --- .../scala/cli/commands/bloop/BloopExit.scala | 2 +- .../cli/commands/bloop/BloopOutput.scala | 2 +- .../scala/cli/commands/bloop/BloopStart.scala | 6 +-- .../scala/cli/commands/config/Config.scala | 2 +- .../scala/cli/commands/default/Default.scala | 2 +- .../cli/commands/github/SecretCreate.scala | 2 +- .../scala/scala/cli/commands/new/New.scala | 2 +- .../cli/commands/pgp/PgpExternalCommand.scala | 2 +- .../scala/cli/commands/pgp/PgpPush.scala | 2 +- .../cli/commands/publish/PublishSetup.scala | 2 +- .../cli/commands/shared/CoursierOptions.scala | 16 +++++-- .../cli/commands/shared/SharedOptions.scala | 6 +-- .../scala/cli/launcher/LauncherCli.scala | 2 +- .../scala/cli/tests/LauncherCliTest.scala | 2 +- .../main/scala/scala/cli/config/Keys.scala | 8 ++++ .../scala/cli/integration/ConfigTests.scala | 43 +++++++++++++++++++ website/docs/guides/power/offline.md | 5 +++ website/docs/reference/commands.md | 1 + .../docs/reference/scala-command/commands.md | 1 + .../scala-command/runner-specification.md | 1 + 20 files changed, 87 insertions(+), 22 deletions(-) diff --git a/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopExit.scala b/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopExit.scala index a2f6612dac..d75d20129b 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopExit.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopExit.scala @@ -18,7 +18,7 @@ object BloopExit extends ScalaCommand[BloopExitOptions] { import opts.* compilationServer.bloopRifleConfig( global.logging.logger, - coursier.coursierCache(global.logging.logger.coursierLogger("Downloading Bloop")), + coursier.coursierCache(global.logging.logger, cacheLoggerPrefix = "Downloading Bloop"), global.logging.verbosity, "java", // shouldn't be used… Directories.directories diff --git a/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopOutput.scala b/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopOutput.scala index b53f155fb1..5bcce2aaf9 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopOutput.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopOutput.scala @@ -20,7 +20,7 @@ object BloopOutput extends ScalaCommand[BloopOutputOptions] { override def runCommand(options: BloopOutputOptions, args: RemainingArgs, logger: Logger): Unit = { val bloopRifleConfig = options.compilationServer.bloopRifleConfig( logger, - CoursierOptions().coursierCache(logger.coursierLogger("Downloading Bloop")), // unused here + CoursierOptions().coursierCache(logger, cacheLoggerPrefix = "Downloading Bloop"), // unused here options.global.logging.verbosity, "unused-java", // unused here Directories.directories diff --git a/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopStart.scala b/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopStart.scala index c68a51411f..2aeb03b57b 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopStart.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/bloop/BloopStart.scala @@ -23,14 +23,12 @@ object BloopStart extends ScalaCommand[BloopStartOptions] { import opts.* val buildOptions = BuildOptions( javaOptions = JvmUtils.javaOptions(jvm).orExit(global.logging.logger), - internal = InternalOptions( - cache = Some(coursier.coursierCache(global.logging.logger.coursierLogger(""))) - ) + internal = InternalOptions(cache = Some(coursier.coursierCache(global.logging.logger))) ) compilationServer.bloopRifleConfig( global.logging.logger, - coursier.coursierCache(global.logging.logger.coursierLogger("Downloading Bloop")), + coursier.coursierCache(global.logging.logger, cacheLoggerPrefix = "Downloading Bloop"), global.logging.verbosity, buildOptions.javaHome().value.javaCommand, Directories.directories diff --git a/modules/cli/src/main/scala/scala/cli/commands/config/Config.scala b/modules/cli/src/main/scala/scala/cli/commands/config/Config.scala index f6b3f0dad2..055f40bd96 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/config/Config.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/config/Config.scala @@ -46,7 +46,7 @@ object Config extends ScalaCommand[ConfigOptions] { ) sys.exit(1) } - val coursierCache = options.coursier.coursierCache(logger.coursierLogger("")) + val coursierCache = options.coursier.coursierCache(logger) val secKeyEntry = Keys.pgpSecretKey val pubKeyEntry = Keys.pgpPublicKey diff --git a/modules/cli/src/main/scala/scala/cli/commands/default/Default.scala b/modules/cli/src/main/scala/scala/cli/commands/default/Default.scala index 4f42a7ca3c..ffa90d094c 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/default/Default.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/default/Default.scala @@ -43,7 +43,7 @@ class Default(actualHelp: => RuntimeCommandsHelp) Version.runCommand( options = VersionOptions( global = options.shared.global, - offline = options.shared.coursier.getOffline().getOrElse(false) + offline = options.shared.coursier.getOffline(logger).getOrElse(false) ), args = args, logger = logger diff --git a/modules/cli/src/main/scala/scala/cli/commands/github/SecretCreate.scala b/modules/cli/src/main/scala/scala/cli/commands/github/SecretCreate.scala index 7a308ad51a..c673a45224 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/github/SecretCreate.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/github/SecretCreate.scala @@ -157,7 +157,7 @@ object SecretCreate extends ScalaCommand[SecretCreateOptions] { ).orExit(logger) } - val cache = options.coursier.coursierCache(logger.coursierLogger("")) + val cache = options.coursier.coursierCache(logger) val archiveCache = ArchiveCache().withCache(cache) LibSodiumJni.init(cache, archiveCache, logger) diff --git a/modules/cli/src/main/scala/scala/cli/commands/new/New.scala b/modules/cli/src/main/scala/scala/cli/commands/new/New.scala index 8f09cd5d39..38839d7ee2 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/new/New.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/new/New.scala @@ -24,7 +24,7 @@ object New extends ScalaCommand[NewOptions] { Seq.empty, Some(scalaParameters), logger, - CoursierOptions().coursierCache(logger.coursierLogger("")), + CoursierOptions().coursierCache(logger), None ) match { case Right(value) => value diff --git a/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpExternalCommand.scala b/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpExternalCommand.scala index 90b5e5bc6c..d9f0dc9674 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpExternalCommand.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpExternalCommand.scala @@ -106,7 +106,7 @@ abstract class PgpExternalCommand extends ExternalCommand { val logger = options.global.logging.logger - val cache = options.coursier.coursierCache(logger.coursierLogger("")) + val cache = options.coursier.coursierCache(logger) val retCode = tryRun( cache, remainingArgs, diff --git a/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpPush.scala b/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpPush.scala index 252fcb132e..6223eeb896 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpPush.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/pgp/PgpPush.scala @@ -31,7 +31,7 @@ object PgpPush extends ScalaCommand[PgpPushOptions] { sys.exit(1) } - lazy val coursierCache = options.coursier.coursierCache(logger.coursierLogger("")) + lazy val coursierCache = options.coursier.coursierCache(logger) for (key <- all) { val path = os.Path(key, os.pwd) diff --git a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishSetup.scala b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishSetup.scala index a01a47196b..de80c9d2c9 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/publish/PublishSetup.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/publish/PublishSetup.scala @@ -42,7 +42,7 @@ object PublishSetup extends ScalaCommand[PublishSetupOptions] { ): Unit = { Publish.maybePrintLicensesAndExit(options.publishParams) - val coursierCache = options.coursier.coursierCache(logger.coursierLogger("")) + val coursierCache = options.coursier.coursierCache(logger) val directories = Directories.directories lazy val configDb = ConfigDbUtils.configDb.orExit(logger) diff --git a/modules/cli/src/main/scala/scala/cli/commands/shared/CoursierOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/shared/CoursierOptions.scala index 18c17ca1b5..9b91dd5709 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/shared/CoursierOptions.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/shared/CoursierOptions.scala @@ -4,9 +4,13 @@ import caseapp.* import com.github.plokhotnyuk.jsoniter_scala.core.* import com.github.plokhotnyuk.jsoniter_scala.macros.* import coursier.cache.{CacheLogger, CachePolicy, FileCache} +import coursier.util.Task +import scala.build.Logger import scala.build.internals.EnvVar import scala.cli.commands.tags +import scala.cli.config.Keys +import scala.cli.util.ConfigDbUtils import scala.concurrent.duration.Duration // format: off @@ -39,8 +43,8 @@ final case class CoursierOptions( private def validateChecksums = coursierValidateChecksums.getOrElse(true) - def coursierCache(logger: CacheLogger) = { - var baseCache = FileCache().withLogger(logger) + def coursierCache(logger: Logger, cacheLogger: CacheLogger): FileCache[Task] = { + var baseCache = FileCache().withLogger(cacheLogger) if (!validateChecksums) baseCache = baseCache.withChecksums(Nil) val ttlOpt = ttl.map(_.trim).filter(_.nonEmpty).map(Duration(_)) @@ -48,15 +52,19 @@ final case class CoursierOptions( baseCache = baseCache.withTtl(ttl0) for (loc <- cache.filter(_.trim.nonEmpty)) baseCache = baseCache.withLocation(loc) - for (isOffline <- getOffline() if isOffline) + for (isOffline <- getOffline(logger) if isOffline) baseCache = baseCache.withCachePolicies(Seq(CachePolicy.LocalOnly)) baseCache } - def getOffline(): Option[Boolean] = offline + def coursierCache(logger: Logger, cacheLoggerPrefix: String = ""): FileCache[Task] = + coursierCache(logger, logger.coursierLogger(cacheLoggerPrefix)) + + def getOffline(logger: Logger): Option[Boolean] = offline .orElse(EnvVar.Coursier.coursierMode.valueOpt.map(_ == "offline")) .orElse(Option(System.getProperty("coursier.mode")).map(_ == "offline")) + .orElse(ConfigDbUtils.getConfigDbOpt(logger).flatMap(_.get(Keys.offline).toOption.flatten)) } object CoursierOptions { diff --git a/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala b/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala index 08b243ac4a..e12c6e5352 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/shared/SharedOptions.scala @@ -432,7 +432,7 @@ final case class SharedOptions( strictBloopJsonCheck = strictBloopJsonCheck, interactive = Some(() => interactive), exclude = exclude.map(Positioned.commandLine), - offline = coursier.getOffline() + offline = coursier.getOffline(logger) ), notForBloopOptions = bo.PostBuildOptions( scalaJsLinkerOptions = linkerOptions(js), @@ -604,11 +604,11 @@ final case class SharedOptions( options => bloopRifleConfig(Some(options)), threads.bloop, strictBloopJsonCheckOrDefault, - coursier.getOffline().getOrElse(false) + coursier.getOffline(logger).getOrElse(false) ) else SimpleScalaCompilerMaker("java", Nil) - lazy val coursierCache = coursier.coursierCache(logging.logger.coursierLogger("")) + lazy val coursierCache: FileCache[Task] = coursier.coursierCache(logging.logger) def inputs( args: Seq[String], diff --git a/modules/cli/src/main/scala/scala/cli/launcher/LauncherCli.scala b/modules/cli/src/main/scala/scala/cli/launcher/LauncherCli.scala index f45d855c3c..ed69bcb520 100644 --- a/modules/cli/src/main/scala/scala/cli/launcher/LauncherCli.scala +++ b/modules/cli/src/main/scala/scala/cli/launcher/LauncherCli.scala @@ -21,7 +21,7 @@ object LauncherCli { def runAndExit(version: String, options: LauncherOptions, remainingArgs: Seq[String]): Nothing = { val logger = LoggingOptions().logger - val cache = CoursierOptions().coursierCache(logger.coursierLogger("")) + val cache = CoursierOptions().coursierCache(logger) val scalaVersion = options.cliScalaVersion.getOrElse(scalaCliScalaVersion(version)) val scalaParameters = ScalaParameters(scalaVersion) val snapshotsRepo = Seq(Repositories.central, Repositories.sonatype("snapshots")) diff --git a/modules/cli/src/test/scala/cli/tests/LauncherCliTest.scala b/modules/cli/src/test/scala/cli/tests/LauncherCliTest.scala index fa9c92a040..723f629594 100644 --- a/modules/cli/src/test/scala/cli/tests/LauncherCliTest.scala +++ b/modules/cli/src/test/scala/cli/tests/LauncherCliTest.scala @@ -12,7 +12,7 @@ class LauncherCliTest extends munit.FunSuite { test("resolve nightly version".flaky) { val logger = TestLogger() - val cache = CoursierOptions().coursierCache(logger.coursierLogger("")) + val cache = CoursierOptions().coursierCache(logger) val scalaParameters = ScalaParameters(Constants.defaultScalaVersion) val nightlyCliVersion = LauncherCli.resolveNightlyScalaCliVersion(cache, scalaParameters) diff --git a/modules/config/src/main/scala/scala/cli/config/Keys.scala b/modules/config/src/main/scala/scala/cli/config/Keys.scala index 4b625d71f6..5ea268dcd3 100644 --- a/modules/config/src/main/scala/scala/cli/config/Keys.scala +++ b/modules/config/src/main/scala/scala/cli/config/Keys.scala @@ -70,6 +70,13 @@ object Keys { description = "Globally enables power mode (the '--power' launcher flag)." ) + val offline = new Key.BooleanEntry( + prefix = Seq.empty, + name = "offline", + specificationLevel = SpecificationLevel.IMPLEMENTATION, + description = "Globally enables offline mode (the '--offline' flag)." + ) + val suppressDirectivesInMultipleFilesWarning = new Key.BooleanEntry( prefix = Seq("suppress-warning"), @@ -172,6 +179,7 @@ object Keys { suppressDirectivesInMultipleFilesWarning, suppressOutdatedDependenciessWarning, suppressExperimentalFeatureWarning, + offline, pgpPublicKey, pgpSecretKey, pgpSecretKeyPassword, diff --git a/modules/integration/src/test/scala/scala/cli/integration/ConfigTests.scala b/modules/integration/src/test/scala/scala/cli/integration/ConfigTests.scala index 327f161350..02f2e5975c 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/ConfigTests.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/ConfigTests.scala @@ -560,4 +560,47 @@ class ConfigTests extends ScalaCliSuite { } } + for { + offlineSetting <- Seq(true, false) + prefillCache <- if (offlineSetting) Seq(true, false) else Seq(false) + caption = s"offline mode: $offlineSetting, " + + (offlineSetting -> prefillCache match { + case (true, true) => "build should succeed when cache was pre-filled" + case (true, false) => "build should fail when cache is empty" + case _ => "dependencies should be downloaded as normal" + }) + } + test(caption) { + TestInputs( + os.rel / "simple.sc" -> "println(dotty.tools.dotc.config.Properties.versionNumberString)" + ) + .fromRoot { root => + val configFile = os.rel / "config" / "config.json" + val localRepoPath = root / "local-repo" + val envs = Map( + "COURSIER_CACHE" -> localRepoPath.toString, + "SCALA_CLI_CONFIG" -> configFile.toString + ) + os.proc(TestUtil.cli, "bloop", "exit", "--power").call(cwd = root) + os.proc(TestUtil.cli, "config", "offline", offlineSetting.toString) + .call(cwd = root, env = envs) + if (prefillCache) for { + artifactName <- Seq( + "scala3-compiler_3", + "scala3-staging_3", + "scala3-tasty-inspector_3", + "scala3-sbt-bridge" + ) + artifact = s"org.scala-lang:$artifactName:${Constants.scala3Next}" + } os.proc(TestUtil.cs, "fetch", "--cache", localRepoPath, artifact).call(cwd = root) + val buildExpectedToSucceed = !offlineSetting || prefillCache + val r = os.proc(TestUtil.cli, "run", "simple.sc", "--with-compiler") + .call(cwd = root, env = envs, check = buildExpectedToSucceed) + if (buildExpectedToSucceed) expect(r.out.trim() == Constants.scala3Next) + else expect(r.exitCode == 1) + os.proc(TestUtil.cli, "config", "offline", "--unset") + .call(cwd = root, env = envs) + } + } + } diff --git a/website/docs/guides/power/offline.md b/website/docs/guides/power/offline.md index 14ca81e925..4bac876ea5 100644 --- a/website/docs/guides/power/offline.md +++ b/website/docs/guides/power/offline.md @@ -39,6 +39,11 @@ or scala-cli -Dcoursier.mode=offline run Main.scala ``` +Finally, it's possible to enable offline mode via global config: +```bash ignore +scala-cli --power config offline true +``` + ## Changes in behaviour ### Scala artifacts diff --git a/website/docs/reference/commands.md b/website/docs/reference/commands.md index 193dfbba8d..b5a99d3ccf 100644 --- a/website/docs/reference/commands.md +++ b/website/docs/reference/commands.md @@ -56,6 +56,7 @@ Available keys: - interactive Globally enables interactive mode (the '--interactive' flag). - interactive-was-suggested Setting indicating if the global interactive mode was already suggested. - java.properties Java properties for Scala CLI's execution. + - offline Globally enables offline mode (the '--offline' flag). - pgp.public-key The PGP public key, used for signing. - pgp.secret-key The PGP secret key, used for signing. - pgp.secret-key-password The PGP secret key password, used for signing. diff --git a/website/docs/reference/scala-command/commands.md b/website/docs/reference/scala-command/commands.md index 8018179432..c0b9e392af 100644 --- a/website/docs/reference/scala-command/commands.md +++ b/website/docs/reference/scala-command/commands.md @@ -55,6 +55,7 @@ Available keys: - interactive Globally enables interactive mode (the '--interactive' flag). - interactive-was-suggested Setting indicating if the global interactive mode was already suggested. - java.properties Java properties for Scala CLI's execution. + - offline Globally enables offline mode (the '--offline' flag). - pgp.public-key The PGP public key, used for signing. - pgp.secret-key The PGP secret key, used for signing. - pgp.secret-key-password The PGP secret key password, used for signing. diff --git a/website/docs/reference/scala-command/runner-specification.md b/website/docs/reference/scala-command/runner-specification.md index 7e1a6a3330..dbc9246c0a 100644 --- a/website/docs/reference/scala-command/runner-specification.md +++ b/website/docs/reference/scala-command/runner-specification.md @@ -663,6 +663,7 @@ Available keys: - interactive Globally enables interactive mode (the '--interactive' flag). - interactive-was-suggested Setting indicating if the global interactive mode was already suggested. - java.properties Java properties for Scala CLI's execution. + - offline Globally enables offline mode (the '--offline' flag). - pgp.public-key The PGP public key, used for signing. - pgp.secret-key The PGP secret key, used for signing. - pgp.secret-key-password The PGP secret key password, used for signing. From 8a9cf6b2ef2e6e3ab082401cf70d541177ffd420 Mon Sep 17 00:00:00 2001 From: Piotr Chabelski Date: Wed, 2 Oct 2024 12:37:36 +0200 Subject: [PATCH 2/2] [debug] increase verbosity in failing tests --- .../scala/cli/integration/RunGistTestDefinitions.scala | 2 +- .../cli/integration/RunPipedSourcesTestDefinitions.scala | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala index 8cfdb61198..389cc726e0 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala @@ -38,7 +38,7 @@ trait RunGistTestDefinitions { _: RunTestDefinitions => "https://gist.github.com/alexarchambault/f972d941bc4a502d70267cfbbc4d6343/raw/2691c01984c9249936a625a42e29a822a357b0f6/Test.java" val message = "Hello from Java GitHub Gist" emptyInputs.fromRoot { root => - val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url)) + val output = os.proc(TestUtil.cli, extraOptions, "-v", "-v", "-v", escapedUrls(url)) .call(cwd = root) .out.trim() expect(output == message) diff --git a/modules/integration/src/test/scala/scala/cli/integration/RunPipedSourcesTestDefinitions.scala b/modules/integration/src/test/scala/scala/cli/integration/RunPipedSourcesTestDefinitions.scala index 0f3972a338..6303a8cc9f 100644 --- a/modules/integration/src/test/scala/scala/cli/integration/RunPipedSourcesTestDefinitions.scala +++ b/modules/integration/src/test/scala/scala/cli/integration/RunPipedSourcesTestDefinitions.scala @@ -64,7 +64,7 @@ trait RunPipedSourcesTestDefinitions { _: RunTestDefinitions => |} |""".stripMargin emptyInputs.fromRoot { root => - val output = os.proc(TestUtil.cli, "_.java", extraOptions) + val output = os.proc(TestUtil.cli, "_.java", "-v", "-v", "-v", extraOptions) .call(cwd = root, stdin = pipedInput) .out.trim() expect(output == expectedOutput) @@ -88,7 +88,7 @@ trait RunPipedSourcesTestDefinitions { _: RunTestDefinitions => |} |""".stripMargin emptyInputs.fromRoot { root => - val output = os.proc(TestUtil.cli, "_.java", extraOptions) + val output = os.proc(TestUtil.cli, "_.java", "-v", "-v", "-v", extraOptions) .call(cwd = root, stdin = pipedInput) .out.trim() expect(output == expectedOutput) @@ -129,6 +129,9 @@ trait RunPipedSourcesTestDefinitions { _: RunTestDefinitions => scalaSnippet, "--java-snippet", javaSnippet, + "-v", + "-v", + "-v", extraOptions ) .call(cwd = root, stdin = pipedInput)