integrate vcllvm into the build

build.sc

@@ -6,6 +6,7 @@ import os._
 import mill.{util => _, _}
 import scalalib.{JavaModule => _, ScalaModule => _, _}
 import contrib.buildinfo.BuildInfo
+import me.pieterbos.mill.cpp.{CMakeModule, LinkableModule}
 import mill.util.Jvm
 import vct.col.ast.structure
 import vct.col.ast.structure.{AllFamilies, FamilyDefinition, Name, NodeDefinition}
@@ -165,6 +166,7 @@ object vercors extends Module {
       def allDefinitionsSource = T.source(analysis / "all-definitions.json")
       def allDefinitions = T { read[Seq[NodeDefinition]](allDefinitionsSource()) }
 
+      // Step 1: compile
       object generators extends VercorsModule {
         def key = "helpers"
         def deps: T[Agg[Dep]] = T {
@@ -266,6 +268,7 @@ object vercors extends Module {
         Seq(
           instantiate[structure.api.AllNodesGeneratorApi]("ProtoAuxTypes")(),
           instantiate[structure.api.AllNodesGeneratorApi]("RewriteHelpers")(),
+          instantiate[structure.api.AllNodesGeneratorApi]("MegaCol")(),
         )
       }
 
@@ -314,7 +317,7 @@ object vercors extends Module {
             gen.generate(T.dest.toNIO, write(declarationFamilies), write(structuralFamilies))
           }
           protoAuxTypes().foreach(_.generate(T.dest.toNIO, write(definitions)))
-          Seq(PathRef(T.dest, quick = true))
+          PathRef(T.dest, quick = true)
         }
       }
 
@@ -351,12 +354,15 @@ object vercors extends Module {
 
       def generatedSources: T[Seq[PathRef]] = T {
         T.traverse(nodeCross)(node(_).generate)() ++
-          T.traverse(familyCross)(family(_).generate)() ++
+          T.traverse(familyCross)(family(_).generate)() :+
           global.generate()
       }
 
+      def megacol: T[PathRef] = T.source(global.generate().path / "col.proto")
+
       def sources: T[PathRef] = T.persistent {
         util.quickCopy(T.dest, generatedSources())
+        os.remove(T.dest / "col.proto")
         PathRef(T.dest, quick = true)
       }
     }
@@ -559,6 +565,100 @@ object vercors extends Module {
     }
   }
 
+  object vcllvm extends CppExecutableModule { outer =>
+    def root: T[os.Path] = T { settings.src / "llvm" }
+
+    object llvm extends LinkableModule {
+      def moduleDeps = Nil
+      def systemLibraryDeps = T { Seq("LLVM-15") }
+      def staticObjects = T { Seq.empty[PathRef] }
+      def dynamicObjects = T { Seq.empty[PathRef] }
+      def exportIncludePaths = T.sources(
+        os.Path("/usr/include/llvm-15"),
+        os.Path("/usr/include/llvm-c-15"),
+      )
+    }
+
+    object json extends LinkableModule {
+      def moduleDeps = Nil
+      def systemLibraryDeps = T { Seq.empty[String] }
+      def staticObjects = T { Seq.empty[PathRef] }
+      def dynamicObjects = T { Seq.empty[PathRef] }
+      def exportIncludePaths = T {
+        os.write(T.dest / "json.tar.xz", requests.get.stream("https://github.com/nlohmann/json/releases/download/v3.11.2/json.tar.xz"))
+        os.proc("tar", "-xf", T.dest / "json.tar.xz").call(cwd = T.dest)
+        Seq(PathRef(T.dest / "json" / "include"))
+      }
+    }
+
+    object origin extends CppModule {
+      def moduleDeps = Seq(llvm, json)
+      def sources = T.sources(vcllvm.root() / "lib" / "origin")
+      def includePaths = T.sources(vcllvm.root() / "include")
+    }
+
+    object passes extends CppModule {
+      def moduleDeps = Seq(llvm, proto.colProto)
+      def sources = T.sources(vcllvm.root() / "lib" / "passes")
+      def includePaths = T.sources(vcllvm.root() / "include")
+    }
+
+    object transform extends CppModule {
+      def moduleDeps = Seq(llvm, proto.colProto)
+      def sources = T.sources(vcllvm.root() / "lib" / "transform")
+      def includePaths = T.sources(vcllvm.root() / "include")
+    }
+
+    object util extends CppModule {
+      def moduleDeps = Seq(llvm)
+      def sources = T.sources(vcllvm.root() / "lib" / "util")
+      def includePaths = T.sources(vcllvm.root() / "include")
+    }
+
+    def moduleDeps = Seq(origin, passes, transform, util, llvm, proto.colProto)
+    def sources = T.sources(vcllvm.root() / "tools" / "vcllvm")
+    def includePaths = T.sources(vcllvm.root() / "include")
+
+    object proto extends CMakeModule {
+      object protobufGit extends GitModule {
+        override def url: T[String] = "https://github.com/protocolbuffers/protobuf"
+        override def commitish: T[String] = "v25.2"
+        override def fetchSubmodulesRecursively = true
+      }
+
+      def root = T.source(protobufGit.repo())
+      def jobs = T { 8 }
+
+      override def cMakeBuild: T[PathRef] = T {
+        os.proc("cmake", "-B", T.dest, "-S", root().path).call(cwd = T.dest)
+        os.proc("make", "-j", jobs(), "all").call(cwd = T.dest)
+        PathRef(T.dest)
+      }
+
+      object libprotobuf extends CMakeLibrary {
+        def target = T { "libprotobuf" }
+      }
+
+      object protoc extends CMakeExecutable {
+        def target = T { "protoc" }
+      }
+
+      def protoPath = T.sources(vercors.col.helpers.megacol().path / os.up, settings.src / "serialize")
+      def proto = T { vercors.col.helpers.megacol() +: os.walk(settings.src / "serialize").filter(_.ext == "proto").map(PathRef(_)) }
+
+      def generate = T {
+        os.proc(protoc.executable().path, protoPath().map(p => "-I=" + p.path.toString),  "--cpp_out=" + T.dest.toString, proto().map(_.path)).call()
+        T.dest
+      }
+
+      object colProto extends CppModule {
+        def moduleDeps = Seq(libprotobuf)
+        def sources = T { Seq(PathRef(generate())) }
+        def includePaths = T { Seq(PathRef(generate())) }
+      }
+    }
+  }
+
   object allTests extends ScalaModule with ReleaseModule {
     def packedResources = T.sources()
     override def moduleDeps: Seq[JavaModule] = Seq(col.test, viperApi.test, main.test)

mill-build/build.sc

@@ -39,7 +39,8 @@ object millbuild extends MillBuildRootModule {
         ivy"com.lihaoyi::mill-main:0.11.0",
         ivy"com.lihaoyi::mill-main-api:0.11.0",
         ivy"com.lihaoyi::mill-scalalib:0.11.0",
-        ivy"com.lihaoyi::mill-contrib-scalapblib:0.11.0"
+        ivy"com.lihaoyi::mill-contrib-scalapblib:0.11.0",
+        ivy"me.pieterbos::mill-cpp_mill0.11::0.0.1",
       )
     }
     def compileResources = T { Seq.empty[PathRef] }

mill-build/util/src/settings.scala

@@ -18,7 +18,6 @@ package object settings {
 
     val common = log ++ Agg(
       ivy"org.scala-lang.modules::scala-parallel-collections:1.0.4",
-      ivy"io.spray::spray-json:1.3.6"
     )
   }
 }

mill-build/util/src/util/CppModule.scala

@@ -0,0 +1,10 @@
+package util
+
+import me.pieterbos.mill.cpp.{CppModule => BaseCppModule, CppExecutableModule => BaseCppExecutableModule, _}
+import mill.T
+
+trait CppModule extends BaseCppModule {
+  override def standard: T[options.CppStandard] = T[options.CppStandard] { options.CppStandard.Cpp20 }
+}
+
+trait CppExecutableModule extends BaseCppExecutableModule with CppModule

mill-build/util/src/util/GitModule.scala

@@ -7,12 +7,16 @@ trait GitModule extends Module {
 
   def commitish: T[String]
 
+  def fetchSubmodulesRecursively: T[Boolean] = false
+
   def repo = T {
     os.proc("git", "init", "-q").call(cwd = T.dest)
     os.proc("git", "remote", "add", "origin", url()).call(cwd = T.dest)
     os.proc("git", "fetch", "--depth", "1", "origin", commitish()).call(cwd = T.dest)
     os.proc("git", "config", "advice.detachedHead", "false").call(cwd = T.dest)
     os.proc("git", "checkout", "FETCH_HEAD").call(cwd = T.dest)
+    if(fetchSubmodulesRecursively())
+      os.proc("git", "submodule", "update", "--init", "--recursive").call(cwd = T.dest)
     os.walk(T.dest).foreach(_.toIO.setWritable(true))
     os.remove.all(T.dest / ".git")
     T.dest

src/helpers/vct/col/ast/helpers/defn/Proto.scala

@@ -8,7 +8,7 @@ import scala.collection.immutable.ListMap
 import scala.collection.mutable
 
 object Proto {
-  val auxBase = Seq("vct", "col", "ast", "serialize")
+  val auxBase = Seq("vct", "col", "ast")
 
   val STANDARD_OPTIONS: Map[String, String] = ListMap(
     "lenses" -> "false",
@@ -143,6 +143,8 @@ object Proto {
   }
 
   case class Message(name: Seq[String], body: MessageBody) {
+    require(name.nonEmpty)
+
     def write(out: Appendable): Unit = {
       out.append("message ").append(name.last).append(" {\n")
       body.write(out)
@@ -157,13 +159,27 @@ object Proto {
     def imports: Seq[Seq[String]] = namedTypes.distinct.map(_.fqName)
   }
 
-  case class Source(imports: Seq[Seq[String]], options: String, message: Message) {
+  object Source {
+    def apply(imports: Seq[Seq[String]], options: String, message: Message): Source =
+      Source(imports, options, Seq(message))
+  }
+
+  case class Source(imports: Seq[Seq[String]], options: String, messages: Seq[Message]) {
+    require(messages.nonEmpty)
+
     def write(out: Appendable): Unit = {
       out.append("syntax = \"proto2\";\n")
       out.append("\n")
 
-      if(message.name.size > 1) {
-        out.append("package ").append(message.name.init.mkString(".")).append(";\n")
+      if(messages.exists(_.name.size > 1)) {
+        val pkg = messages.collectFirst { case msg if msg.name.size > 1 => msg.name.init }.get
+
+        val wrongPackageMessages = messages.filter(msg => msg.name.size != pkg.size + 1 || msg.name.init != pkg)
+        require(wrongPackageMessages.isEmpty,
+          out.append(s"Messages rendered jointly in one source must have the same package. Wrong: ${
+            wrongPackageMessages.map(_.name.mkString(".")).mkString("{", ", ", "}")}"))
+
+        out.append("package ").append(pkg.mkString(".")).append(";\n")
         out.append("\n")
       }
 
@@ -176,13 +192,16 @@ object Proto {
 
       if(!options.isBlank) {
         out.append(options)
-        out.append("\n\n")
+        out.append("\n")
       }
 
-      message.write(out)
+      for(message <- messages) {
+        out.append("\n")
+        message.write(out)
+      }
     }
 
     def opaqueNodes: Source =
-      copy(message = message.opaqueNodes)
+      copy(messages = messages.map(_.opaqueNodes))
   }
 }

src/helpers/vct/col/ast/helpers/generator/MegaCol.scala

@@ -0,0 +1,30 @@
+package vct.col.ast.helpers.generator
+
+import vct.col.ast.helpers.defn.Proto
+import vct.col.ast.structure.{AllNodesGenerator, NodeDefinition}
+
+import java.nio.file.{Files, Path}
+import scala.util.Using
+
+class MegaCol extends AllNodesGenerator {
+  override def generate(out: Path, definitions: Seq[NodeDefinition]): Unit = {
+    val families = definitions.groupBy(_.family)
+
+    val genNode = new ProtoNode()
+    val genFamily = new ProtoFamily()
+    val genAux = new ProtoAuxTypes()
+
+    val messages =
+      definitions.map(node => genNode.message(node)) ++
+        families.flatMap(family => genFamily.message(family._1, family._2.map(_.name)).toSeq) ++
+        genAux.messages(definitions)
+
+    Using(Files.newBufferedWriter(out.resolve("col.proto"))) { writer =>
+      Proto.Source(
+        messages.flatMap(_.namedTypes.collect { case t: Proto.StandardType => t.fqName }).distinct,
+        options = "",
+        messages = messages,
+      ).write(writer)
+    }
+  }
+}

src/helpers/vct/col/ast/helpers/generator/ProtoAuxTypes.scala

@@ -7,16 +7,17 @@ import java.nio.file.{Files, Path}
 import scala.util.Using
 
 class ProtoAuxTypes extends AllNodesGenerator {
+  def messages(definitions: Seq[NodeDefinition]): Seq[Proto.Message] =
+    definitions
+      .flatMap(_.fields)
+      .map(_._2)
+      .map(ProtoNaming.getType)
+      .flatMap(_.auxs)
+      .distinct
+      .sortBy(_.name.headOption)
+
   override def generate(out: Path, definitions: Seq[NodeDefinition]): Unit = {
-    val auxs =
-      definitions
-        .flatMap(_.fields)
-        .map(_._2)
-        .map(ProtoNaming.getType)
-        .flatMap(_.auxs)
-        .distinct
-        .map(_.opaqueNodes)
-        .sortBy(_.name.headOption)
+    val auxs = messages(definitions).map(_.opaqueNodes)
 
     for(aux <- auxs) {
       val dir = aux.name.init.foldLeft(out)(_.resolve(_))

src/helpers/vct/col/ast/helpers/generator/ProtoFamily.scala

@@ -7,30 +7,37 @@ import java.nio.file.{Files, Path}
 import scala.util.Using
 
 class ProtoFamily extends FamilyGenerator {
-  override def generate(out: Path, family: Name, kind: NodeKind, nodes: Seq[Name]): Unit = {
-    if(nodes == Seq(family)) return
+  def message(family: Name, nodes: Seq[Name]): Option[Proto.Message] = {
+    if(nodes == Seq(family))
+      return None
 
     val fqNames = nodes.map(ProtoNaming.getTypeName)
-
     val name = ProtoNaming.getTypeName(family)
-    val dir = name.init.foldLeft(out)(_.resolve(_))
-    Files.createDirectories(dir)
 
-    Using(Files.newBufferedWriter(dir.resolve(family.base + ".proto"))) { writer =>
-      Proto.Source(
-        imports = Seq("scalapb", "scalapb") +: fqNames,
-        options = Proto.renderOptions(Proto.STANDARD_OPTIONS.updated("package_name", ProtoNaming.scalaPackageOption(name))),
-        message = Proto.Message(name, Proto.MessageOneOf(
-          oneOfName = "v",
-          fields = fqNames.zipWithIndex.map {
-            case (name, idx) => Proto.Field(
-              name = ProtoNaming.snake(name.last),
-              index = idx + 1,
-              t = Proto.UnspecifiedArity(Proto.FamilyType(name))
-            )
-          }
-        ))
-      ).write(writer)
+    Some(Proto.Message(name, Proto.MessageOneOf(
+      oneOfName = "v",
+      fields = fqNames.zipWithIndex.map {
+        case (name, idx) => Proto.Field(
+          name = ProtoNaming.snake(name.last),
+          index = idx + 1,
+          t = Proto.UnspecifiedArity(Proto.FamilyType(name))
+        )
+      }
+    )))
+  }
+
+  override def generate(out: Path, family: Name, kind: NodeKind, nodes: Seq[Name]): Unit = {
+    message(family, nodes).foreach { message =>
+      val dir = message.name.init.foldLeft(out)(_.resolve(_))
+      Files.createDirectories(dir)
+
+      Using(Files.newBufferedWriter(dir.resolve(message.name.last + ".proto"))) { writer =>
+        Proto.Source(
+          imports = Seq("scalapb", "scalapb") +: message.imports,
+          options = Proto.renderOptions(Proto.STANDARD_OPTIONS.updated("package_name", ProtoNaming.scalaPackageOption(message.name))),
+          message = message,
+        ).write(writer)
+      }
     }
   }
 }