chore(ci): Have Vector master use main of VRL (#21417)

So that nightly releases can you unreleased VRL changes for testing. This was requested by a user.
I can see the usefulness to be able to validate VRL changes without having to build Vector.

As part of the release, we'll switch it to the latest released version (which should be the same
SHA).

Signed-off-by: Jesse Szwedko <[email protected]>

.github/ISSUE_TEMPLATE/minor-release.md

@@ -13,6 +13,7 @@ The week before the release:
   - `git fetch && git checkout origin/master && git checkout -b v0.<new version number> && git push -u`
 - [ ] Create a new release preparation branch from `master`
   - `git checkout -b prepare-v0.<new version number> && git push -u`
+- [ ] Pin VRL to latest released version rather than `main`
 - [ ] Check if there is a newer version of Alpine or Debian available to update the release images
       in `distribution/docker/`. Update if so.
 - [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release

Cargo.toml

@@ -149,7 +149,7 @@ toml = { version = "0.8.19", default-features = false, features = ["display", "p
 tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] }
 tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] }
 uuid = { version = "1.10.0", features = ["v4", "v7", "serde"] }
-vrl = { version = "0.19.0", features = ["arbitrary", "cli", "test", "test_framework"] }
+vrl = { git = "https://github.com/vectordotdev/vrl", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] }
 
 [dependencies]
 pin-project.workspace = true

lib/vector-vrl/web-playground/Cargo.toml

@@ -19,4 +19,4 @@ vector-vrl-functions = { path = "../functions" }
 enrichment = { path = "../../enrichment" }
 
 [build-dependencies]
-cargo_toml = "0.20.5"
+cargo-lock = "9.0.0"

lib/vector-vrl/web-playground/build.rs

@@ -1,11 +1,12 @@
-use cargo_toml::Manifest;
 use std::fs::File;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::{env, fs, io};
 
-fn get_vector_toml_path() -> PathBuf {
+use cargo_lock::{package::SourceKind, Lockfile};
+
+fn get_vector_lock_path() -> PathBuf {
     let path = fs::canonicalize(env::var("CARGO_MANIFEST_DIR").unwrap()).unwrap();
 
     // Remove the "lib/vector-vrl/web-playground" suffix
@@ -15,7 +16,7 @@ fn get_vector_toml_path() -> PathBuf {
         .and_then(|p| p.parent());
     parent_path
         .expect("Failed to find vector repo root")
-        .join("Cargo.toml")
+        .join("Cargo.lock")
         .to_path_buf()
 }
 
@@ -47,27 +48,39 @@ fn write_vector_constants(output_file: &mut File) {
         .expect("Failed to write Vector version constant");
 }
 
-fn write_vrl_constants(manifest: &Manifest, output_file: &mut File) {
-    let vrl_dep = manifest
-        .dependencies
-        .get("vrl")
-        .expect("missing VRL dependency")
-        .detail()
-        .expect("expected detail dependency format");
+fn write_vrl_constants(lockfile: &Lockfile, output_file: &mut File) {
+    let vrl_dep = lockfile
+        .packages
+        .iter()
+        .find(|&package| package.name.as_str() == "vrl")
+        .expect("missing VRL dependency");
+
+    let vrl_source = vrl_dep.source.clone().expect("missing VRL source id");
 
-    let (version, link) = match &vrl_dep.version {
-        None => {
-            let repo = vrl_dep
-                .git
-                .as_ref()
-                .expect("VRL dependency should use 'version' or 'git'");
-            let version = vrl_dep
-                .rev
-                .as_ref()
-                .expect("VRL git revision not specified");
-            (version.clone(), format!("{repo}/tree/{version}"))
+    let (version, link) = match vrl_source.kind() {
+        SourceKind::Git(_) => {
+            let precise = vrl_source
+                .precise()
+                .expect("git reference should have precise")
+                .to_string();
+            (
+                precise.clone(),
+                format!("{}/tree/{precise}", vrl_source.url()),
+            )
+        }
+        SourceKind::Registry if vrl_source.is_default_registry() => {
+            let version = vrl_dep.version.to_string();
+            (
+                version.to_string(),
+                format!("https://crates.io/crates/vrl/{version}"),
+            )
         }
-        Some(v) => (v.clone(), format!("https://crates.io/crates/vrl/{v}")),
+        SourceKind::Path
+        | SourceKind::Registry
+        | SourceKind::SparseRegistry
+        | SourceKind::LocalRegistry
+        | SourceKind::Directory => unimplemented!("unhandled source kind: {:?}", vrl_source.kind()),
+        _ => unimplemented!("unknown source kind: {:?}", vrl_source.kind()),
     };
 
     output_file
@@ -79,19 +92,19 @@ fn write_vrl_constants(manifest: &Manifest, output_file: &mut File) {
         .expect("Failed to write VRL_LINK constant");
 }
 
-fn write_build_constants(manifest: &Manifest, dest_path: &Path) -> io::Result<()> {
+fn write_build_constants(lockfile: &Lockfile, dest_path: &Path) -> io::Result<()> {
     let mut output_file = File::create(dest_path)?;
     output_file.write_all(
         "// AUTOGENERATED CONSTANTS. SEE BUILD.RS AT REPOSITORY ROOT. DO NOT MODIFY.\n".as_ref(),
     )?;
     write_vector_constants(&mut output_file);
-    write_vrl_constants(manifest, &mut output_file);
+    write_vrl_constants(lockfile, &mut output_file);
     Ok(())
 }
 
 fn main() {
-    let manifest =
-        Manifest::from_path(get_vector_toml_path()).expect("Failed to load Vector Cargo.toml");
+    let lockfile =
+        Lockfile::load(get_vector_lock_path()).expect("Failed to load Vector Cargo.lock");
     let dst = Path::new(&env::var("OUT_DIR").unwrap()).join("built.rs");
-    write_build_constants(&manifest, &dst).expect("Failed to write constants");
+    write_build_constants(&lockfile, &dst).expect("Failed to write constants");
 }