From f9b07dba806c0118ef6022dd55847d68cf198bf3 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Fri, 4 Oct 2024 12:00:45 -0700 Subject: [PATCH] chore(ci): Have Vector `master` use `main` of VRL (#21417) So that nightly releases can you unreleased VRL changes for testing. This was requested by a user. I can see the usefulness to be able to validate VRL changes without having to build Vector. As part of the release, we'll switch it to the latest released version (which should be the same SHA). Signed-off-by: Jesse Szwedko --- .github/ISSUE_TEMPLATE/minor-release.md | 1 + Cargo.lock | 48 +++++++++++------ Cargo.toml | 2 +- lib/vector-vrl/web-playground/Cargo.toml | 2 +- lib/vector-vrl/web-playground/build.rs | 67 ++++++++++++++---------- 5 files changed, 75 insertions(+), 45 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 945a16ed3da01..59f17a239b698 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -13,6 +13,7 @@ The week before the release: - `git fetch && git checkout origin/master && git checkout -b v0. && git push -u` - [ ] Create a new release preparation branch from `master` - `git checkout -b prepare-v0. && git push -u` +- [ ] Pin VRL to latest released version rather than `main` - [ ] Check if there is a newer version of Alpine or Debian available to update the release images in `distribution/docker/`. Update if so. - [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release diff --git a/Cargo.lock b/Cargo.lock index 7c83b17dad013..01721c1d6714d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1844,13 +1844,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" [[package]] -name = "cargo_toml" -version = "0.20.5" +name = "cargo-lock" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88da5a13c620b4ca0078845707ea9c3faf11edbc3ffd8497d11d686211cd1ac0" +checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72" dependencies = [ + "semver 1.0.23", "serde", - "toml", + "toml 0.7.8", + "url", ] [[package]] @@ -2274,7 +2276,7 @@ dependencies = [ "directories", "serde", "thiserror", - "toml", + "toml 0.8.19", ] [[package]] @@ -5117,11 +5119,12 @@ checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" [[package]] name = "lalrpop-util" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "108dc8f5dabad92c65a03523055577d847f5dcc00f3e7d3a68bc4d48e01d8fe1" +checksum = "feee752d43abd0f4807a921958ab4131f692a44d4d599733d4419c5d586176ce" dependencies = [ "regex-automata 0.4.8", + "rustversion", ] [[package]] @@ -8437,7 +8440,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93b4e415d6bff989e5e48649ca9b8b4d4997cb069a0c90a84bfd38c7df5e3968" dependencies = [ - "toml", + "toml 0.8.19", ] [[package]] @@ -9614,6 +9617,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + [[package]] name = "toml" version = "0.8.19" @@ -9642,6 +9657,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap 2.5.0", + "serde", + "serde_spanned", "toml_datetime", "winnow 0.5.18", ] @@ -10351,7 +10368,7 @@ dependencies = [ "serde_yaml 0.9.34+deprecated", "sha2", "tempfile", - "toml", + "toml 0.8.19", ] [[package]] @@ -10522,7 +10539,7 @@ dependencies = [ "tokio-test", "tokio-tungstenite 0.20.1", "tokio-util", - "toml", + "toml 0.8.19", "tonic", "tonic-build 0.11.0", "tower", @@ -10662,7 +10679,7 @@ dependencies = [ "serde_json", "serde_with 3.10.0", "snafu 0.7.5", - "toml", + "toml 0.8.19", "tracing 0.1.40", "url", "vector-config-common", @@ -10766,7 +10783,7 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-util", - "toml", + "toml 0.8.19", "tonic", "tracing 0.1.40", "tracing-core 0.1.32", @@ -10901,7 +10918,7 @@ dependencies = [ name = "vector-vrl-web-playground" version = "0.1.0" dependencies = [ - "cargo_toml", + "cargo-lock", "enrichment", "getrandom 0.2.15", "gloo-utils", @@ -10927,8 +10944,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c22ec61cbd43e563df185521f9a2fb2f42f6ab96604a574c82f6564049fb431" +source = "git+https://github.com/vectordotdev/vrl?branch=main#3295458dc84142abab4637095248771ca6087800" dependencies = [ "aes", "ansi_term", @@ -10971,7 +10987,7 @@ dependencies = [ "influxdb-line-protocol", "itertools 0.13.0", "lalrpop", - "lalrpop-util 0.21.0", + "lalrpop-util 0.22.0", "md-5", "mlua", "nom", diff --git a/Cargo.toml b/Cargo.toml index a5f6f0069626e..c31345e5891dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,7 +149,7 @@ toml = { version = "0.8.19", default-features = false, features = ["display", "p tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } uuid = { version = "1.10.0", features = ["v4", "v7", "serde"] } -vrl = { version = "0.19.0", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { git = "https://github.com/vectordotdev/vrl", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] pin-project.workspace = true diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 60ffe4e5d9f8c..fd54e801578d6 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -19,4 +19,4 @@ vector-vrl-functions = { path = "../functions" } enrichment = { path = "../../enrichment" } [build-dependencies] -cargo_toml = "0.20.5" +cargo-lock = "9.0.0" diff --git a/lib/vector-vrl/web-playground/build.rs b/lib/vector-vrl/web-playground/build.rs index d6abdbbd3090e..b6bc46a33cc80 100644 --- a/lib/vector-vrl/web-playground/build.rs +++ b/lib/vector-vrl/web-playground/build.rs @@ -1,11 +1,12 @@ -use cargo_toml::Manifest; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, fs, io}; -fn get_vector_toml_path() -> PathBuf { +use cargo_lock::{package::SourceKind, Lockfile}; + +fn get_vector_lock_path() -> PathBuf { let path = fs::canonicalize(env::var("CARGO_MANIFEST_DIR").unwrap()).unwrap(); // Remove the "lib/vector-vrl/web-playground" suffix @@ -15,7 +16,7 @@ fn get_vector_toml_path() -> PathBuf { .and_then(|p| p.parent()); parent_path .expect("Failed to find vector repo root") - .join("Cargo.toml") + .join("Cargo.lock") .to_path_buf() } @@ -47,27 +48,39 @@ fn write_vector_constants(output_file: &mut File) { .expect("Failed to write Vector version constant"); } -fn write_vrl_constants(manifest: &Manifest, output_file: &mut File) { - let vrl_dep = manifest - .dependencies - .get("vrl") - .expect("missing VRL dependency") - .detail() - .expect("expected detail dependency format"); +fn write_vrl_constants(lockfile: &Lockfile, output_file: &mut File) { + let vrl_dep = lockfile + .packages + .iter() + .find(|&package| package.name.as_str() == "vrl") + .expect("missing VRL dependency"); + + let vrl_source = vrl_dep.source.clone().expect("missing VRL source id"); - let (version, link) = match &vrl_dep.version { - None => { - let repo = vrl_dep - .git - .as_ref() - .expect("VRL dependency should use 'version' or 'git'"); - let version = vrl_dep - .rev - .as_ref() - .expect("VRL git revision not specified"); - (version.clone(), format!("{repo}/tree/{version}")) + let (version, link) = match vrl_source.kind() { + SourceKind::Git(_) => { + let precise = vrl_source + .precise() + .expect("git reference should have precise") + .to_string(); + ( + precise.clone(), + format!("{}/tree/{precise}", vrl_source.url()), + ) + } + SourceKind::Registry if vrl_source.is_default_registry() => { + let version = vrl_dep.version.to_string(); + ( + version.to_string(), + format!("https://crates.io/crates/vrl/{version}"), + ) } - Some(v) => (v.clone(), format!("https://crates.io/crates/vrl/{v}")), + SourceKind::Path + | SourceKind::Registry + | SourceKind::SparseRegistry + | SourceKind::LocalRegistry + | SourceKind::Directory => unimplemented!("unhandled source kind: {:?}", vrl_source.kind()), + _ => unimplemented!("unknown source kind: {:?}", vrl_source.kind()), }; output_file @@ -79,19 +92,19 @@ fn write_vrl_constants(manifest: &Manifest, output_file: &mut File) { .expect("Failed to write VRL_LINK constant"); } -fn write_build_constants(manifest: &Manifest, dest_path: &Path) -> io::Result<()> { +fn write_build_constants(lockfile: &Lockfile, dest_path: &Path) -> io::Result<()> { let mut output_file = File::create(dest_path)?; output_file.write_all( "// AUTOGENERATED CONSTANTS. SEE BUILD.RS AT REPOSITORY ROOT. DO NOT MODIFY.\n".as_ref(), )?; write_vector_constants(&mut output_file); - write_vrl_constants(manifest, &mut output_file); + write_vrl_constants(lockfile, &mut output_file); Ok(()) } fn main() { - let manifest = - Manifest::from_path(get_vector_toml_path()).expect("Failed to load Vector Cargo.toml"); + let lockfile = + Lockfile::load(get_vector_lock_path()).expect("Failed to load Vector Cargo.lock"); let dst = Path::new(&env::var("OUT_DIR").unwrap()).join("built.rs"); - write_build_constants(&manifest, &dst).expect("Failed to write constants"); + write_build_constants(&lockfile, &dst).expect("Failed to write constants"); }