feat!: upgrade wasmparser & wasm-encoder to v0.212.0 (#266)

.github/workflows/main.yml

@@ -9,32 +9,21 @@ jobs:
       matrix:
         rust: [stable, beta, nightly]
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Install Rust
       run: rustup update ${{ matrix.rust }} && rustup default ${{ matrix.rust }}
-
-    # Build an up-to-date version of wabt since the releases don't always have
-    # all the features we want.
-    - uses: actions/checkout@v2
-      with:
-        repository: WebAssembly/wabt
-        ref: aa0515b3c808da880942db8658abeaa969534667
-        path: wabt
-    - name: Build wabt
+    - name: Install wabt
       run: |
         set -e
-        cd wabt
-        cmake . -DBUILD_TESTS=OFF
-        make -j$(nproc) wast2json spectest-interp wasm-interp
-        echo `pwd` > $GITHUB_PATH
-
+        curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.35/wabt-1.0.35-ubuntu-20.04.tar.gz | tar xzf -
+        echo "`pwd`/wabt-1.0.35/bin" > $GITHUB_PATH
     - name: Install binaryen
       run: |
         set -e
-        curl -L https://github.com/WebAssembly/binaryen/releases/download/1.39.1/binaryen-1.39.1-x86_64-linux.tar.gz | tar xzf -
-        echo "`pwd`/binaryen-1.39.1" > $GITHUB_PATH
+        curl -L https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-linux.tar.gz | tar xzf -
+        echo "`pwd`/binaryen-version_117/bin" > $GITHUB_PATH
     - run: cargo build --all
     - run: cargo test --all
     - run: cargo check --benches
@@ -45,19 +34,19 @@ jobs:
     name: Fuzz Crate
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v4
     - name: Install Rust
       run: rustup update stable && rustup default stable
     - name: Install wabt
       run: |
         set -e
-        curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.13/wabt-1.0.13-linux.tar.gz | tar xzf -
-        echo "`pwd`/wabt-1.0.13" > $GITHUB_PATH
+        curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.35/wabt-1.0.35-ubuntu-20.04.tar.gz | tar xzf -
+        echo "`pwd`/wabt-1.0.35/bin" > $GITHUB_PATH
     - name: Install binaryen
       run: |
         set -e
-        curl -L https://github.com/WebAssembly/binaryen/releases/download/1.39.1/binaryen-1.39.1-x86_64-linux.tar.gz | tar xzf -
-        echo "`pwd`/binaryen-1.39.1" > $GITHUB_PATH
+        curl -L https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-linux.tar.gz | tar xzf -
+        echo "`pwd`/binaryen-version_117/bin" > $GITHUB_PATH
     - name: Run fuzzer
       run: cargo test -p walrus-fuzz-utils > fuzz.log || (tail -n 1000 fuzz.log && exit 1)
       env:
@@ -71,20 +60,20 @@ jobs:
       matrix:
         test: [watgen, wasm-opt-ttf, raw]
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v4
     - name: Install Rust
       run: rustup update nightly && rustup default nightly
     - run: cargo install cargo-fuzz
     - name: Install wabt
       run: |
         set -e
-        curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.13/wabt-1.0.13-linux.tar.gz | tar xzf -
-        echo "`pwd`/wabt-1.0.13" > $GITHUB_PATH
+        curl -L https://github.com/WebAssembly/wabt/releases/download/1.0.35/wabt-1.0.35-ubuntu-20.04.tar.gz | tar xzf -
+        echo "`pwd`/wabt-1.0.35/bin" > $GITHUB_PATH
     - name: Install binaryen
       run: |
         set -e
-        curl -L https://github.com/WebAssembly/binaryen/releases/download/1.39.1/binaryen-1.39.1-x86_64-linux.tar.gz | tar xzf -
-        echo "`pwd`/binaryen-1.39.1" > $GITHUB_PATH
+        curl -L https://github.com/WebAssembly/binaryen/releases/download/version_117/binaryen-version_117-x86_64-linux.tar.gz | tar xzf -
+        echo "`pwd`/binaryen-version_117/bin" > $GITHUB_PATH
     - name: Run fuzzer
       run: |
         cargo fuzz run ${{ matrix.test }} -- -max_total_time=300 -rss_limit_mb=4096 > fuzz.log 2>&1 || (tail -n 1000 fuzz.log && exit 1)
@@ -93,7 +82,7 @@ jobs:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v4
     - name: Install Rust
       run: rustup update stable && rustup default stable && rustup component add rustfmt
     - run: cargo fmt -- --check

Cargo.toml

@@ -29,8 +29,8 @@ leb128 = "0.2.4"
 log = "0.4.8"
 rayon = { version = "1.1.0", optional = true }
 walrus-macro = { path = './crates/macro', version = '=0.19.0' }
-wasm-encoder = "0.41.0"
-wasmparser = "0.80.2"
+wasm-encoder = "0.212.0"
+wasmparser = "0.212.0"
 gimli = "0.26.0"
 
 [features]

crates/macro/Cargo.toml

@@ -19,4 +19,4 @@ quote = "1.0.2"
 syn = { version = "2.0.0", features = ['extra-traits'] }
 
 [lib]
-proc_macro = true
+proc-macro = true

crates/tests/tests/spec-tests.rs

@@ -22,34 +22,20 @@ fn run(wast: &Path) -> Result<(), anyhow::Error> {
         .skip(1)
         .next()
         .map(|s| s.to_str().unwrap());
-    let extra_args: &[&str] = match proposal {
-        // stable features
-        None
-        | Some("multi-value")
-        | Some("nontrapping-float-to-int-conversions")
-        | Some("sign-extension-ops")
-        | Some("mutable-global") => &[],
-
-        Some("simd") => &["--enable-simd"],
-        Some("bulk-memory-operations") => &["--enable-bulk-memory"],
 
-        Some("reference-types") => &["--enable-reference-types", "--enable-bulk-memory"],
-
-        // TODO: should get threads working
-        Some("threads") => return Ok(()),
-        // TODO: should get tail-call working
-        Some("tail-call") => return Ok(()),
-
-        // not a walrus thing, but not implemented in wabt fully yet anyway
+    let extra_args: &[&str] = match proposal {
+        None => &[],
         Some("annotations") => return Ok(()),
-
-        // not implemented in walrus yet
-        Some("function-references") => return Ok(()),
         Some("exception-handling") => return Ok(()),
+        Some("extended-const") => return Ok(()),
+        Some("function-references") => return Ok(()),
+        Some("gc") => return Ok(()),
         Some("memory64") => return Ok(()),
-
-        // Some("threads") => &["--enable-threads"],
-        Some(other) => panic!("unknown wasm proposal: {}", other),
+        Some("multi-memory") => &["--enable-multi-memory"],
+        Some("relaxed-simd") => return Ok(()),
+        Some("tail-call") => return Ok(()),
+        Some("threads") => return Ok(()),
+        Some(other) => bail!("unknown wasm proposal: {}", other),
     };
 
     let tempdir = TempDir::new()?;
@@ -63,8 +49,6 @@ fn run(wast: &Path) -> Result<(), anyhow::Error> {
         .context("executing `wast2json`")?;
     assert!(status.success());
 
-    let wabt_ok = run_spectest_interp(tempdir.path(), extra_args).is_ok();
-
     let contents = fs::read_to_string(&json).context("failed to read file")?;
     let test: Test = serde_json::from_str(&contents).context("failed to parse file")?;
     let mut files = Vec::new();
@@ -74,72 +58,77 @@ fn run(wast: &Path) -> Result<(), anyhow::Error> {
         config.only_stable_features(true);
     }
 
+    let wabt_ok = run_spectest_interp(tempdir.path(), extra_args).is_ok();
+
+    let mut should_not_parse = vec![];
+    let mut non_deterministic = vec![];
     for command in test.commands {
         let filename = match command.get("filename") {
             Some(name) => name.as_str().unwrap().to_string(),
             None => continue,
         };
-        let line = &command["line"];
+        let line = command["line"].as_u64().unwrap();
         let path = tempdir.path().join(filename);
         match command["type"].as_str().unwrap() {
             "assert_invalid" | "assert_malformed" => {
-                // Skip tests that are actually valid with various in-flight proposals
-                let text = command["text"].as_str().unwrap();
-                if text == "invalid result arity"
-                    || text == "multiple memories"
-                    || text == "multiple tables"
-                {
+                // The multiple-memory feature is on (from wasmparser::WasmFeatures::default()).
+                // In imports.wast and memory.wast, following cases will be parsed which should not.
+                if proposal.is_none() && command["text"] == "multiple memories" {
                     continue;
                 }
+                // In binary.wast, following cases will be parsed which should not.
+                if proposal.is_none() && command["text"] == "zero byte expected" {
+                    continue;
+                }
+
                 let wasm = fs::read(&path)?;
-                println!("{:?}", command);
                 if config.parse(&wasm).is_ok() {
-                    // A few spec tests assume multi-value isn't implemented,
-                    // but we implement it, so basically just skip those tests.
-                    let message = command["text"].as_str().unwrap();
-                    if message.contains("invalid result arity") {
-                        continue;
-                    }
-
-                    // MVP wasm considers this tests to fail, but
-                    // reference-types-enhanced wasm considers this test to
-                    // pass. We implement the reference-types semantics, so
-                    // let's go forward with that.
-                    if wast.ends_with("unreached-invalid.wast") && line == 539 {
-                        continue;
-                    }
-                    panic!("wasm parsed when it shouldn't (line {})", line);
+                    should_not_parse.push(line);
                 }
             }
-            "assert_unlinkable" if wast.file_name() == Some("elem.wast".as_ref()) => {
-                // The `elem.wast` file has some unlinkable modules which place
-                // table elements at massive (aka negative) offsets. Our
-                // representation means that we try to allocate a massive amount
-                // of space for null elements. For now we skip these tests as an
-                // implementation detail. This is arguably a bug on our end
-                // where we should improve our representation to not allocate so
-                // much, but that's another bug for another day.
-            }
             cmd => {
-                let wasm = fs::read(&path)?;
-                let mut wasm = config
-                    .parse(&wasm)
+                // The bytes read from the original spec test case
+                let bytes0 = fs::read(&path)?;
+                // The module parsed from bytes0
+                let mut wasm1 = config
+                    .parse(&bytes0)
                     .with_context(|| format!("error parsing wasm (line {})", line))?;
-                let wasm1 = wasm.emit_wasm();
-                fs::write(&path, &wasm1)?;
-                let wasm2 = config
-                    .parse(&wasm1)
-                    .map(|mut m| m.emit_wasm())
+                // The bytes emitted from wasm1
+                let bytes1 = wasm1.emit_wasm();
+                fs::write(&path, &bytes1)?;
+                // The module parsed from bytes1
+                let mut wasm2 = config
+                    .parse(&bytes1)
                     .with_context(|| format!("error re-parsing wasm (line {})", line))?;
-                if wasm1 != wasm2 {
-                    panic!("wasm module at line {} isn't deterministic", line);
+                // The bytes emitted from wasm2
+                let bytes2 = wasm2.emit_wasm();
+
+                if bytes1 != bytes2 {
+                    non_deterministic.push(line);
                 }
                 files.push((cmd.to_string(), path.to_path_buf()));
                 continue;
             }
         }
     }
 
+    let mut message = String::new();
+    if !should_not_parse.is_empty() {
+        message.push_str(&format!(
+            "wasm parsed when it shouldn't at line: {:?}",
+            should_not_parse
+        ));
+    }
+    if !non_deterministic.is_empty() {
+        message.push_str(&format!(
+            "wasm isn't deterministic at line: {:?}",
+            non_deterministic
+        ));
+    }
+    if !message.is_empty() {
+        panic!("{}", message);
+    }
+
     // If wabt didn't succeed before we ran walrus there's no hope of it passing
     // after we run walrus.
     if !wabt_ok {

src/init_expr.rs → src/const_expr.rs

@@ -3,37 +3,51 @@
 use crate::emit::EmitContext;
 use crate::ir::Value;
 use crate::parse::IndicesToIds;
-use crate::ValType;
+use crate::RefType;
 use crate::{FunctionId, GlobalId, Result};
 use anyhow::bail;
 
 /// A constant which is produced in WebAssembly, typically used in global
 /// initializers or element/data offsets.
 #[derive(Debug, Copy, Clone)]
-pub enum InitExpr {
+pub enum ConstExpr {
     /// An immediate constant value
     Value(Value),
     /// A constant value referenced by the global specified
     Global(GlobalId),
     /// A null reference
-    RefNull(ValType),
+    RefNull(RefType),
     /// A function initializer
     RefFunc(FunctionId),
 }
 
-impl InitExpr {
-    pub(crate) fn eval(init: &wasmparser::InitExpr, ids: &IndicesToIds) -> Result<InitExpr> {
+impl ConstExpr {
+    pub(crate) fn eval(init: &wasmparser::ConstExpr, ids: &IndicesToIds) -> Result<ConstExpr> {
         use wasmparser::Operator::*;
         let mut reader = init.get_operators_reader();
         let val = match reader.read()? {
-            I32Const { value } => InitExpr::Value(Value::I32(value)),
-            I64Const { value } => InitExpr::Value(Value::I64(value)),
-            F32Const { value } => InitExpr::Value(Value::F32(f32::from_bits(value.bits()))),
-            F64Const { value } => InitExpr::Value(Value::F64(f64::from_bits(value.bits()))),
-            V128Const { value } => InitExpr::Value(Value::V128(v128_to_u128(&value))),
-            GlobalGet { global_index } => InitExpr::Global(ids.get_global(global_index)?),
-            RefNull { ty } => InitExpr::RefNull(ValType::parse(&ty)?),
-            RefFunc { function_index } => InitExpr::RefFunc(ids.get_func(function_index)?),
+            I32Const { value } => ConstExpr::Value(Value::I32(value)),
+            I64Const { value } => ConstExpr::Value(Value::I64(value)),
+            F32Const { value } => ConstExpr::Value(Value::F32(f32::from_bits(value.bits()))),
+            F64Const { value } => ConstExpr::Value(Value::F64(f64::from_bits(value.bits()))),
+            V128Const { value } => ConstExpr::Value(Value::V128(v128_to_u128(&value))),
+            GlobalGet { global_index } => ConstExpr::Global(ids.get_global(global_index)?),
+            RefNull { hty } => {
+                let val_type = match hty {
+                    wasmparser::HeapType::Abstract { shared: _, ty } => match ty {
+                        wasmparser::AbstractHeapType::Func => RefType::Funcref,
+                        wasmparser::AbstractHeapType::Extern => RefType::Externref,
+                        other => bail!(
+                            "unsupported abstract heap type in constant expression: {other:?}"
+                        ),
+                    },
+                    wasmparser::HeapType::Concrete(_) => {
+                        bail!("unsupported concrete heap type in constant expression")
+                    }
+                };
+                ConstExpr::RefNull(val_type)
+            }
+            RefFunc { function_index } => ConstExpr::RefFunc(ids.get_func(function_index)?),
             _ => bail!("invalid constant expression"),
         };
         match reader.read()? {
@@ -46,22 +60,27 @@ impl InitExpr {
 
     pub(crate) fn to_wasmencoder_type(&self, cx: &EmitContext) -> wasm_encoder::ConstExpr {
         match self {
-            InitExpr::Value(v) => match v {
+            ConstExpr::Value(v) => match v {
                 Value::I32(v) => wasm_encoder::ConstExpr::i32_const(*v),
                 Value::I64(v) => wasm_encoder::ConstExpr::i64_const(*v),
                 Value::F32(v) => wasm_encoder::ConstExpr::f32_const(*v),
                 Value::F64(v) => wasm_encoder::ConstExpr::f64_const(*v),
                 Value::V128(v) => wasm_encoder::ConstExpr::v128_const(*v as i128),
             },
-            InitExpr::Global(g) => {
+            ConstExpr::Global(g) => {
                 wasm_encoder::ConstExpr::global_get(cx.indices.get_global_index(*g))
             }
-            InitExpr::RefNull(ty) => wasm_encoder::ConstExpr::ref_null(match ty {
-                ValType::Externref => wasm_encoder::HeapType::Extern,
-                ValType::Funcref => wasm_encoder::HeapType::Func,
-                _ => unreachable!(),
+            ConstExpr::RefNull(ty) => wasm_encoder::ConstExpr::ref_null(match ty {
+                RefType::Externref => wasm_encoder::HeapType::Abstract {
+                    shared: false,
+                    ty: wasm_encoder::AbstractHeapType::Extern,
+                },
+                RefType::Funcref => wasm_encoder::HeapType::Abstract {
+                    shared: false,
+                    ty: wasm_encoder::AbstractHeapType::Func,
+                },
             }),
-            InitExpr::RefFunc(f) => {
+            ConstExpr::RefFunc(f) => {
                 wasm_encoder::ConstExpr::ref_func(cx.indices.get_func_index(*f))
             }
         }