diff --git a/crates/polkavm-assembler/src/amd64.rs b/crates/polkavm-assembler/src/amd64.rs index ba4831db..01a52bd0 100644 --- a/crates/polkavm-assembler/src/amd64.rs +++ b/crates/polkavm-assembler/src/amd64.rs @@ -573,16 +573,16 @@ impl Inst { self.modrm |= base.modrm_rm_bits(); - if offset != 0 || matches!(base, Reg::rbp | Reg::r13) { - if offset <= i8::MAX as i32 && offset >= i8::MIN as i32 { - self.modrm |= 0b01000000; - self.displacement = offset as u8 as u32; - self.displacement_length = 8; - } else { - self.modrm |= 0b10000000; - self.displacement = offset as u32; - self.displacement_length = 32; - } + let set_displacement = (offset != 0) | matches!(base, Reg::rbp | Reg::r13); + let set_displacement_mask = (-(set_displacement as i32)) as u32; + if offset <= i8::MAX as i32 && offset >= i8::MIN as i32 { + self.modrm |= 0b01000000 & set_displacement_mask as u8; + self.displacement = (offset as u8 as u32) & set_displacement_mask; + self.displacement_length = 8 & set_displacement_mask; + } else { + self.modrm |= 0b10000000 & set_displacement_mask as u8; + self.displacement = (offset as u32) & set_displacement_mask; + self.displacement_length = 32 & set_displacement_mask; } self.override_segment = segment; @@ -602,16 +602,16 @@ impl Inst { self.sib |= base.modrm_rm_bits(); self.sib |= ((scale as usize) << 6) as u8; - if offset != 0 || matches!(base, Reg::rbp | Reg::r13) { - if offset <= i8::MAX as i32 && offset >= i8::MIN as i32 { - self.modrm |= 0b01000000; - self.displacement = offset as u8 as u32; - self.displacement_length = 8; - } else { - self.modrm |= 0b10000000; - self.displacement = offset as u32; - self.displacement_length = 32; - } + let set_displacement = (offset != 0) | matches!(base, Reg::rbp | Reg::r13); + let set_displacement_mask = (-(set_displacement as i32)) as u32; + if offset <= i8::MAX as i32 && offset >= i8::MIN as i32 { + self.modrm |= 0b01000000 & set_displacement_mask as u8; + self.displacement = (offset as u8 as u32) & set_displacement_mask; + self.displacement_length = 8 & set_displacement_mask; + } else { + self.modrm |= 0b10000000 & set_displacement_mask as u8; + self.displacement = (offset as u32) & set_displacement_mask; + self.displacement_length = 32 & set_displacement_mask; } self.override_segment = segment; diff --git a/crates/polkavm-assembler/src/assembler.rs b/crates/polkavm-assembler/src/assembler.rs index 4919eabf..209e5fd7 100644 --- a/crates/polkavm-assembler/src/assembler.rs +++ b/crates/polkavm-assembler/src/assembler.rs @@ -48,6 +48,107 @@ impl<'a> Drop for AssembledCode<'a> { } } +/// # Safety +/// +/// `VALUE` must be non-zero, and `Self::Next::VALUE` must be `VALUE - 1` if `Self::Next` is `NonZero`. +pub unsafe trait NonZero { + const VALUE: usize; + type Next; +} + +pub struct U0; + +macro_rules! impl_non_zero { + ($(($name:ident = $value:expr, $next:ident))*) => { + $( + pub struct $name; + + const _: () = { + assert!($value != 0); + }; + + /// SAFETY: `VALUE` is non-zero. + unsafe impl NonZero for $name { + const VALUE: usize = $value; + type Next = $next; + } + )* + } +} + +impl_non_zero! { + (U1 = 1, U0) + (U2 = 2, U1) + (U3 = 3, U2) + (U4 = 4, U3) + (U5 = 5, U4) + (U6 = 6, U5) +} + +#[repr(transparent)] +pub struct ReservedAssembler<'a, R>(&'a mut Assembler, core::marker::PhantomData); + +impl<'a> ReservedAssembler<'a, U0> { + #[allow(clippy::unused_self)] + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn assert_reserved_exactly_as_needed(self) {} +} + +impl<'a, R> ReservedAssembler<'a, R> { + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn push(self, instruction: Instruction) -> ReservedAssembler<'a, R::Next> + where + R: NonZero, + T: core::fmt::Display, + { + // SAFETY: `R: NonZero`, so we still have space in the buffer. + unsafe { + self.0.push_unchecked(instruction); + } + + ReservedAssembler(self.0, core::marker::PhantomData) + } + + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn push_if(self, condition: bool, instruction: Instruction) -> ReservedAssembler<'a, R::Next> + where + R: NonZero, + T: core::fmt::Display, + { + if condition { + // SAFETY: `R: NonZero`, so we still have space in the buffer. + unsafe { + self.0.push_unchecked(instruction); + } + } + + ReservedAssembler(self.0, core::marker::PhantomData) + } + + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn push_none(self) -> ReservedAssembler<'a, R::Next> + where + R: NonZero, + { + ReservedAssembler(self.0, core::marker::PhantomData) + } + + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn get_label_origin_offset(&self, label: Label) -> Option { + self.0.get_label_origin_offset(label) + } + + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn len(&self) -> usize { + self.0.len() + } + + #[cfg_attr(not(debug_assertions), inline(always))] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + impl Assembler { pub const fn new() -> Self { Assembler { @@ -140,9 +241,14 @@ impl Assembler { } #[inline(always)] - pub fn reserve(&mut self) { - InstBuf::reserve::(&mut self.code); - self.guaranteed_capacity = INSTRUCTIONS; + pub fn reserve(&mut self) -> ReservedAssembler + where + T: NonZero, + { + InstBuf::reserve(&mut self.code, T::VALUE); + + self.guaranteed_capacity = T::VALUE; + ReservedAssembler(self, core::marker::PhantomData) } #[cfg_attr(not(debug_assertions), inline(always))] @@ -150,17 +256,28 @@ impl Assembler { where T: core::fmt::Display, { - #[cfg(debug_assertions)] - log::trace!("{:08x}: {}", self.origin + self.code.len() as u64, instruction); - if self.guaranteed_capacity == 0 { - InstBuf::reserve::<1>(&mut self.code); + InstBuf::reserve_const::<1>(&mut self.code); self.guaranteed_capacity = 1; } + // SAFETY: We've reserved space for at least one instruction. + unsafe { self.push_unchecked(instruction) } + } + + // SAFETY: The buffer *must* have space for at least one instruction. + #[cfg_attr(not(debug_assertions), inline(always))] + unsafe fn push_unchecked(&mut self, instruction: Instruction) -> &mut Self + where + T: core::fmt::Display, + { + #[cfg(debug_assertions)] + log::trace!("{:08x}: {}", self.origin + self.code.len() as u64, instruction); + + debug_assert!(self.guaranteed_capacity > 0); let instruction_offset = self.code.len(); - // SAFETY: We've reserved space for at least one instruction. + // SAFETY: The caller reserved space for at least one instruction. unsafe { instruction.bytes.encode_into_vec_unsafe(&mut self.code); } @@ -404,8 +521,13 @@ impl InstBuf { } #[inline(always)] - fn reserve(output: &mut Vec) { - let count = INSTRUCTIONS.checked_mul(MAXIMUM_INSTRUCTION_SIZE).unwrap(); + fn reserve_const(output: &mut Vec) { + Self::reserve(output, INSTRUCTIONS); + } + + #[inline(always)] + fn reserve(output: &mut Vec, count: usize) { + let count = count.checked_mul(MAXIMUM_INSTRUCTION_SIZE).unwrap(); if output.spare_capacity_mut().len() < count { Self::reserve_impl(output, count); if output.spare_capacity_mut().len() < count { diff --git a/crates/polkavm-assembler/src/lib.rs b/crates/polkavm-assembler/src/lib.rs index ff87dbc8..72034215 100644 --- a/crates/polkavm-assembler/src/lib.rs +++ b/crates/polkavm-assembler/src/lib.rs @@ -8,4 +8,4 @@ mod assembler; extern crate alloc; -pub use crate::assembler::{Assembler, Instruction, Label}; +pub use crate::assembler::{Assembler, Instruction, Label, NonZero, ReservedAssembler, U0, U1, U2, U3, U4, U5, U6}; diff --git a/crates/polkavm-common/src/program.rs b/crates/polkavm-common/src/program.rs index c23647e9..84835e11 100644 --- a/crates/polkavm-common/src/program.rs +++ b/crates/polkavm-common/src/program.rs @@ -1,8 +1,69 @@ use crate::abi::{VM_CODE_ADDRESS_ALIGNMENT, VM_MAXIMUM_CODE_SIZE, VM_MAXIMUM_IMPORT_COUNT, VM_MAXIMUM_JUMP_TABLE_ENTRIES}; use crate::utils::CowBytes; -use crate::varint::{read_simple_varint_fast, read_varint, read_varint_fast, write_simple_varint, write_varint, MAX_VARINT_LENGTH}; +use crate::varint::{read_simple_varint, read_simple_varint_length_minus_1, read_varint, write_simple_varint, MAX_VARINT_LENGTH}; use core::ops::Range; +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct RawReg(u32); + +impl Eq for RawReg {} +impl PartialEq for RawReg { + fn eq(&self, rhs: &Self) -> bool { + self.get() == rhs.get() + } +} + +impl RawReg { + #[inline] + pub const fn get(self) -> Reg { + let mut value = self.0 & 0b1111; + if value > 12 { + value = 12; + } + + match value { + 0 => Reg::RA, + 1 => Reg::SP, + 2 => Reg::T0, + 3 => Reg::T1, + 4 => Reg::T2, + 5 => Reg::S0, + 6 => Reg::S1, + 7 => Reg::A0, + 8 => Reg::A1, + 9 => Reg::A2, + 10 => Reg::A3, + 11 => Reg::A4, + 12 => Reg::A5, + _ => unreachable!(), + } + } + + #[inline] + pub const fn raw_unparsed(self) -> u32 { + self.0 + } +} + +impl From for RawReg { + fn from(reg: Reg) -> Self { + Self(reg as u32) + } +} + +impl core::fmt::Debug for RawReg { + fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(fmt, "{} (0x{:x})", self.get(), self.0) + } +} + +impl core::fmt::Display for RawReg { + fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { + self.get().fmt(fmt) + } +} + #[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)] #[repr(u32)] pub enum Reg { @@ -23,23 +84,8 @@ pub enum Reg { impl Reg { #[inline] - pub const fn from_u8(value: u8) -> Option { - match value { - 0 => Some(Reg::RA), - 1 => Some(Reg::SP), - 2 => Some(Reg::T0), - 3 => Some(Reg::T1), - 4 => Some(Reg::T2), - 5 => Some(Reg::S0), - 6 => Some(Reg::S1), - 7 => Some(Reg::A0), - 8 => Some(Reg::A1), - 9 => Some(Reg::A2), - 10 => Some(Reg::A3), - 11 => Some(Reg::A4), - 12 => Some(Reg::A5), - _ => None, - } + pub const fn raw(self) -> RawReg { + RawReg(self as u32) } pub const fn name(self) -> &'static str { @@ -83,48 +129,7 @@ impl core::fmt::Display for Reg { #[allow(clippy::partial_pub_fields)] #[doc(hidden)] pub struct VisitorHelper { - chunk: u128, pub visitor: T, - args_length: usize, - instruction_offset: u32, - opcode: u8, -} - -macro_rules! skip { - ($chunk:expr, $count:expr) => { - $chunk >> ($count << 3) - }; -} - -macro_rules! read_reg { - ($chunk:ident) => {{ - let reg = Reg::from_u8($chunk as u8)?; - (reg, skip!($chunk, 1)) - }}; -} - -macro_rules! read_reg2 { - ($chunk:ident) => {{ - let value = $chunk as u8; - let reg1 = Reg::from_u8(value & 0b1111)?; - let reg2 = Reg::from_u8(value >> 4)?; - (reg1, reg2, skip!($chunk, 1)) - }}; -} - -macro_rules! read_simple_varint { - ($chunk:expr, $length:expr) => {{ - let imm = read_simple_varint_fast($chunk as u32, $length as u32); - (imm, skip!($chunk, $length)) - }}; -} - -macro_rules! read_varint { - ($chunk:ident) => {{ - let (imm_length, imm) = read_varint_fast($chunk as u64)?; - let imm_length = imm_length as usize; - (imm, skip!($chunk, imm_length), imm_length) - }}; } impl VisitorHelper { @@ -134,28 +139,28 @@ impl VisitorHelper { fn step_slow( &mut self, code: &[u8], + bitmask: &[u8], instruction_offset: usize, - instruction_length: usize, - decode_table: &[fn(state: &mut Self) -> ::ReturnTy], - ) -> Option<::ReturnTy> + decode_table: &[fn(state: &mut Self, chunk: u128, instruction_offset: u32, args_length: u32) -> ::ReturnTy], + ) -> Option<(usize, ::ReturnTy)> where T: ParsingVisitor, { - let chunk = code.get(instruction_offset..instruction_offset + instruction_length)?; + let (next_offset, args_length) = parse_bitmask_slow(bitmask, instruction_offset)?; + let chunk_length = core::cmp::min(16, args_length + 1); + let chunk = code.get(instruction_offset..instruction_offset + chunk_length)?; let opcode = chunk[0]; - let args = &chunk[1..instruction_length]; + let mut t: [u8; 16] = [0; 16]; - t[..instruction_length].copy_from_slice(chunk); - self.chunk = u128::from_le_bytes([ + t[..chunk_length].copy_from_slice(chunk); + let chunk = u128::from_le_bytes([ t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], t[10], t[11], t[12], t[13], t[14], t[15], ]) >> 8; - self.opcode = opcode; - self.args_length = args.len(); - self.instruction_offset = instruction_offset as u32; - - self.visitor.on_pre_visit(instruction_offset, instruction_length, opcode); - Some(decode_table[opcode as usize](self)) + Some(( + next_offset, + decode_table[opcode as usize](self, chunk, instruction_offset as u32, args_length as u32), + )) } #[allow(clippy::type_complexity)] @@ -164,51 +169,46 @@ impl VisitorHelper { &mut self, code: &[u8], bitmask: &[u8], - offset: &mut usize, - decode_table: &[fn(state: &mut Self) -> ::ReturnTy], - ) -> Option<::ReturnTy> + instruction_offset: usize, + decode_table: &[fn(state: &mut Self, chunk: u128, instruction_offset: u32, args_length: u32) -> ::ReturnTy], + ) -> Option<(usize, ::ReturnTy)> where T: ParsingVisitor, { - let instruction_offset = *offset; - let args_length = parse_bitmask(bitmask, offset)?; - let instruction_length = args_length + 1; - - let padded_length = core::cmp::max(instruction_length, 16); - let Some(chunk) = code.get(instruction_offset..instruction_offset + padded_length) else { - return self.step_slow(code, instruction_offset, instruction_length, decode_table); - }; - - assert!(chunk.len() >= 16); - let opcode = chunk[0]; - - // NOTE: This should produce the same assembly as the unsafe `read_unaligned`. - self.chunk = u128::from_le_bytes([ - chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], - chunk[12], chunk[13], chunk[14], chunk[15], - ]) >> 8; - - self.opcode = opcode; - self.args_length = instruction_length - 1; - self.instruction_offset = instruction_offset as u32; + if let Some((next_offset, args_length)) = parse_bitmask_fast(bitmask, instruction_offset) { + debug_assert!(args_length <= 31); + if let Some(chunk) = code.get(instruction_offset..instruction_offset + 32) { + assert!(chunk.len() >= 32); + let opcode = chunk[0]; + + // NOTE: This should produce the same assembly as the unsafe `read_unaligned`. + let chunk = u128::from_le_bytes([ + chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], chunk[8], chunk[9], chunk[10], chunk[11], + chunk[12], chunk[13], chunk[14], chunk[15], chunk[16], + ]); + + return Some(( + next_offset, + decode_table[opcode as usize](self, chunk, instruction_offset as u32, args_length as u32), + )); + } + } - self.visitor.on_pre_visit(instruction_offset, instruction_length, opcode); - Some(decode_table[opcode as usize](self)) + self.step_slow(code, bitmask, instruction_offset, decode_table) } #[inline] pub fn new(visitor: T) -> Self { - VisitorHelper { - chunk: 0, - visitor, - args_length: 0, - instruction_offset: 0, - opcode: 0, - } + VisitorHelper { visitor } } + #[allow(clippy::type_complexity)] #[inline] - pub fn run(mut self, blob: &ProgramBlob, decode_table: &[fn(&mut Self) -> ::ReturnTy; 256]) -> T + pub fn run( + mut self, + blob: &ProgramBlob, + decode_table: &[fn(&mut Self, chunk: u128, instruction_offset: u32, args_length: u32) -> ::ReturnTy; 256], + ) -> T where T: ParsingVisitor, { @@ -217,100 +217,127 @@ impl VisitorHelper { debug_assert_eq!(bitmask[0] & 0b1, 1); let mut offset = 0; - while self.step(code, bitmask, &mut offset, decode_table).is_some() {} - self.visitor - } + loop { + let Some((next_offset, ())) = self.step(code, bitmask, offset, decode_table) else { + break; + }; + offset = next_offset; + } - #[inline] - pub fn opcode(&self) -> u8 { - self.opcode + self.visitor } #[inline(always)] - pub fn read_args_offset(&mut self) -> Option { - let (imm, _) = read_simple_varint!(self.chunk, self.args_length); - Some(self.instruction_offset.wrapping_add(imm)) + pub fn read_args_offset(&mut self, chunk: u128, instruction_offset: u32, args_length: u32) -> Option { + let imm = read_simple_varint(chunk as u32, args_length); + Some(instruction_offset.wrapping_add(imm)) } #[inline(always)] - pub fn read_args_imm(&mut self) -> Option { - let (imm, _) = read_simple_varint!(self.chunk, self.args_length); + pub fn read_args_imm(&mut self, chunk: u128, args_length: u32) -> Option { + let imm = read_simple_varint(chunk as u32, args_length); Some(imm) } #[inline(always)] - pub fn read_args_imm2(&mut self) -> Option<(u32, u32)> { - let chunk = self.chunk; - let (imm1, chunk, imm1_length) = read_varint!(chunk); - let (imm2, _) = read_simple_varint!(chunk, self.args_length - imm1_length); + pub fn read_args_imm2(&mut self, chunk: u128, args_length: u32) -> Option<(u32, u32)> { + let imm1_length = u32::from(chunk as u8); + let chunk = chunk >> 8; + let chunk = chunk as u64; + let imm1 = read_simple_varint(chunk as u32, imm1_length); + let chunk = chunk >> (imm1_length << 3); + let imm2 = read_simple_varint_length_minus_1(chunk as u32, args_length.wrapping_sub(imm1_length)); Some((imm1, imm2)) } #[inline(always)] - pub fn read_args_reg_imm(&mut self) -> Option<(Reg, u32)> { - let chunk = self.chunk; - let (reg, chunk) = read_reg!(chunk); - let (imm, _) = read_simple_varint!(chunk, self.args_length - 1); + pub fn read_args_reg_imm(&mut self, chunk: u128, args_length: u32) -> Option<(RawReg, u32)> { + let chunk = chunk as u64; + let reg = RawReg(chunk as u32); + let chunk = chunk >> 8; + let imm = read_simple_varint_length_minus_1(chunk as u32, args_length); Some((reg, imm)) } #[inline(always)] - pub fn read_args_reg_imm_offset(&mut self) -> Option<(Reg, u32, u32)> { - let chunk = self.chunk; - let (reg, chunk) = read_reg!(chunk); - let (imm1, chunk, imm1_length) = read_varint!(chunk); - let (imm2, _) = read_simple_varint!(chunk, self.args_length - 1 - imm1_length); - let imm2 = self.instruction_offset.wrapping_add(imm2); + pub fn read_args_reg_imm_offset(&mut self, chunk: u128, instruction_offset: u32, args_length: u32) -> Option<(RawReg, u32, u32)> { + let (reg, imm1_length) = { + let value = chunk as u32; + (RawReg(value), (value >> 4) & 0b1111) + }; + let chunk = chunk >> 8; + let chunk = chunk as u64; + let imm1 = read_simple_varint(chunk as u32, imm1_length); + let chunk = chunk >> (imm1_length << 3); + let imm2 = read_simple_varint_length_minus_1(chunk as u32, args_length.wrapping_sub(imm1_length)); + let imm2 = instruction_offset.wrapping_add(imm2); Some((reg, imm1, imm2)) } #[inline(always)] - pub fn read_args_reg_imm2(&mut self) -> Option<(Reg, u32, u32)> { - let chunk = self.chunk; - let (reg, chunk) = read_reg!(chunk); - let (imm1, chunk, imm1_length) = read_varint!(chunk); - let (imm2, _) = read_simple_varint!(chunk, self.args_length - 1 - imm1_length); + pub fn read_args_reg_imm2(&mut self, chunk: u128, args_length: u32) -> Option<(RawReg, u32, u32)> { + let (reg, imm1_length) = { + let value = chunk as u32; + (RawReg(value), (value >> 4) & 0b1111) + }; + let chunk = chunk >> 8; + let chunk = chunk as u64; + let imm1 = read_simple_varint(chunk as u32, imm1_length); + let chunk = chunk >> (imm1_length << 3); + let imm2 = read_simple_varint_length_minus_1(chunk as u32, args_length.wrapping_sub(imm1_length)); Some((reg, imm1, imm2)) } #[inline(always)] - pub fn read_args_regs2_imm2(&mut self) -> Option<(Reg, Reg, u32, u32)> { - let chunk = self.chunk; - let (reg1, reg2, chunk) = read_reg2!(chunk); - let (imm1, chunk, imm1_length) = read_varint!(chunk); - let (imm2, _) = read_simple_varint!(chunk, self.args_length - 1 - imm1_length); + pub fn read_args_regs2_imm2(&mut self, chunk: u128, args_length: u32) -> Option<(RawReg, RawReg, u32, u32)> { + let (reg1, reg2, imm1_length) = { + let value = chunk as u32; + (RawReg(value), RawReg(value >> 4), u32::from((value >> 8) as u8)) + }; + let chunk = chunk >> 16; + let chunk = chunk as u64; + let imm1 = read_simple_varint(chunk as u32, imm1_length); + let chunk = chunk >> (imm1_length << 3); + let imm2 = read_simple_varint_length_minus_1(chunk as u32, args_length.wrapping_sub(imm1_length + 1)); Some((reg1, reg2, imm1, imm2)) } #[inline(always)] - pub fn read_args_regs2_imm(&mut self) -> Option<(Reg, Reg, u32)> { - let chunk = self.chunk; - let (reg1, reg2, chunk) = read_reg2!(chunk); - let (imm, _) = read_simple_varint!(chunk, self.args_length - 1); + pub fn read_args_regs2_imm(&mut self, chunk: u128, args_length: u32) -> Option<(RawReg, RawReg, u32)> { + let chunk = chunk as u64; + let (reg1, reg2) = { + let value = chunk as u32; + (RawReg(value), RawReg(value >> 4)) + }; + let chunk = chunk >> 8; + let imm = read_simple_varint_length_minus_1(chunk as u32, args_length); Some((reg1, reg2, imm)) } #[inline(always)] - pub fn read_args_regs2_offset(&mut self) -> Option<(Reg, Reg, u32)> { - let chunk = self.chunk; - let (reg1, reg2, chunk) = read_reg2!(chunk); - let (imm, _) = read_simple_varint!(chunk, self.args_length - 1); - let imm = self.instruction_offset.wrapping_add(imm); + pub fn read_args_regs2_offset(&mut self, chunk: u128, instruction_offset: u32, args_length: u32) -> Option<(RawReg, RawReg, u32)> { + let chunk = chunk as u64; + let (reg1, reg2) = { + let value = chunk as u32; + (RawReg(value), RawReg(value >> 4)) + }; + let chunk = chunk >> 8; + let imm = read_simple_varint_length_minus_1(chunk as u32, args_length); + let imm = instruction_offset.wrapping_add(imm); Some((reg1, reg2, imm)) } #[inline(always)] - pub fn read_args_regs3(&mut self) -> Option<(Reg, Reg, Reg)> { - let chunk = self.chunk; - let (reg1, reg2, chunk) = read_reg2!(chunk); - let (reg3, _) = read_reg!(chunk); + pub fn read_args_regs3(&mut self, chunk: u128) -> Option<(RawReg, RawReg, RawReg)> { + let chunk = chunk as u32; + let (reg1, reg2, reg3) = (RawReg(chunk), RawReg(chunk >> 4), RawReg(chunk >> 8)); Some((reg1, reg2, reg3)) } #[inline(always)] - pub fn read_args_regs2(&mut self) -> Option<(Reg, Reg)> { - let chunk = self.chunk; - let (reg1, reg2, _) = read_reg2!(chunk); + pub fn read_args_regs2(&mut self, chunk: u128) -> Option<(RawReg, RawReg)> { + let chunk = chunk as u32; + let (reg1, reg2) = (RawReg(chunk), RawReg(chunk >> 4)); Some((reg1, reg2)) } } @@ -386,98 +413,69 @@ macro_rules! define_opcodes { [$($name_reg_reg:ident = $value_reg_reg:expr,)+] [$($name_reg_reg_imm_imm:ident = $value_reg_reg_imm_imm:expr,)+] ) => { - pub trait ParsingVisitor: InstructionVisitor { - fn on_pre_visit(&mut self, _offset: usize, _length: usize, _opcode: u8); + pub trait ParsingVisitor { + type ReturnTy; + + $(fn $name_argless(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_imm_offset(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_offset(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+ + $(fn $name_offset(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_imm(&mut self, offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_imm_imm(&mut self, offset: u32, args_length: u32, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_imm_imm(&mut self, offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + + #[inline(never)] + #[cold] + fn invalid(&mut self, offset: u32, args_length: u32) -> Self::ReturnTy { + self.trap(offset, args_length) + } } pub trait InstructionVisitor { type ReturnTy; $(fn $name_argless(&mut self) -> Self::ReturnTy;)+ - $(fn $name_reg_imm(&mut self, reg: Reg, imm: u32) -> Self::ReturnTy;)+ - $(fn $name_reg_imm_offset(&mut self, reg: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ - $(fn $name_reg_imm_imm(&mut self, reg: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ - $(fn $name_reg_reg_imm(&mut self, reg1: Reg, reg2: Reg, imm: u32) -> Self::ReturnTy;)+ - $(fn $name_reg_reg_offset(&mut self, reg1: Reg, reg2: Reg, imm: u32) -> Self::ReturnTy;)+ - $(fn $name_reg_reg_reg(&mut self, reg1: Reg, reg2: Reg, reg3: Reg) -> Self::ReturnTy;)+ + $(fn $name_reg_imm(&mut self, reg: RawReg, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_imm_offset(&mut self, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_imm_imm(&mut self, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_imm(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_offset(&mut self, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_reg(&mut self, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy;)+ $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy;)+ $(fn $name_imm(&mut self, imm: u32) -> Self::ReturnTy;)+ $(fn $name_imm_imm(&mut self, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ - $(fn $name_reg_reg(&mut self, reg1: Reg, reg2: Reg) -> Self::ReturnTy;)+ - $(fn $name_reg_reg_imm_imm(&mut self, reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ + $(fn $name_reg_reg(&mut self, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy;)+ + $(fn $name_reg_reg_imm_imm(&mut self, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy;)+ #[inline(never)] #[cold] - fn invalid(&mut self, _opcode: u8) -> Self::ReturnTy { + fn invalid(&mut self) -> Self::ReturnTy { self.trap() } } - #[macro_export] - macro_rules! implement_instruction_visitor { - (impl<$d($visitor_ty_params:tt),*> $visitor_ty:ty, $method:ident) => { - impl<$d($visitor_ty_params),*> polkavm_common::program::InstructionVisitor for $visitor_ty { - type ReturnTy = (); - - $(fn $name_argless(&mut self) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_argless); - })+ - $(fn $name_reg_imm(&mut self, reg: Reg, imm: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_imm(reg, imm)); - })+ - $(fn $name_reg_imm_offset(&mut self, reg: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_imm_offset(reg, imm1, imm2)); - })+ - $(fn $name_reg_imm_imm(&mut self, reg: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_imm_imm(reg, imm1, imm2)); - })+ - $(fn $name_reg_reg_imm(&mut self, reg1: Reg, reg2: Reg, imm: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_reg_imm(reg1, reg2, imm)); - })+ - $(fn $name_reg_reg_offset(&mut self, reg1: Reg, reg2: Reg, imm: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_reg_offset(reg1, reg2, imm)); - })+ - $(fn $name_reg_reg_reg(&mut self, reg1: Reg, reg2: Reg, reg3: Reg) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_reg_reg(reg1, reg2, reg3)); - })+ - $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_offset(imm)); - })+ - $(fn $name_imm(&mut self, imm: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_imm(imm)); - })+ - $(fn $name_imm_imm(&mut self, imm1: u32, imm2: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_imm_imm(imm1, imm2)); - })+ - $(fn $name_reg_reg(&mut self, reg1: Reg, reg2: Reg) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_reg(reg1, reg2)); - })+ - $(fn $name_reg_reg_imm_imm(&mut self, reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy { - self.$method(polkavm_common::program::Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2)); - })+ - } - } - } - - pub use implement_instruction_visitor; - #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[allow(non_camel_case_types)] #[repr(u32)] pub enum Instruction { $($name_argless = $value_argless,)+ - $($name_reg_imm(Reg, u32) = $value_reg_imm,)+ - $($name_reg_imm_offset(Reg, u32, u32) = $value_reg_imm_offset,)+ - $($name_reg_imm_imm(Reg, u32, u32) = $value_reg_imm_imm,)+ - $($name_reg_reg_imm(Reg, Reg, u32) = $value_reg_reg_imm,)+ - $($name_reg_reg_offset(Reg, Reg, u32) = $value_reg_reg_offset,)+ - $($name_reg_reg_reg(Reg, Reg, Reg) = $value_reg_reg_reg,)+ + $($name_reg_imm(RawReg, u32) = $value_reg_imm,)+ + $($name_reg_imm_offset(RawReg, u32, u32) = $value_reg_imm_offset,)+ + $($name_reg_imm_imm(RawReg, u32, u32) = $value_reg_imm_imm,)+ + $($name_reg_reg_imm(RawReg, RawReg, u32) = $value_reg_reg_imm,)+ + $($name_reg_reg_offset(RawReg, RawReg, u32) = $value_reg_reg_offset,)+ + $($name_reg_reg_reg(RawReg, RawReg, RawReg) = $value_reg_reg_reg,)+ $($name_offset(u32) = $value_offset,)+ $($name_imm(u32) = $value_imm,)+ $($name_imm_imm(u32, u32) = $value_imm_imm,)+ - $($name_reg_reg(Reg, Reg) = $value_reg_reg,)+ - $($name_reg_reg_imm_imm(Reg, Reg, u32, u32) = $value_reg_reg_imm_imm,)+ - invalid(u8) = 88, + $($name_reg_reg(RawReg, RawReg) = $value_reg_reg,)+ + $($name_reg_reg_imm_imm(RawReg, RawReg, u32, u32) = $value_reg_reg_imm_imm,)+ + invalid = 88, } impl Instruction { @@ -495,7 +493,7 @@ macro_rules! define_opcodes { $(Self::$name_imm_imm(imm1, imm2) => visitor.$name_imm_imm(imm1, imm2),)+ $(Self::$name_reg_reg(reg1, reg2) => visitor.$name_reg_reg(reg1, reg2),)+ $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => visitor.$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2),)+ - Self::invalid(opcode) => visitor.invalid(opcode), + Self::invalid => visitor.invalid(), } } @@ -513,7 +511,7 @@ macro_rules! define_opcodes { $(Self::$name_imm_imm(imm1, imm2) => Self::serialize_imm_imm(buffer, Opcode::$name_imm_imm, imm1, imm2),)+ $(Self::$name_reg_reg(reg1, reg2) => Self::serialize_reg_reg(buffer, Opcode::$name_reg_reg, reg1, reg2),)+ $(Self::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) => Self::serialize_reg_reg_imm_imm(buffer, Opcode::$name_reg_reg_imm_imm, reg1, reg2, imm1, imm2),)+ - Self::invalid(..) => Self::serialize_argless(buffer, Opcode::trap), + Self::invalid => Self::serialize_argless(buffer, Opcode::trap), } } @@ -532,17 +530,11 @@ macro_rules! define_opcodes { $(Self::$name_imm_imm(..) => Opcode::$name_imm_imm,)+ $(Self::$name_reg_reg(..) => Opcode::$name_reg_reg,)+ $(Self::$name_reg_reg_imm_imm(..) => Opcode::$name_reg_reg_imm_imm,)+ - Self::invalid(..) => Opcode::trap, + Self::invalid => Opcode::trap, } } } - impl core::fmt::Display for Instruction { - fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { - self.visit(fmt) - } - } - pub mod asm { use super::{Instruction, Reg}; @@ -554,37 +546,37 @@ macro_rules! define_opcodes { $( pub fn $name_reg_imm(reg: Reg, imm: u32) -> Instruction { - Instruction::$name_reg_imm(reg, imm) + Instruction::$name_reg_imm(reg.into(), imm) } )+ $( pub fn $name_reg_imm_offset(reg: Reg, imm1: u32, imm2: u32) -> Instruction { - Instruction::$name_reg_imm_offset(reg, imm1, imm2) + Instruction::$name_reg_imm_offset(reg.into(), imm1, imm2) } )+ $( pub fn $name_reg_imm_imm(reg: Reg, imm1: u32, imm2: u32) -> Instruction { - Instruction::$name_reg_imm_imm(reg, imm1, imm2) + Instruction::$name_reg_imm_imm(reg.into(), imm1, imm2) } )+ $( pub fn $name_reg_reg_imm(reg1: Reg, reg2: Reg, imm: u32) -> Instruction { - Instruction::$name_reg_reg_imm(reg1, reg2, imm) + Instruction::$name_reg_reg_imm(reg1.into(), reg2.into(), imm) } )+ $( pub fn $name_reg_reg_offset(reg1: Reg, reg2: Reg, imm: u32) -> Instruction { - Instruction::$name_reg_reg_offset(reg1, reg2, imm) + Instruction::$name_reg_reg_offset(reg1.into(), reg2.into(), imm) } )+ $( pub fn $name_reg_reg_reg(reg1: Reg, reg2: Reg, reg3: Reg) -> Instruction { - Instruction::$name_reg_reg_reg(reg1, reg2, reg3) + Instruction::$name_reg_reg_reg(reg1.into(), reg2.into(), reg3.into()) } )+ @@ -608,13 +600,13 @@ macro_rules! define_opcodes { $( pub fn $name_reg_reg(reg1: Reg, reg2: Reg) -> Instruction { - Instruction::$name_reg_reg(reg1, reg2) + Instruction::$name_reg_reg(reg1.into(), reg2.into()) } )+ $( pub fn $name_reg_reg_imm_imm(reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> Instruction { - Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) + Instruction::$name_reg_reg_imm_imm(reg1.into(), reg2.into(), imm1, imm2) } )+ @@ -627,12 +619,12 @@ macro_rules! define_opcodes { macro_rules! prepare_visitor { (@define_table $table_name:ident, $visitor_ty:ident<$d($visitor_ty_params:tt),*>) => { use $crate::program::{ - InstructionVisitor, + ParsingVisitor, VisitorHelper, }; - type ReturnTy<$d($visitor_ty_params),*> = <$visitor_ty<$d($visitor_ty_params),*> as InstructionVisitor>::ReturnTy; - type VisitFn<$d($visitor_ty_params),*> = fn(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>; + type ReturnTy<$d($visitor_ty_params),*> = <$visitor_ty<$d($visitor_ty_params),*> as ParsingVisitor>::ReturnTy; + type VisitFn<$d($visitor_ty_params),*> = fn(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>; #[allow(unsafe_code)] static $table_name: [VisitFn; 256] = { @@ -643,8 +635,8 @@ macro_rules! define_opcodes { // compiler and the linker to put all of this code near each other, minimizing // instruction cache misses. #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_argless<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - state.visitor.$name_argless() + fn $name_argless<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, _chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + state.visitor.$name_argless(instruction_offset, args_length) } table[$value_argless] = $name_argless; @@ -652,12 +644,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg, imm)) = state.read_args_reg_imm() { - return state.visitor.$name_reg_imm(reg, imm); - }; - - state.visitor.invalid($value_reg_imm) + fn $name_reg_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg, imm)) = state.read_args_reg_imm(chunk, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_imm(instruction_offset, args_length, reg, imm) } table[$value_reg_imm] = $name_reg_imm; @@ -665,12 +654,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_imm_offset<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg, imm1, imm2)) = state.read_args_reg_imm_offset() { - return state.visitor.$name_reg_imm_offset(reg, imm1, imm2); - } - - state.visitor.invalid($value_reg_imm_offset) + fn $name_reg_imm_offset<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg, imm1, imm2)) = state.read_args_reg_imm_offset(chunk, instruction_offset, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_imm_offset(instruction_offset, args_length, reg, imm1, imm2) } table[$value_reg_imm_offset] = $name_reg_imm_offset; @@ -678,12 +664,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg, imm1, imm2)) = state.read_args_reg_imm2() { - return state.visitor.$name_reg_imm_imm(reg, imm1, imm2); - } - - state.visitor.invalid($value_reg_imm_imm) + fn $name_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg, imm1, imm2)) = state.read_args_reg_imm2(chunk, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_imm_imm(instruction_offset, args_length, reg, imm1, imm2) } table[$value_reg_imm_imm] = $name_reg_imm_imm; @@ -691,12 +674,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_reg_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg1, reg2, imm)) = state.read_args_regs2_imm() { - return state.visitor.$name_reg_reg_imm(reg1, reg2, imm); - } - - state.visitor.invalid($value_reg_reg_imm) + fn $name_reg_reg_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg1, reg2, imm)) = state.read_args_regs2_imm(chunk, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_reg_imm(instruction_offset, args_length, reg1, reg2, imm) } table[$value_reg_reg_imm] = $name_reg_reg_imm; @@ -704,12 +684,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_reg_offset<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg1, reg2, imm)) = state.read_args_regs2_offset() { - return state.visitor.$name_reg_reg_offset(reg1, reg2, imm); - } - - state.visitor.invalid($value_reg_reg_offset) + fn $name_reg_reg_offset<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg1, reg2, imm)) = state.read_args_regs2_offset(chunk, instruction_offset, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_reg_offset(instruction_offset, args_length, reg1, reg2, imm) } table[$value_reg_reg_offset] = $name_reg_reg_offset; @@ -717,12 +694,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_reg_reg<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg1, reg2, reg3)) = state.read_args_regs3() { - return state.visitor.$name_reg_reg_reg(reg1, reg2, reg3); - } - - state.visitor.invalid($value_reg_reg_reg) + fn $name_reg_reg_reg<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg1, reg2, reg3)) = state.read_args_regs3(chunk) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_reg_reg(instruction_offset, args_length, reg1, reg2, reg3) } table[$value_reg_reg_reg] = $name_reg_reg_reg; @@ -730,12 +704,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_offset<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some(imm) = state.read_args_offset() { - return state.visitor.$name_offset(imm); - } - - state.visitor.invalid($value_offset) + fn $name_offset<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some(imm) = state.read_args_offset(chunk, instruction_offset, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_offset(instruction_offset, args_length, imm) } table[$value_offset] = $name_offset; @@ -743,12 +714,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some(imm) = state.read_args_imm() { - return state.visitor.$name_imm(imm); - } - - state.visitor.invalid($value_imm) + fn $name_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some(imm) = state.read_args_imm(chunk, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_imm(instruction_offset, args_length, imm) } table[$value_imm] = $name_imm; @@ -756,12 +724,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_imm_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((imm1, imm2)) = state.read_args_imm2() { - return state.visitor.$name_imm_imm(imm1, imm2); - } - - state.visitor.invalid($value_imm_imm) + fn $name_imm_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((imm1, imm2)) = state.read_args_imm2(chunk, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_imm_imm(instruction_offset, args_length, imm1, imm2) } table[$value_imm_imm] = $name_imm_imm; @@ -769,12 +734,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_reg<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg1, reg2)) = state.read_args_regs2() { - return state.visitor.$name_reg_reg(reg1, reg2); - } - - state.visitor.invalid($value_reg_reg) + fn $name_reg_reg<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg1, reg2)) = state.read_args_regs2(chunk) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_reg(instruction_offset, args_length, reg1, reg2) } table[$value_reg_reg] = $name_reg_reg; @@ -782,12 +744,9 @@ macro_rules! define_opcodes { $({ #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] - fn $name_reg_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - if let Some((reg1, reg2, imm1, imm2)) = state.read_args_regs2_imm2() { - return state.visitor.$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2); - } - - state.visitor.invalid($value_reg_reg_imm_imm) + fn $name_reg_reg_imm_imm<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + let Some((reg1, reg2, imm1, imm2)) = state.read_args_regs2_imm2(chunk, args_length) else { return state.visitor.invalid(instruction_offset, args_length) }; + state.visitor.$name_reg_reg_imm_imm(instruction_offset, args_length, reg1, reg2, imm1, imm2) } table[$value_reg_reg_imm_imm] = $name_reg_reg_imm_imm; @@ -795,8 +754,8 @@ macro_rules! define_opcodes { #[cfg_attr(target_os = "linux", link_section = concat!(".text.", stringify!($table_name)))] #[cold] - fn invalid_instruction<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>) -> ReturnTy<$d($visitor_ty_params),*>{ - state.visitor.invalid(state.opcode()) + fn invalid_instruction<$d($visitor_ty_params),*>(state: &mut VisitorHelper<$visitor_ty<$d($visitor_ty_params),*>>, _chunk: u128, instruction_offset: u32, args_length: u32) -> ReturnTy<$d($visitor_ty_params),*>{ + state.visitor.invalid(instruction_offset, args_length) } table @@ -828,75 +787,59 @@ macro_rules! define_opcodes { pub use prepare_visitor; - struct ToEnumVisitor<'a>(core::marker::PhantomData<&'a ()>); - impl<'a> ParsingVisitor for ToEnumVisitor<'a> { - fn on_pre_visit(&mut self, _offset: usize, _length: usize, _opcode: u8) {} - } - - impl<'a> InstructionVisitor for ToEnumVisitor<'a> { type ReturnTy = Instruction; - $(fn $name_argless(&mut self) -> Self::ReturnTy { + $(fn $name_argless(&mut self, _offset: u32, args_length: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_argless })+ - $(fn $name_reg_imm(&mut self, reg: Reg, imm: u32) -> Self::ReturnTy { + $(fn $name_reg_imm(&mut self, _offset: u32, args_length: u32, reg: RawReg, imm: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_imm(reg, imm) })+ - $(fn $name_reg_imm_offset(&mut self, reg: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy { + $(fn $name_reg_imm_offset(&mut self, _offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_imm_offset(reg, imm1, imm2) })+ - $(fn $name_reg_imm_imm(&mut self, reg: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy { + $(fn $name_reg_imm_imm(&mut self, _offset: u32, args_length: u32, reg: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_imm_imm(reg, imm1, imm2) })+ - $(fn $name_reg_reg_imm(&mut self, reg1: Reg, reg2: Reg, imm: u32) -> Self::ReturnTy { + $(fn $name_reg_reg_imm(&mut self, _offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_reg_imm(reg1, reg2, imm) })+ - $(fn $name_reg_reg_offset(&mut self, reg1: Reg, reg2: Reg, imm: u32) -> Self::ReturnTy { + $(fn $name_reg_reg_offset(&mut self, _offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_reg_offset(reg1, reg2, imm) })+ - $(fn $name_reg_reg_reg(&mut self, reg1: Reg, reg2: Reg, reg3: Reg) -> Self::ReturnTy { + $(fn $name_reg_reg_reg(&mut self, _offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_reg_reg(reg1, reg2, reg3) })+ - $(fn $name_offset(&mut self, imm: u32) -> Self::ReturnTy { + $(fn $name_offset(&mut self, _offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_offset(imm) })+ - $(fn $name_imm(&mut self, imm: u32) -> Self::ReturnTy { + $(fn $name_imm(&mut self, _offset: u32, args_length: u32, imm: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_imm(imm) })+ - $(fn $name_imm_imm(&mut self, imm1: u32, imm2: u32) -> Self::ReturnTy { + $(fn $name_imm_imm(&mut self, _offset: u32, args_length: u32, imm1: u32, imm2: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_imm_imm(imm1, imm2) })+ - $(fn $name_reg_reg(&mut self, reg1: Reg, reg2: Reg) -> Self::ReturnTy { + $(fn $name_reg_reg(&mut self, _offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_reg(reg1, reg2) })+ - $(fn $name_reg_reg_imm_imm(&mut self, reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> Self::ReturnTy { + $(fn $name_reg_reg_imm_imm(&mut self, _offset: u32, args_length: u32, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> Self::ReturnTy { + self.args_length = args_length; Instruction::$name_reg_reg_imm_imm(reg1, reg2, imm1, imm2) })+ } - #[inline] - fn parse_instruction(code: &[u8], bitmask: &[u8], offset: &mut usize) -> Option { - prepare_visitor!(@define_table TO_ENUM_VISITOR, ToEnumVisitor<'a>); - - let decode_table: &[VisitFn; 256] = &TO_ENUM_VISITOR; - - #[allow(unsafe_code)] - // SAFETY: Here we transmute the lifetimes which were unnecessarily extended to be 'static due to the table here being a `static`. - let decode_table: &[VisitFn; 256] = unsafe { core::mem::transmute(decode_table) }; - - let mut helper = VisitorHelper::new(ToEnumVisitor(core::marker::PhantomData)); - let origin = *offset; - let instruction = helper.step(code, bitmask, offset, decode_table)?; - let length = helper.args_length + 1; - - Some(ParsedInstruction { - kind: instruction, - offset: origin as u32, - length: length as u32, - }) - } - define_opcodes!( @impl_shared $($name_argless = $value_argless,)+ @@ -915,6 +858,147 @@ macro_rules! define_opcodes { } } +struct ToEnumVisitor<'a> { + args_length: u32, + phantom: core::marker::PhantomData<&'a ()>, +} + +#[inline] +fn parse_instruction(code: &[u8], bitmask: &[u8], offset: &mut usize) -> Option { + prepare_visitor!(@define_table TO_ENUM_VISITOR, ToEnumVisitor<'a>); + + let decode_table: &[VisitFn; 256] = &TO_ENUM_VISITOR; + + #[allow(unsafe_code)] + // SAFETY: Here we transmute the lifetimes which were unnecessarily extended to be 'static due to the table here being a `static`. + let decode_table: &[VisitFn; 256] = unsafe { core::mem::transmute(decode_table) }; + + let mut helper = VisitorHelper::new(ToEnumVisitor { + args_length: 0, + phantom: core::marker::PhantomData, + }); + + let origin = *offset; + let (next_offset, instruction) = helper.step(code, bitmask, origin, decode_table)?; + *offset = next_offset; + + let length = helper.visitor.args_length + 1; + Some(ParsedInstruction { + kind: instruction, + offset: origin as u32, + length, + }) +} + +#[test] +fn test_parse_instruction() { + // Instruction with no arguments. + assert_eq!( + parse_instruction(&[Opcode::fallthrough as u8], &[0b11111111], &mut 0), + Some(ParsedInstruction { + kind: Instruction::fallthrough, + offset: 0, + length: 1 + }) + ); + + // Instruction with no arguments, overparametrized. + assert_eq!( + parse_instruction(&[Opcode::fallthrough as u8, 0xff], &[0b00000101], &mut 0), + Some(ParsedInstruction { + kind: Instruction::fallthrough, + offset: 0, + length: 2 + }) + ); + + // Instruction with no arguments, overparametrized, truncated code. + assert_eq!(parse_instruction(&[Opcode::fallthrough as u8], &[0b00000101], &mut 0), None); + + // Instruction with no arguments, overparametrized until end of code. + assert_eq!( + parse_instruction( + &[Opcode::fallthrough as u8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], + &[0b00000001], + &mut 0 + ), + Some(ParsedInstruction { + kind: Instruction::fallthrough, + offset: 0, + length: 8 + }) + ); + + // Instruction with one immediate argument. + assert_eq!( + parse_instruction(&[Opcode::ecalli as u8], &[0b00000011], &mut 0), + Some(ParsedInstruction { + kind: Instruction::ecalli(0), + offset: 0, + length: 1 + }) + ); + + assert_eq!( + parse_instruction(&[Opcode::ecalli as u8, 0xff, 0xff, 0xff, 0xff], &[0b00100001], &mut 0), + Some(ParsedInstruction { + kind: Instruction::ecalli(0x80000000), + offset: 0, + length: 5 + }) + ); + + // Instruction with one immediate argument, overparametrized. + assert_eq!( + parse_instruction(&[Opcode::ecalli as u8, 0xff, 0xff, 0xff, 0xff, 0x66], &[0b01000001], &mut 0), + Some(ParsedInstruction { + kind: Instruction::ecalli(0x80000000), + offset: 0, + length: 6 + }) + ); + + // Instruction with two registers and one immediate argument. + assert_eq!( + parse_instruction(&[Opcode::add_imm as u8, 0x00, 0xff, 0xff, 0xff, 0xff], &[0b01000001], &mut 0), + Some(ParsedInstruction { + kind: Instruction::add_imm(Reg::RA.into(), Reg::RA.into(), 0x80000000), + offset: 0, + length: 6 + }) + ); + + // Instruction with two registers and one immediate argument, overparametrized. + assert_eq!( + parse_instruction(&[Opcode::add_imm as u8, 0x00, 0xff, 0xff, 0xff, 0xff, 0x66], &[0b10000001], &mut 0), + Some(ParsedInstruction { + kind: Instruction::add_imm(Reg::RA.into(), Reg::RA.into(), 0x80000000), + offset: 0, + length: 7 + }) + ); + + extern crate alloc; + use alloc::vec; + use alloc::vec::Vec; + + let length = 512; + let mut bitmask = vec![0; length / 8]; + bitmask[0] = 0b00000001; + let mut code = Vec::new(); + code.resize(length, 0xff); + code[0] = Opcode::add_imm as u8; + code[1] = 0x00; + assert_eq!( + parse_instruction(&code, &bitmask, &mut 0), + Some(ParsedInstruction { + kind: Instruction::add_imm(Reg::RA.into(), Reg::RA.into(), 0x80000000), + offset: 0, + length: 512 + }) + ); +} + // NOTE: The opcodes here are assigned roughly in the order of how common a given instruction is, // except the `trap` which is deliberately hardcoded as zero. define_opcodes! { @@ -1089,6 +1173,12 @@ impl Opcode { } } +impl core::fmt::Display for Instruction { + fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { + self.visit(fmt) + } +} + impl Instruction { pub fn starts_new_basic_block(self) -> bool { self.opcode().starts_new_basic_block() @@ -1099,56 +1189,60 @@ impl Instruction { 1 } - fn serialize_reg_imm_offset(buffer: &mut [u8], position: u32, opcode: Opcode, reg: Reg, imm1: u32, imm2: u32) -> usize { + fn serialize_reg_imm_offset(buffer: &mut [u8], position: u32, opcode: Opcode, reg: RawReg, imm1: u32, imm2: u32) -> usize { let imm2 = imm2.wrapping_sub(position); buffer[0] = opcode as u8; - buffer[1] = reg as u8; let mut position = 2; - position += write_varint(imm1, &mut buffer[position..]); + let imm1_length = write_simple_varint(imm1, &mut buffer[position..]); + position += imm1_length; + buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8; position += write_simple_varint(imm2, &mut buffer[position..]); position } - fn serialize_reg_imm_imm(buffer: &mut [u8], opcode: Opcode, reg: Reg, imm1: u32, imm2: u32) -> usize { + fn serialize_reg_imm_imm(buffer: &mut [u8], opcode: Opcode, reg: RawReg, imm1: u32, imm2: u32) -> usize { buffer[0] = opcode as u8; - buffer[1] = reg as u8; let mut position = 2; - position += write_varint(imm1, &mut buffer[position..]); + let imm1_length = write_simple_varint(imm1, &mut buffer[position..]); + position += imm1_length; + buffer[1] = reg.0 as u8 | (imm1_length << 4) as u8; position += write_simple_varint(imm2, &mut buffer[position..]); position } - fn serialize_reg_reg_imm_imm(buffer: &mut [u8], opcode: Opcode, reg1: Reg, reg2: Reg, imm1: u32, imm2: u32) -> usize { + fn serialize_reg_reg_imm_imm(buffer: &mut [u8], opcode: Opcode, reg1: RawReg, reg2: RawReg, imm1: u32, imm2: u32) -> usize { buffer[0] = opcode as u8; - buffer[1] = reg1 as u8 | (reg2 as u8) << 4; - let mut position = 2; - position += write_varint(imm1, &mut buffer[position..]); + buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4; + let mut position = 3; + let imm1_length = write_simple_varint(imm1, &mut buffer[position..]); + buffer[2] = imm1_length as u8; + position += imm1_length; position += write_simple_varint(imm2, &mut buffer[position..]); position } - fn serialize_reg_reg_reg(buffer: &mut [u8], opcode: Opcode, reg1: Reg, reg2: Reg, reg3: Reg) -> usize { + fn serialize_reg_reg_reg(buffer: &mut [u8], opcode: Opcode, reg1: RawReg, reg2: RawReg, reg3: RawReg) -> usize { buffer[0] = opcode as u8; - buffer[1] = reg1 as u8 | (reg2 as u8) << 4; - buffer[2] = reg3 as u8; + buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4; + buffer[2] = reg3.0 as u8; 3 } - fn serialize_reg_reg_imm(buffer: &mut [u8], opcode: Opcode, reg1: Reg, reg2: Reg, imm: u32) -> usize { + fn serialize_reg_reg_imm(buffer: &mut [u8], opcode: Opcode, reg1: RawReg, reg2: RawReg, imm: u32) -> usize { buffer[0] = opcode as u8; - buffer[1] = reg1 as u8 | (reg2 as u8) << 4; + buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4; write_simple_varint(imm, &mut buffer[2..]) + 2 } - fn serialize_reg_reg_offset(buffer: &mut [u8], position: u32, opcode: Opcode, reg1: Reg, reg2: Reg, imm: u32) -> usize { + fn serialize_reg_reg_offset(buffer: &mut [u8], position: u32, opcode: Opcode, reg1: RawReg, reg2: RawReg, imm: u32) -> usize { let imm = imm.wrapping_sub(position); buffer[0] = opcode as u8; - buffer[1] = reg1 as u8 | (reg2 as u8) << 4; + buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4; write_simple_varint(imm, &mut buffer[2..]) + 2 } - fn serialize_reg_imm(buffer: &mut [u8], opcode: Opcode, reg: Reg, imm: u32) -> usize { + fn serialize_reg_imm(buffer: &mut [u8], opcode: Opcode, reg: RawReg, imm: u32) -> usize { buffer[0] = opcode as u8; - buffer[1] = reg as u8; + buffer[1] = reg.0 as u8; write_simple_varint(imm, &mut buffer[2..]) + 2 } @@ -1165,15 +1259,17 @@ impl Instruction { fn serialize_imm_imm(buffer: &mut [u8], opcode: Opcode, imm1: u32, imm2: u32) -> usize { buffer[0] = opcode as u8; - let mut position = 1; - position += write_varint(imm1, &mut buffer[position..]); + let mut position = 2; + let imm1_length = write_simple_varint(imm1, &mut buffer[position..]); + buffer[1] = imm1_length as u8; + position += imm1_length; position += write_simple_varint(imm2, &mut buffer[position..]); position } - fn serialize_reg_reg(buffer: &mut [u8], opcode: Opcode, reg1: Reg, reg2: Reg) -> usize { + fn serialize_reg_reg(buffer: &mut [u8], opcode: Opcode, reg1: RawReg, reg2: RawReg) -> usize { buffer[0] = opcode as u8; - buffer[1] = reg1 as u8 | (reg2 as u8) << 4; + buffer[1] = reg1.0 as u8 | (reg2.0 as u8) << 4; 2 } } @@ -1191,7 +1287,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { write!(self, "fallthrough") } - fn sbrk(&mut self, d: Reg, s: Reg) -> Self::ReturnTy { + fn sbrk(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy { write!(self, "{d} = sbrk {s}") } @@ -1199,167 +1295,167 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { write!(self, "ecalli {nth_import}") } - fn set_less_than_unsigned(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn set_less_than_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} Self::ReturnTy { + fn set_less_than_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} Self::ReturnTy { + fn shift_logical_right(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} >> {s2}") } - fn shift_arithmetic_right(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn shift_arithmetic_right(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} >>a {s2}") } - fn shift_logical_left(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn shift_logical_left(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} << {s2}") } - fn xor(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn xor(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} ^ {s2}") } - fn and(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn and(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} & {s2}") } - fn or(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn or(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} | {s2}") } - fn add(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn add(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} + {s2}") } - fn sub(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn sub(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} - {s2}") } - fn mul(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn mul(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} * {s2}") } - fn mul_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn mul_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} * {s2}") } - fn mul_upper_signed_signed(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn mul_upper_signed_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = ({s1} as i64 * {s2} as i64) >> 32") } - fn mul_upper_signed_signed_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn mul_upper_signed_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = ({s1} as i64 * {s2} as i64) >> 32", s2 = s2 as i32) } - fn mul_upper_unsigned_unsigned(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn mul_upper_unsigned_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = ({s1} as u64 * {s2} as u64) >> 32") } - fn mul_upper_unsigned_unsigned_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn mul_upper_unsigned_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = ({s1} as u64 * {s2} as u64) >> 32") } - fn mul_upper_signed_unsigned(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn mul_upper_signed_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = ({s1} as i64 * {s2} as u64) >> 32") } - fn div_unsigned(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn div_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} /u {s2}") } - fn div_signed(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn div_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} /s {s2}") } - fn rem_unsigned(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn rem_unsigned(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} %u {s2}") } - fn rem_signed(&mut self, d: Reg, s1: Reg, s2: Reg) -> Self::ReturnTy { + fn rem_signed(&mut self, d: RawReg, s1: RawReg, s2: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s1} %s {s2}") } - fn set_less_than_unsigned_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn set_less_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} Self::ReturnTy { + fn set_greater_than_unsigned_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} >u 0x{s2:x}") } - fn set_less_than_signed_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn set_less_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} Self::ReturnTy { + fn set_greater_than_signed_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} >s {s2}", s2 = s2 as i32) } - fn shift_logical_right_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn shift_logical_right_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} >> {s2}") } - fn shift_logical_right_imm_alt(&mut self, d: Reg, s2: Reg, s1: u32) -> Self::ReturnTy { + fn shift_logical_right_imm_alt(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} >> {s2}") } - fn shift_arithmetic_right_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn shift_arithmetic_right_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} >>a {s2}") } - fn shift_arithmetic_right_imm_alt(&mut self, d: Reg, s2: Reg, s1: u32) -> Self::ReturnTy { + fn shift_arithmetic_right_imm_alt(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} >>a {s2}") } - fn shift_logical_left_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn shift_logical_left_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} << {s2}") } - fn shift_logical_left_imm_alt(&mut self, d: Reg, s2: Reg, s1: u32) -> Self::ReturnTy { + fn shift_logical_left_imm_alt(&mut self, d: RawReg, s2: RawReg, s1: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} << {s2}") } - fn or_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn or_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} | 0x{s2:x}") } - fn and_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn and_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} & 0x{s2:x}") } - fn xor_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn xor_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { write!(self, "{d} = {s1} ^ 0x{s2:x}") } - fn load_imm(&mut self, d: Reg, a: u32) -> Self::ReturnTy { + fn load_imm(&mut self, d: RawReg, a: u32) -> Self::ReturnTy { write!(self, "{d} = 0x{a:x}") } - fn move_reg(&mut self, d: Reg, s: Reg) -> Self::ReturnTy { + fn move_reg(&mut self, d: RawReg, s: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s}") } - fn cmov_if_zero(&mut self, d: Reg, s: Reg, c: Reg) -> Self::ReturnTy { + fn cmov_if_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s} if {c} == 0") } - fn cmov_if_not_zero(&mut self, d: Reg, s: Reg, c: Reg) -> Self::ReturnTy { + fn cmov_if_not_zero(&mut self, d: RawReg, s: RawReg, c: RawReg) -> Self::ReturnTy { write!(self, "{d} = {s} if {c} != 0") } - fn cmov_if_zero_imm(&mut self, d: Reg, c: Reg, s: u32) -> Self::ReturnTy { + fn cmov_if_zero_imm(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy { write!(self, "{d} = {s} if {c} == 0") } - fn cmov_if_not_zero_imm(&mut self, d: Reg, c: Reg, s: u32) -> Self::ReturnTy { + fn cmov_if_not_zero_imm(&mut self, d: RawReg, c: RawReg, s: u32) -> Self::ReturnTy { write!(self, "{d} = {s} if {c} != 0") } - fn add_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn add_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { if (s2 as i32) < 0 && (s2 as i32) > -4096 { write!(self, "{d} = {s1} - {s2}", s2 = -(s2 as i32)) } else { @@ -1367,7 +1463,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn negate_and_add_imm(&mut self, d: Reg, s1: Reg, s2: u32) -> Self::ReturnTy { + fn negate_and_add_imm(&mut self, d: RawReg, s1: RawReg, s2: u32) -> Self::ReturnTy { if s2 == 0 { write!(self, "{d} = -{s1}") } else { @@ -1375,19 +1471,19 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn store_imm_indirect_u8(&mut self, base: Reg, offset: u32, value: u32) -> Self::ReturnTy { + fn store_imm_indirect_u8(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy { write!(self, "u8 [{base} + {offset}] = {value}") } - fn store_imm_indirect_u16(&mut self, base: Reg, offset: u32, value: u32) -> Self::ReturnTy { + fn store_imm_indirect_u16(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy { write!(self, "u16 [{base} + {offset}] = {value}") } - fn store_imm_indirect_u32(&mut self, base: Reg, offset: u32, value: u32) -> Self::ReturnTy { + fn store_imm_indirect_u32(&mut self, base: RawReg, offset: u32, value: u32) -> Self::ReturnTy { write!(self, "u32 [{base} + {offset}] = {value}") } - fn store_indirect_u8(&mut self, src: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn store_indirect_u8(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "u8 [{base} + {offset}] = {src}") } else { @@ -1395,7 +1491,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn store_indirect_u16(&mut self, src: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn store_indirect_u16(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "u16 [{base} + {offset}] = {src}") } else { @@ -1403,7 +1499,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn store_indirect_u32(&mut self, src: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn store_indirect_u32(&mut self, src: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "u32 [{base} + {offset}] = {src}") } else { @@ -1411,31 +1507,31 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn store_imm_u8(&mut self, value: u32, offset: u32) -> Self::ReturnTy { + fn store_imm_u8(&mut self, offset: u32, value: u32) -> Self::ReturnTy { write!(self, "u8 [0x{offset:x}] = {value}") } - fn store_imm_u16(&mut self, value: u32, offset: u32) -> Self::ReturnTy { + fn store_imm_u16(&mut self, offset: u32, value: u32) -> Self::ReturnTy { write!(self, "u16 [0x{offset:x}] = {value}") } - fn store_imm_u32(&mut self, value: u32, offset: u32) -> Self::ReturnTy { + fn store_imm_u32(&mut self, offset: u32, value: u32) -> Self::ReturnTy { write!(self, "u32 [0x{offset:x}] = {value}") } - fn store_u8(&mut self, src: Reg, offset: u32) -> Self::ReturnTy { + fn store_u8(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "u8 [0x{offset:x}] = {src}") } - fn store_u16(&mut self, src: Reg, offset: u32) -> Self::ReturnTy { + fn store_u16(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "u16 [0x{offset:x}] = {src}") } - fn store_u32(&mut self, src: Reg, offset: u32) -> Self::ReturnTy { + fn store_u32(&mut self, src: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "u32 [0x{offset:x}] = {src}") } - fn load_indirect_u8(&mut self, dst: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn load_indirect_u8(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "{} = u8 [{} + {}]", dst, base, offset) } else { @@ -1443,7 +1539,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn load_indirect_i8(&mut self, dst: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn load_indirect_i8(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "{} = i8 [{} + {}]", dst, base, offset) } else { @@ -1451,7 +1547,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn load_indirect_u16(&mut self, dst: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn load_indirect_u16(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "{} = u16 [{} + {}]", dst, base, offset) } else { @@ -1459,7 +1555,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn load_indirect_i16(&mut self, dst: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn load_indirect_i16(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "{} = i16 [{} + {}]", dst, base, offset) } else { @@ -1467,7 +1563,7 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn load_indirect_u32(&mut self, dst: Reg, base: Reg, offset: u32) -> Self::ReturnTy { + fn load_indirect_u32(&mut self, dst: RawReg, base: RawReg, offset: u32) -> Self::ReturnTy { if offset != 0 { write!(self, "{} = u32 [{} + {}]", dst, base, offset) } else { @@ -1475,87 +1571,87 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn load_u8(&mut self, dst: Reg, offset: u32) -> Self::ReturnTy { + fn load_u8(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "{} = u8 [0x{:x}]", dst, offset) } - fn load_i8(&mut self, dst: Reg, offset: u32) -> Self::ReturnTy { + fn load_i8(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "{} = i8 [0x{:x}]", dst, offset) } - fn load_u16(&mut self, dst: Reg, offset: u32) -> Self::ReturnTy { + fn load_u16(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "{} = u16 [0x{:x}]", dst, offset) } - fn load_i16(&mut self, dst: Reg, offset: u32) -> Self::ReturnTy { + fn load_i16(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "{} = i16 [0x{:x}]", dst, offset) } - fn load_u32(&mut self, dst: Reg, offset: u32) -> Self::ReturnTy { + fn load_u32(&mut self, dst: RawReg, offset: u32) -> Self::ReturnTy { write!(self, "{} = u32 [0x{:x}]", dst, offset) } - fn branch_less_unsigned(&mut self, s1: Reg, s2: Reg, imm: u32) -> Self::ReturnTy { + fn branch_less_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} Self::ReturnTy { + fn branch_less_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} Self::ReturnTy { + fn branch_less_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} Self::ReturnTy { + fn branch_less_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} Self::ReturnTy { + fn branch_greater_or_equal_unsigned(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} >=u {}", imm, s1, s2) } - fn branch_greater_or_equal_signed(&mut self, s1: Reg, s2: Reg, imm: u32) -> Self::ReturnTy { + fn branch_greater_or_equal_signed(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} >=s {}", imm, s1, s2) } - fn branch_greater_or_equal_unsigned_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_greater_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} >=u {}", imm, s1, s2) } - fn branch_greater_or_equal_signed_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_greater_or_equal_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} >=s {}", imm, s1, s2) } - fn branch_eq(&mut self, s1: Reg, s2: Reg, imm: u32) -> Self::ReturnTy { + fn branch_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} == {}", imm, s1, s2) } - fn branch_not_eq(&mut self, s1: Reg, s2: Reg, imm: u32) -> Self::ReturnTy { + fn branch_not_eq(&mut self, s1: RawReg, s2: RawReg, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} != {}", imm, s1, s2) } - fn branch_eq_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_eq_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} == {}", imm, s1, s2) } - fn branch_not_eq_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_not_eq_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} != {}", imm, s1, s2) } - fn branch_less_or_equal_unsigned_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_less_or_equal_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} <=u {}", imm, s1, s2) } - fn branch_less_or_equal_signed_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_less_or_equal_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} <=s {}", imm, s1, s2) } - fn branch_greater_unsigned_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_greater_unsigned_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} >u {}", imm, s1, s2) } - fn branch_greater_signed_imm(&mut self, s1: Reg, s2: u32, imm: u32) -> Self::ReturnTy { + fn branch_greater_signed_imm(&mut self, s1: RawReg, s2: u32, imm: u32) -> Self::ReturnTy { write!(self, "jump {} if {} >s {}", imm, s1, s2) } @@ -1563,20 +1659,19 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { write!(self, "jump {}", target) } - fn load_imm_and_jump(&mut self, ra: Reg, value: u32, target: u32) -> Self::ReturnTy { + fn load_imm_and_jump(&mut self, ra: RawReg, value: u32, target: u32) -> Self::ReturnTy { write!(self, "{ra} = {value}, jump {target}") } - fn jump_indirect(&mut self, base: Reg, offset: u32) -> Self::ReturnTy { - use Reg::*; + fn jump_indirect(&mut self, base: RawReg, offset: u32) -> Self::ReturnTy { match (base, offset) { - (RA, 0) => write!(self, "ret"), + (_, 0) if base == Reg::RA.into() => write!(self, "ret"), (_, 0) => write!(self, "jump [{}]", base), (_, _) => write!(self, "jump [{} + {}]", base, offset), } } - fn load_imm_and_jump_indirect(&mut self, ra: Reg, base: Reg, value: u32, offset: u32) -> Self::ReturnTy { + fn load_imm_and_jump_indirect(&mut self, ra: RawReg, base: RawReg, value: u32, offset: u32) -> Self::ReturnTy { if offset == 0 { write!(self, "jump [{base}], {ra} = {value}") } else { @@ -1584,8 +1679,8 @@ impl<'a> InstructionVisitor for core::fmt::Formatter<'a> { } } - fn invalid(&mut self, opcode: u8) -> Self::ReturnTy { - write!(self, "invalid 0x{opcode:02x}") + fn invalid(&mut self) -> Self::ReturnTy { + write!(self, "invalid") } } @@ -2033,12 +2128,12 @@ impl<'a> Iterator for JumpTableIter<'a> { #[inline(never)] #[cold] -fn parse_bitmask_slow(bitmask: &[u8], code_offset: &mut usize) -> Option { +fn parse_bitmask_slow(bitmask: &[u8], mut offset: usize) -> Option<(usize, usize)> { if bitmask.is_empty() { return None; } - let mut offset = *code_offset + 1; + offset += 1; let mut args_length = 0; while let Some(&byte) = bitmask.get(offset >> 3) { let shift = offset & 7; @@ -2059,47 +2154,54 @@ fn parse_bitmask_slow(bitmask: &[u8], code_offset: &mut usize) -> Option offset += length; } - *code_offset = offset; - Some(args_length) + Some((offset, args_length)) } #[cfg_attr(not(debug_assertions), inline(always))] -pub(crate) fn parse_bitmask(bitmask: &[u8], code_offset: &mut usize) -> Option { - let mut offset = *code_offset + 1; +pub(crate) fn parse_bitmask_fast(bitmask: &[u8], mut offset: usize) -> Option<(usize, usize)> { + offset += 1; let Some(bitmask) = bitmask.get(offset >> 3..(offset >> 3) + 4) else { - return parse_bitmask_slow(bitmask, code_offset); + return None; }; let shift = offset & 7; let mask = u32::from_le_bytes([bitmask[0], bitmask[1], bitmask[2], bitmask[3]]) >> shift; - if mask == 0 { - return parse_bitmask_slow(bitmask, code_offset); + return None; } let args_length = mask.trailing_zeros() as usize; offset += args_length; - *code_offset = offset; - Some(args_length) + Some((offset, args_length)) } #[test] fn test_parse_bitmask() { - fn p(bitmask: &[u8]) -> (Option, usize) { - let mut offset = 0; - (parse_bitmask(bitmask, &mut offset), offset) + fn p(bitmask: &[u8]) -> Option<(usize, usize)> { + let result_fast = parse_bitmask_fast(bitmask, 0); + let result_slow = parse_bitmask_slow(bitmask, 0); + if result_fast.is_some() { + assert_eq!(result_fast, result_slow); + } + + result_slow } - assert_eq!(p(&[0b00000001, 0, 0, 0]), (Some(31), 32)); - assert_eq!(p(&[0b00000001, 0, 0]), (Some(23), 24)); - assert_eq!(p(&[0b00000001, 0]), (Some(15), 16)); - assert_eq!(p(&[0b00000001]), (Some(7), 8)); - assert_eq!(p(&[0b00000011]), (Some(0), 1)); - assert_eq!(p(&[0b00000011, 0]), (Some(0), 1)); - assert_eq!(p(&[0b00000101]), (Some(1), 2)); - assert_eq!(p(&[0b10000001]), (Some(6), 7)); - assert_eq!(p(&[0b00000001, 1]), (Some(7), 8)); + assert_eq!(p(&[0b00000001, 0, 0, 0]), Some((32, 31))); + assert_eq!(p(&[0b00000001, 0, 0]), Some((24, 23))); + assert_eq!(p(&[0b00000001, 0]), Some((16, 15))); + assert_eq!(p(&[0b00000001]), Some((8, 7))); + assert_eq!(p(&[0b00000011]), Some((1, 0))); + assert_eq!(p(&[0b00000011, 0]), Some((1, 0))); + assert_eq!(p(&[0b00000101]), Some((2, 1))); + assert_eq!(p(&[0b10000001]), Some((7, 6))); + assert_eq!(p(&[0b00000001, 1]), Some((8, 7))); + + assert_eq!(parse_bitmask_fast(&[1, 0, 1, 0, 0, 0, 0, 0], 0), Some((16, 15))); + assert_eq!(parse_bitmask_fast(&[1, 0, 0, 1, 0, 0, 0, 0], 0), Some((24, 23))); + assert_eq!(parse_bitmask_fast(&[1, 0, 0, 0b10000000, 0, 0, 0, 0], 0), Some((31, 30))); + assert_eq!(parse_bitmask_fast(&[1, 0, 0, 0, 1, 0, 0, 0], 0), None); } #[derive(Clone)] @@ -2109,7 +2211,7 @@ pub struct Instructions<'a> { offset: usize, } -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct ParsedInstruction { pub kind: Instruction, pub offset: u32, diff --git a/crates/polkavm-common/src/varint.rs b/crates/polkavm-common/src/varint.rs index e3f24002..ba12127f 100644 --- a/crates/polkavm-common/src/varint.rs +++ b/crates/polkavm-common/src/varint.rs @@ -7,20 +7,6 @@ fn get_varint_length(leading_zeros: u32) -> u32 { pub const MAX_VARINT_LENGTH: usize = 5; -#[inline(always)] -pub(crate) fn read_varint_fast(chunk: u64) -> Option<(u32, u32)> { - let first_byte = chunk as u8; - let length = (!first_byte).leading_zeros(); - if length > 4 { - return None; - } - - let upper_mask = 0b11111111_u32 >> length; - let lower_mask = ((1_u64 << (length << 3)) - 1) as u32; - let value = (upper_mask & u32::from(first_byte)).wrapping_shl(length * 8) | (((chunk >> 8) as u32) & lower_mask); - Some((length + 1, value)) -} - #[inline] pub(crate) fn read_varint(input: &[u8], first_byte: u8) -> Option<(usize, u32)> { let length = (!first_byte).leading_zeros(); @@ -90,24 +76,71 @@ proptest::proptest! { } #[inline(always)] -pub(crate) fn read_simple_varint_fast(chunk: u32, length: u32) -> u32 { - let mask = ((1_u64 << (length << 3)) - 1) as u32; - let value = chunk & mask; +fn apply_zigzag(value: u32) -> u32 { + let value = value as i32; + let value = (value << 1) ^ (value >> 31); + value as u32 +} + +#[inline(always)] +fn undo_zigzag(value: u32) -> u32 { (value >> 1) ^ (-((value & 1) as i32)) as u32 } +const fn bit_length_to_mask_u32_slow(length: u32) -> u32 { + if length >= 32 { + 0xffffffff + } else { + (1_u32 << length) - 1 + } +} + +static LENGTH_MINUS_1_TO_MASK: [u32; 256] = { + let mut output = [0; 256]; + let mut length = 0_u32; + while length < 256 { + let length_effective = length.wrapping_sub(1); + let length_effective = if length_effective < 4 { length_effective } else { 4 }; + output[length as usize] = bit_length_to_mask_u32_slow(length_effective * 8); + length += 1; + } + output +}; + +#[inline(always)] +pub(crate) fn read_simple_varint_length_minus_1(chunk: u32, length: u32) -> u32 { + let mask = LENGTH_MINUS_1_TO_MASK[length as u8 as usize]; + undo_zigzag(chunk & mask) +} + +static LENGTH_TO_MASK: [u32; 256] = { + let mut output = [0; 256]; + let mut length = 0_u32; + while length < 256 { + let length_effective = if length < 4 { length } else { 4 }; + output[length as usize] = bit_length_to_mask_u32_slow(length_effective * 8); + length += 1; + } + output +}; + +#[inline(always)] +pub(crate) fn read_simple_varint(chunk: u32, length: u32) -> u32 { + let mask = LENGTH_TO_MASK[length as u8 as usize]; + undo_zigzag(chunk & mask) +} + #[inline] -fn get_simple_varint_length(leading_zeros: u32) -> u32 { +fn get_bytes_required(value: u32) -> u32 { + let leading_zeros = value.leading_zeros(); let bits_required = 32 - leading_zeros; bits_required / 8 + (if leading_zeros % 8 > 0 { 1 } else { 0 }) } #[inline] pub(crate) fn write_simple_varint(value: u32, buffer: &mut [u8]) -> usize { - let value = value as i32; - let value = (value << 1) ^ (value >> 31); - let value = value as u32; - let varint_length = get_simple_varint_length(value.leading_zeros()); + let value = apply_zigzag(value); + let varint_length = get_bytes_required(value); match varint_length { 0 => {} 1 => { @@ -142,7 +175,7 @@ proptest::proptest! { #[allow(clippy::ignored_unit_patterns)] #[test] fn simple_varint_serialization(value in 0u32..=0xffffffff) { - fn read_simple_varint(input: &[u8]) -> Option { + fn read_simple_varint_slow(input: &[u8]) -> Option { let value = match input.len() { 0 => 0, 1 => u32::from(input[0]), @@ -156,9 +189,16 @@ proptest::proptest! { Some(value) } - let mut buffer = [0; MAX_VARINT_LENGTH]; - let length = write_simple_varint(value, &mut buffer); - let parsed_value = read_simple_varint(&buffer[..length]).unwrap(); - assert_eq!(parsed_value, value, "value mismatch"); + let mut t = [0; 4]; + let length = write_simple_varint(value, &mut t); + assert_eq!(read_simple_varint_slow(&t[..length]).unwrap(), value, "value mismatch"); + + let chunk = u32::from_le_bytes([t[0], t[1], t[2], t[3]]); + assert_eq!(read_simple_varint(chunk, length as u32), value); + assert_eq!(read_simple_varint(chunk, length as u32 + 1), value); + assert_eq!(read_simple_varint(chunk, length as u32 + 2), value); + assert_eq!(read_simple_varint(chunk, length as u32 + 3), value); + assert_eq!(read_simple_varint(chunk, length as u32 + 4), value); + assert_eq!(read_simple_varint(chunk, 0xffffffff), value); } } diff --git a/crates/polkavm-common/src/writer.rs b/crates/polkavm-common/src/writer.rs index 09701a46..e4a6e2f5 100644 --- a/crates/polkavm-common/src/writer.rs +++ b/crates/polkavm-common/src/writer.rs @@ -271,7 +271,7 @@ impl ProgramBlobBuilder { assert_eq!(parsed.len(), instructions.len()); for ((offset, mut instruction), entry) in parsed.into_iter().zip(instructions.into_iter()) { - assert_eq!(instruction, entry.instruction); + assert_eq!(instruction, entry.instruction, "broken serialization: {:?}", entry.bytes.bytes); assert_eq!(entry.position, offset); if let Some(target) = instruction.target_mut() { assert!(offsets.contains(target)); diff --git a/crates/polkavm-disassembler/src/lib.rs b/crates/polkavm-disassembler/src/lib.rs index b4f6841c..c7f36d16 100644 --- a/crates/polkavm-disassembler/src/lib.rs +++ b/crates/polkavm-disassembler/src/lib.rs @@ -426,7 +426,7 @@ mod tests { builder.add_import("hostcall".into()); builder.set_code( &[ - asm::store_imm_u32(0x12345678, memory_map.rw_data_address()), + asm::store_imm_u32(memory_map.rw_data_address(), 0x12345678), asm::add(S0, A0, A1), asm::ecalli(0), asm::add(A0, A0, S0), diff --git a/crates/polkavm-linker/src/program_from_elf.rs b/crates/polkavm-linker/src/program_from_elf.rs index ea75fb50..077af93c 100644 --- a/crates/polkavm-linker/src/program_from_elf.rs +++ b/crates/polkavm-linker/src/program_from_elf.rs @@ -5104,7 +5104,7 @@ fn emit_code( is_optimized: bool, ) -> Result, ProgramFromElfError> { use polkavm_common::program::Reg as PReg; - fn conv_reg(reg: Reg) -> PReg { + fn conv_reg(reg: Reg) -> polkavm_common::program::RawReg { match reg { Reg::RA => PReg::RA, Reg::SP => PReg::SP, @@ -5123,6 +5123,7 @@ fn emit_code( unreachable!("internal error: temporary register was not spilled into memory"); } } + .into() } let can_fallthrough_to_next_block = calculate_whether_can_fallthrough(all_blocks, used_blocks); @@ -5216,7 +5217,7 @@ fn emit_code( } RegImm::Imm(value) => { codegen! { - args = (value, target), + args = (target, value), kind = kind, { StoreKind::U32 => store_imm_u32, diff --git a/crates/polkavm/src/compiler.rs b/crates/polkavm/src/compiler.rs index a1299eda..34f0fcb8 100644 --- a/crates/polkavm/src/compiler.rs +++ b/crates/polkavm/src/compiler.rs @@ -4,7 +4,7 @@ use std::sync::Mutex; use core::marker::PhantomData; use polkavm_assembler::{Assembler, Label}; -use polkavm_common::program::{ParsedInstruction, ProgramExport, Instructions, JumpTable, Reg}; +use polkavm_common::program::{ParsedInstruction, ProgramExport, Instructions, JumpTable, RawReg, InstructionVisitor}; use polkavm_common::zygote::{ VM_COMPILER_MAXIMUM_EPILOGUE_LENGTH, VM_COMPILER_MAXIMUM_INSTRUCTION_LENGTH, }; @@ -33,10 +33,6 @@ struct Cache { pub(crate) struct CompilerCache(Mutex); pub(crate) struct CompilerVisitor<'a, S> where S: Sandbox { - current_code_offset: u32, - next_code_offset: u32, - is_last_instruction: bool, - init: GuestInit<'a>, jump_table: JumpTable<'a>, code: &'a [u8], @@ -133,9 +129,6 @@ impl<'a, S> CompilerVisitor<'a, S> where S: Sandbox { asm.set_origin(native_code_address); let mut visitor = CompilerVisitor { - current_code_offset: 0, - next_code_offset: 0, - is_last_instruction: false, gas_visitor: GasVisitor::default(), asm, exports, @@ -167,7 +160,7 @@ impl<'a, S> CompilerVisitor<'a, S> where S: Sandbox { ArchVisitor(&mut visitor).emit_trace_trampoline(); } - visitor.start_new_basic_block::(); + visitor.force_start_new_basic_block(0); Ok((visitor, address_space)) } @@ -272,58 +265,61 @@ impl<'a, S> CompilerVisitor<'a, S> where S: Sandbox { } #[inline(always)] - fn start_new_basic_block(&mut self) { - if !IS_INITIAL && self.gas_metering.is_some() { + fn start_new_basic_block(&mut self, code_offset: u32, args_length: u32) { + if self.gas_metering.is_some() { let cost = self.gas_visitor.take_block_cost().unwrap(); self.gas_cost_for_basic_block.push(cost); } - if !self.is_last_instruction { - let code_offset = self.next_code_offset; + let next_code_offset = code_offset + args_length + 1; + let is_last_instruction = next_code_offset as usize >= self.code.len(); + if !is_last_instruction { + self.force_start_new_basic_block(next_code_offset); + } + } - log::trace!("Starting new basic block at: {code_offset}"); - if let Some(label) = self.code_offset_to_label.get(code_offset) { - log::trace!("Label: {label} -> {code_offset} -> {:08x}", self.asm.current_address()); - self.asm.define_label(label); - } else { - let label = self.asm.create_label(); - log::trace!("Label: {label} -> {code_offset} -> {:08x}", self.asm.current_address()); - self.code_offset_to_label.insert(code_offset, label); - } + #[inline(always)] + fn force_start_new_basic_block(&mut self, next_code_offset: u32) { + log::trace!("Starting new basic block at: {next_code_offset}"); + if let Some(label) = self.code_offset_to_label.get(next_code_offset) { + log::trace!("Label: {label} -> {next_code_offset} -> {:08x}", self.asm.current_address()); + self.asm.define_label(label); + } else { + let label = self.asm.create_label(); + log::trace!("Label: {label} -> {next_code_offset} -> {:08x}", self.asm.current_address()); + self.code_offset_to_label.insert(next_code_offset, label); + } - if let Some(gas_metering) = self.gas_metering { - self.gas_metering_stub_offsets.push(self.asm.len()); - ArchVisitor(self).emit_gas_metering_stub(gas_metering); - } + if let Some(gas_metering) = self.gas_metering { + self.gas_metering_stub_offsets.push(self.asm.len()); + ArchVisitor(self).emit_gas_metering_stub(gas_metering); } } #[inline(always)] - fn before_instruction(&mut self) { - self.code_offset_to_native_code_offset.push((self.current_code_offset, self.asm.len() as u32)); + fn before_instruction(&mut self, code_offset: u32) { + self.code_offset_to_native_code_offset.push((code_offset, self.asm.len() as u32)); if log::log_enabled!(log::Level::Trace) { - self.trace_compiled_instruction(); + self.trace_compiled_instruction(code_offset); } if self.debug_trace_execution { - ArchVisitor(self).trace_execution(); + ArchVisitor(self).trace_execution(code_offset); } - - self.asm.reserve::<8>(); } - fn after_instruction(&mut self) { + fn after_instruction(&mut self, code_offset: u32) { if cfg!(debug_assertions) && !self.debug_trace_execution { let offset = self.code_offset_to_native_code_offset.last().unwrap().1 as usize; let instruction_length = self.asm.len() - offset; if instruction_length > VM_COMPILER_MAXIMUM_INSTRUCTION_LENGTH as usize { - self.panic_on_too_long_instruction(instruction_length) + self.panic_on_too_long_instruction(code_offset, instruction_length) } } } - fn current_instruction(&self) -> impl core::fmt::Display { + fn current_instruction(&self, code_offset: u32) -> impl core::fmt::Display { struct MaybeInstruction(Option); impl core::fmt::Display for MaybeInstruction { fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result { @@ -334,34 +330,38 @@ impl<'a, S> CompilerVisitor<'a, S> where S: Sandbox { } } } - MaybeInstruction(Instructions::new(self.code, self.bitmask, self.current_code_offset).next()) + MaybeInstruction(Instructions::new(self.code, self.bitmask, code_offset).next()) } #[cold] - fn panic_on_too_long_instruction(&self, instruction_length: usize) -> ! { + fn panic_on_too_long_instruction(&self, code_offset: u32, instruction_length: usize) -> ! { panic!( "maximum instruction length of {} exceeded with {} bytes for instruction: {}", VM_COMPILER_MAXIMUM_INSTRUCTION_LENGTH, instruction_length, - self.current_instruction(), + self.current_instruction(code_offset), ); } #[inline(never)] #[cold] - fn trace_compiled_instruction(&self) { - log::trace!("Compiling {}", self.current_instruction()); + fn trace_compiled_instruction(&self, code_offset: u32) { + log::trace!("Compiling {}", self.current_instruction(code_offset)); } - fn get_or_forward_declare_label(&mut self, code_offset: u32) -> Label { + fn get_or_forward_declare_label(&mut self, code_offset: u32) -> Option