-
Notifications
You must be signed in to change notification settings - Fork 40
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #417 from msft-jlange/tlb_flush
cpu/tlb: do not broadcast per-cpu TLB flushes
- Loading branch information
Showing
7 changed files
with
63 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,8 @@ | |
// Author: Joerg Roedel <[email protected]> | ||
|
||
use crate::address::{Address, VirtAddr}; | ||
use crate::cpu::control_regs::{read_cr4, write_cr4, CR4Flags}; | ||
|
||
use core::arch::asm; | ||
|
||
const INVLPGB_VALID_VA: u64 = 1u64 << 0; | ||
|
@@ -50,6 +52,21 @@ pub fn flush_tlb_global_sync() { | |
do_tlbsync(); | ||
} | ||
|
||
pub fn flush_tlb_global_percpu() { | ||
let cr4 = read_cr4(); | ||
write_cr4(cr4 ^ CR4Flags::PGE); | ||
write_cr4(cr4); | ||
} | ||
|
||
pub fn flush_address_percpu(va: VirtAddr) { | ||
let va: u64 = va.page_align().bits() as u64; | ||
unsafe { | ||
asm!("invlpg (%rax)", | ||
in("rax") va, | ||
options(att_syntax)); | ||
} | ||
} | ||
|
||
pub fn flush_address(va: VirtAddr) { | ||
let rax: u64 = (va.page_align().bits() as u64) | ||
| INVLPGB_VALID_VA | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,7 +5,7 @@ | |
// Author: Joerg Roedel <[email protected]> | ||
|
||
use crate::address::{Address, VirtAddr}; | ||
use crate::cpu::flush_tlb_global_sync; | ||
use crate::cpu::{flush_tlb_global_percpu, flush_tlb_global_sync}; | ||
use crate::error::SvsmError; | ||
use crate::locking::RWLock; | ||
use crate::mm::pagetable::{PTEntryFlags, PageTable, PageTablePart, PageTableRef}; | ||
|
@@ -59,6 +59,10 @@ pub struct VMR { | |
/// [`PTEntryFlags`] global to all mappings in this region. This is a | ||
/// combination of [`PTEntryFlags::GLOBAL`] and [`PTEntryFlags::USER`]. | ||
pt_flags: PTEntryFlags, | ||
|
||
/// Indicates that this [`struct VMR`] is visible only on a single CPU | ||
/// and therefore TLB flushes do not require broadcast. | ||
per_cpu: bool, | ||
} | ||
|
||
impl VMR { | ||
|
@@ -82,9 +86,16 @@ impl VMR { | |
tree: RWLock::new(RBTree::new(VMMAdapter::new())), | ||
pgtbl_parts: RWLock::new(Vec::new()), | ||
pt_flags: flags, | ||
per_cpu: false, | ||
} | ||
} | ||
|
||
/// Marks a [`struct VMR`] as being associated with only a single CPU | ||
/// so that TLB flushes do not require broadcast. | ||
pub fn set_per_cpu(&mut self, per_cpu: bool) { | ||
self.per_cpu = per_cpu; | ||
} | ||
|
||
/// Allocated all [`PageTablePart`]s needed to map this region | ||
/// | ||
/// # Returns | ||
|
@@ -425,7 +436,11 @@ impl VMR { | |
let mut cursor = tree.find_mut(&addr); | ||
if let Some(node) = cursor.get() { | ||
self.unmap_vmm(node); | ||
flush_tlb_global_sync(); | ||
if self.per_cpu { | ||
flush_tlb_global_percpu(); | ||
} else { | ||
flush_tlb_global_sync(); | ||
} | ||
} | ||
cursor.remove().ok_or(SvsmError::Mem) | ||
} | ||
|