From 2c12a5ae512689bedf378e9a2270b3fa123a7466 Mon Sep 17 00:00:00 2001 From: aarkegz Date: Tue, 4 Aug 2026 17:18:05 +0000 Subject: [PATCH] feat(addr): add LA57-aware virtual address validity Add sealed fixed LA48, fixed LA57, and runtime virtual-address validity policies, with const construction for fixed-width addresses and runtime validation against CR4.LA57. Propagate address validity through the directly affected descriptor, interrupt, page, range, TLB, and register APIs while keeping existing four-level page-table traversal semantics explicit. Preserve the crate's Rust 1.59 configurations and architectural structure layouts, and add coverage for canonicality, arithmetic, conversions, and generic API behavior. --- Changelog.md | 37 + src/addr.rs | 809 ++++++++++++++---- src/instructions/tables.rs | 49 +- src/instructions/tlb.rs | 37 +- src/lib.rs | 10 +- src/structures/gdt.rs | 187 ++-- src/structures/idt.rs | 326 ++++--- src/structures/mod.rs | 40 +- .../paging/mapper/mapped_page_table.rs | 74 +- src/structures/paging/mapper/mod.rs | 53 +- .../paging/mapper/offset_page_table.rs | 61 +- .../paging/mapper/recursive_page_table.rs | 108 ++- src/structures/paging/page.rs | 279 +++--- src/structures/tss.rs | 69 +- testing/src/tests.rs | 14 + 15 files changed, 1552 insertions(+), 601 deletions(-) diff --git a/Changelog.md b/Changelog.md index cd17fadd5..7ea19e2f8 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,42 @@ # Unreleased +## New Features + +- Add sealed `FixedValidity<48>`, `FixedValidity<57>`, and `RuntimeValidity` policies for virtual + addresses. +- Add the `VirtAddr48`, `VirtAddr57`, and `VirtAddrRT` aliases. `VirtAddr` and all validity-aware + aggregate types use runtime validity by default. +- Propagate virtual-address validity through pages, descriptor pointers, TSS, GDT, IDT, handler + types, interrupt stack frames, TLB commands, and CET legacy bitmap pages. +- Make hardware address APIs use runtime-valid addresses through their existing names. These APIs + are available only on `x86_64` with the `instructions` feature and require ring 0 when they read + `CR4.LA57`. +- Add `is_valid_current` for explicitly checking an existing address against the active mode. + +## Compatibility Notes + +- `VirtAddr::new`, `try_new`, `new_truncate`, and `from_ptr` now construct runtime-valid addresses + and are not const. Fixed 48-bit and 57-bit construction uses `new_const`, `try_new_const`, + `new_truncate_const`, and `from_ptr_const`. +- Checked default construction is unavailable without the `instructions` feature and on non-x86 + targets. Storage-only operations such as `zero`, `new_unsafe`, formatting, and comparison remain + available. +- Hardware APIs that previously returned or accepted `VirtAddr`, `Page`, or descriptor pointers + retain their names but their concrete function-pointer signatures now use runtime validity. +- `Segment64` keeps its existing declaration, but its unparameterized `VirtAddr` methods now use + runtime validity. `HandlerFuncType`, handler aliases, and `InterruptStackFrame` likewise default + to runtime validity. +- Installing a handler or encoding a default TSS descriptor checks the containing object's address + by reading `CR4.LA57`, so these operations require ring 0. Ring 3 callers can use an explicit + fixed validity policy when the address-space contract is fixed. +- Unparameterized `Page` and page ranges are runtime-valid. The existing mapper stack still accepts + only `Page<_, FixedValidity<48>>`, so mapper call sites must use an explicit fixed page policy. +- Aggregate type inference changes wherever a validity parameter was previously omitted, including + GDT, IDT, TSS, descriptor pointers, TLB commands, and CET legacy bitmap pages. +- Validity is checked when a value is created. Later address-space mode changes do not + retroactively invalidate existing values. +- The mapper stack remains limited to four-level page tables and explicitly accepts VA48 pages. + # 0.15.5 – 2026-07-11 This release is compatible with Rust nightlies starting with `nightly-2026-07-10` (this only applies when the `nightly` feature is used). diff --git a/src/addr.rs b/src/addr.rs index e3645034b..113ad5141 100644 --- a/src/addr.rs +++ b/src/addr.rs @@ -4,6 +4,7 @@ use core::convert::TryFrom; use core::fmt; #[cfg(feature = "step_trait")] use core::iter::Step; +use core::marker::PhantomData; use core::ops::{Add, AddAssign, Sub, SubAssign}; #[cfg(feature = "memory_encryption")] use core::sync::atomic::Ordering; @@ -13,10 +14,153 @@ use crate::structures::mem_encrypt::ENC_BIT_MASK; use crate::structures::paging::page_table::PageTableLevel; use crate::structures::paging::{PageOffset, PageTableIndex}; -use bit_field::BitField; use dep_const_fn::const_fn; -const ADDRESS_SPACE_SIZE: u64 = 0x1_0000_0000_0000; +/// A policy for virtual-address validity. +/// +/// This trait is sealed and cannot be implemented outside this crate. Three validities are +/// supported: +/// +/// - [`FixedValidity<48>`]: 48-bit fixed width. +/// - [`FixedValidity<57>`]: 57-bit fixed width. +/// - [`RuntimeValidity`]: Runtime validity. +/// +/// This trait is used to select the validity policy for a [`VirtAddr`]. +/// +/// # Examples +/// +/// ``` +/// use x86_64::{FixedValidity, VirtAddr}; +/// +/// let addr = VirtAddr::>::new_const(0x1000); +/// ``` +/// +/// The set of validity policies is closed: +/// +/// ```compile_fail +/// struct CustomValidity; +/// +/// let _ = x86_64::VirtAddr::::zero(); +/// ``` +pub trait VirtAddrValidity: crate::sealed::VirtAddrValiditySealed {} + +/// A fixed-width virtual-address validity policy. +/// +/// Only `FixedValidity<48>` and `FixedValidity<57>` are supported. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FixedValidity; + +/// The runtime virtual-address validity policy. +/// +/// This policy checks the currently active address-space mode by reading `CR4.LA57`. The policy +/// type itself and operations that do not consult the active mode are available on all targets. +/// Checked construction, canonicalization, and address-producing arithmetic require the +/// `instructions` feature and an `x86_64` target, and they must execute in Ring 0. +#[cfg_attr( + not(all(feature = "instructions", target_arch = "x86_64")), + doc = r#" +Address-producing arithmetic is unavailable when the current address-space mode cannot be read: + +```compile_fail +use x86_64::VirtAddrRT; + +let address = VirtAddrRT::zero(); +let _ = address + 1u64; +``` +"# +)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RuntimeValidity; + +impl VirtAddrValidity for FixedValidity<48> {} +impl VirtAddrValidity for FixedValidity<57> {} +impl VirtAddrValidity for RuntimeValidity {} + +/// A validity policy for which address-producing arithmetic is available. +pub(crate) trait VirtAddrArithmeticValidity: VirtAddrValidity {} + +impl VirtAddrArithmeticValidity for FixedValidity where + FixedValidity: VirtAddrValidity +{ +} + +#[cfg(all(feature = "instructions", target_arch = "x86_64"))] +impl VirtAddrArithmeticValidity for RuntimeValidity {} + +impl crate::sealed::VirtAddrValiditySealed for FixedValidity { + #[inline] + fn bits() -> usize { + BITS + } +} + +impl crate::sealed::VirtAddrValiditySealed for RuntimeValidity { + #[inline] + fn bits() -> usize { + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + { + current_virtual_address_bits() + } + + #[cfg(not(all(feature = "instructions", target_arch = "x86_64")))] + { + // All callers of this function are expected to be disabled on non-x86_64 targets or + // when the instructions feature is disabled. + unreachable!( + "runtime virtual-address width requires x86_64 and the instructions feature" + ) + } + } +} + +/// Returns the number of valid bits for the given validity policy. +#[inline] +fn validity_bits() -> usize { + ::bits() +} + +/// Canonicalizes the given address with the given number of bits. +#[inline] +const fn canonicalize_with_bits(addr: u64, bits: usize) -> u64 { + let shift = 64 - bits; + ((addr << shift) as i64 >> shift) as u64 +} + +/// Tries to create a new canonical virtual address with the given number of bits. +#[inline] +#[rustversion::attr(since(1.61), const)] +fn try_new_with_bits( + addr: u64, + bits: usize, +) -> Result, VirtAddrNotValid> { + let canonicalized = canonicalize_with_bits(addr, bits); + if canonicalized == addr { + Ok(VirtAddr(canonicalized, PhantomData)) + } else { + Err(VirtAddrNotValid(addr)) + } +} + +/// Creates a canonical virtual address by discarding invalid high bits, with the given number of +/// bits. +#[inline] +#[rustversion::attr(since(1.61), const)] +fn new_truncate_with_bits(addr: u64, bits: usize) -> VirtAddr { + VirtAddr(canonicalize_with_bits(addr, bits), PhantomData) +} + +/// Returns the number of valid bits for the currently active address-space mode. +#[cfg(all(feature = "instructions", target_arch = "x86_64"))] +#[inline] +fn current_virtual_address_bits() -> usize { + use crate::registers::control::{Cr4, Cr4Flags}; + + if Cr4::read().contains(Cr4Flags::L5_PAGING) { + 57 + } else { + 48 + } +} /// A canonical 64-bit virtual memory address. /// @@ -25,12 +169,45 @@ const ADDRESS_SPACE_SIZE: u64 = 0x1_0000_0000_0000; /// [`TryFrom`](https://doc.rust-lang.org/std/convert/trait.TryFrom.html) trait can be used for performing conversions /// between `u64` and `usize`. /// -/// On `x86_64`, only the 48 lower bits of a virtual address can be used. The top 16 bits need -/// to be copies of bit 47, i.e. the most significant bit. Addresses that fulfil this criterion -/// are called “canonical”. This type guarantees that it always represents a canonical address. +/// On `x86_64`, virtual addresses are canonical when all bits above the most significant valid bit +/// are copies of that bit. Currently, two address-space modes are supported on `x86_64`: +/// +/// - Four-level paging (48-bit): The most significant valid bit is bit 47. +/// - Five-level paging (57-bit): The most significant valid bit is bit 56. +/// +/// [`VirtAddr`] uses [`VirtAddrValidity`] to create different types of virtual addresses for +/// different modes: +/// +/// - [`VirtAddr48`]: A virtual address that is canonical under four-level paging. (A 48-bit +/// canonical virtual address.) +/// - [`VirtAddr57`]: A virtual address that is canonical under five-level paging. (A 57-bit +/// canonical virtual address) +/// - [`VirtAddrRT`]: A virtual address that is canonical under the currently active address-space +/// mode. Validity is checked only when an address is created. A later address-space mode change +/// does not invalidate existing values. +/// +/// [`VirtAddr48`] and [`VirtAddr57`] provide const-capable constructors and accessors. +/// [`VirtAddrRT`] can be stored, compared, formatted, inspected, and created through +/// [`zero`](Self::zero) or unsafe [`new_unsafe`](Self::new_unsafe) on all targets. Operations that +/// check the current address-space mode or produce a new runtime-valid address require the +/// `instructions` feature and an `x86_64` target, and they must execute in Ring 0. +/// +/// Validity is checked only when an address is created. A later address-space mode change does not +/// invalidate existing values. Operations that subsequently produce a new address check the +/// result against the mode active at that time. Use `is_valid_currently` to explicitly revalidate +/// an existing address when current-mode checks are available. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(transparent)] -pub struct VirtAddr(u64); +pub struct VirtAddr(u64, PhantomData); + +/// A virtual address that is canonical under four-level paging. +pub type VirtAddr48 = VirtAddr>; + +/// A virtual address that is canonical under five-level paging. +pub type VirtAddr57 = VirtAddr>; + +/// A virtual address checked against the current address-space mode when created. +pub type VirtAddrRT = VirtAddr; /// A 64-bit physical memory address. /// @@ -47,10 +224,8 @@ pub struct PhysAddr(u64); /// A passed `u64` was not a valid virtual address. /// -/// This means that bits 48 to 64 are not -/// a valid sign extension and are not null either. So automatic sign extension would have -/// overwritten possibly meaningful bits. This likely indicates a bug, for example an invalid -/// address calculation. +/// Automatic sign extension for the selected validity policy would have overwritten possibly +/// meaningful bits. This likely indicates a bug, for example an invalid address calculation. /// /// Contains the invalid address. pub struct VirtAddrNotValid(pub u64); @@ -63,121 +238,145 @@ impl core::fmt::Debug for VirtAddrNotValid { } } -impl VirtAddr { - /// Creates a new canonical virtual address. +impl VirtAddr> +where + FixedValidity: VirtAddrValidity, +{ + /// Creates a new canonical virtual address, with provided fixed width. /// /// The provided address should already be canonical. If you want to check - /// whether an address is canonical, use [`try_new`](Self::try_new). + /// whether an address is canonical, use [`try_new_const`](Self::try_new_const). /// /// ## Panics /// - /// This function panics if the bits in the range 48 to 64 are invalid - /// (i.e. are not a proper sign extension of bit 47). + /// This function panics if the address is not canonical for the selected fixed width. #[inline] - pub const fn new(addr: u64) -> VirtAddr { + #[rustversion::attr(since(1.61), const)] + pub fn new_const(addr: u64) -> Self { // TODO: Replace with .ok().expect(msg) when that works on stable. - match Self::try_new(addr) { + match Self::try_new_const(addr) { Ok(v) => v, - Err(_) => panic!("virtual address must be sign extended in bits 48 to 64"), + Err(_) => panic!("virtual address must be canonical for the selected fixed width"), } } - /// Tries to create a new canonical virtual address. + /// Tries to create a new canonical virtual address, with provided fixed width. /// - /// This function checks whether the given address is canonical - /// and returns an error otherwise. An address is canonical - /// if bits 48 to 64 are a correct sign - /// extension (i.e. copies of bit 47). - #[inline] - pub const fn try_new(addr: u64) -> Result { - let v = Self::new_truncate(addr); - if v.0 == addr { - Ok(v) - } else { - Err(VirtAddrNotValid(addr)) - } + /// This function checks whether the given address is canonical for the selected fixed width + /// and returns an error otherwise. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn try_new_const(addr: u64) -> Result { + try_new_with_bits(addr, BITS) } - /// Creates a new canonical virtual address, throwing out bits 48..64. + /// Creates a canonical virtual address by discarding invalid high bits, with provided fixed + /// width. /// - /// This function performs sign extension of bit 47 to make the address - /// canonical, overwriting bits 48 to 64. If you want to check whether an - /// address is canonical, use [`new`](Self::new) or [`try_new`](Self::try_new). + /// This function sign-extends the selected fixed-width sign bit. If you want to check whether + /// an address is canonical, use [`new_const`](Self::new_const) or + /// [`try_new_const`](Self::try_new_const). #[inline] - pub const fn new_truncate(addr: u64) -> VirtAddr { - // By doing the right shift as a signed operation (on a i64), it will - // sign extend the value, repeating the leftmost bit. - VirtAddr(((addr << 16) as i64 >> 16) as u64) + #[rustversion::attr(since(1.61), const)] + pub fn new_truncate_const(addr: u64) -> Self { + new_truncate_with_bits(addr, BITS) } - /// Creates a new virtual address, without any checks. + /// Creates a fixed-width virtual address from the given pointer. /// - /// ## Safety + /// The pointer address must be canonical under the selected fixed validity policy. + #[cfg(target_pointer_width = "64")] + #[inline] + pub fn from_ptr_const(ptr: *const T) -> Self { + Self::new_const(ptr as *const () as u64) + } + + /// Aligns the virtual address upwards to the given alignment. /// - /// You must make sure bits 48..64 are equal to bit 47. This is not checked. + /// See the [`align_up`] function for more information. #[inline] - pub const unsafe fn new_unsafe(addr: u64) -> VirtAddr { - VirtAddr(addr) + pub fn align_up(self, align: U) -> Self + where + U: Into, + { + Self::new_truncate_const(align_up(self.0, align.into())) } - /// Creates a virtual address that points to `0`. + /// Aligns the virtual address downwards to the given alignment. + /// + /// See the [`align_down`] function for more information. #[inline] - pub const fn zero() -> VirtAddr { - VirtAddr(0) + pub fn align_down(self, align: U) -> Self + where + U: Into, + { + self.align_down_u64(align.into()) } - /// Converts the address to an `u64`. + /// Aligns the virtual address downwards to the given alignment. + /// + /// This variant accepts the alignment as a `u64` for internal users. #[inline] - pub const fn as_u64(self) -> u64 { - self.0 + #[rustversion::attr(since(1.61), const)] + pub(crate) fn align_down_u64(self, align: u64) -> Self { + Self::new_truncate_const(align_down(self.0, align)) } +} - /// Creates a virtual address from the given pointer - #[cfg(target_pointer_width = "64")] +#[cfg(all(feature = "instructions", target_arch = "x86_64"))] +impl VirtAddr { + /// Creates a new virtual address valid in the current address-space mode. + /// + /// # Panics + /// + /// This function panics if the address is not canonical under the currently active mode. #[inline] - pub fn from_ptr(ptr: *const T) -> Self { - Self::new(ptr as *const () as u64) + pub fn new(addr: u64) -> Self { + match Self::try_new(addr) { + Ok(address) => address, + Err(_) => panic!("virtual address must be canonical in the current address-space mode"), + } } - /// Converts the address to a raw pointer. - #[cfg(target_pointer_width = "64")] + /// Tries to create a virtual address valid in the current address-space mode. + /// + /// This function reads `CR4.LA57` and checks the address using the active canonical width. #[inline] - pub const fn as_ptr(self) -> *const T { - self.as_u64() as *const T + pub fn try_new(addr: u64) -> Result { + try_new_with_bits(addr, current_virtual_address_bits()) } - /// Converts the address to a mutable raw pointer. - #[cfg(target_pointer_width = "64")] + /// Creates a virtual address by canonicalizing it for the current address-space mode. + /// + /// This function reads `CR4.LA57` and sign-extends the active canonical sign bit. #[inline] - pub const fn as_mut_ptr(self) -> *mut T { - self.as_ptr::() as *mut T + pub fn new_truncate(addr: u64) -> Self { + new_truncate_with_bits(addr, current_virtual_address_bits()) } - /// Convenience method for checking if a virtual address is null. + /// Creates a virtual address from the given pointer. + /// + /// The pointer address must be canonical in the current address-space mode. + #[cfg(target_pointer_width = "64")] #[inline] - pub const fn is_null(self) -> bool { - self.0 == 0 + pub fn from_ptr(ptr: *const T) -> Self { + Self::new(ptr as *const () as u64) } /// Aligns the virtual address upwards to the given alignment. /// - /// See the `align_up` function for more information. - /// - /// # Panics - /// - /// This function panics if the resulting address is higher than - /// `0xffff_ffff_ffff_ffff`. + /// The result is canonicalized using the current address-space mode. #[inline] pub fn align_up(self, align: U) -> Self where U: Into, { - VirtAddr::new_truncate(align_up(self.0, align.into())) + Self::new_truncate(align_up(self.0, align.into())) } /// Aligns the virtual address downwards to the given alignment. /// - /// See the `align_down` function for more information. + /// The result is canonicalized using the current address-space mode. #[inline] pub fn align_down(self, align: U) -> Self where @@ -188,62 +387,180 @@ impl VirtAddr { /// Aligns the virtual address downwards to the given alignment. /// - /// See the `align_down` function for more information. + /// This variant accepts the alignment as a `u64` for internal users. #[inline] - pub(crate) const fn align_down_u64(self, align: u64) -> Self { - VirtAddr::new_truncate(align_down(self.0, align)) + pub(crate) fn align_down_u64(self, align: u64) -> Self { + Self::new_truncate(align_down(self.0, align)) } +} - /// Checks whether the virtual address has the demanded alignment. +impl VirtAddr { + /// Creates a new virtual address, without any checks. + /// + /// ## Safety + /// + /// The caller must ensure that `addr` is valid for `V`. This is not checked. #[inline] - pub fn is_aligned(self, align: U) -> bool - where - U: Into, - { - self.is_aligned_u64(align.into()) + #[rustversion::attr(since(1.61), const)] + pub unsafe fn new_unsafe(addr: u64) -> Self { + VirtAddr(addr, PhantomData) } - /// Checks whether the virtual address has the demanded alignment. + /// Creates a virtual address that points to `0`. #[inline] - pub(crate) const fn is_aligned_u64(self, align: u64) -> bool { - self.align_down_u64(align).as_u64() == self.as_u64() + #[rustversion::attr(since(1.61), const)] + pub fn zero() -> Self { + VirtAddr(0, PhantomData) + } + + /// Converts the address to an `u64`. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn as_u64(self) -> u64 { + self.0 + } + + /// Converts the address to a raw pointer. + #[cfg(target_pointer_width = "64")] + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn as_ptr(self) -> *const T { + self.as_u64() as *const T + } + + /// Converts the address to a mutable raw pointer. + #[cfg(target_pointer_width = "64")] + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn as_mut_ptr(self) -> *mut T { + self.as_ptr::() as *mut T + } + + /// Convenience method for checking if a virtual address is null. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn is_null(self) -> bool { + self.0 == 0 } /// Returns the 12-bit page offset of this virtual address. #[inline] - pub const fn page_offset(self) -> PageOffset { + #[rustversion::attr(since(1.61), const)] + pub fn page_offset(self) -> PageOffset { PageOffset::new_truncate(self.0 as u16) } /// Returns the 9-bit level 1 page table index. #[inline] - pub const fn p1_index(self) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn p1_index(self) -> PageTableIndex { PageTableIndex::new_truncate((self.0 >> 12) as u16) } /// Returns the 9-bit level 2 page table index. #[inline] - pub const fn p2_index(self) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn p2_index(self) -> PageTableIndex { PageTableIndex::new_truncate((self.0 >> 12 >> 9) as u16) } /// Returns the 9-bit level 3 page table index. #[inline] - pub const fn p3_index(self) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn p3_index(self) -> PageTableIndex { PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9) as u16) } /// Returns the 9-bit level 4 page table index. #[inline] - pub const fn p4_index(self) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn p4_index(self) -> PageTableIndex { PageTableIndex::new_truncate((self.0 >> 12 >> 9 >> 9 >> 9) as u16) } /// Returns the 9-bit level page table index. #[inline] - pub const fn page_table_index(self, level: PageTableLevel) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn page_table_index(self, level: PageTableLevel) -> PageTableIndex { PageTableIndex::new_truncate((self.0 >> 12 >> ((level as u8 - 1) * 9)) as u16) } +} + +impl VirtAddr { + /// Checks whether the virtual address has the demanded alignment. + #[inline] + pub fn is_aligned(self, align: U) -> bool + where + U: Into, + { + self.is_aligned_u64(align.into()) + } + + /// Checks whether the virtual address has the demanded alignment. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub(crate) fn is_aligned_u64(self, align: u64) -> bool { + align_down(self.0, align) == self.0 + } + + /// Checks whether the address is canonical in the currently active address-space mode. + /// + /// This method checks the address again even though it was valid for its policy when created. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub fn is_valid_currently(self) -> bool { + new_truncate_with_bits::(self.0, current_virtual_address_bits()).0 + == self.0 + } + + /// Creates a checked virtual address for an internal policy-generic API. + /// + /// Runtime policies read the current address-space mode during this construction. + #[inline] + #[cfg_attr( + not(all(feature = "instructions", target_arch = "x86_64")), + allow(dead_code) + )] + pub(crate) fn new_with_validity(addr: u64) -> Self { + match Self::try_new_with_validity(addr) { + Ok(address) => address, + Err(_) => panic!("virtual address must be canonical for its validity policy"), + } + } + + /// Returns the first address in the upper canonical half for this policy. + #[inline] + pub(crate) fn upper_half_start() -> Self { + Self::new_truncate_with_validity(1u64 << (validity_bits::() - 1)) + } + + /// Returns the final address in the lower canonical half for this policy. + #[inline] + pub(crate) fn lower_half_end() -> Self { + unsafe { Self::new_unsafe((1u64 << (validity_bits::() - 1)) - 1) } + } + + /// Returns the greatest canonical address for this policy. + #[inline] + pub(crate) fn max_value() -> Self { + unsafe { Self::new_unsafe(u64::MAX) } + } + + /// Creates a virtual address from the given pointer. + /// + /// The pointer address must be canonical under the selected validity policy. + #[inline] + pub(crate) fn try_new_with_validity(addr: u64) -> Result { + try_new_with_bits(addr, validity_bits::()) + } + + #[inline] + fn new_truncate_with_validity(addr: u64) -> Self { + VirtAddr( + canonicalize_with_bits(addr, validity_bits::()), + PhantomData, + ) + } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. #[cfg(feature = "step_trait")] @@ -260,12 +577,8 @@ impl VirtAddr { /// function always returns the exact bound, so it doesn't need to return a /// lower and upper bound like steps_between does. pub(crate) fn steps_between_u64(start: &Self, end: &Self) -> Option { - let mut steps = end.0.checked_sub(start.0)?; - - // Mask away extra bits that appear while jumping the gap. - steps &= 0xffff_ffff_ffff; - - Some(steps) + let mask = (1u64 << validity_bits::()) - 1; + (end.0 & mask).checked_sub(start.0 & mask) } // FIXME: Move this into the `Step` impl, once `Step` is stabilized. @@ -277,54 +590,26 @@ impl VirtAddr { /// An implementation of forward_checked that takes u64 instead of usize. #[inline] pub(crate) fn forward_checked_u64(start: Self, count: u64) -> Option { - if count > ADDRESS_SPACE_SIZE { - return None; - } - - let mut addr = start.0.checked_add(count)?; - - match addr.get_bits(47..) { - 0x1 => { - // Jump the gap by sign extending the 47th bit. - addr.set_bits(47.., 0x1ffff); - } - 0x2 => { - // Address overflow - return None; - } - _ => {} + let mask = (1u64 << validity_bits::()) - 1; + let addr = (start.0 & mask).checked_add(count)?; + if addr > mask { + None + } else { + Some(Self::new_truncate_with_validity(addr)) } - - Some(unsafe { Self::new_unsafe(addr) }) } /// An implementation of backward_checked that takes u64 instead of usize. #[cfg(feature = "step_trait")] #[inline] pub(crate) fn backward_checked_u64(start: Self, count: u64) -> Option { - if count > ADDRESS_SPACE_SIZE { - return None; - } - - let mut addr = start.0.checked_sub(count)?; - - match addr.get_bits(47..) { - 0x1fffe => { - // Jump the gap by sign extending the 47th bit. - addr.set_bits(47.., 0); - } - 0x1fffd => { - // Address underflow - return None; - } - _ => {} - } - - Some(unsafe { Self::new_unsafe(addr) }) + let mask = (1u64 << validity_bits::()) - 1; + let addr = (start.0 & mask).checked_sub(count)?; + Some(Self::new_truncate_with_validity(addr)) } } -impl fmt::Debug for VirtAddr { +impl fmt::Debug for VirtAddr { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_tuple("VirtAddr") .field(&format_args!("{:#x}", self.0)) @@ -332,42 +617,42 @@ impl fmt::Debug for VirtAddr { } } -impl fmt::Binary for VirtAddr { +impl fmt::Binary for VirtAddr { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Binary::fmt(&self.0, f) } } -impl fmt::LowerHex for VirtAddr { +impl fmt::LowerHex for VirtAddr { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::LowerHex::fmt(&self.0, f) } } -impl fmt::Octal for VirtAddr { +impl fmt::Octal for VirtAddr { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Octal::fmt(&self.0, f) } } -impl fmt::UpperHex for VirtAddr { +impl fmt::UpperHex for VirtAddr { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::UpperHex::fmt(&self.0, f) } } -impl fmt::Pointer for VirtAddr { +impl fmt::Pointer for VirtAddr { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fmt::Pointer::fmt(&(self.0 as *const ()), f) } } -impl Add for VirtAddr { +impl Add for VirtAddr { type Output = Self; #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] @@ -383,7 +668,7 @@ impl Add for VirtAddr { /// canonical address. #[inline] fn add(self, rhs: u64) -> Self::Output { - VirtAddr::try_new( + Self::try_new_with_validity( self.0 .checked_add(rhs) .expect("attempt to add with overflow"), @@ -392,7 +677,7 @@ impl Add for VirtAddr { } } -impl AddAssign for VirtAddr { +impl AddAssign for VirtAddr { #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] /// Add an offset to a virtual address. /// @@ -410,7 +695,7 @@ impl AddAssign for VirtAddr { } } -impl Sub for VirtAddr { +impl Sub for VirtAddr { type Output = Self; #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] @@ -426,7 +711,7 @@ impl Sub for VirtAddr { /// canonical address. #[inline] fn sub(self, rhs: u64) -> Self::Output { - VirtAddr::try_new( + Self::try_new_with_validity( self.0 .checked_sub(rhs) .expect("attempt to subtract with overflow"), @@ -435,7 +720,7 @@ impl Sub for VirtAddr { } } -impl SubAssign for VirtAddr { +impl SubAssign for VirtAddr { #[cfg_attr(not(feature = "step_trait"), allow(rustdoc::broken_intra_doc_links))] /// Subtract an offset from a virtual address. /// @@ -453,7 +738,7 @@ impl SubAssign for VirtAddr { } } -impl Sub for VirtAddr { +impl Sub> for VirtAddr { type Output = u64; /// Returns the difference between two addresses. @@ -462,15 +747,64 @@ impl Sub for VirtAddr { /// /// This function will panic on overflow. #[inline] - fn sub(self, rhs: VirtAddr) -> Self::Output { + fn sub(self, rhs: VirtAddr) -> Self::Output { self.as_u64() .checked_sub(rhs.as_u64()) .expect("attempt to subtract with overflow") } } +impl From for VirtAddr57 { + #[inline] + fn from(address: VirtAddr48) -> Self { + unsafe { Self::new_unsafe(address.as_u64()) } + } +} + +impl From for VirtAddrRT { + #[inline] + fn from(address: VirtAddr48) -> Self { + unsafe { Self::new_unsafe(address.as_u64()) } + } +} + +impl From for VirtAddr57 { + #[inline] + fn from(address: VirtAddrRT) -> Self { + unsafe { Self::new_unsafe(address.as_u64()) } + } +} + +impl TryFrom for VirtAddr48 { + type Error = VirtAddrNotValid; + + #[inline] + fn try_from(address: VirtAddr57) -> Result { + Self::try_new_const(address.as_u64()) + } +} + +impl TryFrom for VirtAddr48 { + type Error = VirtAddrNotValid; + + #[inline] + fn try_from(address: VirtAddrRT) -> Result { + Self::try_new_const(address.as_u64()) + } +} + +#[cfg(all(feature = "instructions", target_arch = "x86_64"))] +impl TryFrom for VirtAddrRT { + type Error = VirtAddrNotValid; + + #[inline] + fn try_from(address: VirtAddr57) -> Result { + Self::try_new(address.as_u64()) + } +} + #[cfg(feature = "step_trait")] -impl Step for VirtAddr { +impl Step for VirtAddr { #[inline] fn steps_between(start: &Self, end: &Self) -> (usize, Option) { Self::steps_between_impl(start, end) @@ -512,7 +846,10 @@ impl Step for VirtAddr { } #[cfg(kani)] -impl kani::Arbitrary for VirtAddr { +impl kani::Arbitrary for VirtAddr> +where + FixedValidity: VirtAddrValidity, +{ fn any() -> Self { Self::new_truncate(kani::any()) } @@ -785,10 +1122,148 @@ pub const fn align_up(addr: u64, align: u64) -> u64 { mod tests { use super::*; + /// Constructs an unchecked VA48 value for tests of internal arithmetic behavior. + /// + /// This helper preserves the concise tuple-constructor spelling used by the original tests. + #[allow(non_snake_case)] + fn VirtAddr(addr: u64) -> VirtAddr48 { + unsafe { VirtAddr48::new_unsafe(addr) } + } + + #[rustversion::since(1.61)] + const UNSAFE_VIRT_ADDR_48: VirtAddr48 = unsafe { VirtAddr48::new_unsafe(0x1234) }; + #[rustversion::since(1.61)] + const UNSAFE_VIRT_ADDR_57: VirtAddr57 = unsafe { VirtAddr57::new_unsafe(0x1234) }; + #[rustversion::since(1.61)] + const UNSAFE_VIRT_ADDR_RT: VirtAddrRT = unsafe { VirtAddrRT::new_unsafe(0x1234) }; + + #[rustversion::since(1.61)] + #[test] + fn virtaddr_default_is_runtime_valid() { + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + let _: fn(u64) -> VirtAddrRT = crate::VirtAddr::new; + + const FIXED48: VirtAddr48 = VirtAddr48::new_const(0x1234); + const FIXED57: VirtAddr57 = VirtAddr57::new_const(0x00ff_0000_0000_0000); + assert_eq!(FIXED48.as_u64(), 0x1234); + assert_eq!(FIXED57.as_u64(), 0x00ff_0000_0000_0000); + } + + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[test] + fn runtime_virtaddr_arithmetic_traits_are_available() { + fn assert_arithmetic() + where + T: Add + AddAssign + Sub + SubAssign, + { + } + + assert_arithmetic::(); + + #[cfg(feature = "step_trait")] + { + fn assert_step() {} + assert_step::(); + } + } + + #[test] + fn fixed_virtaddr_canonicality() { + assert!(VirtAddr48::try_new_const(0x0000_7fff_ffff_ffff).is_ok()); + assert!(VirtAddr48::try_new_const(0x0000_8000_0000_0000).is_err()); + assert!(VirtAddr48::try_new_const(0xffff_8000_0000_0000).is_ok()); + + assert!(VirtAddr57::try_new_const(0x00ff_ffff_ffff_ffff).is_ok()); + assert!(VirtAddr57::try_new_const(0x0100_0000_0000_0000).is_err()); + assert!(VirtAddr57::try_new_const(0xff00_0000_0000_0000).is_ok()); + assert!(VirtAddr57::try_new_const(0x0000_8000_0000_0000).is_ok()); + } + + #[test] + fn pure_canonicalization_uses_selected_width() { + assert_eq!(canonicalize_with_bits(1 << 47, 48), 0xffff_8000_0000_0000); + assert_eq!(canonicalize_with_bits(1 << 56, 57), 0xff00_0000_0000_0000); + assert_eq!(canonicalize_with_bits((1 << 47) - 1, 48), (1 << 47) - 1); + assert_eq!(canonicalize_with_bits((1 << 56) - 1, 57), (1 << 56) - 1); + } + + #[test] + #[cfg(feature = "step_trait")] + fn fixed_virtaddr_operations_use_policy_width() { + let low_end = VirtAddr57::new_const(0x00ff_ffff_ffff_fffe); + assert_eq!((low_end + 1).as_u64(), 0x00ff_ffff_ffff_ffff); + assert_eq!( + Step::forward(low_end + 1, 1).as_u64(), + 0xff00_0000_0000_0000 + ); + assert_eq!( + Step::backward(VirtAddr57::new_const(0xff00_0000_0000_0000), 1).as_u64(), + 0x00ff_ffff_ffff_ffff + ); + assert_eq!( + VirtAddr57::new_const(0x00ff_ffff_ffff_ffff) + .align_up(2u64) + .as_u64(), + 0xff00_0000_0000_0000 + ); + } + + #[test] + fn fixed_virtaddr_conversions_preserve_or_check_values() { + let address48 = VirtAddr48::new_const(0xffff_8000_0000_1234); + let address57 = VirtAddr57::from(address48); + let address_rt = VirtAddrRT::from(address48); + + assert_eq!(address57.as_u64(), address48.as_u64()); + assert_eq!(address_rt.as_u64(), address48.as_u64()); + assert_eq!(VirtAddr48::try_from(address57).unwrap(), address48); + assert_eq!(VirtAddr48::try_from(address_rt).unwrap(), address48); + assert_eq!(VirtAddr57::from(address_rt), address57); + + let la57_only = VirtAddr57::new_const(0x0000_8000_0000_0000); + assert!(VirtAddr48::try_from(la57_only).is_err()); + } + + #[test] + fn virtaddr_policy_layout_is_transparent() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::() + ); + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::() + ); + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::() + ); + } + + #[rustversion::since(1.61)] + #[test] + fn new_unsafe_is_const_for_all_policies() { + assert_eq!(UNSAFE_VIRT_ADDR_48.as_u64(), 0x1234); + assert_eq!(UNSAFE_VIRT_ADDR_57.as_u64(), 0x1234); + assert_eq!(UNSAFE_VIRT_ADDR_RT.as_u64(), 0x1234); + } + #[test] #[should_panic] pub fn add_overflow_virtaddr() { - let _ = VirtAddr::new(0xffff_ffff_ffff_ffff) + 1; + let _ = VirtAddr48::new_const(0xffff_ffff_ffff_ffff) + 1; } #[test] @@ -800,7 +1275,7 @@ mod tests { #[test] #[should_panic] pub fn sub_underflow_virtaddr() { - let _ = VirtAddr::new(0) - 1; + let _ = VirtAddr48::new_const(0) - 1; } #[test] @@ -811,10 +1286,16 @@ mod tests { #[test] pub fn virtaddr_new_truncate() { - assert_eq!(VirtAddr::new_truncate(0), VirtAddr(0)); - assert_eq!(VirtAddr::new_truncate(1 << 47), VirtAddr(0xfffff << 47)); - assert_eq!(VirtAddr::new_truncate(123), VirtAddr(123)); - assert_eq!(VirtAddr::new_truncate(123 << 47), VirtAddr(0xfffff << 47)); + assert_eq!(VirtAddr48::new_truncate_const(0), VirtAddr(0)); + assert_eq!( + VirtAddr48::new_truncate_const(1 << 47), + VirtAddr(0xfffff << 47) + ); + assert_eq!(VirtAddr48::new_truncate_const(123), VirtAddr(123)); + assert_eq!( + VirtAddr48::new_truncate_const(123 << 47), + VirtAddr(0xfffff << 47) + ); } #[test] @@ -1002,8 +1483,8 @@ mod tests { fn test_virt_addr_align_up() { // Make sure the 47th bit is extended. assert_eq!( - VirtAddr::new(0x7fff_ffff_ffff).align_up(2u64), - VirtAddr::new(0xffff_8000_0000_0000) + VirtAddr48::new_const(0x7fff_ffff_ffff).align_up(2u64), + VirtAddr48::new_const(0xffff_8000_0000_0000) ); } @@ -1011,15 +1492,15 @@ mod tests { fn test_virt_addr_align_down() { // Make sure the 47th bit is extended. assert_eq!( - VirtAddr::new(0xffff_8000_0000_0000).align_down(1u64 << 48), - VirtAddr::new(0) + VirtAddr48::new_const(0xffff_8000_0000_0000).align_down(1u64 << 48), + VirtAddr48::new_const(0) ); } #[test] #[should_panic] fn test_virt_addr_align_up_overflow() { - VirtAddr::new(0xffff_ffff_ffff_ffff).align_up(2u64); + VirtAddr48::new_const(0xffff_ffff_ffff_ffff).align_up(2u64); } #[test] @@ -1034,8 +1515,8 @@ mod tests { let slice = &[1, 2, 3, 4, 5]; // Make sure that from_ptr(slice) is the address of the first element assert_eq!( - VirtAddr::from_ptr(slice.as_slice()), - VirtAddr::from_ptr(&slice[0]) + VirtAddr48::from_ptr_const(slice.as_slice()), + VirtAddr48::from_ptr_const(&slice[0]) ); } } @@ -1075,7 +1556,7 @@ mod proofs { }; if let Some(expected) = expected { // Verify that `expected` is a valid address. - assert!(VirtAddr::try_new(expected).is_ok()); + assert!(VirtAddr48::try_new_const(expected).is_ok()); } // Verify `forward_checked`. let next = Step::forward_checked(start, 1); diff --git a/src/instructions/tables.rs b/src/instructions/tables.rs index 611d61175..34be735f6 100644 --- a/src/instructions/tables.rs +++ b/src/instructions/tables.rs @@ -42,30 +42,51 @@ pub unsafe fn lidt(idt: &DescriptorTablePointer) { } } +/// A raw descriptor-table register value. +/// +/// Assembly writes plain integers here before the base is checked and wrapped in a semantic type. +#[repr(C, packed(2))] +struct RawDescriptorTablePointer { + limit: u16, + base: u64, +} + +#[inline] +fn read_raw_gdt() -> RawDescriptorTablePointer { + let mut pointer = RawDescriptorTablePointer { limit: 0, base: 0 }; + unsafe { + asm!("sgdt [{}]", in(reg) &mut pointer, options(nostack, preserves_flags)); + } + pointer +} + +#[inline] +fn read_raw_idt() -> RawDescriptorTablePointer { + let mut pointer = RawDescriptorTablePointer { limit: 0, base: 0 }; + unsafe { + asm!("sidt [{}]", in(reg) &mut pointer, options(nostack, preserves_flags)); + } + pointer +} + /// Get the address of the current GDT. #[inline] pub fn sgdt() -> DescriptorTablePointer { - let mut gdt: DescriptorTablePointer = DescriptorTablePointer { - limit: 0, - base: VirtAddr::new(0), - }; - unsafe { - asm!("sgdt [{}]", in(reg) &mut gdt, options(nostack, preserves_flags)); + let raw = read_raw_gdt(); + DescriptorTablePointer { + limit: raw.limit, + base: VirtAddr::new(raw.base), } - gdt } /// Get the address of the current IDT. #[inline] pub fn sidt() -> DescriptorTablePointer { - let mut idt: DescriptorTablePointer = DescriptorTablePointer { - limit: 0, - base: VirtAddr::new(0), - }; - unsafe { - asm!("sidt [{}]", in(reg) &mut idt, options(nostack, preserves_flags)); + let raw = read_raw_idt(); + DescriptorTablePointer { + limit: raw.limit, + base: VirtAddr::new(raw.base), } - idt } /// Load the task state register using the `ltr` instruction. diff --git a/src/instructions/tlb.rs b/src/instructions/tlb.rs index d96cc8955..cd501e54f 100644 --- a/src/instructions/tlb.rs +++ b/src/instructions/tlb.rs @@ -8,7 +8,7 @@ use crate::{ page::{NotGiantPageSize, PageRange}, Page, PageSize, Size2MiB, Size4KiB, }, - PrivilegeLevel, VirtAddr, + PrivilegeLevel, RuntimeValidity, VirtAddr, VirtAddrValidity, }; use core::{arch::asm, cmp, convert::TryFrom, fmt}; @@ -30,9 +30,9 @@ pub fn flush_all() { /// The Invalidate PCID Command to execute. #[derive(Debug)] -pub enum InvPcidCommand { +pub enum InvPcidCommand { /// The logical processor invalidates mappings—except global translations—for the linear address and PCID specified. - Address(VirtAddr, Pcid), + Address(VirtAddr, Pcid), /// The logical processor invalidates all mappings—except global translations—associated with the PCID. Single(Pcid), @@ -47,7 +47,7 @@ pub enum InvPcidCommand { // TODO: Remove this in the next breaking release. #[deprecated = "please use `InvPcidCommand` instead"] #[doc(hidden)] -pub type InvPicdCommand = InvPcidCommand; +pub type InvPicdCommand = InvPcidCommand; /// The INVPCID descriptor comprises 128 bits and consists of a PCID and a linear address. /// For INVPCID type 0, the processor uses the full 64 bits of the linear address even outside 64-bit mode; the linear address is not used for other INVPCID types. @@ -99,6 +99,11 @@ impl fmt::Display for PcidTooBig { /// This function is unsafe as it requires CPUID.(EAX=07H, ECX=0H):EBX.INVPCID to be 1. #[inline] pub unsafe fn flush_pcid(command: InvPcidCommand) { + unsafe { flush_pcid_inner(command) } +} + +#[inline] +unsafe fn flush_pcid_inner(command: InvPcidCommand) { let mut desc = InvpcidDescriptor { pcid: 0, address: 0, @@ -225,12 +230,13 @@ impl Invlpgb { /// A builder struct to construct the parameters for the `invlpgb` instruction. #[derive(Debug, Clone)] #[must_use] -pub struct InvlpgbFlushBuilder<'a, S = Size4KiB> +pub struct InvlpgbFlushBuilder<'a, S = Size4KiB, V = RuntimeValidity> where S: NotGiantPageSize, + V: VirtAddrValidity, { invlpgb: &'a Invlpgb, - page_range: Option>, + page_range: Option>, pcid: Option, asid: Option, include_global: bool, @@ -238,15 +244,16 @@ where include_nested_translations: bool, } -impl<'a, S> InvlpgbFlushBuilder<'a, S> +impl<'a, S, V> InvlpgbFlushBuilder<'a, S, V> where S: NotGiantPageSize, + V: VirtAddrValidity, { /// Flush a range of pages. /// /// If the range doesn't fit within `invlpgb_count_max`, `invlpgb` is /// executed multiple times. - pub fn pages(self, page_range: PageRange) -> InvlpgbFlushBuilder<'a, T> + pub fn pages(self, page_range: PageRange) -> InvlpgbFlushBuilder<'a, T, V> where T: NotGiantPageSize, { @@ -317,11 +324,12 @@ where if let Some(mut pages) = self.page_range { while !pages.is_empty() { // Calculate out how many pages we still need to flush. - let count = Page::::steps_between_impl(&pages.start, &pages.end).0; + let count = Page::::steps_between_impl(&pages.start, &pages.end).0; // Make sure that we never jump the gap in the address space when flushing. - let second_half_start = - Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let second_half_start = unsafe { + Page::::from_start_address_unchecked(VirtAddr::::upper_half_start()) + }; let count = if pages.start < second_half_start { let count_to_second_half = Page::steps_between_impl(&pages.start, &second_half_start).0; @@ -355,7 +363,7 @@ where } } else { unsafe { - flush_broadcast::( + flush_broadcast::( None, self.pcid, self.asid, @@ -389,8 +397,8 @@ impl fmt::Display for AsidOutOfRangeError { /// See `INVLPGB` in AMD64 Architecture Programmer's Manual Volume 3 #[inline] -unsafe fn flush_broadcast( - va_and_count: Option<(Page, u16)>, +unsafe fn flush_broadcast( + va_and_count: Option<(Page, u16)>, pcid: Option, asid: Option, include_global: bool, @@ -398,6 +406,7 @@ unsafe fn flush_broadcast( include_nested_translations: bool, ) where S: NotGiantPageSize, + V: VirtAddrValidity, { let mut rax = 0; let mut ecx = 0; diff --git a/src/lib.rs b/src/lib.rs index 6e682ac4d..433f99eb2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,10 @@ #![deny(missing_debug_implementations)] #![deny(unsafe_op_in_unsafe_fn)] -pub use crate::addr::{align_down, align_up, PhysAddr, VirtAddr}; +pub use crate::addr::{ + align_down, align_up, FixedValidity, PhysAddr, RuntimeValidity, VirtAddr, VirtAddr48, + VirtAddr57, VirtAddrRT, VirtAddrValidity, +}; pub mod addr; pub mod instructions; @@ -70,4 +73,9 @@ impl PrivilegeLevel { pub(crate) mod sealed { pub trait Sealed {} + + pub trait VirtAddrValiditySealed: Copy + Ord + core::hash::Hash { + /// Returns the number of valid bits in the virtual address. + fn bits() -> usize; + } } diff --git a/src/structures/gdt.rs b/src/structures/gdt.rs index bf267e5bd..b163b39b4 100644 --- a/src/structures/gdt.rs +++ b/src/structures/gdt.rs @@ -2,10 +2,10 @@ pub use crate::registers::segmentation::SegmentSelector; use crate::structures::tss::{InvalidIoMap, TaskStateSegment}; -use crate::PrivilegeLevel; +use crate::{PrivilegeLevel, RuntimeValidity, VirtAddrValidity}; use bit_field::BitField; use bitflags::bitflags; -use core::{cmp, fmt, mem}; +use core::{cmp, fmt, marker::PhantomData, mem}; // imports for intra-doc links #[cfg(doc)] use crate::registers::segmentation::{Segment, CS, SS}; @@ -105,29 +105,30 @@ impl fmt::Debug for Entry { /// ``` #[derive(Debug, Clone)] -pub struct GlobalDescriptorTable { +pub struct GlobalDescriptorTable { table: [Entry; MAX], len: usize, + validity: PhantomData, } -impl GlobalDescriptorTable { +impl GlobalDescriptorTable<8, RuntimeValidity> { /// Creates an empty GDT with the default length of 8. pub const fn new() -> Self { Self::empty() } } -impl Default for GlobalDescriptorTable { +impl Default for GlobalDescriptorTable { #[inline] fn default() -> Self { - Self::new() + Self::empty_with_validity() } } -impl GlobalDescriptorTable { +impl GlobalDescriptorTable { /// Creates an empty GDT which can hold `MAX` number of [`Entry`]s. #[inline] - pub const fn empty() -> Self { + pub const fn empty_with_validity() -> Self { // TODO: Replace with compiler error when feature(generic_const_exprs) is stable. assert!(MAX > 0, "A GDT cannot have 0 entries"); assert!(MAX <= (1 << 13), "A GDT can only have at most 2^13 entries"); @@ -138,6 +139,7 @@ impl GlobalDescriptorTable { Self { table: [NULL; MAX], len: 1, + validity: PhantomData, } } @@ -158,9 +160,9 @@ impl GlobalDescriptorTable { allow(rustdoc::broken_intra_doc_links) )] #[inline] - pub const fn from_raw_entries(slice: &[u64]) -> Self { + pub const fn from_raw_entries_with_validity(slice: &[u64]) -> Self { let len = slice.len(); - let mut table = Self::empty().table; + let mut table = Self::empty_with_validity().table; let mut idx = 0; assert!(len > 0, "cannot initialize GDT with empty slice"); @@ -175,7 +177,11 @@ impl GlobalDescriptorTable { idx += 1; } - Self { table, len } + Self { + table, + len, + validity: PhantomData, + } } /// Get a reference to the internal [`Entry`] table. @@ -214,37 +220,6 @@ impl GlobalDescriptorTable { SegmentSelector::new(index as u16, entry.dpl()) } - /// Loads the GDT in the CPU using the `lgdt` instruction. This does **not** alter any of the - /// segment registers; you **must** (re)load them yourself using [the appropriate - /// functions](crate::instructions::segmentation): - /// [`SS::set_reg()`] and [`CS::set_reg()`]. - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub fn load(&'static self) { - // SAFETY: static lifetime ensures no modification after loading. - unsafe { self.load_unsafe() }; - } - - /// Loads the GDT in the CPU using the `lgdt` instruction. This does **not** alter any of the - /// segment registers; you **must** (re)load them yourself using [the appropriate - /// functions](crate::instructions::segmentation): - /// [`SS::set_reg()`] and [`CS::set_reg()`]. - /// - /// # Safety - /// - /// Unlike `load` this function will not impose a static lifetime constraint - /// this means its up to the user to ensure that there will be no modifications - /// after loading and that the GDT will live for as long as it's loaded. - /// - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub unsafe fn load_unsafe(&self) { - use crate::instructions::tables::lgdt; - unsafe { - lgdt(&self.pointer()); - } - } - #[inline] #[rustversion::attr(since(1.83), const)] fn push(&mut self, value: u64) -> usize { @@ -265,14 +240,55 @@ impl GlobalDescriptorTable { /// Creates the descriptor pointer for this table. This pointer can only be /// safely used if the table is never modified or destroyed while in use. #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - fn pointer(&self) -> super::DescriptorTablePointer { + fn pointer(&self) -> super::DescriptorTablePointer + where + V: VirtAddrValidity, + { super::DescriptorTablePointer { - base: crate::VirtAddr::new(self.table.as_ptr() as u64), + base: crate::VirtAddr::::new_with_validity(self.table.as_ptr() as u64), limit: self.limit(), } } } +impl GlobalDescriptorTable { + /// Loads the GDT in the CPU using the `lgdt` instruction. + /// + /// The static lifetime ensures that the table is not destroyed while loaded. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub fn load(&'static self) { + unsafe { self.load_unsafe() }; + } + + /// Loads the GDT without imposing a static lifetime. + /// + /// # Safety + /// + /// The caller must keep the GDT alive and unmodified while it is loaded. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub unsafe fn load_unsafe(&self) { + unsafe { crate::instructions::tables::lgdt(&self.pointer()) }; + } + + /// Creates an empty runtime-valid GDT with the selected capacity. + /// + /// This method preserves the legacy `GlobalDescriptorTable::::empty` API. + #[inline] + pub const fn empty() -> Self { + Self::empty_with_validity() + } + + /// Forms a runtime-valid GDT from a slice of raw entries. + /// + /// This method preserves the legacy `from_raw_entries` API. + #[inline] + pub const fn from_raw_entries(slice: &[u64]) -> Self { + Self::from_raw_entries_with_validity(slice) + } +} + /// A 64-bit mode segment descriptor. /// /// Segmentation is no longer supported in 64-bit mode, so most of the descriptor @@ -431,6 +447,16 @@ impl Descriptor { unsafe { Self::tss_segment_unchecked(tss) } } + /// Creates a TSS system descriptor with the selected validity type. + #[inline] + pub fn tss_segment_with_validity(tss: &'static TaskStateSegment) -> Descriptor + where + V: VirtAddrValidity, + { + // SAFETY: The pointer is derived from a &'static reference, which ensures its validity. + unsafe { Self::tss_segment_unchecked_with_validity(tss) } + } + /// Similar to [`Descriptor::tss_segment`], but unsafe since it does not enforce a lifetime /// constraint on the provided TSS. /// @@ -443,6 +469,22 @@ impl Descriptor { unsafe { Self::tss_segment_raw(tss, 0) } } + /// Creates a TSS descriptor with the selected validity type from a raw pointer. + /// + /// # Safety + /// The caller must ensure that the passed pointer is valid for as long as the descriptor is + /// being used. + #[inline] + pub unsafe fn tss_segment_unchecked_with_validity( + tss: *const TaskStateSegment, + ) -> Descriptor + where + V: VirtAddrValidity, + { + // SAFETY: if iomap_size is zero, there are no requirements to uphold. + unsafe { Self::tss_segment_raw(tss, 0) } + } + /// Creates a TSS system descriptor for the given TSS, setting up the IO permissions bitmap. /// /// # Example @@ -450,25 +492,38 @@ impl Descriptor { /// ``` /// use x86_64::structures::gdt::Descriptor; /// use x86_64::structures::tss::TaskStateSegment; + /// use x86_64::FixedValidity; /// /// /// A helper that places some I/O map bytes behind a TSS. /// #[repr(C)] /// struct TssWithIOMap { - /// tss: TaskStateSegment, + /// tss: TaskStateSegment>, /// iomap: [u8; 5], /// } /// - /// static TSS: TssWithIOMap = TssWithIOMap { - /// tss: TaskStateSegment::new(), + /// let tss = Box::leak(Box::new(TssWithIOMap { + /// tss: TaskStateSegment::new_with_validity(), /// iomap: [0xff, 0xff, 0x00, 0x80, 0xff], - /// }; + /// })); /// - /// let tss = Descriptor::tss_segment_with_iomap(&TSS.tss, &TSS.iomap).unwrap(); + /// let descriptor = + /// Descriptor::tss_segment_with_iomap_with_validity(&tss.tss, &tss.iomap).unwrap(); /// ``` pub fn tss_segment_with_iomap( tss: &'static TaskStateSegment, iomap: &'static [u8], ) -> Result { + Self::tss_segment_with_iomap_with_validity(tss, iomap) + } + + /// Creates a TSS descriptor with an I/O bitmap and the selected validity type. + pub fn tss_segment_with_iomap_with_validity( + tss: &'static TaskStateSegment, + iomap: &'static [u8], + ) -> Result + where + V: VirtAddrValidity, + { if iomap.len() > 8193 { return Err(InvalidIoMap::TooLong { len: iomap.len() }); } @@ -508,10 +563,13 @@ impl Descriptor { /// There must be a valid IO map at `(tss as *const u8).offset(tss.iomap_base)` /// of length `iomap_size`, with the terminating `0xFF` byte. Additionally, `iomap_base` must /// not exceed `0xDFFF`. - unsafe fn tss_segment_raw(tss: *const TaskStateSegment, iomap_size: u16) -> Descriptor { + unsafe fn tss_segment_raw(tss: *const TaskStateSegment, iomap_size: u16) -> Descriptor + where + V: VirtAddrValidity, + { use self::DescriptorFlags as Flags; - let ptr = tss as u64; + let ptr = crate::VirtAddr::::new_with_validity(tss as u64).as_u64(); let mut low = Flags::PRESENT.bits(); // base @@ -521,7 +579,7 @@ impl Descriptor { let iomap_limit = u64::from(unsafe { (*tss).iomap_base }) + u64::from(iomap_size); low.set_bits( 0..16, - cmp::max(mem::size_of::() as u64, iomap_limit) - 1, + cmp::max(mem::size_of::>() as u64, iomap_limit) - 1, ); // type (0b1001 = available 64-bit tss) low.set_bits(40..44, 0b1001); @@ -538,6 +596,23 @@ mod tests { use super::DescriptorFlags as Flags; use super::*; + #[test] + fn policy_does_not_change_gdt_layout() { + assert_eq!(mem::size_of::(), 72); + assert_eq!( + mem::size_of::>>(), + 72 + ); + assert_eq!( + mem::size_of::>(), + 72 + ); + assert_eq!( + mem::align_of::>(), + 8 + ); + } + #[test] #[rustfmt::skip] pub fn linux_kernel_defaults() { @@ -563,11 +638,13 @@ mod tests { gdt } - static TSS: TaskStateSegment = TaskStateSegment::new(); + fn tss() -> &'static TaskStateSegment> { + Box::leak(Box::new(TaskStateSegment::new_with_validity())) + } fn make_full_gdt() -> GlobalDescriptorTable { let mut gdt = make_six_entry_gdt(); - gdt.append(Descriptor::tss_segment(&TSS)); + gdt.append(Descriptor::tss_segment_with_validity(tss())); assert_eq!(gdt.len, 8); gdt } @@ -597,7 +674,7 @@ mod tests { let mut gdt = make_six_entry_gdt(); gdt.append(Descriptor::user_data_segment()); // We have one free slot, but the GDT requires two - gdt.append(Descriptor::tss_segment(&TSS)); + gdt.append(Descriptor::tss_segment_with_validity(tss())); } #[test] diff --git a/src/structures/idt.rs b/src/structures/idt.rs index f15cedcce..ea990667d 100644 --- a/src/structures/idt.rs +++ b/src/structures/idt.rs @@ -21,7 +21,7 @@ //! These types are defined for the compatibility with the Nightly Rust build. use crate::registers::rflags::RFlags; -use crate::{PrivilegeLevel, VirtAddr}; +use crate::{PrivilegeLevel, RuntimeValidity, VirtAddr, VirtAddrValidity}; use bit_field::BitField; use bitflags::bitflags; use core::convert::TryFrom; @@ -54,7 +54,7 @@ use super::gdt::SegmentSelector; #[derive(Clone, Debug)] #[repr(C)] #[repr(align(16))] -pub struct InterruptDescriptorTable { +pub struct InterruptDescriptorTable { /// A divide error (`#DE`) occurs when the denominator of a DIV instruction or /// an IDIV instruction is 0. A `#DE` also occurs if the result is too large to be /// represented in the destination. @@ -62,7 +62,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#DE`. /// /// The vector number of the `#DE` exception is 0. - pub divide_error: Entry, + pub divide_error: Entry, V>, /// When the debug-exception mechanism is enabled, a `#DB` exception can occur under any /// of the following circumstances: @@ -94,7 +94,7 @@ pub struct InterruptDescriptorTable { /// instruction pointer points to the instruction after the one that caused the `#DB`. /// /// The vector number of the `#DB` exception is 1. - pub debug: Entry, + pub debug: Entry, V>, /// An non maskable interrupt exception (NMI) occurs as a result of system logic /// signaling a non-maskable interrupt to the processor. @@ -104,7 +104,7 @@ pub struct InterruptDescriptorTable { /// boundary where the NMI was recognized. /// /// The vector number of the NMI exception is 2. - pub non_maskable_interrupt: Entry, + pub non_maskable_interrupt: Entry, V>, /// A breakpoint (`#BP`) exception occurs when an `INT3` instruction is executed. The /// `INT3` is normally used by debug software to set instruction breakpoints by replacing @@ -112,7 +112,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the byte after the `INT3` instruction. /// /// The vector number of the `#BP` exception is 3. - pub breakpoint: Entry, + pub breakpoint: Entry, V>, /// An overflow exception (`#OF`) occurs as a result of executing an `INTO` instruction /// while the overflow bit in `RFLAGS` is set to 1. @@ -121,7 +121,7 @@ pub struct InterruptDescriptorTable { /// instruction that caused the `#OF`. /// /// The vector number of the `#OF` exception is 4. - pub overflow: Entry, + pub overflow: Entry, V>, /// A bound-range exception (`#BR`) exception can occur as a result of executing /// the `BOUND` instruction. The `BOUND` instruction compares an array index (first @@ -131,7 +131,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the `BOUND` instruction that caused the `#BR`. /// /// The vector number of the `#BR` exception is 5. - pub bound_range_exceeded: Entry, + pub bound_range_exceeded: Entry, V>, /// An invalid opcode exception (`#UD`) occurs when an attempt is made to execute an /// invalid or undefined opcode. The validity of an opcode often depends on the @@ -165,7 +165,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#UD`. /// /// The vector number of the `#UD` exception is 6. - pub invalid_opcode: Entry, + pub invalid_opcode: Entry, V>, /// A device not available exception (`#NM`) occurs under any of the following conditions: /// @@ -182,7 +182,7 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#NM`. /// /// The vector number of the `#NM` exception is 7. - pub device_not_available: Entry, + pub device_not_available: Entry, V>, /// A double fault (`#DF`) exception can occur when a second exception occurs during /// the handling of a prior (first) exception or interrupt handler. @@ -216,14 +216,14 @@ pub struct InterruptDescriptorTable { /// and the program cannot be restarted. /// /// The vector number of the `#DF` exception is 8. - pub double_fault: Entry, + pub double_fault: Entry, V>, /// This interrupt vector is reserved. It is for a discontinued exception originally used /// by processors that supported external x87-instruction coprocessors. On those processors, /// the exception condition is caused by an invalid-segment or invalid-page access on an /// x87-instruction coprocessor-instruction operand. On current processors, this condition /// causes a general-protection exception to occur. - coprocessor_segment_overrun: Entry, + coprocessor_segment_overrun: Entry, V>, /// An invalid TSS exception (`#TS`) occurs only as a result of a control transfer through /// a gate descriptor that results in an invalid stack-segment reference using an `SS` @@ -233,7 +233,7 @@ pub struct InterruptDescriptorTable { /// points to the control-transfer instruction that caused the `#TS`. /// /// The vector number of the `#TS` exception is 10. - pub invalid_tss: Entry, + pub invalid_tss: Entry, V>, /// An segment-not-present exception (`#NP`) occurs when an attempt is made to load a /// segment or gate with a clear present bit. @@ -243,7 +243,7 @@ pub struct InterruptDescriptorTable { /// that loaded the segment selector resulting in the `#NP`. /// /// The vector number of the `#NP` exception is 11. - pub segment_not_present: Entry, + pub segment_not_present: Entry, V>, /// An stack segment exception (`#SS`) can occur in the following situations: /// @@ -260,7 +260,7 @@ pub struct InterruptDescriptorTable { /// caused the `#SS`. /// /// The vector number of the `#NP` exception is 12. - pub stack_segment_fault: Entry, + pub stack_segment_fault: Entry, V>, /// A general protection fault (`#GP`) can occur in various situations. Common causes include: /// @@ -276,7 +276,7 @@ pub struct InterruptDescriptorTable { /// the instruction that caused the `#GP`. /// /// The vector number of the `#GP` exception is 13. - pub general_protection_fault: Entry, + pub general_protection_fault: Entry, V>, /// A page fault (`#PF`) can occur during a memory access in any of the following situations: /// @@ -297,10 +297,10 @@ pub struct InterruptDescriptorTable { /// [`PageFaultErrorCode`](struct.PageFaultErrorCode.html) struct. /// /// The vector number of the `#PF` exception is 14. - pub page_fault: Entry, + pub page_fault: Entry, V>, /// vector nr. 15 - reserved_1: Entry, + reserved_1: Entry, V>, /// The x87 Floating-Point Exception-Pending exception (`#MF`) is used to handle unmasked x87 /// floating-point exceptions. In 64-bit mode, the x87 floating point unit is not used @@ -308,7 +308,7 @@ pub struct InterruptDescriptorTable { /// compatibility mode. /// /// The vector number of the `#MF` exception is 16. - pub x87_floating_point: Entry, + pub x87_floating_point: Entry, V>, /// An alignment check exception (`#AC`) occurs when an unaligned-memory data reference /// is performed while alignment checking is enabled. An `#AC` can occur only when CPL=3. @@ -317,7 +317,7 @@ pub struct InterruptDescriptorTable { /// instruction that caused the `#AC`. /// /// The vector number of the `#AC` exception is 17. - pub alignment_check: Entry, + pub alignment_check: Entry, V>, /// The machine check exception (`#MC`) is model specific. Processor implementations /// are not required to support the `#MC` exception, and those implementations that do @@ -326,7 +326,7 @@ pub struct InterruptDescriptorTable { /// There is no reliable way to restart the program. /// /// The vector number of the `#MC` exception is 18. - pub machine_check: Entry, + pub machine_check: Entry, V>, /// The SIMD Floating-Point Exception (`#XF`) is used to handle unmasked SSE /// floating-point exceptions. The SSE floating-point exceptions reported by @@ -342,10 +342,10 @@ pub struct InterruptDescriptorTable { /// The saved instruction pointer points to the instruction that caused the `#XF`. /// /// The vector number of the `#XF` exception is 19. - pub simd_floating_point: Entry, + pub simd_floating_point: Entry, V>, /// vector nr. 20 - pub virtualization: Entry, + pub virtualization: Entry, V>, /// A #CP exception is generated when shadow stacks are enabled and mismatch /// scenarios are detected (possible error code cases below). @@ -358,10 +358,10 @@ pub struct InterruptDescriptorTable { /// - A missing ENDBRANCH instruction if indirect branch tracking is enabled. /// /// vector nr. 21 - pub cp_protection_exception: Entry, + pub cp_protection_exception: Entry, V>, /// vector nr. 22-27 - reserved_2: [Entry; 6], + reserved_2: [Entry, V>; 6], /// The Hypervisor Injection Exception (`#HV`) is injected by a hypervisor /// as a doorbell to inform an `SEV-SNP` enabled guest running with the @@ -378,7 +378,7 @@ pub struct InterruptDescriptorTable { /// software-managed para-virtualization interface. /// /// The vector number of the ``#HV`` exception is 28. - pub hv_injection_exception: Entry, + pub hv_injection_exception: Entry, V>, /// The VMM Communication Exception (`#VC`) is always generated by hardware when an `SEV-ES` /// enabled guest is running and an `NAE` event occurs. @@ -407,7 +407,7 @@ pub struct InterruptDescriptorTable { /// setting intercept bits for events that would occur in the `#VC` handler (such as `IRET`). /// /// The vector number of the ``#VC`` exception is 29. - pub vmm_communication_exception: Entry, + pub vmm_communication_exception: Entry, V>, /// The Security Exception (`#SX`) signals security-sensitive events that occur while /// executing the VMM, in the form of an exception so that the VMM may take appropriate @@ -418,10 +418,10 @@ pub struct InterruptDescriptorTable { /// The only error code currently defined is 1, and indicates redirection of INIT has occurred. /// /// The vector number of the ``#SX`` exception is 30. - pub security_exception: Entry, + pub security_exception: Entry, V>, /// vector nr. 31 - reserved_3: Entry, + reserved_3: Entry, V>, /// User-defined interrupts can be initiated either by system logic or software. They occur /// when: @@ -443,14 +443,14 @@ pub struct InterruptDescriptorTable { /// external interrupt was recognized. /// - If the interrupt occurs as a result of executing the INTn instruction, the saved /// instruction pointer points to the instruction after the INTn. - interrupts: [Entry; 256 - 32], + interrupts: [Entry, V>; 256 - 32], } -impl InterruptDescriptorTable { +impl InterruptDescriptorTable { /// Creates a new IDT filled with non-present entries. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn new() -> InterruptDescriptorTable { + pub fn new_with_validity() -> Self { InterruptDescriptorTable { divide_error: Entry::missing(), debug: Entry::missing(), @@ -486,42 +486,19 @@ impl InterruptDescriptorTable { /// Resets all entries of this IDT in place. #[inline] pub fn reset(&mut self) { - *self = Self::new(); - } - - /// Loads the IDT in the CPU using the `lidt` command. - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub fn load(&'static self) { - unsafe { self.load_unsafe() } - } - - /// Loads the IDT in the CPU using the `lidt` command. - /// - /// # Safety - /// - /// As long as it is the active IDT, you must ensure that: - /// - /// - `self` is never destroyed. - /// - `self` always stays at the same memory location. It is recommended to wrap it in - /// a `Box`. - /// - #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - #[inline] - pub unsafe fn load_unsafe(&self) { - use crate::instructions::tables::lidt; - unsafe { - lidt(&self.pointer()); - } + *self = Self::new_with_validity(); } /// Creates the descriptor pointer for this table. This pointer can only be /// safely used if the table is never modified or destroyed while in use. #[cfg(all(feature = "instructions", target_arch = "x86_64"))] - fn pointer(&self) -> crate::structures::DescriptorTablePointer { + fn pointer(&self) -> crate::structures::DescriptorTablePointer + where + V: VirtAddrValidity, + { use core::mem::size_of; crate::structures::DescriptorTablePointer { - base: VirtAddr::new(self as *const _ as u64), + base: VirtAddr::::new_with_validity(self as *const _ as u64), limit: (size_of::() - 1) as u16, } } @@ -551,7 +528,7 @@ impl InterruptDescriptorTable { /// /// Panics if the entry is an exception. #[inline] - pub fn slice(&self, bounds: impl RangeBounds) -> &[Entry] { + pub fn slice(&self, bounds: impl RangeBounds) -> &[Entry, V>] { let (lower_idx, upper_idx) = self.condition_slice_bounds(bounds); &self.interrupts[(lower_idx - 32)..(upper_idx - 32)] } @@ -560,21 +537,52 @@ impl InterruptDescriptorTable { /// /// Panics if the entry is an exception. #[inline] - pub fn slice_mut(&mut self, bounds: impl RangeBounds) -> &mut [Entry] { + pub fn slice_mut(&mut self, bounds: impl RangeBounds) -> &mut [Entry, V>] { let (lower_idx, upper_idx) = self.condition_slice_bounds(bounds); &mut self.interrupts[(lower_idx - 32)..(upper_idx - 32)] } } -impl Default for InterruptDescriptorTable { +impl InterruptDescriptorTable { + /// Creates a new runtime-valid IDT filled with non-present entries. + /// + /// Handler addresses assigned later retain their creation-time validity guarantees. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn new() -> Self { + Self::new_with_validity() + } + + /// Loads the IDT in the CPU using the `lidt` command. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub fn load(&'static self) { + unsafe { self.load_unsafe() } + } + + /// Loads the IDT without imposing a static lifetime. + /// + /// # Safety + /// + /// The caller must keep the IDT alive and unmodified while it is loaded. + #[cfg(all(feature = "instructions", target_arch = "x86_64"))] + #[inline] + pub unsafe fn load_unsafe(&self) { + use crate::instructions::tables::lidt; + + unsafe { lidt(&self.pointer()) } + } +} + +impl Default for InterruptDescriptorTable { #[inline] fn default() -> Self { - Self::new() + Self::new_with_validity() } } -impl Index for InterruptDescriptorTable { - type Output = Entry; +impl Index for InterruptDescriptorTable { + type Output = Entry, V>; /// Returns the IDT entry with the specified index. /// @@ -605,7 +613,7 @@ impl Index for InterruptDescriptorTable { } } -impl IndexMut for InterruptDescriptorTable { +impl IndexMut for InterruptDescriptorTable { /// Returns a mutable reference to the IDT entry with the specified index. /// /// Panics if the entry is an exception that pushes an error code (use the struct fields for accessing these entries). @@ -637,8 +645,8 @@ impl IndexMut for InterruptDescriptorTable { macro_rules! impl_index_for_idt { ($ty:ty) => { - impl Index<$ty> for InterruptDescriptorTable { - type Output = [Entry]; + impl Index<$ty> for InterruptDescriptorTable { + type Output = [Entry, V>]; /// Returns the IDT entry with the specified index. /// @@ -650,7 +658,7 @@ macro_rules! impl_index_for_idt { } } - impl IndexMut<$ty> for InterruptDescriptorTable { + impl IndexMut<$ty> for InterruptDescriptorTable { /// Returns a mutable reference to the IDT entry with the specified index. /// /// Panics if the entry is an exception that pushes an error code (use the struct fields for accessing these entries). @@ -682,16 +690,16 @@ impl_index_for_idt!(RangeFull); /// The generic parameter is some [`HandlerFuncType`], depending on the interrupt vector. #[derive(Clone, Copy)] #[repr(C)] -pub struct Entry { +pub struct Entry { pointer_low: u16, options: EntryOptions, pointer_middle: u16, pointer_high: u32, reserved: u32, - phantom: PhantomData, + phantom: PhantomData<(F, V)>, } -impl fmt::Debug for Entry { +impl fmt::Debug for Entry { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("Entry") .field("handler_addr", &format_args!("{:#x}", self.handler_addr())) @@ -700,7 +708,7 @@ impl fmt::Debug for Entry { } } -impl PartialEq for Entry { +impl PartialEq for Entry { fn eq(&self, other: &Self) -> bool { self.pointer_low == other.pointer_low && self.options == other.options @@ -717,14 +725,14 @@ impl PartialEq for Entry { any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type HandlerFunc = extern "x86-interrupt" fn(InterruptStackFrame); +pub type HandlerFunc = extern "x86-interrupt" fn(InterruptStackFrame); /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct HandlerFunc(()); +pub struct HandlerFunc(PhantomData); /// A handler function for an exception that pushes an error code. /// @@ -733,14 +741,15 @@ pub struct HandlerFunc(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type HandlerFuncWithErrCode = extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64); +pub type HandlerFuncWithErrCode = + extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64); /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct HandlerFuncWithErrCode(()); +pub struct HandlerFuncWithErrCode(PhantomData); /// A page fault handler function that pushes a page fault error code. /// @@ -749,15 +758,15 @@ pub struct HandlerFuncWithErrCode(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type PageFaultHandlerFunc = - extern "x86-interrupt" fn(InterruptStackFrame, error_code: PageFaultErrorCode); +pub type PageFaultHandlerFunc = + extern "x86-interrupt" fn(InterruptStackFrame, error_code: PageFaultErrorCode); /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct PageFaultHandlerFunc(()); +pub struct PageFaultHandlerFunc(PhantomData); /// A handler function that must not return, e.g. for a machine check exception. /// @@ -766,14 +775,15 @@ pub struct PageFaultHandlerFunc(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type DivergingHandlerFunc = extern "x86-interrupt" fn(InterruptStackFrame) -> !; +pub type DivergingHandlerFunc = + extern "x86-interrupt" fn(InterruptStackFrame) -> !; /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct DivergingHandlerFunc(()); +pub struct DivergingHandlerFunc(PhantomData); /// A handler function with an error code that must not return, e.g. for a double fault exception. /// @@ -782,20 +792,21 @@ pub struct DivergingHandlerFunc(()); any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] -pub type DivergingHandlerFuncWithErrCode = - extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64) -> !; +pub type DivergingHandlerFuncWithErrCode = + extern "x86-interrupt" fn(InterruptStackFrame, error_code: u64) -> !; /// This type is not usable without the `abi_x86_interrupt` feature. #[cfg(not(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" )))] #[derive(Copy, Clone, Debug)] -pub struct DivergingHandlerFuncWithErrCode(()); +pub struct DivergingHandlerFuncWithErrCode(PhantomData); /// A general handler function for an interrupt or an exception with the interrupt/exceptions's index and an optional error code. -pub type GeneralHandlerFunc = fn(InterruptStackFrame, index: u8, error_code: Option); +pub type GeneralHandlerFunc = + fn(InterruptStackFrame, index: u8, error_code: Option); -impl Entry { +impl Entry { /// Creates a non-present IDT entry (but sets the must-be-one bits). #[inline] pub const fn missing() -> Self { @@ -808,7 +819,9 @@ impl Entry { phantom: PhantomData, } } +} +impl Entry { /// Sets the handler address for the IDT entry and sets the following defaults: /// - The code selector is the code segment currently active in the CPU /// - The present bit is set @@ -825,7 +838,7 @@ impl Entry { /// and the signature of such a function is correct for the entry type. #[cfg(all(feature = "instructions", target_arch = "x86_64"))] #[inline] - pub unsafe fn set_handler_addr(&mut self, addr: VirtAddr) -> &mut EntryOptions { + pub unsafe fn set_handler_addr(&mut self, addr: VirtAddr) -> &mut EntryOptions { use crate::instructions::segmentation::{Segment, CS}; let addr = addr.as_u64(); @@ -842,18 +855,21 @@ impl Entry { /// Returns the virtual address of this IDT entry's handler function. #[inline] - pub fn handler_addr(&self) -> VirtAddr { + pub fn handler_addr(&self) -> VirtAddr { let addr = self.pointer_low as u64 | ((self.pointer_middle as u64) << 16) | ((self.pointer_high as u64) << 32); // addr is a valid VirtAddr, as the pointer members are either all zero, // or have been set by set_handler_addr (which takes a VirtAddr). - VirtAddr::new_truncate(addr) + unsafe { VirtAddr::::new_unsafe(addr) } } } #[cfg(all(feature = "instructions", target_arch = "x86_64"))] -impl Entry { +impl Entry +where + F: HandlerFuncType, +{ /// Sets the handler function for the IDT entry and sets the following defaults: /// - The code selector is the code segment currently active in the CPU /// - The present bit is set @@ -877,27 +893,30 @@ impl Entry { /// # Safety /// /// Implementors have to ensure that `to_virt_addr` returns a valid address. -pub unsafe trait HandlerFuncType { +pub unsafe trait HandlerFuncType { /// Get the virtual address of the handler function. - fn to_virt_addr(self) -> VirtAddr; + fn to_virt_addr(self) -> VirtAddr; } macro_rules! impl_handler_func_type { - ($f:ty) => { + ($f:ident) => { #[cfg(all( any(target_arch = "x86", target_arch = "x86_64"), feature = "abi_x86_interrupt" ))] - unsafe impl HandlerFuncType for $f { + unsafe impl HandlerFuncType for $f + where + V: VirtAddrValidity, + { #[inline] - fn to_virt_addr(self) -> VirtAddr { + fn to_virt_addr(self) -> VirtAddr { // Casting a function pointer to u64 is fine, if the pointer // width doesn't exceed 64 bits. #[cfg_attr( any(target_pointer_width = "32", target_pointer_width = "64"), allow(clippy::fn_to_numeric_cast) )] - VirtAddr::new(self as u64) + VirtAddr::::new_with_validity(self as u64) } } }; @@ -1015,16 +1034,16 @@ impl EntryOptions { /// occurs, which can cause undefined behavior (see the [`as_mut`](InterruptStackFrame::as_mut) /// method for more information). #[repr(transparent)] -pub struct InterruptStackFrame(InterruptStackFrameValue); +pub struct InterruptStackFrame(InterruptStackFrameValue); -impl InterruptStackFrame { +impl InterruptStackFrame { /// Creates a new interrupt stack frame with the given values. #[inline] pub fn new( - instruction_pointer: VirtAddr, + instruction_pointer: VirtAddr, code_segment: SegmentSelector, cpu_flags: RFlags, - stack_pointer: VirtAddr, + stack_pointer: VirtAddr, stack_segment: SegmentSelector, ) -> Self { Self(InterruptStackFrameValue::new( @@ -1051,13 +1070,13 @@ impl InterruptStackFrame { /// Also, it is not fully clear yet whether modifications of the interrupt stack frame are /// officially supported by LLVM's x86 interrupt calling convention. #[inline] - pub unsafe fn as_mut(&mut self) -> Volatile<&mut InterruptStackFrameValue> { + pub unsafe fn as_mut(&mut self) -> Volatile<&mut InterruptStackFrameValue> { Volatile::new(&mut self.0) } } -impl Deref for InterruptStackFrame { - type Target = InterruptStackFrameValue; +impl Deref for InterruptStackFrame { + type Target = InterruptStackFrameValue; #[inline] fn deref(&self) -> &Self::Target { @@ -1065,7 +1084,7 @@ impl Deref for InterruptStackFrame { } } -impl fmt::Debug for InterruptStackFrame { +impl fmt::Debug for InterruptStackFrame { #[inline] fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { self.0.fmt(f) @@ -1075,33 +1094,33 @@ impl fmt::Debug for InterruptStackFrame { /// Represents the interrupt stack frame pushed by the CPU on interrupt or exception entry. #[derive(Clone, Copy)] #[repr(C)] -pub struct InterruptStackFrameValue { +pub struct InterruptStackFrameValue { /// This value points to the instruction that should be executed when the interrupt /// handler returns. For most interrupts, this value points to the instruction immediately /// following the last executed instruction. However, for some exceptions (e.g., page faults), /// this value points to the faulting instruction, so that the instruction is restarted on /// return. See the documentation of the [`InterruptDescriptorTable`] fields for more details. - pub instruction_pointer: VirtAddr, + pub instruction_pointer: VirtAddr, /// The code segment selector at the time of the interrupt. pub code_segment: SegmentSelector, _reserved1: [u8; 6], /// The flags register before the interrupt handler was invoked. pub cpu_flags: RFlags, /// The stack pointer at the time of the interrupt. - pub stack_pointer: VirtAddr, + pub stack_pointer: VirtAddr, /// The stack segment descriptor at the time of the interrupt (often zero in 64-bit mode). pub stack_segment: SegmentSelector, _reserved2: [u8; 6], } -impl InterruptStackFrameValue { +impl InterruptStackFrameValue { /// Creates a new interrupt stack frame with the given values. #[inline] pub fn new( - instruction_pointer: VirtAddr, + instruction_pointer: VirtAddr, code_segment: SegmentSelector, cpu_flags: RFlags, - stack_pointer: VirtAddr, + stack_pointer: VirtAddr, stack_segment: SegmentSelector, ) -> Self { Self { @@ -1148,7 +1167,7 @@ impl InterruptStackFrameValue { } } -impl fmt::Debug for InterruptStackFrameValue { +impl fmt::Debug for InterruptStackFrameValue { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut s = f.debug_struct("InterruptStackFrame"); s.field("instruction_pointer", &self.instruction_pointer); @@ -1419,7 +1438,7 @@ impl TryFrom for ExceptionVector { ))] #[macro_export] /// Set a general handler in an [`InterruptDescriptorTable`]. -/// ``` +/// ```no_run /// #![feature(abi_x86_interrupt)] /// use x86_64::set_general_handler; /// use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame}; @@ -1666,9 +1685,66 @@ mod test { fn size_test() { use core::mem::size_of; assert_eq!(size_of::>(), 16); + assert_eq!( + size_of::>, crate::FixedValidity<57>>>(), + 16 + ); + assert_eq!( + size_of::, crate::RuntimeValidity>>(), + 16 + ); assert_eq!(size_of::(), 256 * 16); + assert_eq!( + size_of::>>(), + 256 * 16 + ); + assert_eq!( + size_of::>(), + 256 * 16 + ); assert_eq!(size_of::(), 40); + assert_eq!( + size_of::>>(), + 40 + ); + assert_eq!(size_of::>(), 40); assert_eq!(size_of::(), 40); + assert_eq!( + size_of::>>(), + 40 + ); + assert_eq!( + size_of::>(), + 40 + ); + } + + #[test] + fn explicit_policy_idt_and_frames_construct() { + let _: InterruptDescriptorTable> = + InterruptDescriptorTable::new_with_validity(); + let _: InterruptDescriptorTable = + InterruptDescriptorTable::new_with_validity(); + + let address57 = crate::VirtAddr57::new_const(0x0000_8000_0000_0000); + let frame57 = InterruptStackFrame::new( + address57, + SegmentSelector(0), + RFlags::empty(), + address57, + SegmentSelector(0), + ); + assert_eq!(frame57.instruction_pointer, address57); + + let address_rt = unsafe { crate::VirtAddrRT::new_unsafe(0x1234) }; + let frame_rt = InterruptStackFrame::new( + address_rt, + SegmentSelector(0), + RFlags::empty(), + address_rt, + SegmentSelector(0), + ); + assert_eq!(frame_rt.stack_pointer, address_rt); } #[cfg(all( @@ -1680,6 +1756,7 @@ mod test { // https://github.com/rust-osdev/x86_64/pull/285#issuecomment-962642984 #[cfg(not(windows))] #[test] + #[ignore = "runtime-valid handler construction requires ring 0"] fn default_handlers() { fn general_handler( _stack_frame: InterruptStackFrame, @@ -1745,15 +1822,16 @@ mod test { #[test] fn isr_frame_manipulation() { - let mut frame = InterruptStackFrame(InterruptStackFrameValue { - instruction_pointer: VirtAddr::new(0x1000), - code_segment: SegmentSelector(0), - cpu_flags: RFlags::empty(), - stack_pointer: VirtAddr::new(0x2000), - stack_segment: SegmentSelector(0), - _reserved1: Default::default(), - _reserved2: Default::default(), - }); + let mut frame: InterruptStackFrame> = + InterruptStackFrame(InterruptStackFrameValue { + instruction_pointer: crate::VirtAddr48::new_const(0x1000), + code_segment: SegmentSelector(0), + cpu_flags: RFlags::empty(), + stack_pointer: crate::VirtAddr48::new_const(0x2000), + stack_segment: SegmentSelector(0), + _reserved1: Default::default(), + _reserved2: Default::default(), + }); unsafe { frame.as_mut().update(|f| f.instruction_pointer += 2u64); diff --git a/src/structures/mod.rs b/src/structures/mod.rs index 084bbafab..1d705f6fc 100644 --- a/src/structures/mod.rs +++ b/src/structures/mod.rs @@ -1,6 +1,6 @@ //! Representations of various x86 specific structures and descriptor tables. -use crate::VirtAddr; +use crate::{RuntimeValidity, VirtAddr, VirtAddrValidity}; pub mod gdt; @@ -14,13 +14,34 @@ pub mod tss; /// A struct describing a pointer to a descriptor table (GDT / IDT). /// This is in a format suitable for giving to 'lgdt' or 'lidt'. -#[derive(Debug, Clone, Copy)] #[repr(C, packed(2))] -pub struct DescriptorTablePointer { +pub struct DescriptorTablePointer { /// Size of the DT in bytes - 1. pub limit: u16, /// Pointer to the memory region containing the DT. - pub base: VirtAddr, + pub base: VirtAddr, +} + +// These traits are implemented manually because Rust 1.59 has limited derive support for generic +// packed structs. They can use derive once the MSRV is raised to Rust 1.69. +impl Copy for DescriptorTablePointer {} + +impl Clone for DescriptorTablePointer { + fn clone(&self) -> Self { + *self + } +} + +impl core::fmt::Debug for DescriptorTablePointer { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let limit = self.limit; + let base = self.base; + + f.debug_struct("DescriptorTablePointer") + .field("limit", &limit) + .field("base", &base) + .finish() + } } #[cfg(test)] @@ -33,10 +54,19 @@ mod tests { // Per the SDM, a descriptor pointer has to be 2+8=10 bytes assert_eq!(size_of::(), 10); // Make sure that we can reference a pointer's limit - let p = DescriptorTablePointer { + let p: DescriptorTablePointer = DescriptorTablePointer { limit: 5, base: VirtAddr::zero(), }; let _: &u16 = &p.limit; + + assert_eq!( + size_of::>>(), + 10 + ); + assert_eq!( + size_of::>(), + 10 + ); } } diff --git a/src/structures/paging/mapper/mapped_page_table.rs b/src/structures/paging/mapper/mapped_page_table.rs index 5f673c55c..1ff0ded0b 100644 --- a/src/structures/paging/mapper/mapped_page_table.rs +++ b/src/structures/paging/mapper/mapped_page_table.rs @@ -4,13 +4,13 @@ use crate::structures::paging::{ page_table::{FrameError, PageTable, PageTableEntry, PageTableLevel}, }; -/// A Mapper implementation that relies on a PhysAddr to VirtAddr conversion function. +/// A Mapper implementation that relies on a PhysAddr to VirtAddr48 conversion function. /// /// This type requires that the all physical page table frames are mapped to some virtual /// address. Normally, this is done by mapping the complete physical address space into /// the virtual address space at some offset. Other mappings between physical and virtual /// memory are possible too, as long as they can be calculated as an `PhysAddr` to -/// `VirtAddr` closure. +/// `VirtAddr48` closure. #[derive(Debug)] pub struct MappedPageTable<'a, P: PageTableFrameMapping> { page_table_walker: PageTableWalker

, @@ -55,7 +55,7 @@ impl Mapper for MappedPageTable<'_, P> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -81,7 +81,7 @@ impl Mapper for MappedPageTable<'_, P> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.level_4_table; let p3 = self @@ -107,7 +107,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.level_4_table; @@ -125,7 +125,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -142,7 +142,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p3_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) @@ -150,13 +150,16 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.level_4_table; let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?; @@ -175,7 +178,7 @@ impl Mapper for MappedPageTable<'_, P> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -206,7 +209,7 @@ impl Mapper for MappedPageTable<'_, P> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.level_4_table; let p3 = self @@ -235,7 +238,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.level_4_table; @@ -257,7 +260,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -274,7 +277,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -294,13 +297,16 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.level_4_table; let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?; let p2 = self.page_table_walker.next_table(&p3[page.p3_index()])?; @@ -320,7 +326,7 @@ impl Mapper for MappedPageTable<'_, P> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -356,7 +362,7 @@ impl Mapper for MappedPageTable<'_, P> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.level_4_table; let p3 = self @@ -382,7 +388,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.level_4_table; @@ -407,7 +413,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -424,7 +430,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -444,7 +450,7 @@ impl Mapper for MappedPageTable<'_, P> { unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.level_4_table; @@ -465,7 +471,10 @@ impl Mapper for MappedPageTable<'_, P> { Ok(MapperFlushAll::new()) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.level_4_table; let p3 = self.page_table_walker.next_table(&p4[page.p4_index()])?; let p2 = self.page_table_walker.next_table(&p3[page.p3_index()])?; @@ -484,7 +493,7 @@ impl Mapper for MappedPageTable<'_, P> { impl Translate for MappedPageTable<'_, P> { #[allow(clippy::inconsistent_digit_grouping)] - fn translate(&self, addr: VirtAddr) -> TranslateResult { + fn translate(&self, addr: VirtAddr48) -> TranslateResult { let p4 = &self.level_4_table; let p3 = match self.page_table_walker.next_table(&p4[addr.p4_index()]) { Ok(page_table) => page_table, @@ -555,8 +564,9 @@ impl CleanUp for MappedPageTable<'_, P> { unsafe { self.clean_up_addr_range( PageRangeInclusive { - start: Page::from_start_address(VirtAddr::new(0)).unwrap(), - end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(), + start: Page::from_start_address(VirtAddr48::new_const(0)).unwrap(), + end: Page::from_start_address(VirtAddr48::new_const(0xffff_ffff_ffff_f000)) + .unwrap(), }, frame_deallocator, ) @@ -565,7 +575,7 @@ impl CleanUp for MappedPageTable<'_, P> { unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator, @@ -574,7 +584,7 @@ impl CleanUp for MappedPageTable<'_, P> { page_table: &mut PageTable, page_table_walker: &PageTableWalker

, level: PageTableLevel, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut impl FrameDeallocator, ) -> bool { if range.is_empty() { @@ -598,15 +608,17 @@ impl CleanUp for MappedPageTable<'_, P> { .skip(usize::from(start)) { if let Ok(page_table) = page_table_walker.next_table_mut(entry) { - let start = VirtAddr::forward_checked_impl( + let start = VirtAddr48::forward_checked_impl( table_addr, (offset_per_entry as usize) * i, ) .unwrap(); let end = start + (offset_per_entry - 1); - let start = Page::::containing_address(start); + let start = + Page::>::containing_address_const(start); let start = start.max(range.start); - let end = Page::::containing_address(end); + let end = + Page::>::containing_address_const(end); let end = end.min(range.end); unsafe { if clean_up( diff --git a/src/structures/paging/mapper/mod.rs b/src/structures/paging/mapper/mod.rs index d0f217167..4ae1917fa 100644 --- a/src/structures/paging/mapper/mod.rs +++ b/src/structures/paging/mapper/mod.rs @@ -12,7 +12,7 @@ use crate::structures::paging::{ page_table::PageTableFlags, Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB, }; -use crate::{PhysAddr, VirtAddr}; +use crate::{FixedValidity, PhysAddr, VirtAddr48}; mod mapped_page_table; mod offset_page_table; @@ -33,7 +33,7 @@ pub trait Translate { /// frame is returned. Otherwise an error value is returned. /// /// This function works with huge pages of all sizes. - fn translate(&self, addr: VirtAddr) -> TranslateResult; + fn translate(&self, addr: VirtAddr48) -> TranslateResult; /// Translates the given virtual address to the physical address that it maps to. /// @@ -42,7 +42,7 @@ pub trait Translate { /// This is a convenience method. For more information about a mapping see the /// [`translate`](Translate::translate) method. #[inline] - fn translate_addr(&self, addr: VirtAddr) -> Option { + fn translate_addr(&self, addr: VirtAddr48) -> Option { match self.translate(addr) { TranslateResult::NotMapped | TranslateResult::InvalidFrameAddress(_) => None, TranslateResult::Mapped { frame, offset, .. } => Some(frame.start_address() + offset), @@ -159,9 +159,10 @@ pub trait Mapper { /// # Mapper, Page, PhysFrame, FrameAllocator, /// # Size4KiB, OffsetPageTable, page_table::PageTableFlags /// # }; + /// # use x86_64::FixedValidity; /// # #[cfg(all(feature = "instructions", target_arch = "x86_64"))] /// # unsafe fn test(mapper: &mut OffsetPageTable, frame_allocator: &mut impl FrameAllocator, - /// # page: Page, frame: PhysFrame) { + /// # page: Page>, frame: PhysFrame) { /// mapper /// .map_to( /// page, @@ -178,7 +179,7 @@ pub trait Mapper { #[inline] unsafe fn map_to( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, frame_allocator: &mut A, @@ -248,9 +249,10 @@ pub trait Mapper { /// # Mapper, PhysFrame, Page, FrameAllocator, /// # Size4KiB, OffsetPageTable, page_table::PageTableFlags /// # }; + /// # use x86_64::FixedValidity; /// # #[cfg(all(feature = "instructions", target_arch = "x86_64"))] /// # unsafe fn test(mapper: &mut OffsetPageTable, frame_allocator: &mut impl FrameAllocator, - /// # page: Page, frame: PhysFrame) { + /// # page: Page>, frame: PhysFrame) { /// mapper /// .map_to_with_table_flags( /// page, @@ -269,7 +271,7 @@ pub trait Mapper { /// ``` unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -282,7 +284,10 @@ pub trait Mapper { /// Removes a mapping from the page table and returns the frame that used to be mapped. /// /// Note that no page tables or pages are deallocated. - fn unmap(&mut self, page: Page) -> Result<(PhysFrame, MapperFlush), UnmapError>; + fn unmap( + &mut self, + page: Page>, + ) -> Result<(PhysFrame, MapperFlush), UnmapError>; /// Updates the flags of an existing mapping. /// @@ -297,7 +302,7 @@ pub trait Mapper { /// spaces. unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError>; @@ -312,7 +317,7 @@ pub trait Mapper { /// spaces. unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result; @@ -327,7 +332,7 @@ pub trait Mapper { /// spaces. unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result; @@ -342,7 +347,7 @@ pub trait Mapper { /// spaces. unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result; @@ -350,7 +355,10 @@ pub trait Mapper { /// /// This function assumes that the page is mapped to a frame of size `S` and returns an /// error otherwise. - fn translate_page(&self, page: Page) -> Result, TranslateError>; + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError>; /// Maps the given frame to the virtual page with the same address. /// @@ -371,7 +379,8 @@ pub trait Mapper { S: PageSize, Self: Mapper, { - let page = Page::containing_address(VirtAddr::new(frame.start_address().as_u64())); + let page = + Page::containing_address_const(VirtAddr48::new_const(frame.start_address().as_u64())); unsafe { self.map_to(page, frame, flags, frame_allocator) } } } @@ -387,7 +396,7 @@ pub trait Mapper { not(all(feature = "instructions", target_arch = "x86_64")), allow(dead_code) )] // FIXME -pub struct MapperFlush(Page); +pub struct MapperFlush(Page>); impl MapperFlush { /// Create a new flush promise @@ -395,7 +404,7 @@ impl MapperFlush { /// Note that this method is intended for implementing the [`Mapper`] trait and no other uses /// are expected. #[inline] - pub fn new(page: Page) -> Self { + pub fn new(page: Page>) -> Self { MapperFlush(page) } @@ -403,7 +412,7 @@ impl MapperFlush { #[cfg(all(feature = "instructions", target_arch = "x86_64"))] #[inline] pub fn flush(self) { - crate::instructions::tlb::flush(self.0.start_address()); + crate::instructions::tlb::flush(self.0.start_address().into()); } /// Don't flush the TLB and silence the “must be used” warning. @@ -412,7 +421,7 @@ impl MapperFlush { /// Returns the page to be flushed. #[inline] - pub fn page(&self) -> Page { + pub fn page(&self) -> Page> { self.0 } } @@ -513,14 +522,14 @@ pub trait CleanUp { /// Remove all empty P1-P3 tables in a certain range /// ``` /// # use core::ops::RangeInclusive; - /// # use x86_64::{VirtAddr, structures::paging::{ + /// # use x86_64::{VirtAddr48, structures::paging::{ /// # FrameDeallocator, Size4KiB, mapper::CleanUp, page::Page, /// # }}; /// # unsafe fn test(page_table: &mut impl CleanUp, frame_deallocator: &mut impl FrameDeallocator) { /// // clean up all page tables in the lower half of the address space /// let lower_half = Page::range_inclusive( - /// Page::containing_address(VirtAddr::new(0)), - /// Page::containing_address(VirtAddr::new(0x0000_7fff_ffff_ffff)), + /// Page::containing_address_const(VirtAddr48::new_const(0)), + /// Page::containing_address_const(VirtAddr48::new_const(0x0000_7fff_ffff_ffff)), /// ); /// page_table.clean_up_addr_range(lower_half, frame_deallocator); /// # } @@ -533,7 +542,7 @@ pub trait CleanUp { /// (e.g. no reference counted page tables or reusing the same page tables for different virtual addresses ranges in the same page table). unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator; diff --git a/src/structures/paging/mapper/offset_page_table.rs b/src/structures/paging/mapper/offset_page_table.rs index 546a82997..56542b7d9 100644 --- a/src/structures/paging/mapper/offset_page_table.rs +++ b/src/structures/paging/mapper/offset_page_table.rs @@ -26,7 +26,7 @@ impl<'a> OffsetPageTable<'a> { /// of a valid page table hierarchy. Otherwise this function might break memory safety, e.g. /// by writing to an illegal memory location. #[inline] - pub unsafe fn new(level_4_table: &'a mut PageTable, phys_offset: VirtAddr) -> Self { + pub unsafe fn new(level_4_table: &'a mut PageTable, phys_offset: VirtAddr48) -> Self { let phys_offset = PhysOffset { offset: phys_offset, }; @@ -46,14 +46,14 @@ impl<'a> OffsetPageTable<'a> { } /// Returns the offset used for converting virtual to physical addresses. - pub fn phys_offset(&self) -> VirtAddr { + pub fn phys_offset(&self) -> VirtAddr48 { self.inner.page_table_frame_mapping().offset } } #[derive(Debug)] struct PhysOffset { - offset: VirtAddr, + offset: VirtAddr48, } unsafe impl PageTableFrameMapping for PhysOffset { @@ -69,7 +69,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -87,7 +87,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { self.inner.unmap(page) } @@ -95,7 +95,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { unsafe { self.inner.update_flags(page, flags) } @@ -104,7 +104,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p4_entry(page, flags) } @@ -113,7 +113,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p3_entry(page, flags) } @@ -122,14 +122,17 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p2_entry(page, flags) } } #[inline] - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { self.inner.translate_page(page) } } @@ -138,7 +141,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -156,7 +159,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { self.inner.unmap(page) } @@ -164,7 +167,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { unsafe { self.inner.update_flags(page, flags) } @@ -173,7 +176,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p4_entry(page, flags) } @@ -182,7 +185,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p3_entry(page, flags) } @@ -191,14 +194,17 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p2_entry(page, flags) } } #[inline] - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { self.inner.translate_page(page) } } @@ -207,7 +213,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -225,7 +231,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { self.inner.unmap(page) } @@ -233,7 +239,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { unsafe { self.inner.update_flags(page, flags) } @@ -242,7 +248,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p4_entry(page, flags) } @@ -251,7 +257,7 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p3_entry(page, flags) } @@ -260,21 +266,24 @@ impl Mapper for OffsetPageTable<'_> { #[inline] unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { unsafe { self.inner.set_flags_p2_entry(page, flags) } } #[inline] - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { self.inner.translate_page(page) } } impl Translate for OffsetPageTable<'_> { #[inline] - fn translate(&self, addr: VirtAddr) -> TranslateResult { + fn translate(&self, addr: VirtAddr48) -> TranslateResult { self.inner.translate(addr) } } @@ -291,7 +300,7 @@ impl CleanUp for OffsetPageTable<'_> { #[inline] unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator, diff --git a/src/structures/paging/mapper/recursive_page_table.rs b/src/structures/paging/mapper/recursive_page_table.rs index bd3c59811..f7b5fc433 100644 --- a/src/structures/paging/mapper/recursive_page_table.rs +++ b/src/structures/paging/mapper/recursive_page_table.rs @@ -55,7 +55,7 @@ impl<'a> RecursivePageTable<'a> { /// and [in the `unsafe-code-guidelines ` repo](https://github.com/rust-lang/unsafe-code-guidelines/issues/420). #[inline] pub fn new(table: &'a mut PageTable) -> Result { - let page = Page::containing_address(VirtAddr::new(table as *const _ as u64)); + let page = Page::containing_address_const(VirtAddr48::new_const(table as *const _ as u64)); let recursive_index = page.p4_index(); if page.p3_index() != recursive_index @@ -116,7 +116,7 @@ impl<'a> RecursivePageTable<'a> { /// in the passed entry. unsafe fn create_next_table<'b, A, S: PageSize>( entry: &'b mut PageTableEntry, - next_table_page: Page, + next_table_page: Page>, insert_flags: PageTableFlags, allocator: &mut A, ) -> Result<&'b mut PageTable, MapToError> @@ -128,7 +128,7 @@ impl<'a> RecursivePageTable<'a> { /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe. fn inner<'b, A, S: PageSize>( entry: &'b mut PageTableEntry, - next_table_page: Page, + next_table_page: Page>, insert_flags: PageTableFlags, allocator: &mut A, ) -> Result<&'b mut PageTable, MapToError> @@ -172,7 +172,7 @@ impl Mapper for RecursivePageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -204,7 +204,7 @@ impl Mapper for RecursivePageTable<'_> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.p4; let p4_entry = &p4[page.p4_index()]; @@ -234,7 +234,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { use crate::structures::paging::PageTableFlags as Flags; @@ -256,7 +256,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -273,7 +273,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p3_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) @@ -281,13 +281,16 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.p4; if p4[page.p4_index()].is_unused() { @@ -310,7 +313,7 @@ impl Mapper for RecursivePageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -352,7 +355,7 @@ impl Mapper for RecursivePageTable<'_> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.p4; let p4_entry = &p4[page.p4_index()]; @@ -388,7 +391,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { use crate::structures::paging::PageTableFlags as Flags; @@ -417,7 +420,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -434,7 +437,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -457,13 +460,16 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p2_entry( &mut self, - _page: Page, + _page: Page>, _flags: PageTableFlags, ) -> Result { Err(FlagUpdateError::ParentEntryHugePage) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.p4; if p4[page.p4_index()].is_unused() { @@ -493,7 +499,7 @@ impl Mapper for RecursivePageTable<'_> { #[inline] unsafe fn map_to_with_table_flags( &mut self, - page: Page, + page: Page>, frame: PhysFrame, flags: PageTableFlags, parent_table_flags: PageTableFlags, @@ -544,7 +550,7 @@ impl Mapper for RecursivePageTable<'_> { fn unmap( &mut self, - page: Page, + page: Page>, ) -> Result<(PhysFrame, MapperFlush), UnmapError> { let p4 = &mut self.p4; let p4_entry = &p4[page.p4_index()]; @@ -581,7 +587,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn update_flags( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result, FlagUpdateError> { let p4 = &mut self.p4; @@ -615,7 +621,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p4_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -632,7 +638,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p3_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -655,7 +661,7 @@ impl Mapper for RecursivePageTable<'_> { unsafe fn set_flags_p2_entry( &mut self, - page: Page, + page: Page>, flags: PageTableFlags, ) -> Result { let p4 = &mut self.p4; @@ -682,7 +688,10 @@ impl Mapper for RecursivePageTable<'_> { Ok(MapperFlushAll::new()) } - fn translate_page(&self, page: Page) -> Result, TranslateError> { + fn translate_page( + &self, + page: Page>, + ) -> Result, TranslateError> { let p4 = &self.p4; if p4[page.p4_index()].is_unused() { @@ -717,8 +726,8 @@ impl Mapper for RecursivePageTable<'_> { impl Translate for RecursivePageTable<'_> { #[allow(clippy::inconsistent_digit_grouping)] - fn translate(&self, addr: VirtAddr) -> TranslateResult { - let page = Page::containing_address(addr); + fn translate(&self, addr: VirtAddr48) -> TranslateResult { + let page = Page::containing_address_const(addr); let p4 = &self.p4; let p4_entry = &p4[addr.p4_index()]; @@ -797,8 +806,9 @@ impl CleanUp for RecursivePageTable<'_> { unsafe { self.clean_up_addr_range( PageRangeInclusive { - start: Page::from_start_address(VirtAddr::new(0)).unwrap(), - end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(), + start: Page::from_start_address(VirtAddr48::new_const(0)).unwrap(), + end: Page::from_start_address(VirtAddr48::new_const(0xffff_ffff_ffff_f000)) + .unwrap(), }, frame_deallocator, ) @@ -807,7 +817,7 @@ impl CleanUp for RecursivePageTable<'_> { unsafe fn clean_up_addr_range( &mut self, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut D, ) where D: FrameDeallocator, @@ -816,7 +826,7 @@ impl CleanUp for RecursivePageTable<'_> { recursive_index: PageTableIndex, page_table: &mut PageTable, level: PageTableLevel, - range: PageRangeInclusive, + range: PageRangeInclusive>, frame_deallocator: &mut impl FrameDeallocator, ) -> bool { if range.is_empty() { @@ -843,15 +853,17 @@ impl CleanUp for RecursivePageTable<'_> { }) { if let Ok(frame) = entry.frame() { - let start = VirtAddr::forward_checked_impl( + let start = VirtAddr48::forward_checked_impl( table_addr, (offset_per_entry as usize) * i, ) .unwrap(); let end = start + (offset_per_entry - 1); - let start = Page::::containing_address(start); + let start = + Page::>::containing_address_const(start); let start = start.max(range.start); - let end = Page::::containing_address(end); + let end = + Page::>::containing_address_const(end); let end = end.min(range.end); let page_table = [p1_ptr, p2_ptr, p3_ptr][level as usize - 2](start, recursive_index); @@ -913,12 +925,18 @@ impl fmt::Display for InvalidPageTable { } #[inline] -fn p3_ptr(page: Page, recursive_index: PageTableIndex) -> *mut PageTable { +fn p3_ptr( + page: Page>, + recursive_index: PageTableIndex, +) -> *mut PageTable { p3_page(page, recursive_index).start_address().as_mut_ptr() } #[inline] -fn p3_page(page: Page, recursive_index: PageTableIndex) -> Page { +fn p3_page( + page: Page>, + recursive_index: PageTableIndex, +) -> Page> { Page::from_page_table_indices( recursive_index, recursive_index, @@ -928,12 +946,18 @@ fn p3_page(page: Page, recursive_index: PageTableIndex) -> Page } #[inline] -fn p2_ptr(page: Page, recursive_index: PageTableIndex) -> *mut PageTable { +fn p2_ptr( + page: Page>, + recursive_index: PageTableIndex, +) -> *mut PageTable { p2_page(page, recursive_index).start_address().as_mut_ptr() } #[inline] -fn p2_page(page: Page, recursive_index: PageTableIndex) -> Page { +fn p2_page( + page: Page>, + recursive_index: PageTableIndex, +) -> Page> { Page::from_page_table_indices( recursive_index, recursive_index, @@ -943,12 +967,18 @@ fn p2_page(page: Page, recursive_index: PageTableIndex) } #[inline] -fn p1_ptr(page: Page, recursive_index: PageTableIndex) -> *mut PageTable { +fn p1_ptr( + page: Page>, + recursive_index: PageTableIndex, +) -> *mut PageTable { p1_page(page, recursive_index).start_address().as_mut_ptr() } #[inline] -fn p1_page(page: Page, recursive_index: PageTableIndex) -> Page { +fn p1_page( + page: Page>, + recursive_index: PageTableIndex, +) -> Page> { Page::from_page_table_indices( recursive_index, page.p4_index(), diff --git a/src/structures/paging/page.rs b/src/structures/paging/page.rs index b4e7a4e6f..4f9fd9469 100644 --- a/src/structures/paging/page.rs +++ b/src/structures/paging/page.rs @@ -1,9 +1,10 @@ //! Abstractions for default-sized and huge virtual memory pages. +use crate::addr::VirtAddrArithmeticValidity; use crate::sealed::Sealed; use crate::structures::paging::page_table::PageTableLevel; use crate::structures::paging::PageTableIndex; -use crate::VirtAddr; +use crate::{FixedValidity, RuntimeValidity, VirtAddr, VirtAddrValidity}; use core::convert::TryFrom; use core::fmt; #[cfg(feature = "step_trait")] @@ -65,12 +66,12 @@ impl Sealed for super::Size1GiB {} /// A virtual memory page. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] #[repr(C)] -pub struct Page { - start_address: VirtAddr, +pub struct Page { + start_address: VirtAddr, size: PhantomData, } -impl Page { +impl Page { /// The page size in bytes. pub const SIZE: u64 = S::SIZE; @@ -79,11 +80,14 @@ impl Page { /// Returns an error if the address is not correctly aligned (i.e. is not a valid page start). #[inline] #[rustversion::attr(since(1.61), const)] - pub fn from_start_address(address: VirtAddr) -> Result { + pub fn from_start_address(address: VirtAddr) -> Result { if !address.is_aligned_u64(S::SIZE) { return Err(AddressNotAligned); } - Ok(Page::containing_address(address)) + Ok(Page { + start_address: address, + size: PhantomData, + }) } /// Returns the page that starts at the given virtual address. @@ -93,27 +97,17 @@ impl Page { /// The address must be correctly aligned. #[inline] #[rustversion::attr(since(1.61), const)] - pub unsafe fn from_start_address_unchecked(start_address: VirtAddr) -> Self { + pub unsafe fn from_start_address_unchecked(start_address: VirtAddr) -> Self { Page { start_address, size: PhantomData, } } - /// Returns the page that contains the given virtual address. - #[inline] - #[rustversion::attr(since(1.61), const)] - pub fn containing_address(address: VirtAddr) -> Self { - Page { - start_address: address.align_down_u64(S::SIZE), - size: PhantomData, - } - } - /// Returns the start address of the page. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn start_address(self) -> VirtAddr { + pub fn start_address(self) -> VirtAddr { self.start_address } @@ -148,20 +142,49 @@ impl Page { /// Returns a range of pages, exclusive `end`. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn range(start: Self, end: Self) -> PageRange { + pub fn range(start: Self, end: Self) -> PageRange { PageRange { start, end } } /// Returns a range of pages, inclusive `end`. #[inline] #[rustversion::attr(since(1.61), const)] - pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { + pub fn range_inclusive(start: Self, end: Self) -> PageRangeInclusive { PageRangeInclusive { start, end } } +} + +impl Page> +where + FixedValidity: VirtAddrValidity, +{ + /// Returns the page that contains the given fixed-width virtual address. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn containing_address_const(address: VirtAddr>) -> Self { + Page { + start_address: address.align_down_u64(S::SIZE), + size: PhantomData, + } + } +} + +#[cfg(all(feature = "instructions", target_arch = "x86_64"))] +impl Page { + /// Returns the page that contains the given runtime-valid virtual address. + #[inline] + pub fn containing_address(address: VirtAddr) -> Self { + Page { + start_address: address.align_down_u64(S::SIZE), + size: PhantomData, + } + } +} +impl Page { // FIXME: Move this into the `Step` impl, once `Step` is stabilized. pub(crate) fn steps_between_u64(start: &Self, end: &Self) -> Option { - VirtAddr::steps_between_u64(&start.start_address(), &end.start_address()) + VirtAddr::::steps_between_u64(&start.start_address(), &end.start_address()) .map(|steps| steps / S::SIZE) } @@ -180,7 +203,7 @@ impl Page { #[cfg(any(feature = "instructions", feature = "step_trait"))] pub(crate) fn forward_checked_impl(start: Self, count: usize) -> Option { let count = u64::try_from(count).ok()?.checked_mul(S::SIZE)?; - let start_address = VirtAddr::forward_checked_u64(start.start_address, count)?; + let start_address = VirtAddr::::forward_checked_u64(start.start_address, count)?; Some(Self { start_address, size: PhantomData, @@ -188,7 +211,7 @@ impl Page { } } -impl Page { +impl Page { /// Returns the level 2 page table index of this page. #[inline] #[rustversion::attr(since(1.61), const)] @@ -197,7 +220,7 @@ impl Page { } } -impl Page { +impl Page> { /// Returns the 1GiB memory page with the specified page table indices. #[inline] #[rustversion::attr(since(1.61), const)] @@ -208,11 +231,11 @@ impl Page { let mut addr = 0; addr |= p4_index.into_u64() << 39; addr |= p3_index.into_u64() << 30; - Page::containing_address(VirtAddr::new_truncate(addr)) + Page::containing_address_const(crate::VirtAddr48::new_truncate_const(addr)) } } -impl Page { +impl Page> { /// Returns the 2MiB memory page with the specified page table indices. #[inline] #[rustversion::attr(since(1.61), const)] @@ -225,11 +248,11 @@ impl Page { addr |= p4_index.into_u64() << 39; addr |= p3_index.into_u64() << 30; addr |= p2_index.into_u64() << 21; - Page::containing_address(VirtAddr::new_truncate(addr)) + Page::containing_address_const(crate::VirtAddr48::new_truncate_const(addr)) } } -impl Page { +impl Page> { /// Returns the 4KiB memory page with the specified page table indices. #[inline] #[rustversion::attr(since(1.61), const)] @@ -244,17 +267,18 @@ impl Page { addr |= p3_index.into_u64() << 30; addr |= p2_index.into_u64() << 21; addr |= p1_index.into_u64() << 12; - Page::containing_address(VirtAddr::new_truncate(addr)) + Page::containing_address_const(crate::VirtAddr48::new_truncate_const(addr)) } /// Returns the level 1 page table index of this page. #[inline] - pub const fn p1_index(self) -> PageTableIndex { + #[rustversion::attr(since(1.61), const)] + pub fn p1_index(self) -> PageTableIndex { self.start_address.p1_index() } } -impl fmt::Debug for Page { +impl fmt::Debug for Page { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_fmt(format_args!( "Page[{}]({:#x})", @@ -264,37 +288,37 @@ impl fmt::Debug for Page { } } -impl Add for Page { +impl Add for Page { type Output = Self; #[inline] fn add(self, rhs: u64) -> Self::Output { - Page::containing_address(self.start_address() + rhs * S::SIZE) + unsafe { Page::from_start_address_unchecked(self.start_address() + rhs * S::SIZE) } } } -impl AddAssign for Page { +impl AddAssign for Page { #[inline] fn add_assign(&mut self, rhs: u64) { *self = *self + rhs; } } -impl Sub for Page { +impl Sub for Page { type Output = Self; #[inline] fn sub(self, rhs: u64) -> Self::Output { - Page::containing_address(self.start_address() - rhs * S::SIZE) + unsafe { Page::from_start_address_unchecked(self.start_address() - rhs * S::SIZE) } } } -impl SubAssign for Page { +impl SubAssign for Page { #[inline] fn sub_assign(&mut self, rhs: u64) { *self = *self - rhs; } } -impl Sub for Page { +impl Sub for Page { type Output = u64; #[inline] fn sub(self, rhs: Self) -> Self::Output { @@ -303,7 +327,7 @@ impl Sub for Page { } #[cfg(feature = "step_trait")] -impl Step for Page { +impl Step for Page { fn steps_between(start: &Self, end: &Self) -> (usize, Option) { Self::steps_between_impl(start, end) } @@ -316,7 +340,7 @@ impl Step for Page { use core::convert::TryFrom; let count = u64::try_from(count).ok()?.checked_mul(S::SIZE)?; - let start_address = VirtAddr::backward_checked_u64(start.start_address, count)?; + let start_address = VirtAddr::::backward_checked_u64(start.start_address, count)?; Some(Self { start_address, size: PhantomData, @@ -349,14 +373,14 @@ impl Step for Page { /// A range of pages with exclusive upper bound. #[derive(Clone, Copy, PartialEq, Eq, Hash)] #[repr(C)] -pub struct PageRange { +pub struct PageRange { /// The start of the range, inclusive. - pub start: Page, + pub start: Page, /// The end of the range, exclusive. - pub end: Page, + pub end: Page, } -impl PageRange { +impl PageRange { /// Returns whether this range contains no pages. #[inline] pub fn is_empty(&self) -> bool { @@ -380,8 +404,8 @@ impl PageRange { } } -impl Iterator for PageRange { - type Item = Page; +impl Iterator for PageRange { + type Item = Page; #[inline] fn next(&mut self) -> Option { @@ -413,7 +437,9 @@ impl Iterator for PageRange { } // Figure out how many steps there are until the address range gap. - let second_half_start = Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let second_half_start = unsafe { + Page::::from_start_address_unchecked(VirtAddr::::upper_half_start()) + }; let steps_until_gap = Page::steps_between_u64(&self.start, &second_half_start) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -436,7 +462,7 @@ impl Iterator for PageRange { } } -impl DoubleEndedIterator for PageRange { +impl DoubleEndedIterator for PageRange { #[inline] fn next_back(&mut self) -> Option { if self.start < self.end { @@ -466,7 +492,11 @@ impl DoubleEndedIterator for PageRange { } // Figure out how many steps there are until the address range gap. - let first_half_end = Page::::containing_address(VirtAddr::new(0x7fff_ffff_f000)); + let first_half_end = unsafe { + Page::::from_start_address_unchecked(VirtAddr::::new_unsafe( + VirtAddr::::lower_half_end().as_u64() & !(S::SIZE - 1), + )) + }; let steps_until_gap = Page::steps_between_u64(&first_half_end, &self.end) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -482,18 +512,18 @@ impl DoubleEndedIterator for PageRange { } } -impl PageRange { +impl PageRange { /// Converts the range of 2MiB pages to a range of 4KiB pages. #[inline] - pub fn as_4kib_page_range(self) -> PageRange { + pub fn as_4kib_page_range(self) -> PageRange { PageRange { - start: Page::containing_address(self.start.start_address()), - end: Page::containing_address(self.end.start_address()), + start: unsafe { Page::from_start_address_unchecked(self.start.start_address()) }, + end: unsafe { Page::from_start_address_unchecked(self.end.start_address()) }, } } } -impl fmt::Debug for PageRange { +impl fmt::Debug for PageRange { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("PageRange") .field("start", &self.start) @@ -505,14 +535,14 @@ impl fmt::Debug for PageRange { /// A range of pages with inclusive upper bound. #[derive(Clone, Copy, PartialEq, Eq, Hash)] #[repr(C)] -pub struct PageRangeInclusive { +pub struct PageRangeInclusive { /// The start of the range, inclusive. - pub start: Page, + pub start: Page, /// The end of the range, inclusive. - pub end: Page, + pub end: Page, } -impl PageRangeInclusive { +impl PageRangeInclusive { /// Returns whether this range contains no pages. #[inline] pub fn is_empty(&self) -> bool { @@ -536,8 +566,8 @@ impl PageRangeInclusive { } } -impl Iterator for PageRangeInclusive { - type Item = Page; +impl Iterator for PageRangeInclusive { + type Item = Page; #[inline] fn next(&mut self) -> Option { @@ -547,7 +577,7 @@ impl Iterator for PageRangeInclusive { // If the end of the inclusive range is the maximum page possible for size S, // incrementing start until it is greater than the end will cause an integer overflow. // So instead, in that case we decrement end rather than incrementing start. - let max_page_addr = VirtAddr::new(u64::MAX) - (S::SIZE - 1); + let max_page_addr = VirtAddr::::max_value() - (S::SIZE - 1); if self.start.start_address() < max_page_addr { self.start += 1; } else { @@ -578,7 +608,9 @@ impl Iterator for PageRangeInclusive { } // Figure out how many steps there are until the address range gap. - let second_half_start = Page::::containing_address(VirtAddr::new(0xffff_8000_0000_0000)); + let second_half_start = unsafe { + Page::::from_start_address_unchecked(VirtAddr::::upper_half_start()) + }; let steps_until_gap = Page::steps_between_u64(&self.start, &second_half_start) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -601,7 +633,7 @@ impl Iterator for PageRangeInclusive { } } -impl DoubleEndedIterator for PageRangeInclusive { +impl DoubleEndedIterator for PageRangeInclusive { #[inline] fn next_back(&mut self) -> Option { if self.start <= self.end { @@ -640,7 +672,11 @@ impl DoubleEndedIterator for PageRangeInclusive { } // Figure out how many steps there are until the address range gap. - let first_half_end = Page::::containing_address(VirtAddr::new(0x7fff_ffff_f000)); + let first_half_end = unsafe { + Page::::from_start_address_unchecked(VirtAddr::::new_unsafe( + VirtAddr::::lower_half_end().as_u64() & !(S::SIZE - 1), + )) + }; let steps_until_gap = Page::steps_between_u64(&first_half_end, &self.end) .filter(|steps| *steps <= n && *steps > 0); if let Some(steps_until_gap) = steps_until_gap { @@ -656,7 +692,7 @@ impl DoubleEndedIterator for PageRangeInclusive { } } -impl fmt::Debug for PageRangeInclusive { +impl fmt::Debug for PageRangeInclusive { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("PageRangeInclusive") .field("start", &self.start) @@ -666,7 +702,10 @@ impl fmt::Debug for PageRangeInclusive { } #[cfg(kani)] -impl kani::Arbitrary for Page { +impl kani::Arbitrary for Page> +where + FixedValidity: VirtAddrValidity, +{ fn any() -> Self { Self::containing_address(kani::any()) } @@ -686,6 +725,52 @@ impl fmt::Display for AddressNotAligned { mod tests { use super::*; + /// A fixed-VA48 page used by Ring 3 arithmetic tests. + type Page = super::Page>; + + /// A fixed-VA48 address used by Ring 3 arithmetic tests. + type VirtAddr = crate::VirtAddr48; + + #[test] + fn page_validity_defaults_and_explicit_policy() { + let _: super::Page = unsafe { + super::Page::from_start_address_unchecked(crate::VirtAddrRT::new_unsafe(0x20_0000)) + }; + + let page57: super::Page> = + super::Page::containing_address_const(crate::VirtAddr57::new_const( + 0x00ff_0000_0000_1000, + )); + assert_eq!(page57.start_address().as_u64(), 0x00ff_0000_0000_1000); + } + + #[test] + #[cfg(feature = "step_trait")] + fn page57_step_uses_la57_gap() { + let low_end = super::Page::>::from_start_address( + crate::VirtAddr57::new_const(0x00ff_ffff_ffff_f000), + ) + .unwrap(); + let upper_start = super::Page::>::from_start_address( + crate::VirtAddr57::new_const(0xff00_0000_0000_0000), + ) + .unwrap(); + + assert_eq!(Step::forward(low_end, 1), upper_start); + assert_eq!(Step::backward(upper_start, 1), low_end); + } + + #[test] + fn p4_constructors_remain_va48() { + let page = super::Page::from_page_table_indices( + PageTableIndex::new(1), + PageTableIndex::new(2), + PageTableIndex::new(3), + PageTableIndex::new(4), + ); + let _: super::Page> = page; + } + fn test_is_hash() {} #[test] @@ -700,15 +785,15 @@ mod tests { let page_size = Size4KiB::SIZE; let number = 1000; - let start_addr = VirtAddr::new(0xdead_beaf); - let start: Page = Page::containing_address(start_addr); + let start_addr = VirtAddr::new_const(0xdead_beaf); + let start: Page = Page::containing_address_const(start_addr); let end = start + number; let mut range = Page::range(start, end); for i in 0..number { assert_eq!( range.next(), - Some(Page::containing_address(start_addr + page_size * i)) + Some(Page::containing_address_const(start_addr + page_size * i)) ); } assert_eq!(range.next(), None); @@ -717,7 +802,7 @@ mod tests { for i in 0..=number { assert_eq!( range_inclusive.next(), - Some(Page::containing_address(start_addr + page_size * i)) + Some(Page::containing_address_const(start_addr + page_size * i)) ); } assert_eq!(range_inclusive.next(), None); @@ -728,15 +813,15 @@ mod tests { let page_size = Size4KiB::SIZE; let number = 1000; - let start_addr = VirtAddr::new(u64::MAX).align_down(page_size) - number * page_size; - let start: Page = Page::containing_address(start_addr); + let start_addr = VirtAddr::new_const(u64::MAX).align_down(page_size) - number * page_size; + let start: Page = Page::containing_address_const(start_addr); let end = start + number; let mut range_inclusive = Page::range_inclusive(start, end); for i in 0..=number { assert_eq!( range_inclusive.next(), - Some(Page::containing_address(start_addr + page_size * i)) + Some(Page::containing_address_const(start_addr + page_size * i)) ); } assert_eq!(range_inclusive.next(), None); @@ -747,8 +832,8 @@ mod tests { fn test_page_range_next_jumping_gap_panics() { let start = 0x7fff_ffff_f000; let end = 0xffff_8000_0000_0000; - let start = VirtAddr::new(start); - let end = VirtAddr::new(end); + let start = VirtAddr::new_const(start); + let end = VirtAddr::new_const(end); let start = Page::::from_start_address(start).unwrap(); let end = Page::from_start_address(end).unwrap(); Page::range(start, end).next(); @@ -760,8 +845,8 @@ mod tests { fn test_page_range_next_back_jumping_gap_panics() { let start = 0x7fff_ffff_f000; let end = 0xffff_8000_0000_0000; - let start = VirtAddr::new(start); - let end = VirtAddr::new(end); + let start = VirtAddr::new_const(start); + let end = VirtAddr::new_const(end); let start = Page::::from_start_address(start).unwrap(); let end = Page::from_start_address(end).unwrap(); Page::range(start, end).next_back(); @@ -772,8 +857,8 @@ mod tests { fn test_page_range_inclusive_next_not_jumping_gap_panics() { let start = 0x7fff_ffff_f000; let end = 0x7fff_ffff_f000; - let start = VirtAddr::new(start); - let end = VirtAddr::new(end); + let start = VirtAddr::new_const(start); + let end = VirtAddr::new_const(end); let start = Page::::from_start_address(start).unwrap(); let end = Page::from_start_address(end).unwrap(); Page::range_inclusive(start, end).next(); @@ -784,8 +869,8 @@ mod tests { fn test_page_range_inclusive_next_back_not_jumping_gap_panics() { let start = 0x7fff_ffff_f000; let end = 0xffff_8000_0000_0000; - let start = VirtAddr::new(start); - let end = VirtAddr::new(end); + let start = VirtAddr::new_const(start); + let end = VirtAddr::new_const(end); let start = Page::::from_start_address(start).unwrap(); let end = Page::from_start_address(end).unwrap(); Page::range_inclusive(start, end).next_back(); @@ -797,8 +882,8 @@ mod tests { fn test_page_range_inclusive_next_jumping_gap_panics() { let start = 0x7fff_ffff_f000; let end = 0x7fff_ffff_f000; - let start = VirtAddr::new(start); - let end = VirtAddr::new(end); + let start = VirtAddr::new_const(start); + let end = VirtAddr::new_const(end); let start = Page::::from_start_address(start).unwrap(); let end = Page::from_start_address(end).unwrap(); Page::range_inclusive(start, end).next(); @@ -810,8 +895,8 @@ mod tests { fn test_page_range_inclusive_next_back_jumping_gap_panics() { let start = 0xffff_8000_0000_0000; let end = 0xffff_8000_0000_0000; - let start = VirtAddr::new(start); - let end = VirtAddr::new(end); + let start = VirtAddr::new_const(start); + let end = VirtAddr::new_const(end); let start = Page::::from_start_address(start).unwrap(); let end = Page::from_start_address(end).unwrap(); Page::range_inclusive(start, end).next_back(); @@ -820,8 +905,8 @@ mod tests { #[test] pub fn test_page_range_len() { - let start_addr = VirtAddr::new(0xdead_beaf); - let start = Page::::containing_address(start_addr); + let start_addr = VirtAddr::new_const(0xdead_beaf); + let start = Page::::containing_address_const(start_addr); let end = start + 50; let range = PageRange { start, end }; @@ -856,9 +941,10 @@ mod tests { (0, 0x10_0000, Some(0x1_0000_0000)), ]; for (start, count, result) in test_cases { - let start = Page::::from_start_address(VirtAddr::new(start)).unwrap(); - let result = result - .map(|result| Page::::from_start_address(VirtAddr::new(result)).unwrap()); + let start = Page::::from_start_address(VirtAddr::new_const(start)).unwrap(); + let result = result.map(|result| { + Page::::from_start_address(VirtAddr::new_const(result)).unwrap() + }); assert_eq!(Step::forward_checked(start, count), result); } } @@ -885,9 +971,10 @@ mod tests { (0x1_0000_0000, 0x10_0000, Some(0)), ]; for (start, count, result) in test_cases { - let start = Page::::from_start_address(VirtAddr::new(start)).unwrap(); - let result = result - .map(|result| Page::::from_start_address(VirtAddr::new(result)).unwrap()); + let start = Page::::from_start_address(VirtAddr::new_const(start)).unwrap(); + let result = result.map(|result| { + Page::::from_start_address(VirtAddr::new_const(result)).unwrap() + }); assert_eq!(Step::backward_checked(start, count), result); } } @@ -931,8 +1018,8 @@ mod tests { (0x0000_0000_0000, 0x1000_0000_0000, usize::MAX, None), ]; for (start, end, lower, upper) in test_cases { - let start = Page::::from_start_address(VirtAddr::new(start)).unwrap(); - let end = Page::from_start_address(VirtAddr::new(end)).unwrap(); + let start = Page::::from_start_address(VirtAddr::new_const(start)).unwrap(); + let end = Page::from_start_address(VirtAddr::new_const(end)).unwrap(); assert_eq!(Step::steps_between(&start, &end), (lower, upper)); } } @@ -940,7 +1027,7 @@ mod tests { #[test] #[cfg(feature = "step_trait")] fn page_step_overflowing() { - let page = |addr| Page::::from_start_address(VirtAddr::new(addr)).unwrap(); + let page = |addr| Page::::from_start_address(VirtAddr::new_const(addr)).unwrap(); assert_eq!( Step::forward_overflowing(page(0x7fff_ffff_f000), 1), diff --git a/src/structures/tss.rs b/src/structures/tss.rs index f0174f3f4..8f5cdf810 100644 --- a/src/structures/tss.rs +++ b/src/structures/tss.rs @@ -1,25 +1,25 @@ //! Provides a type for the task state segment structure. -use crate::VirtAddr; use core::{ fmt::{self, Display}, mem::size_of, }; +use crate::{RuntimeValidity, VirtAddr, VirtAddrValidity}; + /// In 64-bit mode the TSS holds information that is not /// directly related to the task-switch mechanism, /// but is used for stack switching when an interrupt or exception occurs. -#[derive(Debug, Clone, Copy)] #[repr(C, packed(4))] -pub struct TaskStateSegment { +pub struct TaskStateSegment { reserved_1: u32, /// The full 64-bit canonical forms of the stack pointers (RSP) for privilege levels 0-2. /// The stack pointers used when a privilege level change occurs from a lower privilege level to a higher one. - pub privilege_stack_table: [VirtAddr; 3], + pub privilege_stack_table: [VirtAddr; 3], reserved_2: u64, /// The full 64-bit canonical forms of the interrupt stack table (IST) pointers. /// The stack pointers used when an entry in the Interrupt Descriptor Table has an IST value other than 0. - pub interrupt_stack_table: [VirtAddr; 7], + pub interrupt_stack_table: [VirtAddr; 7], reserved_3: u64, reserved_4: u16, /// The 16-bit offset to the I/O permission bit map from the 64-bit TSS base. It must not @@ -27,7 +27,7 @@ pub struct TaskStateSegment { pub iomap_base: u16, } -impl TaskStateSegment { +impl TaskStateSegment { /// Creates a new TSS with zeroed privilege and interrupt stack table and an /// empty I/O-Permission Bitmap. /// @@ -35,11 +35,12 @@ impl TaskStateSegment { /// `size_of::() - 1`, this means that `iomap_base` is /// initialized to `size_of::()`. #[inline] - pub const fn new() -> TaskStateSegment { + #[rustversion::attr(since(1.61), const)] + pub fn new_with_validity() -> Self { TaskStateSegment { privilege_stack_table: [VirtAddr::zero(); 3], interrupt_stack_table: [VirtAddr::zero(); 7], - iomap_base: size_of::() as u16, + iomap_base: size_of::() as u16, reserved_1: 0, reserved_2: 0, reserved_3: 0, @@ -48,10 +49,53 @@ impl TaskStateSegment { } } -impl Default for TaskStateSegment { +// These traits are implemented manually because Rust 1.59 has limited derive support for generic +// packed structs. They can use derive once the MSRV is raised to Rust 1.69. +impl Copy for TaskStateSegment {} + +impl Clone for TaskStateSegment { + fn clone(&self) -> Self { + *self + } +} + +impl fmt::Debug for TaskStateSegment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let reserved_1 = self.reserved_1; + let privilege_stack_table = self.privilege_stack_table; + let reserved_2 = self.reserved_2; + let interrupt_stack_table = self.interrupt_stack_table; + let reserved_3 = self.reserved_3; + let reserved_4 = self.reserved_4; + let iomap_base = self.iomap_base; + + f.debug_struct("TaskStateSegment") + .field("reserved_1", &reserved_1) + .field("privilege_stack_table", &privilege_stack_table) + .field("reserved_2", &reserved_2) + .field("interrupt_stack_table", &interrupt_stack_table) + .field("reserved_3", &reserved_3) + .field("reserved_4", &reserved_4) + .field("iomap_base", &iomap_base) + .finish() + } +} + +impl TaskStateSegment { + /// Creates a new runtime-valid TSS. + /// + /// Stack addresses assigned later retain their creation-time validity guarantees. + #[inline] + #[rustversion::attr(since(1.61), const)] + pub fn new() -> Self { + Self::new_with_validity() + } +} + +impl Default for TaskStateSegment { #[inline] fn default() -> Self { - Self::new() + Self::new_with_validity() } } @@ -123,5 +167,10 @@ mod tests { // Per the SDM, the minimum size of a TSS is 0x68 bytes, giving a // minimum limit of 0x67. assert_eq!(size_of::(), 0x68); + assert_eq!( + size_of::>>(), + 0x68 + ); + assert_eq!(size_of::>(), 0x68); } } diff --git a/testing/src/tests.rs b/testing/src/tests.rs index d221fefcc..a6ea84d9f 100644 --- a/testing/src/tests.rs +++ b/testing/src/tests.rs @@ -4,3 +4,17 @@ fn example_test() { assert_eq!(0, 0); serial_println!("[ok]"); } + +#[test_case] +fn runtime_virtual_address_validity_in_la48() { + use x86_64::registers::control::{Cr4, Cr4Flags}; + use x86_64::{VirtAddr48, VirtAddr57, VirtAddrRT}; + + serial_print!("runtime_virtual_address_validity_in_la48... "); + assert!(!Cr4::read().contains(Cr4Flags::L5_PAGING)); + assert!(VirtAddrRT::try_new(0x0000_7fff_ffff_ffff).is_ok()); + assert!(VirtAddrRT::try_new(0x00ff_ffff_ffff_ffff).is_err()); + assert!(VirtAddr48::new_const(0x1234).is_valid_currently()); + assert!(!VirtAddr57::new_const(0x00ff_0000_0000_0000).is_valid_currently()); + serial_println!("[ok]"); +}