85 if (0_u64 != page) [[likely]] {
103 for (
auto current {
LEVELS - 1_usize}; current > level; --current) {
105 if (
nullptr == entries) [[unlikely]] {
115 if (0_u64 == page) [[unlikely]] {
129 return (
nullptr != entries) ? &entries[
paging::index(virt, level)] :
nullptr;
152 if (
nullptr == page) [[unlikely]] {
189 if (
nullptr != page) {
201 for (
auto step {0_usize}; step <
LEVELS; ++step) {
202 const auto level {
LEVELS - 1_usize - step};
204 if (
nullptr == entries) [[unlikely]] {
245 const auto*
const virt {std::bit_cast<const igros_byte_t*>(
static_cast<igros_usize_t>(phys) + Platform::kernelOffset())};
247 if ((virt >= Platform::textStart()) && (virt < Platform::rodataStart())) {
251 if ((virt >= Platform::rodataStart()) && (virt < Platform::dataStart())) {
270 if (0_u64 == top) [[unlikely]] {
271 klib::kprintf(
"Paging:\t\tFAILED (no memory for page tables)");
297 if (!mapped) [[unlikely]] {
298 klib::kprintf(
"Paging:\t\tFAILED (can't build kernel mapping)");
310 "Paging:\t\t%s (%zu levels, root at 0x%08llx, direct map %llu MB%s)",
313 static_cast<unsigned long long>(top),
314 static_cast<unsigned long long>(end >> 20),
315 (0_usize != noExecute) ?
", NX" :
""
327 if (0_u64 == phys) [[unlikely]] {
331 auto*
const alias {std::bit_cast<volatile igros_byte_t*>(
paging::table(phys))};
332 if (
nullptr == alias) [[unlikely]] {
343 static_cast<volatile igros_byte_t*
>(page)[0x123] = 0x5A_u8;
344 passed = (0x5A_u8 == alias[0x123]);
345 alias[0x456] = 0xA5_u8;
346 passed = passed && (0xA5_u8 ==
static_cast<volatile igros_byte_t*
>(page)[0x456]);
366Reason: %s page, %s access, %s mode%s%s
378 cpu::dumpRegisters(regs);
static constexpr auto check(const T value) noexcept -> bool
Check alignment.
static constexpr auto up(const T value) noexcept -> T
Align up.
static constexpr auto from(const U value) noexcept -> kFlags
From raw value (hardware register, table entry).
static auto end() noexcept -> phys_t
End of directly mapped physical memory.
static void setEnd(const phys_t end) noexcept
Set end of directly mapped physical memory (paging mapped more memory).
static auto pointer(const phys_t phys, const phys_t size=sizeof(T)) noexcept -> T *
Pointer to physical memory (nullptr if range is not directly mapped).
static void free(const phys_t page) noexcept
Free physical page.
static auto alloc(const phys_t limit=MEMORY_LIMIT) noexcept -> phys_t
Allocate physical page below limit (0 if no memory left).
static auto top() noexcept -> phys_t
End of highest memory ever marked free (top of usable RAM, clamped to MEMORY_LIMIT).
static constexpr void install() noexcept
Install exceptions handler.
static constexpr std::array< const char *const, 32_usize > NAME
Exceptions names.
static void disable() noexcept
Disable interrupts.
static void unmap(const mem::virt_t virt) noexcept
Unmap 4 KB page.
igros_usize_t entry_t
Page table entry (register-wide).
static auto table(const mem::phys_t phys) noexcept -> entry_t *
Table at physical address (through direct map).
static auto selfTest() noexcept -> bool
Check mapping on a scratch page.
static auto root() noexcept -> mem::phys_t
Current top level table physical address.
static auto allocTable() noexcept -> mem::phys_t
Allocate empty table (reachable through direct map, 0 if no memory).
static auto directMapEnd() noexcept -> mem::phys_t
End of physical memory reachable through direct map.
static auto kernelFlags(const mem::phys_t phys) noexcept -> klib::kFlags< FLAGS >
Rights of kernel offset mapping of low memory physical page (kernel sections).
static auto entry(const mem::phys_t top, const mem::virt_t virt, const bool create, const bool user, const igros_usize_t level=0_usize) noexcept -> entry_t *
Entry of virtual address at given level (0 - page table) of given hierarchy.
@ PROTECTED
Protection violation (else page not present).
@ FETCH
Instruction fetch.
@ USER
User mode access (else kernel).
@ RESERVED
Reserved bit set in entry.
@ WRITE
Write access (else read).
static constexpr auto ENTRIES
Entries per table.
static auto map(const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags< FLAGS > flags) noexcept -> bool
Map 4 KB page (false if structures can't be allocated or address is in a large page).
static constexpr auto LEVELS
Table levels.
static void exHandler(const register_t *regs) noexcept
Page fault handler.
static constexpr auto LARGE_PAGE_SIZE
Large page size (4 MB on i386, 2 MB on x86_64).
static constexpr auto PAGE_SIZE
Page size in bytes.
static auto makeEntry(const mem::phys_t phys, const klib::kFlags< FLAGS > flags) noexcept -> entry_t
Entry value for physical address with flags (drops NX if not enabled).
static auto dynamicStart() noexcept -> igros_byte_t *
Start of kernel dynamic mappings (right above direct map).
static void init() noexcept
Build kernel page tables and switch to them:
static auto index(const mem::virt_t virt, const igros_usize_t level) noexcept -> igros_usize_t
Index of virtual address in table of given level (0 - page table).
static constexpr auto INVALID
Address isn't mapped.
static constexpr auto INDEX_BITS
static constexpr auto DIRECT_MAP_LIMIT
Physical memory mapped at kernel offset (-2 GB .. -1 GB), rest is for dynamic mappings.
static constexpr auto ADDRESS_MASK
Physical address bits of entry (52-bit physical addresses).
static constexpr auto PAGE_SHIFT
Page size bits.
FLAGS
Page table entry flags.
@ NO_EXECUTE
Instruction fetch not allowed (EFER.NXE).
@ HUGE
Large page (directory levels).
static auto translate(const mem::virt_t virt) noexcept -> mem::phys_t
Physical address of virtual address (INVALID if not mapped).
static auto mapLarge(const mem::phys_t top, const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags< FLAGS > flags) noexcept -> bool
Map large page in given hierarchy (fresh directory entry only).
CR0 - CR4 registers operations.
void pageInvalidate(const void *const page) noexcept
Invalidate TLB entry of page.
auto readCR2() noexcept -> igros::igros_usize_t
Read CR2 register.
auto readCR0() noexcept -> igros::igros_usize_t
Read CR0 register.
auto readCR3() noexcept -> igros::igros_usize_t
Read CR3 register.
void writeCR3(const igros::igros_usize_t value) noexcept
Write CR3 register.
void writeCR0(const igros::igros_usize_t value) noexcept
Write CR0 register.
Direct map of physical memory.
Exceptions low-level operations.
auto cpuEnableNoExecute() noexcept -> bool
Enable no-execute page protection (EFER.NXE) if CPU supports it.
std::size_t igros_usize_t
Unsigned size.
std::uint8_t igros_byte_t
8-bit unsigned integer
Kernel alignment functions definitions.
Kernel text print functions.
void kprintf(const char *const format,...) noexcept
Kernel printf function (arguments are checked against format at compile time).
constexpr auto make_kflags(const Args... flags) noexcept -> kFlags< T >
Flags creation helper function from set flags.
Memory management: physical page allocator, direct map, virtual addresses.
constexpr auto LOW_MEMORY_LIMIT
Physical memory mapped by boot code at kernel virtual offset (see boot.s).
igros_quad_t phys_t
Physical address (64-bit: physical memory may lie above 4 GB).
auto virtOf(const T *const object) noexcept -> virt_t
Virtual address of object.
virt_t
Virtual address: translated by paging, never dereferenced through this type.
constexpr auto address(const virt_t virt) noexcept -> igros_usize_t
Numeric value of virtual address.
constinit paging::entry_t noExecute
NX bit value if no-execute is enabled (else 0: NX can't be used).
void invalidate(const mem::virt_t virt) noexcept
Invalidate TLB entry of virtual address (invlpg takes memory operand).
Code shared by i386 and x86_64: ports, interrupts, exceptions, paging.
static constexpr auto flag(const paging::FLAGS value) noexcept -> paging::entry_t
Entry flag value.
static constexpr auto hasFlag(const paging::entry_t entry, const paging::FLAGS value) noexcept -> bool
Is flag set in entry.
@ WP
Write protect (kernel can't write to read-only pages).
x86 family current (native) architecture selection
Paging (i386 2-level and x86_64 4-level page tables).
Interrupts low-level operations.