IgrOS-Kernel 0.1.34.545
Simple C++20 Kernel for i386/x86_64
Loading...
Searching...
No Matches
paging.cpp
Go to the documentation of this file.
1
11
12
13// C++
14#include <algorithm>
15#include <bit>
16#include <span>
17// IgrOS-Kernel arch x86
18#include <arch/x86/cr.hpp>
20#include <arch/x86/features.hpp>
21#include <arch/x86/irq.hpp>
22#include <arch/x86/native.hpp>
23#include <arch/x86/paging.hpp>
24// IgrOS-Kernel library
25#include <klib/kAlign.hpp>
26#include <klib/kPrint.hpp>
27// IgrOS-Kernel memory
28#include <mem/direct.hpp>
29// IgrOS-Kernel platform
30#include <platform/platform.hpp>
31
32
33// x86 namespace
34namespace igros::x86 {
35
36
38 namespace {
39
40
43
44
45
48 void invalidate(const mem::virt_t virt) noexcept {
49 ::pageInvalidate(std::bit_cast<const igros_byte_t*>(mem::address(virt)));
50 }
51
52
53 } // namespace
54
55
59 [[nodiscard]]
60 constexpr static auto flag(const paging::FLAGS value) noexcept -> paging::entry_t {
61 return static_cast<paging::entry_t>(value);
62 }
63
68 [[nodiscard]]
69 constexpr static auto hasFlag(const paging::entry_t entry, const paging::FLAGS value) noexcept -> bool {
70 return klib::kFlags<paging::FLAGS>::from(entry).isSet(value);
71 }
72
73
74 // Table at physical address (through direct map)
75 [[nodiscard]]
76 auto paging::table(const mem::phys_t phys) noexcept -> entry_t* {
77 // Only directly mapped memory is reachable through kernel offset
79 }
80
81 // Allocate empty table (reachable through direct map, 0 if no memory)
82 [[nodiscard]]
83 auto paging::allocTable() noexcept -> mem::phys_t {
84 const auto page {mem::phys::alloc(mem::direct::end())};
85 if (0_u64 != page) [[likely]] {
86 std::ranges::fill(std::span {paging::table(page), ENTRIES}, entry_t {0});
87 }
88 return page;
89 }
90
91 // Index of virtual address in table of given level (0 - page table)
92 [[nodiscard]]
93 auto paging::index(const mem::virt_t virt, const igros_usize_t level) noexcept -> igros_usize_t {
94 return (mem::address(virt) >> (PAGE_SHIFT + (INDEX_BITS * level))) & (ENTRIES - 1_usize);
95 }
96
97
98 // Entry of virtual address at given level of given hierarchy (create - allocate missing tables)
99 [[nodiscard]]
100 auto paging::entry(const mem::phys_t top, const mem::virt_t virt, const bool create, const bool user, const igros_usize_t level) noexcept -> entry_t* {
101 auto* entries {paging::table(top)};
102 // Walk down to table of requested level
103 for (auto current {LEVELS - 1_usize}; current > level; --current) {
104 // Check entries of current level
105 if (nullptr == entries) [[unlikely]] {
106 return nullptr;
107 }
108 auto &next {entries[paging::index(virt, current)]};
109 if (!hasFlag(next, FLAGS::PRESENT)) {
110 // Missing table
111 if (!create) {
112 return nullptr;
113 }
114 const auto page {paging::allocTable()};
115 if (0_u64 == page) [[unlikely]] {
116 return nullptr;
117 }
118 // Access rights are checked on every level: keep upper levels permissive
119 next = static_cast<entry_t>(page) | flag(FLAGS::PRESENT) | flag(FLAGS::WRITABLE) | (user ? flag(FLAGS::USER) : 0_usize);
120 } else if (hasFlag(next, FLAGS::HUGE)) {
121 // Address belongs to large page
122 return nullptr;
123 } else if (user) {
124 next |= flag(FLAGS::USER);
125 }
126 entries = paging::table(next & ADDRESS_MASK);
127 }
128 // Entry of requested level
129 return (nullptr != entries) ? &entries[paging::index(virt, level)] : nullptr;
130 }
131
132 // Entry value for physical address with flags (drops NX if not enabled)
133 [[nodiscard]]
134 auto paging::makeEntry(const mem::phys_t phys, const klib::kFlags<FLAGS> flags) noexcept -> entry_t {
135 const auto rights {flags.value() & ~ADDRESS_MASK & ~flag(FLAGS::HUGE) & ~(flag(FLAGS::NO_EXECUTE) & ~noExecute)};
136 return static_cast<entry_t>(phys) | rights | flag(FLAGS::PRESENT);
137 }
138
139 // Map 4 KB page in given hierarchy
140 [[nodiscard]]
141 // NOLINTNEXTLINE(bugprone-easily-swappable-parameters): table hierarchy first, like every paging::entry() walk
142 auto paging::map(const mem::phys_t top, const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags<FLAGS> flags) noexcept -> bool {
143 // Page aligned addresses only
145 return false;
146 }
147 // Physical address must fit into entry
148 if (0_u64 != (phys & ~static_cast<mem::phys_t>(ADDRESS_MASK))) [[unlikely]] {
149 return false;
150 }
151 auto* const page {paging::entry(top, virt, true, flags.isSet(FLAGS::USER))};
152 if (nullptr == page) [[unlikely]] {
153 return false;
154 }
155 *page = paging::makeEntry(phys, flags);
156 invalidate(virt);
157 return true;
158 }
159
160 // Map large page in given hierarchy (fresh directory entry only)
161 [[nodiscard]]
162 // NOLINTNEXTLINE(bugprone-easily-swappable-parameters): table hierarchy first, like every paging::entry() walk
163 auto paging::mapLarge(const mem::phys_t top, const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags<FLAGS> flags) noexcept -> bool {
164 // Large page aligned addresses only, physical address must fit into entry
166 (0_u64 != (phys & ~static_cast<mem::phys_t>(ADDRESS_MASK)))) [[unlikely]] {
167 return false;
168 }
169 auto* const large {paging::entry(top, virt, true, flags.isSet(FLAGS::USER), 1_usize)};
170 // Don't replace page table (its pages would leak)
171 if ((nullptr == large) || (hasFlag(*large, FLAGS::PRESENT) && !hasFlag(*large, FLAGS::HUGE))) [[unlikely]] {
172 return false;
173 }
174 *large = paging::makeEntry(phys, flags) | flag(FLAGS::HUGE);
175 invalidate(virt);
176 return true;
177 }
178
179
180 // Map 4 KB page
181 [[nodiscard]]
182 auto paging::map(const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags<FLAGS> flags) noexcept -> bool {
183 return paging::map(paging::root(), phys, virt, flags);
184 }
185
186 // Unmap 4 KB page
187 void paging::unmap(const mem::virt_t virt) noexcept {
188 auto* const page {paging::entry(paging::root(), virt, false, false)};
189 if (nullptr != page) {
190 *page = 0_usize;
191 invalidate(virt);
192 }
193 }
194
195
196 // Physical address of virtual address (INVALID if not mapped)
197 [[nodiscard]]
198 auto paging::translate(const mem::virt_t virt) noexcept -> mem::phys_t {
199 const auto address {static_cast<mem::phys_t>(mem::address(virt))};
200 const auto* entries {paging::table(paging::root())};
201 for (auto step {0_usize}; step < LEVELS; ++step) {
202 const auto level {LEVELS - 1_usize - step};
203 // Check entries of current level
204 if (nullptr == entries) [[unlikely]] {
205 return INVALID;
206 }
207 const auto current {entries[paging::index(virt, level)]};
208 if (!hasFlag(current, FLAGS::PRESENT)) {
209 return INVALID;
210 }
211 // Page (or large page) covering address
212 if ((0_usize == level) || hasFlag(current, FLAGS::HUGE)) {
213 const auto size {mem::phys_t {1} << (PAGE_SHIFT + (INDEX_BITS * level))};
214 return (static_cast<mem::phys_t>(current & ADDRESS_MASK) & ~(size - 1_u64)) | (address & (size - 1_u64));
215 }
216 entries = paging::table(current & ADDRESS_MASK);
217 }
218 return INVALID;
219 }
220
221 // Current top level table physical address
222 [[nodiscard]]
223 auto paging::root() noexcept -> mem::phys_t {
224 return static_cast<mem::phys_t>(::readCR3() & ADDRESS_MASK);
225 }
226
227
228 // End of physical memory reachable through direct map
229 [[nodiscard]]
230 auto paging::directMapEnd() noexcept -> mem::phys_t {
231 return mem::direct::end();
232 }
233
234 // Start of kernel dynamic mappings (right above direct map)
235 [[nodiscard]]
236 auto paging::dynamicStart() noexcept -> igros_byte_t* {
237 return std::bit_cast<igros_byte_t*>(platform::Platform::kernelOffset() + static_cast<igros_usize_t>(DIRECT_MAP_LIMIT));
238 }
239
240
241 // Rights of kernel offset mapping of low memory physical page (kernel sections)
242 [[nodiscard]]
244 using platform::Platform;
245 const auto* const virt {std::bit_cast<const igros_byte_t*>(static_cast<igros_usize_t>(phys) + Platform::kernelOffset())};
246 // Code: read + execute
247 if ((virt >= Platform::textStart()) && (virt < Platform::rodataStart())) {
249 }
250 // Read-only data: read
251 if ((virt >= Platform::rodataStart()) && (virt < Platform::dataStart())) {
253 }
254 // Data, bss, page tables, low memory, video memory: read + write
256 }
257
258
259 // Build kernel page tables and switch to them
260 void paging::init() noexcept {
261
262 // Page fault handler
264
265 // No-execute protection (before any NX bit is written: it's reserved until enabled)
266 noExecute = ::cpuEnableNoExecute() ? flag(FLAGS::NO_EXECUTE) : 0_usize;
267
268 // Top level table (tables come from memory mapped by current page tables)
269 const auto top {paging::allocTable()};
270 if (0_u64 == top) [[unlikely]] {
271 klib::kprintf("Paging:\t\tFAILED (no memory for page tables)");
272 return;
273 }
274
275 // Low memory: at kernel offset with kernel section rights and identity (boot data);
276 // identity null page stays unmapped to catch null pointer dereference,
277 // boot stack guard page stays unmapped to catch kernel stack overflow
279 const auto offset {platform::Platform::kernelOffset()};
280 const auto guard {static_cast<mem::phys_t>(std::bit_cast<igros_usize_t>(platform::Platform::stackGuard()) - offset)};
281 auto mapped {true};
282 for (auto phys {0_u64}; phys < mem::LOW_MEMORY_LIMIT; phys += PAGE_SIZE) {
283 const auto page {static_cast<igros_usize_t>(phys)};
284 if (guard != phys) {
285 mapped = mapped && paging::map(top, phys, static_cast<mem::virt_t>(page + offset), paging::kernelFlags(phys));
286 }
287 if (0_u64 != phys) {
288 mapped = mapped && paging::map(top, phys, static_cast<mem::virt_t>(page), data);
289 }
290 }
291
292 // Rest of RAM at kernel offset (direct map) with large pages
294 for (auto phys {mem::LOW_MEMORY_LIMIT}; phys < end; phys += LARGE_PAGE_SIZE) {
295 mapped = mapped && paging::mapLarge(top, phys, static_cast<mem::virt_t>(static_cast<igros_usize_t>(phys) + offset), data);
296 }
297 if (!mapped) [[unlikely]] {
298 klib::kprintf("Paging:\t\tFAILED (can't build kernel mapping)");
299 return;
300 }
301
302 // Switch to new tables: whole direct map is reachable from now on
303 ::writeCR3(static_cast<igros_usize_t>(top));
305 // Kernel can't write to read-only pages (code and read-only data)
307
308 // Check mapping
310 "Paging:\t\t%s (%zu levels, root at 0x%08llx, direct map %llu MB%s)",
311 paging::selfTest() ? "OK" : "SELF-TEST FAILED",
312 LEVELS,
313 static_cast<unsigned long long>(top),
314 static_cast<unsigned long long>(end >> 20),
315 (0_usize != noExecute) ? ", NX" : ""
316 );
317 }
318
319
320 // Check mapping on a scratch page
321 [[nodiscard]]
322 auto paging::selfTest() noexcept -> bool {
323 // Scratch page in dynamic mappings area (right above direct map)
324 auto* const page {paging::dynamicStart()};
325 const auto virt {mem::virtOf(page)};
326 const auto phys {mem::phys::alloc(mem::direct::end())};
327 if (0_u64 == phys) [[unlikely]] {
328 return false;
329 }
330 // Direct map alias of scratch page
331 auto* const alias {std::bit_cast<volatile igros_byte_t*>(paging::table(phys))};
332 if (nullptr == alias) [[unlikely]] {
333 mem::phys::free(phys);
334 return false;
335 }
336 auto passed =
337 (INVALID == paging::translate(virt)) &&
340 (phys + 0x123_u64 == paging::translate(virt + 0x123));
341 if (passed) {
342 // Write through new mapping, read through direct map alias
343 static_cast<volatile igros_byte_t*>(page)[0x123] = 0x5A_u8;
344 passed = (0x5A_u8 == alias[0x123]);
345 alias[0x456] = 0xA5_u8;
346 passed = passed && (0xA5_u8 == static_cast<volatile igros_byte_t*>(page)[0x456]);
347 }
348 paging::unmap(virt);
349 passed = passed && (INVALID == paging::translate(virt));
350 mem::phys::free(phys);
351 return passed;
352 }
353
354
355 // Page fault handler
356 [[noreturn]]
357 void paging::exHandler(const register_t* regs) noexcept {
358 // Disable interrupts
359 irq::disable();
360 // Error code bits
361 const auto error {klib::kFlags<FAULT>::from(static_cast<igros_usize_t>(regs->param))};
362 klib::kprintf(R"exception(
363EXCEPTION [#%zu]
364Name: %s
365Address: 0x%016zx
366Reason: %s page, %s access, %s mode%s%s
367)exception",
368 regs->number,
369 except::NAME[regs->number],
370 ::readCR2(),
371 error.isSet(FAULT::PROTECTED) ? "protected" : "missing",
372 error.isSet(FAULT::WRITE) ? "write" : "read",
373 error.isSet(FAULT::USER) ? "user" : "kernel",
374 error.isSet(FAULT::RESERVED) ? ", reserved bit set" : "",
375 error.isSet(FAULT::FETCH) ? ", instruction fetch" : ""
376 );
377 // Dump registers and hang CPU
378 cpu::dumpRegisters(regs);
379 cpu::halt();
380 }
381
382
383} // namespace igros::x86
384
static constexpr auto check(const T value) noexcept -> bool
Check alignment.
Definition kAlign.hpp:155
static constexpr auto up(const T value) noexcept -> T
Align up.
Definition kAlign.hpp:163
Bit flags of enum type.
Definition kFlags.hpp:29
static constexpr auto from(const U value) noexcept -> kFlags
From raw value (hardware register, table entry).
Definition kFlags.hpp:55
static auto end() noexcept -> phys_t
End of directly mapped physical memory.
Definition direct.hpp:92
static void setEnd(const phys_t end) noexcept
Set end of directly mapped physical memory (paging mapped more memory).
Definition direct.hpp:97
static auto pointer(const phys_t phys, const phys_t size=sizeof(T)) noexcept -> T *
Pointer to physical memory (nullptr if range is not directly mapped).
Definition direct.hpp:112
static void free(const phys_t page) noexcept
Free physical page.
Definition phys.cpp:185
static auto alloc(const phys_t limit=MEMORY_LIMIT) noexcept -> phys_t
Allocate physical page below limit (0 if no memory left).
Definition phys.cpp:162
static auto top() noexcept -> phys_t
End of highest memory ever marked free (top of usable RAM, clamped to MEMORY_LIMIT).
Definition phys.cpp:209
Platform desciption structure.
Definition platform.hpp:42
static auto kernelOffset() noexcept -> igros_usize_t
Get kernel virtual memory offset (kernel virtual address = physical address + offset).
Definition platform.hpp:346
static constexpr auto stackGuard() noexcept -> const igros_byte_t *
Get boot stack guard page (one page right below boot stack, never mapped).
Definition platform.hpp:339
static constexpr void install() noexcept
Install exceptions handler.
static constexpr std::array< const char *const, 32_usize > NAME
Exceptions names.
static void disable() noexcept
Disable interrupts.
Definition irq.cpp:70
static void unmap(const mem::virt_t virt) noexcept
Unmap 4 KB page.
Definition paging.cpp:187
igros_usize_t entry_t
Page table entry (register-wide).
Definition paging.hpp:50
static auto table(const mem::phys_t phys) noexcept -> entry_t *
Table at physical address (through direct map).
Definition paging.cpp:76
static auto selfTest() noexcept -> bool
Check mapping on a scratch page.
Definition paging.cpp:322
static auto root() noexcept -> mem::phys_t
Current top level table physical address.
Definition paging.cpp:223
static auto allocTable() noexcept -> mem::phys_t
Allocate empty table (reachable through direct map, 0 if no memory).
Definition paging.cpp:83
static auto directMapEnd() noexcept -> mem::phys_t
End of physical memory reachable through direct map.
Definition paging.cpp:230
static auto kernelFlags(const mem::phys_t phys) noexcept -> klib::kFlags< FLAGS >
Rights of kernel offset mapping of low memory physical page (kernel sections).
Definition paging.cpp:243
static auto entry(const mem::phys_t top, const mem::virt_t virt, const bool create, const bool user, const igros_usize_t level=0_usize) noexcept -> entry_t *
Entry of virtual address at given level (0 - page table) of given hierarchy.
Definition paging.cpp:100
@ PROTECTED
Protection violation (else page not present).
Definition paging.hpp:156
@ FETCH
Instruction fetch.
Definition paging.hpp:160
@ USER
User mode access (else kernel).
Definition paging.hpp:158
@ RESERVED
Reserved bit set in entry.
Definition paging.hpp:159
@ WRITE
Write access (else read).
Definition paging.hpp:157
static constexpr auto ENTRIES
Entries per table.
Definition paging.hpp:97
static auto map(const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags< FLAGS > flags) noexcept -> bool
Map 4 KB page (false if structures can't be allocated or address is in a large page).
Definition paging.cpp:182
static constexpr auto LEVELS
Table levels.
Definition paging.hpp:87
static void exHandler(const register_t *regs) noexcept
Page fault handler.
Definition paging.cpp:357
static constexpr auto LARGE_PAGE_SIZE
Large page size (4 MB on i386, 2 MB on x86_64).
Definition paging.hpp:99
static constexpr auto PAGE_SIZE
Page size in bytes.
Definition paging.hpp:72
static auto makeEntry(const mem::phys_t phys, const klib::kFlags< FLAGS > flags) noexcept -> entry_t
Entry value for physical address with flags (drops NX if not enabled).
Definition paging.cpp:134
static auto dynamicStart() noexcept -> igros_byte_t *
Start of kernel dynamic mappings (right above direct map).
Definition paging.cpp:236
static void init() noexcept
Build kernel page tables and switch to them:
Definition paging.cpp:260
static auto index(const mem::virt_t virt, const igros_usize_t level) noexcept -> igros_usize_t
Index of virtual address in table of given level (0 - page table).
Definition paging.cpp:93
static constexpr auto INVALID
Address isn't mapped.
Definition paging.hpp:102
static constexpr auto INDEX_BITS
Definition paging.hpp:88
static constexpr auto DIRECT_MAP_LIMIT
Physical memory mapped at kernel offset (-2 GB .. -1 GB), rest is for dynamic mappings.
Definition paging.hpp:92
static constexpr auto ADDRESS_MASK
Physical address bits of entry (52-bit physical addresses).
Definition paging.hpp:90
static constexpr auto PAGE_SHIFT
Page size bits.
Definition paging.hpp:71
FLAGS
Page table entry flags.
Definition paging.hpp:53
@ NO_EXECUTE
Instruction fetch not allowed (EFER.NXE).
Definition paging.hpp:63
@ USER
User mode access.
Definition paging.hpp:56
@ WRITABLE
Write access.
Definition paging.hpp:55
@ PRESENT
Entry is valid.
Definition paging.hpp:54
@ HUGE
Large page (directory levels).
Definition paging.hpp:61
static auto translate(const mem::virt_t virt) noexcept -> mem::phys_t
Physical address of virtual address (INVALID if not mapped).
Definition paging.cpp:198
static auto mapLarge(const mem::phys_t top, const mem::phys_t phys, const mem::virt_t virt, const klib::kFlags< FLAGS > flags) noexcept -> bool
Map large page in given hierarchy (fresh directory entry only).
Definition paging.cpp:163
CR0 - CR4 registers operations.
void pageInvalidate(const void *const page) noexcept
Invalidate TLB entry of page.
auto readCR2() noexcept -> igros::igros_usize_t
Read CR2 register.
auto readCR0() noexcept -> igros::igros_usize_t
Read CR0 register.
auto readCR3() noexcept -> igros::igros_usize_t
Read CR3 register.
void writeCR3(const igros::igros_usize_t value) noexcept
Write CR3 register.
void writeCR0(const igros::igros_usize_t value) noexcept
Write CR0 register.
Direct map of physical memory.
Exceptions low-level operations.
auto cpuEnableNoExecute() noexcept -> bool
Enable no-execute page protection (EFER.NXE) if CPU supports it.
Definition features.cpp:36
CPU feature control.
std::size_t igros_usize_t
Unsigned size.
Definition types.hpp:51
std::uint8_t igros_byte_t
8-bit unsigned integer
Definition types.hpp:41
Kernel alignment functions definitions.
Kernel text print functions.
void kprintf(const char *const format,...) noexcept
Kernel printf function (arguments are checked against format at compile time).
Definition kPrint.cpp:358
constexpr auto make_kflags(const Args... flags) noexcept -> kFlags< T >
Flags creation helper function from set flags.
Definition kFlags.hpp:94
Memory management: physical page allocator, direct map, virtual addresses.
constexpr auto LOW_MEMORY_LIMIT
Physical memory mapped by boot code at kernel virtual offset (see boot.s).
Definition phys.hpp:43
igros_quad_t phys_t
Physical address (64-bit: physical memory may lie above 4 GB).
Definition phys.hpp:36
auto virtOf(const T *const object) noexcept -> virt_t
Virtual address of object.
Definition virt.hpp:39
virt_t
Virtual address: translated by paging, never dereferenced through this type.
Definition virt.hpp:30
constexpr auto address(const virt_t virt) noexcept -> igros_usize_t
Numeric value of virtual address.
Definition virt.hpp:47
constinit paging::entry_t noExecute
NX bit value if no-execute is enabled (else 0: NX can't be used).
Definition paging.cpp:42
void invalidate(const mem::virt_t virt) noexcept
Invalidate TLB entry of virtual address (invlpg takes memory operand).
Definition paging.cpp:48
Code shared by i386 and x86_64: ports, interrupts, exceptions, paging.
static constexpr auto flag(const paging::FLAGS value) noexcept -> paging::entry_t
Entry flag value.
Definition paging.cpp:60
static constexpr auto hasFlag(const paging::entry_t entry, const paging::FLAGS value) noexcept -> bool
Is flag set in entry.
Definition paging.cpp:69
@ WP
Write protect (kernel can't write to read-only pages).
Definition cr.hpp:32
x86 family current (native) architecture selection
Paging (i386 2-level and x86_64 4-level page tables).
IgrOS platform description.
Interrupts low-level operations.