2020-01-18 08:38:21 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions are met:
|
|
|
|
*
|
|
|
|
* 1. Redistributions of source code must retain the above copyright notice, this
|
|
|
|
* list of conditions and the following disclaimer.
|
|
|
|
*
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
* this list of conditions and the following disclaimer in the documentation
|
|
|
|
* and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
|
|
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
|
|
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
|
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
|
|
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
2018-10-17 21:13:55 +00:00
|
|
|
#pragma once
|
|
|
|
|
2019-05-28 09:53:16 +00:00
|
|
|
#include <AK/HashTable.h>
|
2019-06-27 11:34:28 +00:00
|
|
|
#include <AK/NonnullRefPtrVector.h>
|
2019-11-29 15:15:30 +00:00
|
|
|
#include <AK/String.h>
|
2019-06-07 18:02:01 +00:00
|
|
|
#include <Kernel/Arch/i386/CPU.h>
|
2020-02-16 00:50:16 +00:00
|
|
|
#include <Kernel/Forward.h>
|
2020-06-27 19:42:28 +00:00
|
|
|
#include <Kernel/SpinLock.h>
|
2020-09-05 21:52:14 +00:00
|
|
|
#include <Kernel/VM/AllocationStrategy.h>
|
2019-04-03 13:13:07 +00:00
|
|
|
#include <Kernel/VM/PhysicalPage.h>
|
|
|
|
#include <Kernel/VM/Region.h>
|
|
|
|
#include <Kernel/VM/VMObject.h>
|
2018-10-18 11:05:00 +00:00
|
|
|
|
2020-02-16 00:27:42 +00:00
|
|
|
namespace Kernel {
|
|
|
|
|
2020-09-03 04:57:09 +00:00
|
|
|
#define PAGE_ROUND_UP(x) ((((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1)))
|
|
|
|
#define PAGE_ROUND_DOWN(x) (((FlatPtr)(x)) & ~(PAGE_SIZE - 1))
|
2019-01-12 23:27:25 +00:00
|
|
|
|
2020-01-17 18:59:20 +00:00
|
|
|
template<typename T>
|
|
|
|
inline T* low_physical_to_virtual(T* physical)
|
|
|
|
{
|
|
|
|
return (T*)(((u8*)physical) + 0xc0000000);
|
|
|
|
}
|
|
|
|
|
|
|
|
inline u32 low_physical_to_virtual(u32 physical)
|
|
|
|
{
|
|
|
|
return physical + 0xc0000000;
|
|
|
|
}
|
|
|
|
|
|
|
|
template<typename T>
|
|
|
|
inline T* virtual_to_low_physical(T* physical)
|
|
|
|
{
|
|
|
|
return (T*)(((u8*)physical) - 0xc0000000);
|
|
|
|
}
|
|
|
|
|
|
|
|
inline u32 virtual_to_low_physical(u32 physical)
|
|
|
|
{
|
|
|
|
return physical - 0xc0000000;
|
|
|
|
}
|
|
|
|
|
2019-08-05 09:35:49 +00:00
|
|
|
class KBuffer;
|
2019-01-18 14:01:40 +00:00
|
|
|
class SynthFSInode;
|
2018-10-18 11:05:00 +00:00
|
|
|
|
2020-02-16 00:27:42 +00:00
|
|
|
#define MM Kernel::MemoryManager::the()
|
2018-10-27 12:56:52 +00:00
|
|
|
|
2020-06-28 22:04:35 +00:00
|
|
|
struct MemoryManagerData {
|
|
|
|
SpinLock<u8> m_quickmap_in_use;
|
2020-07-03 11:19:50 +00:00
|
|
|
u32 m_quickmap_prev_flags;
|
2020-10-31 23:19:50 +00:00
|
|
|
|
|
|
|
PhysicalAddress m_last_quickmap_pd;
|
|
|
|
PhysicalAddress m_last_quickmap_pt;
|
2020-06-28 22:04:35 +00:00
|
|
|
};
|
|
|
|
|
2020-07-06 13:27:22 +00:00
|
|
|
extern RecursiveSpinLock s_mm_lock;
|
|
|
|
|
2018-10-17 21:13:55 +00:00
|
|
|
class MemoryManager {
|
2018-10-31 22:19:15 +00:00
|
|
|
AK_MAKE_ETERNAL
|
2018-12-31 13:58:03 +00:00
|
|
|
friend class PageDirectory;
|
2018-11-05 09:23:00 +00:00
|
|
|
friend class PhysicalPage;
|
2019-06-11 11:13:02 +00:00
|
|
|
friend class PhysicalRegion;
|
2020-09-05 21:52:14 +00:00
|
|
|
friend class AnonymousVMObject;
|
2018-11-08 13:35:30 +00:00
|
|
|
friend class Region;
|
2018-11-08 20:20:09 +00:00
|
|
|
friend class VMObject;
|
2020-12-18 13:10:10 +00:00
|
|
|
friend OwnPtr<KBuffer> procfs$memstat(InodeIdentifier);
|
2019-05-28 09:53:16 +00:00
|
|
|
|
2018-10-17 21:13:55 +00:00
|
|
|
public:
|
2019-07-16 11:44:41 +00:00
|
|
|
static MemoryManager& the();
|
2020-08-29 22:41:30 +00:00
|
|
|
static bool is_initialized();
|
2018-10-17 21:13:55 +00:00
|
|
|
|
2020-06-28 22:04:35 +00:00
|
|
|
static void initialize(u32 cpu);
|
2020-09-18 07:49:51 +00:00
|
|
|
|
2020-06-28 22:04:35 +00:00
|
|
|
static inline MemoryManagerData& get_data()
|
|
|
|
{
|
|
|
|
return Processor::current().get_mm_data();
|
|
|
|
}
|
2018-10-18 11:05:00 +00:00
|
|
|
|
2018-11-05 09:29:19 +00:00
|
|
|
PageFaultResponse handle_page_fault(const PageFault&);
|
2018-10-18 11:05:00 +00:00
|
|
|
|
2018-11-01 12:15:46 +00:00
|
|
|
void enter_process_paging_scope(Process&);
|
2018-11-01 08:01:51 +00:00
|
|
|
|
2019-11-17 11:11:43 +00:00
|
|
|
bool validate_user_stack(const Process&, VirtualAddress) const;
|
2020-02-21 12:05:39 +00:00
|
|
|
|
2019-06-07 15:13:23 +00:00
|
|
|
enum class ShouldZeroFill {
|
2019-05-28 09:53:16 +00:00
|
|
|
No,
|
|
|
|
Yes
|
|
|
|
};
|
2019-01-31 02:57:06 +00:00
|
|
|
|
2020-09-05 03:12:25 +00:00
|
|
|
bool commit_user_physical_pages(size_t);
|
|
|
|
void uncommit_user_physical_pages(size_t);
|
|
|
|
NonnullRefPtr<PhysicalPage> allocate_committed_user_physical_page(ShouldZeroFill = ShouldZeroFill::Yes);
|
2020-09-01 19:40:34 +00:00
|
|
|
RefPtr<PhysicalPage> allocate_user_physical_page(ShouldZeroFill = ShouldZeroFill::Yes, bool* did_purge = nullptr);
|
2019-06-21 16:37:47 +00:00
|
|
|
RefPtr<PhysicalPage> allocate_supervisor_physical_page();
|
2020-05-08 18:15:01 +00:00
|
|
|
NonnullRefPtrVector<PhysicalPage> allocate_contiguous_supervisor_physical_pages(size_t size);
|
2020-08-22 03:49:50 +00:00
|
|
|
void deallocate_user_physical_page(const PhysicalPage&);
|
|
|
|
void deallocate_supervisor_physical_page(const PhysicalPage&);
|
2018-11-05 09:23:00 +00:00
|
|
|
|
2020-03-07 17:24:41 +00:00
|
|
|
OwnPtr<Region> allocate_contiguous_kernel_region(size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
2020-09-05 21:52:14 +00:00
|
|
|
OwnPtr<Region> allocate_kernel_region(size_t, const StringView& name, u8 access, bool user_accessible = false, AllocationStrategy strategy = AllocationStrategy::Reserve, bool cacheable = true);
|
2020-03-02 12:04:17 +00:00
|
|
|
OwnPtr<Region> allocate_kernel_region(PhysicalAddress, size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
2020-06-02 04:55:09 +00:00
|
|
|
OwnPtr<Region> allocate_kernel_region_identity(PhysicalAddress, size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
2020-03-02 12:04:17 +00:00
|
|
|
OwnPtr<Region> allocate_kernel_region_with_vmobject(VMObject&, size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
|
|
|
OwnPtr<Region> allocate_kernel_region_with_vmobject(const Range&, VMObject&, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
|
|
|
OwnPtr<Region> allocate_user_accessible_kernel_region(size_t, const StringView& name, u8 access, bool cacheable = true);
|
2019-05-14 09:51:00 +00:00
|
|
|
|
2019-06-11 11:13:02 +00:00
|
|
|
unsigned user_physical_pages() const { return m_user_physical_pages; }
|
|
|
|
unsigned user_physical_pages_used() const { return m_user_physical_pages_used; }
|
2021-01-05 03:40:38 +00:00
|
|
|
unsigned user_physical_pages_committed() const { return m_user_physical_pages_committed; }
|
|
|
|
unsigned user_physical_pages_uncommitted() const { return m_user_physical_pages_uncommitted; }
|
2019-06-11 11:13:02 +00:00
|
|
|
unsigned super_physical_pages() const { return m_super_physical_pages; }
|
|
|
|
unsigned super_physical_pages_used() const { return m_super_physical_pages_used; }
|
|
|
|
|
2019-08-08 08:43:44 +00:00
|
|
|
template<typename Callback>
|
|
|
|
static void for_each_vmobject(Callback callback)
|
|
|
|
{
|
2019-08-08 11:40:58 +00:00
|
|
|
for (auto& vmobject : MM.m_vmobjects) {
|
|
|
|
if (callback(vmobject) == IterationDecision::Break)
|
2019-08-08 08:43:44 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-08 20:10:47 +00:00
|
|
|
template<typename T, typename Callback>
|
|
|
|
static void for_each_vmobject_of_type(Callback callback)
|
|
|
|
{
|
|
|
|
for (auto& vmobject : MM.m_vmobjects) {
|
|
|
|
if (!is<T>(vmobject))
|
|
|
|
continue;
|
|
|
|
if (callback(static_cast<T&>(vmobject)) == IterationDecision::Break)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-30 21:52:28 +00:00
|
|
|
static Region* find_region_from_vaddr(Process&, VirtualAddress);
|
|
|
|
static const Region* find_region_from_vaddr(const Process&, VirtualAddress);
|
2019-11-29 15:15:30 +00:00
|
|
|
|
2020-01-18 07:34:28 +00:00
|
|
|
void dump_kernel_regions();
|
|
|
|
|
2020-02-15 12:12:02 +00:00
|
|
|
PhysicalPage& shared_zero_page() { return *m_shared_zero_page; }
|
2020-09-05 03:12:25 +00:00
|
|
|
PhysicalPage& lazy_committed_page() { return *m_lazy_committed_page; }
|
2020-02-15 12:12:02 +00:00
|
|
|
|
2020-06-02 04:55:09 +00:00
|
|
|
PageDirectory& kernel_page_directory() { return *m_kernel_page_directory; }
|
|
|
|
|
2018-10-17 21:13:55 +00:00
|
|
|
private:
|
2020-08-22 15:53:34 +00:00
|
|
|
MemoryManager();
|
2018-10-17 21:13:55 +00:00
|
|
|
~MemoryManager();
|
|
|
|
|
2020-02-16 00:27:42 +00:00
|
|
|
enum class AccessSpace { Kernel,
|
|
|
|
User };
|
|
|
|
enum class AccessType { Read,
|
|
|
|
Write };
|
2020-01-02 01:09:25 +00:00
|
|
|
template<AccessSpace, AccessType>
|
|
|
|
bool validate_range(const Process&, VirtualAddress, size_t) const;
|
|
|
|
|
2019-12-19 18:13:44 +00:00
|
|
|
void register_vmobject(VMObject&);
|
|
|
|
void unregister_vmobject(VMObject&);
|
2018-11-08 21:24:02 +00:00
|
|
|
void register_region(Region&);
|
|
|
|
void unregister_region(Region&);
|
2018-11-08 20:20:09 +00:00
|
|
|
|
2020-01-01 00:56:58 +00:00
|
|
|
void detect_cpu_features();
|
2020-01-17 21:07:20 +00:00
|
|
|
void protect_kernel_image();
|
2020-01-17 20:03:52 +00:00
|
|
|
void parse_memory_map();
|
2020-07-06 13:27:22 +00:00
|
|
|
static void flush_tlb_local(VirtualAddress, size_t page_count = 1);
|
2021-01-02 19:27:38 +00:00
|
|
|
static void flush_tlb(const PageDirectory*, VirtualAddress, size_t page_count = 1);
|
2018-10-17 21:13:55 +00:00
|
|
|
|
2019-08-06 08:31:20 +00:00
|
|
|
static Region* user_region_from_vaddr(Process&, VirtualAddress);
|
|
|
|
static Region* kernel_region_from_vaddr(VirtualAddress);
|
|
|
|
|
2020-07-30 21:52:28 +00:00
|
|
|
static Region* find_region_from_vaddr(VirtualAddress);
|
2019-08-06 09:19:16 +00:00
|
|
|
|
2020-09-05 03:12:25 +00:00
|
|
|
RefPtr<PhysicalPage> find_free_user_physical_page(bool);
|
2019-07-03 19:17:35 +00:00
|
|
|
u8* quickmap_page(PhysicalPage&);
|
2018-11-05 12:48:07 +00:00
|
|
|
void unquickmap_page();
|
|
|
|
|
2020-01-17 18:59:20 +00:00
|
|
|
PageDirectoryEntry* quickmap_pd(PageDirectory&, size_t pdpt_index);
|
|
|
|
PageTableEntry* quickmap_pt(PhysicalAddress);
|
|
|
|
|
2020-10-31 23:19:18 +00:00
|
|
|
PageTableEntry* pte(PageDirectory&, VirtualAddress);
|
2020-09-01 22:10:54 +00:00
|
|
|
PageTableEntry* ensure_pte(PageDirectory&, VirtualAddress);
|
2020-08-28 03:29:17 +00:00
|
|
|
void release_pte(PageDirectory&, VirtualAddress, bool);
|
2018-10-17 21:13:55 +00:00
|
|
|
|
2019-06-21 16:37:47 +00:00
|
|
|
RefPtr<PageDirectory> m_kernel_page_directory;
|
2020-01-17 21:02:10 +00:00
|
|
|
RefPtr<PhysicalPage> m_low_page_table;
|
2018-10-18 11:05:00 +00:00
|
|
|
|
2020-02-15 12:12:02 +00:00
|
|
|
RefPtr<PhysicalPage> m_shared_zero_page;
|
2020-09-05 03:12:25 +00:00
|
|
|
RefPtr<PhysicalPage> m_lazy_committed_page;
|
2020-02-15 12:12:02 +00:00
|
|
|
|
2021-01-03 23:58:50 +00:00
|
|
|
Atomic<unsigned, AK::MemoryOrder::memory_order_relaxed> m_user_physical_pages { 0 };
|
|
|
|
Atomic<unsigned, AK::MemoryOrder::memory_order_relaxed> m_user_physical_pages_used { 0 };
|
|
|
|
Atomic<unsigned, AK::MemoryOrder::memory_order_relaxed> m_user_physical_pages_committed { 0 };
|
|
|
|
Atomic<unsigned, AK::MemoryOrder::memory_order_relaxed> m_user_physical_pages_uncommitted { 0 };
|
|
|
|
Atomic<unsigned, AK::MemoryOrder::memory_order_relaxed> m_super_physical_pages { 0 };
|
|
|
|
Atomic<unsigned, AK::MemoryOrder::memory_order_relaxed> m_super_physical_pages_used { 0 };
|
2019-06-11 11:13:02 +00:00
|
|
|
|
2019-06-27 11:34:28 +00:00
|
|
|
NonnullRefPtrVector<PhysicalRegion> m_user_physical_regions;
|
|
|
|
NonnullRefPtrVector<PhysicalRegion> m_super_physical_regions;
|
2018-11-08 20:20:09 +00:00
|
|
|
|
2019-08-08 08:53:24 +00:00
|
|
|
InlineLinkedList<Region> m_user_regions;
|
|
|
|
InlineLinkedList<Region> m_kernel_regions;
|
2019-01-27 09:17:27 +00:00
|
|
|
|
2019-08-08 08:43:44 +00:00
|
|
|
InlineLinkedList<VMObject> m_vmobjects;
|
|
|
|
|
2020-04-13 20:35:37 +00:00
|
|
|
RefPtr<PhysicalPage> m_low_pseudo_identity_mapping_pages[4];
|
2018-10-17 21:13:55 +00:00
|
|
|
};
|
2018-11-01 10:44:21 +00:00
|
|
|
|
2019-12-09 19:06:03 +00:00
|
|
|
template<typename Callback>
|
|
|
|
void VMObject::for_each_region(Callback callback)
|
|
|
|
{
|
2020-08-25 23:12:18 +00:00
|
|
|
ScopedSpinLock lock(s_mm_lock);
|
2019-12-09 19:06:03 +00:00
|
|
|
// FIXME: Figure out a better data structure so we don't have to walk every single region every time an inode changes.
|
|
|
|
// Perhaps VMObject could have a Vector<Region*> with all of his mappers?
|
|
|
|
for (auto& region : MM.m_user_regions) {
|
|
|
|
if (®ion.vmobject() == this)
|
|
|
|
callback(region);
|
|
|
|
}
|
|
|
|
for (auto& region : MM.m_kernel_regions) {
|
|
|
|
if (®ion.vmobject() == this)
|
|
|
|
callback(region);
|
|
|
|
}
|
|
|
|
}
|
2020-01-02 19:49:21 +00:00
|
|
|
|
|
|
|
inline bool is_user_address(VirtualAddress vaddr)
|
|
|
|
{
|
2020-01-19 07:58:07 +00:00
|
|
|
return vaddr.get() < 0xc0000000;
|
2020-01-02 19:49:21 +00:00
|
|
|
}
|
2020-01-19 08:14:14 +00:00
|
|
|
|
|
|
|
inline bool is_user_range(VirtualAddress vaddr, size_t size)
|
|
|
|
{
|
|
|
|
if (vaddr.offset(size) < vaddr)
|
|
|
|
return false;
|
|
|
|
return is_user_address(vaddr) && is_user_address(vaddr.offset(size));
|
|
|
|
}
|
2020-02-15 12:12:02 +00:00
|
|
|
|
|
|
|
inline bool PhysicalPage::is_shared_zero_page() const
|
|
|
|
{
|
|
|
|
return this == &MM.shared_zero_page();
|
|
|
|
}
|
2020-02-16 00:27:42 +00:00
|
|
|
|
2020-09-05 03:12:25 +00:00
|
|
|
inline bool PhysicalPage::is_lazy_committed_page() const
|
|
|
|
{
|
|
|
|
return this == &MM.lazy_committed_page();
|
|
|
|
}
|
|
|
|
|
2020-02-16 00:27:42 +00:00
|
|
|
}
|