From 32840dfa17494f15f9a77ecde9e2cfa8f198dcff Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Sun, 27 Jun 2021 19:49:19 +0200 Subject: [PATCH] Kernel: Implement more x86_64 context switching functionality --- Kernel/Arch/x86/common/Processor.cpp | 71 +++++++++++++++++++++++- Kernel/Arch/x86/i386/CPU.cpp | 82 ---------------------------- Kernel/Arch/x86/i386/Processor.cpp | 3 +- Kernel/Arch/x86/x86_64/CPU.cpp | 37 ------------- Kernel/Arch/x86/x86_64/Processor.cpp | 71 ++++++++++++++++++++++-- Kernel/CMakeLists.txt | 1 - Kernel/Process.cpp | 2 +- Kernel/Scheduler.cpp | 16 +++++- 8 files changed, 151 insertions(+), 132 deletions(-) delete mode 100644 Kernel/Arch/x86/i386/CPU.cpp delete mode 100644 Kernel/Arch/x86/x86_64/CPU.cpp diff --git a/Kernel/Arch/x86/common/Processor.cpp b/Kernel/Arch/x86/common/Processor.cpp index a4d3af0cfb4..48b9bbef7bf 100644 --- a/Kernel/Arch/x86/common/Processor.cpp +++ b/Kernel/Arch/x86/common/Processor.cpp @@ -40,9 +40,11 @@ Atomic Processor::s_idle_cpu_mask { 0 }; extern "C" void thread_context_first_enter(void); extern "C" void exit_kernel_thread(void); -// The compiler can't see the calls to this function inside assembly. -// Declare it, to avoid dead code warnings. +// The compiler can't see the calls to these functions inside assembly. +// Declare them, to avoid dead code warnings. extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, TrapFrame* trap) __attribute__((used)); +extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used)); +extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used)); UNMAP_AFTER_INIT static void sse_init() { @@ -1165,4 +1167,69 @@ extern "C" void context_first_init([[maybe_unused]] Thread* from_thread, [[maybe Scheduler::leave_on_first_switch(flags & ~0x200); } +extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) +{ + VERIFY(from_thread == to_thread || from_thread->state() != Thread::Running); + VERIFY(to_thread->state() == Thread::Running); + + bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR); + Processor::set_current_thread(*to_thread); + + auto& from_regs = from_thread->regs(); + auto& to_regs = to_thread->regs(); + + if (has_fxsr) + asm volatile("fxsave %0" + : "=m"(from_thread->fpu_state())); + else + asm volatile("fnsave %0" + : "=m"(from_thread->fpu_state())); + +#if ARCH(I386) + from_regs.fs = get_fs(); + from_regs.gs = get_gs(); + set_fs(to_regs.fs); + set_gs(to_regs.gs); +#endif + + if (from_thread->process().is_traced()) + read_debug_registers_into(from_thread->debug_register_state()); + + if (to_thread->process().is_traced()) { + write_debug_registers_from(to_thread->debug_register_state()); + } else { + clear_debug_registers(); + } + + auto& processor = Processor::current(); +#if ARCH(I386) + auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS); + tls_descriptor.set_base(to_thread->thread_specific_data()); + tls_descriptor.set_limit(to_thread->thread_specific_region_size()); +#endif + + if (from_regs.cr3 != to_regs.cr3) + write_cr3(to_regs.cr3); + + to_thread->set_cpu(processor.get_id()); + processor.restore_in_critical(to_thread->saved_critical()); + + if (has_fxsr) + asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state())); + else + asm volatile("frstor %0" ::"m"(to_thread->fpu_state())); + + // TODO: ioperm? +} + +extern "C" u32 do_init_context(Thread* thread, u32 flags) +{ + VERIFY_INTERRUPTS_DISABLED(); +#if ARCH(I386) + thread->regs().eflags = flags; +#else + thread->regs().rflags = flags; +#endif + return Processor::current().init_context(*thread, true); +} } diff --git a/Kernel/Arch/x86/i386/CPU.cpp b/Kernel/Arch/x86/i386/CPU.cpp deleted file mode 100644 index 27636b9ad9e..00000000000 --- a/Kernel/Arch/x86/i386/CPU.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace Kernel { - -// The compiler can't see the calls to these functions inside assembly. -// Declare them, to avoid dead code warnings. -extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used)); -extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used)); - -extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) -{ - VERIFY(from_thread == to_thread || from_thread->state() != Thread::Running); - VERIFY(to_thread->state() == Thread::Running); - - bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR); - Processor::set_current_thread(*to_thread); - - auto& from_regs = from_thread->regs(); - auto& to_regs = to_thread->regs(); - - if (has_fxsr) - asm volatile("fxsave %0" - : "=m"(from_thread->fpu_state())); - else - asm volatile("fnsave %0" - : "=m"(from_thread->fpu_state())); - - from_regs.fs = get_fs(); - from_regs.gs = get_gs(); - set_fs(to_regs.fs); - set_gs(to_regs.gs); - - if (from_thread->process().is_traced()) - read_debug_registers_into(from_thread->debug_register_state()); - - if (to_thread->process().is_traced()) { - write_debug_registers_from(to_thread->debug_register_state()); - } else { - clear_debug_registers(); - } - - auto& processor = Processor::current(); - auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS); - tls_descriptor.set_base(to_thread->thread_specific_data()); - tls_descriptor.set_limit(to_thread->thread_specific_region_size()); - - if (from_regs.cr3 != to_regs.cr3) - write_cr3(to_regs.cr3); - - to_thread->set_cpu(processor.get_id()); - processor.restore_in_critical(to_thread->saved_critical()); - - if (has_fxsr) - asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state())); - else - asm volatile("frstor %0" ::"m"(to_thread->fpu_state())); - - // TODO: ioperm? -} - -extern "C" u32 do_init_context(Thread* thread, u32 flags) -{ - VERIFY_INTERRUPTS_DISABLED(); - thread->regs().eflags = flags; - return Processor::current().init_context(*thread, true); -} - -} diff --git a/Kernel/Arch/x86/i386/Processor.cpp b/Kernel/Arch/x86/i386/Processor.cpp index 2719c52cb3a..f2d46fa9434 100644 --- a/Kernel/Arch/x86/i386/Processor.cpp +++ b/Kernel/Arch/x86/i386/Processor.cpp @@ -193,8 +193,7 @@ void Processor::switch_context(Thread*& from_thread, Thread*& to_thread) // Switch to new thread context, passing from_thread and to_thread // through to the new context using registers edx and eax asm volatile( - // NOTE: changing how much we push to the stack affects - // SWITCH_CONTEXT_TO_STACK_SIZE and thread_context_first_enter()! + // NOTE: changing how much we push to the stack affects thread_context_first_enter()! "pushfl \n" "pushl %%ebx \n" "pushl %%esi \n" diff --git a/Kernel/Arch/x86/x86_64/CPU.cpp b/Kernel/Arch/x86/x86_64/CPU.cpp deleted file mode 100644 index 4c02848a522..00000000000 --- a/Kernel/Arch/x86/x86_64/CPU.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace Kernel { - -// The compiler can't see the calls to these functions inside assembly. -// Declare them, to avoid dead code warnings. -extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used)); -extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used)); - -extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) -{ - (void)from_thread; - (void)to_thread; - TODO(); -} - -extern "C" u32 do_init_context(Thread* thread, u32 flags) -{ - (void)thread; - (void)flags; - TODO(); -} -} diff --git a/Kernel/Arch/x86/x86_64/Processor.cpp b/Kernel/Arch/x86/x86_64/Processor.cpp index 0828506d36e..104202cb954 100644 --- a/Kernel/Arch/x86/x86_64/Processor.cpp +++ b/Kernel/Arch/x86/x86_64/Processor.cpp @@ -91,9 +91,8 @@ u32 Processor::init_context(Thread& thread, bool leave_crit) auto& regs = thread.regs(); bool return_to_user = (regs.cs & 3) != 0; - stack_top -= 2 * sizeof(u64); - *reinterpret_cast(kernel_stack_top - 2 * sizeof(u64)) = regs.rsp; - *reinterpret_cast(kernel_stack_top - 3 * sizeof(u64)) = FlatPtr(&exit_kernel_thread); + stack_top -= 1 * sizeof(u64); + *reinterpret_cast(kernel_stack_top - 2 * sizeof(u64)) = FlatPtr(&exit_kernel_thread); stack_top -= sizeof(RegisterState); @@ -167,7 +166,71 @@ void Processor::switch_context(Thread*& from_thread, Thread*& to_thread) dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context --> switching out of: {} {}", VirtualAddress(from_thread), *from_thread); from_thread->save_critical(m_in_critical); - PANIC("Context switching not implemented."); + // clang-format off + // Switch to new thread context, passing from_thread and to_thread + // through to the new context using registers rdx and rax + asm volatile( + // NOTE: changing how much we push to the stack affects thread_context_first_enter()! + "pushfq \n" + "pushq %%rbx \n" + "pushq %%rcx \n" + "pushq %%rbp \n" + "pushq %%rsi \n" + "pushq %%rdi \n" + "pushq %%r8 \n" + "pushq %%r9 \n" + "pushq %%r10 \n" + "pushq %%r11 \n" + "pushq %%r12 \n" + "pushq %%r13 \n" + "pushq %%r14 \n" + "pushq %%r15 \n" + "movq %%rsp, %[from_rsp] \n" + "movabs $1f, %%rbx \n" + "movq %%rbx, %[from_rip] \n" + "movq %[to_rsp0], %%rbx \n" + "movl %%ebx, %[tss_rsp0l] \n" + "shrq $32, %%rbx \n" + "movl %%ebx, %[tss_rsp0h] \n" + "movq %[to_rsp], %%rsp \n" + "pushq %[to_thread] \n" + "pushq %[from_thread] \n" + "pushq %[to_rip] \n" + "cld \n" + "movq 16(%%rsp), %%rsi \n" + "movq 8(%%rsp), %%rdi \n" + "jmp enter_thread_context \n" + "1: \n" + "popq %%rdx \n" + "popq %%rax \n" + "popq %%r15 \n" + "popq %%r14 \n" + "popq %%r13 \n" + "popq %%r12 \n" + "popq %%r11 \n" + "popq %%r10 \n" + "popq %%r9 \n" + "popq %%r8 \n" + "popq %%rdi \n" + "popq %%rsi \n" + "popq %%rbp \n" + "popq %%rcx \n" + "popq %%rbx \n" + "popfq \n" + : [from_rsp] "=m" (from_thread->regs().rsp), + [from_rip] "=m" (from_thread->regs().rip), + [tss_rsp0l] "=m" (m_tss.rsp0l), + [tss_rsp0h] "=m" (m_tss.rsp0h), + "=d" (from_thread), // needed so that from_thread retains the correct value + "=a" (to_thread) // needed so that to_thread retains the correct value + : [to_rsp] "g" (to_thread->regs().rsp), + [to_rsp0] "g" (to_thread->regs().rsp0), + [to_rip] "c" (to_thread->regs().rip), + [from_thread] "d" (from_thread), + [to_thread] "a" (to_thread) + : "memory", "rbx" + ); + // clang-format on dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {}", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread); diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index 0080ad7d832..f2c8ae77f71 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -277,7 +277,6 @@ set(KERNEL_SOURCES ${KERNEL_SOURCES} ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/ASM_wrapper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/Boot/boot.S - ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/CPU.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/InterruptEntry.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/Processor.cpp ) diff --git a/Kernel/Process.cpp b/Kernel/Process.cpp index 427606bf7b5..6e54f624839 100644 --- a/Kernel/Process.cpp +++ b/Kernel/Process.cpp @@ -185,7 +185,7 @@ RefPtr Process::create_kernel_process(RefPtr& first_thread, Str first_thread->regs().esp = FlatPtr(entry_data); // entry function argument is expected to be in regs.esp #else first_thread->regs().rip = (FlatPtr)entry; - first_thread->regs().rsp = FlatPtr(entry_data); // entry function argument is expected to be in regs.rsp + first_thread->regs().rdi = FlatPtr(entry_data); // entry function argument is expected to be in regs.rdi #endif if (process->pid() != 0) { diff --git a/Kernel/Scheduler.cpp b/Kernel/Scheduler.cpp index b972bd8a00b..30cb3de59fb 100644 --- a/Kernel/Scheduler.cpp +++ b/Kernel/Scheduler.cpp @@ -354,8 +354,13 @@ bool Scheduler::context_switch(Thread* thread) from_thread->set_state(Thread::Runnable); #ifdef LOG_EVERY_CONTEXT_SWITCH +# if ARCH(I386) dbgln("Scheduler[{}]: {} -> {} [prio={}] {:04x}:{:08x}", Processor::id(), from_thread->tid().value(), thread->tid().value(), thread->priority(), thread->regs().cs, thread->regs().eip); +# else + dbgln("Scheduler[{}]: {} -> {} [prio={}] {:04x}:{:16x}", Processor::id(), from_thread->tid().value(), + thread->tid().value(), thread->priority(), thread->regs().cs, thread->regs().rip); +# endif #endif } @@ -375,14 +380,19 @@ bool Scheduler::context_switch(Thread* thread) enter_current(*from_thread, false); VERIFY(thread == Thread::current()); -#if ARCH(I386) if (thread->process().is_user_process()) { - auto iopl = get_iopl_from_eflags(Thread::current()->get_register_dump_from_stack().eflags); + FlatPtr flags; + auto& regs = Thread::current()->get_register_dump_from_stack(); +#if ARCH(I386) + flags = regs.eflags; +#else + flags = regs.rflags; +#endif + auto iopl = get_iopl_from_eflags(flags); if (iopl != 0) { PANIC("Switched to thread {} with non-zero IOPL={}", Thread::current()->tid().value(), iopl); } } -#endif return true; }