Kernel: Add support for vmmouse

We add this feature together with the VMWareBackdoor class.
VMWareBackdoor class is responsible for enabling the vmmouse, and then
controlling it from the PS2 mouse IRQ handler.

Kernel/Devices/PS2MouseDevice.cpp

@@ -24,8 +24,9 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "PS2MouseDevice.h"
-#include "IO.h"
+#include <Kernel/Devices/PS2MouseDevice.h>
+#include <Kernel/Devices/VMWareBackdoor.h>
+#include <Kernel/IO.h>
 
 #define IRQ_MOUSE 12
 #define I8042_BUFFER 0x60
@@ -47,6 +48,14 @@
 #define PS2MOUSE_RESEND 0xFE
 #define PS2MOUSE_RESET 0xFF
 
+#define PS2MOUSE_LEFT_CLICK 0x01
+#define PS2MOUSE_RIGHT_CLICK 0x02
+#define PS2MOUSE_MIDDLE_CLICK 0x04
+
+#define VMMOUSE_LEFT_CLICK 0x20
+#define VMMOUSE_RIGHT_CLICK 0x10
+#define VMMOUSE_MIDDLE_CLICK 0x08
+
 #define PS2MOUSE_INTELLIMOUSE_ID 0x03
 
 //#define PS2MOUSE_DEBUG
@@ -70,8 +79,68 @@ PS2MouseDevice& PS2MouseDevice::the()
     return *s_the;
 }
 
+void PS2MouseDevice::handle_vmmouse_absolute_pointer()
+{
+    VMWareCommand command;
+    command.bx = 0;
+    command.command = VMMOUSE_STATUS;
+    VMWareBackdoor::the().send(command);
+    if (command.ax == 0xFFFF0000) {
+#ifdef PS2MOUSE_DEBUG
+        dbgprintf("Reset vmmouse.\n");
+#endif
+        VMWareBackdoor::the().disable_absolute_vmmouse();
+        VMWareBackdoor::the().enable_absolute_vmmouse();
+        return;
+    }
+    int words = command.ax & 0xFFFF;
+
+    if (!words || words % 4)
+        return;
+    command.size = 4;
+    command.command = VMMOUSE_DATA;
+    VMWareBackdoor::the().send(command);
+
+    int buttons = (command.ax & 0xFFFF);
+    int x = (command.bx);
+    int y = (command.cx);
+    int z = (command.dx);
+
+#ifdef PS2MOUSE_DEBUG
+    dbgprintf("Absolute Mouse: Buttons %x\n", buttons);
+    dbgprintf("Mouse: X %d, Y %d, Z %d\n", x, y, z);
+#endif
+    MousePacket packet;
+    packet.x = x;
+    packet.y = y;
+    packet.z = z;
+    if (buttons & VMMOUSE_LEFT_CLICK) {
+        packet.buttons |= PS2MOUSE_LEFT_CLICK;
+    }
+    if (buttons & VMMOUSE_RIGHT_CLICK) {
+        packet.buttons |= PS2MOUSE_RIGHT_CLICK;
+    }
+    if (buttons & VMMOUSE_MIDDLE_CLICK) {
+        packet.buttons |= PS2MOUSE_MIDDLE_CLICK;
+    }
+    packet.is_relative = false;
+    m_queue.enqueue(packet);
+}
+
 void PS2MouseDevice::handle_irq()
 {
+
+    if (VMWareBackdoor::the().vmmouse_is_absolute()) {
+#ifdef PS2MOUSE_DEBUG
+        dbgprintf("Handling PS2 vmmouse.\n");
+#endif
+        IO::in8(I8042_BUFFER);
+        handle_vmmouse_absolute_pointer();
+        return;
+    }
+#ifdef PS2MOUSE_DEBUG
+    dbgprintf("Handling classical PS2 mouse.\n");
+#endif
     for (;;) {
         u8 status = IO::in8(I8042_STATUS);
         if (!(((status & I8042_WHICH_BUFFER) == I8042_MOUSE_BUFFER) && (status & I8042_BUFFER_FULL)))
@@ -143,7 +212,11 @@ void PS2MouseDevice::parse_data_packet()
     packet.y = y;
     packet.z = z;
     packet.buttons = m_data[0] & 0x07;
-
+    packet.is_relative = true;
+#ifdef PS2MOUSE_DEBUG
+    dbgprintf("PS2 Relative Mouse: Buttons %x\n", packet.buttons);
+    dbgprintf("Mouse: X %d, Y %d, Z %d\n", packet.x, packet.y, packet.z);
+#endif
     m_queue.enqueue(packet);
 }
 
@@ -296,6 +369,11 @@ ssize_t PS2MouseDevice::read(FileDescription&, u8* buffer, ssize_t size)
         if ((size - nread) < (ssize_t)sizeof(MousePacket))
             break;
         auto packet = m_queue.dequeue();
+#ifdef PS2MOUSE_DEBUG
+        dbgprintf("PS2 Mouse Read: Buttons %x\n", packet.buttons);
+        dbgprintf("PS2 Mouse Read: X %d, Y %d, Z %d, Relative %d\n", packet.x, packet.y, packet.z, packet.buttons);
+        dbgprintf("PS2 Mouse Read: Filter packets\n");
+#endif
         memcpy(buffer, &packet, sizeof(MousePacket));
         nread += sizeof(MousePacket);
     }

Kernel/Devices/PS2MouseDevice.h

@@ -48,6 +48,7 @@ public:
 private:
     // ^IRQHandler
     virtual void handle_irq() override;
+    void handle_vmmouse_absolute_pointer();
 
     // ^CharacterDevice
     virtual const char* class_name() const override { return "PS2MouseDevice"; }

Kernel/Devices/VMWareBackdoor.cpp

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2020, Liav A. <liavalb@hotmail.co.il>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <AK/Assertions.h>
+#include <Kernel/Arch/i386/CPU.h>
+#include <Kernel/Devices/VMWareBackdoor.h>
+#include <Kernel/IO.h>
+
+#define VMWARE_CMD_GETVERSION 0x0a
+
+#define VMMOUSE_READ_ID 0x45414552
+#define VMMOUSE_DISABLE 0x000000f5
+#define VMMOUSE_REQUEST_RELATIVE 0x4c455252
+#define VMMOUSE_REQUEST_ABSOLUTE 0x53424152
+
+static VMWareBackdoor* s_vmware_backdoor;
+
+static bool is_initialized()
+{
+    return s_vmware_backdoor != nullptr;
+}
+
+void VMWareBackdoor::initialize()
+{
+    if (!is_initialized())
+        s_vmware_backdoor = new VMWareBackdoor;
+}
+
+VMWareBackdoor& VMWareBackdoor::the()
+{
+    ASSERT(s_vmware_backdoor != nullptr);
+    return *s_vmware_backdoor;
+}
+
+VMWareBackdoor::VMWareBackdoor()
+{
+    if (!detect_presence()) {
+        kprintf("VMWare Backdoor: Not supported!\n");
+        m_supported = false;
+        return;
+    }
+    kprintf("VMWare Backdoor: Supported.\n");
+    m_supported = true;
+}
+bool VMWareBackdoor::detect_presence()
+{
+    VMWareCommand command;
+    command.bx = ~VMWARE_MAGIC;
+    command.command = VMWARE_CMD_GETVERSION;
+    IO::vmware_out(command);
+    if (command.bx != VMWARE_MAGIC || command.ax == 0xFFFFFFFF)
+        return false;
+    return true;
+}
+
+bool VMWareBackdoor::supported()
+{
+    return m_supported;
+}
+
+bool VMWareBackdoor::detect_vmmouse()
+{
+    if (!supported())
+        return false;
+    VMWareCommand command;
+    command.bx = 0;
+    command.command = VMMOUSE_GETVERSION;
+    send(command);
+    // Apparently after a VMMOUSE_GETVERSION command, we should see that EBX register = VMWARE_MAGIC
+    if (command.bx != VMWARE_MAGIC || command.ax == 0xFFFFFFFF)
+        return false;
+    return true;
+}
+bool VMWareBackdoor::vmmouse_is_absolute()
+{
+    return m_vmmouse_absolute;
+}
+
+void VMWareBackdoor::enable_absolute_vmmouse()
+{
+    InterruptDisabler disabler;
+    if (!supported() || !detect_vmmouse())
+        return;
+    dbgprintf("Enabling vmmouse, absolute mode\n");
+
+    VMWareCommand command;
+
+    // Enable vmmouse
+    command.bx = VMMOUSE_READ_ID;
+    command.command = VMMOUSE_COMMAND;
+    send(command);
+
+    // Status
+    command.bx = 0;
+    command.command = VMMOUSE_STATUS;
+
+    send(command);
+
+    // Read Data
+    command.bx = 1;
+    command.command = VMMOUSE_DATA;
+    send(command);
+
+    // Enable absolute vmmouse
+    command.bx = VMMOUSE_REQUEST_ABSOLUTE;
+    command.command = VMMOUSE_COMMAND;
+    send(command);
+    m_vmmouse_absolute = true;
+}
+void VMWareBackdoor::disable_absolute_vmmouse()
+{
+    InterruptDisabler disabler;
+    if (!supported())
+        return;
+    VMWareCommand command;
+    command.bx = VMMOUSE_REQUEST_RELATIVE;
+    command.command = VMMOUSE_COMMAND;
+    send(command);
+    m_vmmouse_absolute = false;
+}
+
+void VMWareBackdoor::send(VMWareCommand& command)
+{
+    if (supported())
+        IO::vmware_out(command);
+}

Kernel/Devices/VMWareBackdoor.h

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2020, Liav A. <liavalb@hotmail.co.il>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/Types.h>
+
+#define VMWARE_MAGIC 0x564D5868
+#define VMWARE_PORT 0x5658
+#define VMWARE_PORT_HIGHBANDWIDTH 0x5659
+
+#define VMMOUSE_GETVERSION 10
+#define VMMOUSE_DATA 39
+#define VMMOUSE_STATUS 40
+#define VMMOUSE_COMMAND 41
+
+struct VMWareCommand {
+    union {
+        u32 ax;
+        u32 magic;
+    };
+    union {
+        u32 bx;
+        u32 size;
+    };
+    union {
+        u32 cx;
+        u32 command;
+    };
+    union {
+        u32 dx;
+        u32 port;
+    };
+    u32 si;
+    u32 di;
+};
+
+class VMWareBackdoor {
+public:
+    static void initialize();
+    static VMWareBackdoor& the();
+    bool supported();
+    bool vmmouse_is_absolute();
+    void enable_absolute_vmmouse();
+    void disable_absolute_vmmouse();
+    void send(VMWareCommand& command);
+
+private:
+    VMWareBackdoor();
+    bool detect_presence();
+    bool detect_vmmouse();
+    bool m_supported;
+    bool m_vmmouse_absolute { false };
+};

Kernel/IO.h

@@ -27,6 +27,8 @@
 #pragma once
 
 #include <AK/Types.h>
+#include <Kernel/Arch/i386/CPU.h>
+#include <Kernel/Devices/VMWareBackdoor.h>
 
 namespace IO {
 
@@ -87,6 +89,16 @@ inline void repeated_out16(u16 port, const u8* data, int data_size)
                  : "d"(port));
 }
 
+inline void vmware_out(VMWareCommand& command)
+{
+    command.magic = VMWARE_MAGIC;
+    command.port = VMWARE_PORT;
+    command.si = 0;
+    command.di = 0;
+    asm volatile("in %%dx, %0"
+                 : "+a"(command.ax), "+b"(command.bx), "+c"(command.cx), "+d"(command.dx), "+S"(command.si), "+D"(command.di));
+}
+
 inline void delay()
 {
     // ~3 microsecs

Kernel/Makefile

@@ -38,6 +38,7 @@ OBJS = \
     Devices/SB16.o \
     Devices/SerialDevice.o \
     Devices/ZeroDevice.o \
+    Devices/VMWareBackdoor.o \
     DoubleBuffer.o \
     FileSystem/Custody.o \
     FileSystem/DevPtsFS.o \

Kernel/MousePacket.h

@@ -26,6 +26,8 @@
 
 #pragma once
 
+#include <AK/Types.h>
+
 struct MousePacket {
     int x { 0 };
     int y { 0 };

Kernel/init.cpp

@@ -55,6 +55,7 @@
 #include <Kernel/Devices/RandomDevice.h>
 #include <Kernel/Devices/SB16.h>
 #include <Kernel/Devices/SerialDevice.h>
+#include <Kernel/Devices/VMWareBackdoor.h>
 #include <Kernel/Devices/ZeroDevice.h>
 #include <Kernel/FileSystem/Ext2FileSystem.h>
 #include <Kernel/FileSystem/VirtualFileSystem.h>
@@ -122,6 +123,10 @@ extern "C" [[noreturn]] void init()
 
     new KeyboardDevice;
     new PS2MouseDevice;
+    VMWareBackdoor::initialize();
+    if (!KParams::the().has("no_vmmouse"))
+        VMWareBackdoor::the().enable_absolute_vmmouse();
+
     new SB16;
     new NullDevice;
     if (!get_serial_debug())