diff --git a/common/rfb/SDesktop.h b/common/rfb/SDesktop.h
index c6e267c..b05f05a 100644
--- a/common/rfb/SDesktop.h
+++ b/common/rfb/SDesktop.h
@@ -78,6 +78,12 @@ namespace rfb {
     // the relevant RFB protocol messages from clients.
     // See InputHandler for method signatures.
 
+    virtual Point directMouseEventWithPosition(int dx, int dy, int buttonMask,
+                                               int scrollX, int scrollY) {
+      directMouseEvent(dx, dy, buttonMask, scrollX, scrollY);
+      return Point();
+    }
+
     // handleClipboardAnnounce() is called to indicate a change in the
     // clipboard on a client. Call VNCServer::requestClipboard() to
     // access the actual data.
diff --git a/common/rfb/SMsgReader.cxx b/common/rfb/SMsgReader.cxx
index 4d9997f..f1a8e9f 100644
--- a/common/rfb/SMsgReader.cxx
+++ b/common/rfb/SMsgReader.cxx
@@ -429,7 +429,8 @@ void SMsgReader::readVideoEncodersRequest() const {
 void SMsgReader::readDirectMouseEvent()
 {
   // Wire format (9 bytes after the type byte):
-  //   byte 0:   button mask (bit 0=left, 1=right, 2=middle, 3=side, 4=extra)
+  //   byte 0:   VNC button mask. Button N uses bit N-1, so the common buttons
+  //             are bit 0=left, bit 1=middle, bit 2=right.
   //   bytes 1-2: dx (int16 BE)
   //   bytes 3-4: dy (int16 BE)
   //   bytes 5-6: scroll dx (int16 BE)
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index cee3bc9..b14b4b0 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -815,16 +815,23 @@ void VNCSConnectionST::pointerEvent(const Point& pos, int buttonMask, const bool
 void VNCSConnectionST::directMouseEvent(int dx, int dy, int buttonMask,
                                          int scrollX, int scrollY)
 {
-  pointerEventTime = lastEventTime = time(0);
+  pointerEventTime = lastEventTime = time(nullptr);
   server->lastUserInputTime = lastEventTime;
   if (!(accessRights & AccessPtrEvents))
     return;
   if (!rfb::Server::acceptPointerEvents)
     return;
+  if (server->pointerClient && server->pointerClient != this)
+    return;
 
-  // Route through the desktop interface, which handles both uinput
-  // writes and X11 cursor updates.
-  server->desktop->directMouseEvent(dx, dy, buttonMask, scrollX, scrollY);
+  if (buttonMask)
+    server->pointerClient = this;
+  else
+    server->pointerClient = nullptr;
+
+  pointerEventPos =
+    server->desktop->directMouseEventWithPosition(dx, dy, buttonMask,
+                                                  scrollX, scrollY);
 }
 
 
diff --git a/unix/xserver/hw/vnc/Input.c b/unix/xserver/hw/vnc/Input.c
index 03f636b..dbbfede 100644
--- a/unix/xserver/hw/vnc/Input.c
+++ b/unix/xserver/hw/vnc/Input.c
@@ -28,9 +28,6 @@
 #include "vncExtInit.h"
 #include "RFBGlue.h"
 
-#ifdef __linux__
-#endif
-
 #include "inputstr.h"
 #if XORG >= 110
 #include "inpututils.h"
@@ -767,7 +764,6 @@ static void vncKeysymKeyboardEvent(KeySym keysym, int down)
 	mieqProcessInputEvents();
 }
 
-
 #if INPUTTHREAD
 /** This function is called in Xserver/os/inputthread.c when starting
     the input thread. */
diff --git a/unix/xserver/hw/vnc/XserverDesktop.cc b/unix/xserver/hw/vnc/XserverDesktop.cc
index 5ecb445..3598a74 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.cc
+++ b/unix/xserver/hw/vnc/XserverDesktop.cc
@@ -513,18 +513,49 @@ void XserverDesktop::pointerEvent(const Point& pos, int buttonMask,
 
 void XserverDesktop::directMouseEvent(int dx, int dy, int buttonMask,
                                        int scrollX, int scrollY)
+{
+  directMouseEventWithPosition(dx, dy, buttonMask, scrollX, scrollY);
+}
+
+rfb::Point XserverDesktop::directMouseEventWithPosition(int dx, int dy,
+                                       int buttonMask, int scrollX, int scrollY)
 {
   // Move the X11 cursor via XTest (relative injection).
   // buttonMask uses standard VNC convention (bit 0=left, 1=middle, 2=right)
-  // which matches X11 button numbering directly — no remapping needed.
+  // which matches X11 button numbering directly, so no remapping is needed.
+  int cursorX, cursorY;
+  vncGetPointerPos(&cursorX, &cursorY);
+
   if (scrollX == 0 && scrollY == 0) {
-    vncPointerMoveRelative(dx, dy,
-                           vncGetScreenX(screenIndex),
-                           vncGetScreenY(screenIndex));
+    const int screenX = vncGetScreenX(screenIndex);
+    const int screenY = vncGetScreenY(screenIndex);
+    const int maxX = screenX + width() - 1;
+    const int maxY = screenY + height() - 1;
+
+    cursorX += dx;
+    cursorY += dy;
+
+    if (cursorX < screenX)
+      cursorX = screenX;
+    else if (cursorX > maxX)
+      cursorX = maxX;
+
+    if (cursorY < screenY)
+      cursorY = screenY;
+    else if (cursorY > maxY)
+      cursorY = maxY;
+
+    vncPointerMoveRelative(dx, dy, cursorX, cursorY);
     vncPointerButtonAction(buttonMask, false, false);
   } else {
     vncScroll(scrollX, scrollY);
   }
+
+  rfb::Point cursorPos(cursorX - vncGetScreenX(screenIndex),
+                       cursorY - vncGetScreenY(screenIndex));
+  oldCursorPos = cursorPos;
+  server->setCursorPos(cursorPos, false);
+  return cursorPos;
 }
 
 unsigned int XserverDesktop::setScreenLayout(int fb_width, int fb_height,
diff --git a/unix/xserver/hw/vnc/XserverDesktop.h b/unix/xserver/hw/vnc/XserverDesktop.h
index beddebd..b9ceadc 100644
--- a/unix/xserver/hw/vnc/XserverDesktop.h
+++ b/unix/xserver/hw/vnc/XserverDesktop.h
@@ -98,6 +98,7 @@ public:
   virtual void pointerEvent(const rfb::Point& pos, int buttonMask,
                             const bool skipClick, const bool skipRelease, int scrollX = 0, int scrollY = 0);
   virtual void directMouseEvent(int dx, int dy, int buttonMask, int scrollX, int scrollY);
+  virtual rfb::Point directMouseEventWithPosition(int dx, int dy, int buttonMask, int scrollX, int scrollY);
   virtual void keyEvent(rdr::U32 keysym, rdr::U32 keycode, bool down);
   virtual unsigned int setScreenLayout(int fb_width, int fb_height,
                                        const rfb::ScreenSet& layout);