VNC-14 Update to relative mouse for better UX and app compatibility

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "kasmweb"]
 	path = kasmweb
 	url = https://github.com/kasmtech/noVNC.git
-	branch = master
+	branch = feature/VNC-14_direct_drive_mouse
 [submodule "kasmvnc-functional-tests"]
 	path = kasmvnc-functional-tests
 	url = git@gitlab.com:kasm-technologies/internal/kasmvnc-functional-tests.git

common/rfb/ConnParams.cxx

@@ -62,6 +62,7 @@ ConnParams::ConnParams()
     supportsSetDesktopSize(false), supportsFence(false),
     supportsContinuousUpdates(false), supportsExtendedClipboard(false),
     supportsDisconnectNotify(false),
+    supportsDirectMouse(false),
     supportsUdp(false),
     compressLevel(2), qualityLevel(-1), fineQualityLevel(-1),
     subsampling(subsampleUndefined), name_(0), cursorPos_(0, 0), verStrPos(0),
@@ -151,6 +152,7 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
   supportsWEBP = false;
   supportsQOI = false;
   supportsDisconnectNotify = false;
+  supportsDirectMouse = false;
   compressLevel = -1;
   qualityLevel = -1;
   fineQualityLevel = -1;
@@ -224,6 +226,10 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
       supportsDisconnectNotify = true;
       clientparlog("disconnectNotify", true);
       break;
+    case pseudoEncodingDirectMouse:
+      supportsDirectMouse = true;
+      clientparlog("directMouse", true);
+      break;
     case pseudoEncodingFence:
       supportsFence = true;
       clientparlog("fence", true);

common/rfb/ConnParams.h

@@ -119,6 +119,7 @@ namespace rfb {
     bool supportsContinuousUpdates;
     bool supportsExtendedClipboard;
     bool supportsDisconnectNotify;
+    bool supportsDirectMouse;
 
     bool supportsUdp;
 

common/rfb/InputHandler.h

@@ -44,6 +44,11 @@ namespace rfb {
                               int scrollY) { }
     virtual void clientCutText(const char* __unused_attr str,
                                int __unused_attr len) { }
+    virtual void directMouseEvent(int __unused_attr dx,
+                                  int __unused_attr dy,
+                                  int __unused_attr buttonMask,
+                                  int __unused_attr scrollX,
+                                  int __unused_attr scrollY) { }
   };
 
 }

common/rfb/SMsgHandler.cxx

@@ -68,6 +68,10 @@ void SMsgHandler::handleClipboardAnnounceBinary(const unsigned, const char mimes
 {
 }
 
+void SMsgHandler::directMouseEvent(int, int, int, int, int)
+{
+}
+
 void SMsgHandler::clearBinaryClipboard()
 {
 }

common/rfb/SMsgHandler.h

@@ -54,6 +54,9 @@ namespace rfb {
     virtual void enableContinuousUpdates(bool enable,
                                          int x, int y, int w, int h) = 0;
 
+    virtual void directMouseEvent(int dx, int dy, int buttonMask,
+                                    int scrollX, int scrollY);
+
     virtual void handleClipboardAnnounceBinary(const unsigned num, const char mimes[][32]);
     virtual void clearBinaryClipboard();
     virtual void addBinaryClipboard(const char mime[], const rdr::U8 *data,

common/rfb/SMsgReader.cxx

@@ -108,6 +108,9 @@ void SMsgReader::readMsg()
   case msgTypeKeepAlive:
     readKeepAlive();
     break;
+  case msgTypeDirectMouseEvent:
+    readDirectMouseEvent();
+    break;
   default:
     fprintf(stderr, "unknown message type %d\n", msgType);
     throw Exception("unknown message type");
@@ -422,3 +425,20 @@ void SMsgReader::readVideoEncodersRequest() const {
 
     handler->videoEncodersRequest(buf);
 }
+
+void SMsgReader::readDirectMouseEvent()
+{
+  // Wire format (9 bytes after the type byte):
+  //   byte 0:   button mask (bit 0=left, 1=right, 2=middle, 3=side, 4=extra)
+  //   bytes 1-2: dx (int16 BE)
+  //   bytes 3-4: dy (int16 BE)
+  //   bytes 5-6: scroll dx (int16 BE)
+  //   bytes 7-8: scroll dy (int16 BE)
+  int buttonMask = is->readU8();
+  int dx = is->readS16();
+  int dy = is->readS16();
+  int scrollX = is->readS16();
+  int scrollY = is->readS16();
+
+  handler->directMouseEvent(dx, dy, buttonMask, scrollX, scrollY);
+}

common/rfb/SMsgReader.h

@@ -68,7 +68,8 @@ namespace rfb {
 
     void readSubscribeUnixRelay();
     void readUnixRelay();
-      void readVideoEncodersRequest() const;
+    void readVideoEncodersRequest() const;
+    void readDirectMouseEvent();
 
     SMsgHandler* handler;
     rdr::InStream* is;

common/rfb/SMsgWriter.cxx

@@ -832,3 +832,9 @@ void SMsgWriter::writeDisconnectNotify(bool graceful, const char *reason)
     os->writeBytes(msg, len);
   endMsg();
 }
+
+void SMsgWriter::writeForceGameMode()
+{
+  startMsg(msgTypeForceGameMode);
+  endMsg();
+}

common/rfb/SMsgWriter.h

@@ -136,6 +136,7 @@ namespace rfb {
     void writeUserJoinedSession(const std::string& username);
     void writeUserLeftSession(const std::string& username);
     void writeDisconnectNotify(bool graceful, const char *reason);
+    void writeForceGameMode();
 
   protected:
     void startMsg(int type);

common/rfb/ServerCore.cxx

@@ -85,6 +85,11 @@ rfb::BoolParameter rfb::Server::acceptPointerEvents
 ("AcceptPointerEvents",
  "Accept pointer press and release events from clients.",
  true);
+rfb::BoolParameter rfb::Server::forceGameMode
+("ForceGameMode",
+ "Send a message to newly connected clients instructing them to enable game mode "
+ "(direct-drive relative mouse). Useful for game-focused workspace images.",
+ false);
 rfb::BoolParameter rfb::Server::acceptCutText
 ("AcceptCutText",
  "Accept clipboard updates from clients.",

common/rfb/ServerCore.h

@@ -84,6 +84,7 @@ namespace rfb {
         static BoolParameter disconnectClients;
         static BoolParameter acceptKeyEvents;
         static BoolParameter acceptPointerEvents;
+        static BoolParameter forceGameMode;
         static BoolParameter acceptCutText;
         static BoolParameter sendCutText;
         static BoolParameter acceptSetDesktopSize;

common/rfb/VNCSConnectionST.cxx

@@ -71,7 +71,8 @@ VNCSConnectionST::VNCSConnectionST(VNCServerST* server_, network::Socket *s, con
     needsPermCheck(false), pointerEventTime(0),
     clientHasCursor(false),
     accessRights(AccessDefault), startTime(time(nullptr)), frameTracking(false),
-    udpFramesSinceFull(0), complainedAboutNoViewRights(false), clientUsername("username_unavailable")
+    udpFramesSinceFull(0), complainedAboutNoViewRights(false), gameModeNotified(false),
+    clientUsername("username_unavailable")
 {
   setStreams(&sock->inStream(), &sock->outStream());
   peerEndpoint.buf = sock->getPeerEndpoint();
@@ -848,6 +849,21 @@ void VNCSConnectionST::pointerEvent(const Point& pos, const Point& abspos, int b
   }
 }
 
+void VNCSConnectionST::directMouseEvent(int dx, int dy, int buttonMask,
+                                         int scrollX, int scrollY)
+{
+  pointerEventTime = lastEventTime = time(0);
+  server->lastUserInputTime = lastEventTime;
+  if (!(accessRights & AccessPtrEvents))
+    return;
+  if (!rfb::Server::acceptPointerEvents)
+    return;
+
+  // Route through the desktop interface, which handles both uinput
+  // writes and X11 cursor updates.
+  server->desktop->directMouseEvent(dx, dy, buttonMask, scrollX, scrollY);
+}
+
 
 class VNCSConnectionSTShiftPresser {
 public:
@@ -1017,6 +1033,15 @@ void VNCSConnectionST::framebufferUpdateRequest(const Rect& r,bool incremental)
 
   if (!(accessRights & AccessView)) return;
 
+  // On the first full update request the client is fully connected and has
+  // sent SetEncodings. If the server is configured to force game mode and the
+  // client supports direct mouse, send the notification exactly once.
+  if (!incremental && !gameModeNotified && rfb::Server::forceGameMode &&
+      cp.supportsDirectMouse) {
+    writer()->writeForceGameMode();
+    gameModeNotified = true;
+  }
+
   SConnection::framebufferUpdateRequest(r, incremental);
 
   // Check that the client isn't sending crappy requests

common/rfb/VNCSConnectionST.h

@@ -247,6 +247,7 @@ namespace rfb {
     virtual void clientInit(bool shared);
     virtual void setPixelFormat(const PixelFormat& pf);
     virtual void pointerEvent(const Point& pos, const Point& abspos,int buttonMask, const bool skipClick, const bool skipRelease, int scrollX, int scrollY);
+    virtual void directMouseEvent(int dx, int dy, int buttonMask, int scrollX, int scrollY);
     virtual void keyEvent(rdr::U32 keysym, rdr::U32 keycode, bool down);
     virtual void framebufferUpdateRequest(const Rect& r, bool incremental);
     virtual void setDesktopSize(int fb_width, int fb_height,
@@ -365,6 +366,7 @@ namespace rfb {
 
     char unixRelaySubscriptions[MAX_UNIX_RELAYS][MAX_UNIX_RELAY_NAME_LEN] = {};
     bool complainedAboutNoViewRights;
+    bool gameModeNotified;
     std::string clientUsername;
     bool pendingClientRefresh{false};
   };

common/rfb/encodings.h

@@ -88,6 +88,7 @@ namespace rfb {
   constexpr int pseudoEncodingVideoOutTimeLevel100 = -1887;
   constexpr int pseudoEncodingQOI = -1886;
   constexpr int pseudoEncodingKasmDisconnectNotify = -1885;
+  constexpr int pseudoEncodingDirectMouse = -1884;
 
     constexpr int pseudoEncodingHardwareProfile0 = -1170;
     constexpr int pseudoEncodingHardwareProfile4 = -1166;

common/rfb/msgTypes.h

@@ -38,6 +38,7 @@ namespace rfb {
   constexpr int msgTypeVideoEncoders = 184;
   constexpr int msgTypeKeepAlive = 185;
   constexpr int msgTypeServerDisconnect = 186;
+  constexpr int msgTypeForceGameMode = 187;
 
   constexpr int msgTypeServerFence = 248;
   constexpr int msgTypeUserAddedToSession = 253;
@@ -67,6 +68,8 @@ namespace rfb {
   //constexpr int msgTypeKeepAlive = 185;
   //constexpr int msgTypeServerDisconnect = 186;
 
+  constexpr int msgTypeDirectMouseEvent = 188;
+
   constexpr int msgTypeClientFence = 248;
 
   constexpr int msgTypeSetDesktopSize = 251;

kasmweb

@@ -1 +1 @@
-Subproject commit 7fc03c8afc4100f3c01ffcd94cd7b373f48300c1
+Subproject commit cf75db39658d264380e9cf893d7627fcb70047da

unix/kasmvnc_defaults.yaml

@@ -44,6 +44,9 @@ keyboard:
 # Mouse, trackpad, etc.
 pointer:
   enabled: true
+  # Force all clients into game mode (direct-drive relative mouse) on connect.
+  # Useful for game-focused workspace images where pointer lock is always desired.
+  game_mode: false
 
 runtime_configuration:
   allow_client_to_override_kasm_server_settings: true

unix/vncserver

@@ -1563,6 +1563,15 @@ sub DefineConfigToCLIConversion {
           })
         ]
     }),
+    KasmVNC::CliOption->new({
+        name => 'ForceGameMode',
+        configKeys => [
+          KasmVNC::ConfigKey->new({
+            name => "pointer.game_mode",
+            type => KasmVNC::ConfigKey::BOOLEAN
+          })
+        ]
+    }),
     KasmVNC::CliOption->new({
         name => 'Log',
         configKeys => [

unix/xserver/hw/vnc/Input.c

@@ -28,6 +28,9 @@
 #include "vncExtInit.h"
 #include "RFBGlue.h"
 
+#ifdef __linux__
+#endif
+
 #include "inputstr.h"
 #if XORG >= 110
 #include "inpututils.h"
@@ -764,6 +767,7 @@ static void vncKeysymKeyboardEvent(KeySym keysym, int down)
 	mieqProcessInputEvents();
 }
 
+
 #if INPUTTHREAD
 /** This function is called in Xserver/os/inputthread.c when starting
     the input thread. */

unix/xserver/hw/vnc/XserverDesktop.cc

@@ -517,6 +517,22 @@ void XserverDesktop::pointerEvent(const Point& pos, const Point& abspos, int but
   }
 }
 
+void XserverDesktop::directMouseEvent(int dx, int dy, int buttonMask,
+                                       int scrollX, int scrollY)
+{
+  // Move the X11 cursor via XTest (relative injection).
+  // buttonMask uses standard VNC convention (bit 0=left, 1=middle, 2=right)
+  // which matches X11 button numbering directly — no remapping needed.
+  if (scrollX == 0 && scrollY == 0) {
+    vncPointerMoveRelative(dx, dy,
+                           0 + vncGetScreenX(screenIndex),
+                           0 + vncGetScreenY(screenIndex));
+    vncPointerButtonAction(buttonMask, false, false);
+  } else {
+    vncScroll(scrollX, scrollY);
+  }
+}
+
 unsigned int XserverDesktop::setScreenLayout(int fb_width, int fb_height,
                                              const rfb::ScreenSet& layout)
 {

unix/xserver/hw/vnc/XserverDesktop.h

@@ -97,6 +97,7 @@ public:
   // rfb::SDesktop callbacks
   virtual void pointerEvent(const rfb::Point& pos, const rfb::Point& abspos, int buttonMask,
                             const bool skipClick, const bool skipRelease, int scrollX = 0, int scrollY = 0);
+  virtual void directMouseEvent(int dx, int dy, int buttonMask, int scrollX, int scrollY);
   virtual void keyEvent(rdr::U32 keysym, rdr::U32 keycode, bool down);
   virtual unsigned int setScreenLayout(int fb_width, int fb_height,
                                        const rfb::ScreenSet& layout);