Merge branch 'feature/VNC-14_direct_drive_mouse' into 'master'

Resolve VNC-14 "Feature/ direct drive mouse"

Closes VNC-14

See merge request kasm-technologies/internal/KasmVNC!236

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "kasmweb"]
 	path = kasmweb
 	url = https://github.com/kasmtech/noVNC.git
-	branch = master
+	branch = feature/VNC-14_direct_drive_mouse
 [submodule "kasmvnc-functional-tests"]
 	path = kasmvnc-functional-tests
 	url = git@gitlab.com:kasm-technologies/internal/kasmvnc-functional-tests.git

common/rfb/ConnParams.cxx

@@ -62,6 +62,7 @@ ConnParams::ConnParams()
     supportsSetDesktopSize(false), supportsFence(false),
     supportsContinuousUpdates(false), supportsExtendedClipboard(false),
     supportsDisconnectNotify(false),
+    supportsDirectMouse(false),
     supportsUdp(false),
     compressLevel(2), qualityLevel(-1), fineQualityLevel(-1),
     subsampling(subsampleUndefined), name_(0), cursorPos_(0, 0), verStrPos(0),
@@ -151,6 +152,7 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
   supportsWEBP = false;
   supportsQOI = false;
   supportsDisconnectNotify = false;
+  supportsDirectMouse = false;
   compressLevel = -1;
   qualityLevel = -1;
   fineQualityLevel = -1;
@@ -224,6 +226,10 @@ void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
       supportsDisconnectNotify = true;
       clientparlog("disconnectNotify", true);
       break;
+    case pseudoEncodingDirectMouse:
+      supportsDirectMouse = true;
+      clientparlog("directMouse", true);
+      break;
     case pseudoEncodingFence:
       supportsFence = true;
       clientparlog("fence", true);

common/rfb/ConnParams.h

@@ -119,6 +119,7 @@ namespace rfb {
     bool supportsContinuousUpdates;
     bool supportsExtendedClipboard;
     bool supportsDisconnectNotify;
+    bool supportsDirectMouse;
 
     bool supportsUdp;
 

common/rfb/InputHandler.h

@@ -36,14 +36,18 @@ namespace rfb {
                           rdr::U32 __unused_attr keycode,
                           bool __unused_attr down) { }
     virtual void pointerEvent(const Point& __unused_attr pos,
-		              const Point& __unused_attr abspos,
+                              int __unused_attr buttonMask,
-		              int __unused_attr buttonMask,
                               const bool __unused_attr skipClick,
                               const bool __unused_attr skipRelease,
                               int scrollX,
                               int scrollY) { }
     virtual void clientCutText(const char* __unused_attr str,
                                int __unused_attr len) { }
+    virtual void directMouseEvent(int __unused_attr dx,
+                                  int __unused_attr dy,
+                                  int __unused_attr buttonMask,
+                                  int __unused_attr scrollX,
+                                  int __unused_attr scrollY) { }
   };
 
 }

common/rfb/SDesktop.h

@@ -78,6 +78,12 @@ namespace rfb {
     // the relevant RFB protocol messages from clients.
     // See InputHandler for method signatures.
 
+    virtual Point directMouseEventWithPosition(int dx, int dy, int buttonMask,
+                                               int scrollX, int scrollY) {
+      directMouseEvent(dx, dy, buttonMask, scrollX, scrollY);
+      return Point();
+    }
+
     // handleClipboardAnnounce() is called to indicate a change in the
     // clipboard on a client. Call VNCServer::requestClipboard() to
     // access the actual data.

common/rfb/SMsgHandler.cxx

@@ -43,12 +43,13 @@ void SMsgHandler::setPixelFormat(const PixelFormat& pf)
 void SMsgHandler::setEncodings(int nEncodings, const rdr::S32* encodings)
 {
   bool firstFence, firstContinuousUpdates, firstLEDState,
-       firstQEMUKeyEvent;
+       firstQEMUKeyEvent, firstDirectMouse;
 
   firstFence = !cp.supportsFence;
   firstContinuousUpdates = !cp.supportsContinuousUpdates;
   firstLEDState = !cp.supportsLEDState;
   firstQEMUKeyEvent = !cp.supportsQEMUKeyEvent;
+  firstDirectMouse = !cp.supportsDirectMouse;
 
   cp.setEncodings(nEncodings, encodings);
 
@@ -62,12 +63,22 @@ void SMsgHandler::setEncodings(int nEncodings, const rdr::S32* encodings)
     supportsLEDState();
   if (cp.supportsQEMUKeyEvent && firstQEMUKeyEvent)
     supportsQEMUKeyEvent();
+  if (cp.supportsDirectMouse && firstDirectMouse)
+    supportsDirectMouse();
+}
+
+void SMsgHandler::supportsDirectMouse()
+{
 }
 
 void SMsgHandler::handleClipboardAnnounceBinary(const unsigned, const char mimes[][32])
 {
 }
 
+void SMsgHandler::directMouseEvent(int, int, int, int, int)
+{
+}
+
 void SMsgHandler::clearBinaryClipboard()
 {
 }

common/rfb/SMsgHandler.h

@@ -54,6 +54,9 @@ namespace rfb {
     virtual void enableContinuousUpdates(bool enable,
                                          int x, int y, int w, int h) = 0;
 
+    virtual void directMouseEvent(int dx, int dy, int buttonMask,
+                                    int scrollX, int scrollY);
+
     virtual void handleClipboardAnnounceBinary(const unsigned num, const char mimes[][32]);
     virtual void clearBinaryClipboard();
     virtual void addBinaryClipboard(const char mime[], const rdr::U8 *data,
@@ -96,6 +99,10 @@ namespace rfb {
     // handler will send a pseudo-rect back, signalling server support.
     virtual void supportsQEMUKeyEvent();
 
+    // supportsDirectMouse() is called the first time we detect that the
+    // client supports the direct mouse extension
+    virtual void supportsDirectMouse();
+
     virtual void udpUpgrade(const char *resp) = 0;
     virtual void udpDowngrade(const bool) = 0;
 

common/rfb/SMsgReader.cxx

@@ -108,6 +108,9 @@ void SMsgReader::readMsg()
   case msgTypeKeepAlive:
     readKeepAlive();
     break;
+  case msgTypeDirectMouseEvent:
+    readDirectMouseEvent();
+    break;
   default:
     fprintf(stderr, "unknown message type %d\n", msgType);
     throw Exception("unknown message type");
@@ -242,7 +245,7 @@ void SMsgReader::readPointerEvent()
   int scrollX = is->readS16();
   int scrollY = is->readS16();
   
-  handler->pointerEvent(Point(x, y), Point(0, 0), mask, false, false, scrollX, scrollY);
+  handler->pointerEvent(Point(x, y), mask, false, false, scrollX, scrollY);
 }
 
 
@@ -422,3 +425,21 @@ void SMsgReader::readVideoEncodersRequest() const {
 
     handler->videoEncodersRequest(buf);
 }
+
+void SMsgReader::readDirectMouseEvent()
+{
+  // Wire format (9 bytes after the type byte):
+  //   byte 0:   VNC button mask. Button N uses bit N-1, so the common buttons
+  //             are bit 0=left, bit 1=middle, bit 2=right.
+  //   bytes 1-2: dx (int16 BE)
+  //   bytes 3-4: dy (int16 BE)
+  //   bytes 5-6: scroll dx (int16 BE)
+  //   bytes 7-8: scroll dy (int16 BE)
+  int buttonMask = is->readU8();
+  int dx = is->readS16();
+  int dy = is->readS16();
+  int scrollX = is->readS16();
+  int scrollY = is->readS16();
+
+  handler->directMouseEvent(dx, dy, buttonMask, scrollX, scrollY);
+}

common/rfb/SMsgReader.h

@@ -68,7 +68,8 @@ namespace rfb {
 
     void readSubscribeUnixRelay();
     void readUnixRelay();
-      void readVideoEncodersRequest() const;
+    void readVideoEncodersRequest() const;
+    void readDirectMouseEvent();
 
     SMsgHandler* handler;
     rdr::InStream* is;

common/rfb/SMsgWriter.cxx

@@ -832,3 +832,9 @@ void SMsgWriter::writeDisconnectNotify(bool graceful, const char *reason)
     os->writeBytes(msg, len);
   endMsg();
 }
+
+void SMsgWriter::writeForceGameMode()
+{
+  startMsg(msgTypeForceGameMode);
+  endMsg();
+}

common/rfb/SMsgWriter.h

@@ -136,6 +136,7 @@ namespace rfb {
     void writeUserJoinedSession(const std::string& username);
     void writeUserLeftSession(const std::string& username);
     void writeDisconnectNotify(bool graceful, const char *reason);
+    void writeForceGameMode();
 
   protected:
     void startMsg(int type);

common/rfb/ServerCore.cxx

@@ -85,6 +85,11 @@ rfb::BoolParameter rfb::Server::acceptPointerEvents
 ("AcceptPointerEvents",
  "Accept pointer press and release events from clients.",
  true);
+rfb::BoolParameter rfb::Server::forceGameMode
+("ForceGameMode",
+ "Send a message to newly connected clients instructing them to enable game mode "
+ "(direct-drive relative mouse). Useful for game-focused workspace images.",
+ false);
 rfb::BoolParameter rfb::Server::acceptCutText
 ("AcceptCutText",
  "Accept clipboard updates from clients.",

common/rfb/ServerCore.h

@@ -84,6 +84,7 @@ namespace rfb {
         static BoolParameter disconnectClients;
         static BoolParameter acceptKeyEvents;
         static BoolParameter acceptPointerEvents;
+        static BoolParameter forceGameMode;
         static BoolParameter acceptCutText;
         static BoolParameter sendCutText;
         static BoolParameter acceptSetDesktopSize;

common/rfb/VNCSConnectionST.cxx

@@ -71,7 +71,8 @@ VNCSConnectionST::VNCSConnectionST(VNCServerST* server_, network::Socket *s, con
     needsPermCheck(false), pointerEventTime(0),
     clientHasCursor(false),
     accessRights(AccessDefault), startTime(time(nullptr)), frameTracking(false),
-    udpFramesSinceFull(0), complainedAboutNoViewRights(false), clientUsername("username_unavailable")
+    udpFramesSinceFull(0), complainedAboutNoViewRights(false),
+    clientUsername("username_unavailable")
 {
   setStreams(&sock->inStream(), &sock->outStream());
   peerEndpoint.buf = sock->getPeerEndpoint();
@@ -772,7 +773,7 @@ void VNCSConnectionST::setPixelFormat(const PixelFormat& pf)
   setCursor();
 }
 
-void VNCSConnectionST::pointerEvent(const Point& pos, const Point& abspos, int buttonMask, const bool skipClick, const bool skipRelease, int scrollX, int scrollY)
+void VNCSConnectionST::pointerEvent(const Point& pos, int buttonMask, const bool skipClick, const bool skipRelease, int scrollX, int scrollY)
 {
   pointerEventTime = lastEventTime = time(nullptr);
   server->lastUserInputTime = lastEventTime;
@@ -785,44 +786,7 @@ void VNCSConnectionST::pointerEvent(const Point& pos, const Point& abspos, int b
   }
   if (!rfb::Server::acceptPointerEvents) return;
   if (!server->pointerClient || server->pointerClient == this) {
-    Point newpos = pos;
+    pointerEventPos = pos;
-    if (pos.x & 0x4000) {
-      newpos.x &= ~0x4000;
-      newpos.y &= ~0x4000;
-
-      if (newpos.x & 0x8000) {
-        newpos.x &= ~0x8000;
-        newpos.x = -newpos.x;
-      }
-      if (newpos.y & 0x8000) {
-        newpos.y &= ~0x8000;
-        newpos.y = -newpos.y;
-      }
-
-      if (newpos.x < 0) {
-        if (pointerEventPos.x + newpos.x >= 0)
-          pointerEventPos.x += newpos.x;
-        else
-          pointerEventPos.x = 0;
-      } else {
-        pointerEventPos.x += newpos.x;
-        if (pointerEventPos.x >= cp.width)
-          pointerEventPos.x = cp.width;
-      }
-
-      if (newpos.y < 0) {
-        if (pointerEventPos.y + newpos.y >= 0)
-          pointerEventPos.y += newpos.y;
-        else
-          pointerEventPos.y = 0;
-      } else {
-        pointerEventPos.y += newpos.y;
-        if (pointerEventPos.y >= cp.height)
-          pointerEventPos.y = cp.height;
-      }
-    } else {
-      pointerEventPos = pos;
-    }
 
     if (buttonMask)
       server->pointerClient = this;
@@ -844,10 +808,34 @@ void VNCSConnectionST::pointerEvent(const Point& pos, const Point& abspos, int b
       }
     }
 
-    server->desktop->pointerEvent(newpos, pointerEventPos, buttonMask, skipclick, skiprelease, scrollX, scrollY);
+    server->desktop->pointerEvent(pos, buttonMask, skipclick, skiprelease, scrollX, scrollY);
   }
 }
 
+void VNCSConnectionST::directMouseEvent(int dx, int dy, int buttonMask,
+                                         int scrollX, int scrollY)
+{
+  pointerEventTime = lastEventTime = time(nullptr);
+  server->lastUserInputTime = lastEventTime;
+  if (!(accessRights & AccessPtrEvents)) {
+    recheckPerms();
+    return;
+  }
+  if (!rfb::Server::acceptPointerEvents)
+    return;
+  if (server->pointerClient && server->pointerClient != this)
+    return;
+
+  if (buttonMask)
+    server->pointerClient = this;
+  else
+    server->pointerClient = nullptr;
+
+  pointerEventPos =
+    server->desktop->directMouseEventWithPosition(dx, dy, buttonMask,
+                                                  scrollX, scrollY);
+}
+
 
 class VNCSConnectionSTShiftPresser {
 public:
@@ -1212,6 +1200,12 @@ void VNCSConnectionST::supportsLEDState()
   writer()->writeLEDState();
 }
 
+void VNCSConnectionST::supportsDirectMouse()
+{
+  if (rfb::Server::forceGameMode)
+    writer()->writeForceGameMode();
+}
+
 
 bool VNCSConnectionST::handleTimeout(Timer* t)
 {

common/rfb/VNCSConnectionST.h

@@ -246,7 +246,8 @@ namespace rfb {
     virtual void queryConnection(const char* userName);
     virtual void clientInit(bool shared);
     virtual void setPixelFormat(const PixelFormat& pf);
-    virtual void pointerEvent(const Point& pos, const Point& abspos,int buttonMask, const bool skipClick, const bool skipRelease, int scrollX, int scrollY);
+    virtual void pointerEvent(const Point& pos, int buttonMask, const bool skipClick, const bool skipRelease, int scrollX, int scrollY);
+    virtual void directMouseEvent(int dx, int dy, int buttonMask, int scrollX, int scrollY);
     virtual void keyEvent(rdr::U32 keysym, rdr::U32 keycode, bool down);
     virtual void framebufferUpdateRequest(const Rect& r, bool incremental);
     virtual void setDesktopSize(int fb_width, int fb_height,
@@ -264,6 +265,7 @@ namespace rfb {
     virtual void supportsFence();
     virtual void supportsContinuousUpdates();
     virtual void supportsLEDState();
+    void supportsDirectMouse() override;
 
     bool canChangeKasmSettings() const override {
         return (accessRights & (AccessPtrEvents | AccessKeyEvents)) ==

common/rfb/VNCServerST.cxx

@@ -997,6 +997,27 @@ void VNCServerST::translateDLPRegion(rdr::U16 &x1, rdr::U16 &y1, rdr::U16 &x2, r
   //slog.info("DLP_Region vals %u,%u %u,%u", x1, y1, x2, y2);
 }
 
+void VNCServerST::getDLPRegionSkipFlags(const Point& pos,
+                                        bool& skipclick, bool& skiprelease) const
+{
+  skipclick = false;
+  skiprelease = false;
+
+  if (!DLPRegion.enabled)
+    return;
+
+  rdr::U16 x1, y1, x2, y2;
+  translateDLPRegion(x1, y1, x2, y2);
+
+  if (pos.x < x1 || pos.x >= x2 ||
+      pos.y < y1 || pos.y >= y2) {
+    if (!Server::DLP_RegionAllowClick)
+      skipclick = true;
+    if (!Server::DLP_RegionAllowRelease)
+      skiprelease = true;
+  }
+}
+
 void VNCServerST::blackOut()
 {
   // Compute the region, since the resolution may have changed

common/rfb/VNCServerST.h

@@ -208,6 +208,11 @@ namespace rfb {
     enum UserActionType {Join, Leave};
     void notifyUserAction(const VNCSConnectionST* newConnection, std::string& user_name, const UserActionType action_type);
 
+    // Compute whether a pointer click/release at the given framebuffer
+    // position should be suppressed by the DLP region policy.
+    void getDLPRegionSkipFlags(const Point& pos,
+                               bool& skipclick, bool& skiprelease) const;
+
   protected:
 
     friend class VNCSConnectionST;

common/rfb/encodings.h

@@ -88,6 +88,7 @@ namespace rfb {
   constexpr int pseudoEncodingVideoOutTimeLevel100 = -1887;
   constexpr int pseudoEncodingQOI = -1886;
   constexpr int pseudoEncodingKasmDisconnectNotify = -1885;
+  constexpr int pseudoEncodingDirectMouse = -1884;
 
     constexpr int pseudoEncodingHardwareProfile0 = -1170;
     constexpr int pseudoEncodingHardwareProfile4 = -1166;

common/rfb/msgTypes.h

@@ -38,6 +38,7 @@ namespace rfb {
   constexpr int msgTypeVideoEncoders = 184;
   constexpr int msgTypeKeepAlive = 185;
   constexpr int msgTypeServerDisconnect = 186;
+  constexpr int msgTypeForceGameMode = 187;
 
   constexpr int msgTypeServerFence = 248;
   constexpr int msgTypeUserAddedToSession = 253;
@@ -67,6 +68,8 @@ namespace rfb {
   //constexpr int msgTypeKeepAlive = 185;
   //constexpr int msgTypeServerDisconnect = 186;
 
+  constexpr int msgTypeDirectMouseEvent = 188;
+
   constexpr int msgTypeClientFence = 248;
 
   constexpr int msgTypeSetDesktopSize = 251;

kasmweb

@@ -1 +1 @@
-Subproject commit 7fc03c8afc4100f3c01ffcd94cd7b373f48300c1
+Subproject commit 7f7d974925c5f99fbc409f3b703519a29b37ba7a

unix/kasmvnc_defaults.yaml

@@ -44,6 +44,9 @@ keyboard:
 # Mouse, trackpad, etc.
 pointer:
   enabled: true
+  # Force all clients into game mode (direct-drive relative mouse) on connect.
+  # Useful for game-focused workspace images where pointer lock is always desired.
+  game_mode: false
 
 runtime_configuration:
   allow_client_to_override_kasm_server_settings: true

unix/vncserver

@@ -1563,6 +1563,15 @@ sub DefineConfigToCLIConversion {
           })
         ]
     }),
+    KasmVNC::CliOption->new({
+        name => 'ForceGameMode',
+        configKeys => [
+          KasmVNC::ConfigKey->new({
+            name => "pointer.game_mode",
+            type => KasmVNC::ConfigKey::BOOLEAN
+          })
+        ]
+    }),
     KasmVNC::CliOption->new({
         name => 'Log',
         configKeys => [

unix/xserver/hw/vnc/XserverDesktop.cc

@@ -500,23 +500,70 @@ void XserverDesktop::approveConnection(uint32_t opaqueId, bool accept,
 //
 // SDesktop callbacks
 
-void XserverDesktop::pointerEvent(const Point& pos, const Point& abspos, int buttonMask,
+void XserverDesktop::pointerEvent(const Point& pos, int buttonMask,
                                   const bool skipClick, const bool skipRelease, int scrollX, int scrollY)
 {
   if (scrollX == 0 && scrollY == 0) {
-    if (pos.equals(abspos)) {
+    vncPointerMove(pos.x + vncGetScreenX(screenIndex), pos.y + vncGetScreenY(screenIndex));
-      vncPointerMove(pos.x + vncGetScreenX(screenIndex), pos.y + vncGetScreenY(screenIndex));
-    } else {
-      vncPointerMoveRelative(pos.x, pos.y,
-                           abspos.x + vncGetScreenX(screenIndex),
-                           abspos.y + vncGetScreenY(screenIndex));
-    }
     vncPointerButtonAction(buttonMask, skipClick, skipRelease);
   } else {
     vncScroll(scrollX, scrollY);
   }
 }
 
+void XserverDesktop::directMouseEvent(int dx, int dy, int buttonMask,
+                                       int scrollX, int scrollY)
+{
+  directMouseEventWithPosition(dx, dy, buttonMask, scrollX, scrollY);
+}
+
+rfb::Point XserverDesktop::directMouseEventWithPosition(int dx, int dy,
+                                       int buttonMask, int scrollX, int scrollY)
+{
+  // Move the X11 cursor via XTest (relative injection).
+  // buttonMask uses standard VNC convention (bit 0=left, 1=middle, 2=right)
+  // which matches X11 button numbering directly, so no remapping is needed.
+  int cursorX, cursorY;
+  vncGetPointerPos(&cursorX, &cursorY);
+
+  if (scrollX == 0 && scrollY == 0) {
+    const int screenX = vncGetScreenX(screenIndex);
+    const int screenY = vncGetScreenY(screenIndex);
+    const int maxX = screenX + width() - 1;
+    const int maxY = screenY + height() - 1;
+
+    cursorX += dx;
+    cursorY += dy;
+
+    if (cursorX < screenX)
+      cursorX = screenX;
+    else if (cursorX > maxX)
+      cursorX = maxX;
+
+    if (cursorY < screenY)
+      cursorY = screenY;
+    else if (cursorY > maxY)
+      cursorY = maxY;
+
+    vncPointerMoveRelative(dx, dy, cursorX, cursorY);
+
+    bool skipclick = false, skiprelease = false;
+    server->getDLPRegionSkipFlags(rfb::Point(cursorX - screenX,
+                                             cursorY - screenY),
+                                  skipclick, skiprelease);
+
+    vncPointerButtonAction(buttonMask, skipclick, skiprelease);
+  } else {
+    vncScroll(scrollX, scrollY);
+  }
+
+  rfb::Point cursorPos(cursorX - vncGetScreenX(screenIndex),
+                       cursorY - vncGetScreenY(screenIndex));
+  oldCursorPos = cursorPos;
+  server->setCursorPos(cursorPos, false);
+  return cursorPos;
+}
+
 unsigned int XserverDesktop::setScreenLayout(int fb_width, int fb_height,
                                              const rfb::ScreenSet& layout)
 {

unix/xserver/hw/vnc/XserverDesktop.h

@@ -95,8 +95,10 @@ public:
                          const char* rejectMsg=0);
 
   // rfb::SDesktop callbacks
-  virtual void pointerEvent(const rfb::Point& pos, const rfb::Point& abspos, int buttonMask,
+  virtual void pointerEvent(const rfb::Point& pos, int buttonMask,
                             const bool skipClick, const bool skipRelease, int scrollX = 0, int scrollY = 0);
+  virtual void directMouseEvent(int dx, int dy, int buttonMask, int scrollX, int scrollY);
+  virtual rfb::Point directMouseEventWithPosition(int dx, int dy, int buttonMask, int scrollX, int scrollY);
   virtual void keyEvent(rdr::U32 keysym, rdr::U32 keycode, bool down);
   virtual unsigned int setScreenLayout(int fb_width, int fb_height,
                                        const rfb::ScreenSet& layout);