From 464510f91b312f5769bbfa411df64b3d0cc1e7e8 Mon Sep 17 00:00:00 2001
From: El <eldargafiyatov@gmail.com>
Date: Fri, 12 Dec 2025 08:47:33 +0000
Subject: [PATCH] VNC-151 Refactor video encoding logic in EncodeManager

Moved video encoding updates to a dedicated updateVideo method. Simplified related conditional logic and reduced code duplication by consolidating encoder management.
---
 common/rfb/EncodeManager.cxx | 83 +++++++++++++++++++++---------------
 common/rfb/EncodeManager.h   |  3 ++
 2 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index ffb435e..d1397b1 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -185,18 +185,16 @@ EncodeManager::EncodeManager(SConnection *conn_, EncCache *encCache_, const FFmp
     encoders[encoderZRLE] = new ZRLEEncoder(conn);
 
     if (ffmpeg_available) {
-        const VideoEncoderParams encoder_params = {conn_->cp.width,
-            conn_->cp.height,
-            static_cast<uint8_t>(Server::frameRate),
-            static_cast<uint8_t>(Server::groupOfPicture),
-            static_cast<uint8_t>(Server::videoQualityCRFCQP)};
-
         encoders[encoderKasmVideo] = new ScreenEncoderManager(ffmpeg,
             encoder_probe.get_best_encoder(),
             encoder_probe.get_available_encoders(),
             conn,
             encoder_probe.get_drm_device_path(),
-            encoder_params);
+            {conn_->cp.width,
+                conn_->cp.height,
+                static_cast<uint8_t>(Server::frameRate),
+                static_cast<uint8_t>(Server::groupOfPicture),
+                static_cast<uint8_t>(Server::videoQualityCRFCQP)});
     }
 
     video_mode_available = ffmpeg_available && Server::videoCodec[0];
@@ -441,35 +439,12 @@ void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
     writeCopyRects(copied, copyDelta);
     writeCopyPassRects(copypassed);
 
-    if (video_mode_available && conn->cp.encoder != KasmVideoEncoders::Encoder::unavailable) {
-        auto *screen_encoder_manager = dynamic_cast<ScreenEncoderManager<> *>(encoders[encoderKasmVideo]);
-        if (screen_encoder_manager) {
-            if (screen_encoder_manager->get_encoder() != conn->cp.encoder) {
-                delete encoders[encoderKasmVideo];
+    bool video_mode = video_mode_available && conn->cp.encoder != KasmVideoEncoders::Encoder::unavailable;
+    if (video_mode) {
+        video_mode = updateVideo(changed, layout, pb);
+    }
 
-                const VideoEncoderParams encoder_params = {0,
-                    0,
-                    static_cast<uint8_t>(Server::frameRate),
-                    static_cast<uint8_t>(Server::groupOfPicture),
-                    static_cast<uint8_t>(Server::videoQualityCRFCQP)};
-
-                encoders[encoderKasmVideo] = new ScreenEncoderManager(ffmpeg,
-                    conn->cp.encoder,
-                    encoder_probe.get_available_encoders(),
-                    conn,
-                    encoder_probe.get_drm_device_path(),
-                    encoder_params);
-            }
-            if (screen_encoder_manager->sync_layout(layout, changed)) {
-                static const Palette palette;
-                screen_encoder_manager->writeRect(pb, palette);
-            }
-        }
-
-        std::vector<Rect> rects;
-        changed.get_rects(&rects);
-        updateVideoStats(rects, pb);
-    } else {
+    if (!video_mode) {
         /*
          * We start by searching for solid rects, which are then removed
          * from the changed region.
@@ -505,6 +480,44 @@ void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
     conn->writer()->writeFramebufferUpdateEnd();
 }
 
+bool EncodeManager::updateVideo(const Region &changed, const ScreenSet &layout, const PixelBuffer *pb) {
+    auto *screen_encoder_manager = dynamic_cast<ScreenEncoderManager<> *>(encoders[encoderKasmVideo]);
+    if (!screen_encoder_manager)
+        return false;
+
+    if (screen_encoder_manager->get_encoder() != conn->cp.encoder) {
+        delete encoders[encoderKasmVideo];
+
+        screen_encoder_manager = new ScreenEncoderManager(ffmpeg,
+            conn->cp.encoder,
+            encoder_probe.get_available_encoders(),
+            conn,
+            encoder_probe.get_drm_device_path(),
+            {0,
+                0,
+                static_cast<uint8_t>(Server::frameRate),
+                static_cast<uint8_t>(Server::groupOfPicture),
+                static_cast<uint8_t>(Server::videoQualityCRFCQP)});
+
+        if (!screen_encoder_manager)
+            return false;
+
+        encoders[encoderKasmVideo] = screen_encoder_manager;
+    }
+
+    if (!screen_encoder_manager->sync_layout(layout, changed))
+        return false;
+
+    static const Palette palette;
+    screen_encoder_manager->writeRect(pb, palette);
+
+    std::vector<Rect> rects;
+    changed.get_rects(&rects);
+    updateVideoStats(rects, pb);
+
+    return true;
+}
+
 void EncodeManager::prepareEncoders(bool allowLossy)
 {
   enum EncoderClass solid, bitmap, bitmapRLE;
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index 08fd7d4..19ec5f0 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -106,6 +106,9 @@ namespace rfb {
                   const ScreenSet &layout,
                   const PixelBuffer* pb,
                   const RenderedCursor* renderedCursor);
+
+    bool updateVideo(const Region& changed, const ScreenSet &layout, const PixelBuffer* pb);
+
     void prepareEncoders(bool allowLossy);
 
     Region getLosslessRefresh(const Region& req, size_t maxUpdateSize);