From c2a9d60664b54344fc63d7d05b215a434a1485ad Mon Sep 17 00:00:00 2001 From: El Date: Sun, 31 May 2026 23:07:58 +0000 Subject: [PATCH] VNC-325 Add support for codec type ID in video encoding and decoding --- common/rfb/encoders/FFMPEGHWEncoder.cxx | 62 ++----------------------- common/rfb/encoders/FFMPEGHWEncoder.h | 1 + common/rfb/encoders/SoftwareEncoder.cxx | 4 +- common/rfb/encoders/SoftwareEncoder.h | 1 + kasmweb | 2 +- 5 files changed, 10 insertions(+), 60 deletions(-) diff --git a/common/rfb/encoders/FFMPEGHWEncoder.cxx b/common/rfb/encoders/FFMPEGHWEncoder.cxx index 9a24115..da8a404 100644 --- a/common/rfb/encoders/FFMPEGHWEncoder.cxx +++ b/common/rfb/encoders/FFMPEGHWEncoder.cxx @@ -37,7 +37,7 @@ namespace rfb { const char *dri_node_, VideoEncoderParams params) : VideoEncoder(layout_.id, conn), layout(layout_), ffmpeg(ffmpeg_), encoder(encoder_), current_params(params), msg_codec_id(KasmVideoEncoders::to_msg_id(encoder)), - dri_node(dri_node_) { + msg_codec_type_id(pseudoEncodingStreamingModeJpegWebp - KasmVideoEncoders::to_encoding(encoder)), dri_node(dri_node_) { AVBufferRef *hw_device_ctx{}; int err{}; @@ -356,14 +356,14 @@ namespace rfb { err = ffmpeg.avcodec_receive_packet(ctx_guard.get(), pkt); if (err == AVERROR(EAGAIN)) { - DEBUG_LOG(vlog, "Encoder buffering frame (EAGAIN) - waiting for more input"); + vlog.error("Encoder buffering frame (EAGAIN) - waiting for more input"); return false; } } if (err == AVERROR_EOF) { - DEBUG_LOG(vlog, "Encoder EOF reached"); + vlog.error("Encoder EOF reached"); return false; } @@ -374,61 +374,6 @@ namespace rfb { if (pkt->flags & AV_PKT_FLAG_KEY) { DEBUG_LOG(vlog, "Key frame %ld", frame->pts); - - // Log codec header (extradata - contains SPS/PPS for H.264) - if (ctx_guard->extradata && ctx_guard->extradata_size > 0) { - vlog.info("=== CODEC HEADER (EXTRADATA) ==="); - vlog.info("Extradata size: %d bytes", ctx_guard->extradata_size); - - // Print full extradata in hex format - char hex_str[256] = {0}; - int offset = 0; - for (int i = 0; i < ctx_guard->extradata_size && i < 64; i++) { - offset += snprintf(hex_str + offset, sizeof(hex_str) - offset, "%02x ", ctx_guard->extradata[i]); - } - vlog.info("Extradata (first 64 bytes): %s", hex_str); - - // Parse AVCC format (most common for extradata) - if (ctx_guard->extradata_size > 7) { - uint8_t* data = ctx_guard->extradata; - - // AVCC format starts with: [configurationVersion, AVCProfileIndication, profile_compatibility, AVCLevelIndication] - uint8_t config_version = data[0]; - uint8_t profile_idc = data[1]; - uint8_t profile_compat = data[2]; - uint8_t level_idc = data[3]; - uint8_t nal_length_size = (data[4] & 0x03) + 1; - uint8_t num_sps = data[5] & 0x1F; - - const char* profile_name = "Unknown"; - if (profile_idc == 66) profile_name = "Baseline"; - else if (profile_idc == 77) profile_name = "Main"; - else if (profile_idc == 88) profile_name = "Extended"; - else if (profile_idc == 100) profile_name = "High"; - else if (profile_idc == 110) profile_name = "High 10"; - else if (profile_idc == 122) profile_name = "High 4:2:2"; - else if (profile_idc == 244) profile_name = "High 4:4:4"; - - vlog.info("AVCC Header Parsed:"); - vlog.info(" Configuration Version: %d", config_version); - vlog.info(" Profile: %s (IDC: %d)", profile_name, profile_idc); - vlog.info(" Profile Compatibility: 0x%02x", profile_compat); - vlog.info(" Level: %d.%d (IDC: %d)", level_idc / 10, level_idc % 10, level_idc); - vlog.info(" NAL Length Size: %d bytes", nal_length_size); - vlog.info(" Number of SPS: %d", num_sps); - - if (num_sps > 0 && ctx_guard->extradata_size > 7) { - uint16_t sps_size = (data[6] << 8) | data[7]; - vlog.info(" SPS Size: %d bytes", sps_size); - } - } - - vlog.info("================================"); - } - - // Also log first bytes of the keyframe packet itself - DEBUG_LOG(vlog, "Keyframe packet size: %d, first bytes: %02x %02x %02x %02x %02x", - pkt->size, pkt->data[0], pkt->data[1], pkt->data[2], pkt->data[3], pkt->data[4]); } return true; @@ -440,6 +385,7 @@ namespace rfb { auto *os = conn->getOutStream(conn->cp.supportsUdp); os->writeU8(layout.id); os->writeU8(msg_codec_id); + os->writeU8(msg_codec_type_id); os->writeU8(pkt->flags & AV_PKT_FLAG_KEY); encoders::write_compact(os, pkt->size); os->writeBytes(&pkt->data[0], pkt->size); diff --git a/common/rfb/encoders/FFMPEGHWEncoder.h b/common/rfb/encoders/FFMPEGHWEncoder.h index 59f01a7..c451655 100644 --- a/common/rfb/encoders/FFMPEGHWEncoder.h +++ b/common/rfb/encoders/FFMPEGHWEncoder.h @@ -41,6 +41,7 @@ class FFMPEGHWEncoder final : public VideoEncoder { KasmVideoEncoders::Encoder encoder; VideoEncoderParams current_params{}; uint8_t msg_codec_id; + uint8_t msg_codec_type_id; int64_t pts{}; int bpp{}; diff --git a/common/rfb/encoders/SoftwareEncoder.cxx b/common/rfb/encoders/SoftwareEncoder.cxx index 36a747e..a83b1fe 100644 --- a/common/rfb/encoders/SoftwareEncoder.cxx +++ b/common/rfb/encoders/SoftwareEncoder.cxx @@ -33,7 +33,8 @@ namespace rfb { SoftwareEncoder::SoftwareEncoder(Screen layout_, const FFmpeg &ffmpeg_, SConnection *conn, KasmVideoEncoders::Encoder encoder_, VideoEncoderParams params) : VideoEncoder(layout_.id, conn), layout(layout_), - ffmpeg(ffmpeg_), encoder(encoder_), current_params(params), msg_codec_id(KasmVideoEncoders::to_msg_id(encoder)) { + ffmpeg(ffmpeg_), encoder(encoder_), current_params(params), msg_codec_id(KasmVideoEncoders::to_msg_id(encoder)), + msg_codec_type_id(pseudoEncodingStreamingModeJpegWebp - KasmVideoEncoders::to_encoding(encoder)) { const auto *enc_name = KasmVideoEncoders::to_string(encoder); codec = ffmpeg.avcodec_find_encoder_by_name(enc_name); if (!codec) @@ -147,6 +148,7 @@ namespace rfb { auto *os = conn->getOutStream(conn->cp.supportsUdp); os->writeU8(layout.id); os->writeU8(msg_codec_id); + os->writeU8(msg_codec_type_id); os->writeU8(pkt->flags & AV_PKT_FLAG_KEY); encoders::write_compact(os, pkt->size); os->writeBytes(&pkt->data[0], pkt->size); diff --git a/common/rfb/encoders/SoftwareEncoder.h b/common/rfb/encoders/SoftwareEncoder.h index aec662c..912e002 100644 --- a/common/rfb/encoders/SoftwareEncoder.h +++ b/common/rfb/encoders/SoftwareEncoder.h @@ -36,6 +36,7 @@ namespace rfb { KasmVideoEncoders::Encoder encoder; VideoEncoderParams current_params{}; uint8_t msg_codec_id; + uint8_t msg_codec_type_id; int64_t pts{}; int bpp{}; diff --git a/kasmweb b/kasmweb index 4cfb1d6..44ccfc4 160000 --- a/kasmweb +++ b/kasmweb @@ -1 +1 @@ -Subproject commit 4cfb1d6f416fe21f7b61697eb67e3aa73a518a1a +Subproject commit 44ccfc4171f2a302c6049248d84d76a2ae146be0