From 084e40a1020d2cad170daca3f40d4b1090b9cc85 Mon Sep 17 00:00:00 2001 From: Lauri Kasanen Date: Mon, 28 Oct 2024 18:31:52 +0500 Subject: [PATCH] Start on new video impl --- common/rfb/CMakeLists.txt | 1 + common/rfb/EncodeManager.cxx | 41 ++++-- common/rfb/EncodeManager.h | 8 +- common/rfb/KasmVideoConstants.h | 25 ++++ common/rfb/KasmVideoEncoder.cxx | 219 ++++++++++++++++++++++++++++++++ common/rfb/KasmVideoEncoder.h | 52 ++++++++ common/rfb/ServerCore.cxx | 9 ++ common/rfb/ServerCore.h | 2 + common/rfb/encodings.cxx | 2 + common/rfb/encodings.h | 1 + unix/xserver/hw/vnc/Makefile.am | 2 +- 11 files changed, 351 insertions(+), 11 deletions(-) create mode 100644 common/rfb/KasmVideoConstants.h create mode 100644 common/rfb/KasmVideoEncoder.cxx create mode 100644 common/rfb/KasmVideoEncoder.h diff --git a/common/rfb/CMakeLists.txt b/common/rfb/CMakeLists.txt index 27f123f..6dca014 100644 --- a/common/rfb/CMakeLists.txt +++ b/common/rfb/CMakeLists.txt @@ -25,6 +25,7 @@ set(RFB_SOURCES HextileEncoder.cxx JpegCompressor.cxx JpegDecompressor.cxx + KasmVideoEncoder.cxx KeyRemapper.cxx LogWriter.cxx Logger.cxx diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx index 2bb21f6..aa54c64 100644 --- a/common/rfb/EncodeManager.cxx +++ b/common/rfb/EncodeManager.cxx @@ -44,6 +44,7 @@ #include #include #include +#include using namespace rfb; @@ -74,6 +75,7 @@ enum EncoderClass { encoderTightWEBP, encoderTightQOI, encoderZRLE, + encoderKasmVideo, encoderClassMax, }; @@ -119,6 +121,8 @@ static const char *encoderClassName(EncoderClass klass) return "Tight (QOI)"; case encoderZRLE: return "ZRLE"; + case encoderKasmVideo: + return "KasmVideo"; case encoderClassMax: break; } @@ -180,6 +184,7 @@ EncodeManager::EncodeManager(SConnection* conn_, EncCache *encCache_) : conn(con encoders[encoderTightWEBP] = new TightWEBPEncoder(conn); encoders[encoderTightQOI] = new TightQOIEncoder(conn); encoders[encoderZRLE] = new ZRLEEncoder(conn); + encoders[encoderKasmVideo] = new KasmVideoEncoder(conn); webpBenchResult = ((TightWEBPEncoder *) encoders[encoderTightWEBP])->benchmark(); vlog.info("WEBP benchmark result: %u ms", webpBenchResult); @@ -421,6 +426,15 @@ void EncodeManager::doUpdate(bool allowLossy, const Region& changed_, writeCopyRects(copied, copyDelta); writeCopyPassRects(copypassed); + // If a video codec is enabled, send that stream constantly + if (Server::videoCodec[0]) { + startRect(pb->getRect(), encoderFullColour, true, STARTRECT_OVERRIDE_KASMVIDEO); + + encoders[encoderKasmVideo]->writeRect(pb, Palette()); + + endRect(false); + } + /* * We start by searching for solid rects, which are then removed * from the changed region. @@ -636,7 +650,7 @@ int EncodeManager::computeNumRects(const Region& changed) // No split necessary? if ((((w*h) < SubRectMaxArea) && (w < SubRectMaxWidth)) || - (videoDetected && !encoders[encoderTightWEBP]->isSupported())) { + (videoDetected && !Server::videoCodec[0] && !encoders[encoderTightWEBP]->isSupported())) { numRects += 1; continue; } @@ -656,15 +670,17 @@ int EncodeManager::computeNumRects(const Region& changed) } Encoder *EncodeManager::startRect(const Rect& rect, int type, const bool trackQuality, - const uint8_t isWebp) + const startRectOverride overrider) { Encoder *encoder; int klass, equiv; activeType = type; klass = activeEncoders[activeType]; - if (isWebp) + if (overrider == STARTRECT_OVERRIDE_WEBP) klass = encoderTightWEBP; + else if (overrider == STARTRECT_OVERRIDE_KASMVIDEO) + klass = encoderKasmVideo; beforeLength = conn->getOutStream(conn->cp.supportsUdp)->length(); @@ -1127,7 +1143,7 @@ void EncodeManager::writeRects(const Region& changed, const PixelBuffer* pb, updateVideoStats(rects, pb); } - if (videoDetected) { + if (videoDetected && !Server::videoCodec[0]) { rects.clear(); rects.push_back(pb->getRect()); } @@ -1143,7 +1159,7 @@ void EncodeManager::writeRects(const Region& changed, const PixelBuffer* pb, // No split necessary? if ((((w*h) < SubRectMaxArea) && (w < SubRectMaxWidth)) || - (videoDetected && !encoders[encoderTightWEBP]->isSupported())) { + (videoDetected && !Server::videoCodec[0] && !encoders[encoderTightWEBP]->isSupported())) { subrects.push_back(rect); trackRectQuality(rect); continue; @@ -1188,7 +1204,7 @@ void EncodeManager::writeRects(const Region& changed, const PixelBuffer* pb, gettimeofday(&scalestart, NULL); const PixelBuffer *scaledpb = NULL; - if (videoDetected && + if (videoDetected && !Server::videoCodec[0] && (maxVideoX < pb->getRect().width() || maxVideoY < pb->getRect().height())) { const float xdiff = maxVideoX / (float) pb->getRect().width(); const float ydiff = maxVideoY / (float) pb->getRect().height(); @@ -1275,10 +1291,13 @@ void EncodeManager::writeRects(const Region& changed, const PixelBuffer* pb, if (webpTookTooLong.load(std::memory_order_relaxed)) activeEncoders[encoderFullColour] = encoderTightJPEG; + if (videoDetected && Server::videoCodec[0]) + activeEncoders[encoderFullColour] = encoderKasmVideo; for (uint32_t i = 0; i < subrects_size; ++i) { if (encCache->enabled && !compresseds[i].empty() && !fromCache[i] && - !encoders[encoderTightQOI]->isSupported()) { + !encoders[encoderTightQOI]->isSupported() && + activeEncoders[encoderFullColour] != encoderKasmVideo) { void *tmp = malloc(compresseds[i].size()); memcpy(tmp, &compresseds[i][0], compresseds[i].size()); encCache->add(isWebp[i] ? encoderTightWEBP : encoderTightJPEG, @@ -1357,7 +1376,9 @@ uint8_t EncodeManager::getEncoderType(const Rect& rect, const PixelBuffer *pb, struct timeval start; gettimeofday(&start, NULL); - if (encCache->enabled && + if (videoDetected && Server::videoCodec[0]) { + // nop, send this as a skip rect + } else if (encCache->enabled && (data = encCache->get(activeEncoders[encoderFullColour], rect.tl.x, rect.tl.y, rect.width(), rect.height(), len))) { @@ -1428,7 +1449,7 @@ void EncodeManager::writeSubRect(const Rect& rect, const PixelBuffer *pb, PixelBuffer *ppb; Encoder *encoder; - encoder = startRect(rect, type, compressed.size() == 0, isWebp); + encoder = startRect(rect, type, compressed.size() == 0, isWebp ? STARTRECT_OVERRIDE_WEBP : STARTRECT_NO_OVERRIDE); if (compressed.size()) { if (isWebp) { @@ -1444,6 +1465,8 @@ void EncodeManager::writeSubRect(const Rect& rect, const PixelBuffer *pb, jpegstats.area += rect.area(); jpegstats.rects++; } + } else if (type == encoderFullColour && activeEncoders[encoderFullColour] == encoderKasmVideo) { + ((KasmVideoEncoder *) encoders[encoderKasmVideo])->writeSkipRect(); } else { if (encoder->flags & EncoderUseNativePF) { ppb = preparePixelBuffer(rect, pb, false); diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h index 7d6774f..03e298d 100644 --- a/common/rfb/EncodeManager.h +++ b/common/rfb/EncodeManager.h @@ -35,6 +35,12 @@ #include #include +enum startRectOverride { + STARTRECT_NO_OVERRIDE, + STARTRECT_OVERRIDE_WEBP, + STARTRECT_OVERRIDE_KASMVIDEO, +}; + namespace rfb { class SConnection; class Encoder; @@ -103,7 +109,7 @@ namespace rfb { int computeNumRects(const Region& changed); Encoder *startRect(const Rect& rect, int type, const bool trackQuality = true, - const uint8_t isWebp = 0); + const enum startRectOverride overrider = STARTRECT_NO_OVERRIDE); void endRect(const uint8_t isWebp = 0); void writeCopyRects(const Region& copied, const Point& delta); diff --git a/common/rfb/KasmVideoConstants.h b/common/rfb/KasmVideoConstants.h new file mode 100644 index 0000000..704e8d4 --- /dev/null +++ b/common/rfb/KasmVideoConstants.h @@ -0,0 +1,25 @@ +/* Copyright (C) 2024 Kasm. All Rights Reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ +#ifndef __RFB_KASMVIDEOCONSTANTS_H__ +#define __RFB_KASMVIDEOCONSTANTS_H__ +namespace rfb { + // Compression control + const unsigned int kasmVideoSkip = 0x00; + const unsigned int kasmVideoVP8 = 0x01; +} +#endif diff --git a/common/rfb/KasmVideoEncoder.cxx b/common/rfb/KasmVideoEncoder.cxx new file mode 100644 index 0000000..0e0bc65 --- /dev/null +++ b/common/rfb/KasmVideoEncoder.cxx @@ -0,0 +1,219 @@ +/* Copyright (C) 2024 Kasm. All Rights Reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +using namespace rfb; + +static LogWriter vlog("KasmVideoEncoder"); +static const PixelFormat pfRGBX(32, 24, false, true, 255, 255, 255, 0, 8, 16); +static const PixelFormat pfBGRX(32, 24, false, true, 255, 255, 255, 16, 8, 0); + +struct vp8_t { + vpx_codec_iface_t *cx; + vpx_image_t raw; + vpx_codec_enc_cfg_t cfg; + vpx_codec_ctx_t codec; + uint32_t frame; + uint8_t header[12]; +}; + +KasmVideoEncoder::KasmVideoEncoder(SConnection* conn) : + Encoder(conn, encodingKasmVideo, (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1), + init(false), sw(0), sh(0), vp8(NULL) +{ + vp8 = new vp8_t; +} + +KasmVideoEncoder::~KasmVideoEncoder() +{ + delete vp8; +} + +bool KasmVideoEncoder::isSupported() +{ + return conn->cp.supportsEncoding(encodingKasmVideo); +} + +static void init_vp8(vp8_t *vp8, + const uint32_t w, const uint32_t h, const uint32_t fps, + const uint32_t bitrate) { + vp8->cx = vpx_codec_vp8_cx(); + if (!vpx_img_alloc(&vp8->raw, VPX_IMG_FMT_I420, w, h, 1)) + vlog.error("Can't allocate vp8 img"); + + if (vpx_codec_enc_config_default(vp8->cx, &vp8->cfg, 0)) + vlog.error("VP8 config"); + + vp8->cfg.g_w = w; + vp8->cfg.g_h = h; + vp8->cfg.g_timebase.num = 1; + vp8->cfg.g_timebase.den = fps; + vp8->cfg.rc_target_bitrate = bitrate; + vp8->cfg.g_error_resilient = 0; + + vp8->cfg.g_lag_in_frames = 0; // realtime + + if (vpx_codec_enc_init(&vp8->codec, vp8->cx, &vp8->cfg, 0)) + vlog.error("VP8 init"); + + vp8->frame = 0; +} + +static void deinit_vp8(vp8_t *vp8) { + //vpx_img_free(&vp8->raw); + vpx_codec_destroy(&vp8->codec); +} + +static void ivf_write_frame_header(uint8_t header[12], int64_t pts, uint32_t frame_size) { + // LE + uint32_t tmp; + memcpy(header, &frame_size, 4); + tmp = (int)(pts & 0xFFFFFFFF); + memcpy(header + 4, &tmp, 4); + tmp = (int)(pts >> 32); + memcpy(header + 8, &tmp, 4); +} + +void KasmVideoEncoder::writeRect(const PixelBuffer* pb, const Palette& palette) +{ + const rdr::U8* buffer; + int stride; + + rdr::OutStream* os; + + buffer = pb->getBuffer(pb->getRect(), &stride); + + // compress + WebPPicture pic; + + WebPPictureInit(&pic); + pic.width = pb->getRect().width(); + pic.height = pb->getRect().height(); + + if (pfRGBX.equal(pb->getPF())) { + WebPPictureImportRGBX(&pic, buffer, stride * 4); + } else if (pfBGRX.equal(pb->getPF())) { + WebPPictureImportBGRX(&pic, buffer, stride * 4); + } else { + rdr::U8* tmpbuf = new rdr::U8[pic.width * pic.height * 3]; + pb->getPF().rgbFromBuffer(tmpbuf, (const rdr::U8 *) buffer, pic.width, stride, pic.height); + stride = pic.width * 3; + + WebPPictureImportRGB(&pic, tmpbuf, stride); + delete [] tmpbuf; + } + + os = conn->getOutStream(conn->cp.supportsUdp); + + if (!strcmp(Server::videoCodec, "vp8")) { + os->writeU8(kasmVideoVP8 << 4); + + if (!init) { + init_vp8(vp8, pb->getRect().width(), pb->getRect().height(), Server::frameRate, + Server::videoBitrate); + init = true; + sw = pb->getRect().width(); + sh = pb->getRect().height(); + } else if (sw != (uint32_t) pb->getRect().width() || sh != (uint32_t) pb->getRect().height()) { + deinit_vp8(vp8); + init_vp8(vp8, pb->getRect().width(), pb->getRect().height(), Server::frameRate, + Server::videoBitrate); + sw = pb->getRect().width(); + sh = pb->getRect().height(); + } + + vp8->raw.planes[0] = pic.y; + vp8->raw.planes[1] = pic.u; + vp8->raw.planes[2] = pic.v; + + vp8->raw.stride[0] = pic.y_stride; + vp8->raw.stride[1] = vp8->raw.stride[2] = pic.uv_stride; + + int flags = 0; + // VPX_EFLAG_FORCE_KF ? + // flush? raw NULL, frame -1 + + vpx_codec_iter_t iter = NULL; + const vpx_codec_cx_pkt_t *pkt = NULL; + const vpx_codec_err_t res = + vpx_codec_encode(&vp8->codec, &vp8->raw, vp8->frame++, 1, flags, VPX_DL_REALTIME); + if (res != VPX_CODEC_OK) + vlog.error("VP8 encoding error %u", res); + + while ((pkt = vpx_codec_get_cx_data(&vp8->codec, &iter)) != NULL) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + const uint8_t keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0; + ivf_write_frame_header(vp8->header, pkt->data.frame.pts, pkt->data.frame.sz); + writeCompact(pkt->data.frame.sz + 12 + 1, os); + os->writeBytes(&keyframe, 1); + os->writeBytes(vp8->header, 12); + os->writeBytes(pkt->data.frame.buf, pkt->data.frame.sz); + } + } + } else { + vlog.error("Unknown videoCodec %s", (const char *) Server::videoCodec); + } + + WebPPictureFree(&pic); +} + +void KasmVideoEncoder::writeSkipRect() +{ + rdr::OutStream* os; + os = conn->getOutStream(conn->cp.supportsUdp); + os->writeU8(kasmVideoSkip << 4); +} + +void KasmVideoEncoder::writeCompact(rdr::U32 value, rdr::OutStream* os) const +{ + // Copied from TightEncoder as it's overkill to inherit just for this + rdr::U8 b; + + b = value & 0x7F; + if (value <= 0x7F) { + os->writeU8(b); + } else { + os->writeU8(b | 0x80); + b = value >> 7 & 0x7F; + if (value <= 0x3FFF) { + os->writeU8(b); + } else { + os->writeU8(b | 0x80); + os->writeU8(value >> 14 & 0xFF); + } + } +} + +void KasmVideoEncoder::writeSolidRect(int width, int height, + const PixelFormat& pf, + const rdr::U8* colour) +{ +// nop +} diff --git a/common/rfb/KasmVideoEncoder.h b/common/rfb/KasmVideoEncoder.h new file mode 100644 index 0000000..62f73f8 --- /dev/null +++ b/common/rfb/KasmVideoEncoder.h @@ -0,0 +1,52 @@ +/* Copyright (C) 2024 Kasm. All Rights Reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this software; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, + * USA. + */ +#ifndef __RFB_KASMVIDEOENCODER_H__ +#define __RFB_KASMVIDEOENCODER_H__ + +#include +#include +#include + +struct vp8_t; + +namespace rfb { + + class KasmVideoEncoder : public Encoder { + public: + KasmVideoEncoder(SConnection* conn); + virtual ~KasmVideoEncoder(); + + virtual bool isSupported(); + + virtual void writeRect(const PixelBuffer* pb, const Palette& palette); + virtual void writeSkipRect(); + virtual void writeSolidRect(int width, int height, + const PixelFormat& pf, + const rdr::U8* colour); + + protected: + void writeCompact(rdr::U32 value, rdr::OutStream* os) const; + + private: + bool init; + uint32_t sw, sh; + + vp8_t *vp8; + }; +} +#endif diff --git a/common/rfb/ServerCore.cxx b/common/rfb/ServerCore.cxx index 22f872c..b1da131 100644 --- a/common/rfb/ServerCore.cxx +++ b/common/rfb/ServerCore.cxx @@ -287,6 +287,15 @@ rfb::IntParameter rfb::Server::udpPort "Which port to use for UDP. Default same as websocket", 0, 0, 65535); +rfb::IntParameter rfb::Server::videoBitrate +("videoBitrate", + "Bitrate in kbps to use when encoding with a -videoCodec, default 300", + 300, 0, 10000); +rfb::StringParameter rfb::Server::videoCodec +("videoCodec", + "If set, use this codec to send a video stream for WebCodecs. Supported options: vp8", + ""); + static void bandwidthPreset() { rfb::Server::dynamicQualityMin.setParam(2); rfb::Server::dynamicQualityMax.setParam(9); diff --git a/common/rfb/ServerCore.h b/common/rfb/ServerCore.h index 82a06b5..0fc4984 100644 --- a/common/rfb/ServerCore.h +++ b/common/rfb/ServerCore.h @@ -70,9 +70,11 @@ namespace rfb { static IntParameter videoScaling; static IntParameter udpFullFrameFrequency; static IntParameter udpPort; + static IntParameter videoBitrate; static StringParameter kasmPasswordFile; static StringParameter publicIP; static StringParameter stunServer; + static StringParameter videoCodec; static BoolParameter printVideoArea; static BoolParameter protocol3_3; static BoolParameter alwaysShared; diff --git a/common/rfb/encodings.cxx b/common/rfb/encodings.cxx index 190e032..cc6d94c 100644 --- a/common/rfb/encodings.cxx +++ b/common/rfb/encodings.cxx @@ -28,6 +28,7 @@ int rfb::encodingNum(const char* name) if (strcasecmp(name, "hextile") == 0) return encodingHextile; if (strcasecmp(name, "ZRLE") == 0) return encodingZRLE; if (strcasecmp(name, "Tight") == 0) return encodingTight; + if (strcasecmp(name, "KasmVideo") == 0) return encodingKasmVideo; return -1; } @@ -41,6 +42,7 @@ const char* rfb::encodingName(int num) case encodingHextile: return "hextile"; case encodingZRLE: return "ZRLE"; case encodingTight: return "Tight"; + case encodingKasmVideo: return "KasmVideo"; default: return "[unknown encoding]"; } } diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h index 7fa6fa3..afb8208 100644 --- a/common/rfb/encodings.h +++ b/common/rfb/encodings.h @@ -29,6 +29,7 @@ namespace rfb { const int encodingTight = 7; const int encodingUdp = 8; const int encodingZRLE = 16; + const int encodingKasmVideo = 17; const int encodingMax = 255; diff --git a/unix/xserver/hw/vnc/Makefile.am b/unix/xserver/hw/vnc/Makefile.am index c3226eb..adf84e6 100644 --- a/unix/xserver/hw/vnc/Makefile.am +++ b/unix/xserver/hw/vnc/Makefile.am @@ -53,7 +53,7 @@ Xvnc_CPPFLAGS = $(XVNC_CPPFLAGS) -DKASMVNC -DNO_MODULE_EXTS \ Xvnc_LDADD = $(XVNC_LIBS) libvnccommon.la $(COMMON_LIBS) \ $(XSERVER_LIBS) $(XSERVER_SYS_LIBS) $(XVNC_SYS_LIBS) -lX11 -lwebp -lsharpyuv -lssl -lcrypto -lcrypt \ - -lfreetype + -lfreetype -lvpx Xvnc_LDFLAGS = $(LD_EXPORT_SYMBOLS_FLAG) -fopenmp