Add new YUV converter with avg color calc (#287)

Add new YUV converter with 2x2 pixel matrix YUV color estimation.

The new YUV color converter, as opposed to the original one, uses more precise color calculations based on four neighboring pixels' average color values which helps a great deal with image aliasing / shimmering artifacts.
By default, it runs in the multithreaded mode with the game frames sliced between 2*(CPU cores) goroutines.
In case if this estimation mode doesn't work as expected it is possible to switch back to the original mode.

The default encoder is switched to x264 since it's faster now.

configs/config.yaml

@@ -152,7 +152,7 @@ encoder:
     frequency: 48000
   video:
     # h264, vpx (VP8)
-    codec: vpx
+    codec: h264
     # see: https://trac.ffmpeg.org/wiki/Encode/H.264
     h264:
       # Constant Rate Factor (CRF) 0-51 (default: 23)

pkg/encoder/pipe.go

@@ -1,6 +1,10 @@
 package encoder
 
-import "log"
+import (
+	"log"
+
+	"github.com/giongto35/cloud-game/v2/pkg/encoder/yuv"
+)
 
 type VideoPipe struct {
 	Input  chan InFrame
@@ -40,9 +44,9 @@ func (vp *VideoPipe) Start() {
 		}
 	}()
 
-	yuv := NewYuvBuffer(vp.w, vp.h)
+	yuvProc := yuv.NewYuvImgProcessor(vp.w, vp.h)
 	for img := range vp.Input {
-		frame := vp.encoder.Encode(yuv.FromRGBA(img.Image).data)
+		frame := vp.encoder.Encode(yuvProc.Process(img.Image).Get())
 		if len(frame) > 0 {
 			vp.Output <- OutFrame{Data: frame, Timestamp: img.Timestamp}
 		}

pkg/encoder/yuv.go

@@ -1,65 +0,0 @@
-package encoder
-
-import (
-	"image"
-	"unsafe"
-)
-
-// see: https://stackoverflow.com/questions/9465815/rgb-to-yuv420-algorithm-efficiency
-// credit to https://github.com/poi5305/go-yuv2webRTC/blob/master/webrtc/webrtc.go
-
-/*
-void rgba2yuv(void * destination, void * source, int width, int height, int stride) {
-  const int image_size = width * height;
-  unsigned char * rgba = source;
-  unsigned char * dst_y = destination;
-  unsigned char * dst_u = destination + image_size;
-  unsigned char * dst_v = destination + image_size + image_size / 4;
-
-  int i, x, y;
-  // Y plane
-  for (y = 0; y < height; ++y) {
-    for (x = 0; x < width; ++x) {
-      i = y * (width + stride) + x;
-      * dst_y++ = ((66 * rgba[4 * i] + 129 * rgba[4 * i + 1] + 25 * rgba[4 * i + 2]) >> 8) + 16;
-    }
-  }
-
-  // U plane
-  for (y = 0; y < height; y += 2) {
-    for (x = 0; x < width; x += 2) {
-      i = y * (width + stride) + x;
-      * dst_u++ = ((-38 * rgba[4 * i] + -74 * rgba[4 * i + 1] + 112 * rgba[4 * i + 2]) >> 8) + 128;
-    }
-  }
-
-  // V plane
-  for (y = 0; y < height; y += 2) {
-    for (x = 0; x < width; x += 2) {
-      i = y * (width + stride) + x;
-      * dst_v++ = ((112 * rgba[4 * i] + -94 * rgba[4 * i + 1] + -18 * rgba[4 * i + 2]) >> 8) + 128;
-    }
-  }
-}
-*/
-import "C"
-
-type Yuv struct {
-	data []byte
-	w, h int
-}
-
-func NewYuvBuffer(w, h int) Yuv {
-	size := int(float32(w*h) * 1.5)
-	return Yuv{
-		data: make([]byte, size, size),
-		w:    w,
-		h:    h,
-	}
-}
-
-// FromRGBA converts RGBA colorspace into YUV I420 format inside the internal buffer.
-func (yuv *Yuv) FromRGBA(rgba *image.RGBA) *Yuv {
-	C.rgba2yuv(unsafe.Pointer(&yuv.data[0]), unsafe.Pointer(&rgba.Pix[0]), C.int(yuv.w), C.int(yuv.h), C.int(0))
-	return yuv
-}

pkg/encoder/yuv/options.go

@@ -0,0 +1,42 @@
+package yuv
+
+type Options struct {
+	ChromaP  ChromaPos
+	Threaded bool
+	Threads  int
+}
+
+func (o *Options) override(options ...Option) {
+	for _, opt := range options {
+		opt(o)
+	}
+}
+
+type Option func(*Options)
+
+func Threaded(t bool) Option {
+	return func(opts *Options) {
+		opts.Threaded = t
+	}
+}
+
+func Threads(t int) Option {
+	return func(opts *Options) {
+		opts.Threads = t
+	}
+}
+
+func ChromaP(cp ChromaPos) Option {
+	return func(opts *Options) {
+		opts.ChromaP = cp
+	}
+}
+
+// WithOptions used for config files.
+func WithOptions(arg Options) Option {
+	return func(args *Options) {
+		args.ChromaP = arg.ChromaP
+		args.Threaded = arg.Threaded
+		args.Threads = arg.Threads
+	}
+}

pkg/encoder/yuv/yuv.c

@@ -0,0 +1,143 @@
+#include "yuv.h"
+
+// based on: https://stackoverflow.com/questions/9465815/rgb-to-yuv420-algorithm-efficiency
+
+// Converts RGBA image to YUV (I420) with BT.601 studio color range.
+void rgbaToYuv(void *destination, void *source, int width, int height, chromaPos chroma) {
+    const int image_size = width * height;
+    unsigned char *rgba = source;
+    unsigned char *dst_y = destination;
+    unsigned char *dst_u = destination + image_size;
+    unsigned char *dst_v = destination + image_size + image_size / 4;
+
+    int r1, g1, b1, stride;
+    // Y plane
+    for (int y = 0; y < height; ++y) {
+        stride = 4 * y * width;
+        for (int x = 0; x < width; ++x) {
+            r1 = 4 * x + stride;
+            g1 = r1 + 1;
+            b1 = g1 + 1;
+            *dst_y++ = ((66 * rgba[r1] + 129 * rgba[g1] + 25 * rgba[b1]) >> 8) + 16;
+        }
+    }
+
+    // U+V plane
+    if (chroma == TOP_LEFT) {
+        for (int y = 0; y < height; y += 2) {
+            stride = 4 * y * width;
+            for (int x = 0; x < width; x += 2) {
+                r1 = 4 * x + stride;
+                g1 = r1 + 1;
+                b1 = g1 + 1;
+                *dst_u++ = ((-38 * rgba[r1] + -74 * rgba[g1] + 112 * rgba[b1]) >> 8) + 128;
+                *dst_v++ = ((112 * rgba[r1] + -94 * rgba[g1] + -18 * rgba[b1]) >> 8) + 128;
+            }
+        }
+    } else if (chroma == BETWEEN_FOUR) {
+        int r2, g2, b2, r3, g3, b3, r4, g4, b4;
+
+        for (int y = 0; y < height; y += 2) {
+            stride = 4 * y * width;
+            for (int x = 0; x < width; x += 2) {
+                // (1 2) x x
+                //  x x  x x
+                r1 = 4 * x + stride;
+                g1 = r1 + 1;
+                b1 = g1 + 1;
+                r2 = r1 + 4;
+                g2 = r2 + 1;
+                b2 = g2 + 1;
+                //  x x  x x
+                // (3 4) x x
+                r3 = r1 + 4 * width;
+
+                g3 = r3 + 1;
+                b3 = g3 + 1;
+                r4 = r3 + 4;
+                g4 = r4 + 1;
+                b4 = g4 + 1;
+                *dst_u++ = (((-38 * rgba[r1] + -74 * rgba[g1] + 112 * rgba[b1]) >> 8) +
+                            ((-38 * rgba[r2] + -74 * rgba[g2] + 112 * rgba[b2]) >> 8) +
+                            ((-38 * rgba[r3] + -74 * rgba[g3] + 112 * rgba[b3]) >> 8) +
+                            ((-38 * rgba[r4] + -74 * rgba[g4] + 112 * rgba[b4]) >> 8) + 512) >> 2;
+                *dst_v++ = (((112 * rgba[r1] + -94 * rgba[g1] + -18 * rgba[b1]) >> 8) +
+                            ((112 * rgba[r2] + -94 * rgba[g2] + -18 * rgba[b2]) >> 8) +
+                            ((112 * rgba[r3] + -94 * rgba[g3] + -18 * rgba[b3]) >> 8) +
+                            ((112 * rgba[r4] + -94 * rgba[g4] + -18 * rgba[b4]) >> 8) + 512) >> 2;
+            }
+        }
+    }
+}
+
+void chroma(void *destination, void *source, int pos, int deu, int dev, int width, int height, chromaPos chroma) {
+    unsigned char *rgba = source + 4 * pos;
+    unsigned char *dst_u = destination + deu + pos / 4;
+    unsigned char *dst_v = destination + dev + pos / 4;
+
+    int r1, g1, b1, stride;
+
+    // U+V plane
+    if (chroma == TOP_LEFT) {
+        for (int y = 0; y < height; y += 2) {
+            stride = 4 * y * width;
+            for (int x = 0; x < width; x += 2) {
+                r1 = 4 * x + stride;
+                g1 = r1 + 1;
+                b1 = g1 + 1;
+                *dst_u++ = ((-38 * rgba[r1] + -74 * rgba[g1] + 112 * rgba[b1]) >> 8) + 128;
+                *dst_v++ = ((112 * rgba[r1] + -94 * rgba[g1] + -18 * rgba[b1]) >> 8) + 128;
+            }
+        }
+    } else if (chroma == BETWEEN_FOUR) {
+        int r2, g2, b2, r3, g3, b3, r4, g4, b4;
+
+        for (int y = 0; y < height; y += 2) {
+            stride = 4 * y * width;
+            for (int x = 0; x < width; x += 2) {
+                // (1 2) x x
+                //  x x  x x
+                r1 = 4 * x + stride;
+                g1 = r1 + 1;
+                b1 = g1 + 1;
+                r2 = r1 + 4;
+                g2 = r2 + 1;
+                b2 = g2 + 1;
+                //  x x  x x
+                // (3 4) x x
+                r3 = r1 + 4 * width;
+                g3 = r3 + 1;
+                b3 = g3 + 1;
+                r4 = r3 + 4;
+                g4 = r4 + 1;
+                b4 = g4 + 1;
+                *dst_u++ = (((-38 * rgba[r1] + -74 * rgba[g1] + 112 * rgba[b1]) >> 8) +
+                            ((-38 * rgba[r2] + -74 * rgba[g2] + 112 * rgba[b2]) >> 8) +
+                            ((-38 * rgba[r3] + -74 * rgba[g3] + 112 * rgba[b3]) >> 8) +
+                            ((-38 * rgba[r4] + -74 * rgba[g4] + 112 * rgba[b4]) >> 8) + 512) >> 2;
+                *dst_v++ = (((112 * rgba[r1] + -94 * rgba[g1] + -18 * rgba[b1]) >> 8) +
+                            ((112 * rgba[r2] + -94 * rgba[g2] + -18 * rgba[b2]) >> 8) +
+                            ((112 * rgba[r3] + -94 * rgba[g3] + -18 * rgba[b3]) >> 8) +
+                            ((112 * rgba[r4] + -94 * rgba[g4] + -18 * rgba[b4]) >> 8) + 512) >> 2;
+            }
+        }
+    }
+}
+
+void luma(void *destination, void *source, int pos, int width, int height) {
+    unsigned char *rgba = source + 4 * pos;
+    unsigned char *dst_y = destination + pos;
+
+    int x, y, r1, g1, b1, stride;
+
+    // Y plane
+    for (y = 0; y < height; ++y) {
+        stride = 4 * y * width;
+        for (x = 0; x < width; ++x) {
+            r1 = 4 * x + stride;
+            g1 = r1 + 1;
+            b1 = g1 + 1;
+            *dst_y++ = 16 + ((66 * rgba[r1] + 129 * rgba[g1] + 25 * rgba[b1]) >> 8);
+        }
+    }
+}

pkg/encoder/yuv/yuv.go

@@ -0,0 +1,138 @@
+package yuv
+
+import (
+	"image"
+	"runtime"
+	"sync"
+	"unsafe"
+)
+
+/*
+#cgo CFLAGS: -Wall -O3
+#include "yuv.h"
+*/
+import "C"
+
+type ImgProcessor interface {
+	Process(rgba *image.RGBA) ImgProcessor
+	Get() []byte
+}
+
+type processor struct {
+	Data []byte
+	w, h int
+	pos  ChromaPos
+
+	// cache
+	dst    unsafe.Pointer
+	ww     C.int
+	chroma C.chromaPos
+}
+
+type threadedProcessor struct {
+	*processor
+
+	// threading
+	threads int
+	chunk   int
+
+	// cache
+	chromaU C.int
+	chromaV C.int
+}
+
+type ChromaPos uint8
+
+const (
+	TopLeft ChromaPos = iota
+	BetweenFour
+)
+
+// NewYuvImgProcessor creates new YUV image converter from RGBA.
+func NewYuvImgProcessor(w, h int, options ...Option) ImgProcessor {
+	opts := &Options{
+		ChromaP:  BetweenFour,
+		Threaded: true,
+		Threads:  runtime.NumCPU(),
+	}
+	opts.override(options...)
+
+	bufSize := int(float32(w*h) * 1.5)
+	buf := make([]byte, bufSize, bufSize)
+
+	processor := processor{
+		Data:   buf,
+		chroma: C.chromaPos(opts.ChromaP),
+		dst:    unsafe.Pointer(&buf[0]),
+		h:      h,
+		pos:    opts.ChromaP,
+		w:      w,
+		ww:     C.int(w),
+	}
+
+	if opts.Threaded {
+		// chunks the image evenly
+		chunk := h / opts.Threads
+		if chunk%2 != 0 {
+			chunk--
+		}
+
+		return &threadedProcessor{
+			chromaU:   C.int(w * h),
+			chromaV:   C.int(w*h + w*h/4),
+			chunk:     chunk,
+			processor: &processor,
+			threads:   opts.Threads,
+		}
+	}
+	return &processor
+}
+
+func (yuv *processor) Get() []byte {
+	return yuv.Data
+}
+
+// Process converts RGBA colorspace into YUV I420 format inside the internal buffer.
+// Non-threaded version.
+func (yuv *processor) Process(rgba *image.RGBA) ImgProcessor {
+	C.rgbaToYuv(yuv.dst, unsafe.Pointer(&rgba.Pix[0]), yuv.ww, C.int(yuv.h), yuv.chroma)
+	return yuv
+}
+
+func (yuv *threadedProcessor) Get() []byte {
+	return yuv.Data
+}
+
+// Process converts RGBA colorspace into YUV I420 format inside the internal buffer.
+// Threaded version.
+//
+// We divide the input image into chunks by the number of available CPUs.
+// Each chunk should contain 2, 4, 6, and etc. rows of the image.
+//
+//        8x4          CPU (2)
+//  x x x x x x x x  | Coroutine 1
+//  x x x x x x x x  | Coroutine 1
+//  x x x x x x x x  | Coroutine 2
+//  x x x x x x x x  | Coroutine 2
+//
+func (yuv *threadedProcessor) Process(rgba *image.RGBA) ImgProcessor {
+	src := unsafe.Pointer(&rgba.Pix[0])
+	wg := sync.WaitGroup{}
+	wg.Add(2 * yuv.threads)
+	for i := 0; i < yuv.threads; i++ {
+		pos, hh := C.int(yuv.w*i*yuv.chunk), C.int(yuv.chunk)
+		// we need to know how many pixels left
+		// if the image can't be divided evenly
+		// between all the threads
+		if i == yuv.threads-1 {
+			hh = C.int(yuv.h - i*yuv.chunk)
+		}
+		go func() { defer wg.Done(); C.luma(yuv.dst, src, pos, yuv.ww, hh) }()
+		go func() {
+			defer wg.Done()
+			C.chroma(yuv.dst, src, pos, yuv.chromaU, yuv.chromaV, yuv.ww, hh, yuv.chroma)
+		}()
+	}
+	wg.Wait()
+	return yuv
+}

pkg/encoder/yuv/yuv.h

@@ -0,0 +1,24 @@
+
+typedef enum {
+    // It will take each TL pixel for chroma values.
+    // XO  X   XO  X
+    // X   X   X   X
+    TOP_LEFT = 0,
+    // It will take an average color from the 2x2 pixel group for chroma values.
+    // X   X   X   X
+    //   O       O
+    // X   X   X   X
+    BETWEEN_FOUR = 1
+} chromaPos;
+
+// Converts RGBA image to YUV (I420) with BT.601 studio color range.
+void rgbaToYuv(void *destination, void *source, int width, int height, chromaPos chroma);
+
+// Converts RGBA image chunk to YUV (I420) chroma with BT.601 studio color range.
+// pos contains a shift value for chunks.
+// deu, dev contains constant shifts for U, V planes in the resulting array.
+// chroma (0, 1) selects chroma estimation algorithm.
+void chroma(void *destination, void *source, int pos, int deu, int dev, int width, int height, chromaPos chroma);
+
+// Converts RGBA image chunk to YUV (I420) luma with BT.601 studio color range.
+void luma(void *destination, void *source, int pos, int width, int height);