From 6b6c391f81c7e5ea81aaa9ee3dfc31760d2c5b75 Mon Sep 17 00:00:00 2001 From: sergystepanov Date: Thu, 24 Nov 2022 13:44:00 +0300 Subject: [PATCH] Faster image processing (#382) * Calculate frame duration in ns * Reuse single OpenGL byte buffer * Add threaded a/v processing * Check min threads in opts * Return missing audio sample --- configs/config.yaml | 4 + pkg/config/emulator/config.go | 1 + pkg/emulator/graphics/opengl.go | 9 +- pkg/emulator/image/color.go | 33 +++---- pkg/emulator/image/draw.go | 98 +++++++++++-------- pkg/emulator/image/rotation.go | 64 ++++++------ pkg/emulator/image/scale.go | 1 - pkg/emulator/libretro/nanoarch/naemulator.go | 12 ++- pkg/emulator/libretro/nanoarch/nanoarch.go | 62 +++++++----- .../libretro/nanoarch/nanoarch_test.go | 7 +- pkg/media/resampler.go | 2 +- pkg/worker/room/room.go | 10 +- 12 files changed, 173 insertions(+), 130 deletions(-) diff --git a/configs/config.yaml b/configs/config.yaml index 5a4d0ff4..85217353 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -92,6 +92,10 @@ emulator: # set output viewport scale factor scale: 1 + # set the total number of threads for the image processing + # (experimental) + threads: 4 + aspectRatio: # enable aspect ratio changing # (experimental) diff --git a/pkg/config/emulator/config.go b/pkg/config/emulator/config.go index ddd1f892..22f35549 100644 --- a/pkg/config/emulator/config.go +++ b/pkg/config/emulator/config.go @@ -8,6 +8,7 @@ import ( type Emulator struct { Scale int + Threads int AspectRatio struct { Keep bool Width int diff --git a/pkg/emulator/graphics/opengl.go b/pkg/emulator/graphics/opengl.go index e732e2ed..36e5e2a7 100644 --- a/pkg/emulator/graphics/opengl.go +++ b/pkg/emulator/graphics/opengl.go @@ -22,7 +22,10 @@ type offscreenSetup struct { hasStencil bool } -var opt = offscreenSetup{} +var ( + opt = offscreenSetup{} + buf []byte +) type PixelFormat int @@ -94,7 +97,7 @@ func destroyFramebuffer() { } func ReadFramebuffer(bytes int, w int, h int) []byte { - data := make([]byte, bytes) + data := buf[:bytes] gl.BindFramebuffer(gl.FRAMEBUFFER, opt.fbo) gl.ReadPixels(0, 0, int32(w), int32(h), opt.pixType, opt.pixFormat, unsafe.Pointer(&data[0])) gl.BindFramebuffer(gl.FRAMEBUFFER, 0) @@ -103,6 +106,8 @@ func ReadFramebuffer(bytes int, w int, h int) []byte { func getFbo() uint32 { return opt.fbo } +func SetBuffer(size int) { buf = make([]byte, size) } + func SetPixelFormat(format PixelFormat) { switch format { case UnsignedShort5551: diff --git a/pkg/emulator/image/color.go b/pkg/emulator/image/color.go index 085e4c09..6104c3be 100644 --- a/pkg/emulator/image/color.go +++ b/pkg/emulator/image/color.go @@ -1,8 +1,6 @@ package image -import ( - "image/color" -) +import "unsafe" const ( // BIT_FORMAT_SHORT_5_5_5_1 has 5 bits R, 5 bits G, 5 bits B, 1 bit alpha @@ -13,24 +11,17 @@ const ( BitFormatShort565 ) -type Format func(data []byte, index int) color.RGBA - -func Rgb565(data []byte, index int) color.RGBA { - pixel := (int)(data[index]) + ((int)(data[index+1]) << 8) - - return color.RGBA{ - R: byte(((pixel>>11)*255 + 15) / 31), - G: byte((((pixel>>5)&0x3F)*255 + 31) / 63), - B: byte(((pixel&0x1F)*255 + 15) / 31), - A: 255, - } +type RGB struct { + R, G, B uint8 } -func Rgba8888(data []byte, index int) color.RGBA { - return color.RGBA{ - R: data[index+2], - G: data[index+1], - B: data[index], - A: 255, - } +type Format func(data []byte, index int) RGB + +func Rgb565(data []byte, index int) RGB { + pixel := *(*uint16)(unsafe.Pointer(&data[index])) + return RGB{R: uint8((pixel >> 8) & 0xf8), G: uint8((pixel >> 3) & 0xfc), B: uint8((pixel << 3) & 0xfc)} +} + +func Rgba8888(data []byte, index int) RGB { + return RGB{R: data[index+2], G: data[index+1], B: data[index]} } diff --git a/pkg/emulator/image/draw.go b/pkg/emulator/image/draw.go index b2a1ba87..0d1358c6 100644 --- a/pkg/emulator/image/draw.go +++ b/pkg/emulator/image/draw.go @@ -2,6 +2,7 @@ package image import ( "image" + "sync" ) type imageCache struct { @@ -10,57 +11,72 @@ type imageCache struct { h int } -var canvas = imageCache{ - image.NewRGBA(image.Rectangle{}), - 0, - 0, +func (i *imageCache) get(w, h int) *image.RGBA { + if i.w == w && i.h == h { + return i.image + } + i.w, i.h = w, h + i.image = image.NewRGBA(image.Rect(0, 0, w, h)) + return i.image } -func DrawRgbaImage(pixFormat Format, rotationFn Rotate, scaleType int, flipV bool, w, h, packedW, bpp int, - data []byte, dw, dh int) *image.RGBA { - if pixFormat == nil { - return nil - } +var ( + canvas1 = imageCache{image.NewRGBA(image.Rectangle{}), 0, 0} + canvas2 = imageCache{image.NewRGBA(image.Rectangle{}), 0, 0} + wg sync.WaitGroup +) +func DrawRgbaImage(pixFormat Format, rot *Rotate, scaleType int, flipV bool, w, h, packedW, bpp int, + data []byte, dw, dh, th int) *image.RGBA { // !to implement own image interfaces img.Pix = bytes[] ww, hh := w, h - if rotationFn.IsEven { + if rot != nil && rot.IsEven { ww, hh = hh, ww } - src := getCanvas(ww, hh) + src := canvas1.get(ww, hh) - drawImage(pixFormat, w, h, packedW, bpp, flipV, rotationFn, data, src) - out := image.NewRGBA(image.Rect(0,0, dw, dh)) - Resize(scaleType, src, out) - return out -} + normY := !flipV + hn := h / th + pwb := packedW * bpp + wg.Add(th) + for i := 0; i < th; i++ { + xx := hn * i + go func() { + for y, yy, l, lx, row := xx, 0, xx+hn, 0, 0; y < l; y++ { + if normY { + yy = y + } else { + yy = (h - 1) - y + } + row = yy * src.Stride + lx = y * pwb + for x, k := 0, 0; x < w; x++ { + if rot == nil { + k = x<<2 + row + } else { + dx, dy := rot.Call(x, yy, w, h) + k = dx<<2 + dy*src.Stride + } + r := pixFormat(data, x*bpp+lx) + src.Pix[k], src.Pix[k+1], src.Pix[k+2], src.Pix[k+3] = r.R, r.G, r.B, 255 + } + } + wg.Done() + }() + } + wg.Wait() -func drawImage(toRGBA Format, w, h, packedW, bpp int, flipV bool, rotationFn Rotate, data []byte, image *image.RGBA) { - for y := 0; y < h; y++ { - yy := y - if flipV { - yy = (h - 1) - y - } - for x := 0; x < w; x++ { - src := toRGBA(data, (x+y*packedW)*bpp) - dx, dy := rotationFn.Call(x, yy, w, h) - i := dx*4 + dy*image.Stride - dst := image.Pix[i : i+4 : i+4] - dst[0] = src.R - dst[1] = src.G - dst[2] = src.B - dst[3] = src.A - } + if ww == dw && hh == dh { + return src + } else { + out := canvas2.get(dw, dh) + Resize(scaleType, src, out) + return out } } -func getCanvas(w, h int) *image.RGBA { - if canvas.w == w && canvas.h == h { - return canvas.image - } - - canvas.w, canvas.h = w, h - canvas.image = image.NewRGBA(image.Rect(0, 0, w, h)) - - return canvas.image +func Clear() { + wg = sync.WaitGroup{} + canvas1.get(0, 0) + canvas2.get(0, 0) } diff --git a/pkg/emulator/image/rotation.go b/pkg/emulator/image/rotation.go index 5f6e0bfd..ba61ed94 100644 --- a/pkg/emulator/image/rotation.go +++ b/pkg/emulator/image/rotation.go @@ -1,5 +1,3 @@ -// This package contains functions for -// Pi/2 step rotation of points in a 2-dimensional space. package image type Angle uint @@ -11,7 +9,7 @@ const ( Angle270 ) -// A helper to choose appropriate rotation by its angle +// Angles is a helper to choose appropriate rotation based on its angle. var Angles = [4]Rotate{ Angle0: {Call: Rotate0, IsEven: false}, Angle90: {Call: Rotate90, IsEven: true}, @@ -23,58 +21,58 @@ func GetRotation(angle Angle) Rotate { return Angles[angle] } -// An interface for rotation of a given point -// with the coordinates x, y in the matrix of w x h. -// Returns a pair of new coordinates x, y in the resulting -// matrix. -// Be aware that w / h values are 0 index-based and -// it's meant to be used with h corresponded +// Rotate is an interface for rotation of a given point. +// +// With the coordinates x, y in the matrix of w x h. +// Returns a pair of new coordinates x, y in the resulting matrix. +// Be aware that w / h values are 0 index-based, +// and it's meant to be used with h corresponded // to matrix height and y coordinate, and with w to x coordinate. type Rotate struct { Call func(x, y, w, h int) (int, int) IsEven bool } -// 0° or the original orientation -/* Example: */ -/* 1 2 3 1 2 3 */ -/* 4 5 6 -> 4 5 6 */ -/* 7 8 9 7 8 9 */ +// Rotate0 is 0° or the original orientation. +// +// 1 2 3 1 2 3 +// 4 5 6 -> 4 5 6 +// 7 8 9 7 8 9 func Rotate0(x, y, _, _ int) (int, int) { return x, y } -// 90° CCW or 270° CW -/* Example: */ -/* 1 2 3 3 6 9 */ -/* 4 5 6 -> 2 5 8 */ -/* 7 8 9 1 4 7 */ +// Rotate90 is 90° CCW or 270° CW. +// +// 1 2 3 3 6 9 +// 4 5 6 -> 2 5 8 +// 7 8 9 1 4 7 func Rotate90(x, y, w, _ int) (int, int) { return y, (w - 1) - x } -// 180° CCW -/* Example: */ -/* 1 2 3 9 8 7 */ -/* 4 5 6 -> 6 5 4 */ -/* 7 8 9 3 2 1 */ +// Rotate180 is 180° CCW. +// +// 1 2 3 9 8 7 +// 4 5 6 -> 6 5 4 +// 7 8 9 3 2 1 func Rotate180(x, y, w, h int) (int, int) { return (w - 1) - x, (h - 1) - y } -// 270° CCW or 90° CW -/* Example: */ -/* 1 2 3 7 4 1 */ -/* 4 5 6 -> 8 5 2 */ -/* 7 8 9 9 6 3 */ +// Rotate270 is 270° CCW or 90° CW. +// +// 1 2 3 7 4 1 +// 4 5 6 -> 8 5 2 +// 7 8 9 9 6 3 func Rotate270(x, y, _, h int) (int, int) { return (h - 1) - y, x } -/* -[1 2 3 4 5 6 7 8 9] -[7 4 1 8 5 2 9 6 3] -*/ +// ExampleRotate is an example of rotation usage. +// +// [1 2 3 4 5 6 7 8 9] +// [7 4 1 8 5 2 9 6 3] func ExampleRotate(data []uint8, w int, h int, angle Angle) []uint8 { dest := make([]uint8, len(data)) rotationFn := Angles[angle] diff --git a/pkg/emulator/image/scale.go b/pkg/emulator/image/scale.go index 8ab8dbf5..2a507c92 100644 --- a/pkg/emulator/image/scale.go +++ b/pkg/emulator/image/scale.go @@ -17,7 +17,6 @@ const ( func Resize(scaleType int, src *image.RGBA, out *image.RGBA) { // !to do set it once instead switching on each iteration - // !to do skip resize if w=vw h=vh switch scaleType { case ScaleBilinear: draw.ApproxBiLinear.Scale(out, out.Bounds(), src, src.Bounds(), draw.Src, nil) diff --git a/pkg/emulator/libretro/nanoarch/naemulator.go b/pkg/emulator/libretro/nanoarch/naemulator.go index 8c48a11b..e8c407db 100644 --- a/pkg/emulator/libretro/nanoarch/naemulator.go +++ b/pkg/emulator/libretro/nanoarch/naemulator.go @@ -69,6 +69,9 @@ type naEmulator struct { // out frame size vw, vh int + // draw threads + th int + players Players done chan struct{} @@ -89,7 +92,7 @@ type GameFrame struct { var NAEmulator *naEmulator // NAEmulator implements CloudEmulator interface based on NanoArch(golang RetroArch) -func NewNAEmulator(roomID string, inputChannel <-chan InputEvent, storage Storage, conf config.LibretroCoreConfig) (*naEmulator, chan GameFrame, chan []int16) { +func NewNAEmulator(roomID string, inputChannel <-chan InputEvent, storage Storage, conf config.LibretroCoreConfig, threads int) (*naEmulator, chan GameFrame, chan []int16) { imageChannel := make(chan GameFrame, 30) audioChannel := make(chan []int16, 30) @@ -110,6 +113,7 @@ func NewNAEmulator(roomID string, inputChannel <-chan InputEvent, storage Storag players: NewPlayerSessionInput(), roomID: roomID, done: make(chan struct{}, 1), + th: threads, }, imageChannel, audioChannel } @@ -142,8 +146,8 @@ func NewVideoExporter(roomID string, imgChannel chan GameFrame) *VideoExporter { // Init initialize new RetroArch cloud emulator // withImageChan returns an image stream as Channel for output else it will write to unix socket -func Init(roomID string, withImageChannel bool, inputChannel <-chan InputEvent, storage Storage, config config.LibretroCoreConfig) (*naEmulator, chan GameFrame, chan []int16) { - emu, imageChannel, audioChannel := NewNAEmulator(roomID, inputChannel, storage, config) +func Init(roomID string, withImageChannel bool, inputChannel <-chan InputEvent, storage Storage, config config.LibretroCoreConfig, threads int) (*naEmulator, chan GameFrame, chan []int16) { + emu, imageChannel, audioChannel := NewNAEmulator(roomID, inputChannel, storage, config, threads) // Set to global NAEmulator NAEmulator = emu if !withImageChannel { @@ -189,7 +193,7 @@ func (na *naEmulator) Start() { ticker := time.NewTicker(time.Second / time.Duration(na.meta.Fps)) defer ticker.Stop() - lastFrameTime = time.Now() + lastFrameTime = time.Now().UnixNano() for { na.Lock() diff --git a/pkg/emulator/libretro/nanoarch/nanoarch.go b/pkg/emulator/libretro/nanoarch/nanoarch.go index 26e61b28..6423ec83 100644 --- a/pkg/emulator/libretro/nanoarch/nanoarch.go +++ b/pkg/emulator/libretro/nanoarch/nanoarch.go @@ -2,6 +2,7 @@ package nanoarch import ( "bufio" + "fmt" "log" "os" "os/user" @@ -57,8 +58,8 @@ void bridge_execute(void *f); */ import "C" -var mu, fmu sync.Mutex -var lastFrameTime time.Time +var mu sync.Mutex +var lastFrameTime int64 var video struct { pitch uint32 @@ -78,7 +79,7 @@ var video struct { // default core pix format converter var pixelFormatConverterFn = image.Rgb565 -var rotationFn = image.GetRotation(image.Angle(0)) +var rotationFn *image.Rotate //const joypadNumKeys = int(C.RETRO_DEVICE_ID_JOYPAD_R3 + 1) //var joy [joypadNumKeys]bool @@ -127,12 +128,6 @@ type CloudEmulator interface { //export coreVideoRefresh func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned, pitch C.size_t) { - t := time.Now() - fmu.Lock() - dt := t.Sub(lastFrameTime) - lastFrameTime = t - fmu.Unlock() - // some cores can return nothing // !to add duplicate if can dup if data == nil { @@ -157,7 +152,7 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned, } // the image is being resized and de-rotated - img := image.DrawRgbaImage( + frame := image.DrawRgbaImage( pixelFormatConverterFn, rotationFn, image.ScaleNearestNeighbour, @@ -166,12 +161,15 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned, data_, NAEmulator.vw, NAEmulator.vh, + NAEmulator.th, ) - // the image is pushed into a channel - // where it will be distributed with fan-out + t := time.Now().UnixNano() + dt := time.Duration(t - lastFrameTime) + lastFrameTime = t + select { - case NAEmulator.imageChannel <- GameFrame{Data: img, Duration: dt}: + case NAEmulator.imageChannel <- GameFrame{Data: frame, Duration: dt}: default: } } @@ -211,13 +209,9 @@ func coreInputState(port C.unsigned, device C.unsigned, index C.unsigned, id C.u } func audioWrite(buf unsafe.Pointer, frames C.size_t) C.size_t { - // !to make it mono/stereo independent - samples := int(frames) * 2 - pcm := (*[(1 << 30) - 1]int16)(buf)[:samples:samples] - + samples := int(frames) << 1 + pcm := (*[4096]int16)(buf)[:samples:samples] p := make([]int16, samples) - // copy because pcm slice refer to buf underlying pointer, - // and buf pointer is the same in continuous frames copy(p, pcm) select { @@ -483,6 +477,8 @@ func slurp(path string, size int64) ([]byte, error) { } func coreLoadGame(filename string) { + lastFrameTime = 0 + file, err := os.Open(filename) if err != nil { panic(err) @@ -513,7 +509,7 @@ func coreLoadGame(filename string) { log.Printf(" block_extract: %v", bool(si.block_extract)) if !si.need_fullpath { - bytes, err := slurp(filename, size) + bytes, err := os.ReadFile(filename) if err != nil { panic(err) } @@ -561,6 +557,9 @@ func coreLoadGame(filename string) { video.baseWidth = int32(avi.geometry.base_width) video.baseHeight = int32(avi.geometry.base_height) if video.isGl { + bufS := int(video.maxWidth * video.maxHeight * int32(video.bpp)) + graphics.SetBuffer(bufS) + log.Printf("Set buffer: %v", byteCountBinary(int64(bufS))) if usesLibCo { C.bridge_execute(C.initVideo_cgo) } else { @@ -626,6 +625,7 @@ func nanoarchShutdown() { for _, element := range coreConfig { C.free(unsafe.Pointer(element)) } + image.Clear() } func nanoarchRun() { @@ -673,7 +673,25 @@ func setRotation(rotation uint) { return } video.rotation = image.Angle(rotation) - rotationFn = image.GetRotation(video.rotation) - NAEmulator.meta.Rotation = rotationFn + r := image.GetRotation(video.rotation) + if rotation > 0 { + rotationFn = &r + } else { + rotationFn = nil + } + NAEmulator.meta.Rotation = r log.Printf("[Env]: the game video is rotated %v°", map[uint]uint{0: 0, 1: 90, 2: 180, 3: 270}[rotation]) } + +func byteCountBinary(b int64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/pkg/emulator/libretro/nanoarch/nanoarch_test.go b/pkg/emulator/libretro/nanoarch/nanoarch_test.go index 239ccf27..515a6381 100644 --- a/pkg/emulator/libretro/nanoarch/nanoarch_test.go +++ b/pkg/emulator/libretro/nanoarch/nanoarch_test.go @@ -3,7 +3,7 @@ package nanoarch import ( "crypto/md5" "fmt" - "io/ioutil" + "io" "log" "os" "path" @@ -85,6 +85,7 @@ func GetEmulatorMock(room string, system string) *EmulatorMock { players: NewPlayerSessionInput(), roomID: room, done: make(chan struct{}, 1), + th: conf.Emulator.Threads, }, core: path.Base(meta.Lib), @@ -175,7 +176,7 @@ func (emu *EmulatorMock) handleInput(handler func(event InputEvent)) { // Locks the emulator. func (emu *EmulatorMock) dumpState() (string, string) { emu.Lock() - bytes, _ := ioutil.ReadFile(emu.paths.save) + bytes, _ := os.ReadFile(emu.paths.save) persistedStateHash := getHash(bytes) emu.Unlock() @@ -213,7 +214,7 @@ func cleanPath(path string) string { // benchmarkEmulator is a generic function for // measuring emulator performance for one emulation frame. func benchmarkEmulator(system string, rom string, b *testing.B) { - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) os.Stdout, _ = os.Open(os.DevNull) s := GetDefaultEmulatorMock("bench_"+system+"_performance", system, rom) diff --git a/pkg/media/resampler.go b/pkg/media/resampler.go index 297aa390..1ada082f 100644 --- a/pkg/media/resampler.go +++ b/pkg/media/resampler.go @@ -18,7 +18,7 @@ func ResampleStretch(pcm []int16, size int) []int16 { l[i] = l[i-1] } } - for i := 0; i < size-1; i += 2 { + for i := 0; i < size; i += 2 { audio[i], audio[i+1] = r[i/2], l[i/2] } return audio diff --git a/pkg/worker/room/room.go b/pkg/worker/room/room.go index 391ab24b..c13e64b1 100644 --- a/pkg/worker/room/room.go +++ b/pkg/worker/room/room.go @@ -170,16 +170,22 @@ func NewRoom(roomID string, game games.GameMetadata, recUser string, rec bool, o emuName := cfg.Emulator.GetEmulator(game.Type, game.Path) libretroConfig := cfg.Emulator.GetLibretroCoreConfig(emuName) + th := cfg.Emulator.Threads + if th == 0 { + th = 1 + } + log.Printf("Image processing threads = %v", th) + if cfg.Encoder.WithoutGame { // Run without game, image stream is communicated over a unix socket imageChannel := NewVideoImporter(roomID) - director, _, audioChannel := nanoarch.Init(roomID, false, inputChannel, store, libretroConfig) + director, _, audioChannel := nanoarch.Init(roomID, false, inputChannel, store, libretroConfig, th) room.imageChannel = imageChannel room.director = director room.audioChannel = audioChannel } else { // Run without game, image stream is communicated over image channel - director, imageChannel, audioChannel := nanoarch.Init(roomID, true, inputChannel, store, libretroConfig) + director, imageChannel, audioChannel := nanoarch.Init(roomID, true, inputChannel, store, libretroConfig, th) room.imageChannel = imageChannel room.director = director room.audioChannel = audioChannel