Faster image processing (#382)

* Calculate frame duration in ns

* Reuse single OpenGL byte buffer

* Add threaded a/v processing

* Check min threads in opts

* Return missing audio sample
This commit is contained in:
sergystepanov 2022-11-24 13:44:00 +03:00 committed by GitHub
parent 7d355611eb
commit 6b6c391f81
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 173 additions and 130 deletions

View file

@ -92,6 +92,10 @@ emulator:
# set output viewport scale factor
scale: 1
# set the total number of threads for the image processing
# (experimental)
threads: 4
aspectRatio:
# enable aspect ratio changing
# (experimental)

View file

@ -8,6 +8,7 @@ import (
type Emulator struct {
Scale int
Threads int
AspectRatio struct {
Keep bool
Width int

View file

@ -22,7 +22,10 @@ type offscreenSetup struct {
hasStencil bool
}
var opt = offscreenSetup{}
var (
opt = offscreenSetup{}
buf []byte
)
type PixelFormat int
@ -94,7 +97,7 @@ func destroyFramebuffer() {
}
func ReadFramebuffer(bytes int, w int, h int) []byte {
data := make([]byte, bytes)
data := buf[:bytes]
gl.BindFramebuffer(gl.FRAMEBUFFER, opt.fbo)
gl.ReadPixels(0, 0, int32(w), int32(h), opt.pixType, opt.pixFormat, unsafe.Pointer(&data[0]))
gl.BindFramebuffer(gl.FRAMEBUFFER, 0)
@ -103,6 +106,8 @@ func ReadFramebuffer(bytes int, w int, h int) []byte {
func getFbo() uint32 { return opt.fbo }
func SetBuffer(size int) { buf = make([]byte, size) }
func SetPixelFormat(format PixelFormat) {
switch format {
case UnsignedShort5551:

View file

@ -1,8 +1,6 @@
package image
import (
"image/color"
)
import "unsafe"
const (
// BIT_FORMAT_SHORT_5_5_5_1 has 5 bits R, 5 bits G, 5 bits B, 1 bit alpha
@ -13,24 +11,17 @@ const (
BitFormatShort565
)
type Format func(data []byte, index int) color.RGBA
func Rgb565(data []byte, index int) color.RGBA {
pixel := (int)(data[index]) + ((int)(data[index+1]) << 8)
return color.RGBA{
R: byte(((pixel>>11)*255 + 15) / 31),
G: byte((((pixel>>5)&0x3F)*255 + 31) / 63),
B: byte(((pixel&0x1F)*255 + 15) / 31),
A: 255,
}
type RGB struct {
R, G, B uint8
}
func Rgba8888(data []byte, index int) color.RGBA {
return color.RGBA{
R: data[index+2],
G: data[index+1],
B: data[index],
A: 255,
}
type Format func(data []byte, index int) RGB
func Rgb565(data []byte, index int) RGB {
pixel := *(*uint16)(unsafe.Pointer(&data[index]))
return RGB{R: uint8((pixel >> 8) & 0xf8), G: uint8((pixel >> 3) & 0xfc), B: uint8((pixel << 3) & 0xfc)}
}
func Rgba8888(data []byte, index int) RGB {
return RGB{R: data[index+2], G: data[index+1], B: data[index]}
}

View file

@ -2,6 +2,7 @@ package image
import (
"image"
"sync"
)
type imageCache struct {
@ -10,57 +11,72 @@ type imageCache struct {
h int
}
var canvas = imageCache{
image.NewRGBA(image.Rectangle{}),
0,
0,
func (i *imageCache) get(w, h int) *image.RGBA {
if i.w == w && i.h == h {
return i.image
}
i.w, i.h = w, h
i.image = image.NewRGBA(image.Rect(0, 0, w, h))
return i.image
}
func DrawRgbaImage(pixFormat Format, rotationFn Rotate, scaleType int, flipV bool, w, h, packedW, bpp int,
data []byte, dw, dh int) *image.RGBA {
if pixFormat == nil {
return nil
}
var (
canvas1 = imageCache{image.NewRGBA(image.Rectangle{}), 0, 0}
canvas2 = imageCache{image.NewRGBA(image.Rectangle{}), 0, 0}
wg sync.WaitGroup
)
func DrawRgbaImage(pixFormat Format, rot *Rotate, scaleType int, flipV bool, w, h, packedW, bpp int,
data []byte, dw, dh, th int) *image.RGBA {
// !to implement own image interfaces img.Pix = bytes[]
ww, hh := w, h
if rotationFn.IsEven {
if rot != nil && rot.IsEven {
ww, hh = hh, ww
}
src := getCanvas(ww, hh)
src := canvas1.get(ww, hh)
drawImage(pixFormat, w, h, packedW, bpp, flipV, rotationFn, data, src)
out := image.NewRGBA(image.Rect(0,0, dw, dh))
Resize(scaleType, src, out)
return out
}
normY := !flipV
hn := h / th
pwb := packedW * bpp
wg.Add(th)
for i := 0; i < th; i++ {
xx := hn * i
go func() {
for y, yy, l, lx, row := xx, 0, xx+hn, 0, 0; y < l; y++ {
if normY {
yy = y
} else {
yy = (h - 1) - y
}
row = yy * src.Stride
lx = y * pwb
for x, k := 0, 0; x < w; x++ {
if rot == nil {
k = x<<2 + row
} else {
dx, dy := rot.Call(x, yy, w, h)
k = dx<<2 + dy*src.Stride
}
r := pixFormat(data, x*bpp+lx)
src.Pix[k], src.Pix[k+1], src.Pix[k+2], src.Pix[k+3] = r.R, r.G, r.B, 255
}
}
wg.Done()
}()
}
wg.Wait()
func drawImage(toRGBA Format, w, h, packedW, bpp int, flipV bool, rotationFn Rotate, data []byte, image *image.RGBA) {
for y := 0; y < h; y++ {
yy := y
if flipV {
yy = (h - 1) - y
}
for x := 0; x < w; x++ {
src := toRGBA(data, (x+y*packedW)*bpp)
dx, dy := rotationFn.Call(x, yy, w, h)
i := dx*4 + dy*image.Stride
dst := image.Pix[i : i+4 : i+4]
dst[0] = src.R
dst[1] = src.G
dst[2] = src.B
dst[3] = src.A
}
if ww == dw && hh == dh {
return src
} else {
out := canvas2.get(dw, dh)
Resize(scaleType, src, out)
return out
}
}
func getCanvas(w, h int) *image.RGBA {
if canvas.w == w && canvas.h == h {
return canvas.image
}
canvas.w, canvas.h = w, h
canvas.image = image.NewRGBA(image.Rect(0, 0, w, h))
return canvas.image
func Clear() {
wg = sync.WaitGroup{}
canvas1.get(0, 0)
canvas2.get(0, 0)
}

View file

@ -1,5 +1,3 @@
// This package contains functions for
// Pi/2 step rotation of points in a 2-dimensional space.
package image
type Angle uint
@ -11,7 +9,7 @@ const (
Angle270
)
// A helper to choose appropriate rotation by its angle
// Angles is a helper to choose appropriate rotation based on its angle.
var Angles = [4]Rotate{
Angle0: {Call: Rotate0, IsEven: false},
Angle90: {Call: Rotate90, IsEven: true},
@ -23,58 +21,58 @@ func GetRotation(angle Angle) Rotate {
return Angles[angle]
}
// An interface for rotation of a given point
// with the coordinates x, y in the matrix of w x h.
// Returns a pair of new coordinates x, y in the resulting
// matrix.
// Be aware that w / h values are 0 index-based and
// it's meant to be used with h corresponded
// Rotate is an interface for rotation of a given point.
//
// With the coordinates x, y in the matrix of w x h.
// Returns a pair of new coordinates x, y in the resulting matrix.
// Be aware that w / h values are 0 index-based,
// and it's meant to be used with h corresponded
// to matrix height and y coordinate, and with w to x coordinate.
type Rotate struct {
Call func(x, y, w, h int) (int, int)
IsEven bool
}
// 0° or the original orientation
/* Example: */
/* 1 2 3 1 2 3 */
/* 4 5 6 -> 4 5 6 */
/* 7 8 9 7 8 9 */
// Rotate0 is 0° or the original orientation.
//
// 1 2 3 1 2 3
// 4 5 6 -> 4 5 6
// 7 8 9 7 8 9
func Rotate0(x, y, _, _ int) (int, int) {
return x, y
}
// 90° CCW or 270° CW
/* Example: */
/* 1 2 3 3 6 9 */
/* 4 5 6 -> 2 5 8 */
/* 7 8 9 1 4 7 */
// Rotate90 is 90° CCW or 270° CW.
//
// 1 2 3 3 6 9
// 4 5 6 -> 2 5 8
// 7 8 9 1 4 7
func Rotate90(x, y, w, _ int) (int, int) {
return y, (w - 1) - x
}
// 180° CCW
/* Example: */
/* 1 2 3 9 8 7 */
/* 4 5 6 -> 6 5 4 */
/* 7 8 9 3 2 1 */
// Rotate180 is 180° CCW.
//
// 1 2 3 9 8 7
// 4 5 6 -> 6 5 4
// 7 8 9 3 2 1
func Rotate180(x, y, w, h int) (int, int) {
return (w - 1) - x, (h - 1) - y
}
// 270° CCW or 90° CW
/* Example: */
/* 1 2 3 7 4 1 */
/* 4 5 6 -> 8 5 2 */
/* 7 8 9 9 6 3 */
// Rotate270 is 270° CCW or 90° CW.
//
// 1 2 3 7 4 1
// 4 5 6 -> 8 5 2
// 7 8 9 9 6 3
func Rotate270(x, y, _, h int) (int, int) {
return (h - 1) - y, x
}
/*
[1 2 3 4 5 6 7 8 9]
[7 4 1 8 5 2 9 6 3]
*/
// ExampleRotate is an example of rotation usage.
//
// [1 2 3 4 5 6 7 8 9]
// [7 4 1 8 5 2 9 6 3]
func ExampleRotate(data []uint8, w int, h int, angle Angle) []uint8 {
dest := make([]uint8, len(data))
rotationFn := Angles[angle]

View file

@ -17,7 +17,6 @@ const (
func Resize(scaleType int, src *image.RGBA, out *image.RGBA) {
// !to do set it once instead switching on each iteration
// !to do skip resize if w=vw h=vh
switch scaleType {
case ScaleBilinear:
draw.ApproxBiLinear.Scale(out, out.Bounds(), src, src.Bounds(), draw.Src, nil)

View file

@ -69,6 +69,9 @@ type naEmulator struct {
// out frame size
vw, vh int
// draw threads
th int
players Players
done chan struct{}
@ -89,7 +92,7 @@ type GameFrame struct {
var NAEmulator *naEmulator
// NAEmulator implements CloudEmulator interface based on NanoArch(golang RetroArch)
func NewNAEmulator(roomID string, inputChannel <-chan InputEvent, storage Storage, conf config.LibretroCoreConfig) (*naEmulator, chan GameFrame, chan []int16) {
func NewNAEmulator(roomID string, inputChannel <-chan InputEvent, storage Storage, conf config.LibretroCoreConfig, threads int) (*naEmulator, chan GameFrame, chan []int16) {
imageChannel := make(chan GameFrame, 30)
audioChannel := make(chan []int16, 30)
@ -110,6 +113,7 @@ func NewNAEmulator(roomID string, inputChannel <-chan InputEvent, storage Storag
players: NewPlayerSessionInput(),
roomID: roomID,
done: make(chan struct{}, 1),
th: threads,
}, imageChannel, audioChannel
}
@ -142,8 +146,8 @@ func NewVideoExporter(roomID string, imgChannel chan GameFrame) *VideoExporter {
// Init initialize new RetroArch cloud emulator
// withImageChan returns an image stream as Channel for output else it will write to unix socket
func Init(roomID string, withImageChannel bool, inputChannel <-chan InputEvent, storage Storage, config config.LibretroCoreConfig) (*naEmulator, chan GameFrame, chan []int16) {
emu, imageChannel, audioChannel := NewNAEmulator(roomID, inputChannel, storage, config)
func Init(roomID string, withImageChannel bool, inputChannel <-chan InputEvent, storage Storage, config config.LibretroCoreConfig, threads int) (*naEmulator, chan GameFrame, chan []int16) {
emu, imageChannel, audioChannel := NewNAEmulator(roomID, inputChannel, storage, config, threads)
// Set to global NAEmulator
NAEmulator = emu
if !withImageChannel {
@ -189,7 +193,7 @@ func (na *naEmulator) Start() {
ticker := time.NewTicker(time.Second / time.Duration(na.meta.Fps))
defer ticker.Stop()
lastFrameTime = time.Now()
lastFrameTime = time.Now().UnixNano()
for {
na.Lock()

View file

@ -2,6 +2,7 @@ package nanoarch
import (
"bufio"
"fmt"
"log"
"os"
"os/user"
@ -57,8 +58,8 @@ void bridge_execute(void *f);
*/
import "C"
var mu, fmu sync.Mutex
var lastFrameTime time.Time
var mu sync.Mutex
var lastFrameTime int64
var video struct {
pitch uint32
@ -78,7 +79,7 @@ var video struct {
// default core pix format converter
var pixelFormatConverterFn = image.Rgb565
var rotationFn = image.GetRotation(image.Angle(0))
var rotationFn *image.Rotate
//const joypadNumKeys = int(C.RETRO_DEVICE_ID_JOYPAD_R3 + 1)
//var joy [joypadNumKeys]bool
@ -127,12 +128,6 @@ type CloudEmulator interface {
//export coreVideoRefresh
func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned, pitch C.size_t) {
t := time.Now()
fmu.Lock()
dt := t.Sub(lastFrameTime)
lastFrameTime = t
fmu.Unlock()
// some cores can return nothing
// !to add duplicate if can dup
if data == nil {
@ -157,7 +152,7 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned,
}
// the image is being resized and de-rotated
img := image.DrawRgbaImage(
frame := image.DrawRgbaImage(
pixelFormatConverterFn,
rotationFn,
image.ScaleNearestNeighbour,
@ -166,12 +161,15 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned,
data_,
NAEmulator.vw,
NAEmulator.vh,
NAEmulator.th,
)
// the image is pushed into a channel
// where it will be distributed with fan-out
t := time.Now().UnixNano()
dt := time.Duration(t - lastFrameTime)
lastFrameTime = t
select {
case NAEmulator.imageChannel <- GameFrame{Data: img, Duration: dt}:
case NAEmulator.imageChannel <- GameFrame{Data: frame, Duration: dt}:
default:
}
}
@ -211,13 +209,9 @@ func coreInputState(port C.unsigned, device C.unsigned, index C.unsigned, id C.u
}
func audioWrite(buf unsafe.Pointer, frames C.size_t) C.size_t {
// !to make it mono/stereo independent
samples := int(frames) * 2
pcm := (*[(1 << 30) - 1]int16)(buf)[:samples:samples]
samples := int(frames) << 1
pcm := (*[4096]int16)(buf)[:samples:samples]
p := make([]int16, samples)
// copy because pcm slice refer to buf underlying pointer,
// and buf pointer is the same in continuous frames
copy(p, pcm)
select {
@ -483,6 +477,8 @@ func slurp(path string, size int64) ([]byte, error) {
}
func coreLoadGame(filename string) {
lastFrameTime = 0
file, err := os.Open(filename)
if err != nil {
panic(err)
@ -513,7 +509,7 @@ func coreLoadGame(filename string) {
log.Printf(" block_extract: %v", bool(si.block_extract))
if !si.need_fullpath {
bytes, err := slurp(filename, size)
bytes, err := os.ReadFile(filename)
if err != nil {
panic(err)
}
@ -561,6 +557,9 @@ func coreLoadGame(filename string) {
video.baseWidth = int32(avi.geometry.base_width)
video.baseHeight = int32(avi.geometry.base_height)
if video.isGl {
bufS := int(video.maxWidth * video.maxHeight * int32(video.bpp))
graphics.SetBuffer(bufS)
log.Printf("Set buffer: %v", byteCountBinary(int64(bufS)))
if usesLibCo {
C.bridge_execute(C.initVideo_cgo)
} else {
@ -626,6 +625,7 @@ func nanoarchShutdown() {
for _, element := range coreConfig {
C.free(unsafe.Pointer(element))
}
image.Clear()
}
func nanoarchRun() {
@ -673,7 +673,25 @@ func setRotation(rotation uint) {
return
}
video.rotation = image.Angle(rotation)
rotationFn = image.GetRotation(video.rotation)
NAEmulator.meta.Rotation = rotationFn
r := image.GetRotation(video.rotation)
if rotation > 0 {
rotationFn = &r
} else {
rotationFn = nil
}
NAEmulator.meta.Rotation = r
log.Printf("[Env]: the game video is rotated %v°", map[uint]uint{0: 0, 1: 90, 2: 180, 3: 270}[rotation])
}
func byteCountBinary(b int64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}

View file

@ -3,7 +3,7 @@ package nanoarch
import (
"crypto/md5"
"fmt"
"io/ioutil"
"io"
"log"
"os"
"path"
@ -85,6 +85,7 @@ func GetEmulatorMock(room string, system string) *EmulatorMock {
players: NewPlayerSessionInput(),
roomID: room,
done: make(chan struct{}, 1),
th: conf.Emulator.Threads,
},
core: path.Base(meta.Lib),
@ -175,7 +176,7 @@ func (emu *EmulatorMock) handleInput(handler func(event InputEvent)) {
// Locks the emulator.
func (emu *EmulatorMock) dumpState() (string, string) {
emu.Lock()
bytes, _ := ioutil.ReadFile(emu.paths.save)
bytes, _ := os.ReadFile(emu.paths.save)
persistedStateHash := getHash(bytes)
emu.Unlock()
@ -213,7 +214,7 @@ func cleanPath(path string) string {
// benchmarkEmulator is a generic function for
// measuring emulator performance for one emulation frame.
func benchmarkEmulator(system string, rom string, b *testing.B) {
log.SetOutput(ioutil.Discard)
log.SetOutput(io.Discard)
os.Stdout, _ = os.Open(os.DevNull)
s := GetDefaultEmulatorMock("bench_"+system+"_performance", system, rom)

View file

@ -18,7 +18,7 @@ func ResampleStretch(pcm []int16, size int) []int16 {
l[i] = l[i-1]
}
}
for i := 0; i < size-1; i += 2 {
for i := 0; i < size; i += 2 {
audio[i], audio[i+1] = r[i/2], l[i/2]
}
return audio

View file

@ -170,16 +170,22 @@ func NewRoom(roomID string, game games.GameMetadata, recUser string, rec bool, o
emuName := cfg.Emulator.GetEmulator(game.Type, game.Path)
libretroConfig := cfg.Emulator.GetLibretroCoreConfig(emuName)
th := cfg.Emulator.Threads
if th == 0 {
th = 1
}
log.Printf("Image processing threads = %v", th)
if cfg.Encoder.WithoutGame {
// Run without game, image stream is communicated over a unix socket
imageChannel := NewVideoImporter(roomID)
director, _, audioChannel := nanoarch.Init(roomID, false, inputChannel, store, libretroConfig)
director, _, audioChannel := nanoarch.Init(roomID, false, inputChannel, store, libretroConfig, th)
room.imageChannel = imageChannel
room.director = director
room.audioChannel = audioChannel
} else {
// Run without game, image stream is communicated over image channel
director, imageChannel, audioChannel := nanoarch.Init(roomID, true, inputChannel, store, libretroConfig)
director, imageChannel, audioChannel := nanoarch.Init(roomID, true, inputChannel, store, libretroConfig, th)
room.imageChannel = imageChannel
room.director = director
room.audioChannel = audioChannel