Add initial support of portrait games (#186)

* Add initial support of portrait games

* Add benchmarks to angle selector

* Remove map from angles
This commit is contained in:
sergystepanov 2020-05-25 13:36:07 +03:00 committed by GitHub
parent c8e6cfcb64
commit a007157e76
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 468 additions and 146 deletions

View file

@ -2,6 +2,7 @@ package config
import (
"flag"
"github.com/giongto35/cloud-game/pkg/emulator/libretro/image"
"time"
)
@ -50,6 +51,7 @@ type EmulatorMeta struct {
BaseWidth int
BaseHeight int
Ratio float64
Rotation image.Rotate
}
var EmulatorConfig = map[string]EmulatorMeta{

View file

@ -15,7 +15,7 @@ const (
type Format func(data []byte, index int) color.RGBA
func rgb565(data []byte, index int) color.RGBA {
func Rgb565(data []byte, index int) color.RGBA {
pixel := (int)(data[index]) + ((int)(data[index+1]) << 8)
return color.RGBA{
@ -26,7 +26,7 @@ func rgb565(data []byte, index int) color.RGBA {
}
}
func rgba8888(data []byte, index int) color.RGBA {
func Rgba8888(data []byte, index int) color.RGBA {
return color.RGBA{
R: data[index+2],
G: data[index+1],

View file

@ -4,15 +4,57 @@ import (
"image"
)
func DrawRgbaImage(pixFormat int, scaleType int, w int, h int, packedW int, vw int, vh int, bpp int, data []byte, image *image.RGBA) {
switch pixFormat {
case BIT_FORMAT_SHORT_5_6_5:
Resize(scaleType, rgb565, w, h, packedW, vw, vh, bpp, data, image)
case BIT_FORMAT_INT_8_8_8_8_REV:
Resize(scaleType, rgba8888, w, h, packedW, vw, vh, bpp, data, image)
case BIT_FORMAT_SHORT_5_5_5_1:
fallthrough
default:
image = nil
type imageCache struct {
image *image.RGBA
w int
h int
}
var canvas = imageCache{
image.NewRGBA(image.Rectangle{}),
0,
0,
}
func DrawRgbaImage(pixFormat Format, rotationFn Rotate, scaleType, w, h, packedW, bpp int, data []byte, dest *image.RGBA) {
if pixFormat == nil {
dest = nil
}
// !to implement own image interfaces img.Pix = bytes[]
ww, hh := w, h
if rotationFn.IsEven {
ww, hh = hh, ww
}
src := getCanvas(ww, hh)
drawImage(pixFormat, w, h, packedW, bpp, rotationFn, data, src)
Resize(scaleType, src, dest)
}
func drawImage(toRGBA Format, w, h, packedW, bpp int, rotationFn Rotate, data []byte, image *image.RGBA) {
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
src := toRGBA(data, (x+y*packedW)*bpp)
dx, dy := rotationFn.Call(x, y, w, h)
i := dx*4 + dy*image.Stride
dst := image.Pix[i : i+4 : i+4]
dst[0] = src.R
dst[1] = src.G
dst[2] = src.B
dst[3] = src.A
}
}
}
func getCanvas(w, h int) *image.RGBA {
if canvas.w == w && canvas.h == h {
return canvas.image
}
canvas.w, canvas.h = w, h
canvas.image = image.NewRGBA(image.Rect(0, 0, w, h))
return canvas.image
}

View file

@ -0,0 +1,96 @@
// This package contains functions for
// Pi/2 step rotation of points in a 2-dimensional space.
package image
type Angle uint
const (
Angle0 Angle = iota
Angle90
Angle180
Angle270
)
// A helper to choose appropriate rotation by its angle
var Angles = [4]Rotate{
Angle0: {Call: Rotate0, IsEven: false},
Angle90: {Call: Rotate90, IsEven: true},
Angle180: {Call: Rotate180, IsEven: false},
Angle270: {Call: Rotate270, IsEven: true},
}
func GetRotation(angle Angle) Rotate {
return Angles[angle]
}
// An interface for rotation of a given point
// with the coordinates x, y in the matrix of w x h.
// Returns a pair of new coordinates x, y in the resulting
// matrix.
// Be aware that w / h values are 0 index-based and
// it's meant to be used with h corresponded
// to matrix height and y coordinate, and with w to x coordinate.
type Rotate struct {
Call func(x, y, w, h int) (int, int)
IsEven bool
}
// 0° or the original orientation
/* Example: */
/* 1 2 3 1 2 3 */
/* 4 5 6 -> 4 5 6 */
/* 7 8 9 7 8 9 */
func Rotate0(x, y, _, _ int) (int, int) {
return x, y
}
// 90° CCW or 270° CW
/* Example: */
/* 1 2 3 3 6 9 */
/* 4 5 6 -> 2 5 8 */
/* 7 8 9 1 4 7 */
func Rotate90(x, y, w, _ int) (int, int) {
return y, (w - 1) - x
}
// 180° CCW
/* Example: */
/* 1 2 3 9 8 7 */
/* 4 5 6 -> 6 5 4 */
/* 7 8 9 3 2 1 */
func Rotate180(x, y, w, h int) (int, int) {
return (w - 1) - x, (h - 1) - y
}
// 270° CCW or 90° CW
/* Example: */
/* 1 2 3 7 4 1 */
/* 4 5 6 -> 8 5 2 */
/* 7 8 9 9 6 3 */
func Rotate270(x, y, _, h int) (int, int) {
return (h - 1) - y, x
}
/*
[1 2 3 4 5 6 7 8 9]
[7 4 1 8 5 2 9 6 3]
*/
func ExampleRotate(data []uint8, w int, h int, angle Angle) []uint8 {
dest := make([]uint8, len(data))
rotationFn := Angles[angle]
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
nx, ny := rotationFn.Call(x, y, w, h)
stride := w
if rotationFn.IsEven {
stride = h
}
//fmt.Printf("%v:%v (%v) -> %v:%v (%v)\n", x, y, n1, nx, ny, n2)
dest[nx+ny*stride] = data[x+y*w]
}
}
return dest
}

View file

@ -0,0 +1,254 @@
package image
import (
"bytes"
"testing"
)
type dimensions struct {
w int
h int
}
func TestRotate(t *testing.T) {
tests := []struct {
// packed bytes from a 2D matrix
input []byte
// original matrix's width
w int
// original matrix's height
h int
// rotation algorithm
rotateHow []Angle
expected [][]byte
}{
{
// a cross
[]byte{
0, 1, 0,
1, 1, 1,
0, 1, 0,
},
3, 3, []Angle{Angle0, Angle90, Angle180, Angle270},
[][]byte{
{
0, 1, 0,
1, 1, 1,
0, 1, 0,
},
{
0, 1, 0,
1, 1, 1,
0, 1, 0,
},
{
0, 1, 0,
1, 1, 1,
0, 1, 0,
},
{
0, 1, 0,
1, 1, 1,
0, 1, 0,
},
},
},
{
[]byte{
1, 2,
3, 4,
5, 6,
7, 8,
},
2, 4, []Angle{Angle0, Angle90, Angle180, Angle270},
[][]byte{
{
1, 2,
3, 4,
5, 6,
7, 8,
},
{
2, 4, 6, 8,
1, 3, 5, 7,
},
{
8, 7,
6, 5,
4, 3,
2, 1,
},
{
7, 5, 3, 1,
8, 6, 4, 2,
},
},
},
{
// a square
[]byte{
1, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 1, 0, 0, 0, 0, 1, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 1,
},
8, 6, []Angle{Angle0, Angle90, Angle180, Angle270},
[][]byte{
{
// L // R
1, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 1, 0, 0, 0, 0, 1, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 1,
},
{
0, 0, 0, 0, 0, 1,
0, 1, 1, 1, 1, 0,
0, 1, 1, 0, 1, 0,
0, 1, 1, 0, 1, 0,
0, 1, 1, 0, 1, 0,
0, 1, 1, 0, 1, 0,
0, 1, 1, 1, 1, 0,
1, 0, 0, 0, 0, 0,
},
{
1, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 1, 0, 0, 0, 0, 1, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 1, 1, 1, 1, 1, 1, 0,
0, 0, 0, 0, 0, 0, 0, 1,
},
{
0, 0, 0, 0, 0, 1,
0, 1, 1, 1, 1, 0,
0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0,
0, 1, 1, 1, 1, 0,
1, 0, 0, 0, 0, 0,
},
},
},
}
for _, test := range tests {
for i, rot := range test.rotateHow {
if output := ExampleRotate(test.input, test.w, test.h, rot); bytes.Compare(output, test.expected[i]) != 0 {
t.Errorf(
"Test fail for angle %v with %v that should be \n%v but it's \n%v",
rot, test.input, test.expected[i], output)
}
}
}
}
func TestBoundsAfterRotation(t *testing.T) {
tests := []struct {
dim []dimensions
rotateHow []Angle
}{
{
// a combinatorics lib would be nice instead
[]dimensions{
// square
{w: 100, h: 100},
// even w/h
{w: 100, h: 50},
// even h/w
{w: 50, h: 100},
// odd even w/h
{w: 77, h: 32},
// even odd h/w
{w: 32, h: 77},
// just odd
{w: 13, h: 19},
},
[]Angle{Angle0, Angle90, Angle180, Angle270},
},
}
for _, test := range tests {
for _, rot := range test.rotateHow {
rotationFn := Angles[rot]
for _, dim := range test.dim {
for y := 0; y < dim.h; y++ {
for x := 0; x < dim.w; x++ {
xx, yy := rotationFn.Call(x, y, dim.w, dim.h)
if rotationFn.IsEven {
yy, xx = xx, yy
}
if xx < 0 || xx > dim.w {
t.Errorf("Rot %v, coordinate x should be in range [0; %v]: %v", rot, dim.w-1, xx)
}
if yy < 0 || yy > dim.h {
t.Errorf("Rot %v, coordinate y should be in range [0; %v]: %v", rot, dim.h-1, yy)
}
}
}
}
}
}
}
func BenchmarkDirect(b *testing.B) {
for i := 0; i < b.N; i++ {
p1, p2 := Rotate90(1, 1, 2, 2)
p1, p2 = p2, p1
}
}
func BenchmarkLiteral(b *testing.B) {
fn := Rotate90
for i := 0; i < b.N; i++ {
p1, p2 := fn(1, 1, 2, 2)
p1, p2 = p2, p1
}
}
func BenchmarkAssign(b *testing.B) {
fn := Angles[Angle90].Call
for i := 0; i < b.N; i++ {
p1, p2 := fn(1, 1, 2, 2)
p1, p2 = p2, p1
}
}
func BenchmarkMapReassign(b *testing.B) {
fn := Angles[Angle90].Call
for i := 0; i < b.N; i++ {
fn2 := fn
p1, p2 := fn2(1, 1, 2, 2)
p1, p2 = p2, p1
}
}
func BenchmarkMapDirect(b *testing.B) {
for i := 0; i < b.N; i++ {
p1, p2 := Angles[Angle90].Call(1, 1, 2, 2)
p1, p2 = p2, p1
}
}
func BenchmarkNewMapDirect(b *testing.B) {
fns := map[Angle]func(x, y, w, h int) (int, int){
Angle90: Rotate90,
}
for i := 0; i < b.N; i++ {
p1, p2 := fns[Angle90](1, 1, 2, 2)
p1, p2 = p2, p1
}
}

View file

@ -1,135 +1,30 @@
package image
import (
"image"
"image/color"
"golang.org/x/image/draw"
"image"
)
const (
// skips image interpolation
ScaleSkip = -1
// initial image interpolation algorithm
ScaleOld = 0
ScaleNot = iota
// nearest neighbour interpolation
ScaleNearestNeighbour = 1
ScaleNearestNeighbour
// bilinear interpolation
ScaleBilinear = 2
ScaleBilinear
)
func Resize(scaleType int, fn Format, w int, h int, packedW int, vw int, vh int, bpp int, data []byte, out *image.RGBA) {
// !to implement own image interfaces img.Pix = bytes[]
src := image.NewRGBA(image.Rect(0, 0, w, h))
toRgba(fn, w, h, packedW, bpp, data, src)
func Resize(scaleType int, src *image.RGBA, out *image.RGBA) {
// !to do set it once instead switching on each iteration
// !to do skip resize if w=vw h=vh
switch scaleType {
case ScaleSkip:
skip(fn, w, h, packedW, vw, vh, bpp, data, src, out)
case ScaleNearestNeighbour:
draw.NearestNeighbor.Scale(out, out.Bounds(), src, src.Bounds(), draw.Src, nil)
//nearest(fn, w, h, packedW, vw, vh, bpp, data, src, out)
case ScaleBilinear:
draw.ApproxBiLinear.Scale(out, out.Bounds(), src, src.Bounds(), draw.Src, nil)
//bilinear(fn, w, h, packedW, vw, vh, bpp, data, src, out)
case ScaleOld:
case ScaleNot:
fallthrough
case ScaleNearestNeighbour:
fallthrough
default:
old(fn, w, h, packedW, vw, vh, bpp, data, src, out)
}
}
func old(fn Format, w int, h int, packedW int, vw int, vh int, bpp int, data []byte, _ *image.RGBA, out *image.RGBA) {
seek := 0
scaleWidth := float64(vw) / float64(w)
scaleHeight := float64(vh) / float64(h)
for y := 0; y < h; y++ {
y2 := int(float64(y) * scaleHeight)
for x := 0; x < packedW; x++ {
x2 := int(float64(x) * scaleWidth)
if x2 < vw {
out.Set(x2, y2, fn(data, seek))
}
seek += bpp
}
}
}
func skip(fn Format, w int, h int, packedW int, _ int, _ int, bpp int, data []byte, _ *image.RGBA, out *image.RGBA) {
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
index := (y * packedW) + x
index *= bpp
out.Set(x, y, fn(data, index))
}
}
}
func nearest(fn Format, w int, h int, packedW int, vw int, vh int, bpp int, data []byte, _ *image.RGBA, out *image.RGBA) {
xRatio := ((w << 16) / vw) + 1
yRatio := ((h << 16) / vh) + 1
for y := 0; y < vh; y++ {
y2 := (y * yRatio) >> 16
for x := 0; x < vw; x++ {
x2 := (x * xRatio) >> 16
index := (y2 * packedW) + x2
index *= bpp
out.Set(x, y, fn(data, index))
}
}
}
// This implementation has some color bleeding issues
func bilinear(fn Format, w int, h int, packedW int, vw int, vh int, bpp int, data []byte, _ *image.RGBA, out *image.RGBA) {
xRatio := float64(w-1) / float64(vw)
yRatio := float64(h-1) / float64(vh)
for y := 0; y < vh; y++ {
y2 := int(yRatio * float64(y))
for x := 0; x < vw; x++ {
x2 := int(xRatio * float64(x))
w := (xRatio * float64(x)) - float64(x2)
h := (yRatio * float64(y)) - float64(y2)
index := (y2 * packedW) + x2
a := fn(data, index*bpp)
b := fn(data, (index+1)*bpp)
c := fn(data, (index+packedW)*bpp)
d := fn(data, (index+packedW+1)*bpp)
out.Set(x, y, color.RGBA{
// don't sink the boat
R: byte(float64(a.R)*(1-w)*(1-h) + float64(b.R)*w*(1-h) + float64(c.R)*h*(1-w) + float64(d.R)*w*h),
G: byte(float64(a.G)*(1-w)*(1-h) + float64(b.G)*w*(1-h) + float64(c.G)*h*(1-w) + float64(d.G)*w*h),
B: byte(float64(a.B)*(1-w)*(1-h) + float64(b.B)*w*(1-h) + float64(c.B)*h*(1-w) + float64(d.B)*w*h),
//A: byte(float64(a.A)*(1-w)*(1-h) + float64(b.A)*w*(1-h) + float64(c.A)*h*(1-w) + float64(d.A)*w*h),
})
}
}
}
func toRgba(fn Format, w int, h int, packedW int, bpp int, data []byte, image *image.RGBA) {
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
index := (y*packedW + x) * bpp
c := fn(data, index)
i := (y-image.Rect.Min.Y)*image.Stride + (x-image.Rect.Min.X)*4
s := image.Pix[i : i+4 : i+4]
s[0] = c.R
s[1] = c.G
s[2] = c.B
s[3] = c.A
}
draw.NearestNeighbor.Scale(out, out.Bounds(), src, src.Bounds(), draw.Src, nil)
}
}

View file

@ -135,9 +135,6 @@ func (na *naEmulator) LoadMeta(path string) config.EmulatorMeta {
func (na *naEmulator) SetViewport(width int, height int) {
// outputImg is tmp img used for decoding and reuse in encoding flow
outputImg = image.NewRGBA(image.Rect(0, 0, width, height))
ewidth = width
eheight = height
}
func (na *naEmulator) Start() {

View file

@ -55,19 +55,23 @@ import "C"
var mu sync.Mutex
var video struct {
program uint32
vao uint32
pitch uint32
pixFmt uint32
pixType uint32
bpp uint32
program uint32
vao uint32
pitch uint32
pixFmt uint32
pixType uint32
bpp uint32
rotation image.Angle
}
// default core pix format converter
var pixelFormatConverterFn = image.Rgb565
var rotationFn = image.GetRotation(image.Angle(0))
const bufSize = 1024 * 4
const joypadNumKeys = int(C.RETRO_DEVICE_ID_JOYPAD_R3 + 1)
var joy [joypadNumKeys]bool
var ewidth, eheight int
var bindKeysMap = map[int]int{
C.RETRO_DEVICE_ID_JOYPAD_A: 0,
@ -106,10 +110,18 @@ func coreVideoRefresh(data unsafe.Pointer, width C.unsigned, height C.unsigned,
bytes := int(height) * packedWidth * int(video.bpp)
data_ := (*[1 << 30]byte)(data)[:bytes:bytes]
// image is resized here and push to channel. On the other side, images will be fan out
image.DrawRgbaImage(int(video.pixFmt), image.ScaleNearestNeighbour, int(width), int(height),
packedWidth, ewidth, eheight, int(video.bpp), data_, outputImg)
// the image is being resized and de-rotated
image.DrawRgbaImage(
pixelFormatConverterFn,
rotationFn,
image.ScaleNearestNeighbour,
int(width), int(height), packedWidth, int(video.bpp),
data_,
outputImg,
)
// the image is pushed into a channel
// where it will be distributed with fan-out
NAEmulator.imageChannel <- outputImg
}
@ -208,10 +220,14 @@ func coreEnvironment(cmd C.unsigned, data unsafe.Pointer) C.bool {
case C.RETRO_ENVIRONMENT_SHUTDOWN:
//window.SetShouldClose(true)
return true
case C.RETRO_ENVIRONMENT_GET_VARIABLE:
variable := (*C.struct_retro_variable)(data)
fmt.Println("[Env]: get variable:", C.GoString(variable.key))
return false
/*
Sets screen rotation of graphics.
Valid values are 0, 1, 2, 3, which rotates screen by 0, 90, 180, 270 degrees
ccw respectively.
*/
case C.RETRO_ENVIRONMENT_SET_ROTATION:
setRotation(*(*int)(data) % 4)
return true
default:
//fmt.Println("[Env]: command not implemented", cmd)
return false
@ -421,6 +437,8 @@ func unserialize(bytes []byte, size uint) error {
func nanoarchShutdown() {
C.bridge_retro_unload_game(retroUnloadGame)
C.bridge_retro_deinit(retroDeinit)
setRotation(0)
}
func nanoarchRun() {
@ -432,14 +450,18 @@ func videoSetPixelFormat(format uint32) C.bool {
case C.RETRO_PIXEL_FORMAT_0RGB1555:
video.pixFmt = image.BIT_FORMAT_SHORT_5_5_5_1
video.bpp = 2
// format is not implemented
pixelFormatConverterFn = nil
break
case C.RETRO_PIXEL_FORMAT_XRGB8888:
video.pixFmt = image.BIT_FORMAT_INT_8_8_8_8_REV
video.bpp = 4
pixelFormatConverterFn = image.Rgba8888
break
case C.RETRO_PIXEL_FORMAT_RGB565:
video.pixFmt = image.BIT_FORMAT_SHORT_5_6_5
video.bpp = 2
pixelFormatConverterFn = image.Rgb565
break
default:
log.Fatalf("Unknown pixel type %v", format)
@ -448,3 +470,10 @@ func videoSetPixelFormat(format uint32) C.bool {
fmt.Printf("Video pixel: %v %v %v %v %v", video, format, C.RETRO_PIXEL_FORMAT_0RGB1555, C.RETRO_PIXEL_FORMAT_XRGB8888, C.RETRO_PIXEL_FORMAT_RGB565)
return true
}
func setRotation(rotation int) {
video.rotation = image.Angle(rotation)
rotationFn = image.GetRotation(video.rotation)
NAEmulator.meta.Rotation = rotationFn
log.Printf("[Env]: the game video is rotated %v°", map[int]int{0: 0, 1: 90, 2: 180, 3: 270}[rotation])
}

View file

@ -131,12 +131,19 @@ func NewRoom(roomID string, gameName string, videoEncoderType string, onlineStor
log.Printf("Viewport size has scaled to %dx%d", nwidth, nheight)
}
room.director.SetViewport(nwidth, nheight)
log.Println("meta: ", gameMeta)
// set resulting game frame size considering
// its orientation
encoderW, encoderH := nwidth, nheight
if gameMeta.Rotation.IsEven {
encoderW, encoderH = nheight, nwidth
}
room.director.SetViewport(encoderW, encoderH)
// Spawn video and audio encoding for webRTC
go room.startVideo(nwidth, nheight, videoEncoderType)
go room.startVideo(encoderW, encoderH, videoEncoderType)
go room.startAudio(gameMeta.AudioSampleRate)
go room.startVoice()
room.director.Start()