Mirrors/cloud-game
1
0
Fork 0
mirror of https://github.com/giongto35/cloud-game.git synced 2026-07-23 01:57:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #19 from giongto35/audio2

Browse source 
Support Audio over data channel
This commit is contained in:
giongto35 2019-04-23 21:05:57 +02:00 • committed by GitHub
commit cdc5980b13
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 551 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile vendored
View file

@ -10,7 +10,7 @@ RUN apt-get update
#RUN go get github.com/gordonklaus/portaudio
#RUN apt-get install portaudio19-dev -y
RUN apt-get install pkg-config libvpx-dev -y
RUN apt-get install pkg-config libvpx-dev libopus-dev libopusfile-dev -y
RUN go get github.com/pion/webrtc
RUN go get github.com/gorilla/websocket
RUN go get github.com/satori/go.uuid

78
main.go
View file

@ -21,6 +21,8 @@ import (
"github.com/gorilla/websocket"
pionRTC "github.com/pion/webrtc"
uuid "github.com/satori/go.uuid"
"gopkg.in/hraban/opus.v2"
)
const (
@ -45,6 +47,7 @@ var upgrader = websocket.Upgrader{}
// A room stores all the channel for interaction between all webRTCs session and emulator
type Room struct {
imageChannel chan *image.RGBA
audioChannel chan float32
inputChannel chan int
// Done channel is to fire exit event when there is no webRTC session running
Done chan struct{}
@ -63,8 +66,6 @@ var serverID = ""
var oclient *Client
func main() {
rand.Seed(time.Now().UTC().UnixNano())
flag.Parse()
log.Println("Usage: ./game [debug]")
if *config.IsDebug {
@ -133,13 +134,15 @@ func initRoom(roomID, gameName string) string {
}
log.Println("Init new room", roomID)
imageChannel := make(chan *image.RGBA, 100)
audioChannel := make(chan float32, ui.SampleRate)
inputChannel := make(chan int, 100)
// create director
director := ui.NewDirector(roomID, imageChannel, inputChannel)
director := ui.NewDirector(roomID, imageChannel, audioChannel, inputChannel)
room := &Room{
imageChannel: imageChannel,
audioChannel: audioChannel,
inputChannel: inputChannel,
rtcSessions: []*webrtc.WebRTC{},
sessionsLock: &sync.Mutex{},
@ -148,7 +151,8 @@ func initRoom(roomID, gameName string) string {
}
rooms[roomID] = room
go room.start()
go room.startVideo()
go room.startAudio()
go director.Start([]string{"games/" + gameName})
return roomID
@ -332,7 +336,7 @@ func generateRoomID() string {
return roomID
}
func (r *Room) start() {
func (r *Room) startVideo() {
// fanout Screen
for {
select {
@ -363,6 +367,68 @@ func (r *Room) start() {
}
}
func (r *Room) startAudio() {
log.Println("Enter fan audio")
enc, err := opus.NewEncoder(ui.SampleRate, ui.Channels, opus.AppAudio)
maxBufferSize := ui.TimeFrame * ui.SampleRate / 1000
pcm := make([]float32, maxBufferSize) // 640 * 1000 / 16000 == 40 ms
idx := 0
if err != nil {
log.Println("[!] Cannot create audio encoder")
return
}
var count byte = 0
// fanout Audio
for {
select {
case <-r.Done:
r.remove()
return
case sample := <-r.audioChannel:
pcm[idx] = sample
idx ++
if idx == len(pcm) {
data := make([]byte, 640)
n, err := enc.EncodeFloat32(pcm, data)
if err != nil {
log.Println("[!] Failed to decode")
continue
}
data = data[:n]
data = append(data, count)
r.sessionsLock.Lock()
for _, webRTC := range r.rtcSessions {
// Client stopped
if webRTC.IsClosed() {
continue
}
// encode frame
// fanout audioChannel
if webRTC.IsConnected() {
// NOTE: can block here
webRTC.AudioChannel <- data
}
//isRoomRunning = true
}
r.sessionsLock.Unlock()
idx = 0
count = (count + 1) & 0xff
}
}
}
}
func (r *Room) remove() {
log.Println("Closing room", r)
r.director.Done <- struct{}{}
@ -543,4 +609,4 @@ func NewOverlordClient(oc *websocket.Conn) *Client {
go oclient.listen()
return oclient
}
}

11
static/index_ws.html vendored
View file

@ -10,7 +10,8 @@ textarea {
<select id="gameOp">
</select>
<button id="play" onclick="window.startGame()">Play</button><br/><br/>
<button id="play" onclick="window.startGame()">Play</button>
<button id="play" onclick="pc.close()">Stop</button>
Your current room: <b><label id="currentRoomID" style="color:blue"></b> <br />
You can join a remote game by roomID.<br />
Room ID: <input type="text" id="roomID">
@ -23,7 +24,7 @@ Play as player(1,2): <select id="playerIndex">
<body scroll="no" style="overflow: hidden">
<div id="remoteVideos">
<video id="game-screen" autoplay=true width=400 height=300 poster="https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif" style="display: none">
<video id="game-screen" autoplay=true width=400 height=300 poster="https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif">
</div> <br />
<h3>Instruction</h3>
@ -51,6 +52,10 @@ Play as player(1,2): <select id="playerIndex">
DEBUG = true;
</script>
<!-- https://rawgit.com/Rillke/opus.js-sample/master/index.xhtml -->
<script src="/static/js/libopus.js"></script>
<script src="/static/js/opus.js"></script>
<script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
<script src="/static/js/const.js"></script>
<script src="/static/js/global.js"></script>
@ -68,6 +73,8 @@ Play as player(1,2): <select id="playerIndex">
$("#gameOp").on("change", function() {
gameIdx = gameOp.selectedIndex;
});
gameIdx = 7;
</script>
</html>

31
static/js/libopus.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

200
static/js/opus.js vendored Normal file
View file

@ -0,0 +1,200 @@
///<reference path="d.ts/asm.d.ts" />
///<reference path="d.ts/libopus.d.ts" />
var OpusApplication;
(function (OpusApplication) {
OpusApplication[OpusApplication["VoIP"] = 2048] = "VoIP";
OpusApplication[OpusApplication["Audio"] = 2049] = "Audio";
OpusApplication[OpusApplication["RestrictedLowDelay"] = 2051] = "RestrictedLowDelay";
})(OpusApplication || (OpusApplication = {}));
var OpusError;
(function (OpusError) {
OpusError[OpusError["OK"] = 0] = "OK";
OpusError[OpusError["BadArgument"] = -1] = "BadArgument";
OpusError[OpusError["BufferTooSmall"] = -2] = "BufferTooSmall";
OpusError[OpusError["InternalError"] = -3] = "InternalError";
OpusError[OpusError["InvalidPacket"] = -4] = "InvalidPacket";
OpusError[OpusError["Unimplemented"] = -5] = "Unimplemented";
OpusError[OpusError["InvalidState"] = -6] = "InvalidState";
OpusError[OpusError["AllocFail"] = -7] = "AllocFail";
})(OpusError || (OpusError = {}));
var Opus = (function () {
function Opus() {
}
Opus.getVersion = function () {
var ptr = _opus_get_version_string();
return Pointer_stringify(ptr);
};
Opus.getMaxFrameSize = function (numberOfStreams) {
if (numberOfStreams === void 0) { numberOfStreams = 1; }
return (1275 * 3 + 7) * numberOfStreams;
};
Opus.getMinFrameDuration = function () {
return 2.5;
};
Opus.getMaxFrameDuration = function () {
return 60;
};
Opus.validFrameDuration = function (x) {
return [2.5, 5, 10, 20, 40, 60].some(function (element) {
return element == x;
});
};
Opus.getMaxSamplesPerChannel = function (sampling_rate) {
return sampling_rate / 1000 * Opus.getMaxFrameDuration();
};
return Opus;
})();
var OpusEncoder = (function () {
function OpusEncoder(sampling_rate, channels, app, frame_duration) {
if (frame_duration === void 0) { frame_duration = 20; }
this.handle = 0;
this.frame_size = 0;
this.in_ptr = 0;
this.in_off = 0;
this.out_ptr = 0;
if (!Opus.validFrameDuration(frame_duration))
throw 'invalid frame duration';
this.frame_size = sampling_rate * frame_duration / 1000;
var err_ptr = allocate(4, 'i32', ALLOC_STACK);
this.handle = _opus_encoder_create(sampling_rate, channels, app, err_ptr);
if (getValue(err_ptr, 'i32') != 0 /* OK */)
throw 'opus_encoder_create failed: ' + getValue(err_ptr, 'i32');
this.in_ptr = _malloc(this.frame_size * channels * 4);
this.in_len = this.frame_size * channels;
this.in_i16 = HEAP16.subarray(this.in_ptr >> 1, (this.in_ptr >> 1) + this.in_len);
this.in_f32 = HEAPF32.subarray(this.in_ptr >> 2, (this.in_ptr >> 2) + this.in_len);
this.out_bytes = Opus.getMaxFrameSize();
this.out_ptr = _malloc(this.out_bytes);
this.out_buf = HEAPU8.subarray(this.out_ptr, this.out_ptr + this.out_bytes);
}
OpusEncoder.prototype.encode = function (pcm) {
var output = [];
var pcm_off = 0;
while (pcm.length - pcm_off >= this.in_len - this.in_off) {
if (this.in_off > 0) {
this.in_i16.set(pcm.subarray(pcm_off, pcm_off + this.in_len - this.in_off), this.in_off);
pcm_off += this.in_len - this.in_off;
this.in_off = 0;
}
else {
this.in_i16.set(pcm.subarray(pcm_off, pcm_off + this.in_len));
pcm_off += this.in_len;
}
var ret = _opus_encode(this.handle, this.in_ptr, this.frame_size, this.out_ptr, this.out_bytes);
if (ret <= 0)
throw 'opus_encode failed: ' + ret;
var packet = new ArrayBuffer(ret);
new Uint8Array(packet).set(this.out_buf.subarray(0, ret));
output.push(packet);
}
if (pcm_off < pcm.length) {
this.in_i16.set(pcm.subarray(pcm_off));
this.in_off = pcm.length - pcm_off;
}
return output;
};
OpusEncoder.prototype.encode_float = function (pcm) {
var output = [];
var pcm_off = 0;
while (pcm.length - pcm_off >= this.in_len - this.in_off) {
if (this.in_off > 0) {
this.in_f32.set(pcm.subarray(pcm_off, pcm_off + this.in_len - this.in_off), this.in_off);
pcm_off += this.in_len - this.in_off;
this.in_off = 0;
}
else {
this.in_f32.set(pcm.subarray(pcm_off, pcm_off + this.in_len));
pcm_off += this.in_len;
}
var ret = _opus_encode_float(this.handle, this.in_ptr, this.frame_size, this.out_ptr, this.out_bytes);
if (ret <= 0)
throw 'opus_encode failed: ' + ret;
var packet = new ArrayBuffer(ret);
new Uint8Array(packet).set(this.out_buf.subarray(0, ret));
output.push(packet);
}
if (pcm_off < pcm.length) {
this.in_f32.set(pcm.subarray(pcm_off));
this.in_off = pcm.length - pcm_off;
}
return output;
};
OpusEncoder.prototype.encode_final = function () {
if (this.in_off == 0)
return new ArrayBuffer(0);
for (var i = this.in_off; i < this.in_len; ++i)
this.in_i16[i] = 0;
var ret = _opus_encode(this.handle, this.in_ptr, this.frame_size, this.out_ptr, this.out_bytes);
if (ret <= 0)
throw 'opus_encode failed: ' + ret;
var packet = new ArrayBuffer(ret);
new Uint8Array(packet).set(this.out_buf.subarray(0, ret));
return packet;
};
OpusEncoder.prototype.encode_float_final = function () {
if (this.in_off == 0)
return new ArrayBuffer(0);
for (var i = this.in_off; i < this.in_len; ++i)
this.in_f32[i] = 0;
var ret = _opus_encode_float(this.handle, this.in_ptr, this.frame_size, this.out_ptr, this.out_bytes);
if (ret <= 0)
throw 'opus_encode failed: ' + ret;
var packet = new ArrayBuffer(ret);
new Uint8Array(packet).set(this.out_buf.subarray(0, ret));
return packet;
};
OpusEncoder.prototype.destroy = function () {
if (!this.handle)
return;
_opus_encoder_destroy(this.handle);
_free(this.in_ptr);
this.handle = this.in_ptr = 0;
};
return OpusEncoder;
})();
var OpusDecoder = (function () {
function OpusDecoder(sampling_rate, channels) {
this.handle = 0;
this.in_ptr = 0;
this.out_ptr = 0;
this.channels = channels;
var err_ptr = allocate(4, 'i32', ALLOC_STACK);
this.handle = _opus_decoder_create(sampling_rate, channels, err_ptr);
if (getValue(err_ptr, 'i32') != 0 /* OK */)
throw 'opus_decoder_create failed: ' + getValue(err_ptr, 'i32');
this.in_ptr = _malloc(Opus.getMaxFrameSize(channels));
this.in_buf = HEAPU8.subarray(this.in_ptr, this.in_ptr + Opus.getMaxFrameSize(channels));
this.out_len = Opus.getMaxSamplesPerChannel(sampling_rate);
var out_bytes = this.out_len * channels * 4;
this.out_ptr = _malloc(out_bytes);
this.out_i16 = HEAP16.subarray(this.out_ptr >> 1, (this.out_ptr + out_bytes) >> 1);
this.out_f32 = HEAPF32.subarray(this.out_ptr >> 2, (this.out_ptr + out_bytes) >> 2);
}
OpusDecoder.prototype.decode = function (packet) {
this.in_buf.set(new Uint8Array(packet));
var ret = _opus_decode(this.handle, this.in_ptr, packet.byteLength, this.out_ptr, this.out_len, 0);
if (ret < 0)
throw 'opus_decode failed: ' + ret;
var samples = new Int16Array(ret * this.channels);
samples.set(this.out_i16.subarray(0, samples.length));
return samples;
};
OpusDecoder.prototype.decode_float = function (packet) {
this.in_buf.set(new Uint8Array(packet));
var ret = _opus_decode_float(this.handle, this.in_ptr, packet.byteLength, this.out_ptr, this.out_len, 0);
if (ret < 0)
throw 'opus_decode failed: ' + ret;
var samples = new Float32Array(ret * this.channels);
samples.set(this.out_f32.subarray(0, samples.length));
return samples;
};
OpusDecoder.prototype.destroy = function () {
if (!this.handle)
return;
_opus_decoder_destroy(this.handle);
_free(this.in_ptr);
_free(this.out_ptr);
this.handle = this.in_ptr = this.out_ptr = 0;
};
return OpusDecoder;
})();

85
static/js/ws.js vendored
View file

@ -9,7 +9,7 @@ conn = new WebSocket(`ws://${location.host}/ws`);
conn.onopen = () => {
log("WebSocket is opened. Send ping");
log("Send ping pong frequently")
pingpongTimer = setInterval(sendPing, 1000 / PINGPONGPS)
// pingpongTimer = setInterval(sendPing, 1000 / PINGPONGPS)
startWebRTC();
}
@ -70,24 +70,82 @@ function startWebRTC() {
// webrtc
pc = new RTCPeerConnection({iceServers: [{urls: 'stun:stun.l.google.com:19302'}]})
// input channel
inputChannel = pc.createDataChannel('foo', {ordered: false})
inputChannel.onclose = () => {
log('inputChannel has closed');
// input channel, ordered + reliable, id 0
inputChannel = pc.createDataChannel('a', {
ordered: true,
negotiated: true,
id: 0,
});
inputChannel.onopen = () => log('inputChannel has opened');
inputChannel.onclose = () => log('inputChannel has closed');
// audio channel, unordered + unreliable, id 1
var audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var isInit = false;
var audioStack = [];
var nextTime = 0;
var packetIdx = 0;
function scheduleBuffers() {
while (audioStack.length) {
var buffer = audioStack.shift();
var source = audioCtx.createBufferSource();
source.buffer = buffer;
source.connect(audioCtx.destination);
// tracking linear time
if (nextTime == 0)
nextTime = audioCtx.currentTime + 0.1; /// add 100ms latency to work well across systems - tune this if you like
source.start(nextTime);
nextTime+=source.buffer.duration; // Make the next buffer wait the length of the last buffer before being played
};
}
inputChannel.onopen = () => {
log('inputChannel has opened');
sampleRate = 16000;
channels = 1;
bitDepth = 16;
decoder = new OpusDecoder(sampleRate, channels);
function decodeChunk(opusChunk) {
pcmChunk = decoder.decode_float(opusChunk);
myBuffer = audioCtx.createBuffer(channels, pcmChunk.length, sampleRate);
nowBuffering = myBuffer.getChannelData(0, bitDepth, sampleRate);
nowBuffering.set(pcmChunk);
return myBuffer;
}
inputChannel.onmessage = e => {
log(`Message from DataChannel '${inputChannel.label}' payload '${e.data}'`);
audioChannel = pc.createDataChannel('b', {
ordered: false,
negotiated: true,
id: 1,
maxRetransmits: 0
})
audioChannel.onopen = () => log('audioChannel has opened');
audioChannel.onclose = () => log('audioChannel has closed');
audioChannel.onmessage = (e) => {
arr = new Uint8Array(e.data);
idx = arr[arr.length - 1];
// only accept missing 5 packets
if (idx < packetIdx && packetIdx - idx < 251) // 256 - 5
return;
packetIdx = idx;
audioStack.push(decodeChunk(e.data));
if (isInit || (audioStack.length > 10)) { // make sure we put at least 10 chunks in the buffer before starting
isInit = true;
scheduleBuffers();
}
}
//
pc.oniceconnectionstatechange = e => {
log(`iceConnectionState: ${pc.iceConnectionState}`);
if (pc.iceConnectionState === "connected") {
//conn.send(JSON.stringify({"id": "start", "data": ""}));
}
else if (pc.iceConnectionState === "disconnected") {
@ -95,9 +153,9 @@ function startWebRTC() {
}
}
// stream channel
// video channel
pc.ontrack = function (event) {
log("New stream, yay!");
document.getElementById("game-screen").srcObject = event.streams[0];
$("#game-screen").show();
}
@ -131,10 +189,10 @@ function startGame() {
log("Starting game screen");
screenState = "game";
conn.send(JSON.stringify({"id": "start", "data": GAME_LIST[gameIdx].nes, "room_id": roomID.value, "player_index": parseInt(playerIndex.value, 10)}));inputTimer
conn.send(JSON.stringify({"id": "start", "data": GAME_LIST[gameIdx].nes, "room_id": roomID.value, "player_index": parseInt(playerIndex.value, 10)}));
// clear menu screen
//endInput();
endInput();
document.getElementById('div').innerHTML = "";
if (!DEBUG) {
$("#menu-screen").fadeOut(400, function() {
@ -142,6 +200,5 @@ function startGame() {
});
}
// end clear
startInput();
}

BIN
static/test/sinewave.raw vendored Normal file
View file

Binary file not shown.

78
static/test/test.html vendored Normal file
View file

@ -0,0 +1,78 @@
<html>
<body>
<button onclick="play()">hihi</button>
<button onclick="alert(1)">hoho</button>
<b>hehe</b>
<script src="/static/js/libopus.js"></script>
<script src="/static/js/opus.js"></script>
<script>
a = new ArrayBuffer(3);
v = new DataView(a);
v.setInt8(0, -28);
v.setInt8(1, -1);
v.setInt8(2, -2);
console.log(a);
b = new Uint8Array(a);
console.log(b);
d = new OpusDecoder(48000, 2);
console.log(d);
console.log(d.decode_float(a));
// Stereo
var audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var channels = 1;
var sampleRate = 48000;
var bitDepth = 16;
// Create an empty two second stereo buffer at the
// sample rate of the AudioContext
var req = new XMLHttpRequest();
var sound;
req.open('GET', "/static/ex3.raw", false);
// req.open('GET', "/static/sinewave.raw", false);
req.overrideMimeType('text\/plain; charset=x-user-defined');
req.onreadystatechange = function (aEvt) {
if (req.readyState == 4) {
if(req.status == 200) {
sound = req.responseText;
console.log("loaded");
} else {
alert('failed loading audio');
}
}
};
req.send(null);
function play(){
frameCount = sound.length / 2;
myAudioBuffer = audioCtx.createBuffer(channels, frameCount, sampleRate);
for (var channel = 0; channel < channels; channel++) {
var nowBuffering = myAudioBuffer.getChannelData(channel, bitDepth, sampleRate);
for (var i = 0; i < frameCount; i++) {
// audio needs to be in [-1.0; 1.0]
// for this reason I also tried to divide it by 32767
// as my pcm sample is in 16-Bit. It plays still the
// same creepy sound less noisy.
var word = (sound.charCodeAt(i * 2) & 0xff) + ((sound.charCodeAt(i * 2 + 1) & 0xff) << 8);
nowBuffering[i] = ((word + 32768) % 65536 - 32768) / 32768.0;
}
}
// Get an AudioBufferSourceNode.
// This is the AudioNode to use when we want to play an AudioBuffer
var source = audioCtx.createBufferSource();
// set the buffer in the AudioBufferSourceNode
source.buffer = myAudioBuffer;
// connect the AudioBufferSourceNode to the
// destination so we can hear the sound
source.connect(audioCtx.destination);
// start the source playing
source.start();
}
</script>
</body>
</html>

16
ui/director.go
View file

@ -6,6 +6,7 @@ import (
"time"
"github.com/giongto35/cloud-game/nes"
// "github.com/gordonklaus/portaudio"
)
type Director struct {
@ -13,6 +14,7 @@ type Director struct {
view *GameView
timestamp float64
imageChannel chan *image.RGBA
audioChannel chan float32
inputChannel chan int
Done chan struct{}
@ -22,10 +24,10 @@ type Director struct {
const FPS = 60
func NewDirector(roomID string, imageChannel chan *image.RGBA, inputChannel chan int) *Director {
func NewDirector(roomID string, imageChannel chan *image.RGBA, audioChannel chan float32, inputChannel chan int) *Director {
director := Director{}
// director.audio = audio
director.Done = make(chan struct{})
director.audioChannel = audioChannel
director.imageChannel = imageChannel
director.inputChannel = inputChannel
director.roomID = roomID
@ -58,6 +60,13 @@ func (d *Director) Step() {
}
func (d *Director) Start(paths []string) {
// portaudio.Initialize()
// defer portaudio.Terminate()
// audio := NewAudio()
// audio.Start()
// d.audio = audio
if len(paths) == 1 {
d.PlayGame(paths[0])
}
@ -68,6 +77,7 @@ func (d *Director) Run() {
c := time.Tick(time.Second / FPS)
L:
for range c {
// for {
// quit game
// TODO: Anyway not using select because it will slow down
select {
@ -96,7 +106,7 @@ func (d *Director) PlayGame(path string) {
log.Fatalln(err)
}
// Set GameView as current view
d.SetView(NewGameView(console, path, hash, d.imageChannel, d.inputChannel))
d.SetView(NewGameView(console, path, hash, d.imageChannel, d.audioChannel, d.inputChannel))
}
func (d *Director) SaveGame() error {

25
ui/gameview.go
View file

@ -29,6 +29,14 @@ const (
)
const NumKeys = 8
// Audio consts
const (
SampleRate = 16000
Channels = 1
TimeFrame = 60
)
type GameView struct {
console *nes.Console
title string
@ -41,16 +49,19 @@ type GameView struct {
loadingPath string
imageChannel chan *image.RGBA
audioChannel chan float32
inputChannel chan int
}
func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, inputChannel chan int) *GameView {
func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, audioChannel chan float32, inputChannel chan int) *GameView {
gameview := &GameView{
console: console,
title: title,
hash: hash,
keyPressed: [NumKeys * 2]bool{false},
imageChannel: imageChannel,
audioChannel: audioChannel,
inputChannel: inputChannel,
}
@ -81,10 +92,8 @@ func (view *GameView) ListenToInputChannel() {
// Enter enter the game view.
func (view *GameView) Enter() {
// Always reset game
// Legacy Audio code. TODO: Add it back to support audio
// view.console.SetAudioChannel(view.director.audio.channel)
// view.console.SetAudioSampleRate(view.director.audio.sampleRate)
view.console.SetAudioSampleRate(SampleRate)
view.console.SetAudioChannel(view.audioChannel)
// load state
if err := view.console.LoadState(savePath(view.hash)); err == nil {
@ -105,8 +114,8 @@ func (view *GameView) Enter() {
// Exit ...
func (view *GameView) Exit() {
// view.console.SetAudioChannel(nil)
// view.console.SetAudioSampleRate(0)
view.console.SetAudioChannel(nil)
view.console.SetAudioSampleRate(0)
// save sram
cartridge := view.console.Cartridge
if cartridge.Battery != 0 {
@ -156,10 +165,10 @@ func (view *GameView) updateControllers() {
// First 8 keys are player 1
var player1Keys [8]bool
copy(player1Keys[:], view.keyPressed[:8])
var player2Keys [8]bool
copy(player2Keys[:], view.keyPressed[8:])
view.console.Controller1.SetButtons(player1Keys)
view.console.Controller2.SetButtons(player2Keys)
}

85
webrtc/webrtc.go
View file

@ -95,6 +95,7 @@ func Decode(in string, obj interface{}) {
func NewWebRTC() *WebRTC {
w := &WebRTC{
ImageChannel: make(chan []byte, 2),
AudioChannel: make(chan []byte, 1000),
InputChannel: make(chan int, 2),
}
return w
@ -113,6 +114,7 @@ type WebRTC struct {
isClosed bool
// for yuvI420 image
ImageChannel chan []byte
AudioChannel chan []byte
InputChannel chan int
Done chan struct{}
@ -159,6 +161,49 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
return "", err
}
// audio track
dfalse := false
dtrue := true
var d0 uint16 = 0
var d1 uint16 = 1
audioTrack, err := w.connection.CreateDataChannel("b", &webrtc.DataChannelInit{
Ordered: &dfalse,
MaxRetransmits: &d0,
Negotiated: &dtrue,
ID: &d1,
})
if err != nil {
return "", err
}
// input channel
inputTrack, err := w.connection.CreateDataChannel("a", &webrtc.DataChannelInit{
Ordered: &dtrue,
Negotiated: &dtrue,
ID: &d0,
})
inputTrack.OnOpen(func() {
log.Printf("Data channel '%s'-'%d' open.\n", inputTrack.Label(), inputTrack.ID())
})
// Register text message handling
inputTrack.OnMessage(func(msg webrtc.DataChannelMessage) {
//layout .:= "2006-01-02T15:04:05.000Z"
//if t, err := time.Parse(layout, string(msg.Data[1])); err == nil {
//fmt.Println("Delay ", time.Now().Sub(t))
//} else {
w.InputChannel <- int(msg.Data[0])
//}
})
inputTrack.OnClose(func() {
fmt.Println("closed webrtc")
w.Done <- struct{}{}
close(w.Done)
})
// WebRTC state callback
w.connection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
log.Printf("ICE Connection State has changed: %s\n", connectionState.String())
@ -166,7 +211,7 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
go func() {
w.isConnected = true
log.Println("ConnectionStateConnected")
w.startStreaming(vp8Track)
w.startStreaming(vp8Track, audioTrack)
}()
}
@ -180,31 +225,7 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
log.Println(iceCandidate)
})
// Data channel callback
w.connection.OnDataChannel(func(d *webrtc.DataChannel) {
log.Printf("New DataChannel %s %d\n", d.Label(), d.ID())
// Register channel opening handling
d.OnOpen(func() {
log.Printf("Data channel '%s'-'%d' open.\n", d.Label(), d.ID())
})
// Register text message handling
d.OnMessage(func(msg webrtc.DataChannelMessage) {
//layout .:= "2006-01-02T15:04:05.000Z"
//if t, err := time.Parse(layout, string(msg.Data[1])); err == nil {
//fmt.Println("Delay ", time.Now().Sub(t))
//} else {
w.InputChannel <- int(msg.Data[0])
//}
})
d.OnClose(func() {
fmt.Println("closed webrtc")
w.Done <- struct{}{}
close(w.Done)
})
})
offer := webrtc.SessionDescription{}
@ -259,7 +280,8 @@ func (w *WebRTC) IsClosed() bool {
return w.isClosed
}
func (w *WebRTC) startStreaming(vp8Track *webrtc.Track) {
// func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, opusTrack *webrtc.Track) {
func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, audioTrack *webrtc.DataChannel) {
log.Println("Start streaming")
// send screenshot
go func() {
@ -281,6 +303,17 @@ func (w *WebRTC) startStreaming(vp8Track *webrtc.Track) {
vp8Track.WriteSample(media.Sample{Data: bs, Samples: 1})
}
}()
// send audio
go func() {
for w.isConnected {
data := <-w.AudioChannel
// time.Sleep()
// time.Sleep(time.Millisecond * time.Duration(rand.Intn(100)))
audioTrack.Send(data)
}
}()
}
func (w *WebRTC) calculateFPS() int {