Mirrors/cloud-game
1
0
Fork 0
mirror of https://github.com/giongto35/cloud-game.git synced 2026-07-22 09:37:09 +00:00
Code Issues Projects Releases Packages Wiki Activity

audio channel unordered + unreliable buffer

Browse source
This commit is contained in:
trichimtrich 2019-04-23 04:49:43 +08:00
parent a98bf7799e
commit 7a3943ae4c
4 changed files with 147 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

69
main.go
View file

@ -21,6 +21,8 @@ import (
"github.com/gorilla/websocket"
pionRTC "github.com/pion/webrtc"
uuid "github.com/satori/go.uuid"
"gopkg.in/hraban/opus.v2"
)
const (
@ -142,7 +144,8 @@ func initRoom(roomID, gameName string) string {
}
rooms[roomID] = room
go room.start()
go room.startVideo()
go room.startAudio()
go director.Start([]string{"games/" + gameName})
return roomID
@ -326,7 +329,7 @@ func generateRoomID() string {
return roomID
}
func (r *Room) start() {
func (r *Room) startVideo() {
// fanout Screen
for {
select {
@ -357,6 +360,68 @@ func (r *Room) start() {
}
}
func (r *Room) startAudio() {
log.Println("Enter fan audio")
enc, err := opus.NewEncoder(ui.SampleRate, ui.Channels, opus.AppAudio)
maxBufferSize := ui.TimeFrame * ui.SampleRate / 1000
pcm := make([]float32, maxBufferSize) // 640 * 1000 / 16000 == 40 ms
idx := 0
if err != nil {
log.Println("[!] Cannot create audio encoder")
return
}
var count byte = 0
// fanout Audio
for {
select {
case <-r.Done:
r.remove()
return
case sample := <-r.audioChannel:
pcm[idx] = sample
idx ++
if idx == len(pcm) {
data := make([]byte, 640)
n, err := enc.EncodeFloat32(pcm, data)
if err != nil {
log.Println("[!] Failed to decode")
continue
}
data = data[:n]
data = append(data, count)
r.sessionsLock.Lock()
for _, webRTC := range r.rtcSessions {
// Client stopped
if webRTC.IsClosed() {
continue
}
// encode frame
// fanout audioChannel
if webRTC.IsConnected() {
// NOTE: can block here
webRTC.AudioChannel <- data
}
//isRoomRunning = true
}
r.sessionsLock.Unlock()
idx = 0
count = (count + 1) & 0xff
}
}
}
}
func (r *Room) remove() {
log.Println("Closing room", r)
r.director.Done <- struct{}{}

8
static/index_ws.html vendored
View file

@ -12,7 +12,6 @@ textarea {
<button id="play" onclick="window.startGame()">Play</button>
<button id="play" onclick="pc.close()">Stop</button>
<button id="play" onclick="window.hihi()">Magic</button><br/><br/>
Your current room: <b><label id="currentRoomID" style="color:blue"></b> <br />
You can join a remote game by roomID.<br />
Room ID: <input type="text" id="roomID">
@ -25,7 +24,7 @@ Play as player(1,2): <select id="playerIndex">
<body scroll="no" style="overflow: hidden">
<div id="remoteVideos">
<video id="game-screen2" autoplay=true width=400 height=300 poster="https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif">
<video id="game-screen" autoplay=true width=400 height=300 poster="https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif">
</div> <br />
<h3>Instruction</h3>
@ -64,11 +63,6 @@ Play as player(1,2): <select id="playerIndex">
<script src="/static/js/ws.js"></script>
<script>
function hihi() {
track = stream.getAudioTracks()[0];
console.log(track);
}
GAME_LIST.forEach(e => {
ee = document.createElement("option");
ee.value = e.nes;

94
static/js/ws.js vendored
View file

@ -70,70 +70,77 @@ function startWebRTC() {
// webrtc
pc = new RTCPeerConnection({iceServers: [{urls: 'stun:stun.l.google.com:19302'}]})
// input channel, ordered + reliable
// input channel, ordered + reliable, id 0
inputChannel = pc.createDataChannel('a', {
ordered: true,
negotiated: true,
id: 0,
});
inputChannel.onopen = () => log('inputChannel has opened');
inputChannel.onclose = () => log('inputChannel has closed');
window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new AudioContext();
var delayTime = 0;
var init = 0;
// audio channel, unordered + unreliable, id 1
var audioCtx = new (window.AudioContext || window.webkitAudioContext)();
var isInit = false;
var audioStack = [];
var nextTime = 0;
var isRun = false;
var packetIdx = 0;
function scheduleBuffers() {
if (isRun) return;
console.log("daaaa");
while ( audioStack.length) {
isRun = true;
while (audioStack.length) {
var buffer = audioStack.shift();
var source = context.createBufferSource();
var source = audioCtx.createBufferSource();
source.buffer = buffer;
source.connect(context.destination);
source.connect(audioCtx.destination);
// tracking linear time
if (nextTime == 0)
nextTime = context.currentTime + 0.05; /// add 50ms latency to work well across systems - tune this if you like
nextTime = audioCtx.currentTime + 0.1; /// add 100ms latency to work well across systems - tune this if you like
source.start(nextTime);
nextTime+=source.buffer.duration; // Make the next buffer wait the length of the last buffer before being played
};
console.log("diii");
isRun = false;
}
sampleRate = 16000;
channels = 1;
bitDepth = 16;
decoder = new OpusDecoder(sampleRate, channels);
function damn(opusChunk) {
function decodeChunk(opusChunk) {
pcmChunk = decoder.decode_float(opusChunk);
myBuffer = context.createBuffer(channels, pcmChunk.length, sampleRate);
myBuffer = audioCtx.createBuffer(channels, pcmChunk.length, sampleRate);
nowBuffering = myBuffer.getChannelData(0, bitDepth, sampleRate);
nowBuffering.set(pcmChunk);
return myBuffer;
}
pc.ondatachannel = function (ev) {
log(`New data channel '${ev.channel.label}'`);
ev.channel.onopen = () => log('channelX has opened');
ev.channel.onclose = () => log('channelX has closed');
ev.channel.onmessage = (e) => {
// source = context.createBufferSource();
// source.buffer = buf;
// source.connect(context.destination);
// source.start(0);
audioStack.push(damn(e.data));
if ((init!=0) || (audioStack.length > 10)) { // make sure we put at least 10 chunks in the buffer before starting
init++;
scheduleBuffers();
}
audioChannel = pc.createDataChannel('b', {
ordered: false,
negotiated: true,
id: 1,
maxRetransmits: 0
})
audioChannel.onopen = () => log('audioChannel has opened');
audioChannel.onclose = () => log('audioChannel has closed');
audioChannel.onmessage = (e) => {
arr = new Uint8Array(e.data);
idx = arr[arr.length - 1];
// only accept missing 5 packets
if (idx < packetIdx && packetIdx - idx < 251) // 256 - 5
return;
packetIdx = idx;
audioStack.push(decodeChunk(e.data));
if (isInit || (audioStack.length > 10)) { // make sure we put at least 10 chunks in the buffer before starting
isInit = true;
scheduleBuffers();
}
}
//
pc.oniceconnectionstatechange = e => {
log(`iceConnectionState: ${pc.iceConnectionState}`);
@ -146,24 +153,11 @@ function startWebRTC() {
}
}
window.stream = new MediaStream();
document.getElementById("game-screen2").srcObject = stream;
// stream channel
// video channel
pc.ontrack = function (event) {
console.log(event);
stream.addTrack(event.track);
// var el = document.createElement(event.track.kind);
// el.srcObject = event.streams[0];
// el.autoplay = true;
// el.width = 800;
// el.height = 600;
// el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif");
// document.getElementById('remoteVideos').appendChild(el)
log("New stream, yay!");
// document.getElementById("game-screen").srcObject = event.streams[0];
// $("#game-screen").show();
document.getElementById("game-screen").srcObject = event.streams[0];
$("#game-screen").show();
}

60
webrtc/webrtc.go
View file

@ -163,11 +163,46 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
// audio track
dfalse := false
dtrue := true
var d0 uint16 = 0
var d1 uint16 = 1
audioTrack, err := w.connection.CreateDataChannel("b", &webrtc.DataChannelInit{
Ordered: &dfalse,
MaxRetransmits: &d0,
Negotiated: &dtrue,
ID: &d1,
})
if err != nil {
return "", err
}
// input channel
inputTrack, err := w.connection.CreateDataChannel("a", &webrtc.DataChannelInit{
Ordered: &dtrue,
Negotiated: &dtrue,
ID: &d0,
})
inputTrack.OnOpen(func() {
log.Printf("Data channel '%s'-'%d' open.\n", inputTrack.Label(), inputTrack.ID())
})
// Register text message handling
inputTrack.OnMessage(func(msg webrtc.DataChannelMessage) {
//layout .:= "2006-01-02T15:04:05.000Z"
//if t, err := time.Parse(layout, string(msg.Data[1])); err == nil {
//fmt.Println("Delay ", time.Now().Sub(t))
//} else {
w.InputChannel <- int(msg.Data[0])
//}
})
inputTrack.OnClose(func() {
fmt.Println("closed webrtc")
w.Done <- struct{}{}
close(w.Done)
})
// WebRTC state callback
w.connection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
@ -191,31 +226,6 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
})
// Data channel callback
w.connection.OnDataChannel(func(d *webrtc.DataChannel) {
log.Printf("New DataChannel %s %d\n", d.Label(), d.ID())
// Register channel opening handling
d.OnOpen(func() {
log.Printf("Data channel '%s'-'%d' open.\n", d.Label(), d.ID())
})
// Register text message handling
d.OnMessage(func(msg webrtc.DataChannelMessage) {
//layout .:= "2006-01-02T15:04:05.000Z"
//if t, err := time.Parse(layout, string(msg.Data[1])); err == nil {
//fmt.Println("Delay ", time.Now().Sub(t))
//} else {
w.InputChannel <- int(msg.Data[0])
//}
})
d.OnClose(func() {
fmt.Println("closed webrtc")
w.Done <- struct{}{}
close(w.Done)
})
})
offer := webrtc.SessionDescription{}