Mirrors/cloud-game
1
0
Fork 0
mirror of https://github.com/giongto35/cloud-game.git synced 2026-07-22 09:37:09 +00:00
Code Issues Projects Releases Packages Wiki Activity

WIP audio

Browse source
This commit is contained in:
hasagi 2019-04-14 00:54:47 +07:00
parent 73f180493f
commit 0ccc7e58a5
7 changed files with 131 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

120
main.go
View file

@ -19,9 +19,76 @@ import (
"github.com/gorilla/websocket"
pionRTC "github.com/pion/webrtc"
"github.com/hraban/opus"
// "github.com/hraban/opus"
"gopkg.in/hraban/opus.v2"
// "github.com/xlab/opus-go/opus"
)
// fanoutAudio fanout outputs to all webrtc in the same room
func fanoutAudio(audioChannel chan float32, roomID string) {
var output float32
pcm := make([]float32, 240)
enc, err := opus.NewEncoder(48000, 1, opus.AppAudio)
if err != nil {
log.Println("[!] Cannot create audio encoder")
return
}
// var err2 int32
// var err error
// enc := opus.EncoderCreate(48000, 1, opus.ApplicationAudio, &err2)
c := time.Tick(time.Millisecond * 5)
for range c {
for i := 0; i < len(pcm); i++ {
select {
case sample := <- audioChannel:
output = sample
default:
output = 0
}
pcm[i] = output
}
data := make([]byte, 1000)
n, err := enc.EncodeFloat32(pcm, data)
// n := opus.EncodeFloat(enc, pcm, int32(c), data, 1000)
if err != nil {
log.Println("[!] Failed to decode")
continue
}
data = data[:n]
isRoomRunning := false
for _, webRTC := range rooms[roomID].rtcSessions {
// Client stopped
if webRTC.IsClosed() {
continue
}
// encode frame
// fanout imageChannel
if webRTC.IsConnected() {
// NOTE: can block here
webRTC.AudioChannel <- data
}
isRoomRunning = true
}
if isRoomRunning == false {
log.Println("Closed room from audio routine", roomID)
rooms[roomID].closedChannel <- true
}
}
}
const (
width = 256
height = 240
@ -310,57 +377,6 @@ func fanoutScreen(imageChannel chan *image.RGBA, roomID string) {
}
}
// fanoutAudio fanout outputs to all webrtc in the same room
func fanoutAudio(audioChanel chan float32, roomID string) {
enc, err := opus.NewEncoder(48000, 1, opus.AppVoIP)
if err != nil {
log.Println("[!] Cannot create audio encoder")
return
}
pcm := make([]float32, 120)
c := 0
for audio := range audioChanel {
if c >= cap(pcm) {
data := make([]byte, 1000)
n, err := enc.EncodeFloat32(pcm, data)
if err != nil {
log.Println("[!] Failed to decode")
continue
}
data = data[:n]
isRoomRunning := false
for _, webRTC := range rooms[roomID].rtcSessions {
// Client stopped
if webRTC.IsClosed() {
continue
}
// encode frame
// fanout imageChannel
if webRTC.IsConnected() {
// NOTE: can block here
webRTC.AudioChannel <- data
}
isRoomRunning = true
}
if isRoomRunning == false {
log.Println("Closed room from audio routine", roomID)
rooms[roomID].closedChannel <- true
}
c = 0
} else {
pcm[c] = audio
c++
}
}
}
// faninInput fan-in of the same room to inputChannel
func faninInput(inputChannel chan int, webRTC *webrtc.WebRTC, playerIndex int) {
for {

4
static/index_ws.html
View file

@ -23,7 +23,7 @@ Play as player(1,2): <select id="playerIndex">
<body scroll="no" style="overflow: hidden">
<div id="remoteVideos">
<video id="game-screen" autoplay=true width=400 height=300 poster="https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif" style="display: none">
<video id="game-screen2" autoplay=true width=400 height=300 poster="https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif">
</div> <br />
<h3>Instruction</h3>
@ -68,6 +68,8 @@ Play as player(1,2): <select id="playerIndex">
$("#gameOp").on("change", function() {
gameIdx = gameOp.selectedIndex;
});
gameIdx = 7;
</script>
</html>

22
static/js/ws.js
View file

@ -84,18 +84,21 @@ function startGame() {
}
}
var stream = new MediaStream();
document.getElementById("game-screen2").srcObject = stream;
// stream channel
pc.ontrack = function (event) {
console.log(event.streams);
var el = document.createElement(event.track.kind);
el.srcObject = event.streams[0];
el.autoplay = true;
el.width = 800;
el.height = 600;
el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif");
document.getElementById('remoteVideos').appendChild(el)
console.log(event);
stream.addTrack(event.track);
// var el = document.createElement(event.track.kind);
// el.srcObject = event.streams[0];
// el.autoplay = true;
// el.width = 800;
// el.height = 600;
// el.poster = new URL("https://orig00.deviantart.net/cdcd/f/2017/276/a/a/october_2nd___gameboy_poltergeist_by_wanyo-dbpdmnd.gif");
// document.getElementById('remoteVideos').appendChild(el)
// log("New stream, yay!");
log("New stream, yay!");
// document.getElementById("game-screen").srcObject = event.streams[0];
// $("#game-screen").show();
}
@ -109,6 +112,7 @@ function startGame() {
localSessionDescription = session;
log("Send SDP to remote peer");
conn.send(JSON.stringify({"id": "sdp", "data": session}));
console.log(session);
} else {
console.log(JSON.stringify(event.candidate));
}

45
ui/audio.go
View file

@ -2,6 +2,10 @@ package ui
import (
"github.com/gordonklaus/portaudio"
"log"
"gopkg.in/hraban/opus.v2"
// "github.com/xlab/opus-go/opus"
)
@ -19,13 +23,17 @@ func NewAudio() *Audio {
}
func (a *Audio) Start() error {
host, err := portaudio.DefaultHostApi()
if err != nil {
return err
}
parameters := portaudio.HighLatencyParameters(nil, host.DefaultOutputDevice)
stream, err := portaudio.OpenStream(parameters, a.Callback)
parameters := portaudio.StreamParameters{}
parameters.SampleRate = 44100
// host, err := portaudio.DefaultHostApi()
// if err != nil {
// return err
// }
// parameters := portaudio.HighLatencyParameters(nil, host.DefaultOutputDevice)
stream, err := portaudio.OpenDefaultStream(0, 1, 48000, 0, a.Callback)
// stream, err := portaudio.OpenStream(parameters, a.Callback)
if err != nil {
return err
}
@ -34,7 +42,11 @@ func (a *Audio) Start() error {
}
a.stream = stream
a.sampleRate = parameters.SampleRate
a.outputChannels = parameters.Output.Channels
// a.outputChannels = parameters.Output.Channels
a.outputChannels = 1
log.Println(a.sampleRate, a.outputChannels, parameters.FramesPerBuffer)
return nil
}
@ -44,8 +56,9 @@ func (a *Audio) Stop() error {
func (a *Audio) Callback(out []float32) {
var output float32
log.Println(len(out))
for i := range out {
if i%a.outputChannels == 0 {
if i%a.outputChannels == 1 {
select {
case sample := <-a.channel:
output = sample
@ -55,4 +68,18 @@ func (a *Audio) Callback(out []float32) {
}
out[i] = output
}
enc, err := opus.NewEncoder(48000, 1, opus.AppVoIP)
if err != nil {
log.Println("[!] Cannot create audio encoder", err)
return
}
data := make([]byte, 1000)
n, err := enc.EncodeFloat32(out, data)
if err != nil {
log.Println("[!] Failed to decode")
return
}
data = data[:n]
}

2
ui/director.go
View file

@ -10,7 +10,6 @@ import (
)
type Director struct {
// audio *Audio
view *GameView
timestamp float64
imageChannel chan *image.RGBA
@ -25,7 +24,6 @@ const FPS = 60
func NewDirector(roomID string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int, closedChannel chan bool) *Director {
director := Director{}
// director.audio = audio
director.imageChannel = imageChannel
director.audioChanel = audioChanel
director.inputChannel = inputChannel

4
ui/gameview.go
View file

@ -40,6 +40,7 @@ type GameView struct {
audioChanel chan float32
inputChannel chan int
}
func NewGameView(console *nes.Console, title, hash string, imageChannel chan *image.RGBA, audioChanel chan float32, inputChannel chan int) *GameView {
@ -71,10 +72,9 @@ func (view *GameView) ListenToInputChannel() {
// Enter enter the game view.
func (view *GameView) Enter() {
// Always reset game
// Legacy Audio code. TODO: Add it back to support audio
// view.console.SetAudioChannel(view.audio.channel)
// view.console.SetAudioSampleRate(view.audio.sampleRate)
view.console.SetAudioSampleRate(48000)
view.console.SetAudioChannel(view.audioChanel)

11
webrtc/webrtc.go
View file

@ -93,7 +93,7 @@ func Decode(in string, obj interface{}) {
func NewWebRTC() *WebRTC {
w := &WebRTC{
ImageChannel: make(chan []byte, 2),
AudioChannel: make(chan []byte, 2),
AudioChannel: make(chan []byte, 1000),
InputChannel: make(chan int, 2),
}
return w
@ -135,6 +135,12 @@ func (w *WebRTC) StartClient(remoteSession string, width, height int) (string, e
log.Println("=== StartClient ===")
w.connection, err = webrtc.NewPeerConnection(config)
// m := webrtc.MediaEngine{}
// m.RegisterCodec(webrtc.NewRTPOpusCodec(webrtc.DefaultPayloadTypeOpus, 48000))
// m.RegisterCodec(webrtc.NewRTPVP8Codec(webrtc.DefaultPayloadTypeVP8, 1))
// api := webrtc.NewAPI(webrtc.WithMediaEngine(m))
// w.connection, err = api.NewPeerConnection(config)
if err != nil {
return "", err
}
@ -271,7 +277,8 @@ func (w *WebRTC) startStreaming(vp8Track *webrtc.Track, opusTrack *webrtc.Track)
go func() {
for w.isConnected {
data := <-w.AudioChannel
opusTrack.WriteSample(media.Sample{Data: data, Samples: 1})
// opusTrack.Write(data)
opusTrack.WriteSample(media.Sample{Data: data, Samples: 1000})
}
}()