diff --git a/assets/cores/mupen64plus_next_libretro.armv7-neon-hf.so b/assets/cores/mupen64plus_next_libretro.armv7-neon-hf.so deleted file mode 100644 index 6c857e1b..00000000 Binary files a/assets/cores/mupen64plus_next_libretro.armv7-neon-hf.so and /dev/null differ diff --git a/assets/cores/mupen64plus_next_libretro.dylib b/assets/cores/mupen64plus_next_libretro.dylib deleted file mode 100644 index 6bdb8520..00000000 Binary files a/assets/cores/mupen64plus_next_libretro.dylib and /dev/null differ diff --git a/configs/config.yaml b/configs/config.yaml index 9ff25ffa..2ad4a99f 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -86,10 +86,16 @@ emulator: repo: # enable auto-download for the list of cores (list->lib) sync: true - type: buildbot - url: https://buildbot.libretro.com/nightly - # if repo has file compression - compression: zip + main: + type: buildbot + url: https://buildbot.libretro.com/nightly + # if repo has file compression + compression: zip + # a secondary repo to use i.e. for not found in the main cores + secondary: + type: github + url: https://github.com/sergystepanov/libretro-spiegel/blob/main + compression: zip # Libretro core configuration # # Available config params: diff --git a/pkg/config/emulator/config.go b/pkg/config/emulator/config.go index 122d94b6..bf0d1658 100644 --- a/pkg/config/emulator/config.go +++ b/pkg/config/emulator/config.go @@ -22,15 +22,20 @@ type LibretroConfig struct { Configs string } Repo struct { - Sync bool - Type string - Url string - Compression string + Sync bool + Main LibretroRepoConfig + Secondary LibretroRepoConfig } List map[string]LibretroCoreConfig } } +type LibretroRepoConfig struct { + Type string + Url string + Compression string +} + type LibretroCoreConfig struct { Lib string Config string diff --git a/pkg/downloader/backend/backend.go b/pkg/downloader/backend/backend.go new file mode 100644 index 00000000..717b0654 --- /dev/null +++ b/pkg/downloader/backend/backend.go @@ -0,0 +1,10 @@ +package backend + +type Download struct { + Key string + Address string +} + +type Client interface { + Request(dest string, urls ...Download) ([]string, []string) +} diff --git a/pkg/downloader/backend/grab.go b/pkg/downloader/backend/grab.go index 32f6178c..3542e957 100644 --- a/pkg/downloader/backend/grab.go +++ b/pkg/downloader/backend/grab.go @@ -18,24 +18,29 @@ func NewGrabDownloader() GrabDownloader { } } -func (d GrabDownloader) Request(dest string, urls ...string) (files []string) { +func (d GrabDownloader) Request(dest string, urls ...Download) (ok []string, nook []string) { reqs := make([]*grab.Request, 0) for _, url := range urls { - req, err := grab.NewRequest(dest, url) + req, err := grab.NewRequest(dest, url.Address) if err != nil { log.Printf("error: couldn't make request URL: %v, %v", url, err) } else { + req.Label = url.Key reqs = append(reqs, req) } } // check each response for resp := range d.client.DoBatch(d.concurrency, reqs...) { + r := resp.Request if err := resp.Err(); err != nil { - log.Printf("error: download failed: %v\n", err) + log.Printf("error: download [%s] %s failed: %v\n", r.Label, r.URL(), err) + if resp.HTTPResponse.StatusCode == 404 { + nook = append(nook, resp.Request.Label) + } } else { - log.Printf("Downloaded [%v] %s\n", resp.HTTPResponse.Status, resp.Filename) - files = append(files, resp.Filename) + log.Printf("Downloaded [%v] [%s] -> %s\n", resp.HTTPResponse.Status, r.Label, resp.Filename) + ok = append(ok, resp.Filename) } } return diff --git a/pkg/downloader/downloader.go b/pkg/downloader/downloader.go index 42e32dd2..66697063 100644 --- a/pkg/downloader/downloader.go +++ b/pkg/downloader/downloader.go @@ -6,7 +6,7 @@ import ( ) type Downloader struct { - backend client + backend backend.Client // pipe contains a sequential list of // operations applied to some files and // each operation will return a list of @@ -14,9 +14,6 @@ type Downloader struct { pipe []Process } -type client interface { - Request(dest string, urls ...string) []string -} type Process func(string, []string) []string @@ -33,10 +30,10 @@ func NewDefaultDownloader() Downloader { // put them into the destination folder. // It will return a partial or full list of downloaded files, // a list of processed files if some pipe processing functions are set. -func (d *Downloader) Download(dest string, urls ...string) []string { - files := d.backend.Request(dest, urls...) +func (d *Downloader) Download(dest string, urls ...backend.Download) ([]string, []string) { + files, fails := d.backend.Request(dest, urls...) for _, op := range d.pipe { files = op(dest, files) } - return files + return files, fails } diff --git a/pkg/emulator/libretro/manager/remotehttp/manager.go b/pkg/emulator/libretro/manager/remotehttp/manager.go index 6589b0f5..422b5db7 100644 --- a/pkg/emulator/libretro/manager/remotehttp/manager.go +++ b/pkg/emulator/libretro/manager/remotehttp/manager.go @@ -7,54 +7,45 @@ import ( "github.com/giongto35/cloud-game/v2/pkg/config/emulator" "github.com/giongto35/cloud-game/v2/pkg/downloader" + "github.com/giongto35/cloud-game/v2/pkg/downloader/backend" "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/manager" "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo" - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/buildbot" - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/github" - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/raw" "github.com/gofrs/flock" ) type Manager struct { manager.BasicManager + arch core.ArchInfo repo repo.Repository client downloader.Downloader fmu *flock.Flock } func NewRemoteHttpManager(conf emulator.LibretroConfig) Manager { - repoConf := conf.Cores.Repo - - var repository repo.Repository - switch repoConf.Type { - case "raw": - repository = raw.NewRawRepo(repoConf.Url) - case "github": - repository = github.NewGithubRepo(repoConf.Url, repoConf.Compression) - case "buildbot": - fallthrough - default: - repository = buildbot.NewBuildbotRepo(repoConf.Url, repoConf.Compression) - } - + repoConf := conf.Cores.Repo.Main // used for synchronization of multiple process fileLock := os.TempDir() + string(os.PathSeparator) + "cloud_game.lock" + arch, err := core.GetCoreExt() + if err != nil { + log.Printf("error: %v", err) + } + return Manager{ BasicManager: manager.BasicManager{ Conf: conf, }, - repo: repository, + arch: arch, + repo: repo.New(repoConf.Type, repoConf.Url, repoConf.Compression, "buildbot"), client: downloader.NewDefaultDownloader(), fmu: flock.New(fileLock), } } -func (m Manager) Sync() error { +func (m *Manager) Sync() error { declared := m.Conf.GetCores() - dir := m.Conf.GetCoresStorePath() // IPC lock if multiple worker processes on the same machine m.fmu.Lock() @@ -63,20 +54,38 @@ func (m Manager) Sync() error { installed := m.GetInstalled() download := diff(declared, installed) - if len(download) > 0 { - log.Printf("Starting Libretro cores download: %v", strings.Join(download, ", ")) - m.client.Download(dir, m.getCoreUrls(download)...) + _, failed := m.download(download) + if len(failed) > 0 { + log.Printf("[core-dl] error: unable to download some cores, trying 2nd repository") + conf := m.Conf.Cores.Repo.Secondary + if conf.Type != "" { + if fallback := repo.New(conf.Type, conf.Url, conf.Compression, ""); fallback != nil { + defer m.setRepo(m.repo) + m.setRepo(fallback) + _, failed = m.download(failed) + } + } } + return nil } -func (m Manager) getCoreUrls(names []string) (urls []string) { - arch, err := core.GetCoreExt() - if err != nil { - return - } +func (m *Manager) getCoreUrls(names []string, repo repo.Repository) (urls []backend.Download) { for _, c := range names { - urls = append(urls, m.repo.GetCoreData(c, arch).Url) + urls = append(urls, backend.Download{Key: c, Address: repo.GetCoreUrl(c, m.arch)}) + } + return +} + +func (m *Manager) setRepo(repo repo.Repository) { + m.repo = repo +} + +func (m *Manager) download(cores []string) (succeeded []string, failed []string) { + if len(cores) > 0 && m.repo != nil { + dir := m.Conf.GetCoresStorePath() + log.Printf("[core-dl] <<< download: %v", strings.Join(cores, ", ")) + _, failed = m.client.Download(dir, m.getCoreUrls(cores, m.repo)...) } return } diff --git a/pkg/emulator/libretro/repo/buildbot/repository.go b/pkg/emulator/libretro/repo/buildbot/repository.go index 28d45690..6ced435c 100644 --- a/pkg/emulator/libretro/repo/buildbot/repository.go +++ b/pkg/emulator/libretro/repo/buildbot/repository.go @@ -4,27 +4,31 @@ import ( "strings" "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo" + "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/raw" ) -type Repo struct { - address string - compression repo.CompressionType +type RepoBuildbot struct { + raw.Repo } -func NewBuildbotRepo(address string, compression string) Repo { - return Repo{address: address, compression: (repo.CompressionType)(compression)} +func NewBuildbotRepo(address string, compression string) RepoBuildbot { + return RepoBuildbot{ + Repo: raw.Repo{ + Address: address, + Compression: compression, + }, + } } -func (r Repo) GetCoreData(file string, info core.ArchInfo) repo.Data { +func (r RepoBuildbot) GetCoreUrl(file string, info core.ArchInfo) string { var sb strings.Builder - sb.WriteString(r.address + "/") + sb.WriteString(r.Address + "/") if info.Vendor != "" { sb.WriteString(info.Vendor + "/") } sb.WriteString(info.Os + "/" + info.Arch + "/latest/" + file + info.LibExt) - if r.compression != "" { - sb.WriteString("." + r.compression.GetExt()) + if r.Compression != "" { + sb.WriteString("." + r.Compression) } - return repo.Data{Url: sb.String(), Compression: r.compression} + return sb.String() } diff --git a/pkg/emulator/libretro/repo/buildbot/repository_test.go b/pkg/emulator/libretro/repo/buildbot/repository_test.go index f129bdaf..37cb302c 100644 --- a/pkg/emulator/libretro/repo/buildbot/repository_test.go +++ b/pkg/emulator/libretro/repo/buildbot/repository_test.go @@ -47,10 +47,10 @@ func TestBuildbotRepo(t *testing.T) { for _, test := range tests { repo := NewBuildbotRepo(testAddress, test.compression) - data := repo.GetCoreData(test.file, test.arch) - if data.Url != test.resultUrl { + url := repo.GetCoreUrl(test.file, test.arch) + if url != test.resultUrl { t.Errorf("seems that expected link address is incorrect (%v) for file %s %+v", - data.Url, test.file, test.arch) + url, test.file, test.arch) } } } diff --git a/pkg/emulator/libretro/repo/compression.go b/pkg/emulator/libretro/repo/compression.go deleted file mode 100644 index 15804fa2..00000000 --- a/pkg/emulator/libretro/repo/compression.go +++ /dev/null @@ -1,7 +0,0 @@ -package repo - -type CompressionType string - -func (c *CompressionType) GetExt() string { - return (string)(*c) -} diff --git a/pkg/emulator/libretro/repo/github/repository.go b/pkg/emulator/libretro/repo/github/repository.go index 48787e68..01687126 100644 --- a/pkg/emulator/libretro/repo/github/repository.go +++ b/pkg/emulator/libretro/repo/github/repository.go @@ -2,19 +2,17 @@ package github import ( "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo" "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/buildbot" ) -type Repo struct { - buildbot.Repo +type RepoGithub struct { + buildbot.RepoBuildbot } -func NewGithubRepo(address string, compression string) Repo { - return Repo{Repo: buildbot.NewBuildbotRepo(address, compression)} +func NewGithubRepo(address string, compression string) RepoGithub { + return RepoGithub{RepoBuildbot: buildbot.NewBuildbotRepo(address, compression)} } -func (r Repo) GetCoreData(file string, info core.ArchInfo) repo.Data { - dat := r.Repo.GetCoreData(file, info) - return repo.Data{Url: dat.Url + "?raw=true", Compression: dat.Compression} +func (r RepoGithub) GetCoreUrl(file string, info core.ArchInfo) string { + return r.RepoBuildbot.GetCoreUrl(file, info) + "?raw=true" } diff --git a/pkg/emulator/libretro/repo/github/repository_test.go b/pkg/emulator/libretro/repo/github/repository_test.go index 0b52a0a0..01dea346 100644 --- a/pkg/emulator/libretro/repo/github/repository_test.go +++ b/pkg/emulator/libretro/repo/github/repository_test.go @@ -47,10 +47,10 @@ func TestBuildbotRepo(t *testing.T) { for _, test := range tests { repo := NewGithubRepo(testAddress, test.compression) - data := repo.GetCoreData(test.file, test.arch) - if data.Url != test.resultUrl { + url := repo.GetCoreUrl(test.file, test.arch) + if url != test.resultUrl { t.Errorf("seems that expected link address is incorrect (%v) for file %s %+v", - data.Url, test.file, test.arch) + url, test.file, test.arch) } } } diff --git a/pkg/emulator/libretro/repo/raw/repository.go b/pkg/emulator/libretro/repo/raw/repository.go index 344e2eb8..a00c3b30 100644 --- a/pkg/emulator/libretro/repo/raw/repository.go +++ b/pkg/emulator/libretro/repo/raw/repository.go @@ -1,24 +1,18 @@ package raw -import ( - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" - "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo" -) +import "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" type Repo struct { - address string - compression repo.CompressionType + Address string + Compression string } // NewRawRepo defines a simple zip file containing // all the cores that will be extracted as is. func NewRawRepo(address string) Repo { - return Repo{ - address: address, - compression: "zip", - } + return Repo{Address: address, Compression: "zip"} } -func (r Repo) GetCoreData(_ string, _ core.ArchInfo) repo.Data { - return repo.Data{Url: r.address, Compression: r.compression} +func (r Repo) GetCoreUrl(_ string, _ core.ArchInfo) string { + return r.Address } diff --git a/pkg/emulator/libretro/repo/repository.go b/pkg/emulator/libretro/repo/repository.go index d412dd2a..f3eaa44c 100644 --- a/pkg/emulator/libretro/repo/repository.go +++ b/pkg/emulator/libretro/repo/repository.go @@ -1,14 +1,36 @@ package repo -import "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" +import ( + "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core" + "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/buildbot" + "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/github" + "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/raw" +) type ( Data struct { Url string - Compression CompressionType + Compression string } Repository interface { - GetCoreData(file string, info core.ArchInfo) Data + GetCoreUrl(file string, info core.ArchInfo) (url string) } ) + +func New(kind string, url string, compression string, defaultRepo string) Repository { + var repository Repository + switch kind { + case "raw": + repository = raw.NewRawRepo(url) + case "github": + repository = github.NewGithubRepo(url, compression) + case "buildbot": + repository = buildbot.NewBuildbotRepo(url, compression) + default: + if defaultRepo != "" { + repository = New(defaultRepo, url, compression, "") + } + } + return repository +}