Add fallback repo for not found cores (#258)

* Add fallback repo for not found cores

configs/config.yaml

@@ -86,10 +86,16 @@ emulator:
       repo:
         # enable auto-download for the list of cores (list->lib)
         sync: true
-        type: buildbot
-        url: https://buildbot.libretro.com/nightly
-        # if repo has file compression
-        compression: zip
+        main:
+          type: buildbot
+          url: https://buildbot.libretro.com/nightly
+          # if repo has file compression
+          compression: zip
+        # a secondary repo to use i.e. for not found in the main cores
+        secondary:
+          type: github
+          url: https://github.com/sergystepanov/libretro-spiegel/blob/main
+          compression: zip
       # Libretro core configuration
       #
       # Available config params:

pkg/config/emulator/config.go

@@ -22,15 +22,20 @@ type LibretroConfig struct {
 			Configs string
 		}
 		Repo struct {
-			Sync        bool
-			Type        string
-			Url         string
-			Compression string
+			Sync      bool
+			Main      LibretroRepoConfig
+			Secondary LibretroRepoConfig
 		}
 		List map[string]LibretroCoreConfig
 	}
 }
 
+type LibretroRepoConfig struct {
+	Type        string
+	Url         string
+	Compression string
+}
+
 type LibretroCoreConfig struct {
 	Lib         string
 	Config      string

pkg/downloader/backend/backend.go

@@ -0,0 +1,10 @@
+package backend
+
+type Download struct {
+	Key     string
+	Address string
+}
+
+type Client interface {
+	Request(dest string, urls ...Download) ([]string, []string)
+}

pkg/downloader/backend/grab.go

@@ -18,24 +18,29 @@ func NewGrabDownloader() GrabDownloader {
 	}
 }
 
-func (d GrabDownloader) Request(dest string, urls ...string) (files []string) {
+func (d GrabDownloader) Request(dest string, urls ...Download) (ok []string, nook []string) {
 	reqs := make([]*grab.Request, 0)
 	for _, url := range urls {
-		req, err := grab.NewRequest(dest, url)
+		req, err := grab.NewRequest(dest, url.Address)
 		if err != nil {
 			log.Printf("error: couldn't make request URL: %v, %v", url, err)
 		} else {
+			req.Label = url.Key
 			reqs = append(reqs, req)
 		}
 	}
 
 	// check each response
 	for resp := range d.client.DoBatch(d.concurrency, reqs...) {
+		r := resp.Request
 		if err := resp.Err(); err != nil {
-			log.Printf("error: download failed: %v\n", err)
+			log.Printf("error: download [%s] %s failed: %v\n", r.Label, r.URL(), err)
+			if resp.HTTPResponse.StatusCode == 404 {
+				nook = append(nook, resp.Request.Label)
+			}
 		} else {
-			log.Printf("Downloaded [%v] %s\n", resp.HTTPResponse.Status, resp.Filename)
-			files = append(files, resp.Filename)
+			log.Printf("Downloaded [%v] [%s] -> %s\n", resp.HTTPResponse.Status, r.Label, resp.Filename)
+			ok = append(ok, resp.Filename)
 		}
 	}
 	return

pkg/downloader/downloader.go

@@ -6,7 +6,7 @@ import (
 )
 
 type Downloader struct {
-	backend client
+	backend backend.Client
 	// pipe contains a sequential list of
 	// operations applied to some files and
 	// each operation will return a list of
@@ -14,9 +14,6 @@ type Downloader struct {
 	pipe []Process
 }
 
-type client interface {
-	Request(dest string, urls ...string) []string
-}
 
 type Process func(string, []string) []string
 
@@ -33,10 +30,10 @@ func NewDefaultDownloader() Downloader {
 // put them into the destination folder.
 // It will return a partial or full list of downloaded files,
 // a list of processed files if some pipe processing functions are set.
-func (d *Downloader) Download(dest string, urls ...string) []string {
-	files := d.backend.Request(dest, urls...)
+func (d *Downloader) Download(dest string, urls ...backend.Download) ([]string, []string) {
+	files, fails := d.backend.Request(dest, urls...)
 	for _, op := range d.pipe {
 		files = op(dest, files)
 	}
-	return files
+	return files, fails
 }

pkg/emulator/libretro/manager/remotehttp/manager.go

@@ -7,54 +7,45 @@ import (
 
 	"github.com/giongto35/cloud-game/v2/pkg/config/emulator"
 	"github.com/giongto35/cloud-game/v2/pkg/downloader"
+	"github.com/giongto35/cloud-game/v2/pkg/downloader/backend"
 	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
 	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/manager"
 	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo"
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/buildbot"
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/github"
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/raw"
 	"github.com/gofrs/flock"
 )
 
 type Manager struct {
 	manager.BasicManager
 
+	arch   core.ArchInfo
 	repo   repo.Repository
 	client downloader.Downloader
 	fmu    *flock.Flock
 }
 
 func NewRemoteHttpManager(conf emulator.LibretroConfig) Manager {
-	repoConf := conf.Cores.Repo
-
-	var repository repo.Repository
-	switch repoConf.Type {
-	case "raw":
-		repository = raw.NewRawRepo(repoConf.Url)
-	case "github":
-		repository = github.NewGithubRepo(repoConf.Url, repoConf.Compression)
-	case "buildbot":
-		fallthrough
-	default:
-		repository = buildbot.NewBuildbotRepo(repoConf.Url, repoConf.Compression)
-	}
-
+	repoConf := conf.Cores.Repo.Main
 	// used for synchronization of multiple process
 	fileLock := os.TempDir() + string(os.PathSeparator) + "cloud_game.lock"
 
+	arch, err := core.GetCoreExt()
+	if err != nil {
+		log.Printf("error: %v", err)
+	}
+
 	return Manager{
 		BasicManager: manager.BasicManager{
 			Conf: conf,
 		},
-		repo:   repository,
+		arch:   arch,
+		repo:   repo.New(repoConf.Type, repoConf.Url, repoConf.Compression, "buildbot"),
 		client: downloader.NewDefaultDownloader(),
 		fmu:    flock.New(fileLock),
 	}
 }
 
-func (m Manager) Sync() error {
+func (m *Manager) Sync() error {
 	declared := m.Conf.GetCores()
-	dir := m.Conf.GetCoresStorePath()
 
 	// IPC lock if multiple worker processes on the same machine
 	m.fmu.Lock()
@@ -63,20 +54,38 @@ func (m Manager) Sync() error {
 	installed := m.GetInstalled()
 	download := diff(declared, installed)
 
-	if len(download) > 0 {
-		log.Printf("Starting Libretro cores download: %v", strings.Join(download, ", "))
-		m.client.Download(dir, m.getCoreUrls(download)...)
+	_, failed := m.download(download)
+	if len(failed) > 0 {
+		log.Printf("[core-dl] error: unable to download some cores, trying 2nd repository")
+		conf := m.Conf.Cores.Repo.Secondary
+		if conf.Type != "" {
+			if fallback := repo.New(conf.Type, conf.Url, conf.Compression, ""); fallback != nil {
+				defer m.setRepo(m.repo)
+				m.setRepo(fallback)
+				_, failed = m.download(failed)
+			}
+		}
 	}
+
 	return nil
 }
 
-func (m Manager) getCoreUrls(names []string) (urls []string) {
-	arch, err := core.GetCoreExt()
-	if err != nil {
-		return
-	}
+func (m *Manager) getCoreUrls(names []string, repo repo.Repository) (urls []backend.Download) {
 	for _, c := range names {
-		urls = append(urls, m.repo.GetCoreData(c, arch).Url)
+		urls = append(urls, backend.Download{Key: c, Address: repo.GetCoreUrl(c, m.arch)})
+	}
+	return
+}
+
+func (m *Manager) setRepo(repo repo.Repository) {
+	m.repo = repo
+}
+
+func (m *Manager) download(cores []string) (succeeded []string, failed []string) {
+	if len(cores) > 0 && m.repo != nil {
+		dir := m.Conf.GetCoresStorePath()
+		log.Printf("[core-dl] <<< download: %v", strings.Join(cores, ", "))
+		_, failed = m.client.Download(dir, m.getCoreUrls(cores, m.repo)...)
 	}
 	return
 }

pkg/emulator/libretro/repo/buildbot/repository.go

@@ -4,27 +4,31 @@ import (
 	"strings"
 
 	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo"
+	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/raw"
 )
 
-type Repo struct {
-	address     string
-	compression repo.CompressionType
+type RepoBuildbot struct {
+	raw.Repo
 }
 
-func NewBuildbotRepo(address string, compression string) Repo {
-	return Repo{address: address, compression: (repo.CompressionType)(compression)}
+func NewBuildbotRepo(address string, compression string) RepoBuildbot {
+	return RepoBuildbot{
+		Repo: raw.Repo{
+			Address:     address,
+			Compression: compression,
+		},
+	}
 }
 
-func (r Repo) GetCoreData(file string, info core.ArchInfo) repo.Data {
+func (r RepoBuildbot) GetCoreUrl(file string, info core.ArchInfo) string {
 	var sb strings.Builder
-	sb.WriteString(r.address + "/")
+	sb.WriteString(r.Address + "/")
 	if info.Vendor != "" {
 		sb.WriteString(info.Vendor + "/")
 	}
 	sb.WriteString(info.Os + "/" + info.Arch + "/latest/" + file + info.LibExt)
-	if r.compression != "" {
-		sb.WriteString("." + r.compression.GetExt())
+	if r.Compression != "" {
+		sb.WriteString("." + r.Compression)
 	}
-	return repo.Data{Url: sb.String(), Compression: r.compression}
+	return sb.String()
 }

pkg/emulator/libretro/repo/buildbot/repository_test.go

@@ -47,10 +47,10 @@ func TestBuildbotRepo(t *testing.T) {
 
 	for _, test := range tests {
 		repo := NewBuildbotRepo(testAddress, test.compression)
-		data := repo.GetCoreData(test.file, test.arch)
-		if data.Url != test.resultUrl {
+		url := repo.GetCoreUrl(test.file, test.arch)
+		if url != test.resultUrl {
 			t.Errorf("seems that expected link address is incorrect (%v) for file %s %+v",
-				data.Url, test.file, test.arch)
+				url, test.file, test.arch)
 		}
 	}
 }

pkg/emulator/libretro/repo/compression.go

@@ -1,7 +0,0 @@
-package repo
-
-type CompressionType string
-
-func (c *CompressionType) GetExt() string {
-	return (string)(*c)
-}

pkg/emulator/libretro/repo/github/repository.go

@@ -2,19 +2,17 @@ package github
 
 import (
 	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo"
 	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/buildbot"
 )
 
-type Repo struct {
-	buildbot.Repo
+type RepoGithub struct {
+	buildbot.RepoBuildbot
 }
 
-func NewGithubRepo(address string, compression string) Repo {
-	return Repo{Repo: buildbot.NewBuildbotRepo(address, compression)}
+func NewGithubRepo(address string, compression string) RepoGithub {
+	return RepoGithub{RepoBuildbot: buildbot.NewBuildbotRepo(address, compression)}
 }
 
-func (r Repo) GetCoreData(file string, info core.ArchInfo) repo.Data {
-	dat := r.Repo.GetCoreData(file, info)
-	return repo.Data{Url: dat.Url + "?raw=true", Compression: dat.Compression}
+func (r RepoGithub) GetCoreUrl(file string, info core.ArchInfo) string {
+	return r.RepoBuildbot.GetCoreUrl(file, info) + "?raw=true"
 }

pkg/emulator/libretro/repo/github/repository_test.go

@@ -47,10 +47,10 @@ func TestBuildbotRepo(t *testing.T) {
 
 	for _, test := range tests {
 		repo := NewGithubRepo(testAddress, test.compression)
-		data := repo.GetCoreData(test.file, test.arch)
-		if data.Url != test.resultUrl {
+		url := repo.GetCoreUrl(test.file, test.arch)
+		if url != test.resultUrl {
 			t.Errorf("seems that expected link address is incorrect (%v) for file %s %+v",
-				data.Url, test.file, test.arch)
+				url, test.file, test.arch)
 		}
 	}
 }

pkg/emulator/libretro/repo/raw/repository.go

@@ -1,24 +1,18 @@
 package raw
 
-import (
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
-	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo"
-)
+import "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
 
 type Repo struct {
-	address     string
-	compression repo.CompressionType
+	Address     string
+	Compression string
 }
 
 // NewRawRepo defines a simple zip file containing
 // all the cores that will be extracted as is.
 func NewRawRepo(address string) Repo {
-	return Repo{
-		address:     address,
-		compression: "zip",
-	}
+	return Repo{Address: address, Compression: "zip"}
 }
 
-func (r Repo) GetCoreData(_ string, _ core.ArchInfo) repo.Data {
-	return repo.Data{Url: r.address, Compression: r.compression}
+func (r Repo) GetCoreUrl(_ string, _ core.ArchInfo) string {
+	return r.Address
 }

pkg/emulator/libretro/repo/repository.go

@@ -1,14 +1,36 @@
 package repo
 
-import "github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
+import (
+	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/core"
+	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/buildbot"
+	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/github"
+	"github.com/giongto35/cloud-game/v2/pkg/emulator/libretro/repo/raw"
+)
 
 type (
 	Data struct {
 		Url         string
-		Compression CompressionType
+		Compression string
 	}
 
 	Repository interface {
-		GetCoreData(file string, info core.ArchInfo) Data
+		GetCoreUrl(file string, info core.ArchInfo) (url string)
 	}
 )
+
+func New(kind string, url string, compression string, defaultRepo string) Repository {
+	var repository Repository
+	switch kind {
+	case "raw":
+		repository = raw.NewRawRepo(url)
+	case "github":
+		repository = github.NewGithubRepo(url, compression)
+	case "buildbot":
+		repository = buildbot.NewBuildbotRepo(url, compression)
+	default:
+		if defaultRepo != "" {
+			repository = New(defaultRepo, url, compression, "")
+		}
+	}
+	return repository
+}