From 4512264e8405bcdde10a20451a2a29ca21535a70 Mon Sep 17 00:00:00 2001 From: BBaoVanC Date: Thu, 23 Dec 2021 01:20:44 -0600 Subject: [PATCH 1/5] Support file locking TODO: Actually check for locks in cleanup.go --- backends/localfs/localfs.go | 27 ++++++++++++++++++++++++++- backends/s3/s3.go | 11 +++++++++++ backends/storage.go | 3 +++ cleanup/cleanup.go | 8 ++++---- server.go | 12 ++++++++++-- upload.go | 16 ++++++++++++++++ 6 files changed, 70 insertions(+), 7 deletions(-) diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index 3f48299..bd68d20 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -16,6 +16,7 @@ import ( type LocalfsBackend struct { metaPath string filesPath string + locksPath string } type MetadataJSON struct { @@ -127,6 +128,29 @@ func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) er return nil } +func (b LocalfsBackend) Lock(filename string) (err error) { + lockPath := path.Join(b.locksPath, filename) + + lock, err := os.Create(lockPath) + if err != nil { + return err + } + + lock.Close() + return +} + +func (b LocalfsBackend) Unlock(filename string) (err error) { + lockPath := path.Join(b.locksPath, filename) + + err = os.Remove(lockPath) + if err != nil { + return err + } + + return +} + func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { filePath := path.Join(b.filesPath, key) @@ -201,9 +225,10 @@ func (b LocalfsBackend) List() ([]string, error) { return output, nil } -func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend { +func NewLocalfsBackend(metaPath string, filesPath string, locksPath string) LocalfsBackend { return LocalfsBackend{ metaPath: metaPath, filesPath: filesPath, + locksPath: locksPath, } } diff --git a/backends/s3/s3.go b/backends/s3/s3.go index 58ccd14..b48837f 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -3,6 +3,7 @@ package s3 import ( "io" "io/ioutil" + "log" "net/http" "os" "strconv" @@ -156,6 +157,16 @@ func unmapMetadata(input map[string]*string) (m backends.Metadata, err error) { return } +func (b S3Backend) Lock(filename string) (err error) { + log.Printf("Locking is not supported on S3") + return +} + +func (b S3Backend) Unlock(filename string) (err error) { + log.Printf("Locking is not supported on S3") + return +} + func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { tmpDst, err := ioutil.TempFile("", "linx-server-upload") if err != nil { diff --git a/backends/storage.go b/backends/storage.go index 4e2ad6b..119ddb6 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -12,6 +12,8 @@ type StorageBackend interface { Exists(key string) (bool, error) Head(key string) (Metadata, error) Get(key string) (Metadata, io.ReadCloser, error) + Lock(filename string) (error) + Unlock(filename string) (error) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (Metadata, error) PutMetadata(key string, m Metadata) error ServeFile(key string, w http.ResponseWriter, r *http.Request) error @@ -25,3 +27,4 @@ type MetaStorageBackend interface { var NotFoundErr = errors.New("File not found.") var FileEmptyError = errors.New("Empty file") +var FileLockedError = errors.New("Locked file") diff --git a/cleanup/cleanup.go b/cleanup/cleanup.go index 5920c22..6230b00 100644 --- a/cleanup/cleanup.go +++ b/cleanup/cleanup.go @@ -8,8 +8,8 @@ import ( "github.com/andreimarcu/linx-server/expiry" ) -func Cleanup(filesDir string, metaDir string, noLogs bool) { - fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir) +func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { + fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir, locksDir) files, err := fileBackend.List() if err != nil { @@ -33,10 +33,10 @@ func Cleanup(filesDir string, metaDir string, noLogs bool) { } } -func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) { +func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, locksDir string, noLogs bool) { c := time.Tick(minutes) for range c { - Cleanup(filesDir, metaDir, noLogs) + Cleanup(filesDir, metaDir, locksDir, noLogs) } } diff --git a/server.go b/server.go index 41cd932..b34dfd4 100644 --- a/server.go +++ b/server.go @@ -43,6 +43,7 @@ var Config struct { bind string filesDir string metaDir string + locksDir string siteName string siteURL string sitePath string @@ -137,6 +138,11 @@ func setup() *web.Mux { log.Fatal("Could not create metadata directory:", err) } + err = os.MkdirAll(Config.locksDir, 0700) + if err != nil { + log.Fatal("Could not create locks directory:", err) + } + if Config.siteURL != "" { // ensure siteURL ends wth '/' if lastChar := Config.siteURL[len(Config.siteURL)-1:]; lastChar != "/" { @@ -161,9 +167,9 @@ func setup() *web.Mux { if Config.s3Bucket != "" { storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle) } else { - storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) + storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir, Config.locksDir) if Config.cleanupEveryMinutes > 0 { - go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs) + go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.locksDir, Config.noLogs) } } @@ -249,6 +255,8 @@ func main() { "path to files directory") flag.StringVar(&Config.metaDir, "metapath", "meta/", "path to metadata directory") + flag.StringVar(&Config.locksDir, "lockspath", "locks/", + "path to metadata directory") flag.BoolVar(&Config.basicAuth, "basicauth", false, "allow logging by basic auth password") flag.BoolVar(&Config.noLogs, "nologs", false, diff --git a/upload.go b/upload.go index 8e5fc91..87ef6ba 100644 --- a/upload.go +++ b/upload.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "net/url" "path" @@ -318,6 +319,13 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { return upload, errors.New("Prohibited filename") } + // Lock the upload + log.Printf("Lock %s", upload.Filename) + err = storageBackend.Lock(upload.Filename) + if err != nil { + return upload, err + } + // Get the rest of the metadata needed for storage var fileExpiry time.Time maxDurationTime := time.Duration(Config.maxDurationTime) * time.Second @@ -341,11 +349,19 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { if Config.disableAccessKey == true { upReq.accessKey = "" } + log.Printf("Write %s", upload.Filename) upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deleteKey, upReq.accessKey, upReq.srcIp) if err != nil { return upload, err } + // Unlock the upload + log.Printf("Unlock %s", upload.Filename) + err = storageBackend.Unlock(upload.Filename) + if err != nil { + return upload, err + } + return } From 4a9a4f7be2084cf8019c45136df0520a5b407602 Mon Sep 17 00:00:00 2001 From: BBaoVanC Date: Thu, 23 Dec 2021 18:24:50 -0600 Subject: [PATCH 2/5] Add check for locks in cleanup.go --- backends/localfs/localfs.go | 13 +++++++++++++ backends/s3/s3.go | 5 +++++ backends/storage.go | 2 +- cleanup/cleanup.go | 11 +++++++++++ linx-cleanup/linx-cleanup.go | 3 +++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index bd68d20..9a72891 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -2,6 +2,7 @@ package localfs import ( "encoding/json" + "errors" "io" "io/ioutil" "net/http" @@ -151,6 +152,18 @@ func (b LocalfsBackend) Unlock(filename string) (err error) { return } +func (b LocalfsBackend) CheckLock(filename string) (locked bool, err error) { + lockPath := path.Join(b.locksPath, filename) + + if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { + return false, nil + } else { + return true, nil + } + + return false, err +} + func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { filePath := path.Join(b.filesPath, key) diff --git a/backends/s3/s3.go b/backends/s3/s3.go index b48837f..17f6725 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -167,6 +167,11 @@ func (b S3Backend) Unlock(filename string) (err error) { return } +func (b S3Backend) CheckLock(filename string) (locked bool, err error) { + log.Printf("Locking is not supported on S3") + return +} + func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { tmpDst, err := ioutil.TempFile("", "linx-server-upload") if err != nil { diff --git a/backends/storage.go b/backends/storage.go index 119ddb6..02e2a57 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -14,6 +14,7 @@ type StorageBackend interface { Get(key string) (Metadata, io.ReadCloser, error) Lock(filename string) (error) Unlock(filename string) (error) + CheckLock(filename string) (bool, error) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (Metadata, error) PutMetadata(key string, m Metadata) error ServeFile(key string, w http.ResponseWriter, r *http.Request) error @@ -27,4 +28,3 @@ type MetaStorageBackend interface { var NotFoundErr = errors.New("File not found.") var FileEmptyError = errors.New("Empty file") -var FileLockedError = errors.New("Locked file") diff --git a/cleanup/cleanup.go b/cleanup/cleanup.go index 6230b00..97789d5 100644 --- a/cleanup/cleanup.go +++ b/cleanup/cleanup.go @@ -17,6 +17,15 @@ func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { } for _, filename := range files { + locked, err := fileBackend.CheckLock(filename) + if err != nil { + log.Printf("Error checking if %s is locked: %s", filename, err) + } + if locked { + log.Printf("%s is locked, it will be ignored", filename) + continue + } + metadata, err := fileBackend.Head(filename) if err != nil { if !noLogs { @@ -36,7 +45,9 @@ func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, locksDir string, noLogs bool) { c := time.Tick(minutes) for range c { + log.Printf("Running periodic cleanup") Cleanup(filesDir, metaDir, locksDir, noLogs) + log.Printf("Finished periodic cleanup") } } diff --git a/linx-cleanup/linx-cleanup.go b/linx-cleanup/linx-cleanup.go index 13b3ef1..7953ab4 100644 --- a/linx-cleanup/linx-cleanup.go +++ b/linx-cleanup/linx-cleanup.go @@ -9,12 +9,15 @@ import ( func main() { var filesDir string var metaDir string + var locksDir string var noLogs bool flag.StringVar(&filesDir, "filespath", "files/", "path to files directory") flag.StringVar(&metaDir, "metapath", "meta/", "path to metadata directory") + flag.StringVar(&metaDir, "lockspath", "locks/", + "path to metadata directory") flag.BoolVar(&noLogs, "nologs", false, "don't log deleted files") flag.Parse() From 953f72da76dc7dcc912cf7480ff1e039ce96549e Mon Sep 17 00:00:00 2001 From: BBaoVanC Date: Thu, 23 Dec 2021 19:04:09 -0600 Subject: [PATCH 3/5] Make the locks dir with 755 instead of 700 --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index b34dfd4..f6a45a9 100644 --- a/server.go +++ b/server.go @@ -138,7 +138,7 @@ func setup() *web.Mux { log.Fatal("Could not create metadata directory:", err) } - err = os.MkdirAll(Config.locksDir, 0700) + err = os.MkdirAll(Config.locksDir, 0755) if err != nil { log.Fatal("Could not create locks directory:", err) } From 181e65bd217dd7a4130b57c826bbaf9a48dd94d9 Mon Sep 17 00:00:00 2001 From: BBaoVanC Date: Thu, 23 Dec 2021 19:21:45 -0600 Subject: [PATCH 4/5] Create locks path in Dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5a73bf..e9d61f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,11 +18,11 @@ ENV SSL_CERT_FILE /etc/ssl/cert.pem COPY static /go/src/github.com/andreimarcu/linx-server/static/ COPY templates /go/src/github.com/andreimarcu/linx-server/templates/ -RUN mkdir -p /data/files && mkdir -p /data/meta && chown -R 65534:65534 /data +RUN mkdir -p /data/files && mkdir -p /data/meta && mkdir -p /data/locks && chown -R 65534:65534 /data -VOLUME ["/data/files", "/data/meta"] +VOLUME ["/data/files", "/data/meta", "/data/locks"] EXPOSE 8080 USER nobody -ENTRYPOINT ["/usr/local/bin/linx-server", "-bind=0.0.0.0:8080", "-filespath=/data/files/", "-metapath=/data/meta/"] +ENTRYPOINT ["/usr/local/bin/linx-server", "-bind=0.0.0.0:8080", "-filespath=/data/files/", "-metapath=/data/meta/", "-lockspath=/data/locks/"] CMD ["-sitename=linx", "-allowhotlink"] From 3b57cef637a3b16295a0a4df528f303c130738f8 Mon Sep 17 00:00:00 2001 From: BBaoVanC Date: Tue, 28 Dec 2021 18:41:56 -0600 Subject: [PATCH 5/5] Fix some naming and other modules that I missed --- csp_test.go | 1 + linx-cleanup/linx-cleanup.go | 6 +++--- server.go | 2 +- server_test.go | 2 ++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/csp_test.go b/csp_test.go index e3dbbdd..b996594 100644 --- a/csp_test.go +++ b/csp_test.go @@ -20,6 +20,7 @@ func TestContentSecurityPolicy(t *testing.T) { Config.siteURL = "http://linx.example.org/" Config.filesDir = path.Join(os.TempDir(), generateBarename()) Config.metaDir = Config.filesDir + "_meta" + Config.locksDir = Config.filesDir + "_locks" Config.maxSize = 1024 * 1024 * 1024 Config.noLogs = true Config.siteName = "linx" diff --git a/linx-cleanup/linx-cleanup.go b/linx-cleanup/linx-cleanup.go index 7953ab4..08d8363 100644 --- a/linx-cleanup/linx-cleanup.go +++ b/linx-cleanup/linx-cleanup.go @@ -16,11 +16,11 @@ func main() { "path to files directory") flag.StringVar(&metaDir, "metapath", "meta/", "path to metadata directory") - flag.StringVar(&metaDir, "lockspath", "locks/", - "path to metadata directory") + flag.StringVar(&locksDir, "lockspath", "locks/", + "path to locks directory") flag.BoolVar(&noLogs, "nologs", false, "don't log deleted files") flag.Parse() - cleanup.Cleanup(filesDir, metaDir, noLogs) + cleanup.Cleanup(filesDir, metaDir, locksDir, noLogs) } diff --git a/server.go b/server.go index f6a45a9..ec66f75 100644 --- a/server.go +++ b/server.go @@ -256,7 +256,7 @@ func main() { flag.StringVar(&Config.metaDir, "metapath", "meta/", "path to metadata directory") flag.StringVar(&Config.locksDir, "lockspath", "locks/", - "path to metadata directory") + "path to locks directory") flag.BoolVar(&Config.basicAuth, "basicauth", false, "allow logging by basic auth password") flag.BoolVar(&Config.noLogs, "nologs", false, diff --git a/server_test.go b/server_test.go index fc225ce..c64cfdc 100644 --- a/server_test.go +++ b/server_test.go @@ -33,6 +33,7 @@ func TestSetup(t *testing.T) { Config.siteURL = "http://linx.example.org/" Config.filesDir = path.Join(os.TempDir(), generateBarename()) Config.metaDir = Config.filesDir + "_meta" + Config.locksDir = Config.filesDir + "_locks" Config.maxSize = 1024 * 1024 * 1024 Config.noLogs = true Config.siteName = "linx" @@ -1255,6 +1256,7 @@ func TestInferSiteURLHTTPSFastCGI(t *testing.T) { func TestShutdown(t *testing.T) { os.RemoveAll(Config.filesDir) os.RemoveAll(Config.metaDir) + os.RemoveAll(Config.locksDir) } func TestPutAndGetCLI(t *testing.T) {