From 9f7e6b6eea01ef3fa19cbcfdfb952021ca6aee53 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Wed, 18 Mar 2020 17:26:43 -0700 Subject: [PATCH 001/130] Torrent: Ranged requests so torrents work --- backends/localfs/localfs.go | 13 +++++++++++++ backends/s3/s3.go | 38 +++++++++++++++++++++++++++++++++++++ backends/storage.go | 2 ++ fileserve.go | 11 +++-------- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index 42e32b8..aaf487f 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -4,6 +4,7 @@ import ( "encoding/json" "io" "io/ioutil" + "net/http" "os" "path" "time" @@ -82,6 +83,18 @@ func (b LocalfsBackend) Get(key string) (metadata backends.Metadata, f io.ReadCl return } +func (b LocalfsBackend) ServeFile(key string, w http.ResponseWriter, r *http.Request) (err error) { + _, err = b.Head(key) + if err != nil { + return + } + + filePath := path.Join(b.filesPath, key) + http.ServeFile(w, r, filePath) + + return +} + func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) error { metaPath := path.Join(b.metaPath, key) diff --git a/backends/s3/s3.go b/backends/s3/s3.go index bfc6e1c..a558779 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -3,6 +3,7 @@ package s3 import ( "io" "io/ioutil" + "net/http" "os" "strconv" "time" @@ -79,6 +80,43 @@ func (b S3Backend) Get(key string) (metadata backends.Metadata, r io.ReadCloser, return } +func (b S3Backend) ServeFile(key string, w http.ResponseWriter, r *http.Request) (err error) { + var result *s3.GetObjectOutput + + if r.Header.Get("Range") != "" { + result, err = b.svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + Range: aws.String(r.Header.Get("Range")), + }) + + w.WriteHeader(206) + w.Header().Set("Content-Range", *result.ContentRange) + w.Header().Set("Content-Length", strconv.FormatInt(*result.ContentLength, 10)) + w.Header().Set("Accept-Ranges", "bytes") + + } else { + result, err = b.svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(key), + }) + + } + + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() == s3.ErrCodeNoSuchKey || aerr.Code() == "NotFound" { + err = backends.NotFoundErr + } + } + return + } + + _, err = io.Copy(w, result.Body) + + return +} + func mapMetadata(m backends.Metadata) map[string]*string { return map[string]*string{ "Expiry": aws.String(strconv.FormatInt(m.Expiry.Unix(), 10)), diff --git a/backends/storage.go b/backends/storage.go index 5d973c4..864d0a1 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -3,6 +3,7 @@ package backends import ( "errors" "io" + "net/http" "time" ) @@ -13,6 +14,7 @@ type StorageBackend interface { Get(key string) (Metadata, io.ReadCloser, error) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string) (Metadata, error) PutMetadata(key string, m Metadata) error + ServeFile(key string, w http.ResponseWriter, r *http.Request) error Size(key string) (int64, error) } diff --git a/fileserve.go b/fileserve.go index 27a28a9..90c1507 100644 --- a/fileserve.go +++ b/fileserve.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io" "net/http" "net/url" "strconv" @@ -61,15 +60,11 @@ func fileServeHandler(c web.C, w http.ResponseWriter, r *http.Request) { } if r.Method != "HEAD" { - _, reader, err := storageBackend.Get(fileName) - if err != nil { - oopsHandler(c, w, r, RespAUTO, "Unable to open file.") - return - } - defer reader.Close() - if _, err = io.CopyN(w, reader, metadata.Size); err != nil { + storageBackend.ServeFile(fileName, w, r) + if err != nil { oopsHandler(c, w, r, RespAUTO, err.Error()) + return } } } From 8edf53c14203bca15b39a9778abe23e14d707a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn?= Date: Mon, 13 Apr 2020 21:36:03 +0200 Subject: [PATCH 002/130] Add systemd unit/timer for the cleanup tool (#210) * add systemd service for the cleanup script * add systemd timer for the cleanup service --- linx-cleanup.service | 12 ++++++++++++ linx-cleanup.timer | 8 ++++++++ 2 files changed, 20 insertions(+) create mode 100644 linx-cleanup.service create mode 100644 linx-cleanup.timer diff --git a/linx-cleanup.service b/linx-cleanup.service new file mode 100644 index 0000000..7a03941 --- /dev/null +++ b/linx-cleanup.service @@ -0,0 +1,12 @@ +[Unit] +Description=Self-hosted file/code/media sharing (expired files cleanup) +After=network.target + +[Service] +User=linx +Group=linx +ExecStart=/usr/bin/linx-cleanup +WorkingDirectory=/srv/linx/ + +[Install] +WantedBy=multi-user.target diff --git a/linx-cleanup.timer b/linx-cleanup.timer new file mode 100644 index 0000000..40d8cda --- /dev/null +++ b/linx-cleanup.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run linx-cleanup every hour + +[Timer] +OnUnitActiveSec=1h + +[Install] +WantedBy=timers.target From bc599ae0188b428dd07cc9e0dc6a1ce53c44dd57 Mon Sep 17 00:00:00 2001 From: mutantmonkey <67266+mutantmonkey@users.noreply.github.com> Date: Thu, 16 Apr 2020 21:18:21 +0000 Subject: [PATCH 003/130] Fix typo in readme (#213) This should say linx-cleanup, not linx-client. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ff22db3..c7e0adf 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ automatically clean up files that have expired, you can use the included `linx-cleanup` utility. To run it automatically, use a cronjob or similar type of scheduled task. -You should be careful to ensure that only one instance of `linx-client` runs at +You should be careful to ensure that only one instance of `linx-cleanup` runs at a time to avoid unexpected behavior. It does not implement any type of locking. From 151515f516c615c15f418c3753c400db21a4bd12 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Wed, 13 May 2020 17:37:33 -0700 Subject: [PATCH 004/130] Cleanup: Integrate expired files periodic cleanup --- .gitignore | 4 +-- README.md | 33 +++++++------------ {linx-cleanup => cleanup}/cleanup.go | 26 +++++++-------- linx-cleanup/README.md | 19 +++++++++++ linx-cleanup/linx-cleanup.go | 23 +++++++++++++ .../linx-cleanup.service | 0 .../linx-cleanup.timer | 0 server.go | 8 +++++ 8 files changed, 75 insertions(+), 38 deletions(-) rename {linx-cleanup => cleanup}/cleanup.go (63%) create mode 100644 linx-cleanup/README.md create mode 100644 linx-cleanup/linx-cleanup.go rename linx-cleanup.service => linx-cleanup/linx-cleanup.service (100%) rename linx-cleanup.timer => linx-cleanup/linx-cleanup.timer (100%) diff --git a/.gitignore b/.gitignore index c6750eb..ec613f1 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,8 @@ _testmain.go *.prof linx-server -linx-cleanup -linx-genkey +linx-cleanup/linx-cleanup +linx-genkey/linx-genkey files/ meta/ binaries/ diff --git a/README.md b/README.md index c7e0adf..567d646 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,17 @@ allowhotlink = true | ```-force-random-filename``` | (optionally) force the use of random filenames | ```-custompagespath "custom_pages"``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation + +#### Cleaning up expired files +When files expire, access is disabled immediately, but the files and metadata +will persist on disk until someone attempts to access them. You can set the following option to run cleanup every few minutes. This can also be done using a separate utility found the linx-cleanup directory. + + +|Option|Description +|------|----------- +| ```-cleanup-every-minutes 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed) + + #### Require API Keys for uploads |Option|Description @@ -127,26 +138,6 @@ The following storage backends are available: |------|----------- | ```-fastcgi``` | serve through fastcgi - -Cleaning up expired files -------------------------- -When files expire, access is disabled immediately, but the files and metadata -will persist on disk until someone attempts to access them. If you'd like to -automatically clean up files that have expired, you can use the included -`linx-cleanup` utility. To run it automatically, use a cronjob or similar type -of scheduled task. - -You should be careful to ensure that only one instance of `linx-cleanup` runs at -a time to avoid unexpected behavior. It does not implement any type of locking. - - -|Option|Description -|------|----------- -| ```-filespath files/``` | Path to stored uploads (default is files/) -| ```-nologs``` | (optionally) disable deletion logs in stdout -| ```-metapath meta/``` | Path to stored information about uploads (default is meta/) - - Deployment ---------- Linx-server supports being deployed in a subdirectory (ie. example.com/mylinx/) as well as on its own (example.com/). @@ -207,4 +198,4 @@ along with this program. If not, see . Author ------- -Andrei Marcu, http://andreim.net/ +Andrei Marcu, https://andreim.net/ diff --git a/linx-cleanup/cleanup.go b/cleanup/cleanup.go similarity index 63% rename from linx-cleanup/cleanup.go rename to cleanup/cleanup.go index 88c2bce..5920c22 100644 --- a/linx-cleanup/cleanup.go +++ b/cleanup/cleanup.go @@ -1,26 +1,14 @@ -package main +package cleanup import ( - "flag" "log" + "time" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/expiry" ) -func main() { - var filesDir string - var metaDir string - var noLogs bool - - flag.StringVar(&filesDir, "filespath", "files/", - "path to files directory") - flag.StringVar(&metaDir, "metapath", "meta/", - "path to metadata directory") - flag.BoolVar(&noLogs, "nologs", false, - "don't log deleted files") - flag.Parse() - +func Cleanup(filesDir string, metaDir string, noLogs bool) { fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir) files, err := fileBackend.List() @@ -44,3 +32,11 @@ func main() { } } } + +func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) { + c := time.Tick(minutes) + for range c { + Cleanup(filesDir, metaDir, noLogs) + } + +} diff --git a/linx-cleanup/README.md b/linx-cleanup/README.md new file mode 100644 index 0000000..7d4f4a3 --- /dev/null +++ b/linx-cleanup/README.md @@ -0,0 +1,19 @@ + +linx-cleanup +------------------------- +When files expire, access is disabled immediately, but the files and metadata +will persist on disk until someone attempts to access them. + +If you'd like to automatically clean up files that have expired, you can use the included `linx-cleanup` utility. To run it automatically, use a cronjob or similar type +of scheduled task. + +You should be careful to ensure that only one instance of `linx-cleanup` runs at +a time to avoid unexpected behavior. It does not implement any type of locking. + + +|Option|Description +|------|----------- +| ```-filespath files/``` | Path to stored uploads (default is files/) +| ```-nologs``` | (optionally) disable deletion logs in stdout +| ```-metapath meta/``` | Path to stored information about uploads (default is meta/) + diff --git a/linx-cleanup/linx-cleanup.go b/linx-cleanup/linx-cleanup.go new file mode 100644 index 0000000..13b3ef1 --- /dev/null +++ b/linx-cleanup/linx-cleanup.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + + "github.com/andreimarcu/linx-server/cleanup" +) + +func main() { + var filesDir string + var metaDir string + var noLogs bool + + flag.StringVar(&filesDir, "filespath", "files/", + "path to files directory") + flag.StringVar(&metaDir, "metapath", "meta/", + "path to metadata directory") + flag.BoolVar(&noLogs, "nologs", false, + "don't log deleted files") + flag.Parse() + + cleanup.Cleanup(filesDir, metaDir, noLogs) +} diff --git a/linx-cleanup.service b/linx-cleanup/linx-cleanup.service similarity index 100% rename from linx-cleanup.service rename to linx-cleanup/linx-cleanup.service diff --git a/linx-cleanup.timer b/linx-cleanup/linx-cleanup.timer similarity index 100% rename from linx-cleanup.timer rename to linx-cleanup/linx-cleanup.timer diff --git a/server.go b/server.go index e1f0c0a..dae3491 100644 --- a/server.go +++ b/server.go @@ -19,6 +19,7 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/backends/s3" + "github.com/andreimarcu/linx-server/cleanup" "github.com/flosch/pongo2" "github.com/vharitonsky/iniflags" "github.com/zenazn/goji/graceful" @@ -71,6 +72,7 @@ var Config struct { forceRandomFilename bool accessKeyCookieExpiry uint64 customPagesDir string + cleanupEveryMinutes uint64 } var Templates = make(map[string]*pongo2.Template) @@ -150,6 +152,10 @@ func setup() *web.Mux { storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle) } else { storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) + if Config.cleanupEveryMinutes > 0 { + go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs) + } + } // Template setup @@ -311,6 +317,8 @@ func main() { flag.Uint64Var(&Config.accessKeyCookieExpiry, "access-cookie-expiry", 0, "Expiration time for access key cookies in seconds (set 0 to use session cookies)") flag.StringVar(&Config.customPagesDir, "custompagespath", "", "path to directory containing .md files to render as custom pages") + flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0, + "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") iniflags.Parse() From e2a65a5b62a42f94d985ca7ffa3fa5e0d3673199 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Thu, 14 May 2020 00:51:19 -0700 Subject: [PATCH 005/130] README: Clarify docker usage and example --- .gitignore | 1 + README.md | 81 ++++++++++++++++++++++------------------ linx-server.conf.example | 12 ++++++ 3 files changed, 57 insertions(+), 37 deletions(-) create mode 100644 linx-server.conf.example diff --git a/.gitignore b/.gitignore index ec613f1..df2bae9 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ _testmain.go linx-server linx-cleanup/linx-cleanup linx-genkey/linx-genkey +linx-server.conf files/ meta/ binaries/ diff --git a/README.md b/README.md index 567d646..d509689 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,14 @@ Getting started ------------------- #### Using Docker +1. Create directories ```files``` and ```meta``` and run ```chown -R 65534:65534 meta && chown -R 65534:65534 files``` +2. Create a config file (example provided in repo), we'll refer to it as __linx-server.conf__ in the following examples + + + Example running ``` -docker run -p 8080:8080 -v /path/to/meta:/data/meta -v /path/to/files:/data/files andreimarcu/linx-server +docker run -p 8080:8080 -v /path/to/linx-server.conf:/data/linx-server.conf -v /path/to/meta:/data/meta -v /path/to/files:/data/files andreimarcu/linx-server -config /data/linx-server.conf ``` Example with docker-compose @@ -40,11 +45,12 @@ services: linx-server: container_name: linx-server image: andreimarcu/linx-server - entrypoint: /usr/local/bin/linx-server -bind=0.0.0.0:8080 -filespath=/data/files/ -metapath=/data/meta/ - command: -sitename=Linx -siteurl=https://linx.example.com + entrypoint: /usr/local/bin/linx-server + command: -config /data/linx-server.conf volumes: - /path/to/files:/data/files - /path/to/meta:/data/meta + - /path/to/linx-server.conf:/data/linx-server.conf network_mode: bridge ports: - "8080:8080" @@ -57,40 +63,41 @@ Ideally, you would use a reverse proxy such as nginx or caddy to handle TLS cert 1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases) 2. Run ```./linx-server``` - Usage ----- #### Configuration -All configuration options are accepted either as arguments or can be placed in an ini-style file as such: +All configuration options are accepted either as arguments or can be placed in a file as such (see example file linx-server.conf.example in repo): ```ini +bind = 127.0.0.1:8080 +sitename = myLinx maxsize = 4294967296 -allowhotlink = true -# etc -``` -...and then invoke ```linx-server -config path/to/config.ini``` +maxexpiry = 86400 +# ... etc +``` +...and then run ```linx-server -config path/to/linx-server.conf``` #### Options |Option|Description |------|----------- -| ```-bind 127.0.0.1:8080``` | what to bind to (default is 127.0.0.1:8080) -| ```-sitename myLinx``` | the site name displayed on top (default is inferred from Host header) -| ```-siteurl "https://mylinx.example.org/"``` | the site url (default is inferred from execution context) -| ```-selifpath "selif"``` | path relative to site base url (the "selif" in mylinx.example.org/selif/image.jpg) where files are accessed directly (default: selif) -| ```-maxsize 4294967296``` | maximum upload file size in bytes (default 4GB) -| ```-maxexpiry 86400``` | maximum expiration time in seconds (default is 0, which is no expiry) -| ```-allowhotlink``` | Allow file hotlinking -| ```-contentsecuritypolicy "..."``` | Content-Security-Policy header for pages (default is "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';") -| ```-filecontentsecuritypolicy "..."``` | Content-Security-Policy header for files (default is "default-src 'none'; img-src 'self'; object-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';") -| ```-refererpolicy "..."``` | Referrer-Policy header for pages (default is "same-origin") -| ```-filereferrerpolicy "..."``` | Referrer-Policy header for files (default is "same-origin") -| ```-xframeoptions "..." ``` | X-Frame-Options header (default is "SAMEORIGIN") -| ```-remoteuploads``` | (optionally) enable remote uploads (/upload?url=https://...) -| ```-nologs``` | (optionally) disable request logs in stdout -| ```-force-random-filename``` | (optionally) force the use of random filenames -| ```-custompagespath "custom_pages"``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation +| ```bind = 127.0.0.1:8080``` | what to bind to (default is 127.0.0.1:8080) +| ```sitename = myLinx``` | the site name displayed on top (default is inferred from Host header) +| ```siteurl = https://mylinx.example.org/``` | the site url (default is inferred from execution context) +| ```selifpath = selif``` | path relative to site base url (the "selif" in mylinx.example.org/selif/image.jpg) where files are accessed directly (default: selif) +| ```maxsize = 4294967296``` | maximum upload file size in bytes (default 4GB) +| ```maxexpiry = 86400``` | maximum expiration time in seconds (default is 0, which is no expiry) +| ```allowhotlink = true``` | Allow file hotlinking +| ```contentsecuritypolicy = "..."``` | Content-Security-Policy header for pages (default is "default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';") +| ```filecontentsecuritypolicy = "..."``` | Content-Security-Policy header for files (default is "default-src 'none'; img-src 'self'; object-src 'self'; media-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self';") +| ```refererpolicy = "..."``` | Referrer-Policy header for pages (default is "same-origin") +| ```filereferrerpolicy = "..."``` | Referrer-Policy header for files (default is "same-origin") +| ```xframeoptions = "..." ``` | X-Frame-Options header (default is "SAMEORIGIN") +| ```remoteuploads = true``` | (optionally) enable remote uploads (/upload?url=https://...) +| ```nologs = true``` | (optionally) disable request logs in stdout +| ```force-random-filename = true``` | (optionally) force the use of random filenames +| ```custompagespath = custom_pages/``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation #### Cleaning up expired files @@ -100,16 +107,16 @@ will persist on disk until someone attempts to access them. You can set the foll |Option|Description |------|----------- -| ```-cleanup-every-minutes 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed) +| ```cleanup-every-minutes = 5``` | How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed) #### Require API Keys for uploads |Option|Description |------|----------- -| ```-authfile path/to/authfile``` | (optionally) require authorization for upload/delete by providing a newline-separated file of scrypted auth keys -| ```-remoteauthfile path/to/remoteauthfile``` | (optionally) require authorization for remote uploads by providing a newline-separated file of scrypted auth keys -| ```-basicauth``` | (optionally) allow basic authorization to upload or paste files from browser when `-authfile` is enabled. When uploading, you will be prompted to enter a user and password - leave the user blank and use your auth key as the password +| ```authfile = path/to/authfile``` | (optionally) require authorization for upload/delete by providing a newline-separated file of scrypted auth keys +| ```remoteauthfile = path/to/remoteauthfile``` | (optionally) require authorization for remote uploads by providing a newline-separated file of scrypted auth keys +| ```basicauth = true``` | (optionally) allow basic authorization to upload or paste files from browser when `-authfile` is enabled. When uploading, you will be prompted to enter a user and password - leave the user blank and use your auth key as the password A helper utility ```linx-genkey``` is provided which hashes keys to the format required in the auth files. @@ -118,25 +125,25 @@ The following storage backends are available: |Name|Notes|Options |----|-----|------- -|LocalFS|Enabled by default, this backend uses the filesystem|```-filespath files/``` -- Path to store uploads (default is files/)
```-metapath meta/``` -- Path to store information about uploads (default is meta/)| -|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```-s3-endpoint https://...``` -- S3 endpoint
```-s3-region us-east-1``` -- S3 region
```-s3-bucket mybucket``` -- S3 bucket to use for files and metadata
```-s3-force-path-style``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| +|LocalFS|Enabled by default, this backend uses the filesystem|```filespath = files/``` -- Path to store uploads (default is files/)
```metapath = meta/``` -- Path to store information about uploads (default is meta/)| +|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| #### SSL with built-in server |Option|Description |------|----------- -| ```-certfile path/to/your.crt``` | Path to the ssl certificate (required if you want to use the https server) -| ```-keyfile path/to/your.key``` | Path to the ssl key (required if you want to use the https server) +| ```certfile = path/to/your.crt``` | Path to the ssl certificate (required if you want to use the https server) +| ```keyfile = path/to/your.key``` | Path to the ssl key (required if you want to use the https server) #### Use with http proxy |Option|Description |------|----------- -| ```-realip``` | let linx-server know you (nginx, etc) are providing the X-Real-IP and/or X-Forwarded-For headers. +| ```realip = true``` | let linx-server know you (nginx, etc) are providing the X-Real-IP and/or X-Forwarded-For headers. #### Use with fastcgi |Option|Description |------|----------- -| ```-fastcgi``` | serve through fastcgi +| ```fastcgi = true``` | serve through fastcgi Deployment ---------- @@ -161,10 +168,10 @@ server { } } ``` -And run linx-server with the ```-fastcgi``` option. +And run linx-server with the ```fastcgi = true``` option. #### 2. Using the built-in https server -Run linx-server with the ```-certfile path/to/cert.file``` and ```-keyfile path/to/key.file``` options. +Run linx-server with the ```certfile = path/to/cert.file``` and ```keyfile = path/to/key.file``` options. #### 3. Using the built-in http server Run linx-server normally. diff --git a/linx-server.conf.example b/linx-server.conf.example new file mode 100644 index 0000000..eb2e1f8 --- /dev/null +++ b/linx-server.conf.example @@ -0,0 +1,12 @@ + +bind = 127.0.0.1:8080 +sitename = myLinx +siteurl = https://mylinx.example.org/ +selifpath = s +maxsize = 4294967296 +maxexpiry = 86400 +allowhotlink = true +remoteuploads = true +nologs = true +force-random-filename = false +cleanup-every-minutes = 5 \ No newline at end of file From 6ce2bd6b9f0685bf5d11e24c6a012258f618ae32 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Thu, 14 May 2020 01:12:24 -0700 Subject: [PATCH 006/130] Display pages: Add OpenGraph tags for media --- display.go | 1 + templates/display/audio.html | 7 +++++-- templates/display/image.html | 6 +++++- templates/display/video.html | 8 ++++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/display.go b/display.go index 6ac87d6..6228216 100644 --- a/display.go +++ b/display.go @@ -122,6 +122,7 @@ func fileDisplayHandler(c web.C, w http.ResponseWriter, r *http.Request, fileNam "forcerandom": Config.forceRandomFilename, "lines": lines, "files": metadata.ArchiveFiles, + "siteurl": strings.TrimSuffix(getSiteURL(r), "/"), }, r, w) if err != nil { diff --git a/templates/display/audio.html b/templates/display/audio.html index b5ae1e3..ba491f0 100644 --- a/templates/display/audio.html +++ b/templates/display/audio.html @@ -1,9 +1,12 @@ {% extends "base.html" %} +{% block head %} + +{% endblock %} + {% block main %} -{% endblock %} - +{% endblock %} \ No newline at end of file diff --git a/templates/display/image.html b/templates/display/image.html index 807b7ad..985c504 100644 --- a/templates/display/image.html +++ b/templates/display/image.html @@ -1,7 +1,11 @@ {% extends "base.html" %} +{% block head %} + +{% endblock %} + {% block main %} -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/templates/display/video.html b/templates/display/video.html index 317664b..54a796f 100644 --- a/templates/display/video.html +++ b/templates/display/video.html @@ -1,8 +1,12 @@ {% extends "base.html" %} +{% block head %} + +{% endblock %} + {% block main %} -{% endblock %} +{% endblock %} \ No newline at end of file From 5eb6f32ff0a0252caab5fbc8e154912b4b5e4676 Mon Sep 17 00:00:00 2001 From: Infinoid Date: Mon, 3 Aug 2020 01:16:47 -0400 Subject: [PATCH 007/130] Switch to a more comprehensive mimetype detection library (#231) --- go.mod | 2 +- go.sum | 4 ++-- helpers/helpers.go | 17 ++++----------- helpers/helpers_test.go | 46 ++++++++++++++++++++++++++++++++++++++++- upload.go | 8 +++---- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 50a48ab..f433699 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 github.com/dustin/go-humanize v1.0.0 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 + github.com/gabriel-vasile/mimetype v1.1.1 github.com/microcosm-cc/bluemonday v1.0.2 github.com/minio/sha256-simd v0.1.1 github.com/russross/blackfriday v1.5.1 @@ -15,5 +16,4 @@ require ( github.com/zeebo/bencode v1.0.0 github.com/zenazn/goji v0.9.0 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 - gopkg.in/h2non/filetype.v1 v1.0.5 ) diff --git a/go.sum b/go.sum index 15a736c..99d63bf 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/gabriel-vasile/mimetype v1.1.1 h1:qbN9MPuRf3bstHu9zkI9jDWNfH//9+9kHxr9oRBBBOA= +github.com/gabriel-vasile/mimetype v1.1.1/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -68,8 +70,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= -gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/helpers/helpers.go b/helpers/helpers.go index f51d998..f13e302 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -7,8 +7,8 @@ import ( "unicode" "github.com/andreimarcu/linx-server/backends" + "github.com/gabriel-vasile/mimetype" "github.com/minio/sha256-simd" - "gopkg.in/h2non/filetype.v1" ) func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { @@ -21,7 +21,7 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { // Get first 512 bytes for mimetype detection header := make([]byte, 512) - _, err = teeReader.Read(header) + headerlen, err := teeReader.Read(header) if err != nil { return } @@ -47,17 +47,8 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { // Use the bytes we extracted earlier and attempt to determine the file // type - kind, err := filetype.Match(header) - if err != nil { - m.Mimetype = "application/octet-stream" - return m, err - } else if kind.MIME.Value != "" { - m.Mimetype = kind.MIME.Value - } else if printable(header) { - m.Mimetype = "text/plain" - } else { - m.Mimetype = "application/octet-stream" - } + kind := mimetype.Detect(header[:headerlen]) + m.Mimetype = kind.String() return } diff --git a/helpers/helpers_test.go b/helpers/helpers_test.go index 800d0d2..d891173 100644 --- a/helpers/helpers_test.go +++ b/helpers/helpers_test.go @@ -1,8 +1,10 @@ package helpers import ( + "bytes" "strings" "testing" + "unicode/utf16" ) func TestGenerateMetadata(t *testing.T) { @@ -17,7 +19,7 @@ func TestGenerateMetadata(t *testing.T) { t.Fatalf("Sha256sum was %q instead of expected value of %q", m.Sha256sum, expectedSha256sum) } - expectedMimetype := "text/plain" + expectedMimetype := "text/plain; charset=utf-8" if m.Mimetype != expectedMimetype { t.Fatalf("Mimetype was %q instead of expected value of %q", m.Mimetype, expectedMimetype) } @@ -27,3 +29,45 @@ func TestGenerateMetadata(t *testing.T) { t.Fatalf("Size was %d instead of expected value of %d", m.Size, expectedSize) } } + +func TestTextCharsets(t *testing.T) { + // verify that different text encodings are detected and passed through + orig := "This is a text string" + utf16 := utf16.Encode([]rune(orig)) + utf16LE := make([]byte, len(utf16)*2+2) + utf16BE := make([]byte, len(utf16)*2+2) + utf8 := []byte(orig) + utf16LE[0] = 0xff + utf16LE[1] = 0xfe + utf16BE[0] = 0xfe + utf16BE[1] = 0xff + for i := 0; i < len(utf16); i++ { + lsb := utf16[i] & 0xff + msb := utf16[i] >> 8 + utf16LE[i*2+2] = byte(lsb) + utf16LE[i*2+3] = byte(msb) + utf16BE[i*2+2] = byte(msb) + utf16BE[i*2+3] = byte(lsb) + } + + testcases := []struct { + data []byte + extension string + mimetype string + }{ + {mimetype: "text/plain; charset=utf-8", data: utf8}, + {mimetype: "text/plain; charset=utf-16le", data: utf16LE}, + {mimetype: "text/plain; charset=utf-16be", data: utf16BE}, + } + + for i, testcase := range testcases { + r := bytes.NewReader(testcase.data) + m, err := GenerateMetadata(r) + if err != nil { + t.Fatalf("[%d] unexpected error return %v\n", i, err) + } + if m.Mimetype != testcase.mimetype { + t.Errorf("[%d] Expected mimetype '%s', got mimetype '%s'\n", i, testcase.mimetype, m.Mimetype) + } + } +} diff --git a/upload.go b/upload.go index 8526260..3cac122 100644 --- a/upload.go +++ b/upload.go @@ -18,8 +18,8 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" + "github.com/gabriel-vasile/mimetype" "github.com/zenazn/goji/web" - "gopkg.in/h2non/filetype.v1" ) var FileTooLargeError = errors.New("File too large.") @@ -263,11 +263,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { header = header[:n] // Determine the type of file from header - kind, err := filetype.Match(header) - if err != nil || kind.Extension == "unknown" { + kind := mimetype.Detect(header) + if len(kind.Extension()) < 2 { extension = "file" } else { - extension = kind.Extension + extension = kind.Extension()[1:] // remove leading "." } } From 8ed205181a5e83b86ff3b782434f2ca6062ab424 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 14 Aug 2020 00:32:55 -0700 Subject: [PATCH 008/130] Add buildx GH action for multi-arch docker images --- .github/workflows/buildx.yaml | 61 +++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/buildx.yaml diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml new file mode 100644 index 0000000..c8b6cfe --- /dev/null +++ b/.github/workflows/buildx.yaml @@ -0,0 +1,61 @@ +name: buildx + +on: + push: + branches: master + tags: + - v* + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Prepare + id: prepare + run: | + DOCKER_IMAGE=crazymax/diun + DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/386 + VERSION=version-${GITHUB_REF#refs/tags/v} + TAGS="--tag ${DOCKER_IMAGE}:${VERSION} --tag ${DOCKER_IMAGE}:latest" + + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./test/Dockerfile ./test + - + name: Set up Docker Buildx + uses: crazy-max/ghaction-docker-buildx@v3 + - + name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - + name: Docker Login + if: success() + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - + name: Docker Buildx (push) + if: success() + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - + name: Docker Check Manifest + if: always() + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - + name: Clear + if: always() + run: | + rm -f ${HOME}/.docker/config.json From 965d5f6c29e584cc84febf70762d95ee27f14d88 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 14 Aug 2020 00:39:53 -0700 Subject: [PATCH 009/130] Fix GH action --- .github/workflows/buildx.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml index c8b6cfe..50e8601 100644 --- a/.github/workflows/buildx.yaml +++ b/.github/workflows/buildx.yaml @@ -2,9 +2,8 @@ name: buildx on: push: - branches: master tags: - - v* + - 'v*' jobs: buildx: @@ -17,7 +16,7 @@ jobs: name: Prepare id: prepare run: | - DOCKER_IMAGE=crazymax/diun + DOCKER_IMAGE=andreimarcu/linx-server DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/386 VERSION=version-${GITHUB_REF#refs/tags/v} TAGS="--tag ${DOCKER_IMAGE}:${VERSION} --tag ${DOCKER_IMAGE}:latest" From a2e00d06e03a1de04e85b872a1cbe27c204acf88 Mon Sep 17 00:00:00 2001 From: mutantmonkey <67266+mutantmonkey@users.noreply.github.com> Date: Fri, 14 Aug 2020 07:40:52 +0000 Subject: [PATCH 010/130] Clarify how metadata is stored with the S3 backend (#223) This was suggested in #221. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d509689..ed90db5 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ The following storage backends are available: |Name|Notes|Options |----|-----|------- |LocalFS|Enabled by default, this backend uses the filesystem|```filespath = files/``` -- Path to store uploads (default is files/)
```metapath = meta/``` -- Path to store information about uploads (default is meta/)| -|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| +|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket). File metadata will be stored as tags on the object in the bucket.

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| #### SSL with built-in server From 456274c1b9d90f8e688d9fb4f4207963dbecf3e4 Mon Sep 17 00:00:00 2001 From: mutantmonkey <67266+mutantmonkey@users.noreply.github.com> Date: Fri, 14 Aug 2020 07:42:45 +0000 Subject: [PATCH 011/130] Split and move auth into a separate package (#224) * Split and move auth into a separate package This change will make it easier to implement additional authentication methods, such as OpenID Connect. For now, only the existing "apikeys" authentication method is supported. * Use absolute site prefix to prevent redirect loop --- auth.go => auth/apikeys/apikeys.go | 77 +++++++++++++------- auth_test.go => auth/apikeys/apikeys_test.go | 8 +- server.go | 27 ++----- upload.go | 12 ++- 4 files changed, 68 insertions(+), 56 deletions(-) rename auth.go => auth/apikeys/apikeys.go (53%) rename auth_test.go => auth/apikeys/apikeys_test.go (64%) diff --git a/auth.go b/auth/apikeys/apikeys.go similarity index 53% rename from auth.go rename to auth/apikeys/apikeys.go index 3dc5ba6..d2a592d 100644 --- a/auth.go +++ b/auth/apikeys/apikeys.go @@ -1,4 +1,4 @@ -package main +package apikeys import ( "bufio" @@ -24,16 +24,18 @@ const ( type AuthOptions struct { AuthFile string UnauthMethods []string + BasicAuth bool + SiteName string + SitePath string } -type auth struct { +type ApiKeysMiddleware struct { successHandler http.Handler - failureHandler http.Handler authKeys []string o AuthOptions } -func readAuthKeys(authFile string) []string { +func ReadAuthKeys(authFile string) []string { var authKeys []string f, err := os.Open(authFile) @@ -55,7 +57,7 @@ func readAuthKeys(authFile string) []string { return authKeys } -func checkAuth(authKeys []string, key string) (result bool, err error) { +func CheckAuth(authKeys []string, key string) (result bool, err error) { checkKey, err := scrypt.Key([]byte(key), []byte(scryptSalt), scryptN, scryptr, scryptp, scryptKeyLen) if err != nil { return @@ -73,53 +75,74 @@ func checkAuth(authKeys []string, key string) (result bool, err error) { return } -func (a auth) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if sliceContains(a.o.UnauthMethods, r.Method) { +func (a ApiKeysMiddleware) getSitePrefix() string { + prefix := a.o.SitePath + if len(prefix) <= 0 || prefix[0] != '/' { + prefix = "/" + prefix + } + return prefix +} + +func (a ApiKeysMiddleware) goodAuthorizationHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", a.getSitePrefix()) + w.WriteHeader(http.StatusFound) +} + +func (a ApiKeysMiddleware) badAuthorizationHandler(w http.ResponseWriter, r *http.Request) { + if a.o.BasicAuth { + rs := "" + if a.o.SiteName != "" { + rs = fmt.Sprintf(` realm="%s"`, a.o.SiteName) + } + w.Header().Set("WWW-Authenticate", `Basic`+rs) + } + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) +} + +func (a ApiKeysMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var successHandler http.Handler + prefix := a.getSitePrefix() + + if r.URL.Path == prefix+"auth" { + successHandler = http.HandlerFunc(a.goodAuthorizationHandler) + } else { + successHandler = a.successHandler + } + + if sliceContains(a.o.UnauthMethods, r.Method) && r.URL.Path != prefix+"auth" { // allow unauthenticated methods - a.successHandler.ServeHTTP(w, r) + successHandler.ServeHTTP(w, r) return } key := r.Header.Get("Linx-Api-Key") - if key == "" && Config.basicAuth { + if key == "" && a.o.BasicAuth { _, password, ok := r.BasicAuth() if ok { key = password } } - result, err := checkAuth(a.authKeys, key) + result, err := CheckAuth(a.authKeys, key) if err != nil || !result { - a.failureHandler.ServeHTTP(w, r) + http.HandlerFunc(a.badAuthorizationHandler).ServeHTTP(w, r) return } - a.successHandler.ServeHTTP(w, r) + successHandler.ServeHTTP(w, r) } -func UploadAuth(o AuthOptions) func(*web.C, http.Handler) http.Handler { +func NewApiKeysMiddleware(o AuthOptions) func(*web.C, http.Handler) http.Handler { fn := func(c *web.C, h http.Handler) http.Handler { - return auth{ + return ApiKeysMiddleware{ successHandler: h, - failureHandler: http.HandlerFunc(badAuthorizationHandler), - authKeys: readAuthKeys(o.AuthFile), + authKeys: ReadAuthKeys(o.AuthFile), o: o, } } return fn } -func badAuthorizationHandler(w http.ResponseWriter, r *http.Request) { - if Config.basicAuth { - rs := "" - if Config.siteName != "" { - rs = fmt.Sprintf(` realm="%s"`, Config.siteName) - } - w.Header().Set("WWW-Authenticate", `Basic`+rs) - } - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) -} - func sliceContains(slice []string, s string) bool { for _, v := range slice { if s == v { diff --git a/auth_test.go b/auth/apikeys/apikeys_test.go similarity index 64% rename from auth_test.go rename to auth/apikeys/apikeys_test.go index ded98b0..3c2b8e6 100644 --- a/auth_test.go +++ b/auth/apikeys/apikeys_test.go @@ -1,4 +1,4 @@ -package main +package apikeys import ( "testing" @@ -10,15 +10,15 @@ func TestCheckAuth(t *testing.T) { "vFpNprT9wbHgwAubpvRxYCCpA2FQMAK6hFqPvAGrdZo=", } - if r, err := checkAuth(authKeys, ""); err != nil && r { + if r, err := CheckAuth(authKeys, ""); err != nil && r { t.Fatal("Authorization passed for empty key") } - if r, err := checkAuth(authKeys, "thisisnotvalid"); err != nil && r { + if r, err := CheckAuth(authKeys, "thisisnotvalid"); err != nil && r { t.Fatal("Authorization passed for invalid key") } - if r, err := checkAuth(authKeys, "haPVipRnGJ0QovA9nyqK"); err != nil && !r { + if r, err := CheckAuth(authKeys, "haPVipRnGJ0QovA9nyqK"); err != nil && !r { t.Fatal("Authorization failed for valid key") } } diff --git a/server.go b/server.go index dae3491..4d06db9 100644 --- a/server.go +++ b/server.go @@ -16,6 +16,7 @@ import ( "time" rice "github.com/GeertJohan/go.rice" + "github.com/andreimarcu/linx-server/auth/apikeys" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/backends/s3" @@ -110,9 +111,12 @@ func setup() *web.Mux { mux.Use(AddHeaders(Config.addHeaders)) if Config.authFile != "" { - mux.Use(UploadAuth(AuthOptions{ + mux.Use(apikeys.NewApiKeysMiddleware(apikeys.AuthOptions{ AuthFile: Config.authFile, UnauthMethods: []string{"GET", "HEAD", "OPTIONS", "TRACE"}, + BasicAuth: Config.basicAuth, + SiteName: Config.siteName, + SitePath: Config.sitePath, })) } @@ -196,29 +200,10 @@ func setup() *web.Mux { mux.Get(Config.sitePath+"upload/", uploadRemote) if Config.remoteAuthFile != "" { - remoteAuthKeys = readAuthKeys(Config.remoteAuthFile) + remoteAuthKeys = apikeys.ReadAuthKeys(Config.remoteAuthFile) } } - if Config.basicAuth { - options := AuthOptions{ - AuthFile: Config.authFile, - UnauthMethods: []string{}, - } - okFunc := func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Location", Config.sitePath) - w.WriteHeader(http.StatusFound) - } - authHandler := auth{ - successHandler: http.HandlerFunc(okFunc), - failureHandler: http.HandlerFunc(badAuthorizationHandler), - authKeys: readAuthKeys(Config.authFile), - o: options, - } - mux.Head(Config.sitePath+"auth", authHandler) - mux.Get(Config.sitePath+"auth", authHandler) - } - mux.Post(Config.sitePath+"upload", uploadPostHandler) mux.Post(Config.sitePath+"upload/", uploadPostHandler) mux.Put(Config.sitePath+"upload", uploadPutHandler) diff --git a/upload.go b/upload.go index 3cac122..bda0d7f 100644 --- a/upload.go +++ b/upload.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/andreimarcu/linx-server/auth/apikeys" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" @@ -166,13 +167,16 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) { key = password } } - result, err := checkAuth(remoteAuthKeys, key) + result, err := apikeys.CheckAuth(remoteAuthKeys, key) if err != nil || !result { if Config.basicAuth { - badAuthorizationHandler(w, r) - } else { - unauthorizedHandler(c, w, r) + rs := "" + if Config.siteName != "" { + rs = fmt.Sprintf(` realm="%s"`, Config.siteName) + } + w.Header().Set("WWW-Authenticate", `Basic`+rs) } + unauthorizedHandler(c, w, r) return } } From 9a5fc11dffe5d2ac6cb6e7edfa97bccd417285ed Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 14 Aug 2020 00:52:25 -0700 Subject: [PATCH 012/130] Fix GH action (again) --- .github/workflows/buildx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml index 50e8601..1938731 100644 --- a/.github/workflows/buildx.yaml +++ b/.github/workflows/buildx.yaml @@ -27,7 +27,7 @@ jobs: --build-arg VERSION=${VERSION} \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file ./test/Dockerfile ./test + ${TAGS} --file Dockerfile . - name: Set up Docker Buildx uses: crazy-max/ghaction-docker-buildx@v3 From ef99024433df488e2aa2d063044ee05f4d01ac78 Mon Sep 17 00:00:00 2001 From: tuxx Date: Sat, 17 Oct 2020 01:55:11 +0200 Subject: [PATCH 013/130] Add LinxShare android client to README.md (#246) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed90db5..62f4d98 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can see what it looks like using the demo: [https://demo.linx-server.net/](h - Display common filetypes (image, video, audio, markdown, pdf) - Display syntax-highlighted code with in-place editing -- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line) +- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line, or [LinxShare](https://github.com/iksteen/LinxShare/) client for android - also available on the (Play Store)[https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare] ) - Torrent download of files using web seeding - File expiry, deletion key, file access key, and random filename options From 91b9885ac6d6c9bf7e2837e8daf7ff3b1facea4c Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 16 Oct 2020 16:57:09 -0700 Subject: [PATCH 014/130] Update README.md --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 62f4d98..622d950 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,20 @@ Self-hosted file/media sharing website. You can see what it looks like using the demo: [https://demo.linx-server.net/](https://demo.linx-server.net/) +### Clients +**Official** +- CLI: **linx-client** - [Source](https://github.com/andreimarcu/linx-client) + +**Unofficial** +- Android: **LinxShare** - [Source](https://github.com/iksteen/LinxShare/) | [Google Play](https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare) +- CLI: **golinx** - [Source](https://github.com/mutantmonkey/golinx) + + ### Features - Display common filetypes (image, video, audio, markdown, pdf) - Display syntax-highlighted code with in-place editing -- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line, or [LinxShare](https://github.com/iksteen/LinxShare/) client for android - also available on the (Play Store)[https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare] ) +- Documented API with keys for restricting uploads - Torrent download of files using web seeding - File expiry, deletion key, file access key, and random filename options @@ -60,8 +69,8 @@ Ideally, you would use a reverse proxy such as nginx or caddy to handle TLS cert #### Using a binary release -1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases) -2. Run ```./linx-server``` +1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases), then run ```go install``` +2. Run ```linx-server -config path/to/linx-server.conf``` Usage From 486cc6ff778171eb2b5b231db2cc52d5f484352f Mon Sep 17 00:00:00 2001 From: Steven Tang Date: Mon, 30 Nov 2020 06:09:53 +1100 Subject: [PATCH 015/130] Remove entrypoint from sample docker-compose.yml (#252) Resolves #225 Entrypoint in Dockerfile specifies default bind, filespath, metapath. Having entrypoint in docker-compose.yml will remove those defaults. Without the entrypoint line, the command executed is: `/usr/local/bin/linx-server -bind=0.0.0.0:8080 -filespath=/data/files/ -metapath=/data/meta/ -config /data/linx-server.conf` --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 622d950..cf285b9 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,6 @@ services: linx-server: container_name: linx-server image: andreimarcu/linx-server - entrypoint: /usr/local/bin/linx-server command: -config /data/linx-server.conf volumes: - /path/to/files:/data/files From 94f63c204506d595194bd52b5f3d3c1ba19923f0 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Mon, 25 Jan 2021 11:29:55 -0800 Subject: [PATCH 016/130] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index cf285b9..16318be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ +## Seeking new maintainer! +If anyone would be interested in taking on this project, I would be willing to transfer it over as development and work on issues has mostly stalled at this point. +Please [respond to this issue](https://github.com/andreimarcu/linx-server/issues/266) if interested! + +--- + linx-server ====== [![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) From 63975c2019d9a943be9f6d6f122239749b0e75f4 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Tue, 9 Feb 2021 20:00:20 -0500 Subject: [PATCH 017/130] Extra footer text block test. --- server.go | 2 ++ templates.go | 1 + templates/base.html | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index 4d06db9..903288f 100644 --- a/server.go +++ b/server.go @@ -304,6 +304,8 @@ func main() { "path to directory containing .md files to render as custom pages") flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0, "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") + flag.StringVar(&Config.extraFooterText, "extrafootertext", "", + "Extra text above the footer for notices.") iniflags.Parse() diff --git a/templates.go b/templates.go index 7d38b51..6dcf852 100644 --- a/templates.go +++ b/templates.go @@ -87,6 +87,7 @@ func renderTemplate(tpl *pongo2.Template, context pongo2.Context, r *http.Reques context["sitepath"] = Config.sitePath context["selifpath"] = Config.selifPath context["custom_pages_names"] = customPagesNames + context["extra_footer_text"] = Config.extraFooterText var a string if Config.authFile == "" { diff --git a/templates/base.html b/templates/base.html index 7e4f82d..847a995 100644 --- a/templates/base.html +++ b/templates/base.html @@ -29,7 +29,9 @@ {% block content %}{% endblock %} - +
+ {% if disable_access_key != true %} - + {% endif %} Randomize filename + {% if default_randomized && !(forcerandom) %} checked {% endif %} /> Randomize filename
From 4579be1048c5faf0761479c1f92c7f5b4580d664 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Sat, 27 Feb 2021 21:42:19 -0500 Subject: [PATCH 112/130] Update index.html --- templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index 01ec49e..ab699e2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,7 +21,7 @@ + {% if default_randomized or not forcerandom %} checked {% endif %} /> Randomize filename
From 59cf8dbfee43831ed7891844b00bc5c0a4c3bdc2 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Sat, 27 Feb 2021 21:53:29 -0500 Subject: [PATCH 113/130] Update index.html --- templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index ab699e2..8060af2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,7 +21,7 @@ + {% if (default_randomize && !( forcerandom)) || forcerandom %} checked {% endif %} /> Randomize filename
From fae511f7d55a06c151d2a50b2739b0cf7501fcd0 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Wed, 3 Mar 2021 00:06:44 -0500 Subject: [PATCH 114/130] Update Readme to include new feature descriptions. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 16318be..2c8fb83 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,11 @@ maxexpiry = 86400 | ```nologs = true``` | (optionally) disable request logs in stdout | ```force-random-filename = true``` | (optionally) force the use of random filenames | ```custompagespath = custom_pages/``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation +| ```extra-footer-text = "..."``` | (optionally) Extra text above the footer for notices. +| ```max-duration-time = 0``` | Time till expiry for files over max-duration-size. (Default is 0 for no-expiry.) +| ```max-duration-size = 4294967296``` | Size of file before max-duration-time is used to determine expiry max time. (Default is 4GB) +| ```disable-access-key = true``` | Disables access key usage. (Default is false.) +| ```default-random-filename = true``` | Makes it so the random filename is not default if set false. (Default is true.) #### Cleaning up expired files From d43dcf4acdbdba303bf417da750067cfdaae501f Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Sun, 7 Mar 2021 23:04:52 -0500 Subject: [PATCH 115/130] Update default text for new features. --- server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server.go b/server.go index 985c769..82a72e3 100644 --- a/server.go +++ b/server.go @@ -313,10 +313,10 @@ func main() { "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") flag.StringVar(&Config.extraFooterText, "extra-footer-text", "", "Extra text above the footer for notices.") - flag.Uint64Var(&Config.maxDurationTime, "max-duration-time", 0, "Time till expiry for files over max-duration-size") - flag.Int64Var(&Config.maxDurationSize, "max-duration-size", 4*1024*1024*1024, "Size of file before max-duration-time is used to determine expiry max time.") - flag.BoolVar(&Config.disableAccessKey, "disable-access-key", false, "Disables access key usage") - flag.BoolVar(&Config.defaultRandomFilename, "default-random-filename", true, "Makes it so the random filename is not default if set false. Default true.") + flag.Uint64Var(&Config.maxDurationTime, "max-duration-time", 0, "Time till expiry for files over max-duration-size. (Default is 0 for no-expiry.)") + flag.Int64Var(&Config.maxDurationSize, "max-duration-size", 4*1024*1024*1024, "Size of file before max-duration-time is used to determine expiry max time. (Default is 4GB)") + flag.BoolVar(&Config.disableAccessKey, "disable-access-key", false, "Disables access key usage. (Default is false.)") + flag.BoolVar(&Config.defaultRandomFilename, "default-random-filename", true, "Makes it so the random filename is not default if set false. (Default is true.)") iniflags.Parse() mux := setup() From ac7c088e91e444ddc05e9e83138d7320e1a47314 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Wed, 17 Mar 2021 19:24:07 -0400 Subject: [PATCH 116/130] Merge remote-tracking branch 'upstream/master' --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 2c8fb83..a603003 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,3 @@ - -## Seeking new maintainer! -If anyone would be interested in taking on this project, I would be willing to transfer it over as development and work on issues has mostly stalled at this point. -Please [respond to this issue](https://github.com/andreimarcu/linx-server/issues/266) if interested! - ---- - linx-server ====== [![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) From 5b774469ea1be43c14602d2f9dd80e4783b88955 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Wed, 17 Mar 2021 19:34:36 -0400 Subject: [PATCH 117/130] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index a603003..12aeaac 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ linx-server ====== -[![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) Self-hosted file/media sharing website. ### Demo -You can see what it looks like using the demo: [https://demo.linx-server.net/](https://demo.linx-server.net/) +You can see what it looks like using the demo: [https://put.icu/](https://put.icu/) ### Clients From 7d0950e54c3015c4dce220e06de7c59b5035c901 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sun, 11 Jul 2021 00:09:57 +0100 Subject: [PATCH 118/130] Update image.html --- templates/display/image.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/display/image.html b/templates/display/image.html index 985c504..8dcd844 100644 --- a/templates/display/image.html +++ b/templates/display/image.html @@ -1,6 +1,7 @@ {% extends "base.html" %} {% block head %} + {% endblock %} @@ -8,4 +9,4 @@ -{% endblock %} \ No newline at end of file +{% endblock %} From 084ac233ffaa51c37b32ced8126bd4fa8c176dc1 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Fri, 20 Aug 2021 12:13:58 -0400 Subject: [PATCH 119/130] Repo Link update. Added link to this fork of Linx-server. --- templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/base.html b/templates/base.html index 17b1ea2..54fd368 100644 --- a/templates/base.html +++ b/templates/base.html @@ -30,7 +30,7 @@ {% block content %}{% endblock %}
From d63a875b12e4bf370c31ec4ee4f1d79b74c8f0e8 Mon Sep 17 00:00:00 2001 From: luckman212 Date: Sat, 9 Oct 2021 11:36:53 -0400 Subject: [PATCH 120/130] add default expiry config parameter --- pages.go | 12 +++++++----- server.go | 3 +++ templates/index.html | 2 +- templates/paste.html | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pages.go b/pages.go index ae8de42..6492651 100644 --- a/pages.go +++ b/pages.go @@ -21,9 +21,10 @@ const ( func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["index.html"], pongo2.Context{ - "maxsize": Config.maxSize, - "expirylist": listExpirationTimes(), - "forcerandom": Config.forceRandomFilename, + "maxsize": Config.maxSize, + "expirylist": listExpirationTimes(), + "expirydefault": Config.defaultExpiry, + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -32,8 +33,9 @@ func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { func pasteHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["paste.html"], pongo2.Context{ - "expirylist": listExpirationTimes(), - "forcerandom": Config.forceRandomFilename, + "expirylist": listExpirationTimes(), + "expirydefault": Config.defaultExpiry, + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { oopsHandler(c, w, r, RespHTML, "") diff --git a/server.go b/server.go index 82a72e3..41cd932 100644 --- a/server.go +++ b/server.go @@ -56,6 +56,7 @@ var Config struct { xFrameOptions string maxSize int64 maxExpiry uint64 + defaultExpiry uint64 realIp bool noLogs bool allowHotlink bool @@ -264,6 +265,8 @@ func main() { "maximum upload file size in bytes (default 4GB)") flag.Uint64Var(&Config.maxExpiry, "maxexpiry", 0, "maximum expiration time in seconds (default is 0, which is no expiry)") + flag.Uint64Var(&Config.defaultExpiry, "default-expiry", 86400, + "default expiration time in seconds (default is 86400, which is 1 day)") flag.StringVar(&Config.certFile, "certfile", "", "path to ssl certificate (for https)") flag.StringVar(&Config.keyFile, "keyfile", "", diff --git a/templates/index.html b/templates/index.html index 8060af2..3958788 100644 --- a/templates/index.html +++ b/templates/index.html @@ -28,7 +28,7 @@