From 5eb6f32ff0a0252caab5fbc8e154912b4b5e4676 Mon Sep 17 00:00:00 2001 From: Infinoid Date: Mon, 3 Aug 2020 01:16:47 -0400 Subject: [PATCH 001/124] Switch to a more comprehensive mimetype detection library (#231) --- go.mod | 2 +- go.sum | 4 ++-- helpers/helpers.go | 17 ++++----------- helpers/helpers_test.go | 46 ++++++++++++++++++++++++++++++++++++++++- upload.go | 8 +++---- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 50a48ab..f433699 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 github.com/dustin/go-humanize v1.0.0 github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 + github.com/gabriel-vasile/mimetype v1.1.1 github.com/microcosm-cc/bluemonday v1.0.2 github.com/minio/sha256-simd v0.1.1 github.com/russross/blackfriday v1.5.1 @@ -15,5 +16,4 @@ require ( github.com/zeebo/bencode v1.0.0 github.com/zenazn/goji v0.9.0 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 - gopkg.in/h2non/filetype.v1 v1.0.5 ) diff --git a/go.sum b/go.sum index 15a736c..99d63bf 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/gabriel-vasile/mimetype v1.1.1 h1:qbN9MPuRf3bstHu9zkI9jDWNfH//9+9kHxr9oRBBBOA= +github.com/gabriel-vasile/mimetype v1.1.1/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -68,8 +70,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= -gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/helpers/helpers.go b/helpers/helpers.go index f51d998..f13e302 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -7,8 +7,8 @@ import ( "unicode" "github.com/andreimarcu/linx-server/backends" + "github.com/gabriel-vasile/mimetype" "github.com/minio/sha256-simd" - "gopkg.in/h2non/filetype.v1" ) func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { @@ -21,7 +21,7 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { // Get first 512 bytes for mimetype detection header := make([]byte, 512) - _, err = teeReader.Read(header) + headerlen, err := teeReader.Read(header) if err != nil { return } @@ -47,17 +47,8 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { // Use the bytes we extracted earlier and attempt to determine the file // type - kind, err := filetype.Match(header) - if err != nil { - m.Mimetype = "application/octet-stream" - return m, err - } else if kind.MIME.Value != "" { - m.Mimetype = kind.MIME.Value - } else if printable(header) { - m.Mimetype = "text/plain" - } else { - m.Mimetype = "application/octet-stream" - } + kind := mimetype.Detect(header[:headerlen]) + m.Mimetype = kind.String() return } diff --git a/helpers/helpers_test.go b/helpers/helpers_test.go index 800d0d2..d891173 100644 --- a/helpers/helpers_test.go +++ b/helpers/helpers_test.go @@ -1,8 +1,10 @@ package helpers import ( + "bytes" "strings" "testing" + "unicode/utf16" ) func TestGenerateMetadata(t *testing.T) { @@ -17,7 +19,7 @@ func TestGenerateMetadata(t *testing.T) { t.Fatalf("Sha256sum was %q instead of expected value of %q", m.Sha256sum, expectedSha256sum) } - expectedMimetype := "text/plain" + expectedMimetype := "text/plain; charset=utf-8" if m.Mimetype != expectedMimetype { t.Fatalf("Mimetype was %q instead of expected value of %q", m.Mimetype, expectedMimetype) } @@ -27,3 +29,45 @@ func TestGenerateMetadata(t *testing.T) { t.Fatalf("Size was %d instead of expected value of %d", m.Size, expectedSize) } } + +func TestTextCharsets(t *testing.T) { + // verify that different text encodings are detected and passed through + orig := "This is a text string" + utf16 := utf16.Encode([]rune(orig)) + utf16LE := make([]byte, len(utf16)*2+2) + utf16BE := make([]byte, len(utf16)*2+2) + utf8 := []byte(orig) + utf16LE[0] = 0xff + utf16LE[1] = 0xfe + utf16BE[0] = 0xfe + utf16BE[1] = 0xff + for i := 0; i < len(utf16); i++ { + lsb := utf16[i] & 0xff + msb := utf16[i] >> 8 + utf16LE[i*2+2] = byte(lsb) + utf16LE[i*2+3] = byte(msb) + utf16BE[i*2+2] = byte(msb) + utf16BE[i*2+3] = byte(lsb) + } + + testcases := []struct { + data []byte + extension string + mimetype string + }{ + {mimetype: "text/plain; charset=utf-8", data: utf8}, + {mimetype: "text/plain; charset=utf-16le", data: utf16LE}, + {mimetype: "text/plain; charset=utf-16be", data: utf16BE}, + } + + for i, testcase := range testcases { + r := bytes.NewReader(testcase.data) + m, err := GenerateMetadata(r) + if err != nil { + t.Fatalf("[%d] unexpected error return %v\n", i, err) + } + if m.Mimetype != testcase.mimetype { + t.Errorf("[%d] Expected mimetype '%s', got mimetype '%s'\n", i, testcase.mimetype, m.Mimetype) + } + } +} diff --git a/upload.go b/upload.go index 8526260..3cac122 100644 --- a/upload.go +++ b/upload.go @@ -18,8 +18,8 @@ import ( "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" + "github.com/gabriel-vasile/mimetype" "github.com/zenazn/goji/web" - "gopkg.in/h2non/filetype.v1" ) var FileTooLargeError = errors.New("File too large.") @@ -263,11 +263,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { header = header[:n] // Determine the type of file from header - kind, err := filetype.Match(header) - if err != nil || kind.Extension == "unknown" { + kind := mimetype.Detect(header) + if len(kind.Extension()) < 2 { extension = "file" } else { - extension = kind.Extension + extension = kind.Extension()[1:] // remove leading "." } } From 8ed205181a5e83b86ff3b782434f2ca6062ab424 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 14 Aug 2020 00:32:55 -0700 Subject: [PATCH 002/124] Add buildx GH action for multi-arch docker images --- .github/workflows/buildx.yaml | 61 +++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/buildx.yaml diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml new file mode 100644 index 0000000..c8b6cfe --- /dev/null +++ b/.github/workflows/buildx.yaml @@ -0,0 +1,61 @@ +name: buildx + +on: + push: + branches: master + tags: + - v* + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Prepare + id: prepare + run: | + DOCKER_IMAGE=crazymax/diun + DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/386 + VERSION=version-${GITHUB_REF#refs/tags/v} + TAGS="--tag ${DOCKER_IMAGE}:${VERSION} --tag ${DOCKER_IMAGE}:latest" + + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./test/Dockerfile ./test + - + name: Set up Docker Buildx + uses: crazy-max/ghaction-docker-buildx@v3 + - + name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - + name: Docker Login + if: success() + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - + name: Docker Buildx (push) + if: success() + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - + name: Docker Check Manifest + if: always() + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - + name: Clear + if: always() + run: | + rm -f ${HOME}/.docker/config.json From 965d5f6c29e584cc84febf70762d95ee27f14d88 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 14 Aug 2020 00:39:53 -0700 Subject: [PATCH 003/124] Fix GH action --- .github/workflows/buildx.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml index c8b6cfe..50e8601 100644 --- a/.github/workflows/buildx.yaml +++ b/.github/workflows/buildx.yaml @@ -2,9 +2,8 @@ name: buildx on: push: - branches: master tags: - - v* + - 'v*' jobs: buildx: @@ -17,7 +16,7 @@ jobs: name: Prepare id: prepare run: | - DOCKER_IMAGE=crazymax/diun + DOCKER_IMAGE=andreimarcu/linx-server DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/386 VERSION=version-${GITHUB_REF#refs/tags/v} TAGS="--tag ${DOCKER_IMAGE}:${VERSION} --tag ${DOCKER_IMAGE}:latest" From a2e00d06e03a1de04e85b872a1cbe27c204acf88 Mon Sep 17 00:00:00 2001 From: mutantmonkey <67266+mutantmonkey@users.noreply.github.com> Date: Fri, 14 Aug 2020 07:40:52 +0000 Subject: [PATCH 004/124] Clarify how metadata is stored with the S3 backend (#223) This was suggested in #221. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d509689..ed90db5 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ The following storage backends are available: |Name|Notes|Options |----|-----|------- |LocalFS|Enabled by default, this backend uses the filesystem|```filespath = files/``` -- Path to store uploads (default is files/)
```metapath = meta/``` -- Path to store information about uploads (default is meta/)| -|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| +|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket). File metadata will be stored as tags on the object in the bucket.

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| #### SSL with built-in server From 456274c1b9d90f8e688d9fb4f4207963dbecf3e4 Mon Sep 17 00:00:00 2001 From: mutantmonkey <67266+mutantmonkey@users.noreply.github.com> Date: Fri, 14 Aug 2020 07:42:45 +0000 Subject: [PATCH 005/124] Split and move auth into a separate package (#224) * Split and move auth into a separate package This change will make it easier to implement additional authentication methods, such as OpenID Connect. For now, only the existing "apikeys" authentication method is supported. * Use absolute site prefix to prevent redirect loop --- auth.go => auth/apikeys/apikeys.go | 77 +++++++++++++------- auth_test.go => auth/apikeys/apikeys_test.go | 8 +- server.go | 27 ++----- upload.go | 12 ++- 4 files changed, 68 insertions(+), 56 deletions(-) rename auth.go => auth/apikeys/apikeys.go (53%) rename auth_test.go => auth/apikeys/apikeys_test.go (64%) diff --git a/auth.go b/auth/apikeys/apikeys.go similarity index 53% rename from auth.go rename to auth/apikeys/apikeys.go index 3dc5ba6..d2a592d 100644 --- a/auth.go +++ b/auth/apikeys/apikeys.go @@ -1,4 +1,4 @@ -package main +package apikeys import ( "bufio" @@ -24,16 +24,18 @@ const ( type AuthOptions struct { AuthFile string UnauthMethods []string + BasicAuth bool + SiteName string + SitePath string } -type auth struct { +type ApiKeysMiddleware struct { successHandler http.Handler - failureHandler http.Handler authKeys []string o AuthOptions } -func readAuthKeys(authFile string) []string { +func ReadAuthKeys(authFile string) []string { var authKeys []string f, err := os.Open(authFile) @@ -55,7 +57,7 @@ func readAuthKeys(authFile string) []string { return authKeys } -func checkAuth(authKeys []string, key string) (result bool, err error) { +func CheckAuth(authKeys []string, key string) (result bool, err error) { checkKey, err := scrypt.Key([]byte(key), []byte(scryptSalt), scryptN, scryptr, scryptp, scryptKeyLen) if err != nil { return @@ -73,53 +75,74 @@ func checkAuth(authKeys []string, key string) (result bool, err error) { return } -func (a auth) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if sliceContains(a.o.UnauthMethods, r.Method) { +func (a ApiKeysMiddleware) getSitePrefix() string { + prefix := a.o.SitePath + if len(prefix) <= 0 || prefix[0] != '/' { + prefix = "/" + prefix + } + return prefix +} + +func (a ApiKeysMiddleware) goodAuthorizationHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", a.getSitePrefix()) + w.WriteHeader(http.StatusFound) +} + +func (a ApiKeysMiddleware) badAuthorizationHandler(w http.ResponseWriter, r *http.Request) { + if a.o.BasicAuth { + rs := "" + if a.o.SiteName != "" { + rs = fmt.Sprintf(` realm="%s"`, a.o.SiteName) + } + w.Header().Set("WWW-Authenticate", `Basic`+rs) + } + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) +} + +func (a ApiKeysMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var successHandler http.Handler + prefix := a.getSitePrefix() + + if r.URL.Path == prefix+"auth" { + successHandler = http.HandlerFunc(a.goodAuthorizationHandler) + } else { + successHandler = a.successHandler + } + + if sliceContains(a.o.UnauthMethods, r.Method) && r.URL.Path != prefix+"auth" { // allow unauthenticated methods - a.successHandler.ServeHTTP(w, r) + successHandler.ServeHTTP(w, r) return } key := r.Header.Get("Linx-Api-Key") - if key == "" && Config.basicAuth { + if key == "" && a.o.BasicAuth { _, password, ok := r.BasicAuth() if ok { key = password } } - result, err := checkAuth(a.authKeys, key) + result, err := CheckAuth(a.authKeys, key) if err != nil || !result { - a.failureHandler.ServeHTTP(w, r) + http.HandlerFunc(a.badAuthorizationHandler).ServeHTTP(w, r) return } - a.successHandler.ServeHTTP(w, r) + successHandler.ServeHTTP(w, r) } -func UploadAuth(o AuthOptions) func(*web.C, http.Handler) http.Handler { +func NewApiKeysMiddleware(o AuthOptions) func(*web.C, http.Handler) http.Handler { fn := func(c *web.C, h http.Handler) http.Handler { - return auth{ + return ApiKeysMiddleware{ successHandler: h, - failureHandler: http.HandlerFunc(badAuthorizationHandler), - authKeys: readAuthKeys(o.AuthFile), + authKeys: ReadAuthKeys(o.AuthFile), o: o, } } return fn } -func badAuthorizationHandler(w http.ResponseWriter, r *http.Request) { - if Config.basicAuth { - rs := "" - if Config.siteName != "" { - rs = fmt.Sprintf(` realm="%s"`, Config.siteName) - } - w.Header().Set("WWW-Authenticate", `Basic`+rs) - } - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) -} - func sliceContains(slice []string, s string) bool { for _, v := range slice { if s == v { diff --git a/auth_test.go b/auth/apikeys/apikeys_test.go similarity index 64% rename from auth_test.go rename to auth/apikeys/apikeys_test.go index ded98b0..3c2b8e6 100644 --- a/auth_test.go +++ b/auth/apikeys/apikeys_test.go @@ -1,4 +1,4 @@ -package main +package apikeys import ( "testing" @@ -10,15 +10,15 @@ func TestCheckAuth(t *testing.T) { "vFpNprT9wbHgwAubpvRxYCCpA2FQMAK6hFqPvAGrdZo=", } - if r, err := checkAuth(authKeys, ""); err != nil && r { + if r, err := CheckAuth(authKeys, ""); err != nil && r { t.Fatal("Authorization passed for empty key") } - if r, err := checkAuth(authKeys, "thisisnotvalid"); err != nil && r { + if r, err := CheckAuth(authKeys, "thisisnotvalid"); err != nil && r { t.Fatal("Authorization passed for invalid key") } - if r, err := checkAuth(authKeys, "haPVipRnGJ0QovA9nyqK"); err != nil && !r { + if r, err := CheckAuth(authKeys, "haPVipRnGJ0QovA9nyqK"); err != nil && !r { t.Fatal("Authorization failed for valid key") } } diff --git a/server.go b/server.go index dae3491..4d06db9 100644 --- a/server.go +++ b/server.go @@ -16,6 +16,7 @@ import ( "time" rice "github.com/GeertJohan/go.rice" + "github.com/andreimarcu/linx-server/auth/apikeys" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/backends/s3" @@ -110,9 +111,12 @@ func setup() *web.Mux { mux.Use(AddHeaders(Config.addHeaders)) if Config.authFile != "" { - mux.Use(UploadAuth(AuthOptions{ + mux.Use(apikeys.NewApiKeysMiddleware(apikeys.AuthOptions{ AuthFile: Config.authFile, UnauthMethods: []string{"GET", "HEAD", "OPTIONS", "TRACE"}, + BasicAuth: Config.basicAuth, + SiteName: Config.siteName, + SitePath: Config.sitePath, })) } @@ -196,29 +200,10 @@ func setup() *web.Mux { mux.Get(Config.sitePath+"upload/", uploadRemote) if Config.remoteAuthFile != "" { - remoteAuthKeys = readAuthKeys(Config.remoteAuthFile) + remoteAuthKeys = apikeys.ReadAuthKeys(Config.remoteAuthFile) } } - if Config.basicAuth { - options := AuthOptions{ - AuthFile: Config.authFile, - UnauthMethods: []string{}, - } - okFunc := func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Location", Config.sitePath) - w.WriteHeader(http.StatusFound) - } - authHandler := auth{ - successHandler: http.HandlerFunc(okFunc), - failureHandler: http.HandlerFunc(badAuthorizationHandler), - authKeys: readAuthKeys(Config.authFile), - o: options, - } - mux.Head(Config.sitePath+"auth", authHandler) - mux.Get(Config.sitePath+"auth", authHandler) - } - mux.Post(Config.sitePath+"upload", uploadPostHandler) mux.Post(Config.sitePath+"upload/", uploadPostHandler) mux.Put(Config.sitePath+"upload", uploadPutHandler) diff --git a/upload.go b/upload.go index 3cac122..bda0d7f 100644 --- a/upload.go +++ b/upload.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/andreimarcu/linx-server/auth/apikeys" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" @@ -166,13 +167,16 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) { key = password } } - result, err := checkAuth(remoteAuthKeys, key) + result, err := apikeys.CheckAuth(remoteAuthKeys, key) if err != nil || !result { if Config.basicAuth { - badAuthorizationHandler(w, r) - } else { - unauthorizedHandler(c, w, r) + rs := "" + if Config.siteName != "" { + rs = fmt.Sprintf(` realm="%s"`, Config.siteName) + } + w.Header().Set("WWW-Authenticate", `Basic`+rs) } + unauthorizedHandler(c, w, r) return } } From 9a5fc11dffe5d2ac6cb6e7edfa97bccd417285ed Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 14 Aug 2020 00:52:25 -0700 Subject: [PATCH 006/124] Fix GH action (again) --- .github/workflows/buildx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml index 50e8601..1938731 100644 --- a/.github/workflows/buildx.yaml +++ b/.github/workflows/buildx.yaml @@ -27,7 +27,7 @@ jobs: --build-arg VERSION=${VERSION} \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file ./test/Dockerfile ./test + ${TAGS} --file Dockerfile . - name: Set up Docker Buildx uses: crazy-max/ghaction-docker-buildx@v3 From ef99024433df488e2aa2d063044ee05f4d01ac78 Mon Sep 17 00:00:00 2001 From: tuxx Date: Sat, 17 Oct 2020 01:55:11 +0200 Subject: [PATCH 007/124] Add LinxShare android client to README.md (#246) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed90db5..62f4d98 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ You can see what it looks like using the demo: [https://demo.linx-server.net/](h - Display common filetypes (image, video, audio, markdown, pdf) - Display syntax-highlighted code with in-place editing -- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line) +- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line, or [LinxShare](https://github.com/iksteen/LinxShare/) client for android - also available on the (Play Store)[https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare] ) - Torrent download of files using web seeding - File expiry, deletion key, file access key, and random filename options From 91b9885ac6d6c9bf7e2837e8daf7ff3b1facea4c Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Fri, 16 Oct 2020 16:57:09 -0700 Subject: [PATCH 008/124] Update README.md --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 62f4d98..622d950 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,20 @@ Self-hosted file/media sharing website. You can see what it looks like using the demo: [https://demo.linx-server.net/](https://demo.linx-server.net/) +### Clients +**Official** +- CLI: **linx-client** - [Source](https://github.com/andreimarcu/linx-client) + +**Unofficial** +- Android: **LinxShare** - [Source](https://github.com/iksteen/LinxShare/) | [Google Play](https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare) +- CLI: **golinx** - [Source](https://github.com/mutantmonkey/golinx) + + ### Features - Display common filetypes (image, video, audio, markdown, pdf) - Display syntax-highlighted code with in-place editing -- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line, or [LinxShare](https://github.com/iksteen/LinxShare/) client for android - also available on the (Play Store)[https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare] ) +- Documented API with keys for restricting uploads - Torrent download of files using web seeding - File expiry, deletion key, file access key, and random filename options @@ -60,8 +69,8 @@ Ideally, you would use a reverse proxy such as nginx or caddy to handle TLS cert #### Using a binary release -1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases) -2. Run ```./linx-server``` +1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases), then run ```go install``` +2. Run ```linx-server -config path/to/linx-server.conf``` Usage From 486cc6ff778171eb2b5b231db2cc52d5f484352f Mon Sep 17 00:00:00 2001 From: Steven Tang Date: Mon, 30 Nov 2020 06:09:53 +1100 Subject: [PATCH 009/124] Remove entrypoint from sample docker-compose.yml (#252) Resolves #225 Entrypoint in Dockerfile specifies default bind, filespath, metapath. Having entrypoint in docker-compose.yml will remove those defaults. Without the entrypoint line, the command executed is: `/usr/local/bin/linx-server -bind=0.0.0.0:8080 -filespath=/data/files/ -metapath=/data/meta/ -config /data/linx-server.conf` --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 622d950..cf285b9 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,6 @@ services: linx-server: container_name: linx-server image: andreimarcu/linx-server - entrypoint: /usr/local/bin/linx-server command: -config /data/linx-server.conf volumes: - /path/to/files:/data/files From 94f63c204506d595194bd52b5f3d3c1ba19923f0 Mon Sep 17 00:00:00 2001 From: Andrei Marcu Date: Mon, 25 Jan 2021 11:29:55 -0800 Subject: [PATCH 010/124] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index cf285b9..16318be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ +## Seeking new maintainer! +If anyone would be interested in taking on this project, I would be willing to transfer it over as development and work on issues has mostly stalled at this point. +Please [respond to this issue](https://github.com/andreimarcu/linx-server/issues/266) if interested! + +--- + linx-server ====== [![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) From 63975c2019d9a943be9f6d6f122239749b0e75f4 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Tue, 9 Feb 2021 20:00:20 -0500 Subject: [PATCH 011/124] Extra footer text block test. --- server.go | 2 ++ templates.go | 1 + templates/base.html | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index 4d06db9..903288f 100644 --- a/server.go +++ b/server.go @@ -304,6 +304,8 @@ func main() { "path to directory containing .md files to render as custom pages") flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0, "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") + flag.StringVar(&Config.extraFooterText, "extrafootertext", "", + "Extra text above the footer for notices.") iniflags.Parse() diff --git a/templates.go b/templates.go index 7d38b51..6dcf852 100644 --- a/templates.go +++ b/templates.go @@ -87,6 +87,7 @@ func renderTemplate(tpl *pongo2.Template, context pongo2.Context, r *http.Reques context["sitepath"] = Config.sitePath context["selifpath"] = Config.selifPath context["custom_pages_names"] = customPagesNames + context["extra_footer_text"] = Config.extraFooterText var a string if Config.authFile == "" { diff --git a/templates/base.html b/templates/base.html index 7e4f82d..847a995 100644 --- a/templates/base.html +++ b/templates/base.html @@ -29,7 +29,9 @@ {% block content %}{% endblock %} - +
+ {% if disable_access_key != true %} - + {% endif %} Randomize filename + {% if default_randomized && !(forcerandom) %} checked {% endif %} /> Randomize filename
From 4579be1048c5faf0761479c1f92c7f5b4580d664 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Sat, 27 Feb 2021 21:42:19 -0500 Subject: [PATCH 106/124] Update index.html --- templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index 01ec49e..ab699e2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,7 +21,7 @@ + {% if default_randomized or not forcerandom %} checked {% endif %} /> Randomize filename
From 59cf8dbfee43831ed7891844b00bc5c0a4c3bdc2 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Sat, 27 Feb 2021 21:53:29 -0500 Subject: [PATCH 107/124] Update index.html --- templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html b/templates/index.html index ab699e2..8060af2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,7 +21,7 @@ + {% if (default_randomize && !( forcerandom)) || forcerandom %} checked {% endif %} /> Randomize filename
From fae511f7d55a06c151d2a50b2739b0cf7501fcd0 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Wed, 3 Mar 2021 00:06:44 -0500 Subject: [PATCH 108/124] Update Readme to include new feature descriptions. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 16318be..2c8fb83 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,11 @@ maxexpiry = 86400 | ```nologs = true``` | (optionally) disable request logs in stdout | ```force-random-filename = true``` | (optionally) force the use of random filenames | ```custompagespath = custom_pages/``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation +| ```extra-footer-text = "..."``` | (optionally) Extra text above the footer for notices. +| ```max-duration-time = 0``` | Time till expiry for files over max-duration-size. (Default is 0 for no-expiry.) +| ```max-duration-size = 4294967296``` | Size of file before max-duration-time is used to determine expiry max time. (Default is 4GB) +| ```disable-access-key = true``` | Disables access key usage. (Default is false.) +| ```default-random-filename = true``` | Makes it so the random filename is not default if set false. (Default is true.) #### Cleaning up expired files From d43dcf4acdbdba303bf417da750067cfdaae501f Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Sun, 7 Mar 2021 23:04:52 -0500 Subject: [PATCH 109/124] Update default text for new features. --- server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server.go b/server.go index 985c769..82a72e3 100644 --- a/server.go +++ b/server.go @@ -313,10 +313,10 @@ func main() { "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") flag.StringVar(&Config.extraFooterText, "extra-footer-text", "", "Extra text above the footer for notices.") - flag.Uint64Var(&Config.maxDurationTime, "max-duration-time", 0, "Time till expiry for files over max-duration-size") - flag.Int64Var(&Config.maxDurationSize, "max-duration-size", 4*1024*1024*1024, "Size of file before max-duration-time is used to determine expiry max time.") - flag.BoolVar(&Config.disableAccessKey, "disable-access-key", false, "Disables access key usage") - flag.BoolVar(&Config.defaultRandomFilename, "default-random-filename", true, "Makes it so the random filename is not default if set false. Default true.") + flag.Uint64Var(&Config.maxDurationTime, "max-duration-time", 0, "Time till expiry for files over max-duration-size. (Default is 0 for no-expiry.)") + flag.Int64Var(&Config.maxDurationSize, "max-duration-size", 4*1024*1024*1024, "Size of file before max-duration-time is used to determine expiry max time. (Default is 4GB)") + flag.BoolVar(&Config.disableAccessKey, "disable-access-key", false, "Disables access key usage. (Default is false.)") + flag.BoolVar(&Config.defaultRandomFilename, "default-random-filename", true, "Makes it so the random filename is not default if set false. (Default is true.)") iniflags.Parse() mux := setup() From ac7c088e91e444ddc05e9e83138d7320e1a47314 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Wed, 17 Mar 2021 19:24:07 -0400 Subject: [PATCH 110/124] Merge remote-tracking branch 'upstream/master' --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 2c8fb83..a603003 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,3 @@ - -## Seeking new maintainer! -If anyone would be interested in taking on this project, I would be willing to transfer it over as development and work on issues has mostly stalled at this point. -Please [respond to this issue](https://github.com/andreimarcu/linx-server/issues/266) if interested! - ---- - linx-server ====== [![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) From 5b774469ea1be43c14602d2f9dd80e4783b88955 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Wed, 17 Mar 2021 19:34:36 -0400 Subject: [PATCH 111/124] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index a603003..12aeaac 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ linx-server ====== -[![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) Self-hosted file/media sharing website. ### Demo -You can see what it looks like using the demo: [https://demo.linx-server.net/](https://demo.linx-server.net/) +You can see what it looks like using the demo: [https://put.icu/](https://put.icu/) ### Clients From 7d0950e54c3015c4dce220e06de7c59b5035c901 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sun, 11 Jul 2021 00:09:57 +0100 Subject: [PATCH 112/124] Update image.html --- templates/display/image.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/display/image.html b/templates/display/image.html index 985c504..8dcd844 100644 --- a/templates/display/image.html +++ b/templates/display/image.html @@ -1,6 +1,7 @@ {% extends "base.html" %} {% block head %} + {% endblock %} @@ -8,4 +9,4 @@ -{% endblock %} \ No newline at end of file +{% endblock %} From 084ac233ffaa51c37b32ced8126bd4fa8c176dc1 Mon Sep 17 00:00:00 2001 From: ZizzyDizzyMC Date: Fri, 20 Aug 2021 12:13:58 -0400 Subject: [PATCH 113/124] Repo Link update. Added link to this fork of Linx-server. --- templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/base.html b/templates/base.html index 17b1ea2..54fd368 100644 --- a/templates/base.html +++ b/templates/base.html @@ -30,7 +30,7 @@ {% block content %}{% endblock %}
From d63a875b12e4bf370c31ec4ee4f1d79b74c8f0e8 Mon Sep 17 00:00:00 2001 From: luckman212 Date: Sat, 9 Oct 2021 11:36:53 -0400 Subject: [PATCH 114/124] add default expiry config parameter --- pages.go | 12 +++++++----- server.go | 3 +++ templates/index.html | 2 +- templates/paste.html | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pages.go b/pages.go index ae8de42..6492651 100644 --- a/pages.go +++ b/pages.go @@ -21,9 +21,10 @@ const ( func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["index.html"], pongo2.Context{ - "maxsize": Config.maxSize, - "expirylist": listExpirationTimes(), - "forcerandom": Config.forceRandomFilename, + "maxsize": Config.maxSize, + "expirylist": listExpirationTimes(), + "expirydefault": Config.defaultExpiry, + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -32,8 +33,9 @@ func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { func pasteHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["paste.html"], pongo2.Context{ - "expirylist": listExpirationTimes(), - "forcerandom": Config.forceRandomFilename, + "expirylist": listExpirationTimes(), + "expirydefault": Config.defaultExpiry, + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { oopsHandler(c, w, r, RespHTML, "") diff --git a/server.go b/server.go index 82a72e3..41cd932 100644 --- a/server.go +++ b/server.go @@ -56,6 +56,7 @@ var Config struct { xFrameOptions string maxSize int64 maxExpiry uint64 + defaultExpiry uint64 realIp bool noLogs bool allowHotlink bool @@ -264,6 +265,8 @@ func main() { "maximum upload file size in bytes (default 4GB)") flag.Uint64Var(&Config.maxExpiry, "maxexpiry", 0, "maximum expiration time in seconds (default is 0, which is no expiry)") + flag.Uint64Var(&Config.defaultExpiry, "default-expiry", 86400, + "default expiration time in seconds (default is 86400, which is 1 day)") flag.StringVar(&Config.certFile, "certfile", "", "path to ssl certificate (for https)") flag.StringVar(&Config.keyFile, "keyfile", "", diff --git a/templates/index.html b/templates/index.html index 8060af2..3958788 100644 --- a/templates/index.html +++ b/templates/index.html @@ -28,7 +28,7 @@