diff --git a/.github/workflows/buildx.yaml b/.github/workflows/buildx.yaml deleted file mode 100644 index 1938731..0000000 --- a/.github/workflows/buildx.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: buildx - -on: - push: - tags: - - 'v*' - -jobs: - buildx: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Prepare - id: prepare - run: | - DOCKER_IMAGE=andreimarcu/linx-server - DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/386 - VERSION=version-${GITHUB_REF#refs/tags/v} - TAGS="--tag ${DOCKER_IMAGE}:${VERSION} --tag ${DOCKER_IMAGE}:latest" - - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=version::${VERSION} - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ - --build-arg VERSION=${VERSION} \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file Dockerfile . - - - name: Set up Docker Buildx - uses: crazy-max/ghaction-docker-buildx@v3 - - - name: Docker Buildx (build) - run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - - - name: Docker Login - if: success() - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: | - echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - - - name: Docker Buildx (push) - if: success() - run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} - - - name: Docker Check Manifest - if: always() - run: | - docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} - - - name: Clear - if: always() - run: | - rm -f ${HOME}/.docker/config.json diff --git a/Dockerfile b/Dockerfile index e9d61f2..b5a73bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,11 +18,11 @@ ENV SSL_CERT_FILE /etc/ssl/cert.pem COPY static /go/src/github.com/andreimarcu/linx-server/static/ COPY templates /go/src/github.com/andreimarcu/linx-server/templates/ -RUN mkdir -p /data/files && mkdir -p /data/meta && mkdir -p /data/locks && chown -R 65534:65534 /data +RUN mkdir -p /data/files && mkdir -p /data/meta && chown -R 65534:65534 /data -VOLUME ["/data/files", "/data/meta", "/data/locks"] +VOLUME ["/data/files", "/data/meta"] EXPOSE 8080 USER nobody -ENTRYPOINT ["/usr/local/bin/linx-server", "-bind=0.0.0.0:8080", "-filespath=/data/files/", "-metapath=/data/meta/", "-lockspath=/data/locks/"] +ENTRYPOINT ["/usr/local/bin/linx-server", "-bind=0.0.0.0:8080", "-filespath=/data/files/", "-metapath=/data/meta/"] CMD ["-sitename=linx", "-allowhotlink"] diff --git a/README.md b/README.md index 6f3b615..d509689 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,19 @@ + linx-server ====== +[![Build Status](https://travis-ci.org/andreimarcu/linx-server.svg?branch=master)](https://travis-ci.org/andreimarcu/linx-server) Self-hosted file/media sharing website. -### Is this still active? - -Yes, though the repo may be old, it's still active and I'll try and fix any major issues that occur with my limited time. - ### Demo -You can see what it looks like using the demo: [https://put.icu/](https://put.icu/) - - -### Clients -**Official** -- CLI: **linx-client** - [Source](https://github.com/andreimarcu/linx-client) - -**Unofficial** -- Android: **LinxShare** - [Source](https://github.com/iksteen/LinxShare/) | [Google Play](https://play.google.com/store/apps/details?id=org.thegraveyard.linxshare) -- CLI: **golinx** - [Source](https://github.com/mutantmonkey/golinx) +You can see what it looks like using the demo: [https://demo.linx-server.net/](https://demo.linx-server.net/) ### Features - Display common filetypes (image, video, audio, markdown, pdf) - Display syntax-highlighted code with in-place editing -- Documented API with keys for restricting uploads +- Documented API with keys if need to restrict uploads (can use [linx-client](https://github.com/andreimarcu/linx-client) for uploading through command-line) - Torrent download of files using web seeding - File expiry, deletion key, file access key, and random filename options @@ -56,6 +45,7 @@ services: linx-server: container_name: linx-server image: andreimarcu/linx-server + entrypoint: /usr/local/bin/linx-server command: -config /data/linx-server.conf volumes: - /path/to/files:/data/files @@ -70,8 +60,8 @@ Ideally, you would use a reverse proxy such as nginx or caddy to handle TLS cert #### Using a binary release -1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases), then run ```go install``` -2. Run ```linx-server -config path/to/linx-server.conf``` +1. Grab the latest binary from the [releases](https://github.com/andreimarcu/linx-server/releases) +2. Run ```./linx-server``` Usage @@ -108,11 +98,6 @@ maxexpiry = 86400 | ```nologs = true``` | (optionally) disable request logs in stdout | ```force-random-filename = true``` | (optionally) force the use of random filenames | ```custompagespath = custom_pages/``` | (optionally) specify path to directory containing markdown pages (must end in .md) that will be added to the site navigation (this can be useful for providing contact/support information and so on). For example, custom_pages/My_Page.md will become My Page in the site navigation -| ```extra-footer-text = "..."``` | (optionally) Extra text above the footer for notices. -| ```max-duration-time = 0``` | Time till expiry for files over max-duration-size. (Default is 0 for no-expiry.) -| ```max-duration-size = 4294967296``` | Size of file before max-duration-time is used to determine expiry max time. (Default is 4GB) -| ```disable-access-key = true``` | Disables access key usage. (Default is false.) -| ```default-random-filename = true``` | Makes it so the random filename is not default if set false. (Default is true.) #### Cleaning up expired files @@ -141,7 +126,7 @@ The following storage backends are available: |Name|Notes|Options |----|-----|------- |LocalFS|Enabled by default, this backend uses the filesystem|```filespath = files/``` -- Path to store uploads (default is files/)
```metapath = meta/``` -- Path to store information about uploads (default is meta/)| -|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket). File metadata will be stored as tags on the object in the bucket.

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| +|S3|Use with any S3-compatible provider.
This implementation will stream files through the linx instance (every download will request and stream the file from the S3 bucket).

For high-traffic environments, one might consider using an external caching layer such as described [in this article](https://blog.sentry.io/2017/03/01/dodging-s3-downtime-with-nginx-and-haproxy.html).|```s3-endpoint = https://...``` -- S3 endpoint
```s3-region = us-east-1``` -- S3 region
```s3-bucket = mybucket``` -- S3 bucket to use for files and metadata
```s3-force-path-style = true``` (optional) -- force path-style addresing (e.g. https://s3.amazonaws.com/linx/example.txt)

Environment variables to provide:
```AWS_ACCESS_KEY_ID``` -- the S3 access key
```AWS_SECRET_ACCESS_KEY ``` -- the S3 secret key
```AWS_SESSION_TOKEN``` (optional) -- the S3 session token| #### SSL with built-in server diff --git a/auth/apikeys/apikeys.go b/auth.go similarity index 53% rename from auth/apikeys/apikeys.go rename to auth.go index d2a592d..3dc5ba6 100644 --- a/auth/apikeys/apikeys.go +++ b/auth.go @@ -1,4 +1,4 @@ -package apikeys +package main import ( "bufio" @@ -24,18 +24,16 @@ const ( type AuthOptions struct { AuthFile string UnauthMethods []string - BasicAuth bool - SiteName string - SitePath string } -type ApiKeysMiddleware struct { +type auth struct { successHandler http.Handler + failureHandler http.Handler authKeys []string o AuthOptions } -func ReadAuthKeys(authFile string) []string { +func readAuthKeys(authFile string) []string { var authKeys []string f, err := os.Open(authFile) @@ -57,7 +55,7 @@ func ReadAuthKeys(authFile string) []string { return authKeys } -func CheckAuth(authKeys []string, key string) (result bool, err error) { +func checkAuth(authKeys []string, key string) (result bool, err error) { checkKey, err := scrypt.Key([]byte(key), []byte(scryptSalt), scryptN, scryptr, scryptp, scryptKeyLen) if err != nil { return @@ -75,74 +73,53 @@ func CheckAuth(authKeys []string, key string) (result bool, err error) { return } -func (a ApiKeysMiddleware) getSitePrefix() string { - prefix := a.o.SitePath - if len(prefix) <= 0 || prefix[0] != '/' { - prefix = "/" + prefix - } - return prefix -} - -func (a ApiKeysMiddleware) goodAuthorizationHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Location", a.getSitePrefix()) - w.WriteHeader(http.StatusFound) -} - -func (a ApiKeysMiddleware) badAuthorizationHandler(w http.ResponseWriter, r *http.Request) { - if a.o.BasicAuth { - rs := "" - if a.o.SiteName != "" { - rs = fmt.Sprintf(` realm="%s"`, a.o.SiteName) - } - w.Header().Set("WWW-Authenticate", `Basic`+rs) - } - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) -} - -func (a ApiKeysMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var successHandler http.Handler - prefix := a.getSitePrefix() - - if r.URL.Path == prefix+"auth" { - successHandler = http.HandlerFunc(a.goodAuthorizationHandler) - } else { - successHandler = a.successHandler - } - - if sliceContains(a.o.UnauthMethods, r.Method) && r.URL.Path != prefix+"auth" { +func (a auth) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if sliceContains(a.o.UnauthMethods, r.Method) { // allow unauthenticated methods - successHandler.ServeHTTP(w, r) + a.successHandler.ServeHTTP(w, r) return } key := r.Header.Get("Linx-Api-Key") - if key == "" && a.o.BasicAuth { + if key == "" && Config.basicAuth { _, password, ok := r.BasicAuth() if ok { key = password } } - result, err := CheckAuth(a.authKeys, key) + result, err := checkAuth(a.authKeys, key) if err != nil || !result { - http.HandlerFunc(a.badAuthorizationHandler).ServeHTTP(w, r) + a.failureHandler.ServeHTTP(w, r) return } - successHandler.ServeHTTP(w, r) + a.successHandler.ServeHTTP(w, r) } -func NewApiKeysMiddleware(o AuthOptions) func(*web.C, http.Handler) http.Handler { +func UploadAuth(o AuthOptions) func(*web.C, http.Handler) http.Handler { fn := func(c *web.C, h http.Handler) http.Handler { - return ApiKeysMiddleware{ + return auth{ successHandler: h, - authKeys: ReadAuthKeys(o.AuthFile), + failureHandler: http.HandlerFunc(badAuthorizationHandler), + authKeys: readAuthKeys(o.AuthFile), o: o, } } return fn } +func badAuthorizationHandler(w http.ResponseWriter, r *http.Request) { + if Config.basicAuth { + rs := "" + if Config.siteName != "" { + rs = fmt.Sprintf(` realm="%s"`, Config.siteName) + } + w.Header().Set("WWW-Authenticate", `Basic`+rs) + } + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) +} + func sliceContains(slice []string, s string) bool { for _, v := range slice { if s == v { diff --git a/auth/apikeys/apikeys_test.go b/auth_test.go similarity index 64% rename from auth/apikeys/apikeys_test.go rename to auth_test.go index 3c2b8e6..ded98b0 100644 --- a/auth/apikeys/apikeys_test.go +++ b/auth_test.go @@ -1,4 +1,4 @@ -package apikeys +package main import ( "testing" @@ -10,15 +10,15 @@ func TestCheckAuth(t *testing.T) { "vFpNprT9wbHgwAubpvRxYCCpA2FQMAK6hFqPvAGrdZo=", } - if r, err := CheckAuth(authKeys, ""); err != nil && r { + if r, err := checkAuth(authKeys, ""); err != nil && r { t.Fatal("Authorization passed for empty key") } - if r, err := CheckAuth(authKeys, "thisisnotvalid"); err != nil && r { + if r, err := checkAuth(authKeys, "thisisnotvalid"); err != nil && r { t.Fatal("Authorization passed for invalid key") } - if r, err := CheckAuth(authKeys, "haPVipRnGJ0QovA9nyqK"); err != nil && !r { + if r, err := checkAuth(authKeys, "haPVipRnGJ0QovA9nyqK"); err != nil && !r { t.Fatal("Authorization failed for valid key") } } diff --git a/backends/localfs/localfs.go b/backends/localfs/localfs.go index 9a72891..aaf487f 100644 --- a/backends/localfs/localfs.go +++ b/backends/localfs/localfs.go @@ -2,7 +2,6 @@ package localfs import ( "encoding/json" - "errors" "io" "io/ioutil" "net/http" @@ -17,7 +16,6 @@ import ( type LocalfsBackend struct { metaPath string filesPath string - locksPath string } type MetadataJSON struct { @@ -27,7 +25,6 @@ type MetadataJSON struct { Mimetype string `json:"mimetype"` Size int64 `json:"size"` Expiry int64 `json:"expiry"` - SrcIp string `json:"srcip,omitempty"` ArchiveFiles []string `json:"archive_files,omitempty"` } @@ -109,8 +106,6 @@ func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) er Sha256sum: metadata.Sha256sum, Expiry: metadata.Expiry.Unix(), Size: metadata.Size, - SrcIp: metadata.SrcIp, - } dst, err := os.Create(metaPath) @@ -129,42 +124,7 @@ func (b LocalfsBackend) writeMetadata(key string, metadata backends.Metadata) er return nil } -func (b LocalfsBackend) Lock(filename string) (err error) { - lockPath := path.Join(b.locksPath, filename) - - lock, err := os.Create(lockPath) - if err != nil { - return err - } - - lock.Close() - return -} - -func (b LocalfsBackend) Unlock(filename string) (err error) { - lockPath := path.Join(b.locksPath, filename) - - err = os.Remove(lockPath) - if err != nil { - return err - } - - return -} - -func (b LocalfsBackend) CheckLock(filename string) (locked bool, err error) { - lockPath := path.Join(b.locksPath, filename) - - if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { - return false, nil - } else { - return true, nil - } - - return false, err -} - -func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { +func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string) (m backends.Metadata, err error) { filePath := path.Join(b.filesPath, key) dst, err := os.Create(filePath) @@ -193,7 +153,6 @@ func (b LocalfsBackend) Put(key string, r io.Reader, expiry time.Time, deleteKey m.Expiry = expiry m.DeleteKey = deleteKey m.AccessKey = accessKey - m.SrcIp = srcIp m.ArchiveFiles, _ = helpers.ListArchiveFiles(m.Mimetype, m.Size, dst) err = b.writeMetadata(key, m) @@ -238,10 +197,9 @@ func (b LocalfsBackend) List() ([]string, error) { return output, nil } -func NewLocalfsBackend(metaPath string, filesPath string, locksPath string) LocalfsBackend { +func NewLocalfsBackend(metaPath string, filesPath string) LocalfsBackend { return LocalfsBackend{ metaPath: metaPath, filesPath: filesPath, - locksPath: locksPath, } } diff --git a/backends/meta.go b/backends/meta.go index 1c5868a..b22276e 100644 --- a/backends/meta.go +++ b/backends/meta.go @@ -12,7 +12,6 @@ type Metadata struct { Mimetype string Size int64 Expiry time.Time - SrcIp string ArchiveFiles []string } diff --git a/backends/s3/s3.go b/backends/s3/s3.go index 17f6725..a558779 100644 --- a/backends/s3/s3.go +++ b/backends/s3/s3.go @@ -3,7 +3,6 @@ package s3 import ( "io" "io/ioutil" - "log" "net/http" "os" "strconv" @@ -126,7 +125,6 @@ func mapMetadata(m backends.Metadata) map[string]*string { "Mimetype": aws.String(m.Mimetype), "Sha256sum": aws.String(m.Sha256sum), "AccessKey": aws.String(m.AccessKey), - "SrcIp": aws.String(m.SrcIp), } } @@ -157,22 +155,7 @@ func unmapMetadata(input map[string]*string) (m backends.Metadata, err error) { return } -func (b S3Backend) Lock(filename string) (err error) { - log.Printf("Locking is not supported on S3") - return -} - -func (b S3Backend) Unlock(filename string) (err error) { - log.Printf("Locking is not supported on S3") - return -} - -func (b S3Backend) CheckLock(filename string) (locked bool, err error) { - log.Printf("Locking is not supported on S3") - return -} - -func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (m backends.Metadata, err error) { +func (b S3Backend) Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string) (m backends.Metadata, err error) { tmpDst, err := ioutil.TempFile("", "linx-server-upload") if err != nil { return m, err diff --git a/backends/storage.go b/backends/storage.go index 02e2a57..864d0a1 100644 --- a/backends/storage.go +++ b/backends/storage.go @@ -12,10 +12,7 @@ type StorageBackend interface { Exists(key string) (bool, error) Head(key string) (Metadata, error) Get(key string) (Metadata, io.ReadCloser, error) - Lock(filename string) (error) - Unlock(filename string) (error) - CheckLock(filename string) (bool, error) - Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string, srcIp string) (Metadata, error) + Put(key string, r io.Reader, expiry time.Time, deleteKey, accessKey string) (Metadata, error) PutMetadata(key string, m Metadata) error ServeFile(key string, w http.ResponseWriter, r *http.Request) error Size(key string) (int64, error) diff --git a/cleanup/cleanup.go b/cleanup/cleanup.go index 97789d5..5920c22 100644 --- a/cleanup/cleanup.go +++ b/cleanup/cleanup.go @@ -8,8 +8,8 @@ import ( "github.com/andreimarcu/linx-server/expiry" ) -func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { - fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir, locksDir) +func Cleanup(filesDir string, metaDir string, noLogs bool) { + fileBackend := localfs.NewLocalfsBackend(metaDir, filesDir) files, err := fileBackend.List() if err != nil { @@ -17,15 +17,6 @@ func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { } for _, filename := range files { - locked, err := fileBackend.CheckLock(filename) - if err != nil { - log.Printf("Error checking if %s is locked: %s", filename, err) - } - if locked { - log.Printf("%s is locked, it will be ignored", filename) - continue - } - metadata, err := fileBackend.Head(filename) if err != nil { if !noLogs { @@ -42,12 +33,10 @@ func Cleanup(filesDir string, metaDir string, locksDir string, noLogs bool) { } } -func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, locksDir string, noLogs bool) { +func PeriodicCleanup(minutes time.Duration, filesDir string, metaDir string, noLogs bool) { c := time.Tick(minutes) for range c { - log.Printf("Running periodic cleanup") - Cleanup(filesDir, metaDir, locksDir, noLogs) - log.Printf("Finished periodic cleanup") + Cleanup(filesDir, metaDir, noLogs) } } diff --git a/csp_test.go b/csp_test.go index b996594..e3dbbdd 100644 --- a/csp_test.go +++ b/csp_test.go @@ -20,7 +20,6 @@ func TestContentSecurityPolicy(t *testing.T) { Config.siteURL = "http://linx.example.org/" Config.filesDir = path.Join(os.TempDir(), generateBarename()) Config.metaDir = Config.filesDir + "_meta" - Config.locksDir = Config.filesDir + "_locks" Config.maxSize = 1024 * 1024 * 1024 Config.noLogs = true Config.siteName = "linx" diff --git a/delete.go b/delete.go index 3f8e429..38e36e3 100644 --- a/delete.go +++ b/delete.go @@ -10,13 +10,9 @@ import ( func deleteHandler(c web.C, w http.ResponseWriter, r *http.Request) { requestKey := r.Header.Get("Linx-Delete-Key") - - if len(r.URL.Query().Get("linx-delete-key")) > 0 { - requestKey = r.URL.Query().Get("linx-delete-key") - } - + filename := c.URLParams["name"] - + // Ensure that file exists and delete key is correct metadata, err := storageBackend.Head(filename) if err == backends.NotFoundErr { diff --git a/expiry.go b/expiry.go index f17f7cd..63b7757 100644 --- a/expiry.go +++ b/expiry.go @@ -11,10 +11,6 @@ var defaultExpiryList = []uint64{ 60, 300, 3600, - 7200, - 14400, - 28800, - 43200, 86400, 604800, 2419200, diff --git a/go.mod b/go.mod index 434e9c6..50a48ab 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,19 @@ module github.com/andreimarcu/linx-server -go 1.21 - -toolchain go1.22.4 +go 1.14 require ( - github.com/GeertJohan/go.rice v1.0.3 - github.com/aws/aws-sdk-go v1.54.9 - github.com/dchest/uniuri v1.2.0 - github.com/dustin/go-humanize v1.0.1 - github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 - github.com/gabriel-vasile/mimetype v1.4.4 - github.com/microcosm-cc/bluemonday v1.0.26 - github.com/minio/sha256-simd v1.0.1 - github.com/russross/blackfriday v1.6.0 + github.com/GeertJohan/go.rice v1.0.0 + github.com/aws/aws-sdk-go v1.29.19 + github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 + github.com/dustin/go-humanize v1.0.0 + github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 + github.com/microcosm-cc/bluemonday v1.0.2 + github.com/minio/sha256-simd v0.1.1 + github.com/russross/blackfriday v1.5.1 github.com/vharitonsky/iniflags v0.0.0-20180513140207-a33cd0b5f3de github.com/zeebo/bencode v1.0.0 - github.com/zenazn/goji v1.0.1 - golang.org/x/crypto v0.24.0 -) - -require ( - github.com/aymerick/douceur v0.2.0 // indirect - github.com/daaku/go.zipexe v1.0.2 // indirect - github.com/gorilla/css v1.0.1 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/kr/text v0.2.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + github.com/zenazn/goji v0.9.0 + golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 + gopkg.in/h2non/filetype.v1 v1.0.5 ) diff --git a/go.sum b/go.sum index d82e16a..15a736c 100644 --- a/go.sum +++ b/go.sum @@ -1,69 +1,76 @@ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.3 h1:k5viR+xGtIhF61125vCE1cmJ5957RQGXG6dmbaWZSmI= -github.com/GeertJohan/go.rice v1.0.3/go.mod h1:XVdrU4pW00M4ikZed5q56tPf1v2KwnIKeIdc9CBYNt4= +github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= +github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/aws/aws-sdk-go v1.54.9 h1:e0Czh9AhrCVPuyaIUnibYmih3cYexJKlqlHSJ2eMKbI= -github.com/aws/aws-sdk-go v1.54.9/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= -github.com/daaku/go.zipexe v1.0.2/go.mod h1:5xWogtqlYnfBXkSB1o9xysukNP9GTvaNkqzUZbt3Bw8= +github.com/aws/aws-sdk-go v1.29.19 h1:+jifYixffn6kzWygtGWFWQMv0tDGyISZHNwugF9V2sE= +github.com/aws/aws-sdk-go v1.29.19/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= +github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY= +github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/uniuri v1.2.0 h1:koIcOUdrTIivZgSLhHQvKgqdWZq5d7KdMEWF1Ud6+5g= -github.com/dchest/uniuri v1.2.0/go.mod h1:fSzm4SLHzNZvWLvWJew423PhAzkpNQYq+uNLq4kxhkY= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 h1:fmFk0Wt3bBxxwZnu48jqMdaOR/IZ4vdtJFuaFV8MpIE= -github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3/go.mod h1:bJWSKrZyQvfTnb2OudyUjurSG4/edverV7n82+K3JiM= -github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= -github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= -github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= -github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 h1:RAV05c0xOkJ3dZGS0JFybxFKZ2WMLabgx3uXnd7rpGs= +github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJm034bVoG30IshVm688T2hi8= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 h1:WQM1NildKThwdP7qWrNAFGzp4ijNLw8RlgENkaI4MJs= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= -github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= -github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday v1.5.1 h1:B8ZN6pD4PVofmlDCDUdELeYrbsVIDM/bpjW3v3zgcRc= +github.com/russross/blackfriday v1.5.1/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/vharitonsky/iniflags v0.0.0-20180513140207-a33cd0b5f3de h1:fkw+7JkxF3U1GzQoX9h69Wvtvxajo5Rbzy6+YMMzPIg= github.com/vharitonsky/iniflags v0.0.0-20180513140207-a33cd0b5f3de/go.mod h1:irMhzlTz8+fVFj6CH2AN2i+WI5S6wWFtK3MBCIxIpyI= github.com/zeebo/bencode v1.0.0 h1:zgop0Wu1nu4IexAZeCZ5qbsjU4O1vMrfCrVgUjbHVuA= github.com/zeebo/bencode v1.0.0/go.mod h1:Ct7CkrWIQuLWAy9M3atFHYq4kG9Ao/SsY5cdtCXmp9Y= -github.com/zenazn/goji v1.0.1 h1:4lbD8Mx2h7IvloP7r2C0D6ltZP6Ufip8Hn0wmSK5LR8= -github.com/zenazn/goji v1.0.1/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/zenazn/goji v0.9.0 h1:RSQQAbXGArQ0dIDEq+PI6WqN6if+5KHu6x2Cx/GXLTQ= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= +gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/helpers/helpers.go b/helpers/helpers.go index f13e302..f51d998 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -7,8 +7,8 @@ import ( "unicode" "github.com/andreimarcu/linx-server/backends" - "github.com/gabriel-vasile/mimetype" "github.com/minio/sha256-simd" + "gopkg.in/h2non/filetype.v1" ) func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { @@ -21,7 +21,7 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { // Get first 512 bytes for mimetype detection header := make([]byte, 512) - headerlen, err := teeReader.Read(header) + _, err = teeReader.Read(header) if err != nil { return } @@ -47,8 +47,17 @@ func GenerateMetadata(r io.Reader) (m backends.Metadata, err error) { // Use the bytes we extracted earlier and attempt to determine the file // type - kind := mimetype.Detect(header[:headerlen]) - m.Mimetype = kind.String() + kind, err := filetype.Match(header) + if err != nil { + m.Mimetype = "application/octet-stream" + return m, err + } else if kind.MIME.Value != "" { + m.Mimetype = kind.MIME.Value + } else if printable(header) { + m.Mimetype = "text/plain" + } else { + m.Mimetype = "application/octet-stream" + } return } diff --git a/helpers/helpers_test.go b/helpers/helpers_test.go index d891173..800d0d2 100644 --- a/helpers/helpers_test.go +++ b/helpers/helpers_test.go @@ -1,10 +1,8 @@ package helpers import ( - "bytes" "strings" "testing" - "unicode/utf16" ) func TestGenerateMetadata(t *testing.T) { @@ -19,7 +17,7 @@ func TestGenerateMetadata(t *testing.T) { t.Fatalf("Sha256sum was %q instead of expected value of %q", m.Sha256sum, expectedSha256sum) } - expectedMimetype := "text/plain; charset=utf-8" + expectedMimetype := "text/plain" if m.Mimetype != expectedMimetype { t.Fatalf("Mimetype was %q instead of expected value of %q", m.Mimetype, expectedMimetype) } @@ -29,45 +27,3 @@ func TestGenerateMetadata(t *testing.T) { t.Fatalf("Size was %d instead of expected value of %d", m.Size, expectedSize) } } - -func TestTextCharsets(t *testing.T) { - // verify that different text encodings are detected and passed through - orig := "This is a text string" - utf16 := utf16.Encode([]rune(orig)) - utf16LE := make([]byte, len(utf16)*2+2) - utf16BE := make([]byte, len(utf16)*2+2) - utf8 := []byte(orig) - utf16LE[0] = 0xff - utf16LE[1] = 0xfe - utf16BE[0] = 0xfe - utf16BE[1] = 0xff - for i := 0; i < len(utf16); i++ { - lsb := utf16[i] & 0xff - msb := utf16[i] >> 8 - utf16LE[i*2+2] = byte(lsb) - utf16LE[i*2+3] = byte(msb) - utf16BE[i*2+2] = byte(msb) - utf16BE[i*2+3] = byte(lsb) - } - - testcases := []struct { - data []byte - extension string - mimetype string - }{ - {mimetype: "text/plain; charset=utf-8", data: utf8}, - {mimetype: "text/plain; charset=utf-16le", data: utf16LE}, - {mimetype: "text/plain; charset=utf-16be", data: utf16BE}, - } - - for i, testcase := range testcases { - r := bytes.NewReader(testcase.data) - m, err := GenerateMetadata(r) - if err != nil { - t.Fatalf("[%d] unexpected error return %v\n", i, err) - } - if m.Mimetype != testcase.mimetype { - t.Errorf("[%d] Expected mimetype '%s', got mimetype '%s'\n", i, testcase.mimetype, m.Mimetype) - } - } -} diff --git a/linx-cleanup/linx-cleanup.go b/linx-cleanup/linx-cleanup.go index 08d8363..13b3ef1 100644 --- a/linx-cleanup/linx-cleanup.go +++ b/linx-cleanup/linx-cleanup.go @@ -9,18 +9,15 @@ import ( func main() { var filesDir string var metaDir string - var locksDir string var noLogs bool flag.StringVar(&filesDir, "filespath", "files/", "path to files directory") flag.StringVar(&metaDir, "metapath", "meta/", "path to metadata directory") - flag.StringVar(&locksDir, "lockspath", "locks/", - "path to locks directory") flag.BoolVar(&noLogs, "nologs", false, "don't log deleted files") flag.Parse() - cleanup.Cleanup(filesDir, metaDir, locksDir, noLogs) + cleanup.Cleanup(filesDir, metaDir, noLogs) } diff --git a/metalog.sh b/metalog.sh deleted file mode 100755 index 9c05b88..0000000 --- a/metalog.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# For setting up cron with a logging function for metadata only. -# Add to crontab with "crontab -e" -# Paste "*/1 * * * * /path/to/metalog.sh /path/to/meta/source/ /path/to/meta-log" -# Change it to source and log directory paths. -# The slash after source/ is important. It means everyhitng *inside* the source directory. -rsync -a ${1} ${2} \ No newline at end of file diff --git a/pages.go b/pages.go index 6492651..ae8de42 100644 --- a/pages.go +++ b/pages.go @@ -21,10 +21,9 @@ const ( func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["index.html"], pongo2.Context{ - "maxsize": Config.maxSize, - "expirylist": listExpirationTimes(), - "expirydefault": Config.defaultExpiry, - "forcerandom": Config.forceRandomFilename, + "maxsize": Config.maxSize, + "expirylist": listExpirationTimes(), + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -33,9 +32,8 @@ func indexHandler(c web.C, w http.ResponseWriter, r *http.Request) { func pasteHandler(c web.C, w http.ResponseWriter, r *http.Request) { err := renderTemplate(Templates["paste.html"], pongo2.Context{ - "expirylist": listExpirationTimes(), - "expirydefault": Config.defaultExpiry, - "forcerandom": Config.forceRandomFilename, + "expirylist": listExpirationTimes(), + "forcerandom": Config.forceRandomFilename, }, r, w) if err != nil { oopsHandler(c, w, r, RespHTML, "") diff --git a/put.icu.sxcu b/put.icu.sxcu deleted file mode 100644 index 79bd301..0000000 --- a/put.icu.sxcu +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Version": "13.1.0", - "DestinationType": "ImageUploader, TextUploader, FileUploader", - "RequestMethod": "PUT", - "RequestURL": "https://put.icu/upload/", - "Headers": { - "Accept": "application/json", - "Linx-Delete-Key": null, - "Linx-Access-Key": null - }, - "Body": "Binary", - "URL": "$json:url$", - "DeletionURL": "https://put.icu/delete/$json:filename$?linx-delete-key=$json:delete_key$" -} \ No newline at end of file diff --git a/server.go b/server.go index ec66f75..dae3491 100644 --- a/server.go +++ b/server.go @@ -16,7 +16,6 @@ import ( "time" rice "github.com/GeertJohan/go.rice" - "github.com/andreimarcu/linx-server/auth/apikeys" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/backends/localfs" "github.com/andreimarcu/linx-server/backends/s3" @@ -43,7 +42,6 @@ var Config struct { bind string filesDir string metaDir string - locksDir string siteName string siteURL string sitePath string @@ -57,7 +55,6 @@ var Config struct { xFrameOptions string maxSize int64 maxExpiry uint64 - defaultExpiry uint64 realIp bool noLogs bool allowHotlink bool @@ -76,11 +73,6 @@ var Config struct { accessKeyCookieExpiry uint64 customPagesDir string cleanupEveryMinutes uint64 - extraFooterText string - maxDurationTime uint64 - maxDurationSize int64 - disableAccessKey bool - defaultRandomFilename bool } var Templates = make(map[string]*pongo2.Template) @@ -118,12 +110,9 @@ func setup() *web.Mux { mux.Use(AddHeaders(Config.addHeaders)) if Config.authFile != "" { - mux.Use(apikeys.NewApiKeysMiddleware(apikeys.AuthOptions{ + mux.Use(UploadAuth(AuthOptions{ AuthFile: Config.authFile, UnauthMethods: []string{"GET", "HEAD", "OPTIONS", "TRACE"}, - BasicAuth: Config.basicAuth, - SiteName: Config.siteName, - SitePath: Config.sitePath, })) } @@ -138,11 +127,6 @@ func setup() *web.Mux { log.Fatal("Could not create metadata directory:", err) } - err = os.MkdirAll(Config.locksDir, 0755) - if err != nil { - log.Fatal("Could not create locks directory:", err) - } - if Config.siteURL != "" { // ensure siteURL ends wth '/' if lastChar := Config.siteURL[len(Config.siteURL)-1:]; lastChar != "/" { @@ -167,9 +151,9 @@ func setup() *web.Mux { if Config.s3Bucket != "" { storageBackend = s3.NewS3Backend(Config.s3Bucket, Config.s3Region, Config.s3Endpoint, Config.s3ForcePathStyle) } else { - storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir, Config.locksDir) + storageBackend = localfs.NewLocalfsBackend(Config.metaDir, Config.filesDir) if Config.cleanupEveryMinutes > 0 { - go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.locksDir, Config.noLogs) + go cleanup.PeriodicCleanup(time.Duration(Config.cleanupEveryMinutes)*time.Minute, Config.filesDir, Config.metaDir, Config.noLogs) } } @@ -212,10 +196,29 @@ func setup() *web.Mux { mux.Get(Config.sitePath+"upload/", uploadRemote) if Config.remoteAuthFile != "" { - remoteAuthKeys = apikeys.ReadAuthKeys(Config.remoteAuthFile) + remoteAuthKeys = readAuthKeys(Config.remoteAuthFile) } } + if Config.basicAuth { + options := AuthOptions{ + AuthFile: Config.authFile, + UnauthMethods: []string{}, + } + okFunc := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Location", Config.sitePath) + w.WriteHeader(http.StatusFound) + } + authHandler := auth{ + successHandler: http.HandlerFunc(okFunc), + failureHandler: http.HandlerFunc(badAuthorizationHandler), + authKeys: readAuthKeys(Config.authFile), + o: options, + } + mux.Head(Config.sitePath+"auth", authHandler) + mux.Get(Config.sitePath+"auth", authHandler) + } + mux.Post(Config.sitePath+"upload", uploadPostHandler) mux.Post(Config.sitePath+"upload/", uploadPostHandler) mux.Put(Config.sitePath+"upload", uploadPutHandler) @@ -223,9 +226,7 @@ func setup() *web.Mux { mux.Put(Config.sitePath+"upload/:name", uploadPutHandler) mux.Delete(Config.sitePath+":name", deleteHandler) - // Adding new delete path method to make linx-server usable with ShareX. - mux.Get(Config.sitePath+"delete/:name", deleteHandler) - + mux.Get(Config.sitePath+"static/*", staticHandler) mux.Get(Config.sitePath+"favicon.ico", staticHandler) mux.Get(Config.sitePath+"robots.txt", staticHandler) @@ -255,8 +256,6 @@ func main() { "path to files directory") flag.StringVar(&Config.metaDir, "metapath", "meta/", "path to metadata directory") - flag.StringVar(&Config.locksDir, "lockspath", "locks/", - "path to locks directory") flag.BoolVar(&Config.basicAuth, "basicauth", false, "allow logging by basic auth password") flag.BoolVar(&Config.noLogs, "nologs", false, @@ -273,8 +272,6 @@ func main() { "maximum upload file size in bytes (default 4GB)") flag.Uint64Var(&Config.maxExpiry, "maxexpiry", 0, "maximum expiration time in seconds (default is 0, which is no expiry)") - flag.Uint64Var(&Config.defaultExpiry, "default-expiry", 86400, - "default expiration time in seconds (default is 86400, which is 1 day)") flag.StringVar(&Config.certFile, "certfile", "", "path to ssl certificate (for https)") flag.StringVar(&Config.keyFile, "keyfile", "", @@ -301,7 +298,7 @@ func main() { flag.StringVar(&Config.fileReferrerPolicy, "filereferrerpolicy", "same-origin", "value of Referrer-Policy header for file access") - flag.StringVar(&Config.xFrameOptions, "xframeoptions", "", + flag.StringVar(&Config.xFrameOptions, "xframeoptions", "SAMEORIGIN", "value of X-Frame-Options header") flag.Var(&Config.addHeaders, "addheader", "Add an arbitrary header to the response. This option can be used multiple times.") @@ -322,12 +319,7 @@ func main() { "path to directory containing .md files to render as custom pages") flag.Uint64Var(&Config.cleanupEveryMinutes, "cleanup-every-minutes", 0, "How often to clean up expired files in minutes (default is 0, which means files will be cleaned up as they are accessed)") - flag.StringVar(&Config.extraFooterText, "extra-footer-text", "", - "Extra text above the footer for notices.") - flag.Uint64Var(&Config.maxDurationTime, "max-duration-time", 0, "Time till expiry for files over max-duration-size. (Default is 0 for no-expiry.)") - flag.Int64Var(&Config.maxDurationSize, "max-duration-size", 4*1024*1024*1024, "Size of file before max-duration-time is used to determine expiry max time. (Default is 4GB)") - flag.BoolVar(&Config.disableAccessKey, "disable-access-key", false, "Disables access key usage. (Default is false.)") - flag.BoolVar(&Config.defaultRandomFilename, "default-random-filename", true, "Makes it so the random filename is not default if set false. (Default is true.)") + iniflags.Parse() mux := setup() diff --git a/server_test.go b/server_test.go index 5b5fb76..fc225ce 100644 --- a/server_test.go +++ b/server_test.go @@ -33,7 +33,6 @@ func TestSetup(t *testing.T) { Config.siteURL = "http://linx.example.org/" Config.filesDir = path.Join(os.TempDir(), generateBarename()) Config.metaDir = Config.filesDir + "_meta" - Config.locksDir = Config.filesDir + "_locks" Config.maxSize = 1024 * 1024 * 1024 Config.noLogs = true Config.siteName = "linx" @@ -447,6 +446,63 @@ func TestPostJSONUpload(t *testing.T) { } } +func TestPostJSONUploadMaxExpiry(t *testing.T) { + mux := setup() + Config.maxExpiry = 300 + + // include 0 to test edge case + // https://github.com/andreimarcu/linx-server/issues/111 + testExpiries := []string{"86400", "-150", "0"} + for _, expiry := range testExpiries { + w := httptest.NewRecorder() + + filename := generateBarename() + ".txt" + + var b bytes.Buffer + mw := multipart.NewWriter(&b) + fw, err := mw.CreateFormFile("file", filename) + if err != nil { + t.Fatal(err) + } + + fw.Write([]byte("File content")) + mw.Close() + + req, err := http.NewRequest("POST", "/upload/", &b) + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("Accept", "application/json") + req.Header.Set("Linx-Expiry", expiry) + if err != nil { + t.Fatal(err) + } + + mux.ServeHTTP(w, req) + + if w.Code != 200 { + t.Log(w.Body.String()) + t.Fatalf("Status code is not 200, but %d", w.Code) + } + + var myjson RespOkJSON + err = json.Unmarshal([]byte(w.Body.String()), &myjson) + if err != nil { + t.Fatal(err) + } + + myExp, err := strconv.ParseInt(myjson.Expiry, 10, 64) + if err != nil { + t.Fatal(err) + } + + expected := time.Now().Add(time.Duration(Config.maxExpiry) * time.Second).Unix() + if myExp != expected { + t.Fatalf("File expiry is not %d but %s", expected, myjson.Expiry) + } + } + + Config.maxExpiry = 0 +} + func TestPostExpiresJSONUpload(t *testing.T) { mux := setup() w := httptest.NewRecorder() @@ -1199,7 +1255,6 @@ func TestInferSiteURLHTTPSFastCGI(t *testing.T) { func TestShutdown(t *testing.T) { os.RemoveAll(Config.filesDir) os.RemoveAll(Config.metaDir) - os.RemoveAll(Config.locksDir) } func TestPutAndGetCLI(t *testing.T) { diff --git a/static/mdpages/Sample.md b/static/mdpages/Sample.md deleted file mode 100644 index 9b3956a..0000000 --- a/static/mdpages/Sample.md +++ /dev/null @@ -1,3 +0,0 @@ -A sample custom page for use with the custom pages plugin. - -`custompagespath = /path/to/linx-server/static/mdpages` diff --git a/static/robots.txt b/static/robots.txt index ec4a551..44ec3ed 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,3 +1,3 @@ User-agent: * Allow: /$ - +Disallow: * diff --git a/templates.go b/templates.go index 458af90..7d38b51 100644 --- a/templates.go +++ b/templates.go @@ -87,11 +87,6 @@ func renderTemplate(tpl *pongo2.Template, context pongo2.Context, r *http.Reques context["sitepath"] = Config.sitePath context["selifpath"] = Config.selifPath context["custom_pages_names"] = customPagesNames - // Add the context for Config.extraFooterText - context["extra_footer_text"] = Config.extraFooterText - // Add the context for Config.disableAccessKey - context["disable_access_key"] = Config.disableAccessKey - context["default_randomize"] = Config.defaultRandomFilename var a string if Config.authFile == "" { diff --git a/templates/base.html b/templates/base.html index 54fd368..7e4f82d 100644 --- a/templates/base.html +++ b/templates/base.html @@ -29,8 +29,9 @@ {% block content %}{% endblock %} + diff --git a/templates/display/audio.html b/templates/display/audio.html index 68d461e..ba491f0 100644 --- a/templates/display/audio.html +++ b/templates/display/audio.html @@ -2,7 +2,6 @@ {% block head %} - {% endblock %} {% block main %} diff --git a/templates/display/image.html b/templates/display/image.html index 8dcd844..985c504 100644 --- a/templates/display/image.html +++ b/templates/display/image.html @@ -1,7 +1,6 @@ {% extends "base.html" %} {% block head %} - {% endblock %} @@ -9,4 +8,4 @@ -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/templates/display/video.html b/templates/display/video.html index 3cf6a81..54a796f 100644 --- a/templates/display/video.html +++ b/templates/display/video.html @@ -1,7 +1,6 @@ {% extends "base.html" %} {% block head %} - {% endblock %} diff --git a/templates/index.html b/templates/index.html index 3958788..34e5d86 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,20 +21,19 @@ + checked /> Randomize filename
- {% if disable_access_key != true %}
@@ -45,7 +44,6 @@
- {% endif %}
diff --git a/templates/paste.html b/templates/paste.html index b22ca96..84335d1 100644 --- a/templates/paste.html +++ b/templates/paste.html @@ -14,15 +14,14 @@ id="extension" class="codebox" name='extension' type='text' value="" placeholder="txt" />
- {% if disable_access_key != true %} - {% endif %} + diff --git a/upload.go b/upload.go index 0485c22..8526260 100644 --- a/upload.go +++ b/upload.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "log" "net/http" "net/url" "path" @@ -16,13 +15,13 @@ import ( "strings" "time" - "github.com/andreimarcu/linx-server/auth/apikeys" "github.com/andreimarcu/linx-server/backends" "github.com/andreimarcu/linx-server/expiry" "github.com/dchest/uniuri" - "github.com/gabriel-vasile/mimetype" "github.com/zenazn/goji/web" + "gopkg.in/h2non/filetype.v1" ) + var FileTooLargeError = errors.New("File too large.") var fileBlacklist = map[string]bool{ "favicon.ico": true, @@ -42,7 +41,6 @@ type UploadRequest struct { deleteKey string // Empty string if not defined randomBarename bool accessKey string // Empty string if not defined - srcIp string // Empty string if not defined } // Metadata associated with a file as it would actually be stored @@ -61,6 +59,7 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { uploadHeaderProcess(r, &upReq) contentType := r.Header.Get("Content-Type") + if strings.HasPrefix(contentType, "multipart/form-data") { file, headers, err := r.FormFile("file") if err != nil { @@ -91,10 +90,11 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { upReq.expiry = parseExpiry(r.PostFormValue("expires")) upReq.accessKey = r.PostFormValue(accessKeyParamName) + if r.PostFormValue("randomize") == "true" { upReq.randomBarename = true } - upReq.srcIp = r.Header.Get("X-Forwarded-For") + upload, err := processUpload(upReq) if strings.EqualFold("application/json", r.Header.Get("Accept")) { @@ -125,11 +125,11 @@ func uploadPostHandler(c web.C, w http.ResponseWriter, r *http.Request) { func uploadPutHandler(c web.C, w http.ResponseWriter, r *http.Request) { upReq := UploadRequest{} uploadHeaderProcess(r, &upReq) - + defer r.Body.Close() upReq.filename = c.URLParams["name"] upReq.src = http.MaxBytesReader(w, r.Body, Config.maxSize) - upReq.srcIp = r.Header.Get("X-Forwarded-For") + upload, err := processUpload(upReq) if strings.EqualFold("application/json", r.Header.Get("Accept")) { @@ -166,16 +166,13 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) { key = password } } - result, err := apikeys.CheckAuth(remoteAuthKeys, key) + result, err := checkAuth(remoteAuthKeys, key) if err != nil || !result { if Config.basicAuth { - rs := "" - if Config.siteName != "" { - rs = fmt.Sprintf(` realm="%s"`, Config.siteName) - } - w.Header().Set("WWW-Authenticate", `Basic`+rs) + badAuthorizationHandler(w, r) + } else { + unauthorizedHandler(c, w, r) } - unauthorizedHandler(c, w, r) return } } @@ -194,14 +191,14 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) { oopsHandler(c, w, r, RespAUTO, "Could not retrieve URL") return } - + upReq.filename = filepath.Base(grabUrl.Path) upReq.src = http.MaxBytesReader(w, resp.Body, Config.maxSize) upReq.deleteKey = r.FormValue("deletekey") upReq.accessKey = r.FormValue(accessKeyParamName) upReq.randomBarename = r.FormValue("randomize") == "yes" upReq.expiry = parseExpiry(r.FormValue("expiry")) - upReq.srcIp = r.Header.Get("X-Forwarded-For") + upload, err := processUpload(upReq) if strings.EqualFold("application/json", r.Header.Get("Accept")) { @@ -230,11 +227,11 @@ func uploadRemote(c web.C, w http.ResponseWriter, r *http.Request) { func uploadHeaderProcess(r *http.Request, upReq *UploadRequest) { if r.Header.Get("Linx-Randomize") == "yes" { upReq.randomBarename = true - } else { - upReq.randomBarename = false } + upReq.deleteKey = r.Header.Get("Linx-Delete-Key") upReq.accessKey = r.Header.Get(accessKeyHeaderName) + // Get seconds until expiry. Non-integer responses never expire. expStr := r.Header.Get("Linx-Expiry") upReq.expiry = parseExpiry(expStr) @@ -266,11 +263,11 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { header = header[:n] // Determine the type of file from header - kind := mimetype.Detect(header) - if len(kind.Extension()) < 2 { + kind, err := filetype.Match(header) + if err != nil || kind.Extension == "unknown" { extension = "file" } else { - extension = kind.Extension()[1:] // remove leading "." + extension = kind.Extension } } @@ -321,45 +318,19 @@ func processUpload(upReq UploadRequest) (upload Upload, err error) { return upload, errors.New("Prohibited filename") } - // Lock the upload - log.Printf("Lock %s", upload.Filename) - err = storageBackend.Lock(upload.Filename) - if err != nil { - return upload, err - } - // Get the rest of the metadata needed for storage var fileExpiry time.Time - maxDurationTime := time.Duration(Config.maxDurationTime) * time.Second if upReq.expiry == 0 { - if upReq.size > Config.maxDurationSize && maxDurationTime > 0 { - fileExpiry = time.Now().Add(maxDurationTime) - } else { - fileExpiry = expiry.NeverExpire - } + fileExpiry = expiry.NeverExpire } else { - if upReq.size > Config.maxDurationSize && upReq.expiry > maxDurationTime { - fileExpiry = time.Now().Add(maxDurationTime) - } else { - fileExpiry = time.Now().Add(upReq.expiry) - } + fileExpiry = time.Now().Add(upReq.expiry) } if upReq.deleteKey == "" { upReq.deleteKey = uniuri.NewLen(30) } - if Config.disableAccessKey == true { - upReq.accessKey = "" - } - log.Printf("Write %s", upload.Filename) - upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deleteKey, upReq.accessKey, upReq.srcIp) - if err != nil { - return upload, err - } - // Unlock the upload - log.Printf("Unlock %s", upload.Filename) - err = storageBackend.Unlock(upload.Filename) + upload.Metadata, err = storageBackend.Put(upload.Filename, io.MultiReader(bytes.NewReader(header), upReq.src), fileExpiry, upReq.deleteKey, upReq.accessKey) if err != nil { return upload, err } @@ -423,11 +394,11 @@ func barePlusExt(filename string) (barename, extension string) { func parseExpiry(expStr string) time.Duration { if expStr == "" { - return time.Duration(Config.defaultExpiry) * time.Second + return time.Duration(Config.maxExpiry) * time.Second } else { fileExpiry, err := strconv.ParseUint(expStr, 10, 64) if err != nil { - return time.Duration(Config.defaultExpiry) * time.Second + return time.Duration(Config.maxExpiry) * time.Second } else { if Config.maxExpiry > 0 && (fileExpiry > Config.maxExpiry || fileExpiry == 0) { fileExpiry = Config.maxExpiry