Upload: Allow to limit the types of files users can upload #4895

Signed-off-by: Michael Mayer <michael@photoprism.app>

frontend/src/component/photo/upload/dialog.vue

@@ -11,7 +11,7 @@
     @after-leave="afterLeave"
   >
     <v-form ref="form" class="p-photo-upload" validate-on="invalid-input" tabindex="1" @submit.prevent="submit">
-      <input ref="upload" type="file" multiple class="d-none input-upload" @change.stop="onUpload()" />
+      <input ref="upload" type="file" multiple :accept="accept" class="d-none input-upload" @change.stop="onUpload()" />
       <v-card :tile="$vuetify.display.mdAndDown">
         <v-toolbar
           v-if="$vuetify.display.mdAndDown"
@@ -151,6 +151,7 @@ export default {
   data() {
     const isDemo = this.$config.get("demo");
     return {
+      accept: this.$config.get("uploadAllow"),
       albums: [],
       selectedAlbums: [],
       selected: [],

internal/api/users_upload.go

@@ -87,10 +87,28 @@ func UploadUserFiles(router *gin.RouterGroup) {
 			return
 		}
 
-		// Save uploaded files.
+		// List of allowed file extensions (all types allowed if empty).
+		allowExt := conf.UploadAllow()
+
+		// Save uploaded files if their extension is allowed.
 		for _, file := range files {
 			fileName := filepath.Base(file.Filename)
 			filePath := path.Join(uploadDir, fileName)
+			fileType := fs.FileType(fileName)
+
+			if fileType == fs.TypeUnknown {
+				log.Warnf("upload: rejected %s due to unknown or unsupported extension", clean.Log(fileName))
+				if err = os.Remove(filePath); err != nil {
+					log.Errorf("upload: %s could not be deleted (%s)", clean.Log(fileName), err)
+				}
+				continue
+			} else if allowExt.Excludes(fileType.DefaultExt()) {
+				log.Warnf("upload: rejected %s because the file type is not allowed", clean.Log(fileName))
+				if err = os.Remove(filePath); err != nil {
+					log.Errorf("upload: %s could not be deleted (%s)", clean.Log(fileName), err)
+				}
+				continue
+			}
 
 			if err = c.SaveUploadedFile(file, filePath); err != nil {
 				log.Errorf("upload: failed saving file %s", clean.Log(fileName))
@@ -105,16 +123,16 @@ func UploadUserFiles(router *gin.RouterGroup) {
 		}
 
 		// Check if uploaded file is safe.
-		if !conf.UploadNSFW() {
+		if len(uploads) > 0 && !conf.UploadNSFW() {
 			nd := get.NsfwDetector()
 
 			containsNSFW := false
 
 			for _, filename := range uploads {
-				labels, err := nd.File(filename)
+				labels, nsfwErr := nd.File(filename)
 
-				if err != nil {
-					log.Debug(err)
+				if nsfwErr != nil {
+					log.Debug(nsfwErr)
 					continue
 				}
 
@@ -235,8 +253,8 @@ func ProcessUserUpload(router *gin.RouterGroup) {
 			log.Infof("upload: imported %s", english.Plural(n, "file", "files"))
 			if moments := get.Moments(); moments == nil {
 				log.Warnf("upload: moments service not set - you may have found a bug")
-			} else if err := moments.Start(); err != nil {
-				log.Warnf("moments: %s", err)
+			} else if workerErr := moments.Start(); workerErr != nil {
+				log.Warnf("moments: %s", workerErr)
 			}
 		}
 
@@ -258,8 +276,8 @@ func ProcessUserUpload(router *gin.RouterGroup) {
 		UpdateClientConfig()
 
 		// Update album, label, and subject cover thumbs.
-		if err := query.UpdateCovers(); err != nil {
-			log.Warnf("upload: %s (update covers)", err)
+		if coversErr := query.UpdateCovers(); coversErr != nil {
+			log.Warnf("upload: %s (update covers)", coversErr)
 		}
 
 		c.JSON(http.StatusOK, i18n.Response{Code: http.StatusOK, Msg: msg})

internal/config/client_config.go

@@ -61,6 +61,7 @@ type ClientConfig struct {
 	Sponsor          bool                `json:"sponsor"`
 	ReadOnly         bool                `json:"readonly"`
 	UploadNSFW       bool                `json:"uploadNSFW"`
+	UploadAllow      string              `json:"uploadAllow"`
 	Public           bool                `json:"public"`
 	AuthMode         string              `json:"authMode"`
 	UsersPath        string              `json:"usersPath"`
@@ -392,6 +393,7 @@ func (c *Config) ClientShare() *ClientConfig {
 		Sponsor:          c.Sponsor(),
 		ReadOnly:         c.ReadOnly(),
 		UploadNSFW:       c.UploadNSFW(),
+		UploadAllow:      c.UploadAllow().Accept(),
 		Public:           c.Public(),
 		AuthMode:         c.AuthMode(),
 		UsersPath:        "",
@@ -491,6 +493,7 @@ func (c *Config) ClientUser(withSettings bool) *ClientConfig {
 		Sponsor:          c.Sponsor(),
 		ReadOnly:         c.ReadOnly(),
 		UploadNSFW:       c.UploadNSFW(),
+		UploadAllow:      c.UploadAllow().Accept(),
 		Public:           c.Public(),
 		AuthMode:         c.AuthMode(),
 		UsersPath:        c.UsersPath(),

internal/config/config_ai.go

@@ -30,8 +30,3 @@ func (c *Config) NSFWModelPath() string {
 func (c *Config) DetectNSFW() bool {
 	return c.options.DetectNSFW
 }
-
-// UploadNSFW checks if NSFW photos can be uploaded.
-func (c *Config) UploadNSFW() bool {
-	return c.options.UploadNSFW
-}

internal/config/config_media_test.go

@@ -113,3 +113,9 @@ func TestConfig_DisableJpegXL(t *testing.T) {
 	assert.True(t, c.DisableJpegXL())
 	c.options.DisableJpegXL = false
 }
+
+func TestConfig_Import(t *testing.T) {
+	c := NewConfig(CliTestContext())
+
+	assert.Equal(t, "avif, avifs, thm", c.SipsExclude())
+}

internal/config/config_storage.go

@@ -337,6 +337,11 @@ func (c *Config) ImportDest() string {
 	return clean.UserPath(c.options.ImportDest)
 }
 
+// ImportAllow returns the file extensions that users are allowed to import.
+func (c *Config) ImportAllow() fs.ExtList {
+	return fs.NewExtList(c.options.ImportAllow)
+}
+
 // SidecarPath returns the storage path for generated sidecar files (relative or absolute).
 func (c *Config) SidecarPath() string {
 	if c.options.SidecarPath == "" {

internal/config/config_storage_test.go

@@ -453,6 +453,19 @@ func TestConfig_ImportPath2(t *testing.T) {
 	}
 }
 
+func TestConfig_ImportAllow(t *testing.T) {
+	c := NewConfig(CliTestContext())
+
+	c.options.ImportAllow = "jpg, PNG,pdf"
+
+	assert.Equal(t, "jpg, pdf, png", c.ImportAllow().String())
+
+	c.options.ImportAllow = ""
+
+	assert.Len(t, c.ImportAllow(), 0)
+	assert.Equal(t, "", c.ImportAllow().String())
+}
+
 func TestConfig_AssetsPath2(t *testing.T) {
 	c := NewConfig(CliTestContext())
 	assert.Equal(t, "/go/src/github.com/photoprism/photoprism/assets", c.AssetsPath())

internal/config/config_upload.go

@@ -0,0 +1,15 @@
+package config
+
+import (
+	"github.com/photoprism/photoprism/pkg/fs"
+)
+
+// UploadNSFW checks if NSFW photos can be uploaded.
+func (c *Config) UploadNSFW() bool {
+	return c.options.UploadNSFW
+}
+
+// UploadAllow returns the file extensions that users are allowed to upload.
+func (c *Config) UploadAllow() fs.ExtList {
+	return fs.NewExtList(c.options.UploadAllow)
+}

internal/config/config_upload_test.go

@@ -0,0 +1,26 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfig_UploadNSFW(t *testing.T) {
+	c := NewConfig(CliTestContext())
+
+	assert.False(t, c.UploadNSFW())
+}
+
+func TestConfig_UploadAllow(t *testing.T) {
+	c := NewConfig(CliTestContext())
+
+	c.options.UploadAllow = "jpg, PNG,pdf"
+
+	assert.Equal(t, "jpg, pdf, png", c.UploadAllow().String())
+
+	c.options.UploadAllow = ""
+
+	assert.Len(t, c.UploadAllow(), 0)
+	assert.Equal(t, "", c.UploadAllow().String())
+}

internal/config/flags.go

@@ -237,6 +237,22 @@ var Flags = CliFlags{
 			Usage:   "relative originals `PATH` to which the files should be imported by default *optional*",
 			EnvVars: EnvVars("IMPORT_DEST"),
 		}}, {
+		Flag: &cli.StringFlag{
+			Name:    "import-allow",
+			Usage:   "allow to import these file types (comma-separated list of `EXTENSIONS`; leave blank to allow all)",
+			EnvVars: EnvVars("IMPORT_ALLOW"),
+		}}, {
+		Flag: &cli.BoolFlag{
+			Name:    "upload-nsfw",
+			Aliases: []string{"n"},
+			Usage:   "allow uploads that might be offensive (detecting unsafe content requires TensorFlow)",
+			EnvVars: EnvVars("UPLOAD_NSFW"),
+		}}, {
+		Flag: &cli.StringFlag{
+			Name:    "upload-allow",
+			Usage:   "allow to upload these file types (comma-separated list of `EXTENSIONS`; leave blank to allow all)",
+			EnvVars: EnvVars("UPLOAD_ALLOW"),
+		}}, {
 		Flag: &cli.StringFlag{
 			Name:    "cache-path",
 			Aliases: []string{"ca"},
@@ -453,18 +469,6 @@ var Flags = CliFlags{
 			Usage:   "flag newly added pictures as private if they might be offensive (requires TensorFlow)",
 			EnvVars: EnvVars("DETECT_NSFW"),
 		}}, {
-		Flag: &cli.BoolFlag{
-			Name:    "upload-nsfw",
-			Aliases: []string{"n"},
-			Usage:   "allow uploads that might be offensive (detecting unsafe content requires TensorFlow)",
-			EnvVars: EnvVars("UPLOAD_NSFW"),
-		}}, {
-		Flag: &cli.BoolFlag{
-			Name:    "upload-allow",
-			Usage:   "allow these file types for web uploads (comma-separated list of extensions; leave blank to allow all)",
-			EnvVars: EnvVars("UPLOAD_ALLOW"),
-			Hidden:  true,
-		}}, {
 		Flag: &cli.StringFlag{
 			Name:    "default-locale",
 			Aliases: []string{"lang"},

internal/config/options.go

@@ -66,6 +66,9 @@ type Options struct {
 	StoragePath            string        `yaml:"StoragePath" json:"-" flag:"storage-path"`
 	ImportPath             string        `yaml:"ImportPath" json:"-" flag:"import-path"`
 	ImportDest             string        `yaml:"ImportDest" json:"-" flag:"import-dest"`
+	ImportAllow            string        `yaml:"ImportAllow" json:"ImportAllow" flag:"import-allow"`
+	UploadNSFW             bool          `yaml:"UploadNSFW" json:"-" flag:"upload-nsfw"`
+	UploadAllow            string        `yaml:"UploadAllow" json:"UploadAllow" flag:"upload-allow"`
 	CachePath              string        `yaml:"CachePath" json:"-" flag:"cache-path"`
 	TempPath               string        `yaml:"TempPath" json:"-" flag:"temp-path"`
 	AssetsPath             string        `yaml:"AssetsPath" json:"-" flag:"assets-path"`
@@ -109,7 +112,6 @@ type Options struct {
 	RawPresets             bool          `yaml:"RawPresets" json:"RawPresets" flag:"raw-presets"`
 	ExifBruteForce         bool          `yaml:"ExifBruteForce" json:"ExifBruteForce" flag:"exif-bruteforce"`
 	DetectNSFW             bool          `yaml:"DetectNSFW" json:"DetectNSFW" flag:"detect-nsfw"`
-	UploadNSFW             bool          `yaml:"UploadNSFW" json:"-" flag:"upload-nsfw"`
 	DefaultLocale          string        `yaml:"DefaultLocale" json:"DefaultLocale" flag:"default-locale"`
 	DefaultTimezone        string        `yaml:"DefaultTimezone" json:"DefaultTimezone" flag:"default-timezone"`
 	DefaultTheme           string        `yaml:"DefaultTheme" json:"DefaultTheme" flag:"default-theme"`

internal/config/report.go

@@ -64,6 +64,9 @@ func (c *Config) Report() (rows [][]string, cols []string) {
 		{"users-storage-path", c.UsersStoragePath()},
 		{"import-path", c.ImportPath()},
 		{"import-dest", c.ImportDest()},
+		{"import-allow", c.ImportAllow().String()},
+		{"upload-nsfw", fmt.Sprintf("%t", c.UploadNSFW())},
+		{"upload-allow", c.UploadAllow().String()},
 		{"cache-path", c.CachePath()},
 		{"cmd-cache-path", c.CmdCachePath()},
 		{"media-cache-path", c.MediaCachePath()},
@@ -131,7 +134,6 @@ func (c *Config) Report() (rows [][]string, cols []string) {
 
 		// TensorFlow.
 		{"detect-nsfw", fmt.Sprintf("%t", c.DetectNSFW())},
-		{"upload-nsfw", fmt.Sprintf("%t", c.UploadNSFW())},
 		{"tensorflow-version", c.TensorFlowVersion()},
 		{"tensorflow-model-path", c.TensorFlowModelPath()},
 

internal/photoprism/import.go

@@ -23,17 +23,19 @@ import (
 
 // Import represents an importer that can copy/move MediaFiles to the originals directory.
 type Import struct {
-	conf    *config.Config
-	index   *Index
-	convert *Convert
+	conf     *config.Config
+	index    *Index
+	convert  *Convert
+	AllowExt fs.ExtList
 }
 
 // NewImport returns a new importer and expects its dependencies as arguments.
 func NewImport(conf *config.Config, index *Index, convert *Convert) *Import {
 	instance := &Import{
-		conf:    conf,
-		index:   index,
-		convert: convert,
+		conf:     conf,
+		index:    index,
+		convert:  convert,
+		AllowExt: conf.ImportAllow(),
 	}
 
 	return instance

internal/photoprism/import_worker.go

@@ -27,6 +27,7 @@ func ImportWorker(jobs <-chan ImportJob) {
 		imp := job.Imp
 		opt := job.ImportOpt
 		src := job.ImportOpt.Path
+		allowExt := job.Imp.AllowExt
 
 		related := job.Related
 
@@ -38,12 +39,16 @@ func ImportWorker(jobs <-chan ImportJob) {
 			continue
 		}
 
-		// Create JSON sidecar file, if needed.
-		if jsonErr := related.Main.CreateExifToolJson(imp.convert); jsonErr != nil {
-			log.Warnf("import: %s", clean.Error(jsonErr))
-		}
-
 		originalName := related.Main.RelName(src)
+		mainFileType := related.Main.FileType()
+
+		if mainFileType == fs.TypeUnknown {
+			log.Warnf("import: skipped %s due to unknown or unsupported extension", clean.Log(originalName))
+			continue
+		} else if allowExt.Excludes(mainFileType.DefaultExt()) {
+			log.Warnf("import: skipped %s because the file type is not allowed", clean.Log(originalName))
+			continue
+		}
 
 		event.Publish("import.file", event.Data{
 			"fileName":  originalName,
@@ -51,6 +56,11 @@ func ImportWorker(jobs <-chan ImportJob) {
 			"subFolder": opt.DestFolder,
 		})
 
+		// Create JSON sidecar file, if needed.
+		if jsonErr := related.Main.CreateExifToolJson(imp.convert); jsonErr != nil {
+			log.Warnf("import: %s", clean.Error(jsonErr))
+		}
+
 		for _, f := range related.Files {
 			relFileName := f.RelName(src)
 

internal/photoprism/import_worker_test.go

@@ -13,30 +13,30 @@ import (
 )
 
 func TestImportWorker_OriginalFileNames(t *testing.T) {
-	conf := config.TestConfig()
+	c := config.TestConfig()
 
-	if err := conf.InitializeTestData(); err != nil {
+	if err := c.InitializeTestData(); err != nil {
 		t.Fatal(err)
 	}
 
-	tf := classify.New(conf.AssetsPath(), conf.DisableTensorFlow())
-	nd := nsfw.New(conf.NSFWModelPath())
-	fn := face.NewNet(conf.FaceNetModelPath(), "", conf.DisableTensorFlow())
-	convert := NewConvert(conf)
-	ind := NewIndex(conf, tf, nd, fn, convert, NewFiles(), NewPhotos())
-	imp := &Import{conf, ind, convert}
+	tf := classify.New(c.AssetsPath(), c.DisableTensorFlow())
+	nd := nsfw.New(c.NSFWModelPath())
+	fn := face.NewNet(c.FaceNetModelPath(), "", c.DisableTensorFlow())
+	convert := NewConvert(c)
+	ind := NewIndex(c, tf, nd, fn, convert, NewFiles(), NewPhotos())
+	imp := &Import{c, ind, convert, c.ImportAllow()}
 
-	mediaFileName := conf.ExamplesPath() + "/beach_sand.jpg"
+	mediaFileName := c.ExamplesPath() + "/beach_sand.jpg"
 	mediaFile, err := NewMediaFile(mediaFileName)
 	if err != nil {
 		t.Fatal(err)
 	}
-	mediaFileName2 := conf.ExamplesPath() + "/beach_wood.jpg"
+	mediaFileName2 := c.ExamplesPath() + "/beach_wood.jpg"
 	mediaFile2, err2 := NewMediaFile(mediaFileName2)
 	if err2 != nil {
 		t.Fatal(err2)
 	}
-	mediaFileName3 := conf.ExamplesPath() + "/beach_colorfilter.jpg"
+	mediaFileName3 := c.ExamplesPath() + "/beach_colorfilter.jpg"
 	mediaFile3, err3 := NewMediaFile(mediaFileName3)
 	if err3 != nil {
 		t.Fatal(err3)
@@ -58,7 +58,7 @@ func TestImportWorker_OriginalFileNames(t *testing.T) {
 		FileName:  mediaFile.FileName(),
 		Related:   relatedFiles,
 		IndexOpt:  IndexOptionsAll(),
-		ImportOpt: ImportOptionsCopy(conf.ImportPath(), conf.ImportDest()),
+		ImportOpt: ImportOptionsCopy(c.ImportPath(), c.ImportDest()),
 		Imp:       imp,
 	}
 

pkg/fs/ext_list.go

@@ -1,6 +1,7 @@
 package fs
 
 import (
+	"sort"
 	"strings"
 )
 
@@ -46,6 +47,15 @@ func (b ExtList) Contains(ext string) bool {
 	return false
 }
 
+// Excludes tests if the extension is not included, or returns false if the list is empty.
+func (b ExtList) Excludes(ext string) bool {
+	if len(b) == 0 {
+		return false
+	}
+
+	return !b.Contains(ext)
+}
+
 // Allow tests if a file extension is NOT listed.
 func (b ExtList) Allow(ext string) bool {
 	return !b.Contains(ext)
@@ -75,3 +85,40 @@ func (b ExtList) Add(ext string) {
 
 	b[ext] = true
 }
+
+// String returns the list as a comma-separated list in alphabetical order.
+func (b ExtList) String() string {
+	if len(b) == 0 {
+		return ""
+	}
+
+	list := make([]string, 0, len(b))
+
+	for s := range b {
+		list = append(list, s)
+	}
+
+	sort.Strings(list)
+
+	return strings.Join(list, ", ")
+}
+
+// Accept returns a comma-separated list in alphabetical order for use as an input accept attribute.
+func (b ExtList) Accept() string {
+	if len(b) == 0 {
+		return ""
+	}
+
+	list := make([]string, 0, len(b))
+
+	for typeExt := range b {
+		allExt := FileTypesLower[FileType("."+typeExt)]
+		for _, s := range allExt {
+			list = append(list, s)
+		}
+	}
+
+	sort.Strings(list)
+
+	return strings.Join(list, ",")
+}

pkg/fs/ext_list_test.go

@@ -49,7 +49,7 @@ func TestExtList_Ok(t *testing.T) {
 }
 
 func TestExtList_Contains(t *testing.T) {
-	t.Run("DNG", func(t *testing.T) {
+	t.Run("Success", func(t *testing.T) {
 		list := NewExtList("dng")
 		assert.True(t, list.Contains("dng"))
 		assert.False(t, list.Contains("cr2"))
@@ -60,8 +60,22 @@ func TestExtList_Contains(t *testing.T) {
 	})
 }
 
+func TestExtList_Excludes(t *testing.T) {
+	t.Run("Success", func(t *testing.T) {
+		list := NewExtList("dng")
+		assert.False(t, list.Excludes("dng"))
+		assert.True(t, list.Excludes("cr2"))
+	})
+	t.Run("Empty", func(t *testing.T) {
+		list := NewExtList("")
+		assert.False(t, list.Excludes(""))
+		assert.False(t, list.Excludes("dng"))
+		assert.False(t, list.Excludes("cr2"))
+	})
+}
+
 func TestExtList_Set(t *testing.T) {
-	t.Run("DNG, CR2", func(t *testing.T) {
+	t.Run("Success", func(t *testing.T) {
 		list := NewExtList("dng")
 		assert.True(t, list.Contains("dng"))
 		assert.False(t, list.Contains("cr2"))
@@ -69,7 +83,7 @@ func TestExtList_Set(t *testing.T) {
 		assert.True(t, list.Contains("dng"))
 		assert.True(t, list.Contains("cr2"))
 	})
-	t.Run("DNG", func(t *testing.T) {
+	t.Run("Empty", func(t *testing.T) {
 		list := NewExtList("dng")
 		assert.True(t, list.Contains("dng"))
 		assert.False(t, list.Contains("cr2"))
@@ -80,7 +94,7 @@ func TestExtList_Set(t *testing.T) {
 }
 
 func TestExtList_Add(t *testing.T) {
-	t.Run("DNG, CR2", func(t *testing.T) {
+	t.Run("Success", func(t *testing.T) {
 		list := NewExtList("dng")
 		assert.True(t, list.Contains("dng"))
 		assert.False(t, list.Contains("cr2"))
@@ -88,7 +102,7 @@ func TestExtList_Add(t *testing.T) {
 		assert.True(t, list.Contains("dng"))
 		assert.True(t, list.Contains("cr2"))
 	})
-	t.Run("DNG", func(t *testing.T) {
+	t.Run("Empty", func(t *testing.T) {
 		list := NewExtList("dng")
 		assert.True(t, list.Contains("dng"))
 		assert.False(t, list.Contains("cr2"))
@@ -97,3 +111,33 @@ func TestExtList_Add(t *testing.T) {
 		assert.False(t, list.Contains("cr2"))
 	})
 }
+
+func TestExtList_String(t *testing.T) {
+	t.Run("One", func(t *testing.T) {
+		list := NewExtList("jpg")
+		assert.Equal(t, "jpg", list.String())
+	})
+	t.Run("Two", func(t *testing.T) {
+		list := NewExtList("dng, CR2")
+		assert.Equal(t, "cr2, dng", list.String())
+	})
+	t.Run("Empty", func(t *testing.T) {
+		list := NewExtList("")
+		assert.Equal(t, "", list.String())
+	})
+}
+
+func TestExtList_Accept(t *testing.T) {
+	t.Run("One", func(t *testing.T) {
+		list := NewExtList("jpg")
+		assert.Equal(t, ".jfi,.jfif,.jif,.jpe,.jpeg,.jpg", list.Accept())
+	})
+	t.Run("Two", func(t *testing.T) {
+		list := NewExtList("mp4, avi")
+		assert.Equal(t, ".avi,.mp,.mp4", list.Accept())
+	})
+	t.Run("Empty", func(t *testing.T) {
+		list := NewExtList("")
+		assert.Equal(t, "", list.Accept())
+	})
+}

pkg/fs/file_types_ext.go

@@ -15,3 +15,4 @@ type TypesExt map[Type][]string
 
 // FileTypes contains the default file type extensions.
 var FileTypes = Extensions.Types(ignoreCase)
+var FileTypesLower = Extensions.Types(true)