Thumbs: Configure max cache size and number of workers for libvips #1474

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/config/config.go

@@ -275,13 +275,13 @@ func (c *Config) Init() error {
 	}
 
 	// Fail if less than 128 MB of memory were detected.
-	if TotalMem < 128*Megabyte {
-		return fmt.Errorf("config: %s of memory detected, %d GB required", humanize.Bytes(TotalMem), MinMem/Gigabyte)
+	if TotalMem < 128*MegaByte {
+		return fmt.Errorf("config: %s of memory detected, %d GB required", humanize.Bytes(TotalMem), MinMem/GigaByte)
 	}
 
 	// Show warning if less than 1 GB RAM was detected.
 	if LowMem {
-		log.Warnf(`config: less than %d GB of memory detected, please upgrade if server becomes unstable or unresponsive`, MinMem/Gigabyte)
+		log.Warnf(`config: less than %d GB of memory detected, please upgrade if server becomes unstable or unresponsive`, MinMem/GigaByte)
 		log.Warnf("config: tensorflow as well as indexing and conversion of RAW images have been disabled automatically")
 	}
 
@@ -290,6 +290,9 @@ func (c *Config) Init() error {
 		log.Infof("config: make sure your server has enough swap configured to prevent restarts when there are memory usage spikes")
 	}
 
+	// Initialize thumb package based on available memory and allowed number of workers.
+	thumb.Init(memory.FreeMemory(), c.IndexWorkers())
+
 	// Show wake-up interval warning if face recognition is activated and the worker runs less than once an hour.
 	if !c.DisableFaces() && !c.Unsafe() && c.WakeupInterval() > time.Hour {
 		log.Warnf("config: the wakeup interval is %s, but must be 1h or less for face recognition to work", c.WakeupInterval().String())

internal/config/config_const.go

@@ -30,17 +30,17 @@ const MaxWakeupInterval = time.Hour * 24          // 1 Day
 const DefaultWakeupIntervalSeconds = int(15 * 60) // 15 Minutes
 const DefaultWakeupInterval = time.Second * time.Duration(DefaultWakeupIntervalSeconds)
 
-// Megabyte defines a megabyte in bytes.
-const Megabyte = 1000 * 1000 // 1,000,000 Bytes
+// MegaByte defines a megabyte in bytes.
+const MegaByte = 1000 * 1000 // 1,000,000 Bytes
 
-// Gigabyte defines gigabyte in bytes.
-const Gigabyte = Megabyte * 1000 // 1,000,000,000 Bytes
+// GigaByte defines gigabyte in bytes.
+const GigaByte = MegaByte * 1000 // 1,000,000,000 Bytes
 
 // MinMem defines the minimum amount of system memory required.
-const MinMem = Gigabyte
+const MinMem = GigaByte
 
 // RecommendedMem defines the recommended amount of system memory.
-const RecommendedMem = 3 * Gigabyte // 3,000,000,000 Bytes
+const RecommendedMem = 3 * GigaByte // 3,000,000,000 Bytes
 
 // DefaultResolutionLimit defines the default resolution limit.
 const DefaultResolutionLimit = 150 // 150 Megapixels

internal/query/file_selection.go

@@ -11,7 +11,7 @@ import (
 	"github.com/photoprism/photoprism/pkg/media"
 )
 
-const MegaByte = 1024 * 1024
+const MiB = 1024 * 1024
 
 // FileSelection represents a selection filter to include/exclude certain files.
 type FileSelection struct {
@@ -78,7 +78,7 @@ func ShareSelection(originals bool) FileSelection {
 		Hidden:    false,
 		Private:   false,
 		Archived:  false,
-		MaxSize:   1024 * MegaByte,
+		MaxSize:   1024 * MiB,
 	}
 }
 

internal/thumb/init.go

@@ -1,12 +1,52 @@
 package thumb
 
-var (
-	ConcurrencyLevel = 1
-	MaxCacheFiles    = 0
-	MaxCacheMem      = 0
-	MaxCacheSize     = 0
+import "github.com/dustin/go-humanize/english"
+
+const (
+	MiB               = 1024 * 1024
+	DefaultCacheMem   = 64 * MiB
+	DefaultCacheSize  = 100
+	DefaultCacheFiles = 0
+	DefaultWorkers    = 1
 )
 
+var (
+	MaxCacheMem   = DefaultCacheMem
+	MaxCacheSize  = DefaultCacheSize
+	MaxCacheFiles = DefaultCacheFiles
+	NumWorkers    = DefaultWorkers
+)
+
+// Init configures the thumb package based on available memory and allowed number of workers.
+func Init(availableMemory uint64, maxWorkers int) {
+	// Set the maximum amount of cached data allowed
+	// before libvips drops cached operations.
+	switch {
+	case availableMemory > 4:
+		MaxCacheMem = 512 * MiB
+	case availableMemory > 2:
+		MaxCacheMem = 256 * MiB
+	case availableMemory > 1:
+		MaxCacheMem = 128 * MiB
+	default:
+		MaxCacheMem = DefaultCacheMem
+	}
+
+	// Set the number of worker threads that libvips can use.
+	if maxWorkers > 0 {
+		// Using the specified number of workers.
+		NumWorkers = maxWorkers
+	} else if maxWorkers < 0 {
+		// Using built-in default.
+		NumWorkers = 0
+	} else {
+		// Default to one worker.
+		NumWorkers = DefaultWorkers
+	}
+
+	log.Debugf("vips: using up to %d MB of cache and %s", MaxCacheMem/MiB, english.Plural(NumWorkers, "worker", "workers"))
+}
+
 // Shutdown shuts down dependencies like libvips.
 func Shutdown() {
 	VipsShutdown()

internal/thumb/init_test.go

@@ -0,0 +1,22 @@
+package thumb
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/pbnjay/memory"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInit(t *testing.T) {
+	t.Run("Defaults", func(t *testing.T) {
+		Init(0, 0)
+		assert.Equal(t, DefaultWorkers, NumWorkers)
+		assert.Equal(t, DefaultCacheMem, MaxCacheMem)
+	})
+	t.Run("Dynamic", func(t *testing.T) {
+		Init(memory.FreeMemory(), runtime.NumCPU())
+		assert.GreaterOrEqual(t, NumWorkers, DefaultWorkers)
+		assert.GreaterOrEqual(t, MaxCacheMem, DefaultCacheMem)
+	})
+}

internal/thumb/thumb_test.go

@@ -1,21 +1,19 @@
 package thumb
 
 import (
-	"bytes"
 	"os"
 	"testing"
 
 	"github.com/sirupsen/logrus"
-
 	"github.com/stretchr/testify/assert"
-)
 
-var logBuffer bytes.Buffer
+	"github.com/photoprism/photoprism/internal/event"
+)
 
 func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
-	log.SetOutput(&logBuffer)
 	log.SetLevel(logrus.TraceLevel)
+	event.AuditLog = log
 
 	code := m.Run()
 
@@ -24,6 +22,8 @@ func TestMain(m *testing.M) {
 	_ = os.RemoveAll("testdata/cache")
 	_ = os.RemoveAll("testdata/vips")
 
+	Shutdown()
+
 	os.Exit(code)
 }
 

internal/thumb/vips_init.go

@@ -1,12 +1,11 @@
 package thumb
 
 import (
+	"strings"
 	"sync"
 
 	"github.com/davidbyttow/govips/v2/vips"
 	"github.com/sirupsen/logrus"
-
-	"github.com/photoprism/photoprism/pkg/clean"
 )
 
 var (
@@ -40,13 +39,11 @@ func vipsInit() {
 	vips.LoggingSettings(func(domain string, level vips.LogLevel, msg string) {
 		switch level {
 		case vips.LogLevelError, vips.LogLevelCritical:
-			log.Errorf("vips: %s › %s", domain, clean.Log(msg))
+			log.Errorf("%s: %s", strings.ToLower(domain), msg)
 		case vips.LogLevelWarning:
-			log.Warnf("vips: %s › %s", domain, clean.Log(msg))
-		case vips.LogLevelInfo, vips.LogLevelMessage:
-			log.Infof("vips: %s › %s", domain, clean.Log(msg))
+			log.Warnf("%s: %s", strings.ToLower(domain), msg)
 		default:
-			log.Tracef("vips: %s › %s", domain, clean.Log(msg))
+			log.Tracef("%s: %s", strings.ToLower(domain), msg)
 		}
 	}, vipsLogLevel())
 
@@ -56,16 +53,14 @@ func vipsInit() {
 
 // vipsConfig provides the config for initializing libvips.
 func vipsConfig() *vips.Config {
-	traceMode := log.GetLevel() == logrus.TraceLevel
-
 	return &vips.Config{
-		ConcurrencyLevel: ConcurrencyLevel,
-		MaxCacheFiles:    MaxCacheFiles,
 		MaxCacheMem:      MaxCacheMem,
 		MaxCacheSize:     MaxCacheSize,
-		ReportLeaks:      traceMode,
+		MaxCacheFiles:    MaxCacheFiles,
+		ConcurrencyLevel: NumWorkers,
+		ReportLeaks:      false,
 		CacheTrace:       false,
-		CollectStats:     traceMode,
+		CollectStats:     false,
 	}
 }
 
@@ -74,11 +69,9 @@ func vipsLogLevel() vips.LogLevel {
 	switch log.GetLevel() {
 	case logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel:
 		return vips.LogLevelError
-	case logrus.WarnLevel:
-		return vips.LogLevelWarning
-	case logrus.InfoLevel:
-		return vips.LogLevelMessage
-	default:
+	case logrus.TraceLevel:
 		return vips.LogLevelDebug
+	default:
+		return vips.LogLevelWarning
 	}
 }

internal/thumb/vips_init_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/davidbyttow/govips/v2/vips"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -16,14 +15,13 @@ func TestVipsInit(t *testing.T) {
 		if conf := vipsConfig(); conf == nil {
 			t.Fatal("vips config is nil")
 		} else {
-			traceMode := log.GetLevel() == logrus.TraceLevel
-			assert.Equal(t, ConcurrencyLevel, conf.ConcurrencyLevel)
 			assert.Equal(t, MaxCacheFiles, conf.MaxCacheFiles)
 			assert.Equal(t, MaxCacheMem, conf.MaxCacheMem)
 			assert.Equal(t, MaxCacheSize, conf.MaxCacheSize)
-			assert.Equal(t, traceMode, conf.ReportLeaks)
-			assert.False(t, conf.CacheTrace)
-			assert.Equal(t, traceMode, conf.CollectStats)
+			assert.Equal(t, NumWorkers, conf.ConcurrencyLevel)
+			assert.Equal(t, false, conf.ReportLeaks)
+			assert.Equal(t, false, conf.CacheTrace)
+			assert.Equal(t, false, conf.CollectStats)
 		}
 	})
 }