Backend: Add ordered list package (pkg/list/ordered) #271 #5324

Signed-off-by: Michael Mayer <michael@photoprism.app>

pkg/list/ordered/README.md

@@ -0,0 +1,165 @@
+## PhotoPrism — Ordered List Package
+
+**Last Updated:** November 17, 2025
+
+### Overview
+
+The `pkg/list/ordered` package provides an ordered associative container that
+combines O(1) lookups with predictable iteration. It underpins features such as
+batch photo edits where we need stable selections, repeatable JSON responses,
+and fast lookups by UID while keeping the insertion order defined by the UI.
+
+Use `Map` when you want deterministic ordering (for example, mirroring the
+selection order that comes from the frontend) and `SyncMap` when multiple
+goroutines need to mutate or read the same ordered state.
+
+### Basic Usage Example
+
+```go
+package main
+
+import (
+	"fmt"
+
+	ordered "github.com/photoprism/photoprism/pkg/list/ordered"
+)
+
+func main() {
+	m := ordered.NewMap[string, int]()
+	m.Set("pq1z9t3", 1)
+	m.Set("px4y2k0", 2)
+	m.ReplaceKey("px4y2k0", "px4y2k9")
+
+	if v, ok := m.Get("px4y2k9"); ok {
+		fmt.Println("latest selection index:", v)
+	}
+
+	fmt.Println("ordered iteration")
+	for el := m.Front(); el != nil; el = el.Next() {
+		fmt.Printf("%s => %d\n", el.Key, el.Value)
+	}
+}
+```
+
+### JSON Serialization Example
+
+Because JSON arrays preserve order, iterating with `Front() … Next()` (or the
+`Keys()` / `Values()` iterators) lets us produce deterministic responses for the
+frontend REST models described in `internal/photoprism/batch/README.md`.
+
+```go
+package photorest
+
+import (
+	"encoding/json"
+
+	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/pkg/list/ordered"
+)
+
+type photoDTO struct {
+	UID   string `json:"UID"`
+	Title string `json:"Title"`
+}
+
+func MarshalPhotosOrdered(m *ordered.Map[string, *entity.Photo]) ([]byte, error) {
+	payload := make([]photoDTO, 0, m.Len())
+	for el := m.Front(); el != nil; el = el.Next() {
+		payload = append(payload, photoDTO{
+			UID:   el.Key,
+			Title: el.Value.PhotoTitle,
+		})
+	}
+	return json.Marshal(payload)
+}
+```
+
+Calling `MarshalPhotosOrdered` guarantees that the frontend receives the photos
+exactly in the order users selected them, which keeps batch edit dialogs and the
+REST `/api/v1/batch/photos/edit` response in sync.
+
+### Batch Edit Integration Example
+
+The following snippet sketches how a batch edit handler can combine
+`ordered.Map`, the entity models in `internal/entity`, and the helpers from
+`internal/photoprism/batch` to preload photos, build the response payload, and
+still offer constant-time lookups by UID:
+
+```go
+package batchhandler
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/entity/query"
+	"github.com/photoprism/photoprism/internal/entity/search"
+	"github.com/photoprism/photoprism/internal/photoprism/batch"
+	ordered "github.com/photoprism/photoprism/pkg/list/ordered"
+)
+
+func BuildBatchResponse(ctx context.Context, uids []string) (*batch.PhotosResponse, error) {
+	photosByUID := ordered.NewMap[string, *entity.Photo]()
+	preloaded := make(map[string]*entity.Photo, len(uids))
+	results := make(search.PhotoResults, 0, len(uids))
+
+	for _, uid := range uids {
+		if uid == "" {
+			continue
+		}
+		photo, err := query.PhotoPreloadByUID(uid)
+		if err != nil || !photo.HasID() {
+			return nil, fmt.Errorf("load photo %s: %w", uid, err)
+		}
+
+		p := photo // capture copy because query returns a value
+		photosByUID.Set(uid, &p)
+		preloaded[uid] = &p
+
+		results = append(results, search.Photo{
+			PhotoUID:     p.PhotoUID,
+			PhotoTitle:   p.PhotoTitle,
+			PhotoCaption: p.PhotoCaption,
+			TakenAt:      p.TakenAt,
+			TimeZone:     p.TimeZone,
+		})
+	}
+
+	resp := &batch.PhotosResponse{
+		Models: results,
+		Values: batch.NewPhotosFormWithEntities(results, preloaded),
+	}
+
+	// Quick lookup later in the request lifecycle (album diffing, ACL checks, etc.).
+	if el := photosByUID.GetElement(resp.Models[0].PhotoUID); el != nil {
+		logTitle := el.Value.PhotoTitle
+		_ = logTitle // use in audit/logging
+	}
+
+	return resp, nil
+}
+```
+
+`photosByUID` keeps the submission order defined by the UI so `resp.Models`
+matches the frontend expectations, while the embedded map lets us jump to a
+specific `entity.Photo` instantly during album/label updates. Passing the
+preloaded map into `batch.NewPhotosFormWithEntities` avoids re-querying the same
+photos, which keeps `/api/v1/batch/photos/edit` fast even for large selections.
+
+### Concurrency Helpers
+
+For long-lived caches that multiple goroutines touch (for example, background
+workers adding or removing photos while HTTP handlers read the same selection),
+wrap the ordered map with `SyncMap`:
+
+```go
+cache := ordered.NewSyncMap[string, *entity.Photo]()
+cache.Set(photo.PhotoUID, photo)
+if _, ok := cache.Get(uid); ok {
+	// safe concurrent read
+}
+```
+
+`SyncMap` applies read/write locking around every operation, so callers do not
+need to sprinkle additional mutex logic around shared ordered selections.

pkg/list/ordered/list.go

@@ -0,0 +1,106 @@
+package ordered
+
+// Element represents a node in the intrusive doubly linked list that backs
+// Map. Every element knows its key and value so the map can mutate values
+// without rebuilding the list.
+type Element[K comparable, V any] struct {
+	// Next and previous pointers in the doubly-linked list of elements.
+	// To simplify the implementation, internally a list l is implemented
+	// as a ring, such that &l.root is both the next element of the last
+	// list element (l.Back()) and the previous element of the first list
+	// element (l.Front()).
+	next, prev *Element[K, V]
+
+	// The key that corresponds to this element in the ordered map.
+	Key K
+
+	// The value stored with this element.
+	Value V
+}
+
+// Next returns the next list element or nil.
+func (e *Element[K, V]) Next() *Element[K, V] {
+	return e.next
+}
+
+// Prev returns the previous list element or nil.
+func (e *Element[K, V]) Prev() *Element[K, V] {
+	return e.prev
+}
+
+// list is a minimal intrusive doubly linked list implementation used to keep
+// insertion order for Map. It is intentionally null terminated (non-circular)
+// so checks such as "el == nil" read naturally when iterating, and the zero
+// value is ready for use because the root sentinel starts empty.
+type list[K comparable, V any] struct {
+	root Element[K, V] // list head and tail
+}
+
+// IsEmpty reports whether the list currently contains no elements.
+func (l *list[K, V]) IsEmpty() bool {
+	return l.root.next == nil
+}
+
+// Front returns the first element of list l or nil if the list is empty.
+func (l *list[K, V]) Front() *Element[K, V] {
+	return l.root.next
+}
+
+// Back returns the last element of list l or nil if the list is empty.
+func (l *list[K, V]) Back() *Element[K, V] {
+	return l.root.prev
+}
+
+// Remove detaches e from the list while keeping the remaining neighbours
+// correctly linked. After removal the element's next/prev references are
+// zeroed so the node can be safely re-used or left for GC without retaining
+// other elements.
+func (l *list[K, V]) Remove(e *Element[K, V]) {
+	if e.prev == nil {
+		l.root.next = e.next
+	} else {
+		e.prev.next = e.next
+	}
+	if e.next == nil {
+		l.root.prev = e.prev
+	} else {
+		e.next.prev = e.prev
+	}
+	e.next = nil // avoid memory leaks
+	e.prev = nil // avoid memory leaks
+}
+
+// PushFront inserts a new element with the given key/value at the head of the
+// list and returns the created element. The first insertion also updates the
+// tail pointer because the list was empty before.
+func (l *list[K, V]) PushFront(key K, value V) *Element[K, V] {
+	e := &Element[K, V]{Key: key, Value: value}
+	if l.root.next == nil {
+		// It's the first element
+		l.root.next = e
+		l.root.prev = e
+		return e
+	}
+
+	e.next = l.root.next
+	l.root.next.prev = e
+	l.root.next = e
+	return e
+}
+
+// PushBack appends a new element to the tail of the list and returns it. When
+// called on an empty list the element becomes both head and tail.
+func (l *list[K, V]) PushBack(key K, value V) *Element[K, V] {
+	e := &Element[K, V]{Key: key, Value: value}
+	if l.root.prev == nil {
+		// It's the first element
+		l.root.next = e
+		l.root.prev = e
+		return e
+	}
+
+	e.prev = l.root.prev
+	l.root.prev.next = e
+	l.root.prev = e
+	return e
+}

pkg/list/ordered/map.go

@@ -0,0 +1,194 @@
+package ordered
+
+import "iter"
+
+// Map maintains key/value pairs and preserves the order in which keys were
+// inserted. Internally it keeps a regular Go map for O(1) lookups plus the
+// intrusive list defined in list.go to make iteration stable.
+type Map[K comparable, V any] struct {
+	kv map[K]*Element[K, V]
+	ll list[K, V]
+}
+
+// NewMap returns an empty ordered map ready for use. The zero value works as
+// well, but this helper is clearer at call sites.
+func NewMap[K comparable, V any]() *Map[K, V] {
+	return &Map[K, V]{
+		kv: make(map[K]*Element[K, V]),
+	}
+}
+
+// NewMapWithCapacity creates a map with enough pre-allocated space to
+// hold the specified number of elements.
+func NewMapWithCapacity[K comparable, V any](capacity int) *Map[K, V] {
+	return &Map[K, V]{
+		kv: make(map[K]*Element[K, V], capacity),
+	}
+}
+
+// NewMapWithElements returns a map pre-populated with the passed elements.
+// Elements are copied into the new map to avoid aliasing external list nodes.
+func NewMapWithElements[K comparable, V any](els ...*Element[K, V]) *Map[K, V] {
+	om := NewMapWithCapacity[K, V](len(els))
+	for _, el := range els {
+		om.Set(el.Key, el.Value)
+	}
+	return om
+}
+
+// Get returns the value for a key. If the key does not exist, the second return
+// parameter will be false and the value will be the zero value for V.
+func (m *Map[K, V]) Get(key K) (value V, ok bool) {
+	v, ok := m.kv[key]
+	if ok {
+		value = v.Value
+	}
+
+	return
+}
+
+// Set will set (or replace) a value for a key. If the key was new, then true
+// will be returned. The returned value will be false if the value was replaced
+// (even if the value was the same).
+func (m *Map[K, V]) Set(key K, value V) bool {
+	_, alreadyExist := m.kv[key]
+	if alreadyExist {
+		m.kv[key].Value = value
+		return false
+	}
+
+	element := m.ll.PushBack(key, value)
+	m.kv[key] = element
+	return true
+}
+
+// ReplaceKey replaces an existing key with a new key while preserving the
+// element's position in the iteration order. This function returns true if the
+// operation succeeds, or false if originalKey is not found OR newKey already
+// exists.
+func (m *Map[K, V]) ReplaceKey(originalKey, newKey K) bool {
+	element, originalExists := m.kv[originalKey]
+	_, newKeyExists := m.kv[newKey]
+	if originalExists && !newKeyExists {
+		delete(m.kv, originalKey)
+		m.kv[newKey] = element
+		element.Key = newKey
+		return true
+	}
+	return false
+}
+
+// GetOrDefault returns the value for a key. If the key does not exist, returns
+// the default value instead.
+func (m *Map[K, V]) GetOrDefault(key K, defaultValue V) V {
+	if value, ok := m.kv[key]; ok {
+		return value.Value
+	}
+
+	return defaultValue
+}
+
+// GetElement returns the element for a key. If the key does not exist, the
+// pointer will be nil.
+func (m *Map[K, V]) GetElement(key K) *Element[K, V] {
+	element, ok := m.kv[key]
+	if ok {
+		return element
+	}
+
+	return nil
+}
+
+// Len returns the number of elements in the map.
+func (m *Map[K, V]) Len() int {
+	return len(m.kv)
+}
+
+// AllFromFront returns a lazy iterator that yields all elements in the map
+// starting at the front (oldest Set element). It stops early if the consumer
+// returns false.
+func (m *Map[K, V]) AllFromFront() iter.Seq2[K, V] {
+	return func(yield func(key K, value V) bool) {
+		for el := m.Front(); el != nil; el = el.Next() {
+			if !yield(el.Key, el.Value) {
+				return
+			}
+		}
+	}
+}
+
+// AllFromBack returns a lazy iterator that yields all elements in the map
+// starting at the back (most recent Set element).
+func (m *Map[K, V]) AllFromBack() iter.Seq2[K, V] {
+	return func(yield func(key K, value V) bool) {
+		for el := m.Back(); el != nil; el = el.Prev() {
+			if !yield(el.Key, el.Value) {
+				return
+			}
+		}
+	}
+}
+
+// Keys returns an iterator that yields all keys in insertion order. Use
+// slices.Collect(m.Keys()) if a materialised slice is required.
+func (m *Map[K, V]) Keys() iter.Seq[K] {
+	return func(yield func(key K) bool) {
+		for el := m.Front(); el != nil; el = el.Next() {
+			if !yield(el.Key) {
+				return
+			}
+		}
+	}
+}
+
+// Values returns an iterator that yields all values in insertion order. Use
+// slices.Collect(m.Values()) when you need a slice copy.
+func (m *Map[K, V]) Values() iter.Seq[V] {
+	return func(yield func(value V) bool) {
+		for el := m.Front(); el != nil; el = el.Next() {
+			if !yield(el.Value) {
+				return
+			}
+		}
+	}
+}
+
+// Delete removes a key from the map and unlinks the corresponding element from
+// the order list. It returns true when the key existed.
+func (m *Map[K, V]) Delete(key K) (didDelete bool) {
+	element, ok := m.kv[key]
+	if ok {
+		m.ll.Remove(element)
+		delete(m.kv, key)
+	}
+
+	return ok
+}
+
+// Front will return the element that is the first (oldest Set element). If
+// there are no elements this will return nil.
+func (m *Map[K, V]) Front() *Element[K, V] {
+	return m.ll.Front()
+}
+
+// Back will return the element that is the last (most recent Set element). If
+// there are no elements this will return nil.
+func (m *Map[K, V]) Back() *Element[K, V] {
+	return m.ll.Back()
+}
+
+// Copy returns a new Map with the same elements. Callers must avoid concurrent
+// writes during the copy or the snapshot may be inconsistent.
+func (m *Map[K, V]) Copy() *Map[K, V] {
+	m2 := NewMapWithCapacity[K, V](m.Len())
+	for el := m.Front(); el != nil; el = el.Next() {
+		m2.Set(el.Key, el.Value)
+	}
+	return m2
+}
+
+// Has checks if a key exists in the map.
+func (m *Map[K, V]) Has(key K) bool {
+	_, exists := m.kv[key]
+	return exists
+}

pkg/list/ordered/ordered.go

@@ -0,0 +1,30 @@
+/*
+Package ordered implements ordered associative containers that combine constant
+time key lookups with predictable iteration by pairing a Go map with an
+intrusive doubly linked list. It ships with SyncMap for concurrent access and is
+used wherever PhotoPrism needs to preserve the exact order of user selections
+while still addressing elements directly by key (for example, batch photo
+editing and ordered REST responses).
+
+Copyright (c) 2018 - 2025 PhotoPrism UG. All rights reserved.
+
+	This program is free software: you can redistribute it and/or modify
+	it under Version 3 of the GNU Affero General Public License (the "AGPL"):
+	<https://docs.photoprism.app/license/agpl>
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	The AGPL is supplemented by our Trademark and Brand Guidelines,
+	which describe how our Brand Assets may be used:
+	<https://www.photoprism.app/trademark>
+
+Feel free to send an email to hello@photoprism.app if you have questions,
+want to support our work, or just want to say hello.
+
+Additional information can be found in our Developer Guide:
+<https://docs.photoprism.app/developer-guide/>
+*/
+package ordered

pkg/list/ordered/sync.go

@@ -0,0 +1,85 @@
+package ordered
+
+import "sync"
+
+// SyncMap wraps Map with an RWMutex so callers can share an ordered map across
+// goroutines without managing locks at every call site.
+type SyncMap[K comparable, V any] struct {
+	Map[K, V]
+	sync.RWMutex
+}
+
+// NewSyncMap returns an empty thread-safe ordered map.
+func NewSyncMap[K comparable, V any]() *SyncMap[K, V] {
+	return &SyncMap[K, V]{*NewMap[K, V](), sync.RWMutex{}}
+}
+
+// NewSyncMapWithCapacity returns a thread-safe map with space pre-allocated for
+// capacity elements.
+func NewSyncMapWithCapacity[K comparable, V any](capacity int) *SyncMap[K, V] {
+	return &SyncMap[K, V]{*NewMapWithCapacity[K, V](capacity), sync.RWMutex{}}
+}
+
+// Get returns the value for key while holding a read lock.
+func (m *SyncMap[K, V]) Get(key K) (value V, ok bool) {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.Map.Get(key)
+}
+
+// Set stores the value for key while holding an exclusive lock.
+func (m *SyncMap[K, V]) Set(key K, value V) bool {
+	m.Lock()
+	defer m.Unlock()
+
+	return m.Map.Set(key, value)
+}
+
+// ReplaceKey safely forwards to Map.ReplaceKey using a write lock.
+func (m *SyncMap[K, V]) ReplaceKey(originalKey, newKey K) bool {
+	m.Lock()
+	defer m.Unlock()
+
+	return m.Map.ReplaceKey(originalKey, newKey)
+}
+
+// GetOrDefault is the concurrent-safe variant of Map.GetOrDefault.
+func (m *SyncMap[K, V]) GetOrDefault(key K, defaultValue V) V {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.Map.GetOrDefault(key, defaultValue)
+}
+
+// Len returns the number of elements while holding a read lock.
+func (m *SyncMap[K, V]) Len() int {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.Map.Len()
+}
+
+// Delete removes a key/value pair while holding an exclusive lock.
+func (m *SyncMap[K, V]) Delete(key K) (didDelete bool) {
+	m.Lock()
+	defer m.Unlock()
+
+	return m.Map.Delete(key)
+}
+
+// Copy takes a consistent snapshot by holding a read lock during the copy.
+func (m *SyncMap[K, V]) Copy() *Map[K, V] {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.Map.Copy()
+}
+
+// Has performs a constant-time key existence check behind a read lock.
+func (m *SyncMap[K, V]) Has(key K) bool {
+	m.RLock()
+	defer m.RUnlock()
+
+	return m.Map.Has(key)
+}

pkg/list/ordered/sync_test.go

@@ -0,0 +1,101 @@
+package ordered
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"testing"
+)
+
+func TestRaceCondition(t *testing.T) {
+	m := NewSyncMap[int, int]()
+	wg := &sync.WaitGroup{}
+
+	var asyncGet = func() {
+		wg.Add(1)
+		go func() {
+			key := rand.Intn(100)
+			m.Get(key)
+			wg.Done()
+		}()
+	}
+
+	var asyncSet = func() {
+		wg.Add(1)
+		go func() {
+			key := rand.Intn(100)
+			value := rand.Intn(100)
+			m.Set(key, value)
+			wg.Done()
+		}()
+	}
+
+	var asyncDelete = func() {
+		wg.Add(1)
+		go func() {
+			key := rand.Intn(100)
+			m.Delete(key)
+			wg.Done()
+		}()
+	}
+
+	var asyncHas = func() {
+		wg.Add(1)
+		go func() {
+			key := rand.Intn(100)
+			m.Has(key)
+			wg.Done()
+		}()
+	}
+
+	var asyncReplaceKEy = func() {
+		wg.Add(1)
+		go func() {
+			key := rand.Intn(100)
+			newKey := rand.Intn(100)
+			m.ReplaceKey(key, newKey)
+			wg.Done()
+		}()
+	}
+
+	var asyncGetOrDefault = func() {
+		wg.Add(1)
+		go func() {
+			key := rand.Intn(100)
+			def := rand.Intn(100)
+			m.GetOrDefault(key, def)
+			wg.Done()
+		}()
+	}
+
+	var asyncLen = func() {
+		wg.Add(1)
+		go func() {
+			m.Len()
+			wg.Done()
+		}()
+	}
+
+	var asyncCopy = func() {
+		wg.Add(1)
+		go func() {
+			m.Copy()
+			wg.Done()
+		}()
+	}
+
+	for i := 0; i < 10000; i++ {
+		asyncSet()
+		asyncGet()
+		asyncDelete()
+		asyncHas()
+		asyncLen()
+		asyncReplaceKEy()
+		asyncGetOrDefault()
+		asyncCopy()
+	}
+
+	wg.Wait()
+	fmt.Println("TestRaceCondition completed")
+	fmt.Printf("SyncMap eventually has %v elements\n", m.Len())
+}