From 35bfe0694b6db459fb2ca872875714b97d0be10d Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Apr 2025 14:13:25 +0200 Subject: [PATCH 01/23] API: Disable gzip compression for /api/v1/dl endpoint #127 #1090 Required for compatibility with the Python HTTP client. In addition, this commit refactors function names and adds tests. Signed-off-by: Michael Mayer --- internal/ai/face/embedding.go | 2 +- internal/ai/face/embeddings.go | 2 +- internal/ai/vision/api_client.go | 2 +- internal/ai/vision/api_client_test.go | 4 +- internal/ai/vision/api_request.go | 4 +- internal/ai/vision/caption.go | 2 +- .../api/{download_file.go => download.go} | 0 ...download_file_test.go => download_test.go} | 0 internal/api/vision_face_test.go | 8 +-- internal/api/vision_labels_test.go | 6 +- internal/api/vision_nsfw_test.go | 6 +- internal/server/start.go | 1 + internal/workers/vision.go | 2 +- pkg/clean/content_type_test.go | 2 +- pkg/fs/fs.go | 17 +++++ pkg/fs/fs_test.go | 8 +++ pkg/media/base64.go | 41 ++++++++++-- pkg/media/base64_test.go | 67 ++++++++++++++++++- pkg/media/data_url.go | 4 +- pkg/media/data_url_test.go | 2 +- pkg/media/http/header/content.go | 1 + pkg/media/http/header/content_test.go | 1 + pkg/media/http/header/webhook.go | 8 +++ 23 files changed, 157 insertions(+), 33 deletions(-) rename internal/api/{download_file.go => download.go} (100%) rename internal/api/{download_file_test.go => download_test.go} (100%) create mode 100644 pkg/media/http/header/webhook.go diff --git a/internal/ai/face/embedding.go b/internal/ai/face/embedding.go index 2e7e0f417..3d6004405 100644 --- a/internal/ai/face/embedding.go +++ b/internal/ai/face/embedding.go @@ -62,7 +62,7 @@ func (m Embedding) Magnitude() float64 { return m.Dist(NullEmbedding) } -// JSON returns the face embedding as JSON bytes. +// JSON returns the face embedding as JSON-encoded bytes. func (m Embedding) JSON() []byte { var noResult = []byte("") diff --git a/internal/ai/face/embeddings.go b/internal/ai/face/embeddings.go index feaefd2fb..ce07c2ef6 100644 --- a/internal/ai/face/embeddings.go +++ b/internal/ai/face/embeddings.go @@ -109,7 +109,7 @@ func (embeddings Embeddings) Dist(other Embedding) (dist float64) { return dist } -// JSON returns the embeddings as JSON bytes. +// JSON returns the embeddings as JSON-encoded bytes. func (embeddings Embeddings) JSON() []byte { var noResult = []byte("") diff --git a/internal/ai/vision/api_client.go b/internal/ai/vision/api_client.go index c3a9abc41..1f2392cc1 100644 --- a/internal/ai/vision/api_client.go +++ b/internal/ai/vision/api_client.go @@ -59,7 +59,7 @@ func PerformApiRequest(apiRequest *ApiRequest, uri, method, key string) (apiResp return apiResponse, errors.New("api request is nil") } - data, jsonErr := apiRequest.MarshalJSON() + data, jsonErr := apiRequest.JSON() if jsonErr != nil { return apiResponse, jsonErr diff --git a/internal/ai/vision/api_client_test.go b/internal/ai/vision/api_client_test.go index 57fdbc219..6fcb59425 100644 --- a/internal/ai/vision/api_client_test.go +++ b/internal/ai/vision/api_client_test.go @@ -22,7 +22,7 @@ func TestNewApiRequest(t *testing.T) { // t.Logf("request: %#v", result) if result != nil { - json, jsonErr := result.MarshalJSON() + json, jsonErr := result.JSON() assert.NoError(t, jsonErr) assert.NotEmpty(t, json) // t.Logf("json: %s", json) @@ -36,7 +36,7 @@ func TestNewApiRequest(t *testing.T) { assert.NotNil(t, result) // t.Logf("request: %#v", result) if result != nil { - json, jsonErr := result.MarshalJSON() + json, jsonErr := result.JSON() assert.NoError(t, jsonErr) assert.NotEmpty(t, json) t.Logf("json: %s", json) diff --git a/internal/ai/vision/api_request.go b/internal/ai/vision/api_request.go index 90bc22f07..f771e5512 100644 --- a/internal/ai/vision/api_request.go +++ b/internal/ai/vision/api_request.go @@ -25,7 +25,7 @@ func (r *ApiRequest) GetId() string { return r.Id } -// MarshalJSON returns request as JSON. -func (r *ApiRequest) MarshalJSON() ([]byte, error) { +// JSON returns the request data as JSON-encoded bytes. +func (r *ApiRequest) JSON() ([]byte, error) { return json.Marshal(*r) } diff --git a/internal/ai/vision/caption.go b/internal/ai/vision/caption.go index edc671874..7790ba2f3 100644 --- a/internal/ai/vision/caption.go +++ b/internal/ai/vision/caption.go @@ -65,7 +65,7 @@ func Caption(imgName string, src media.Src) (result CaptionResult, err error) { Url: imgUrl, } - /* if json, _ := apiRequest.MarshalJSON(); len(json) > 0 { + /* if json, _ := apiRequest.JSON(); len(json) > 0 { log.Debugf("request: %s", json) } */ diff --git a/internal/api/download_file.go b/internal/api/download.go similarity index 100% rename from internal/api/download_file.go rename to internal/api/download.go diff --git a/internal/api/download_file_test.go b/internal/api/download_test.go similarity index 100% rename from internal/api/download_file_test.go rename to internal/api/download_test.go diff --git a/internal/api/vision_face_test.go b/internal/api/vision_face_test.go index aeed919a3..21600d425 100644 --- a/internal/api/vision_face_test.go +++ b/internal/api/vision_face_test.go @@ -28,7 +28,7 @@ func TestPostVisionFace(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -71,7 +71,7 @@ func TestPostVisionFace(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -107,7 +107,7 @@ func TestPostVisionFace(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -148,7 +148,7 @@ func TestPostVisionFace(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) diff --git a/internal/api/vision_labels_test.go b/internal/api/vision_labels_test.go index 77cfffd07..eb72fba5f 100644 --- a/internal/api/vision_labels_test.go +++ b/internal/api/vision_labels_test.go @@ -28,7 +28,7 @@ func TestPostVisionLabels(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -65,7 +65,7 @@ func TestPostVisionLabels(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -99,7 +99,7 @@ func TestPostVisionLabels(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) diff --git a/internal/api/vision_nsfw_test.go b/internal/api/vision_nsfw_test.go index 20672e12c..07d5b8d17 100644 --- a/internal/api/vision_nsfw_test.go +++ b/internal/api/vision_nsfw_test.go @@ -28,7 +28,7 @@ func TestPostVisionNsfw(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -80,7 +80,7 @@ func TestPostVisionNsfw(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) @@ -114,7 +114,7 @@ func TestPostVisionNsfw(t *testing.T) { t.Fatal(err) } - jsonReq, jsonErr := req.MarshalJSON() + jsonReq, jsonErr := req.JSON() if jsonErr != nil { t.Fatal(err) diff --git a/internal/server/start.go b/internal/server/start.go index 89431456d..45ddb9772 100644 --- a/internal/server/start.go +++ b/internal/server/start.go @@ -74,6 +74,7 @@ func Start(ctx context.Context, conf *config.Config) { conf.BaseUri("/health"), conf.BaseUri(config.ApiUri + "/t"), conf.BaseUri(config.ApiUri + "/folders/t"), + conf.BaseUri(config.ApiUri + "/dl"), conf.BaseUri(config.ApiUri + "/zip"), conf.BaseUri(config.ApiUri + "/albums"), conf.BaseUri(config.ApiUri + "/labels"), diff --git a/internal/workers/vision.go b/internal/workers/vision.go index 272ada2d9..faec978b3 100644 --- a/internal/workers/vision.go +++ b/internal/workers/vision.go @@ -154,7 +154,7 @@ func (w *Vision) Start(q string, models []string, force bool) (err error) { if (entity.SrcPriority[caption.Source] > entity.SrcPriority[m.CaptionSrc]) || !m.HasCaption() { m.SetCaption(caption.Text, caption.Source) changed = true - log.Infof("vision: changed caption of %s to %t", photoName, clean.Log(m.PhotoCaption)) + log.Infof("vision: changed caption of %s to %s", photoName, clean.Log(m.PhotoCaption)) } } } diff --git a/pkg/clean/content_type_test.go b/pkg/clean/content_type_test.go index c0d3d2451..3caff9a82 100644 --- a/pkg/clean/content_type_test.go +++ b/pkg/clean/content_type_test.go @@ -23,7 +23,7 @@ func TestContentType(t *testing.T) { result := ContentType("invalid") assert.Equal(t, "invalid", result) }) - t.Run("Json", func(t *testing.T) { + t.Run("JSON", func(t *testing.T) { result := ContentType("text/json") assert.Equal(t, "application/json; charset=utf-8", result) }) diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 1837bfc9a..476fb53a9 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -89,6 +89,23 @@ func FileExistsNotEmpty(fileName string) bool { return err == nil && !info.IsDir() && info.Size() > 0 } +// FileSize returns the size of a file in bytes or -1 in case of an error. +func FileSize(fileName string) int64 { + if fileName == "" { + return -1 + } + + info, err := os.Stat(fileName) + + if err != nil || info == nil { + return -1 + } else if info.IsDir() { + return -1 + } + + return info.Size() +} + // PathExists tests if a path exists, and is a directory or symlink. func PathExists(path string) bool { if path == "" { diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go index 3b595a65b..16b932328 100644 --- a/pkg/fs/fs_test.go +++ b/pkg/fs/fs_test.go @@ -37,6 +37,14 @@ func TestFileExistsNotEmpty(t *testing.T) { assert.False(t, FileExistsNotEmpty("")) } +func TestFileSize(t *testing.T) { + assert.Equal(t, 10990, int(FileSize("./testdata/test.jpg"))) + assert.Equal(t, 10990, int(FileSize("./testdata/test.jpg"))) + assert.Equal(t, 0, int(FileSize("./testdata/empty.jpg"))) + assert.Equal(t, -1, int(FileSize("./foo.jpg"))) + assert.Equal(t, -1, int(FileSize(""))) +} + func TestPathExists(t *testing.T) { assert.True(t, PathExists("./testdata")) assert.False(t, PathExists("./testdata/test.jpg")) diff --git a/pkg/media/base64.go b/pkg/media/base64.go index 19abf11b2..64fb6bb89 100644 --- a/pkg/media/base64.go +++ b/pkg/media/base64.go @@ -5,19 +5,46 @@ import ( "io" ) -// EncodeBase64 returns the base64 encoding of bin. -func EncodeBase64(bin []byte) string { +// EncodeBase64String returns the base64 encoding of bin. +func EncodeBase64String(bin []byte) string { return base64.StdEncoding.EncodeToString(bin) } +// EncodedLenBase64 returns the length in bytes of the base64 encoding of an input buffer of length n. +func EncodedLenBase64(decodedBytes int) int { + return base64.StdEncoding.EncodedLen(decodedBytes) +} + +// DecodeBase64String returns the bytes represented by the base64 string s. +// If the input is malformed, it returns the partially decoded data and +// [CorruptInputError]. Newline characters (\r and \n) are ignored. +func DecodeBase64String(s string) ([]byte, error) { + return base64.StdEncoding.DecodeString(s) +} + // ReadBase64 returns a new reader that decodes base64 and returns binary data. func ReadBase64(stream io.Reader) io.Reader { return base64.NewDecoder(base64.StdEncoding, stream) } -// DecodeBase64 returns the bytes represented by the base64 string s. -// If the input is malformed, it returns the partially decoded data and -// [CorruptInputError]. Newline characters (\r and \n) are ignored. -func DecodeBase64(s string) ([]byte, error) { - return base64.StdEncoding.DecodeString(s) +// EncodeBase64Bytes encodes src, writing EncodedLenBase64 bytes to dst. +// +// The encoding pads the output to a multiple of 4 bytes, +// so Encode is not appropriate for use on individual blocks +// of a large data stream. +func EncodeBase64Bytes(dst, src []byte) { + base64.StdEncoding.Encode(dst, src) +} + +// DecodedLenBase64 returns the maximum length in bytes of the decoded data +// corresponding to n bytes of base64-encoded data. +func DecodedLenBase64(encodedBytes int) int { + return base64.StdEncoding.DecodedLen(encodedBytes) +} + +// DecodeBase64Bytes decodes src, writing at most DecodedLenBase64 bytes to dst. +// If src contains invalid base64 data, it returns the number of bytes successfully +// written. New line characters (\r and \n) are ignored. +func DecodeBase64Bytes(dst, src []byte) (n int, err error) { + return base64.StdEncoding.Decode(dst, src) } diff --git a/pkg/media/base64_test.go b/pkg/media/base64_test.go index 592a27cdb..de258a60d 100644 --- a/pkg/media/base64_test.go +++ b/pkg/media/base64_test.go @@ -1,15 +1,76 @@ package media import ( + "io" + "strings" "testing" + "github.com/gabriel-vasile/mimetype" "github.com/stretchr/testify/assert" ) func TestBase64(t *testing.T) { - t.Run("Gopher", func(t *testing.T) { - data, err := DecodeBase64(gopher) + t.Run("DecodeString", func(t *testing.T) { + data, err := DecodeBase64String(gopher) assert.NoError(t, err) - assert.Equal(t, gopher, EncodeBase64(data)) + + if mime := mimetype.Detect(data); mime == nil { + t.Fatal("mimetype image/png expected") + } else { + assert.Equal(t, "image/png", mime.String()) + } + }) + t.Run("DecodeString", func(t *testing.T) { + data, err := DecodeBase64String(gopher) + assert.NoError(t, err) + assert.Equal(t, gopher, EncodeBase64String(data)) + }) + t.Run("Read", func(t *testing.T) { + reader := ReadBase64(strings.NewReader(gopher)) + + if data, err := io.ReadAll(reader); err != nil { + t.Fatal(err) + } else if decodeData, decodeErr := DecodeBase64String(gopher); decodeErr != nil { + t.Fatal(decodeErr) + } else { + assert.Equal(t, data, decodeData) + assert.Equal(t, EncodeBase64String(data), gopher) + } + }) + t.Run("DecodeBytes", func(t *testing.T) { + encoded := []byte(gopher) + encodedLen := len(encoded) + decodedLen := DecodedLenBase64(encodedLen) + binary := make([]byte, decodedLen) + + if n, err := DecodeBase64Bytes(binary, encoded); err != nil { + t.Fatal(err) + } else { + assert.GreaterOrEqual(t, decodedLen, n) + } + }) + t.Run("EncodeBytes", func(t *testing.T) { + encoded := []byte(gopher) + encodedLen := len(encoded) + decodedLen := DecodedLenBase64(encodedLen) + binary := make([]byte, decodedLen) + + if n, err := DecodeBase64Bytes(binary, encoded); err != nil { + t.Fatal(err) + } else { + binary = binary[:n] + assert.GreaterOrEqual(t, decodedLen, n) + } + + binaryEncodedLen := EncodedLenBase64(len(binary)) + binaryEncoded := make([]byte, binaryEncodedLen) + + EncodeBase64Bytes(binaryEncoded, binary) + assert.Equal(t, encoded, binaryEncoded) + assert.Equal(t, gopher, string(binaryEncoded)) + + data, err := DecodeBase64String(string(binaryEncoded)) + assert.NoError(t, err) + assert.Equal(t, gopher, EncodeBase64String(data)) }) } diff --git a/pkg/media/data_url.go b/pkg/media/data_url.go index a10c2aeee..f9b807d13 100644 --- a/pkg/media/data_url.go +++ b/pkg/media/data_url.go @@ -42,7 +42,7 @@ func DataUrl(r io.Reader) string { } // Generate data URL. - return fmt.Sprintf("data:%s;base64,%s", mimeType, EncodeBase64(data)) + return fmt.Sprintf("data:%s;base64,%s", mimeType, EncodeBase64String(data)) } // ReadUrl reads binary data from a regular file path, @@ -85,7 +85,7 @@ func ReadUrl(fileUrl string, schemes []string) (data []byte, err error) { if _, binaryData, found := strings.Cut(u.Opaque, ";base64,"); !found || len(binaryData) == 0 { return data, fmt.Errorf("invalid %s url", u.Scheme) } else { - return DecodeBase64(binaryData) + return DecodeBase64String(binaryData) } case scheme.File: if data, err = os.ReadFile(fileUrl); err != nil { diff --git a/pkg/media/data_url_test.go b/pkg/media/data_url_test.go index d7a71aa85..aa094c798 100644 --- a/pkg/media/data_url_test.go +++ b/pkg/media/data_url_test.go @@ -25,7 +25,7 @@ func TestReadUrl(t *testing.T) { if data, err := ReadUrl(dataUrl, []string{"https", "data"}); err != nil { t.Fatal(err) } else { - expected, _ := DecodeBase64(gopher) + expected, _ := DecodeBase64String(gopher) assert.Equal(t, expected, data) } }) diff --git a/pkg/media/http/header/content.go b/pkg/media/http/header/content.go index 03bbebe19..bcc35070f 100644 --- a/pkg/media/http/header/content.go +++ b/pkg/media/http/header/content.go @@ -9,6 +9,7 @@ const ( ContentType = "Content-Type" ContentDisposition = "Content-Disposition" ContentEncoding = "Content-Encoding" + ContentLength = "Content-Length" ContentRange = "Content-Range" Location = "Location" Origin = "Origin" diff --git a/pkg/media/http/header/content_test.go b/pkg/media/http/header/content_test.go index 3375c80f2..dd88cc3a1 100644 --- a/pkg/media/http/header/content_test.go +++ b/pkg/media/http/header/content_test.go @@ -16,6 +16,7 @@ func TestContent(t *testing.T) { assert.Equal(t, "Content-Type", ContentType) assert.Equal(t, "Content-Disposition", ContentDisposition) assert.Equal(t, "Content-Encoding", ContentEncoding) + assert.Equal(t, "Content-Length", ContentLength) assert.Equal(t, "Content-Range", ContentRange) assert.Equal(t, "Location", Location) assert.Equal(t, "Origin", Origin) diff --git a/pkg/media/http/header/webhook.go b/pkg/media/http/header/webhook.go new file mode 100644 index 000000000..72750fd67 --- /dev/null +++ b/pkg/media/http/header/webhook.go @@ -0,0 +1,8 @@ +package header + +const ( + WebhookID string = "webhook-id" + WebhookSignature string = "webhook-signature" + WebhookTimestamp string = "webhook-timestamp" + WebhookSecretPrefix string = "whsec_" +) From f2ffb0fdce3dbe9cdcac0a98731eafb57f00b239 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Apr 2025 18:41:54 +0200 Subject: [PATCH 02/23] AI: Add Webhook endpoint and refactor ACL for Vision API #127 #1090 Signed-off-by: Michael Mayer --- internal/ai/vision/api_client.go | 5 +- internal/ai/vision/caption.go | 6 +- internal/ai/vision/vision_test.go | 2 + internal/api/download/cache.go | 3 +- internal/api/download/download_test.go | 24 +++ internal/api/download/find.go | 1 - internal/api/download/paths.go | 28 +++ internal/api/download/register.go | 31 ++- internal/api/download/register_test.go | 19 +- internal/api/hooks/hooks.go | 31 +++ internal/api/hooks/payload_test.go | 37 ++++ internal/api/hooks/payloard.go | 27 +++ internal/api/hooks/secret.go | 152 +++++++++++++ internal/api/hooks/secret_test.go | 260 +++++++++++++++++++++++ internal/api/swagger.json | 196 +++++++++++++++-- internal/api/vision_caption.go | 2 +- internal/api/vision_face.go | 2 +- internal/api/vision_labels.go | 2 +- internal/api/vision_nsfw.go | 2 +- internal/api/webhook.go | 120 +++++++++++ internal/api/webhook_test.go | 64 ++++++ internal/auth/acl/const.go | 6 +- internal/auth/acl/grant.go | 14 +- internal/auth/acl/resource_names.go | 2 + internal/auth/acl/rules.go | 10 +- internal/config/config.go | 8 + internal/entity/auth_session_fixtures.go | 4 +- internal/server/routes.go | 1 + 28 files changed, 1010 insertions(+), 49 deletions(-) create mode 100644 internal/api/download/download_test.go create mode 100644 internal/api/download/paths.go create mode 100644 internal/api/hooks/hooks.go create mode 100644 internal/api/hooks/payload_test.go create mode 100644 internal/api/hooks/payloard.go create mode 100644 internal/api/hooks/secret.go create mode 100644 internal/api/hooks/secret_test.go create mode 100644 internal/api/webhook.go create mode 100644 internal/api/webhook_test.go diff --git a/internal/ai/vision/api_client.go b/internal/ai/vision/api_client.go index 1f2392cc1..8aafc2254 100644 --- a/internal/ai/vision/api_client.go +++ b/internal/ai/vision/api_client.go @@ -30,10 +30,11 @@ func NewApiRequest(images Files, fileScheme string) (*ApiRequest, error) { for i := range images { switch fileScheme { case scheme.Https: - if id, err := download.Register(images[i]); err != nil { + fileUuid := rnd.UUID() + if err := download.Register(fileUuid, images[i]); err != nil { return nil, fmt.Errorf("%s (create download url)", err) } else { - imageUrls[i] = fmt.Sprintf("%s/%s", DownloadUrl, id) + imageUrls[i] = fmt.Sprintf("%s/%s", DownloadUrl, fileUuid) } case scheme.Data: if file, err := os.Open(images[i]); err != nil { diff --git a/internal/ai/vision/caption.go b/internal/ai/vision/caption.go index 7790ba2f3..068427153 100644 --- a/internal/ai/vision/caption.go +++ b/internal/ai/vision/caption.go @@ -39,13 +39,13 @@ func Caption(imgName string, src media.Src) (result CaptionResult, err error) { imgUrl = media.DataUrl(file) } */ - dlId, dlErr := download.Register(imgName) + fileUuid := rnd.UUID() - if dlErr != nil { + if dlErr := download.Register(imgName, fileUuid); dlErr != nil { return result, fmt.Errorf("%s (create download url)", err) } - imgUrl = fmt.Sprintf("%s/%s", DownloadUrl, dlId) + imgUrl = fmt.Sprintf("%s/%s", DownloadUrl, fileUuid) case media.SrcRemote: var u *url.URL if u, err = url.Parse(imgName); err != nil { diff --git a/internal/ai/vision/vision_test.go b/internal/ai/vision/vision_test.go index 428e8ddd0..717e98e6e 100644 --- a/internal/ai/vision/vision_test.go +++ b/internal/ai/vision/vision_test.go @@ -6,6 +6,7 @@ import ( "github.com/sirupsen/logrus" + "github.com/photoprism/photoprism/internal/api/download" "github.com/photoprism/photoprism/internal/event" ) @@ -14,6 +15,7 @@ func TestMain(m *testing.M) { log = logrus.StandardLogger() log.SetLevel(logrus.TraceLevel) event.AuditLog = log + download.AllowedPaths = append(download.AllowedPaths, AssetsPath) // Set test config values. DownloadUrl = "https://app.localssl.dev/api/v1/dl" diff --git a/internal/api/download/cache.go b/internal/api/download/cache.go index 90df13113..f72f04128 100644 --- a/internal/api/download/cache.go +++ b/internal/api/download/cache.go @@ -6,7 +6,8 @@ import ( gc "github.com/patrickmn/go-cache" ) -var cache = gc.New(time.Minute*15, 5*time.Minute) +var expires = time.Minute * 15 +var cache = gc.New(expires, 5*time.Minute) // Flush resets the download cache. func Flush() { diff --git a/internal/api/download/download_test.go b/internal/api/download/download_test.go new file mode 100644 index 000000000..dd273bba7 --- /dev/null +++ b/internal/api/download/download_test.go @@ -0,0 +1,24 @@ +package download + +import ( + "os" + "testing" + + "github.com/sirupsen/logrus" + + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/pkg/fs" +) + +func TestMain(m *testing.M) { + // Init test logger. + log = logrus.StandardLogger() + log.SetLevel(logrus.TraceLevel) + event.AuditLog = log + AllowedPaths = append(AllowedPaths, fs.Abs("./testdata")) + + // Run unit tests. + code := m.Run() + + os.Exit(code) +} diff --git a/internal/api/download/find.go b/internal/api/download/find.go index d4f0a119c..d878f6303 100644 --- a/internal/api/download/find.go +++ b/internal/api/download/find.go @@ -9,7 +9,6 @@ import ( // Find returns the fileName for the given download id or an error if the id is invalid. func Find(uniqueId string) (fileName string, err error) { - if uniqueId == "" || !rnd.IsUUID(uniqueId) { return fileName, fmt.Errorf("id has an invalid format") } diff --git a/internal/api/download/paths.go b/internal/api/download/paths.go new file mode 100644 index 000000000..d874a25da --- /dev/null +++ b/internal/api/download/paths.go @@ -0,0 +1,28 @@ +package download + +import ( + "path/filepath" + "strings" + + "github.com/photoprism/photoprism/pkg/fs" +) + +var AllowedPaths []string + +// Deny checks if the filename may not be registered for download. +func Deny(fileName string) bool { + if len(AllowedPaths) == 0 || fileName == "" { + return true + } else if fileName = fs.Abs(fileName); strings.HasPrefix(fileName, "/etc") || + strings.HasPrefix(filepath.Base(fileName), ".") { + return true + } + + for _, dir := range AllowedPaths { + if dir != "" && strings.HasPrefix(fileName, dir+"/") { + return false + } + } + + return true +} diff --git a/internal/api/download/register.go b/internal/api/download/register.go index 16c2cbbf1..2002ef35f 100644 --- a/internal/api/download/register.go +++ b/internal/api/download/register.go @@ -1,22 +1,33 @@ package download import ( - "fmt" + "errors" - "github.com/photoprism/photoprism/pkg/clean" + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/pkg/authn" "github.com/photoprism/photoprism/pkg/fs" "github.com/photoprism/photoprism/pkg/rnd" ) -// Register makes the specified file available for download with the -// returned id until the cache expires, or the server is restarted. -func Register(fileName string) (string, error) { - if !fs.FileExists(fileName) { - return "", fmt.Errorf("%s does not exists", clean.Log(fileName)) +// Register generated an event to make the specified file available +// for download until the cache expires, or the server is restarted. +func Register(fileUuid, fileName string) error { + if !rnd.IsUUID(fileUuid) { + event.AuditWarn([]string{"api", "create download token", "%s", authn.Failed}, fileName) + return errors.New("invalid file uuid") } - uniqueId := rnd.UUID() - cache.SetDefault(uniqueId, fileName) + if fileName = fs.Abs(fileName); !fs.FileExists(fileName) { + event.AuditWarn([]string{"api", "create download token", "%s", authn.Failed}, fileName) + return errors.New("file not found") + } else if Deny(fileName) { + event.AuditErr([]string{"api", "create download token", "%s", authn.Denied}, fileName) + return errors.New("forbidden file path") + } - return uniqueId, nil + event.AuditInfo([]string{"api", "create download token", "%s", authn.Succeeded}, fileName, expires.String()) + + cache.SetDefault(fileUuid, fileName) + + return nil } diff --git a/internal/api/download/register_test.go b/internal/api/download/register_test.go index 4beb18c06..b6ff7c8ec 100644 --- a/internal/api/download/register_test.go +++ b/internal/api/download/register_test.go @@ -11,27 +11,34 @@ import ( func TestRegister(t *testing.T) { t.Run("Success", func(t *testing.T) { + fileUuid := rnd.UUID() fileName := fs.Abs("./testdata/image.jpg") - uniqueId, err := Register(fileName) + err := Register(fileUuid, fileName) assert.NoError(t, err) - assert.True(t, rnd.IsUUID(uniqueId)) + assert.True(t, rnd.IsUUID(fileUuid)) - findName, findErr := Find(uniqueId) + findName, findErr := Find(fileUuid) assert.NoError(t, findErr) assert.Equal(t, fileName, findName) Flush() - findName, findErr = Find(uniqueId) + findName, findErr = Find(fileUuid) assert.Error(t, findErr) assert.Equal(t, "", findName) }) t.Run("NotFound", func(t *testing.T) { + fileUuid := rnd.UUID() fileName := fs.Abs("./testdata/invalid.jpg") - uniqueId, err := Register(fileName) + err := Register(fileUuid, fileName) assert.Error(t, err) - assert.Equal(t, "", uniqueId) + assert.True(t, rnd.IsUUID(fileUuid)) + + findName, findErr := Find(fileUuid) + + assert.Error(t, findErr) + assert.Equal(t, "", findName) }) } diff --git a/internal/api/hooks/hooks.go b/internal/api/hooks/hooks.go new file mode 100644 index 000000000..a925e1664 --- /dev/null +++ b/internal/api/hooks/hooks.go @@ -0,0 +1,31 @@ +/* +Package hooks provides webhook authentication and payload handlers. + +Copyright (c) 2018 - 2025 PhotoPrism UG. All rights reserved. + + This program is free software: you can redistribute it and/or modify + it under Version 3 of the GNU Affero General Public License (the "AGPL"): + + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + The AGPL is supplemented by our Trademark and Brand Guidelines, + which describe how our Brand Assets may be used: + + +Feel free to send an email to hello@photoprism.app if you have questions, +want to support our work, or just want to say hello. + +Additional information can be found in our Developer Guide: + +*/ +package hooks + +import ( + "github.com/photoprism/photoprism/internal/event" +) + +var log = event.Log diff --git a/internal/api/hooks/payload_test.go b/internal/api/hooks/payload_test.go new file mode 100644 index 000000000..ecef60ed1 --- /dev/null +++ b/internal/api/hooks/payload_test.go @@ -0,0 +1,37 @@ +package hooks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/photoprism/photoprism/internal/event" +) + +func TestPayload_Json(t *testing.T) { + t.Run("Success", func(t *testing.T) { + timeStamp, timeErr := time.Parse(time.RFC3339Nano, "2025-04-11T11:48:58.540199797Z") + + if timeErr != nil { + t.Fatal(timeErr) + } + + id := "49b8a329-5aa6-4b76-ba62-bb3adb001817" + + payload := &Payload{ + Type: "foo.bar", + Timestamp: timeStamp.UTC(), + Data: event.Data{ + "id": id, + "hello": "World!", + "number": 42, + }, + } + + result := payload.JSON() + expected := `{"type":"foo.bar","timestamp":"2025-04-11T11:48:58.540199797Z","data":{"hello":"World!","id":"49b8a329-5aa6-4b76-ba62-bb3adb001817","number":42}}` + + assert.Equal(t, expected, string(result)) + }) +} diff --git a/internal/api/hooks/payloard.go b/internal/api/hooks/payloard.go new file mode 100644 index 000000000..d9ce363e5 --- /dev/null +++ b/internal/api/hooks/payloard.go @@ -0,0 +1,27 @@ +package hooks + +import ( + "encoding/json" + "time" + + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/pkg/clean" +) + +// Payload represents a webhook payload. +type Payload struct { + Type string `form:"type" json:"type"` + Timestamp time.Time `form:"timestamp" json:"timestamp,omitempty"` + Data event.Data `form:"data" json:"data"` +} + +// JSON returns the payload data as JSON-encoded bytes. +func (p *Payload) JSON() (b []byte) { + b, jsonErr := json.Marshal(p) + + if jsonErr != nil { + log.Warningf("hook: %s (json encode)", clean.Error(jsonErr)) + } + + return b +} diff --git a/internal/api/hooks/secret.go b/internal/api/hooks/secret.go new file mode 100644 index 000000000..5d1f821be --- /dev/null +++ b/internal/api/hooks/secret.go @@ -0,0 +1,152 @@ +package hooks + +import ( + "crypto/hmac" + "crypto/sha256" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/photoprism/photoprism/pkg/media" + "github.com/photoprism/photoprism/pkg/media/http/header" +) + +var timeTolerance = 5 * time.Minute + +var ( + ErrRequiredHeaders = errors.New("missing required headers") + ErrInvalidHeaders = errors.New("invalid signature headers") + ErrNoMatchingSignature = errors.New("no matching signature found") + ErrMessageTooOld = errors.New("message timestamp too old") + ErrMessageTooNew = errors.New("message timestamp too new") +) + +type Secret struct { + key []byte +} + +func NewSecret(secret string) (*Secret, error) { + key, err := media.DecodeBase64String(strings.TrimPrefix(secret, header.WebhookSecretPrefix)) + if err != nil { + return nil, fmt.Errorf("unable to create webhook, err: %w", err) + } + return &Secret{ + key: key, + }, nil +} + +func NewWebhookRaw(secret []byte) (*Secret, error) { + return &Secret{ + key: secret, + }, nil +} + +// Verify validates the payload against the webhook signature headers +// using the webhooks signing secret. +// +// Returns an error if the body or headers are missing/unreadable +// or if the signature doesn't match. +func (wh *Secret) Verify(payload []byte, headers http.Header) error { + return wh.verify(payload, headers, true) +} + +// VerifyIgnoringTimestamp validates the payload against the webhook signature headers +// using the webhooks signing secret. +// +// Returns an error if the body or headers are missing/unreadable +// or if the signature doesn't match. +// +// WARNING: This function does not check the signature's timestamp. +// We recommend using the `Verify` function instead. +func (wh *Secret) VerifyIgnoringTimestamp(payload []byte, headers http.Header) error { + return wh.verify(payload, headers, false) +} + +func (wh *Secret) verify(payload []byte, headers http.Header, enforceTolerance bool) error { + msgId := headers.Get(header.WebhookID) + msgSignature := headers.Get(header.WebhookSignature) + msgTimestamp := headers.Get(header.WebhookTimestamp) + if msgId == "" || msgSignature == "" || msgTimestamp == "" { + return fmt.Errorf("unable to verify payload, err: %w", ErrRequiredHeaders) + } + + timestamp, err := parseTimestampHeader(msgTimestamp) + if err != nil { + return fmt.Errorf("unable to verify payload, err: %w", err) + } + + if enforceTolerance { + if err := verifyTimestamp(timestamp); err != nil { + return fmt.Errorf("unable to verify payload, err: %w", err) + } + } + + _, expectedSignature, err := wh.sign(msgId, timestamp, payload) + if err != nil { + return fmt.Errorf("unable to verify payload, err: %w", err) + } + + passedSignatures := strings.Split(msgSignature, " ") + for _, versionedSignature := range passedSignatures { + sigParts := strings.Split(versionedSignature, ",") + if len(sigParts) < 2 { + continue + } + + version := sigParts[0] + + if version != "v1" { + continue + } + + signature := []byte(sigParts[1]) + + if hmac.Equal(signature, expectedSignature) { + return nil + } + } + + return fmt.Errorf("unable to verify payload, err: %w", ErrNoMatchingSignature) +} + +func (wh *Secret) Sign(msgId string, timestamp time.Time, payload []byte) (string, error) { + version, signature, err := wh.sign(msgId, timestamp, payload) + return fmt.Sprintf("%s,%s", version, signature), err +} + +func (wh *Secret) sign(msgId string, timestamp time.Time, payload []byte) (version string, signature []byte, err error) { + toSign := fmt.Sprintf("%s.%d.%s", msgId, timestamp.Unix(), payload) + + h := hmac.New(sha256.New, wh.key) + h.Write([]byte(toSign)) + sig := make([]byte, media.EncodedLenBase64(h.Size())) + media.EncodeBase64Bytes(sig, h.Sum(nil)) + + return "v1", sig, nil +} + +func parseTimestampHeader(timestampHeader string) (time.Time, error) { + timeInt, err := strconv.ParseInt(timestampHeader, 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("unable to parse timestamp header, err: %w", errors.Join(err, ErrInvalidHeaders)) + } + timestamp := time.Unix(timeInt, 0) + return timestamp, nil +} + +func verifyTimestamp(timestamp time.Time) error { + now := time.Now() + + if now.Sub(timestamp) > timeTolerance { + return ErrMessageTooOld + } + + if timestamp.After(now.Add(timeTolerance)) { + return ErrMessageTooNew + } + + return nil +} diff --git a/internal/api/hooks/secret_test.go b/internal/api/hooks/secret_test.go new file mode 100644 index 000000000..f86d96c5f --- /dev/null +++ b/internal/api/hooks/secret_test.go @@ -0,0 +1,260 @@ +package hooks + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/photoprism/photoprism/pkg/media/http/header" +) + +var defaultMsgID = "msg_p5jXN8AQM9LWM0D4loKWxJek" +var defaultPayload = []byte(`{"test": 2432232314}`) +var defaultSecret = "MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" + +type testPayload struct { + id string + timestamp time.Time + header http.Header + secret string + payload []byte + signature string +} + +func newTestPayload(timestamp time.Time) *testPayload { + tp := &testPayload{} + tp.id = defaultMsgID + tp.timestamp = timestamp + + tp.payload = defaultPayload + tp.secret = defaultSecret + + wh, _ := NewSecret(tp.secret) + tp.signature, _ = wh.Sign(tp.id, tp.timestamp, tp.payload) + + tp.header = http.Header{} + tp.header.Set(header.WebhookID, tp.id) + tp.header.Set(header.WebhookSignature, tp.signature) + tp.header.Set(header.WebhookTimestamp, fmt.Sprint(tp.timestamp.Unix())) + + return tp +} + +func TestWebhook(t *testing.T) { + + testCases := []struct { + name string + testPayload *testPayload + modifyPayload func(*testPayload) + noEnforceTimestamp bool + expectedErr bool + }{ + { + name: "valid signature is valid", + testPayload: newTestPayload(time.Now()), + expectedErr: false, + }, + { + name: "missing id returns error", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + tp.header.Del("webhook-id") + }, + expectedErr: true, + }, + { + name: "missing timestamp returns error", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + tp.header.Del("webhook-timestamp") + }, + expectedErr: true, + }, + { + name: "missing signature returns error", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + tp.header.Del("webhook-signature") + }, + expectedErr: true, + }, + { + name: "invalid signature is invalid", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + tp.header.Set("webhook-signature", "v1,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=") + }, + expectedErr: true, + }, + { + name: "partial signature is invalid", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + tp.header.Set("webhook-signature", "v1,") + }, + expectedErr: true, + }, + { + name: "old timestamp fails", + testPayload: newTestPayload(time.Now().Add(timeTolerance * -1)), + expectedErr: true, + }, + { + name: "new timestamp fails", + testPayload: newTestPayload(time.Now().Add(timeTolerance + time.Second)), + expectedErr: true, + }, + { + name: "valid multi sig is valid", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + sigs := []string{ + "v1,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=", + "v2,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=", + tp.header.Get("webhook-signature"), // valid signature + "v1,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=", + } + tp.header.Set("webhook-signature", strings.Join(sigs, " ")) + }, + expectedErr: false, + }, + { + name: "old timestamp passes when ignoring tolerance", + testPayload: newTestPayload(time.Now().Add(timeTolerance * -1)), + noEnforceTimestamp: true, + expectedErr: false, + }, + { + name: "new timestamp passes when ignoring tolerance", + testPayload: newTestPayload(time.Now().Add(timeTolerance * 1)), + noEnforceTimestamp: true, + expectedErr: false, + }, + { + name: "valid timestamp passes when ignoring tolerance", + testPayload: newTestPayload(time.Now()), + noEnforceTimestamp: true, + expectedErr: false, + }, + { + name: "invalid timestamp fails when ignoring tolerance", + testPayload: newTestPayload(time.Now()), + modifyPayload: func(tp *testPayload) { + tp.header.Set("webhook-timestamp", fmt.Sprint(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).Unix())) + }, + noEnforceTimestamp: true, + expectedErr: true, + }, + } + + for _, tc := range testCases { + if tc.modifyPayload != nil { + tc.modifyPayload(tc.testPayload) + } + + wh, err := NewSecret(tc.testPayload.secret) + if err != nil { + t.Error(err) + continue + } + if tc.noEnforceTimestamp { + err = wh.VerifyIgnoringTimestamp(tc.testPayload.payload, tc.testPayload.header) + } else { + err = wh.Verify(tc.testPayload.payload, tc.testPayload.header) + } + if err != nil && !tc.expectedErr { + t.Errorf("%s: failed with err %s but shouldn't have", tc.name, err.Error()) + } else if err == nil && tc.expectedErr { + t.Errorf("%s: didn't error but should have", tc.name) + } + } +} + +func TestWebhookPrefix(t *testing.T) { + tp := newTestPayload(time.Now()) + + wh, err := NewSecret(tp.secret) + if err != nil { + t.Fatal(err) + } + + err = wh.Verify(tp.payload, tp.header) + if err != nil { + t.Fatal(err) + } + + wh, err = NewSecret(fmt.Sprintf("whsec_%s", tp.secret)) + if err != nil { + t.Fatal(err) + } + + err = wh.Verify(tp.payload, tp.header) + if err != nil { + t.Fatal(err) + } +} + +func TestWebhookSign(t *testing.T) { + key := "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" + msgID := "msg_p5jXN8AQM9LWM0D4loKWxJek" + timestamp := time.Unix(1614265330, 0) + payload := []byte(`{"test": 2432232314}`) + expected := "v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=" + + wh, err := NewSecret(key) + if err != nil { + t.Fatal(err) + } + + signature, err := wh.Sign(msgID, timestamp, payload) + if err != nil { + t.Fatal(err) + } + + if signature != expected { + t.Fatalf("signature %s != expected signature %s", signature, expected) + } + +} + +// A complete example flow for signing and verifying a webhook payload, +// including timestamp verification. +func TestSignatureFlow(t *testing.T) { + const secretKey = "MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" + + var ( + ts = time.Now() + id = "1234567890" + ) + + wh, err := NewSecret(secretKey) + if err != nil { + t.Fatal(err) + } + + payload := `{"type": "example.created", "timestamp":"2023-09-28T19:20:22+00:00", "data":{"str":"string","bool":true,"int":42}}` + + // signing the payload with the webhook handler + signature, err := wh.Sign(id, ts, []byte(payload)) + if err != nil { + t.Fatal(err) + } + + // generating the http header carrier + head := http.Header{} + head.Set(header.WebhookID, id) + head.Set(header.WebhookSignature, signature) + head.Set(header.WebhookTimestamp, fmt.Sprint(ts.Unix())) + + // http request is sent to consumer + + // consumer verifies the signature + err = wh.Verify([]byte(payload), head) + if err != nil { + t.Fatal(err) + } + + // Done. +} diff --git a/internal/api/swagger.json b/internal/api/swagger.json index e2e5a3132..9b5c5de10 100644 --- a/internal/api/swagger.json +++ b/internal/api/swagger.json @@ -1602,7 +1602,7 @@ } } }, - "/api/v1/dl/{hash}": { + "/api/v1/dl/{file}": { "get": { "produces": [ "application/octet-stream" @@ -1616,8 +1616,8 @@ "parameters": [ { "type": "string", - "description": "File Hash", - "name": "hash", + "description": "file hash or unique download id", + "name": "file", "in": "path", "required": true } @@ -4911,7 +4911,7 @@ } } }, - "/api/v1/vision/faces": { + "/api/v1/vision/face": { "post": { "produces": [ "application/json" @@ -4919,8 +4919,8 @@ "tags": [ "Vision" ], - "summary": "returns the positions and embeddings of detected faces", - "operationId": "PostVisionFaces", + "summary": "returns the embeddings of a face image", + "operationId": "PostVisionFace", "parameters": [ { "description": "list of image file urls", @@ -5018,6 +5018,95 @@ } } }, + "/api/v1/vision/nsfw": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Vision" + ], + "summary": "checks the specified images for inappropriate content", + "operationId": "PostVisionNsfw", + "parameters": [ + { + "description": "list of image file urls", + "name": "images", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/vision.ApiRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/vision.ApiResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/i18n.Response" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/i18n.Response" + } + }, + "429": { + "description": "Too Many Requests", + "schema": { + "$ref": "#/definitions/i18n.Response" + } + } + } + } + }, + "/api/v1/webhook/{channel}": { + "post": { + "consumes": [ + "application/json" + ], + "tags": [ + "Webhook" + ], + "summary": "listens for webhook events and checks their authorization", + "operationId": "Webhook", + "parameters": [ + { + "description": "webhook event data", + "name": "payload", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/hooks.Payload" + } + } + ], + "responses": { + "200": { + "description": "OK" + }, + "401": { + "description": "Unauthorized" + }, + "403": { + "description": "Forbidden" + }, + "429": { + "description": "Too Many Requests" + } + } + } + }, "/api/v1/zip": { "post": { "tags": [ @@ -6568,6 +6657,10 @@ } } }, + "event.Data": { + "type": "object", + "additionalProperties": true + }, "form.Album": { "type": "object", "properties": { @@ -7041,6 +7134,20 @@ "type": "object", "additionalProperties": {} }, + "hooks.Payload": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/event.Data" + }, + "timestamp": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, "i18n.Response": { "type": "object", "properties": { @@ -7058,6 +7165,26 @@ } } }, + "nsfw.Result": { + "type": "object", + "properties": { + "drawing": { + "type": "number" + }, + "hentai": { + "type": "number" + }, + "neutral": { + "type": "number" + }, + "porn": { + "type": "number" + }, + "sexy": { + "type": "number" + } + } + }, "search.Album": { "type": "object", "properties": { @@ -7652,11 +7779,8 @@ "model": { "type": "string" }, - "videos": { - "type": "array", - "items": { - "type": "string" - } + "url": { + "type": "string" } } }, @@ -7686,10 +7810,16 @@ "caption": { "$ref": "#/definitions/vision.CaptionResult" }, - "faces": { + "embeddings": { "type": "array", "items": { - "type": "string" + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } } }, "labels": { @@ -7697,6 +7827,12 @@ "items": { "$ref": "#/definitions/vision.LabelResult" } + }, + "nsfw": { + "type": "array", + "items": { + "$ref": "#/definitions/nsfw.Result" + } } } }, @@ -7706,6 +7842,9 @@ "confidence": { "type": "number" }, + "source": { + "type": "string" + }, "text": { "type": "string" } @@ -7714,8 +7853,11 @@ "vision.LabelResult": { "type": "object", "properties": { - "category": { - "type": "string" + "categories": { + "type": "array", + "items": { + "type": "string" + } }, "confidence": { "type": "number" @@ -7723,6 +7865,12 @@ "name": { "type": "string" }, + "priority": { + "type": "integer" + }, + "source": { + "type": "string" + }, "topicality": { "type": "number" } @@ -7737,10 +7885,28 @@ "resolution": { "type": "integer" }, + "type": { + "$ref": "#/definitions/vision.ModelType" + }, "version": { "type": "string" } } + }, + "vision.ModelType": { + "type": "string", + "enum": [ + "labels", + "nsfw", + "face", + "caption" + ], + "x-enum-varnames": [ + "ModelTypeLabels", + "ModelTypeNsfw", + "ModelTypeFace", + "ModelTypeCaption" + ] } }, "externalDocs": { diff --git a/internal/api/vision_caption.go b/internal/api/vision_caption.go index 6edb49aea..c683a16c0 100644 --- a/internal/api/vision_caption.go +++ b/internal/api/vision_caption.go @@ -23,7 +23,7 @@ import ( // @Router /api/v1/vision/caption [post] func PostVisionCaption(router *gin.RouterGroup) { router.POST("/vision/caption", func(c *gin.Context) { - s := Auth(c, acl.ResourceVision, acl.Use) + s := Auth(c, acl.ResourceVision, acl.ActionUse) // Abort if permission is not granted. if s.Abort(c) { diff --git a/internal/api/vision_face.go b/internal/api/vision_face.go index f316f93d2..a781dcae9 100644 --- a/internal/api/vision_face.go +++ b/internal/api/vision_face.go @@ -26,7 +26,7 @@ import ( // @Router /api/v1/vision/face [post] func PostVisionFace(router *gin.RouterGroup) { router.POST("/vision/face", func(c *gin.Context) { - s := Auth(c, acl.ResourceVision, acl.Use) + s := Auth(c, acl.ResourceVision, acl.ActionUse) // Abort if permission is not granted. if s.Abort(c) { diff --git a/internal/api/vision_labels.go b/internal/api/vision_labels.go index a9a5cd05f..7154f33d8 100644 --- a/internal/api/vision_labels.go +++ b/internal/api/vision_labels.go @@ -25,7 +25,7 @@ import ( // @Router /api/v1/vision/labels [post] func PostVisionLabels(router *gin.RouterGroup) { router.POST("/vision/labels", func(c *gin.Context) { - s := Auth(c, acl.ResourceVision, acl.Use) + s := Auth(c, acl.ResourceVision, acl.ActionUse) // Abort if permission is not granted. if s.Abort(c) { diff --git a/internal/api/vision_nsfw.go b/internal/api/vision_nsfw.go index 39105af21..3816bd8bf 100644 --- a/internal/api/vision_nsfw.go +++ b/internal/api/vision_nsfw.go @@ -25,7 +25,7 @@ import ( // @Router /api/v1/vision/nsfw [post] func PostVisionNsfw(router *gin.RouterGroup) { router.POST("/vision/nsfw", func(c *gin.Context) { - s := Auth(c, acl.ResourceVision, acl.Use) + s := Auth(c, acl.ResourceVision, acl.ActionUse) // Abort if permission is not granted. if s.Abort(c) { diff --git a/internal/api/webhook.go b/internal/api/webhook.go new file mode 100644 index 000000000..8ad575c30 --- /dev/null +++ b/internal/api/webhook.go @@ -0,0 +1,120 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/photoprism/photoprism/internal/api/download" + "github.com/photoprism/photoprism/internal/api/hooks" + "github.com/photoprism/photoprism/internal/auth/acl" + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/internal/photoprism/get" + "github.com/photoprism/photoprism/pkg/authn" + "github.com/photoprism/photoprism/pkg/clean" + "github.com/photoprism/photoprism/pkg/i18n" + "github.com/photoprism/photoprism/pkg/media/http/header" +) + +// Webhook listens for webhook events and checks their authorization. +// +// @Summary listens for webhook events and checks their authorization +// @Id Webhook +// @Tags Webhook +// @Accept json +// @Success 200 +// @Failure 401,403,429 +// @Param payload body hooks.Payload true "webhook event data" +// @Router /api/v1/webhook/{channel} [post] +func Webhook(router *gin.RouterGroup) { + requestHandler := func(c *gin.Context) { + // Prevent API response caching. + c.Header(header.CacheControl, header.CacheControlNoStore) + + // Only the instance channel is currently implemented. + if !acl.ChannelInstance.Equal(clean.Token(c.Param("channel"))) { + AbortNotImplemented(c) + return + } + + // For security reasons, this endpoint is not available in public or demo mode. + if conf := get.Config(); conf.Public() || conf.Demo() { + Abort(c, http.StatusForbidden, i18n.ErrFeatureDisabled) + return + } + + s := Auth(c, acl.ResourceWebhooks, acl.ActionPublish) + + if s.Abort(c) { + return + } + + var request hooks.Payload + + // Assign and validate request form values. + if c.Request.Method == http.MethodGet { + if err := c.BindQuery(&request); err != nil { + event.AuditErr([]string{ClientIP(c), "session %s", "webhook", "%s"}, s.RefID, err) + AbortBadRequest(c) + return + } + } else { + if err := c.BindJSON(&request); err != nil { + event.AuditErr([]string{ClientIP(c), "session %s", "webhook", "%s"}, s.RefID, err) + AbortBadRequest(c) + return + } + } + + eventType := clean.TypeLowerUnderscore(request.Type) + + if eventType == "" { + event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "missing type"}, s.RefID) + AbortBadRequest(c) + return + } + + if request.Data == nil { + event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "missing data"}, s.RefID) + AbortBadRequest(c) + return + } + + resource, resourceEv, found := strings.Cut(eventType, ".") + + if !found || resource == "" || resourceEv == "" { + event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) + AbortBadRequest(c) + return + } + + if s.IsClient() { + if acl.Rules.Deny(acl.Resource(resource), s.ClientRole(), acl.ActionPublish) { + event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) + AbortForbidden(c) + return + } + } else { + if acl.Rules.Deny(acl.Resource(resource), s.UserRole(), acl.ActionPublish) { + event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) + AbortForbidden(c) + return + } + } + + ev := "instance." + eventType + + switch ev { + case "instance.api.downloads.register": + _ = download.Register(fmt.Sprintf("%v", request.Data["uuid"]), fmt.Sprintf("%v", request.Data["filename"])) + default: + event.Publish(ev, request.Data) + } + + } + + router.GET("/webhook/:channel", requestHandler) + router.POST("/webhook/:channel", requestHandler) +} diff --git a/internal/api/webhook_test.go b/internal/api/webhook_test.go new file mode 100644 index 000000000..22c3a8011 --- /dev/null +++ b/internal/api/webhook_test.go @@ -0,0 +1,64 @@ +package api + +import ( + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/photoprism/photoprism/internal/api/hooks" + "github.com/photoprism/photoprism/internal/config" + "github.com/photoprism/photoprism/internal/event" + "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/pkg/rnd" +) + +func TestPostWebhook(t *testing.T) { + app, router, conf := NewApiTest() + conf.SetAuthMode(config.AuthModePasswd) + defer conf.SetAuthMode(config.AuthModePublic) + Webhook(router) + t.Run("Success", func(t *testing.T) { + payload := hooks.Payload{ + Type: "api.downloads.register", + Timestamp: time.Now().UTC(), + Data: event.Data{ + "uuid": rnd.UUID(), + "filename": fs.Abs("./testdata/cat_224x224.jpg"), + }, + } + + body := payload.JSON() + token := "9d8b8801ffa23eb52e08ca7766283799ddfd8dd368212123" + + t.Logf("request: %s", string(body)) + + response := AuthenticatedRequestWithBody(app, http.MethodPost, "/api/v1/webhook/instance", string(body), token) + + assert.Equal(t, http.StatusOK, response.Code) + }) + t.Run("InvalidData", func(t *testing.T) { + payload := hooks.Payload{ + Type: "api.downloads.register", + Timestamp: time.Now().UTC(), + Data: event.Data{ + "uuid": 12345, + "filename": fs.Abs("./testdata/green_224x224.jpg"), + }, + } + + body := payload.JSON() + token := "778f0f7d80579a072836c65b786145d6e0127505194cc51e" + + t.Logf("request: %s", string(body)) + + response := AuthenticatedRequestWithBody(app, http.MethodPost, "/api/v1/webhook/instance", string(body), token) + + assert.Equal(t, http.StatusOK, response.Code) + }) + t.Run("Unauthorized", func(t *testing.T) { + r := PerformRequest(app, http.MethodPost, "/api/v1/webhook/instance") + assert.Equal(t, http.StatusUnauthorized, r.Code) + }) +} diff --git a/internal/auth/acl/const.go b/internal/auth/acl/const.go index 06ee8a099..a04e6a8f3 100644 --- a/internal/auth/acl/const.go +++ b/internal/auth/acl/const.go @@ -20,7 +20,7 @@ const ( AccessPrivate Permission = "access_private" AccessOwn Permission = "access_own" AccessAll Permission = "access_all" - Use Permission = "use" + ActionUse Permission = "use" ActionSearch Permission = "search" ActionView Permission = "view" ActionUpload Permission = "upload" @@ -31,6 +31,7 @@ const ( ActionDelete Permission = "delete" ActionRate Permission = "rate" ActionReact Permission = "react" + ActionPublish Permission = "publish" ActionSubscribe Permission = "subscribe" ActionManage Permission = "manage" ActionManageOwn Permission = "manage_own" @@ -58,7 +59,9 @@ const ( ResourceUsers Resource = "users" ResourceSessions Resource = "sessions" ResourceLogs Resource = "logs" + ResourceApi Resource = "api" ResourceWebDAV Resource = "webdav" + ResourceWebhooks Resource = "webhooks" ResourceMetrics Resource = "metrics" ResourceVision Resource = "vision" ResourceFeedback Resource = "feedback" @@ -86,4 +89,5 @@ const ( ChannelSubjects Resource = "subjects" ChannelPeople Resource = "people" ChannelSync Resource = "sync" + ChannelInstance Resource = "instance" ) diff --git a/internal/auth/acl/grant.go b/internal/auth/acl/grant.go index 8a8cad846..e4c2ead72 100644 --- a/internal/auth/acl/grant.go +++ b/internal/auth/acl/grant.go @@ -11,6 +11,7 @@ var ( AccessOwn: true, AccessShared: true, AccessLibrary: true, + ActionUse: true, ActionView: true, ActionCreate: true, ActionUpdate: true, @@ -20,6 +21,7 @@ var ( ActionRate: true, ActionReact: true, ActionManage: true, + ActionPublish: true, ActionSubscribe: true, } GrantUploadAccess = Grant{ @@ -42,10 +44,12 @@ var ( GrantAll = Grant{ AccessAll: true, AccessOwn: true, + ActionUse: true, ActionView: true, ActionCreate: true, ActionUpdate: true, ActionDelete: true, + ActionPublish: true, ActionSubscribe: true, } GrantManageOwn = Grant{ @@ -122,9 +126,13 @@ var ( AccessAll: true, ActionSubscribe: true, } - GrantUse = Grant{ - Use: true, - ActionCreate: true, + GrantPublishOwn = Grant{ + AccessOwn: true, + ActionPublish: true, + } + GrantUseOwn = Grant{ + AccessOwn: true, + ActionUse: true, } GrantNone = Grant{} ) diff --git a/internal/auth/acl/resource_names.go b/internal/auth/acl/resource_names.go index 7822ddbc5..92180dbeb 100644 --- a/internal/auth/acl/resource_names.go +++ b/internal/auth/acl/resource_names.go @@ -22,7 +22,9 @@ var ResourceNames = []Resource{ ResourceUsers, ResourceSessions, ResourceLogs, + ResourceApi, ResourceWebDAV, + ResourceWebhooks, ResourceMetrics, ResourceVision, ResourceFeedback, diff --git a/internal/auth/acl/rules.go b/internal/auth/acl/rules.go index 0d2c96d55..aa4b4bbc2 100644 --- a/internal/auth/acl/rules.go +++ b/internal/auth/acl/rules.go @@ -87,17 +87,25 @@ var Rules = ACL{ RoleAdmin: GrantFullAccess, RoleClient: GrantFullAccess, }, + ResourceApi: Roles{ + RoleAdmin: GrantFullAccess, + RoleClient: GrantPublishOwn, + }, ResourceWebDAV: Roles{ RoleAdmin: GrantFullAccess, RoleClient: GrantFullAccess, }, + ResourceWebhooks: Roles{ + RoleAdmin: GrantFullAccess, + RoleClient: GrantPublishOwn, + }, ResourceMetrics: Roles{ RoleAdmin: GrantFullAccess, RoleClient: GrantViewAll, }, ResourceVision: Roles{ RoleAdmin: GrantFullAccess, - RoleClient: GrantUse, + RoleClient: GrantUseOwn, }, ResourceFeedback: Roles{ RoleAdmin: GrantFullAccess, diff --git a/internal/config/config.go b/internal/config/config.go index 995dab894..5dc7ca7f3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -46,6 +46,7 @@ import ( "github.com/photoprism/photoprism/internal/ai/face" "github.com/photoprism/photoprism/internal/ai/vision" + "github.com/photoprism/photoprism/internal/api/download" "github.com/photoprism/photoprism/internal/config/customize" "github.com/photoprism/photoprism/internal/config/ttl" "github.com/photoprism/photoprism/internal/entity" @@ -289,6 +290,13 @@ func (c *Config) Propagate() { vision.ServiceKey = c.VisionKey() vision.DownloadUrl = c.DownloadUrl() + // Set allowed path in download package. + download.AllowedPaths = []string{ + c.SidecarPath(), + c.OriginalsPath(), + c.ThumbCachePath(), + } + // Set cache expiration defaults. ttl.CacheDefault = c.HttpCacheMaxAge() ttl.CacheVideo = c.HttpVideoMaxAge() diff --git a/internal/entity/auth_session_fixtures.go b/internal/entity/auth_session_fixtures.go index 566ccc520..cfd1457d0 100644 --- a/internal/entity/auth_session_fixtures.go +++ b/internal/entity/auth_session_fixtures.go @@ -91,7 +91,7 @@ var SessionFixtures = SessionMap{ RefID: "sessjr0ge18d", SessTimeout: 0, SessExpires: unix.Now() + unix.Day, - AuthScope: clean.Scope("metrics photos albums videos"), + AuthScope: clean.Scope("metrics photos albums videos api webhooks"), AuthProvider: authn.ProviderAccessToken.String(), AuthMethod: authn.MethodDefault.String(), GrantType: authn.GrantPassword.String(), @@ -229,7 +229,7 @@ var SessionFixtures = SessionMap{ RefID: "sessgh6123yt", SessTimeout: 0, SessExpires: unix.Now() + unix.Week, - AuthScope: clean.Scope("statistics"), + AuthScope: clean.Scope("statistics api webhooks"), AuthProvider: authn.ProviderClient.String(), AuthMethod: authn.MethodOAuth2.String(), GrantType: authn.GrantCLI.String(), diff --git a/internal/server/routes.go b/internal/server/routes.go index 3ee5ca05a..c5252061e 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -197,6 +197,7 @@ func registerRoutes(router *gin.Engine, conf *config.Config) { api.DeleteErrors(APIv1) api.SendFeedback(APIv1) api.Connect(APIv1) + api.Webhook(APIv1) api.WebSocket(APIv1) api.GetMetrics(APIv1) api.Echo(APIv1) From 63d72ee8678cef8ec389a85808eae8301df538f8 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Apr 2025 19:22:54 +0200 Subject: [PATCH 03/23] AI: Refactor Webhook endpoint and improve related unit tests #127 #1090 Signed-off-by: Michael Mayer --- internal/api/webhook.go | 67 ++++++++++++++++++++---------------- internal/api/webhook_test.go | 38 ++++++++++---------- 2 files changed, 56 insertions(+), 49 deletions(-) diff --git a/internal/api/webhook.go b/internal/api/webhook.go index 8ad575c30..97b618a2a 100644 --- a/internal/api/webhook.go +++ b/internal/api/webhook.go @@ -1,13 +1,11 @@ package api import ( - "fmt" "net/http" "strings" "github.com/gin-gonic/gin" - "github.com/photoprism/photoprism/internal/api/download" "github.com/photoprism/photoprism/internal/api/hooks" "github.com/photoprism/photoprism/internal/auth/acl" "github.com/photoprism/photoprism/internal/event" @@ -18,9 +16,9 @@ import ( "github.com/photoprism/photoprism/pkg/media/http/header" ) -// Webhook listens for webhook events and checks their authorization. +// Webhook handles web event hooks and checks their authorization. // -// @Summary listens for webhook events and checks their authorization +// @Summary handles web event hooks and checks their authorization // @Id Webhook // @Tags Webhook // @Accept json @@ -29,22 +27,7 @@ import ( // @Param payload body hooks.Payload true "webhook event data" // @Router /api/v1/webhook/{channel} [post] func Webhook(router *gin.RouterGroup) { - requestHandler := func(c *gin.Context) { - // Prevent API response caching. - c.Header(header.CacheControl, header.CacheControlNoStore) - - // Only the instance channel is currently implemented. - if !acl.ChannelInstance.Equal(clean.Token(c.Param("channel"))) { - AbortNotImplemented(c) - return - } - - // For security reasons, this endpoint is not available in public or demo mode. - if conf := get.Config(); conf.Public() || conf.Demo() { - Abort(c, http.StatusForbidden, i18n.ErrFeatureDisabled) - return - } - + instanceRequestHandler := func(c *gin.Context) { s := Auth(c, acl.ResourceWebhooks, acl.ActionPublish) if s.Abort(c) { @@ -90,31 +73,55 @@ func Webhook(router *gin.RouterGroup) { return } + ev := "instance." + eventType + if s.IsClient() { if acl.Rules.Deny(acl.Resource(resource), s.ClientRole(), acl.ActionPublish) { - event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) + event.AuditWarn([]string{ClientIP(c), "client %s", "session %s", "webhook", "%s", authn.Denied}, clean.Log(s.ClientInfo()), s.RefID, ev) + AbortForbidden(c) + return + } else if s.InsufficientScope(acl.Resource(resource), acl.Permissions{acl.ActionPublish}) { + event.AuditErr([]string{ClientIP(c), "client %s", "session %s", "webhook", "%s", authn.ErrInsufficientScope.Error()}, clean.Log(s.ClientInfo()), s.RefID, ev) AbortForbidden(c) return } } else { - if acl.Rules.Deny(acl.Resource(resource), s.UserRole(), acl.ActionPublish) { - event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) - AbortForbidden(c) - return - } + event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) + AbortForbidden(c) + return } - ev := "instance." + eventType - switch ev { - case "instance.api.downloads.register": - _ = download.Register(fmt.Sprintf("%v", request.Data["uuid"]), fmt.Sprintf("%v", request.Data["filename"])) + // case "instance.api.downloads.register": + // _ = download.Register(fmt.Sprintf("%v", request.Data["uuid"]), fmt.Sprintf("%v", request.Data["filename"])) default: event.Publish(ev, request.Data) } } + requestHandler := func(c *gin.Context) { + // Prevent API response caching. + c.Header(header.CacheControl, header.CacheControlNoStore) + + // For security reasons, this endpoint is not available in public or demo mode. + if conf := get.Config(); conf.Public() || conf.Demo() { + Abort(c, http.StatusForbidden, i18n.ErrFeatureDisabled) + return + } + + // Only the instance channel is currently implemented. + channel := clean.Token(c.Param("channel")) + + switch channel { + case acl.ChannelInstance.String(): + instanceRequestHandler(c) + default: + AbortNotImplemented(c) + return + } + } + router.GET("/webhook/:channel", requestHandler) router.POST("/webhook/:channel", requestHandler) } diff --git a/internal/api/webhook_test.go b/internal/api/webhook_test.go index 22c3a8011..1728a43d4 100644 --- a/internal/api/webhook_test.go +++ b/internal/api/webhook_test.go @@ -29,25 +29,6 @@ func TestPostWebhook(t *testing.T) { }, } - body := payload.JSON() - token := "9d8b8801ffa23eb52e08ca7766283799ddfd8dd368212123" - - t.Logf("request: %s", string(body)) - - response := AuthenticatedRequestWithBody(app, http.MethodPost, "/api/v1/webhook/instance", string(body), token) - - assert.Equal(t, http.StatusOK, response.Code) - }) - t.Run("InvalidData", func(t *testing.T) { - payload := hooks.Payload{ - Type: "api.downloads.register", - Timestamp: time.Now().UTC(), - Data: event.Data{ - "uuid": 12345, - "filename": fs.Abs("./testdata/green_224x224.jpg"), - }, - } - body := payload.JSON() token := "778f0f7d80579a072836c65b786145d6e0127505194cc51e" @@ -57,6 +38,25 @@ func TestPostWebhook(t *testing.T) { assert.Equal(t, http.StatusOK, response.Code) }) + t.Run("InsufficientScope", func(t *testing.T) { + payload := hooks.Payload{ + Type: "api.downloads.register", + Timestamp: time.Now().UTC(), + Data: event.Data{ + "uuid": rnd.UUID(), + "filename": fs.Abs("./testdata/green_224x224.jpg"), + }, + } + + body := payload.JSON() + token := "9d8b8801ffa23eb52e08ca7766283799ddfd8dd368212123" + + t.Logf("request: %s", string(body)) + + response := AuthenticatedRequestWithBody(app, http.MethodPost, "/api/v1/webhook/instance", string(body), token) + + assert.Equal(t, http.StatusForbidden, response.Code) + }) t.Run("Unauthorized", func(t *testing.T) { r := PerformRequest(app, http.MethodPost, "/api/v1/webhook/instance") assert.Equal(t, http.StatusUnauthorized, r.Code) From 07e22b0c14298b49b3639114349168f7b2c289d2 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Apr 2025 19:39:39 +0200 Subject: [PATCH 04/23] API: Remove unused Webhook endpoint due to architecture change Signed-off-by: Michael Mayer --- internal/api/webhook.go | 127 ----------------------------------- internal/api/webhook_test.go | 64 ------------------ internal/server/routes.go | 1 - 3 files changed, 192 deletions(-) delete mode 100644 internal/api/webhook.go delete mode 100644 internal/api/webhook_test.go diff --git a/internal/api/webhook.go b/internal/api/webhook.go deleted file mode 100644 index 97b618a2a..000000000 --- a/internal/api/webhook.go +++ /dev/null @@ -1,127 +0,0 @@ -package api - -import ( - "net/http" - "strings" - - "github.com/gin-gonic/gin" - - "github.com/photoprism/photoprism/internal/api/hooks" - "github.com/photoprism/photoprism/internal/auth/acl" - "github.com/photoprism/photoprism/internal/event" - "github.com/photoprism/photoprism/internal/photoprism/get" - "github.com/photoprism/photoprism/pkg/authn" - "github.com/photoprism/photoprism/pkg/clean" - "github.com/photoprism/photoprism/pkg/i18n" - "github.com/photoprism/photoprism/pkg/media/http/header" -) - -// Webhook handles web event hooks and checks their authorization. -// -// @Summary handles web event hooks and checks their authorization -// @Id Webhook -// @Tags Webhook -// @Accept json -// @Success 200 -// @Failure 401,403,429 -// @Param payload body hooks.Payload true "webhook event data" -// @Router /api/v1/webhook/{channel} [post] -func Webhook(router *gin.RouterGroup) { - instanceRequestHandler := func(c *gin.Context) { - s := Auth(c, acl.ResourceWebhooks, acl.ActionPublish) - - if s.Abort(c) { - return - } - - var request hooks.Payload - - // Assign and validate request form values. - if c.Request.Method == http.MethodGet { - if err := c.BindQuery(&request); err != nil { - event.AuditErr([]string{ClientIP(c), "session %s", "webhook", "%s"}, s.RefID, err) - AbortBadRequest(c) - return - } - } else { - if err := c.BindJSON(&request); err != nil { - event.AuditErr([]string{ClientIP(c), "session %s", "webhook", "%s"}, s.RefID, err) - AbortBadRequest(c) - return - } - } - - eventType := clean.TypeLowerUnderscore(request.Type) - - if eventType == "" { - event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "missing type"}, s.RefID) - AbortBadRequest(c) - return - } - - if request.Data == nil { - event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "missing data"}, s.RefID) - AbortBadRequest(c) - return - } - - resource, resourceEv, found := strings.Cut(eventType, ".") - - if !found || resource == "" || resourceEv == "" { - event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) - AbortBadRequest(c) - return - } - - ev := "instance." + eventType - - if s.IsClient() { - if acl.Rules.Deny(acl.Resource(resource), s.ClientRole(), acl.ActionPublish) { - event.AuditWarn([]string{ClientIP(c), "client %s", "session %s", "webhook", "%s", authn.Denied}, clean.Log(s.ClientInfo()), s.RefID, ev) - AbortForbidden(c) - return - } else if s.InsufficientScope(acl.Resource(resource), acl.Permissions{acl.ActionPublish}) { - event.AuditErr([]string{ClientIP(c), "client %s", "session %s", "webhook", "%s", authn.ErrInsufficientScope.Error()}, clean.Log(s.ClientInfo()), s.RefID, ev) - AbortForbidden(c) - return - } - } else { - event.AuditWarn([]string{ClientIP(c), "session %s", "webhook", "%s", authn.Denied}, s.RefID, eventType) - AbortForbidden(c) - return - } - - switch ev { - // case "instance.api.downloads.register": - // _ = download.Register(fmt.Sprintf("%v", request.Data["uuid"]), fmt.Sprintf("%v", request.Data["filename"])) - default: - event.Publish(ev, request.Data) - } - - } - - requestHandler := func(c *gin.Context) { - // Prevent API response caching. - c.Header(header.CacheControl, header.CacheControlNoStore) - - // For security reasons, this endpoint is not available in public or demo mode. - if conf := get.Config(); conf.Public() || conf.Demo() { - Abort(c, http.StatusForbidden, i18n.ErrFeatureDisabled) - return - } - - // Only the instance channel is currently implemented. - channel := clean.Token(c.Param("channel")) - - switch channel { - case acl.ChannelInstance.String(): - instanceRequestHandler(c) - default: - AbortNotImplemented(c) - return - } - } - - router.GET("/webhook/:channel", requestHandler) - router.POST("/webhook/:channel", requestHandler) -} diff --git a/internal/api/webhook_test.go b/internal/api/webhook_test.go deleted file mode 100644 index 1728a43d4..000000000 --- a/internal/api/webhook_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package api - -import ( - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/photoprism/photoprism/internal/api/hooks" - "github.com/photoprism/photoprism/internal/config" - "github.com/photoprism/photoprism/internal/event" - "github.com/photoprism/photoprism/pkg/fs" - "github.com/photoprism/photoprism/pkg/rnd" -) - -func TestPostWebhook(t *testing.T) { - app, router, conf := NewApiTest() - conf.SetAuthMode(config.AuthModePasswd) - defer conf.SetAuthMode(config.AuthModePublic) - Webhook(router) - t.Run("Success", func(t *testing.T) { - payload := hooks.Payload{ - Type: "api.downloads.register", - Timestamp: time.Now().UTC(), - Data: event.Data{ - "uuid": rnd.UUID(), - "filename": fs.Abs("./testdata/cat_224x224.jpg"), - }, - } - - body := payload.JSON() - token := "778f0f7d80579a072836c65b786145d6e0127505194cc51e" - - t.Logf("request: %s", string(body)) - - response := AuthenticatedRequestWithBody(app, http.MethodPost, "/api/v1/webhook/instance", string(body), token) - - assert.Equal(t, http.StatusOK, response.Code) - }) - t.Run("InsufficientScope", func(t *testing.T) { - payload := hooks.Payload{ - Type: "api.downloads.register", - Timestamp: time.Now().UTC(), - Data: event.Data{ - "uuid": rnd.UUID(), - "filename": fs.Abs("./testdata/green_224x224.jpg"), - }, - } - - body := payload.JSON() - token := "9d8b8801ffa23eb52e08ca7766283799ddfd8dd368212123" - - t.Logf("request: %s", string(body)) - - response := AuthenticatedRequestWithBody(app, http.MethodPost, "/api/v1/webhook/instance", string(body), token) - - assert.Equal(t, http.StatusForbidden, response.Code) - }) - t.Run("Unauthorized", func(t *testing.T) { - r := PerformRequest(app, http.MethodPost, "/api/v1/webhook/instance") - assert.Equal(t, http.StatusUnauthorized, r.Code) - }) -} diff --git a/internal/server/routes.go b/internal/server/routes.go index c5252061e..3ee5ca05a 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -197,7 +197,6 @@ func registerRoutes(router *gin.Engine, conf *config.Config) { api.DeleteErrors(APIv1) api.SendFeedback(APIv1) api.Connect(APIv1) - api.Webhook(APIv1) api.WebSocket(APIv1) api.GetMetrics(APIv1) api.Echo(APIv1) From 63fb46578a4197dfb09b75297cb4a0b05d6402c7 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 11 Apr 2025 20:02:08 +0200 Subject: [PATCH 05/23] API: Remove unused "api/hooks" package Signed-off-by: Michael Mayer --- internal/api/hooks/hooks.go | 31 ---- internal/api/hooks/payload_test.go | 37 ---- internal/api/hooks/payloard.go | 27 --- internal/api/hooks/secret.go | 152 ----------------- internal/api/hooks/secret_test.go | 260 ----------------------------- 5 files changed, 507 deletions(-) delete mode 100644 internal/api/hooks/hooks.go delete mode 100644 internal/api/hooks/payload_test.go delete mode 100644 internal/api/hooks/payloard.go delete mode 100644 internal/api/hooks/secret.go delete mode 100644 internal/api/hooks/secret_test.go diff --git a/internal/api/hooks/hooks.go b/internal/api/hooks/hooks.go deleted file mode 100644 index a925e1664..000000000 --- a/internal/api/hooks/hooks.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Package hooks provides webhook authentication and payload handlers. - -Copyright (c) 2018 - 2025 PhotoPrism UG. All rights reserved. - - This program is free software: you can redistribute it and/or modify - it under Version 3 of the GNU Affero General Public License (the "AGPL"): - - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - The AGPL is supplemented by our Trademark and Brand Guidelines, - which describe how our Brand Assets may be used: - - -Feel free to send an email to hello@photoprism.app if you have questions, -want to support our work, or just want to say hello. - -Additional information can be found in our Developer Guide: - -*/ -package hooks - -import ( - "github.com/photoprism/photoprism/internal/event" -) - -var log = event.Log diff --git a/internal/api/hooks/payload_test.go b/internal/api/hooks/payload_test.go deleted file mode 100644 index ecef60ed1..000000000 --- a/internal/api/hooks/payload_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package hooks - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/photoprism/photoprism/internal/event" -) - -func TestPayload_Json(t *testing.T) { - t.Run("Success", func(t *testing.T) { - timeStamp, timeErr := time.Parse(time.RFC3339Nano, "2025-04-11T11:48:58.540199797Z") - - if timeErr != nil { - t.Fatal(timeErr) - } - - id := "49b8a329-5aa6-4b76-ba62-bb3adb001817" - - payload := &Payload{ - Type: "foo.bar", - Timestamp: timeStamp.UTC(), - Data: event.Data{ - "id": id, - "hello": "World!", - "number": 42, - }, - } - - result := payload.JSON() - expected := `{"type":"foo.bar","timestamp":"2025-04-11T11:48:58.540199797Z","data":{"hello":"World!","id":"49b8a329-5aa6-4b76-ba62-bb3adb001817","number":42}}` - - assert.Equal(t, expected, string(result)) - }) -} diff --git a/internal/api/hooks/payloard.go b/internal/api/hooks/payloard.go deleted file mode 100644 index d9ce363e5..000000000 --- a/internal/api/hooks/payloard.go +++ /dev/null @@ -1,27 +0,0 @@ -package hooks - -import ( - "encoding/json" - "time" - - "github.com/photoprism/photoprism/internal/event" - "github.com/photoprism/photoprism/pkg/clean" -) - -// Payload represents a webhook payload. -type Payload struct { - Type string `form:"type" json:"type"` - Timestamp time.Time `form:"timestamp" json:"timestamp,omitempty"` - Data event.Data `form:"data" json:"data"` -} - -// JSON returns the payload data as JSON-encoded bytes. -func (p *Payload) JSON() (b []byte) { - b, jsonErr := json.Marshal(p) - - if jsonErr != nil { - log.Warningf("hook: %s (json encode)", clean.Error(jsonErr)) - } - - return b -} diff --git a/internal/api/hooks/secret.go b/internal/api/hooks/secret.go deleted file mode 100644 index 5d1f821be..000000000 --- a/internal/api/hooks/secret.go +++ /dev/null @@ -1,152 +0,0 @@ -package hooks - -import ( - "crypto/hmac" - "crypto/sha256" - "errors" - "fmt" - "net/http" - "strconv" - "strings" - "time" - - "github.com/photoprism/photoprism/pkg/media" - "github.com/photoprism/photoprism/pkg/media/http/header" -) - -var timeTolerance = 5 * time.Minute - -var ( - ErrRequiredHeaders = errors.New("missing required headers") - ErrInvalidHeaders = errors.New("invalid signature headers") - ErrNoMatchingSignature = errors.New("no matching signature found") - ErrMessageTooOld = errors.New("message timestamp too old") - ErrMessageTooNew = errors.New("message timestamp too new") -) - -type Secret struct { - key []byte -} - -func NewSecret(secret string) (*Secret, error) { - key, err := media.DecodeBase64String(strings.TrimPrefix(secret, header.WebhookSecretPrefix)) - if err != nil { - return nil, fmt.Errorf("unable to create webhook, err: %w", err) - } - return &Secret{ - key: key, - }, nil -} - -func NewWebhookRaw(secret []byte) (*Secret, error) { - return &Secret{ - key: secret, - }, nil -} - -// Verify validates the payload against the webhook signature headers -// using the webhooks signing secret. -// -// Returns an error if the body or headers are missing/unreadable -// or if the signature doesn't match. -func (wh *Secret) Verify(payload []byte, headers http.Header) error { - return wh.verify(payload, headers, true) -} - -// VerifyIgnoringTimestamp validates the payload against the webhook signature headers -// using the webhooks signing secret. -// -// Returns an error if the body or headers are missing/unreadable -// or if the signature doesn't match. -// -// WARNING: This function does not check the signature's timestamp. -// We recommend using the `Verify` function instead. -func (wh *Secret) VerifyIgnoringTimestamp(payload []byte, headers http.Header) error { - return wh.verify(payload, headers, false) -} - -func (wh *Secret) verify(payload []byte, headers http.Header, enforceTolerance bool) error { - msgId := headers.Get(header.WebhookID) - msgSignature := headers.Get(header.WebhookSignature) - msgTimestamp := headers.Get(header.WebhookTimestamp) - if msgId == "" || msgSignature == "" || msgTimestamp == "" { - return fmt.Errorf("unable to verify payload, err: %w", ErrRequiredHeaders) - } - - timestamp, err := parseTimestampHeader(msgTimestamp) - if err != nil { - return fmt.Errorf("unable to verify payload, err: %w", err) - } - - if enforceTolerance { - if err := verifyTimestamp(timestamp); err != nil { - return fmt.Errorf("unable to verify payload, err: %w", err) - } - } - - _, expectedSignature, err := wh.sign(msgId, timestamp, payload) - if err != nil { - return fmt.Errorf("unable to verify payload, err: %w", err) - } - - passedSignatures := strings.Split(msgSignature, " ") - for _, versionedSignature := range passedSignatures { - sigParts := strings.Split(versionedSignature, ",") - if len(sigParts) < 2 { - continue - } - - version := sigParts[0] - - if version != "v1" { - continue - } - - signature := []byte(sigParts[1]) - - if hmac.Equal(signature, expectedSignature) { - return nil - } - } - - return fmt.Errorf("unable to verify payload, err: %w", ErrNoMatchingSignature) -} - -func (wh *Secret) Sign(msgId string, timestamp time.Time, payload []byte) (string, error) { - version, signature, err := wh.sign(msgId, timestamp, payload) - return fmt.Sprintf("%s,%s", version, signature), err -} - -func (wh *Secret) sign(msgId string, timestamp time.Time, payload []byte) (version string, signature []byte, err error) { - toSign := fmt.Sprintf("%s.%d.%s", msgId, timestamp.Unix(), payload) - - h := hmac.New(sha256.New, wh.key) - h.Write([]byte(toSign)) - sig := make([]byte, media.EncodedLenBase64(h.Size())) - media.EncodeBase64Bytes(sig, h.Sum(nil)) - - return "v1", sig, nil -} - -func parseTimestampHeader(timestampHeader string) (time.Time, error) { - timeInt, err := strconv.ParseInt(timestampHeader, 10, 64) - if err != nil { - return time.Time{}, fmt.Errorf("unable to parse timestamp header, err: %w", errors.Join(err, ErrInvalidHeaders)) - } - timestamp := time.Unix(timeInt, 0) - return timestamp, nil -} - -func verifyTimestamp(timestamp time.Time) error { - now := time.Now() - - if now.Sub(timestamp) > timeTolerance { - return ErrMessageTooOld - } - - if timestamp.After(now.Add(timeTolerance)) { - return ErrMessageTooNew - } - - return nil -} diff --git a/internal/api/hooks/secret_test.go b/internal/api/hooks/secret_test.go deleted file mode 100644 index f86d96c5f..000000000 --- a/internal/api/hooks/secret_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package hooks - -import ( - "fmt" - "net/http" - "strings" - "testing" - "time" - - "github.com/photoprism/photoprism/pkg/media/http/header" -) - -var defaultMsgID = "msg_p5jXN8AQM9LWM0D4loKWxJek" -var defaultPayload = []byte(`{"test": 2432232314}`) -var defaultSecret = "MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" - -type testPayload struct { - id string - timestamp time.Time - header http.Header - secret string - payload []byte - signature string -} - -func newTestPayload(timestamp time.Time) *testPayload { - tp := &testPayload{} - tp.id = defaultMsgID - tp.timestamp = timestamp - - tp.payload = defaultPayload - tp.secret = defaultSecret - - wh, _ := NewSecret(tp.secret) - tp.signature, _ = wh.Sign(tp.id, tp.timestamp, tp.payload) - - tp.header = http.Header{} - tp.header.Set(header.WebhookID, tp.id) - tp.header.Set(header.WebhookSignature, tp.signature) - tp.header.Set(header.WebhookTimestamp, fmt.Sprint(tp.timestamp.Unix())) - - return tp -} - -func TestWebhook(t *testing.T) { - - testCases := []struct { - name string - testPayload *testPayload - modifyPayload func(*testPayload) - noEnforceTimestamp bool - expectedErr bool - }{ - { - name: "valid signature is valid", - testPayload: newTestPayload(time.Now()), - expectedErr: false, - }, - { - name: "missing id returns error", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - tp.header.Del("webhook-id") - }, - expectedErr: true, - }, - { - name: "missing timestamp returns error", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - tp.header.Del("webhook-timestamp") - }, - expectedErr: true, - }, - { - name: "missing signature returns error", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - tp.header.Del("webhook-signature") - }, - expectedErr: true, - }, - { - name: "invalid signature is invalid", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - tp.header.Set("webhook-signature", "v1,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=") - }, - expectedErr: true, - }, - { - name: "partial signature is invalid", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - tp.header.Set("webhook-signature", "v1,") - }, - expectedErr: true, - }, - { - name: "old timestamp fails", - testPayload: newTestPayload(time.Now().Add(timeTolerance * -1)), - expectedErr: true, - }, - { - name: "new timestamp fails", - testPayload: newTestPayload(time.Now().Add(timeTolerance + time.Second)), - expectedErr: true, - }, - { - name: "valid multi sig is valid", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - sigs := []string{ - "v1,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=", - "v2,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=", - tp.header.Get("webhook-signature"), // valid signature - "v1,Ceo5qEr07ixe2NLpvHk3FH9bwy/WavXrAFQ/9tdO6mc=", - } - tp.header.Set("webhook-signature", strings.Join(sigs, " ")) - }, - expectedErr: false, - }, - { - name: "old timestamp passes when ignoring tolerance", - testPayload: newTestPayload(time.Now().Add(timeTolerance * -1)), - noEnforceTimestamp: true, - expectedErr: false, - }, - { - name: "new timestamp passes when ignoring tolerance", - testPayload: newTestPayload(time.Now().Add(timeTolerance * 1)), - noEnforceTimestamp: true, - expectedErr: false, - }, - { - name: "valid timestamp passes when ignoring tolerance", - testPayload: newTestPayload(time.Now()), - noEnforceTimestamp: true, - expectedErr: false, - }, - { - name: "invalid timestamp fails when ignoring tolerance", - testPayload: newTestPayload(time.Now()), - modifyPayload: func(tp *testPayload) { - tp.header.Set("webhook-timestamp", fmt.Sprint(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).Unix())) - }, - noEnforceTimestamp: true, - expectedErr: true, - }, - } - - for _, tc := range testCases { - if tc.modifyPayload != nil { - tc.modifyPayload(tc.testPayload) - } - - wh, err := NewSecret(tc.testPayload.secret) - if err != nil { - t.Error(err) - continue - } - if tc.noEnforceTimestamp { - err = wh.VerifyIgnoringTimestamp(tc.testPayload.payload, tc.testPayload.header) - } else { - err = wh.Verify(tc.testPayload.payload, tc.testPayload.header) - } - if err != nil && !tc.expectedErr { - t.Errorf("%s: failed with err %s but shouldn't have", tc.name, err.Error()) - } else if err == nil && tc.expectedErr { - t.Errorf("%s: didn't error but should have", tc.name) - } - } -} - -func TestWebhookPrefix(t *testing.T) { - tp := newTestPayload(time.Now()) - - wh, err := NewSecret(tp.secret) - if err != nil { - t.Fatal(err) - } - - err = wh.Verify(tp.payload, tp.header) - if err != nil { - t.Fatal(err) - } - - wh, err = NewSecret(fmt.Sprintf("whsec_%s", tp.secret)) - if err != nil { - t.Fatal(err) - } - - err = wh.Verify(tp.payload, tp.header) - if err != nil { - t.Fatal(err) - } -} - -func TestWebhookSign(t *testing.T) { - key := "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" - msgID := "msg_p5jXN8AQM9LWM0D4loKWxJek" - timestamp := time.Unix(1614265330, 0) - payload := []byte(`{"test": 2432232314}`) - expected := "v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=" - - wh, err := NewSecret(key) - if err != nil { - t.Fatal(err) - } - - signature, err := wh.Sign(msgID, timestamp, payload) - if err != nil { - t.Fatal(err) - } - - if signature != expected { - t.Fatalf("signature %s != expected signature %s", signature, expected) - } - -} - -// A complete example flow for signing and verifying a webhook payload, -// including timestamp verification. -func TestSignatureFlow(t *testing.T) { - const secretKey = "MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw" - - var ( - ts = time.Now() - id = "1234567890" - ) - - wh, err := NewSecret(secretKey) - if err != nil { - t.Fatal(err) - } - - payload := `{"type": "example.created", "timestamp":"2023-09-28T19:20:22+00:00", "data":{"str":"string","bool":true,"int":42}}` - - // signing the payload with the webhook handler - signature, err := wh.Sign(id, ts, []byte(payload)) - if err != nil { - t.Fatal(err) - } - - // generating the http header carrier - head := http.Header{} - head.Set(header.WebhookID, id) - head.Set(header.WebhookSignature, signature) - head.Set(header.WebhookTimestamp, fmt.Sprint(ts.Unix())) - - // http request is sent to consumer - - // consumer verifies the signature - err = wh.Verify([]byte(payload), head) - if err != nil { - t.Fatal(err) - } - - // Done. -} From 43c794d24283d97cafa3dbf19f1d8dc78f6a2fdf Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 11:49:03 +0200 Subject: [PATCH 06/23] AI: Add script to install the latest qdrant database binary #127 #1090 Signed-off-by: Michael Mayer --- scripts/dist/install-qdrant.sh | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 scripts/dist/install-qdrant.sh diff --git a/scripts/dist/install-qdrant.sh b/scripts/dist/install-qdrant.sh new file mode 100755 index 000000000..6fd05dec0 --- /dev/null +++ b/scripts/dist/install-qdrant.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Installs the qdrant binary, a vector search engine, on Linux. +# bash <(curl -s https://raw.githubusercontent.com/photoprism/photoprism/develop/scripts/dist/install-qdrant.sh) + +set -e + +# Show usage information if first argument is --help. +if [[ ${1} == "--help" ]]; then + echo "Usage: ${0##*/} [destdir] [version]" 1>&2 + exit 0 +fi + +# You can provide a custom installation directory as the first argument. +DESTDIR=$(realpath "${1:-/usr/local}") + +# Determine target architecture. +if [[ $PHOTOPRISM_ARCH ]]; then + SYSTEM_ARCH=$PHOTOPRISM_ARCH +else + SYSTEM_ARCH=$(uname -m) +fi + +DESTARCH=${BUILD_ARCH:-$SYSTEM_ARCH} + +case $DESTARCH in + amd64 | AMD64 | x86_64 | x86-64) + DESTARCH=x86_64 + ;; + + arm64 | ARM64 | aarch64) + DESTARCH=aarch64 + ;; + + *) + echo "Unsupported Machine Architecture: \"$DESTARCH\"" 1>&2 + exit 1 + ;; +esac + +. /etc/os-release + +# Abort if not executed as root. +if [[ $(id -u) != "0" ]] && [[ $DESTDIR == "/usr" || $DESTDIR == "/usr/local" ]]; then + echo "Error: Run ${0##*/} as root to install in a system directory!" 1>&2 + exit 1 +fi + +echo "Installing Qdrant for ${DESTARCH^^}..." + +# Alternatively, users can specify a custom version to install as the second argument. +GITHUB_LATEST=$(curl --silent "https://api.github.com/repos/qdrant/qdrant/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') +VERSION=${2:-$GITHUB_LATEST} +ARCHIVE="qdrant-${DESTARCH}-unknown-linux-musl.tar.gz" +GITHUB_URL="https://github.com/qdrant/qdrant/releases/download/${VERSION}/${ARCHIVE}" + +echo "------------------------------------------------" +echo "VERSION : ${VERSION}" +echo "LATEST : ${GITHUB_LATEST}" +echo "DOWNLOAD: ${GITHUB_URL}" +echo "DESTDIR : ${DESTDIR}" +echo "------------------------------------------------" + +# Adjust the installation path because the archive does not contain a bin directory. +DESTDIR="${DESTDIR}/bin" + +echo "Extracting the qdrant binary in \"${ARCHIVE}\" to \"${DESTDIR}\"..." +mkdir -p "${DESTDIR}" +curl -fsSL "${GITHUB_URL}" | tar --overwrite --mode=755 -xz -C "${DESTDIR}" --wildcards --no-anchored "qdrant" + +echo "Done." From 1831261cfd077ae562bb89cd2ffc017a9ead8248 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 11:50:12 +0200 Subject: [PATCH 07/23] Dist: Add script to install the latest nats-server binary Signed-off-by: Michael Mayer --- scripts/dist/install-nats.sh | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100755 scripts/dist/install-nats.sh diff --git a/scripts/dist/install-nats.sh b/scripts/dist/install-nats.sh new file mode 100755 index 000000000..b1d45e240 --- /dev/null +++ b/scripts/dist/install-nats.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# Installs the nats-server binary, a cloud-native messaging system, on Linux. +# bash <(curl -s https://raw.githubusercontent.com/photoprism/photoprism/develop/scripts/dist/install-nats.sh) + +set -e + +# Show usage information if first argument is --help. +if [[ ${1} == "--help" ]]; then + echo "Usage: ${0##*/} [destdir] [version]" 1>&2 + exit 0 +fi + +# You can provide a custom installation directory as the first argument. +DESTDIR=$(realpath "${1:-/usr/local}") + +# Determine target architecture. +if [[ $PHOTOPRISM_ARCH ]]; then + SYSTEM_ARCH=$PHOTOPRISM_ARCH +else + SYSTEM_ARCH=$(uname -m) +fi + +DESTARCH=${BUILD_ARCH:-$SYSTEM_ARCH} + +case $DESTARCH in + amd64 | AMD64 | x86_64 | x86-64) + DESTARCH=amd64 + ;; + + arm64 | ARM64 | aarch64) + DESTARCH=arm64 + ;; + + arm | ARM | aarch | armv7l | armhf) + DESTARCH=arm7 + ;; + + *) + echo "Unsupported Machine Architecture: \"$DESTARCH\"" 1>&2 + exit 1 + ;; +esac + +. /etc/os-release + +# Abort if not executed as root. +if [[ $(id -u) != "0" ]] && [[ $DESTDIR == "/usr" || $DESTDIR == "/usr/local" ]]; then + echo "Error: Run ${0##*/} as root to install in a system directory!" 1>&2 + exit 1 +fi + +echo "Installing NATS for ${DESTARCH^^}..." + +# Alternatively, users can specify a custom version to install as the second argument. +GITHUB_LATEST=$(curl --silent "https://api.github.com/repos/nats-io/nats-server/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') +VERSION=${2:-$GITHUB_LATEST} +ARCHIVE="nats-server-${VERSION}-linux-${DESTARCH}.tar.gz" +GITHUB_URL="https://github.com/nats-io/nats-server/releases/download/${VERSION}/${ARCHIVE}" + +echo "------------------------------------------------" +echo "VERSION : ${VERSION}" +echo "LATEST : ${GITHUB_LATEST}" +echo "DOWNLOAD: ${GITHUB_URL}" +echo "DESTDIR : ${DESTDIR}" +echo "------------------------------------------------" + +# Adjust the installation path because the archive does not contain a bin directory. +DESTDIR="${DESTDIR}/bin" + +echo "Extracting the nats-server binary in \"${ARCHIVE}\" to \"${DESTDIR}\"..." +mkdir -p "${DESTDIR}" +curl -fsSL "${GITHUB_URL}" | tar --overwrite --mode=755 -xz -C "${DESTDIR}" --strip-components=1 --wildcards --no-anchored "nats-server" + +echo "Done." From b03f06a1a2c0b76f61b45f234b40284307dac8b2 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 12:22:30 +0200 Subject: [PATCH 08/23] Dist: Refactor install-s6.sh script Signed-off-by: Michael Mayer --- scripts/dist/install-s6.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/dist/install-s6.sh b/scripts/dist/install-s6.sh index 10f397696..262f5fdad 100755 --- a/scripts/dist/install-s6.sh +++ b/scripts/dist/install-s6.sh @@ -58,27 +58,29 @@ set -eu S6_OVERLAY_LATEST=$(curl --silent "https://api.github.com/repos/just-containers/s6-overlay/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') S6_OVERLAY_VERSION=${1:-$S6_OVERLAY_LATEST} -S6_ARCH_URL="https://github.com/just-containers/s6-overlay/releases/download/${S6_OVERLAY_VERSION}/s6-overlay-${S6_OVERLAY_ARCH}.tar.xz" -S6_NOARCH_URL="https://github.com/just-containers/s6-overlay/releases/download/${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz" +ARCHIVE_NOARCH="s6-overlay-noarch.tar.xz" +ARCHIVE_BINARY="s6-overlay-${S6_OVERLAY_ARCH}.tar.xz" +S6_NOARCH_URL="https://github.com/just-containers/s6-overlay/releases/download/${S6_OVERLAY_VERSION}/${ARCHIVE_NOARCH}" +S6_BINARY_URL="https://github.com/just-containers/s6-overlay/releases/download/${S6_OVERLAY_VERSION}/${ARCHIVE_BINARY}" -echo "Installing S6 Overlay..." +echo "Installing S6 Overlay for ${S6_OVERLAY_ARCH^^}..." echo "------------------------------------------------" -echo "VERSION : ${S6_OVERLAY_VERSION}" -echo "LATEST : ${S6_OVERLAY_LATEST}" -echo "DESTDIR : ${S6_OVERLAY_DESTDIR}" -echo "BINARY URL: ${S6_ARCH_URL}" -echo "NOARCH URL: ${S6_NOARCH_URL}" +echo "VERSION: ${S6_OVERLAY_VERSION}" +echo "LATEST : ${S6_OVERLAY_LATEST}" +echo "NOARCH : ${ARCHIVE_NOARCH}" +echo "BINARY : ${ARCHIVE_BINARY}" +echo "DESTDIR: ${S6_OVERLAY_DESTDIR}" echo "------------------------------------------------" # Create the destination directory if it does not already exist. mkdir -p "${S6_OVERLAY_DESTDIR}" # Download and install the s6-overlay release from GitHub. -echo "Extracting \"$S6_ARCH_URL\" to \"$S6_OVERLAY_DESTDIR\"." -curl -fsSL "$S6_ARCH_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp - -echo "Extracting \"$S6_NOARCH_URL\" to \"$S6_OVERLAY_DESTDIR\"." +echo "Extracting \"$S6_NOARCH_URL\" to \"$S6_OVERLAY_DESTDIR\"..." curl -fsSL "$S6_NOARCH_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp +echo "Extracting \"$S6_BINARY_URL\" to \"$S6_OVERLAY_DESTDIR\"..." +curl -fsSL "$S6_BINARY_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp + echo "Done." From 06de8e2b66b9b1da080527039181b940b9161c5f Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 14:06:39 +0200 Subject: [PATCH 09/23] Docker: Refactor service initialization and reduce startup log verbosity Signed-off-by: Michael Mayer --- docker/develop/armv7/Dockerfile | 1 + docker/develop/jammy-slim/Dockerfile | 6 ++++-- docker/develop/jammy/Dockerfile | 1 + docker/develop/oracular-slim/Dockerfile | 6 ++++-- docker/develop/oracular/Dockerfile | 1 + docker/photoprism/armv7/Dockerfile | 6 ++++-- docker/photoprism/jammy/Dockerfile | 3 --- docker/photoprism/oracular/Dockerfile | 3 --- scripts/dist/install-s6.sh | 7 +++++++ scripts/dist/services/photoprism/dependencies.d/base | 0 scripts/dist/services/photoprism/run | 3 +++ scripts/dist/services/photoprism/type | 1 + 12 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 scripts/dist/services/photoprism/dependencies.d/base create mode 100644 scripts/dist/services/photoprism/run create mode 100644 scripts/dist/services/photoprism/type diff --git a/docker/develop/armv7/Dockerfile b/docker/develop/armv7/Dockerfile index 67c5c6831..5c0f45830 100644 --- a/docker/develop/armv7/Dockerfile +++ b/docker/develop/armv7/Dockerfile @@ -39,6 +39,7 @@ ENV PHOTOPRISM_ARCH=$TARGETARCH \ CGO_CFLAGS="-g -O2 -Wno-return-local-addr" \ PROG="photoprism" \ S6_KEEP_ENV=1 \ + S6_VERBOSITY=0 \ S6_LOGGING=0 # Copy scripts and package sources config. diff --git a/docker/develop/jammy-slim/Dockerfile b/docker/develop/jammy-slim/Dockerfile index d5b1ea534..d642f8539 100644 --- a/docker/develop/jammy-slim/Dockerfile +++ b/docker/develop/jammy-slim/Dockerfile @@ -33,7 +33,8 @@ ENV PHOTOPRISM_ARCH=$TARGETARCH \ TF_ENABLE_ONEDNN_OPTS=1 \ MALLOC_ARENA_MAX=4 \ PROG="photoprism" \ - S6_KEEP_ENV=1 \ + S6_KEEP_ENV=0 \ + S6_VERBOSITY=0 \ S6_LOGGING=0 # Copy scripts and package sources config. @@ -73,6 +74,8 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \ /photoprism/storage/config \ /photoprism/storage/cache && \ /scripts/install-s6.sh && \ + ln -sf /scripts/services/photoprism /etc/s6-overlay/s6-rc.d/photoprism && \ + touch /etc/s6-overlay/s6-rc.d/user/contents.d/photoprism && \ /scripts/cleanup.sh # Set default working directory. @@ -83,4 +86,3 @@ EXPOSE 2342 2442 2443 # Set default entrypoint and command. ENTRYPOINT ["/init"] -CMD ["/scripts/cmd.sh", "tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/docker/develop/jammy/Dockerfile b/docker/develop/jammy/Dockerfile index b3590faa1..87b6c95b4 100644 --- a/docker/develop/jammy/Dockerfile +++ b/docker/develop/jammy/Dockerfile @@ -39,6 +39,7 @@ ENV PHOTOPRISM_ARCH=$TARGETARCH \ CGO_CFLAGS="-g -O2 -Wno-return-local-addr" \ PROG="photoprism" \ S6_KEEP_ENV=1 \ + S6_VERBOSITY=0 \ S6_LOGGING=0 # Copy scripts and package sources config. diff --git a/docker/develop/oracular-slim/Dockerfile b/docker/develop/oracular-slim/Dockerfile index 4b4a117cf..a270604a1 100644 --- a/docker/develop/oracular-slim/Dockerfile +++ b/docker/develop/oracular-slim/Dockerfile @@ -33,7 +33,8 @@ ENV PHOTOPRISM_ARCH=$TARGETARCH \ TF_ENABLE_ONEDNN_OPTS=1 \ MALLOC_ARENA_MAX=4 \ PROG="photoprism" \ - S6_KEEP_ENV=1 \ + S6_KEEP_ENV=0 \ + S6_VERBOSITY=0 \ S6_LOGGING=0 # Copy scripts and package sources config. @@ -74,6 +75,8 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \ /photoprism/storage/config \ /photoprism/storage/cache && \ /scripts/install-s6.sh && \ + ln -sf /scripts/services/photoprism /etc/s6-overlay/s6-rc.d/photoprism && \ + touch /etc/s6-overlay/s6-rc.d/user/contents.d/photoprism && \ /scripts/cleanup.sh # Set default working directory. @@ -84,4 +87,3 @@ EXPOSE 2342 2442 2443 # Set default entrypoint and command. ENTRYPOINT ["/init"] -CMD ["/scripts/cmd.sh", "tail", "-f", "/dev/null"] diff --git a/docker/develop/oracular/Dockerfile b/docker/develop/oracular/Dockerfile index 5cc72728c..c571bdd03 100644 --- a/docker/develop/oracular/Dockerfile +++ b/docker/develop/oracular/Dockerfile @@ -39,6 +39,7 @@ ENV PHOTOPRISM_ARCH=$TARGETARCH \ CGO_CFLAGS="-g -O2 -Wno-return-local-addr" \ PROG="photoprism" \ S6_KEEP_ENV=1 \ + S6_VERBOSITY=0 \ S6_LOGGING=0 # Copy scripts and package sources config. diff --git a/docker/photoprism/armv7/Dockerfile b/docker/photoprism/armv7/Dockerfile index 4ce528ec8..9b4a69d6f 100644 --- a/docker/photoprism/armv7/Dockerfile +++ b/docker/photoprism/armv7/Dockerfile @@ -92,7 +92,8 @@ ENV PHOTOPRISM_ARCH=$TARGETARCH \ PHOTOPRISM_AUTO_INDEX="300" \ PHOTOPRISM_AUTO_IMPORT="-1" \ PHOTOPRISM_INIT="https" \ - S6_KEEP_ENV=1 \ + S6_KEEP_ENV=0 \ + S6_VERBOSITY=0 \ S6_LOGGING=0 # Copy dist files, scripts, and debian backports sources list. @@ -132,6 +133,8 @@ RUN echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80retries && \ /photoprism/storage/config \ /photoprism/storage/cache && \ /scripts/install-s6.sh && \ + ln -sf /scripts/services/photoprism /etc/s6-overlay/s6-rc.d/photoprism && \ + touch /etc/s6-overlay/s6-rc.d/user/contents.d/photoprism && \ /scripts/cleanup.sh # Set default working directory. @@ -142,4 +145,3 @@ EXPOSE 2342 2443 # Set default entrypoint and command. ENTRYPOINT ["/init"] -CMD ["/scripts/cmd.sh", "/opt/photoprism/bin/photoprism", "start"] diff --git a/docker/photoprism/jammy/Dockerfile b/docker/photoprism/jammy/Dockerfile index 0af53b8d5..ca5b5fcea 100644 --- a/docker/photoprism/jammy/Dockerfile +++ b/docker/photoprism/jammy/Dockerfile @@ -101,6 +101,3 @@ EXPOSE 2342 2443 # Copy app files. COPY --from=build --chown=root:root --chmod=755 /opt/photoprism/ /opt/photoprism - -# Start app. -CMD ["/scripts/cmd.sh", "/opt/photoprism/bin/photoprism", "start"] \ No newline at end of file diff --git a/docker/photoprism/oracular/Dockerfile b/docker/photoprism/oracular/Dockerfile index 6df7f34c5..5a55af43c 100644 --- a/docker/photoprism/oracular/Dockerfile +++ b/docker/photoprism/oracular/Dockerfile @@ -101,6 +101,3 @@ EXPOSE 2342 2443 # Copy app files. COPY --from=build --chown=root:root --chmod=755 /opt/photoprism/ /opt/photoprism - -# Start app. -CMD ["/scripts/cmd.sh", "/opt/photoprism/bin/photoprism", "start"] \ No newline at end of file diff --git a/scripts/dist/install-s6.sh b/scripts/dist/install-s6.sh index 262f5fdad..c5fd0a2fa 100755 --- a/scripts/dist/install-s6.sh +++ b/scripts/dist/install-s6.sh @@ -83,4 +83,11 @@ curl -fsSL "$S6_NOARCH_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp echo "Extracting \"$S6_BINARY_URL\" to \"$S6_OVERLAY_DESTDIR\"..." curl -fsSL "$S6_BINARY_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp +S6_USER2_BUNDLE="${S6_OVERLAY_DESTDIR}etc/s6-overlay/s6-rc.d/user2" + +if [ -d "$S6_USER2_BUNDLE" ]; then + echo "Removing \"${S6_USER2_BUNDLE}\"..." + rm -rf "${S6_OVERLAY_DESTDIR}etc/s6-overlay/s6-rc.d/user2" +fi + echo "Done." diff --git a/scripts/dist/services/photoprism/dependencies.d/base b/scripts/dist/services/photoprism/dependencies.d/base new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/dist/services/photoprism/run b/scripts/dist/services/photoprism/run new file mode 100644 index 000000000..87dfc7bd9 --- /dev/null +++ b/scripts/dist/services/photoprism/run @@ -0,0 +1,3 @@ +#!/command/execlineb -P +with-contenv +/scripts/cmd.sh /opt/photoprism/bin/photoprism start diff --git a/scripts/dist/services/photoprism/type b/scripts/dist/services/photoprism/type new file mode 100644 index 000000000..1780f9f44 --- /dev/null +++ b/scripts/dist/services/photoprism/type @@ -0,0 +1 @@ +longrun \ No newline at end of file From 52cc9b1cd3d9e4a7aa29f214f9a60ff7c6f730a6 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 15:10:37 +0200 Subject: [PATCH 10/23] Docker: Update install-s6.sh script Signed-off-by: Michael Mayer --- scripts/dist/install-s6.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/dist/install-s6.sh b/scripts/dist/install-s6.sh index c5fd0a2fa..262f5fdad 100755 --- a/scripts/dist/install-s6.sh +++ b/scripts/dist/install-s6.sh @@ -83,11 +83,4 @@ curl -fsSL "$S6_NOARCH_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp echo "Extracting \"$S6_BINARY_URL\" to \"$S6_OVERLAY_DESTDIR\"..." curl -fsSL "$S6_BINARY_URL" | tar -C "${S6_OVERLAY_DESTDIR}" -Jxp -S6_USER2_BUNDLE="${S6_OVERLAY_DESTDIR}etc/s6-overlay/s6-rc.d/user2" - -if [ -d "$S6_USER2_BUNDLE" ]; then - echo "Removing \"${S6_USER2_BUNDLE}\"..." - rm -rf "${S6_OVERLAY_DESTDIR}etc/s6-overlay/s6-rc.d/user2" -fi - echo "Done." From 494f75917c6c6018abab8a4d119b9a5b3e1022ed Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 15:11:09 +0200 Subject: [PATCH 11/23] Develop: Update service configuration in compose.yaml files Signed-off-by: Michael Mayer --- Makefile | 4 ++-- compose.nvidia.yaml | 21 ++++----------------- compose.yaml | 4 +++- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 482df8a22..c62342c42 100644 --- a/Makefile +++ b/Makefile @@ -399,9 +399,9 @@ docker-build: $(DOCKER_COMPOSE) build --pull docker-nvidia: docker-nvidia-up docker-nvidia-up: - docker compose --profile=vision -f compose.nvidia.yaml up + docker compose --profile=qdrant -f compose.nvidia.yaml up docker-nvidia-build: - docker compose --profile=vision -f compose.nvidia.yaml build + docker compose --profile=qdrant -f compose.nvidia.yaml build docker-intel: docker-intel-up docker-intel-up: docker compose -f compose.intel.yaml up diff --git a/compose.nvidia.yaml b/compose.nvidia.yaml index 1706df1f3..e6d125138 100644 --- a/compose.nvidia.yaml +++ b/compose.nvidia.yaml @@ -146,27 +146,14 @@ services: extends: file: ./compose.yaml service: mariadb - photoprism-vision: - profiles: ["all", "vision"] - environment: - TF_CPP_MIN_LOG_LEVEL: 2 - NVIDIA_VISIBLE_DEVICES: "all" - NVIDIA_DRIVER_CAPABILITIES: "all" - deploy: - resources: - reservations: - devices: - - driver: "nvidia" - count: 1 - capabilities: [ gpu ] - extends: - file: ./compose.yaml - service: photoprism-vision qdrant: - profiles: ["all", "vision"] extends: file: ./compose.yaml service: qdrant + photoprism-vision: + extends: + file: ./compose.yaml + service: photoprism-vision traefik: extends: file: ./compose.yaml diff --git a/compose.yaml b/compose.yaml index 75a3c18da..08c73354d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -172,7 +172,7 @@ services: ## Web UI: https://qdrant.localssl.dev/dashboard qdrant: image: qdrant/qdrant:latest - profiles: ["all", "vision"] + profiles: ["all", "qdrant"] links: - "traefik:localssl.dev" - "traefik:app.localssl.dev" @@ -199,6 +199,8 @@ services: ## See: https://github.com/photoprism/photoprism-vision photoprism-vision: image: photoprism/vision:latest + entrypoint: [ "/app/venv/bin/flask" ] + command: [ "--app", "app", "run", "--debug", "--host", "0.0.0.0" ] profiles: ["all", "vision"] stop_grace_period: 5s working_dir: "/app" From 8068e471d1d770882952103eedee1b750fec4202 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 15:30:22 +0200 Subject: [PATCH 12/23] Develop: Update Makefile and compose.nvidia.yaml Signed-off-by: Michael Mayer --- Makefile | 2 ++ compose.nvidia.yaml | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c62342c42..a29ee5679 100644 --- a/Makefile +++ b/Makefile @@ -400,6 +400,8 @@ docker-build: docker-nvidia: docker-nvidia-up docker-nvidia-up: docker compose --profile=qdrant -f compose.nvidia.yaml up +docker-nvidia-down: + docker compose --profile=qdrant -f compose.nvidia.yaml down --remove-orphans docker-nvidia-build: docker compose --profile=qdrant -f compose.nvidia.yaml build docker-intel: docker-intel-up diff --git a/compose.nvidia.yaml b/compose.nvidia.yaml index e6d125138..af5033dc0 100644 --- a/compose.nvidia.yaml +++ b/compose.nvidia.yaml @@ -185,9 +185,3 @@ volumes: driver: local mariadb: driver: local - -## Create shared "photoprism-develop" network for connecting with services in other compose.yaml files -networks: - default: - name: photoprism - driver: bridge From 128df65f5486fdcee2511bb8dcda942aed4d4d38 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 15:30:37 +0200 Subject: [PATCH 13/23] Develop: Upgrade base image from 250407-oracular to 250412-oracular Signed-off-by: Michael Mayer --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 242e28851..d0621df1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Ubuntu 24.10 (Oracular Oriole) -FROM photoprism/develop:250407-oracular +FROM photoprism/develop:250412-oracular ## Alternative Environments: # FROM photoprism/develop:armv7 # ARMv7 (32bit) From 377c7ed1d585073165ab8b502e436f11f1e43971 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 17:53:46 +0200 Subject: [PATCH 14/23] Config: Change the order of options in settings.yml Signed-off-by: Michael Mayer --- internal/config/customize/album.go | 2 +- internal/config/customize/settings.go | 2 +- .../config/customize/testdata/settings.yml | 26 +++++++++---------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/config/customize/album.go b/internal/config/customize/album.go index a6e720fc1..0cb767d28 100644 --- a/internal/config/customize/album.go +++ b/internal/config/customize/album.go @@ -6,8 +6,8 @@ import ( // AlbumsSettings represents album defaults and preferences. type AlbumsSettings struct { - Order AlbumsOrder `json:"order" yaml:"Order"` Download DownloadSettings `json:"download" yaml:"Download"` + Order AlbumsOrder `json:"order" yaml:"Order"` } // AlbumsOrder represents default album sort orders. diff --git a/internal/config/customize/settings.go b/internal/config/customize/settings.go index 4297094c7..2af8171fc 100644 --- a/internal/config/customize/settings.go +++ b/internal/config/customize/settings.go @@ -19,7 +19,6 @@ const ( type Settings struct { UI UISettings `json:"ui" yaml:"UI"` Search SearchSettings `json:"search" yaml:"Search"` - Albums AlbumsSettings `json:"albums" yaml:"Albums"` Maps MapsSettings `json:"maps" yaml:"Maps"` Features FeatureSettings `json:"features" yaml:"Features"` Import ImportSettings `json:"import" yaml:"Import"` @@ -27,6 +26,7 @@ type Settings struct { Stack StackSettings `json:"stack" yaml:"Stack"` Share ShareSettings `json:"share" yaml:"Share"` Download DownloadSettings `json:"download" yaml:"Download"` + Albums AlbumsSettings `json:"albums" yaml:"Albums"` Templates TemplateSettings `json:"templates" yaml:"Templates"` } diff --git a/internal/config/customize/testdata/settings.yml b/internal/config/customize/testdata/settings.yml index f1ebea0ac..9c80dbcf0 100755 --- a/internal/config/customize/testdata/settings.yml +++ b/internal/config/customize/testdata/settings.yml @@ -10,19 +10,6 @@ Search: ListView: true ShowTitles: true ShowCaptions: true -Albums: - Order: - Album: oldest - Folder: added - Moment: oldest - State: newest - Month: oldest - Download: - Name: share - Disabled: false - Originals: true - MediaRaw: false - MediaSidecar: false Maps: Animate: 0 Style: "" @@ -75,5 +62,18 @@ Download: Originals: true MediaRaw: false MediaSidecar: false +Albums: + Download: + Name: share + Disabled: false + Originals: true + MediaRaw: false + MediaSidecar: false + Order: + Album: oldest + Folder: added + Moment: oldest + State: newest + Month: oldest Templates: Default: index.gohtml From 6f3ad2dea335d3b598c16470bc65ae57501ab24d Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Sat, 12 Apr 2025 19:03:01 +0200 Subject: [PATCH 15/23] UX: Replace v-snackbar component with div elements Signed-off-by: Michael Mayer --- frontend/src/app.vue | 3 +- frontend/src/component/defaults.js | 2 - frontend/src/component/notify.vue | 59 +++++++++++++++------------- frontend/src/css/notify.css | 62 ++++++++++++++++++++++-------- 4 files changed, 80 insertions(+), 46 deletions(-) diff --git a/frontend/src/app.vue b/frontend/src/app.vue index 223d90987..4979871e4 100644 --- a/frontend/src/app.vue +++ b/frontend/src/app.vue @@ -2,8 +2,6 @@
- - @@ -13,6 +11,7 @@ +
diff --git a/frontend/src/component/defaults.js b/frontend/src/component/defaults.js index fdb660c90..0835d0ab8 100644 --- a/frontend/src/component/defaults.js +++ b/frontend/src/component/defaults.js @@ -194,11 +194,9 @@ export default { attach: document.body, }, VOverlay: { - scrim: true, transition: false, openDelay: 0, closeDelay: 0, - attach: document.body, }, VExpansionPanel: { tile: true, diff --git a/frontend/src/component/notify.vue b/frontend/src/component/notify.vue index 16391db66..0c717c416 100644 --- a/frontend/src/component/notify.vue +++ b/frontend/src/component/notify.vue @@ -1,31 +1,36 @@