API: Disable gzip compression for /api/v1/dl endpoint #127 #1090

Required for compatibility with the Python HTTP client. In addition,
this commit refactors function names and adds tests.

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/ai/face/embedding.go

@@ -62,7 +62,7 @@ func (m Embedding) Magnitude() float64 {
 	return m.Dist(NullEmbedding)
 }
 
-// JSON returns the face embedding as JSON bytes.
+// JSON returns the face embedding as JSON-encoded bytes.
 func (m Embedding) JSON() []byte {
 	var noResult = []byte("")
 

internal/ai/face/embeddings.go

@@ -109,7 +109,7 @@ func (embeddings Embeddings) Dist(other Embedding) (dist float64) {
 	return dist
 }
 
-// JSON returns the embeddings as JSON bytes.
+// JSON returns the embeddings as JSON-encoded bytes.
 func (embeddings Embeddings) JSON() []byte {
 	var noResult = []byte("")
 

internal/ai/vision/api_client.go

@@ -59,7 +59,7 @@ func PerformApiRequest(apiRequest *ApiRequest, uri, method, key string) (apiResp
 		return apiResponse, errors.New("api request is nil")
 	}
 
-	data, jsonErr := apiRequest.MarshalJSON()
+	data, jsonErr := apiRequest.JSON()
 
 	if jsonErr != nil {
 		return apiResponse, jsonErr

internal/ai/vision/api_client_test.go

@@ -22,7 +22,7 @@ func TestNewApiRequest(t *testing.T) {
 		// t.Logf("request: %#v", result)
 
 		if result != nil {
-			json, jsonErr := result.MarshalJSON()
+			json, jsonErr := result.JSON()
 			assert.NoError(t, jsonErr)
 			assert.NotEmpty(t, json)
 			// t.Logf("json: %s", json)
@@ -36,7 +36,7 @@ func TestNewApiRequest(t *testing.T) {
 		assert.NotNil(t, result)
 		// t.Logf("request: %#v", result)
 		if result != nil {
-			json, jsonErr := result.MarshalJSON()
+			json, jsonErr := result.JSON()
 			assert.NoError(t, jsonErr)
 			assert.NotEmpty(t, json)
 			t.Logf("json: %s", json)

internal/ai/vision/api_request.go

@@ -25,7 +25,7 @@ func (r *ApiRequest) GetId() string {
 	return r.Id
 }
 
-// MarshalJSON returns request as JSON.
-func (r *ApiRequest) MarshalJSON() ([]byte, error) {
+// JSON returns the request data as JSON-encoded bytes.
+func (r *ApiRequest) JSON() ([]byte, error) {
 	return json.Marshal(*r)
 }

internal/ai/vision/caption.go

@@ -65,7 +65,7 @@ func Caption(imgName string, src media.Src) (result CaptionResult, err error) {
 				Url:   imgUrl,
 			}
 
-			/* if json, _ := apiRequest.MarshalJSON(); len(json) > 0 {
+			/* if json, _ := apiRequest.JSON(); len(json) > 0 {
 				log.Debugf("request: %s", json)
 			} */
 

internal/api/vision_face_test.go

@@ -28,7 +28,7 @@ func TestPostVisionFace(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -71,7 +71,7 @@ func TestPostVisionFace(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -107,7 +107,7 @@ func TestPostVisionFace(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -148,7 +148,7 @@ func TestPostVisionFace(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)

internal/api/vision_labels_test.go

@@ -28,7 +28,7 @@ func TestPostVisionLabels(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -65,7 +65,7 @@ func TestPostVisionLabels(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -99,7 +99,7 @@ func TestPostVisionLabels(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)

internal/api/vision_nsfw_test.go

@@ -28,7 +28,7 @@ func TestPostVisionNsfw(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -80,7 +80,7 @@ func TestPostVisionNsfw(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)
@@ -114,7 +114,7 @@ func TestPostVisionNsfw(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		jsonReq, jsonErr := req.MarshalJSON()
+		jsonReq, jsonErr := req.JSON()
 
 		if jsonErr != nil {
 			t.Fatal(err)

internal/server/start.go

@@ -74,6 +74,7 @@ func Start(ctx context.Context, conf *config.Config) {
 				conf.BaseUri("/health"),
 				conf.BaseUri(config.ApiUri + "/t"),
 				conf.BaseUri(config.ApiUri + "/folders/t"),
+				conf.BaseUri(config.ApiUri + "/dl"),
 				conf.BaseUri(config.ApiUri + "/zip"),
 				conf.BaseUri(config.ApiUri + "/albums"),
 				conf.BaseUri(config.ApiUri + "/labels"),

internal/workers/vision.go

@@ -154,7 +154,7 @@ func (w *Vision) Start(q string, models []string, force bool) (err error) {
 					if (entity.SrcPriority[caption.Source] > entity.SrcPriority[m.CaptionSrc]) || !m.HasCaption() {
 						m.SetCaption(caption.Text, caption.Source)
 						changed = true
-						log.Infof("vision: changed caption of %s to %t", photoName, clean.Log(m.PhotoCaption))
+						log.Infof("vision: changed caption of %s to %s", photoName, clean.Log(m.PhotoCaption))
 					}
 				}
 			}

pkg/clean/content_type_test.go

@@ -23,7 +23,7 @@ func TestContentType(t *testing.T) {
 		result := ContentType("invalid")
 		assert.Equal(t, "invalid", result)
 	})
-	t.Run("Json", func(t *testing.T) {
+	t.Run("JSON", func(t *testing.T) {
 		result := ContentType("text/json")
 		assert.Equal(t, "application/json; charset=utf-8", result)
 	})

pkg/fs/fs.go

@@ -89,6 +89,23 @@ func FileExistsNotEmpty(fileName string) bool {
 	return err == nil && !info.IsDir() && info.Size() > 0
 }
 
+// FileSize returns the size of a file in bytes or -1 in case of an error.
+func FileSize(fileName string) int64 {
+	if fileName == "" {
+		return -1
+	}
+
+	info, err := os.Stat(fileName)
+
+	if err != nil || info == nil {
+		return -1
+	} else if info.IsDir() {
+		return -1
+	}
+
+	return info.Size()
+}
+
 // PathExists tests if a path exists, and is a directory or symlink.
 func PathExists(path string) bool {
 	if path == "" {

pkg/fs/fs_test.go

@@ -37,6 +37,14 @@ func TestFileExistsNotEmpty(t *testing.T) {
 	assert.False(t, FileExistsNotEmpty(""))
 }
 
+func TestFileSize(t *testing.T) {
+	assert.Equal(t, 10990, int(FileSize("./testdata/test.jpg")))
+	assert.Equal(t, 10990, int(FileSize("./testdata/test.jpg")))
+	assert.Equal(t, 0, int(FileSize("./testdata/empty.jpg")))
+	assert.Equal(t, -1, int(FileSize("./foo.jpg")))
+	assert.Equal(t, -1, int(FileSize("")))
+}
+
 func TestPathExists(t *testing.T) {
 	assert.True(t, PathExists("./testdata"))
 	assert.False(t, PathExists("./testdata/test.jpg"))

pkg/media/base64.go

@@ -5,19 +5,46 @@ import (
 	"io"
 )
 
-// EncodeBase64 returns the base64 encoding of bin.
-func EncodeBase64(bin []byte) string {
+// EncodeBase64String returns the base64 encoding of bin.
+func EncodeBase64String(bin []byte) string {
 	return base64.StdEncoding.EncodeToString(bin)
 }
 
+// EncodedLenBase64 returns the length in bytes of the base64 encoding of an input buffer of length n.
+func EncodedLenBase64(decodedBytes int) int {
+	return base64.StdEncoding.EncodedLen(decodedBytes)
+}
+
+// DecodeBase64String returns the bytes represented by the base64 string s.
+// If the input is malformed, it returns the partially decoded data and
+// [CorruptInputError]. Newline characters (\r and \n) are ignored.
+func DecodeBase64String(s string) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s)
+}
+
 // ReadBase64 returns a new reader that decodes base64 and returns binary data.
 func ReadBase64(stream io.Reader) io.Reader {
 	return base64.NewDecoder(base64.StdEncoding, stream)
 }
 
-// DecodeBase64 returns the bytes represented by the base64 string s.
-// If the input is malformed, it returns the partially decoded data and
-// [CorruptInputError]. Newline characters (\r and \n) are ignored.
-func DecodeBase64(s string) ([]byte, error) {
-	return base64.StdEncoding.DecodeString(s)
+// EncodeBase64Bytes encodes src, writing EncodedLenBase64 bytes to dst.
+//
+// The encoding pads the output to a multiple of 4 bytes,
+// so Encode is not appropriate for use on individual blocks
+// of a large data stream.
+func EncodeBase64Bytes(dst, src []byte) {
+	base64.StdEncoding.Encode(dst, src)
+}
+
+// DecodedLenBase64 returns the maximum length in bytes of the decoded data
+// corresponding to n bytes of base64-encoded data.
+func DecodedLenBase64(encodedBytes int) int {
+	return base64.StdEncoding.DecodedLen(encodedBytes)
+}
+
+// DecodeBase64Bytes decodes src, writing at most DecodedLenBase64 bytes to dst.
+// If src contains invalid base64 data, it returns the number of bytes successfully
+// written. New line characters (\r and \n) are ignored.
+func DecodeBase64Bytes(dst, src []byte) (n int, err error) {
+	return base64.StdEncoding.Decode(dst, src)
 }

pkg/media/base64_test.go

@@ -1,15 +1,76 @@
 package media
 
 import (
+	"io"
+	"strings"
 	"testing"
 
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestBase64(t *testing.T) {
-	t.Run("Gopher", func(t *testing.T) {
-		data, err := DecodeBase64(gopher)
+	t.Run("DecodeString", func(t *testing.T) {
+		data, err := DecodeBase64String(gopher)
 		assert.NoError(t, err)
-		assert.Equal(t, gopher, EncodeBase64(data))
+
+		if mime := mimetype.Detect(data); mime == nil {
+			t.Fatal("mimetype image/png expected")
+		} else {
+			assert.Equal(t, "image/png", mime.String())
+		}
+	})
+	t.Run("DecodeString", func(t *testing.T) {
+		data, err := DecodeBase64String(gopher)
+		assert.NoError(t, err)
+		assert.Equal(t, gopher, EncodeBase64String(data))
+	})
+	t.Run("Read", func(t *testing.T) {
+		reader := ReadBase64(strings.NewReader(gopher))
+
+		if data, err := io.ReadAll(reader); err != nil {
+			t.Fatal(err)
+		} else if decodeData, decodeErr := DecodeBase64String(gopher); decodeErr != nil {
+			t.Fatal(decodeErr)
+		} else {
+			assert.Equal(t, data, decodeData)
+			assert.Equal(t, EncodeBase64String(data), gopher)
+		}
+	})
+	t.Run("DecodeBytes", func(t *testing.T) {
+		encoded := []byte(gopher)
+		encodedLen := len(encoded)
+		decodedLen := DecodedLenBase64(encodedLen)
+		binary := make([]byte, decodedLen)
+
+		if n, err := DecodeBase64Bytes(binary, encoded); err != nil {
+			t.Fatal(err)
+		} else {
+			assert.GreaterOrEqual(t, decodedLen, n)
+		}
+	})
+	t.Run("EncodeBytes", func(t *testing.T) {
+		encoded := []byte(gopher)
+		encodedLen := len(encoded)
+		decodedLen := DecodedLenBase64(encodedLen)
+		binary := make([]byte, decodedLen)
+
+		if n, err := DecodeBase64Bytes(binary, encoded); err != nil {
+			t.Fatal(err)
+		} else {
+			binary = binary[:n]
+			assert.GreaterOrEqual(t, decodedLen, n)
+		}
+
+		binaryEncodedLen := EncodedLenBase64(len(binary))
+		binaryEncoded := make([]byte, binaryEncodedLen)
+
+		EncodeBase64Bytes(binaryEncoded, binary)
+		assert.Equal(t, encoded, binaryEncoded)
+		assert.Equal(t, gopher, string(binaryEncoded))
+
+		data, err := DecodeBase64String(string(binaryEncoded))
+		assert.NoError(t, err)
+		assert.Equal(t, gopher, EncodeBase64String(data))
 	})
 }

pkg/media/data_url.go

@@ -42,7 +42,7 @@ func DataUrl(r io.Reader) string {
 	}
 
 	// Generate data URL.
-	return fmt.Sprintf("data:%s;base64,%s", mimeType, EncodeBase64(data))
+	return fmt.Sprintf("data:%s;base64,%s", mimeType, EncodeBase64String(data))
 }
 
 // ReadUrl reads binary data from a regular file path,
@@ -85,7 +85,7 @@ func ReadUrl(fileUrl string, schemes []string) (data []byte, err error) {
 		if _, binaryData, found := strings.Cut(u.Opaque, ";base64,"); !found || len(binaryData) == 0 {
 			return data, fmt.Errorf("invalid %s url", u.Scheme)
 		} else {
-			return DecodeBase64(binaryData)
+			return DecodeBase64String(binaryData)
 		}
 	case scheme.File:
 		if data, err = os.ReadFile(fileUrl); err != nil {

pkg/media/data_url_test.go

@@ -25,7 +25,7 @@ func TestReadUrl(t *testing.T) {
 		if data, err := ReadUrl(dataUrl, []string{"https", "data"}); err != nil {
 			t.Fatal(err)
 		} else {
-			expected, _ := DecodeBase64(gopher)
+			expected, _ := DecodeBase64String(gopher)
 			assert.Equal(t, expected, data)
 		}
 	})

pkg/media/http/header/content.go

@@ -9,6 +9,7 @@ const (
 	ContentType        = "Content-Type"
 	ContentDisposition = "Content-Disposition"
 	ContentEncoding    = "Content-Encoding"
+	ContentLength      = "Content-Length"
 	ContentRange       = "Content-Range"
 	Location           = "Location"
 	Origin             = "Origin"

pkg/media/http/header/content_test.go

@@ -16,6 +16,7 @@ func TestContent(t *testing.T) {
 		assert.Equal(t, "Content-Type", ContentType)
 		assert.Equal(t, "Content-Disposition", ContentDisposition)
 		assert.Equal(t, "Content-Encoding", ContentEncoding)
+		assert.Equal(t, "Content-Length", ContentLength)
 		assert.Equal(t, "Content-Range", ContentRange)
 		assert.Equal(t, "Location", Location)
 		assert.Equal(t, "Origin", Origin)

pkg/media/http/header/webhook.go

@@ -0,0 +1,8 @@
+package header
+
+const (
+	WebhookID           string = "webhook-id"
+	WebhookSignature    string = "webhook-signature"
+	WebhookTimestamp    string = "webhook-timestamp"
+	WebhookSecretPrefix string = "whsec_"
+)