Convert: Gate RAW render rejection by file format #5673

Apply the RawTherapee stderr rejection (raw.DecoderErrors) only for a
gated set of formats (default .cr3) instead of every RAW format.
RawTherapee prints "Cannot use camera white balance" for any sensor it
cannot read, including bodies whose default-WB render is fine, so for
formats it alone can decode (e.g. .raw, .kdc) the warning-based discard
left nothing to index. CR3 is unchanged: a magenta render warns and
falls back to the embedded preview.

internal/photoprism/convert_image_jpeg.go

@@ -68,14 +68,22 @@ func (w *Convert) JpegConvertCmds(f *MediaFile, jpegName string, xmpName string)
 				result = append(result, NewConvertCmd(cmd))
 			}
 
-			// Render the RAW with RawTherapee when Darktable is unavailable or fails. Output is
-			// rejected on an untrustworthy decode (raw.DecoderErrors) so the embedded preview wins.
+			// Render the RAW with RawTherapee when Darktable is unavailable or fails. For formats in the
+			// discard set (e.g. CR3) the output is rejected on an untrustworthy decode (raw.DecoderErrors)
+			// so the embedded preview wins; other RAW formats (e.g. .raw/.kdc) keep the render because
+			// RawTherapee may be their only working decoder.
 			if w.conf.RawTherapeeEnabled() && w.rawTherapeeExclude.Allow(fileExt) {
 				profile := filepath.Join(conf.AssetsPath(), "profiles", "raw.pp3")
 
-				result = append(result, NewConvertCmd(
+				rtCmd := NewConvertCmd(
 					raw.TherapeeCmd(w.conf.RawTherapeeBin(), f.FileName(), jpegName, profile, int(w.conf.JpegQuality())),
-				).WithStderrRejection(raw.DecoderErrors...))
+				)
+
+				if raw.DiscardRenderOnWarning(fileExt) {
+					rtCmd = rtCmd.WithStderrRejection(raw.DecoderErrors...)
+				}
+
+				result = append(result, rtCmd)
 			}
 		}
 

internal/photoprism/convert_image_test.go

@@ -390,7 +390,70 @@ func TestConvert_JpegConvertCmds_RawEmbeddedPreview(t *testing.T) {
 	if cnf.RawTherapeeEnabled() {
 		assert.GreaterOrEqual(t, rawTherapee, 0, "expected a RawTherapee command")
 		assert.Less(t, rawTherapee, jpgFromRaw, "RawTherapee must be tried before the embedded preview")
-		assert.Contains(t, rawTherapeeCmd.RejectStderr, raw.WhiteBalanceError, "RawTherapee output must be rejected on a white-balance failure")
+		assert.Empty(t, rawTherapeeCmd.RejectStderr, "a non-gated RAW format (.dng) must keep its render, so no stderr rejection is attached")
+	}
+}
+
+// TestConvert_JpegConvertCmds_DiscardRenderGate verifies that the RawTherapee stderr rejection is
+// attached only for formats in the discard set (raw.DiscardRenderOnWarning, e.g. CR3), so a magenta
+// CR3 falls back to its embedded preview, while formats RawTherapee alone can decode (e.g. .raw/.kdc)
+// keep their render.
+func TestConvert_JpegConvertCmds_DiscardRenderGate(t *testing.T) {
+	cnf := config.TestConfig()
+
+	if !cnf.RawTherapeeEnabled() {
+		t.Skip("RawTherapee must be available for the discard-render gate")
+	}
+
+	convert := NewConvert(cnf)
+	dir := t.TempDir()
+
+	cases := []struct {
+		name, ext string
+		gated     bool
+	}{
+		{"Cr3", ".cr3", true},
+		{"Raw", ".raw", false},
+		{"Kdc", ".kdc", false},
+		{"Cr2", ".cr2", false},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			rawFile := filepath.Join(dir, "sample"+tc.ext)
+			if err := os.WriteFile(rawFile, []byte("raw"), fs.ModeFile); err != nil {
+				t.Fatal(err)
+			}
+
+			mediaFile, err := NewMediaFile(rawFile)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.True(t, mediaFile.IsRaw(), "%s must be recognized as RAW", tc.ext)
+
+			cmds, _, err := convert.JpegConvertCmds(mediaFile, rawFile+".jpg", "")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var rtCmd *ConvertCmd
+			for _, cmd := range cmds {
+				if filepath.Base(cmd.Cmd.Path) == filepath.Base(cnf.RawTherapeeBin()) {
+					rtCmd = cmd
+					break
+				}
+			}
+
+			if rtCmd == nil {
+				t.Fatalf("expected a RawTherapee command for %s", tc.ext)
+			}
+
+			if tc.gated {
+				assert.Contains(t, rtCmd.RejectStderr, raw.WhiteBalanceError, "%s is gated, so its render must be rejected on a decode warning", tc.ext)
+			} else {
+				assert.Empty(t, rtCmd.RejectStderr, "%s is not gated, so its render must be kept", tc.ext)
+			}
+		})
 	}
 }
 

internal/raw/README.md

@@ -22,7 +22,7 @@ For a RAW input, `JpegConvertCmds` appends commands in this order, and the conve
 2. **RawTherapee** — full RAW developer (when Darktable is unavailable or fails).
 3. **Embedded camera preview** via ExifTool (largest first) — the last resort.
 
-The embedded preview is last because a full render is higher quality, but it has correct colors when the RAW developers cannot identify the sensor (e.g. very recent Canon CR3 bodies otherwise come out magenta), so it wins whenever the developers are unavailable or untrustworthy.
+The embedded preview is last because a full render is higher quality, but it has correct colors when the RAW developers cannot identify the sensor (e.g. very recent Canon CR3 bodies otherwise come out magenta), so it wins when the developers are unavailable or — for the gated formats described below — produce an untrustworthy render.
 
 ### Conversion Gating (`--disable-raw`)
 
@@ -36,11 +36,17 @@ The embedded preview is last because a full render is higher quality, but it has
 The three tools fail differently, so they are guarded differently:
 
 - **Darktable** signals an unsupported sensor with a **non-zero exit code** and writes no file (its diagnostics go to stdout, e.g. `[libraw_open] detected unsupported image`). The convert loop's exit-code check skips it, so no stderr inspection is needed.
-- **RawTherapee** can **exit 0 yet produce wrong colors** (e.g. a default white balance for an unidentified sensor) while printing a dcraw-derived message to stderr. Its command is therefore tagged with `ConvertCmd.WithStderrRejection(raw.DecoderErrors...)`; a match discards the output and the loop falls through to the embedded preview. `WhiteBalanceError` is the canonical case; `DecoderErrors` adds the other untrustworthy-decode messages (sourced from `dcraw.c`).
+- **RawTherapee** can **exit 0 yet produce wrong colors** (e.g. a default white balance for an unidentified sensor) while printing a dcraw-derived message to stderr. For formats in the discard set (`DiscardRenderOnWarning`, default `.cr3`) its command is tagged with `ConvertCmd.WithStderrRejection(raw.DecoderErrors...)`; a match discards the output and the loop falls through to the embedded preview. `WhiteBalanceError` is the canonical case; `DecoderErrors` adds the other untrustworthy-decode messages (sourced from `dcraw.c`). Other RAW formats keep the render — see *Discard-on-Warning Gate* below.
 - **Embedded previews** can be header-valid yet fail the thumbnailer (a bogus Huffman table only surfaces during shrink-on-load), so they are tagged `WithImageVerification`, which runs `thumb.Verify` to force the decode before acceptance.
 
 Because of this, `StderrRejected` is RawTherapee-specific by design — Darktable does not need it (it fails via exit code, not via accepted-but-wrong output).
 
+### Discard-on-Warning Gate
+
+`DiscardRenderOnWarning(ext)` decides, per RAW format, whether a RawTherapee render is discarded when its stderr matches `DecoderErrors`. It defaults to Canon CR3 (`.cr3`): recent CR3 bodies, which neither RAW developer can demosaic, render magenta, but they reliably embed a good preview to fall back to. The white-balance warning alone cannot tell a magenta render from a fine one — RawTherapee emits it for any sensor whose coefficients it cannot read, including obscure bodies whose default-WB render is perfectly fine — so the rejection is gated on the format as well as the warning.
+
+Formats RawTherapee alone can decode (e.g. `.raw`, `.kdc`) are deliberately left out: they have no embedded preview to fall back to, so discarding the render would leave nothing to index. The set is overridable via `PHOTOPRISM_RAW_DISCARD_ON_WARNING_EXT` (a comma-separated list that **replaces** the default), an escape hatch — outside the CLI/config surface — for adding a newly found magenta-prone format without a rebuild. A gated format must allow embedded-preview extraction (`PreviewExtAllowed`); otherwise its render would be discarded with no fallback, so PhotoPrism logs a warning at startup if an override names such a format.
+
 ### Preview-Unsafe Formats
 
 `previewUnsafeExt` lists RAW extensions whose embedded JPEG preview is known to be unusable, exposed via `PreviewExtAllowed(ext)`. Currently `.mos` (Leaf), whose preview is a bogus-Huffman JPEG that passes a MIME sniff but fails thumbnailing; skipping it forces a full RAW render.

internal/raw/discard.go

@@ -0,0 +1,77 @@
+package raw
+
+import (
+	"os"
+	"sort"
+	"strings"
+)
+
+// discardOnWarningEnv overrides the RAW extensions whose RawTherapee render is discarded on a decode
+// warning, as a comma-separated list that replaces the default set (e.g. "cr3, cr2"). It is an escape
+// hatch outside the CLI/config surface for adding a newly found magenta-prone format without a rebuild.
+const discardOnWarningEnv = "PHOTOPRISM_RAW_DISCARD_ON_WARNING_EXT"
+
+// discardRenderOnWarning holds the RAW extensions (lowercase, leading dot) whose RawTherapee render is
+// discarded when its stderr matches DecoderErrors, so the embedded preview wins (see README.md).
+// A gated format must allow preview extraction (PreviewExtAllowed), or its render has no fallback.
+var discardRenderOnWarning = defaultDiscardExt()
+
+// init warns when a configured gated format has no usable embedded preview, since a discarded render
+// would then have no fallback and the file would fail to index.
+func init() {
+	for _, ext := range discardExtMissingPreview() {
+		log.Warnf("raw: discard-on-warning format %s has no embedded preview to fall back to", ext)
+	}
+}
+
+// defaultDiscardExt returns the gated-extension set, reading discardOnWarningEnv when set and falling
+// back to Canon CR3 otherwise.
+func defaultDiscardExt() map[string]bool {
+	if env := strings.TrimSpace(os.Getenv(discardOnWarningEnv)); env != "" {
+		return parseDiscardExt(env)
+	}
+
+	return parseDiscardExt(".cr3")
+}
+
+// parseDiscardExt parses a comma-separated extension list into a normalized set (lowercase, leading
+// dot). An override that yields no valid entry returns an empty set, disabling the gate entirely.
+func parseDiscardExt(s string) map[string]bool {
+	set := make(map[string]bool)
+
+	for _, item := range strings.Split(s, ",") {
+		ext := strings.ToLower(strings.TrimSpace(item))
+		if ext == "" {
+			continue
+		}
+		if !strings.HasPrefix(ext, ".") {
+			ext = "." + ext
+		}
+		set[ext] = true
+	}
+
+	return set
+}
+
+// discardExtMissingPreview returns the gated extensions that have no usable embedded preview, sorted, so
+// a discarded render would have no fallback. The default set always returns none.
+func discardExtMissingPreview() []string {
+	var missing []string
+
+	for ext := range discardRenderOnWarning {
+		if !PreviewExtAllowed(ext) {
+			missing = append(missing, ext)
+		}
+	}
+
+	sort.Strings(missing)
+
+	return missing
+}
+
+// DiscardRenderOnWarning reports whether a RawTherapee render of the extension should be discarded when
+// its stderr reports a decode warning (DecoderErrors), preferring the embedded preview. The extension
+// must be lowercase with a leading dot (e.g. ".cr3").
+func DiscardRenderOnWarning(ext string) bool {
+	return discardRenderOnWarning[ext]
+}

internal/raw/discard_test.go

@@ -0,0 +1,101 @@
+package raw
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDiscardRenderOnWarning(t *testing.T) {
+	// Pin the gated set so the predicate is independent of any process-level env override.
+	orig := discardRenderOnWarning
+	discardRenderOnWarning = parseDiscardExt(".cr3")
+	t.Cleanup(func() { discardRenderOnWarning = orig })
+
+	t.Run("Gated", func(t *testing.T) {
+		assert.True(t, DiscardRenderOnWarning(".cr3"))
+	})
+	t.Run("NotGated", func(t *testing.T) {
+		assert.False(t, DiscardRenderOnWarning(".raw"))
+		assert.False(t, DiscardRenderOnWarning(".kdc"))
+		assert.False(t, DiscardRenderOnWarning(".cr2"))
+		assert.False(t, DiscardRenderOnWarning(".dng"))
+	})
+	t.Run("Empty", func(t *testing.T) {
+		assert.False(t, DiscardRenderOnWarning(""))
+	})
+}
+
+func TestDefaultDiscardExt(t *testing.T) {
+	t.Run("Default", func(t *testing.T) {
+		set := defaultDiscardExt()
+		assert.True(t, set[".cr3"])
+		assert.Len(t, set, 1)
+	})
+	t.Run("EnvOverrideReplaces", func(t *testing.T) {
+		// The override replaces the default set rather than extending it.
+		t.Setenv(discardOnWarningEnv, "cr2, kdc")
+		set := defaultDiscardExt()
+		assert.True(t, set[".cr2"])
+		assert.True(t, set[".kdc"])
+		assert.False(t, set[".cr3"])
+	})
+	t.Run("EnvBlankFallsBack", func(t *testing.T) {
+		t.Setenv(discardOnWarningEnv, "   ")
+		set := defaultDiscardExt()
+		assert.True(t, set[".cr3"])
+		assert.Len(t, set, 1)
+	})
+	t.Run("EnvDisablesGate", func(t *testing.T) {
+		// A comma-only value yields no valid extension and disables the gate.
+		t.Setenv(discardOnWarningEnv, ",")
+		assert.Empty(t, defaultDiscardExt())
+	})
+}
+
+func TestParseDiscardExt(t *testing.T) {
+	t.Run("AddsLeadingDot", func(t *testing.T) {
+		set := parseDiscardExt("cr3")
+		assert.True(t, set[".cr3"])
+	})
+	t.Run("KeepsLeadingDot", func(t *testing.T) {
+		set := parseDiscardExt(".cr3")
+		assert.True(t, set[".cr3"])
+	})
+	t.Run("TrimsAndLowercases", func(t *testing.T) {
+		set := parseDiscardExt("  .CR3 , KDC ")
+		assert.True(t, set[".cr3"])
+		assert.True(t, set[".kdc"])
+		assert.Len(t, set, 2)
+	})
+	t.Run("SkipsBlankEntries", func(t *testing.T) {
+		set := parseDiscardExt("cr3, , ,cr2")
+		assert.True(t, set[".cr3"])
+		assert.True(t, set[".cr2"])
+		assert.Len(t, set, 2)
+	})
+	t.Run("EmptyInput", func(t *testing.T) {
+		assert.Empty(t, parseDiscardExt(""))
+	})
+}
+
+func TestDiscardExtMissingPreview(t *testing.T) {
+	orig := discardRenderOnWarning
+	t.Cleanup(func() { discardRenderOnWarning = orig })
+
+	t.Run("ShippedDefaultIsSafe", func(t *testing.T) {
+		// The default gated set must never include a preview-unsafe format.
+		for ext := range defaultDiscardExt() {
+			assert.Truef(t, PreviewExtAllowed(ext), "default gated format %s must allow preview extraction", ext)
+		}
+	})
+	t.Run("DefaultClean", func(t *testing.T) {
+		discardRenderOnWarning = parseDiscardExt(".cr3")
+		assert.Empty(t, discardExtMissingPreview())
+	})
+	t.Run("DetectsPreviewUnsafe", func(t *testing.T) {
+		// A preview-unsafe override (.mos) would discard the render with no fallback.
+		discardRenderOnWarning = parseDiscardExt(".cr3, .mos")
+		assert.Equal(t, []string{".mos"}, discardExtMissingPreview())
+	})
+}

internal/raw/raw.go

@@ -1,7 +1,8 @@
 /*
 Package raw provides command builders and helpers for converting camera RAW
 images to JPEG with Darktable, RawTherapee, and ExifTool, including the
-heuristics for detecting an untrustworthy decode and skipping unusable previews.
+heuristics that decide, per format, when to discard an untrustworthy decode and
+when to skip an unusable embedded preview.
 
 Copyright (c) 2018 - 2026 PhotoPrism UG. All rights reserved.
 
@@ -25,3 +26,9 @@ Additional information can be found in our Developer Guide:
 <https://docs.photoprism.app/developer-guide/>
 */
 package raw
+
+import (
+	"github.com/photoprism/photoprism/internal/event"
+)
+
+var log = event.Log