From 2b62928b4296648929ee7a95553fe3572983182d Mon Sep 17 00:00:00 2001 From: SergeantPanda Date: Sat, 18 Jul 2026 17:33:02 +0000 Subject: [PATCH] feat(settings): implement Redis caching for CoreSettings and enhance network access handling This update introduces Redis caching for grouped settings in the CoreSettings model, improving performance by reducing database queries. A new method, `get_network_access_settings`, is added to streamline access to network settings. Additionally, the `setting_flag_enabled` function is introduced to handle boolean settings more robustly. The cache is invalidated on save and delete operations, ensuring data consistency. Tests have been added to validate the caching behavior and the new settings functionality. --- core/developer_notifications.py | 4 +- core/models.py | 89 ++++++++++++++++++++++- core/signals.py | 56 ++++++++------- core/tests/test_core.py | 121 +++++++++++++++++++++++++++++++- core/utils.py | 8 +-- dispatcharr/utils.py | 7 +- 6 files changed, 245 insertions(+), 40 deletions(-) diff --git a/core/developer_notifications.py b/core/developer_notifications.py index f4214b43..a121d655 100644 --- a/core/developer_notifications.py +++ b/core/developer_notifications.py @@ -82,10 +82,10 @@ def check_network_access_is_default(user, endpoint: str = 'M3U_EPG') -> bool: Returns: True if the notification should show (insecure settings detected) """ - from core.models import CoreSettings, NETWORK_ACCESS_KEY + from core.models import CoreSettings try: - network_settings = CoreSettings._get_group(NETWORK_ACCESS_KEY, {}) + network_settings = CoreSettings.get_network_access_settings() # Empty settings are secure (defaults to local network only) if not network_settings: diff --git a/core/models.py b/core/models.py index ad9677c6..762c8529 100644 --- a/core/models.py +++ b/core/models.py @@ -196,6 +196,34 @@ SYSTEM_SETTINGS_KEY = "system_settings" EPG_SETTINGS_KEY = "epg_settings" USER_LIMITS_SETTINGS_KEY = "user_limit_settings" +# Redis cache for CoreSettings JSON groups. Primary invalidation is post_save / +# post_delete; TTL is a safety net if a writer bypasses signals. +_GROUP_CACHE_PREFIX = "coresettings:group:" +_GROUP_CACHE_TTL_SECONDS = 300 + + +def setting_flag_enabled(value, *, default=True): + """Parse a settings/JSON flag that should default to *default* when unset. + + Explicit ``False`` / ``0`` / common false strings disable. Explicit + ``True`` / ``1`` / common true strings enable. Other types fall back to + *default* so garbage payloads do not accidentally enable a kill-switch. + """ + if value is None: + return default + if value is False or value == 0: + return False + if value is True or value == 1: + return True + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ("false", "0", "no", "off", ""): + return False + if lowered in ("true", "1", "yes", "on"): + return True + return default + return default + class CoreSettings(models.Model): key = models.CharField( @@ -213,14 +241,55 @@ class CoreSettings(models.Model): def __str__(self): return "Core Settings" + @classmethod + def group_cache_key(cls, key): + return f"{_GROUP_CACHE_PREFIX}{key}" + + @classmethod + def invalidate_group_cache(cls, key): + """Drop the cached JSON for a settings group (all workers share Redis).""" + from django.core.cache import cache + + cache.delete(cls.group_cache_key(key)) + if key == PROXY_SETTINGS_KEY: + # Proxy workers also keep a short process-local copy. + try: + from apps.proxy.config import BaseConfig + + BaseConfig.clear_proxy_settings_cache() + except Exception: + pass + # Helper methods to get/set grouped settings @classmethod def _get_group(cls, key, defaults=None): - """Get a settings group, returning defaults if not found.""" + """Get a settings group, returning defaults if not found. + + Results are cached in Redis so hot paths (proxy, XC, catchup) do not + hit Postgres on every client request. Mutations go through ``save`` / + ``_update_group``, which invalidate via CoreSettings post_save / + post_delete signals. + """ + import copy + + from django.core.cache import cache + + defaults = defaults or {} + cache_key = cls.group_cache_key(key) + cached = cache.get(cache_key) + if isinstance(cached, dict): + return copy.deepcopy(cached) + try: - return cls.objects.get(key=key).value or (defaults or {}) + value = cls.objects.get(key=key).value or defaults + if not isinstance(value, dict): + value = defaults except cls.DoesNotExist: - return defaults or {} + value = defaults + + value = copy.deepcopy(value) + cache.set(cache_key, value, timeout=_GROUP_CACHE_TTL_SECONDS) + return copy.deepcopy(value) @classmethod def _update_group(cls, key, name, updates): @@ -399,6 +468,11 @@ class CoreSettings(models.Model): "new_client_behind_seconds": 5, }) + @classmethod + def get_network_access_settings(cls): + """CIDR allowlists per endpoint type (UI, STREAMS, XC_API, M3U_EPG, ...).""" + return cls._get_group(NETWORK_ACCESS_KEY, {}) + # System Settings @classmethod def get_system_settings(cls): @@ -409,8 +483,17 @@ class CoreSettings(models.Model): "preferred_region": None, "auto_import_mapped_files": True, "enable_ip_lookup": True, + "catchup_enabled": True, }) + @classmethod + def get_catchup_enabled(cls): + """Whether catch-up / timeshift is enabled system-wide (default True).""" + return setting_flag_enabled( + cls.get_system_settings().get("catchup_enabled"), + default=True, + ) + @classmethod def get_system_time_zone(cls): return cls.get_system_settings().get("time_zone") or getattr(settings, "TIME_ZONE", "UTC") or "UTC" diff --git a/core/signals.py b/core/signals.py index bbb73dd9..25c5ed71 100644 --- a/core/signals.py +++ b/core/signals.py @@ -1,4 +1,4 @@ -from django.db.models.signals import pre_delete, post_save +from django.db.models.signals import pre_delete, post_delete, post_save from django.dispatch import receiver from django.core.exceptions import ValidationError from .models import StreamProfile, CoreSettings, NETWORK_ACCESS_KEY @@ -8,32 +8,40 @@ def prevent_deletion_if_locked(sender, instance, **kwargs): if instance.locked: raise ValidationError("This profile is locked and cannot be deleted.") +@receiver(post_save, sender=CoreSettings) +@receiver(post_delete, sender=CoreSettings) +def handle_coresettings_cache_invalidation(sender, instance, **kwargs): + """Drop Redis group cache whenever a CoreSettings row is saved or deleted.""" + CoreSettings.invalidate_group_cache(instance.key) + @receiver(post_save, sender=CoreSettings) def handle_network_access_update(sender, instance, **kwargs): - """Invalidate cache and sync notifications when network access settings change.""" - if instance.key == NETWORK_ACCESS_KEY: - from django.core.cache import cache - from core.developer_notifications import sync_developer_notifications - import logging + """Sync developer notifications when network access settings change.""" + if instance.key != NETWORK_ACCESS_KEY: + return - logger = logging.getLogger(__name__) + from django.core.cache import cache + from core.developer_notifications import sync_developer_notifications + import logging - # Invalidate all notification condition caches + logger = logging.getLogger(__name__) + + # Invalidate all notification condition caches + try: + cache.delete_pattern('dev_notif_condition_*') + logger.info("Invalidated notification condition cache due to network access settings update") + except Exception as e: + logger.warning(f"Failed to delete cache pattern: {e}") + # Fallback: try to clear entire cache (if delete_pattern not supported) try: - cache.delete_pattern('dev_notif_condition_*') - logger.info("Invalidated notification condition cache due to network access settings update") - except Exception as e: - logger.warning(f"Failed to delete cache pattern: {e}") - # Fallback: try to clear entire cache (if delete_pattern not supported) - try: - cache.clear() - except Exception: - pass + cache.clear() + except Exception: + pass - # Re-sync developer notifications to re-evaluate conditions - # (websocket notification is sent by sync_developer_notifications) - try: - sync_developer_notifications() - logger.info("Re-synced developer notifications after network access settings update") - except Exception as e: - logger.error(f"Failed to sync developer notifications: {e}") + # Re-sync developer notifications to re-evaluate conditions + # (websocket notification is sent by sync_developer_notifications) + try: + sync_developer_notifications() + logger.info("Re-synced developer notifications after network access settings update") + except Exception as e: + logger.error(f"Failed to sync developer notifications: {e}") diff --git a/core/tests/test_core.py b/core/tests/test_core.py index 2af52f07..d045512e 100644 --- a/core/tests/test_core.py +++ b/core/tests/test_core.py @@ -1,9 +1,128 @@ from unittest.mock import patch, MagicMock +from django.core.cache import cache from django.test import TestCase, SimpleTestCase from apps.epg.models import EPGSource, EPGSourceIndex -from core.models import CoreSettings, DVR_SETTINGS_KEY, EPG_SETTINGS_KEY +from core.models import ( + CoreSettings, + DVR_SETTINGS_KEY, + EPG_SETTINGS_KEY, + SYSTEM_SETTINGS_KEY, + setting_flag_enabled, +) + + +class SettingFlagEnabledTests(SimpleTestCase): + def test_defaults_when_missing(self): + self.assertTrue(setting_flag_enabled(None, default=True)) + self.assertFalse(setting_flag_enabled(None, default=False)) + + def test_bool_and_int(self): + self.assertFalse(setting_flag_enabled(False)) + self.assertTrue(setting_flag_enabled(True)) + self.assertFalse(setting_flag_enabled(0)) + self.assertTrue(setting_flag_enabled(1)) + + def test_common_strings(self): + self.assertFalse(setting_flag_enabled("false")) + self.assertFalse(setting_flag_enabled("0")) + self.assertFalse(setting_flag_enabled("off")) + self.assertTrue(setting_flag_enabled("true")) + self.assertTrue(setting_flag_enabled("1")) + self.assertTrue(setting_flag_enabled("yes")) + + def test_garbage_falls_back_to_default(self): + self.assertTrue(setting_flag_enabled("maybe", default=True)) + self.assertFalse(setting_flag_enabled([], default=False)) + + +class CoreSettingsGroupCacheTests(TestCase): + """_get_group Redis cache: hit after first read, invalidate on save.""" + + def setUp(self): + cache.clear() + CoreSettings.objects.filter(key=SYSTEM_SETTINGS_KEY).delete() + + def test_second_read_does_not_query_database(self): + CoreSettings.objects.create( + key=SYSTEM_SETTINGS_KEY, + name="System Settings", + value={"catchup_enabled": False}, + ) + self.assertFalse(CoreSettings.get_catchup_enabled()) + + with self.assertNumQueries(0): + self.assertFalse(CoreSettings.get_catchup_enabled()) + + def test_save_invalidates_cache(self): + obj = CoreSettings.objects.create( + key=SYSTEM_SETTINGS_KEY, + name="System Settings", + value={"catchup_enabled": True}, + ) + self.assertTrue(CoreSettings.get_catchup_enabled()) + + obj.value = {"catchup_enabled": False} + obj.save() + self.assertFalse(CoreSettings.get_catchup_enabled()) + + def test_delete_invalidates_cache(self): + obj = CoreSettings.objects.create( + key=SYSTEM_SETTINGS_KEY, + name="System Settings", + value={"catchup_enabled": False}, + ) + self.assertFalse(CoreSettings.get_catchup_enabled()) + + obj.delete() + # Row gone: defaults apply (catchup enabled) + self.assertTrue(CoreSettings.get_catchup_enabled()) + + def test_nested_mutation_does_not_poison_cache(self): + from core.models import DVR_SETTINGS_KEY + + obj, _ = CoreSettings.objects.get_or_create( + key=DVR_SETTINGS_KEY, + defaults={"name": "DVR Settings", "value": {}}, + ) + obj.value = {**(obj.value if isinstance(obj.value, dict) else {}), "series_rules": [{"tvg_id": "a"}]} + obj.save() + CoreSettings.invalidate_group_cache(DVR_SETTINGS_KEY) + + rules = CoreSettings.get_dvr_settings()["series_rules"] + rules.append({"tvg_id": "mutated"}) + + again = CoreSettings.get_dvr_settings()["series_rules"] + self.assertEqual(len(again), 1) + self.assertEqual(again[0]["tvg_id"], "a") + + @patch("apps.proxy.config.BaseConfig.clear_proxy_settings_cache") + def test_invalidate_clears_proxy_process_cache(self, clear_mock): + from core.models import PROXY_SETTINGS_KEY + + CoreSettings.invalidate_group_cache(PROXY_SETTINGS_KEY) + clear_mock.assert_called_once_with() + + def test_network_access_allowed_uses_cached_settings(self): + from django.test import RequestFactory + + from core.models import NETWORK_ACCESS_KEY + from dispatcharr.utils import network_access_allowed + + CoreSettings.objects.update_or_create( + key=NETWORK_ACCESS_KEY, + defaults={ + "name": "Network Access", + "value": {"STREAMS": "0.0.0.0/0,::/0"}, + }, + ) + request = RequestFactory().get("/") + request.META["REMOTE_ADDR"] = "1.2.3.4" + + self.assertTrue(network_access_allowed(request, "STREAMS")) + with self.assertNumQueries(0): + self.assertTrue(network_access_allowed(request, "STREAMS")) class DispatcharrUserAgentTests(TestCase): diff --git a/core/utils.py b/core/utils.py index 4ffa90fa..5ab643f4 100644 --- a/core/utils.py +++ b/core/utils.py @@ -876,11 +876,9 @@ def log_system_event(event_type, channel_id=None, channel_name=None, **details): # Get max events from settings (default 100) try: from .models import CoreSettings - system_settings = CoreSettings.objects.filter(key='system_settings').first() - if system_settings and isinstance(system_settings.value, dict): - max_events = int(system_settings.value.get('max_system_events', 100)) - else: - max_events = 100 + max_events = int( + CoreSettings.get_system_settings().get("max_system_events", 100) or 100 + ) except Exception: max_events = 100 diff --git a/dispatcharr/utils.py b/dispatcharr/utils.py index c31e56e0..0e93baaf 100644 --- a/dispatcharr/utils.py +++ b/dispatcharr/utils.py @@ -3,7 +3,7 @@ import json import ipaddress from django.http import JsonResponse from django.core.exceptions import ValidationError -from core.models import CoreSettings, NETWORK_ACCESS_KEY +from core.models import CoreSettings def json_error_response(message, status=400): @@ -33,10 +33,7 @@ def get_client_ip(request): def network_access_allowed(request, settings_key, user=None): - try: - network_access = CoreSettings.objects.get(key=NETWORK_ACCESS_KEY).value - except CoreSettings.DoesNotExist: - network_access = {} + network_access = CoreSettings.get_network_access_settings() local_cidrs = ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7", "fe80::/10"] # Set defaults based on endpoint type if settings_key == "M3U_EPG":