diff --git a/CHANGELOG.md b/CHANGELOG.md index b57879d6..d63bab5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Schedules Direct poster proxy no longer ignores daily image download limits or missing-image errors.** SD returns codes 5002/5003 as HTTP 200 with a JSON body; the proxy previously only inspected HTTP 400 and kept requesting images after the limit, which can get accounts blocked. It now detects 5002/5003 (subscriber and trial), stops further image fetches for that source until the next midnight UTC (persisted on the EPG source so all workers honor it), and on explicit IMAGE_NOT_FOUND (code 5000) clears that program's `sd_icon` so the same dead URI is not retried. Transient bare HTTP 404s do not blacklist the URI. Successful SD posters use a dedicated nginx cache under `/data/cache/sd_posters` (14 days); channel/VOD logos remain at `/data/cache/logos` (24 hours). On startup, an existing `/data/logo_cache` directory is moved to `/data/cache/logos` once so warm logo entries are kept. API and XMLTV icon URLs include a `?v=` hash of the stored SD image URI so a new artwork link after refresh uses a new cache key without waiting for TTL expiry. +- **Schedules Direct poster proxy no longer ignores daily image download limits or missing-image errors.** SD returns codes 5002/5003 as HTTP 200 with a JSON body; the proxy previously only inspected HTTP 400 and kept requesting images after the limit, which can get accounts blocked. It now detects 5002/5003 (subscriber and trial), stops further image fetches for that source until the next midnight UTC (persisted on the EPG source so all workers honor it), and on explicit IMAGE_NOT_FOUND (code 5000) clears that program's `sd_icon` so the same dead URI is not retried. Transient bare HTTP 404s do not blacklist the URI. Successful SD posters use a dedicated nginx cache under `/data/cache/sd_posters` (14 days); channel/VOD logos remain at `/data/cache/logos` (24 hours). On startup, an existing `/data/logo_cache` directory is moved to `/data/cache/logos` once so warm logo entries are kept. API and XMLTV icon URLs include a `?v=` hash of the stored SD image URI so a new artwork link after refresh uses a new cache key without waiting for TTL expiry. Selecting programs that still need artwork no longer uses a negated JSON `AND` that dropped every row when `sd_icon_missing` / `sd_poster_style` keys were absent (Postgres NULL), which had prevented poster links from being saved after refresh. - **Schedules Direct auth and lineup change handling better match SD guidelines.** Token codes 3000/3001 (offline/busy) stop as idle with clear "do not retry" messaging instead of looking like credential failures; 4010 (too many unique IPs) and related account codes get explicit messages. Lineup add/delete respects a known daily change lockout before calling SD, handles 4100 on deletes, and the UI blocks remove when no changes remain (adds and deletes both count toward the 6/day limit). - **Schedules Direct stops retrying `/token` on auth failures that will not clear by themselves.** Codes 4002/4003 (bad credentials), 4001/4005/4007/4008 (account state), 4009/4010 (login/IP limits), and 4004 (15-minute account lock) persist a cooldown on the EPG source. Soft codes 3000/3001 use a 1-hour idle cooldown. Lockouts clear when username/password change or the cooldown expires. Token auth is centralized in `sd_obtain_token` for refresh, lineup/form, and poster paths. Redis-cached SD tokens are bound to a credential fingerprint (and cleared when the EPG source username/password change) so switching accounts cannot reuse another session. - **Schedules Direct code split out of general EPG modules.** Refresh pipeline and Celery tasks live in `apps/epg/sd_tasks.py`; lineup/poster API mixins in `apps/epg/sd_api.py`; shared protocol helpers remain in `sd_utils.py`. Progress WebSocket updates (`send_epg_update`) live in `apps/epg/utils.py` so XMLTV and SD tasks can import cleanly. `tasks.py` / `api_views.py` re-export or inherit so existing imports and Celery task names are unchanged. diff --git a/apps/epg/sd_tasks.py b/apps/epg/sd_tasks.py index feb37e6d..5356400b 100644 --- a/apps/epg/sd_tasks.py +++ b/apps/epg/sd_tasks.py @@ -263,6 +263,34 @@ def _sd_pick_poster_url(images, poster_style=SD_POSTER_STYLE_DEFAULT): return None +def _sd_program_ids_needing_posters(mapped_epg_ids, poster_style): + """ + Return program_id values that still need SD artwork for this style. + + Skips programs marked sd_icon_missing for the same poster_style (SD code 5000). + """ + needs_poster_q = ( + Q(custom_properties__isnull=True) + | ~Q(custom_properties__has_key='sd_icon') + | ~Q(custom_properties__sd_poster_style=poster_style) + ) + poster_qs = ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + ).filter(needs_poster_q) + skip_missing_ids = ProgramData.objects.filter( + epg_id__in=mapped_epg_ids, + program_id__isnull=False, + custom_properties__sd_icon_missing=True, + custom_properties__sd_poster_style=poster_style, + ).values_list('id', flat=True) + return set( + poster_qs.exclude(id__in=skip_missing_ids).values_list( + 'program_id', flat=True + ) + ) + + def _sd_fetch_lineup_country(token, sd_headers_fn): """Return country code prefix from the first subscribed lineup (poster metadata).""" try: @@ -587,29 +615,21 @@ def fetch_schedules_direct( return sd_headers_for_source(source, token=token, content_type=content_type) def _sd_post_refresh_tasks(mapped_epg_ids, program_metadata, today): - """Poster fetch, logo auto-apply, and pruning — runs even when schedules are unchanged.""" + """Poster fetch, logo auto-apply, and pruning. Runs even when schedules are unchanged. + + Returns the number of ProgramData rows updated with poster artwork. + """ from apps.epg.models import SDProgramMD5 + posters_updated = 0 fetch_posters = (source.custom_properties or {}).get('fetch_posters', False) poster_style = (source.custom_properties or {}).get('poster_style', SD_POSTER_STYLE_DEFAULT) poster_program_ids = set() if fetch_posters: - needs_poster_q = ( - Q(custom_properties__isnull=True) - | ~Q(custom_properties__has_key='sd_icon') - | ~Q(custom_properties__sd_poster_style=poster_style) - ) # Do not re-request artwork for URIs that SD already reported missing # (code 5000), unless the user changed poster_style (explicit retry). - needs_poster_q = needs_poster_q & ~( - Q(custom_properties__sd_icon_missing=True) - & Q(custom_properties__sd_poster_style=poster_style) - ) - poster_program_ids = set( - ProgramData.objects.filter( - epg_id__in=mapped_epg_ids, - program_id__isnull=False, - ).filter(needs_poster_q).values_list('program_id', flat=True) + poster_program_ids = _sd_program_ids_needing_posters( + mapped_epg_ids, poster_style ) if poster_program_ids: logger.info( @@ -694,7 +714,8 @@ def fetch_schedules_direct( ProgramData.objects.bulk_update( programs_to_update, ['custom_properties'], batch_size=1000 ) - logger.info(f"Updated {len(programs_to_update)} programs with poster artwork.") + posters_updated = len(programs_to_update) + logger.info(f"Updated {posters_updated} programs with poster artwork.") else: logger.info("No poster artwork matched committed programs.") else: @@ -746,6 +767,8 @@ def fetch_schedules_direct( except Exception as prune_err: logger.warning(f"Failed to prune stale SDProgramMD5 records: {prune_err}") + return posters_updated + # ------------------------------------------------------------------------- # Step 1: Authenticate and obtain session token # The SD API requires the password to be SHA1-hashed before transmission. @@ -1183,17 +1206,22 @@ def fetch_schedules_direct( if not changed_by_station: logger.info("No schedule changes detected, skipping schedule and program downloads.") - _sd_post_refresh_tasks(mapped_epg_ids, {}, today) + posters_updated = _sd_post_refresh_tasks(mapped_epg_ids, {}, today) + if posters_updated: + msg = ( + f"No schedule changes detected. Updated poster artwork for " + f"{posters_updated} program{'s' if posters_updated != 1 else ''}." + ) + else: + msg = "No schedule changes detected. Guide data is up to date." if lightweight_sd_fetch: - msg = "No schedule updates needed; guide data is up to date." source.last_message = msg source.save(update_fields=['last_message']) send_epg_update(source.id, "parsing_programs", 100, status="success", message=msg) return - send_epg_update(source.id, "parsing_programs", 100, status="success", - message="No schedule changes detected since last refresh. Guide data is up to date.") + send_epg_update(source.id, "parsing_programs", 100, status="success", message=msg) source.status = EPGSource.STATUS_SUCCESS - source.last_message = "No schedule changes detected. Guide data is up to date." + source.last_message = msg source.updated_at = timezone.now() source.save(update_fields=['status', 'last_message', 'updated_at']) return diff --git a/apps/epg/tests/test_schedules_direct.py b/apps/epg/tests/test_schedules_direct.py index f89ebfa2..195ee677 100644 --- a/apps/epg/tests/test_schedules_direct.py +++ b/apps/epg/tests/test_schedules_direct.py @@ -1539,6 +1539,65 @@ class SDSourceSignalTests(TestCase): # Poster selection tests # --------------------------------------------------------------------------- +class SDPosterNeedsQueryTests(TestCase): + """Programs needing artwork must not be dropped by JSON NULL NOT quirks.""" + + def setUp(self): + self.source = EPGSource.objects.create( + name='SD Poster Needs', + source_type='schedules_direct', + username='u', + password='p', + ) + self.epg = EPGData.objects.create( + tvg_id='station-needs', + name='Needs Station', + epg_source=self.source, + ) + self.now = timezone.now() + + def _prog(self, program_id, **cp): + return ProgramData.objects.create( + epg=self.epg, + title='Show', + start_time=self.now, + end_time=self.now + timedelta(hours=1), + program_id=program_id, + custom_properties=cp or {}, + ) + + def test_programs_without_sd_icon_are_selected(self): + from apps.epg.sd_tasks import _sd_program_ids_needing_posters + + self._prog('EP000000010001', date='2020', categories=['Drama']) + self._prog('EP000000010002', season=1, episode=2) + needed = _sd_program_ids_needing_posters({self.epg.id}, 'sd_recommended') + self.assertEqual(needed, {'EP000000010001', 'EP000000010002'}) + + def test_sd_icon_missing_for_same_style_is_skipped(self): + from apps.epg.sd_tasks import _sd_program_ids_needing_posters + + self._prog( + 'EP000000010003', + sd_icon_missing=True, + sd_poster_style='sd_recommended', + ) + self._prog('EP000000010004', categories=['News']) + needed = _sd_program_ids_needing_posters({self.epg.id}, 'sd_recommended') + self.assertEqual(needed, {'EP000000010004'}) + + def test_sd_icon_missing_retries_after_style_change(self): + from apps.epg.sd_tasks import _sd_program_ids_needing_posters + + self._prog( + 'EP000000010005', + sd_icon_missing=True, + sd_poster_style='portrait_iconic', + ) + needed = _sd_program_ids_needing_posters({self.epg.id}, 'sd_recommended') + self.assertEqual(needed, {'EP000000010005'}) + + class SDPosterSelectionTests(TestCase): """_sd_pick_poster_url must honour style preference with sensible fallbacks."""