From 429b01b569f742748bac664c6890f1c63bb984f6 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 11 Aug 2025 16:28:23 +0100 Subject: [PATCH 1/2] prevent memory issues by implementing a chunked extractor --- apps/epg/tasks.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 4fcf5706..452478a2 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -28,6 +28,8 @@ from core.utils import acquire_task_lock, release_task_lock, send_websocket_upda logger = logging.getLogger(__name__) +MAX_EXTRACT_CHUNK_SIZE = 65536 # 64kb (base2) + def send_epg_update(source_id, action, progress, **kwargs): """Send WebSocket update about EPG download/parsing progress""" @@ -641,7 +643,11 @@ def extract_compressed_file(file_path, output_path=None, delete_original=False): # Reset file pointer and extract the content gz_file.seek(0) with open(extracted_path, 'wb') as out_file: - out_file.write(gz_file.read()) + while True: + chunk = gz_file.read(MAX_EXTRACT_CHUNK_SIZE) + if not chunk: + break + out_file.write(chunk) except Exception as e: logger.error(f"Error extracting GZIP file: {e}", exc_info=True) return None @@ -685,9 +691,13 @@ def extract_compressed_file(file_path, output_path=None, delete_original=False): return None # Extract the first XML file - xml_content = zip_file.read(xml_files[0]) with open(extracted_path, 'wb') as out_file: - out_file.write(xml_content) + with zip_file.open(xml_files[0], "r") as xml_file: + while True: + chunk = xml_file.read(MAX_EXTRACT_CHUNK_SIZE) + if not chunk: + break + out_file.write(chunk) logger.info(f"Successfully extracted zip file to: {extracted_path}") From 72fee02ec47f675666f5a58684bda5da2a6c8dc5 Mon Sep 17 00:00:00 2001 From: Jordan Date: Tue, 12 Aug 2025 11:26:50 +0100 Subject: [PATCH 2/2] ensure chunk is either null or empty to exit loop --- apps/epg/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/epg/tasks.py b/apps/epg/tasks.py index 452478a2..087d9fba 100644 --- a/apps/epg/tasks.py +++ b/apps/epg/tasks.py @@ -645,7 +645,7 @@ def extract_compressed_file(file_path, output_path=None, delete_original=False): with open(extracted_path, 'wb') as out_file: while True: chunk = gz_file.read(MAX_EXTRACT_CHUNK_SIZE) - if not chunk: + if not chunk or len(chunk) == 0: break out_file.write(chunk) except Exception as e: @@ -695,7 +695,7 @@ def extract_compressed_file(file_path, output_path=None, delete_original=False): with zip_file.open(xml_files[0], "r") as xml_file: while True: chunk = xml_file.read(MAX_EXTRACT_CHUNK_SIZE) - if not chunk: + if not chunk or len(chunk) == 0: break out_file.write(chunk)