Replace django-background-tasks with huey (#657)

* Replace django-background-tasks with huey

* Add command for migrating tasks

* Add custom admin view

* fix dockerfile

* fix tests

* fix tests in CI

* fix task tests

* implement retries

* improve config

* workaround to avoid running singlefile tasks in parallel

* properly kill single-file sub-processes

* use period task for HTML snapshots

* clear task lock when starting task consumer

* remove obsolete cleanup task command

bookmarks/services/singlefile.py

@@ -1,6 +1,8 @@
 import gzip
+import logging
 import os
 import shutil
+import signal
 import subprocess
 
 from django.conf import settings
@@ -10,16 +12,21 @@ class SingeFileError(Exception):
     pass
 
 
+logger = logging.getLogger(__name__)
+
+
 def create_snapshot(url: str, filepath: str):
     singlefile_path = settings.LD_SINGLEFILE_PATH
-    singlefile_options = settings.LD_SINGLEFILE_OPTIONS
+    # singlefile_options = settings.LD_SINGLEFILE_OPTIONS
     temp_filepath = filepath + ".tmp"
 
+    args = [singlefile_path, url, temp_filepath]
     try:
-        command = f"{singlefile_path} '{url}' {singlefile_options} {temp_filepath}"
-        subprocess.run(command, check=True, shell=True)
+        # Use start_new_session=True to create a new process group
+        process = subprocess.Popen(args, start_new_session=True)
+        process.wait(timeout=60)
 
-        # single-file doesn't return exit codes apparently, so check if the file was created
+        # check if the file was created
         if not os.path.exists(temp_filepath):
             raise SingeFileError("Failed to create snapshot")
 
@@ -29,5 +36,20 @@ def create_snapshot(url: str, filepath: str):
             shutil.copyfileobj(raw_file, gz_file)
 
         os.remove(temp_filepath)
+    except subprocess.TimeoutExpired:
+        # First try to terminate properly
+        try:
+            logger.error(
+                "Timeout expired while creating snapshot. Terminating process..."
+            )
+            process.terminate()
+            process.wait(timeout=20)
+            raise SingeFileError("Timeout expired while creating snapshot")
+        except subprocess.TimeoutExpired:
+            # Kill the whole process group, which should also clean up any chromium
+            # processes spawned by single-file
+            logger.error("Timeout expired while terminating. Killing process...")
+            os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+            raise SingeFileError("Timeout expired while creating snapshot")
     except subprocess.CalledProcessError as error:
         raise SingeFileError(f"Failed to create snapshot: {error.stderr}")