Replace django-background-tasks with huey (#657)

* Replace django-background-tasks with huey

* Add command for migrating tasks

* Add custom admin view

* fix dockerfile

* fix tests

* fix tests in CI

* fix task tests

* implement retries

* improve config

* workaround to avoid running singlefile tasks in parallel

* properly kill single-file sub-processes

* use period task for HTML snapshots

* clear task lock when starting task consumer

* remove obsolete cleanup task command
This commit is contained in:
Sascha Ißbrücker
2024-04-07 11:11:14 +02:00
committed by GitHub
parent 68c163d943
commit a6f35119cd
22 changed files with 566 additions and 403 deletions

View File

@@ -1,7 +1,7 @@
import gzip
import os
from unittest import mock
import subprocess
from unittest import mock
from django.test import TestCase
@@ -24,9 +24,11 @@ class SingleFileServiceTestCase(TestCase):
file.write(self.html_content)
def test_create_snapshot(self):
with mock.patch("subprocess.run") as mock_run:
mock_run.side_effect = self.create_test_file
mock_process = mock.Mock()
mock_process.wait.return_value = 0
self.create_test_file()
with mock.patch("subprocess.Popen", return_value=mock_process):
singlefile.create_snapshot("http://example.com", self.html_filepath)
self.assertTrue(os.path.exists(self.html_filepath))
@@ -38,13 +40,13 @@ class SingleFileServiceTestCase(TestCase):
def test_create_snapshot_failure(self):
# subprocess fails - which it probably doesn't as single-file doesn't return exit codes
with mock.patch("subprocess.run") as mock_run:
mock_run.side_effect = subprocess.CalledProcessError(1, "command")
with mock.patch("subprocess.Popen") as mock_popen:
mock_popen.side_effect = subprocess.CalledProcessError(1, "command")
with self.assertRaises(singlefile.SingeFileError):
singlefile.create_snapshot("http://example.com", self.html_filepath)
# so also check that it raises error if output file isn't created
with mock.patch("subprocess.run") as mock_run:
with mock.patch("subprocess.Popen"):
with self.assertRaises(singlefile.SingeFileError):
singlefile.create_snapshot("http://example.com", self.html_filepath)