marcel/linkding
1
0
Fork 0
mirror of https://github.com/sissbruecker/linkding.git synced 2025-08-10 04:07:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Bump waybackpy to 3.0.6 (#281)

Browse Source 
* fix wayback

* fix tests

* Reuse user agent from website loader

Co-authored-by: Sascha Ißbrücker <sascha.issbruecker@gmail.com>
This commit is contained in:
Dustin Blackman
2022-07-03 00:26:16 -04:00
committed by GitHub
parent 75c0429973
commit b53bd9f112
5 changed files with 18 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bookmarks/services/tasks.py
View File

@@ -8,6 +8,7 @@ from django.contrib.auth.models import User
from waybackpy.exceptions import WaybackError
from bookmarks.models import Bookmark, UserProfile
from bookmarks.services.website_loader import DEFAULT_USER_AGENT
logger = logging.getLogger(__name__)
@@ -38,10 +39,10 @@ def _create_web_archive_snapshot_task(bookmark_id: int, force_update: bool):
logger.debug(f'Create web archive link for bookmark: {bookmark}...')
wayback = waybackpy.Url(bookmark.url)
archive = waybackpy.WaybackMachineSaveAPI(bookmark.url, DEFAULT_USER_AGENT)
try:
archive = wayback.save()
archive.save()
except WaybackError as error:
logger.exception(f'Error creating web archive link for bookmark: {bookmark}...', exc_info=error)
raise

5
bookmarks/services/website_loader.py
View File

@@ -45,11 +45,14 @@ def load_page(url: str):
return str(results.best())
DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'
def fake_request_headers():
return {
"Accept": "text/html,application/xhtml+xml,application/xml",
"Accept-Encoding": "gzip, deflate",
"Dnt": "1",
"Upgrade-Insecure-Requests": "1",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36",
"User-Agent": DEFAULT_USER_AGENT,
}

20
bookmarks/tests/test_bookmarks_tasks.py
View File

@@ -10,15 +10,13 @@ from bookmarks.services import tasks
from bookmarks.tests.helpers import BookmarkFactoryMixin, disable_logging
class MockWaybackUrl:
class MockWaybackMachineSaveAPI:
def __init__(self, archive_url: str):
self.archive_url = archive_url
def save(self):
return self
class MockWaybackUrlWithSaveError:
def save(self):
raise NotImplementedError
@@ -52,7 +50,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
def test_create_web_archive_snapshot_should_update_snapshot_url(self):
bookmark = self.setup_bookmark()
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://example.com')):
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://example.com')):
tasks.create_web_archive_snapshot(self.get_or_create_test_user(), bookmark, False)
self.run_pending_task(tasks._create_web_archive_snapshot_task)
bookmark.refresh_from_db()
@@ -60,7 +58,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
self.assertEqual(bookmark.web_archive_snapshot_url, 'https://example.com')
def test_create_web_archive_snapshot_should_handle_missing_bookmark_id(self):
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://example.com')) as mock_wayback_url:
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://example.com')) as mock_wayback_url:
tasks._create_web_archive_snapshot_task(123, False)
self.run_pending_task(tasks._create_web_archive_snapshot_task)
@@ -69,7 +67,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
def test_create_web_archive_snapshot_should_handle_wayback_save_error(self):
bookmark = self.setup_bookmark()
with patch.object(waybackpy, 'Url',
with patch.object(waybackpy, 'WaybackMachineSaveAPI',
return_value=MockWaybackUrlWithSaveError()):
with self.assertRaises(NotImplementedError):
tasks.create_web_archive_snapshot(self.get_or_create_test_user(), bookmark, False)
@@ -78,7 +76,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
def test_create_web_archive_snapshot_should_skip_if_snapshot_exists(self):
bookmark = self.setup_bookmark(web_archive_snapshot_url='https://example.com')
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://other.com')):
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://other.com')):
tasks.create_web_archive_snapshot(self.get_or_create_test_user(), bookmark, False)
self.run_pending_task(tasks._create_web_archive_snapshot_task)
bookmark.refresh_from_db()
@@ -88,7 +86,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
def test_create_web_archive_snapshot_should_force_update_snapshot(self):
bookmark = self.setup_bookmark(web_archive_snapshot_url='https://example.com')
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://other.com')):
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://other.com')):
tasks.create_web_archive_snapshot(self.get_or_create_test_user(), bookmark, True)
self.run_pending_task(tasks._create_web_archive_snapshot_task)
bookmark.refresh_from_db()
@@ -117,7 +115,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
self.setup_bookmark()
self.setup_bookmark()
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://example.com')):
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://example.com')):
tasks.schedule_bookmarks_without_snapshots(user)
self.run_pending_task(tasks._schedule_bookmarks_without_snapshots_task)
self.run_all_pending_tasks(tasks._create_web_archive_snapshot_task)
@@ -131,7 +129,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
self.setup_bookmark(web_archive_snapshot_url='https://example.com')
self.setup_bookmark(web_archive_snapshot_url='https://example.com')
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://other.com')):
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://other.com')):
tasks.schedule_bookmarks_without_snapshots(user)
self.run_pending_task(tasks._schedule_bookmarks_without_snapshots_task)
self.run_all_pending_tasks(tasks._create_web_archive_snapshot_task)
@@ -149,7 +147,7 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
self.setup_bookmark(user=other_user)
self.setup_bookmark(user=other_user)
with patch.object(waybackpy, 'Url', return_value=MockWaybackUrl('https://example.com')):
with patch.object(waybackpy, 'WaybackMachineSaveAPI', return_value=MockWaybackMachineSaveAPI('https://example.com')):
tasks.schedule_bookmarks_without_snapshots(user)
self.run_pending_task(tasks._schedule_bookmarks_without_snapshots_task)
self.run_all_pending_tasks(tasks._create_web_archive_snapshot_task)

2
requirements.prod.txt
View File

@@ -22,4 +22,4 @@ supervisor==4.2.2
typing-extensions==3.10.0.0
urllib3==1.26.6
uWSGI==2.0.18
waybackpy==2.4.3
waybackpy==3.0.6

2
requirements.txt
View File

@@ -28,4 +28,4 @@ soupsieve==1.9.2
sqlparse==0.4.2
typing-extensions==3.10.0.0
urllib3==1.26.6
waybackpy==2.4.3
waybackpy==3.0.6