Generate fallback URLs for web archive links (#804)

* generate fallback web archive URL if none exists

* remove fallback web archive snapshot creation

* fix test

bookmarks/tests/test_bookmark_index_view.py

@@ -6,11 +6,7 @@ from django.test import TestCase
 from django.urls import reverse
 
 from bookmarks.models import Bookmark, BookmarkSearch, Tag, UserProfile
-from bookmarks.tests.helpers import (
-    BookmarkFactoryMixin,
-    HtmlTestMixin,
-    collapse_whitespace,
-)
+from bookmarks.tests.helpers import BookmarkFactoryMixin, HtmlTestMixin
 
 
 class BookmarkIndexViewTestCase(TestCase, BookmarkFactoryMixin, HtmlTestMixin):

bookmarks/tests/test_bookmarks_list_template.py

@@ -1,3 +1,4 @@
+import datetime
 from typing import Type
 
 from dateutil.relativedelta import relativedelta
@@ -36,15 +37,6 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
             html,
         )
 
-    def assertDateLabel(self, html: str, label_content: str):
-        self.assertInHTML(
-            f"""
-        <span>{label_content}</span>
-        <span>|</span>
-        """,
-            html,
-        )
-
     def assertWebArchiveLink(
         self, html: str, label_content: str, url: str, link_target: str = "_blank"
     ):
@@ -465,15 +457,6 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
         style = bookmark_list["style"]
         self.assertIn("--ld-bookmark-description-max-lines:3;", style)
 
-    def test_should_respect_absolute_date_setting(self):
-        bookmark = self.setup_date_format_test(
-            UserProfile.BOOKMARK_DATE_DISPLAY_ABSOLUTE
-        )
-        html = self.render_template()
-        formatted_date = formats.date_format(bookmark.date_added, "SHORT_DATE_FORMAT")
-
-        self.assertDateLabel(html, formatted_date)
-
     def test_should_render_web_archive_link_with_absolute_date_setting(self):
         bookmark = self.setup_date_format_test(
             UserProfile.BOOKMARK_DATE_DISPLAY_ABSOLUTE,
@@ -486,12 +469,6 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
             html, formatted_date, bookmark.web_archive_snapshot_url
         )
 
-    def test_should_respect_relative_date_setting(self):
-        self.setup_date_format_test(UserProfile.BOOKMARK_DATE_DISPLAY_RELATIVE)
-        html = self.render_template()
-
-        self.assertDateLabel(html, "1 week ago")
-
     def test_should_render_web_archive_link_with_relative_date_setting(self):
         bookmark = self.setup_date_format_test(
             UserProfile.BOOKMARK_DATE_DISPLAY_RELATIVE,
@@ -501,6 +478,27 @@ class BookmarkListTemplateTest(TestCase, BookmarkFactoryMixin, HtmlTestMixin):
 
         self.assertWebArchiveLink(html, "1 week ago", bookmark.web_archive_snapshot_url)
 
+    def test_should_render_generated_web_archive_link_without_saved_snapshot_url(self):
+        user = self.get_or_create_test_user()
+        user.profile.bookmark_date_display = UserProfile.BOOKMARK_DATE_DISPLAY_ABSOLUTE
+        user.profile.save()
+
+        date_added = timezone.datetime(
+            2023, 8, 11, 21, 45, 11, tzinfo=datetime.timezone.utc
+        )
+        bookmark = self.setup_bookmark(
+            url="https://example.com/article", added=date_added
+        )
+
+        html = self.render_template()
+        formatted_date = formats.date_format(bookmark.date_added, "SHORT_DATE_FORMAT")
+
+        self.assertWebArchiveLink(
+            html,
+            formatted_date,
+            "https://web.archive.org/web/20230811214511/https://example.com/article",
+        )
+
     def test_bookmark_link_target_should_be_blank_by_default(self):
         bookmark = self.setup_bookmark()
         html = self.render_template()

bookmarks/tests/test_bookmarks_tasks.py

@@ -1,6 +1,4 @@
-import datetime
 import os.path
-from dataclasses import dataclass
 from unittest import mock
 
 import waybackpy
@@ -10,8 +8,6 @@ from django.test import TestCase, override_settings
 from huey.contrib.djhuey import HUEY as huey
 from waybackpy.exceptions import WaybackError
 
-import bookmarks.services.favicon_loader
-import bookmarks.services.wayback
 from bookmarks.models import BookmarkAsset, UserProfile
 from bookmarks.services import tasks, singlefile
 from bookmarks.tests.helpers import BookmarkFactoryMixin
@@ -29,12 +25,6 @@ def create_wayback_machine_save_api_mock(
     return mock_api
 
 
-@dataclass
-class MockCdxSnapshot:
-    archive_url: str
-    datetime_timestamp: datetime.datetime
-
-
 class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
     def setUp(self):
@@ -50,17 +40,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
         )
         self.mock_save_api_patcher.start()
 
-        self.mock_cdx_api = mock.Mock()
-        self.mock_cdx_api.newest.return_value = MockCdxSnapshot(
-            "https://example.com/newest_snapshot", datetime.datetime.now()
-        )
-        self.mock_cdx_api_patcher = mock.patch.object(
-            bookmarks.services.wayback,
-            "CustomWaybackMachineCDXServerAPI",
-            return_value=self.mock_cdx_api,
-        )
-        self.mock_cdx_api_patcher.start()
-
         self.mock_load_favicon_patcher = mock.patch(
             "bookmarks.services.favicon_loader.load_favicon"
         )
@@ -90,7 +69,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
     def tearDown(self):
         self.mock_save_api_patcher.stop()
-        self.mock_cdx_api_patcher.stop()
         self.mock_load_favicon_patcher.stop()
         self.mock_singlefile_create_snapshot_patcher.stop()
         self.mock_load_preview_image_patcher.stop()
@@ -142,45 +120,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
         self.assertEqual(bookmark.web_archive_snapshot_url, "https://other.com")
 
-    def test_create_web_archive_snapshot_should_use_newest_snapshot_as_fallback(self):
-        bookmark = self.setup_bookmark()
-        self.mock_save_api.save.side_effect = WaybackError
-
-        tasks.create_web_archive_snapshot(
-            self.get_or_create_test_user(), bookmark, False
-        )
-
-        bookmark.refresh_from_db()
-        self.mock_cdx_api.newest.assert_called_once()
-        self.assertEqual(
-            "https://example.com/newest_snapshot",
-            bookmark.web_archive_snapshot_url,
-        )
-
-    def test_create_web_archive_snapshot_should_ignore_missing_newest_snapshot(self):
-        bookmark = self.setup_bookmark()
-        self.mock_save_api.save.side_effect = WaybackError
-        self.mock_cdx_api.newest.return_value = None
-
-        tasks.create_web_archive_snapshot(
-            self.get_or_create_test_user(), bookmark, False
-        )
-
-        bookmark.refresh_from_db()
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
-    def test_create_web_archive_snapshot_should_ignore_newest_snapshot_errors(self):
-        bookmark = self.setup_bookmark()
-        self.mock_save_api.save.side_effect = WaybackError
-        self.mock_cdx_api.newest.side_effect = WaybackError
-
-        tasks.create_web_archive_snapshot(
-            self.get_or_create_test_user(), bookmark, False
-        )
-
-        bookmark.refresh_from_db()
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
     def test_create_web_archive_snapshot_should_not_save_stale_bookmark_data(self):
         bookmark = self.setup_bookmark()
 
@@ -203,69 +142,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
             bookmark.web_archive_snapshot_url,
         )
 
-    def test_load_web_archive_snapshot_should_update_snapshot_url(self):
-        bookmark = self.setup_bookmark()
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        bookmark.refresh_from_db()
-
-        self.assertEqual(self.executed_count(), 1)
-        self.mock_cdx_api.newest.assert_called_once()
-        self.assertEqual(
-            "https://example.com/newest_snapshot", bookmark.web_archive_snapshot_url
-        )
-
-    def test_load_web_archive_snapshot_should_handle_missing_bookmark_id(self):
-        tasks._load_web_archive_snapshot_task(123)
-
-        self.assertEqual(self.executed_count(), 1)
-        self.mock_cdx_api.newest.assert_not_called()
-
-    def test_load_web_archive_snapshot_should_skip_if_snapshot_exists(self):
-        bookmark = self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        self.assertEqual(self.executed_count(), 1)
-        self.mock_cdx_api.newest.assert_not_called()
-
-    def test_load_web_archive_snapshot_should_handle_missing_snapshot(self):
-        bookmark = self.setup_bookmark()
-        self.mock_cdx_api.newest.return_value = None
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
-    def test_load_web_archive_snapshot_should_handle_wayback_errors(self):
-        bookmark = self.setup_bookmark()
-        self.mock_cdx_api.newest.side_effect = WaybackError
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-
-        self.assertEqual("", bookmark.web_archive_snapshot_url)
-
-    def test_load_web_archive_snapshot_should_not_save_stale_bookmark_data(self):
-        bookmark = self.setup_bookmark()
-
-        # update bookmark during API call to check that saving
-        # the snapshot does not overwrite updated bookmark data
-        def mock_newest_impl():
-            bookmark.title = "Updated title"
-            bookmark.save()
-            return mock.DEFAULT
-
-        self.mock_cdx_api.newest.side_effect = mock_newest_impl
-
-        tasks._load_web_archive_snapshot_task(bookmark.id)
-        bookmark.refresh_from_db()
-
-        self.assertEqual("Updated title", bookmark.title)
-        self.assertEqual(
-            "https://example.com/newest_snapshot", bookmark.web_archive_snapshot_url
-        )
-
     @override_settings(LD_DISABLE_BACKGROUND_TASKS=True)
     def test_create_web_archive_snapshot_should_not_run_when_background_tasks_are_disabled(
         self,
@@ -292,59 +168,6 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
 
         self.assertEqual(self.executed_count(), 0)
 
-    def test_schedule_bookmarks_without_snapshots_should_load_snapshot_for_all_bookmarks_without_snapshot(
-        self,
-    ):
-        user = self.get_or_create_test_user()
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-        self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-        self.setup_bookmark(web_archive_snapshot_url="https://example.com")
-
-        tasks.schedule_bookmarks_without_snapshots(user)
-
-        self.assertEqual(self.executed_count(), 4)
-        self.assertEqual(self.mock_cdx_api.newest.call_count, 3)
-
-    def test_schedule_bookmarks_without_snapshots_should_only_update_user_owned_bookmarks(
-        self,
-    ):
-        user = self.get_or_create_test_user()
-        other_user = User.objects.create_user(
-            "otheruser", "otheruser@example.com", "password123"
-        )
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark()
-        self.setup_bookmark(user=other_user)
-        self.setup_bookmark(user=other_user)
-        self.setup_bookmark(user=other_user)
-
-        tasks.schedule_bookmarks_without_snapshots(user)
-
-        self.assertEqual(self.mock_cdx_api.newest.call_count, 3)
-
-    @override_settings(LD_DISABLE_BACKGROUND_TASKS=True)
-    def test_schedule_bookmarks_without_snapshots_should_not_run_when_background_tasks_are_disabled(
-        self,
-    ):
-        tasks.schedule_bookmarks_without_snapshots(self.user)
-
-        self.assertEqual(self.executed_count(), 0)
-
-    def test_schedule_bookmarks_without_snapshots_should_not_run_when_web_archive_integration_is_disabled(
-        self,
-    ):
-        self.user.profile.web_archive_integration = (
-            UserProfile.WEB_ARCHIVE_INTEGRATION_DISABLED
-        )
-        self.user.profile.save()
-        tasks.schedule_bookmarks_without_snapshots(self.user)
-
-        self.assertEqual(self.executed_count(), 0)
-
     def test_load_favicon_should_create_favicon_file(self):
         bookmark = self.setup_bookmark()
 

bookmarks/tests/test_importer.py

@@ -444,17 +444,6 @@ class ImporterTestCase(TestCase, BookmarkFactoryMixin, ImportTestMixin):
         self.assertEqual(Bookmark.objects.all()[0].description, "Example description")
         self.assertEqual(Bookmark.objects.all()[0].notes, "Updated notes")
 
-    def test_schedule_snapshot_creation(self):
-        user = self.get_or_create_test_user()
-        test_html = self.render_html(tags_html="")
-
-        with patch.object(
-            tasks, "schedule_bookmarks_without_snapshots"
-        ) as mock_schedule_bookmarks_without_snapshots:
-            import_netscape_html(test_html, user)
-
-            mock_schedule_bookmarks_without_snapshots.assert_called_once_with(user)
-
     def test_schedule_favicon_loading(self):
         user = self.get_or_create_test_user()
         test_html = self.render_html(tags_html="")

bookmarks/tests/test_signals.py

@@ -1,17 +0,0 @@
-from unittest.mock import patch
-
-from django.test import TestCase
-
-from bookmarks.services import tasks
-from bookmarks.tests.helpers import BookmarkFactoryMixin
-
-
-class SignalsTestCase(TestCase, BookmarkFactoryMixin):
-    def test_login_should_schedule_snapshot_creation(self):
-        user = self.get_or_create_test_user()
-
-        with patch.object(
-            tasks, "schedule_bookmarks_without_snapshots"
-        ) as mock_schedule_bookmarks_without_snapshots:
-            self.client.force_login(user)
-            mock_schedule_bookmarks_without_snapshots.assert_called_once_with(user)