Add option for showing bookmark favicons (#390)

* Implement favicon loader

* Implement load favicon task

* Show favicons in bookmark list

* Add missing migration

* Load missing favicons on import

* Automatically refresh favicons

* Add enable favicon setting

* Update uwsgi config to host favicons

* Improve settings wording

* Fix favicon loader test setup

* Document LD_FAVICON_PROVIDER setting

* Add refresh favicons button

bookmarks/services/bookmarks.py

@@ -30,6 +30,8 @@ def create_bookmark(bookmark: Bookmark, tag_string: str, current_user: User):
     bookmark.save()
     # Create snapshot on web archive
     tasks.create_web_archive_snapshot(current_user, bookmark, False)
+    # Load favicon
+    tasks.load_favicon(current_user, bookmark)
 
     return bookmark
 
@@ -43,6 +45,9 @@ def update_bookmark(bookmark: Bookmark, tag_string, current_user: User):
     # Update dates
     bookmark.date_modified = timezone.now()
     bookmark.save()
+    # Update favicon
+    tasks.load_favicon(current_user, bookmark)
+
     if has_url_changed:
         # Update web archive snapshot, if URL changed
         tasks.create_web_archive_snapshot(current_user, bookmark, True)

bookmarks/services/favicon_loader.py

@@ -0,0 +1,57 @@
+import os.path
+import re
+import shutil
+import time
+from pathlib import Path
+from urllib.parse import urlparse
+
+import requests
+from django.conf import settings
+
+max_file_age = 60 * 60 * 24  # 1 day
+
+
+def _ensure_favicon_folder():
+    Path(settings.LD_FAVICON_FOLDER).mkdir(parents=True, exist_ok=True)
+
+
+def _url_to_filename(url: str) -> str:
+    name = re.sub(r'\W+', '_', url)
+    return f'{name}.png'
+
+
+def _get_base_url(url: str) -> str:
+    parsed_uri = urlparse(url)
+    return f'{parsed_uri.scheme}://{parsed_uri.hostname}'
+
+
+def _get_favicon_path(favicon_file: str) -> Path:
+    return Path(os.path.join(settings.LD_FAVICON_FOLDER, favicon_file))
+
+
+def _is_stale(path: Path) -> bool:
+    stat = path.stat()
+    file_age = time.time() - stat.st_mtime
+    return file_age >= max_file_age
+
+
+def load_favicon(url: str) -> str:
+    # Get base URL so that we can reuse favicons for multiple bookmarks with the same host
+    base_url = _get_base_url(url)
+    favicon_name = _url_to_filename(base_url)
+    favicon_path = _get_favicon_path(favicon_name)
+
+    # Load icon if it doesn't exist yet or has become stale
+    if not favicon_path.exists() or _is_stale(favicon_path):
+        # Create favicon folder if not exists
+        _ensure_favicon_folder()
+        # Load favicon from provider, save to file
+        favicon_url = settings.LD_FAVICON_PROVIDER.format(url=base_url)
+        response = requests.get(favicon_url, stream=True)
+
+        with open(favicon_path, 'wb') as file:
+            shutil.copyfileobj(response.raw, file)
+
+        del response
+
+    return favicon_name

bookmarks/services/importer.py

@@ -74,6 +74,8 @@ def import_netscape_html(html: str, user: User):
 
     # Create snapshots for newly imported bookmarks
     tasks.schedule_bookmarks_without_snapshots(user)
+    # Load favicons for newly imported bookmarks
+    tasks.schedule_bookmarks_without_favicons(user)
 
     end = timezone.now()
     logger.debug(f'Import duration: {end - import_start}')

bookmarks/services/tasks.py

@@ -2,6 +2,7 @@ import logging
 
 import waybackpy
 from background_task import background
+from background_task.models import Task
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
@@ -10,6 +11,7 @@ from waybackpy.exceptions import WaybackError, TooManyRequestsError, NoCDXRecord
 import bookmarks.services.wayback
 from bookmarks.models import Bookmark, UserProfile
 from bookmarks.services.website_loader import DEFAULT_USER_AGENT
+from bookmarks.services import favicon_loader
 
 logger = logging.getLogger(__name__)
 
@@ -72,7 +74,8 @@ def _create_web_archive_snapshot_task(bookmark_id: int, force_update: bool):
         logger.error(
             f'Failed to create snapshot due to rate limiting, trying to load newest snapshot as fallback. url={bookmark.url}')
     except WaybackError as error:
-        logger.error(f'Failed to create snapshot, trying to load newest snapshot as fallback. url={bookmark.url}', exc_info=error)
+        logger.error(f'Failed to create snapshot, trying to load newest snapshot as fallback. url={bookmark.url}',
+                     exc_info=error)
 
     # Load the newest snapshot as fallback
     _load_newest_snapshot(bookmark)
@@ -105,3 +108,67 @@ def _schedule_bookmarks_without_snapshots_task(user_id: int):
         # To prevent rate limit errors from the Wayback API only try to load the latest snapshots instead of creating
         # new ones when processing bookmarks in bulk
         _load_web_archive_snapshot_task(bookmark.id)
+
+
+def is_favicon_feature_active(user: User) -> bool:
+    background_tasks_enabled = not settings.LD_DISABLE_BACKGROUND_TASKS
+
+    return background_tasks_enabled and user.profile.enable_favicons
+
+
+def load_favicon(user: User, bookmark: Bookmark):
+    if is_favicon_feature_active(user):
+        _load_favicon_task(bookmark.id)
+
+
+@background()
+def _load_favicon_task(bookmark_id: int):
+    try:
+        bookmark = Bookmark.objects.get(id=bookmark_id)
+    except Bookmark.DoesNotExist:
+        return
+
+    logger.info(f'Load favicon for bookmark. url={bookmark.url}')
+
+    new_favicon = favicon_loader.load_favicon(bookmark.url)
+
+    if new_favicon != bookmark.favicon_file:
+        bookmark.favicon_file = new_favicon
+        bookmark.save()
+        logger.info(f'Successfully updated favicon for bookmark. url={bookmark.url} icon={new_favicon}')
+
+
+def schedule_bookmarks_without_favicons(user: User):
+    if is_favicon_feature_active(user):
+        _schedule_bookmarks_without_favicons_task(user.id)
+
+
+@background()
+def _schedule_bookmarks_without_favicons_task(user_id: int):
+    user = get_user_model().objects.get(id=user_id)
+    bookmarks = Bookmark.objects.filter(favicon_file__exact='', owner=user)
+    tasks = []
+
+    for bookmark in bookmarks:
+        task = Task.objects.new_task(task_name='bookmarks.services.tasks._load_favicon_task', args=(bookmark.id,))
+        tasks.append(task)
+
+    Task.objects.bulk_create(tasks)
+
+
+def schedule_refresh_favicons(user: User):
+    if is_favicon_feature_active(user) and settings.LD_ENABLE_REFRESH_FAVICONS:
+        _schedule_refresh_favicons_task(user.id)
+
+
+@background()
+def _schedule_refresh_favicons_task(user_id: int):
+    user = get_user_model().objects.get(id=user_id)
+    bookmarks = Bookmark.objects.filter(owner=user)
+    tasks = []
+
+    for bookmark in bookmarks:
+        task = Task.objects.new_task(task_name='bookmarks.services.tasks._load_favicon_task', args=(bookmark.id,))
+        tasks.append(task)
+
+    Task.objects.bulk_create(tasks)