From 82067058765e2e785881cf1d4566875a80ce496d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sascha=20I=C3=9Fbr=C3=BCcker?= Date: Sun, 20 Aug 2023 11:44:53 +0200 Subject: [PATCH] Add support for PRIVATE flag in import and export (#505) * Add support for PRIVATE attribute in import * Add support for PRIVATE attribute in export * Update import sync tests --- bookmarks/services/exporter.py | 3 +- bookmarks/services/importer.py | 31 +++++++++--- bookmarks/services/parser.py | 7 ++- bookmarks/templates/settings/general.html | 14 ++++++ bookmarks/tests/helpers.py | 13 +++-- bookmarks/tests/test_exporter.py | 26 ++++++++++ bookmarks/tests/test_importer.py | 53 +++++++++++++++++--- bookmarks/tests/test_parser.py | 26 ++++++++++ bookmarks/tests/test_settings_import_view.py | 28 +++++++++++ bookmarks/views/settings.py | 3 +- 10 files changed, 184 insertions(+), 20 deletions(-) diff --git a/bookmarks/services/exporter.py b/bookmarks/services/exporter.py index 289aa67..fcc775d 100644 --- a/bookmarks/services/exporter.py +++ b/bookmarks/services/exporter.py @@ -33,9 +33,10 @@ def append_bookmark(doc: BookmarkDocument, bookmark: Bookmark): desc = html.escape(bookmark.resolved_description or '') tags = ','.join(bookmark.tag_names) toread = '1' if bookmark.unread else '0' + private = '0' if bookmark.shared else '1' added = int(bookmark.date_added.timestamp()) - doc.append(f'
{title}') + doc.append(f'
{title}') if desc: doc.append(f'
{desc}') diff --git a/bookmarks/services/importer.py b/bookmarks/services/importer.py index 588c86c..f130dfa 100644 --- a/bookmarks/services/importer.py +++ b/bookmarks/services/importer.py @@ -20,6 +20,11 @@ class ImportResult: failed: int = 0 +@dataclass +class ImportOptions: + map_private_flag: bool = False + + class TagCache: def __init__(self, user: User): self.user = user @@ -50,7 +55,7 @@ class TagCache: self.cache[tag.name.lower()] = tag -def import_netscape_html(html: str, user: User): +def import_netscape_html(html: str, user: User, options: ImportOptions = ImportOptions()) -> ImportResult: result = ImportResult() import_start = timezone.now() @@ -70,7 +75,7 @@ def import_netscape_html(html: str, user: User): # Split bookmarks to import into batches, to keep memory usage for bulk operations manageable batches = _get_batches(netscape_bookmarks, 200) for batch in batches: - _import_batch(batch, user, tag_cache, result) + _import_batch(batch, user, options, tag_cache, result) # Create snapshots for newly imported bookmarks tasks.schedule_bookmarks_without_snapshots(user) @@ -114,7 +119,11 @@ def _get_batches(items: List, batch_size: int): return batches -def _import_batch(netscape_bookmarks: List[NetscapeBookmark], user: User, tag_cache: TagCache, result: ImportResult): +def _import_batch(netscape_bookmarks: List[NetscapeBookmark], + user: User, + options: ImportOptions, + tag_cache: TagCache, + result: ImportResult): # Query existing bookmarks batch_urls = [bookmark.href for bookmark in netscape_bookmarks] existing_bookmarks = Bookmark.objects.filter(owner=user, url__in=batch_urls) @@ -135,7 +144,7 @@ def _import_batch(netscape_bookmarks: List[NetscapeBookmark], user: User, tag_ca else: is_update = True # Copy data from parsed bookmark - _copy_bookmark_data(netscape_bookmark, bookmark) + _copy_bookmark_data(netscape_bookmark, bookmark, options) # Validate bookmark fields, exclude owner to prevent n+1 database query, # also there is no specific validation on owner bookmark.clean_fields(exclude=['owner']) @@ -152,8 +161,14 @@ def _import_batch(netscape_bookmarks: List[NetscapeBookmark], user: User, tag_ca result.failed = result.failed + 1 # Bulk update bookmarks in DB - Bookmark.objects.bulk_update(bookmarks_to_update, - ['url', 'date_added', 'date_modified', 'unread', 'title', 'description', 'owner']) + Bookmark.objects.bulk_update(bookmarks_to_update, ['url', + 'date_added', + 'date_modified', + 'unread', + 'shared', + 'title', + 'description', + 'owner']) # Bulk insert new bookmarks into DB Bookmark.objects.bulk_create(bookmarks_to_create) @@ -187,7 +202,7 @@ def _import_batch(netscape_bookmarks: List[NetscapeBookmark], user: User, tag_ca BookmarkToTagRelationShip.objects.bulk_create(relationships, ignore_conflicts=True) -def _copy_bookmark_data(netscape_bookmark: NetscapeBookmark, bookmark: Bookmark): +def _copy_bookmark_data(netscape_bookmark: NetscapeBookmark, bookmark: Bookmark, options: ImportOptions): bookmark.url = netscape_bookmark.href if netscape_bookmark.date_added: bookmark.date_added = parse_timestamp(netscape_bookmark.date_added) @@ -199,3 +214,5 @@ def _copy_bookmark_data(netscape_bookmark: NetscapeBookmark, bookmark: Bookmark) bookmark.title = netscape_bookmark.title if netscape_bookmark.description: bookmark.description = netscape_bookmark.description + if options.map_private_flag and not netscape_bookmark.private: + bookmark.shared = True diff --git a/bookmarks/services/parser.py b/bookmarks/services/parser.py index 7d27e7a..b757507 100644 --- a/bookmarks/services/parser.py +++ b/bookmarks/services/parser.py @@ -11,6 +11,7 @@ class NetscapeBookmark: date_added: str tag_string: str to_read: bool + private: bool class BookmarkParser(HTMLParser): @@ -26,6 +27,7 @@ class BookmarkParser(HTMLParser): self.title = '' self.description = '' self.toread = '' + self.private = '' def handle_starttag(self, tag: str, attrs: list): name = 'handle_start_' + tag.lower() @@ -58,7 +60,9 @@ class BookmarkParser(HTMLParser): description='', date_added=self.add_date, tag_string=self.tags, - to_read=self.toread == '1' + to_read=self.toread == '1', + # Mark as private by default, also when attribute is not specified + private=self.private != '0', ) def handle_a_data(self, data): @@ -79,6 +83,7 @@ class BookmarkParser(HTMLParser): self.title = '' self.description = '' self.toread = '' + self.private = '' def parse(html: str) -> List[NetscapeBookmark]: diff --git a/bookmarks/templates/settings/general.html b/bookmarks/templates/settings/general.html index 7070b3b..1da58c6 100644 --- a/bookmarks/templates/settings/general.html +++ b/bookmarks/templates/settings/general.html @@ -144,6 +144,16 @@ added and existing ones are updated.

{% csrf_token %} +
+ +
+ When importing bookmarks from a service that supports marking bookmarks as public or private (using the PRIVATE attribute), enabling this option will import all bookmarks that are marked as not private as shared bookmarks. + Otherwise, all bookmarks will be imported as private bookmarks. +
+
@@ -171,6 +181,10 @@

Export

Export all bookmarks in Netscape HTML format.

+

+ Note that exporting bookmark notes is currently not supported due to limitations of the format. + For proper backups please use a database backup as described in the documentation. +

Download (.html) {% if export_error %}
diff --git a/bookmarks/tests/helpers.py b/bookmarks/tests/helpers.py index d68875a..e72f2fd 100644 --- a/bookmarks/tests/helpers.py +++ b/bookmarks/tests/helpers.py @@ -1,5 +1,6 @@ import random import logging +import datetime from typing import List from bs4 import BeautifulSoup @@ -35,6 +36,7 @@ class BookmarkFactoryMixin: website_description: str = '', web_archive_snapshot_url: str = '', favicon_file: str = '', + added: datetime = None, ): if not title: title = get_random_string(length=32) @@ -45,6 +47,8 @@ class BookmarkFactoryMixin: if not url: unique_id = get_random_string(length=32) url = 'https://example.com/' + unique_id + if added is None: + added = timezone.now() bookmark = Bookmark( url=url, title=title, @@ -52,7 +56,7 @@ class BookmarkFactoryMixin: notes=notes, website_title=website_title, website_description=website_description, - date_added=timezone.now(), + date_added=added, date_modified=timezone.now(), owner=user, is_archived=is_archived, @@ -125,13 +129,15 @@ class BookmarkHtmlTag: description: str = '', add_date: str = '', tags: str = '', - to_read: bool = False): + to_read: bool = False, + private: bool = True): self.href = href self.title = title self.description = description self.add_date = add_date self.tags = tags self.to_read = to_read + self.private = private class ImportTestMixin: @@ -141,7 +147,8 @@ class ImportTestMixin: + TOREAD="{1 if tag.to_read else 0}" + PRIVATE="{1 if tag.private else 0}"> {tag.title if tag.title else ''} {f'
{tag.description}' if tag.description else ''} diff --git a/bookmarks/tests/test_exporter.py b/bookmarks/tests/test_exporter.py index aab46af..74f7b06 100644 --- a/bookmarks/tests/test_exporter.py +++ b/bookmarks/tests/test_exporter.py @@ -1,10 +1,36 @@ from django.test import TestCase +from django.utils import timezone from bookmarks.services import exporter from bookmarks.tests.helpers import BookmarkFactoryMixin class ExporterTestCase(TestCase, BookmarkFactoryMixin): + def test_export_bookmarks(self): + added = timezone.now() + timestamp = int(added.timestamp()) + + bookmarks = [ + self.setup_bookmark(url='https://example.com/1', title='Title 1', added=added, + description='Example description'), + self.setup_bookmark(url='https://example.com/2', title='Title 2', added=added, + tags=[self.setup_tag(name='tag1'), self.setup_tag(name='tag2'), + self.setup_tag(name='tag3')]), + self.setup_bookmark(url='https://example.com/3', title='Title 3', added=added, unread=True), + self.setup_bookmark(url='https://example.com/4', title='Title 4', added=added, shared=True), + + ] + html = exporter.export_netscape_html(bookmarks) + + lines = [ + f'
Title 1', + '
Example description', + f'
Title 2', + f'
Title 3', + f'
Title 4', + ] + self.assertIn('\n\r'.join(lines), html) + def test_escape_html_in_title_and_description(self): bookmark = self.setup_bookmark( title='