Bump version

* Promoting singlefile timeout to env variable

* Promoting singlefile timeout to env variable

* add tests

* Add LD_SINGLEFILE_OPTIONS support

* add tests

---------

Co-authored-by: Sascha Ißbrücker <sascha.issbruecker@gmail.com>

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v1.27.1 (07/04/2024)
+
+### What's Changed
+* Fix HTML snapshot errors related to single-file-cli by @sissbruecker in https://github.com/sissbruecker/linkding/pull/683
+* Replace django-background-tasks with huey by @sissbruecker in https://github.com/sissbruecker/linkding/pull/657
+* Add Authelia OIDC example to docs by @hugo-vrijswijk in https://github.com/sissbruecker/linkding/pull/675
+
+
+**Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.27.0...v1.27.1
+
+---
+
 ## v1.27.0 (01/04/2024)
 
 ### What's Changed

bookmarks/frontend/components/TagAutocomplete.svelte

@@ -150,17 +150,27 @@
         display: block;
     }
 
+    .form-autocomplete-input {
+        box-sizing: border-box;
+        height: var(--control-size);
+        min-height: var(--control-size);
+        padding: 0;
+    }
+
+    .form-autocomplete-input input {
+        width: 100%;
+        height: 100%;
+        border: none;
+        margin: 0;
+    }
+
     .form-autocomplete.small .form-autocomplete-input {
         height: var(--control-size-sm);
         min-height: var(--control-size-sm);
-        padding: 0.05rem 0.3rem;
     }
 
     .form-autocomplete.small .form-autocomplete-input input {
-        width: 100%;
-        height: 100%;
-        margin: 0;
-        padding: 0;
+        padding: 0.05rem 0.3rem;
         font-size: var(--font-size-sm);
     }
 

bookmarks/management/commands/backup.py

@@ -24,3 +24,8 @@ class Command(BaseCommand):
         source_db.close()
 
         self.stdout.write(self.style.SUCCESS(f"Backup created at {destination}"))
+        self.stdout.write(
+            self.style.WARNING(
+                "This backup method is deprecated and may be removed in the future. Please use the full_backup command instead, which creates backup zip file with all contents of the data folder."
+            )
+        )

bookmarks/management/commands/full_backup.py

@@ -0,0 +1,62 @@
+import sqlite3
+import os
+import tempfile
+import zipfile
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Creates a backup of the linkding data folder"
+
+    def add_arguments(self, parser):
+        parser.add_argument("backup_file", type=str, help="Backup zip file destination")
+
+    def handle(self, *args, **options):
+        backup_file = options["backup_file"]
+        with zipfile.ZipFile(backup_file, "w", zipfile.ZIP_DEFLATED) as zip_file:
+            # Backup the database
+            self.stdout.write("Create database backup...")
+            with tempfile.TemporaryDirectory() as temp_dir:
+                backup_db_file = os.path.join(temp_dir, "db.sqlite3")
+                self.backup_database(backup_db_file)
+                zip_file.write(backup_db_file, "db.sqlite3")
+
+            # Backup the assets folder
+            if not os.path.exists(os.path.join("data", "assets")):
+                self.stdout.write(
+                    self.style.WARNING("No assets folder found. Skipping...")
+                )
+            else:
+                self.stdout.write("Backup bookmark assets...")
+                assets_folder = os.path.join("data", "assets")
+                for root, _, files in os.walk(assets_folder):
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        zip_file.write(file_path, os.path.join("assets", file))
+
+            # Backup the favicons folder
+            if not os.path.exists(os.path.join("data", "favicons")):
+                self.stdout.write(
+                    self.style.WARNING("No favicons folder found. Skipping...")
+                )
+            else:
+                self.stdout.write("Backup bookmark favicons...")
+                favicons_folder = os.path.join("data", "favicons")
+                for root, _, files in os.walk(favicons_folder):
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        zip_file.write(file_path, os.path.join("favicons", file))
+
+        self.stdout.write(self.style.SUCCESS(f"Backup created at {backup_file}"))
+
+    def backup_database(self, backup_db_file):
+        def progress(status, remaining, total):
+            self.stdout.write(f"Copied {total-remaining} of {total} pages...")
+
+        source_db = sqlite3.connect(os.path.join("data", "db.sqlite3"))
+        backup_db = sqlite3.connect(backup_db_file)
+        with backup_db:
+            source_db.backup(backup_db, pages=50, progress=progress)
+        backup_db.close()
+        source_db.close()

bookmarks/services/singlefile.py

@@ -1,6 +1,7 @@
 import gzip
 import logging
 import os
+import shlex
 import shutil
 import signal
 import subprocess
@@ -17,14 +18,15 @@ logger = logging.getLogger(__name__)
 
 def create_snapshot(url: str, filepath: str):
     singlefile_path = settings.LD_SINGLEFILE_PATH
-    # singlefile_options = settings.LD_SINGLEFILE_OPTIONS
+    # parse string to list of arguments
+    singlefile_options = shlex.split(settings.LD_SINGLEFILE_OPTIONS)
     temp_filepath = filepath + ".tmp"
-
-    args = [singlefile_path, url, temp_filepath]
+    # concat lists
+    args = [singlefile_path] + singlefile_options + [url, temp_filepath]
     try:
         # Use start_new_session=True to create a new process group
         process = subprocess.Popen(args, start_new_session=True)
-        process.wait(timeout=60)
+        process.wait(timeout=settings.LD_SINGLEFILE_TIMEOUT_SEC)
 
         # check if the file was created
         if not os.path.exists(temp_filepath):

bookmarks/services/tasks.py

@@ -239,6 +239,9 @@ def create_html_snapshot(bookmark: Bookmark):
     asset.save()
 
 
+MAX_SNAPSHOT_FILENAME_LENGTH = 192
+
+
 def _generate_snapshot_filename(asset: BookmarkAsset) -> str:
     def sanitize_char(char):
         if char.isalnum() or char in ("-", "_", "."):
@@ -249,6 +252,13 @@ def _generate_snapshot_filename(asset: BookmarkAsset) -> str:
     formatted_datetime = asset.date_created.strftime("%Y-%m-%d_%H%M%S")
     sanitized_url = "".join(sanitize_char(char) for char in asset.bookmark.url)
 
+    # Calculate the length of the non-URL parts of the filename
+    non_url_length = len(f"{asset.asset_type}{formatted_datetime}__.html.gz")
+    # Calculate the maximum length for the URL part
+    max_url_length = MAX_SNAPSHOT_FILENAME_LENGTH - non_url_length
+    # Truncate the URL if necessary
+    sanitized_url = sanitized_url[:max_url_length]
+
     return f"{asset.asset_type}_{formatted_datetime}_{sanitized_url}.html.gz"
 
 

bookmarks/tests/test_bookmarks_tasks.py

@@ -556,6 +556,21 @@ class BookmarkTasksTestCase(TestCase, BookmarkFactoryMixin):
             self.assertEqual(asset.file, expected_filename)
             self.assertTrue(asset.gzip)
 
+    @override_settings(LD_ENABLE_SNAPSHOTS=True)
+    def test_create_html_snapshot_truncate_filename(self):
+        # Create a bookmark with a very long URL
+        long_url = "http://" + "a" * 300 + ".com"
+        bookmark = self.setup_bookmark(url=long_url)
+
+        tasks.create_html_snapshot(bookmark)
+        BookmarkAsset.objects.get(bookmark=bookmark)
+
+        # Run periodic task to process the snapshot
+        tasks._schedule_html_snapshots_task()
+
+        asset = BookmarkAsset.objects.get(bookmark=bookmark)
+        self.assertEqual(len(asset.file), 192)
+
     @override_settings(LD_ENABLE_SNAPSHOTS=True)
     def test_create_html_snapshot_should_handle_error(self):
         bookmark = self.setup_bookmark(url="https://example.com")

bookmarks/tests/test_oidc_support.py

@@ -49,3 +49,15 @@ class OidcSupportTest(TestCase):
             base_settings.AUTHENTICATION_BACKENDS,
         )
         del os.environ["LD_ENABLE_OIDC"]  # Remove the temporary environment variable
+
+    def test_default_settings(self):
+        os.environ["LD_ENABLE_OIDC"] = "True"
+        base_settings = importlib.import_module("siteroot.settings.base")
+        importlib.reload(base_settings)
+
+        self.assertEqual(
+            True,
+            base_settings.OIDC_VERIFY_SSL,
+        )
+
+        del os.environ["LD_ENABLE_OIDC"]

bookmarks/tests/test_singlefile_service.py

@@ -3,7 +3,7 @@ import os
 import subprocess
 from unittest import mock
 
-from django.test import TestCase
+from django.test import TestCase, override_settings
 
 from bookmarks.services import singlefile
 
@@ -50,3 +50,62 @@ class SingleFileServiceTestCase(TestCase):
         with mock.patch("subprocess.Popen"):
             with self.assertRaises(singlefile.SingeFileError):
                 singlefile.create_snapshot("http://example.com", self.html_filepath)
+
+    def test_create_snapshot_empty_options(self):
+        mock_process = mock.Mock()
+        mock_process.wait.return_value = 0
+        self.create_test_file()
+
+        with mock.patch("subprocess.Popen") as mock_popen:
+            singlefile.create_snapshot("http://example.com", self.html_filepath)
+
+            expected_args = [
+                "single-file",
+                "http://example.com",
+                self.html_filepath + ".tmp",
+            ]
+            mock_popen.assert_called_with(expected_args, start_new_session=True)
+
+    @override_settings(
+        LD_SINGLEFILE_OPTIONS='--some-option "some value" --another-option "another value" --third-option="third value"'
+    )
+    def test_create_snapshot_custom_options(self):
+        mock_process = mock.Mock()
+        mock_process.wait.return_value = 0
+        self.create_test_file()
+
+        with mock.patch("subprocess.Popen") as mock_popen:
+            singlefile.create_snapshot("http://example.com", self.html_filepath)
+
+            expected_args = [
+                "single-file",
+                "--some-option",
+                "some value",
+                "--another-option",
+                "another value",
+                "--third-option=third value",
+                "http://example.com",
+                self.html_filepath + ".tmp",
+            ]
+            mock_popen.assert_called_with(expected_args, start_new_session=True)
+
+    def test_create_snapshot_default_timeout_setting(self):
+        mock_process = mock.Mock()
+        mock_process.wait.return_value = 0
+        self.create_test_file()
+
+        with mock.patch("subprocess.Popen", return_value=mock_process):
+            singlefile.create_snapshot("http://example.com", self.html_filepath)
+
+            mock_process.wait.assert_called_with(timeout=60)
+
+    @override_settings(LD_SINGLEFILE_TIMEOUT_SEC=120)
+    def test_create_snapshot_custom_timeout_setting(self):
+        mock_process = mock.Mock()
+        mock_process.wait.return_value = 0
+        self.create_test_file()
+
+        with mock.patch("subprocess.Popen", return_value=mock_process):
+            singlefile.create_snapshot("http://example.com", self.html_filepath)
+
+            mock_process.wait.assert_called_with(timeout=120)

docs/Options.md

@@ -118,6 +118,7 @@ The following options can be configured:
 - `OIDC_RP_CLIENT_SECRET` - The client secret of the application.
 - `OIDC_RP_SIGN_ALGO` - The algorithm the OIDC provider uses to sign ID tokens. Default is `RS256`.
 - `OIDC_USE_PKCE` - Whether to use PKCE for the OIDC flow. Default is `True`.
+- `OIDC_VERIFY_SSL` - Whether to verify the SSL certificate of the OIDC provider. Set to `False` if using self-signed certificates or custom certificate authority. Default is `True`.
 
 <details>
 
@@ -245,3 +246,20 @@ See the default URL for how to insert the placeholder to the favicon provider UR
 
 Alternative favicon providers:
 - DuckDuckGo: `https://icons.duckduckgo.com/ip3/{domain}.ico`
+
+
+### `LD_SINGLEFILE_TIMEOUT_SEC`
+
+Values: `Float` | Default =  60.0
+
+When creating HTML archive snapshots, control the timeout for how long to wait for the snapshot to complete, in `seconds`.
+Defaults to 60 seconds; on lower-powered hardware you may need to increase this value. 
+
+### `LD_SINGLEFILE_OPTIONS`
+
+Values: `String` | Default = None
+
+When creating HTML archive snapshots, pass additional options to the `single-file` application that is used to create snapshots.
+See `single-file --help` for complete list of arguments, or browse source: https://github.com/gildas-lormeau/single-file-cli/blob/master/options.js
+
+Example: `LD_SINGLEFILE_OPTIONS=--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:124.0) Gecko/20100101 Firefox/124.0"`

docs/backup.md

@@ -4,24 +4,56 @@ Linkding stores all data in the application's data folder.
 The full path to that folder in the Docker container is `/etc/linkding/data`.
 As described in the installation docs, you should mount the `/etc/linkding/data` folder to a folder on your host system.
 
-The data folder contains the following contents:
+The data folder contains the following contents that are relevant for backups:
 - `db.sqlite3` - the SQLite database
+- `assets` - folder that contains HTML snapshots of bookmarks
 - `favicons` - folder that contains downloaded favicons
 
 The following sections explain how to back up the individual contents.
 
-## Database
+## Full backup
 
-This section describes several methods on how to back up the contents of the SQLite database.
+linkding provides a CLI command to create a full backup of the data folder. This creates a zip file that contains backups of the database, assets, and favicons.
+
+> [!NOTE]
+> This method assumes that you are using the default SQLite database.
+> If you are using a different database, such as Postgres, you'll have to back up the database and other contents of the data folder manually.
+
+To create a full backup, execute the following command:
+```shell
+docker exec -it linkding python manage.py full_backup /etc/linkding/data/backup.zip
+```
+This creates a `backup.zip` file in the Docker container under `/etc/linkding/data`.
+
+To copy the backup file to your host system, execute the following command:
+```shell
+docker cp linkding:/etc/linkding/data/backup.zip backup.zip
+```
+This copies the backup file from the Docker container to the current folder on your host system.
+Now you can move that file to your backup location.
+
+To restore a backup: 
+- Extract the zip file in a folder of your new installation.
+- Rename the extracted folder to `data`.
+- When starting the Docker container, mount that folder to `/etc/linkding/data` as explained in the README.
+- Then start the Docker container.
+
+## Alternative backup methods
+
+If you can't use the full backup method, this section describes alternatives how to back up the individual contents of the data folder.
+
+### SQLite database backup
+
+linkding includes a CLI command for creating a backup copy of the database.
 
 > [!WARNING]
 > While the SQLite database is just a single file, it is not recommended to just copy that file.
 > This method is not transaction safe and may result in a [corrupted database](https://www.sqlite.org/howtocorrupt.html).
 > Use one of the backup methods described below.
 
-### Using the backup command
-
-linkding includes a CLI command for creating a backup copy of the database.
+> [!WARNING]
+> This method is deprecated and may be removed in the future.
+> Please use the full backup method described above.
 
 To create a backup, execute the following command:
 ```shell
@@ -38,12 +70,12 @@ Now you can move that file to your backup location.
 
 To restore the backup, just copy the backup file to the data folder of your new installation and rename it to `db.sqlite3`. Then start the Docker container.
 
-### Using the SQLite dump function
+### SQLite database SQL dump
 
 Requires [SQLite](https://www.sqlite.org/index.html) to be installed on your host system.
 
 With this method you create a plain text file with the SQL statements to recreate the SQLite database.
-To create a backup, execute the following command in the data folder:
+To create a backup, execute the following command in the data folder on your host system:
 ```shell
 sqlite3 db.sqlite3 .dump > backup.sql
 ```
@@ -56,8 +88,8 @@ Using git, you can commit the changes, followed by a git push to a remote reposi
 This is the least technical option to back up bookmarks, but has several limitations:
 - It does not export user profiles.
 - It only exports your own bookmarks, not those of other users.
-- It does not export archived bookmarks.
 - It does not export URLs of snapshots on the Internet Archive Wayback machine.
+- It does not export HTML snapshots of bookmarks. Even if you backup and restore the assets folder, the bookmarks will not be linked to the snapshots anymore.
 - It does not export favicons.
 
 Only use this method if you are fine with the above limitations.
@@ -70,7 +102,16 @@ To restore bookmarks, open the general settings on your new installation.
 In the Import section, click on the *Choose file* button to select the HTML file you downloaded before.
 Then click on the *Import* button to import the bookmarks.
 
-## Favicons
+### Assets
+
+If you are using the HTML snapshots feature, you should also do backups of the `assets` folder.
+It contains the HTML snapshots files of your bookmarks which are referenced from the database.
+
+To back up the assets, then you have to copy the `assets` folder to your backup location.
+
+To restore the assets, copy the `assets` folder back to the data folder of your new installation.
+
+### Favicons
 
 Doing a backup of the icons is optional, as they can be downloaded again.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "linkding",
-  "version": "1.27.1",
+  "version": "1.28.0",
   "description": "",
   "main": "index.js",
   "scripts": {

siteroot/settings/base.py

@@ -212,6 +212,7 @@ if LD_ENABLE_OIDC:
     OIDC_RP_CLIENT_SECRET = os.getenv("OIDC_RP_CLIENT_SECRET")
     OIDC_RP_SIGN_ALGO = os.getenv("OIDC_RP_SIGN_ALGO", "RS256")
     OIDC_USE_PKCE = os.getenv("OIDC_USE_PKCE", True) in (True, "True", "1")
+    OIDC_VERIFY_SSL = os.getenv("OIDC_VERIFY_SSL", True) in (True, "True", "1")
 
 # Enable authentication proxy support if configured
 LD_ENABLE_AUTH_PROXY = os.getenv("LD_ENABLE_AUTH_PROXY", False) in (True, "True", "1")
@@ -294,6 +295,7 @@ LD_ENABLE_SNAPSHOTS = os.getenv("LD_ENABLE_SNAPSHOTS", False) in (
 )
 LD_SINGLEFILE_PATH = os.getenv("LD_SINGLEFILE_PATH", "single-file")
 LD_SINGLEFILE_OPTIONS = os.getenv("LD_SINGLEFILE_OPTIONS", "")
+LD_SINGLEFILE_TIMEOUT_SEC = float(os.getenv("LD_SINGLEFILE_TIMEOUT_SEC", 60))
 
 # Monolith isn't used at the moment, as the local snapshot implementation
 # switched to single-file after the prototype. Keeping this around in case

version.txt

@@ -1 +1 @@
-1.27.1
+1.28.0