Add date filters for REST API (#1080)

* Add modified_since query parameter

* Add added_since parameter

* update date_modified when assets change
This commit is contained in:
Sascha Ißbrücker
2025-05-30 10:24:19 +02:00
committed by GitHub
parent 578680c3c1
commit bb796c9bdb
6 changed files with 172 additions and 5 deletions

View File

@@ -171,7 +171,7 @@ class BookmarkSearch:
FILTER_UNREAD_YES = "yes" FILTER_UNREAD_YES = "yes"
FILTER_UNREAD_NO = "no" FILTER_UNREAD_NO = "no"
params = ["q", "user", "sort", "shared", "unread"] params = ["q", "user", "sort", "shared", "unread", "modified_since", "added_since"]
preferences = ["sort", "shared", "unread"] preferences = ["sort", "shared", "unread"]
defaults = { defaults = {
"q": "", "q": "",
@@ -179,6 +179,8 @@ class BookmarkSearch:
"sort": SORT_ADDED_DESC, "sort": SORT_ADDED_DESC,
"shared": FILTER_SHARED_OFF, "shared": FILTER_SHARED_OFF,
"unread": FILTER_UNREAD_OFF, "unread": FILTER_UNREAD_OFF,
"modified_since": None,
"added_since": None,
} }
def __init__( def __init__(
@@ -188,6 +190,8 @@ class BookmarkSearch:
sort: str = None, sort: str = None,
shared: str = None, shared: str = None,
unread: str = None, unread: str = None,
modified_since: str = None,
added_since: str = None,
preferences: dict = None, preferences: dict = None,
): ):
if not preferences: if not preferences:
@@ -199,6 +203,8 @@ class BookmarkSearch:
self.sort = sort or self.defaults["sort"] self.sort = sort or self.defaults["sort"]
self.shared = shared or self.defaults["shared"] self.shared = shared or self.defaults["shared"]
self.unread = unread or self.defaults["unread"] self.unread = unread or self.defaults["unread"]
self.modified_since = modified_since or self.defaults["modified_since"]
self.added_since = added_since or self.defaults["added_since"]
def is_modified(self, param): def is_modified(self, param):
value = self.__dict__[param] value = self.__dict__[param]
@@ -268,6 +274,8 @@ class BookmarkSearchForm(forms.Form):
sort = forms.ChoiceField(choices=SORT_CHOICES) sort = forms.ChoiceField(choices=SORT_CHOICES)
shared = forms.ChoiceField(choices=FILTER_SHARED_CHOICES, widget=forms.RadioSelect) shared = forms.ChoiceField(choices=FILTER_SHARED_CHOICES, widget=forms.RadioSelect)
unread = forms.ChoiceField(choices=FILTER_UNREAD_CHOICES, widget=forms.RadioSelect) unread = forms.ChoiceField(choices=FILTER_UNREAD_CHOICES, widget=forms.RadioSelect)
modified_since = forms.CharField(required=False)
added_since = forms.CharField(required=False)
def __init__( def __init__(
self, self,

View File

@@ -2,6 +2,7 @@ from typing import Optional
from django.conf import settings from django.conf import settings
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.db.models import Q, QuerySet, Exists, OuterRef, Case, When, CharField from django.db.models import Q, QuerySet, Exists, OuterRef, Case, When, CharField
from django.db.models.expressions import RawSQL from django.db.models.expressions import RawSQL
from django.db.models.functions import Lower from django.db.models.functions import Lower
@@ -44,6 +45,22 @@ def _base_bookmarks_query(
if user: if user:
query_set = query_set.filter(owner=user) query_set = query_set.filter(owner=user)
# Filter by modified_since if provided
if search.modified_since:
try:
query_set = query_set.filter(date_modified__gt=search.modified_since)
except ValidationError:
# If the date format is invalid, ignore the filter
pass
# Filter by added_since if provided
if search.added_since:
try:
query_set = query_set.filter(date_added__gt=search.added_since)
except ValidationError:
# If the date format is invalid, ignore the filter
pass
# Split query into search terms and tags # Split query into search terms and tags
query = parse_query_string(search.q) query = parse_query_string(search.q)

View File

@@ -53,6 +53,7 @@ def create_snapshot(asset: BookmarkAsset):
asset.save() asset.save()
asset.bookmark.latest_snapshot = asset asset.bookmark.latest_snapshot = asset
asset.bookmark.date_modified = timezone.now()
asset.bookmark.save() asset.bookmark.save()
except Exception as error: except Exception as error:
asset.status = BookmarkAsset.STATUS_FAILURE asset.status = BookmarkAsset.STATUS_FAILURE
@@ -75,6 +76,7 @@ def upload_snapshot(bookmark: Bookmark, html: bytes):
asset.save() asset.save()
asset.bookmark.latest_snapshot = asset asset.bookmark.latest_snapshot = asset
asset.bookmark.date_modified = timezone.now()
asset.bookmark.save() asset.bookmark.save()
return asset return asset
@@ -100,6 +102,10 @@ def upload_asset(bookmark: Bookmark, upload_file: UploadedFile):
asset.file = filename asset.file = filename
asset.file_size = upload_file.size asset.file_size = upload_file.size
asset.save() asset.save()
asset.bookmark.date_modified = timezone.now()
asset.bookmark.save()
logger.info( logger.info(
f"Successfully uploaded asset file. bookmark={bookmark} file={upload_file.name}" f"Successfully uploaded asset file. bookmark={bookmark} file={upload_file.name}"
) )
@@ -128,9 +134,10 @@ def remove_asset(asset: BookmarkAsset):
) )
bookmark.latest_snapshot = latest bookmark.latest_snapshot = latest
bookmark.save()
asset.delete() asset.delete()
bookmark.date_modified = timezone.now()
bookmark.save()
def _generate_asset_filename( def _generate_asset_filename(

View File

@@ -55,7 +55,12 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
self.assertIsNone(asset.id) self.assertIsNone(asset.id)
def test_create_snapshot(self): def test_create_snapshot(self):
bookmark = self.setup_bookmark(url="https://example.com") initial_modified = timezone.datetime(
2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
)
bookmark = self.setup_bookmark(
url="https://example.com", modified=initial_modified
)
asset = assets.create_snapshot_asset(bookmark) asset = assets.create_snapshot_asset(bookmark)
asset.save() asset.save()
asset.date_created = timezone.datetime( asset.date_created = timezone.datetime(
@@ -91,6 +96,9 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
self.assertEqual(asset.file, expected_filename) self.assertEqual(asset.file, expected_filename)
self.assertTrue(asset.gzip) self.assertTrue(asset.gzip)
# should update bookmark modified date
bookmark.refresh_from_db()
def test_create_snapshot_failure(self): def test_create_snapshot_failure(self):
bookmark = self.setup_bookmark(url="https://example.com") bookmark = self.setup_bookmark(url="https://example.com")
asset = assets.create_snapshot_asset(bookmark) asset = assets.create_snapshot_asset(bookmark)
@@ -120,7 +128,12 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
self.assertTrue(saved_file.endswith("aaaa.html.gz")) self.assertTrue(saved_file.endswith("aaaa.html.gz"))
def test_upload_snapshot(self): def test_upload_snapshot(self):
bookmark = self.setup_bookmark(url="https://example.com") initial_modified = timezone.datetime(
2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
)
bookmark = self.setup_bookmark(
url="https://example.com", modified=initial_modified
)
asset = assets.upload_snapshot(bookmark, self.html_content.encode()) asset = assets.upload_snapshot(bookmark, self.html_content.encode())
# should create gzip file in asset folder # should create gzip file in asset folder
@@ -145,6 +158,10 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
self.assertEqual(asset.file, saved_file_name) self.assertEqual(asset.file, saved_file_name)
self.assertTrue(asset.gzip) self.assertTrue(asset.gzip)
# should update bookmark modified date
bookmark.refresh_from_db()
self.assertGreater(bookmark.date_modified, initial_modified)
def test_upload_snapshot_failure(self): def test_upload_snapshot_failure(self):
bookmark = self.setup_bookmark(url="https://example.com") bookmark = self.setup_bookmark(url="https://example.com")
@@ -173,7 +190,10 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
@disable_logging @disable_logging
def test_upload_asset(self): def test_upload_asset(self):
bookmark = self.setup_bookmark() initial_modified = timezone.datetime(
2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
)
bookmark = self.setup_bookmark(modified=initial_modified)
file_content = b"test content" file_content = b"test content"
upload_file = SimpleUploadedFile( upload_file = SimpleUploadedFile(
"test_file.txt", file_content, content_type="text/plain" "test_file.txt", file_content, content_type="text/plain"
@@ -204,6 +224,10 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
self.assertEqual(asset.file_size, len(file_content)) self.assertEqual(asset.file_size, len(file_content))
self.assertFalse(asset.gzip) self.assertFalse(asset.gzip)
# should update bookmark modified date
bookmark.refresh_from_db()
self.assertGreater(bookmark.date_modified, initial_modified)
@disable_logging @disable_logging
def test_upload_asset_truncates_asset_file_name(self): def test_upload_asset_truncates_asset_file_name(self):
# Create a bookmark with a very long URL # Create a bookmark with a very long URL
@@ -409,3 +433,36 @@ class AssetServiceTestCase(TestCase, BookmarkFactoryMixin):
# Verify that latest_snapshot hasn't changed # Verify that latest_snapshot hasn't changed
self.assertEqual(bookmark.latest_snapshot, latest_asset) self.assertEqual(bookmark.latest_snapshot, latest_asset)
@disable_logging
def test_remove_asset(self):
initial_modified = timezone.datetime(
2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc
)
bookmark = self.setup_bookmark(modified=initial_modified)
file_content = b"test content for removal"
upload_file = SimpleUploadedFile(
"test_remove_file.txt", file_content, content_type="text/plain"
)
asset = assets.upload_asset(bookmark, upload_file)
asset_filepath = os.path.join(self.assets_dir, asset.file)
# Verify asset and file exist
self.assertTrue(BookmarkAsset.objects.filter(id=asset.id).exists())
self.assertTrue(os.path.exists(asset_filepath))
bookmark.date_modified = initial_modified
bookmark.save()
# Remove the asset
assets.remove_asset(asset)
# Verify asset is removed from DB
self.assertFalse(BookmarkAsset.objects.filter(id=asset.id).exists())
# Verify file is removed from disk
self.assertFalse(os.path.exists(asset_filepath))
# Verify bookmark modified date is updated
bookmark.refresh_from_db()
self.assertGreater(bookmark.date_modified, initial_modified)

View File

@@ -1211,3 +1211,79 @@ class QueriesTestCase(TestCase, BookmarkFactoryMixin):
query = queries.query_bookmarks(self.user, self.profile, search) query = queries.query_bookmarks(self.user, self.profile, search)
self.assertEqual(list(query), sorted_bookmarks) self.assertEqual(list(query), sorted_bookmarks)
def test_query_bookmarks_filter_modified_since(self):
# Create bookmarks with different modification dates
older_bookmark = self.setup_bookmark(title="old bookmark")
recent_bookmark = self.setup_bookmark(title="recent bookmark")
# Modify date field on bookmark directly to test modified_since
older_bookmark.date_modified = timezone.datetime(
2025, 1, 1, tzinfo=datetime.timezone.utc
)
older_bookmark.save()
recent_bookmark.date_modified = timezone.datetime(
2025, 5, 15, tzinfo=datetime.timezone.utc
)
recent_bookmark.save()
# Test with date between the two bookmarks
search = BookmarkSearch(modified_since="2025-03-01T00:00:00Z")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [recent_bookmark])
# Test with date before both bookmarks
search = BookmarkSearch(modified_since="2024-12-31T00:00:00Z")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [older_bookmark, recent_bookmark])
# Test with date after both bookmarks
search = BookmarkSearch(modified_since="2025-05-16T00:00:00Z")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [])
# Test with no modified_since - should return all bookmarks
search = BookmarkSearch()
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [older_bookmark, recent_bookmark])
# Test with invalid date format - should be ignored
search = BookmarkSearch(modified_since="invalid-date")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [older_bookmark, recent_bookmark])
def test_query_bookmarks_filter_added_since(self):
# Create bookmarks with different dates
older_bookmark = self.setup_bookmark(
title="old bookmark",
added=timezone.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc),
)
recent_bookmark = self.setup_bookmark(
title="recent bookmark",
added=timezone.datetime(2025, 5, 15, tzinfo=datetime.timezone.utc),
)
# Test with date between the two bookmarks
search = BookmarkSearch(added_since="2025-03-01T00:00:00Z")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [recent_bookmark])
# Test with date before both bookmarks
search = BookmarkSearch(added_since="2024-12-31T00:00:00Z")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [older_bookmark, recent_bookmark])
# Test with date after both bookmarks
search = BookmarkSearch(added_since="2025-05-16T00:00:00Z")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [])
# Test with no added_since - should return all bookmarks
search = BookmarkSearch()
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [older_bookmark, recent_bookmark])
# Test with invalid date format - should be ignored
search = BookmarkSearch(added_since="invalid-date")
query = queries.query_bookmarks(self.user, self.profile, search)
self.assertCountEqual(list(query), [older_bookmark, recent_bookmark])

View File

@@ -35,6 +35,8 @@ Parameters:
- `q` - Filters results using a search phrase using the same logic as through the UI - `q` - Filters results using a search phrase using the same logic as through the UI
- `limit` - Limits the max. number of results. Default is `100`. - `limit` - Limits the max. number of results. Default is `100`.
- `offset` - Index from which to start returning results - `offset` - Index from which to start returning results
- `modified_since` - Filter results to only include bookmarks modified after the specified date (format: ISO 8601, e.g. "2025-01-01T00:00:00Z")
- `added_since` - Filter results to only include bookmarks added after the specified date (format: ISO 8601, e.g. "2025-05-29T00:00:00Z")
Example response: Example response: