marcel/linkding
1
0
Fork 0
mirror of https://github.com/sissbruecker/linkding.git synced 2025-08-08 11:18:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

Cache website metadata to avoid duplicate scraping (#401)

Browse Source 
* Cache website metadata to avoid duplicate scraping

* fix test setup
This commit is contained in:
Sascha Ißbrücker
2023-01-20 22:28:44 +01:00
committed by GitHub
parent da99b8b034
commit 30da1880a5
2 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bookmarks/services/website_loader.py
View File

@@ -1,5 +1,6 @@
import logging import logging
from dataclasses import dataclass from dataclasses import dataclass
from functools import lru_cache
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@@ -23,6 +24,9 @@ class WebsiteMetadata:
} }
# Caching metadata avoids scraping again when saving bookmarks, in case the
# metadata was already scraped to show preview values in the bookmark form
@lru_cache(maxsize=10)
def load_website_metadata(url: str): def load_website_metadata(url: str):
title = None title = None
description = None description = None

4
bookmarks/tests/test_website_loader.py
View File

@@ -25,6 +25,10 @@ class MockStreamingResponse:
class WebsiteLoaderTestCase(TestCase): class WebsiteLoaderTestCase(TestCase):
def setUp(self):
# clear cached metadata before test run
website_loader.load_website_metadata.cache_clear()
def render_html_document(self, title, description): def render_html_document(self, title, description):
return f''' return f'''
<!DOCTYPE html> <!DOCTYPE html>