beenull/mwmbl
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Temporary just select some URLs at random for initialization

Browse source
This commit is contained in:
Daoud Clarke 2023-09-29 22:32:31 +01:00
parent 826d3d6ba9
commit 7c3aea5ca0
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
mwmbl/crawler/urls.py
View file

@ -132,10 +132,12 @@ class URLDatabase:
sql = f"""
SELECT url, status, user_id_hash, score, updated FROM urls
WHERE status = %(status)s
ORDER BY score DESC
LIMIT %(num_urls)s
"""
# TODO: reinstate this line once performance issue is resolved:
# ORDER BY score DESC
with self.connection.cursor() as cursor:
cursor.execute(sql, {'status': status.value, 'num_urls': num_urls})
results = cursor.fetchall()