From bd017079d5ad9a48c0edffb8d8e02498a10fa34b Mon Sep 17 00:00:00 2001 From: Daoud Clarke Date: Tue, 24 Oct 2023 10:32:06 +0100 Subject: [PATCH 01/55] Add login using allauth --- mwmbl/settings_bg_prod.py | 2 +- mwmbl/settings_common.py | 16 +++ mwmbl/settings_dev.py | 4 +- mwmbl/templates/base.html | 22 ++++ mwmbl/templates/home.html | 5 + mwmbl/templates/profile.html | 8 ++ mwmbl/templates/registration/login.html | 26 ++++ mwmbl/templates/signup.html | 10 ++ mwmbl/urls.py | 14 ++- mwmbl/views.py | 24 ++++ poetry.lock | 158 +++++++++++++++++++++++- pyproject.toml | 1 + 12 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 mwmbl/templates/base.html create mode 100644 mwmbl/templates/home.html create mode 100644 mwmbl/templates/profile.html create mode 100644 mwmbl/templates/registration/login.html create mode 100644 mwmbl/templates/signup.html create mode 100644 mwmbl/views.py diff --git a/mwmbl/settings_bg_prod.py b/mwmbl/settings_bg_prod.py index 070bbf9..51d035e 100644 --- a/mwmbl/settings_bg_prod.py +++ b/mwmbl/settings_bg_prod.py @@ -5,4 +5,4 @@ ALLOWED_HOSTS = ["api.mwmbl.org", "mwmbl.org"] DATA_PATH = "/app/storage" RUN_BACKGROUND_PROCESSES = True -NUM_PAGES = 10240000 +NUM_PAGES = 10240000 \ No newline at end of file diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py index 645ebd4..14a1e5a 100644 --- a/mwmbl/settings_common.py +++ b/mwmbl/settings_common.py @@ -33,6 +33,10 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'mwmbl', + + 'allauth', + 'allauth.account', + 'allauth.socialaccount', ] MIDDLEWARE = [ @@ -43,6 +47,8 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + + "allauth.account.middleware.AccountMiddleware", ] ROOT_URLCONF = 'mwmbl.urls' @@ -120,3 +126,13 @@ print("Static files", STATICFILES_DIRS) DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +AUTHENTICATION_BACKENDS = [ + # Needed to login by username in Django admin, regardless of `allauth` + 'django.contrib.auth.backends.ModelBackend', + + # `allauth` specific authentication methods, such as login by email + 'allauth.account.auth_backends.AuthenticationBackend', +] + +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py index ee82351..c7cd281 100644 --- a/mwmbl/settings_dev.py +++ b/mwmbl/settings_dev.py @@ -3,7 +3,9 @@ from mwmbl.settings_common import * DEBUG = True ALLOWED_HOSTS = ["localhost", "127.0.0.1"] +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + DATA_PATH = "./devdata" -RUN_BACKGROUND_PROCESSES = True +RUN_BACKGROUND_PROCESSES = False NUM_PAGES = 2560 diff --git a/mwmbl/templates/base.html b/mwmbl/templates/base.html new file mode 100644 index 0000000..b397195 --- /dev/null +++ b/mwmbl/templates/base.html @@ -0,0 +1,22 @@ + + + + + {% block title %}Simple is Better Than Complex{% endblock %} + + +
+

My Site

+ {% if user.is_authenticated %} + logout + {% else %} + login / signup + {% endif %} +
+
+
+ {% block content %} + {% endblock %} +
+ + diff --git a/mwmbl/templates/home.html b/mwmbl/templates/home.html new file mode 100644 index 0000000..ed59579 --- /dev/null +++ b/mwmbl/templates/home.html @@ -0,0 +1,5 @@ +{% extends 'base.html' %} + +{% block content %} +

Welcome, {{ user.username }}!

+{% endblock %} diff --git a/mwmbl/templates/profile.html b/mwmbl/templates/profile.html new file mode 100644 index 0000000..2923d40 --- /dev/null +++ b/mwmbl/templates/profile.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} +{% block title %}Profile Page{% endblock title %} +{% block content %} +
+

This is the profile page for {{user.username}}

+
+ +{% endblock content %} diff --git a/mwmbl/templates/registration/login.html b/mwmbl/templates/registration/login.html new file mode 100644 index 0000000..a5f87a3 --- /dev/null +++ b/mwmbl/templates/registration/login.html @@ -0,0 +1,26 @@ +{% extends 'base.html' %} + +{% block content %} +

Log in to My Site

+ {% if form.errors %} +

Your username and password didn't match. Please try again.

+ {% endif %} +
+ {% csrf_token %} + + {% for field in form %} +

+ {{ field.label_tag }}
+ {{ field }}
+ {% for error in field.errors %} +

{{ error }}

+ {% endfor %} + {% if field.help_text %} +

{{ field.help_text }}

+ {% endif %} +

+ {% endfor %} + + New to My Site? Sign up +
+{% endblock %} diff --git a/mwmbl/templates/signup.html b/mwmbl/templates/signup.html new file mode 100644 index 0000000..b858ff6 --- /dev/null +++ b/mwmbl/templates/signup.html @@ -0,0 +1,10 @@ +{% extends 'base.html' %} + +{% block content %} +

Sign up

+
+ {% csrf_token %} + {{ form.as_p }} + +
+{% endblock %} diff --git a/mwmbl/urls.py b/mwmbl/urls.py index 7d8bff6..b416085 100644 --- a/mwmbl/urls.py +++ b/mwmbl/urls.py @@ -15,12 +15,22 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.contrib.auth import login, logout +from django.template.defaulttags import url +from django.urls import path, include from mwmbl.api import api_original as api, api_v1 +from mwmbl.views import signup, profile urlpatterns = [ path('admin/', admin.site.urls), path('', api.urls), - path('api/v1/', api_v1.urls) + path('api/v1/', api_v1.urls), + path('accounts/', include('allauth.urls')), + + # path("accounts/", include("django.contrib.auth.urls")), + # path('accounts/new/', signup, name='signup'), + path('accounts/profile/', profile, name='profile'), + # path('login/', login, {'template_name': 'login.html'}, name='login'), + # path('logout/', logout, {'next_page': 'login'}, name='logout'), ] diff --git a/mwmbl/views.py b/mwmbl/views.py new file mode 100644 index 0000000..0ae3ba8 --- /dev/null +++ b/mwmbl/views.py @@ -0,0 +1,24 @@ +from django.contrib.auth import authenticate, login +from django.contrib.auth.decorators import login_required +from django.contrib.auth.forms import UserCreationForm +from django.shortcuts import redirect, render + + +def signup(request): + if request.method == 'POST': + form = UserCreationForm(request.POST) + if form.is_valid(): + form.save() + username = form.cleaned_data.get('username') + raw_password = form.cleaned_data.get('password1') + user = authenticate(username=username, password=raw_password) + login(request, user) + return redirect('/') + else: + form = UserCreationForm() + return render(request, 'signup.html', {'form': form}) + + +@login_required +def profile(request): + return render(request, 'profile.html') diff --git a/poetry.lock b/poetry.lock index d7a0647..f4395b2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -419,6 +419,52 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "41.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "cymem" version = "2.0.8" @@ -462,6 +508,18 @@ files = [ {file = "cymem-2.0.8.tar.gz", hash = "sha256:8fb09d222e21dcf1c7e907dc85cf74501d4cea6c4ed4ac6c9e016f98fb59cbbf"}, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + [[package]] name = "django" version = "4.2.6" @@ -483,6 +541,28 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""} argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-allauth" +version = "0.57.0" +description = "Integrated set of Django applications addressing authentication, registration, account management as well as 3rd party (social) account authentication." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "django-allauth-0.57.0.tar.gz", hash = "sha256:a095ef0db7de305d9175772c78e765ebd5fceb004ae61c1383d7fc1af0f7c5b1"}, +] + +[package.dependencies] +Django = ">=3.2" +pyjwt = {version = ">=1.7", extras = ["crypto"]} +python3-openid = ">=3.0.8" +requests = ">=2.0.0" +requests-oauthlib = ">=0.3.0" + +[package.extras] +mfa = ["qrcode (>=7.0.0)"] +saml = ["python3-saml (>=1.15.0,<2.0.0)"] + [[package]] name = "django-ninja" version = "0.22.2" @@ -1104,6 +1184,23 @@ files = [ {file = "numpy-1.26.0.tar.gz", hash = "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"}, ] +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "packaging" version = "23.2" @@ -1454,6 +1551,27 @@ typing-extensions = ">=3.7.4.3" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pyspark" version = "3.2.0" @@ -1530,6 +1648,25 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python3-openid" +version = "3.2.0" +description = "OpenID support for modern servers and consumers." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "python3-openid-3.2.0.tar.gz", hash = "sha256:33fbf6928f401e0b790151ed2b5290b02545e8775f982485205a066f874aaeaf"}, + {file = "python3_openid-3.2.0-py3-none-any.whl", hash = "sha256:6626f771e0417486701e0b4daff762e7212e820ca5b29fcc0d05f6f8736dfa6b"}, +] + +[package.dependencies] +defusedxml = "*" + +[package.extras] +mysql = ["mysql-connector-python"] +postgresql = ["psycopg2"] + [[package]] name = "pytz" version = "2023.3.post1" @@ -1732,6 +1869,25 @@ redis = ["redis (>=3)"] security = ["itsdangerous (>=2.0)"] yaml = ["pyyaml (>=5.4)"] +[[package]] +name = "requests-oauthlib" +version = "1.3.1" +description = "OAuthlib authentication support for Requests." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, + {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + [[package]] name = "s3transfer" version = "0.7.0" @@ -2443,4 +2599,4 @@ indexer = ["ujson", "warcio", "idna", "beautifulsoup4", "lxml", "langdetect", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "fe5f238c57ec2d09acb6bdf8f46f33c7bbe499f68a7e34ab7bca1336e0ae881c" +content-hash = "13572a7df206102ce30a6deb1eecd22b5b217a96f864a7dd6c558a7ca263d520" diff --git a/pyproject.toml b/pyproject.toml index 4a4a725..a0de7fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ django = "^4.2.4" django-ninja = "^0.22.2" requests-cache = "^1.1.0" redis = {extras = ["hiredis"], version = "^5.0.1"} +django-allauth = "^0.57.0" [tool.poetry.extras] indexer = [ From bb9e6aa4bd555adcef8ae512a48830144dd62475 Mon Sep 17 00:00:00 2001 From: Daoud Clarke Date: Wed, 25 Oct 2023 16:39:42 +0100 Subject: [PATCH 02/55] Implement curation API using Django Ninja --- mwmbl/admin.py | 8 ++ mwmbl/api.py | 7 +- mwmbl/apps.py | 15 +-- mwmbl/migrations/0001_initial.py | 58 ++++++++++ mwmbl/migrations/__init__.py | 0 mwmbl/models.py | 15 +++ mwmbl/platform/curate.py | 82 +++++++++++++ mwmbl/platform/data.py | 45 ++++++++ mwmbl/platform/user.py | 190 ------------------------------- mwmbl/settings_common.py | 6 +- 10 files changed, 227 insertions(+), 199 deletions(-) create mode 100644 mwmbl/admin.py create mode 100644 mwmbl/migrations/0001_initial.py create mode 100644 mwmbl/migrations/__init__.py create mode 100644 mwmbl/models.py create mode 100644 mwmbl/platform/curate.py create mode 100644 mwmbl/platform/data.py delete mode 100644 mwmbl/platform/user.py diff --git a/mwmbl/admin.py b/mwmbl/admin.py new file mode 100644 index 0000000..74af7ac --- /dev/null +++ b/mwmbl/admin.py @@ -0,0 +1,8 @@ +from django.contrib.admin import ModelAdmin +from django.contrib.auth.admin import UserAdmin +from django.contrib import admin + +from mwmbl.models import MwmblUser, UserCuration + +admin.site.register(MwmblUser, UserAdmin) +admin.site.register(UserCuration, ModelAdmin) diff --git a/mwmbl/api.py b/mwmbl/api.py index 4a9cf08..bb2fe75 100644 --- a/mwmbl/api.py +++ b/mwmbl/api.py @@ -7,6 +7,7 @@ from ninja import NinjaAPI import mwmbl.crawler.app as crawler from mwmbl.indexer.batch_cache import BatchCache from mwmbl.indexer.paths import INDEX_NAME, BATCH_DIR_NAME +from mwmbl.platform import curate from mwmbl.tinysearchengine import search from mwmbl.tinysearchengine.completer import Completer from mwmbl.tinysearchengine.indexer import TinyIndex, Document @@ -24,13 +25,17 @@ batch_cache = BatchCache(Path(settings.DATA_PATH) / BATCH_DIR_NAME) def create_api(version): - api = NinjaAPI(version=version) + # Set csrf to True to all cookie-based authentication + api = NinjaAPI(version=version, csrf=True) search_router = search.create_router(ranker) api.add_router("/search/", search_router) crawler_router = crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches) api.add_router("/crawler/", crawler_router) + + curation_router = curate.create_router(index_path) + api.add_router("/curation/", curation_router) return api diff --git a/mwmbl/apps.py b/mwmbl/apps.py index f829ecc..dff27b6 100644 --- a/mwmbl/apps.py +++ b/mwmbl/apps.py @@ -6,19 +6,20 @@ from pathlib import Path from django.apps import AppConfig from django.conf import settings -from mwmbl.api import queued_batches -from mwmbl import background -from mwmbl.indexer.paths import INDEX_NAME -from mwmbl.indexer.update_urls import update_urls_continuously -from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE -from mwmbl.url_queue import update_queue_continuously - class MwmblConfig(AppConfig): name = "mwmbl" verbose_name = "Mwmbl Application" def ready(self): + # Imports here to avoid AppRegistryNotReady exception + from mwmbl.api import queued_batches + from mwmbl import background + from mwmbl.indexer.paths import INDEX_NAME + from mwmbl.indexer.update_urls import update_urls_continuously + from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE + from mwmbl.url_queue import update_queue_continuously + index_path = Path(settings.DATA_PATH) / INDEX_NAME try: existing_index = TinyIndex(item_factory=Document, index_path=index_path) diff --git a/mwmbl/migrations/0001_initial.py b/mwmbl/migrations/0001_initial.py new file mode 100644 index 0000000..1dc8d43 --- /dev/null +++ b/mwmbl/migrations/0001_initial.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.6 on 2023-10-25 11:55 + +from django.conf import settings +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='MwmblUser', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + migrations.CreateModel( + name='UserCuration', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField()), + ('url', models.CharField(max_length=300)), + ('results', models.JSONField()), + ('curation_type', models.CharField(max_length=20)), + ('curation', models.JSONField()), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/mwmbl/migrations/__init__.py b/mwmbl/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mwmbl/models.py b/mwmbl/models.py new file mode 100644 index 0000000..30d6e01 --- /dev/null +++ b/mwmbl/models.py @@ -0,0 +1,15 @@ +from django.db import models +from django.contrib.auth.models import AbstractUser + + +class MwmblUser(AbstractUser): + pass + + +class UserCuration(models.Model): + user = models.ForeignKey(MwmblUser, on_delete=models.CASCADE) + timestamp = models.DateTimeField() + url = models.CharField(max_length=300) + results = models.JSONField() + curation_type = models.CharField(max_length=20) + curation = models.JSONField() diff --git a/mwmbl/platform/curate.py b/mwmbl/platform/curate.py new file mode 100644 index 0000000..92b1ebd --- /dev/null +++ b/mwmbl/platform/curate.py @@ -0,0 +1,82 @@ +import json +from urllib.parse import urljoin, parse_qs + +import requests +from ninja import Router +from ninja.security import django_auth + +from mwmbl.indexer.update_urls import get_datetime_from_timestamp +from mwmbl.models import UserCuration +from mwmbl.platform.data import CurateBegin, CurateMove, CurateDelete, CurateAdd, CurateValidate, Curation +from mwmbl.tinysearchengine.indexer import TinyIndex, Document +from mwmbl.tokenizer import tokenize + + +RESULT_URL = "https://mwmbl.org/?q=" +MAX_CURATED_SCORE = 1_111_111.0 + + +def create_router(index_path: str) -> Router: + router = Router(tags=["user"]) + + @router.post("/begin", auth=django_auth) + def user_begin_curate(request, curate_begin: CurateBegin): + return _curate(request, "curate_begin", curate_begin) + + @router.post("/move", auth=django_auth) + def user_move_result(request, curate_move: Curation[CurateMove]): + return _curate(request, "curate_move", curate_move) + + @router.post("/delete", auth=django_auth) + def user_delete_result(request, curate_delete: Curation[CurateDelete]): + return _curate(request, "curate_delete", curate_delete) + + @router.post("/add", auth=django_auth) + def user_add_result(request, curate_add: Curation[CurateAdd]): + return _curate(request, "curate_add", curate_add) + + @router.post("/validate", auth=django_auth) + def user_add_result(request, curate_validate: Curation[CurateValidate]): + return _curate(request, "curate_validate", curate_validate) + + def _curate(request, curation_type: str, curation: Curation): + user_curation = UserCuration( + user=request.user, + timestamp=get_datetime_from_timestamp(curation.timestamp), + url=curation.url, + results=curation.dict()["results"], + curation_type=curation_type, + curation=curation.curation.dict(), + ) + user_curation.save() + + with TinyIndex(Document, index_path, 'w') as indexer: + query_string = parse_qs(curation.url) + if len(query_string) > 1: + raise ValueError(f"Should be one query string in the URL: {curation.url}") + + queries = next(iter(query_string.values())) + if len(queries) > 1: + raise ValueError(f"Should be one query value in the URL: {curation.url}") + + query = queries[0] + print("Query", query) + tokens = tokenize(query) + print("Tokens", tokens) + term = " ".join(tokens) + print("Key", term) + + documents = [ + Document(result.title, result.url, result.extract, MAX_CURATED_SCORE - i, term, result.curated) + for i, result in enumerate(curation.results) + ] + page_index = indexer.get_key_page_index(term) + print("Page index", page_index) + print("Storing documents", documents) + indexer.store_in_page(page_index, documents) + + return {"curation": "ok"} + + return router + + diff --git a/mwmbl/platform/data.py b/mwmbl/platform/data.py new file mode 100644 index 0000000..1f0b0c1 --- /dev/null +++ b/mwmbl/platform/data.py @@ -0,0 +1,45 @@ +from datetime import datetime +from typing import TypeVar, Generic + +from ninja import Schema + + +class Result(Schema): + url: str + title: str + extract: str + curated: bool + + +class CurateBegin(Schema): + pass + + +class CurateMove(Schema): + old_index: int + new_index: int + + +class CurateDelete(Schema): + delete_index: int + + +class CurateAdd(Schema): + insert_index: int + url: str + + +class CurateValidate(Schema): + validate_index: int + is_validated: bool + + +T = TypeVar('T', CurateBegin, CurateAdd, CurateDelete, CurateMove, CurateValidate) + + +class Curation(Schema, Generic[T]): + timestamp: int + url: str + results: list[Result] + curation: T + diff --git a/mwmbl/platform/user.py b/mwmbl/platform/user.py deleted file mode 100644 index bbdcb0e..0000000 --- a/mwmbl/platform/user.py +++ /dev/null @@ -1,190 +0,0 @@ -import json -import os -from typing import TypeVar, Generic -from urllib.parse import urljoin, parse_qs - -import requests -from fastapi import APIRouter, Response -from pydantic import BaseModel - -from mwmbl.tinysearchengine.indexer import TinyIndex, Document -from mwmbl.tokenizer import tokenize - - -LEMMY_URL = os.environ["LEMMY_URL"] -RESULT_URL = "https://mwmbl.org/?q=" -MAX_CURATED_SCORE = 1_111_111.0 - - -class Register(BaseModel): - username: str - email: str - password: str - password_verify: str - - -class Login(BaseModel): - username_or_email: str - password: str - - -class Result(BaseModel): - url: str - title: str - extract: str - curated: bool - - -class BeginCurate(BaseModel): - auth: str - url: str - results: list[Result] - - -class CurateMove(BaseModel): - old_index: int - new_index: int - - -class CurateDelete(BaseModel): - delete_index: int - - -class CurateAdd(BaseModel): - insert_index: int - url: str - - -class CurateValidate(BaseModel): - validate_index: int - is_validated: bool - - -T = TypeVar('T', CurateAdd, CurateDelete, CurateMove, CurateValidate) - - -class Curation(BaseModel, Generic[T]): - auth: str - curation_id: int - url: str - results: list[Result] - curation: T - - -def create_router(index_path: str) -> APIRouter: - router = APIRouter(prefix="/user", tags=["user"]) - - # TODO: reinstate - # community_id = get_community_id() - community_id = 0 - - @router.post("/register") - def user_register(register: Register) -> Response: - lemmy_register = { - "username": register.username, - "email": register.email, - "password": register.password, - "password_verify": register.password_verify, - "answer": "not applicable", - "captcha_answer": None, - "captcha_uuid": None, - "honeypot": None, - "show_nsfw": False, - } - request = requests.post(urljoin(LEMMY_URL, "api/v3/user/register"), json=lemmy_register) - if request.status_code != 200: - return Response(content=request.content, status_code=request.status_code, media_type="text/json") - - @router.post("/login") - def user_login(login: Login) -> Response: - request = requests.post(urljoin(LEMMY_URL, "api/v3/user/login"), json=login.dict()) - return Response(content=request.content, status_code=request.status_code, media_type="text/json") - - @router.post("/curation/begin") - def user_begin_curate(begin_curate: BeginCurate): - results = begin_curate.dict()["results"] - body = json.dumps({"original_results": results}, indent=2) - create_post = { - "auth": begin_curate.auth, - "body": body, - "community_id": community_id, - "honeypot": None, - "language_id": None, - "name": begin_curate.url, - "nsfw": None, - "url": begin_curate.url, - } - request = requests.post(urljoin(LEMMY_URL, "api/v3/post"), json=create_post) - if request.status_code != 200: - return Response(content=request.content, status_code=request.status_code, media_type="text/json") - data = request.json() - curation_id = data["post_view"]["post"]["id"] - return {"curation_id": curation_id} - - @router.post("/curation/move") - def user_move_result(curate_move: Curation[CurateMove]): - return _curate("curate_move", curate_move) - - @router.post("/curation/delete") - def user_delete_result(curate_delete: Curation[CurateDelete]): - return _curate("curate_delete", curate_delete) - - @router.post("/curation/add") - def user_add_result(curate_add: Curation[CurateAdd]): - return _curate("curate_add", curate_add) - - @router.post("/curation/validate") - def user_add_result(curate_validate: Curation[CurateValidate]): - return _curate("curate_validate", curate_validate) - - def _curate(curation_type: str, curation: Curation): - content = json.dumps({ - "curation_type": curation_type, - "curation": curation.curation.dict(), - }, indent=2) - create_comment = { - "auth": curation.auth, - "content": json.dumps(content, indent=2), - "form_id": None, - "language_id": None, - "parent_id": None, - "post_id": curation.curation_id, - } - request = requests.post(urljoin(LEMMY_URL, "api/v3/comment"), json=create_comment) - - with TinyIndex(Document, index_path, 'w') as indexer: - query_string = parse_qs(curation.url) - if len(query_string) > 1: - raise ValueError(f"Should be one query string in the URL: {curation.url}") - - queries = next(iter(query_string.values())) - if len(queries) > 1: - raise ValueError(f"Should be one query value in the URL: {curation.url}") - - query = queries[0] - print("Query", query) - tokens = tokenize(query) - print("Tokens", tokens) - term = " ".join(tokens) - print("Key", term) - - documents = [ - Document(result.title, result.url, result.extract, MAX_CURATED_SCORE - i, term, result.curated) - for i, result in enumerate(curation.results) - ] - page_index = indexer.get_key_page_index(term) - print("Page index", page_index) - print("Storing documents", documents) - indexer.store_in_page(page_index, documents) - - return Response(content=request.content, status_code=request.status_code, media_type="text/json") - - return router - - -def get_community_id() -> str: - request = requests.get(urljoin(LEMMY_URL, "api/v3/community?name=main")) - community = request.json() - return community["community_view"]["community"]["id"] - - diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py index 14a1e5a..e223c1b 100644 --- a/mwmbl/settings_common.py +++ b/mwmbl/settings_common.py @@ -119,7 +119,6 @@ USE_TZ = True STATIC_URL = 'static/' STATICFILES_DIRS = [str(Path(__file__).parent.parent / "front-end" / "dist")] -print("Static files", STATICFILES_DIRS) # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field @@ -134,5 +133,10 @@ AUTHENTICATION_BACKENDS = [ 'allauth.account.auth_backends.AuthenticationBackend', ] + +AUTH_USER_MODEL = "mwmbl.MwmblUser" + + ACCOUNT_EMAIL_REQUIRED = True ACCOUNT_EMAIL_VERIFICATION = "mandatory" + From 4d823497a68512830258dd895783ea8e3b14ecab Mon Sep 17 00:00:00 2001 From: Daoud Clarke Date: Wed, 25 Oct 2023 19:17:02 +0100 Subject: [PATCH 03/55] Add original curation front-end --- front-end/assets/css/global.css | 94 ++++++++++ front-end/assets/opensearch.xml | 10 +- front-end/config.js | 1 + front-end/package-lock.json | 82 +++------ front-end/package.json | 3 - front-end/src/components/app.js | 14 +- front-end/src/components/login.js | 69 ++++++++ .../src/components/molecules/add-button.js | 24 +++ .../src/components/molecules/add-result.js | 69 ++++++++ .../src/components/molecules/delete-button.js | 35 ++++ front-end/src/components/molecules/result.js | 22 ++- .../components/molecules/validate-button.js | 53 ++++++ front-end/src/components/organisms/footer.js | 10 +- front-end/src/components/organisms/results.js | 166 +++++++++++++++++- front-end/src/components/organisms/save.js | 122 +++++++++++++ .../src/components/organisms/search-bar.js | 6 +- front-end/src/components/register.js | 84 +++++++++ front-end/src/index.html | 13 +- front-end/src/index.js | 3 + front-end/src/stats/index.html | 12 +- front-end/vite.config.js | 9 +- 21 files changed, 795 insertions(+), 106 deletions(-) create mode 100644 front-end/src/components/login.js create mode 100644 front-end/src/components/molecules/add-button.js create mode 100644 front-end/src/components/molecules/add-result.js create mode 100644 front-end/src/components/molecules/delete-button.js create mode 100644 front-end/src/components/molecules/validate-button.js create mode 100644 front-end/src/components/organisms/save.js create mode 100644 front-end/src/components/register.js diff --git a/front-end/assets/css/global.css b/front-end/assets/css/global.css index d21eff3..b6f04dd 100644 --- a/front-end/assets/css/global.css +++ b/front-end/assets/css/global.css @@ -117,6 +117,10 @@ mwmbl-results, footer { padding: 10px; } +.result { + min-height: 120px; +} + .result a { display: block; text-decoration: none; @@ -229,4 +233,94 @@ a { font-weight: var(--bold-font-weight); color: var(--primary-color); text-decoration: underline; +} + +.result-container { + display: flex; +} + +.curation-buttons { + padding: 20px; +} + +.curation-button { + opacity: 0; + color: inherit; + border: none; + padding: 0; + font: inherit; + outline: inherit; + cursor: pointer; + + background: darkgrey; + box-shadow: 3px 3px 3px lightgrey; + width: 20px; + height: 20px; + border-radius: 50%; + display: flex; /* or inline-flex */ + align-items: center; + justify-content: center; + margin: 10px 0 10px 0; +} + +.result:hover .curation-button { + opacity: 70%; + transition: + opacity 200ms ease-in-out; +} + +.result:hover .curation-button:hover { + opacity: 100%; +} + +.curate-delete { + margin-top: 0; +} + +.validated { + background: lightgreen; + opacity: 100%; +} + +.curate-add { + margin-bottom: 0; +} + + +.modal { + /*display: none; !* Hidden by default *!*/ + position: fixed; /* Stay in place */ + z-index: 100; /* Sit on top */ + left: 0; + top: 0; + width: 100%; /* Full width */ + height: 100%; /* Full height */ + overflow: auto; /* Enable scroll if needed */ + background-color: rgb(0,0,0); /* Fallback color */ + background-color: rgba(0,0,0,0.4); /* Black w/ opacity */ +} + +/* Modal Content/Box */ +.modal-content { + background-color: #fefefe; + margin: 15% auto; /* 15% from the top and centered */ + padding: 20px; + border: 1px solid #888; + max-width: 800px; + width: 80%; /* Could be more or less, depending on screen size */ +} + +/* The Close Button */ +.close { + color: #aaa; + float: right; + font-size: 28px; + font-weight: bold; +} + +.close:hover, +.close:focus { + color: black; + text-decoration: none; + cursor: pointer; } \ No newline at end of file diff --git a/front-end/assets/opensearch.xml b/front-end/assets/opensearch.xml index bf0fe92..a9bdbcf 100644 --- a/front-end/assets/opensearch.xml +++ b/front-end/assets/opensearch.xml @@ -1,10 +1,10 @@ - MWMBL - Search MWMBL - - - MWMBL Search + MWMBL Local + Search MWMBL Local + + + MWMBL Search Local data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+Cjxzdmcgd2lkdGg9IjEwMCUiIGhlaWdodD0iMTAwJSIgdmlld0JveD0iMCAwIDkzNzUgOTM3NSIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4bWw6c3BhY2U9InByZXNlcnZlIiB4bWxuczpzZXJpZj0iaHR0cDovL3d3dy5zZXJpZi5jb20vIiBzdHlsZT0iZmlsbC1ydWxlOmV2ZW5vZGQ7Y2xpcC1ydWxlOmV2ZW5vZGQ7c3Ryb2tlLWxpbmVjYXA6cm91bmQ7c3Ryb2tlLWxpbmVqb2luOnJvdW5kO3N0cm9rZS1taXRlcmxpbWl0OjEuNTsiPgo8c3R5bGU+CgkJcGF0aCB7CgkJCWZpbGw6ICMwMDA7CgkJfQoJCUBtZWRpYSAoIHByZWZlcnMtY29sb3Itc2NoZW1lOiBkYXJrICkgewoJCQlwYXRoIHsKCQkJCWZpbGw6ICNmZmYgIWltcG9ydGFudDsKCQkJfQoJCX0KCTwvc3R5bGU+CjxwYXRoIGQ9Ik02MTI4LjcyLDgyNTEuNTZjNDk1LjY1LDAgOTE5LjY5NywtMTc2LjIyMiAxMjcyLjEzLC01MjguNjU5YzM1Mi40MzcsLTM1Mi40MzggNTI4LjY1OSwtNzc2LjQ4NCA1MjguNjU5LC0xMjcyLjEzbC0wLC0zMzU4Ljc1Yy0wLC05NC42NDQgLTM1LjQ5MiwtMTc2Ljg0MSAtMTA2LjQ4MiwtMjQ2LjU4MWMtNzAuOTg1LC02OS43MzkgLTE1My44MDEsLTEwNC42MTIgLTI0OC40NDUsLTEwNC42MTJjLTk5LjYzNCwtMCAtMTg0LjMxNCwzNC44NzMgLTI1NC4wNTQsMTA0LjYxMmMtNjkuNzQ2LDY5Ljc0IC0xMDQuNjEyLDE1MS45MzcgLTEwNC42MTIsMjQ2LjU4MWwtMCwzMzU4Ljc1Yy0wLDMwMS4zNzMgLTEwNS44NTcsNTU3LjkyMyAtMzE3LjU3MSw3NjkuNjNjLTIxMS43MDgsMjExLjcxNCAtNDY4LjI1MSwzMTcuNTcxIC03NjkuNjMsMzE3LjU3MWMtMjk4Ljg5LDAgLTU1NC44MDgsLTEwNS44NTcgLTc2Ny43NjYsLTMxNy41NzFjLTIxMi45NTgsLTIxMS43MDcgLTMxOS40MzQsLTQ2OC4yNTcgLTMxOS40MzQsLTc2OS42M2wtMCwtMzM1OC43NWMtMCwtOTQuNjQ0IC0zNC44NzMsLTE3Ni44NDEgLTEwNC42MTMsLTI0Ni41ODFjLTY5LjczOSwtNjkuNzM5IC0xNTQuNDI2LC0xMDQuNjEyIC0yNTQuMDU0LC0xMDQuNjEyYy05NC42NDksLTAgLTE3Ni44NDEsMzQuODczIC0yNDYuNTgsMTA0LjYxMmMtNjkuNzQsNjkuNzQgLTEwNC42MTMsMTUxLjkzNyAtMTA0LjYxMywyNDYuNTgxbDAsMzM1OC43NWMwLDMwMS4zNzMgLTEwNi40NzYsNTU3LjkyMyAtMzE5LjQzNCw3NjkuNjNjLTIxMi45NTksMjExLjcxNCAtNDY4Ljg4MywzMTcuNTcxIC03NjcuNzY2LDMxNy41NzFjLTMwMS4zNzksMCAtNTU3LjkyMywtMTA1Ljg1NyAtNzY5LjYzNiwtMzE3LjU3MWMtMjExLjcwOCwtMjExLjcwNyAtMzE3LjU2NSwtNDY4LjI1NyAtMzE3LjU2NSwtNzY5LjYzbDAsLTMzNTguNzVjMCwtOTQuNjQ0IC0zNC44NzMsLTE3Ni44NDEgLTEwNC42MTIsLTI0Ni41ODFjLTY5Ljc0LC02OS43MzkgLTE1NC40MjcsLTEwNC42MTIgLTI1NC4wNTQsLTEwNC42MTJjLTk0LjY1LC0wIC0xNzYuODQxLDM0Ljg3MyAtMjQ2LjU4MSwxMDQuNjEyYy02OS43MzksNjkuNzQgLTEwNC42MTIsMTUxLjkzNyAtMTA0LjYxMiwyNDYuNTgxbC0wLDMzNTguNzVjLTAsMzI2LjI4MyA4MC4zMjcsNjI3LjY2MiAyNDAuOTc2LDkwNC4xMzFjMTYwLjY1NiwyNzYuNDY5IDM3OC41OTMsNDk1LjAzMSA2NTMuODE3LDY1NS42ODZjMjc1LjIyNCwxNjAuNjQ5IDU3NS45ODQsMjQwLjk3NyA5MDIuMjY3LDI0MC45NzdjMjkxLjQxNiwwIDU2My41MjUsLTY0Ljc2MSA4MTYuMzM1LC0xOTQuMjc3YzI1Mi44MSwtMTI5LjUxNyA0NjAuMTU4LC0zMDcuNjA4IDYyMi4wNTgsLTUzNC4yNjNjMTY2Ljg3OCwyMjYuNjU1IDM3Ni43MjIsNDA0Ljc0NiA2MjkuNTMyLDUzNC4yNjNjMjUyLjgwOSwxMjkuNTE2IDUyNC45MTksMTk0LjI3NyA4MTYuMzM1LDE5NC4yNzdabS0wLjk2LC0xNjE3LjM5bC0wLjU4MiwtMGMtOTkuNjI3LC0wIC0xODQuMzE0LC0zNC44NzMgLTI1NC4wNTQsLTEwNC42MTJjLTY5LjczOSwtNjkuNzQgLTEwNC42MTIsLTE1MS45MzggLTEwNC42MTIsLTI0Ni41ODFsLTAsLTMzNTguNzRjLTAsLTMwMS4zNzMgLTEwNS44NTcsLTU1Ny45MjMgLTMxNy41NjUsLTc2OS42M2MtMjEwLjY5OCwtMjEwLjY5OSAtNDY1Ljc5OSwtMzE2LjU0OSAtNzY1LjMyLC0zMTcuNTU5Yy0yOTkuNTIxLDEuMDEgLTU1NC42MjIsMTA2Ljg2IC03NjUuMzE0LDMxNy41NTljLTIxMS43MTQsMjExLjcwNyAtMzE3LjU3MSw0NjguMjU3IC0zMTcuNTcxLDc2OS42M2wwLDMzNTguNzVjMCw5NC42NDQgLTM0Ljg2NiwxNzYuODQxIC0xMDQuNjA2LDI0Ni41ODFjLTY5LjczOSw2OS43MzkgLTE1NC40MjYsMTA0LjYxMiAtMjU0LjA1NCwxMDQuNjEybC04LjYzOCwwYy05NC42NDMsMCAtMTc2Ljg0MSwtMzQuODczIC0yNDYuNTgsLTEwNC42MTJjLTY5Ljc0LC02OS43NCAtMTA0LjYxMywtMTUxLjkzNyAtMTA0LjYxMywtMjQ2LjU4MWwwLC0zMzU4Ljc1YzAsLTMwMS4zNzMgLTEwNi40NzYsLTU1Ny45MjMgLTMxOS40MzQsLTc2OS42M2MtMjEyLjk1OSwtMjExLjcxNCAtNDY4Ljg3NiwtMzE3LjU3MSAtNzY3Ljc2NiwtMzE3LjU3MWMtMzAxLjM3OSwtMCAtNTU3LjkyMiwxMDUuODU3IC03NjkuNjMsMzE3LjU3MWMtMjExLjcxNCwyMTEuNzA3IC0zMTcuNTcxLDQ2OC4yNTcgLTMxNy41NzEsNzY5LjYzbDAsMzM1OC43NWMwLDk0LjY0NCAtMzQuODY3LDE3Ni44NDEgLTEwNC42MTIsMjQ2LjU4MWMtNjkuNzQsNjkuNzM5IC0xNTQuNDIsMTA0LjYxMiAtMjU0LjA1NCwxMDQuNjEyYy05NC42NDQsMCAtMTc2Ljg0MSwtMzQuODczIC0yNDYuNTgxLC0xMDQuNjEyYy02OS43MzksLTY5Ljc0IC0xMDQuNjA2LC0xNTEuOTM3IC0xMDQuNjA2LC0yNDYuNTgxbDAsLTMzNTguNzVjMCwtMzI2LjI4MyA4MC4zMjEsLTYyNy42NjIgMjQwLjk3NywtOTA0LjEzMWMxNjAuNjQ5LC0yNzYuNDY5IDM3OC41ODYsLTQ5NS4wMzEgNjUzLjgxNiwtNjU1LjY4NmMyNzUuMjI0LC0xNjAuNjQ5IDU3NS45NzgsLTI0MC45NzcgOTAyLjI2MSwtMjQwLjk3N2MyOTEuNDE2LC0wIDU2My41MjYsNjQuNzYxIDgxNi4zMzUsMTk0LjI3N2MyNTIuODEsMTI5LjUxNyA0NjAuMTY0LDMwNy42MDggNjIyLjA1OCw1MzQuMjYzYzE2Ni44NzgsLTIyNi42NTUgMzc2LjcyMiwtNDA0Ljc0NiA2MjkuNTMyLC01MzQuMjYzYzI1Mi44MDksLTEyOS41MTYgNTI0LjkxOSwtMTk0LjI3NyA4MTYuMzM1LC0xOTQuMjc3bDguNjM4LC0wYzE2NC44MjIsLTAgMzIzLjQ3MiwyMC43MTggNDc1Ljk0MSw2Mi4xNTRsNS4yMzksMS40MzFjNDEuMTE0LDExLjI2MyA4MS42MDksMjQuMDI0IDEyMS40OTcsMzguMjg0YzcyLjY4NywyNS44NyAxNDMuOTA3LDU2LjY3NSAyMTMuNjUyLDkyLjQwOGMyNTAuNjM2LDEyOC40MDggNDU2LjU5MiwzMDQuNTQ5IDYxNy44NjYsNTI4LjQxMmw0LjMyOCw1LjY2NWMxNjYuODcyLC0yMjYuNTggMzc2LjY2NywtNDA0LjU5OCA2MjkuMzk2LC01MzQuMDc3YzI1Mi44MDksLTEyOS41MTYgNTI0LjkyNSwtMTk0LjI3NyA4MTYuMzM1LC0xOTQuMjc3YzQ5NS42NTcsLTAgOTE5LjcwNCwxNzYuMjIyIDEyNzIuMTQsNTI4LjY1OWMzNTIuNDM3LDM1Mi40MzggNTI4LjY1Myw3NzYuNDg0IDUyOC42NTMsMTI3Mi4xM2wwLDMzNTguNzVjMCw5NC42NDQgLTM1LjQ5MiwxNzYuODQxIC0xMDYuNDc2LDI0Ni41ODFjLTcwLjk4NCw2OS43MzkgLTE1My44MDEsMTA0LjYxMiAtMjQ4LjQ1MSwxMDQuNjEyYy05OS42MjcsMCAtMTg0LjMxNCwtMzQuODczIC0yNTQuMDU0LC0xMDQuNjEyYy02OS43MzksLTY5Ljc0IC0xMDQuNjEyLC0xNTEuOTM3IC0xMDQuNjEyLC0yNDYuNTgxbC0wLC0zMzU4Ljc1Yy0wLC0zMDEuMzczIC0xMDUuODUxLC01NTcuOTIzIC0zMTcuNTY1LC03NjkuNjNjLTIxMS43MTMsLTIxMS43MTQgLTQ2OC4yNTcsLTMxNy41NzEgLTc2OS42MzYsLTMxNy41NzFjLTI5OC44ODMsLTAgLTU1NC44MDcsMTA1Ljg1NyAtNzY3Ljc2NiwzMTcuNTcxYy0yMTIuOTUyLDIxMS43MDcgLTMxOS40MzQsNDY4LjI1NyAtMzE5LjQzNCw3NjkuNjNsLTAsMzM1OC43NWMtMCw5NC42NDQgLTM0Ljg2NywxNzYuODQxIC0xMDQuNjA2LDI0Ni41ODFjLTY5Ljc0Niw2OS43MzkgLTE1NC40MjcsMTA0LjYxMiAtMjU0LjA1NSwxMDQuNjEybC0wLjU4MiwtMC4wMDZaIiBzdHlsZT0ic3Ryb2tlOiMxODVBREI7c3Ryb2tlLXdpZHRoOjQuMTdweDsiLz48L3N2Zz4K open diff --git a/front-end/config.js b/front-end/config.js index c42b4a8..1c7c23c 100644 --- a/front-end/config.js +++ b/front-end/config.js @@ -9,6 +9,7 @@ export default { componentPrefix: 'mwmbl', publicApiURL: 'https://api.mwmbl.org/', + // publicApiURL: 'http://localhost:5000/', searchQueryParam: 'q', footerLinks: [ { diff --git a/front-end/package-lock.json b/front-end/package-lock.json index 285df52..277dfb4 100644 --- a/front-end/package-lock.json +++ b/front-end/package-lock.json @@ -5,9 +5,6 @@ "packages": { "": { "name": "front-end", - "dependencies": { - "chart.js": "^4.4.0" - }, "devDependencies": { "@vitejs/plugin-legacy": "^2.3.1", "terser": "^5.16.1", @@ -113,11 +110,6 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, - "node_modules/@kurkle/color": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz", - "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==" - }, "node_modules/@vitejs/plugin-legacy": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@vitejs/plugin-legacy/-/plugin-legacy-2.3.1.tgz", @@ -156,17 +148,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "node_modules/chart.js": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.0.tgz", - "integrity": "sha512-vQEj6d+z0dcsKLlQvbKIMYFHd3t8W/7L2vfJIbYcfyPcRx92CsHqECpueN8qVGNlKyDcr5wBrYAYKnfu/9Q1hQ==", - "dependencies": { - "@kurkle/color": "^0.3.0" - }, - "engines": { - "pnpm": ">=7" - } - }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -598,16 +579,10 @@ } }, "node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -628,9 +603,9 @@ "dev": true }, "node_modules/postcss": { - "version": "8.4.31", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", - "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "version": "8.4.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.19.tgz", + "integrity": "sha512-h+pbPsyhlYj6N2ozBmHhHrs9DzGmbaarbLvWipMRO7RLS+v4onj26MPFXA5OBYFxyqYhUJK456SwDcY9H2/zsA==", "dev": true, "funding": [ { @@ -640,14 +615,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" } ], "dependencies": { - "nanoid": "^3.3.6", + "nanoid": "^3.3.4", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" }, @@ -765,9 +736,9 @@ } }, "node_modules/vite": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/vite/-/vite-3.2.7.tgz", - "integrity": "sha512-29pdXjk49xAP0QBr0xXqu2s5jiQIXNvE/xwd0vUizYT2Hzqe4BksNNoWllFVXJf4eLZ+UlVQmXfB4lWrc+t18g==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-3.2.5.tgz", + "integrity": "sha512-4mVEpXpSOgrssFZAOmGIr85wPHKvaDAcXqxVxVRZhljkJOMZi1ibLibzjLHzJvcok8BMguLc7g1W6W/GqZbLdQ==", "dev": true, "dependencies": { "esbuild": "^0.15.9", @@ -884,11 +855,6 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, - "@kurkle/color": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz", - "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==" - }, "@vitejs/plugin-legacy": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@vitejs/plugin-legacy/-/plugin-legacy-2.3.1.tgz", @@ -914,14 +880,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "chart.js": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.0.tgz", - "integrity": "sha512-vQEj6d+z0dcsKLlQvbKIMYFHd3t8W/7L2vfJIbYcfyPcRx92CsHqECpueN8qVGNlKyDcr5wBrYAYKnfu/9Q1hQ==", - "requires": { - "@kurkle/color": "^0.3.0" - } - }, "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -1145,9 +1103,9 @@ } }, "nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", "dev": true }, "path-parse": { @@ -1163,12 +1121,12 @@ "dev": true }, "postcss": { - "version": "8.4.31", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", - "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "version": "8.4.19", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.19.tgz", + "integrity": "sha512-h+pbPsyhlYj6N2ozBmHhHrs9DzGmbaarbLvWipMRO7RLS+v4onj26MPFXA5OBYFxyqYhUJK456SwDcY9H2/zsA==", "dev": true, "requires": { - "nanoid": "^3.3.6", + "nanoid": "^3.3.4", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } @@ -1252,9 +1210,9 @@ } }, "vite": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/vite/-/vite-3.2.7.tgz", - "integrity": "sha512-29pdXjk49xAP0QBr0xXqu2s5jiQIXNvE/xwd0vUizYT2Hzqe4BksNNoWllFVXJf4eLZ+UlVQmXfB4lWrc+t18g==", + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-3.2.5.tgz", + "integrity": "sha512-4mVEpXpSOgrssFZAOmGIr85wPHKvaDAcXqxVxVRZhljkJOMZi1ibLibzjLHzJvcok8BMguLc7g1W6W/GqZbLdQ==", "dev": true, "requires": { "esbuild": "^0.15.9", diff --git a/front-end/package.json b/front-end/package.json index 53a57e0..76e8d53 100644 --- a/front-end/package.json +++ b/front-end/package.json @@ -11,8 +11,5 @@ "@vitejs/plugin-legacy": "^2.3.1", "terser": "^5.16.1", "vite": "^3.2.3" - }, - "dependencies": { - "chart.js": "^4.4.0" } } diff --git a/front-end/src/components/app.js b/front-end/src/components/app.js index 9644a6b..d53cb3b 100644 --- a/front-end/src/components/app.js +++ b/front-end/src/components/app.js @@ -1,16 +1,22 @@ import define from '../utils/define.js'; +import addResult from "./molecules/add-result.js"; +import save from "./organisms/save.js"; const template = () => /*html*/`
+
    +
  • +
- mwmbl logo + mwmbl logo MWMBL
-
- -
+
+ +
+
`; diff --git a/front-end/src/components/login.js b/front-end/src/components/login.js new file mode 100644 index 0000000..9546695 --- /dev/null +++ b/front-end/src/components/login.js @@ -0,0 +1,69 @@ +import define from '../utils/define.js'; +import config from "../../config.js"; + +const template = () => /*html*/` +
+
Login
+
+ +
+ +
+
+
+ +
+ + +
+
+
+ +
+
+`; + +export default define('login', class extends HTMLElement { + constructor() { + super(); + this.loginForm = null; + this.emailOrUsernameInput = null; + this.passwordInput = null; + this.__setup(); + this.__events(); + } + + __setup() { + this.innerHTML = template(); + this.loginForm = this.querySelector('form'); + this.emailOrUsernameInput = this.querySelector('#login-email-or-username'); + this.passwordInput = this.querySelector('#login-password'); + } + + __events() { + this.loginForm.addEventListener('submit', (e) => { + e.preventDefault(); + this.__handleLogin(e); + }); + } + + __handleLogin = async () => { + const response = await fetch(`${config.publicApiURL}user/login`, { + method: 'POST', + cache: 'no-cache', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + "username_or_email": this.emailOrUsernameInput.value, + "password": this.passwordInput.value, + }) + }); + if (response.status === 200) { + const loginData = await response.json(); + console.log("Login data", loginData); + document.cookie = `jwt=${loginData["jwt"]}; SameSite=Strict`; + console.log("Login success"); + } else { + console.log("Login error", response); + } + } +}); \ No newline at end of file diff --git a/front-end/src/components/molecules/add-button.js b/front-end/src/components/molecules/add-button.js new file mode 100644 index 0000000..d59884a --- /dev/null +++ b/front-end/src/components/molecules/add-button.js @@ -0,0 +1,24 @@ +import define from "../../utils/define.js"; +import {globalBus} from "../../utils/events.js"; +import addResult from "./add-result.js"; +import emptyResult from "./empty-result.js"; + + +export default define('add-button', class extends HTMLButtonElement { + constructor() { + super(); + this.__setup(); + } + + __setup() { + this.__events(); + } + + __events() { + this.addEventListener('click', (e) => { + console.log("Add button"); + document.querySelector('.modal').style.display = 'block'; + document.querySelector('.modal input').focus(); + }) + } +}, { extends: 'button' }); diff --git a/front-end/src/components/molecules/add-result.js b/front-end/src/components/molecules/add-result.js new file mode 100644 index 0000000..6e9ce17 --- /dev/null +++ b/front-end/src/components/molecules/add-result.js @@ -0,0 +1,69 @@ +import define from '../../utils/define.js'; +import config from "../../../config.js"; +import {globalBus} from "../../utils/events.js"; + + +const FETCH_URL = `${config['publicApiURL']}crawler/fetch?` + + +const template = () => /*html*/` +
+ × + + +
+`; + +export default define('add-result', class extends HTMLDivElement { + constructor() { + super(); + this.classList.add('modal'); + this.__setup(); + } + + __setup() { + this.innerHTML = template(); + this.__events(); + this.style.display = 'none'; + } + + __events() { + this.querySelector('.close').addEventListener('click', e => { + if (e.target === this) { + this.style.display = 'none'; + } + }); + + this.addEventListener('click', e => { + this.style.display = 'none'; + }); + + this.querySelector('form').addEventListener('click', e => { + // Clicking on the form shouldn't close it + e.stopPropagation(); + }); + + this.addEventListener('submit', this.__urlSubmitted.bind(this)); + } + + async __urlSubmitted(e) { + e.preventDefault(); + const value = this.querySelector('input').value; + console.log("Input value", value); + + const query = document.querySelector('.search-bar input').value; + + const url = `${FETCH_URL}url=${encodeURIComponent(value)}&query=${encodeURIComponent(query)}`; + const response = await fetch(url); + if (response.status === 200) { + const data = await response.json(); + console.log("Data", data); + + const addResultEvent = new CustomEvent('curate-add-result', {detail: data}); + globalBus.dispatch(addResultEvent); + } else { + console.log("Bad response", response); + // TODO + } + } +}, { extends: 'div' }); diff --git a/front-end/src/components/molecules/delete-button.js b/front-end/src/components/molecules/delete-button.js new file mode 100644 index 0000000..1914684 --- /dev/null +++ b/front-end/src/components/molecules/delete-button.js @@ -0,0 +1,35 @@ +import define from "../../utils/define.js"; +import {globalBus} from "../../utils/events.js"; + + +export default define('delete-button', class extends HTMLButtonElement { + constructor() { + super(); + this.__setup(); + } + + __setup() { + this.__events(); + } + + __events() { + this.addEventListener('click', (e) => { + console.log("Delete button"); + + const result = this.closest('.result'); + const parent = result.parentNode; + + const index = Array.prototype.indexOf.call(parent.children, result); + console.log("Delete index", index); + + const beginCuratingEvent = new CustomEvent('curate-delete-result', { + detail: { + data: { + delete_index: index + } + } + }); + globalBus.dispatch(beginCuratingEvent); + }) + } +}, { extends: 'button' }); diff --git a/front-end/src/components/molecules/result.js b/front-end/src/components/molecules/result.js index 696a7a1..2a2859e 100644 --- a/front-end/src/components/molecules/result.js +++ b/front-end/src/components/molecules/result.js @@ -1,13 +1,25 @@ import define from '../../utils/define.js'; import escapeString from '../../utils/escapeString.js'; import { globalBus } from '../../utils/events.js'; +import deleteButton from "./delete-button.js"; +import validateButton from "./validate-button.js"; +import addButton from "./add-button.js"; const template = ({ data }) => /*html*/` - -

${data.url}

-

${data.title}

-

${data.extract}

- +
+
+ + + +
+
+ +

${data.url}

+

${data.title}

+

${data.extract}

+ +
+
`; export default define('result', class extends HTMLLIElement { diff --git a/front-end/src/components/molecules/validate-button.js b/front-end/src/components/molecules/validate-button.js new file mode 100644 index 0000000..65e3b10 --- /dev/null +++ b/front-end/src/components/molecules/validate-button.js @@ -0,0 +1,53 @@ +import define from "../../utils/define.js"; +import {globalBus} from "../../utils/events.js"; + + +const VALIDATED_CLASS = "validated"; + +export default define('validate-button', class extends HTMLButtonElement { + constructor() { + super(); + this.__setup(); + } + + __setup() { + this.__events(); + } + + __events() { + this.addEventListener('click', (e) => { + console.log("Validate button"); + + const result = this.closest('.result'); + const parent = result.parentNode; + + const index = Array.prototype.indexOf.call(parent.children, result); + console.log("Validate index", index); + + const curationValidateEvent = new CustomEvent('curate-validate-result', { + detail: { + data: { + validate_index: index + } + } + }); + globalBus.dispatch(curationValidateEvent); + }) + } + + isValidated() { + return this.classList.contains(VALIDATED_CLASS); + } + + validate() { + this.classList.add(VALIDATED_CLASS); + } + + unvalidate() { + this.classList.remove(VALIDATED_CLASS); + } + + toggleValidate() { + this.classList.toggle(VALIDATED_CLASS); + } +}, { extends: 'button' }); diff --git a/front-end/src/components/organisms/footer.js b/front-end/src/components/organisms/footer.js index 73344cb..4b8d62f 100644 --- a/front-end/src/components/organisms/footer.js +++ b/front-end/src/components/organisms/footer.js @@ -4,14 +4,14 @@ import config from '../../../config.js'; const template = ({ data }) => /*html*/`

Find more on

    - ${data.links.map(link => /*html*/` + ${ data.links.map(link => /*html*/`
  • - + ${link.name}
    • - `).join('')} + `).join('') }
`; @@ -22,7 +22,7 @@ export default define('footer', class extends HTMLElement { } __setup() { - this.innerHTML = template({ + this.innerHTML = template({ data: { links: config.footerLinks } @@ -31,6 +31,6 @@ export default define('footer', class extends HTMLElement { } __events() { - + } }, { extends: 'footer' }); \ No newline at end of file diff --git a/front-end/src/components/organisms/results.js b/front-end/src/components/organisms/results.js index 40f558c..37fefe5 100644 --- a/front-end/src/components/organisms/results.js +++ b/front-end/src/components/organisms/results.js @@ -1,5 +1,5 @@ import define from '../../utils/define.js'; -import { globalBus } from '../../utils/events.js'; +import {globalBus} from '../../utils/events.js'; // Components import result from '../molecules/result.js'; @@ -17,6 +17,8 @@ export default define('results', class extends HTMLElement { constructor() { super(); this.results = null; + this.oldIndex = null; + this.curating = false; this.__setup(); } @@ -31,7 +33,7 @@ export default define('results', class extends HTMLElement { this.results.innerHTML = ''; let resultsHTML = ''; if (!e.detail.error) { - // If there is no details the input is empty + // If there is no details the input is empty if (!e.detail.results) { resultsHTML = /*html*/`
  • @@ -42,7 +44,7 @@ export default define('results', class extends HTMLElement { for(const resultData of e.detail.results) { resultsHTML += /*html*/`
  • { this.results.firstElementChild.firstElementChild.focus(); - }) + }); + + globalBus.on('curate-delete-result', (e) => { + console.log("Curate delete result event", e); + this.__beginCurating.bind(this)(); + + const children = this.results.getElementsByClassName('result'); + let deleteIndex = e.detail.data.delete_index; + const child = children[deleteIndex]; + this.results.removeChild(child); + const newResults = this.__getResults(); + + const curationSaveEvent = new CustomEvent('save-curation', { + detail: { + type: 'delete', + data: { + url: document.location.href, + results: newResults, + curation: { + delete_index: deleteIndex + } + } + } + }); + globalBus.dispatch(curationSaveEvent); + }); + + globalBus.on('curate-validate-result', (e) => { + console.log("Curate validate result event", e); + this.__beginCurating.bind(this)(); + + const children = this.results.getElementsByClassName('result'); + const validateChild = children[e.detail.data.validate_index]; + validateChild.querySelector('.curate-approve').toggleValidate(); + + const newResults = this.__getResults(); + + const curationStartEvent = new CustomEvent('save-curation', { + detail: { + type: 'validate', + data: { + url: document.location.href, + results: newResults, + curation: e.detail.data + } + } + }); + globalBus.dispatch(curationStartEvent); + }); + + globalBus.on('begin-curating-results', (e) => { + // We might not be online, or logged in, so save the curation in local storage in case: + console.log("Begin curation event", e); + this.__beginCurating.bind(this)(); + }); + + globalBus.on('curate-add-result', (e) => { + console.log("Add result", e); + this.__beginCurating(); + const resultData = e.detail; + const resultHTML = /*html*/` +
  • + `; + this.results.insertAdjacentHTML('afterbegin', resultHTML); + + const newResults = this.__getResults(); + + const curationSaveEvent = new CustomEvent('save-curation', { + detail: { + type: 'add', + data: { + url: document.location.href, + results: newResults, + curation: { + insert_index: 0, + url: e.detail.url + } + } + } + }); + globalBus.dispatch(curationSaveEvent); + + }); + } + + __sortableActivate(event, ui) { + console.log("Sortable activate", ui); + this.__beginCurating(); + this.oldIndex = ui.item.index(); + } + + __beginCurating() { + if (!this.curating) { + const results = this.__getResults(); + const curationStartEvent = new CustomEvent('save-curation', { + detail: { + type: 'begin', + data: { + url: document.location.href, + results: results + } + } + }); + globalBus.dispatch(curationStartEvent); + this.curating = true; + } + } + + __getResults() { + const resultsElements = document.querySelectorAll('.results .result:not(.ui-sortable-placeholder)'); + const results = []; + for (let resultElement of resultsElements) { + const result = { + url: resultElement.querySelector('a').href, + title: resultElement.querySelector('.title').innerText, + extract: resultElement.querySelector('.extract').innerText, + curated: resultElement.querySelector('.curate-approve').isValidated() + } + results.push(result); + } + console.log("Results", results); + return results; + } + + __sortableDeactivate(event, ui) { + const newIndex = ui.item.index(); + console.log('Sortable deactivate', ui, this.oldIndex, newIndex); + + const newResults = this.__getResults(); + + const curationMoveEvent = new CustomEvent('save-curation', { + detail: { + type: 'move', + data: { + url: document.location.href, + results: newResults, + curation: { + old_index: this.oldIndex, + new_index: newIndex, + } + } + } + }); + globalBus.dispatch(curationMoveEvent); } }); \ No newline at end of file diff --git a/front-end/src/components/organisms/save.js b/front-end/src/components/organisms/save.js new file mode 100644 index 0000000..dc2f883 --- /dev/null +++ b/front-end/src/components/organisms/save.js @@ -0,0 +1,122 @@ +import define from '../../utils/define.js'; +import {globalBus} from "../../utils/events.js"; +import config from "../../../config.js"; + + +const CURATION_KEY_PREFIX = "curation-"; +const CURATION_URL = config.publicApiURL + "user/curation/"; + + +const template = () => /*html*/` + 🖫 +`; + + +export default define('save', class extends HTMLLIElement { + constructor() { + super(); + this.currentCurationId = null; + this.classList.add('save'); + this.sendId = 0; + this.sending = false; + this.__setup(); + } + + __setup() { + this.innerHTML = template(); + this.__events(); + // TODO: figure out when to call __sendToApi() + // setInterval(this.__sendToApi.bind(this), 1000); + } + + __events() { + globalBus.on('save-curation', (e) => { + // We might not be online, or logged in, so save the curation in local storage in case: + console.log("Curation event", e); + this.__setCuration(e.detail); + this.__sendToApi(); + }); + } + + __setCuration(curation) { + this.sendId += 1; + const key = CURATION_KEY_PREFIX + this.sendId; + localStorage.setItem(key, JSON.stringify(curation)); + } + + __getOldestCurationKey() { + let oldestId = Number.MAX_SAFE_INTEGER; + let oldestKey = null; + for (let i=0; i row.startsWith('jwt=')) + ?.split('=')[1]; + + if (!auth) { + console.log("No auth"); + return; + } + + if (localStorage.length > 0) { + const key = this.__getOldestCurationKey(); + const value = JSON.parse(localStorage.getItem(key)); + console.log("Value", value); + const url = CURATION_URL + value['type']; + + let data = value['data']; + if (value.type !== 'begin') { + if (this.currentCurationId === null) { + throw ReferenceError("No current curation found"); + } + data['curation_id'] = this.currentCurationId; + } + data['auth'] = auth; + + console.log("Data", data); + const response = await fetch(url, { + method: 'POST', + cache: 'no-cache', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify(data), + }); + + console.log("Save curation API response", response); + + if (response.status === 200) { + localStorage.removeItem(key); + } else { + console.log("Bad response, skipping"); + return; + } + + const responseData = await response.json(); + console.log("Response data", responseData); + if (responseData["curation_id"]) { + this.currentCurationId = responseData["curation_id"]; + } + + // There may be more to send, wait a second and see + setTimeout(this.__sendToApi.bind(this), 1000); + } + this.sending = false; + } +}, { extends: 'li' }); + diff --git a/front-end/src/components/organisms/search-bar.js b/front-end/src/components/organisms/search-bar.js index f700fda..5adf52b 100644 --- a/front-end/src/components/organisms/search-bar.js +++ b/front-end/src/components/organisms/search-bar.js @@ -144,8 +144,12 @@ export default define('search-bar', class extends HTMLElement { // Focus search bar when pressing `ctrl + k` or `/` document.addEventListener('keydown', (e) => { - if ((e.key === 'k' && e.ctrlKey) || e.key === '/' || e.key === 'Escape') { + if ((e.key === 'k' && e.ctrlKey) || e.key === 'Escape') { e.preventDefault(); + + // Remove the modal if it's visible + document.querySelector('.modal').style.display = 'none'; + this.searchInput.focus(); } }); diff --git a/front-end/src/components/register.js b/front-end/src/components/register.js new file mode 100644 index 0000000..ff5bfc0 --- /dev/null +++ b/front-end/src/components/register.js @@ -0,0 +1,84 @@ +import define from '../utils/define.js'; +import config from "../../config.js"; + +const template = () => /*html*/` +
    +
    Register
    +
    + +
    + +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    +
    +`; + +export default define('register', class extends HTMLElement { + constructor() { + super(); + this.registerForm = null; + this.emailInput = null; + this.usernameInput = null; + this.passwordInput = null; + this.passwordVerifyInput = null; + this.__setup(); + this.__events(); + } + + __setup() { + this.innerHTML = template(); + this.registerForm = this.querySelector('form'); + this.emailInput = this.querySelector('#register-email'); + this.usernameInput = this.querySelector('#register-username'); + this.passwordInput = this.querySelector('#register-password'); + this.passwordVerifyInput = this.querySelector('#register-password-verify'); + } + + __events() { + this.registerForm.addEventListener('submit', (e) => { + e.preventDefault(); + this.__handleRegister(e); + }); + } + + __handleRegister = async () => { + const response = await fetch(`${config.publicApiURL}user/register`, { + method: 'POST', + cache: 'no-cache', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + "username": this.usernameInput.value, + "email": this.emailInput.value, + "password": this.passwordInput.value, + "password_verify": this.passwordVerifyInput.value, + }) + }); + if (response.status === 200) { + const registerData = await response.json(); + console.log("Register data", registerData); + document.cookie = `jwt=${registerData["jwt"]}; SameSite=Strict`; + console.log("Register success"); + } else { + console.log("Register error", response); + } + } +}); \ No newline at end of file diff --git a/front-end/src/index.html b/front-end/src/index.html index 23c0b52..843bfc8 100644 --- a/front-end/src/index.html +++ b/front-end/src/index.html @@ -35,21 +35,28 @@ - + + + + + + + +