From 8f339923f87a989c58589251856e207a73e684d1 Mon Sep 17 00:00:00 2001 From: Carlos Quintana Date: Thu, 24 Feb 2022 15:05:05 +0100 Subject: [PATCH 01/35] Make nameservers configurable --- app/config.py | 17 ++++++++++++++++- app/dns_utils.py | 15 +++++++-------- example.env | 6 +++++- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/app/config.py b/app/config.py index 2148eb42..7578f01f 100644 --- a/app/config.py +++ b/app/config.py @@ -4,7 +4,7 @@ import socket import string import subprocess from ast import literal_eval -from typing import Callable, List +from typing import Callable, List, Optional from urllib.parse import urlparse from dotenv import load_dotenv @@ -37,6 +37,13 @@ def sl_getenv(env_var: str, default_factory: Callable = None): return literal_eval(value) +def env(var: str, default: Optional[str] = None) -> Optional[str]: + if var in os.environ: + return os.environ[var] + else: + return default + + config_file = os.environ.get("CONFIG") if config_file: config_file = get_abs_path(config_file) @@ -431,3 +438,11 @@ def get_allowed_redirect_domains() -> List[str]: ALLOWED_REDIRECT_DOMAINS = get_allowed_redirect_domains() + + +def setup_nameservers(): + nameservers = env("NAMESERVERS", "1.1.1.1") + return nameservers.split(",") + + +NAMESERVERS = setup_nameservers() diff --git a/app/dns_utils.py b/app/dns_utils.py index 0c27b939..80e25043 100644 --- a/app/dns_utils.py +++ b/app/dns_utils.py @@ -1,3 +1,4 @@ +from app import config from typing import Optional, List, Tuple import dns.resolver @@ -5,16 +6,14 @@ import dns.resolver def _get_dns_resolver(): my_resolver = dns.resolver.Resolver() - - # 1.1.1.1 is CloudFlare's public DNS server - my_resolver.nameservers = ["1.1.1.1"] + my_resolver.nameservers = config.NAMESERVERS return my_resolver def get_ns(hostname) -> [str]: try: - answers = _get_dns_resolver().resolve(hostname, "NS") + answers = _get_dns_resolver().resolve(hostname, "NS", search=True) except Exception: return [] return [a.to_text() for a in answers] @@ -23,7 +22,7 @@ def get_ns(hostname) -> [str]: def get_cname_record(hostname) -> Optional[str]: """Return the CNAME record if exists for a domain, WITHOUT the trailing period at the end""" try: - answers = _get_dns_resolver().resolve(hostname, "CNAME") + answers = _get_dns_resolver().resolve(hostname, "CNAME", search=True) except Exception: return None @@ -39,7 +38,7 @@ def get_mx_domains(hostname) -> [(int, str)]: domain name ends with a "." at the end. """ try: - answers = _get_dns_resolver().resolve(hostname, "MX") + answers = _get_dns_resolver().resolve(hostname, "MX", search=True) except Exception: return [] @@ -60,7 +59,7 @@ _include_spf = "include:" def get_spf_domain(hostname) -> [str]: """return all domains listed in *include:*""" try: - answers = _get_dns_resolver().resolve(hostname, "TXT") + answers = _get_dns_resolver().resolve(hostname, "TXT", search=True) except Exception: return [] @@ -82,7 +81,7 @@ def get_spf_domain(hostname) -> [str]: def get_txt_record(hostname) -> [str]: """return all domains listed in *include:*""" try: - answers = _get_dns_resolver().resolve(hostname, "TXT") + answers = _get_dns_resolver().resolve(hostname, "TXT", search=True) except Exception: return [] diff --git a/example.env b/example.env index e3b9cadb..31429f75 100644 --- a/example.env +++ b/example.env @@ -174,4 +174,8 @@ DISABLE_ONBOARDING=true #ALIAS_AUTOMATIC_DISABLE=true # domains that can be present in the &next= section when using absolute urls -ALLOWED_REDIRECT_DOMAINS=[] \ No newline at end of file +ALLOWED_REDIRECT_DOMAINS=[] + +# DNS nameservers to be used by the app +# Multiple nameservers can be specified, separated by ',' +NAMESERVERS="1.1.1.1" From 01cc65bdcaf2cf008124a2ae386d4392d3256989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 24 Feb 2022 17:23:45 +0100 Subject: [PATCH 02/35] Allow to have lower priority MX servers --- app/dns_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/dns_utils.py b/app/dns_utils.py index 0c27b939..34f0e8eb 100644 --- a/app/dns_utils.py +++ b/app/dns_utils.py @@ -112,10 +112,10 @@ def is_mx_equivalent( ref_mx_domains, key=lambda priority_domain: priority_domain[0] ) - if len(mx_domains) != len(ref_mx_domains): + if len(mx_domains) < len(ref_mx_domains): return False - for i in range(0, len(mx_domains)): + for i in range(0, len(ref_mx_domains)): if mx_domains[i][1] != ref_mx_domains[i][1]: return False From 77cf5d9620c7e3ea36f2a9a89f90785b4f9f316f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 24 Feb 2022 17:25:48 +0100 Subject: [PATCH 03/35] Added tests --- tests/test_dns_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_dns_utils.py b/tests/test_dns_utils.py index 0772fce5..e8e9285f 100644 --- a/tests/test_dns_utils.py +++ b/tests/test_dns_utils.py @@ -38,3 +38,9 @@ def test_is_mx_equivalent(): assert is_mx_equivalent( [(5, "domain1"), (10, "domain2")], [(10, "domain1"), (20, "domain2")] ) + assert is_mx_equivalent( + [(5, "domain1"), (10, "domain2"), (20, "domain3")], [(10, "domain1"), (20, "domain2")] + ) + assert not is_mx_equivalent( + [(5, "domain1"), (10, "domain2")], [(10, "domain1"), (20, "domain2"), (20, "domain3")] + ) From 0c008edc820565b850e88808666d21f859b5544e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 24 Feb 2022 17:30:07 +0100 Subject: [PATCH 04/35] Format --- tests/test_dns_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_dns_utils.py b/tests/test_dns_utils.py index e8e9285f..46a6ccdd 100644 --- a/tests/test_dns_utils.py +++ b/tests/test_dns_utils.py @@ -39,8 +39,10 @@ def test_is_mx_equivalent(): [(5, "domain1"), (10, "domain2")], [(10, "domain1"), (20, "domain2")] ) assert is_mx_equivalent( - [(5, "domain1"), (10, "domain2"), (20, "domain3")], [(10, "domain1"), (20, "domain2")] + [(5, "domain1"), (10, "domain2"), (20, "domain3")], + [(10, "domain1"), (20, "domain2")], ) assert not is_mx_equivalent( - [(5, "domain1"), (10, "domain2")], [(10, "domain1"), (20, "domain2"), (20, "domain3")] + [(5, "domain1"), (10, "domain2")], + [(10, "domain1"), (20, "domain2"), (20, "domain3")], ) From 3d498b4eae3814004d09b114627c1f310a272a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 24 Feb 2022 18:28:30 +0100 Subject: [PATCH 05/35] Allow drag and drop of keys into the text area --- templates/dashboard/contact_detail.html | 37 ++++++++++++++++++++++- templates/dashboard/mailbox_detail.html | 39 +++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/templates/dashboard/contact_detail.html b/templates/dashboard/contact_detail.html index 25a79776..ea5da431 100644 --- a/templates/dashboard/contact_detail.html +++ b/templates/dashboard/contact_detail.html @@ -49,8 +49,10 @@ +
You can drag and drop the pgp key into the text area
+ + + + +
Alias Import
From 6c8d4310e5c3ac365e5de16009f3536cdd4c7f03 Mon Sep 17 00:00:00 2001 From: Son Date: Fri, 25 Feb 2022 12:22:09 +0100 Subject: [PATCH 09/35] only set the X-SimpleLogin-Envelope-From header if user has this option enabled --- email_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 3f36eabf..525a30f9 100644 --- a/email_handler.py +++ b/email_handler.py @@ -801,7 +801,8 @@ def forward_email_to_mailbox( add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") msg[headers.SL_EMAIL_LOG_ID] = str(email_log.id) - msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from + if user.include_header_email_header: + msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from # when an alias isn't in the To: header, there's no way for users to know what alias has received the email msg[headers.SL_ENVELOPE_TO] = alias.email From 61d16555298abd7bddd2a81026dde243c9f8f8e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 25 Feb 2022 14:58:38 +0100 Subject: [PATCH 10/35] Move all js to a source file --- static/js/utils/drag-drop-into-text.js | 28 +++++++++++++++++ templates/dashboard/contact_detail.html | 34 ++------------------- templates/dashboard/mailbox_detail.html | 40 +++---------------------- 3 files changed, 35 insertions(+), 67 deletions(-) create mode 100644 static/js/utils/drag-drop-into-text.js diff --git a/static/js/utils/drag-drop-into-text.js b/static/js/utils/drag-drop-into-text.js new file mode 100644 index 00000000..b1a4464f --- /dev/null +++ b/static/js/utils/drag-drop-into-text.js @@ -0,0 +1,28 @@ +const MAX_BYTES = 1240; // 10KiB + +function enableDragDropForPGPKeys(inputID) { + function drop(event) { + event.stopPropagation(); + event.preventDefault(); + + let files = event.dataTransfer.files; + for (let i = 0; i < files.length; i++) { + let file = files[i]; + if(file.type !== 'text/plain'){ + toastr.warning(`File ${file.name} is not a public key file`); + continue; + } + let reader = new FileReader(); + reader.onloadend = onFileLoaded; + reader.readAsBinaryString(file); + } + } + + function onFileLoaded(event) { + const initialData = event.currentTarget.result.substr(0, MAX_BYTES); + $(inputID).val(initialData); + } + + const dropArea = $(inputID).get(0); + dropArea.addEventListener("drop", drop, false); +} \ No newline at end of file diff --git a/templates/dashboard/contact_detail.html b/templates/dashboard/contact_detail.html index ea5da431..f6fc405b 100644 --- a/templates/dashboard/contact_detail.html +++ b/templates/dashboard/contact_detail.html @@ -51,7 +51,7 @@ {% if not current_user.is_premium() %} disabled {% endif %} class="form-control" rows=10 id="pgp-public-key" placeholder="-----BEGIN PGP PUBLIC KEY BLOCK-----">{{ contact.pgp_public_key or "" }} -
You can drag and drop the pgp key into the text area
+
You can drag and drop the pgp key file into the text area
@@ -73,36 +73,8 @@ {% endblock %} {% block script %} - {% if current_user.is_premium() %} + - {% endif %} {% endblock %} diff --git a/templates/dashboard/mailbox_detail.html b/templates/dashboard/mailbox_detail.html index 683d2e32..a63394b5 100644 --- a/templates/dashboard/mailbox_detail.html +++ b/templates/dashboard/mailbox_detail.html @@ -125,7 +125,7 @@ {% if not current_user.is_premium() %} disabled {% endif %} class="form-control" rows=10 id="pgp-public-key" placeholder="-----BEGIN PGP PUBLIC KEY BLOCK-----">{{ mailbox.pgp_public_key or "" }} -
You can drag and drop the pgp key into the text area
+
You can drag and drop the pgp key file into the text area
@@ -264,44 +264,12 @@ {% endblock %} {% block script %} - + + - - {% if current_user.is_premium() %} - - {% endif %} {% endblock %} From c2ae38ec8fa5ab8caa9e2a202372b2e9f975b2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 25 Feb 2022 15:01:17 +0100 Subject: [PATCH 11/35] typo --- static/js/utils/drag-drop-into-text.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/js/utils/drag-drop-into-text.js b/static/js/utils/drag-drop-into-text.js index b1a4464f..ab4b062f 100644 --- a/static/js/utils/drag-drop-into-text.js +++ b/static/js/utils/drag-drop-into-text.js @@ -1,4 +1,4 @@ -const MAX_BYTES = 1240; // 10KiB +const MAX_BYTES = 10240; // 10KiB function enableDragDropForPGPKeys(inputID) { function drop(event) { From 9c67aad34da366b3d2853c73d63807bb69df4e88 Mon Sep 17 00:00:00 2001 From: Son Date: Sat, 26 Feb 2022 15:29:33 +0100 Subject: [PATCH 12/35] remove "reply to this email" --- templates/emails/transactional/cycle-email.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/templates/emails/transactional/cycle-email.html b/templates/emails/transactional/cycle-email.html index 7c705cf8..cc68a678 100644 --- a/templates/emails/transactional/cycle-email.html +++ b/templates/emails/transactional/cycle-email.html @@ -19,9 +19,5 @@ The email is automatically deleted in 7 days. {% endcall %} - {% call text() %} - If you have any question, please reply to this email. - {% endcall %} - {{ render_text('Thanks,
SimpleLogin Team.') }} {% endblock %} From fa95f4273d655c51f7292ace87e22f704a6092d4 Mon Sep 17 00:00:00 2001 From: Son Date: Sat, 26 Feb 2022 16:12:44 +0100 Subject: [PATCH 13/35] ui tweak --- templates/dashboard/contact_detail.html | 3 ++- templates/dashboard/mailbox_detail.html | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/dashboard/contact_detail.html b/templates/dashboard/contact_detail.html index f6fc405b..df09090e 100644 --- a/templates/dashboard/contact_detail.html +++ b/templates/dashboard/contact_detail.html @@ -44,6 +44,7 @@ {% endif %} +
You can drag and drop the pgp key file into the text area
@@ -51,7 +52,7 @@ {% if not current_user.is_premium() %} disabled {% endif %} class="form-control" rows=10 id="pgp-public-key" placeholder="-----BEGIN PGP PUBLIC KEY BLOCK-----">{{ contact.pgp_public_key or "" }} -
You can drag and drop the pgp key file into the text area
+
diff --git a/templates/dashboard/mailbox_detail.html b/templates/dashboard/mailbox_detail.html index a63394b5..9af4983e 100644 --- a/templates/dashboard/mailbox_detail.html +++ b/templates/dashboard/mailbox_detail.html @@ -117,6 +117,7 @@ {% endif %} +
You can drag and drop the pgp key file into the text area
@@ -125,7 +126,7 @@ {% if not current_user.is_premium() %} disabled {% endif %} class="form-control" rows=10 id="pgp-public-key" placeholder="-----BEGIN PGP PUBLIC KEY BLOCK-----">{{ mailbox.pgp_public_key or "" }} -
You can drag and drop the pgp key file into the text area
+
From 4faf0d763646695769e6a4b9243440c8e9d84d30 Mon Sep 17 00:00:00 2001 From: Son Date: Sat, 26 Feb 2022 17:34:53 +0100 Subject: [PATCH 14/35] optimize dashboard page: load custom domain using joinedload() instead of explicit join --- app/api/serializer.py | 11 +++++------ templates/dashboard/index.html | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/app/api/serializer.py b/app/api/serializer.py index 2d9546b6..21b801de 100644 --- a/app/api/serializer.py +++ b/app/api/serializer.py @@ -204,7 +204,7 @@ def get_alias_infos_with_pagination_v3( q = list(q.limit(page_limit).offset(page_id * page_size)) ret = [] - for alias, contact, email_log, custom_domain, nb_reply, nb_blocked, nb_forward in q: + for alias, contact, email_log, nb_reply, nb_blocked, nb_forward in q: ret.append( AliasInfo( alias=alias, @@ -215,7 +215,7 @@ def get_alias_infos_with_pagination_v3( nb_reply=nb_reply, latest_email_log=email_log, latest_contact=contact, - custom_domain=custom_domain, + custom_domain=alias.custom_domain, ) ) @@ -318,7 +318,7 @@ def get_alias_info_v3(user: User, alias_id: int) -> AliasInfo: q = construct_alias_query(user) q = q.filter(Alias.id == alias_id) - for alias, contact, email_log, custom_domain, nb_reply, nb_blocked, nb_forward in q: + for alias, contact, email_log, nb_reply, nb_blocked, nb_forward in q: return AliasInfo( alias=alias, mailbox=alias.mailbox, @@ -328,7 +328,7 @@ def get_alias_info_v3(user: User, alias_id: int) -> AliasInfo: nb_reply=nb_reply, latest_email_log=email_log, latest_contact=contact, - custom_domain=custom_domain, + custom_domain=alias.custom_domain, ) @@ -379,14 +379,13 @@ def construct_alias_query(user: User): Alias, Contact, EmailLog, - CustomDomain, alias_activity_subquery.c.nb_reply, alias_activity_subquery.c.nb_blocked, alias_activity_subquery.c.nb_forward, ) .options(joinedload(Alias.hibp_breaches)) + .options(joinedload(Alias.custom_domain)) .join(Contact, Alias.id == Contact.alias_id, isouter=True) - .join(CustomDomain, Alias.custom_domain_id == CustomDomain.id, isouter=True) .join(EmailLog, Contact.id == EmailLog.contact_id, isouter=True) .filter(Alias.id == alias_activity_subquery.c.id) .filter(Alias.id == alias_contact_subquery.c.id) diff --git a/templates/dashboard/index.html b/templates/dashboard/index.html index 853206fb..225c3cf0 100644 --- a/templates/dashboard/index.html +++ b/templates/dashboard/index.html @@ -284,7 +284,7 @@ {% endif %} - {% if alias_info.custom_domain and not alias_info.custom_domain.verified %} + {% if alias.custom_domain and not alias.custom_domain.verified %} {% endif %} From 205d8d7d3f1e629d5ad86fa7be3aa30604dae8d8 Mon Sep 17 00:00:00 2001 From: Son Date: Sat, 26 Feb 2022 17:51:50 +0100 Subject: [PATCH 15/35] add index for Alias custom_domain_id and directory_id columns --- app/models.py | 4 +-- .../versions/2022_022617_5047fcbd57c7_.py | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 migrations/versions/2022_022617_5047fcbd57c7_.py diff --git a/app/models.py b/app/models.py index 512a3050..4c05683d 100644 --- a/app/models.py +++ b/app/models.py @@ -1160,7 +1160,7 @@ class Alias(Base, ModelMixin): enabled = sa.Column(sa.Boolean(), default=True, nullable=False) custom_domain_id = sa.Column( - sa.ForeignKey("custom_domain.id", ondelete="cascade"), nullable=True + sa.ForeignKey("custom_domain.id", ondelete="cascade"), nullable=True, index=True ) custom_domain = orm.relationship("CustomDomain", foreign_keys=[custom_domain_id]) @@ -1172,7 +1172,7 @@ class Alias(Base, ModelMixin): # to know whether an alias belongs to a directory directory_id = sa.Column( - sa.ForeignKey("directory.id", ondelete="cascade"), nullable=True + sa.ForeignKey("directory.id", ondelete="cascade"), nullable=True, index=True ) note = sa.Column(sa.Text, default=None, nullable=True) diff --git a/migrations/versions/2022_022617_5047fcbd57c7_.py b/migrations/versions/2022_022617_5047fcbd57c7_.py new file mode 100644 index 00000000..0485d29d --- /dev/null +++ b/migrations/versions/2022_022617_5047fcbd57c7_.py @@ -0,0 +1,31 @@ +"""empty message + +Revision ID: 5047fcbd57c7 +Revises: 9282e982bc05 +Create Date: 2022-02-26 17:51:03.379676 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5047fcbd57c7' +down_revision = '9282e982bc05' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('ix_alias_custom_domain_id'), 'alias', ['custom_domain_id'], unique=False) + op.create_index(op.f('ix_alias_directory_id'), 'alias', ['directory_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_alias_directory_id'), table_name='alias') + op.drop_index(op.f('ix_alias_custom_domain_id'), table_name='alias') + # ### end Alembic commands ### From 3d1a960702c4fb4126ff3e433ac74ac78f41f5a2 Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 28 Feb 2022 10:13:19 +0100 Subject: [PATCH 16/35] fix duplicated stats --- cron.py | 56 +++++++++++++++++------------------------------------ crontab.yml | 8 +------- 2 files changed, 19 insertions(+), 45 deletions(-) diff --git a/cron.py b/cron.py index ced9a6b9..0c64eac4 100644 --- a/cron.py +++ b/cron.py @@ -465,7 +465,7 @@ def alias_creation_report() -> List[Tuple[str, int]]: return res -def growth_stats(): +def stats(): """send admin stats everyday""" if not ADMIN_EMAIL: LOG.w("ADMIN_EMAIL not set, nothing to do") @@ -480,7 +480,7 @@ def growth_stats(): today = arrow.now().format() - report = f""" + growth_stats = f""" Growth Stats for {today} nb_user: {stats_today.nb_user} - {increase_percent(stats_yesterday.nb_user, stats_today.nb_user)} @@ -507,32 +507,16 @@ nb_referred_user: {stats_today.nb_referred_user} - {increase_percent(stats_yeste nb_referred_user_upgrade: {stats_today.nb_referred_user_paid} - {increase_percent(stats_yesterday.nb_referred_user_paid, stats_today.nb_referred_user_paid)} """ - LOG.d("report email: %s", report) + LOG.d("growth_stats email: %s", growth_stats) send_email( ADMIN_EMAIL, subject=f"SimpleLogin Growth Stats for {today}", - plaintext=report, + plaintext=growth_stats, retries=3, ) - -def daily_monitoring_report(): - """send monitoring stats of the previous day""" - if not MONITORING_EMAIL: - LOG.w("MONITORING_EMAIL not set, nothing to do") - return - - stats_today = compute_metric2() - stats_yesterday = ( - Metric2.filter(Metric2.date < stats_today.date) - .order_by(Metric2.date.desc()) - .first() - ) - - today = arrow.now().format() - - report = f""" + monitoring_report = f""" Monitoring Stats for {today} nb_alias: {stats_today.nb_alias} - {increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)} @@ -545,32 +529,32 @@ nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_p """ - report += "\n====================================\n" - report += f""" + monitoring_report += "\n====================================\n" + monitoring_report += f""" # Account bounce report: """ for email, bounces in bounce_report(): - report += f"{email}: {bounces}\n" + monitoring_report += f"{email}: {bounces}\n" - report += f"""\n + monitoring_report += f"""\n # Alias creation report: """ for email, nb_alias, date in alias_creation_report(): - report += f"{email}, {date}: {nb_alias}\n" + monitoring_report += f"{email}, {date}: {nb_alias}\n" - report += f"""\n + monitoring_report += f"""\n # Full bounce detail report: """ - report += all_bounce_report() + monitoring_report += all_bounce_report() - LOG.d("report email: %s", report) + LOG.d("monitoring_report email: %s", monitoring_report) send_email( MONITORING_EMAIL, subject=f"SimpleLogin Monitoring Report for {today}", - plaintext=report, + plaintext=monitoring_report, retries=3, ) @@ -1040,8 +1024,7 @@ if __name__ == "__main__": help="Choose a cron job to run", type=str, choices=[ - "growth_stats", - "daily_monitoring_report", + "stats", "notify_trial_end", "notify_manual_subscription_end", "notify_premium_end", @@ -1057,12 +1040,9 @@ if __name__ == "__main__": args = parser.parse_args() # wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc with create_light_app().app_context(): - if args.job == "growth_stats": - LOG.d("Compute growth Stats") - growth_stats() - if args.job == "daily_monitoring_report": - LOG.d("Send out daily monitoring stats") - daily_monitoring_report() + if args.job == "stats": + LOG.d("Compute growth and daily monitoring stats") + stats() elif args.job == "notify_trial_end": LOG.d("Notify users with trial ending soon") notify_trial_end() diff --git a/crontab.yml b/crontab.yml index adf463cb..f52dd908 100644 --- a/crontab.yml +++ b/crontab.yml @@ -1,12 +1,6 @@ jobs: - name: SimpleLogin growth stats - command: python /code/cron.py -j growth_stats - shell: /bin/bash - schedule: "0 1 * * *" - captureStderr: true - - - name: SimpleLogin monitoring stats - command: python /code/cron.py -j daily_monitoring_report + command: python /code/cron.py -j stats shell: /bin/bash schedule: "0 0 * * *" captureStderr: true From 8502e1666b1260395049dbc64d1b7be516b9ed13 Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 28 Feb 2022 11:14:59 +0100 Subject: [PATCH 17/35] fix migration --- migrations/versions/2022_022512_4729b7096d12_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/versions/2022_022512_4729b7096d12_.py b/migrations/versions/2022_022512_4729b7096d12_.py index 9800b92b..66c8c721 100644 --- a/migrations/versions/2022_022512_4729b7096d12_.py +++ b/migrations/versions/2022_022512_4729b7096d12_.py @@ -12,7 +12,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. revision = '4729b7096d12' -down_revision = '9282e982bc05' +down_revision = '5047fcbd57c7' branch_labels = None depends_on = None From 627b2e56d93212b68723cfccabd94eba4e822a0c Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 28 Feb 2022 16:40:07 +0100 Subject: [PATCH 18/35] comment out some admin pages --- app/admin_model.py | 56 +++++++++++++++++++++++----------------------- server.py | 12 ---------- 2 files changed, 28 insertions(+), 40 deletions(-) diff --git a/app/admin_model.py b/app/admin_model.py index 1e967b75..75e56f3f 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -3,7 +3,7 @@ from flask import redirect, url_for, request, flash from flask_admin import expose, AdminIndexView from flask_admin.actions import action from flask_admin.contrib import sqla -from flask_login import current_user, login_user +from flask_login import current_user from app.db import Session from app.models import User, ManualSubscription, Fido, Subscription, AppleSubscription @@ -126,20 +126,20 @@ class UserAdmin(SLModelView): Session.commit() - @action( - "login_as", - "Login as this user", - "Login as this user?", - ) - def login_as(self, ids): - if len(ids) != 1: - flash("only 1 user can be selected", "error") - return - - for user in User.filter(User.id.in_(ids)): - login_user(user) - flash(f"Login as user {user}", "success") - return redirect("/") + # @action( + # "login_as", + # "Login as this user", + # "Login as this user?", + # ) + # def login_as(self, ids): + # if len(ids) != 1: + # flash("only 1 user can be selected", "error") + # return + # + # for user in User.filter(User.id.in_(ids)): + # login_user(user) + # flash(f"Login as user {user}", "success") + # return redirect("/") def manual_upgrade(way: str, ids: [int], is_giveaway: bool): @@ -204,9 +204,9 @@ class MailboxAdmin(SLModelView): column_filters = ["id", "user.email", "email"] -class LifetimeCouponAdmin(SLModelView): - can_edit = True - can_create = True +# class LifetimeCouponAdmin(SLModelView): +# can_edit = True +# can_create = True class CouponAdmin(SLModelView): @@ -231,10 +231,10 @@ class ManualSubscriptionAdmin(SLModelView): Session.commit() -class ClientAdmin(SLModelView): - column_searchable_list = ["name", "description", "user.email"] - column_exclude_list = ["oauth_client_secret", "home_url"] - can_edit = True +# class ClientAdmin(SLModelView): +# column_searchable_list = ["name", "description", "user.email"] +# column_exclude_list = ["oauth_client_secret", "home_url"] +# can_edit = True class CustomDomainAdmin(SLModelView): @@ -254,9 +254,9 @@ class ReferralAdmin(SLModelView): return ret -class PayoutAdmin(SLModelView): - column_searchable_list = ["id", "user.email"] - column_filters = ["id", "user.email"] - can_edit = True - can_create = True - can_delete = True +# class PayoutAdmin(SLModelView): +# column_searchable_list = ["id", "user.email"] +# column_filters = ["id", "user.email"] +# can_edit = True +# can_create = True +# can_delete = True diff --git a/server.py b/server.py index 3128ff2a..8d6c9426 100644 --- a/server.py +++ b/server.py @@ -33,11 +33,7 @@ from app.admin_model import ( EmailLogAdmin, AliasAdmin, MailboxAdmin, - LifetimeCouponAdmin, ManualSubscriptionAdmin, - ClientAdmin, - ReferralAdmin, - PayoutAdmin, CouponAdmin, CustomDomainAdmin, ) @@ -81,20 +77,16 @@ from app.fake_data import fake_data from app.jose_utils import get_jwk_key from app.log import LOG from app.models import ( - Client, User, Alias, Subscription, PlanEnum, CustomDomain, - LifetimeCoupon, Mailbox, - Referral, CoinbaseSubscription, EmailLog, Contact, ManualSubscription, - Payout, Coupon, ) from app.monitor.base import monitor_bp @@ -693,13 +685,9 @@ def init_admin(app): admin.add_view(AliasAdmin(Alias, Session)) admin.add_view(MailboxAdmin(Mailbox, Session)) admin.add_view(EmailLogAdmin(EmailLog, Session)) - admin.add_view(LifetimeCouponAdmin(LifetimeCoupon, Session)) admin.add_view(CouponAdmin(Coupon, Session)) admin.add_view(ManualSubscriptionAdmin(ManualSubscription, Session)) - admin.add_view(ClientAdmin(Client, Session)) admin.add_view(CustomDomainAdmin(CustomDomain, Session)) - admin.add_view(ReferralAdmin(Referral, Session)) - admin.add_view(PayoutAdmin(Payout, Session)) def register_custom_commands(app): From b2d8f5a01704abd65df8f027bb26775d8c596473 Mon Sep 17 00:00:00 2001 From: Son Date: Wed, 2 Mar 2022 19:04:30 +0100 Subject: [PATCH 19/35] disable edition on admin --- app/admin_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/admin_model.py b/app/admin_model.py index 75e56f3f..2ea0eb95 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -45,7 +45,7 @@ class UserAdmin(SLModelView): "fido_uuid", "profile_picture", ] - can_edit = True + can_edit = False def scaffold_list_columns(self): ret = super().scaffold_list_columns() @@ -210,12 +210,12 @@ class MailboxAdmin(SLModelView): class CouponAdmin(SLModelView): - can_edit = True + can_edit = False can_create = True class ManualSubscriptionAdmin(SLModelView): - can_edit = True + can_edit = False column_searchable_list = ["id", "user.email"] @action( From 52a911f9d31de31bcc1858b96f1f8c44289338d1 Mon Sep 17 00:00:00 2001 From: Son Date: Wed, 2 Mar 2022 19:04:45 +0100 Subject: [PATCH 20/35] add extend subscription for 1 month to admin --- app/admin_model.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/admin_model.py b/app/admin_model.py index 2ea0eb95..5bc7a2d9 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -226,7 +226,19 @@ class ManualSubscriptionAdmin(SLModelView): def extend_1y(self, ids): for ms in ManualSubscription.filter(ManualSubscription.id.in_(ids)): ms.end_at = ms.end_at.shift(years=1) - flash(f"Extend subscription for {ms.user}", "success") + flash(f"Extend subscription for 1 year for {ms.user}", "success") + + Session.commit() + + @action( + "extend_1m", + "Extend for 1 month", + "Extend 1 month more?", + ) + def extend_1m(self, ids): + for ms in ManualSubscription.filter(ManualSubscription.id.in_(ids)): + ms.end_at = ms.end_at.shift(months=1) + flash(f"Extend subscription for 1 month for {ms.user}", "success") Session.commit() From f7ba3873d0f02bd51c97644f509a8b7cddc1399d Mon Sep 17 00:00:00 2001 From: Son Date: Wed, 2 Mar 2022 19:05:17 +0100 Subject: [PATCH 21/35] add adhoc upgrade on admin --- app/admin_model.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/admin_model.py b/app/admin_model.py index 5bc7a2d9..8a1d8e58 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -92,6 +92,14 @@ class UserAdmin(SLModelView): def action_monero_upgrade(self, ids): manual_upgrade("Crypto", ids, is_giveaway=False) + @action( + "adhoc_upgrade", + "Adhoc upgrade - for exceptional case", + "Are you sure you want to crypto-upgrade selected users?", + ) + def action_adhoc_upgrade(self, ids): + manual_upgrade("Adhoc", ids, is_giveaway=False) + @action( "extend_trial_1w", "Extend trial for 1 week more", From 71136669e9da9cff859e0b5c239a3de9545eab20 Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 7 Mar 2022 15:44:27 +0100 Subject: [PATCH 22/35] return the block reason in should_disable() --- app/email_utils.py | 44 ++++++++++++++++----------------------- email_handler.py | 7 +++++-- tests/test_email_utils.py | 20 +++++++++--------- 3 files changed, 33 insertions(+), 38 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index 69957e40..d4799575 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1118,15 +1118,17 @@ def normalize_reply_email(reply_email: str) -> str: return "".join(ret) -def should_disable(alias: Alias) -> bool: - """Disable an alias if it has too many bounces recently""" +def should_disable(alias: Alias) -> (bool, str): + """ + Return whether an alias should be disabled and if yes, the reason why + """ # Bypass the bounce rule if alias.cannot_be_disabled: LOG.w("%s cannot be disabled", alias) - return False + return False, "" if not ALIAS_AUTOMATIC_DISABLE: - return False + return False, "" yesterday = arrow.now().shift(days=-1) nb_bounced_last_24h = ( @@ -1141,8 +1143,7 @@ def should_disable(alias: Alias) -> bool: ) # if more than 12 bounces in 24h -> disable alias if nb_bounced_last_24h > 12: - LOG.d("more than 12 bounces in the last 24h, disable alias %s", alias) - return True + return True, "more than 12 bounces in the last 24h" # if more than 5 bounces but has bounces last week -> disable alias elif nb_bounced_last_24h > 5: @@ -1159,15 +1160,13 @@ def should_disable(alias: Alias) -> bool: .count() ) if nb_bounced_7d_1d > 1: - LOG.d( - "more than 5 bounces in the last 24h and more than 1 bounces in the last 7 days, " - "disable alias %s", - alias, + return ( + True, + "more than 5 bounces in the last 24h and more than 1 bounces in the last 7 days", ) - return True else: # alias level - # if bounces at least 9 days in the last 10 days -> disable alias + # if bounces happen for at least 9 days in the last 10 days -> disable alias query = ( Session.query( func.date(EmailLog.created_at).label("date"), @@ -1183,11 +1182,7 @@ def should_disable(alias: Alias) -> bool: ) if query.count() >= 9: - LOG.d( - "Bounces every day for at least 9 days in the last 10 days, disable alias %s", - alias, - ) - return True + return True, "Bounces every day for at least 9 days in the last 10 days" # account level query = ( @@ -1206,16 +1201,13 @@ def should_disable(alias: Alias) -> bool: # if an account has more than 10 bounces every day for at least 4 days in the last 10 days, disable alias date_bounces: List[Tuple[arrow.Arrow, int]] = list(query) - if len(date_bounces) > 4: - if all([v > 10 for _, v in date_bounces]): - LOG.d( - "+10 bounces for +4 days in the last 10 days on %s, disable alias %s", - alias.user, - alias, - ) - return True + more_than_10_bounces = [ + (d, nb_bounce) for d, nb_bounce in date_bounces if nb_bounce > 10 + ] + if len(more_than_10_bounces) > 4: + return True, "+10 bounces for +4 days in the last 10 days" - return False + return False, "" def parse_id_from_bounce(email_address: str) -> int: diff --git a/email_handler.py b/email_handler.py index 525a30f9..8dc9f4b9 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1396,7 +1396,8 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}" # inform user of this bounce - if not should_disable(alias): + alias_will_be_disabled, reason = should_disable(alias) + if not alias_will_be_disabled: LOG.d( "Inform user %s about a bounce from contact %s to alias %s", user, @@ -1447,7 +1448,9 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): ignore_smtp_error=True, ) else: - LOG.w("Disable alias %s %s. Last contact %s", alias, user, contact) + LOG.w( + f"Disable alias {alias} because {reason}. {alias.mailboxes} {alias.user}. Last contact {contact}" + ) alias.enabled = False Session.commit() diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 48216d5f..fe183164 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -610,7 +610,7 @@ def test_should_disable(flask_client): alias = Alias.create_new_random(user) Session.commit() - assert not should_disable(alias) + assert not should_disable(alias)[0] # create a lot of bounce on this alias contact = Contact.create( @@ -629,12 +629,12 @@ def test_should_disable(flask_client): bounced=True, ) - assert should_disable(alias) + assert should_disable(alias)[0] # should not affect another alias alias2 = Alias.create_new_random(user) Session.commit() - assert not should_disable(alias2) + assert not should_disable(alias2)[0] def test_should_disable_bounces_every_day(flask_client): @@ -643,7 +643,7 @@ def test_should_disable_bounces_every_day(flask_client): alias = Alias.create_new_random(user) Session.commit() - assert not should_disable(alias) + assert not should_disable(alias)[0] # create a lot of bounce on this alias contact = Contact.create( @@ -663,7 +663,7 @@ def test_should_disable_bounces_every_day(flask_client): created_at=arrow.now().shift(days=-i), ) - assert should_disable(alias) + assert should_disable(alias)[0] def test_should_disable_bounces_account(flask_client): @@ -682,8 +682,8 @@ def test_should_disable_bounces_account(flask_client): commit=True, ) - for day in range(6): - for _ in range(10): + for day in range(5): + for _ in range(11): EmailLog.create( user_id=user.id, contact_id=contact.id, @@ -694,7 +694,7 @@ def test_should_disable_bounces_account(flask_client): ) alias2 = Alias.create_new_random(user) - assert should_disable(alias2) + assert should_disable(alias2)[0] def test_should_disable_bounce_consecutive_days(flask_client): @@ -719,7 +719,7 @@ def test_should_disable_bounce_consecutive_days(flask_client): commit=True, bounced=True, ) - assert not should_disable(alias) + assert not should_disable(alias)[0] # create 2 bounces in the last 7 days: alias should be disabled for _ in range(2): @@ -731,7 +731,7 @@ def test_should_disable_bounce_consecutive_days(flask_client): bounced=True, created_at=arrow.now().shift(days=-3), ) - assert should_disable(alias) + assert should_disable(alias)[0] def test_parse_id_from_bounce(): From 99dc45e09af41f0f056634c852113461846b50eb Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 7 Mar 2022 15:45:36 +0100 Subject: [PATCH 23/35] refactor --- email_handler.py | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/email_handler.py b/email_handler.py index 8dc9f4b9..6c077921 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1397,7 +1397,34 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): # inform user of this bounce alias_will_be_disabled, reason = should_disable(alias) - if not alias_will_be_disabled: + if alias_will_be_disabled: + LOG.w( + f"Disable alias {alias} because {reason}. {alias.mailboxes} {alias.user}. Last contact {contact}" + ) + alias.enabled = False + Session.commit() + + send_email_with_rate_control( + user, + ALERT_BOUNCE_EMAIL, + user.email, + f"Alias {alias.email} has been disabled due to multiple bounces", + render( + "transactional/bounce/automatic-disable-alias.txt", + alias=alias, + refused_email_url=refused_email_url, + mailbox_email=mailbox.email, + ), + render( + "transactional/bounce/automatic-disable-alias.html", + alias=alias, + refused_email_url=refused_email_url, + mailbox_email=mailbox.email, + ), + max_nb_alert=10, + ignore_smtp_error=True, + ) + else: LOG.d( "Inform user %s about a bounce from contact %s to alias %s", user, @@ -1447,33 +1474,6 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): # smtp error can happen if user mailbox is unreachable, that might explain the bounce ignore_smtp_error=True, ) - else: - LOG.w( - f"Disable alias {alias} because {reason}. {alias.mailboxes} {alias.user}. Last contact {contact}" - ) - alias.enabled = False - Session.commit() - - send_email_with_rate_control( - user, - ALERT_BOUNCE_EMAIL, - user.email, - f"Alias {alias.email} has been disabled due to multiple bounces", - render( - "transactional/bounce/automatic-disable-alias.txt", - alias=alias, - refused_email_url=refused_email_url, - mailbox_email=mailbox.email, - ), - render( - "transactional/bounce/automatic-disable-alias.html", - alias=alias, - refused_email_url=refused_email_url, - mailbox_email=mailbox.email, - ), - max_nb_alert=10, - ignore_smtp_error=True, - ) def handle_hotmail_complaint(msg: Message) -> bool: From 350f498b94d5cd4db11ea8a24087077b8a1eb70b Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 7 Mar 2022 15:50:58 +0100 Subject: [PATCH 24/35] lessen alias automatic disable check --- app/email_utils.py | 10 +++++----- email_handler.py | 1 - tests/test_email_utils.py | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/email_utils.py b/app/email_utils.py index d4799575..23ca2c2f 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1143,11 +1143,11 @@ def should_disable(alias: Alias) -> (bool, str): ) # if more than 12 bounces in 24h -> disable alias if nb_bounced_last_24h > 12: - return True, "more than 12 bounces in the last 24h" + return True, "+12 bounces in the last 24h" - # if more than 5 bounces but has bounces last week -> disable alias + # if more than 5 bounces but has +10 bounces last week -> disable alias elif nb_bounced_last_24h > 5: - one_week_ago = arrow.now().shift(days=-8) + one_week_ago = arrow.now().shift(days=-7) nb_bounced_7d_1d = ( Session.query(EmailLog) .filter( @@ -1159,10 +1159,10 @@ def should_disable(alias: Alias) -> (bool, str): .filter(EmailLog.alias_id == alias.id) .count() ) - if nb_bounced_7d_1d > 1: + if nb_bounced_7d_1d > 10: return ( True, - "more than 5 bounces in the last 24h and more than 1 bounces in the last 7 days", + "+5 bounces in the last 24h and +10 bounces in the last 7 days", ) else: # alias level diff --git a/email_handler.py b/email_handler.py index 6c077921..324b4d09 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1395,7 +1395,6 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}" - # inform user of this bounce alias_will_be_disabled, reason = should_disable(alias) if alias_will_be_disabled: LOG.w( diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index fe183164..dc7c41f5 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -721,8 +721,8 @@ def test_should_disable_bounce_consecutive_days(flask_client): ) assert not should_disable(alias)[0] - # create 2 bounces in the last 7 days: alias should be disabled - for _ in range(2): + # create +10 bounces in the last 7 days: alias should be disabled + for _ in range(11): EmailLog.create( user_id=user.id, contact_id=contact.id, From a64a70cbc8716d7d45febb85011b681d4780bd99 Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 7 Mar 2022 15:57:29 +0100 Subject: [PATCH 25/35] use Date instead of date for header value --- app/email/headers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/email/headers.py b/app/email/headers.py index 3b5be9e8..e1aaabd6 100644 --- a/app/email/headers.py +++ b/app/email/headers.py @@ -2,7 +2,7 @@ MESSAGE_ID = "Message-ID" IN_REPLY_TO = "In-Reply-To" REFERENCES = "References" -DATE = "date" +DATE = "Date" SUBJECT = "Subject" FROM = "From" TO = "To" From ed089109bb86f281dd9f577b8c62f37017d94aaa Mon Sep 17 00:00:00 2001 From: Son Date: Mon, 7 Mar 2022 17:52:16 +0100 Subject: [PATCH 26/35] make sure to close session in monitoring --- monitoring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monitoring.py b/monitoring.py index b10c6122..cf4fb87a 100644 --- a/monitoring.py +++ b/monitoring.py @@ -50,6 +50,7 @@ if __name__ == "__main__": while True: log_postfix_metrics() log_nb_db_connection() + Session.close() # 1 min sleep(60) From 89218fab7fdc3781e2124c5a94b436c205b905a8 Mon Sep 17 00:00:00 2001 From: Son Date: Tue, 8 Mar 2022 10:30:29 +0100 Subject: [PATCH 27/35] fix "local variable 'alias_id' referenced before assignment" --- email_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 324b4d09..65cc8ec8 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1906,7 +1906,7 @@ def handle_unsubscribe(envelope: Envelope, msg: Message) -> str: return status.E507 if not alias: - LOG.w("No such alias %s", alias_id) + LOG.w("Cannot parse alias from subject %s", subject) return status.E508 mail_from = envelope.mail_from From b711743d6eb4a0ec2654acba8bd0f8eabb6a6396 Mon Sep 17 00:00:00 2001 From: Son Date: Tue, 8 Mar 2022 10:31:20 +0100 Subject: [PATCH 28/35] fix --- email_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 65cc8ec8..9469eeb9 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1902,11 +1902,11 @@ def handle_unsubscribe(envelope: Envelope, msg: Message) -> str: alias_id = int(subject) alias = Alias.get(alias_id) except Exception: - LOG.w("Cannot parse alias from subject %s", msg[headers.SUBJECT]) + LOG.w("Wrong format subject %s", msg[headers.SUBJECT]) return status.E507 if not alias: - LOG.w("Cannot parse alias from subject %s", subject) + LOG.w("Cannot get alias from subject %s", subject) return status.E508 mail_from = envelope.mail_from From 6f80edfd64d265ca820fd9dd1bfdbbf25338184d Mon Sep 17 00:00:00 2001 From: Son Date: Tue, 8 Mar 2022 16:38:03 +0100 Subject: [PATCH 29/35] fix discover page --- app/discover/views/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/discover/views/index.py b/app/discover/views/index.py index a8017c61..2ed84704 100644 --- a/app/discover/views/index.py +++ b/app/discover/views/index.py @@ -8,5 +8,5 @@ from app.models import Client @discover_bp.route("/", methods=["GET", "POST"]) @login_required def index(): - clients = Client.filter_by(published=True).all() + clients = Client.filter_by(approved=True).all() return render_template("discover/index.html", clients=clients) From b6b917eba847237b655d3579dc6f303021189c2b Mon Sep 17 00:00:00 2001 From: Son Date: Tue, 8 Mar 2022 18:35:18 +0100 Subject: [PATCH 30/35] add more log --- email_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/email_handler.py b/email_handler.py index 9469eeb9..e76de0ec 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1913,7 +1913,12 @@ def handle_unsubscribe(envelope: Envelope, msg: Message) -> str: # Only alias's owning mailbox can send the unsubscribe request mailbox = get_mailbox_from_mail_from(mail_from, alias) if not mailbox: - LOG.d("%s cannot disable alias %s", envelope.mail_from, alias) + LOG.d( + "%s cannot disable alias %s. Alias authorized addresses:%s", + envelope.mail_from, + alias, + alias.authorized_addresses, + ) return status.E509 user = alias.user From b35b13b76430136ffaa5064f95a5c27ab3e4ae3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Wed, 9 Mar 2022 09:45:09 +0100 Subject: [PATCH 31/35] Use plausible outbound link tracking --- templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/base.html b/templates/base.html index 0767fd5e..364c2fa2 100644 --- a/templates/base.html +++ b/templates/base.html @@ -87,7 +87,7 @@ {% if PLAUSIBLE_HOST and PLAUSIBLE_DOMAIN %} - + {% endif %} From 0e3a5c3d3cc0ec4c2a0da9b971f8a9d51879d3f2 Mon Sep 17 00:00:00 2001 From: Son Date: Wed, 9 Mar 2022 17:58:26 +0100 Subject: [PATCH 32/35] mark a notification as read when user arrives on the notification page --- app/dashboard/views/notification.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/dashboard/views/notification.py b/app/dashboard/views/notification.py index 82270e4f..e22c8e2c 100644 --- a/app/dashboard/views/notification.py +++ b/app/dashboard/views/notification.py @@ -23,6 +23,10 @@ def notification_route(notification_id): ) return redirect(url_for("dashboard.index")) + if not notification.read: + notification.read = True + Session.commit() + if request.method == "POST": notification_title = notification.title or notification.message[:20] Notification.delete(notification_id) From fb00c18d5ae7b42ae0101f34be8d49f7371787e4 Mon Sep 17 00:00:00 2001 From: Son Date: Wed, 9 Mar 2022 17:59:02 +0100 Subject: [PATCH 33/35] create a notification when an alias is disabled --- email_handler.py | 9 +++++++++ templates/notification/alias-disable.html | 10 ++++++++++ 2 files changed, 19 insertions(+) create mode 100644 templates/notification/alias-disable.html diff --git a/email_handler.py b/email_handler.py index e76de0ec..53831062 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1403,6 +1403,15 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): alias.enabled = False Session.commit() + Notification.create( + user_id=user.id, + title=f"{alias.email} has been disabled due to multiple bounces", + message=Notification.render( + "notification/alias-disable.html", alias=alias, mailbox=mailbox + ), + commit=True, + ) + send_email_with_rate_control( user, ALERT_BOUNCE_EMAIL, diff --git a/templates/notification/alias-disable.html b/templates/notification/alias-disable.html new file mode 100644 index 00000000..9c2b6720 --- /dev/null +++ b/templates/notification/alias-disable.html @@ -0,0 +1,10 @@ +
+ There are several emails sent to your alias {{ alias.email }} that have been bounced by your + mailbox {{ mailbox.email }}. +
+ +
+ As security measure, we have disabled the alias {{ alias.email }}. + +
+ From e0b5bd36a6099517e7c77beb7d97c82bde91fa4d Mon Sep 17 00:00:00 2001 From: Son Date: Wed, 9 Mar 2022 17:59:42 +0100 Subject: [PATCH 34/35] show "more" only when a notification has a title. Show either title or message. Use bold font when a notification isn't read --- templates/header.html | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/templates/header.html b/templates/header.html index 3e602142..656317ba 100644 --- a/templates/header.html +++ b/templates/header.html @@ -38,15 +38,13 @@
-
-
-
- -
+ From ed12e470778b59ba03d82d41fb7867275b2746ee Mon Sep 17 00:00:00 2001 From: Son Date: Thu, 10 Mar 2022 08:33:26 +0100 Subject: [PATCH 35/35] reduce nb of commit --- email_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/email_handler.py b/email_handler.py index 53831062..67ba6a12 100644 --- a/email_handler.py +++ b/email_handler.py @@ -1401,7 +1401,6 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): f"Disable alias {alias} because {reason}. {alias.mailboxes} {alias.user}. Last contact {contact}" ) alias.enabled = False - Session.commit() Notification.create( user_id=user.id, @@ -1409,9 +1408,10 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): message=Notification.render( "notification/alias-disable.html", alias=alias, mailbox=mailbox ), - commit=True, ) + Session.commit() + send_email_with_rate_control( user, ALERT_BOUNCE_EMAIL,