From 08db23658a8dbc204c984469ca89df59351c2510 Mon Sep 17 00:00:00 2001
From: Son NK <>
Date: Sat, 15 Aug 2020 16:33:48 +0200
Subject: [PATCH 1/6] add SPAMASSASSIN_HOST param
---
app/config.py | 4 ++++
example.env | 5 ++++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/app/config.py b/app/config.py
index 0e5dbf53..8f8d2024 100644
--- a/app/config.py
+++ b/app/config.py
@@ -302,3 +302,7 @@ PLAUSIBLE_DOMAIN = os.environ.get("PLAUSIBLE_DOMAIN")
# server host
HOST = socket.gethostname()
+
+# by default use a tolerant score
+MAX_SPAM_SCORE = 10
+SPAMASSASSIN_HOST = os.environ.get("SPAMASSASSIN_HOST")
diff --git a/example.env b/example.env
index ca552674..baed77d7 100644
--- a/example.env
+++ b/example.env
@@ -153,4 +153,7 @@ DISABLE_ONBOARDING=true
# Set the 2 below variables to enable Plausible Analytics
# PLAUSIBLE_HOST=https://plausible.io
-# PLAUSIBLE_DOMAIN=yourdomain.com
\ No newline at end of file
+# PLAUSIBLE_DOMAIN=yourdomain.com
+
+# Spamassassin server
+# SPAMASSASSIN_HOST = 127.0.0.1
\ No newline at end of file
From c4dd980cf6b35caa066b4e84140d3ce0b5fa72c5 Mon Sep 17 00:00:00 2001
From: Son NK <>
Date: Sat, 15 Aug 2020 16:35:18 +0200
Subject: [PATCH 2/6] add aiospamc to requirement
---
requirements.in | 3 ++-
requirements.txt | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/requirements.in b/requirements.in
index 11fb6e49..6dc20d42 100644
--- a/requirements.in
+++ b/requirements.in
@@ -43,4 +43,5 @@ pyspf
Flask-Limiter
memory_profiler
gevent
-aiocontextvars
\ No newline at end of file
+aiocontextvars
+aiospamc
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 26806347..b1e2a90d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,6 +8,7 @@ aiocontextvars==0.2.2 # via -r requirements.in
aiohttp==3.5.4 # via raven-aiohttp, yacron
aiosmtpd==1.2 # via -r requirements.in
aiosmtplib==1.0.6 # via yacron
+aiospamc==0.6.1 # via -r requirements.in
alembic==1.0.10 # via flask-migrate
appnope==0.1.0 # via ipython
arrow==0.14.2 # via -r requirements.in
@@ -23,7 +24,7 @@ boto3==1.9.167 # via -r requirements.in, watchtower
botocore==1.12.167 # via boto3, s3transfer
cachetools==4.0.0 # via google-auth
cbor2==5.1.0 # via webauthn
-certifi==2019.3.9 # via requests, sentry-sdk
+certifi==2019.11.28 # via aiospamc, requests, sentry-sdk
cffi==1.12.3 # via bcrypt, cryptography
chardet==3.0.4 # via aiohttp, requests
click==7.0 # via flask, pip-tools
From 673b08712caad5626612b0dfb5f4043bf1a8baf5 Mon Sep 17 00:00:00 2001
From: Son NK <>
Date: Sat, 15 Aug 2020 16:38:16 +0200
Subject: [PATCH 3/6] use SPAMASSASSIN server if available
---
email_handler.py | 34 +++++++++++++++++++++++++++-------
1 file changed, 27 insertions(+), 7 deletions(-)
diff --git a/email_handler.py b/email_handler.py
index 9bd4a90f..7322aced 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -43,6 +43,7 @@ from io import BytesIO
from smtplib import SMTP
from typing import List, Tuple
+import aiospamc
import arrow
import spf
from aiosmtpd.controller import Controller
@@ -66,6 +67,8 @@ from app.config import (
POSTFIX_PORT,
SENDER,
SENDER_DIR,
+ SPAMASSASSIN_HOST,
+ MAX_SPAM_SCORE,
)
from app.email_utils import (
send_email,
@@ -359,7 +362,7 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
return msg
-def handle_forward(
+async def handle_forward(
envelope, smtp: SMTP, msg: Message, rcpt_to: str
) -> List[Tuple[bool, str]]:
"""return whether an email has been delivered and
@@ -391,7 +394,7 @@ def handle_forward(
ret = []
for mailbox in alias.mailboxes:
ret.append(
- forward_email_to_mailbox(
+ await forward_email_to_mailbox(
alias, copy(msg), email_log, contact, envelope, smtp, mailbox, user
)
)
@@ -399,7 +402,7 @@ def handle_forward(
return ret
-def forward_email_to_mailbox(
+async def forward_email_to_mailbox(
alias,
msg: Message,
email_log: EmailLog,
@@ -421,7 +424,19 @@ def forward_email_to_mailbox(
)
return False, "550 SL E14"
- is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
+ spam_status = ""
+ is_spam = False
+
+ if SPAMASSASSIN_HOST:
+ spam_score = await get_spam_score(msg)
+ if (user.max_spam_score and spam_score > user.max_spam_score) or (
+ not user.max_spam_score and spam_score > MAX_SPAM_SCORE
+ ):
+ is_spam = True
+ spam_status = "Spam detected by SpamAssassin server"
+ else:
+ is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
+
if is_spam:
LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact)
email_log.is_spam = True
@@ -1065,7 +1080,7 @@ def handle_sender_email(envelope: Envelope):
return "250 email to sender accepted"
-def handle(envelope: Envelope, smtp: SMTP) -> str:
+async def handle(envelope: Envelope, smtp: SMTP) -> str:
"""Return SMTP status"""
# unsubscribe request
if UNSUBSCRIBER and envelope.rcpt_tos == [UNSUBSCRIBER]:
@@ -1106,7 +1121,7 @@ def handle(envelope: Envelope, smtp: SMTP) -> str:
msg["From"],
rcpt_to,
)
- for is_delivered, smtp_status in handle_forward(
+ for is_delivered, smtp_status in await handle_forward(
envelope, smtp, msg, rcpt_to
):
res.append((is_delivered, smtp_status))
@@ -1120,6 +1135,11 @@ def handle(envelope: Envelope, smtp: SMTP) -> str:
return res[0][1]
+async def get_spam_score(message) -> float:
+ response = await aiospamc.check(message, host=SPAMASSASSIN_HOST)
+ return response.headers["Spam"].score
+
+
class MailHandler:
async def handle_DATA(self, server, session, envelope: Envelope):
start = time.time()
@@ -1137,7 +1157,7 @@ class MailHandler:
app = new_app()
with app.app_context():
- ret = handle(envelope, smtp)
+ ret = await handle(envelope, smtp)
LOG.debug("takes %s seconds <<===", time.time() - start)
return ret
From bf555ed6059a535ad16cb850b4038ad92e34ed10 Mon Sep 17 00:00:00 2001
From: Son NK <>
Date: Sat, 15 Aug 2020 16:53:57 +0200
Subject: [PATCH 4/6] detect spam in reply phase
---
app/config.py | 2 +
email_handler.py | 122 +++++++++++++-----
.../transactional/spam-email-reply-phase.html | 17 +++
.../transactional/spam-email-reply-phase.txt | 15 +++
4 files changed, 125 insertions(+), 31 deletions(-)
create mode 100644 templates/emails/transactional/spam-email-reply-phase.html
create mode 100644 templates/emails/transactional/spam-email-reply-phase.txt
diff --git a/app/config.py b/app/config.py
index 8f8d2024..f4e2afe3 100644
--- a/app/config.py
+++ b/app/config.py
@@ -306,3 +306,5 @@ HOST = socket.gethostname()
# by default use a tolerant score
MAX_SPAM_SCORE = 10
SPAMASSASSIN_HOST = os.environ.get("SPAMASSASSIN_HOST")
+# use a more restrictive score when replying
+MAX_REPLY_PHASE_SPAM_SCORE = 5
diff --git a/email_handler.py b/email_handler.py
index 7322aced..9b784d99 100644
--- a/email_handler.py
+++ b/email_handler.py
@@ -69,6 +69,7 @@ from app.config import (
SENDER_DIR,
SPAMASSASSIN_HOST,
MAX_SPAM_SCORE,
+ MAX_REPLY_PHASE_SPAM_SCORE,
)
from app.email_utils import (
send_email,
@@ -424,6 +425,7 @@ async def forward_email_to_mailbox(
)
return False, "550 SL E14"
+ # Spam check
spam_status = ""
is_spam = False
@@ -524,7 +526,7 @@ async def forward_email_to_mailbox(
return True, "250 Message accepted for delivery"
-def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
+async def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
"""
return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
@@ -577,6 +579,33 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str
# cannot use 4** here as sender will retry. 5** because that generates bounce report
return True, "250 SL E11"
+ # Spam check
+ spam_status = ""
+ is_spam = False
+
+ # do not use user.max_spam_score here
+ if SPAMASSASSIN_HOST:
+ spam_score = await get_spam_score(msg)
+ if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
+ is_spam = True
+ spam_status = "Spam detected by SpamAssassin server"
+ else:
+ is_spam, spam_status = get_spam_info(msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE)
+
+ if is_spam:
+ LOG.exception(
+ "Reply phase - email sent from %s to %s detected as spam", alias, contact
+ )
+ email_log = EmailLog.create(
+ contact_id=contact.id, is_reply=True, user_id=contact.user_id
+ )
+ email_log.is_spam = True
+ email_log.spam_status = spam_status
+ db.session.commit()
+
+ handle_spam(contact, alias, msg, user, mailbox.email, email_log, is_reply=True)
+ return False, "550 SL E15 Email detected as spam"
+
delete_header(msg, _IP_HEADER)
delete_header(msg, "DKIM-Signature")
@@ -941,6 +970,7 @@ def handle_spam(
user: User,
mailbox_email: str,
email_log: EmailLog,
+ is_reply=False, # whether the email is in forward or reply phase
):
# Store the report & original email
orig_msg = get_orig_message_from_spamassassin_report(msg)
@@ -972,35 +1002,65 @@ def handle_spam(
)
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
- # inform user
- LOG.d(
- "Inform user %s about spam email sent by %s to alias %s",
- user,
- contact.website_email,
- alias.email,
- )
- send_email_with_rate_control(
- user,
- ALERT_SPAM_EMAIL,
- mailbox_email,
- f"Email from {contact.website_email} to {alias.email} is detected as spam",
- render(
- "transactional/spam-email.txt",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- ),
- render(
- "transactional/spam-email.html",
- name=user.name,
- alias=alias,
- website_email=contact.website_email,
- disable_alias_link=disable_alias_link,
- refused_email_url=refused_email_url,
- ),
- )
+ if is_reply:
+ LOG.d(
+ "Inform user %s about spam email sent from alias %s to %s",
+ user,
+ alias,
+ contact,
+ )
+ send_email_with_rate_control(
+ user,
+ ALERT_SPAM_EMAIL,
+ mailbox_email,
+ f"Email from {contact.website_email} to {alias.email} is detected as spam",
+ render(
+ "transactional/spam-email-reply-phase.txt",
+ name=user.name,
+ alias=alias,
+ website_email=contact.website_email,
+ disable_alias_link=disable_alias_link,
+ refused_email_url=refused_email_url,
+ ),
+ render(
+ "transactional/spam-email-reply-phase.html",
+ name=user.name,
+ alias=alias,
+ website_email=contact.website_email,
+ disable_alias_link=disable_alias_link,
+ refused_email_url=refused_email_url,
+ ),
+ )
+ else:
+ # inform user
+ LOG.d(
+ "Inform user %s about spam email sent by %s to alias %s",
+ user,
+ contact,
+ alias,
+ )
+ send_email_with_rate_control(
+ user,
+ ALERT_SPAM_EMAIL,
+ mailbox_email,
+ f"Email from {contact.website_email} to {alias.email} is detected as spam",
+ render(
+ "transactional/spam-email.txt",
+ name=user.name,
+ alias=alias,
+ website_email=contact.website_email,
+ disable_alias_link=disable_alias_link,
+ refused_email_url=refused_email_url,
+ ),
+ render(
+ "transactional/spam-email.html",
+ name=user.name,
+ alias=alias,
+ website_email=contact.website_email,
+ disable_alias_link=disable_alias_link,
+ refused_email_url=refused_email_url,
+ ),
+ )
def handle_unsubscribe(envelope: Envelope):
@@ -1112,7 +1172,7 @@ async def handle(envelope: Envelope, smtp: SMTP) -> str:
LOG.debug(
">>> Reply phase %s(%s) -> %s", envelope.mail_from, msg["From"], rcpt_to
)
- is_delivered, smtp_status = handle_reply(envelope, smtp, msg, rcpt_to)
+ is_delivered, smtp_status = await handle_reply(envelope, smtp, msg, rcpt_to)
res.append((is_delivered, smtp_status))
else: # Forward case
LOG.debug(
diff --git a/templates/emails/transactional/spam-email-reply-phase.html b/templates/emails/transactional/spam-email-reply-phase.html
new file mode 100644
index 00000000..3ec3c76c
--- /dev/null
+++ b/templates/emails/transactional/spam-email-reply-phase.html
@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+
+{% block content %}
+ {{ render_text("Hi " + name) }}
+ {{ render_text("An email sent from your alias " + alias.email + " to " + website_email + " is detected as spam by our Spam Detection Engine (SpamAssassin).") }}
+
+ {{ render_text('In most of the cases, the email will be refused by your contact.') }}
+
+ {{ render_button("View the email", refused_email_url) }}
+
+ {{ render_text('The email is automatically deleted in 7 days.') }}
+
+ {{ render_text('Please let us know if you have any question by replying to this email.') }}
+
+ {{ render_text('Thanks,
SimpleLogin Team.') }}
+ {{ raw_url(disable_alias_link) }}
+{% endblock %}
diff --git a/templates/emails/transactional/spam-email-reply-phase.txt b/templates/emails/transactional/spam-email-reply-phase.txt
new file mode 100644
index 00000000..792db3f9
--- /dev/null
+++ b/templates/emails/transactional/spam-email-reply-phase.txt
@@ -0,0 +1,15 @@
+Hi {{name}}
+
+An email sent from your alias {{alias.email}} to {{website_email}} is detected as spam by our Spam Detection Engine (SpamAssassin).
+
+In most of the cases, the email will be refused by your contact.
+
+You can view this email here:
+{{ refused_email_url }}
+
+The email is automatically deleted in 7 days.
+
+Please let us know if you have any question by replying to this email.
+
+Best,
+SimpleLogin team.
\ No newline at end of file
From f9300009e5acd537bf708a4dc379c43470e33eed Mon Sep 17 00:00:00 2001
From: Son NK <>
Date: Sat, 15 Aug 2020 16:56:16 +0200
Subject: [PATCH 5/6] refactor: rename forward -> contact
---
app/dashboard/templates/dashboard/refused_email.html | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/app/dashboard/templates/dashboard/refused_email.html b/app/dashboard/templates/dashboard/refused_email.html
index 4d273b99..a32ecda4 100644
--- a/app/dashboard/templates/dashboard/refused_email.html
+++ b/app/dashboard/templates/dashboard/refused_email.html
@@ -30,15 +30,15 @@
{% for email_log in email_logs %}
{% set refused_email = email_log.refused_email %}
- {% set forward = email_log.forward %}
- {% set alias = forward.alias %}
+ {% set contact = email_log.contact %}
+ {% set alias = contact.alias %}