Merge pull request #264 from simple-login/spamassassin

Enable Spamassassin server

app/config.py

@@ -302,3 +302,9 @@ PLAUSIBLE_DOMAIN = os.environ.get("PLAUSIBLE_DOMAIN")
 
 # server host
 HOST = socket.gethostname()
+
+# by default use a tolerant score
+MAX_SPAM_SCORE = 10
+SPAMASSASSIN_HOST = os.environ.get("SPAMASSASSIN_HOST")
+# use a more restrictive score when replying
+MAX_REPLY_PHASE_SPAM_SCORE = 5

app/dashboard/templates/dashboard/refused_email.html

@@ -30,21 +30,26 @@
 
     {% for email_log in email_logs %}
       {% set refused_email = email_log.refused_email %}
-      {% set forward = email_log.forward %}
+      {% set contact = email_log.contact %}
-      {% set alias = forward.alias %}
+      {% set alias = contact.alias %}
 
       <div class="card p-4 shadow-sm {% if email_log.id == highlight_id %} highlight-row {% endif %}">
         <div class="small-text">
           Sent {{ refused_email.created_at | dt }}
         </div>
 
-        From: {{ forward.website_email }} <br>
+        {% if email_log.is_reply %}
+          From: {{ alias.email }} <br>
+          To: {{ contact.website_email }}
+        {% else %}
+          From: {{ contact.website_email }} <br>
 
-        <span>
+          <span>
           To: {{ alias.email }}
           <a href='{{ url_for("dashboard.index", highlight_alias_id=alias.id) }}'
              class="btn btn-sm btn-outline-danger">Disable Alias</a>
         </span>
+        {% endif %}
 
         {% if refused_email.deleted %}
           <div>
@@ -53,7 +58,7 @@
         {% else %}
           <a href="{{ refused_email.get_url() }}" download
              class="mt-4">Download →</a>
-          <div class="small-text">This will download a ".eml" file that you can open in your favorite email client</div>
+          <div class="small-text">This will download a ".eml" file that you can open in your email client</div>
         {% endif %}
 
       </div>

email_handler.py

@@ -43,6 +43,7 @@ from io import BytesIO
 from smtplib import SMTP
 from typing import List, Tuple
 
+import aiospamc
 import arrow
 import spf
 from aiosmtpd.controller import Controller
@@ -66,6 +67,9 @@ from app.config import (
     POSTFIX_PORT,
     SENDER,
     SENDER_DIR,
+    SPAMASSASSIN_HOST,
+    MAX_SPAM_SCORE,
+    MAX_REPLY_PHASE_SPAM_SCORE,
 )
 from app.email_utils import (
     send_email,
@@ -359,7 +363,7 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
     return msg
 
 
-def handle_forward(
+async def handle_forward(
     envelope, smtp: SMTP, msg: Message, rcpt_to: str
 ) -> List[Tuple[bool, str]]:
     """return whether an email has been delivered and
@@ -391,7 +395,7 @@ def handle_forward(
     ret = []
     for mailbox in alias.mailboxes:
         ret.append(
-            forward_email_to_mailbox(
+            await forward_email_to_mailbox(
                 alias, copy(msg), email_log, contact, envelope, smtp, mailbox, user
             )
         )
@@ -399,7 +403,7 @@ def handle_forward(
     return ret
 
 
-def forward_email_to_mailbox(
+async def forward_email_to_mailbox(
     alias,
     msg: Message,
     email_log: EmailLog,
@@ -421,7 +425,20 @@ def forward_email_to_mailbox(
         )
         return False, "550 SL E14"
 
-    is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
+    # Spam check
+    spam_status = ""
+    is_spam = False
+
+    if SPAMASSASSIN_HOST:
+        spam_score = await get_spam_score(msg)
+        if (user.max_spam_score and spam_score > user.max_spam_score) or (
+            not user.max_spam_score and spam_score > MAX_SPAM_SCORE
+        ):
+            is_spam = True
+            spam_status = "Spam detected by SpamAssassin server"
+    else:
+        is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
+
     if is_spam:
         LOG.warning("Email detected as spam. Alias: %s, from: %s", alias, contact)
         email_log.is_spam = True
@@ -509,7 +526,7 @@ def forward_email_to_mailbox(
     return True, "250 Message accepted for delivery"
 
 
-def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
+async def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
     """
     return whether an email has been delivered and
     the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
@@ -562,6 +579,33 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str
             # cannot use 4** here as sender will retry. 5** because that generates bounce report
             return True, "250 SL E11"
 
+    # Spam check
+    spam_status = ""
+    is_spam = False
+
+    # do not use user.max_spam_score here
+    if SPAMASSASSIN_HOST:
+        spam_score = await get_spam_score(msg)
+        if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
+            is_spam = True
+            spam_status = "Spam detected by SpamAssassin server"
+    else:
+        is_spam, spam_status = get_spam_info(msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE)
+
+    if is_spam:
+        LOG.exception(
+            "Reply phase - email sent from %s to %s detected as spam", alias, contact
+        )
+        email_log = EmailLog.create(
+            contact_id=contact.id, is_reply=True, user_id=contact.user_id
+        )
+        email_log.is_spam = True
+        email_log.spam_status = spam_status
+        db.session.commit()
+
+        handle_spam(contact, alias, msg, user, mailbox.email, email_log, is_reply=True)
+        return False, "550 SL E15 Email detected as spam"
+
     delete_header(msg, _IP_HEADER)
 
     delete_header(msg, "DKIM-Signature")
@@ -926,6 +970,7 @@ def handle_spam(
     user: User,
     mailbox_email: str,
     email_log: EmailLog,
+    is_reply=False,  # whether the email is in forward or reply phase
 ):
     # Store the report & original email
     orig_msg = get_orig_message_from_spamassassin_report(msg)
@@ -957,35 +1002,65 @@ def handle_spam(
     )
     disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
 
-    # inform user
+    if is_reply:
-    LOG.d(
+        LOG.d(
-        "Inform user %s about spam email sent by %s to alias %s",
+            "Inform user %s about spam email sent from alias %s to %s",
-        user,
+            user,
-        contact.website_email,
+            alias,
-        alias.email,
+            contact,
-    )
+        )
-    send_email_with_rate_control(
+        send_email_with_rate_control(
-        user,
+            user,
-        ALERT_SPAM_EMAIL,
+            ALERT_SPAM_EMAIL,
-        mailbox_email,
+            mailbox_email,
-        f"Email from {contact.website_email} to {alias.email} is detected as spam",
+            f"Email from {contact.website_email} to {alias.email} is detected as spam",
-        render(
+            render(
-            "transactional/spam-email.txt",
+                "transactional/spam-email-reply-phase.txt",
-            name=user.name,
+                name=user.name,
-            alias=alias,
+                alias=alias,
-            website_email=contact.website_email,
+                website_email=contact.website_email,
-            disable_alias_link=disable_alias_link,
+                disable_alias_link=disable_alias_link,
-            refused_email_url=refused_email_url,
+                refused_email_url=refused_email_url,
-        ),
+            ),
-        render(
+            render(
-            "transactional/spam-email.html",
+                "transactional/spam-email-reply-phase.html",
-            name=user.name,
+                name=user.name,
-            alias=alias,
+                alias=alias,
-            website_email=contact.website_email,
+                website_email=contact.website_email,
-            disable_alias_link=disable_alias_link,
+                disable_alias_link=disable_alias_link,
-            refused_email_url=refused_email_url,
+                refused_email_url=refused_email_url,
-        ),
+            ),
-    )
+        )
+    else:
+        # inform user
+        LOG.d(
+            "Inform user %s about spam email sent by %s to alias %s",
+            user,
+            contact,
+            alias,
+        )
+        send_email_with_rate_control(
+            user,
+            ALERT_SPAM_EMAIL,
+            mailbox_email,
+            f"Email from {contact.website_email} to {alias.email} is detected as spam",
+            render(
+                "transactional/spam-email.txt",
+                name=user.name,
+                alias=alias,
+                website_email=contact.website_email,
+                disable_alias_link=disable_alias_link,
+                refused_email_url=refused_email_url,
+            ),
+            render(
+                "transactional/spam-email.html",
+                name=user.name,
+                alias=alias,
+                website_email=contact.website_email,
+                disable_alias_link=disable_alias_link,
+                refused_email_url=refused_email_url,
+            ),
+        )
 
 
 def handle_unsubscribe(envelope: Envelope):
@@ -1065,7 +1140,7 @@ def handle_sender_email(envelope: Envelope):
     return "250 email to sender accepted"
 
 
-def handle(envelope: Envelope, smtp: SMTP) -> str:
+async def handle(envelope: Envelope, smtp: SMTP) -> str:
     """Return SMTP status"""
     # unsubscribe request
     if UNSUBSCRIBER and envelope.rcpt_tos == [UNSUBSCRIBER]:
@@ -1097,7 +1172,7 @@ def handle(envelope: Envelope, smtp: SMTP) -> str:
             LOG.debug(
                 ">>> Reply phase %s(%s) -> %s", envelope.mail_from, msg["From"], rcpt_to
             )
-            is_delivered, smtp_status = handle_reply(envelope, smtp, msg, rcpt_to)
+            is_delivered, smtp_status = await handle_reply(envelope, smtp, msg, rcpt_to)
             res.append((is_delivered, smtp_status))
         else:  # Forward case
             LOG.debug(
@@ -1106,7 +1181,7 @@ def handle(envelope: Envelope, smtp: SMTP) -> str:
                 msg["From"],
                 rcpt_to,
             )
-            for is_delivered, smtp_status in handle_forward(
+            for is_delivered, smtp_status in await handle_forward(
                 envelope, smtp, msg, rcpt_to
             ):
                 res.append((is_delivered, smtp_status))
@@ -1120,6 +1195,11 @@ def handle(envelope: Envelope, smtp: SMTP) -> str:
     return res[0][1]
 
 
+async def get_spam_score(message) -> float:
+    response = await aiospamc.check(message, host=SPAMASSASSIN_HOST)
+    return response.headers["Spam"].score
+
+
 class MailHandler:
     async def handle_DATA(self, server, session, envelope: Envelope):
         start = time.time()
@@ -1137,7 +1217,7 @@ class MailHandler:
 
         app = new_app()
         with app.app_context():
-            ret = handle(envelope, smtp)
+            ret = await handle(envelope, smtp)
             LOG.debug("takes %s seconds <<===", time.time() - start)
             return ret
 

example.env

@@ -153,4 +153,7 @@ DISABLE_ONBOARDING=true
 
 # Set the 2 below variables to enable Plausible Analytics
 # PLAUSIBLE_HOST=https://plausible.io
-# PLAUSIBLE_DOMAIN=yourdomain.com
+# PLAUSIBLE_DOMAIN=yourdomain.com
+
+# Spamassassin server
+# SPAMASSASSIN_HOST = 127.0.0.1

requirements.in

@@ -43,4 +43,5 @@ pyspf
 Flask-Limiter
 memory_profiler
 gevent
-aiocontextvars
+aiocontextvars
+aiospamc

requirements.txt

@@ -8,6 +8,7 @@ aiocontextvars==0.2.2     # via -r requirements.in
 aiohttp==3.5.4            # via raven-aiohttp, yacron
 aiosmtpd==1.2             # via -r requirements.in
 aiosmtplib==1.0.6         # via yacron
+aiospamc==0.6.1           # via -r requirements.in
 alembic==1.0.10           # via flask-migrate
 appnope==0.1.0            # via ipython
 arrow==0.14.2             # via -r requirements.in
@@ -23,7 +24,7 @@ boto3==1.9.167            # via -r requirements.in, watchtower
 botocore==1.12.167        # via boto3, s3transfer
 cachetools==4.0.0         # via google-auth
 cbor2==5.1.0              # via webauthn
-certifi==2019.3.9         # via requests, sentry-sdk
+certifi==2019.11.28       # via aiospamc, requests, sentry-sdk
 cffi==1.12.3              # via bcrypt, cryptography
 chardet==3.0.4            # via aiohttp, requests
 click==7.0                # via flask, pip-tools

templates/emails/transactional/spam-email-reply-phase.html

@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+
+{% block content %}
+  {{ render_text("Hi " + name) }}
+  {{ render_text("An email sent from your alias <b>" + alias.email + "</b> to <b>" + website_email + "</b> is detected as spam by our Spam Detection Engine (SpamAssassin).") }}
+
+  {{ render_text('In most of the cases, the email will be refused by your contact.') }}
+
+  {{ render_button("View the email", refused_email_url) }}
+
+  {{ render_text('The email is automatically deleted in 7 days.') }}
+
+  {{ render_text('Please let us know if you have any question by replying to this email.') }}
+
+  {{ render_text('Thanks, <br />SimpleLogin Team.') }}
+  {{ raw_url(disable_alias_link) }}
+{% endblock %}

templates/emails/transactional/spam-email-reply-phase.txt

@@ -0,0 +1,15 @@
+Hi {{name}}
+
+An email sent from your alias {{alias.email}} to {{website_email}} is detected as spam by our Spam Detection Engine (SpamAssassin).
+
+In most of the cases, the email will be refused by your contact.
+
+You can view this email here:
+{{ refused_email_url }}
+
+The email is automatically deleted in 7 days.
+
+Please let us know if you have any question by replying to this email.
+
+Best,
+SimpleLogin team.