diff --git a/app/config.py b/app/config.py index 4a83f985..82626d41 100644 --- a/app/config.py +++ b/app/config.py @@ -304,6 +304,9 @@ MAX_ALERT_24H = 4 # When a reverse-alias receives emails from un unknown mailbox ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX = "reverse_alias_unknown_mailbox" +# When somebody is trying to spoof a reply +ALERT_DMARC_FAILED_REPLY_PHASE = "dmarc_failed_reply_phase" + # When a forwarding email is bounced ALERT_BOUNCE_EMAIL = "bounce" diff --git a/app/email_utils.py b/app/email_utils.py index 26194158..6b43dd72 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -73,9 +73,6 @@ from app.models import ( TransactionalEmail, IgnoreBounceSender, InvalidMailboxDomain, - DmarcCheckResult, - SpamdResult, - SPFCheckResult, ) from app.utils import ( random_string, @@ -1465,28 +1462,3 @@ def save_envelope_for_debugging(envelope: Envelope, file_name_prefix=None) -> st return file_name return "" - - -def get_spamd_result(msg: Message) -> Optional[SpamdResult]: - spam_result_header = msg.get_all(headers.SPAMD_RESULT) - if not spam_result_header: - newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"}) - return None - - spam_entries = [entry.strip() for entry in str(spam_result_header[-1]).split("\n")] - for entry_pos in range(len(spam_entries)): - sep = spam_entries[entry_pos].find("(") - if sep > -1: - spam_entries[entry_pos] = spam_entries[entry_pos][:sep] - - spamd_result = SpamdResult() - - for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items(): - if header_value in spam_entries: - spamd_result.set_dmarc_result(dmarc_result) - for header_value, spf_result in SPFCheckResult.get_string_dict().items(): - if header_value in spam_entries: - spamd_result.set_spf_result(spf_result) - - newrelic.agent.record_custom_event("SpamdCheck", spamd_result.event_data()) - return spamd_result diff --git a/app/errors.py b/app/errors.py index e041c763..26be6091 100644 --- a/app/errors.py +++ b/app/errors.py @@ -56,8 +56,3 @@ class MailSentFromReverseAlias(SLException): """raised when receiving an email sent from a reverse alias""" pass - - -class DmarcSoftFail(SLException): - - pass diff --git a/tests/email/__init__.py b/app/handler/__init__.py similarity index 100% rename from tests/email/__init__.py rename to app/handler/__init__.py diff --git a/app/handler/dmarc.py b/app/handler/dmarc.py new file mode 100644 index 00000000..8ba6d0a9 --- /dev/null +++ b/app/handler/dmarc.py @@ -0,0 +1,155 @@ +import uuid +from io import BytesIO +from typing import Optional, Tuple + +from aiosmtpd.handlers import Message +from aiosmtpd.smtp import Envelope + +from app import s3 +from app.config import ( + DMARC_CHECK_ENABLED, + ALERT_QUARANTINE_DMARC, + ALERT_DMARC_FAILED_REPLY_PHASE, +) +from app.email import headers, status +from app.email_utils import ( + get_header_unicode, + send_email_with_rate_control, + render, + add_or_replace_header, + to_bytes, + add_header, +) +from app.handler.spamd_result import SpamdResult, Phase, DmarcCheckResult +from app.log import LOG +from app.models import Alias, Contact, Notification, EmailLog, RefusedEmail + + +def apply_dmarc_policy_for_forward_phase( + alias: Alias, contact: Contact, envelope: Envelope, msg: Message +) -> Tuple[Message, Optional[str]]: + spam_result = SpamdResult.extract_from_headers(msg, Phase.forward) + if not DMARC_CHECK_ENABLED or not spam_result: + return msg, None + + from_header = get_header_unicode(msg[headers.FROM]) + + if spam_result.dmarc == DmarcCheckResult.soft_fail: + LOG.w( + f"dmarc forward: soft_fail from contact {contact.email} to alias {alias.email}." + f"mail_from:{envelope.mail_from}, from_header: {from_header}" + ) + changed_msg = add_header( + msg, + f"""This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content.""", + f""" +
+ This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content. +
+ """, + ) + return changed_msg, None + + if spam_result.dmarc in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + ): + LOG.w( + f"dmarc forward: put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, " + f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" + ) + email_log = quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) + Notification.create( + user_id=alias.user_id, + title=f"{alias.email} has a new mail in quarantine", + message=Notification.render( + "notification/message-quarantine.html", alias=alias + ), + commit=True, + ) + user = alias.user + send_email_with_rate_control( + user, + ALERT_QUARANTINE_DMARC, + user.email, + f"An email sent to {alias.email} has been quarantined", + render( + "transactional/message-quarantine-dmarc.txt.jinja2", + from_header=from_header, + alias=alias, + refused_email_url=email_log.get_dashboard_url(), + ), + render( + "transactional/message-quarantine-dmarc.html", + from_header=from_header, + alias=alias, + refused_email_url=email_log.get_dashboard_url(), + ), + max_nb_alert=10, + ignore_smtp_error=True, + ) + return msg, status.E215 + + return msg, None + + +def quarantine_dmarc_failed_forward_email(alias, contact, envelope, msg) -> EmailLog: + add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") + msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from + random_name = str(uuid.uuid4()) + s3_report_path = f"refused-emails/full-{random_name}.eml" + s3.upload_email_from_bytesio( + s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}" + ) + refused_email = RefusedEmail.create( + full_report_path=s3_report_path, user_id=alias.user_id, flush=True + ) + return EmailLog.create( + user_id=alias.user_id, + mailbox_id=alias.mailbox_id, + contact_id=contact.id, + alias_id=alias.id, + message_id=str(msg[headers.MESSAGE_ID]), + refused_email_id=refused_email.id, + is_spam=True, + blocked=True, + commit=True, + ) + + +def apply_dmarc_policy_for_reply_phase( + alias_from: Alias, contact_recipient: Contact, envelope: Envelope, msg: Message +) -> Optional[str]: + spam_result = SpamdResult.extract_from_headers(msg, Phase.reply) + if not DMARC_CHECK_ENABLED or not spam_result: + return None + + if spam_result.dmarc not in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + DmarcCheckResult.soft_fail, + ): + return None + LOG.w( + f"dmarc reply: Put email from {alias_from.email} to {contact_recipient} into quarantine. {spam_result.event_data()}, " + f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" + ) + send_email_with_rate_control( + alias_from.user, + ALERT_DMARC_FAILED_REPLY_PHASE, + alias_from.user.email, + f"Attempt to send an email to your contact {contact_recipient.email} from {envelope.mail_from}", + render( + "transactional/spoof-reply.txt", + contact=contact_recipient, + alias=alias_from, + sender=envelope.mail_from, + ), + render( + "transactional/spoof-reply.html", + contact=contact_recipient, + alias=alias_from, + sender=envelope.mail_from, + ), + ) + return status.E215 diff --git a/app/handler/spamd_result.py b/app/handler/spamd_result.py new file mode 100644 index 00000000..57cc250b --- /dev/null +++ b/app/handler/spamd_result.py @@ -0,0 +1,127 @@ +from __future__ import annotations +from typing import Dict, Optional + +import newrelic + +from app.email import headers +from app.models import EnumE +from email.message import Message + + +class Phase(EnumE): + unknown = 0 + forward = 1 + reply = 2 + + +class DmarcCheckResult(EnumE): + allow = 0 + soft_fail = 1 + quarantine = 2 + reject = 3 + not_available = 4 + bad_policy = 5 + + @staticmethod + def get_string_dict(): + return { + "DMARC_POLICY_ALLOW": DmarcCheckResult.allow, + "DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail, + "DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine, + "DMARC_POLICY_REJECT": DmarcCheckResult.reject, + "DMARC_NA": DmarcCheckResult.not_available, + "DMARC_BAD_POLICY": DmarcCheckResult.bad_policy, + } + + +class SPFCheckResult(EnumE): + allow = 0 + fail = 1 + soft_fail = 1 + neutral = 2 + temp_error = 3 + not_available = 4 + perm_error = 5 + + @staticmethod + def get_string_dict(): + return { + "R_SPF_ALLOW": SPFCheckResult.allow, + "R_SPF_FAIL": SPFCheckResult.fail, + "R_SPF_SOFTFAIL": SPFCheckResult.soft_fail, + "R_SPF_NEUTRAL": SPFCheckResult.neutral, + "R_SPF_DNSFAIL": SPFCheckResult.temp_error, + "R_SPF_NA": SPFCheckResult.not_available, + "R_SPF_PERMFAIL": SPFCheckResult.perm_error, + } + + +class SpamdResult: + def __init__(self, phase: Phase = Phase.unknown): + self.phase: Phase = phase + self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available + self.spf: SPFCheckResult = SPFCheckResult.not_available + + def set_dmarc_result(self, dmarc_result: DmarcCheckResult): + self.dmarc = dmarc_result + + def set_spf_result(self, spf_result: SPFCheckResult): + self.spf = spf_result + + def event_data(self) -> Dict: + return { + "header": "present", + "dmarc": self.dmarc.name, + "spf": self.spf.name, + "phase": self.phase.name, + } + + @classmethod + def extract_from_headers( + cls, msg: Message, phase: Phase = Phase.unknown + ) -> Optional[SpamdResult]: + cached = cls._get_from_message(msg) + if cached: + return cached + + spam_result_header = msg.get_all(headers.SPAMD_RESULT) + if not spam_result_header: + return None + + spam_entries = [ + entry.strip() for entry in str(spam_result_header[-1]).split("\n") + ] + for entry_pos in range(len(spam_entries)): + sep = spam_entries[entry_pos].find("(") + if sep > -1: + spam_entries[entry_pos] = spam_entries[entry_pos][:sep] + + spamd_result = SpamdResult(phase) + + for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items(): + if header_value in spam_entries: + spamd_result.set_dmarc_result(dmarc_result) + break + for header_value, spf_result in SPFCheckResult.get_string_dict().items(): + if header_value in spam_entries: + spamd_result.set_spf_result(spf_result) + break + + cls._store_in_message(spamd_result, msg) + return spamd_result + + @classmethod + def _store_in_message(cls, check: SpamdResult, msg: Message): + msg.spamd_check = check + + @classmethod + def _get_from_message(cls, msg: Message) -> Optional[SpamdResult]: + return getattr(msg, "spamd_check", None) + + @classmethod + def send_to_new_relic(cls, msg: Message): + check = cls._get_from_message(msg) + if check: + newrelic.agent.record_custom_event("SpamdCheck", check.event_data()) + else: + newrelic.agent.record_custom_event("SpamdCheck", {"header": "missing"}) diff --git a/app/models.py b/app/models.py index b40ca741..3f77b506 100644 --- a/app/models.py +++ b/app/models.py @@ -3,7 +3,7 @@ import os import random import uuid from email.utils import formataddr -from typing import List, Tuple, Optional, Dict +from typing import List, Tuple, Optional import arrow import sqlalchemy as sa @@ -237,63 +237,6 @@ class AuditLogActionEnum(EnumE): extend_subscription = 7 -class DmarcCheckResult(EnumE): - allow = 0 - soft_fail = 1 - quarantine = 2 - reject = 3 - not_available = 4 - bad_policy = 5 - - @staticmethod - def get_string_dict(): - return { - "DMARC_POLICY_ALLOW": DmarcCheckResult.allow, - "DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail, - "DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine, - "DMARC_POLICY_REJECT": DmarcCheckResult.reject, - "DMARC_NA": DmarcCheckResult.not_available, - "DMARC_BAD_POLICY": DmarcCheckResult.bad_policy, - } - - -class SPFCheckResult(EnumE): - allow = 0 - fail = 1 - soft_fail = 1 - neutral = 2 - temp_error = 3 - not_available = 4 - perm_error = 5 - - @staticmethod - def get_string_dict(): - return { - "R_SPF_ALLOW": SPFCheckResult.allow, - "R_SPF_FAIL": SPFCheckResult.fail, - "R_SPF_SOFTFAIL": SPFCheckResult.soft_fail, - "R_SPF_NEUTRAL": SPFCheckResult.neutral, - "R_SPF_DNSFAIL": SPFCheckResult.temp_error, - "R_SPF_NA": SPFCheckResult.not_available, - "R_SPF_PERMFAIL": SPFCheckResult.perm_error, - } - - -class SpamdResult: - def __init__(self): - self.dmarc: DmarcCheckResult = DmarcCheckResult.not_available - self.spf: SPFCheckResult = SPFCheckResult.not_available - - def set_dmarc_result(self, dmarc_result: DmarcCheckResult): - self.dmarc = dmarc_result - - def set_spf_result(self, spf_result: SPFCheckResult): - self.spf = spf_result - - def event_data(self) -> Dict: - return {"header": "present", "dmarc": self.dmarc, "spf": self.spf} - - class Hibp(Base, ModelMixin): __tablename__ = "hibp" name = sa.Column(sa.String(), nullable=False, unique=True, index=True) diff --git a/email_handler.py b/email_handler.py index a66b6f95..8a6cbe8c 100644 --- a/email_handler.py +++ b/email_handler.py @@ -87,10 +87,16 @@ from app.config import ( OLD_UNSUBSCRIBER, ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS, ALERT_TO_NOREPLY, - DMARC_CHECK_ENABLED, - ALERT_QUARANTINE_DMARC, ) from app.db import Session +from app.handler.dmarc import ( + apply_dmarc_policy_for_reply_phase, + apply_dmarc_policy_for_forward_phase, +) +from app.handler.spamd_result import ( + SpamdResult, + SPFCheckResult, +) from app.email import status, headers from app.email.rate_limit import rate_limited from app.email.spam import get_spam_score @@ -130,7 +136,6 @@ from app.email_utils import ( get_orig_message_from_yahoo_complaint, get_mailbox_bounce_info, save_email_for_debugging, - get_spamd_result, save_envelope_for_debugging, ) from app.errors import ( @@ -139,7 +144,6 @@ from app.errors import ( VERPForward, VERPReply, CannotCreateContactForReverseAlias, - DmarcSoftFail, ) from app.log import LOG, set_message_id from app.models import ( @@ -157,8 +161,6 @@ from app.models import ( DeletedAlias, DomainDeletedAlias, Notification, - DmarcCheckResult, - SPFCheckResult, ) from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data from app.utils import sanitize_email @@ -543,99 +545,6 @@ def handle_email_sent_to_ourself(alias, from_addr: str, msg: Message, user): ) -def apply_dmarc_policy( - alias: Alias, contact: Contact, envelope: Envelope, msg: Message -) -> Optional[str]: - spam_result = get_spamd_result(msg) - if not DMARC_CHECK_ENABLED or not spam_result: - return None - - from_header = get_header_unicode(msg[headers.FROM]) - - if spam_result.dmarc == DmarcCheckResult.soft_fail: - LOG.w( - f"dmarc soft_fail from contact {contact.email} to alias {alias.email}." - f"mail_from:{envelope.mail_from}, from_header: {from_header}" - ) - raise DmarcSoftFail - - if spam_result.dmarc in ( - DmarcCheckResult.quarantine, - DmarcCheckResult.reject, - ): - LOG.w( - f"put email from {contact} to {alias} to quarantine. {spam_result.event_data()}, " - f"mail_from:{envelope.mail_from}, from_header: {msg[headers.FROM]}" - ) - email_log = quarantine_dmarc_failed_email(alias, contact, envelope, msg) - Notification.create( - user_id=alias.user_id, - title=f"{alias.email} has a new mail in quarantine", - message=Notification.render( - "notification/message-quarantine.html", alias=alias - ), - commit=True, - ) - user = alias.user - send_email_with_rate_control( - user, - ALERT_QUARANTINE_DMARC, - user.email, - f"An email sent to {alias.email} has been quarantined", - render( - "transactional/message-quarantine-dmarc.txt.jinja2", - from_header=from_header, - alias=alias, - refused_email_url=email_log.get_dashboard_url(), - ), - render( - "transactional/message-quarantine-dmarc.html", - from_header=from_header, - alias=alias, - refused_email_url=email_log.get_dashboard_url(), - ), - max_nb_alert=10, - ignore_smtp_error=True, - ) - return status.E215 - - return None - - -def quarantine_dmarc_failed_email(alias, contact, envelope, msg) -> EmailLog: - add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") - msg[headers.SL_ENVELOPE_TO] = alias.email - msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from - add_or_replace_header(msg, "From", contact.new_addr()) - # replace CC & To emails by reverse-alias for all emails that are not alias - try: - replace_header_when_forward(msg, alias, "Cc") - replace_header_when_forward(msg, alias, "To") - except CannotCreateContactForReverseAlias: - Session.commit() - raise - - random_name = str(uuid.uuid4()) - s3_report_path = f"refused-emails/full-{random_name}.eml" - s3.upload_email_from_bytesio( - s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}" - ) - refused_email = RefusedEmail.create( - full_report_path=s3_report_path, user_id=alias.user_id, flush=True - ) - return EmailLog.create( - user_id=alias.user_id, - mailbox_id=alias.mailbox_id, - contact_id=contact.id, - alias_id=alias.id, - message_id=str(msg[headers.MESSAGE_ID]), - refused_email_id=refused_email.id, - is_spam=True, - blocked=True, - commit=True, - ) - - def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return an array of SMTP status (is_success, smtp_status) is_success indicates whether an email has been delivered and @@ -717,20 +626,11 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str return [(True, res_status)] # Check if we need to reject or quarantine based on dmarc - try: - dmarc_delivery_status = apply_dmarc_policy(alias, contact, envelope, msg) - if dmarc_delivery_status is not None: - return [(False, dmarc_delivery_status)] - except DmarcSoftFail: - msg = add_header( - msg, - f"""This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content.""", - f""" -- This email failed anti-phishing checks when it was received by SimpleLogin, be careful with its content. -
-""", - ) + msg, dmarc_delivery_status = apply_dmarc_policy_for_forward_phase( + alias, contact, envelope, msg + ) + if dmarc_delivery_status is not None: + return [(False, dmarc_delivery_status)] ret = [] mailboxes = alias.mailboxes @@ -1042,6 +942,7 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): Return whether an email has been delivered and the smtp status ("250 Message accepted", "550 Non-existent email address", etc) """ + reply_email = rcpt_to # reply_email must end with EMAIL_DOMAIN @@ -1077,7 +978,14 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): alias, contact, ) - return [(False, status.E504)] + return False, status.E504 + + # Check if we need to reject or quarantine based on dmarc + dmarc_delivery_status = apply_dmarc_policy_for_reply_phase( + alias, contact, envelope, msg + ) + if dmarc_delivery_status is not None: + return False, dmarc_delivery_status # Anti-spoofing mailbox = get_mailbox_from_mail_from(mail_from, alias) @@ -2613,9 +2521,9 @@ class MailHandler: return_status = handle(envelope, msg) elapsed = time.time() - start # Only bounce messages if the return-path passes the spf check. Otherwise black-hole it. + spamd_result = SpamdResult.extract_from_headers(msg) if return_status[0] == "5": - spamd_result = get_spamd_result(msg) - if spamd_result and get_spamd_result(msg).spf in ( + if spamd_result and spamd_result.spf in ( SPFCheckResult.fail, SPFCheckResult.soft_fail, ): @@ -2631,6 +2539,8 @@ class MailHandler: elapsed, return_status, ) + + SpamdResult.send_to_new_relic(msg) newrelic.agent.record_custom_metric("Custom/email_handler_time", elapsed) newrelic.agent.record_custom_metric("Custom/number_incoming_email", 1) return return_status diff --git a/templates/emails/transactional/spoof-reply.html b/templates/emails/transactional/spoof-reply.html new file mode 100644 index 00000000..83766dba --- /dev/null +++ b/templates/emails/transactional/spoof-reply.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} + +{% block content %} + + {% call text() %} +