diff --git a/app/config.py b/app/config.py index 82626d41..fb77039f 100644 --- a/app/config.py +++ b/app/config.py @@ -74,7 +74,6 @@ MONITORING_EMAIL = os.environ.get("MONITORING_EMAIL") # VERP: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX BOUNCE_PREFIX = os.environ.get("BOUNCE_PREFIX") or "bounce+" BOUNCE_SUFFIX = os.environ.get("BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}" -BOUNCE_EMAIL = BOUNCE_PREFIX + "{}" + BOUNCE_SUFFIX # Used for VERP during reply phase. It's similar to BOUNCE_PREFIX. # It's needed when sending emails from custom domain to respect DMARC. @@ -93,9 +92,6 @@ TRANSACTIONAL_BOUNCE_PREFIX = ( TRANSACTIONAL_BOUNCE_SUFFIX = ( os.environ.get("TRANSACTIONAL_BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}" ) -TRANSACTIONAL_BOUNCE_EMAIL = ( - TRANSACTIONAL_BOUNCE_PREFIX + "{}" + TRANSACTIONAL_BOUNCE_SUFFIX -) try: MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"]) @@ -169,6 +165,8 @@ DB_URI = os.environ["DB_URI"] # Flask secret FLASK_SECRET = os.environ["FLASK_SECRET"] +if not FLASK_SECRET: + raise RuntimeError("FLASK_SECRET is empty. Please define it.") SESSION_COOKIE_NAME = "slapp" MAILBOX_SECRET = FLASK_SECRET + "mailbox" CUSTOM_ALIAS_SECRET = FLASK_SECRET + "custom_alias" @@ -429,6 +427,18 @@ ZENDESK_ENABLED = "ZENDESK_ENABLED" in os.environ DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ +# Bounces can happen after 5 days +VERP_MESSAGE_LIFETIME = 5 * 86400 +VERP_PREFIX = os.environ.get("VERP_PREFIX") or "sl" +# Generate with python3 -c 'import secrets; print(secrets.token_hex(28))' +VERP_EMAIL_SECRET = os.environ.get("VERP_EMAIL_SECRET") or ( + FLASK_SECRET + "pleasegenerateagoodrandomtoken" +) +if len(VERP_EMAIL_SECRET) < 32: + raise RuntimeError( + "Please, set VERP_EMAIL_SECRET to a random string at least 32 chars long" + ) + def get_allowed_redirect_domains() -> List[str]: allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list) diff --git a/app/email_utils.py b/app/email_utils.py index 6b43dd72..bef720e6 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -1,5 +1,7 @@ import base64 import enum +import hmac +import json import os import quopri import random @@ -49,13 +51,15 @@ from app.config import ( LANDING_PAGE_URL, EMAIL_DOMAIN, ALERT_DIRECTORY_DISABLED_ALIAS_CREATION, - TRANSACTIONAL_BOUNCE_EMAIL, ALERT_SPF, ALERT_INVALID_TOTP_LOGIN, TEMP_DIR, ALIAS_AUTOMATIC_DISABLE, RSPAMD_SIGN_DKIM, NOREPLY, + VERP_PREFIX, + VERP_MESSAGE_LIFETIME, + VERP_EMAIL_SECRET, ) from app.db import Session from app.dns_utils import get_mx_domains @@ -73,6 +77,7 @@ from app.models import ( TransactionalEmail, IgnoreBounceSender, InvalidMailboxDomain, + VerpType, ) from app.utils import ( random_string, @@ -324,7 +329,7 @@ def send_email( # use a different envelope sender for each transactional email (aka VERP) sl_sendmail( - TRANSACTIONAL_BOUNCE_EMAIL.format(transaction.id), + generate_verp_email(VerpType.transactional, transaction.id), to_email, msg, retries=retries, @@ -1462,3 +1467,49 @@ def save_envelope_for_debugging(envelope: Envelope, file_name_prefix=None) -> st return file_name return "" + + +def generate_verp_email( + verp_type: VerpType, object_id: int, sender_domain: Optional[str] = None +) -> str: + # Encoded as a list to minimize size of email address + data = [verp_type.bounce_forward.value, object_id, int(time.time())] + json_payload = json.dumps(data).encode("utf-8") + # Signing without itsdangereous because it uses base64 that includes +/= symbols and lower and upper case letters. + # We need to encode in base32 + payload_hmac = hmac.new( + VERP_EMAIL_SECRET.encode("utf-8"), json_payload, "shake128" + ).digest() + encoded_payload = base64.b32encode(json_payload).rstrip(b"=").decode("utf-8") + encoded_signature = base64.b32encode(payload_hmac).rstrip(b"=").decode("utf-8") + return "{}.{}.{}@{}".format( + VERP_PREFIX, encoded_payload, encoded_signature, sender_domain or EMAIL_DOMAIN + ).lower() + + +# This method processes the email address, checks if it's a signed verp email generated by us to receive bounces +# and extracts the type of verp email and associated email log id/transactional email id stored as object_id +def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]: + idx = email.find("@") + if idx == -1: + return None + username = email[:idx] + fields = username.split(".") + if len(fields) != 3 or fields[0] != VERP_PREFIX: + return None + padding = 8 - (len(fields[1]) % 8) + payload = base64.b32decode(fields[1].encode("utf-8").upper() + (b"=" * padding)) + padding = 8 - (len(fields[2]) % 8) + signature = base64.b32decode(fields[2].encode("utf-8").upper() + (b"=" * padding)) + expected_signature = hmac.new( + VERP_EMAIL_SECRET.encode("utf-8"), payload, "shake128" + ).digest() + if expected_signature != signature: + return None + data = json.loads(payload) + # verp type, object_id, time + if len(data) != 3: + return None + if data[2] > time.time() + VERP_MESSAGE_LIFETIME: + return None + return VerpType(data[0]), data[1] diff --git a/app/models.py b/app/models.py index 3f77b506..ed30ca52 100644 --- a/app/models.py +++ b/app/models.py @@ -237,6 +237,12 @@ class AuditLogActionEnum(EnumE): extend_subscription = 7 +class VerpType(EnumE): + bounce_forward = 0 + bounce_reply = 1 + transactional = 2 + + class Hibp(Base, ModelMixin): __tablename__ = "hibp" name = sa.Column(sa.String(), nullable=False, unique=True, index=True) diff --git a/email_handler.py b/email_handler.py index 8a8ea7c8..2ea3375e 100644 --- a/email_handler.py +++ b/email_handler.py @@ -72,7 +72,6 @@ from app.config import ( PGP_SENDER_PRIVATE_KEY, ALERT_BOUNCE_EMAIL_REPLY_PHASE, NOREPLY, - BOUNCE_EMAIL, BOUNCE_PREFIX, BOUNCE_SUFFIX, TRANSACTIONAL_BOUNCE_PREFIX, @@ -137,6 +136,8 @@ from app.email_utils import ( get_mailbox_bounce_info, save_email_for_debugging, save_envelope_for_debugging, + get_verp_info_from_email, + generate_verp_email, ) from app.errors import ( NonReverseAliasInReplyPhase, @@ -161,6 +162,7 @@ from app.models import ( DeletedAlias, DomainDeletedAlias, Notification, + VerpType, ) from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data from app.utils import sanitize_email @@ -876,7 +878,7 @@ def forward_email_to_mailbox( try: sl_sendmail( # use a different envelope sender for each forward (aka VERP) - BOUNCE_EMAIL.format(email_log.id), + generate_verp_email(VerpType.bounce_forward, email_log.id), mailbox.email, msg, envelope.mail_options, @@ -1167,12 +1169,9 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str): if should_add_dkim_signature(alias_domain): add_dkim_signature(msg, alias_domain) - # generate a mail_from for VERP - verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}" - try: sl_sendmail( - verp_mail_from, + generate_verp_email(VerpType.bounce_reply, email_log.id, alias_domain), contact.website_email, msg, envelope.mail_options, @@ -2013,11 +2012,13 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str: return status.E202 -def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to): +def handle_transactional_bounce( + envelope: Envelope, msg, rcpt_to, transactional_id=None +): LOG.d("handle transactional bounce sent to %s", rcpt_to) # parse the TransactionalEmail - transactional_id = parse_id_from_bounce(rcpt_to) + transactional_id = transactional_id or parse_id_from_bounce(rcpt_to) transactional = TransactionalEmail.get(transactional_id) # a transaction might have been deleted in delete_logs() @@ -2211,15 +2212,18 @@ def handle(envelope: Envelope, msg: Message) -> str: return handle_unsubscribe(envelope, msg) # region mail sent to VERP + verp_info = get_verp_info_from_email(rcpt_tos[0]) # sent to transactional VERP. Either bounce emails or out-of-office if ( len(rcpt_tos) == 1 and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX) and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX) - ): + ) or (verp_info and verp_info[0] == VerpType.transactional): if is_bounce(envelope, msg): - handle_transactional_bounce(envelope, msg, rcpt_tos[0]) + handle_transactional_bounce( + envelope, msg, rcpt_tos[0], verp_info and verp_info[1] + ) return status.E205 elif is_automatic_out_of_office(msg): LOG.d( @@ -2234,8 +2238,8 @@ def handle(envelope: Envelope, msg: Message) -> str: len(rcpt_tos) == 1 and rcpt_tos[0].startswith(BOUNCE_PREFIX) and rcpt_tos[0].endswith(BOUNCE_SUFFIX) - ): - email_log_id = parse_id_from_bounce(rcpt_tos[0]) + ) or (verp_info and verp_info[0] == VerpType.bounce_forward): + email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0]) email_log = EmailLog.get(email_log_id) if not email_log: @@ -2250,10 +2254,12 @@ def handle(envelope: Envelope, msg: Message) -> str: raise VERPForward # sent to reply VERP, can be either bounce or out-of-office - if len(rcpt_tos) == 1 and rcpt_tos[0].startswith( - f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+" + if ( + len(rcpt_tos) == 1 + and rcpt_tos[0].startswith(f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+") + or (verp_info and verp_info[0] == VerpType.bounce_reply) ): - email_log_id = parse_id_from_bounce(rcpt_tos[0]) + email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0]) email_log = EmailLog.get(email_log_id) if not email_log: @@ -2272,12 +2278,13 @@ def handle(envelope: Envelope, msg: Message) -> str: ) # iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias + verp_info = get_verp_info_from_email(mail_from[0]) if ( len(rcpt_tos) == 1 and mail_from.startswith(BOUNCE_PREFIX) and mail_from.endswith(BOUNCE_SUFFIX) - ): - email_log_id = parse_id_from_bounce(mail_from) + ) or (verp_info and verp_info[0] == VerpType.bounce_forward): + email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(mail_from) email_log = EmailLog.get(email_log_id) alias = Alias.get_by(email=rcpt_tos[0]) LOG.w( diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index 524a7f18..a52af3cf 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -6,15 +6,17 @@ import pytest from aiosmtpd.smtp import Envelope import email_handler -from app.config import BOUNCE_EMAIL, EMAIL_DOMAIN, ALERT_DMARC_FAILED_REPLY_PHASE +from app.config import EMAIL_DOMAIN, ALERT_DMARC_FAILED_REPLY_PHASE from app.db import Session from app.email import headers, status +from app.email_utils import generate_verp_email from app.models import ( Alias, AuthorizedAddress, IgnoredEmail, EmailLog, Notification, + VerpType, Contact, SentAlert, ) @@ -119,10 +121,11 @@ def test_prevent_5xx_from_spf(flask_client): {"alias_email": alias.email, "spf_result": "R_SPF_FAIL"}, ) envelope = Envelope() - envelope.mail_from = BOUNCE_EMAIL.format(999999999999999999) - envelope.rcpt_tos = [msg["to"]] + envelope.mail_from = msg["from"] + # Ensure invalid email log + envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)] result = email_handler.MailHandler()._handle(envelope, msg) - assert result == status.E216 + assert status.E216 == result def test_preserve_5xx_with_valid_spf(flask_client): @@ -133,10 +136,11 @@ def test_preserve_5xx_with_valid_spf(flask_client): {"alias_email": alias.email, "spf_result": "R_SPF_ALLOW"}, ) envelope = Envelope() - envelope.mail_from = BOUNCE_EMAIL.format(999999999999999999) - envelope.rcpt_tos = [msg["to"]] + envelope.mail_from = msg["from"] + # Ensure invalid email log + envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)] result = email_handler.MailHandler()._handle(envelope, msg) - assert result == status.E512 + assert status.E512 == result def test_preserve_5xx_with_no_header(flask_client): @@ -147,10 +151,11 @@ def test_preserve_5xx_with_no_header(flask_client): {"alias_email": alias.email}, ) envelope = Envelope() - envelope.mail_from = BOUNCE_EMAIL.format(999999999999999999) - envelope.rcpt_tos = [msg["to"]] + envelope.mail_from = msg["from"] + # Ensure invalid email log + envelope.rcpt_tos = [generate_verp_email(VerpType.bounce_forward, 99999999999999)] result = email_handler.MailHandler()._handle(envelope, msg) - assert result == status.E512 + assert status.E512 == result def generate_dmarc_result() -> List: diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index 94b57e0b..28ced81a 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -5,7 +5,7 @@ from email.message import EmailMessage import arrow import pytest -from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR +from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, ROOT_DIR from app.db import Session from app.email_utils import ( get_email_domain_part, @@ -36,6 +36,8 @@ from app.email_utils import ( get_orig_message_from_bounce, get_mailbox_bounce_info, is_invalid_mailbox_domain, + generate_verp_email, + get_verp_info_from_email, ) from app.models import ( CustomDomain, @@ -44,6 +46,7 @@ from app.models import ( EmailLog, IgnoreBounceSender, InvalidMailboxDomain, + VerpType, ) # flake8: noqa: E101, W191 @@ -712,7 +715,6 @@ def test_should_disable_bounce_consecutive_days(flask_client): def test_parse_id_from_bounce(): assert parse_id_from_bounce("bounces+1234+@local") == 1234 assert parse_id_from_bounce("anything+1234+@local") == 1234 - assert parse_id_from_bounce(BOUNCE_EMAIL.format(1234)) == 1234 def test_get_queue_id(): @@ -768,6 +770,14 @@ def test_is_invalid_mailbox_domain(flask_client): assert not is_invalid_mailbox_domain("xy.zt") +def test_generate_verp_email(): + generated_email = generate_verp_email(VerpType.bounce_forward, 1, "somewhere.net") + print(generated_email) + info = get_verp_info_from_email(generated_email.lower()) + assert info[0] == VerpType.bounce_forward + assert info[1] == 1 + + def test_add_header_multipart_with_invalid_part(): msg = load_eml_file("multipart_alternative.eml") parts = msg.get_payload() + ["invalid"]