diff --git a/app/email_utils.py b/app/email_utils.py index b11109a8..7f2403e1 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -3,7 +3,7 @@ from email.message import Message from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import make_msgid, formatdate +from email.utils import make_msgid, formatdate, parseaddr, formataddr from smtplib import SMTP from typing import Optional @@ -363,3 +363,33 @@ def get_orig_message_from_bounce(msg: Message) -> Message: # 7th is original message if i == 7: return part + + +def new_addr(old_addr, new_email) -> str: + """replace First Last by + first@example.com by SimpleLogin + + `new_email` is a special reply address + """ + name, old_email = parseaddr(old_addr) + new_name = f"{old_email} via SimpleLogin" + new_addr = formataddr((new_name, new_email)).strip() + + return new_addr.strip() + + +def get_addrs_from_header(msg: Message, header) -> [str]: + """Get all addresses contained in `header` + Used for To or CC header. + """ + ret = [] + header_content = msg.get_all(header) + if not header_content: + return ret + + for addrs in header_content: + for addr in addrs.split(","): + ret.append(addr.strip()) + + # do not return empty string + return [r for r in ret if r] diff --git a/app/models.py b/app/models.py index 0068e986..7999b28d 100644 --- a/app/models.py +++ b/app/models.py @@ -729,6 +729,9 @@ class Contact(db.Model, ModelMixin): # it has the prefix "reply+" to distinguish with other email reply_email = db.Column(db.String(512), nullable=False) + # whether a contact is created via CC + is_cc = db.Column(db.Boolean, nullable=False, default=False, server_default="0") + alias = db.relationship(Alias, backref="contacts") def website_send_to(self): @@ -757,6 +760,9 @@ class Contact(db.Model, ModelMixin): .first() ) + def __repr__(self): + return f"" + class EmailLog(db.Model, ModelMixin): user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False) diff --git a/email_handler.py b/email_handler.py index 5a2e9b12..0c739661 100644 --- a/email_handler.py +++ b/email_handler.py @@ -38,7 +38,7 @@ from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.parser import Parser from email.policy import SMTPUTF8 -from email.utils import parseaddr, formataddr +from email.utils import parseaddr from io import BytesIO from smtplib import SMTP from typing import Optional @@ -65,6 +65,8 @@ from app.email_utils import ( render, get_orig_message_from_bounce, delete_all_headers_except, + new_addr, + get_addrs_from_header, ) from app.extensions import db from app.log import LOG @@ -226,14 +228,7 @@ def get_or_create_contact(website_from_header: str, alias: Alias) -> Contact: website_from_header, ) - # generate a reply_email, make sure it is unique - # not use while loop to avoid infinite loop - reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}" - for _ in range(1000): - if not Contact.get_by(reply_email=reply_email): - # found! - break - reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}" + reply_email = generate_reply_email() contact = Contact.create( user_id=alias.user_id, @@ -247,8 +242,118 @@ def get_or_create_contact(website_from_header: str, alias: Alias) -> Contact: return contact +def replace_header_when_forward(msg: Message, alias: Alias, header: str): + """ + Replace CC or To header by Reply emails in forward phase + """ + addrs = get_addrs_from_header(msg, header) + + # Nothing to do + if not addrs: + return + + new_addrs: [str] = [] + need_replace = False + + for addr in addrs: + name, email = parseaddr(addr) + + # no transformation when alias is already in the header + if email == alias.email: + new_addrs.append(addr) + continue + + contact = Contact.get_by(alias_id=alias.id, website_email=email) + if contact: + # update the website_from if needed + if contact.website_from != addr: + LOG.d("Update website_from for %s to %s", contact, addr) + contact.website_from = addr + db.session.commit() + else: + LOG.debug( + "create contact for alias %s and email %s, header %s", + alias, + email, + header, + ) + + reply_email = generate_reply_email() + + contact = Contact.create( + user_id=alias.user_id, + alias_id=alias.id, + website_email=email, + website_from=addr, + reply_email=reply_email, + is_cc=header.lower() == "cc", + ) + db.session.commit() + + new_addrs.append(new_addr(contact.website_from, contact.reply_email)) + need_replace = True + + if need_replace: + new_header = ",".join(new_addrs) + LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header) + add_or_replace_header(msg, header, new_header) + else: + LOG.d("No need to replace %s header", header) + + +def replace_header_when_reply(msg: Message, alias: Alias, header: str): + """ + Replace CC or To Reply emails by original emails + """ + addrs = get_addrs_from_header(msg, header) + + # Nothing to do + if not addrs: + return + + new_addrs: [str] = [] + need_replace = False + + for addr in addrs: + name, email = parseaddr(addr) + + # no transformation when alias is already in the header + if email == alias.email: + continue + + contact = Contact.get_by(reply_email=email) + if not contact: + LOG.warning("CC email in reply phase %s must be reply emails", email) + # still keep this email in header + new_addrs.append(addr) + continue + + new_addrs.append(contact.website_from or contact.website_email) + need_replace = True + + if need_replace: + new_header = ",".join(new_addrs) + LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header) + add_or_replace_header(msg, header, new_header) + else: + LOG.d("No need to replace %s header", header) + + +def generate_reply_email(): + # generate a reply_email, make sure it is unique + # not use while loop to avoid infinite loop + reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}" + for _ in range(1000): + if not Contact.get_by(reply_email=reply_email): + # found! + break + reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}" + + return reply_email + + def should_append_alias(msg: Message, address: str): - """whether an alias should be appened to TO header in message""" + """whether an alias should be appended to TO header in message""" if msg["To"] and address in msg["To"]: return False @@ -294,8 +399,10 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str): return msg -def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: - """return *status_code message*""" +def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): + """return whether an email has been delivered and + the smtp status ("250 Message accepted", "550 Non-existent email address", etc) + """ address = rcpt_to.lower() # alias@SL alias = Alias.get_by(email=address) @@ -304,7 +411,7 @@ def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: alias = try_auto_create(address) if not alias: LOG.d("alias %s cannot be created on-the-fly, return 550", address) - return "550 SL Email not exist" + return False, "550 SL Email not exist" mailbox = alias.mailbox mailbox_email = mailbox.email @@ -331,15 +438,13 @@ def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: # replace the email part in from: header contact_from_header = msg["From"] contact_name, contact_email = parseaddr(contact_from_header) - new_contact_name = f"{contact_email} via SimpleLogin" - new_from_header = formataddr((new_contact_name, contact.reply_email)).strip() + new_from_header = new_addr(contact_from_header, contact.reply_email) add_or_replace_header(msg, "From", new_from_header) - LOG.d( - "new from header:%s, contact_name %s, contact_email %s", - new_from_header, - contact_name, - contact_email, - ) + LOG.d("new_from_header:%s, old header %s", new_from_header, contact_from_header) + + # replace CC & To emails by reply-email for all emails that are not alias + replace_header_when_forward(msg, alias, "Cc") + replace_header_when_forward(msg, alias, "To") # append alias into the TO header if it's not present in To or CC if should_append_alias(msg, alias.email): @@ -383,31 +488,35 @@ def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: forward_log.blocked = True db.session.commit() - return "250 Message accepted for delivery" + return True, "250 Message accepted for delivery" -def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: +def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str): + """ + return whether an email has been delivered and + the smtp status ("250 Message accepted", "550 Non-existent email address", etc) + """ reply_email = rcpt_to.lower() # reply_email must end with EMAIL_DOMAIN if not reply_email.endswith(EMAIL_DOMAIN): LOG.warning(f"Reply email {reply_email} has wrong domain") - return "550 SL wrong reply email" + return False, "550 SL wrong reply email" contact = Contact.get_by(reply_email=reply_email) if not contact: LOG.warning(f"No such forward-email with {reply_email} as reply-email") - return "550 SL wrong reply email" + return False, "550 SL wrong reply email" + alias = contact.alias address: str = contact.alias.email alias_domain = address[address.find("@") + 1 :] # alias must end with one of the ALIAS_DOMAINS or custom-domain - if not email_belongs_to_alias_domains(address): + if not email_belongs_to_alias_domains(alias.email): if not CustomDomain.get_by(domain=alias_domain): - return "550 SL alias unknown by SimpleLogin" + return False, "550 SL alias unknown by SimpleLogin" - alias = contact.alias user = alias.user mailbox_email = alias.mailbox_email() @@ -418,13 +527,13 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: if envelope.mail_from == "<>": LOG.error( "Bounce when sending to alias %s from %s, user %s", - address, + alias, contact.website_from, alias.user, ) handle_bounce(contact, alias, msg, user, mailbox_email) - return "550 SL ignored" + return False, "550 SL ignored" # only mailbox can send email to the reply-email if envelope.mail_from.lower() != mailbox_email.lower(): @@ -437,21 +546,20 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: reply_email, ) - user = alias.user send_email( mailbox_email, - f"Reply from your alias {address} only works from your mailbox", + f"Reply from your alias {alias.email} only works from your mailbox", render( "transactional/reply-must-use-personal-email.txt", name=user.name, - alias=address, + alias=alias.email, sender=envelope.mail_from, mailbox_email=mailbox_email, ), render( "transactional/reply-must-use-personal-email.html", name=user.name, - alias=address, + alias=alias.email, sender=envelope.mail_from, mailbox_email=mailbox_email, ), @@ -469,12 +577,12 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: "", ) - return "550 SL ignored" + return False, "550 SL ignored" delete_header(msg, "DKIM-Signature") - # the email comes from alias - add_or_replace_header(msg, "From", address) + # make the email comes from alias + add_or_replace_header(msg, "From", alias.email) # some email providers like ProtonMail adds automatically the Reply-To field # make sure to delete it @@ -483,19 +591,15 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: # remove sender header if present as this could reveal user real email delete_header(msg, "Sender") - add_or_replace_header(msg, "To", contact.website_email) - - # add List-Unsubscribe header - unsubscribe_link = f"{URL}/dashboard/unsubscribe/{contact.alias_id}" - add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>") - add_or_replace_header(msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click") + replace_header_when_reply(msg, alias, "To") + replace_header_when_reply(msg, alias, "Cc") # Received-SPF is injected by postfix-policyd-spf-python can reveal user original email delete_header(msg, "Received-SPF") LOG.d( "send email from %s to %s, mail_options:%s,rcpt_options:%s", - address, + alias.email, contact.website_email, envelope.mail_options, envelope.rcpt_options, @@ -511,7 +615,7 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: msg_raw = msg.as_string().encode() smtp.sendmail( - address, + alias.email, contact.website_email, msg_raw, envelope.mail_options, @@ -521,7 +625,7 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str: EmailLog.create(contact_id=contact.id, is_reply=True, user_id=contact.user_id) db.session.commit() - return "250 Message accepted for delivery" + return True, "250 Message accepted for delivery" def handle_bounce( @@ -639,11 +743,11 @@ def handle_bounce( class MailHandler: async def handle_DATA(self, server, session, envelope): - LOG.debug(">>> New message <<<") - - LOG.debug("Mail from %s", envelope.mail_from) - LOG.debug("Rcpt to %s", envelope.rcpt_tos) - message_data = envelope.content.decode("utf8", errors="replace") + LOG.debug( + "===>> New message, mail from %s, rctp tos %s ", + envelope.mail_from, + envelope.rcpt_tos, + ) if POSTFIX_SUBMISSION_TLS: smtp = SMTP(POSTFIX_SERVER, 587) @@ -651,23 +755,42 @@ class MailHandler: else: smtp = SMTP(POSTFIX_SERVER, 25) - msg = Parser(policy=SMTPUTF8).parsestr(message_data) + # result of all deliveries + # each element is a couple of whether the delivery is successful and the smtp status + res: [(bool, str)] = [] for rcpt_to in envelope.rcpt_tos: + message_data = envelope.content.decode("utf8", errors="replace") + msg = Parser(policy=SMTPUTF8).parsestr(message_data) + # Reply case # recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"): - LOG.debug("Reply phase") + LOG.debug(">>> Reply phase %s -> %s", envelope.mail_from, rcpt_to) app = new_app() with app.app_context(): - return handle_reply(envelope, smtp, msg, rcpt_to) + is_delivered, smtp_status = handle_reply( + envelope, smtp, msg, rcpt_to + ) + res.append((is_delivered, smtp_status)) else: # Forward case - LOG.debug("Forward phase") + LOG.debug(">>> Forward phase %s -> %s", envelope.mail_from, rcpt_to) app = new_app() with app.app_context(): - return handle_forward(envelope, smtp, msg, rcpt_to) + is_delivered, smtp_status = handle_forward( + envelope, smtp, msg, rcpt_to + ) + res.append((is_delivered, smtp_status)) + + for (is_success, smtp_status) in res: + # Consider all deliveries successful if 1 delivery is successful + if is_success: + return smtp_status + + # Failed delivery for all, return the first failure + return res[0][1] if __name__ == "__main__": diff --git a/migrations/versions/2020_032223_67c61eead8d2_.py b/migrations/versions/2020_032223_67c61eead8d2_.py new file mode 100644 index 00000000..9bb6a08e --- /dev/null +++ b/migrations/versions/2020_032223_67c61eead8d2_.py @@ -0,0 +1,29 @@ +"""empty message + +Revision ID: 67c61eead8d2 +Revises: 541ce53ab6e9 +Create Date: 2020-03-22 23:58:02.672562 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '67c61eead8d2' +down_revision = '541ce53ab6e9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('contact', sa.Column('is_cc', sa.Boolean(), server_default='0', nullable=False)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('contact', 'is_cc') + # ### end Alembic commands ###