beenull/simple-login
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #120 from simple-login/cc

Browse source 
Handle CC and multiple recipients
This commit is contained in:
Son Nguyen Kim 2020-03-28 21:48:40 +01:00 committed by GitHub
parent f10d18c020 5771eaeb63
commit 09fd21eda7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 245 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
app/email_utils.py
View file

@ -3,7 +3,7 @@ from email.message import Message
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import make_msgid, formatdate
from email.utils import make_msgid, formatdate, parseaddr, formataddr
from smtplib import SMTP
from typing import Optional
@ -363,3 +363,33 @@ def get_orig_message_from_bounce(msg: Message) -> Message:
# 7th is original message
if i == 7:
return part
def new_addr(old_addr, new_email) -> str:
"""replace First Last <first@example.com> by
first@example.com by SimpleLogin <new_email>
`new_email` is a special reply address
"""
name, old_email = parseaddr(old_addr)
new_name = f"{old_email} via SimpleLogin"
new_addr = formataddr((new_name, new_email)).strip()
return new_addr.strip()
def get_addrs_from_header(msg: Message, header) -> [str]:
"""Get all addresses contained in `header`
Used for To or CC header.
"""
ret = []
header_content = msg.get_all(header)
if not header_content:
return ret
for addrs in header_content:
for addr in addrs.split(","):
ret.append(addr.strip())
# do not return empty string
return [r for r in ret if r]

6
app/models.py
View file

@ -729,6 +729,9 @@ class Contact(db.Model, ModelMixin):
# it has the prefix "reply+" to distinguish with other email
reply_email = db.Column(db.String(512), nullable=False)
# whether a contact is created via CC
is_cc = db.Column(db.Boolean, nullable=False, default=False, server_default="0")
alias = db.relationship(Alias, backref="contacts")
def website_send_to(self):
@ -757,6 +760,9 @@ class Contact(db.Model, ModelMixin):
.first()
)
def __repr__(self):
return f"<Contact {self.id} {self.website_email} {self.alias_id}>"
class EmailLog(db.Model, ModelMixin):
user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False)

235
email_handler.py
View file

@ -38,7 +38,7 @@ from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.parser import Parser
from email.policy import SMTPUTF8
from email.utils import parseaddr, formataddr
from email.utils import parseaddr
from io import BytesIO
from smtplib import SMTP
from typing import Optional
@ -65,6 +65,8 @@ from app.email_utils import (
render,
get_orig_message_from_bounce,
delete_all_headers_except,
new_addr,
get_addrs_from_header,
)
from app.extensions import db
from app.log import LOG
@ -226,14 +228,7 @@ def get_or_create_contact(website_from_header: str, alias: Alias) -> Contact:
website_from_header,
)
# generate a reply_email, make sure it is unique
# not use while loop to avoid infinite loop
reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
for _ in range(1000):
if not Contact.get_by(reply_email=reply_email):
# found!
break
reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
reply_email = generate_reply_email()
contact = Contact.create(
user_id=alias.user_id,
@ -247,8 +242,118 @@ def get_or_create_contact(website_from_header: str, alias: Alias) -> Contact:
return contact
def replace_header_when_forward(msg: Message, alias: Alias, header: str):
"""
Replace CC or To header by Reply emails in forward phase
"""
addrs = get_addrs_from_header(msg, header)
# Nothing to do
if not addrs:
return
new_addrs: [str] = []
need_replace = False
for addr in addrs:
name, email = parseaddr(addr)
# no transformation when alias is already in the header
if email == alias.email:
new_addrs.append(addr)
continue
contact = Contact.get_by(alias_id=alias.id, website_email=email)
if contact:
# update the website_from if needed
if contact.website_from != addr:
LOG.d("Update website_from for %s to %s", contact, addr)
contact.website_from = addr
db.session.commit()
else:
LOG.debug(
"create contact for alias %s and email %s, header %s",
alias,
email,
header,
)
reply_email = generate_reply_email()
contact = Contact.create(
user_id=alias.user_id,
alias_id=alias.id,
website_email=email,
website_from=addr,
reply_email=reply_email,
is_cc=header.lower() == "cc",
)
db.session.commit()
new_addrs.append(new_addr(contact.website_from, contact.reply_email))
need_replace = True
if need_replace:
new_header = ",".join(new_addrs)
LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
add_or_replace_header(msg, header, new_header)
else:
LOG.d("No need to replace %s header", header)
def replace_header_when_reply(msg: Message, alias: Alias, header: str):
"""
Replace CC or To Reply emails by original emails
"""
addrs = get_addrs_from_header(msg, header)
# Nothing to do
if not addrs:
return
new_addrs: [str] = []
need_replace = False
for addr in addrs:
name, email = parseaddr(addr)
# no transformation when alias is already in the header
if email == alias.email:
continue
contact = Contact.get_by(reply_email=email)
if not contact:
LOG.warning("CC email in reply phase %s must be reply emails", email)
# still keep this email in header
new_addrs.append(addr)
continue
new_addrs.append(contact.website_from or contact.website_email)
need_replace = True
if need_replace:
new_header = ",".join(new_addrs)
LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
add_or_replace_header(msg, header, new_header)
else:
LOG.d("No need to replace %s header", header)
def generate_reply_email():
# generate a reply_email, make sure it is unique
# not use while loop to avoid infinite loop
reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
for _ in range(1000):
if not Contact.get_by(reply_email=reply_email):
# found!
break
reply_email = f"reply+{random_string(30)}@{EMAIL_DOMAIN}"
return reply_email
def should_append_alias(msg: Message, address: str):
"""whether an alias should be appened to TO header in message"""
"""whether an alias should be appended to TO header in message"""
if msg["To"] and address in msg["To"]:
return False
@ -294,8 +399,10 @@ def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
return msg
def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
"""return *status_code message*"""
def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
"""return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
address = rcpt_to.lower() # alias@SL
alias = Alias.get_by(email=address)
@ -304,7 +411,7 @@ def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
alias = try_auto_create(address)
if not alias:
LOG.d("alias %s cannot be created on-the-fly, return 550", address)
return "550 SL Email not exist"
return False, "550 SL Email not exist"
mailbox = alias.mailbox
mailbox_email = mailbox.email
@ -331,15 +438,13 @@ def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
# replace the email part in from: header
contact_from_header = msg["From"]
contact_name, contact_email = parseaddr(contact_from_header)
new_contact_name = f"{contact_email} via SimpleLogin"
new_from_header = formataddr((new_contact_name, contact.reply_email)).strip()
new_from_header = new_addr(contact_from_header, contact.reply_email)
add_or_replace_header(msg, "From", new_from_header)
LOG.d(
"new from header:%s, contact_name %s, contact_email %s",
new_from_header,
contact_name,
contact_email,
)
LOG.d("new_from_header:%s, old header %s", new_from_header, contact_from_header)
# replace CC & To emails by reply-email for all emails that are not alias
replace_header_when_forward(msg, alias, "Cc")
replace_header_when_forward(msg, alias, "To")
# append alias into the TO header if it's not present in To or CC
if should_append_alias(msg, alias.email):
@ -383,31 +488,35 @@ def handle_forward(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
forward_log.blocked = True
db.session.commit()
return "250 Message accepted for delivery"
return True, "250 Message accepted for delivery"
def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> (bool, str):
"""
return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to.lower()
# reply_email must end with EMAIL_DOMAIN
if not reply_email.endswith(EMAIL_DOMAIN):
LOG.warning(f"Reply email {reply_email} has wrong domain")
return "550 SL wrong reply email"
return False, "550 SL wrong reply email"
contact = Contact.get_by(reply_email=reply_email)
if not contact:
LOG.warning(f"No such forward-email with {reply_email} as reply-email")
return "550 SL wrong reply email"
return False, "550 SL wrong reply email"
alias = contact.alias
address: str = contact.alias.email
alias_domain = address[address.find("@") + 1 :]
# alias must end with one of the ALIAS_DOMAINS or custom-domain
if not email_belongs_to_alias_domains(address):
if not email_belongs_to_alias_domains(alias.email):
if not CustomDomain.get_by(domain=alias_domain):
return "550 SL alias unknown by SimpleLogin"
return False, "550 SL alias unknown by SimpleLogin"
alias = contact.alias
user = alias.user
mailbox_email = alias.mailbox_email()
@ -418,13 +527,13 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
if envelope.mail_from == "<>":
LOG.error(
"Bounce when sending to alias %s from %s, user %s",
address,
alias,
contact.website_from,
alias.user,
)
handle_bounce(contact, alias, msg, user, mailbox_email)
return "550 SL ignored"
return False, "550 SL ignored"
# only mailbox can send email to the reply-email
if envelope.mail_from.lower() != mailbox_email.lower():
@ -437,21 +546,20 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
reply_email,
)
user = alias.user
send_email(
mailbox_email,
f"Reply from your alias {address} only works from your mailbox",
f"Reply from your alias {alias.email} only works from your mailbox",
render(
"transactional/reply-must-use-personal-email.txt",
name=user.name,
alias=address,
alias=alias.email,
sender=envelope.mail_from,
mailbox_email=mailbox_email,
),
render(
"transactional/reply-must-use-personal-email.html",
name=user.name,
alias=address,
alias=alias.email,
sender=envelope.mail_from,
mailbox_email=mailbox_email,
),
@ -469,12 +577,12 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
"",
)
return "550 SL ignored"
return False, "550 SL ignored"
delete_header(msg, "DKIM-Signature")
# the email comes from alias
add_or_replace_header(msg, "From", address)
# make the email comes from alias
add_or_replace_header(msg, "From", alias.email)
# some email providers like ProtonMail adds automatically the Reply-To field
# make sure to delete it
@ -483,19 +591,15 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
# remove sender header if present as this could reveal user real email
delete_header(msg, "Sender")
add_or_replace_header(msg, "To", contact.website_email)
# add List-Unsubscribe header
unsubscribe_link = f"{URL}/dashboard/unsubscribe/{contact.alias_id}"
add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
add_or_replace_header(msg, "List-Unsubscribe-Post", "List-Unsubscribe=One-Click")
replace_header_when_reply(msg, alias, "To")
replace_header_when_reply(msg, alias, "Cc")
# Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
delete_header(msg, "Received-SPF")
LOG.d(
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
address,
alias.email,
contact.website_email,
envelope.mail_options,
envelope.rcpt_options,
@ -511,7 +615,7 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
msg_raw = msg.as_string().encode()
smtp.sendmail(
address,
alias.email,
contact.website_email,
msg_raw,
envelope.mail_options,
@ -521,7 +625,7 @@ def handle_reply(envelope, smtp: SMTP, msg: Message, rcpt_to: str) -> str:
EmailLog.create(contact_id=contact.id, is_reply=True, user_id=contact.user_id)
db.session.commit()
return "250 Message accepted for delivery"
return True, "250 Message accepted for delivery"
def handle_bounce(
@ -639,11 +743,11 @@ def handle_bounce(
class MailHandler:
async def handle_DATA(self, server, session, envelope):
LOG.debug(">>> New message <<<")
LOG.debug("Mail from %s", envelope.mail_from)
LOG.debug("Rcpt to %s", envelope.rcpt_tos)
message_data = envelope.content.decode("utf8", errors="replace")
LOG.debug(
"===>> New message, mail from %s, rctp tos %s ",
envelope.mail_from,
envelope.rcpt_tos,
)
if POSTFIX_SUBMISSION_TLS:
smtp = SMTP(POSTFIX_SERVER, 587)
@ -651,23 +755,42 @@ class MailHandler:
else:
smtp = SMTP(POSTFIX_SERVER, 25)
msg = Parser(policy=SMTPUTF8).parsestr(message_data)
# result of all deliveries
# each element is a couple of whether the delivery is successful and the smtp status
res: [(bool, str)] = []
for rcpt_to in envelope.rcpt_tos:
message_data = envelope.content.decode("utf8", errors="replace")
msg = Parser(policy=SMTPUTF8).parsestr(message_data)
# Reply case
# recipient starts with "reply+" or "ra+" (ra=reverse-alias) prefix
if rcpt_to.startswith("reply+") or rcpt_to.startswith("ra+"):
LOG.debug("Reply phase")
LOG.debug(">>> Reply phase %s -> %s", envelope.mail_from, rcpt_to)
app = new_app()
with app.app_context():
return handle_reply(envelope, smtp, msg, rcpt_to)
is_delivered, smtp_status = handle_reply(
envelope, smtp, msg, rcpt_to
)
res.append((is_delivered, smtp_status))
else: # Forward case
LOG.debug("Forward phase")
LOG.debug(">>> Forward phase %s -> %s", envelope.mail_from, rcpt_to)
app = new_app()
with app.app_context():
return handle_forward(envelope, smtp, msg, rcpt_to)
is_delivered, smtp_status = handle_forward(
envelope, smtp, msg, rcpt_to
)
res.append((is_delivered, smtp_status))
for (is_success, smtp_status) in res:
# Consider all deliveries successful if 1 delivery is successful
if is_success:
return smtp_status
# Failed delivery for all, return the first failure
return res[0][1]
if __name__ == "__main__":

29
migrations/versions/2020_032223_67c61eead8d2_.py Normal file
View file

@ -0,0 +1,29 @@
"""empty message
Revision ID: 67c61eead8d2
Revises: 541ce53ab6e9
Create Date: 2020-03-22 23:58:02.672562
"""
import sqlalchemy_utils
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '67c61eead8d2'
down_revision = '541ce53ab6e9'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('contact', sa.Column('is_cc', sa.Boolean(), server_default='0', nullable=False))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('contact', 'is_cc')
# ### end Alembic commands ###