2019-11-07 16:49:26 +00:00
|
|
|
"""
|
2019-11-08 10:05:34 +00:00
|
|
|
Handle the email *forward* and *reply*. phase. There are 3 actors:
|
2020-03-26 10:19:20 +00:00
|
|
|
- contact: who sends emails to alias@sl.co address
|
2019-11-07 16:49:26 +00:00
|
|
|
- SL email handler (this script)
|
2020-03-26 10:19:20 +00:00
|
|
|
- user personal email: to be protected. Should never leak to contact.
|
2019-11-07 16:49:26 +00:00
|
|
|
|
|
|
|
This script makes sure that in the forward phase, the email that is forwarded to user personal email has the following
|
|
|
|
envelope and header fields:
|
|
|
|
Envelope:
|
2020-03-26 10:19:20 +00:00
|
|
|
mail from: @contact
|
2019-11-08 10:05:34 +00:00
|
|
|
rcpt to: @personal_email
|
2019-11-07 16:49:26 +00:00
|
|
|
Header:
|
2020-03-26 10:19:20 +00:00
|
|
|
From: @contact
|
2019-11-08 10:05:34 +00:00
|
|
|
To: alias@sl.co # so user knows this email is sent to alias
|
|
|
|
Reply-to: special@sl.co # magic HERE
|
2019-11-07 16:49:26 +00:00
|
|
|
|
|
|
|
And in the reply phase:
|
|
|
|
Envelope:
|
2020-03-26 10:19:20 +00:00
|
|
|
mail from: @contact
|
|
|
|
rcpt to: @contact
|
2019-11-07 16:49:26 +00:00
|
|
|
|
|
|
|
Header:
|
2020-03-26 10:19:20 +00:00
|
|
|
From: alias@sl.co # so for contact the email comes from alias. magic HERE
|
|
|
|
To: @contact
|
2019-11-07 16:49:26 +00:00
|
|
|
|
|
|
|
The special@sl.co allows to hide user personal email when user clicks "Reply" to the forwarded email.
|
|
|
|
It should contain the following info:
|
|
|
|
- alias
|
2020-03-26 10:19:20 +00:00
|
|
|
- @contact
|
2019-11-07 16:49:26 +00:00
|
|
|
|
|
|
|
|
|
|
|
"""
|
2020-09-02 15:36:11 +00:00
|
|
|
import argparse
|
2020-04-02 16:10:08 +00:00
|
|
|
import email
|
2021-09-22 07:58:40 +00:00
|
|
|
import os
|
2019-11-07 16:49:26 +00:00
|
|
|
import time
|
2020-03-15 21:29:53 +00:00
|
|
|
import uuid
|
2020-03-08 22:07:23 +00:00
|
|
|
from email import encoders
|
2020-11-02 18:09:57 +00:00
|
|
|
from email.encoders import encode_noop
|
2020-01-04 09:25:19 +00:00
|
|
|
from email.message import Message
|
2020-03-08 22:07:23 +00:00
|
|
|
from email.mime.application import MIMEApplication
|
|
|
|
from email.mime.multipart import MIMEMultipart
|
2020-11-21 18:15:02 +00:00
|
|
|
from email.utils import formataddr, make_msgid, formatdate, getaddresses
|
2020-03-14 15:34:23 +00:00
|
|
|
from io import BytesIO
|
2021-03-17 09:09:02 +00:00
|
|
|
from smtplib import SMTPRecipientsRefused
|
2020-09-28 15:41:16 +00:00
|
|
|
from typing import List, Tuple, Optional
|
2019-11-07 16:49:26 +00:00
|
|
|
|
2021-07-23 13:48:43 +00:00
|
|
|
import newrelic.agent
|
2020-09-30 09:05:21 +00:00
|
|
|
from aiosmtpd.controller import Controller
|
2020-06-07 09:41:35 +00:00
|
|
|
from aiosmtpd.smtp import Envelope
|
2021-09-10 14:51:36 +00:00
|
|
|
from email_validator import validate_email, EmailNotValidError
|
|
|
|
from flanker.addresslib import address
|
|
|
|
from flanker.addresslib.address import EmailAddress
|
2020-09-03 17:42:52 +00:00
|
|
|
from sqlalchemy.exc import IntegrityError
|
2020-06-07 09:41:35 +00:00
|
|
|
|
2020-03-14 15:34:23 +00:00
|
|
|
from app import pgp_utils, s3
|
2020-04-04 13:24:27 +00:00
|
|
|
from app.alias_utils import try_auto_create
|
2020-02-15 10:04:22 +00:00
|
|
|
from app.config import (
|
|
|
|
EMAIL_DOMAIN,
|
|
|
|
URL,
|
2020-03-28 22:19:25 +00:00
|
|
|
UNSUBSCRIBER,
|
2020-04-14 10:45:47 +00:00
|
|
|
LOAD_PGP_EMAIL_HANDLER,
|
2020-05-07 11:28:04 +00:00
|
|
|
ENFORCE_SPF,
|
2020-05-09 18:45:04 +00:00
|
|
|
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
|
|
|
|
ALERT_BOUNCE_EMAIL,
|
|
|
|
ALERT_SPAM_EMAIL,
|
2020-08-15 14:38:16 +00:00
|
|
|
SPAMASSASSIN_HOST,
|
|
|
|
MAX_SPAM_SCORE,
|
2020-08-15 14:53:57 +00:00
|
|
|
MAX_REPLY_PHASE_SPAM_SCORE,
|
2020-08-25 10:51:05 +00:00
|
|
|
ALERT_SEND_EMAIL_CYCLE,
|
2020-08-30 17:06:50 +00:00
|
|
|
ALERT_MAILBOX_IS_ALIAS,
|
2020-11-02 18:09:57 +00:00
|
|
|
PGP_SENDER_PRIVATE_KEY,
|
2020-11-04 11:32:15 +00:00
|
|
|
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
|
2020-11-14 14:55:53 +00:00
|
|
|
NOREPLY,
|
2021-01-11 13:55:55 +00:00
|
|
|
BOUNCE_EMAIL,
|
|
|
|
BOUNCE_PREFIX,
|
|
|
|
BOUNCE_SUFFIX,
|
2021-01-26 08:59:08 +00:00
|
|
|
TRANSACTIONAL_BOUNCE_PREFIX,
|
|
|
|
TRANSACTIONAL_BOUNCE_SUFFIX,
|
2021-03-26 09:00:48 +00:00
|
|
|
ENABLE_SPAM_ASSASSIN,
|
2021-05-25 15:58:45 +00:00
|
|
|
BOUNCE_PREFIX_FOR_REPLY_PHASE,
|
2021-07-23 13:48:43 +00:00
|
|
|
NEWRELIC_CONFIG_PATH,
|
2021-09-06 17:44:18 +00:00
|
|
|
POSTMASTER,
|
|
|
|
ALERT_HOTMAIL_COMPLAINT,
|
2021-09-13 17:49:40 +00:00
|
|
|
ALERT_YAHOO_COMPLAINT,
|
2021-09-22 07:58:40 +00:00
|
|
|
TEMP_DIR,
|
2020-02-15 10:04:22 +00:00
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
from app.db import Session
|
2021-10-11 10:00:37 +00:00
|
|
|
from app.email import status, headers
|
2021-07-23 13:48:43 +00:00
|
|
|
from app.email.rate_limit import rate_limited
|
2021-03-17 09:07:13 +00:00
|
|
|
from app.email.spam import get_spam_score
|
2019-12-17 16:48:06 +00:00
|
|
|
from app.email_utils import (
|
|
|
|
send_email,
|
2021-09-25 16:47:15 +00:00
|
|
|
add_dkim_signature,
|
2020-01-07 18:50:36 +00:00
|
|
|
add_or_replace_header,
|
|
|
|
delete_header,
|
2020-02-11 15:46:53 +00:00
|
|
|
render,
|
2020-03-14 15:34:23 +00:00
|
|
|
get_orig_message_from_bounce,
|
2020-03-14 21:24:02 +00:00
|
|
|
delete_all_headers_except,
|
2020-03-30 20:05:31 +00:00
|
|
|
get_spam_info,
|
|
|
|
get_orig_message_from_spamassassin_report,
|
2020-05-09 18:45:04 +00:00
|
|
|
send_email_with_rate_control,
|
2020-06-09 15:16:32 +00:00
|
|
|
get_email_domain_part,
|
2020-08-21 10:03:23 +00:00
|
|
|
copy,
|
|
|
|
to_bytes,
|
2020-08-27 08:43:48 +00:00
|
|
|
send_email_at_most_times,
|
2020-10-15 14:21:31 +00:00
|
|
|
is_valid_alias_address_domain,
|
2021-09-25 16:47:15 +00:00
|
|
|
should_add_dkim_signature,
|
2020-11-07 12:00:45 +00:00
|
|
|
add_header,
|
2020-11-09 20:16:50 +00:00
|
|
|
get_header_unicode,
|
2020-11-16 18:15:09 +00:00
|
|
|
generate_reply_email,
|
2021-10-19 10:14:16 +00:00
|
|
|
is_reverse_alias,
|
2020-11-22 12:07:09 +00:00
|
|
|
normalize_reply_email,
|
2020-11-25 14:20:42 +00:00
|
|
|
is_valid_email,
|
2020-11-30 14:15:44 +00:00
|
|
|
replace,
|
2020-12-16 17:48:10 +00:00
|
|
|
should_disable,
|
2021-01-26 08:46:47 +00:00
|
|
|
parse_id_from_bounce,
|
2021-03-17 09:09:02 +00:00
|
|
|
spf_pass,
|
|
|
|
sl_sendmail,
|
2021-03-17 09:59:13 +00:00
|
|
|
sanitize_header,
|
2021-06-04 15:15:59 +00:00
|
|
|
get_queue_id,
|
2021-08-02 09:33:58 +00:00
|
|
|
should_ignore_bounce,
|
2021-09-13 17:50:15 +00:00
|
|
|
get_orig_message_from_hotmail_complaint,
|
2021-09-10 15:31:29 +00:00
|
|
|
parse_full_address,
|
2021-09-13 17:49:40 +00:00
|
|
|
get_orig_message_from_yahoo_complaint,
|
2021-10-14 13:46:52 +00:00
|
|
|
get_mailbox_bounce_info,
|
2021-11-02 13:32:16 +00:00
|
|
|
save_email_for_debugging,
|
2020-01-07 18:50:36 +00:00
|
|
|
)
|
2021-03-15 18:41:42 +00:00
|
|
|
from app.log import LOG, set_message_id
|
2020-01-30 07:43:31 +00:00
|
|
|
from app.models import (
|
2020-03-17 10:51:40 +00:00
|
|
|
Alias,
|
2020-03-17 09:56:59 +00:00
|
|
|
Contact,
|
2020-03-17 10:10:50 +00:00
|
|
|
EmailLog,
|
2020-01-30 07:43:31 +00:00
|
|
|
User,
|
2020-03-14 15:34:23 +00:00
|
|
|
RefusedEmail,
|
2020-05-07 11:28:04 +00:00
|
|
|
Mailbox,
|
2021-01-26 08:56:13 +00:00
|
|
|
Bounce,
|
|
|
|
TransactionalEmail,
|
2021-06-22 15:52:24 +00:00
|
|
|
IgnoredEmail,
|
2021-10-18 15:34:11 +00:00
|
|
|
MessageIDMatching,
|
2020-01-30 07:43:31 +00:00
|
|
|
)
|
2020-11-02 18:09:57 +00:00
|
|
|
from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data
|
2021-03-17 09:59:13 +00:00
|
|
|
from app.utils import sanitize_email
|
2020-04-14 10:45:47 +00:00
|
|
|
from init_app import load_pgp_public_keys
|
2021-10-12 13:03:16 +00:00
|
|
|
from server import create_light_app
|
2019-11-07 16:49:26 +00:00
|
|
|
|
2021-07-23 13:48:43 +00:00
|
|
|
newrelic_app = None
|
|
|
|
if NEWRELIC_CONFIG_PATH:
|
|
|
|
newrelic.agent.initialize(NEWRELIC_CONFIG_PATH)
|
|
|
|
|
|
|
|
newrelic_app = newrelic.agent.register_application()
|
|
|
|
|
2020-11-02 18:09:57 +00:00
|
|
|
|
2020-12-05 17:15:53 +00:00
|
|
|
def get_or_create_contact(from_header: str, mail_from: str, alias: Alias) -> Contact:
|
2020-02-19 15:49:40 +00:00
|
|
|
"""
|
2020-04-05 13:24:09 +00:00
|
|
|
contact_from_header is the RFC 2047 format FROM header
|
2020-02-19 15:49:40 +00:00
|
|
|
"""
|
2021-09-10 15:31:29 +00:00
|
|
|
try:
|
|
|
|
contact_name, contact_email = parse_full_address(from_header)
|
|
|
|
except ValueError:
|
|
|
|
contact_name, contact_email = "", ""
|
|
|
|
|
2020-11-25 16:50:25 +00:00
|
|
|
if not is_valid_email(contact_email):
|
2020-05-15 13:46:37 +00:00
|
|
|
# From header is wrongly formatted, try with mail_from
|
2020-11-14 14:55:53 +00:00
|
|
|
if mail_from and mail_from != "<>":
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2021-09-10 15:06:38 +00:00
|
|
|
"Cannot parse email from from_header %s, use mail_from %s",
|
2020-12-05 17:15:53 +00:00
|
|
|
from_header,
|
2020-11-25 14:20:00 +00:00
|
|
|
mail_from,
|
|
|
|
)
|
2021-09-10 15:06:38 +00:00
|
|
|
contact_email = mail_from
|
2020-05-13 20:35:27 +00:00
|
|
|
|
2020-11-25 16:50:25 +00:00
|
|
|
if not is_valid_email(contact_email):
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2020-11-25 14:20:42 +00:00
|
|
|
"invalid contact email %s. Parse from %s %s",
|
|
|
|
contact_email,
|
2020-12-05 17:15:53 +00:00
|
|
|
from_header,
|
2020-11-25 14:20:42 +00:00
|
|
|
mail_from,
|
|
|
|
)
|
|
|
|
# either reuse a contact with empty email or create a new contact with empty email
|
|
|
|
contact_email = ""
|
|
|
|
|
2021-01-11 11:27:02 +00:00
|
|
|
contact_email = sanitize_email(contact_email)
|
|
|
|
|
2021-09-27 08:21:49 +00:00
|
|
|
if contact_name and "\x00" in contact_name:
|
|
|
|
LOG.w("issue with contact name %s", contact_name)
|
|
|
|
contact_name = ""
|
|
|
|
|
2020-04-04 18:06:35 +00:00
|
|
|
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
|
2020-03-17 09:56:59 +00:00
|
|
|
if contact:
|
2020-04-05 13:24:09 +00:00
|
|
|
if contact.name != contact_name:
|
|
|
|
LOG.d(
|
2020-08-27 08:20:48 +00:00
|
|
|
"Update contact %s name %s to %s",
|
|
|
|
contact,
|
|
|
|
contact.name,
|
|
|
|
contact_name,
|
2020-04-05 13:24:09 +00:00
|
|
|
)
|
|
|
|
contact.name = contact_name
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-10-27 09:40:44 +00:00
|
|
|
|
2020-11-25 14:20:00 +00:00
|
|
|
# contact created in the past does not have mail_from and from_header field
|
|
|
|
if not contact.mail_from and mail_from:
|
2020-10-27 09:40:44 +00:00
|
|
|
LOG.d(
|
2020-11-25 14:20:00 +00:00
|
|
|
"Set contact mail_from %s: %s to %s",
|
2020-10-27 09:40:44 +00:00
|
|
|
contact,
|
|
|
|
contact.mail_from,
|
|
|
|
mail_from,
|
|
|
|
)
|
|
|
|
contact.mail_from = mail_from
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-02-19 15:17:13 +00:00
|
|
|
else:
|
2020-11-25 14:20:00 +00:00
|
|
|
LOG.d(
|
|
|
|
"create contact %s for alias %s",
|
|
|
|
contact_email,
|
2020-08-27 08:20:48 +00:00
|
|
|
alias,
|
2020-02-19 15:17:13 +00:00
|
|
|
)
|
2020-02-19 14:50:38 +00:00
|
|
|
|
2020-09-03 17:42:52 +00:00
|
|
|
try:
|
|
|
|
contact = Contact.create(
|
|
|
|
user_id=alias.user_id,
|
|
|
|
alias_id=alias.id,
|
|
|
|
website_email=contact_email,
|
|
|
|
name=contact_name,
|
2020-09-14 16:22:26 +00:00
|
|
|
mail_from=mail_from,
|
2020-12-06 18:37:20 +00:00
|
|
|
reply_email=generate_reply_email(contact_email, alias.user)
|
2020-11-25 16:50:25 +00:00
|
|
|
if is_valid_email(contact_email)
|
2020-11-18 10:48:09 +00:00
|
|
|
else NOREPLY,
|
2020-09-03 17:42:52 +00:00
|
|
|
)
|
2020-11-18 10:48:09 +00:00
|
|
|
if not contact_email:
|
2020-11-14 14:55:53 +00:00
|
|
|
LOG.d("Create a contact with invalid email for %s", alias)
|
|
|
|
contact.invalid_email = True
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-09-03 17:42:52 +00:00
|
|
|
except IntegrityError:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("Contact %s %s already exist", alias, contact_email)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.rollback()
|
2020-09-03 17:42:52 +00:00
|
|
|
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
|
2020-02-19 14:50:38 +00:00
|
|
|
|
2020-03-17 09:56:59 +00:00
|
|
|
return contact
|
2020-02-19 15:49:40 +00:00
|
|
|
|
|
|
|
|
2021-07-28 07:12:52 +00:00
|
|
|
def get_or_create_reply_to_contact(
|
|
|
|
reply_to_header: str, alias: Alias
|
|
|
|
) -> Optional[Contact]:
|
|
|
|
"""
|
|
|
|
Get or create the contact for the Reply-To header
|
|
|
|
"""
|
2021-09-10 15:31:29 +00:00
|
|
|
try:
|
|
|
|
contact_name, contact_address = parse_full_address(reply_to_header)
|
|
|
|
except ValueError:
|
|
|
|
return
|
2021-07-28 07:12:52 +00:00
|
|
|
|
2021-09-10 15:31:29 +00:00
|
|
|
if not is_valid_email(contact_address):
|
2021-07-28 07:12:52 +00:00
|
|
|
LOG.w(
|
|
|
|
"invalid reply-to address %s. Parse from %s",
|
2021-09-10 15:31:29 +00:00
|
|
|
contact_address,
|
2021-07-28 07:12:52 +00:00
|
|
|
reply_to_header,
|
|
|
|
)
|
|
|
|
return None
|
|
|
|
|
2021-09-10 15:31:29 +00:00
|
|
|
contact = Contact.get_by(alias_id=alias.id, website_email=contact_address)
|
2021-07-28 07:12:52 +00:00
|
|
|
if contact:
|
|
|
|
return contact
|
|
|
|
else:
|
|
|
|
LOG.d(
|
|
|
|
"create contact %s for alias %s via reply-to header",
|
2021-09-10 15:31:29 +00:00
|
|
|
contact_address,
|
2021-07-28 07:12:52 +00:00
|
|
|
alias,
|
|
|
|
reply_to_header,
|
|
|
|
)
|
|
|
|
|
|
|
|
try:
|
|
|
|
contact = Contact.create(
|
|
|
|
user_id=alias.user_id,
|
|
|
|
alias_id=alias.id,
|
2021-09-10 15:31:29 +00:00
|
|
|
website_email=contact_address,
|
|
|
|
name=contact_name,
|
|
|
|
reply_email=generate_reply_email(contact_address, alias.user),
|
2021-07-28 07:12:52 +00:00
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-07-28 07:12:52 +00:00
|
|
|
except IntegrityError:
|
2021-09-10 15:31:29 +00:00
|
|
|
LOG.w("Contact %s %s already exist", alias, contact_address)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.rollback()
|
2021-09-10 15:31:29 +00:00
|
|
|
contact = Contact.get_by(alias_id=alias.id, website_email=contact_address)
|
2021-07-28 07:12:52 +00:00
|
|
|
|
|
|
|
return contact
|
|
|
|
|
|
|
|
|
2020-03-28 18:16:55 +00:00
|
|
|
def replace_header_when_forward(msg: Message, alias: Alias, header: str):
|
|
|
|
"""
|
|
|
|
Replace CC or To header by Reply emails in forward phase
|
|
|
|
"""
|
|
|
|
new_addrs: [str] = []
|
2020-11-26 08:49:03 +00:00
|
|
|
headers = msg.get_all(header, [])
|
2021-01-15 10:30:43 +00:00
|
|
|
# headers can be an array of Header, convert it to string here
|
2021-09-10 14:51:36 +00:00
|
|
|
headers = [get_header_unicode(h) for h in headers]
|
2021-01-11 11:27:02 +00:00
|
|
|
|
2021-09-10 14:51:36 +00:00
|
|
|
full_addresses: [EmailAddress] = []
|
|
|
|
for h in headers:
|
|
|
|
full_addresses += address.parse_list(h)
|
|
|
|
|
|
|
|
for full_address in full_addresses:
|
|
|
|
contact_email = sanitize_email(full_address.address)
|
2020-03-28 18:16:55 +00:00
|
|
|
|
|
|
|
# no transformation when alias is already in the header
|
2020-04-05 10:59:36 +00:00
|
|
|
if contact_email == alias.email:
|
2021-09-10 14:51:36 +00:00
|
|
|
new_addrs.append(full_address.full_spec())
|
2020-03-28 18:16:55 +00:00
|
|
|
continue
|
|
|
|
|
2021-09-10 14:51:36 +00:00
|
|
|
try:
|
|
|
|
# NOT allow unicode for contact address
|
2021-09-10 15:06:38 +00:00
|
|
|
validate_email(
|
|
|
|
contact_email, check_deliverability=False, allow_smtputf8=False
|
|
|
|
)
|
2021-09-10 14:51:36 +00:00
|
|
|
except EmailNotValidError:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("invalid contact email %s. %s. Skip", contact_email, headers)
|
2020-11-25 14:21:01 +00:00
|
|
|
continue
|
|
|
|
|
2020-04-05 10:59:36 +00:00
|
|
|
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
|
2020-03-28 18:16:55 +00:00
|
|
|
if contact:
|
2020-04-05 13:39:48 +00:00
|
|
|
# update the contact name if needed
|
2021-09-10 14:51:36 +00:00
|
|
|
if contact.name != full_address.display_name:
|
2020-04-05 12:50:12 +00:00
|
|
|
LOG.d(
|
|
|
|
"Update contact %s name %s to %s",
|
|
|
|
contact,
|
|
|
|
contact.name,
|
2021-09-10 14:51:36 +00:00
|
|
|
full_address.display_name,
|
2020-04-05 12:50:12 +00:00
|
|
|
)
|
2021-09-10 14:51:36 +00:00
|
|
|
contact.name = full_address.display_name
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-03-28 18:16:55 +00:00
|
|
|
else:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.d(
|
2020-03-28 18:16:55 +00:00
|
|
|
"create contact for alias %s and email %s, header %s",
|
|
|
|
alias,
|
2020-04-05 10:59:36 +00:00
|
|
|
contact_email,
|
2020-03-28 18:16:55 +00:00
|
|
|
header,
|
|
|
|
)
|
|
|
|
|
2020-09-09 15:00:07 +00:00
|
|
|
try:
|
|
|
|
contact = Contact.create(
|
|
|
|
user_id=alias.user_id,
|
|
|
|
alias_id=alias.id,
|
|
|
|
website_email=contact_email,
|
2021-09-10 14:51:36 +00:00
|
|
|
name=full_address.display_name,
|
2020-12-06 18:37:20 +00:00
|
|
|
reply_email=generate_reply_email(contact_email, alias.user),
|
2020-09-09 15:00:07 +00:00
|
|
|
is_cc=header.lower() == "cc",
|
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-09-09 15:00:07 +00:00
|
|
|
except IntegrityError:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("Contact %s %s already exist", alias, contact_email)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.rollback()
|
2020-09-09 15:00:07 +00:00
|
|
|
contact = Contact.get_by(alias_id=alias.id, website_email=contact_email)
|
2020-03-28 18:16:55 +00:00
|
|
|
|
2020-04-05 13:24:09 +00:00
|
|
|
new_addrs.append(contact.new_addr())
|
2020-03-28 18:16:55 +00:00
|
|
|
|
2020-08-25 10:48:28 +00:00
|
|
|
if new_addrs:
|
2020-03-28 18:16:55 +00:00
|
|
|
new_header = ",".join(new_addrs)
|
|
|
|
LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
|
|
|
|
add_or_replace_header(msg, header, new_header)
|
|
|
|
else:
|
2020-08-25 10:48:28 +00:00
|
|
|
LOG.d("Delete %s header, old value %s", header, msg[header])
|
|
|
|
delete_header(msg, header)
|
2020-03-28 18:16:55 +00:00
|
|
|
|
|
|
|
|
|
|
|
def replace_header_when_reply(msg: Message, alias: Alias, header: str):
|
|
|
|
"""
|
|
|
|
Replace CC or To Reply emails by original emails
|
|
|
|
"""
|
|
|
|
new_addrs: [str] = []
|
2021-01-15 10:30:43 +00:00
|
|
|
headers = msg.get_all(header, [])
|
|
|
|
# headers can be an array of Header, convert it to string here
|
|
|
|
headers = [str(h) for h in headers]
|
2020-03-28 18:16:55 +00:00
|
|
|
|
2021-01-15 10:30:43 +00:00
|
|
|
for _, reply_email in getaddresses(headers):
|
2020-03-28 18:16:55 +00:00
|
|
|
# no transformation when alias is already in the header
|
2020-04-05 13:27:35 +00:00
|
|
|
if reply_email == alias.email:
|
2020-03-28 18:16:55 +00:00
|
|
|
continue
|
|
|
|
|
2020-04-05 13:27:35 +00:00
|
|
|
contact = Contact.get_by(reply_email=reply_email)
|
2020-03-28 18:16:55 +00:00
|
|
|
if not contact:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2020-04-05 13:27:35 +00:00
|
|
|
"%s email in reply phase %s must be reply emails", header, reply_email
|
2020-03-30 19:45:18 +00:00
|
|
|
)
|
2020-03-28 18:16:55 +00:00
|
|
|
# still keep this email in header
|
2020-11-21 18:15:02 +00:00
|
|
|
new_addrs.append(reply_email)
|
2020-04-05 13:27:35 +00:00
|
|
|
else:
|
|
|
|
new_addrs.append(formataddr((contact.name, contact.website_email)))
|
2020-03-28 18:16:55 +00:00
|
|
|
|
2020-08-25 10:48:28 +00:00
|
|
|
if new_addrs:
|
|
|
|
new_header = ",".join(new_addrs)
|
|
|
|
LOG.d("Replace %s header, old: %s, new: %s", header, msg[header], new_header)
|
|
|
|
add_or_replace_header(msg, header, new_header)
|
|
|
|
else:
|
|
|
|
LOG.d("delete the %s header. Old value %s", header, msg[header])
|
|
|
|
delete_header(msg, header)
|
2020-03-28 18:16:55 +00:00
|
|
|
|
|
|
|
|
2020-11-02 18:09:57 +00:00
|
|
|
def prepare_pgp_message(
|
|
|
|
orig_msg: Message, pgp_fingerprint: str, public_key: str, can_sign: bool = False
|
2020-11-07 12:00:45 +00:00
|
|
|
) -> Message:
|
2020-03-08 22:07:23 +00:00
|
|
|
msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
|
|
|
|
|
2020-11-01 17:06:05 +00:00
|
|
|
# clone orig message to avoid modifying it
|
|
|
|
clone_msg = copy(orig_msg)
|
|
|
|
|
2020-04-14 18:49:48 +00:00
|
|
|
# copy all headers from original message except all standard MIME headers
|
2020-11-01 17:06:05 +00:00
|
|
|
for i in reversed(range(len(clone_msg._headers))):
|
|
|
|
header_name = clone_msg._headers[i][0].lower()
|
2021-10-11 10:19:21 +00:00
|
|
|
if header_name.lower() not in headers.MIME_HEADERS:
|
2020-11-01 17:06:05 +00:00
|
|
|
msg[header_name] = clone_msg._headers[i][1]
|
2020-03-08 22:07:23 +00:00
|
|
|
|
2020-11-02 18:09:57 +00:00
|
|
|
# Delete unnecessary headers in clone_msg except _MIME_HEADERS to save space
|
2020-03-14 21:24:02 +00:00
|
|
|
delete_all_headers_except(
|
2020-11-01 17:06:05 +00:00
|
|
|
clone_msg,
|
2021-10-11 10:19:21 +00:00
|
|
|
headers.MIME_HEADERS,
|
2020-03-14 21:24:02 +00:00
|
|
|
)
|
|
|
|
|
2021-10-11 10:10:18 +00:00
|
|
|
if clone_msg[headers.CONTENT_TYPE] is None:
|
2020-11-01 17:06:28 +00:00
|
|
|
LOG.d("Content-Type missing")
|
2021-10-11 10:10:18 +00:00
|
|
|
clone_msg[headers.CONTENT_TYPE] = "text/plain"
|
2020-11-01 17:06:28 +00:00
|
|
|
|
2021-10-11 10:10:18 +00:00
|
|
|
if clone_msg[headers.MIME_VERSION] is None:
|
2020-11-01 17:06:28 +00:00
|
|
|
LOG.d("Mime-Version missing")
|
2021-10-11 10:10:18 +00:00
|
|
|
clone_msg[headers.MIME_VERSION] = "1.0"
|
2020-11-01 17:06:28 +00:00
|
|
|
|
2020-03-08 22:07:23 +00:00
|
|
|
first = MIMEApplication(
|
|
|
|
_subtype="pgp-encrypted", _encoder=encoders.encode_7or8bit, _data=""
|
|
|
|
)
|
|
|
|
first.set_payload("Version: 1")
|
|
|
|
msg.attach(first)
|
|
|
|
|
2020-11-02 18:09:57 +00:00
|
|
|
if can_sign and PGP_SENDER_PRIVATE_KEY:
|
|
|
|
LOG.d("Sign msg")
|
|
|
|
clone_msg = sign_msg(clone_msg)
|
|
|
|
|
|
|
|
# use pgpy as fallback
|
2020-09-08 09:10:22 +00:00
|
|
|
second = MIMEApplication(
|
|
|
|
"octet-stream", _encoder=encoders.encode_7or8bit, name="encrypted.asc"
|
|
|
|
)
|
|
|
|
second.add_header("Content-Disposition", 'inline; filename="encrypted.asc"')
|
2020-10-28 10:50:14 +00:00
|
|
|
|
2020-11-02 18:09:57 +00:00
|
|
|
# encrypt
|
2020-10-28 16:07:53 +00:00
|
|
|
# use pgpy as fallback
|
2020-11-09 16:02:10 +00:00
|
|
|
msg_bytes = to_bytes(clone_msg)
|
2020-10-28 16:07:53 +00:00
|
|
|
try:
|
2020-11-01 17:06:05 +00:00
|
|
|
encrypted_data = pgp_utils.encrypt_file(BytesIO(msg_bytes), pgp_fingerprint)
|
2020-10-28 16:07:53 +00:00
|
|
|
second.set_payload(encrypted_data)
|
|
|
|
except PGPException:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("Cannot encrypt using python-gnupg, use pgpy")
|
2020-11-01 17:06:05 +00:00
|
|
|
encrypted = pgp_utils.encrypt_file_with_pgpy(msg_bytes, public_key)
|
2020-10-28 16:07:53 +00:00
|
|
|
second.set_payload(str(encrypted))
|
2020-10-28 10:50:14 +00:00
|
|
|
|
2020-03-08 22:07:23 +00:00
|
|
|
msg.attach(second)
|
|
|
|
|
|
|
|
return msg
|
|
|
|
|
|
|
|
|
2020-11-02 18:09:57 +00:00
|
|
|
def sign_msg(msg: Message) -> Message:
|
|
|
|
container = MIMEMultipart(
|
|
|
|
"signed", protocol="application/pgp-signature", micalg="pgp-sha256"
|
|
|
|
)
|
|
|
|
container.attach(msg)
|
|
|
|
|
|
|
|
signature = MIMEApplication(
|
|
|
|
_subtype="pgp-signature", name="signature.asc", _data="", _encoder=encode_noop
|
|
|
|
)
|
|
|
|
signature.add_header("Content-Disposition", 'attachment; filename="signature.asc"')
|
|
|
|
|
|
|
|
try:
|
2020-11-09 16:02:10 +00:00
|
|
|
signature.set_payload(sign_data(to_bytes(msg).replace(b"\n", b"\r\n")))
|
2020-11-02 18:09:57 +00:00
|
|
|
except Exception:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Cannot sign, try using pgpy")
|
2020-11-02 18:09:57 +00:00
|
|
|
signature.set_payload(
|
2020-11-09 16:02:10 +00:00
|
|
|
sign_data_with_pgpy(to_bytes(msg).replace(b"\n", b"\r\n"))
|
2020-11-02 18:09:57 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
container.attach(signature)
|
|
|
|
|
|
|
|
return container
|
|
|
|
|
|
|
|
|
2020-08-25 10:51:05 +00:00
|
|
|
def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
|
|
|
|
# store the refused email
|
|
|
|
random_name = str(uuid.uuid4())
|
|
|
|
full_report_path = f"refused-emails/cycle-{random_name}.eml"
|
2020-11-09 16:02:10 +00:00
|
|
|
s3.upload_email_from_bytesio(full_report_path, BytesIO(to_bytes(msg)), random_name)
|
2020-08-25 10:51:05 +00:00
|
|
|
refused_email = RefusedEmail.create(
|
|
|
|
path=None, full_report_path=full_report_path, user_id=alias.user_id
|
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-08-25 10:51:05 +00:00
|
|
|
LOG.d("Create refused email %s", refused_email)
|
|
|
|
# link available for 6 days as it gets deleted in 7 days
|
|
|
|
refused_email_url = refused_email.get_url(expires_in=518400)
|
|
|
|
|
2020-08-27 08:43:48 +00:00
|
|
|
send_email_at_most_times(
|
2020-08-25 10:51:05 +00:00
|
|
|
user,
|
|
|
|
ALERT_SEND_EMAIL_CYCLE,
|
|
|
|
mailbox.email,
|
2020-08-27 09:10:16 +00:00
|
|
|
f"Email sent to {alias.email} from its own mailbox {mailbox.email}",
|
2020-08-25 10:51:05 +00:00
|
|
|
render(
|
2021-11-02 10:46:41 +00:00
|
|
|
"transactional/cycle-email.txt.jinja2",
|
2020-08-25 10:51:05 +00:00
|
|
|
alias=alias,
|
|
|
|
mailbox=mailbox,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/cycle-email.html",
|
|
|
|
alias=alias,
|
|
|
|
mailbox=mailbox,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2020-09-30 09:05:21 +00:00
|
|
|
def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]:
|
2020-08-30 17:22:21 +00:00
|
|
|
"""return an array of SMTP status (is_success, smtp_status)
|
|
|
|
is_success indicates whether an email has been delivered and
|
|
|
|
smtp_status is the SMTP Status ("250 Message accepted", "550 Non-existent email address", etc)
|
2020-03-28 20:24:43 +00:00
|
|
|
"""
|
2021-09-10 15:48:36 +00:00
|
|
|
alias_address = rcpt_to # alias@SL
|
2020-02-19 15:49:40 +00:00
|
|
|
|
2021-09-10 15:48:36 +00:00
|
|
|
alias = Alias.get_by(email=alias_address)
|
2020-03-17 10:51:40 +00:00
|
|
|
if not alias:
|
2021-09-10 15:48:36 +00:00
|
|
|
LOG.d(
|
|
|
|
"alias %s not exist. Try to see if it can be created on the fly",
|
|
|
|
alias_address,
|
|
|
|
)
|
|
|
|
alias = try_auto_create(alias_address)
|
2020-03-17 10:51:40 +00:00
|
|
|
if not alias:
|
2021-09-10 15:48:36 +00:00
|
|
|
LOG.d("alias %s cannot be created on-the-fly, return 550", alias_address)
|
2021-08-02 09:33:58 +00:00
|
|
|
if should_ignore_bounce(envelope.mail_from):
|
|
|
|
return [(True, status.E207)]
|
|
|
|
else:
|
|
|
|
return [(False, status.E515)]
|
2020-02-19 15:49:40 +00:00
|
|
|
|
2020-11-04 11:32:15 +00:00
|
|
|
user = alias.user
|
|
|
|
|
|
|
|
if user.disabled:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("User %s disabled, disable forwarding emails for %s", user, alias)
|
2021-08-02 09:33:58 +00:00
|
|
|
if should_ignore_bounce(envelope.mail_from):
|
|
|
|
return [(True, status.E207)]
|
|
|
|
else:
|
|
|
|
return [(False, status.E504)]
|
2020-10-04 10:49:27 +00:00
|
|
|
|
2021-09-10 16:15:22 +00:00
|
|
|
if user.ignore_loop_email:
|
|
|
|
mail_from = envelope.mail_from
|
|
|
|
for mb in alias.mailboxes:
|
2021-09-13 14:04:32 +00:00
|
|
|
# email sent from a mailbox to its alias
|
2021-09-10 16:15:22 +00:00
|
|
|
if mb.email == mail_from:
|
|
|
|
LOG.w("cycle email sent from %s to %s", mb, alias)
|
|
|
|
handle_email_sent_to_ourself(alias, mb, msg, user)
|
|
|
|
return [(True, status.E209)]
|
2020-08-25 10:51:05 +00:00
|
|
|
|
2021-10-11 10:10:18 +00:00
|
|
|
from_header = get_header_unicode(msg[headers.FROM])
|
2021-07-28 07:12:52 +00:00
|
|
|
LOG.d("Create or get contact for from_header:%s", from_header)
|
2020-12-05 17:15:53 +00:00
|
|
|
contact = get_or_create_contact(from_header, envelope.mail_from, alias)
|
2020-11-04 11:32:15 +00:00
|
|
|
|
2021-07-28 07:12:52 +00:00
|
|
|
reply_to_contact = None
|
2021-10-11 10:10:18 +00:00
|
|
|
if msg[headers.REPLY_TO]:
|
|
|
|
reply_to = get_header_unicode(msg[headers.REPLY_TO])
|
2021-07-28 07:12:52 +00:00
|
|
|
LOG.d("Create or get contact for from_header:%s", reply_to)
|
|
|
|
# ignore when reply-to = alias
|
|
|
|
if reply_to == alias.email:
|
|
|
|
LOG.i("Reply-to same as alias %s", alias)
|
|
|
|
else:
|
|
|
|
reply_to_contact = get_or_create_reply_to_contact(reply_to, alias)
|
|
|
|
|
2021-10-28 08:19:58 +00:00
|
|
|
if not alias.enabled or contact.block_forward:
|
2020-04-27 16:18:40 +00:00
|
|
|
LOG.d("%s is disabled, do not forward", alias)
|
2020-11-24 15:38:34 +00:00
|
|
|
EmailLog.create(
|
2021-07-11 10:28:23 +00:00
|
|
|
contact_id=contact.id,
|
|
|
|
user_id=contact.user_id,
|
|
|
|
blocked=True,
|
|
|
|
alias_id=contact.alias_id,
|
|
|
|
commit=True,
|
2020-11-24 15:38:34 +00:00
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-10-28 08:19:58 +00:00
|
|
|
# do not return 5** to allow user to receive emails later when alias is enabled or contact is unblocked
|
2021-06-23 17:47:06 +00:00
|
|
|
return [(True, status.E200)]
|
2020-03-30 20:05:31 +00:00
|
|
|
|
2020-05-10 14:57:47 +00:00
|
|
|
ret = []
|
2020-08-21 08:41:50 +00:00
|
|
|
mailboxes = alias.mailboxes
|
2020-08-30 17:22:21 +00:00
|
|
|
|
|
|
|
# no valid mailbox
|
|
|
|
if not mailboxes:
|
2021-08-02 09:33:58 +00:00
|
|
|
if should_ignore_bounce(envelope.mail_from):
|
|
|
|
return [(True, status.E207)]
|
|
|
|
else:
|
|
|
|
return [(False, status.E516)]
|
2020-08-30 17:22:21 +00:00
|
|
|
|
2020-08-21 08:41:50 +00:00
|
|
|
# no need to create a copy of message
|
2020-11-04 13:55:54 +00:00
|
|
|
for mailbox in mailboxes:
|
2020-09-10 07:38:30 +00:00
|
|
|
if not mailbox.verified:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.d("%s unverified, do not forward", mailbox)
|
2021-06-23 17:47:06 +00:00
|
|
|
ret.append((False, status.E517))
|
2020-09-10 07:38:30 +00:00
|
|
|
else:
|
2020-11-04 13:55:54 +00:00
|
|
|
# create a copy of message for each forward
|
2020-09-10 07:38:30 +00:00
|
|
|
ret.append(
|
2020-09-30 09:05:21 +00:00
|
|
|
forward_email_to_mailbox(
|
2021-07-28 07:12:52 +00:00
|
|
|
alias, copy(msg), contact, envelope, mailbox, user, reply_to_contact
|
2020-09-10 07:38:30 +00:00
|
|
|
)
|
2020-05-10 14:57:47 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
2020-09-30 09:05:21 +00:00
|
|
|
def forward_email_to_mailbox(
|
2020-05-10 14:57:47 +00:00
|
|
|
alias,
|
|
|
|
msg: Message,
|
|
|
|
contact: Contact,
|
|
|
|
envelope,
|
|
|
|
mailbox,
|
|
|
|
user,
|
2021-07-28 07:12:52 +00:00
|
|
|
reply_to_contact: Optional[Contact],
|
2020-05-10 14:57:47 +00:00
|
|
|
) -> (bool, str):
|
|
|
|
LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
|
2020-06-09 15:16:32 +00:00
|
|
|
|
2020-10-12 11:28:21 +00:00
|
|
|
if mailbox.disabled:
|
2021-08-02 09:33:58 +00:00
|
|
|
LOG.d("%s disabled, do not forward")
|
|
|
|
if should_ignore_bounce(envelope.mail_from):
|
|
|
|
return True, status.E207
|
|
|
|
else:
|
|
|
|
return False, status.E518
|
2020-10-12 11:28:21 +00:00
|
|
|
|
2020-06-09 15:16:32 +00:00
|
|
|
# sanity check: make sure mailbox is not actually an alias
|
|
|
|
if get_email_domain_part(alias.email) == get_email_domain_part(mailbox.email):
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2020-06-09 15:16:32 +00:00
|
|
|
"Mailbox has the same domain as alias. %s -> %s -> %s",
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
mailbox,
|
|
|
|
)
|
2020-08-30 17:06:50 +00:00
|
|
|
mailbox_url = f"{URL}/dashboard/mailbox/{mailbox.id}/"
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_MAILBOX_IS_ALIAS,
|
|
|
|
user.email,
|
2020-12-21 08:39:26 +00:00
|
|
|
f"Your mailbox {mailbox.email} and alias {alias.email} use the same domain",
|
2020-08-30 17:06:50 +00:00
|
|
|
render(
|
2021-11-02 10:37:46 +00:00
|
|
|
"transactional/mailbox-invalid.txt.jinja2",
|
2020-08-30 17:06:50 +00:00
|
|
|
mailbox=mailbox,
|
|
|
|
mailbox_url=mailbox_url,
|
2020-12-21 10:57:12 +00:00
|
|
|
alias=alias,
|
2020-08-30 17:06:50 +00:00
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/mailbox-invalid.html",
|
|
|
|
mailbox=mailbox,
|
|
|
|
mailbox_url=mailbox_url,
|
2020-12-21 10:57:12 +00:00
|
|
|
alias=alias,
|
2020-08-30 17:06:50 +00:00
|
|
|
),
|
2020-08-31 15:32:46 +00:00
|
|
|
max_nb_alert=1,
|
2020-08-30 17:06:50 +00:00
|
|
|
)
|
|
|
|
|
2020-08-30 17:08:53 +00:00
|
|
|
# retry later
|
|
|
|
# so when user fixes the mailbox, the email can be delivered
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E405
|
2020-06-09 15:16:32 +00:00
|
|
|
|
2020-11-24 15:38:34 +00:00
|
|
|
email_log = EmailLog.create(
|
2021-07-11 10:28:23 +00:00
|
|
|
contact_id=contact.id,
|
|
|
|
user_id=user.id,
|
|
|
|
mailbox_id=mailbox.id,
|
|
|
|
alias_id=contact.alias_id,
|
2021-11-10 08:38:20 +00:00
|
|
|
message_id=str(msg[headers.MESSAGE_ID]),
|
2021-07-11 10:28:23 +00:00
|
|
|
commit=True,
|
2020-11-24 15:38:34 +00:00
|
|
|
)
|
2021-05-24 10:08:30 +00:00
|
|
|
LOG.d("Create %s for %s, %s, %s", email_log, contact, user, mailbox)
|
2020-11-24 15:38:34 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
if ENABLE_SPAM_ASSASSIN:
|
|
|
|
# Spam check
|
|
|
|
spam_status = ""
|
|
|
|
is_spam = False
|
2020-08-15 14:38:16 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
if SPAMASSASSIN_HOST:
|
|
|
|
start = time.time()
|
|
|
|
spam_score, spam_report = get_spam_score(msg, email_log)
|
|
|
|
LOG.d(
|
|
|
|
"%s -> %s - spam score:%s in %s seconds. Spam report %s",
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
spam_score,
|
|
|
|
time.time() - start,
|
|
|
|
spam_report,
|
|
|
|
)
|
|
|
|
email_log.spam_score = spam_score
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-08-20 09:58:46 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
if (user.max_spam_score and spam_score > user.max_spam_score) or (
|
|
|
|
not user.max_spam_score and spam_score > MAX_SPAM_SCORE
|
|
|
|
):
|
|
|
|
is_spam = True
|
|
|
|
# only set the spam report for spam
|
|
|
|
email_log.spam_report = spam_report
|
|
|
|
else:
|
|
|
|
is_spam, spam_status = get_spam_info(msg, max_score=user.max_spam_score)
|
2020-08-15 14:38:16 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
if is_spam:
|
|
|
|
LOG.w(
|
|
|
|
"Email detected as spam. %s -> %s. Spam Score: %s, Spam Report: %s",
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
email_log.spam_score,
|
|
|
|
email_log.spam_report,
|
|
|
|
)
|
|
|
|
email_log.is_spam = True
|
|
|
|
email_log.spam_status = spam_status
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-05-15 14:34:07 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
handle_spam(contact, alias, msg, user, mailbox, email_log)
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E519
|
2020-03-30 20:05:31 +00:00
|
|
|
|
2020-11-14 14:55:53 +00:00
|
|
|
if contact.invalid_email:
|
|
|
|
LOG.d("add noreply information %s %s", alias, mailbox)
|
|
|
|
msg = add_header(
|
|
|
|
msg,
|
|
|
|
f"""Email sent to {alias.email} from an invalid address and cannot be replied""",
|
|
|
|
f"""Email sent to {alias.email} from an invalid address and cannot be replied""",
|
|
|
|
)
|
|
|
|
|
2021-01-28 15:34:24 +00:00
|
|
|
delete_all_headers_except(
|
|
|
|
msg,
|
|
|
|
[
|
2021-10-11 10:19:21 +00:00
|
|
|
headers.FROM,
|
|
|
|
headers.TO,
|
|
|
|
headers.CC,
|
|
|
|
headers.SUBJECT,
|
2021-10-11 10:00:37 +00:00
|
|
|
# do not delete original message id
|
|
|
|
headers.MESSAGE_ID,
|
2021-01-28 15:34:24 +00:00
|
|
|
# References and In-Reply-To are used for keeping the email thread
|
2021-10-11 10:00:37 +00:00
|
|
|
headers.REFERENCES,
|
|
|
|
headers.IN_REPLY_TO,
|
2021-01-28 15:34:24 +00:00
|
|
|
]
|
2021-10-11 10:19:21 +00:00
|
|
|
+ headers.MIME_HEADERS,
|
2021-01-28 15:34:24 +00:00
|
|
|
)
|
|
|
|
|
2020-03-08 22:07:23 +00:00
|
|
|
# create PGP email if needed
|
2020-11-24 10:28:14 +00:00
|
|
|
if mailbox.pgp_enabled() and user.is_premium() and not alias.disable_pgp:
|
2020-03-08 22:07:23 +00:00
|
|
|
LOG.d("Encrypt message using mailbox %s", mailbox)
|
2020-11-07 12:00:45 +00:00
|
|
|
if mailbox.generic_subject:
|
|
|
|
LOG.d("Use a generic subject for %s", mailbox)
|
2021-10-11 10:10:18 +00:00
|
|
|
orig_subject = msg[headers.SUBJECT]
|
2020-11-09 20:16:50 +00:00
|
|
|
orig_subject = get_header_unicode(orig_subject)
|
2020-11-07 12:00:45 +00:00
|
|
|
add_or_replace_header(msg, "Subject", mailbox.generic_subject)
|
|
|
|
msg = add_header(
|
|
|
|
msg,
|
2020-11-07 16:23:28 +00:00
|
|
|
f"""Forwarded by SimpleLogin to {alias.email} with "{orig_subject}" as subject""",
|
|
|
|
f"""Forwarded by SimpleLogin to {alias.email} with <b>{orig_subject}</b> as subject""",
|
2020-11-07 12:00:45 +00:00
|
|
|
)
|
|
|
|
|
2020-06-08 11:54:42 +00:00
|
|
|
try:
|
2020-10-28 10:50:14 +00:00
|
|
|
msg = prepare_pgp_message(
|
2020-11-02 18:09:57 +00:00
|
|
|
msg, mailbox.pgp_finger_print, mailbox.pgp_public_key, can_sign=True
|
2020-10-28 10:50:14 +00:00
|
|
|
)
|
2020-06-08 11:54:42 +00:00
|
|
|
except PGPException:
|
2021-06-23 17:47:06 +00:00
|
|
|
LOG.e(
|
2020-06-08 11:54:42 +00:00
|
|
|
"Cannot encrypt message %s -> %s. %s %s", contact, alias, mailbox, user
|
|
|
|
)
|
|
|
|
# so the client can retry later
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E406
|
2020-03-08 22:07:23 +00:00
|
|
|
|
2020-04-27 16:18:40 +00:00
|
|
|
# add custom header
|
2021-10-11 10:19:21 +00:00
|
|
|
add_or_replace_header(msg, headers.SL_DIRECTION, "Forward")
|
2020-04-27 16:18:40 +00:00
|
|
|
|
2021-10-11 10:19:21 +00:00
|
|
|
msg[headers.SL_EMAIL_LOG_ID] = str(email_log.id)
|
|
|
|
msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from
|
2021-05-12 08:46:07 +00:00
|
|
|
# when an alias isn't in the To: header, there's no way for users to know what alias has received the email
|
2021-10-11 10:19:21 +00:00
|
|
|
msg[headers.SL_ENVELOPE_TO] = alias.email
|
2021-10-11 10:00:37 +00:00
|
|
|
|
|
|
|
if not msg[headers.DATE]:
|
|
|
|
LOG.w("missing date header, create one")
|
|
|
|
msg[headers.DATE] = formatdate()
|
2020-11-02 13:53:22 +00:00
|
|
|
|
2021-11-01 17:45:10 +00:00
|
|
|
replace_sl_message_id_by_original_message_id(msg)
|
2021-10-18 15:35:16 +00:00
|
|
|
|
2020-11-01 17:13:50 +00:00
|
|
|
# change the from header so the sender comes from a reverse-alias
|
2020-04-27 16:18:40 +00:00
|
|
|
# so it can pass DMARC check
|
|
|
|
# replace the email part in from: header
|
2021-10-11 10:10:18 +00:00
|
|
|
contact_from_header = msg[headers.FROM]
|
2020-04-27 16:18:40 +00:00
|
|
|
new_from_header = contact.new_addr()
|
|
|
|
add_or_replace_header(msg, "From", new_from_header)
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.d("From header, new:%s, old:%s", new_from_header, contact_from_header)
|
2020-04-27 16:18:40 +00:00
|
|
|
|
2021-07-28 07:12:52 +00:00
|
|
|
if reply_to_contact:
|
2021-10-11 10:10:18 +00:00
|
|
|
reply_to_header = msg[headers.REPLY_TO]
|
2021-07-28 07:12:52 +00:00
|
|
|
new_reply_to_header = reply_to_contact.new_addr()
|
|
|
|
add_or_replace_header(msg, "Reply-To", new_reply_to_header)
|
|
|
|
LOG.d("Reply-To header, new:%s, old:%s", new_reply_to_header, reply_to_header)
|
|
|
|
|
2020-11-01 17:13:50 +00:00
|
|
|
# replace CC & To emails by reverse-alias for all emails that are not alias
|
2020-04-27 16:18:40 +00:00
|
|
|
replace_header_when_forward(msg, alias, "Cc")
|
|
|
|
replace_header_when_forward(msg, alias, "To")
|
|
|
|
|
|
|
|
# add List-Unsubscribe header
|
2021-11-02 14:44:43 +00:00
|
|
|
if user.one_click_unsubscribe_block_sender:
|
|
|
|
unsubscribe_link, via_email = alias.unsubscribe_link(contact)
|
|
|
|
else:
|
|
|
|
unsubscribe_link, via_email = alias.unsubscribe_link()
|
|
|
|
|
2021-11-02 13:36:37 +00:00
|
|
|
add_or_replace_header(msg, headers.LIST_UNSUBSCRIBE, f"<{unsubscribe_link}>")
|
2020-10-22 08:37:02 +00:00
|
|
|
if not via_email:
|
2020-04-27 16:18:40 +00:00
|
|
|
add_or_replace_header(
|
2021-11-02 13:36:37 +00:00
|
|
|
msg, headers.LIST_UNSUBSCRIBE_POST, "List-Unsubscribe=One-Click"
|
2019-11-19 09:23:06 +00:00
|
|
|
)
|
2019-12-15 09:18:33 +00:00
|
|
|
|
2021-09-25 16:47:15 +00:00
|
|
|
add_dkim_signature(msg, EMAIL_DOMAIN)
|
|
|
|
|
2020-04-27 16:18:40 +00:00
|
|
|
LOG.d(
|
2021-03-16 08:17:23 +00:00
|
|
|
"Forward mail from %s to %s, mail_options:%s, rcpt_options:%s ",
|
2020-04-27 16:18:40 +00:00
|
|
|
contact.website_email,
|
2020-05-10 14:32:54 +00:00
|
|
|
mailbox.email,
|
2020-04-27 16:18:40 +00:00
|
|
|
envelope.mail_options,
|
|
|
|
envelope.rcpt_options,
|
|
|
|
)
|
|
|
|
|
2020-09-02 08:16:13 +00:00
|
|
|
try:
|
2020-09-29 10:57:14 +00:00
|
|
|
sl_sendmail(
|
2021-01-11 13:55:55 +00:00
|
|
|
# use a different envelope sender for each forward (aka VERP)
|
|
|
|
BOUNCE_EMAIL.format(email_log.id),
|
2020-09-02 08:16:13 +00:00
|
|
|
mailbox.email,
|
2020-09-29 10:57:14 +00:00
|
|
|
msg,
|
2020-09-02 08:16:13 +00:00
|
|
|
envelope.mail_options,
|
|
|
|
envelope.rcpt_options,
|
2021-01-28 12:50:24 +00:00
|
|
|
is_forward=True,
|
2020-09-02 08:16:13 +00:00
|
|
|
)
|
|
|
|
except SMTPRecipientsRefused:
|
|
|
|
# that means the mailbox is maybe invalid
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2020-09-02 08:16:13 +00:00
|
|
|
"SMTPRecipientsRefused forward phase %s -> %s -> %s",
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
mailbox,
|
|
|
|
)
|
2021-08-02 09:33:58 +00:00
|
|
|
if should_ignore_bounce(envelope.mail_from):
|
|
|
|
return True, status.E207
|
|
|
|
else:
|
|
|
|
return False, status.E521
|
2020-09-02 08:16:13 +00:00
|
|
|
else:
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-06-23 17:47:06 +00:00
|
|
|
return True, status.E200
|
2019-11-19 09:23:06 +00:00
|
|
|
|
|
|
|
|
2021-11-01 17:45:10 +00:00
|
|
|
def replace_sl_message_id_by_original_message_id(msg):
|
|
|
|
# Replace SL Message-ID by original one in In-Reply-To header
|
|
|
|
if msg[headers.IN_REPLY_TO]:
|
|
|
|
matching: MessageIDMatching = MessageIDMatching.get_by(
|
2021-11-08 10:21:01 +00:00
|
|
|
sl_message_id=str(msg[headers.IN_REPLY_TO])
|
2021-11-01 17:45:10 +00:00
|
|
|
)
|
|
|
|
if matching:
|
|
|
|
LOG.d(
|
|
|
|
"replace SL message id by original one in in-reply-to header, %s -> %s",
|
|
|
|
msg[headers.IN_REPLY_TO],
|
|
|
|
matching.original_message_id,
|
|
|
|
)
|
|
|
|
del msg[headers.IN_REPLY_TO]
|
|
|
|
msg[headers.IN_REPLY_TO] = matching.original_message_id
|
|
|
|
|
|
|
|
# Replace SL Message-ID by original Message-ID in References header
|
|
|
|
if msg[headers.REFERENCES]:
|
|
|
|
message_ids = msg[headers.REFERENCES].split()
|
|
|
|
new_message_ids = []
|
|
|
|
for message_id in message_ids:
|
|
|
|
matching = MessageIDMatching.get_by(sl_message_id=message_id)
|
|
|
|
if matching:
|
|
|
|
LOG.d(
|
|
|
|
"replace SL message id by original one in references header, %s -> %s",
|
|
|
|
message_id,
|
|
|
|
matching.original_message_id,
|
|
|
|
)
|
|
|
|
new_message_ids.append(matching.original_message_id)
|
|
|
|
else:
|
|
|
|
new_message_ids.append(message_id)
|
|
|
|
|
|
|
|
del msg[headers.REFERENCES]
|
|
|
|
msg[headers.REFERENCES] = " ".join(new_message_ids)
|
|
|
|
|
|
|
|
|
2020-09-30 09:05:21 +00:00
|
|
|
def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
|
2020-03-28 20:24:43 +00:00
|
|
|
"""
|
2021-01-28 15:34:24 +00:00
|
|
|
Return whether an email has been delivered and
|
2020-03-28 20:24:43 +00:00
|
|
|
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
|
|
|
|
"""
|
2020-09-14 15:38:48 +00:00
|
|
|
reply_email = rcpt_to
|
2019-11-19 09:23:06 +00:00
|
|
|
|
2020-02-19 15:17:13 +00:00
|
|
|
# reply_email must end with EMAIL_DOMAIN
|
|
|
|
if not reply_email.endswith(EMAIL_DOMAIN):
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(f"Reply email {reply_email} has wrong domain")
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E501
|
2019-11-19 09:23:06 +00:00
|
|
|
|
2020-11-22 12:07:09 +00:00
|
|
|
# handle case where reply email is generated with non-allowed char
|
|
|
|
reply_email = normalize_reply_email(reply_email)
|
2020-11-18 15:11:00 +00:00
|
|
|
|
2020-03-17 09:56:59 +00:00
|
|
|
contact = Contact.get_by(reply_email=reply_email)
|
|
|
|
if not contact:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(f"No such forward-email with {reply_email} as reply-email")
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E502
|
2019-12-18 16:07:20 +00:00
|
|
|
|
2020-03-28 18:16:55 +00:00
|
|
|
alias = contact.alias
|
2020-03-17 10:51:40 +00:00
|
|
|
address: str = contact.alias.email
|
|
|
|
alias_domain = address[address.find("@") + 1 :]
|
2019-11-19 09:23:06 +00:00
|
|
|
|
2020-10-15 14:21:31 +00:00
|
|
|
# Sanity check: verify alias domain is managed by SimpleLogin
|
|
|
|
# scenario: a user have removed a domain but due to a bug, the aliases are still there
|
|
|
|
if not is_valid_alias_address_domain(alias.email):
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("%s domain isn't known", alias)
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E503
|
2019-11-19 09:23:06 +00:00
|
|
|
|
2020-03-17 10:51:40 +00:00
|
|
|
user = alias.user
|
2020-09-14 15:38:48 +00:00
|
|
|
mail_from = envelope.mail_from
|
2020-02-19 15:17:13 +00:00
|
|
|
|
2020-10-04 10:49:27 +00:00
|
|
|
if user.disabled:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e(
|
2020-10-04 10:49:27 +00:00
|
|
|
"User %s disabled, disable sending emails from %s to %s",
|
|
|
|
user,
|
|
|
|
alias,
|
|
|
|
contact,
|
|
|
|
)
|
2021-06-23 17:47:06 +00:00
|
|
|
return [(False, status.E504)]
|
2020-10-04 10:49:27 +00:00
|
|
|
|
2020-09-28 15:43:09 +00:00
|
|
|
# Anti-spoofing
|
|
|
|
mailbox = get_mailbox_from_mail_from(mail_from, alias)
|
|
|
|
if not mailbox:
|
2020-08-26 12:39:51 +00:00
|
|
|
if alias.disable_email_spoofing_check:
|
|
|
|
# ignore this error, use default alias mailbox
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2020-08-26 12:39:51 +00:00
|
|
|
"ignore unknown sender to reverse-alias %s: %s -> %s",
|
|
|
|
mail_from,
|
|
|
|
alias,
|
|
|
|
contact,
|
|
|
|
)
|
|
|
|
mailbox = alias.mailbox
|
|
|
|
else:
|
|
|
|
# only mailbox can send email to the reply-email
|
2020-09-16 15:24:42 +00:00
|
|
|
handle_unknown_mailbox(envelope, msg, reply_email, user, alias, contact)
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E505
|
2020-05-10 16:19:29 +00:00
|
|
|
|
2020-08-26 12:39:51 +00:00
|
|
|
if ENFORCE_SPF and mailbox.force_spf and not alias.disable_email_spoofing_check:
|
2021-03-17 09:16:09 +00:00
|
|
|
if not spf_pass(envelope, mailbox, user, alias, contact.website_email, msg):
|
2021-06-23 17:47:06 +00:00
|
|
|
# cannot use 4** here as sender will retry.
|
|
|
|
# cannot use 5** because that generates bounce report
|
|
|
|
return True, status.E201
|
2020-05-07 11:28:04 +00:00
|
|
|
|
2020-08-16 12:28:47 +00:00
|
|
|
email_log = EmailLog.create(
|
2020-11-24 15:38:34 +00:00
|
|
|
contact_id=contact.id,
|
2021-07-11 10:28:23 +00:00
|
|
|
alias_id=contact.alias_id,
|
2020-11-24 15:38:34 +00:00
|
|
|
is_reply=True,
|
|
|
|
user_id=contact.user_id,
|
|
|
|
mailbox_id=mailbox.id,
|
2021-10-18 15:35:16 +00:00
|
|
|
message_id=msg[headers.MESSAGE_ID],
|
2020-12-11 10:13:19 +00:00
|
|
|
commit=True,
|
2020-08-16 12:28:47 +00:00
|
|
|
)
|
2021-05-24 10:08:30 +00:00
|
|
|
LOG.d("Create %s for %s, %s, %s", email_log, contact, user, mailbox)
|
2020-08-16 12:28:47 +00:00
|
|
|
|
2020-08-15 14:53:57 +00:00
|
|
|
# Spam check
|
2021-03-26 09:00:48 +00:00
|
|
|
if ENABLE_SPAM_ASSASSIN:
|
|
|
|
spam_status = ""
|
|
|
|
is_spam = False
|
|
|
|
|
|
|
|
# do not use user.max_spam_score here
|
|
|
|
if SPAMASSASSIN_HOST:
|
|
|
|
start = time.time()
|
|
|
|
spam_score, spam_report = get_spam_score(msg, email_log)
|
|
|
|
LOG.d(
|
|
|
|
"%s -> %s - spam score %s in %s seconds. Spam report %s",
|
|
|
|
alias,
|
|
|
|
contact,
|
|
|
|
spam_score,
|
|
|
|
time.time() - start,
|
|
|
|
spam_report,
|
|
|
|
)
|
|
|
|
email_log.spam_score = spam_score
|
|
|
|
if spam_score > MAX_REPLY_PHASE_SPAM_SCORE:
|
|
|
|
is_spam = True
|
|
|
|
# only set the spam report for spam
|
|
|
|
email_log.spam_report = spam_report
|
|
|
|
else:
|
|
|
|
is_spam, spam_status = get_spam_info(
|
|
|
|
msg, max_score=MAX_REPLY_PHASE_SPAM_SCORE
|
|
|
|
)
|
2020-08-15 14:53:57 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
if is_spam:
|
|
|
|
LOG.w(
|
|
|
|
"Email detected as spam. Reply phase. %s -> %s. Spam Score: %s, Spam Report: %s",
|
|
|
|
alias,
|
|
|
|
contact,
|
|
|
|
email_log.spam_score,
|
|
|
|
email_log.spam_report,
|
|
|
|
)
|
2020-08-16 12:28:47 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
email_log.is_spam = True
|
|
|
|
email_log.spam_status = spam_status
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-08-15 14:53:57 +00:00
|
|
|
|
2021-03-26 09:00:48 +00:00
|
|
|
handle_spam(contact, alias, msg, user, mailbox, email_log, is_reply=True)
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E506
|
2020-08-15 14:53:57 +00:00
|
|
|
|
2020-11-02 13:51:37 +00:00
|
|
|
delete_all_headers_except(
|
|
|
|
msg,
|
|
|
|
[
|
2021-10-11 10:19:21 +00:00
|
|
|
headers.FROM,
|
|
|
|
headers.TO,
|
|
|
|
headers.CC,
|
|
|
|
headers.SUBJECT,
|
2021-10-11 10:00:37 +00:00
|
|
|
# do not delete original message id
|
|
|
|
headers.MESSAGE_ID,
|
2020-12-11 10:13:19 +00:00
|
|
|
# References and In-Reply-To are used for keeping the email thread
|
2021-10-11 10:00:37 +00:00
|
|
|
headers.REFERENCES,
|
|
|
|
headers.IN_REPLY_TO,
|
2020-11-02 13:51:37 +00:00
|
|
|
]
|
2021-10-11 10:19:21 +00:00
|
|
|
+ headers.MIME_HEADERS,
|
2020-11-02 13:51:37 +00:00
|
|
|
)
|
2020-11-01 17:12:09 +00:00
|
|
|
|
2021-10-25 12:33:42 +00:00
|
|
|
# replace the reverse-alias by the contact email in the email body
|
2020-11-01 17:02:43 +00:00
|
|
|
# as this is usually included when replying
|
|
|
|
if user.replace_reverse_alias:
|
2020-11-30 14:15:44 +00:00
|
|
|
LOG.d("Replace reverse-alias %s by contact email %s", reply_email, contact)
|
|
|
|
msg = replace(msg, reply_email, contact.website_email)
|
2020-11-01 17:02:43 +00:00
|
|
|
|
|
|
|
# create PGP email if needed
|
|
|
|
if contact.pgp_finger_print and user.is_premium():
|
|
|
|
LOG.d("Encrypt message for contact %s", contact)
|
|
|
|
try:
|
|
|
|
msg = prepare_pgp_message(
|
|
|
|
msg, contact.pgp_finger_print, contact.pgp_public_key
|
|
|
|
)
|
|
|
|
except PGPException:
|
2021-06-23 17:55:41 +00:00
|
|
|
LOG.e(
|
2020-11-01 17:02:43 +00:00
|
|
|
"Cannot encrypt message %s -> %s. %s %s", alias, contact, mailbox, user
|
|
|
|
)
|
2021-11-02 09:59:17 +00:00
|
|
|
# programming error, user shouldn't see a new email log
|
2020-12-11 10:03:52 +00:00
|
|
|
EmailLog.delete(email_log.id)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-11-01 17:02:43 +00:00
|
|
|
# return 421 so the client can retry later
|
2021-06-23 17:47:06 +00:00
|
|
|
return False, status.E402
|
2020-11-01 17:02:43 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-11-04 11:32:15 +00:00
|
|
|
|
2020-03-28 18:16:55 +00:00
|
|
|
# make the email comes from alias
|
2020-05-03 14:05:34 +00:00
|
|
|
from_header = alias.email
|
|
|
|
# add alias name from alias
|
2020-04-26 08:41:24 +00:00
|
|
|
if alias.name:
|
|
|
|
LOG.d("Put alias name in from header")
|
|
|
|
from_header = formataddr((alias.name, alias.email))
|
2020-05-03 14:05:34 +00:00
|
|
|
elif alias.custom_domain:
|
|
|
|
LOG.d("Put domain default alias name in from header")
|
|
|
|
|
|
|
|
# add alias name from domain
|
|
|
|
if alias.custom_domain.name:
|
|
|
|
from_header = formataddr((alias.custom_domain.name, alias.email))
|
|
|
|
|
2021-10-11 10:13:24 +00:00
|
|
|
add_or_replace_header(msg, headers.FROM, from_header)
|
2020-02-10 16:24:14 +00:00
|
|
|
|
2021-10-11 10:13:24 +00:00
|
|
|
replace_header_when_reply(msg, alias, headers.TO)
|
|
|
|
replace_header_when_reply(msg, alias, headers.CC)
|
2019-12-15 16:04:46 +00:00
|
|
|
|
2021-11-01 17:43:19 +00:00
|
|
|
replace_original_message_id(alias, email_log, msg)
|
2021-10-11 10:13:24 +00:00
|
|
|
|
|
|
|
if not msg[headers.DATE]:
|
|
|
|
date_header = formatdate()
|
|
|
|
LOG.w("missing date header, add one")
|
|
|
|
msg[headers.DATE] = date_header
|
2020-08-25 10:47:28 +00:00
|
|
|
|
2021-10-11 10:19:21 +00:00
|
|
|
msg[headers.SL_DIRECTION] = "Reply"
|
|
|
|
msg[headers.SL_EMAIL_LOG_ID] = str(email_log.id)
|
2020-08-25 10:47:28 +00:00
|
|
|
|
2020-02-19 15:17:13 +00:00
|
|
|
LOG.d(
|
|
|
|
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
|
2020-03-28 18:16:55 +00:00
|
|
|
alias.email,
|
2020-03-17 09:56:59 +00:00
|
|
|
contact.website_email,
|
2020-02-19 15:17:13 +00:00
|
|
|
envelope.mail_options,
|
|
|
|
envelope.rcpt_options,
|
|
|
|
)
|
2019-12-17 16:48:06 +00:00
|
|
|
|
2021-09-25 16:47:15 +00:00
|
|
|
if should_add_dkim_signature(alias_domain):
|
|
|
|
add_dkim_signature(msg, alias_domain)
|
|
|
|
|
2021-05-25 15:58:45 +00:00
|
|
|
# generate a mail_from for VERP
|
|
|
|
verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}"
|
|
|
|
|
2020-06-20 14:19:01 +00:00
|
|
|
try:
|
2020-09-29 10:57:14 +00:00
|
|
|
sl_sendmail(
|
2021-05-25 15:58:45 +00:00
|
|
|
verp_mail_from,
|
2020-06-20 14:19:01 +00:00
|
|
|
contact.website_email,
|
2020-09-29 10:57:14 +00:00
|
|
|
msg,
|
2020-06-20 14:19:01 +00:00
|
|
|
envelope.mail_options,
|
|
|
|
envelope.rcpt_options,
|
2021-01-28 12:50:24 +00:00
|
|
|
is_forward=False,
|
2020-06-20 14:19:01 +00:00
|
|
|
)
|
|
|
|
except Exception:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("Cannot send email from %s to %s", alias, contact)
|
2020-06-20 14:19:01 +00:00
|
|
|
send_email(
|
|
|
|
mailbox.email,
|
|
|
|
f"Email cannot be sent to {contact.email} from {alias.email}",
|
|
|
|
render(
|
2021-11-02 09:58:51 +00:00
|
|
|
"transactional/reply-error.txt.jinja2",
|
2020-06-20 14:19:01 +00:00
|
|
|
user=user,
|
|
|
|
alias=alias,
|
|
|
|
contact=contact,
|
|
|
|
contact_domain=get_email_domain_part(contact.email),
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/reply-error.html",
|
|
|
|
user=user,
|
|
|
|
alias=alias,
|
|
|
|
contact=contact,
|
|
|
|
contact_domain=get_email_domain_part(contact.email),
|
|
|
|
),
|
|
|
|
)
|
2020-01-07 18:14:36 +00:00
|
|
|
|
2020-09-02 08:16:13 +00:00
|
|
|
# return 250 even if error as user is already informed of the incident and can retry sending the email
|
2021-06-23 17:47:06 +00:00
|
|
|
return True, status.E200
|
2019-11-20 17:52:49 +00:00
|
|
|
|
2020-01-08 11:44:29 +00:00
|
|
|
|
2021-11-01 17:43:19 +00:00
|
|
|
def replace_original_message_id(alias: Alias, email_log: EmailLog, msg: Message):
|
|
|
|
"""
|
|
|
|
Replace original Message-ID by SL-Message-ID during the reply phase
|
|
|
|
for "message-id" and "References" headers
|
|
|
|
"""
|
|
|
|
original_message_id = msg[headers.MESSAGE_ID]
|
2021-11-05 08:43:58 +00:00
|
|
|
if original_message_id:
|
|
|
|
matching = MessageIDMatching.get_by(original_message_id=original_message_id)
|
|
|
|
# can happen when a user replies to multiple recipient from their alias
|
|
|
|
# a SL Message_id will be created for the first recipient
|
|
|
|
# it should be reused for other recipients
|
|
|
|
if matching:
|
|
|
|
sl_message_id = matching.sl_message_id
|
|
|
|
LOG.d("reuse the sl_message_id %s", sl_message_id)
|
|
|
|
else:
|
|
|
|
sl_message_id = make_msgid(
|
|
|
|
str(email_log.id), get_email_domain_part(alias.email)
|
|
|
|
)
|
|
|
|
LOG.d("create a new sl_message_id %s", sl_message_id)
|
|
|
|
try:
|
|
|
|
MessageIDMatching.create(
|
|
|
|
sl_message_id=sl_message_id,
|
|
|
|
original_message_id=original_message_id,
|
|
|
|
email_log_id=email_log.id,
|
|
|
|
commit=True,
|
|
|
|
)
|
|
|
|
except IntegrityError:
|
|
|
|
LOG.w(
|
|
|
|
"another matching with original_message_id %s was created in the mean time",
|
|
|
|
original_message_id,
|
|
|
|
)
|
|
|
|
Session.rollback()
|
|
|
|
matching = MessageIDMatching.get_by(
|
|
|
|
original_message_id=original_message_id
|
|
|
|
)
|
|
|
|
sl_message_id = matching.sl_message_id
|
2021-11-01 17:43:19 +00:00
|
|
|
else:
|
|
|
|
sl_message_id = make_msgid(
|
|
|
|
str(email_log.id), get_email_domain_part(alias.email)
|
|
|
|
)
|
2021-11-05 08:43:58 +00:00
|
|
|
LOG.d("no original_message_id, create a new sl_message_id %s", sl_message_id)
|
2021-11-01 19:36:15 +00:00
|
|
|
|
2021-11-01 17:43:19 +00:00
|
|
|
del msg[headers.MESSAGE_ID]
|
|
|
|
msg[headers.MESSAGE_ID] = sl_message_id
|
2021-11-01 19:36:15 +00:00
|
|
|
|
2021-11-01 17:43:19 +00:00
|
|
|
email_log.sl_message_id = sl_message_id
|
|
|
|
Session.commit()
|
|
|
|
|
|
|
|
# Replace all original headers in References header by SL Message ID header if needed
|
|
|
|
if msg[headers.REFERENCES]:
|
|
|
|
message_ids = msg[headers.REFERENCES].split()
|
|
|
|
new_message_ids = []
|
|
|
|
for message_id in message_ids:
|
|
|
|
matching = MessageIDMatching.get_by(original_message_id=message_id)
|
|
|
|
if matching:
|
|
|
|
LOG.d(
|
|
|
|
"replace original message id by SL one, %s -> %s",
|
|
|
|
message_id,
|
|
|
|
matching.sl_message_id,
|
|
|
|
)
|
|
|
|
new_message_ids.append(matching.sl_message_id)
|
|
|
|
else:
|
|
|
|
new_message_ids.append(message_id)
|
|
|
|
|
|
|
|
del msg[headers.REFERENCES]
|
|
|
|
msg[headers.REFERENCES] = " ".join(new_message_ids)
|
|
|
|
|
|
|
|
|
2020-09-28 15:41:16 +00:00
|
|
|
def get_mailbox_from_mail_from(mail_from: str, alias) -> Optional[Mailbox]:
|
|
|
|
"""return the corresponding mailbox given the mail_from and alias
|
|
|
|
Usually the mail_from=mailbox.email but it can also be one of the authorized address
|
|
|
|
"""
|
|
|
|
for mailbox in alias.mailboxes:
|
|
|
|
if mailbox.email == mail_from:
|
|
|
|
return mailbox
|
|
|
|
|
2021-09-10 15:10:05 +00:00
|
|
|
for addr in mailbox.authorized_addresses:
|
|
|
|
if addr.email == mail_from:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.d(
|
2020-09-28 15:41:16 +00:00
|
|
|
"Found an authorized address for %s %s %s", alias, mailbox, address
|
|
|
|
)
|
|
|
|
return mailbox
|
|
|
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2020-09-16 15:24:42 +00:00
|
|
|
def handle_unknown_mailbox(
|
|
|
|
envelope, msg, reply_email: str, user: User, alias: Alias, contact: Contact
|
|
|
|
):
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2020-12-06 12:54:59 +00:00
|
|
|
"Reply email can only be used by mailbox. "
|
|
|
|
"Actual mail_from: %s. msg from header: %s, reverse-alias %s, %s %s %s",
|
2020-05-09 21:12:30 +00:00
|
|
|
envelope.mail_from,
|
2021-10-11 10:10:18 +00:00
|
|
|
msg[headers.FROM],
|
2020-05-09 21:12:30 +00:00
|
|
|
reply_email,
|
2020-05-10 16:19:29 +00:00
|
|
|
alias,
|
|
|
|
user,
|
2020-09-16 15:24:42 +00:00
|
|
|
contact,
|
2020-05-09 21:12:30 +00:00
|
|
|
)
|
|
|
|
|
2020-09-29 09:00:50 +00:00
|
|
|
authorize_address_link = (
|
|
|
|
f"{URL}/dashboard/mailbox/{alias.mailbox_id}/#authorized-address"
|
|
|
|
)
|
2020-09-29 11:03:15 +00:00
|
|
|
mailbox_emails = [mailbox.email for mailbox in alias.mailboxes]
|
2020-05-09 21:12:30 +00:00
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
|
2020-05-10 16:19:29 +00:00
|
|
|
user.email,
|
2020-08-27 09:12:48 +00:00
|
|
|
f"Attempt to use your alias {alias.email} from {envelope.mail_from}",
|
2020-05-09 21:12:30 +00:00
|
|
|
render(
|
|
|
|
"transactional/reply-must-use-personal-email.txt",
|
2020-05-10 16:19:29 +00:00
|
|
|
alias=alias,
|
2020-05-09 21:12:30 +00:00
|
|
|
sender=envelope.mail_from,
|
2020-09-29 09:00:50 +00:00
|
|
|
authorize_address_link=authorize_address_link,
|
2020-09-29 11:11:04 +00:00
|
|
|
mailbox_emails=mailbox_emails,
|
2020-05-09 21:12:30 +00:00
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/reply-must-use-personal-email.html",
|
2020-05-10 16:19:29 +00:00
|
|
|
alias=alias,
|
2020-05-09 21:12:30 +00:00
|
|
|
sender=envelope.mail_from,
|
2020-09-29 09:00:50 +00:00
|
|
|
authorize_address_link=authorize_address_link,
|
2020-09-29 11:11:04 +00:00
|
|
|
mailbox_emails=mailbox_emails,
|
2020-05-09 21:12:30 +00:00
|
|
|
),
|
|
|
|
)
|
|
|
|
|
|
|
|
# Notify sender that they cannot send emails to this address
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_REVERSE_ALIAS_UNKNOWN_MAILBOX,
|
|
|
|
envelope.mail_from,
|
|
|
|
f"Your email ({envelope.mail_from}) is not allowed to send emails to {reply_email}",
|
|
|
|
render(
|
|
|
|
"transactional/send-from-alias-from-unknown-sender.txt",
|
|
|
|
sender=envelope.mail_from,
|
|
|
|
reply_email=reply_email,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/send-from-alias-from-unknown-sender.html",
|
|
|
|
sender=envelope.mail_from,
|
|
|
|
reply_email=reply_email,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2021-01-11 13:55:55 +00:00
|
|
|
def handle_bounce_forward_phase(msg: Message, email_log: EmailLog):
|
2020-11-04 11:32:15 +00:00
|
|
|
"""
|
2021-01-11 13:55:55 +00:00
|
|
|
Handle forward phase bounce
|
|
|
|
Happens when an email cannot be sent to a mailbox
|
2020-11-04 11:32:15 +00:00
|
|
|
"""
|
2021-01-11 13:55:55 +00:00
|
|
|
contact = email_log.contact
|
|
|
|
alias = contact.alias
|
2021-01-04 13:22:07 +00:00
|
|
|
user = alias.user
|
2021-01-11 14:53:08 +00:00
|
|
|
mailbox = email_log.mailbox
|
|
|
|
|
|
|
|
# email_log.mailbox should be set during the forward phase
|
|
|
|
if not mailbox:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Use %s default mailbox %s", alias, alias.mailbox)
|
2021-01-11 14:53:08 +00:00
|
|
|
mailbox = alias.mailbox
|
|
|
|
|
2021-10-14 13:46:52 +00:00
|
|
|
bounce_info = get_mailbox_bounce_info(msg)
|
|
|
|
if bounce_info:
|
2021-10-19 10:05:35 +00:00
|
|
|
Bounce.create(
|
|
|
|
email=mailbox.email, info=bounce_info.as_bytes().decode(), commit=True
|
|
|
|
)
|
2021-10-14 13:46:52 +00:00
|
|
|
else:
|
2021-11-02 13:32:16 +00:00
|
|
|
LOG.w("cannot get bounce info, debug at %s", save_email_for_debugging(msg))
|
2021-10-14 13:46:52 +00:00
|
|
|
Bounce.create(email=mailbox.email, commit=True)
|
2021-01-26 08:56:13 +00:00
|
|
|
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.d(
|
2021-01-11 14:53:08 +00:00
|
|
|
"Handle forward bounce %s -> %s -> %s. %s", contact, alias, mailbox, email_log
|
|
|
|
)
|
2021-01-04 13:22:07 +00:00
|
|
|
|
2021-01-11 13:55:55 +00:00
|
|
|
# Store the bounced email, generate a name for the email
|
|
|
|
random_name = str(uuid.uuid4())
|
2020-11-04 14:38:26 +00:00
|
|
|
|
2021-01-11 13:55:55 +00:00
|
|
|
full_report_path = f"refused-emails/full-{random_name}.eml"
|
2021-01-11 14:25:54 +00:00
|
|
|
s3.upload_email_from_bytesio(
|
|
|
|
full_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}"
|
|
|
|
)
|
2021-01-11 13:55:55 +00:00
|
|
|
|
|
|
|
file_path = None
|
|
|
|
|
|
|
|
orig_msg = get_orig_message_from_bounce(msg)
|
|
|
|
if not orig_msg:
|
|
|
|
# Some MTA does not return the original message in bounce message
|
|
|
|
# nothing we can do here
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w(
|
2021-01-11 13:55:55 +00:00
|
|
|
"Cannot parse original message from bounce message %s %s %s %s",
|
2020-11-04 14:38:26 +00:00
|
|
|
alias,
|
|
|
|
user,
|
2021-01-11 13:55:55 +00:00
|
|
|
contact,
|
|
|
|
full_report_path,
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
file_path = f"refused-emails/{random_name}.eml"
|
2021-01-11 14:25:54 +00:00
|
|
|
s3.upload_email_from_bytesio(
|
|
|
|
file_path, BytesIO(to_bytes(orig_msg)), random_name
|
|
|
|
)
|
2021-01-11 13:55:55 +00:00
|
|
|
|
|
|
|
refused_email = RefusedEmail.create(
|
|
|
|
path=file_path, full_report_path=full_report_path, user_id=user.id
|
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.flush()
|
2021-01-11 13:55:55 +00:00
|
|
|
LOG.d("Create refused email %s", refused_email)
|
|
|
|
|
|
|
|
email_log.bounced = True
|
|
|
|
email_log.refused_email_id = refused_email.id
|
|
|
|
email_log.bounced_mailbox_id = mailbox.id
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-01-11 13:55:55 +00:00
|
|
|
|
|
|
|
refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}"
|
|
|
|
|
|
|
|
# inform user of this bounce
|
|
|
|
if not should_disable(alias):
|
|
|
|
LOG.d(
|
|
|
|
"Inform user %s about a bounce from contact %s to alias %s",
|
|
|
|
user,
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
)
|
|
|
|
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
|
2021-10-28 16:41:36 +00:00
|
|
|
block_sender_link = f"{URL}/dashboard/alias_contact_manager/{alias.id}?highlight_contact_id={contact.id}"
|
2021-01-11 13:55:55 +00:00
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_BOUNCE_EMAIL,
|
|
|
|
user.email,
|
|
|
|
f"Email from {contact.website_email} to {alias.email} cannot be delivered to your mailbox",
|
|
|
|
render(
|
2021-10-28 09:43:44 +00:00
|
|
|
"transactional/bounce/bounced-email.txt.jinja2",
|
2021-01-11 13:55:55 +00:00
|
|
|
alias=alias,
|
|
|
|
website_email=contact.website_email,
|
|
|
|
disable_alias_link=disable_alias_link,
|
2021-10-28 09:43:44 +00:00
|
|
|
block_sender_link=block_sender_link,
|
2021-01-11 13:55:55 +00:00
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
mailbox_email=mailbox.email,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/bounce/bounced-email.html",
|
|
|
|
alias=alias,
|
|
|
|
website_email=contact.website_email,
|
|
|
|
disable_alias_link=disable_alias_link,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
mailbox_email=mailbox.email,
|
|
|
|
),
|
|
|
|
max_nb_alert=10,
|
2021-09-20 11:51:16 +00:00
|
|
|
# smtp error can happen if user mailbox is unreachable, that might explain the bounce
|
|
|
|
ignore_smtp_error=True,
|
2021-01-11 13:55:55 +00:00
|
|
|
)
|
|
|
|
else:
|
2021-10-14 13:46:52 +00:00
|
|
|
LOG.w("Disable alias %s %s. Last contact %s", alias, user, contact)
|
2021-01-11 13:55:55 +00:00
|
|
|
alias.enabled = False
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-01-11 13:55:55 +00:00
|
|
|
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_BOUNCE_EMAIL,
|
|
|
|
user.email,
|
2021-01-19 09:36:11 +00:00
|
|
|
f"Alias {alias.email} has been disabled due to multiple bounces",
|
2021-01-11 13:55:55 +00:00
|
|
|
render(
|
|
|
|
"transactional/bounce/automatic-disable-alias.txt",
|
|
|
|
alias=alias,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
mailbox_email=mailbox.email,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/bounce/automatic-disable-alias.html",
|
|
|
|
alias=alias,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
mailbox_email=mailbox.email,
|
|
|
|
),
|
|
|
|
max_nb_alert=10,
|
2021-09-20 11:51:16 +00:00
|
|
|
ignore_smtp_error=True,
|
2020-11-04 14:38:26 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
2021-09-09 16:54:54 +00:00
|
|
|
def handle_hotmail_complaint(msg: Message) -> bool:
|
2021-09-06 17:44:18 +00:00
|
|
|
"""
|
|
|
|
Handle hotmail complaint sent to postmaster
|
2021-09-09 16:54:54 +00:00
|
|
|
Return True if the complaint can be handled, False otherwise
|
2021-09-06 17:44:18 +00:00
|
|
|
"""
|
2021-09-13 17:50:15 +00:00
|
|
|
orig_msg = get_orig_message_from_hotmail_complaint(msg)
|
2021-10-11 10:10:18 +00:00
|
|
|
to_header = orig_msg[headers.TO]
|
2021-10-26 10:16:57 +00:00
|
|
|
from_header = orig_msg[headers.FROM]
|
2021-11-04 09:40:12 +00:00
|
|
|
alias = None
|
2021-10-26 10:16:57 +00:00
|
|
|
|
2021-11-04 09:40:12 +00:00
|
|
|
# try parsing the from header which might contain the reverse alias
|
2021-10-25 08:36:50 +00:00
|
|
|
try:
|
2021-10-26 10:16:57 +00:00
|
|
|
_, reverse_alias = parse_full_address(get_header_unicode(from_header))
|
|
|
|
contact = Contact.get_by(reply_email=reverse_alias)
|
|
|
|
if contact:
|
|
|
|
alias = contact.alias
|
|
|
|
LOG.d("find %s through %s", alias, contact)
|
|
|
|
else:
|
2021-11-04 09:40:12 +00:00
|
|
|
LOG.d("No contact found for %s", reverse_alias)
|
|
|
|
except ValueError:
|
|
|
|
LOG.w("Cannot parse %s", from_header)
|
2021-09-06 17:44:18 +00:00
|
|
|
|
2021-11-04 09:40:12 +00:00
|
|
|
# try parsing the to_header which is usually the alias
|
2021-09-06 17:44:18 +00:00
|
|
|
if not alias:
|
2021-11-04 09:40:12 +00:00
|
|
|
try:
|
|
|
|
_, alias_address = parse_full_address(get_header_unicode(to_header))
|
|
|
|
alias = Alias.get_by(email=alias_address)
|
|
|
|
except ValueError:
|
|
|
|
LOG.w("Cannot parse %s", to_header)
|
|
|
|
|
|
|
|
if not alias:
|
|
|
|
LOG.e(
|
2021-10-26 10:16:57 +00:00
|
|
|
"Cannot parse alias from to header %s and from header %s",
|
|
|
|
to_header,
|
|
|
|
from_header,
|
|
|
|
)
|
2021-09-09 16:54:54 +00:00
|
|
|
return False
|
2021-09-06 17:44:18 +00:00
|
|
|
|
|
|
|
user = alias.user
|
2021-11-04 09:40:12 +00:00
|
|
|
LOG.d("Handle hotmail complaint for %s %s %s", alias, user, alias.mailboxes)
|
2021-09-06 17:44:18 +00:00
|
|
|
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_HOTMAIL_COMPLAINT,
|
|
|
|
user.email,
|
|
|
|
f"Hotmail abuse report",
|
|
|
|
render(
|
|
|
|
"transactional/hotmail-complaint.txt.jinja2",
|
|
|
|
alias=alias,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/hotmail-complaint.html",
|
|
|
|
alias=alias,
|
|
|
|
),
|
2021-11-10 09:57:22 +00:00
|
|
|
max_nb_alert=1,
|
|
|
|
nb_day=7,
|
2021-09-06 17:44:18 +00:00
|
|
|
)
|
|
|
|
|
2021-09-09 16:54:54 +00:00
|
|
|
return True
|
|
|
|
|
2021-09-06 17:44:18 +00:00
|
|
|
|
2021-09-13 17:49:40 +00:00
|
|
|
def handle_yahoo_complaint(msg: Message) -> bool:
|
|
|
|
"""
|
|
|
|
Handle yahoo complaint sent to postmaster
|
|
|
|
Return True if the complaint can be handled, False otherwise
|
|
|
|
"""
|
|
|
|
orig_msg = get_orig_message_from_yahoo_complaint(msg)
|
2021-10-11 10:10:18 +00:00
|
|
|
to_header = orig_msg[headers.TO]
|
2021-09-13 17:49:40 +00:00
|
|
|
if not to_header:
|
|
|
|
LOG.e("cannot find the alias")
|
|
|
|
return False
|
|
|
|
|
|
|
|
_, alias_address = parse_full_address(get_header_unicode(to_header))
|
|
|
|
alias = Alias.get_by(email=alias_address)
|
|
|
|
|
|
|
|
if not alias:
|
|
|
|
LOG.w("No alias for %s", alias_address)
|
|
|
|
return False
|
|
|
|
|
|
|
|
user = alias.user
|
|
|
|
LOG.w("Handle yahoo complaint for %s %s %s", alias, user, alias.mailboxes)
|
|
|
|
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_YAHOO_COMPLAINT,
|
|
|
|
user.email,
|
|
|
|
f"Yahoo abuse report",
|
|
|
|
render(
|
|
|
|
"transactional/yahoo-complaint.txt.jinja2",
|
|
|
|
alias=alias,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/yahoo-complaint.html",
|
|
|
|
alias=alias,
|
|
|
|
),
|
|
|
|
max_nb_alert=2,
|
|
|
|
)
|
|
|
|
|
|
|
|
return True
|
|
|
|
|
|
|
|
|
2021-03-06 16:44:31 +00:00
|
|
|
def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog):
|
2021-01-11 13:55:55 +00:00
|
|
|
"""
|
|
|
|
Handle reply phase bounce
|
|
|
|
Happens when an email cannot be sent from an alias to a contact
|
|
|
|
"""
|
2021-01-26 08:56:13 +00:00
|
|
|
contact: Contact = email_log.contact
|
2021-01-11 13:55:55 +00:00
|
|
|
alias = contact.alias
|
|
|
|
user = alias.user
|
2021-01-11 14:53:08 +00:00
|
|
|
mailbox = email_log.mailbox or alias.mailbox
|
|
|
|
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.d("Handle reply bounce %s -> %s -> %s.%s", mailbox, alias, contact, email_log)
|
2021-01-11 13:55:55 +00:00
|
|
|
|
2021-10-14 13:46:52 +00:00
|
|
|
bounce_info = get_mailbox_bounce_info(msg)
|
|
|
|
if bounce_info:
|
|
|
|
Bounce.create(
|
|
|
|
email=sanitize_email(contact.website_email),
|
2021-10-19 10:05:35 +00:00
|
|
|
info=bounce_info.as_bytes().decode(),
|
2021-10-14 13:46:52 +00:00
|
|
|
commit=True,
|
|
|
|
)
|
|
|
|
else:
|
2021-11-02 13:32:16 +00:00
|
|
|
LOG.w("cannot get bounce info, debug at %s", save_email_for_debugging(msg))
|
2021-10-14 13:46:52 +00:00
|
|
|
Bounce.create(email=sanitize_email(contact.website_email), commit=True)
|
2021-01-26 08:56:13 +00:00
|
|
|
|
2020-11-04 11:32:15 +00:00
|
|
|
# Store the bounced email
|
|
|
|
# generate a name for the email
|
|
|
|
random_name = str(uuid.uuid4())
|
|
|
|
|
|
|
|
full_report_path = f"refused-emails/full-{random_name}.eml"
|
2020-11-09 16:02:10 +00:00
|
|
|
s3.upload_email_from_bytesio(full_report_path, BytesIO(to_bytes(msg)), random_name)
|
2020-11-04 11:32:15 +00:00
|
|
|
|
|
|
|
orig_msg = get_orig_message_from_bounce(msg)
|
2020-11-04 14:38:26 +00:00
|
|
|
file_path = None
|
|
|
|
if orig_msg:
|
|
|
|
file_path = f"refused-emails/{random_name}.eml"
|
|
|
|
s3.upload_email_from_bytesio(
|
2020-11-09 16:02:10 +00:00
|
|
|
file_path, BytesIO(to_bytes(orig_msg)), random_name
|
2020-11-04 14:38:26 +00:00
|
|
|
)
|
2020-11-04 11:32:15 +00:00
|
|
|
|
2020-11-04 14:38:26 +00:00
|
|
|
refused_email = RefusedEmail.create(
|
|
|
|
path=file_path, full_report_path=full_report_path, user_id=user.id, commit=True
|
|
|
|
)
|
|
|
|
LOG.d("Create refused email %s", refused_email)
|
2020-11-04 11:32:15 +00:00
|
|
|
|
2020-11-04 14:38:26 +00:00
|
|
|
email_log.bounced = True
|
|
|
|
email_log.refused_email_id = refused_email.id
|
2020-11-04 11:32:15 +00:00
|
|
|
|
2020-11-24 15:50:55 +00:00
|
|
|
email_log.bounced_mailbox_id = mailbox.id
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-11-04 11:32:15 +00:00
|
|
|
|
2020-12-06 12:54:59 +00:00
|
|
|
refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}"
|
2020-11-04 11:32:15 +00:00
|
|
|
|
|
|
|
LOG.d(
|
|
|
|
"Inform user %s about bounced email sent by %s to %s",
|
|
|
|
user,
|
|
|
|
alias,
|
|
|
|
contact,
|
|
|
|
)
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_BOUNCE_EMAIL_REPLY_PHASE,
|
|
|
|
mailbox.email,
|
|
|
|
f"Email cannot be sent to { contact.email } from your alias { alias.email }",
|
|
|
|
render(
|
2021-01-04 13:22:07 +00:00
|
|
|
"transactional/bounce/bounce-email-reply-phase.txt",
|
2020-11-04 11:32:15 +00:00
|
|
|
alias=alias,
|
|
|
|
contact=contact,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
render(
|
2021-01-04 13:22:07 +00:00
|
|
|
"transactional/bounce/bounce-email-reply-phase.html",
|
2020-11-04 11:32:15 +00:00
|
|
|
alias=alias,
|
|
|
|
contact=contact,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2020-03-30 20:05:31 +00:00
|
|
|
def handle_spam(
|
|
|
|
contact: Contact,
|
|
|
|
alias: Alias,
|
|
|
|
msg: Message,
|
|
|
|
user: User,
|
2020-08-21 08:18:58 +00:00
|
|
|
mailbox: Mailbox,
|
2020-04-27 16:18:40 +00:00
|
|
|
email_log: EmailLog,
|
2020-08-15 14:53:57 +00:00
|
|
|
is_reply=False, # whether the email is in forward or reply phase
|
2020-03-30 20:05:31 +00:00
|
|
|
):
|
|
|
|
# Store the report & original email
|
|
|
|
orig_msg = get_orig_message_from_spamassassin_report(msg)
|
|
|
|
# generate a name for the email
|
|
|
|
random_name = str(uuid.uuid4())
|
|
|
|
|
2020-04-02 16:09:05 +00:00
|
|
|
full_report_path = f"spams/full-{random_name}.eml"
|
2020-11-09 16:02:10 +00:00
|
|
|
s3.upload_email_from_bytesio(full_report_path, BytesIO(to_bytes(msg)), random_name)
|
2020-03-30 20:05:31 +00:00
|
|
|
|
|
|
|
file_path = None
|
|
|
|
if orig_msg:
|
2020-04-02 16:09:05 +00:00
|
|
|
file_path = f"spams/{random_name}.eml"
|
2020-03-30 20:05:31 +00:00
|
|
|
s3.upload_email_from_bytesio(
|
2020-11-09 16:02:10 +00:00
|
|
|
file_path, BytesIO(to_bytes(orig_msg)), random_name
|
2020-03-30 20:05:31 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
refused_email = RefusedEmail.create(
|
|
|
|
path=file_path, full_report_path=full_report_path, user_id=user.id
|
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.flush()
|
2020-03-30 20:05:31 +00:00
|
|
|
|
|
|
|
email_log.refused_email_id = refused_email.id
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-03-30 20:05:31 +00:00
|
|
|
|
|
|
|
LOG.d("Create spam email %s", refused_email)
|
|
|
|
|
2020-12-06 12:54:59 +00:00
|
|
|
refused_email_url = f"{URL}/dashboard/refused_email?highlight_id={email_log.id}"
|
2020-03-30 20:05:31 +00:00
|
|
|
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
|
|
|
|
|
2020-08-15 14:53:57 +00:00
|
|
|
if is_reply:
|
|
|
|
LOG.d(
|
2020-12-02 11:40:29 +00:00
|
|
|
"Inform %s (%s) about spam email sent from alias %s to %s. %s",
|
2020-08-21 08:20:08 +00:00
|
|
|
mailbox,
|
2020-08-15 14:53:57 +00:00
|
|
|
user,
|
|
|
|
alias,
|
|
|
|
contact,
|
2020-12-02 11:40:29 +00:00
|
|
|
refused_email,
|
2020-08-15 14:53:57 +00:00
|
|
|
)
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_SPAM_EMAIL,
|
2020-08-21 08:18:58 +00:00
|
|
|
mailbox.email,
|
2020-11-18 15:16:37 +00:00
|
|
|
f"Email from {alias.email} to {contact.website_email} is detected as spam",
|
2020-08-15 14:53:57 +00:00
|
|
|
render(
|
|
|
|
"transactional/spam-email-reply-phase.txt",
|
|
|
|
alias=alias,
|
|
|
|
website_email=contact.website_email,
|
|
|
|
disable_alias_link=disable_alias_link,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/spam-email-reply-phase.html",
|
|
|
|
alias=alias,
|
|
|
|
website_email=contact.website_email,
|
|
|
|
disable_alias_link=disable_alias_link,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
# inform user
|
|
|
|
LOG.d(
|
2020-08-21 08:20:08 +00:00
|
|
|
"Inform %s (%s) about spam email sent by %s to alias %s",
|
|
|
|
mailbox,
|
2020-08-15 14:53:57 +00:00
|
|
|
user,
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
)
|
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
ALERT_SPAM_EMAIL,
|
2020-08-21 08:18:58 +00:00
|
|
|
mailbox.email,
|
2020-08-15 14:53:57 +00:00
|
|
|
f"Email from {contact.website_email} to {alias.email} is detected as spam",
|
|
|
|
render(
|
|
|
|
"transactional/spam-email.txt",
|
|
|
|
alias=alias,
|
|
|
|
website_email=contact.website_email,
|
|
|
|
disable_alias_link=disable_alias_link,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/spam-email.html",
|
|
|
|
alias=alias,
|
|
|
|
website_email=contact.website_email,
|
|
|
|
disable_alias_link=disable_alias_link,
|
|
|
|
refused_email_url=refused_email_url,
|
|
|
|
),
|
|
|
|
)
|
2020-03-30 20:05:31 +00:00
|
|
|
|
|
|
|
|
2021-03-17 09:20:10 +00:00
|
|
|
def handle_unsubscribe(envelope: Envelope, msg: Message) -> str:
|
2020-10-22 08:34:52 +00:00
|
|
|
"""return the SMTP status"""
|
2020-03-28 22:19:25 +00:00
|
|
|
# format: alias_id:
|
2021-10-11 10:10:18 +00:00
|
|
|
subject = msg[headers.SUBJECT]
|
2021-11-02 14:44:43 +00:00
|
|
|
alias, contact = None, None
|
|
|
|
|
2020-03-28 22:19:25 +00:00
|
|
|
try:
|
2020-04-24 07:09:11 +00:00
|
|
|
# subject has the format {alias.id}=
|
|
|
|
if subject.endswith("="):
|
|
|
|
alias_id = int(subject[:-1])
|
2021-11-02 14:44:43 +00:00
|
|
|
alias = Alias.get(alias_id)
|
|
|
|
# {contact.id}_
|
|
|
|
elif subject.endswith("_"):
|
|
|
|
contact_id = int(subject[:-1])
|
|
|
|
contact = Contact.get(contact_id)
|
|
|
|
if contact:
|
|
|
|
alias = contact.alias
|
2020-10-22 08:34:52 +00:00
|
|
|
# {user.id}*
|
|
|
|
elif subject.endswith("*"):
|
|
|
|
user_id = int(subject[:-1])
|
|
|
|
return handle_unsubscribe_user(user_id, envelope.mail_from)
|
2020-04-24 07:09:11 +00:00
|
|
|
# some email providers might strip off the = suffix
|
|
|
|
else:
|
|
|
|
alias_id = int(subject)
|
2021-11-02 14:44:43 +00:00
|
|
|
alias = Alias.get(alias_id)
|
2020-03-28 22:19:25 +00:00
|
|
|
except Exception:
|
2021-10-11 10:10:18 +00:00
|
|
|
LOG.w("Cannot parse alias from subject %s", msg[headers.SUBJECT])
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E507
|
2020-03-28 22:19:25 +00:00
|
|
|
|
|
|
|
if not alias:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("No such alias %s", alias_id)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E508
|
2020-03-28 22:19:25 +00:00
|
|
|
|
2020-09-14 15:38:48 +00:00
|
|
|
mail_from = envelope.mail_from
|
2020-10-23 11:29:20 +00:00
|
|
|
# Only alias's owning mailbox can send the unsubscribe request
|
|
|
|
mailbox = get_mailbox_from_mail_from(mail_from, alias)
|
|
|
|
if not mailbox:
|
2020-03-28 22:19:25 +00:00
|
|
|
LOG.d("%s cannot disable alias %s", envelope.mail_from, alias)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E509
|
2020-03-28 22:19:25 +00:00
|
|
|
|
|
|
|
user = alias.user
|
|
|
|
|
2021-11-02 14:44:43 +00:00
|
|
|
if contact:
|
|
|
|
contact.block_forward = True
|
|
|
|
Session.commit()
|
|
|
|
unblock_contact_url = (
|
|
|
|
URL
|
2021-11-03 09:11:52 +00:00
|
|
|
+ f"/dashboard/alias_contact_manager/{alias.id}?highlight_contact_id={contact.id}"
|
2020-05-10 16:23:43 +00:00
|
|
|
)
|
2021-11-02 14:44:43 +00:00
|
|
|
for mailbox in alias.mailboxes:
|
|
|
|
send_email(
|
|
|
|
mailbox.email,
|
|
|
|
f"Emails from {contact.website_email} to {alias.email} are now blocked",
|
|
|
|
render(
|
|
|
|
"transactional/unsubscribe-block-contact.txt.jinja2",
|
|
|
|
user=user,
|
|
|
|
alias=alias,
|
|
|
|
contact=contact,
|
|
|
|
unblock_contact_url=unblock_contact_url,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
alias.enabled = False
|
|
|
|
Session.commit()
|
|
|
|
enable_alias_url = URL + f"/dashboard/?highlight_alias_id={alias.id}"
|
|
|
|
for mailbox in alias.mailboxes:
|
|
|
|
send_email(
|
|
|
|
mailbox.email,
|
|
|
|
f"Alias {alias.email} has been disabled successfully",
|
|
|
|
render(
|
|
|
|
"transactional/unsubscribe-disable-alias.txt",
|
|
|
|
user=user,
|
|
|
|
alias=alias.email,
|
|
|
|
enable_alias_url=enable_alias_url,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/unsubscribe-disable-alias.html",
|
|
|
|
user=user,
|
|
|
|
alias=alias.email,
|
|
|
|
enable_alias_url=enable_alias_url,
|
|
|
|
),
|
|
|
|
)
|
2020-03-28 22:19:25 +00:00
|
|
|
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E202
|
2020-03-28 22:19:25 +00:00
|
|
|
|
|
|
|
|
2020-10-22 08:34:52 +00:00
|
|
|
def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
|
|
|
|
"""return the SMTP status"""
|
|
|
|
user = User.get(user_id)
|
|
|
|
if not user:
|
2021-09-20 14:59:27 +00:00
|
|
|
LOG.w("No such user %s %s", user_id, mail_from)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E510
|
2020-10-22 08:34:52 +00:00
|
|
|
|
|
|
|
if mail_from != user.email:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Unauthorized mail_from %s %s", user, mail_from)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E511
|
2020-10-22 08:34:52 +00:00
|
|
|
|
|
|
|
user.notification = False
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-10-22 08:34:52 +00:00
|
|
|
|
|
|
|
send_email(
|
|
|
|
user.email,
|
2020-12-06 12:54:59 +00:00
|
|
|
"You have been unsubscribed from SimpleLogin newsletter",
|
2020-10-22 08:34:52 +00:00
|
|
|
render(
|
|
|
|
"transactional/unsubscribe-newsletter.txt",
|
|
|
|
user=user,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/unsubscribe-newsletter.html",
|
|
|
|
user=user,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E202
|
2020-10-22 10:26:45 +00:00
|
|
|
|
2020-10-22 08:34:52 +00:00
|
|
|
|
2021-10-14 13:46:52 +00:00
|
|
|
def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
|
2021-01-26 08:59:08 +00:00
|
|
|
LOG.d("handle transactional bounce sent to %s", rcpt_to)
|
2020-06-10 11:57:23 +00:00
|
|
|
|
2021-01-26 08:59:08 +00:00
|
|
|
# parse the TransactionalEmail
|
|
|
|
transactional_id = parse_id_from_bounce(rcpt_to)
|
|
|
|
transactional = TransactionalEmail.get(transactional_id)
|
2020-06-10 11:57:23 +00:00
|
|
|
|
2021-02-06 15:00:32 +00:00
|
|
|
# a transaction might have been deleted in delete_logs()
|
2021-01-26 08:59:08 +00:00
|
|
|
if transactional:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.i("Create bounce for %s", transactional.email)
|
2021-10-14 13:46:52 +00:00
|
|
|
bounce_info = get_mailbox_bounce_info(msg)
|
|
|
|
if bounce_info:
|
|
|
|
Bounce.create(
|
2021-10-19 10:05:35 +00:00
|
|
|
email=transactional.email,
|
|
|
|
info=bounce_info.as_bytes().decode(),
|
|
|
|
commit=True,
|
2021-10-14 13:46:52 +00:00
|
|
|
)
|
|
|
|
else:
|
2021-11-02 13:32:16 +00:00
|
|
|
LOG.w("cannot get bounce info, debug at %s", save_email_for_debugging(msg))
|
2021-10-14 13:46:52 +00:00
|
|
|
Bounce.create(email=transactional.email, commit=True)
|
2020-06-10 11:57:23 +00:00
|
|
|
|
|
|
|
|
2021-06-02 09:38:25 +00:00
|
|
|
def handle_bounce(envelope, email_log: EmailLog, msg: Message) -> str:
|
2021-03-17 09:09:54 +00:00
|
|
|
"""
|
|
|
|
Return SMTP status, e.g. "500 Error"
|
|
|
|
"""
|
|
|
|
|
|
|
|
if not email_log:
|
2021-06-02 09:38:25 +00:00
|
|
|
LOG.w("No such email log")
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E512
|
2021-03-17 09:09:54 +00:00
|
|
|
|
2021-06-02 09:38:25 +00:00
|
|
|
contact: Contact = email_log.contact
|
|
|
|
alias = contact.alias
|
|
|
|
LOG.d(
|
|
|
|
"handle bounce for %s, phase=%s, contact=%s, alias=%s",
|
|
|
|
email_log,
|
|
|
|
email_log.get_phase(),
|
|
|
|
contact,
|
|
|
|
alias,
|
|
|
|
)
|
|
|
|
|
2021-03-17 09:09:54 +00:00
|
|
|
if email_log.is_reply:
|
|
|
|
content_type = msg.get_content_type().lower()
|
|
|
|
|
|
|
|
if content_type != "multipart/report" or envelope.mail_from != "<>":
|
|
|
|
# forward the email again to the alias
|
2021-05-30 18:02:41 +00:00
|
|
|
LOG.i(
|
|
|
|
"Handle auto reply %s %s",
|
2021-03-17 09:09:54 +00:00
|
|
|
content_type,
|
|
|
|
envelope.mail_from,
|
|
|
|
)
|
|
|
|
|
|
|
|
contact: Contact = email_log.contact
|
|
|
|
alias = contact.alias
|
|
|
|
|
|
|
|
email_log.auto_replied = True
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-03-17 09:09:54 +00:00
|
|
|
|
|
|
|
# replace the BOUNCE_EMAIL by alias in To field
|
|
|
|
add_or_replace_header(msg, "To", alias.email)
|
|
|
|
envelope.rcpt_tos = [alias.email]
|
|
|
|
|
|
|
|
# same as handle()
|
|
|
|
# result of all deliveries
|
|
|
|
# each element is a couple of whether the delivery is successful and the smtp status
|
|
|
|
res: [(bool, str)] = []
|
|
|
|
|
|
|
|
for is_delivered, smtp_status in handle_forward(envelope, msg, alias.email):
|
|
|
|
res.append((is_delivered, smtp_status))
|
|
|
|
|
|
|
|
for (is_success, smtp_status) in res:
|
|
|
|
# Consider all deliveries successful if 1 delivery is successful
|
|
|
|
if is_success:
|
|
|
|
return smtp_status
|
|
|
|
|
|
|
|
# Failed delivery for all, return the first failure
|
|
|
|
return res[0][1]
|
|
|
|
|
2021-11-13 10:21:19 +00:00
|
|
|
handle_bounce_reply_phase(envelope, msg, email_log)
|
|
|
|
return status.E212
|
2021-03-17 09:09:54 +00:00
|
|
|
else: # forward phase
|
|
|
|
handle_bounce_forward_phase(msg, email_log)
|
2021-11-13 10:21:19 +00:00
|
|
|
return status.E211
|
2021-03-17 09:09:54 +00:00
|
|
|
|
|
|
|
|
2021-06-22 15:52:24 +00:00
|
|
|
def should_ignore(mail_from: str, rcpt_tos: List[str]) -> bool:
|
|
|
|
if len(rcpt_tos) != 1:
|
|
|
|
return False
|
|
|
|
|
|
|
|
rcpt_to = rcpt_tos[0]
|
|
|
|
if IgnoredEmail.get_by(mail_from=mail_from, rcpt_to=rcpt_to):
|
|
|
|
return True
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
2021-10-13 08:27:59 +00:00
|
|
|
def handle(envelope: Envelope) -> str:
|
2020-04-04 14:09:24 +00:00
|
|
|
"""Return SMTP status"""
|
2020-09-14 15:19:29 +00:00
|
|
|
|
|
|
|
# sanitize mail_from, rcpt_tos
|
2021-01-11 11:27:02 +00:00
|
|
|
mail_from = sanitize_email(envelope.mail_from)
|
|
|
|
rcpt_tos = [sanitize_email(rcpt_to) for rcpt_to in envelope.rcpt_tos]
|
2020-09-14 15:19:29 +00:00
|
|
|
envelope.mail_from = mail_from
|
|
|
|
envelope.rcpt_tos = rcpt_tos
|
|
|
|
|
2021-03-16 08:17:23 +00:00
|
|
|
msg = email.message_from_bytes(envelope.original_content)
|
2021-06-04 15:15:59 +00:00
|
|
|
postfix_queue_id = get_queue_id(msg)
|
|
|
|
if postfix_queue_id:
|
|
|
|
set_message_id(postfix_queue_id)
|
2021-06-23 16:19:13 +00:00
|
|
|
else:
|
2021-10-15 08:37:22 +00:00
|
|
|
LOG.d(
|
|
|
|
"Cannot parse Postfix queue ID from %s %s",
|
|
|
|
msg.get_all(headers.RECEIVED),
|
|
|
|
msg[headers.RECEIVED],
|
|
|
|
)
|
2021-06-23 16:19:13 +00:00
|
|
|
|
|
|
|
if should_ignore(mail_from, rcpt_tos):
|
2021-06-23 17:47:51 +00:00
|
|
|
LOG.w("Ignore email mail_from=%s rcpt_to=%s", mail_from, rcpt_tos)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E204
|
2021-03-17 09:19:27 +00:00
|
|
|
|
|
|
|
# sanitize email headers
|
2021-03-17 09:59:13 +00:00
|
|
|
sanitize_header(msg, "from")
|
|
|
|
sanitize_header(msg, "to")
|
|
|
|
sanitize_header(msg, "cc")
|
|
|
|
sanitize_header(msg, "reply-to")
|
2021-03-17 09:19:27 +00:00
|
|
|
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.d(
|
|
|
|
"==>> Handle mail_from:%s, rcpt_tos:%s, header_from:%s, header_to:%s, "
|
2021-10-19 10:05:41 +00:00
|
|
|
"cc:%s, reply-to:%s, message_id:%s, mail_options:%s, rcpt_options:%s",
|
2021-03-16 08:17:23 +00:00
|
|
|
mail_from,
|
|
|
|
rcpt_tos,
|
2021-10-11 10:10:18 +00:00
|
|
|
msg[headers.FROM],
|
|
|
|
msg[headers.TO],
|
|
|
|
msg[headers.CC],
|
|
|
|
msg[headers.REPLY_TO],
|
2021-10-19 10:05:41 +00:00
|
|
|
msg[headers.MESSAGE_ID],
|
2021-03-16 08:17:23 +00:00
|
|
|
envelope.mail_options,
|
|
|
|
envelope.rcpt_options,
|
|
|
|
)
|
|
|
|
|
2021-03-15 18:55:22 +00:00
|
|
|
contact = Contact.get_by(reply_email=mail_from)
|
|
|
|
if contact:
|
2021-09-08 16:25:40 +00:00
|
|
|
LOG.w(
|
2021-09-08 09:25:53 +00:00
|
|
|
"email can't be sent from a reverse-alias:%s, contact email:%s, %s, %s",
|
|
|
|
contact.reply_email,
|
2021-03-15 18:55:22 +00:00
|
|
|
contact.website_email,
|
2021-09-08 09:25:53 +00:00
|
|
|
contact.alias,
|
|
|
|
contact.user,
|
2021-03-15 18:55:22 +00:00
|
|
|
)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E203
|
2021-03-15 18:55:22 +00:00
|
|
|
|
2020-04-04 14:09:24 +00:00
|
|
|
# unsubscribe request
|
2020-09-14 15:19:29 +00:00
|
|
|
if UNSUBSCRIBER and rcpt_tos == [UNSUBSCRIBER]:
|
|
|
|
LOG.d("Handle unsubscribe request from %s", mail_from)
|
2021-03-17 09:20:10 +00:00
|
|
|
return handle_unsubscribe(envelope, msg)
|
2020-04-04 14:09:24 +00:00
|
|
|
|
2020-06-10 11:57:23 +00:00
|
|
|
# emails sent to sender. Probably bounce emails
|
2021-01-26 08:59:08 +00:00
|
|
|
if (
|
|
|
|
len(rcpt_tos) == 1
|
|
|
|
and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX)
|
|
|
|
and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
|
|
|
|
):
|
2020-09-14 15:19:29 +00:00
|
|
|
LOG.d("Handle email sent to sender from %s", mail_from)
|
2021-10-14 13:46:52 +00:00
|
|
|
handle_transactional_bounce(envelope, msg, rcpt_tos[0])
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E205
|
2020-06-10 11:57:23 +00:00
|
|
|
|
2021-09-06 17:44:18 +00:00
|
|
|
if (
|
|
|
|
len(rcpt_tos) == 1
|
|
|
|
and mail_from == "staff@hotmail.com"
|
|
|
|
and rcpt_tos[0] == POSTMASTER
|
|
|
|
):
|
|
|
|
LOG.w("Handle hotmail complaint")
|
2021-09-09 16:54:54 +00:00
|
|
|
|
|
|
|
# if the complaint cannot be handled, forward it normally
|
|
|
|
if handle_hotmail_complaint(msg):
|
|
|
|
return status.E208
|
2021-09-06 17:44:18 +00:00
|
|
|
|
2021-09-13 17:49:40 +00:00
|
|
|
if (
|
|
|
|
len(rcpt_tos) == 1
|
|
|
|
and mail_from == "feedback@arf.mail.yahoo.com"
|
|
|
|
and rcpt_tos[0] == POSTMASTER
|
|
|
|
):
|
|
|
|
LOG.w("Handle yahoo complaint")
|
|
|
|
|
|
|
|
# if the complaint cannot be handled, forward it normally
|
|
|
|
if handle_yahoo_complaint(msg):
|
|
|
|
return status.E210
|
|
|
|
|
2021-06-02 09:38:25 +00:00
|
|
|
# Handle bounce
|
2021-01-11 13:55:55 +00:00
|
|
|
if (
|
|
|
|
len(rcpt_tos) == 1
|
|
|
|
and rcpt_tos[0].startswith(BOUNCE_PREFIX)
|
|
|
|
and rcpt_tos[0].endswith(BOUNCE_SUFFIX)
|
|
|
|
):
|
2021-06-02 09:38:25 +00:00
|
|
|
email_log_id = parse_id_from_bounce(rcpt_tos[0])
|
|
|
|
email_log = EmailLog.get(email_log_id)
|
|
|
|
return handle_bounce(envelope, email_log, msg)
|
2021-05-25 15:59:09 +00:00
|
|
|
|
|
|
|
if len(rcpt_tos) == 1 and rcpt_tos[0].startswith(
|
|
|
|
f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+"
|
|
|
|
):
|
2021-06-02 09:38:25 +00:00
|
|
|
email_log_id = parse_id_from_bounce(rcpt_tos[0])
|
|
|
|
email_log = EmailLog.get(email_log_id)
|
|
|
|
return handle_bounce(envelope, email_log, msg)
|
2021-01-11 13:55:55 +00:00
|
|
|
|
2021-06-02 09:46:00 +00:00
|
|
|
# iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
|
2021-06-02 14:27:48 +00:00
|
|
|
if (
|
|
|
|
len(rcpt_tos) == 1
|
|
|
|
and mail_from.startswith(BOUNCE_PREFIX)
|
|
|
|
and mail_from.endswith(BOUNCE_SUFFIX)
|
|
|
|
):
|
2021-06-02 09:46:00 +00:00
|
|
|
email_log_id = parse_id_from_bounce(mail_from)
|
|
|
|
email_log = EmailLog.get(email_log_id)
|
|
|
|
alias = Alias.get_by(email=rcpt_tos[0])
|
2021-06-04 13:23:48 +00:00
|
|
|
LOG.w(
|
2021-06-02 09:46:00 +00:00
|
|
|
"iCloud bounces %s %s msg=%s",
|
|
|
|
email_log,
|
|
|
|
alias,
|
|
|
|
msg.as_string(),
|
|
|
|
)
|
|
|
|
return handle_bounce(envelope, email_log, msg)
|
|
|
|
|
2021-09-22 07:58:40 +00:00
|
|
|
# case where From: header is a reverse alias which should never happen
|
2021-10-11 10:10:18 +00:00
|
|
|
from_header = get_header_unicode(msg[headers.FROM])
|
2021-09-22 07:58:40 +00:00
|
|
|
if from_header:
|
2021-09-22 14:39:21 +00:00
|
|
|
try:
|
|
|
|
_, from_header_address = parse_full_address(from_header)
|
|
|
|
except ValueError:
|
|
|
|
LOG.d("cannot parse the From header %s", from_header)
|
|
|
|
else:
|
2021-10-19 10:14:16 +00:00
|
|
|
if is_reverse_alias(from_header_address):
|
2021-09-22 14:39:21 +00:00
|
|
|
LOG.e("email sent from a reverse alias %s", from_header_address)
|
|
|
|
# get more info for debug
|
|
|
|
contact = Contact.get_by(reply_email=from_header_address)
|
|
|
|
if contact:
|
|
|
|
LOG.d("%s %s %s", contact.user, contact.alias, contact)
|
|
|
|
|
|
|
|
# To investigate. todo: remove
|
|
|
|
if TEMP_DIR:
|
|
|
|
file_name = str(uuid.uuid4()) + ".eml"
|
|
|
|
with open(os.path.join(TEMP_DIR, file_name), "wb") as f:
|
|
|
|
f.write(msg.as_bytes())
|
|
|
|
|
|
|
|
LOG.d("email saved to %s", file_name)
|
|
|
|
return status.E523
|
2021-09-22 07:58:40 +00:00
|
|
|
|
2021-06-24 07:47:01 +00:00
|
|
|
if rate_limited(mail_from, rcpt_tos):
|
2021-10-13 08:27:54 +00:00
|
|
|
LOG.w("Rate Limiting applied for mail_from:%s rcpt_tos:%s", mail_from, rcpt_tos)
|
|
|
|
|
|
|
|
# add more logging info. TODO: remove
|
|
|
|
if len(rcpt_tos) == 1:
|
|
|
|
alias = Alias.get_by(email=rcpt_tos[0])
|
|
|
|
if alias:
|
|
|
|
LOG.w(
|
|
|
|
"total number email log on %s, %s is %s, %s",
|
|
|
|
alias,
|
|
|
|
alias.user,
|
|
|
|
EmailLog.filter(EmailLog.alias_id == alias.id).count(),
|
|
|
|
EmailLog.filter(EmailLog.user_id == alias.user_id).count(),
|
|
|
|
)
|
2021-10-12 12:53:30 +00:00
|
|
|
|
2021-10-13 08:27:54 +00:00
|
|
|
if should_ignore_bounce(envelope.mail_from):
|
|
|
|
return status.E207
|
|
|
|
else:
|
|
|
|
return status.E522
|
2020-04-04 14:27:22 +00:00
|
|
|
|
2021-05-30 17:58:08 +00:00
|
|
|
# Handle "out of office" auto notice. An automatic response is sent for every forwarded email
|
|
|
|
# todo: remove logging
|
2021-10-19 10:14:16 +00:00
|
|
|
if len(rcpt_tos) == 1 and is_reverse_alias(rcpt_tos[0]) and mail_from == "<>":
|
2021-05-30 18:02:41 +00:00
|
|
|
LOG.w(
|
|
|
|
"out-of-office email to reverse alias %s. %s", rcpt_tos[0], msg.as_string()
|
|
|
|
)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E206
|
2021-05-30 17:58:08 +00:00
|
|
|
|
2020-04-04 14:09:24 +00:00
|
|
|
# result of all deliveries
|
|
|
|
# each element is a couple of whether the delivery is successful and the smtp status
|
|
|
|
res: [(bool, str)] = []
|
|
|
|
|
2021-05-06 15:08:30 +00:00
|
|
|
nb_rcpt_tos = len(rcpt_tos)
|
|
|
|
for rcpt_index, rcpt_to in enumerate(rcpt_tos):
|
2020-11-14 14:55:53 +00:00
|
|
|
if rcpt_to == NOREPLY:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.e("email sent to noreply address from %s", mail_from)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E514
|
2020-11-14 14:55:53 +00:00
|
|
|
|
2021-05-06 15:08:30 +00:00
|
|
|
# create a copy of msg for each recipient except the last one
|
|
|
|
# as copy() is a slow function
|
|
|
|
if rcpt_index < nb_rcpt_tos - 1:
|
|
|
|
LOG.d("copy message for rcpt %s", rcpt_to)
|
|
|
|
copy_msg = copy(msg)
|
|
|
|
else:
|
|
|
|
copy_msg = msg
|
|
|
|
|
2021-10-25 12:33:42 +00:00
|
|
|
# Reply case: the recipient is a reverse alias. Usually starts with "reply+" or "ra+"
|
2021-10-19 10:14:16 +00:00
|
|
|
if is_reverse_alias(rcpt_to):
|
2021-10-11 10:10:18 +00:00
|
|
|
LOG.d(
|
|
|
|
"Reply phase %s(%s) -> %s", mail_from, copy_msg[headers.FROM], rcpt_to
|
|
|
|
)
|
2021-05-06 15:08:30 +00:00
|
|
|
is_delivered, smtp_status = handle_reply(envelope, copy_msg, rcpt_to)
|
2020-04-04 14:09:24 +00:00
|
|
|
res.append((is_delivered, smtp_status))
|
|
|
|
else: # Forward case
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.d(
|
2020-11-04 18:42:20 +00:00
|
|
|
"Forward phase %s(%s) -> %s",
|
2020-09-14 15:19:29 +00:00
|
|
|
mail_from,
|
2021-10-11 10:10:18 +00:00
|
|
|
copy_msg[headers.FROM],
|
2020-07-23 08:32:10 +00:00
|
|
|
rcpt_to,
|
|
|
|
)
|
2021-05-06 15:08:30 +00:00
|
|
|
for is_delivered, smtp_status in handle_forward(
|
|
|
|
envelope, copy_msg, rcpt_to
|
|
|
|
):
|
2020-05-10 14:57:47 +00:00
|
|
|
res.append((is_delivered, smtp_status))
|
2020-04-04 14:09:24 +00:00
|
|
|
|
|
|
|
for (is_success, smtp_status) in res:
|
|
|
|
# Consider all deliveries successful if 1 delivery is successful
|
|
|
|
if is_success:
|
|
|
|
return smtp_status
|
|
|
|
|
|
|
|
# Failed delivery for all, return the first failure
|
|
|
|
return res[0][1]
|
|
|
|
|
|
|
|
|
2020-02-19 15:17:13 +00:00
|
|
|
class MailHandler:
|
2020-04-02 16:10:08 +00:00
|
|
|
async def handle_DATA(self, server, session, envelope: Envelope):
|
2020-08-17 09:40:58 +00:00
|
|
|
try:
|
2021-10-13 08:27:59 +00:00
|
|
|
ret = self._handle(envelope)
|
2020-08-17 09:40:58 +00:00
|
|
|
return ret
|
|
|
|
except Exception:
|
2021-06-23 17:57:21 +00:00
|
|
|
LOG.e(
|
2020-08-27 08:20:48 +00:00
|
|
|
"email handling fail %s -> %s",
|
|
|
|
envelope.mail_from,
|
|
|
|
envelope.rcpt_tos,
|
2020-08-17 09:40:58 +00:00
|
|
|
)
|
2021-06-23 17:47:06 +00:00
|
|
|
return status.E404
|
2020-08-16 09:10:01 +00:00
|
|
|
|
2021-07-23 13:48:43 +00:00
|
|
|
@newrelic.agent.background_task(application=newrelic_app)
|
2021-10-13 08:27:59 +00:00
|
|
|
def _handle(self, envelope: Envelope):
|
2020-09-30 09:05:21 +00:00
|
|
|
start = time.time()
|
2021-05-06 15:20:33 +00:00
|
|
|
|
|
|
|
# generate a different message_id to keep track of an email lifecycle
|
|
|
|
message_id = str(uuid.uuid4())
|
|
|
|
set_message_id(message_id)
|
|
|
|
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.i(
|
2020-09-30 09:05:21 +00:00
|
|
|
"===>> New message, mail from %s, rctp tos %s ",
|
|
|
|
envelope.mail_from,
|
|
|
|
envelope.rcpt_tos,
|
|
|
|
)
|
2019-12-17 19:43:31 +00:00
|
|
|
|
2021-10-26 08:52:28 +00:00
|
|
|
with create_light_app().app_context():
|
2021-10-13 08:27:59 +00:00
|
|
|
ret = handle(envelope)
|
|
|
|
elapsed = time.time() - start
|
|
|
|
LOG.i(
|
|
|
|
"Finish mail from %s, rctp tos %s, takes %s seconds <<===",
|
|
|
|
envelope.mail_from,
|
|
|
|
envelope.rcpt_tos,
|
|
|
|
elapsed,
|
|
|
|
)
|
|
|
|
newrelic.agent.record_custom_metric(
|
|
|
|
"Custom/email_handler_time", elapsed, newrelic_app
|
|
|
|
)
|
|
|
|
newrelic.agent.record_custom_metric(
|
|
|
|
"Custom/number_incoming_email", 1, newrelic_app
|
|
|
|
)
|
|
|
|
return ret
|
2019-11-07 16:49:26 +00:00
|
|
|
|
|
|
|
|
2020-09-30 09:05:21 +00:00
|
|
|
def main(port: int):
|
|
|
|
"""Use aiosmtpd Controller"""
|
|
|
|
controller = Controller(MailHandler(), hostname="0.0.0.0", port=port)
|
2020-09-02 15:36:11 +00:00
|
|
|
|
2020-09-30 09:05:21 +00:00
|
|
|
controller.start()
|
|
|
|
LOG.d("Start mail controller %s %s", controller.hostname, controller.port)
|
2020-09-02 15:36:11 +00:00
|
|
|
|
2020-09-30 09:05:21 +00:00
|
|
|
if LOAD_PGP_EMAIL_HANDLER:
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.w("LOAD PGP keys")
|
2021-10-12 12:47:01 +00:00
|
|
|
load_pgp_public_keys()
|
2020-09-30 09:05:21 +00:00
|
|
|
|
|
|
|
while True:
|
|
|
|
time.sleep(2)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument(
|
|
|
|
"-p", "--port", help="SMTP port to listen for", type=int, default=20381
|
|
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
2021-03-16 08:17:23 +00:00
|
|
|
LOG.i("Listen for port %s", args.port)
|
2020-09-30 09:05:21 +00:00
|
|
|
main(port=args.port)
|