diff --git a/app/config.py b/app/config.py index 9caf5396..a21c8dfb 100644 --- a/app/config.py +++ b/app/config.py @@ -422,6 +422,8 @@ ZENDESK_HOST = os.environ.get("ZENDESK_HOST") ZENDESK_API_TOKEN = os.environ.get("ZENDESK_API_TOKEN") ZENDESK_ENABLED = "ZENDESK_ENABLED" in os.environ +DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ + def get_allowed_redirect_domains() -> List[str]: allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list) diff --git a/app/email/headers.py b/app/email/headers.py index d639272c..3edb4dce 100644 --- a/app/email/headers.py +++ b/app/email/headers.py @@ -12,6 +12,8 @@ CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding" MIME_VERSION = "Mime-Version" REPLY_TO = "Reply-To" RECEIVED = "Received" +RSPAMD_QUEUE_ID = "X-Rspamd-Queue-Id" +SPAMD_RESULT = "X-Spamd-Result" CC = "Cc" DKIM_SIGNATURE = "DKIM-Signature" X_SPAM_STATUS = "X-Spam-Status" diff --git a/app/email_utils.py b/app/email_utils.py index 114f6b61..0673fb1a 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -70,6 +70,7 @@ from app.models import ( TransactionalEmail, IgnoreBounceSender, InvalidMailboxDomain, + DmarcCheckResult, ) from app.utils import ( random_string, @@ -1375,6 +1376,11 @@ def sl_sendmail( def get_queue_id(msg: Message) -> Optional[str]: """Get the Postfix queue-id from a message""" + header_values = msg.get_all(headers.RSPAMD_QUEUE_ID) + if header_values: + # Get last in case somebody tries to inject a header + return header_values[-1] + received_header = str(msg[headers.RECEIVED]) if not received_header: return @@ -1433,3 +1439,18 @@ def save_email_for_debugging(msg: Message, file_name_prefix=None) -> str: return file_name return "" + + +def get_dmarc_status(msg: Message) -> Optional[DmarcCheckResult]: + spam_result = msg.get_all(headers.SPAMD_RESULT) + if not spam_result: + return None + spam_entries = [entry.strip() for entry in spam_result[-1].split("\n")] + for entry_pos in range(len(spam_entries)): + sep = spam_entries[entry_pos].find("(") + if sep > -1: + spam_entries[entry_pos] = spam_entries[entry_pos][:sep] + for header_value, dmarc_result in DmarcCheckResult.get_string_dict().items(): + if header_value in spam_entries: + return dmarc_result + return None diff --git a/app/models.py b/app/models.py index 2a7cf9ca..66bfff5f 100644 --- a/app/models.py +++ b/app/models.py @@ -237,6 +237,26 @@ class AuditLogActionEnum(EnumE): extend_subscription = 7 +class DmarcCheckResult(EnumE): + allow = 0 + soft_fail = 1 + quarantine = 2 + reject = 3 + not_available = 4 + bad_policy = 5 + + @staticmethod + def get_string_dict(): + return { + "DMARC_POLICY_ALLOW": DmarcCheckResult.allow, + "DMARC_POLICY_SOFTFAIL": DmarcCheckResult.soft_fail, + "DMARC_POLICY_QUARANTINE": DmarcCheckResult.quarantine, + "DMARC_POLICY_REJECT": DmarcCheckResult.reject, + "DMARC_NA": DmarcCheckResult.not_available, + "DMARC_BAD_POLICY": DmarcCheckResult.bad_policy, + } + + class Hibp(Base, ModelMixin): __tablename__ = "hibp" name = sa.Column(sa.String(), nullable=False, unique=True, index=True) diff --git a/email_handler.py b/email_handler.py index 753d8758..af42d9e8 100644 --- a/email_handler.py +++ b/email_handler.py @@ -87,6 +87,7 @@ from app.config import ( OLD_UNSUBSCRIBER, ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS, ALERT_TO_NOREPLY, + DMARC_CHECK_ENABLED, ) from app.db import Session from app.email import status, headers @@ -128,6 +129,7 @@ from app.email_utils import ( get_orig_message_from_yahoo_complaint, get_mailbox_bounce_info, save_email_for_debugging, + get_dmarc_status, ) from app.errors import ( NonReverseAliasInReplyPhase, @@ -152,6 +154,7 @@ from app.models import ( DeletedAlias, DomainDeletedAlias, Notification, + DmarcCheckResult, ) from app.pgp_utils import PGPException, sign_data_with_pgpy, sign_data from app.utils import sanitize_email @@ -536,6 +539,83 @@ def handle_email_sent_to_ourself(alias, from_addr: str, msg: Message, user): ) +def apply_dmarc_policy( + alias: Alias, contact: Contact, envelope: Envelope, msg: Message +) -> Optional[str]: + dmarc_result = get_dmarc_status(msg) + newrelic.agent.record_custom_event( + "Custom/dmarc_check", {"result": dmarc_result.name} + ) + if not DMARC_CHECK_ENABLED: + return None + if dmarc_result in ( + DmarcCheckResult.quarantine, + DmarcCheckResult.reject, + DmarcCheckResult.soft_fail, + ): + quarantine_dmarc_failed_email(alias, contact, envelope, msg) + add_quarantine_notification_for_alias(alias) + return status.E519 + return None + + +def quarantine_dmarc_failed_email(alias, contact, envelope, msg): + add_or_replace_header(msg, headers.SL_DIRECTION, "Forward") + msg[headers.SL_ENVELOPE_TO] = alias.email + msg[headers.SL_ENVELOPE_FROM] = envelope.mail_from + add_or_replace_header(msg, "From", contact.new_addr()) + # replace CC & To emails by reverse-alias for all emails that are not alias + try: + replace_header_when_forward(msg, alias, "Cc") + replace_header_when_forward(msg, alias, "To") + except CannotCreateContactForReverseAlias: + Session.commit() + raise + random_name = str(uuid.uuid4()) + s3_report_path = f"refused-emails/full-{random_name}.eml" + s3.upload_email_from_bytesio( + s3_report_path, BytesIO(to_bytes(msg)), f"full-{random_name}" + ) + refused_email = RefusedEmail.create( + full_report_path=s3_report_path, user_id=alias.user_id, flush=True + ) + EmailLog.create( + user_id=alias.user_id, + mailbox_id=alias.mailbox_id, + contact_id=contact.id, + alias_id=alias.id, + message_id=str(msg[headers.MESSAGE_ID]), + refused_email_id=refused_email.id, + is_spam=True, + blocked=True, + commit=True, + ) + + +def add_quarantine_notification_for_alias(alias: Alias): + notification_title = f"{alias.email} has a new mail in quarantine" + notifications = ( + Notification.filter_by(user_id=alias.user_id) + .order_by(Notification.read, Notification.created_at.desc()) + .limit(10) + .all() + ) # load a record more to know whether there's more + already_notified = False + for notification in notifications: + if notification.title == notification_title: + already_notified = True + break + if not already_notified: + Notification.create( + user_id=alias.user_id, + title=notification_title, + message=Notification.render( + "notification/message-quarantine.html", alias=alias + ), + commit=True, + ) + + def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str]]: """return an array of SMTP status (is_success, smtp_status) is_success indicates whether an email has been delivered and @@ -616,6 +696,11 @@ def handle_forward(envelope, msg: Message, rcpt_to: str) -> List[Tuple[bool, str # do not return 5** to allow user to receive emails later when alias is enabled or contact is unblocked return [(True, res_status)] + # Check if we need to reject or quarantine based on dmarc + dmarc_delivery_status = apply_dmarc_policy(alias, contact, envelope, msg) + if dmarc_delivery_status is not None: + return [(False, dmarc_delivery_status)] + ret = [] mailboxes = alias.mailboxes diff --git a/templates/notification/message-quarantine.html b/templates/notification/message-quarantine.html new file mode 100644 index 00000000..4970572b --- /dev/null +++ b/templates/notification/message-quarantine.html @@ -0,0 +1,4 @@ +
+ There is a new quarantined message for {{ alias.email }}. Please go to your quarantined emails to review it. +
+ diff --git a/tests/api/test_serializer.py b/tests/api/test_serializer.py index b8e2c27a..56340bf2 100644 --- a/tests/api/test_serializer.py +++ b/tests/api/test_serializer.py @@ -2,7 +2,7 @@ from app.api.serializer import get_alias_infos_with_pagination_v3 from app.config import PAGE_LIMIT from app.db import Session from app.models import User, Alias, Mailbox, Contact -from tests.utils import create_user +from tests.utils import create_random_user def test_get_alias_infos_with_pagination_v3(flask_client): @@ -147,7 +147,7 @@ def test_get_alias_infos_with_pagination_v3_no_duplicate_when_empty_contact( """ Make sure an alias is returned once when it has 2 contacts that have no email log activity """ - user = create_user(flask_client) + user = create_random_user() alias = Alias.first() Contact.create( diff --git a/tests/example_emls/dmarc_allow.eml b/tests/example_emls/dmarc_allow.eml new file mode 100644 index 00000000..4006bef3 --- /dev/null +++ b/tests/example_emls/dmarc_allow.eml @@ -0,0 +1,27 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: {{ alias_email }} +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_ALLOW(0.10); + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] + +This is a test mailing diff --git a/tests/example_emls/dmarc_bad_policy.eml b/tests/example_emls/dmarc_bad_policy.eml new file mode 100644 index 00000000..522ef9f7 --- /dev/null +++ b/tests/example_emls/dmarc_bad_policy.eml @@ -0,0 +1,27 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: {{ alias_email }} +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_BAD_POLICY(0.10); + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] + +This is a test mailing diff --git a/tests/example_emls/dmarc_gmail_softfail.eml b/tests/example_emls/dmarc_gmail_softfail.eml new file mode 100644 index 00000000..b1d9f5e6 --- /dev/null +++ b/tests/example_emls/dmarc_gmail_softfail.eml @@ -0,0 +1,41 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: {{ alias_email }} +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_SOFTFAIL(0.10)[gmail.com : No valid SPF, No valid DKIM,none]; + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + FROM_EQ_ENVFROM(0.00)[]; + ASN(0.00)[asn:16276, ipnet:34.59.0.0/16, country:FR]; + R_DKIM_NA(0.00)[]; + RCVD_COUNT_ZERO(0.00)[0]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + FROM_NO_DN(0.00)[]; + R_SPF_SOFTFAIL(0.00)[~all]; + FORCE_ACTION_SL_SPF_FAIL_ADD_HEADER(0.00)[add header]; + RCPT_COUNT_ONE(0.00)[1]; + FREEMAIL_FROM(0.00)[gmail.com]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] +X-Rspamd-Pre-Result: action=add header; + module=force_actions; + unknown reason +X-Spam: Yes + +This is a test mailing diff --git a/tests/example_emls/dmarc_na.eml b/tests/example_emls/dmarc_na.eml new file mode 100644 index 00000000..c5eb71da --- /dev/null +++ b/tests/example_emls/dmarc_na.eml @@ -0,0 +1,27 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: {{ alias_email }} +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_NA(0.10); + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] + +This is a test mailing diff --git a/tests/example_emls/dmarc_quarantine.eml b/tests/example_emls/dmarc_quarantine.eml new file mode 100644 index 00000000..129b07e9 --- /dev/null +++ b/tests/example_emls/dmarc_quarantine.eml @@ -0,0 +1,41 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: {{ alias_email }} +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_QUARANTINE(0.10)[gmail.com : No valid SPF, No valid DKIM,none]; + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + FROM_EQ_ENVFROM(0.00)[]; + ASN(0.00)[asn:16276, ipnet:34.59.0.0/16, country:FR]; + R_DKIM_NA(0.00)[]; + RCVD_COUNT_ZERO(0.00)[0]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + FROM_NO_DN(0.00)[]; + R_SPF_SOFTFAIL(0.00)[~all]; + FORCE_ACTION_SL_SPF_FAIL_ADD_HEADER(0.00)[add header]; + RCPT_COUNT_ONE(0.00)[1]; + FREEMAIL_FROM(0.00)[gmail.com]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] +X-Rspamd-Pre-Result: action=add header; + module=force_actions; + unknown reason +X-Spam: Yes + +This is a test mailing diff --git a/tests/example_emls/dmarc_reject.eml b/tests/example_emls/dmarc_reject.eml new file mode 100644 index 00000000..4807b1f2 --- /dev/null +++ b/tests/example_emls/dmarc_reject.eml @@ -0,0 +1,41 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: {{ alias_email }} +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_REJECT(0.10)[gmail.com : No valid SPF, No valid DKIM,none]; + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + FROM_EQ_ENVFROM(0.00)[]; + ASN(0.00)[asn:16276, ipnet:34.59.0.0/16, country:FR]; + R_DKIM_NA(0.00)[]; + RCVD_COUNT_ZERO(0.00)[0]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + FROM_NO_DN(0.00)[]; + R_SPF_SOFTFAIL(0.00)[~all]; + FORCE_ACTION_SL_SPF_FAIL_ADD_HEADER(0.00)[add header]; + RCPT_COUNT_ONE(0.00)[1]; + FREEMAIL_FROM(0.00)[gmail.com]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] +X-Rspamd-Pre-Result: action=add header; + module=force_actions; + unknown reason +X-Spam: Yes + +This is a test mailing diff --git a/tests/example_emls/double_queue_id_header.eml b/tests/example_emls/double_queue_id_header.eml new file mode 100644 index 00000000..1a576750 --- /dev/null +++ b/tests/example_emls/double_queue_id_header.eml @@ -0,0 +1,42 @@ +X-SimpleLogin-Client-IP: 54.39.200.130 +Received-SPF: Softfail (mailfrom) identity=mailfrom; client-ip=34.59.200.130; + helo=relay.somewhere.net; envelope-from=everwaste@gmail.com; + receiver= +Received: from relay.somewhere.net (relay.somewhere.net [34.59.200.130]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mx1.sldev.ovh (Postfix) with ESMTPS id 6D8C13F069 + for ; Thu, 17 Mar 2022 16:50:20 +0000 (UTC) +Date: Thu, 17 Mar 2022 16:50:18 +0000 +To: wehrman_mannequin@sldev.ovh +From: spoofedemailsource@gmail.com +Subject: test Thu, 17 Mar 2022 16:50:18 +0000 +Message-Id: <20220317165018.000191@somewhere-5488dd4b6b-7crp6> +X-Mailer: swaks v20201014.0 jetmore.org/john/code/swaks/ +X-Rspamd-Queue-Id: INVALIDVALUE +X-Rspamd-Queue-Id: 6D8C13F069 +X-Rspamd-Server: staging1 +X-Spamd-Result: default: False [0.50 / 13.00]; + MID_RHS_NOT_FQDN(0.50)[]; + DMARC_POLICY_SOFTFAIL(0.10)[gmail.com : No valid SPF, No valid DKIM,none]; + MIME_GOOD(-0.10)[text/plain]; + MIME_TRACE(0.00)[0:+]; + FROM_EQ_ENVFROM(0.00)[]; + ASN(0.00)[asn:16276, ipnet:34.59.0.0/16, country:FR]; + R_DKIM_NA(0.00)[]; + RCVD_COUNT_ZERO(0.00)[0]; + FREEMAIL_ENVFROM(0.00)[gmail.com]; + FROM_NO_DN(0.00)[]; + R_SPF_SOFTFAIL(0.00)[~all]; + FORCE_ACTION_SL_SPF_FAIL_ADD_HEADER(0.00)[add header]; + RCPT_COUNT_ONE(0.00)[1]; + FREEMAIL_FROM(0.00)[gmail.com]; + TO_DN_NONE(0.00)[]; + TO_MATCH_ENVRCPT_ALL(0.00)[]; + ARC_NA(0.00)[] +X-Rspamd-Pre-Result: action=add header; + module=force_actions; + unknown reason +X-Spam: Yes + +This is a test mailing diff --git a/tests/test.env b/tests/test.env index 06f3d5d2..ae3dff84 100644 --- a/tests/test.env +++ b/tests/test.env @@ -52,4 +52,6 @@ FACEBOOK_CLIENT_SECRET=to_fill PGP_SENDER_PRIVATE_KEY_PATH=local_data/private-pgp.asc ALIAS_AUTOMATIC_DISABLE=true -ALLOWED_REDIRECT_DOMAINS=["test.simplelogin.local"] \ No newline at end of file +ALLOWED_REDIRECT_DOMAINS=["test.simplelogin.local"] + +DMARC_CHECK_ENABLED=true \ No newline at end of file diff --git a/tests/test_email_handler.py b/tests/test_email_handler.py index bea243ba..5f88c8b9 100644 --- a/tests/test_email_handler.py +++ b/tests/test_email_handler.py @@ -1,12 +1,23 @@ from email.message import EmailMessage -from app.email import headers -from app.models import User, Alias, AuthorizedAddress, IgnoredEmail +from aiosmtpd.smtp import Envelope + +import email_handler +from app.email import headers, status +from app.models import ( + User, + Alias, + AuthorizedAddress, + IgnoredEmail, + EmailLog, + Notification, +) from email_handler import ( get_mailbox_from_mail_from, should_ignore, is_automatic_out_of_office, ) +from tests.utils import load_eml_file, create_random_user def test_get_mailbox_from_mail_from(flask_client): @@ -61,3 +72,46 @@ def test_is_automatic_out_of_office(): msg[headers.AUTO_SUBMITTED] = "auto-generated" assert is_automatic_out_of_office(msg) + + +def test_dmarc_quarantine(flask_client): + user = create_random_user() + alias = Alias.create_new_random(user) + msg = load_eml_file("dmarc_quarantine.eml", {"alias_email": alias.email}) + envelope = Envelope() + envelope.mail_from = msg["from"] + envelope.rcpt_tos = [msg["to"]] + result = email_handler.handle(envelope, msg) + assert result == status.E519 + email_logs = ( + EmailLog.filter_by(user_id=user.id, alias_id=alias.id) + .order_by(EmailLog.id.desc()) + .all() + ) + assert len(email_logs) == 1 + email_log = email_logs[0] + assert email_log.blocked + assert email_log.refused_email_id + notifications = Notification.filter_by(user_id=user.id).all() + assert len(notifications) == 1 + assert f"{alias.email} has a new mail in quarantine" == notifications[0].title + + +def test_gmail_dmarc_softfail(flask_client): + user = create_random_user() + alias = Alias.create_new_random(user) + msg = load_eml_file("dmarc_gmail_softfail.eml", {"alias_email": alias.email}) + envelope = Envelope() + envelope.mail_from = msg["from"] + envelope.rcpt_tos = [msg["to"]] + result = email_handler.handle(envelope, msg) + assert result == status.E519 + email_logs = ( + EmailLog.filter_by(user_id=user.id, alias_id=alias.id) + .order_by(EmailLog.id.desc()) + .all() + ) + assert len(email_logs) == 1 + email_log = email_logs[0] + assert email_log.blocked + assert email_log.refused_email_id diff --git a/tests/test_email_utils.py b/tests/test_email_utils.py index dc7c41f5..466f860c 100644 --- a/tests/test_email_utils.py +++ b/tests/test_email_utils.py @@ -36,6 +36,7 @@ from app.email_utils import ( get_orig_message_from_bounce, get_mailbox_bounce_info, is_invalid_mailbox_domain, + get_dmarc_status, ) from app.models import ( User, @@ -45,10 +46,11 @@ from app.models import ( EmailLog, IgnoreBounceSender, InvalidMailboxDomain, + DmarcCheckResult, ) # flake8: noqa: E101, W191 -from tests.utils import login +from tests.utils import login, load_eml_file def test_get_email_domain_part(): @@ -748,6 +750,11 @@ def test_get_queue_id(): assert get_queue_id(msg) == "4FxQmw1DXdz2vK2" +def test_get_queue_id_from_double_header(): + msg = load_eml_file("double_queue_id_header.eml") + assert get_queue_id(msg) == "6D8C13F069" + + def test_should_ignore_bounce(flask_client): assert not should_ignore_bounce("not-exist") @@ -786,3 +793,33 @@ def test_is_invalid_mailbox_domain(flask_client): assert is_invalid_mailbox_domain("sub1.sub2.ab.cd") assert not is_invalid_mailbox_domain("xy.zt") + + +def test_dmarc_result_softfail(): + msg = load_eml_file("dmarc_gmail_softfail.eml") + assert DmarcCheckResult.soft_fail == get_dmarc_status(msg) + + +def test_dmarc_result_quarantine(): + msg = load_eml_file("dmarc_quarantine.eml") + assert DmarcCheckResult.quarantine == get_dmarc_status(msg) + + +def test_dmarc_result_reject(): + msg = load_eml_file("dmarc_reject.eml") + assert DmarcCheckResult.reject == get_dmarc_status(msg) + + +def test_dmarc_result_allow(): + msg = load_eml_file("dmarc_allow.eml") + assert DmarcCheckResult.allow == get_dmarc_status(msg) + + +def test_dmarc_result_na(): + msg = load_eml_file("dmarc_na.eml") + assert DmarcCheckResult.not_available == get_dmarc_status(msg) + + +def test_dmarc_result_bad_policy(): + msg = load_eml_file("dmarc_bad_policy.eml") + assert DmarcCheckResult.bad_policy == get_dmarc_status(msg) diff --git a/tests/utils.py b/tests/utils.py index 5de1a4a4..3621d9f5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,12 @@ +import email import json +import os +import random +import string +from email.message import EmailMessage +from typing import Optional, Dict +import jinja2 from flask import url_for from app.models import User @@ -27,10 +34,14 @@ def login(flask_client) -> User: return user -def create_user(flask_client) -> User: - # create user, user is activated +def random_token(length: int = 10) -> str: + return "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) + + +def create_random_user() -> User: + email = "{}@{}.com".format(random_token(), random_token()) return User.create( - email="a@b.c", + email=email, password="password", name="Test User", activated=True, @@ -41,3 +52,18 @@ def create_user(flask_client) -> User: def pretty(d): """pretty print as json""" print(json.dumps(d, indent=2)) + + +def load_eml_file( + filename: str, template_values: Optional[Dict[str, str]] = None +) -> EmailMessage: + emails_dir = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "example_emls" + ) + fullpath = os.path.join(emails_dir, filename) + with open(fullpath) as fd: + template = jinja2.Template(fd.read()) + if not template_values: + template_values = {} + rendered = template.render(**template_values) + return email.message_from_string(rendered)