From c573ef655e9036523b949a38486a82495c389731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 14 Apr 2022 18:46:11 +0200 Subject: [PATCH 01/12] Store bounces in the reply phase to prevent abuse --- app/admin_model.py | 86 ++++++- app/config.py | 7 +- app/email_utils.py | 24 -- app/handler/spamd_result.py | 8 +- app/handler/transactional_complaint.py | 229 ++++++++++++++++++ app/models.py | 37 ++- app/s3.py | 18 ++ email_handler.py | 197 +-------------- ...475_store_transactional_complaints_for_.py | 36 +++ server.py | 3 + .../hotmail-complaint-reply-phase.txt.jinja2 | 15 -- .../transactional/hotmail-complaint.html | 41 ---- .../hotmail-complaint.txt.jinja2 | 20 -- .../hotmail-transactional-complaint.html | 42 ---- ...hotmail-transactional-complaint.txt.jinja2 | 23 -- ...ransactional-complaint-forward-phase.html} | 9 +- ...tional-complaint-forward-phase.txt.jinja2} | 6 +- ...sactional-complaint-reply-phase.txt.jinja2 | 15 ++ ...l => transactional-complaint-to-user.html} | 2 +- ...ransactional-complaint-to-user.txt.jinja2} | 2 +- .../handler/test_transactional_complaints.py | 92 +++++++ tests/utils.py | 8 +- 22 files changed, 531 insertions(+), 389 deletions(-) create mode 100644 app/handler/transactional_complaint.py create mode 100644 migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py delete mode 100644 templates/emails/transactional/hotmail-complaint-reply-phase.txt.jinja2 delete mode 100644 templates/emails/transactional/hotmail-complaint.html delete mode 100644 templates/emails/transactional/hotmail-complaint.txt.jinja2 delete mode 100644 templates/emails/transactional/hotmail-transactional-complaint.html delete mode 100644 templates/emails/transactional/hotmail-transactional-complaint.txt.jinja2 rename templates/emails/transactional/{yahoo-transactional-complaint.html => transactional-complaint-forward-phase.html} (73%) rename templates/emails/transactional/{yahoo-transactional-complaint.txt.jinja2 => transactional-complaint-forward-phase.txt.jinja2} (70%) create mode 100644 templates/emails/transactional/transactional-complaint-reply-phase.txt.jinja2 rename templates/emails/transactional/{yahoo-complaint.html => transactional-complaint-to-user.html} (88%) rename templates/emails/transactional/{yahoo-complaint.txt.jinja2 => transactional-complaint-to-user.txt.jinja2} (84%) create mode 100644 tests/handler/test_transactional_complaints.py diff --git a/app/admin_model.py b/app/admin_model.py index 9d6b8e19..17f28133 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -1,8 +1,12 @@ +from typing import Optional + import arrow import sqlalchemy +from flask_admin.model.template import EndpointLinkRowAction +from markupsafe import Markup -from app import models -from flask import redirect, url_for, request, flash +from app import models, s3 +from flask import redirect, url_for, request, flash, Response from flask_admin import expose, AdminIndexView from flask_admin.actions import action from flask_admin.contrib import sqla @@ -17,6 +21,9 @@ from app.models import ( AppleSubscription, AdminAuditLog, AuditLogActionEnum, + TransactionalComplaintState, + Phase, + TransactionalComplaint, ) @@ -366,3 +373,78 @@ class AdminAuditLogAdmin(SLModelView): "action": _admin_action_formatter, "created_at": _admin_created_at_formatter, } + + +def _transactionalcomplaint_state_formatter(view, context, model, name): + return "{} ({})".format(TransactionalComplaintState(model.state).name, model.state) + + +def _transactionalcomplaint_phase_formatter(view, context, model, name): + return Phase(model.phase).name + + +def _transactionalcomplaint_refused_email_id_formatter(view, context, model, name): + markupstring = "{}".format( + url_for(".download_eml", id=model.id), model.refused_email.full_report_path + ) + return Markup(markupstring) + + +class TransactionalComplaintAdmin(SLModelView): + column_searchable_list = ["id", "user.id", "created_at"] + column_filters = ["user.id", "state"] + column_hide_backrefs = False + can_edit = False + can_create = False + can_delete = False + + column_formatters = { + "created_at": _admin_created_at_formatter, + "updated_at": _admin_created_at_formatter, + "state": _transactionalcomplaint_state_formatter, + "phase": _transactionalcomplaint_phase_formatter, + "refused_email": _transactionalcomplaint_refused_email_id_formatter, + } + + column_extra_row_actions = [ # Add a new action button + EndpointLinkRowAction("fa fa-check-square", ".mark_ok"), + ] + + def _get_complaint(self) -> Optional[TransactionalComplaint]: + complain_id = request.args.get("id") + if complain_id is None: + flash("Missing id", "error") + return None + complaint = TransactionalComplaint.get_by(id=complain_id) + if not complaint: + flash("Could not find complaint", "error") + return None + return complaint + + @expose("/mark_ok", methods=["GET"]) + def mark_ok(self): + complaint = self._get_complaint() + if not complaint: + return redirect("/admin/transactionalcomplaint/") + complaint.state = TransactionalComplaintState.reviewed.value + Session.commit() + return redirect("/admin/transactionalcomplaint/") + + @expose("/download_eml", methods=["GET"]) + def download_eml(self): + complaint = self._get_complaint() + if not complaint: + return redirect("/admin/transactionalcomplaint/") + eml_path = complaint.refused_email.full_report_path + eml_data = s3.download_email(eml_path) + AdminAuditLog.downloaded_transactional_complaint(current_user.id, complaint.id) + Session.commit() + return Response( + eml_data, + mimetype="message/rfc822", + headers={ + "Content-Disposition": "attachment;filename={}".format( + complaint.refused_email.path + ) + }, + ) diff --git a/app/config.py b/app/config.py index 475a661e..9fcbdb70 100644 --- a/app/config.py +++ b/app/config.py @@ -336,10 +336,9 @@ AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN = "custom_domain_mx_record_issue" # alert when a new alias is about to be created on a disabled directory ALERT_DIRECTORY_DISABLED_ALIAS_CREATION = "alert_directory_disabled_alias_creation" -ALERT_HOTMAIL_COMPLAINT = "alert_hotmail_complaint" -ALERT_HOTMAIL_COMPLAINT_REPLY_PHASE = "alert_hotmail_complaint_reply_phase" -ALERT_HOTMAIL_COMPLAINT_TRANSACTIONAL = "alert_hotmail_complaint_transactional" -ALERT_YAHOO_COMPLAINT = "alert_yahoo_complaint" +ALERT_COMPLAINT_REPLY_PHASE = "alert_complaint_reply_phase" +ALERT_COMPLAINT_FORWARD_PHASE = "alert_complaint_forward_phase" +ALERT_COMPLAINT_TO_USER = "alert_complaint_to_user" ALERT_QUARANTINE_DMARC = "alert_quarantine_dmarc" diff --git a/app/email_utils.py b/app/email_utils.py index c837077b..0561e399 100644 --- a/app/email_utils.py +++ b/app/email_utils.py @@ -699,30 +699,6 @@ def get_mailbox_bounce_info(bounce_report: Message) -> Optional[Message]: return part -def get_orig_message_from_hotmail_complaint(msg: Message) -> Optional[Message]: - i = 0 - for part in msg.walk(): - i += 1 - - # 1st part is the container - # 2nd part is the empty body - # 3rd is original message - if i == 3: - return part - - -def get_orig_message_from_yahoo_complaint(msg: Message) -> Optional[Message]: - i = 0 - for part in msg.walk(): - i += 1 - - # 1st part is the container - # 2nd part is the empty body - # 6th is original message - if i == 6: - return part - - def get_header_from_bounce(msg: Message, header: str) -> str: """using regex to get header value from bounce message get_orig_message_from_bounce is better. This should be the last option diff --git a/app/handler/spamd_result.py b/app/handler/spamd_result.py index 57cc250b..e3bd3cb6 100644 --- a/app/handler/spamd_result.py +++ b/app/handler/spamd_result.py @@ -4,16 +4,10 @@ from typing import Dict, Optional import newrelic from app.email import headers -from app.models import EnumE +from app.models import EnumE, Phase from email.message import Message -class Phase(EnumE): - unknown = 0 - forward = 1 - reply = 2 - - class DmarcCheckResult(EnumE): allow = 0 soft_fail = 1 diff --git a/app/handler/transactional_complaint.py b/app/handler/transactional_complaint.py new file mode 100644 index 00000000..ad0bd105 --- /dev/null +++ b/app/handler/transactional_complaint.py @@ -0,0 +1,229 @@ +import uuid +from abc import ABC, abstractmethod +from io import BytesIO +from mailbox import Message +from typing import Optional + +from app import s3 +from app.config import ( + ALERT_COMPLAINT_REPLY_PHASE, + ALERT_COMPLAINT_TO_USER, + ALERT_COMPLAINT_FORWARD_PHASE, +) +from app.email import headers +from app.email_utils import ( + get_header_unicode, + parse_full_address, + save_email_for_debugging, + to_bytes, + render, + send_email_with_rate_control, +) +from app.log import LOG +from app.models import ( + User, + Alias, + DeletedAlias, + DomainDeletedAlias, + Contact, + TransactionalComplaint, + Phase, + TransactionalComplaintState, + RefusedEmail, +) + + +class TransactionalComplaintOrigin(ABC): + @classmethod + @abstractmethod + def get_original_message(cls, message: Message) -> Optional[Message]: + pass + + @classmethod + @abstractmethod + def name(cls): + pass + + +class TransactionalYahooOrigin(TransactionalComplaintOrigin): + @classmethod + def get_original_message(cls, message: Message) -> Optional[Message]: + wanted_part = 6 + for part in message.walk(): + wanted_part -= 1 + if wanted_part == 0: + return part + return None + + @classmethod + def name(cls): + return "yahoo" + + +class TransactionalHotmailOrigin(TransactionalComplaintOrigin): + @classmethod + def get_original_message(cls, message: Message) -> Optional[Message]: + wanted_part = 3 + for part in message.walk(): + wanted_part -= 1 + if wanted_part == 0: + return part + return None + + @classmethod + def name(cls): + return "hotmail" + + +def handle_hotmail_complaint(message: Message) -> bool: + return handle_complaint(message, TransactionalHotmailOrigin()) + + +def handle_yahoo_complaint(message: Message) -> bool: + return handle_complaint(message, TransactionalYahooOrigin()) + + +def find_alias_with_address(address: str) -> Optional[Alias]: + return ( + Alias.get_by(email=address) + or DeletedAlias.get_by(email=address) + or DomainDeletedAlias.get_by(email=address) + ) + + +def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> bool: + original_message = origin.get_original_message(message) + + try: + _, to_address = parse_full_address( + get_header_unicode(original_message[headers.TO]) + ) + _, from_address = parse_full_address( + get_header_unicode(original_message[headers.FROM]) + ) + except ValueError: + saved_file = save_email_for_debugging(message, "FromParseFailed") + LOG.w("Cannot parse from header. Saved to {}".format(saved_file or "nowhere")) + return True + + user = User.get_by(email=to_address) + if user: + LOG.d("Handle transactional {} complaint for {}".format(origin.name(), user)) + report_complaint_to_user(user, origin) + return True + + alias = find_alias_with_address(from_address) + # the email is during a reply phase, from=alias and to=destination + if alias: + LOG.i( + "Complaint from {} during reply phase {} -> {}, {}".format( + origin.name(), alias, to_address, user + ) + ) + report_complaint_to_user_in_reply_phase(alias, to_address, origin) + store_transactional_complaint(alias, message) + return True + + contact = Contact.get_by(reply_email=from_address) + if contact: + alias = contact.alias + else: + alias = find_alias_with_address(to_address) + + if not alias: + LOG.e( + f"Cannot find alias from address {to_address} or contact with reply {from_address}" + ) + return False + + report_complaint_to_user_in_forward_phase(alias, origin) + return True + + +def report_complaint_to_user_in_reply_phase( + alias: Alias, to_address: str, origin: TransactionalComplaintOrigin +): + capitalized_name = origin.name().capitalize() + send_email_with_rate_control( + alias.user, + f"{ALERT_COMPLAINT_REPLY_PHASE}_{origin.name()}", + alias.user.email, + f"Abuse report from {capitalized_name}", + render( + "transactional/transactional-complaint-reply-phase.txt.jinja2", + user=alias.user, + alias=alias, + destination=to_address, + provider=capitalized_name, + ), + max_nb_alert=1, + nb_day=7, + ) + + +def report_complaint_to_user(user: User, origin: TransactionalComplaintOrigin): + capitalized_name = origin.name().capitalize() + send_email_with_rate_control( + user, + f"{ALERT_COMPLAINT_TO_USER}_{origin.name()}", + user.email, + f"Abuse report from {capitalized_name}", + render( + "transactional/transactional-complaint-to-user.txt.jinja2", + user=user, + provider=capitalized_name, + ), + render( + "transactional/transactional-complaint-to-user.html", + user=user, + provider=capitalized_name, + ), + max_nb_alert=1, + nb_day=7, + ) + + +def report_complaint_to_user_in_forward_phase( + alias: Alias, origin: TransactionalComplaintOrigin +): + capitalized_name = origin.name().capitalize() + user = alias.user + send_email_with_rate_control( + user, + f"{ALERT_COMPLAINT_FORWARD_PHASE}_{origin.name()}", + user.email, + f"Abuse report from {capitalized_name}", + render( + "transactional/transactional-complaint-forward-phase.txt.jinja2", + user=user, + provider=capitalized_name, + ), + render( + "transactional/transactional-complaint-forward-phase.html", + user=user, + provider=capitalized_name, + ), + max_nb_alert=1, + nb_day=7, + ) + + +def store_transactional_complaint(alias, message): + email_name = f"reply-{uuid.uuid4().hex}.eml" + full_report_path = f"transactional_complaint/{email_name}" + s3.upload_email_from_bytesio( + full_report_path, BytesIO(to_bytes(message)), email_name + ) + refused_email = RefusedEmail.create( + full_report_path=full_report_path, + user_id=alias.user_id, + path=email_name, + commit=True, + ) + TransactionalComplaint.create( + user_id=alias.user_id, + state=TransactionalComplaintState.new.value, + phase=Phase.reply.value, + refused_email_id=refused_email.id, + commit=True, + ) diff --git a/app/models.py b/app/models.py index ed30ca52..67109903 100644 --- a/app/models.py +++ b/app/models.py @@ -235,6 +235,13 @@ class AuditLogActionEnum(EnumE): disable_2fa = 5 logged_as_user = 6 extend_subscription = 7 + download_transactional_complaint = 8 + + +class Phase(EnumE): + unknown = 0 + forward = 1 + reply = 2 class VerpType(EnumE): @@ -2967,6 +2974,7 @@ class AdminAuditLog(Base): action=AuditLogActionEnum.logged_as_user.value, model="User", model_id=user_id, + data={}, ) @classmethod @@ -2988,5 +2996,32 @@ class AdminAuditLog(Base): }, ) + @classmethod + def downloaded_transactional_complaint(cls, admin_user_id: int, complaint_id: int): + cls.create( + admin_user_id=admin_user_id, + action=AuditLogActionEnum.download_transactional_complaint.value, + model="TransactionalComplaint", + model_id=complaint_id, + data={}, + ) -# endregion + +class TransactionalComplaintState(EnumE): + new = 0 + reviewed = 1 + + +class TransactionalComplaint(Base, ModelMixin): + __tablename__ = "transactional_complaint" + + user_id = sa.Column(sa.ForeignKey("users.id"), nullable=False) + state = sa.Column(sa.Integer, nullable=False) + phase = sa.Column(sa.Integer, nullable=False) + # Point to the email that has been refused + refused_email_id = sa.Column( + sa.ForeignKey("refused_email.id", ondelete="cascade"), nullable=True + ) + + user = orm.relationship(User, foreign_keys=[user_id]) + refused_email = orm.relationship(RefusedEmail, foreign_keys=[refused_email_id]) diff --git a/app/s3.py b/app/s3.py index 6c4ae2d8..5a639992 100644 --- a/app/s3.py +++ b/app/s3.py @@ -1,5 +1,6 @@ import os from io import BytesIO +from typing import Optional import boto3 import requests @@ -61,6 +62,23 @@ def upload_email_from_bytesio(path: str, bs: BytesIO, filename): ) +def download_email(path: str) -> Optional[str]: + if LOCAL_FILE_UPLOAD: + file_path = os.path.join(UPLOAD_DIR, path) + with open(file_path, "rb") as f: + return f.read() + resp = ( + _session.resource("s3") + .Bucket(BUCKET) + .get_object( + Key=path, + ) + ) + if not resp or "Body" not in resp: + return None + return resp["Body"].read + + def upload_from_url(url: str, upload_path): r = requests.get(url) upload_from_bytesio(upload_path, BytesIO(r.content)) diff --git a/email_handler.py b/email_handler.py index 25be0ba1..d64dce45 100644 --- a/email_handler.py +++ b/email_handler.py @@ -79,10 +79,6 @@ from app.config import ( ENABLE_SPAM_ASSASSIN, BOUNCE_PREFIX_FOR_REPLY_PHASE, POSTMASTER, - ALERT_HOTMAIL_COMPLAINT, - ALERT_YAHOO_COMPLAINT, - ALERT_HOTMAIL_COMPLAINT_TRANSACTIONAL, - ALERT_HOTMAIL_COMPLAINT_REPLY_PHASE, OLD_UNSUBSCRIBER, ALERT_FROM_ADDRESS_IS_REVERSE_ALIAS, ALERT_TO_NOREPLY, @@ -122,9 +118,7 @@ from app.email_utils import ( sanitize_header, get_queue_id, should_ignore_bounce, - get_orig_message_from_hotmail_complaint, parse_full_address, - get_orig_message_from_yahoo_complaint, get_mailbox_bounce_info, save_email_for_debugging, save_envelope_for_debugging, @@ -146,6 +140,10 @@ from app.handler.spamd_result import ( SpamdResult, SPFCheckResult, ) +from app.handler.transactional_complaint import ( + handle_hotmail_complaint, + handle_yahoo_complaint, +) from app.log import LOG, set_message_id from app.models import ( Alias, @@ -159,8 +157,6 @@ from app.models import ( TransactionalEmail, IgnoredEmail, MessageIDMatching, - DeletedAlias, - DomainDeletedAlias, Notification, VerpType, ) @@ -1518,191 +1514,6 @@ def handle_bounce_forward_phase(msg: Message, email_log: EmailLog): ) -def handle_hotmail_complaint(msg: Message) -> bool: - """ - Handle hotmail complaint sent to postmaster - Return True if the complaint can be handled, False otherwise - """ - orig_msg = get_orig_message_from_hotmail_complaint(msg) - to_header = orig_msg[headers.TO] - from_header = orig_msg[headers.FROM] - - user = User.get_by(email=to_header) - if user: - LOG.d("Handle transactional hotmail complaint for %s", user) - handle_hotmail_complain_for_transactional_email(user) - return True - - try: - _, from_address = parse_full_address(get_header_unicode(from_header)) - alias = Alias.get_by(email=from_address) - - # the email is during a reply phase, from=alias and to=destination - if alias: - user = alias.user - LOG.i( - "Hotmail complaint during reply phase %s -> %s, %s", - alias, - to_header, - user, - ) - send_email_with_rate_control( - user, - ALERT_HOTMAIL_COMPLAINT_REPLY_PHASE, - user.email, - f"Hotmail abuse report", - render( - "transactional/hotmail-complaint-reply-phase.txt.jinja2", - user=user, - alias=alias, - destination=to_header, - ), - max_nb_alert=1, - nb_day=7, - ) - return True - - except ValueError: - LOG.w("Cannot parse %s", from_header) - - alias = None - - # try parsing the from_header which might contain the reverse alias - try: - _, reverse_alias = parse_full_address(get_header_unicode(from_header)) - contact = Contact.get_by(reply_email=reverse_alias) - if contact: - alias = contact.alias - LOG.d("find %s through %s", alias, contact) - else: - LOG.d("No contact found for %s", reverse_alias) - except ValueError: - LOG.w("Cannot parse %s", from_header) - - # try parsing the to_header which is usually the alias - if not alias: - try: - _, alias_address = parse_full_address(get_header_unicode(to_header)) - except ValueError: - LOG.w("Cannot parse %s", to_header) - else: - alias = Alias.get_by(email=alias_address) - if not alias: - if DeletedAlias.get_by( - email=alias_address - ) or DomainDeletedAlias.get_by(email=alias_address): - LOG.w("Alias %s is deleted", alias_address) - return True - - if not alias: - LOG.e( - "Cannot parse alias from to header %s and from header %s", - to_header, - from_header, - ) - return False - - user = alias.user - LOG.d("Handle hotmail complaint for %s %s %s", alias, user, alias.mailboxes) - - send_email_with_rate_control( - user, - ALERT_HOTMAIL_COMPLAINT, - user.email, - f"Hotmail abuse report", - render( - "transactional/hotmail-complaint.txt.jinja2", - alias=alias, - ), - render( - "transactional/hotmail-complaint.html", - alias=alias, - ), - max_nb_alert=1, - nb_day=7, - ) - - return True - - -def handle_hotmail_complain_for_transactional_email(user): - """Handle the case when a transactional email is set as Spam by user or by HotMail""" - send_email_with_rate_control( - user, - ALERT_HOTMAIL_COMPLAINT_TRANSACTIONAL, - user.email, - f"Hotmail abuse report", - render("transactional/hotmail-transactional-complaint.txt.jinja2", user=user), - render("transactional/hotmail-transactional-complaint.html", user=user), - max_nb_alert=1, - nb_day=7, - ) - - return True - - -def handle_yahoo_complaint(msg: Message) -> bool: - """ - Handle yahoo complaint sent to postmaster - Return True if the complaint can be handled, False otherwise - """ - orig_msg = get_orig_message_from_yahoo_complaint(msg) - to_header = orig_msg[headers.TO] - if not to_header: - LOG.e("cannot find the alias") - return False - - user = User.get_by(email=to_header) - if user: - LOG.d("Handle transactional yahoo complaint for %s", user) - handle_yahoo_complain_for_transactional_email(user) - return True - - _, alias_address = parse_full_address(get_header_unicode(to_header)) - alias = Alias.get_by(email=alias_address) - - if not alias: - LOG.w("No alias for %s", alias_address) - return False - - user = alias.user - LOG.w("Handle yahoo complaint for %s %s %s", alias, user, alias.mailboxes) - - send_email_with_rate_control( - user, - ALERT_YAHOO_COMPLAINT, - user.email, - f"Yahoo abuse report", - render( - "transactional/yahoo-complaint.txt.jinja2", - alias=alias, - ), - render( - "transactional/yahoo-complaint.html", - alias=alias, - ), - max_nb_alert=2, - ) - - return True - - -def handle_yahoo_complain_for_transactional_email(user): - """Handle the case when a transactional email is set as Spam by user or by Yahoo""" - send_email_with_rate_control( - user, - ALERT_YAHOO_COMPLAINT, - user.email, - f"Yahoo abuse report", - render("transactional/yahoo-transactional-complaint.txt.jinja2", user=user), - render("transactional/yahoo-transactional-complaint.html", user=user), - max_nb_alert=1, - nb_day=7, - ) - - return True - - def handle_bounce_reply_phase(envelope, msg: Message, email_log: EmailLog): """ Handle reply phase bounce diff --git a/migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py b/migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py new file mode 100644 index 00000000..5c6a9a61 --- /dev/null +++ b/migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py @@ -0,0 +1,36 @@ +"""Store transactional complaints for admins to verify + +Revision ID: 45588d9bb475 +Revises: b500363567e3 +Create Date: 2022-04-19 16:17:42.798792 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '45588d9bb475' +down_revision = 'b500363567e3' +branch_labels = None +depends_on = None + +def upgrade(): + op.create_table( + "transactional_complaint", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("created_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), + sa.Column("updated_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), + sa.Column("user_id", sa.Integer, nullable=False), + sa.Column("state", sa.Integer, nullable=False), + sa.Column("phase", sa.Integer, nullable=False), + sa.Column("refused_email_id", sa.Integer, nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'), + sa.ForeignKeyConstraint(['refused_email_id'], ['refused_email.id'], ondelete='cascade'), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade(): + op.drop_table("transactional_complaint") diff --git a/server.py b/server.py index d05308c1..8b7106af 100644 --- a/server.py +++ b/server.py @@ -37,6 +37,7 @@ from app.admin_model import ( CouponAdmin, CustomDomainAdmin, AdminAuditLogAdmin, + TransactionalComplaintAdmin, ) from app.api.base import api_bp from app.auth.base import auth_bp @@ -90,6 +91,7 @@ from app.models import ( ManualSubscription, Coupon, AdminAuditLog, + TransactionalComplaint, ) from app.monitor.base import monitor_bp from app.oauth.base import oauth_bp @@ -691,6 +693,7 @@ def init_admin(app): admin.add_view(ManualSubscriptionAdmin(ManualSubscription, Session)) admin.add_view(CustomDomainAdmin(CustomDomain, Session)) admin.add_view(AdminAuditLogAdmin(AdminAuditLog, Session)) + admin.add_view(TransactionalComplaintAdmin(TransactionalComplaint, Session)) def register_custom_commands(app): diff --git a/templates/emails/transactional/hotmail-complaint-reply-phase.txt.jinja2 b/templates/emails/transactional/hotmail-complaint-reply-phase.txt.jinja2 deleted file mode 100644 index 934a96d1..00000000 --- a/templates/emails/transactional/hotmail-complaint-reply-phase.txt.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "base.txt.jinja2" %} - -{% block content %} -Hi, - -This is SimpleLogin team. - -Hotmail has informed us about an email sent from your alias {{ alias.email }} to {{ destination }} that might have been considered as spam, either by the recipient or by their Hotmail spam filter. - -Please note that sending non-solicited from a SimpleLogin alias infringes our terms and condition as it severely affects SimpleLogin email delivery. - -If somehow the recipient's Hotmail considers a forwarded email as Spam, it helps us a lot if you can ask them to move the email out of their Spam folder. - -Don't hesitate to get in touch with us if you need more information. -{% endblock %} diff --git a/templates/emails/transactional/hotmail-complaint.html b/templates/emails/transactional/hotmail-complaint.html deleted file mode 100644 index fbc59520..00000000 --- a/templates/emails/transactional/hotmail-complaint.html +++ /dev/null @@ -1,41 +0,0 @@ -{% extends "base.html" %} - -{% block content %} - {% call text() %} - This is SimpleLogin team. - {% endcall %} - - {% call text() %} - Hotmail has informed us about an email sent to {{ alias.email }} that might have been considered as spam, - either by you or by Hotmail spam filter. - {% endcall %} - - {% call text() %} - Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, - has a negative effect for all users and - is a violation of our terms and condition. - {% endcall %} - - {% call text() %} - If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. - {% endcall %} - - {% call text() %} - If somehow Hotmail considers a forwarded email as Spam, it will help us if you can move the email out of the Spam - folder. - You can also set up a filter to avoid this from happening in the future using this guide at - https://simplelogin.io/help/ - {% endcall %} - - {% call text() %} - Don't hesitate to get in touch with us if you need more information. - {% endcall %} - - {% call text() %} - Best,
- SimpleLogin Team. - {% endcall %} - -{% endblock %} - - diff --git a/templates/emails/transactional/hotmail-complaint.txt.jinja2 b/templates/emails/transactional/hotmail-complaint.txt.jinja2 deleted file mode 100644 index e27a96e0..00000000 --- a/templates/emails/transactional/hotmail-complaint.txt.jinja2 +++ /dev/null @@ -1,20 +0,0 @@ -{% extends "base.txt.jinja2" %} - -{% block content %} -Hi, - -This is SimpleLogin team. - -Hotmail has informed us about an email sent to {{ alias.email }} that might have been considered as spam, -either by you or by Hotmail spam filter. - -Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, -has a negative effect for all users and is a violation of our terms and condition. - -If that’s the case, please disable the alias instead if you don't want to receive the emails sent to this alias. - -If somehow Hotmail considers a forwarded email as Spam, it will help us if you can move the email out of the Spam folder. -You can also set up a filter to avoid this from happening in the future using this guide at https://simplelogin.io/help/ - -Don't hesitate to get in touch with us if you need more information. -{% endblock %} diff --git a/templates/emails/transactional/hotmail-transactional-complaint.html b/templates/emails/transactional/hotmail-transactional-complaint.html deleted file mode 100644 index 0d794bab..00000000 --- a/templates/emails/transactional/hotmail-transactional-complaint.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base.html" %} - -{% block content %} - {% call text() %} - This is SimpleLogin team. - {% endcall %} - - {% call text() %} - Hotmail has informed us about an email sent to {{ user.email }} that might have been considered as spam, - either by you or by Hotmail spam filter. - {% endcall %} - - {% call text() %} - Please note that explicitly marking a SimpleLogin's forwarded email as Spam - affects SimpleLogin email delivery, - has a negative effect for all users and is a violation of our terms and condition. - {% endcall %} - - {% call text() %} - If somehow Hotmail considers a forwarded email as Spam, it helps us if you can move the email - out of the Spam folder. You can also set up a filter to avoid this - from happening in the future using this guide at - https://simplelogin.io/docs/getting-started/troubleshooting/ - - {% endcall %} - - {% call text() %} - Please don't put our emails into the Spam folder. This can end up in your account being disabled on SimpleLogin. - {% endcall %} - - {% call text() %} - Don't hesitate to get in touch with us if you need more information. - {% endcall %} - - {% call text() %} - Best,
- SimpleLogin Team. - {% endcall %} - -{% endblock %} - - diff --git a/templates/emails/transactional/hotmail-transactional-complaint.txt.jinja2 b/templates/emails/transactional/hotmail-transactional-complaint.txt.jinja2 deleted file mode 100644 index 7969130c..00000000 --- a/templates/emails/transactional/hotmail-transactional-complaint.txt.jinja2 +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "base.txt.jinja2" %} - -{% block content %} -Hi, - -This is SimpleLogin team. - -Hotmail has informed us about an email sent to {{ user.email }} that might have been considered as spam, -either by you or by Hotmail. - -Please note that explicitly marking a SimpleLogin's forwarded email as Spam - affects SimpleLogin email delivery, - has a negative effect for all users and is a violation of our terms and condition. - -If somehow Hotmail considers a forwarded email as Spam, it helps us if you can move the email - out of the Spam folder. You can also set up a filter to avoid this - from happening in the future using this guide at - https://simplelogin.io/docs/getting-started/troubleshooting/ - -Please don't put our emails into the Spam folder. This can end up in your account being disabled on SimpleLogin. - -Don't hesitate to get in touch with us if you need more information. -{% endblock %} diff --git a/templates/emails/transactional/yahoo-transactional-complaint.html b/templates/emails/transactional/transactional-complaint-forward-phase.html similarity index 73% rename from templates/emails/transactional/yahoo-transactional-complaint.html rename to templates/emails/transactional/transactional-complaint-forward-phase.html index 9d8db38f..a1502b1f 100644 --- a/templates/emails/transactional/yahoo-transactional-complaint.html +++ b/templates/emails/transactional/transactional-complaint-forward-phase.html @@ -6,18 +6,17 @@ {% endcall %} {% call text() %} - Yahoo has informed us about an email sent to {{ user.email }} that might have been considered as spam, - either by you or by Yahoo spam filter. + {{ provider }} has informed us about an email sent to {{ user.email }} that might have been considered as spam, + either by you or by {{ provider }} spam filter. {% endcall %} {% call text() %} - Please note that explicitly marking a SimpleLogin's forwarded email as Spam - affects SimpleLogin email delivery, + Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, has a negative effect for all users and is a violation of our terms and condition. {% endcall %} {% call text() %} - If somehow Yahoo considers a forwarded email as Spam, it helps us if you can move the email + If somehow {{ provider }} considers a forwarded email as Spam, it helps us if you can move the email out of the Spam folder. You can also set up a filter to avoid this from happening in the future using this guide at https://simplelogin.io/docs/getting-started/troubleshooting/ diff --git a/templates/emails/transactional/yahoo-transactional-complaint.txt.jinja2 b/templates/emails/transactional/transactional-complaint-forward-phase.txt.jinja2 similarity index 70% rename from templates/emails/transactional/yahoo-transactional-complaint.txt.jinja2 rename to templates/emails/transactional/transactional-complaint-forward-phase.txt.jinja2 index 46b18b31..428674e1 100644 --- a/templates/emails/transactional/yahoo-transactional-complaint.txt.jinja2 +++ b/templates/emails/transactional/transactional-complaint-forward-phase.txt.jinja2 @@ -5,14 +5,14 @@ Hi, This is SimpleLogin team. -Yahoo has informed us about an email sent to {{ user.email }} that might have been considered as spam, -either by you or by Yahoo. +{{ provider }} has informed us about an email sent to {{ user.email }} that might have been considered as spam, +either by you or by {{ provider }}. Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, has a negative effect for all users and is a violation of our terms and condition. -If somehow Yahoo considers a forwarded email as Spam, it helps us if you can move the email +If somehow {{ provider }} considers a forwarded email as Spam, it helps us if you can move the email out of the Spam folder. You can also set up a filter to avoid this from happening in the future using this guide at https://simplelogin.io/docs/getting-started/troubleshooting/ diff --git a/templates/emails/transactional/transactional-complaint-reply-phase.txt.jinja2 b/templates/emails/transactional/transactional-complaint-reply-phase.txt.jinja2 new file mode 100644 index 00000000..76e5fbda --- /dev/null +++ b/templates/emails/transactional/transactional-complaint-reply-phase.txt.jinja2 @@ -0,0 +1,15 @@ +{% extends "base.txt.jinja2" %} + +{% block content %} +Hi, + +This is SimpleLogin team. + +We have received a report from {{ provider }} informing us about an email sent from your alias {{ alias.email }} to {{ destination }} that might have been considered as spam, either by the recipient or by their spam filter. + +Please note that sending non-solicited from a SimpleLogin alias infringes our terms and condition as it severely affects SimpleLogin email delivery. + +If somehow the recipient's provider considers a forwarded email as Spam, it helps us a lot if you can ask them to move the email out of their Spam folder. + +Don't hesitate to get in touch with us if you need more information. +{% endblock %} diff --git a/templates/emails/transactional/yahoo-complaint.html b/templates/emails/transactional/transactional-complaint-to-user.html similarity index 88% rename from templates/emails/transactional/yahoo-complaint.html rename to templates/emails/transactional/transactional-complaint-to-user.html index c9814ce1..d224f98b 100644 --- a/templates/emails/transactional/yahoo-complaint.html +++ b/templates/emails/transactional/transactional-complaint-to-user.html @@ -6,7 +6,7 @@ {% endcall %} {% call text() %} - Yahoo has informed us about an email sent to {{ alias.email }} that might have been marked as spam. + {{ provider }} has informed us about an email sent to {{ user.email }} that might have been marked as spam. {% endcall %} {% call text() %} diff --git a/templates/emails/transactional/yahoo-complaint.txt.jinja2 b/templates/emails/transactional/transactional-complaint-to-user.txt.jinja2 similarity index 84% rename from templates/emails/transactional/yahoo-complaint.txt.jinja2 rename to templates/emails/transactional/transactional-complaint-to-user.txt.jinja2 index 8007e985..a90add48 100644 --- a/templates/emails/transactional/yahoo-complaint.txt.jinja2 +++ b/templates/emails/transactional/transactional-complaint-to-user.txt.jinja2 @@ -5,7 +5,7 @@ Hi, This is SimpleLogin team. -Yahoo has informed us about an email sent to {{ alias.email }} that might have been marked as spam. +{{ provider }} has informed us about an email sent to {{ user.email }} that might have been marked as spam. Please note that explicitly marking a SimpleLogin's forwarded email as Spam affects SimpleLogin email delivery, has a negative effect for all users and is a violation of our terms and condition. diff --git a/tests/handler/test_transactional_complaints.py b/tests/handler/test_transactional_complaints.py new file mode 100644 index 00000000..49dd5004 --- /dev/null +++ b/tests/handler/test_transactional_complaints.py @@ -0,0 +1,92 @@ +import email +from email.message import Message +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +import pytest + +from app.config import ( + ALERT_COMPLAINT_FORWARD_PHASE, + ALERT_COMPLAINT_REPLY_PHASE, + ALERT_COMPLAINT_TO_USER, +) +from app.db import Session +from app.email import headers +from app.handler.transactional_complaint import ( + handle_hotmail_complaint, + handle_yahoo_complaint, +) +from app.models import Alias, TransactionalComplaint, SentAlert +from tests.utils import create_new_user + +origins = [ + [handle_yahoo_complaint, "yahoo", 6], + [handle_hotmail_complaint, "hotmail", 3], +] + + +def prepare_complaint(message: Message, part_num: int) -> Message: + complaint = MIMEMultipart("related") + # When walking, part 0 is the full message so we -1, and we want to be part N so -1 again + for i in range(part_num - 2): + document = MIMEText("text", "plain") + document.set_payload(f"Part {i}") + complaint.attach(document) + complaint.attach(message) + + return email.message_from_bytes(complaint.as_bytes()) + + +@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) +def test_transactional_to_user(flask_client, handle_ftor, provider, part_num): + user = create_new_user() + original_message = Message() + original_message[headers.TO] = user.email + original_message[headers.FROM] = "nobody@nowhere.net" + original_message.set_payload("Contents") + + complaint = prepare_complaint(original_message, part_num) + assert handle_ftor(complaint) + found = TransactionalComplaint.filter_by(user_id=user.id).all() + assert len(found) == 0 + alerts = SentAlert.filter_by(user_id=user.id).all() + assert len(alerts) == 1 + assert alerts[0].alert_type == f"{ALERT_COMPLAINT_TO_USER}_{provider}" + + +@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) +def test_transactional_forward_phase(flask_client, handle_ftor, provider, part_num): + user = create_new_user() + alias = Alias.create_new_random(user) + Session.commit() + original_message = Message() + original_message[headers.TO] = "nobody@nowhere.net" + original_message[headers.FROM] = alias.email + original_message.set_payload("Contents") + + complaint = prepare_complaint(original_message, part_num) + assert handle_ftor(complaint) + found = TransactionalComplaint.filter_by(user_id=user.id).all() + assert len(found) == 1 + alerts = SentAlert.filter_by(user_id=user.id).all() + assert len(alerts) == 1 + assert alerts[0].alert_type == f"{ALERT_COMPLAINT_REPLY_PHASE}_{provider}" + + +@pytest.mark.parametrize("handle_ftor,provider,part_num", origins) +def test_transactional_reply_phase(flask_client, handle_ftor, provider, part_num): + user = create_new_user() + alias = Alias.create_new_random(user) + Session.commit() + original_message = Message() + original_message[headers.TO] = alias.email + original_message[headers.FROM] = "no@no.no" + original_message.set_payload("Contents") + + complaint = prepare_complaint(original_message, part_num) + assert handle_ftor(complaint) + found = TransactionalComplaint.filter_by(user_id=user.id).all() + assert len(found) == 0 + alerts = SentAlert.filter_by(user_id=user.id).all() + assert len(alerts) == 1 + assert alerts[0].alert_type == f"{ALERT_COMPLAINT_FORWARD_PHASE}_{provider}" diff --git a/tests/utils.py b/tests/utils.py index 9d4b5d46..758d65ac 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -11,17 +11,11 @@ from flask import url_for from app.models import User -# keep track of the number of user -_nb_user = 0 - def create_new_user() -> User: - global _nb_user - _nb_user += 1 - # new user has a different email address user = User.create( - email=f"{_nb_user}@mailbox.test", + email=f"user{random.random()}@mailbox.test", password="password", name="Test User", activated=True, From 89d94963d7d851544c09fc0d036af24a11454ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 22 Apr 2022 14:49:03 +0200 Subject: [PATCH 02/12] PR comments --- app/handler/transactional_complaint.py | 28 +++++++++++++++----------- app/models.py | 1 - 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/app/handler/transactional_complaint.py b/app/handler/transactional_complaint.py index ad0bd105..f103bb4b 100644 --- a/app/handler/transactional_complaint.py +++ b/app/handler/transactional_complaint.py @@ -48,10 +48,13 @@ class TransactionalComplaintOrigin(ABC): class TransactionalYahooOrigin(TransactionalComplaintOrigin): @classmethod def get_original_message(cls, message: Message) -> Optional[Message]: - wanted_part = 6 + # 1st part is the container + # 2nd has empty body + # 6th is the original message + current_part = 0 for part in message.walk(): - wanted_part -= 1 - if wanted_part == 0: + current_part += 1 + if current_part == 6: return part return None @@ -63,10 +66,13 @@ class TransactionalYahooOrigin(TransactionalComplaintOrigin): class TransactionalHotmailOrigin(TransactionalComplaintOrigin): @classmethod def get_original_message(cls, message: Message) -> Optional[Message]: - wanted_part = 3 + # 1st part is the container + # 2nd has empty body + # 3rd is the original message + current_part = 0 for part in message.walk(): - wanted_part -= 1 - if wanted_part == 0: + current_part += 1 + if current_part == 3: return part return None @@ -103,12 +109,12 @@ def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> ) except ValueError: saved_file = save_email_for_debugging(message, "FromParseFailed") - LOG.w("Cannot parse from header. Saved to {}".format(saved_file or "nowhere")) - return True + LOG.w(f"Cannot parse from header. Saved to {saved_file or 'nowhere'}") + return False user = User.get_by(email=to_address) if user: - LOG.d("Handle transactional {} complaint for {}".format(origin.name(), user)) + LOG.d(f"Handle transactional {origin.name()} complaint for {user}") report_complaint_to_user(user, origin) return True @@ -116,9 +122,7 @@ def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> # the email is during a reply phase, from=alias and to=destination if alias: LOG.i( - "Complaint from {} during reply phase {} -> {}, {}".format( - origin.name(), alias, to_address, user - ) + f"Complaint from {origin.name} during reply phase {alias} -> {to_address}, {user}" ) report_complaint_to_user_in_reply_phase(alias, to_address, origin) store_transactional_complaint(alias, message) diff --git a/app/models.py b/app/models.py index 67109903..ddea131f 100644 --- a/app/models.py +++ b/app/models.py @@ -2974,7 +2974,6 @@ class AdminAuditLog(Base): action=AuditLogActionEnum.logged_as_user.value, model="User", model_id=user_id, - data={}, ) @classmethod From fcd2ab6fed92cae4d0f867e4d698de2d0aa8271d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 22 Apr 2022 14:53:04 +0200 Subject: [PATCH 03/12] Set data to non-nullable --- app/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index ddea131f..445cdb82 100644 --- a/app/models.py +++ b/app/models.py @@ -2914,7 +2914,7 @@ class AdminAuditLog(Base): action = sa.Column(sa.Integer, nullable=False) model = sa.Column(sa.Text, nullable=False) model_id = sa.Column(sa.Integer, nullable=True) - data = sa.Column(sa.JSON, nullable=True) + data = sa.Column(sa.JSON, nullable=False) admin = orm.relationship(User, foreign_keys=[admin_user_id]) @@ -2974,6 +2974,7 @@ class AdminAuditLog(Base): action=AuditLogActionEnum.logged_as_user.value, model="User", model_id=user_id, + data={}, ) @classmethod From 5208c549fab88e8dae332104ee6c5824a629414b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Mon, 25 Apr 2022 14:40:42 +0200 Subject: [PATCH 04/12] Rename TransactionalComplaint to ProviderComplaint --- app/admin_model.py | 16 ++++---- ...nal_complaint.py => provider_complaint.py} | 40 +++++++++---------- app/models.py | 14 +++---- email_handler.py | 2 +- ...45588d9bb475_store_provider_complaints.py} | 4 +- server.py | 6 +-- ... => provider-complaint-forward-phase.html} | 0 ...ovider-complaint-forward-phase.txt.jinja2} | 0 ...provider-complaint-reply-phase.txt.jinja2} | 0 ...r.html => provider-complaint-to-user.html} | 0 ... => provider-complaint-to-user.txt.jinja2} | 0 ...plaints.py => test_provider_complaints.py} | 16 ++++---- 12 files changed, 49 insertions(+), 49 deletions(-) rename app/handler/{transactional_complaint.py => provider_complaint.py} (81%) rename migrations/versions/{2022_041916_45588d9bb475_store_transactional_complaints_for_.py => 2022_041916_45588d9bb475_store_provider_complaints.py} (93%) rename templates/emails/transactional/{transactional-complaint-forward-phase.html => provider-complaint-forward-phase.html} (100%) rename templates/emails/transactional/{transactional-complaint-forward-phase.txt.jinja2 => provider-complaint-forward-phase.txt.jinja2} (100%) rename templates/emails/transactional/{transactional-complaint-reply-phase.txt.jinja2 => provider-complaint-reply-phase.txt.jinja2} (100%) rename templates/emails/transactional/{transactional-complaint-to-user.html => provider-complaint-to-user.html} (100%) rename templates/emails/transactional/{transactional-complaint-to-user.txt.jinja2 => provider-complaint-to-user.txt.jinja2} (100%) rename tests/handler/{test_transactional_complaints.py => test_provider_complaints.py} (83%) diff --git a/app/admin_model.py b/app/admin_model.py index 17f28133..3351641e 100644 --- a/app/admin_model.py +++ b/app/admin_model.py @@ -21,9 +21,9 @@ from app.models import ( AppleSubscription, AdminAuditLog, AuditLogActionEnum, - TransactionalComplaintState, + ProviderComplaintState, Phase, - TransactionalComplaint, + ProviderComplaint, ) @@ -376,7 +376,7 @@ class AdminAuditLogAdmin(SLModelView): def _transactionalcomplaint_state_formatter(view, context, model, name): - return "{} ({})".format(TransactionalComplaintState(model.state).name, model.state) + return "{} ({})".format(ProviderComplaintState(model.state).name, model.state) def _transactionalcomplaint_phase_formatter(view, context, model, name): @@ -390,7 +390,7 @@ def _transactionalcomplaint_refused_email_id_formatter(view, context, model, nam return Markup(markupstring) -class TransactionalComplaintAdmin(SLModelView): +class ProviderComplaintAdmin(SLModelView): column_searchable_list = ["id", "user.id", "created_at"] column_filters = ["user.id", "state"] column_hide_backrefs = False @@ -410,12 +410,12 @@ class TransactionalComplaintAdmin(SLModelView): EndpointLinkRowAction("fa fa-check-square", ".mark_ok"), ] - def _get_complaint(self) -> Optional[TransactionalComplaint]: + def _get_complaint(self) -> Optional[ProviderComplaint]: complain_id = request.args.get("id") if complain_id is None: flash("Missing id", "error") return None - complaint = TransactionalComplaint.get_by(id=complain_id) + complaint = ProviderComplaint.get_by(id=complain_id) if not complaint: flash("Could not find complaint", "error") return None @@ -426,7 +426,7 @@ class TransactionalComplaintAdmin(SLModelView): complaint = self._get_complaint() if not complaint: return redirect("/admin/transactionalcomplaint/") - complaint.state = TransactionalComplaintState.reviewed.value + complaint.state = ProviderComplaintState.reviewed.value Session.commit() return redirect("/admin/transactionalcomplaint/") @@ -437,7 +437,7 @@ class TransactionalComplaintAdmin(SLModelView): return redirect("/admin/transactionalcomplaint/") eml_path = complaint.refused_email.full_report_path eml_data = s3.download_email(eml_path) - AdminAuditLog.downloaded_transactional_complaint(current_user.id, complaint.id) + AdminAuditLog.downloaded_provider_complaint(current_user.id, complaint.id) Session.commit() return Response( eml_data, diff --git a/app/handler/transactional_complaint.py b/app/handler/provider_complaint.py similarity index 81% rename from app/handler/transactional_complaint.py rename to app/handler/provider_complaint.py index f103bb4b..86d25b45 100644 --- a/app/handler/transactional_complaint.py +++ b/app/handler/provider_complaint.py @@ -26,14 +26,14 @@ from app.models import ( DeletedAlias, DomainDeletedAlias, Contact, - TransactionalComplaint, + ProviderComplaint, Phase, - TransactionalComplaintState, + ProviderComplaintState, RefusedEmail, ) -class TransactionalComplaintOrigin(ABC): +class ProviderComplaintOrigin(ABC): @classmethod @abstractmethod def get_original_message(cls, message: Message) -> Optional[Message]: @@ -45,7 +45,7 @@ class TransactionalComplaintOrigin(ABC): pass -class TransactionalYahooOrigin(TransactionalComplaintOrigin): +class TransactionalYahooOrigin(ProviderComplaintOrigin): @classmethod def get_original_message(cls, message: Message) -> Optional[Message]: # 1st part is the container @@ -63,7 +63,7 @@ class TransactionalYahooOrigin(TransactionalComplaintOrigin): return "yahoo" -class TransactionalHotmailOrigin(TransactionalComplaintOrigin): +class TransactionalHotmailOrigin(ProviderComplaintOrigin): @classmethod def get_original_message(cls, message: Message) -> Optional[Message]: # 1st part is the container @@ -97,7 +97,7 @@ def find_alias_with_address(address: str) -> Optional[Alias]: ) -def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> bool: +def handle_complaint(message: Message, origin: ProviderComplaintOrigin) -> bool: original_message = origin.get_original_message(message) try: @@ -114,7 +114,7 @@ def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> user = User.get_by(email=to_address) if user: - LOG.d(f"Handle transactional {origin.name()} complaint for {user}") + LOG.d(f"Handle provider {origin.name()} complaint for {user}") report_complaint_to_user(user, origin) return True @@ -125,7 +125,7 @@ def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> f"Complaint from {origin.name} during reply phase {alias} -> {to_address}, {user}" ) report_complaint_to_user_in_reply_phase(alias, to_address, origin) - store_transactional_complaint(alias, message) + store_provider_complaint(alias, message) return True contact = Contact.get_by(reply_email=from_address) @@ -145,7 +145,7 @@ def handle_complaint(message: Message, origin: TransactionalComplaintOrigin) -> def report_complaint_to_user_in_reply_phase( - alias: Alias, to_address: str, origin: TransactionalComplaintOrigin + alias: Alias, to_address: str, origin: ProviderComplaintOrigin ): capitalized_name = origin.name().capitalize() send_email_with_rate_control( @@ -154,7 +154,7 @@ def report_complaint_to_user_in_reply_phase( alias.user.email, f"Abuse report from {capitalized_name}", render( - "transactional/transactional-complaint-reply-phase.txt.jinja2", + "transactional/provider-complaint-reply-phase.txt.jinja2", user=alias.user, alias=alias, destination=to_address, @@ -165,7 +165,7 @@ def report_complaint_to_user_in_reply_phase( ) -def report_complaint_to_user(user: User, origin: TransactionalComplaintOrigin): +def report_complaint_to_user(user: User, origin: ProviderComplaintOrigin): capitalized_name = origin.name().capitalize() send_email_with_rate_control( user, @@ -173,12 +173,12 @@ def report_complaint_to_user(user: User, origin: TransactionalComplaintOrigin): user.email, f"Abuse report from {capitalized_name}", render( - "transactional/transactional-complaint-to-user.txt.jinja2", + "transactional/provider-complaint-to-user.txt.jinja2", user=user, provider=capitalized_name, ), render( - "transactional/transactional-complaint-to-user.html", + "transactional/provider-complaint-to-user.html", user=user, provider=capitalized_name, ), @@ -188,7 +188,7 @@ def report_complaint_to_user(user: User, origin: TransactionalComplaintOrigin): def report_complaint_to_user_in_forward_phase( - alias: Alias, origin: TransactionalComplaintOrigin + alias: Alias, origin: ProviderComplaintOrigin ): capitalized_name = origin.name().capitalize() user = alias.user @@ -198,12 +198,12 @@ def report_complaint_to_user_in_forward_phase( user.email, f"Abuse report from {capitalized_name}", render( - "transactional/transactional-complaint-forward-phase.txt.jinja2", + "transactional/provider-complaint-forward-phase.txt.jinja2", user=user, provider=capitalized_name, ), render( - "transactional/transactional-complaint-forward-phase.html", + "transactional/provider-complaint-forward-phase.html", user=user, provider=capitalized_name, ), @@ -212,9 +212,9 @@ def report_complaint_to_user_in_forward_phase( ) -def store_transactional_complaint(alias, message): +def store_provider_complaint(alias, message): email_name = f"reply-{uuid.uuid4().hex}.eml" - full_report_path = f"transactional_complaint/{email_name}" + full_report_path = f"provider_complaint/{email_name}" s3.upload_email_from_bytesio( full_report_path, BytesIO(to_bytes(message)), email_name ) @@ -224,9 +224,9 @@ def store_transactional_complaint(alias, message): path=email_name, commit=True, ) - TransactionalComplaint.create( + ProviderComplaint.create( user_id=alias.user_id, - state=TransactionalComplaintState.new.value, + state=ProviderComplaintState.new.value, phase=Phase.reply.value, refused_email_id=refused_email.id, commit=True, diff --git a/app/models.py b/app/models.py index 445cdb82..1331c233 100644 --- a/app/models.py +++ b/app/models.py @@ -235,7 +235,7 @@ class AuditLogActionEnum(EnumE): disable_2fa = 5 logged_as_user = 6 extend_subscription = 7 - download_transactional_complaint = 8 + download_provider_complaint = 8 class Phase(EnumE): @@ -2997,23 +2997,23 @@ class AdminAuditLog(Base): ) @classmethod - def downloaded_transactional_complaint(cls, admin_user_id: int, complaint_id: int): + def downloaded_provider_complaint(cls, admin_user_id: int, complaint_id: int): cls.create( admin_user_id=admin_user_id, - action=AuditLogActionEnum.download_transactional_complaint.value, - model="TransactionalComplaint", + action=AuditLogActionEnum.download_provider_complaint.value, + model="ProviderComplaint", model_id=complaint_id, data={}, ) -class TransactionalComplaintState(EnumE): +class ProviderComplaintState(EnumE): new = 0 reviewed = 1 -class TransactionalComplaint(Base, ModelMixin): - __tablename__ = "transactional_complaint" +class ProviderComplaint(Base, ModelMixin): + __tablename__ = "provider_complaint" user_id = sa.Column(sa.ForeignKey("users.id"), nullable=False) state = sa.Column(sa.Integer, nullable=False) diff --git a/email_handler.py b/email_handler.py index d64dce45..e90bef9a 100644 --- a/email_handler.py +++ b/email_handler.py @@ -140,7 +140,7 @@ from app.handler.spamd_result import ( SpamdResult, SPFCheckResult, ) -from app.handler.transactional_complaint import ( +from app.handler.provider_complaint import ( handle_hotmail_complaint, handle_yahoo_complaint, ) diff --git a/migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py b/migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py similarity index 93% rename from migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py rename to migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py index 5c6a9a61..e5b7e2e9 100644 --- a/migrations/versions/2022_041916_45588d9bb475_store_transactional_complaints_for_.py +++ b/migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py @@ -18,7 +18,7 @@ depends_on = None def upgrade(): op.create_table( - "transactional_complaint", + "provider_complaint", sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("created_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), sa.Column("updated_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), @@ -33,4 +33,4 @@ def upgrade(): def downgrade(): - op.drop_table("transactional_complaint") + op.drop_table("provider_complaint") diff --git a/server.py b/server.py index 8b7106af..c09127e7 100644 --- a/server.py +++ b/server.py @@ -37,7 +37,7 @@ from app.admin_model import ( CouponAdmin, CustomDomainAdmin, AdminAuditLogAdmin, - TransactionalComplaintAdmin, + ProviderComplaintAdmin, ) from app.api.base import api_bp from app.auth.base import auth_bp @@ -91,7 +91,7 @@ from app.models import ( ManualSubscription, Coupon, AdminAuditLog, - TransactionalComplaint, + ProviderComplaint, ) from app.monitor.base import monitor_bp from app.oauth.base import oauth_bp @@ -693,7 +693,7 @@ def init_admin(app): admin.add_view(ManualSubscriptionAdmin(ManualSubscription, Session)) admin.add_view(CustomDomainAdmin(CustomDomain, Session)) admin.add_view(AdminAuditLogAdmin(AdminAuditLog, Session)) - admin.add_view(TransactionalComplaintAdmin(TransactionalComplaint, Session)) + admin.add_view(ProviderComplaintAdmin(ProviderComplaint, Session)) def register_custom_commands(app): diff --git a/templates/emails/transactional/transactional-complaint-forward-phase.html b/templates/emails/transactional/provider-complaint-forward-phase.html similarity index 100% rename from templates/emails/transactional/transactional-complaint-forward-phase.html rename to templates/emails/transactional/provider-complaint-forward-phase.html diff --git a/templates/emails/transactional/transactional-complaint-forward-phase.txt.jinja2 b/templates/emails/transactional/provider-complaint-forward-phase.txt.jinja2 similarity index 100% rename from templates/emails/transactional/transactional-complaint-forward-phase.txt.jinja2 rename to templates/emails/transactional/provider-complaint-forward-phase.txt.jinja2 diff --git a/templates/emails/transactional/transactional-complaint-reply-phase.txt.jinja2 b/templates/emails/transactional/provider-complaint-reply-phase.txt.jinja2 similarity index 100% rename from templates/emails/transactional/transactional-complaint-reply-phase.txt.jinja2 rename to templates/emails/transactional/provider-complaint-reply-phase.txt.jinja2 diff --git a/templates/emails/transactional/transactional-complaint-to-user.html b/templates/emails/transactional/provider-complaint-to-user.html similarity index 100% rename from templates/emails/transactional/transactional-complaint-to-user.html rename to templates/emails/transactional/provider-complaint-to-user.html diff --git a/templates/emails/transactional/transactional-complaint-to-user.txt.jinja2 b/templates/emails/transactional/provider-complaint-to-user.txt.jinja2 similarity index 100% rename from templates/emails/transactional/transactional-complaint-to-user.txt.jinja2 rename to templates/emails/transactional/provider-complaint-to-user.txt.jinja2 diff --git a/tests/handler/test_transactional_complaints.py b/tests/handler/test_provider_complaints.py similarity index 83% rename from tests/handler/test_transactional_complaints.py rename to tests/handler/test_provider_complaints.py index 49dd5004..b6994e3d 100644 --- a/tests/handler/test_transactional_complaints.py +++ b/tests/handler/test_provider_complaints.py @@ -12,11 +12,11 @@ from app.config import ( ) from app.db import Session from app.email import headers -from app.handler.transactional_complaint import ( +from app.handler.provider_complaint import ( handle_hotmail_complaint, handle_yahoo_complaint, ) -from app.models import Alias, TransactionalComplaint, SentAlert +from app.models import Alias, ProviderComplaint, SentAlert from tests.utils import create_new_user origins = [ @@ -38,7 +38,7 @@ def prepare_complaint(message: Message, part_num: int) -> Message: @pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_transactional_to_user(flask_client, handle_ftor, provider, part_num): +def test_provider_to_user(flask_client, handle_ftor, provider, part_num): user = create_new_user() original_message = Message() original_message[headers.TO] = user.email @@ -47,7 +47,7 @@ def test_transactional_to_user(flask_client, handle_ftor, provider, part_num): complaint = prepare_complaint(original_message, part_num) assert handle_ftor(complaint) - found = TransactionalComplaint.filter_by(user_id=user.id).all() + found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 alerts = SentAlert.filter_by(user_id=user.id).all() assert len(alerts) == 1 @@ -55,7 +55,7 @@ def test_transactional_to_user(flask_client, handle_ftor, provider, part_num): @pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_transactional_forward_phase(flask_client, handle_ftor, provider, part_num): +def test_provider_forward_phase(flask_client, handle_ftor, provider, part_num): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() @@ -66,7 +66,7 @@ def test_transactional_forward_phase(flask_client, handle_ftor, provider, part_n complaint = prepare_complaint(original_message, part_num) assert handle_ftor(complaint) - found = TransactionalComplaint.filter_by(user_id=user.id).all() + found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 1 alerts = SentAlert.filter_by(user_id=user.id).all() assert len(alerts) == 1 @@ -74,7 +74,7 @@ def test_transactional_forward_phase(flask_client, handle_ftor, provider, part_n @pytest.mark.parametrize("handle_ftor,provider,part_num", origins) -def test_transactional_reply_phase(flask_client, handle_ftor, provider, part_num): +def test_provider_reply_phase(flask_client, handle_ftor, provider, part_num): user = create_new_user() alias = Alias.create_new_random(user) Session.commit() @@ -85,7 +85,7 @@ def test_transactional_reply_phase(flask_client, handle_ftor, provider, part_num complaint = prepare_complaint(original_message, part_num) assert handle_ftor(complaint) - found = TransactionalComplaint.filter_by(user_id=user.id).all() + found = ProviderComplaint.filter_by(user_id=user.id).all() assert len(found) == 0 alerts = SentAlert.filter_by(user_id=user.id).all() assert len(alerts) == 1 From 7fd9bdc5a73009cb04c2c3d6bd631df814f28bca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 28 Apr 2022 15:23:52 +0200 Subject: [PATCH 05/12] PR comments --- app/handler/provider_complaint.py | 8 ++++---- app/models.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 86d25b45..57bcd12b 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -45,7 +45,7 @@ class ProviderComplaintOrigin(ABC): pass -class TransactionalYahooOrigin(ProviderComplaintOrigin): +class ProviderComplaintYahoo(ProviderComplaintOrigin): @classmethod def get_original_message(cls, message: Message) -> Optional[Message]: # 1st part is the container @@ -63,7 +63,7 @@ class TransactionalYahooOrigin(ProviderComplaintOrigin): return "yahoo" -class TransactionalHotmailOrigin(ProviderComplaintOrigin): +class ProviderComplaintHotmail(ProviderComplaintOrigin): @classmethod def get_original_message(cls, message: Message) -> Optional[Message]: # 1st part is the container @@ -82,11 +82,11 @@ class TransactionalHotmailOrigin(ProviderComplaintOrigin): def handle_hotmail_complaint(message: Message) -> bool: - return handle_complaint(message, TransactionalHotmailOrigin()) + return handle_complaint(message, ProviderComplaintHotmail()) def handle_yahoo_complaint(message: Message) -> bool: - return handle_complaint(message, TransactionalYahooOrigin()) + return handle_complaint(message, ProviderComplaintYahoo()) def find_alias_with_address(address: str) -> Optional[Alias]: diff --git a/app/models.py b/app/models.py index 1331c233..57199479 100644 --- a/app/models.py +++ b/app/models.py @@ -2905,6 +2905,9 @@ class PhoneMessage(Base, ModelMixin): number = orm.relationship(PhoneNumber) +# endregion + + class AdminAuditLog(Base): __tablename__ = "admin_audit_log" @@ -3017,7 +3020,7 @@ class ProviderComplaint(Base, ModelMixin): user_id = sa.Column(sa.ForeignKey("users.id"), nullable=False) state = sa.Column(sa.Integer, nullable=False) - phase = sa.Column(sa.Integer, nullable=False) + phase = sa.Column(sa.Integer, nullable=False, server_default=Phase.unknown.value) # Point to the email that has been refused refused_email_id = sa.Column( sa.ForeignKey("refused_email.id", ondelete="cascade"), nullable=True From 74b31eac663d0df41b64b1860fb2c3d4636607eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Thu, 28 Apr 2022 15:24:45 +0200 Subject: [PATCH 06/12] PR comments --- app/handler/provider_complaint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 57bcd12b..1e89534e 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -115,7 +115,7 @@ def handle_complaint(message: Message, origin: ProviderComplaintOrigin) -> bool: user = User.get_by(email=to_address) if user: LOG.d(f"Handle provider {origin.name()} complaint for {user}") - report_complaint_to_user(user, origin) + report_complaint_to_user_in_transactional_phase(user, origin) return True alias = find_alias_with_address(from_address) @@ -165,7 +165,7 @@ def report_complaint_to_user_in_reply_phase( ) -def report_complaint_to_user(user: User, origin: ProviderComplaintOrigin): +def report_complaint_to_user_in_transactional_phase(user: User, origin: ProviderComplaintOrigin): capitalized_name = origin.name().capitalize() send_email_with_rate_control( user, From cca709ed480370cd99e5ae31ea233c03cfb63139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 29 Apr 2022 15:50:52 +0200 Subject: [PATCH 07/12] formatting --- app/handler/provider_complaint.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 1e89534e..0eeb181d 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -165,7 +165,9 @@ def report_complaint_to_user_in_reply_phase( ) -def report_complaint_to_user_in_transactional_phase(user: User, origin: ProviderComplaintOrigin): +def report_complaint_to_user_in_transactional_phase( + user: User, origin: ProviderComplaintOrigin +): capitalized_name = origin.name().capitalize() send_email_with_rate_control( user, From baddc0fe677657855acc181d77a52e516322e726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 29 Apr 2022 15:58:48 +0200 Subject: [PATCH 08/12] Fix: sqlalchemy only suports str as server_default --- app/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index a9d823b6..c8d69924 100644 --- a/app/models.py +++ b/app/models.py @@ -3023,7 +3023,7 @@ class ProviderComplaint(Base, ModelMixin): user_id = sa.Column(sa.ForeignKey("users.id"), nullable=False) state = sa.Column(sa.Integer, nullable=False) - phase = sa.Column(sa.Integer, nullable=False, server_default=Phase.unknown.value) + phase = sa.Column(sa.Integer, nullable=False, server_default=str(Phase.unknown.value)) # Point to the email that has been refused refused_email_id = sa.Column( sa.ForeignKey("refused_email.id", ondelete="cascade"), nullable=True From ba46ce5208159f1df5d113b184c2fc8fd29c6f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Fri, 29 Apr 2022 16:02:45 +0200 Subject: [PATCH 09/12] Format --- app/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models.py b/app/models.py index c8d69924..21276207 100644 --- a/app/models.py +++ b/app/models.py @@ -3023,7 +3023,9 @@ class ProviderComplaint(Base, ModelMixin): user_id = sa.Column(sa.ForeignKey("users.id"), nullable=False) state = sa.Column(sa.Integer, nullable=False) - phase = sa.Column(sa.Integer, nullable=False, server_default=str(Phase.unknown.value)) + phase = sa.Column( + sa.Integer, nullable=False, server_default=str(Phase.unknown.value) + ) # Point to the email that has been refused refused_email_id = sa.Column( sa.ForeignKey("refused_email.id", ondelete="cascade"), nullable=True From 56159765d9ca1723d5eee9483cad37c334dd39f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Mon, 2 May 2022 11:53:32 +0200 Subject: [PATCH 10/12] Rename --- app/config.py | 2 +- app/handler/provider_complaint.py | 4 ++-- tests/handler/test_provider_complaints.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/config.py b/app/config.py index 9fcbdb70..fa041a1d 100644 --- a/app/config.py +++ b/app/config.py @@ -338,7 +338,7 @@ ALERT_DIRECTORY_DISABLED_ALIAS_CREATION = "alert_directory_disabled_alias_creati ALERT_COMPLAINT_REPLY_PHASE = "alert_complaint_reply_phase" ALERT_COMPLAINT_FORWARD_PHASE = "alert_complaint_forward_phase" -ALERT_COMPLAINT_TO_USER = "alert_complaint_to_user" +ALERT_COMPLAINT_TRANSACTIONAL_PHASE = "alert_complaint_transactional_phase" ALERT_QUARANTINE_DMARC = "alert_quarantine_dmarc" diff --git a/app/handler/provider_complaint.py b/app/handler/provider_complaint.py index 0eeb181d..ce2befec 100644 --- a/app/handler/provider_complaint.py +++ b/app/handler/provider_complaint.py @@ -7,7 +7,7 @@ from typing import Optional from app import s3 from app.config import ( ALERT_COMPLAINT_REPLY_PHASE, - ALERT_COMPLAINT_TO_USER, + ALERT_COMPLAINT_TRANSACTIONAL_PHASE, ALERT_COMPLAINT_FORWARD_PHASE, ) from app.email import headers @@ -171,7 +171,7 @@ def report_complaint_to_user_in_transactional_phase( capitalized_name = origin.name().capitalize() send_email_with_rate_control( user, - f"{ALERT_COMPLAINT_TO_USER}_{origin.name()}", + f"{ALERT_COMPLAINT_TRANSACTIONAL_PHASE}_{origin.name()}", user.email, f"Abuse report from {capitalized_name}", render( diff --git a/tests/handler/test_provider_complaints.py b/tests/handler/test_provider_complaints.py index b6994e3d..00b9a6a5 100644 --- a/tests/handler/test_provider_complaints.py +++ b/tests/handler/test_provider_complaints.py @@ -8,7 +8,7 @@ import pytest from app.config import ( ALERT_COMPLAINT_FORWARD_PHASE, ALERT_COMPLAINT_REPLY_PHASE, - ALERT_COMPLAINT_TO_USER, + ALERT_COMPLAINT_TRANSACTIONAL_PHASE, ) from app.db import Session from app.email import headers @@ -51,7 +51,7 @@ def test_provider_to_user(flask_client, handle_ftor, provider, part_num): assert len(found) == 0 alerts = SentAlert.filter_by(user_id=user.id).all() assert len(alerts) == 1 - assert alerts[0].alert_type == f"{ALERT_COMPLAINT_TO_USER}_{provider}" + assert alerts[0].alert_type == f"{ALERT_COMPLAINT_TRANSACTIONAL_PHASE}_{provider}" @pytest.mark.parametrize("handle_ftor,provider,part_num", origins) From beea14ef14976c916059b06dee7b51bb8b828464 Mon Sep 17 00:00:00 2001 From: Son Nguyen Kim Date: Mon, 2 May 2022 16:41:37 +0200 Subject: [PATCH 11/12] Update provider-complaint-reply-phase.txt.jinja2 --- .../transactional/provider-complaint-reply-phase.txt.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/emails/transactional/provider-complaint-reply-phase.txt.jinja2 b/templates/emails/transactional/provider-complaint-reply-phase.txt.jinja2 index 76e5fbda..a035e15a 100644 --- a/templates/emails/transactional/provider-complaint-reply-phase.txt.jinja2 +++ b/templates/emails/transactional/provider-complaint-reply-phase.txt.jinja2 @@ -7,7 +7,7 @@ This is SimpleLogin team. We have received a report from {{ provider }} informing us about an email sent from your alias {{ alias.email }} to {{ destination }} that might have been considered as spam, either by the recipient or by their spam filter. -Please note that sending non-solicited from a SimpleLogin alias infringes our terms and condition as it severely affects SimpleLogin email delivery. +Please note that sending non-solicited email from a SimpleLogin alias infringes our terms and condition as it severely affects SimpleLogin email delivery. If somehow the recipient's provider considers a forwarded email as Spam, it helps us a lot if you can ask them to move the email out of their Spam folder. From 6936d9977993bccca9edb20d20f2f83c5885ea89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Casaj=C3=BAs?= Date: Tue, 3 May 2022 14:15:50 +0200 Subject: [PATCH 12/12] Set default state for provider complaint --- app/models.py | 4 +- ..._45588d9bb475_store_provider_complaints.py | 36 ------------- ..._28b9b14c9664_store_provider_complaints.py | 50 +++++++++++++++++++ 3 files changed, 53 insertions(+), 37 deletions(-) delete mode 100644 migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py create mode 100644 migrations/versions/2022_050314_28b9b14c9664_store_provider_complaints.py diff --git a/app/models.py b/app/models.py index 21276207..0054be73 100644 --- a/app/models.py +++ b/app/models.py @@ -3022,7 +3022,9 @@ class ProviderComplaint(Base, ModelMixin): __tablename__ = "provider_complaint" user_id = sa.Column(sa.ForeignKey("users.id"), nullable=False) - state = sa.Column(sa.Integer, nullable=False) + state = sa.Column( + sa.Integer, nullable=False, server_default=str(ProviderComplaintState.new.value) + ) phase = sa.Column( sa.Integer, nullable=False, server_default=str(Phase.unknown.value) ) diff --git a/migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py b/migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py deleted file mode 100644 index e5b7e2e9..00000000 --- a/migrations/versions/2022_041916_45588d9bb475_store_provider_complaints.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Store transactional complaints for admins to verify - -Revision ID: 45588d9bb475 -Revises: b500363567e3 -Create Date: 2022-04-19 16:17:42.798792 - -""" -import sqlalchemy_utils -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '45588d9bb475' -down_revision = 'b500363567e3' -branch_labels = None -depends_on = None - -def upgrade(): - op.create_table( - "provider_complaint", - sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), - sa.Column("created_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=False), - sa.Column("updated_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=True), - sa.Column("user_id", sa.Integer, nullable=False), - sa.Column("state", sa.Integer, nullable=False), - sa.Column("phase", sa.Integer, nullable=False), - sa.Column("refused_email_id", sa.Integer, nullable=False), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'), - sa.ForeignKeyConstraint(['refused_email_id'], ['refused_email.id'], ondelete='cascade'), - sa.PrimaryKeyConstraint("id"), - ) - - -def downgrade(): - op.drop_table("provider_complaint") diff --git a/migrations/versions/2022_050314_28b9b14c9664_store_provider_complaints.py b/migrations/versions/2022_050314_28b9b14c9664_store_provider_complaints.py new file mode 100644 index 00000000..ee0689d6 --- /dev/null +++ b/migrations/versions/2022_050314_28b9b14c9664_store_provider_complaints.py @@ -0,0 +1,50 @@ +"""store provider complaints + +Revision ID: 28b9b14c9664 +Revises: b500363567e3 +Create Date: 2022-05-03 14:14:23.288929 + +""" +import sqlalchemy_utils +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "28b9b14c9664" +down_revision = "b500363567e3" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "provider_complaint", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=False + ), + sa.Column( + "updated_at", sqlalchemy_utils.types.arrow.ArrowType(), nullable=True + ), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("state", sa.Integer(), server_default="0", nullable=False), + sa.Column("phase", sa.Integer(), server_default="0", nullable=False), + sa.Column("refused_email_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["refused_email_id"], ["refused_email.id"], ondelete="cascade" + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("provider_complaint") + # ### end Alembic commands ###