Merge remote-tracking branch 'origin/master' into ac-dmarc-reply-phase

* origin/master:
  Only send enum name for events intead of the full class.enum
  Also track login and register events from the api routes
  typo
  revert changes
  Added fix for parts that are not messages
  Add missing formatting place
  Revert unwanted changes
  Do not show an error if we receive an unsubscribe from a different address
  Revert changes to pgp_utils
  Send newrelic events on login and register

app/api/views/auth.py

@@ -19,6 +19,7 @@ from app.email_utils import (
     send_email,
     render,
 )
+from app.events.auth_event import LoginEvent, RegisterEvent
 from app.extensions import limiter
 from app.log import LOG
 from app.models import User, ApiKey, SocialAuth, AccountActivation
@@ -55,16 +56,20 @@ def auth_login():
     user = User.filter_by(email=email).first()
 
     if not user or not user.check_password(password):
+        LoginEvent(LoginEvent.ActionType.failed, LoginEvent.Source.api).send()
         return jsonify(error="Email or password incorrect"), 400
     elif user.disabled:
+        LoginEvent(LoginEvent.ActionType.disabled_login, LoginEvent.Source.api).send()
         return jsonify(error="Account disabled"), 400
     elif not user.activated:
+        LoginEvent(LoginEvent.ActionType.not_activated, LoginEvent.Source.api).send()
         return jsonify(error="Account not activated"), 422
     elif user.fido_enabled():
         # allow user who has TOTP enabled to continue using the mobile app
         if not user.enable_otp:
             return jsonify(error="Currently we don't support FIDO on mobile yet"), 403
 
+    LoginEvent(LoginEvent.ActionType.success, LoginEvent.Source.api).send()
     return jsonify(**auth_payload(user, device)), 200
 
 
@@ -88,14 +93,20 @@ def auth_register():
     password = data.get("password")
 
     if DISABLE_REGISTRATION:
+        RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
         return jsonify(error="registration is closed"), 400
     if not email_can_be_used_as_mailbox(email) or personal_email_already_used(email):
+        RegisterEvent(
+            RegisterEvent.ActionType.invalid_email, RegisterEvent.Source.api
+        ).send()
         return jsonify(error=f"cannot use {email} as personal inbox"), 400
 
     if not password or len(password) < 8:
+        RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
         return jsonify(error="password too short"), 400
 
     if len(password) > 100:
+        RegisterEvent(RegisterEvent.ActionType.failed, RegisterEvent.Source.api).send()
         return jsonify(error="password too long"), 400
 
     LOG.d("create user %s", email)
@@ -114,6 +125,7 @@ def auth_register():
         render("transactional/code-activation.html", code=code),
     )
 
+    RegisterEvent(RegisterEvent.ActionType.success, RegisterEvent.Source.api).send()
     return jsonify(msg="User needs to confirm their account"), 200
 
 

app/auth/views/login.py

@@ -5,6 +5,7 @@ from wtforms import StringField, validators
 
 from app.auth.base import auth_bp
 from app.auth.views.login_utils import after_login
+from app.events.auth_event import LoginEvent
 from app.extensions import limiter
 from app.log import LOG
 from app.models import User
@@ -43,18 +44,22 @@ def login():
             g.deduct_limit = True
             form.password.data = None
             flash("Email or password incorrect", "error")
+            LoginEvent(LoginEvent.ActionType.failed).send()
         elif user.disabled:
             flash(
                 "Your account is disabled. Please contact SimpleLogin team to re-enable your account.",
                 "error",
             )
+            LoginEvent(LoginEvent.ActionType.disabled_login).send()
         elif not user.activated:
             show_resend_activation = True
             flash(
                 "Please check your inbox for the activation email. You can also have this email re-sent",
                 "error",
             )
+            LoginEvent(LoginEvent.ActionType.not_activated).send()
         else:
+            LoginEvent(LoginEvent.ActionType.success).send()
             return after_login(user, next_url)
 
     return render_template(

app/auth/views/register.py

@@ -13,6 +13,7 @@ from app.email_utils import (
     email_can_be_used_as_mailbox,
     personal_email_already_used,
 )
+from app.events.auth_event import RegisterEvent
 from app.log import LOG
 from app.models import User, ActivationCode
 from app.utils import random_string, encode_url, sanitize_email
@@ -60,6 +61,7 @@ def register():
                     hcaptcha_res,
                 )
                 flash("Wrong Captcha", "error")
+                RegisterEvent(RegisterEvent.ActionType.catpcha_failed).send()
                 return render_template(
                     "auth/register.html",
                     form=form,
@@ -70,10 +72,11 @@ def register():
         email = sanitize_email(form.email.data)
         if not email_can_be_used_as_mailbox(email):
             flash("You cannot use this email address as your personal inbox.", "error")
-
+            RegisterEvent(RegisterEvent.ActionType.email_in_use).send()
         else:
             if personal_email_already_used(email):
                 flash(f"Email {email} already used", "error")
+                RegisterEvent(RegisterEvent.ActionType.email_in_use).send()
             else:
                 LOG.d("create user %s", email)
                 user = User.create(
@@ -86,8 +89,10 @@ def register():
 
                 try:
                     send_activation_email(user, next_url)
+                    RegisterEvent(RegisterEvent.ActionType.success).send()
                 except Exception:
                     flash("Invalid email, are you sure the email is correct?", "error")
+                    RegisterEvent(RegisterEvent.ActionType.invalid_email).send()
                     return redirect(url_for("auth.register"))
 
                 return render_template("auth/register_waiting_activation.html")

app/email_utils.py

@@ -967,7 +967,10 @@ def add_header(msg: Message, text_header, html_header) -> Message:
     elif content_type in ("multipart/alternative", "multipart/related"):
         new_parts = []
         for part in msg.get_payload():
-            new_parts.append(add_header(part, text_header, html_header))
+            if isinstance(part, Message):
+                new_parts.append(add_header(part, text_header, html_header))
+            else:
+                new_parts.append(part)
         clone_msg = copy(msg)
         clone_msg.set_payload(new_parts)
         return clone_msg

app/events/auth_event.py

@@ -0,0 +1,46 @@
+import newrelic
+
+from app.models import EnumE
+
+
+class LoginEvent:
+    class ActionType(EnumE):
+        success = 0
+        failed = 1
+        disabled_login = 2
+        not_activated = 3
+
+    class Source(EnumE):
+        web = 0
+        api = 1
+
+    def __init__(self, action: ActionType, source: Source = Source.web):
+        self.action = action
+        self.source = source
+
+    def send(self):
+        newrelic.agent.record_custom_event(
+            "LoginEvent", {"action": self.action.name, "source": self.source.name}
+        )
+
+
+class RegisterEvent:
+    class ActionType(EnumE):
+        success = 0
+        failed = 1
+        catpcha_failed = 2
+        email_in_use = 3
+        invalid_email = 4
+
+    class Source(EnumE):
+        web = 0
+        api = 1
+
+    def __init__(self, action: ActionType, source: Source = Source.web):
+        self.action = action
+        self.source = source
+
+    def send(self):
+        newrelic.agent.record_custom_event(
+            "RegisterEvent", {"action": self.action.name, "source": self.source.name}
+        )

email_handler.py

@@ -285,7 +285,7 @@ def get_or_create_reply_to_contact(
         return contact
     else:
         LOG.d(
-            "create contact %s for alias %s via reply-to header",
+            "create contact %s for alias %s via reply-to header %s",
             contact_address,
             alias,
             reply_to_header,
@@ -1991,7 +1991,7 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
         return status.E510
 
     if mail_from != user.email:
-        LOG.e("Unauthorized mail_from %s %s", user, mail_from)
+        LOG.w("Unauthorized mail_from %s %s", user, mail_from)
         return status.E511
 
     user.notification = False

tests/example_emls/multipart_alternative.eml

@@ -0,0 +1,25 @@
+Content-Type: multipart/alternative; boundary="===============5006593052976639648=="
+MIME-Version: 1.0
+Subject: My subject
+From: foo@example.org
+To: bar@example.net
+
+--===============5006593052976639648==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+This is HTML
+--===============5006593052976639648==
+Content-Type: text/html; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+<html>
+          <body>
+          This is <i>HTML</i>
+          </body>
+          </html>
+
+--===============5006593052976639648==--
+

tests/test_email_utils.py

@@ -791,3 +791,45 @@ def test_is_invalid_mailbox_domain(flask_client):
     assert is_invalid_mailbox_domain("sub1.sub2.ab.cd")
 
     assert not is_invalid_mailbox_domain("xy.zt")
+
+
+def test_dmarc_result_softfail():
+    msg = load_eml_file("dmarc_gmail_softfail.eml")
+    assert DmarcCheckResult.soft_fail == get_spamd_result(msg).dmarc
+
+
+def test_dmarc_result_quarantine():
+    msg = load_eml_file("dmarc_quarantine.eml")
+    assert DmarcCheckResult.quarantine == get_spamd_result(msg).dmarc
+
+
+def test_dmarc_result_reject():
+    msg = load_eml_file("dmarc_reject.eml")
+    assert DmarcCheckResult.reject == get_spamd_result(msg).dmarc
+
+
+def test_dmarc_result_allow():
+    msg = load_eml_file("dmarc_allow.eml")
+    assert DmarcCheckResult.allow == get_spamd_result(msg).dmarc
+
+
+def test_dmarc_result_na():
+    msg = load_eml_file("dmarc_na.eml")
+    assert DmarcCheckResult.not_available == get_spamd_result(msg).dmarc
+
+
+def test_dmarc_result_bad_policy():
+    msg = load_eml_file("dmarc_bad_policy.eml")
+    assert DmarcCheckResult.bad_policy == get_spamd_result(msg).dmarc
+
+
+def test_add_header_multipart_with_invalid_part():
+    msg = load_eml_file("multipart_alternative.eml")
+    parts = msg.get_payload() + ["invalid"]
+    msg.set_payload(parts)
+    msg = add_header(msg, "INJECT", "INJECT")
+    for i, part in enumerate(msg.get_payload()):
+        if i < 2:
+            assert part.get_payload().index("INJECT") > -1
+        else:
+            assert part == "invalid"