add more fake data to test HIBP

.flake8

@@ -15,7 +15,9 @@ exclude =
     templates,
     # migrations are generated by alembic
     migrations,
-    docs
+    docs,
+    shell.py
+
 
 per-file-ignores =
     # ignore unused imports in __init__

app/api/serializer.py

@@ -322,6 +322,12 @@ def get_alias_infos_with_pagination_v3(
         q = q.filter(Alias.enabled)
     elif alias_filter == "disabled":
         q = q.filter(Alias.enabled.is_(False))
+    elif alias_filter == "hibp":
+        q = q.filter(
+            or_(
+                Alias.hibp_breaches_notified_user, Alias.hibp_breaches_not_notified_user
+            )
+        )
 
     q = q.order_by(Alias.pinned.desc())
 

app/dashboard/templates/dashboard/index.html

@@ -143,6 +143,9 @@
               <option value="disabled" {% if filter == "disabled" %} selected {% endif %}>
                 Only Disabled Aliases
               </option>
+              <option value="hibp" {% if filter == "hibp" %} selected {% endif %}>
+                Only Aliases Found In Data Breaches
+              </option>
             </select>
 
             <input type="search" name="query" placeholder="Enter to search for alias"

app/models.py

@@ -162,9 +162,23 @@ class AliasGeneratorEnum(EnumE):
     uuid = 2  # aliases are generated based on uuid
 
 
+class HibpDataClass(db.Model, ModelMixin):
+    __tablename__ = "hibpdataclass"
+    attribute = db.Column(db.Text, nullable=False, unique=True, index=True)
+
+    breaches = db.relationship("Hibp", secondary="hibp_hibpdataclass")
+
+    def __repr__(self):
+        return f"<HIBP Data Class {self.id} {self.description}>"
+
+
 class Hibp(db.Model, ModelMixin):
     __tablename__ = "hibp"
     name = db.Column(db.String(), nullable=False, unique=True, index=True)
+    description = db.Column(db.Text)
+    date = db.Column(ArrowType, nullable=True)
+    data_classes = db.relationship("HibpDataClass", secondary="hibp_hibpdataclass")
+
     breached_aliases = db.relationship("Alias", secondary="alias_hibp")
 
     def __repr__(self):
@@ -1062,11 +1076,16 @@ class Alias(db.Model, ModelMixin):
 
     # have I been pwned
     hibp_last_check = db.Column(ArrowType, default=None)
-    hibp_breaches = db.relationship("Hibp", secondary="alias_hibp")
+    hibp_breaches_notified_user = db.relationship("Hibp", secondary="alias_hibp")
+    hibp_breaches_not_notified_user = db.relationship("Hibp", secondary="alias_hibp")
 
     user = db.relationship(User, foreign_keys=[user_id])
     mailbox = db.relationship("Mailbox", lazy="joined")
 
+    @property
+    def hibp_breaches(self):
+        return self.hibp_breaches_notified_user + self.hibp_breaches_not_notified_user
+
     @property
     def mailboxes(self):
         ret = [self.mailbox]
@@ -2061,6 +2080,27 @@ class DomainMailbox(db.Model, ModelMixin):
     )
 
 
+class HibpHibpDataClass(db.Model, ModelMixin):
+    __tablename__ = "hibp_hibpdataclass"
+
+    __table_args__ = (
+        db.UniqueConstraint(
+            "hibp_id", "hibp_dataclass_id", name="uq_hibp_hibpdataclass"
+        ),
+    )
+
+    hibp_id = db.Column(db.Integer(), db.ForeignKey("hibp.id"), index=True)
+    hibp_dataclass_id = db.Column(db.Integer(), db.ForeignKey("hibpdataclass.id"))
+
+    hibp = db.relationship(
+        "Hibp", backref=db.backref("hibp_hibpdataclass", cascade="all, delete-orphan")
+    )
+    hibpdataclass = db.relationship(
+        "HibpDataClass",
+        backref=db.backref("hibp_hibpdataclass", cascade="all, delete-orphan"),
+    )
+
+
 _NB_RECOVERY_CODE = 8
 _RECOVERY_CODE_LENGTH = 8
 

cron.py

@@ -56,6 +56,7 @@ from app.models import (
     DeletedAlias,
     DomainDeletedAlias,
     Hibp,
+    HibpDataClass,
 )
 from app.utils import sanitize_email
 from server import create_app
@@ -793,14 +794,12 @@ async def _hibp_check(api_key, queue):
 
         if r.status_code == 200:
             # Breaches found
-            alias.hibp_breaches = [
-                Hibp.get_by(name=entry["Name"]) for entry in r.json()
-            ]
+            breaches = [Hibp.get_by(name=entry["Name"]) for entry in r.json()]
             if len(alias.hibp_breaches) > 0:
                 LOG.w("%s appears in HIBP breaches %s", alias, alias.hibp_breaches)
         elif r.status_code == 404:
             # No breaches found
-            alias.hibp_breaches = []
+            breaches = []
         else:
             LOG.error(
                 "An error occured while checking alias %s: %s - %s",
@@ -810,6 +809,17 @@ async def _hibp_check(api_key, queue):
             )
             return
 
+        alias.hibp_breaches_not_notified_user = [
+            breach
+            for breach in breaches
+            if breach not in alias.hibp_breaches_notified_user
+        ]
+        alias.hibp_breaches_notified_user = [
+            breach
+            for breach in breaches
+            if breach not in alias.hibp_breaches_not_notified_user
+        ]
+
         alias.hibp_last_check = arrow.utcnow()
         db.session.add(alias)
         db.session.commit()
@@ -829,10 +839,23 @@ async def check_hibp():
         LOG.exception("No HIBP API keys")
         return
 
+    LOG.d("Updating list of known breach types")
+    r = requests.get("https://haveibeenpwned.com/api/v3/dataclasses")
+    for entry in r.json():
+        HibpDataClass.get_or_create(attribute=entry)
+
+    db.session.commit()
+
     LOG.d("Updating list of known breaches")
     r = requests.get("https://haveibeenpwned.com/api/v3/breaches")
     for entry in r.json():
-        Hibp.get_or_create(name=entry["Name"])
+        hibp_entry = Hibp.get_or_create(name=entry["Name"])
+        hibp_entry.date = arrow.get(entry["BreachDate"])
+        hibp_entry.description = entry["Description"]
+        hibp_entry.data_classes = [
+            HibpDataClass.get_by(attribute=attribute)
+            for attribute in entry["DataClasses"]
+        ]
 
     db.session.commit()
     LOG.d("Updated list of known breaches")
@@ -872,6 +895,48 @@ async def check_hibp():
     LOG.d("Done checking HIBP API for aliases in breaches")
 
 
+def notify_hibp():
+    """
+    Send aggregated email reports for HIBP breaches
+    """
+    for user in User.query.all():
+        new_breaches = (
+            Alias.query.filter(Alias.user_id == user.id)
+            .filter(Alias.hibp_breaches_not_notified_user)
+            .all()
+        )
+
+        if len(new_breaches) == 0:
+            return
+
+        LOG.d(f"Send new breaches found email to user {user}")
+
+        send_email(
+            user.email,
+            f"You were in a data breach",
+            render(
+                "transactional/hibp-new-breaches.txt.jinja2",
+                user=user,
+                breached_aliases=new_breaches,
+            ),
+            render(
+                "transactional/hibp-new-breaches.html",
+                user=user,
+                breached_aliases=new_breaches,
+            ),
+        )
+
+        for alias in new_breaches:
+            alias.hibp_breaches_notified_user = (
+                alias.hibp_breaches_notified_user
+                + alias.hibp_breaches_not_notified_user
+            )
+            alias.hibp_breaches_not_notified_user = []
+            db.session.add(alias)
+
+        db.session.commit()
+
+
 if __name__ == "__main__":
     LOG.d("Start running cronjob")
     parser = argparse.ArgumentParser()
@@ -891,6 +956,7 @@ if __name__ == "__main__":
             "delete_old_monitoring",
             "check_custom_domain",
             "check_hibp",
+            "notify_hibp",
         ],
     )
     args = parser.parse_args()
@@ -928,3 +994,6 @@ if __name__ == "__main__":
         elif args.job == "check_hibp":
             LOG.d("Check HIBP")
             asyncio.run(check_hibp())
+        elif args.job == "notify_hibp":
+            LOG.d("Notify users with new HIBP breaches")
+            notify_hibp()

crontab.yml

@@ -58,4 +58,11 @@ jobs:
     shell: /bin/bash
     schedule: "0 18 * * *"
     captureStderr: true
+    concurrencyPolicy: Forbid
+
+  - name: SimpleLogin HIBP report
+    command: python /code/cron.py -j notify_hibp
+    shell: /bin/bash
+    schedule: "0 16 * * *"
+    captureStderr: true
     concurrencyPolicy: Forbid

example.env

@@ -171,5 +171,5 @@ DISABLE_ONBOARDING=true
 # ENABLE_SPAM_ASSASSIN = 1
 
 # Have I Been Pwned
-# HIBP_SCAN_INTERVAL_DAYS = 7
+# HIBP_SCAN_INTERVAL_DAYS = 1
 # HIBP_API_KEYS=[]

migrations/versions/2021_060119_4d501f682763_.py

@@ -0,0 +1,55 @@
+"""empty message
+
+Revision ID: 4d501f682763
+Revises: 6cc7f073b358
+Create Date: 2021-06-01 19:30:15.904980
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4d501f682763'
+down_revision = '6cc7f073b358'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('hibpdataclass',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+    sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
+    sa.Column('attribute', sa.Text(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_hibpdataclass_attribute'), 'hibpdataclass', ['attribute'], unique=True)
+    op.create_table('hibp_hibpdataclass',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+    sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
+    sa.Column('hibp_id', sa.Integer(), nullable=True),
+    sa.Column('hibp_dataclass_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['hibp_dataclass_id'], ['hibpdataclass.id'], ),
+    sa.ForeignKeyConstraint(['hibp_id'], ['hibp.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('hibp_id', 'hibp_dataclass_id', name='uq_hibp_hibpdataclass')
+    )
+    op.create_index(op.f('ix_hibp_hibpdataclass_hibp_id'), 'hibp_hibpdataclass', ['hibp_id'], unique=False)
+    op.add_column('hibp', sa.Column('date', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True))
+    op.add_column('hibp', sa.Column('description', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('hibp', 'description')
+    op.drop_column('hibp', 'date')
+    op.drop_index(op.f('ix_hibp_hibpdataclass_hibp_id'), table_name='hibp_hibpdataclass')
+    op.drop_table('hibp_hibpdataclass')
+    op.drop_index(op.f('ix_hibpdataclass_attribute'), table_name='hibpdataclass')
+    op.drop_table('hibpdataclass')
+    # ### end Alembic commands ###

server.py

@@ -298,8 +298,9 @@ def fake_data():
     m1.pgp_finger_print = load_public_key(pgp_public_key)
     db.session.commit()
 
-    # example@example.com is in a LOT of data breaches
+    # example@example.com and hey@example.com is in a LOT of data breaches
     Alias.create(email="example@example.com", user_id=user.id, mailbox_id=m1.id)
+    Alias.create(email="hey@example.com", user_id=user.id, mailbox_id=m1.id)
 
     for i in range(3):
         if i % 2 == 0:

shell.py

@@ -6,12 +6,7 @@ from sqlalchemy_utils import create_database, database_exists, drop_database
 
 from app.config import DB_URI
 from app.email_utils import send_email, render
-from app.extensions import db
-from app.log import LOG
-from app.models import (
-    User,
-    Mailbox,
-)
+from app.models import *
 from job_runner import (
     onboarding_pgp,
     onboarding_browser_extension,

templates/emails/transactional/hibp-new-breaches.html

@@ -0,0 +1,44 @@
+{% extends "base.html" %}
+
+{% block content %}
+  {% call text() %}
+    <h1>
+      {{ breached_aliases|count }} of your aliases are found in data breaches.
+    </h1>
+  {% endcall %}
+
+  <ol>
+    {%- for alias in breached_aliases[:10] %}
+      <li>{% call text() %}
+        <b>{{ alias.email }}</b> was found in {{ alias.hibp_breaches_not_notified_user|count }} data breaches. <br>
+
+        <ul>
+          {% set breaches = alias.hibp_breaches_not_notified_user|sort(attribute='date', reverse=True) %}
+          {%- for breach in breaches[:4] %}
+            <li>
+              <b>{{ breach.name }}</b> ({{ breach.date.format('YYYY-MM-DD') }}).
+              {{ breach.description }}
+              Breached data includes {{ breach.data_classes|join(', ', attribute='attribute') }}.
+            </li>
+          {%- endfor %}
+        </ul>
+
+        {% if  breaches|length > 4 %}
+          And {{ breaches|length - 4 }} more data breaches...
+        {% endif %}
+
+      {% endcall %}</li>
+    {%- endfor %}
+
+  </ol>
+
+  {% if  breached_aliases|length > 10 %}
+    {% call text() %}
+      And {{ breached_aliases|length - 10 }} more aliases...
+    {% endcall %}
+  {% endif %}
+
+  {{ render_text("For more information, please check <a href='https://haveibeenpwned.com/'>HaveIBeenPwned.com</a>.") }}
+
+  {{ render_text('Best, <br />SimpleLogin Team.') }}
+{% endblock %}

templates/emails/transactional/hibp-new-breaches.txt.jinja2

@@ -0,0 +1,24 @@
+{{ breached_aliases|count }} of your aliases are found in data breaches.
+
+{% for alias in breached_aliases[:10] %}
+  {{ loop.index }} )  {{ alias.email }} was found in {{ alias.hibp_breaches_not_notified_user|count }} data breaches.
+
+  {%- set breaches = alias.hibp_breaches_not_notified_user|sort(attribute='date', reverse=True) %}
+  {% for breach in breaches[:4] %}
+    - {{ breach.name }} ({{ breach.date.format('YYYY-MM-DD') }}). Breached data includes {{ breach.data_classes|join(', ', attribute='attribute') }}.
+  {%- endfor %}
+
+
+  {%- if  breaches|length > 4 %}
+    And {{ breaches|length - 4 }} more data breaches...
+  {% endif %}
+{% endfor %}
+
+{%- if  breached_aliases|length > 10 %}
+  And {{ breached_aliases|length - 10 }} more aliases...
+{%- endif %}
+
+For more information, please check https://haveibeenpwned.com/.
+
+Best,
+SimpleLogin Team.