Export Data
diff --git a/app/dashboard/views/batch_import.py b/app/dashboard/views/batch_import.py
new file mode 100644
index 00000000..00e4a363
--- /dev/null
+++ b/app/dashboard/views/batch_import.py
@@ -0,0 +1,53 @@
+import arrow
+from flask import render_template, flash, request, redirect, url_for
+from flask_login import login_required, current_user
+from flask_wtf import FlaskForm
+from wtforms import StringField, validators
+
+from app import s3
+from app.config import JOB_BATCH_IMPORT
+from app.dashboard.base import dashboard_bp
+from app.extensions import db
+from app.log import LOG
+from app.models import CustomDomain, File, BatchImport, Job
+from app.utils import random_string
+
+
+@dashboard_bp.route("/batch_import", methods=["GET", "POST"])
+@login_required
+def batch_import_route():
+ # only for users who have custom domains
+ if not current_user.verified_custom_domains():
+ flash("Alias batch import is only available for custom domains", "warning")
+
+ batch_imports = BatchImport.query.filter_by(user_id=current_user.id).all()
+
+ if request.method == "POST":
+ alias_file = request.files["alias-file"]
+
+ file_path = random_string(20) + ".csv"
+ file = File.create(user_id=current_user.id, path=file_path)
+ s3.upload_from_bytesio(file_path, alias_file)
+ db.session.flush()
+ LOG.d("upload file %s to s3 at %s", file, file_path)
+
+ bi = BatchImport.create(user_id=current_user.id, file_id=file.id)
+ db.session.flush()
+ LOG.debug("Add a batch import job %s for %s", bi, current_user)
+
+ # Schedule batch import job
+ Job.create(
+ name=JOB_BATCH_IMPORT,
+ payload={"batch_import_id": bi.id},
+ run_at=arrow.now(),
+ )
+ db.session.commit()
+
+ flash(
+ "The file has been uploaded successfully and the import will start shortly",
+ "success",
+ )
+
+ return redirect(url_for("dashboard.batch_import_route"))
+
+ return render_template("dashboard/batch_import.html", batch_imports=batch_imports)
diff --git a/app/models.py b/app/models.py
index 27c464e4..ae3854c2 100644
--- a/app/models.py
+++ b/app/models.py
@@ -94,6 +94,9 @@ class File(db.Model, ModelMixin):
def get_url(self, expires_in=3600):
return s3.get_url(self.path, expires_in)
+ def __repr__(self):
+ return f"
"
+
class EnumE(enum.Enum):
@classmethod
@@ -817,6 +820,13 @@ class Alias(db.Model, ModelMixin):
db.Boolean, nullable=False, default=False, server_default="0"
)
+ # to know whether an alias is added using a batch import
+ batch_import_id = db.Column(
+ db.ForeignKey("batch_import.id", ondelete="SET NULL"),
+ nullable=True,
+ default=None,
+ )
+
user = db.relationship(User)
mailbox = db.relationship("Mailbox", lazy="joined")
@@ -1742,3 +1752,19 @@ class Monitoring(db.Model, ModelMixin):
incoming_queue = db.Column(db.Integer, nullable=False)
active_queue = db.Column(db.Integer, nullable=False)
deferred_queue = db.Column(db.Integer, nullable=False)
+
+
+class BatchImport(db.Model, ModelMixin):
+ user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False)
+ file_id = db.Column(db.ForeignKey(File.id, ondelete="cascade"), nullable=False)
+ processed = db.Column(db.Boolean, nullable=False, default=False)
+ summary = db.Column(db.Text, nullable=True, default=None)
+
+ file = db.relationship(File)
+ user = db.relationship(User)
+
+ def nb_alias(self):
+ return Alias.query.filter_by(batch_import_id=self.id).count()
+
+ def __repr__(self):
+ return f""
diff --git a/job_runner.py b/job_runner.py
index 70c580b5..c015731b 100644
--- a/job_runner.py
+++ b/job_runner.py
@@ -2,20 +2,36 @@
Run scheduled jobs.
Not meant for running job at precise time (+- 1h)
"""
+import csv
import time
import arrow
+import requests
+from app import s3
from app.config import (
JOB_ONBOARDING_1,
JOB_ONBOARDING_2,
JOB_ONBOARDING_3,
JOB_ONBOARDING_4,
+ JOB_BATCH_IMPORT,
+)
+from app.email_utils import (
+ send_email,
+ render,
+ get_email_domain_part,
)
-from app.email_utils import send_email, render
from app.extensions import db
from app.log import LOG
-from app.models import User, Job
+from app.models import (
+ User,
+ Job,
+ BatchImport,
+ Alias,
+ DeletedAlias,
+ DomainDeletedAlias,
+ CustomDomain,
+)
from server import create_app
@@ -71,6 +87,55 @@ def onboarding_mailbox(user):
)
+def handle_batch_import(batch_import: BatchImport):
+ user = batch_import.user
+
+ batch_import.processed = True
+ db.session.commit()
+
+ LOG.debug("Start batch import for %s %s", batch_import, user)
+ file_url = s3.get_url(batch_import.file.path)
+
+ LOG.d("Download file %s from %s", batch_import.file, file_url)
+ r = requests.get(file_url)
+ lines = [l.decode() for l in r.iter_lines()]
+ reader = csv.DictReader(lines)
+
+ for row in reader:
+ full_alias = row["alias"].lower().strip().replace(" ", "")
+ note = row["note"]
+
+ alias_domain = get_email_domain_part(full_alias)
+ custom_domain = CustomDomain.get_by(domain=alias_domain)
+
+ if (
+ not custom_domain
+ or not custom_domain.verified
+ or custom_domain.user_id != user.id
+ ):
+ LOG.debug("domain %s can't be used %s", alias_domain, user)
+ continue
+
+ if (
+ Alias.get_by(email=full_alias)
+ or DeletedAlias.get_by(email=full_alias)
+ or DomainDeletedAlias.get_by(email=full_alias)
+ ):
+ LOG.d("alias already used %s", full_alias)
+ continue
+
+ alias = Alias.create(
+ user_id=user.id,
+ email=full_alias,
+ note=note,
+ mailbox_id=user.default_mailbox_id,
+ custom_domain_id=custom_domain.id,
+ batch_import_id=batch_import.id,
+ )
+ db.session.commit()
+ LOG.d("Create %s", alias)
+
+
if __name__ == "__main__":
while True:
# run a job 1h earlier or later is not a big deal ...
@@ -129,6 +194,11 @@ if __name__ == "__main__":
LOG.d("send onboarding pgp email to user %s", user)
onboarding_pgp(user)
+ elif job.name == JOB_BATCH_IMPORT:
+ batch_import_id = job.payload.get("batch_import_id")
+ batch_import = BatchImport.get(batch_import_id)
+ handle_batch_import(batch_import)
+
else:
LOG.exception("Unknown job name %s", job.name)
diff --git a/migrations/versions/2020_091020_84471852b610_.py b/migrations/versions/2020_091020_84471852b610_.py
new file mode 100644
index 00000000..f2acc52b
--- /dev/null
+++ b/migrations/versions/2020_091020_84471852b610_.py
@@ -0,0 +1,44 @@
+"""empty message
+
+Revision ID: 84471852b610
+Revises: b82bcad9accf
+Create Date: 2020-09-10 20:15:10.956801
+
+"""
+import sqlalchemy_utils
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '84471852b610'
+down_revision = 'b82bcad9accf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('batch_import',
+ sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+ sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
+ sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
+ sa.Column('user_id', sa.Integer(), nullable=False),
+ sa.Column('file_id', sa.Integer(), nullable=False),
+ sa.Column('processed', sa.Boolean(), nullable=False),
+ sa.Column('summary', sa.Text(), nullable=True),
+ sa.ForeignKeyConstraint(['file_id'], ['file.id'], ondelete='cascade'),
+ sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.add_column('alias', sa.Column('batch_import_id', sa.Integer(), nullable=True))
+ op.create_foreign_key(None, 'alias', 'batch_import', ['batch_import_id'], ['id'], ondelete='SET NULL')
+ # ### end Alembic commands ###
+
+
+def downgrade():
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_constraint(None, 'alias', type_='foreignkey')
+ op.drop_column('alias', 'batch_import_id')
+ op.drop_table('batch_import')
+ # ### end Alembic commands ###
diff --git a/static/batch_import_template.csv b/static/batch_import_template.csv
new file mode 100644
index 00000000..c9af19f0
--- /dev/null
+++ b/static/batch_import_template.csv
@@ -0,0 +1,3 @@
+"alias","note"
+"ebay@my-domain.com","Used on eBay"
+"facebook@my-domain.com","Used on Facebook, Instagram."
\ No newline at end of file