Merge pull request #281 from simple-login/batch-import
User can batch import aliases
This commit is contained in:
commit
43babcf2d9
|
@ -225,6 +225,7 @@ JOB_ONBOARDING_1 = "onboarding-1"
|
||||||
JOB_ONBOARDING_2 = "onboarding-2"
|
JOB_ONBOARDING_2 = "onboarding-2"
|
||||||
JOB_ONBOARDING_3 = "onboarding-3"
|
JOB_ONBOARDING_3 = "onboarding-3"
|
||||||
JOB_ONBOARDING_4 = "onboarding-4"
|
JOB_ONBOARDING_4 = "onboarding-4"
|
||||||
|
JOB_BATCH_IMPORT = "batch-import"
|
||||||
|
|
||||||
# for pagination
|
# for pagination
|
||||||
PAGE_LIMIT = 20
|
PAGE_LIMIT = 20
|
||||||
|
|
|
@ -24,4 +24,5 @@ from .views import (
|
||||||
recovery_code,
|
recovery_code,
|
||||||
contact_detail,
|
contact_detail,
|
||||||
setup_done,
|
setup_done,
|
||||||
|
batch_import,
|
||||||
)
|
)
|
||||||
|
|
57
app/dashboard/templates/dashboard/batch_import.html
Normal file
57
app/dashboard/templates/dashboard/batch_import.html
Normal file
|
@ -0,0 +1,57 @@
|
||||||
|
{% extends 'default.html' %}
|
||||||
|
{% set active_page = "setting" %}
|
||||||
|
{% block title %}
|
||||||
|
Alias Batch Import
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block default_content %}
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-body">
|
||||||
|
<h1 class="h3">Alias Batch Import</h1>
|
||||||
|
<p>
|
||||||
|
The import can take several minutes.
|
||||||
|
Please come back to this page to verify the import status. <br>
|
||||||
|
Only aliases created with <b>your verified domains</b> can be imported.<br>
|
||||||
|
If an alias already exists, it won't be imported.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<a href="/static/batch_import_template.csv" download>Download CSV Template</a>
|
||||||
|
|
||||||
|
<hr>
|
||||||
|
|
||||||
|
<form method="post" enctype="multipart/form-data" class="mt-4">
|
||||||
|
<input required type="file"
|
||||||
|
name="alias-file"
|
||||||
|
accept=".csv"
|
||||||
|
class="form-control-file">
|
||||||
|
<label>Only <b>.csv</b> file is supported. </label> <br>
|
||||||
|
<button class="btn btn-success mt-2">Upload</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
{% if batch_imports %}
|
||||||
|
<hr>
|
||||||
|
<h2 class="h3 mt-7">Batch imports</h2>
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col">Uploaded</th>
|
||||||
|
<th scope="col">Number Alias Imported</th>
|
||||||
|
<th scope="col">Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for batch_import in batch_imports %}
|
||||||
|
<tr>
|
||||||
|
<td>{{ batch_import.created_at | dt }}</td>
|
||||||
|
<td>{{ batch_import.nb_alias() }}</td>
|
||||||
|
<td>{% if batch_import.processed %} Processed ✅ {% else %} Pending {% endif %}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endblock %}
|
|
@ -8,11 +8,11 @@
|
||||||
|
|
||||||
{% block head %}
|
{% block head %}
|
||||||
<style>
|
<style>
|
||||||
.card-title {
|
.card-title {
|
||||||
font-size: 22px;
|
font-size: 22px;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
margin-bottom: 3px;
|
margin-bottom: 3px;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
|
@ -156,9 +156,9 @@
|
||||||
<select class="form-control mr-sm-2" name="random-alias-default-domain">
|
<select class="form-control mr-sm-2" name="random-alias-default-domain">
|
||||||
{% for is_public, domain in current_user.available_domains_for_random_alias() %}
|
{% for is_public, domain in current_user.available_domains_for_random_alias() %}
|
||||||
<option value="{{ domain }}"
|
<option value="{{ domain }}"
|
||||||
{% if current_user.default_random_alias_domain() == domain %} selected {% endif %} >
|
{% if current_user.default_random_alias_domain() == domain %} selected {% endif %} >
|
||||||
{{ domain }} ({% if is_public %} SimpleLogin domain {% else %} your domain {% endif %})
|
{{ domain }} ({% if is_public %} SimpleLogin domain {% else %} your domain {% endif %})
|
||||||
</option>
|
</option>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</select>
|
</select>
|
||||||
<button class="btn btn-outline-primary">Update</button>
|
<button class="btn btn-outline-primary">Update</button>
|
||||||
|
@ -296,6 +296,19 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="card-title">Import alias</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
You can import your aliases created on other platforms into SimpleLogin.
|
||||||
|
</div>
|
||||||
|
<a href="{{ url_for('dashboard.batch_import_route') }}" class="btn btn-outline-primary">
|
||||||
|
Batch Import
|
||||||
|
</a>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="card">
|
<div class="card">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<div class="card-title">Export Data</div>
|
<div class="card-title">Export Data</div>
|
||||||
|
|
53
app/dashboard/views/batch_import.py
Normal file
53
app/dashboard/views/batch_import.py
Normal file
|
@ -0,0 +1,53 @@
|
||||||
|
import arrow
|
||||||
|
from flask import render_template, flash, request, redirect, url_for
|
||||||
|
from flask_login import login_required, current_user
|
||||||
|
from flask_wtf import FlaskForm
|
||||||
|
from wtforms import StringField, validators
|
||||||
|
|
||||||
|
from app import s3
|
||||||
|
from app.config import JOB_BATCH_IMPORT
|
||||||
|
from app.dashboard.base import dashboard_bp
|
||||||
|
from app.extensions import db
|
||||||
|
from app.log import LOG
|
||||||
|
from app.models import CustomDomain, File, BatchImport, Job
|
||||||
|
from app.utils import random_string
|
||||||
|
|
||||||
|
|
||||||
|
@dashboard_bp.route("/batch_import", methods=["GET", "POST"])
|
||||||
|
@login_required
|
||||||
|
def batch_import_route():
|
||||||
|
# only for users who have custom domains
|
||||||
|
if not current_user.verified_custom_domains():
|
||||||
|
flash("Alias batch import is only available for custom domains", "warning")
|
||||||
|
|
||||||
|
batch_imports = BatchImport.query.filter_by(user_id=current_user.id).all()
|
||||||
|
|
||||||
|
if request.method == "POST":
|
||||||
|
alias_file = request.files["alias-file"]
|
||||||
|
|
||||||
|
file_path = random_string(20) + ".csv"
|
||||||
|
file = File.create(user_id=current_user.id, path=file_path)
|
||||||
|
s3.upload_from_bytesio(file_path, alias_file)
|
||||||
|
db.session.flush()
|
||||||
|
LOG.d("upload file %s to s3 at %s", file, file_path)
|
||||||
|
|
||||||
|
bi = BatchImport.create(user_id=current_user.id, file_id=file.id)
|
||||||
|
db.session.flush()
|
||||||
|
LOG.debug("Add a batch import job %s for %s", bi, current_user)
|
||||||
|
|
||||||
|
# Schedule batch import job
|
||||||
|
Job.create(
|
||||||
|
name=JOB_BATCH_IMPORT,
|
||||||
|
payload={"batch_import_id": bi.id},
|
||||||
|
run_at=arrow.now(),
|
||||||
|
)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
flash(
|
||||||
|
"The file has been uploaded successfully and the import will start shortly",
|
||||||
|
"success",
|
||||||
|
)
|
||||||
|
|
||||||
|
return redirect(url_for("dashboard.batch_import_route"))
|
||||||
|
|
||||||
|
return render_template("dashboard/batch_import.html", batch_imports=batch_imports)
|
|
@ -94,6 +94,9 @@ class File(db.Model, ModelMixin):
|
||||||
def get_url(self, expires_in=3600):
|
def get_url(self, expires_in=3600):
|
||||||
return s3.get_url(self.path, expires_in)
|
return s3.get_url(self.path, expires_in)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<File {self.path}>"
|
||||||
|
|
||||||
|
|
||||||
class EnumE(enum.Enum):
|
class EnumE(enum.Enum):
|
||||||
@classmethod
|
@classmethod
|
||||||
|
@ -817,6 +820,13 @@ class Alias(db.Model, ModelMixin):
|
||||||
db.Boolean, nullable=False, default=False, server_default="0"
|
db.Boolean, nullable=False, default=False, server_default="0"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# to know whether an alias is added using a batch import
|
||||||
|
batch_import_id = db.Column(
|
||||||
|
db.ForeignKey("batch_import.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
user = db.relationship(User)
|
user = db.relationship(User)
|
||||||
mailbox = db.relationship("Mailbox", lazy="joined")
|
mailbox = db.relationship("Mailbox", lazy="joined")
|
||||||
|
|
||||||
|
@ -1742,3 +1752,19 @@ class Monitoring(db.Model, ModelMixin):
|
||||||
incoming_queue = db.Column(db.Integer, nullable=False)
|
incoming_queue = db.Column(db.Integer, nullable=False)
|
||||||
active_queue = db.Column(db.Integer, nullable=False)
|
active_queue = db.Column(db.Integer, nullable=False)
|
||||||
deferred_queue = db.Column(db.Integer, nullable=False)
|
deferred_queue = db.Column(db.Integer, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class BatchImport(db.Model, ModelMixin):
|
||||||
|
user_id = db.Column(db.ForeignKey(User.id, ondelete="cascade"), nullable=False)
|
||||||
|
file_id = db.Column(db.ForeignKey(File.id, ondelete="cascade"), nullable=False)
|
||||||
|
processed = db.Column(db.Boolean, nullable=False, default=False)
|
||||||
|
summary = db.Column(db.Text, nullable=True, default=None)
|
||||||
|
|
||||||
|
file = db.relationship(File)
|
||||||
|
user = db.relationship(User)
|
||||||
|
|
||||||
|
def nb_alias(self):
|
||||||
|
return Alias.query.filter_by(batch_import_id=self.id).count()
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f"<BatchImport {self.id}>"
|
||||||
|
|
|
@ -2,20 +2,36 @@
|
||||||
Run scheduled jobs.
|
Run scheduled jobs.
|
||||||
Not meant for running job at precise time (+- 1h)
|
Not meant for running job at precise time (+- 1h)
|
||||||
"""
|
"""
|
||||||
|
import csv
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import arrow
|
import arrow
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from app import s3
|
||||||
from app.config import (
|
from app.config import (
|
||||||
JOB_ONBOARDING_1,
|
JOB_ONBOARDING_1,
|
||||||
JOB_ONBOARDING_2,
|
JOB_ONBOARDING_2,
|
||||||
JOB_ONBOARDING_3,
|
JOB_ONBOARDING_3,
|
||||||
JOB_ONBOARDING_4,
|
JOB_ONBOARDING_4,
|
||||||
|
JOB_BATCH_IMPORT,
|
||||||
|
)
|
||||||
|
from app.email_utils import (
|
||||||
|
send_email,
|
||||||
|
render,
|
||||||
|
get_email_domain_part,
|
||||||
)
|
)
|
||||||
from app.email_utils import send_email, render
|
|
||||||
from app.extensions import db
|
from app.extensions import db
|
||||||
from app.log import LOG
|
from app.log import LOG
|
||||||
from app.models import User, Job
|
from app.models import (
|
||||||
|
User,
|
||||||
|
Job,
|
||||||
|
BatchImport,
|
||||||
|
Alias,
|
||||||
|
DeletedAlias,
|
||||||
|
DomainDeletedAlias,
|
||||||
|
CustomDomain,
|
||||||
|
)
|
||||||
from server import create_app
|
from server import create_app
|
||||||
|
|
||||||
|
|
||||||
|
@ -71,6 +87,55 @@ def onboarding_mailbox(user):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def handle_batch_import(batch_import: BatchImport):
|
||||||
|
user = batch_import.user
|
||||||
|
|
||||||
|
batch_import.processed = True
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
LOG.debug("Start batch import for %s %s", batch_import, user)
|
||||||
|
file_url = s3.get_url(batch_import.file.path)
|
||||||
|
|
||||||
|
LOG.d("Download file %s from %s", batch_import.file, file_url)
|
||||||
|
r = requests.get(file_url)
|
||||||
|
lines = [l.decode() for l in r.iter_lines()]
|
||||||
|
reader = csv.DictReader(lines)
|
||||||
|
|
||||||
|
for row in reader:
|
||||||
|
full_alias = row["alias"].lower().strip().replace(" ", "")
|
||||||
|
note = row["note"]
|
||||||
|
|
||||||
|
alias_domain = get_email_domain_part(full_alias)
|
||||||
|
custom_domain = CustomDomain.get_by(domain=alias_domain)
|
||||||
|
|
||||||
|
if (
|
||||||
|
not custom_domain
|
||||||
|
or not custom_domain.verified
|
||||||
|
or custom_domain.user_id != user.id
|
||||||
|
):
|
||||||
|
LOG.debug("domain %s can't be used %s", alias_domain, user)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if (
|
||||||
|
Alias.get_by(email=full_alias)
|
||||||
|
or DeletedAlias.get_by(email=full_alias)
|
||||||
|
or DomainDeletedAlias.get_by(email=full_alias)
|
||||||
|
):
|
||||||
|
LOG.d("alias already used %s", full_alias)
|
||||||
|
continue
|
||||||
|
|
||||||
|
alias = Alias.create(
|
||||||
|
user_id=user.id,
|
||||||
|
email=full_alias,
|
||||||
|
note=note,
|
||||||
|
mailbox_id=user.default_mailbox_id,
|
||||||
|
custom_domain_id=custom_domain.id,
|
||||||
|
batch_import_id=batch_import.id,
|
||||||
|
)
|
||||||
|
db.session.commit()
|
||||||
|
LOG.d("Create %s", alias)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
while True:
|
while True:
|
||||||
# run a job 1h earlier or later is not a big deal ...
|
# run a job 1h earlier or later is not a big deal ...
|
||||||
|
@ -129,6 +194,11 @@ if __name__ == "__main__":
|
||||||
LOG.d("send onboarding pgp email to user %s", user)
|
LOG.d("send onboarding pgp email to user %s", user)
|
||||||
onboarding_pgp(user)
|
onboarding_pgp(user)
|
||||||
|
|
||||||
|
elif job.name == JOB_BATCH_IMPORT:
|
||||||
|
batch_import_id = job.payload.get("batch_import_id")
|
||||||
|
batch_import = BatchImport.get(batch_import_id)
|
||||||
|
handle_batch_import(batch_import)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
LOG.exception("Unknown job name %s", job.name)
|
LOG.exception("Unknown job name %s", job.name)
|
||||||
|
|
||||||
|
|
44
migrations/versions/2020_091020_84471852b610_.py
Normal file
44
migrations/versions/2020_091020_84471852b610_.py
Normal file
|
@ -0,0 +1,44 @@
|
||||||
|
"""empty message
|
||||||
|
|
||||||
|
Revision ID: 84471852b610
|
||||||
|
Revises: b82bcad9accf
|
||||||
|
Create Date: 2020-09-10 20:15:10.956801
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy_utils
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '84471852b610'
|
||||||
|
down_revision = 'b82bcad9accf'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table('batch_import',
|
||||||
|
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column('created_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=False),
|
||||||
|
sa.Column('updated_at', sqlalchemy_utils.types.arrow.ArrowType(), nullable=True),
|
||||||
|
sa.Column('user_id', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('file_id', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('processed', sa.Boolean(), nullable=False),
|
||||||
|
sa.Column('summary', sa.Text(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(['file_id'], ['file.id'], ondelete='cascade'),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='cascade'),
|
||||||
|
sa.PrimaryKeyConstraint('id')
|
||||||
|
)
|
||||||
|
op.add_column('alias', sa.Column('batch_import_id', sa.Integer(), nullable=True))
|
||||||
|
op.create_foreign_key(None, 'alias', 'batch_import', ['batch_import_id'], ['id'], ondelete='SET NULL')
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_constraint(None, 'alias', type_='foreignkey')
|
||||||
|
op.drop_column('alias', 'batch_import_id')
|
||||||
|
op.drop_table('batch_import')
|
||||||
|
# ### end Alembic commands ###
|
3
static/batch_import_template.csv
vendored
Normal file
3
static/batch_import_template.csv
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
"alias","note"
|
||||||
|
"ebay@my-domain.com","Used on eBay"
|
||||||
|
"facebook@my-domain.com","Used on Facebook, Instagram."
|
|
Loading…
Reference in a new issue