2020-02-02 15:17:04 +00:00
|
|
|
import argparse
|
2021-05-18 19:18:07 +00:00
|
|
|
import asyncio
|
2021-05-13 20:44:16 +00:00
|
|
|
import urllib.parse
|
2020-06-11 22:09:53 +00:00
|
|
|
from time import sleep
|
2021-03-23 09:02:14 +00:00
|
|
|
from typing import List, Tuple
|
2020-02-02 15:17:04 +00:00
|
|
|
|
create BaseForm to enable CSRF
register page
redirect user to dashboard if they are logged in
enable csrf for login page
Set models more strict
bootstrap developer page
add helper method to ModelMixin, remove CRUDMixin
display list of clients on developer index, add copy client-secret to clipboard using clipboardjs
add toastr and use jquery non slim
display a toast when user copies the client-secret
create new client, generate client-id using unidecode
client detail page: can edit client
add delete client
implement /oauth/authorize and /oauth/allow-deny
implement /oauth/token
add /oauth/user_info endpoint
handle scopes: wip
take into account scope: display scope, return user data according to scope
create virtual-domain, gen email, client_user model WIP
create authorize_nonlogin_user page
user can choose to generate a new email
no need to interfere with root logger
log for before and after request
if user has already allowed a client: generate a auth-code and redirect user to client
get_user_info takes into account gen email
display list of clients that have user has authorised
use yk-client domain instead of localhost as cookie depends on the domain name
use wtforms instead of flask_wtf
Dockerfile
delete virtual domain
EMAIL_DOMAIN can come from env var
bind to host 0.0.0.0
fix signup error: use session as default csrf_context
rename yourkey to simplelogin
add python-dotenv, ipython, sqlalchemy_utils
create DB_URI, FLASK_SECRET. Load config from CONFIG file if exist
add shortcuts to logging
create shell
add psycopg2
do not add local data in Dockerfile
add drop_db into shell
add shell.prepare_db()
fix prepare_db
setup sentry
copy assets from tabler/dist
add icon downloaded from https://commons.wikimedia.org/wiki/File:Simpleicons_Interface_key-tool-1.svg
integrate tabler - login and register page
add favicon
template: default, header. Use gravatar for user avatar url
use default template for dashboard, developer page
use another icon
add clipboard and notie
prettify dashboard
add notie css
add fake gen email and client-user
prettify list client page, use notie for toast
add email, name scope to new client
display client scope in client list
prettify new-client, client-detail
add sentry-sdk and blinker
add arrow, add dt jinja filter, prettify logout, dashboard
comment "last used" in dashboard for now
prettify date display
add copy email to clipboard to dashboard
use "users" as table name for User as "user" is reserved key in postgres
call prepare_db() when creating new db
error page 400, 401, 403, 404
prettify authorize_login_user
create already_authorize.html for user who has already authorized a client
user can generate new email
display all other generated emails
add ENV variable, only reset DB when ENV=local
fix: not return other users gen emails
display nb users for each client
refactor shell: remove prepare_db()
add sendgrid
add /favicon.ico route
add new config: URL, SUPPORT_EMAIL, SENDGRID_API_KEY
user needs to activate their account before login
create copy button on dashboard
client can have multiple redirect uris, in client detail can add/remove redirect-uri,
use redirect_uri passed in /authorize
refactor: move get_user_info into ClientUser model
dashboard: display all apps, all generated emails
add "id" into user_info
add trigger email button
invalidate the session at each new version by changing the secret
centralize Client creation into Client.create_new
user can enable/disable email forwarding
setup auto dismiss alert: just add .alert-auto-dismiss
move name down in register form
add shell.add_real_data
move blueprint template to its own package
prettify authorize page for non-authenticated user
update readme, return error if not redirect_uri
add flask-wtf, use psycopg2-binary
use flask-wtf FlaskForm instead of Form
rename email -> email_utils
add AWS_REGION, BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY to config
add s3 module
add File model, add Client.icon_id
handle client icon update
can create client with icon
display client icon in client list page
add Client.home_url
take into account Client.home_url
add boto3
register: ask name first
only show "trigger test email" if email forwarding is enabled
display gen email in alphabetical order, client in client.name alphabetical order
better error page
the modal does not get close when user clicks outside of modal
add Client.published column
discover page that displays all published Client
add missing bootstrap.bundle.min.js.map
developer can publish/unpublish their app in discover
use notie for display flash message
create hotmail account
fix missing jquery
add footer, add global jinja2 variable
strengthen model: use nullable=False whenever possible,
rename client_id to oauth_client_id, client_secret to oauth_client_secret
add flask-migrate
init migrate
1st migrate version
fix rename client_id -> oauth_client_id
prettify UI
use flask_migrate.upgrade() instead of db.create_all()
make sure requirejs.config is called for all page
enable sentry for js, use uppercase for global jinja2 variables
add flask-admin
add User.is_admin column
setup flask admin, only accessible to admin user
fix migration: add server_default
replace session[redirect_after_login] by "next" request args
add pyproject.toml: ignore migrations/ in black
add register waiting_activation_email page
better email wording
add pytest
add get_host_name_and_scheme and tests
example fail test
fix test
fix client-id display
add flask-cors
/user_info supports cors, add /me as /user_info synonym
return client in /me
support implicit flow
no need to use with "app.app_context()"
add watchtower to requirement
add param ENABLE_CLOUDWATCH, CLOUDWATCH_LOG_GROUP, CLOUDWATCH_LOG_STREAM
add cloudwatch logger if cloudwatch is enabled
add 500 error page
add help text for list of used client
display list of app/website that an email has been used
click on client name brings to client detail page
create style.css to add additional style, append its url with the current sha1 to avoid cache
POC on how to send email using postfix
add sqlalchemy-utils
use arrow instead of datetime
add new params STRIPE_API, STRIPE_YEARLY_SKU, STRIPE_MONTHLY_PLAN
show full error in local
add plan, plan_expiration to User, need to create enum directly in migration script, cf https://github.com/sqlalchemy/alembic/issues/67
reformat all html files: use space instead of tab
new user will have trial plan for 15 days
add new param MAX_NB_EMAIL_FREE_PLAN
only user with enough quota can create new email
if user cannot create new gen email, pick randomly one from existing gen emails. Use flush instead of commit
rename STRIPE_YEARLY_SKU -> STRIPE_YEARLY_PLAN
open client page in discover in a new tab
add stripe
not logging /static call: disable flask logging, replace by after_request
add param STRIPE_SECRET_KEY
add 3 columns stripe_customer_id, stripe_card_token, stripe_subscription_id
user can upgrade their pricing
add setting page as coming-soon
add GenEmail, ClientUser to admin
ignore /admin/static logging
add more fake data
add ondelete="cascade" whenever possible
rename plan_expiration -> trial_expiration
reset migration: delete old migrations, create new one
rename test_send_email -> poc_send_email to avoid the file being called by pytest
add new param LYRA_ANALYTICS_ID, add lyra analytics
add how to create new migration into readme
add drift to base.html
notify admin when new user signs up or pays subscription
log exception in case of 500
use sendgrid to notify admin
add alias /userinfo to user_info endpoint
add change_password to shell
add info on how payment is handled
invite user to retry if card not working
remove drift and add "contact us" link
move poc_send_email into poc/
support getting client-id, client-secret from form-data in addition to basic auth
client-id, client-secret is passed in form-data by passport-oauth2 for ex
add jwtRS256 private and public key
add jwk-jws-jwt poc
add new param OPENID_PRIVATE_KEY_PATH, OPENID_PRIVATE_KEY_PATH
add scope, redirect_url to AuthorizationCode and OauthToken
take into scope when creating oauth-token, authorization-code
add jwcrypto
add jose_utils: make_id_token and verify_id_token
add &scope to redirect uri
add "email_verified": True into user_info
fix user not activated
add /oauth2 as alias for /oauth
handle case where scope and state are empty
remove threaded=False
Use Email Alias as wording
remove help text
user can re-send activation email
add "expired" into ActivationCode
Handle the case activation code is expired
reformat: use form.validate_on_submit instead of request.method == post && form.validate
use error text instead of flash()
display client oauth-id and oauth-secret on client detail page
not display oauth-secret on client listing
fix expiration check
improve page title, footer
add /jwks and /.well-known/openid-configuration
init properly tests, fix blueprint conflict bug in flask-admin
create oauth_models module
rename Scope -> ScopeE to distinguish with Scope DB model
set app.url_map.strict_slashes = False
use ScopeE instead of SCOPE_NAME, ...
support access_token passed as args in /userinfo
merge /allow-deny into /authorize
improve wording
take into account the case response_type=code and openid is in scope
take into account response_type=id_token, id_token token, id_token code
make sure to use in-memory db in test
fix scope can be null
allow cross_origin for /.well-known/openid-configuration and /jwks
fix footer link
center authorize form
rename trial_expiration to plan_expiration
move stripe init to create_app()
use real email to be able to receive email notification
add user.profile_picture_id column
use user profile picture and fallback to gravatar
use nguyenkims+local@gm to distinguish with staging
handle plan cancel, reactivation, user profile update
fix can_create_new_email
create cron.py that set plan to free when expired
add crontab.yml
add yacron
use notify_admin instead of LOG.error
add ResetPasswordCode model
user can change password in setting
increase display time for notie
add forgot_password page
If login error: redirect to this page upon success login.
hide discover tab
add column user.is_developer
only show developer menu to developer
comment out the publish button
set local user to developer
make sure only developer can access /developer blueprint
User is invited to upgrade if they are in free plan or their trial ends soon
not sending email when in local mode
create Partner model
create become partner page
use normal error handling on local
fix migration
add "import sqlalchemy_utils" into migration template
small refactoring on setting page
handle promo code. TODO: add migration file
add migration for user.promo_codes
move email alias on top of apps in dashboard
add introjs
move encode_url to utils
create GenEmail.create_new_gen_email
create a first alias mail to show user how to use when they login
show intro when user visits the website the first time
fix register
2019-07-02 07:20:12 +00:00
|
|
|
import arrow
|
2021-05-13 20:44:16 +00:00
|
|
|
import requests
|
2021-05-17 16:15:12 +00:00
|
|
|
from sqlalchemy import func, desc, or_
|
2021-07-29 07:35:00 +00:00
|
|
|
from sqlalchemy.orm import joinedload
|
create BaseForm to enable CSRF
register page
redirect user to dashboard if they are logged in
enable csrf for login page
Set models more strict
bootstrap developer page
add helper method to ModelMixin, remove CRUDMixin
display list of clients on developer index, add copy client-secret to clipboard using clipboardjs
add toastr and use jquery non slim
display a toast when user copies the client-secret
create new client, generate client-id using unidecode
client detail page: can edit client
add delete client
implement /oauth/authorize and /oauth/allow-deny
implement /oauth/token
add /oauth/user_info endpoint
handle scopes: wip
take into account scope: display scope, return user data according to scope
create virtual-domain, gen email, client_user model WIP
create authorize_nonlogin_user page
user can choose to generate a new email
no need to interfere with root logger
log for before and after request
if user has already allowed a client: generate a auth-code and redirect user to client
get_user_info takes into account gen email
display list of clients that have user has authorised
use yk-client domain instead of localhost as cookie depends on the domain name
use wtforms instead of flask_wtf
Dockerfile
delete virtual domain
EMAIL_DOMAIN can come from env var
bind to host 0.0.0.0
fix signup error: use session as default csrf_context
rename yourkey to simplelogin
add python-dotenv, ipython, sqlalchemy_utils
create DB_URI, FLASK_SECRET. Load config from CONFIG file if exist
add shortcuts to logging
create shell
add psycopg2
do not add local data in Dockerfile
add drop_db into shell
add shell.prepare_db()
fix prepare_db
setup sentry
copy assets from tabler/dist
add icon downloaded from https://commons.wikimedia.org/wiki/File:Simpleicons_Interface_key-tool-1.svg
integrate tabler - login and register page
add favicon
template: default, header. Use gravatar for user avatar url
use default template for dashboard, developer page
use another icon
add clipboard and notie
prettify dashboard
add notie css
add fake gen email and client-user
prettify list client page, use notie for toast
add email, name scope to new client
display client scope in client list
prettify new-client, client-detail
add sentry-sdk and blinker
add arrow, add dt jinja filter, prettify logout, dashboard
comment "last used" in dashboard for now
prettify date display
add copy email to clipboard to dashboard
use "users" as table name for User as "user" is reserved key in postgres
call prepare_db() when creating new db
error page 400, 401, 403, 404
prettify authorize_login_user
create already_authorize.html for user who has already authorized a client
user can generate new email
display all other generated emails
add ENV variable, only reset DB when ENV=local
fix: not return other users gen emails
display nb users for each client
refactor shell: remove prepare_db()
add sendgrid
add /favicon.ico route
add new config: URL, SUPPORT_EMAIL, SENDGRID_API_KEY
user needs to activate their account before login
create copy button on dashboard
client can have multiple redirect uris, in client detail can add/remove redirect-uri,
use redirect_uri passed in /authorize
refactor: move get_user_info into ClientUser model
dashboard: display all apps, all generated emails
add "id" into user_info
add trigger email button
invalidate the session at each new version by changing the secret
centralize Client creation into Client.create_new
user can enable/disable email forwarding
setup auto dismiss alert: just add .alert-auto-dismiss
move name down in register form
add shell.add_real_data
move blueprint template to its own package
prettify authorize page for non-authenticated user
update readme, return error if not redirect_uri
add flask-wtf, use psycopg2-binary
use flask-wtf FlaskForm instead of Form
rename email -> email_utils
add AWS_REGION, BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY to config
add s3 module
add File model, add Client.icon_id
handle client icon update
can create client with icon
display client icon in client list page
add Client.home_url
take into account Client.home_url
add boto3
register: ask name first
only show "trigger test email" if email forwarding is enabled
display gen email in alphabetical order, client in client.name alphabetical order
better error page
the modal does not get close when user clicks outside of modal
add Client.published column
discover page that displays all published Client
add missing bootstrap.bundle.min.js.map
developer can publish/unpublish their app in discover
use notie for display flash message
create hotmail account
fix missing jquery
add footer, add global jinja2 variable
strengthen model: use nullable=False whenever possible,
rename client_id to oauth_client_id, client_secret to oauth_client_secret
add flask-migrate
init migrate
1st migrate version
fix rename client_id -> oauth_client_id
prettify UI
use flask_migrate.upgrade() instead of db.create_all()
make sure requirejs.config is called for all page
enable sentry for js, use uppercase for global jinja2 variables
add flask-admin
add User.is_admin column
setup flask admin, only accessible to admin user
fix migration: add server_default
replace session[redirect_after_login] by "next" request args
add pyproject.toml: ignore migrations/ in black
add register waiting_activation_email page
better email wording
add pytest
add get_host_name_and_scheme and tests
example fail test
fix test
fix client-id display
add flask-cors
/user_info supports cors, add /me as /user_info synonym
return client in /me
support implicit flow
no need to use with "app.app_context()"
add watchtower to requirement
add param ENABLE_CLOUDWATCH, CLOUDWATCH_LOG_GROUP, CLOUDWATCH_LOG_STREAM
add cloudwatch logger if cloudwatch is enabled
add 500 error page
add help text for list of used client
display list of app/website that an email has been used
click on client name brings to client detail page
create style.css to add additional style, append its url with the current sha1 to avoid cache
POC on how to send email using postfix
add sqlalchemy-utils
use arrow instead of datetime
add new params STRIPE_API, STRIPE_YEARLY_SKU, STRIPE_MONTHLY_PLAN
show full error in local
add plan, plan_expiration to User, need to create enum directly in migration script, cf https://github.com/sqlalchemy/alembic/issues/67
reformat all html files: use space instead of tab
new user will have trial plan for 15 days
add new param MAX_NB_EMAIL_FREE_PLAN
only user with enough quota can create new email
if user cannot create new gen email, pick randomly one from existing gen emails. Use flush instead of commit
rename STRIPE_YEARLY_SKU -> STRIPE_YEARLY_PLAN
open client page in discover in a new tab
add stripe
not logging /static call: disable flask logging, replace by after_request
add param STRIPE_SECRET_KEY
add 3 columns stripe_customer_id, stripe_card_token, stripe_subscription_id
user can upgrade their pricing
add setting page as coming-soon
add GenEmail, ClientUser to admin
ignore /admin/static logging
add more fake data
add ondelete="cascade" whenever possible
rename plan_expiration -> trial_expiration
reset migration: delete old migrations, create new one
rename test_send_email -> poc_send_email to avoid the file being called by pytest
add new param LYRA_ANALYTICS_ID, add lyra analytics
add how to create new migration into readme
add drift to base.html
notify admin when new user signs up or pays subscription
log exception in case of 500
use sendgrid to notify admin
add alias /userinfo to user_info endpoint
add change_password to shell
add info on how payment is handled
invite user to retry if card not working
remove drift and add "contact us" link
move poc_send_email into poc/
support getting client-id, client-secret from form-data in addition to basic auth
client-id, client-secret is passed in form-data by passport-oauth2 for ex
add jwtRS256 private and public key
add jwk-jws-jwt poc
add new param OPENID_PRIVATE_KEY_PATH, OPENID_PRIVATE_KEY_PATH
add scope, redirect_url to AuthorizationCode and OauthToken
take into scope when creating oauth-token, authorization-code
add jwcrypto
add jose_utils: make_id_token and verify_id_token
add &scope to redirect uri
add "email_verified": True into user_info
fix user not activated
add /oauth2 as alias for /oauth
handle case where scope and state are empty
remove threaded=False
Use Email Alias as wording
remove help text
user can re-send activation email
add "expired" into ActivationCode
Handle the case activation code is expired
reformat: use form.validate_on_submit instead of request.method == post && form.validate
use error text instead of flash()
display client oauth-id and oauth-secret on client detail page
not display oauth-secret on client listing
fix expiration check
improve page title, footer
add /jwks and /.well-known/openid-configuration
init properly tests, fix blueprint conflict bug in flask-admin
create oauth_models module
rename Scope -> ScopeE to distinguish with Scope DB model
set app.url_map.strict_slashes = False
use ScopeE instead of SCOPE_NAME, ...
support access_token passed as args in /userinfo
merge /allow-deny into /authorize
improve wording
take into account the case response_type=code and openid is in scope
take into account response_type=id_token, id_token token, id_token code
make sure to use in-memory db in test
fix scope can be null
allow cross_origin for /.well-known/openid-configuration and /jwks
fix footer link
center authorize form
rename trial_expiration to plan_expiration
move stripe init to create_app()
use real email to be able to receive email notification
add user.profile_picture_id column
use user profile picture and fallback to gravatar
use nguyenkims+local@gm to distinguish with staging
handle plan cancel, reactivation, user profile update
fix can_create_new_email
create cron.py that set plan to free when expired
add crontab.yml
add yacron
use notify_admin instead of LOG.error
add ResetPasswordCode model
user can change password in setting
increase display time for notie
add forgot_password page
If login error: redirect to this page upon success login.
hide discover tab
add column user.is_developer
only show developer menu to developer
comment out the publish button
set local user to developer
make sure only developer can access /developer blueprint
User is invited to upgrade if they are in free plan or their trial ends soon
not sending email when in local mode
create Partner model
create become partner page
use normal error handling on local
fix migration
add "import sqlalchemy_utils" into migration template
small refactoring on setting page
handle promo code. TODO: add migration file
add migration for user.promo_codes
move email alias on top of apps in dashboard
add introjs
move encode_url to utils
create GenEmail.create_new_gen_email
create a first alias mail to show user how to use when they login
show intro when user visits the website the first time
fix register
2019-07-02 07:20:12 +00:00
|
|
|
|
2020-03-15 10:11:16 +00:00
|
|
|
from app import s3
|
2020-08-30 17:59:39 +00:00
|
|
|
from app.alias_utils import nb_email_log_for_mailbox
|
2020-04-19 09:18:27 +00:00
|
|
|
from app.api.views.apple import verify_receipt
|
2020-04-29 13:50:06 +00:00
|
|
|
from app.config import (
|
|
|
|
ADMIN_EMAIL,
|
|
|
|
MACAPP_APPLE_API_SECRET,
|
|
|
|
APPLE_API_SECRET,
|
2020-08-29 17:05:32 +00:00
|
|
|
EMAIL_SERVERS_WITH_PRIORITY,
|
|
|
|
URL,
|
2020-08-31 15:36:27 +00:00
|
|
|
AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN,
|
2021-05-14 17:59:32 +00:00
|
|
|
HIBP_API_KEYS,
|
2021-05-17 16:15:12 +00:00
|
|
|
HIBP_SCAN_INTERVAL_DAYS,
|
2020-04-29 13:50:06 +00:00
|
|
|
)
|
2021-10-12 12:36:47 +00:00
|
|
|
from app.db import Session
|
2020-08-29 17:05:32 +00:00
|
|
|
from app.dns_utils import get_mx_domains
|
2020-06-09 15:12:34 +00:00
|
|
|
from app.email_utils import (
|
|
|
|
send_email,
|
|
|
|
send_trial_end_soon_email,
|
|
|
|
render,
|
2020-10-15 14:21:31 +00:00
|
|
|
email_can_be_used_as_mailbox,
|
2020-08-31 15:36:27 +00:00
|
|
|
send_email_with_rate_control,
|
2020-11-22 12:07:09 +00:00
|
|
|
normalize_reply_email,
|
2020-11-25 14:26:17 +00:00
|
|
|
is_valid_email,
|
2021-04-12 18:20:55 +00:00
|
|
|
get_email_domain_part,
|
2020-06-09 15:12:34 +00:00
|
|
|
)
|
create BaseForm to enable CSRF
register page
redirect user to dashboard if they are logged in
enable csrf for login page
Set models more strict
bootstrap developer page
add helper method to ModelMixin, remove CRUDMixin
display list of clients on developer index, add copy client-secret to clipboard using clipboardjs
add toastr and use jquery non slim
display a toast when user copies the client-secret
create new client, generate client-id using unidecode
client detail page: can edit client
add delete client
implement /oauth/authorize and /oauth/allow-deny
implement /oauth/token
add /oauth/user_info endpoint
handle scopes: wip
take into account scope: display scope, return user data according to scope
create virtual-domain, gen email, client_user model WIP
create authorize_nonlogin_user page
user can choose to generate a new email
no need to interfere with root logger
log for before and after request
if user has already allowed a client: generate a auth-code and redirect user to client
get_user_info takes into account gen email
display list of clients that have user has authorised
use yk-client domain instead of localhost as cookie depends on the domain name
use wtforms instead of flask_wtf
Dockerfile
delete virtual domain
EMAIL_DOMAIN can come from env var
bind to host 0.0.0.0
fix signup error: use session as default csrf_context
rename yourkey to simplelogin
add python-dotenv, ipython, sqlalchemy_utils
create DB_URI, FLASK_SECRET. Load config from CONFIG file if exist
add shortcuts to logging
create shell
add psycopg2
do not add local data in Dockerfile
add drop_db into shell
add shell.prepare_db()
fix prepare_db
setup sentry
copy assets from tabler/dist
add icon downloaded from https://commons.wikimedia.org/wiki/File:Simpleicons_Interface_key-tool-1.svg
integrate tabler - login and register page
add favicon
template: default, header. Use gravatar for user avatar url
use default template for dashboard, developer page
use another icon
add clipboard and notie
prettify dashboard
add notie css
add fake gen email and client-user
prettify list client page, use notie for toast
add email, name scope to new client
display client scope in client list
prettify new-client, client-detail
add sentry-sdk and blinker
add arrow, add dt jinja filter, prettify logout, dashboard
comment "last used" in dashboard for now
prettify date display
add copy email to clipboard to dashboard
use "users" as table name for User as "user" is reserved key in postgres
call prepare_db() when creating new db
error page 400, 401, 403, 404
prettify authorize_login_user
create already_authorize.html for user who has already authorized a client
user can generate new email
display all other generated emails
add ENV variable, only reset DB when ENV=local
fix: not return other users gen emails
display nb users for each client
refactor shell: remove prepare_db()
add sendgrid
add /favicon.ico route
add new config: URL, SUPPORT_EMAIL, SENDGRID_API_KEY
user needs to activate their account before login
create copy button on dashboard
client can have multiple redirect uris, in client detail can add/remove redirect-uri,
use redirect_uri passed in /authorize
refactor: move get_user_info into ClientUser model
dashboard: display all apps, all generated emails
add "id" into user_info
add trigger email button
invalidate the session at each new version by changing the secret
centralize Client creation into Client.create_new
user can enable/disable email forwarding
setup auto dismiss alert: just add .alert-auto-dismiss
move name down in register form
add shell.add_real_data
move blueprint template to its own package
prettify authorize page for non-authenticated user
update readme, return error if not redirect_uri
add flask-wtf, use psycopg2-binary
use flask-wtf FlaskForm instead of Form
rename email -> email_utils
add AWS_REGION, BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY to config
add s3 module
add File model, add Client.icon_id
handle client icon update
can create client with icon
display client icon in client list page
add Client.home_url
take into account Client.home_url
add boto3
register: ask name first
only show "trigger test email" if email forwarding is enabled
display gen email in alphabetical order, client in client.name alphabetical order
better error page
the modal does not get close when user clicks outside of modal
add Client.published column
discover page that displays all published Client
add missing bootstrap.bundle.min.js.map
developer can publish/unpublish their app in discover
use notie for display flash message
create hotmail account
fix missing jquery
add footer, add global jinja2 variable
strengthen model: use nullable=False whenever possible,
rename client_id to oauth_client_id, client_secret to oauth_client_secret
add flask-migrate
init migrate
1st migrate version
fix rename client_id -> oauth_client_id
prettify UI
use flask_migrate.upgrade() instead of db.create_all()
make sure requirejs.config is called for all page
enable sentry for js, use uppercase for global jinja2 variables
add flask-admin
add User.is_admin column
setup flask admin, only accessible to admin user
fix migration: add server_default
replace session[redirect_after_login] by "next" request args
add pyproject.toml: ignore migrations/ in black
add register waiting_activation_email page
better email wording
add pytest
add get_host_name_and_scheme and tests
example fail test
fix test
fix client-id display
add flask-cors
/user_info supports cors, add /me as /user_info synonym
return client in /me
support implicit flow
no need to use with "app.app_context()"
add watchtower to requirement
add param ENABLE_CLOUDWATCH, CLOUDWATCH_LOG_GROUP, CLOUDWATCH_LOG_STREAM
add cloudwatch logger if cloudwatch is enabled
add 500 error page
add help text for list of used client
display list of app/website that an email has been used
click on client name brings to client detail page
create style.css to add additional style, append its url with the current sha1 to avoid cache
POC on how to send email using postfix
add sqlalchemy-utils
use arrow instead of datetime
add new params STRIPE_API, STRIPE_YEARLY_SKU, STRIPE_MONTHLY_PLAN
show full error in local
add plan, plan_expiration to User, need to create enum directly in migration script, cf https://github.com/sqlalchemy/alembic/issues/67
reformat all html files: use space instead of tab
new user will have trial plan for 15 days
add new param MAX_NB_EMAIL_FREE_PLAN
only user with enough quota can create new email
if user cannot create new gen email, pick randomly one from existing gen emails. Use flush instead of commit
rename STRIPE_YEARLY_SKU -> STRIPE_YEARLY_PLAN
open client page in discover in a new tab
add stripe
not logging /static call: disable flask logging, replace by after_request
add param STRIPE_SECRET_KEY
add 3 columns stripe_customer_id, stripe_card_token, stripe_subscription_id
user can upgrade their pricing
add setting page as coming-soon
add GenEmail, ClientUser to admin
ignore /admin/static logging
add more fake data
add ondelete="cascade" whenever possible
rename plan_expiration -> trial_expiration
reset migration: delete old migrations, create new one
rename test_send_email -> poc_send_email to avoid the file being called by pytest
add new param LYRA_ANALYTICS_ID, add lyra analytics
add how to create new migration into readme
add drift to base.html
notify admin when new user signs up or pays subscription
log exception in case of 500
use sendgrid to notify admin
add alias /userinfo to user_info endpoint
add change_password to shell
add info on how payment is handled
invite user to retry if card not working
remove drift and add "contact us" link
move poc_send_email into poc/
support getting client-id, client-secret from form-data in addition to basic auth
client-id, client-secret is passed in form-data by passport-oauth2 for ex
add jwtRS256 private and public key
add jwk-jws-jwt poc
add new param OPENID_PRIVATE_KEY_PATH, OPENID_PRIVATE_KEY_PATH
add scope, redirect_url to AuthorizationCode and OauthToken
take into scope when creating oauth-token, authorization-code
add jwcrypto
add jose_utils: make_id_token and verify_id_token
add &scope to redirect uri
add "email_verified": True into user_info
fix user not activated
add /oauth2 as alias for /oauth
handle case where scope and state are empty
remove threaded=False
Use Email Alias as wording
remove help text
user can re-send activation email
add "expired" into ActivationCode
Handle the case activation code is expired
reformat: use form.validate_on_submit instead of request.method == post && form.validate
use error text instead of flash()
display client oauth-id and oauth-secret on client detail page
not display oauth-secret on client listing
fix expiration check
improve page title, footer
add /jwks and /.well-known/openid-configuration
init properly tests, fix blueprint conflict bug in flask-admin
create oauth_models module
rename Scope -> ScopeE to distinguish with Scope DB model
set app.url_map.strict_slashes = False
use ScopeE instead of SCOPE_NAME, ...
support access_token passed as args in /userinfo
merge /allow-deny into /authorize
improve wording
take into account the case response_type=code and openid is in scope
take into account response_type=id_token, id_token token, id_token code
make sure to use in-memory db in test
fix scope can be null
allow cross_origin for /.well-known/openid-configuration and /jwks
fix footer link
center authorize form
rename trial_expiration to plan_expiration
move stripe init to create_app()
use real email to be able to receive email notification
add user.profile_picture_id column
use user profile picture and fallback to gravatar
use nguyenkims+local@gm to distinguish with staging
handle plan cancel, reactivation, user profile update
fix can_create_new_email
create cron.py that set plan to free when expired
add crontab.yml
add yacron
use notify_admin instead of LOG.error
add ResetPasswordCode model
user can change password in setting
increase display time for notie
add forgot_password page
If login error: redirect to this page upon success login.
hide discover tab
add column user.is_developer
only show developer menu to developer
comment out the publish button
set local user to developer
make sure only developer can access /developer blueprint
User is invited to upgrade if they are in free plan or their trial ends soon
not sending email when in local mode
create Partner model
create become partner page
use normal error handling on local
fix migration
add "import sqlalchemy_utils" into migration template
small refactoring on setting page
handle promo code. TODO: add migration file
add migration for user.promo_codes
move email alias on top of apps in dashboard
add introjs
move encode_url to utils
create GenEmail.create_new_gen_email
create a first alias mail to show user how to use when they login
show intro when user visits the website the first time
fix register
2019-07-02 07:20:12 +00:00
|
|
|
from app.log import LOG
|
2019-12-15 07:52:20 +00:00
|
|
|
from app.models import (
|
|
|
|
Subscription,
|
|
|
|
User,
|
2020-03-17 10:51:40 +00:00
|
|
|
Alias,
|
2020-03-17 10:10:50 +00:00
|
|
|
EmailLog,
|
2019-12-15 07:52:20 +00:00
|
|
|
CustomDomain,
|
2019-12-15 07:57:24 +00:00
|
|
|
Client,
|
2020-02-23 10:01:23 +00:00
|
|
|
ManualSubscription,
|
2020-03-15 10:11:16 +00:00
|
|
|
RefusedEmail,
|
2020-04-19 09:18:27 +00:00
|
|
|
AppleSubscription,
|
2020-06-09 15:12:34 +00:00
|
|
|
Mailbox,
|
2020-08-17 12:02:59 +00:00
|
|
|
Monitoring,
|
2020-09-14 16:22:08 +00:00
|
|
|
Contact,
|
2020-12-13 18:18:58 +00:00
|
|
|
CoinbaseSubscription,
|
2021-01-26 08:54:58 +00:00
|
|
|
TransactionalEmail,
|
|
|
|
Bounce,
|
2021-03-23 09:23:40 +00:00
|
|
|
Metric2,
|
2021-04-12 18:20:55 +00:00
|
|
|
SLDomain,
|
|
|
|
DeletedAlias,
|
|
|
|
DomainDeletedAlias,
|
2021-05-14 17:59:32 +00:00
|
|
|
Hibp,
|
2021-07-29 07:55:36 +00:00
|
|
|
HibpNotifiedAlias,
|
2021-11-17 16:43:59 +00:00
|
|
|
Directory,
|
|
|
|
DeletedDirectory,
|
|
|
|
DeletedSubdomain,
|
2019-12-15 07:57:24 +00:00
|
|
|
)
|
2021-03-23 09:50:49 +00:00
|
|
|
from app.utils import sanitize_email
|
2021-10-26 08:52:28 +00:00
|
|
|
from server import create_light_app
|
create BaseForm to enable CSRF
register page
redirect user to dashboard if they are logged in
enable csrf for login page
Set models more strict
bootstrap developer page
add helper method to ModelMixin, remove CRUDMixin
display list of clients on developer index, add copy client-secret to clipboard using clipboardjs
add toastr and use jquery non slim
display a toast when user copies the client-secret
create new client, generate client-id using unidecode
client detail page: can edit client
add delete client
implement /oauth/authorize and /oauth/allow-deny
implement /oauth/token
add /oauth/user_info endpoint
handle scopes: wip
take into account scope: display scope, return user data according to scope
create virtual-domain, gen email, client_user model WIP
create authorize_nonlogin_user page
user can choose to generate a new email
no need to interfere with root logger
log for before and after request
if user has already allowed a client: generate a auth-code and redirect user to client
get_user_info takes into account gen email
display list of clients that have user has authorised
use yk-client domain instead of localhost as cookie depends on the domain name
use wtforms instead of flask_wtf
Dockerfile
delete virtual domain
EMAIL_DOMAIN can come from env var
bind to host 0.0.0.0
fix signup error: use session as default csrf_context
rename yourkey to simplelogin
add python-dotenv, ipython, sqlalchemy_utils
create DB_URI, FLASK_SECRET. Load config from CONFIG file if exist
add shortcuts to logging
create shell
add psycopg2
do not add local data in Dockerfile
add drop_db into shell
add shell.prepare_db()
fix prepare_db
setup sentry
copy assets from tabler/dist
add icon downloaded from https://commons.wikimedia.org/wiki/File:Simpleicons_Interface_key-tool-1.svg
integrate tabler - login and register page
add favicon
template: default, header. Use gravatar for user avatar url
use default template for dashboard, developer page
use another icon
add clipboard and notie
prettify dashboard
add notie css
add fake gen email and client-user
prettify list client page, use notie for toast
add email, name scope to new client
display client scope in client list
prettify new-client, client-detail
add sentry-sdk and blinker
add arrow, add dt jinja filter, prettify logout, dashboard
comment "last used" in dashboard for now
prettify date display
add copy email to clipboard to dashboard
use "users" as table name for User as "user" is reserved key in postgres
call prepare_db() when creating new db
error page 400, 401, 403, 404
prettify authorize_login_user
create already_authorize.html for user who has already authorized a client
user can generate new email
display all other generated emails
add ENV variable, only reset DB when ENV=local
fix: not return other users gen emails
display nb users for each client
refactor shell: remove prepare_db()
add sendgrid
add /favicon.ico route
add new config: URL, SUPPORT_EMAIL, SENDGRID_API_KEY
user needs to activate their account before login
create copy button on dashboard
client can have multiple redirect uris, in client detail can add/remove redirect-uri,
use redirect_uri passed in /authorize
refactor: move get_user_info into ClientUser model
dashboard: display all apps, all generated emails
add "id" into user_info
add trigger email button
invalidate the session at each new version by changing the secret
centralize Client creation into Client.create_new
user can enable/disable email forwarding
setup auto dismiss alert: just add .alert-auto-dismiss
move name down in register form
add shell.add_real_data
move blueprint template to its own package
prettify authorize page for non-authenticated user
update readme, return error if not redirect_uri
add flask-wtf, use psycopg2-binary
use flask-wtf FlaskForm instead of Form
rename email -> email_utils
add AWS_REGION, BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY to config
add s3 module
add File model, add Client.icon_id
handle client icon update
can create client with icon
display client icon in client list page
add Client.home_url
take into account Client.home_url
add boto3
register: ask name first
only show "trigger test email" if email forwarding is enabled
display gen email in alphabetical order, client in client.name alphabetical order
better error page
the modal does not get close when user clicks outside of modal
add Client.published column
discover page that displays all published Client
add missing bootstrap.bundle.min.js.map
developer can publish/unpublish their app in discover
use notie for display flash message
create hotmail account
fix missing jquery
add footer, add global jinja2 variable
strengthen model: use nullable=False whenever possible,
rename client_id to oauth_client_id, client_secret to oauth_client_secret
add flask-migrate
init migrate
1st migrate version
fix rename client_id -> oauth_client_id
prettify UI
use flask_migrate.upgrade() instead of db.create_all()
make sure requirejs.config is called for all page
enable sentry for js, use uppercase for global jinja2 variables
add flask-admin
add User.is_admin column
setup flask admin, only accessible to admin user
fix migration: add server_default
replace session[redirect_after_login] by "next" request args
add pyproject.toml: ignore migrations/ in black
add register waiting_activation_email page
better email wording
add pytest
add get_host_name_and_scheme and tests
example fail test
fix test
fix client-id display
add flask-cors
/user_info supports cors, add /me as /user_info synonym
return client in /me
support implicit flow
no need to use with "app.app_context()"
add watchtower to requirement
add param ENABLE_CLOUDWATCH, CLOUDWATCH_LOG_GROUP, CLOUDWATCH_LOG_STREAM
add cloudwatch logger if cloudwatch is enabled
add 500 error page
add help text for list of used client
display list of app/website that an email has been used
click on client name brings to client detail page
create style.css to add additional style, append its url with the current sha1 to avoid cache
POC on how to send email using postfix
add sqlalchemy-utils
use arrow instead of datetime
add new params STRIPE_API, STRIPE_YEARLY_SKU, STRIPE_MONTHLY_PLAN
show full error in local
add plan, plan_expiration to User, need to create enum directly in migration script, cf https://github.com/sqlalchemy/alembic/issues/67
reformat all html files: use space instead of tab
new user will have trial plan for 15 days
add new param MAX_NB_EMAIL_FREE_PLAN
only user with enough quota can create new email
if user cannot create new gen email, pick randomly one from existing gen emails. Use flush instead of commit
rename STRIPE_YEARLY_SKU -> STRIPE_YEARLY_PLAN
open client page in discover in a new tab
add stripe
not logging /static call: disable flask logging, replace by after_request
add param STRIPE_SECRET_KEY
add 3 columns stripe_customer_id, stripe_card_token, stripe_subscription_id
user can upgrade their pricing
add setting page as coming-soon
add GenEmail, ClientUser to admin
ignore /admin/static logging
add more fake data
add ondelete="cascade" whenever possible
rename plan_expiration -> trial_expiration
reset migration: delete old migrations, create new one
rename test_send_email -> poc_send_email to avoid the file being called by pytest
add new param LYRA_ANALYTICS_ID, add lyra analytics
add how to create new migration into readme
add drift to base.html
notify admin when new user signs up or pays subscription
log exception in case of 500
use sendgrid to notify admin
add alias /userinfo to user_info endpoint
add change_password to shell
add info on how payment is handled
invite user to retry if card not working
remove drift and add "contact us" link
move poc_send_email into poc/
support getting client-id, client-secret from form-data in addition to basic auth
client-id, client-secret is passed in form-data by passport-oauth2 for ex
add jwtRS256 private and public key
add jwk-jws-jwt poc
add new param OPENID_PRIVATE_KEY_PATH, OPENID_PRIVATE_KEY_PATH
add scope, redirect_url to AuthorizationCode and OauthToken
take into scope when creating oauth-token, authorization-code
add jwcrypto
add jose_utils: make_id_token and verify_id_token
add &scope to redirect uri
add "email_verified": True into user_info
fix user not activated
add /oauth2 as alias for /oauth
handle case where scope and state are empty
remove threaded=False
Use Email Alias as wording
remove help text
user can re-send activation email
add "expired" into ActivationCode
Handle the case activation code is expired
reformat: use form.validate_on_submit instead of request.method == post && form.validate
use error text instead of flash()
display client oauth-id and oauth-secret on client detail page
not display oauth-secret on client listing
fix expiration check
improve page title, footer
add /jwks and /.well-known/openid-configuration
init properly tests, fix blueprint conflict bug in flask-admin
create oauth_models module
rename Scope -> ScopeE to distinguish with Scope DB model
set app.url_map.strict_slashes = False
use ScopeE instead of SCOPE_NAME, ...
support access_token passed as args in /userinfo
merge /allow-deny into /authorize
improve wording
take into account the case response_type=code and openid is in scope
take into account response_type=id_token, id_token token, id_token code
make sure to use in-memory db in test
fix scope can be null
allow cross_origin for /.well-known/openid-configuration and /jwks
fix footer link
center authorize form
rename trial_expiration to plan_expiration
move stripe init to create_app()
use real email to be able to receive email notification
add user.profile_picture_id column
use user profile picture and fallback to gravatar
use nguyenkims+local@gm to distinguish with staging
handle plan cancel, reactivation, user profile update
fix can_create_new_email
create cron.py that set plan to free when expired
add crontab.yml
add yacron
use notify_admin instead of LOG.error
add ResetPasswordCode model
user can change password in setting
increase display time for notie
add forgot_password page
If login error: redirect to this page upon success login.
hide discover tab
add column user.is_developer
only show developer menu to developer
comment out the publish button
set local user to developer
make sure only developer can access /developer blueprint
User is invited to upgrade if they are in free plan or their trial ends soon
not sending email when in local mode
create Partner model
create become partner page
use normal error handling on local
fix migration
add "import sqlalchemy_utils" into migration template
small refactoring on setting page
handle promo code. TODO: add migration file
add migration for user.promo_codes
move email alias on top of apps in dashboard
add introjs
move encode_url to utils
create GenEmail.create_new_gen_email
create a first alias mail to show user how to use when they login
show intro when user visits the website the first time
fix register
2019-07-02 07:20:12 +00:00
|
|
|
|
|
|
|
|
2020-02-02 15:17:04 +00:00
|
|
|
def notify_trial_end():
|
2021-10-12 12:36:47 +00:00
|
|
|
for user in User.filter(
|
2020-12-06 12:36:32 +00:00
|
|
|
User.activated.is_(True), User.trial_end.isnot(None), User.lifetime.is_(False)
|
2020-02-05 14:21:56 +00:00
|
|
|
).all():
|
2020-02-15 09:29:44 +00:00
|
|
|
if user.in_trial() and arrow.now().shift(
|
|
|
|
days=3
|
|
|
|
) > user.trial_end >= arrow.now().shift(days=2):
|
2020-01-30 07:19:09 +00:00
|
|
|
LOG.d("Send trial end email to user %s", user)
|
|
|
|
send_trial_end_soon_email(user)
|
|
|
|
|
|
|
|
|
2021-01-26 08:54:58 +00:00
|
|
|
def delete_logs():
|
|
|
|
"""delete everything that are considered logs"""
|
|
|
|
delete_refused_emails()
|
|
|
|
delete_old_monitoring()
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for t in TransactionalEmail.filter(
|
2021-01-26 08:54:58 +00:00
|
|
|
TransactionalEmail.created_at < arrow.now().shift(days=-7)
|
|
|
|
):
|
|
|
|
TransactionalEmail.delete(t.id)
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for b in Bounce.filter(Bounce.created_at < arrow.now().shift(days=-7)):
|
2021-01-26 08:54:58 +00:00
|
|
|
Bounce.delete(b.id)
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-01-26 08:54:58 +00:00
|
|
|
|
2021-07-14 15:15:28 +00:00
|
|
|
LOG.d("Delete EmailLog older than 2 weeks")
|
|
|
|
|
|
|
|
max_dt = arrow.now().shift(weeks=-2)
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_deleted = EmailLog.filter(EmailLog.created_at < max_dt).delete()
|
|
|
|
Session.commit()
|
2021-07-14 15:15:28 +00:00
|
|
|
|
|
|
|
LOG.i("Delete %s email logs", nb_deleted)
|
|
|
|
|
2021-01-26 08:54:58 +00:00
|
|
|
|
2020-03-15 10:11:16 +00:00
|
|
|
def delete_refused_emails():
|
2021-10-12 12:36:47 +00:00
|
|
|
for refused_email in RefusedEmail.filter_by(deleted=False).all():
|
2020-03-15 11:14:43 +00:00
|
|
|
if arrow.now().shift(days=1) > refused_email.delete_at >= arrow.now():
|
2020-03-15 10:11:16 +00:00
|
|
|
LOG.d("Delete refused email %s", refused_email)
|
2020-04-02 16:06:08 +00:00
|
|
|
if refused_email.path:
|
|
|
|
s3.delete(refused_email.path)
|
|
|
|
|
2020-03-15 10:11:16 +00:00
|
|
|
s3.delete(refused_email.full_report_path)
|
|
|
|
|
|
|
|
# do not set path and full_report_path to null
|
|
|
|
# so we can check later that the files are indeed deleted
|
|
|
|
refused_email.deleted = True
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-03-15 10:11:16 +00:00
|
|
|
|
|
|
|
LOG.d("Finish delete_refused_emails")
|
|
|
|
|
|
|
|
|
2020-03-05 08:35:20 +00:00
|
|
|
def notify_premium_end():
|
|
|
|
"""sent to user who has canceled their subscription and who has their subscription ending soon"""
|
2021-10-12 12:36:47 +00:00
|
|
|
for sub in Subscription.filter_by(cancelled=True).all():
|
2020-03-05 08:35:20 +00:00
|
|
|
if (
|
|
|
|
arrow.now().shift(days=3).date()
|
|
|
|
> sub.next_bill_date
|
|
|
|
>= arrow.now().shift(days=2).date()
|
|
|
|
):
|
|
|
|
user = sub.user
|
2021-09-27 13:51:09 +00:00
|
|
|
|
|
|
|
if user.lifetime:
|
|
|
|
continue
|
|
|
|
|
2020-03-05 08:35:20 +00:00
|
|
|
LOG.d(f"Send subscription ending soon email to user {user}")
|
|
|
|
|
|
|
|
send_email(
|
|
|
|
user.email,
|
2021-01-11 09:22:39 +00:00
|
|
|
f"Your subscription will end soon",
|
2020-04-19 08:54:05 +00:00
|
|
|
render(
|
|
|
|
"transactional/subscription-end.txt",
|
|
|
|
user=user,
|
|
|
|
next_bill_date=sub.next_bill_date.strftime("%Y-%m-%d"),
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/subscription-end.html",
|
|
|
|
user=user,
|
|
|
|
next_bill_date=sub.next_bill_date.strftime("%Y-%m-%d"),
|
|
|
|
),
|
2020-03-05 08:35:20 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
2020-02-23 10:01:23 +00:00
|
|
|
def notify_manual_sub_end():
|
2021-10-12 12:36:47 +00:00
|
|
|
for manual_sub in ManualSubscription.all():
|
2021-10-31 18:03:49 +00:00
|
|
|
manual_sub: ManualSubscription
|
2020-02-23 10:01:23 +00:00
|
|
|
need_reminder = False
|
|
|
|
if arrow.now().shift(days=14) > manual_sub.end_at > arrow.now().shift(days=13):
|
|
|
|
need_reminder = True
|
|
|
|
elif arrow.now().shift(days=4) > manual_sub.end_at > arrow.now().shift(days=3):
|
|
|
|
need_reminder = True
|
|
|
|
|
|
|
|
if need_reminder:
|
|
|
|
user = manual_sub.user
|
2021-10-31 18:03:49 +00:00
|
|
|
|
|
|
|
# user can have a (free) manual subscription but has taken a paid subscription via
|
|
|
|
# Paddle, Coinbase or Apple since then
|
|
|
|
if manual_sub.is_giveaway:
|
2021-11-02 14:47:31 +00:00
|
|
|
if user.lifetime:
|
|
|
|
LOG.d("%s has a lifetime licence", user)
|
|
|
|
continue
|
|
|
|
|
2021-10-31 18:03:49 +00:00
|
|
|
if user.get_subscription():
|
|
|
|
LOG.d("%s has a active Paddle subscription", user)
|
|
|
|
continue
|
|
|
|
|
|
|
|
coinbase_subscription: CoinbaseSubscription = (
|
|
|
|
CoinbaseSubscription.get_by(user_id=user.id)
|
|
|
|
)
|
|
|
|
if coinbase_subscription and coinbase_subscription.is_active():
|
|
|
|
LOG.d("%s has a active Coinbase subscription", user)
|
|
|
|
continue
|
|
|
|
|
|
|
|
apple_sub: AppleSubscription = AppleSubscription.get_by(user_id=user.id)
|
|
|
|
if apple_sub and apple_sub.is_valid():
|
|
|
|
LOG.d("%s has a active Apple subscription", user)
|
|
|
|
continue
|
|
|
|
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.d("Remind user %s that their manual sub is ending soon", user)
|
2020-02-23 10:01:23 +00:00
|
|
|
send_email(
|
|
|
|
user.email,
|
2021-01-11 09:22:39 +00:00
|
|
|
f"Your subscription will end soon",
|
2020-02-23 10:01:23 +00:00
|
|
|
render(
|
|
|
|
"transactional/manual-subscription-end.txt",
|
|
|
|
user=user,
|
|
|
|
manual_sub=manual_sub,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/manual-subscription-end.html",
|
|
|
|
user=user,
|
|
|
|
manual_sub=manual_sub,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
2020-12-14 10:34:59 +00:00
|
|
|
extend_subscription_url = URL + "/dashboard/coinbase_checkout"
|
2021-10-12 12:36:47 +00:00
|
|
|
for coinbase_subscription in CoinbaseSubscription.all():
|
2020-12-13 18:18:58 +00:00
|
|
|
need_reminder = False
|
|
|
|
if (
|
|
|
|
arrow.now().shift(days=14)
|
|
|
|
> coinbase_subscription.end_at
|
|
|
|
> arrow.now().shift(days=13)
|
|
|
|
):
|
|
|
|
need_reminder = True
|
|
|
|
elif (
|
|
|
|
arrow.now().shift(days=4)
|
|
|
|
> coinbase_subscription.end_at
|
|
|
|
> arrow.now().shift(days=3)
|
|
|
|
):
|
|
|
|
need_reminder = True
|
|
|
|
|
|
|
|
if need_reminder:
|
|
|
|
user = coinbase_subscription.user
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.d(
|
2020-12-13 18:18:58 +00:00
|
|
|
"Remind user %s that their coinbase subscription is ending soon", user
|
|
|
|
)
|
|
|
|
send_email(
|
|
|
|
user.email,
|
2020-12-13 18:41:42 +00:00
|
|
|
"Your SimpleLogin subscription will end soon",
|
2020-12-13 18:18:58 +00:00
|
|
|
render(
|
|
|
|
"transactional/coinbase/reminder-subscription.txt",
|
|
|
|
coinbase_subscription=coinbase_subscription,
|
|
|
|
extend_subscription_url=extend_subscription_url,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/coinbase/reminder-subscription.html",
|
|
|
|
coinbase_subscription=coinbase_subscription,
|
|
|
|
extend_subscription_url=extend_subscription_url,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
2020-02-23 10:01:23 +00:00
|
|
|
|
2020-04-19 09:18:27 +00:00
|
|
|
def poll_apple_subscription():
|
|
|
|
"""Poll Apple API to update AppleSubscription"""
|
|
|
|
# todo: only near the end of the subscription
|
2021-10-12 12:36:47 +00:00
|
|
|
for apple_sub in AppleSubscription.all():
|
2020-04-19 09:18:27 +00:00
|
|
|
user = apple_sub.user
|
2020-04-29 13:50:06 +00:00
|
|
|
verify_receipt(apple_sub.receipt_data, user, APPLE_API_SECRET)
|
|
|
|
verify_receipt(apple_sub.receipt_data, user, MACAPP_APPLE_API_SECRET)
|
2020-04-19 09:18:27 +00:00
|
|
|
|
|
|
|
LOG.d("Finish poll_apple_subscription")
|
|
|
|
|
|
|
|
|
2021-03-24 16:39:16 +00:00
|
|
|
def compute_metric2() -> Metric2:
|
|
|
|
now = arrow.now()
|
2021-07-28 16:31:59 +00:00
|
|
|
_24h_ago = now.shift(days=-1)
|
|
|
|
|
2021-03-23 09:50:32 +00:00
|
|
|
nb_referred_user_paid = 0
|
2021-10-12 12:36:47 +00:00
|
|
|
for user in User.filter(User.referral_id.isnot(None)):
|
2021-03-23 09:50:32 +00:00
|
|
|
if user.is_paid():
|
|
|
|
nb_referred_user_paid += 1
|
|
|
|
|
|
|
|
return Metric2.create(
|
|
|
|
date=now,
|
|
|
|
# user stats
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_user=User.count(),
|
|
|
|
nb_activated_user=User.filter_by(activated=True).count(),
|
2021-03-23 09:50:32 +00:00
|
|
|
# subscription stats
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_premium=Subscription.filter(Subscription.cancelled.is_(False)).count(),
|
|
|
|
nb_cancelled_premium=Subscription.filter(
|
2021-03-23 09:50:32 +00:00
|
|
|
Subscription.cancelled.is_(True)
|
|
|
|
).count(),
|
|
|
|
# todo: filter by expires_date > now
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_apple_premium=AppleSubscription.count(),
|
|
|
|
nb_manual_premium=ManualSubscription.filter(
|
2021-03-23 09:50:32 +00:00
|
|
|
ManualSubscription.end_at > now,
|
|
|
|
ManualSubscription.is_giveaway.is_(False),
|
|
|
|
).count(),
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_coinbase_premium=CoinbaseSubscription.filter(
|
2021-03-23 09:50:32 +00:00
|
|
|
CoinbaseSubscription.end_at > now
|
|
|
|
).count(),
|
|
|
|
# referral stats
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_referred_user=User.filter(User.referral_id.isnot(None)).count(),
|
2021-03-23 09:50:32 +00:00
|
|
|
nb_referred_user_paid=nb_referred_user_paid,
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_alias=Alias.count(),
|
2021-03-23 09:50:32 +00:00
|
|
|
# email log stats
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_forward_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
|
2021-07-28 16:31:59 +00:00
|
|
|
.filter_by(bounced=False, is_spam=False, is_reply=False, blocked=False)
|
|
|
|
.count(),
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_bounced_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
|
2021-07-28 16:31:59 +00:00
|
|
|
.filter_by(bounced=True)
|
|
|
|
.count(),
|
2021-10-15 08:47:07 +00:00
|
|
|
nb_total_bounced_last_24h=Bounce.filter(Bounce.created_at > _24h_ago).count(),
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_reply_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
|
2021-07-28 16:31:59 +00:00
|
|
|
.filter_by(is_reply=True)
|
|
|
|
.count(),
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_block_last_24h=EmailLog.filter(EmailLog.created_at > _24h_ago)
|
2021-07-28 16:31:59 +00:00
|
|
|
.filter_by(blocked=True)
|
|
|
|
.count(),
|
|
|
|
# other stats
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_verified_custom_domain=CustomDomain.filter_by(verified=True).count(),
|
2021-11-17 16:43:59 +00:00
|
|
|
nb_subdomain=CustomDomain.filter_by(is_sl_subdomain=True).count(),
|
|
|
|
nb_directory=Directory.count(),
|
|
|
|
nb_deleted_directory=DeletedDirectory.count(),
|
|
|
|
nb_deleted_subdomain=DeletedSubdomain.count(),
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_app=Client.count(),
|
2021-03-23 09:50:32 +00:00
|
|
|
commit=True,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2020-04-23 20:11:43 +00:00
|
|
|
def increase_percent(old, new) -> str:
|
|
|
|
if old == 0:
|
|
|
|
return "N/A"
|
|
|
|
|
2021-07-29 09:32:15 +00:00
|
|
|
if not old or not new:
|
|
|
|
return "N/A"
|
|
|
|
|
2020-04-23 20:11:43 +00:00
|
|
|
increase = (new - old) / old * 100
|
2021-07-29 07:55:36 +00:00
|
|
|
return f"{increase:.1f}%. Delta: {new - old}"
|
2020-04-23 20:11:43 +00:00
|
|
|
|
|
|
|
|
2021-03-23 09:02:14 +00:00
|
|
|
def bounce_report() -> List[Tuple[str, int]]:
|
|
|
|
"""return the accounts that have most bounces, e.g.
|
|
|
|
(email1, 30)
|
|
|
|
(email2, 20)
|
|
|
|
|
|
|
|
Produce this query
|
|
|
|
|
|
|
|
```
|
|
|
|
SELECT
|
|
|
|
count(*) AS c,
|
|
|
|
users.email
|
|
|
|
FROM
|
|
|
|
email_log,
|
|
|
|
users
|
|
|
|
WHERE
|
|
|
|
email_log.user_id = users.id
|
|
|
|
AND email_log.created_at > '2021-3-20'
|
|
|
|
and email_log.bounced = true
|
|
|
|
GROUP BY
|
|
|
|
users.email
|
|
|
|
ORDER BY
|
|
|
|
c DESC;
|
|
|
|
```
|
|
|
|
|
|
|
|
"""
|
|
|
|
min_dt = arrow.now().shift(days=-1)
|
|
|
|
query = (
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.query(User.email, func.count(EmailLog.id).label("count"))
|
2021-03-23 09:02:14 +00:00
|
|
|
.join(EmailLog, EmailLog.user_id == User.id)
|
|
|
|
.filter(EmailLog.bounced, EmailLog.created_at > min_dt)
|
|
|
|
.group_by(User.email)
|
|
|
|
.having(func.count(EmailLog.id) > 5)
|
|
|
|
.order_by(desc("count"))
|
|
|
|
)
|
|
|
|
|
|
|
|
res = []
|
|
|
|
for email, count in query:
|
|
|
|
res.append((email, count))
|
|
|
|
|
|
|
|
return res
|
|
|
|
|
|
|
|
|
2021-10-23 16:40:02 +00:00
|
|
|
def all_bounce_report() -> str:
|
|
|
|
"""
|
|
|
|
Return a report for all mailboxes that have most bounces. Using this query to get mailboxes that have bounces.
|
|
|
|
For each mailbox in the list, return the first bounce info.
|
|
|
|
|
|
|
|
```
|
|
|
|
SELECT
|
|
|
|
email,
|
|
|
|
count(*) AS nb_bounce
|
|
|
|
FROM
|
|
|
|
bounce
|
|
|
|
WHERE
|
|
|
|
created_at > '2021-10-16'
|
|
|
|
GROUP BY
|
|
|
|
email
|
|
|
|
ORDER BY
|
|
|
|
nb_bounce DESC
|
|
|
|
```
|
|
|
|
|
|
|
|
"""
|
2021-10-26 11:03:57 +00:00
|
|
|
res = ""
|
2021-10-23 16:40:02 +00:00
|
|
|
min_dt = arrow.now().shift(days=-1)
|
|
|
|
query = (
|
|
|
|
Session.query(Bounce.email, func.count(Bounce.id).label("nb_bounce"))
|
|
|
|
.filter(Bounce.created_at > min_dt)
|
|
|
|
.group_by(Bounce.email)
|
|
|
|
# not return mailboxes that have too little bounces
|
2021-11-06 11:41:37 +00:00
|
|
|
.having(func.count(Bounce.id) > 3)
|
2021-10-23 16:40:02 +00:00
|
|
|
.order_by(desc("nb_bounce"))
|
|
|
|
)
|
|
|
|
|
|
|
|
for email, count in query:
|
|
|
|
res += f"{email}: {count} bounces. "
|
|
|
|
most_recent: Bounce = (
|
|
|
|
Bounce.filter(Bounce.email == email)
|
|
|
|
.order_by(Bounce.created_at.desc())
|
|
|
|
.first()
|
|
|
|
)
|
2021-11-04 13:28:19 +00:00
|
|
|
res += f"Most recent cause: \n{most_recent.info or ''}"
|
|
|
|
res += "\n----\n"
|
2021-10-23 16:40:02 +00:00
|
|
|
|
|
|
|
return res
|
|
|
|
|
|
|
|
|
2021-03-29 08:10:25 +00:00
|
|
|
def alias_creation_report() -> List[Tuple[str, int]]:
|
|
|
|
"""return the accounts that have created most aliases in the last 7 days, e.g.
|
|
|
|
(email1, 2021-3-21, 30)
|
|
|
|
(email2, 2021-3-20, 20)
|
|
|
|
|
|
|
|
Produce this query
|
|
|
|
|
|
|
|
```
|
|
|
|
SELECT
|
|
|
|
count(*) AS c,
|
|
|
|
users.email,
|
|
|
|
date(alias.created_at) AS d
|
|
|
|
FROM
|
|
|
|
alias,
|
|
|
|
users
|
|
|
|
WHERE
|
|
|
|
alias.user_id = users.id
|
|
|
|
AND alias.created_at > '2021-3-22'
|
|
|
|
GROUP BY
|
|
|
|
users.email,
|
|
|
|
d
|
|
|
|
HAVING
|
|
|
|
count(*) > 50
|
|
|
|
ORDER BY
|
|
|
|
c DESC;
|
|
|
|
```
|
|
|
|
|
|
|
|
"""
|
|
|
|
min_dt = arrow.now().shift(days=-7)
|
|
|
|
query = (
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.query(
|
2021-03-29 08:10:25 +00:00
|
|
|
User.email,
|
|
|
|
func.count(Alias.id).label("count"),
|
|
|
|
func.date(Alias.created_at).label("date"),
|
|
|
|
)
|
|
|
|
.join(Alias, Alias.user_id == User.id)
|
|
|
|
.filter(Alias.created_at > min_dt)
|
|
|
|
.group_by(User.email, "date")
|
|
|
|
.having(func.count(Alias.id) > 50)
|
|
|
|
.order_by(desc("count"))
|
|
|
|
)
|
|
|
|
|
|
|
|
res = []
|
|
|
|
for email, count, date in query:
|
|
|
|
res.append((email, count, date))
|
|
|
|
|
|
|
|
return res
|
|
|
|
|
|
|
|
|
2020-04-23 20:11:43 +00:00
|
|
|
def stats():
|
|
|
|
"""send admin stats everyday"""
|
|
|
|
if not ADMIN_EMAIL:
|
|
|
|
# nothing to do
|
|
|
|
return
|
|
|
|
|
2021-03-24 16:39:16 +00:00
|
|
|
stats_today = compute_metric2()
|
2021-03-24 16:38:17 +00:00
|
|
|
stats_yesterday = (
|
2021-10-12 12:36:47 +00:00
|
|
|
Metric2.filter(Metric2.date < stats_today.date)
|
2021-03-24 16:38:17 +00:00
|
|
|
.order_by(Metric2.date.desc())
|
|
|
|
.first()
|
|
|
|
)
|
2020-04-23 20:11:43 +00:00
|
|
|
|
|
|
|
nb_user_increase = increase_percent(stats_yesterday.nb_user, stats_today.nb_user)
|
|
|
|
nb_alias_increase = increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)
|
|
|
|
nb_forward_increase = increase_percent(
|
|
|
|
stats_yesterday.nb_forward, stats_today.nb_forward
|
|
|
|
)
|
2020-03-30 20:11:19 +00:00
|
|
|
|
2019-11-24 18:42:29 +00:00
|
|
|
today = arrow.now().format()
|
|
|
|
|
2021-10-26 10:48:01 +00:00
|
|
|
report = f"""
|
|
|
|
Stats for {today}
|
|
|
|
|
|
|
|
nb_user: {stats_today.nb_user} - {increase_percent(stats_yesterday.nb_user, stats_today.nb_user)}
|
|
|
|
nb_premium: {stats_today.nb_premium} - {increase_percent(stats_yesterday.nb_premium, stats_today.nb_premium)}
|
|
|
|
nb_cancelled_premium: {stats_today.nb_cancelled_premium} - {increase_percent(stats_yesterday.nb_cancelled_premium, stats_today.nb_cancelled_premium)}
|
|
|
|
nb_apple_premium: {stats_today.nb_apple_premium} - {increase_percent(stats_yesterday.nb_apple_premium, stats_today.nb_apple_premium)}
|
|
|
|
nb_manual_premium: {stats_today.nb_manual_premium} - {increase_percent(stats_yesterday.nb_manual_premium, stats_today.nb_manual_premium)}
|
|
|
|
nb_coinbase_premium: {stats_today.nb_coinbase_premium} - {increase_percent(stats_yesterday.nb_coinbase_premium, stats_today.nb_coinbase_premium)}
|
|
|
|
nb_alias: {stats_today.nb_alias} - {increase_percent(stats_yesterday.nb_alias, stats_today.nb_alias)}
|
|
|
|
|
|
|
|
nb_forward_last_24h: {stats_today.nb_forward_last_24h} - {increase_percent(stats_yesterday.nb_forward_last_24h, stats_today.nb_forward_last_24h)}
|
|
|
|
nb_reply_last_24h: {stats_today.nb_reply_last_24h} - {increase_percent(stats_yesterday.nb_reply_last_24h, stats_today.nb_reply_last_24h)}
|
|
|
|
nb_block_last_24h: {stats_today.nb_block_last_24h} - {increase_percent(stats_yesterday.nb_block_last_24h, stats_today.nb_block_last_24h)}
|
|
|
|
nb_bounced_last_24h: {stats_today.nb_bounced_last_24h} - {increase_percent(stats_yesterday.nb_bounced_last_24h, stats_today.nb_bounced_last_24h)}
|
|
|
|
nb_total_bounced_last_24h: {stats_today.nb_total_bounced_last_24h} - {increase_percent(stats_yesterday.nb_total_bounced_last_24h, stats_today.nb_total_bounced_last_24h)}
|
|
|
|
|
|
|
|
nb_custom_domain: {stats_today.nb_verified_custom_domain} - {increase_percent(stats_yesterday.nb_verified_custom_domain, stats_today.nb_verified_custom_domain)}
|
2021-11-17 16:43:59 +00:00
|
|
|
nb_subdomain: {stats_today.nb_subdomain} - {increase_percent(stats_yesterday.nb_subdomain, stats_today.nb_subdomain)}
|
|
|
|
nb_directory: {stats_today.nb_directory} - {increase_percent(stats_yesterday.nb_directory, stats_today.nb_directory)}
|
|
|
|
nb_deleted_directory: {stats_today.nb_deleted_directory} - {increase_percent(stats_yesterday.nb_deleted_directory, stats_today.nb_deleted_directory)}
|
|
|
|
nb_deleted_subdomain: {stats_today.nb_deleted_subdomain} - {increase_percent(stats_yesterday.nb_deleted_subdomain, stats_today.nb_deleted_subdomain)}
|
|
|
|
|
2021-10-26 10:48:01 +00:00
|
|
|
nb_app: {stats_today.nb_app} - {increase_percent(stats_yesterday.nb_app, stats_today.nb_app)}
|
|
|
|
nb_referred_user: {stats_today.nb_referred_user} - {increase_percent(stats_yesterday.nb_referred_user, stats_today.nb_referred_user)}
|
|
|
|
nb_referred_user_upgrade: {stats_today.nb_referred_user_paid} - {increase_percent(stats_yesterday.nb_referred_user_paid, stats_today.nb_referred_user_paid)}
|
2021-03-23 08:47:57 +00:00
|
|
|
"""
|
|
|
|
|
2021-11-04 13:28:19 +00:00
|
|
|
report += "\n====================================\n"
|
|
|
|
report += f"""
|
|
|
|
# Account bounce report:
|
|
|
|
"""
|
|
|
|
|
|
|
|
for email, bounces in bounce_report():
|
2021-11-06 11:41:37 +00:00
|
|
|
report += f"{email}: {bounces}\n"
|
2021-11-04 13:28:19 +00:00
|
|
|
|
|
|
|
report += f"""\n
|
|
|
|
# Alias creation report:
|
|
|
|
"""
|
|
|
|
|
|
|
|
for email, nb_alias, date in alias_creation_report():
|
|
|
|
report += f"{email}, {date}: {nb_alias}\n"
|
|
|
|
|
|
|
|
report += f"""\n
|
|
|
|
# Full bounce detail report:
|
|
|
|
"""
|
|
|
|
report += all_bounce_report()
|
2021-10-24 08:39:47 +00:00
|
|
|
|
2021-10-26 10:48:01 +00:00
|
|
|
LOG.d("report email: %s", report)
|
2021-03-23 09:50:32 +00:00
|
|
|
|
2021-03-23 08:47:57 +00:00
|
|
|
send_email(
|
|
|
|
ADMIN_EMAIL,
|
|
|
|
subject=f"SimpleLogin Stats for {today}, {nb_user_increase} users, {nb_alias_increase} aliases, {nb_forward_increase} forwards",
|
2021-10-26 10:48:01 +00:00
|
|
|
plaintext=report,
|
2019-11-24 18:42:29 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
2021-04-12 18:20:55 +00:00
|
|
|
def migrate_domain_trash():
|
|
|
|
"""Move aliases from global trash to domain trash if applicable"""
|
2021-10-12 12:36:47 +00:00
|
|
|
for deleted_alias in DeletedAlias.all():
|
2021-04-12 18:20:55 +00:00
|
|
|
alias_domain = get_email_domain_part(deleted_alias.email)
|
|
|
|
if not SLDomain.get_by(domain=alias_domain):
|
|
|
|
custom_domain = CustomDomain.get_by(domain=alias_domain)
|
|
|
|
if custom_domain:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("move %s to domain %s trash", deleted_alias, custom_domain)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.add(
|
2021-04-12 18:20:55 +00:00
|
|
|
DomainDeletedAlias(
|
|
|
|
user_id=custom_domain.user_id,
|
|
|
|
email=deleted_alias.email,
|
|
|
|
domain_id=custom_domain.id,
|
|
|
|
created_at=deleted_alias.created_at,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
DeletedAlias.delete(deleted_alias.id)
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-04-12 18:20:55 +00:00
|
|
|
|
|
|
|
|
|
|
|
def set_custom_domain_for_alias():
|
|
|
|
"""Go through all aliases and make sure custom_domain is correctly set"""
|
2021-10-12 12:36:47 +00:00
|
|
|
sl_domains = [sl_domain.domain for sl_domain in SLDomain.all()]
|
|
|
|
for alias in Alias.filter(Alias.custom_domain_id.is_(None)):
|
2021-04-12 18:20:55 +00:00
|
|
|
if (
|
|
|
|
not any(alias.email.endswith(f"@{sl_domain}") for sl_domain in sl_domains)
|
|
|
|
and not alias.custom_domain_id
|
|
|
|
):
|
|
|
|
alias_domain = get_email_domain_part(alias.email)
|
|
|
|
custom_domain = CustomDomain.get_by(domain=alias_domain)
|
|
|
|
if custom_domain:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("set %s for %s", custom_domain, alias)
|
2021-04-12 18:20:55 +00:00
|
|
|
alias.custom_domain_id = custom_domain.id
|
|
|
|
else: # phantom domain
|
|
|
|
LOG.d("phantom domain %s %s %s", alias.user, alias, alias.enabled)
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-04-12 18:20:55 +00:00
|
|
|
|
|
|
|
|
2020-06-09 15:12:34 +00:00
|
|
|
def sanity_check():
|
2020-06-10 07:19:52 +00:00
|
|
|
"""
|
|
|
|
#TODO: investigate why DNS sometimes not working
|
|
|
|
Different sanity checks
|
2020-06-09 15:12:34 +00:00
|
|
|
- detect if there's mailbox that's using a invalid domain
|
|
|
|
"""
|
2020-10-12 11:28:21 +00:00
|
|
|
mailbox_ids = (
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.query(Mailbox.id)
|
2020-12-06 12:36:32 +00:00
|
|
|
.filter(Mailbox.verified.is_(True), Mailbox.disabled.is_(False))
|
2020-10-12 11:28:21 +00:00
|
|
|
.all()
|
|
|
|
)
|
2020-09-25 08:06:50 +00:00
|
|
|
mailbox_ids = [e[0] for e in mailbox_ids]
|
|
|
|
|
|
|
|
# iterate over id instead of mailbox directly
|
|
|
|
# as a mailbox can be deleted during the sleep time
|
|
|
|
for mailbox_id in mailbox_ids:
|
|
|
|
mailbox = Mailbox.get(mailbox_id)
|
|
|
|
# a mailbox has been deleted
|
|
|
|
if not mailbox:
|
|
|
|
continue
|
|
|
|
|
2020-06-11 22:09:53 +00:00
|
|
|
# hack to not query DNS too often
|
|
|
|
sleep(1)
|
2020-09-25 08:06:50 +00:00
|
|
|
|
2020-10-15 14:21:31 +00:00
|
|
|
if not email_can_be_used_as_mailbox(mailbox.email):
|
2020-06-28 09:18:06 +00:00
|
|
|
mailbox.nb_failed_checks += 1
|
2020-08-30 17:59:39 +00:00
|
|
|
nb_email_log = nb_email_log_for_mailbox(mailbox)
|
2020-10-12 11:28:21 +00:00
|
|
|
|
|
|
|
# send a warning
|
|
|
|
if mailbox.nb_failed_checks == 5:
|
|
|
|
if mailbox.user.email != mailbox.email:
|
|
|
|
send_email(
|
|
|
|
mailbox.user.email,
|
|
|
|
f"Mailbox {mailbox.email} is disabled",
|
|
|
|
render(
|
2021-11-02 10:40:02 +00:00
|
|
|
"transactional/disable-mailbox-warning.txt.jinja2",
|
|
|
|
mailbox=mailbox,
|
2020-10-12 11:28:21 +00:00
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/disable-mailbox-warning.html",
|
|
|
|
mailbox=mailbox,
|
|
|
|
),
|
|
|
|
)
|
2020-08-30 17:59:39 +00:00
|
|
|
|
|
|
|
# alert if too much fail and nb_email_log > 100
|
|
|
|
if mailbox.nb_failed_checks > 10 and nb_email_log > 100:
|
2020-10-12 11:28:21 +00:00
|
|
|
mailbox.disabled = True
|
|
|
|
|
|
|
|
if mailbox.user.email != mailbox.email:
|
|
|
|
send_email(
|
|
|
|
mailbox.user.email,
|
|
|
|
f"Mailbox {mailbox.email} is disabled",
|
2021-11-02 10:41:01 +00:00
|
|
|
render(
|
|
|
|
"transactional/disable-mailbox.txt.jinja2", mailbox=mailbox
|
|
|
|
),
|
2020-10-12 11:28:21 +00:00
|
|
|
render("transactional/disable-mailbox.html", mailbox=mailbox),
|
|
|
|
)
|
2020-06-28 09:18:06 +00:00
|
|
|
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.w(
|
2020-06-09 15:12:34 +00:00
|
|
|
"issue with mailbox %s domain. #alias %s, nb email log %s",
|
|
|
|
mailbox,
|
|
|
|
mailbox.nb_alias(),
|
2020-08-30 17:59:39 +00:00
|
|
|
nb_email_log,
|
2020-06-09 15:12:34 +00:00
|
|
|
)
|
2020-06-28 09:18:06 +00:00
|
|
|
else: # reset nb check
|
|
|
|
mailbox.nb_failed_checks = 0
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-06-09 15:12:34 +00:00
|
|
|
|
2020-06-11 21:36:06 +00:00
|
|
|
for user in User.filter_by(activated=True).all():
|
2021-01-11 11:29:40 +00:00
|
|
|
if sanitize_email(user.email) != user.email:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("%s does not have sanitized email", user)
|
2020-06-09 15:12:34 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for alias in Alias.all():
|
2021-01-11 11:29:40 +00:00
|
|
|
if sanitize_email(alias.email) != alias.email:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Alias %s email not sanitized", alias)
|
2020-09-14 16:22:08 +00:00
|
|
|
|
2020-11-20 17:39:23 +00:00
|
|
|
if alias.name and "\n" in alias.name:
|
2020-11-27 19:37:55 +00:00
|
|
|
alias.name = alias.name.replace("\n", "")
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Alias %s name contains linebreak %s", alias, alias.name)
|
2020-11-20 17:39:23 +00:00
|
|
|
|
2021-01-11 11:35:54 +00:00
|
|
|
contact_email_sanity_date = arrow.get("2021-01-12")
|
2021-10-12 12:36:47 +00:00
|
|
|
for contact in Contact.all():
|
2021-01-11 11:29:40 +00:00
|
|
|
if sanitize_email(contact.reply_email) != contact.reply_email:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Contact %s reply-email not sanitized", contact)
|
2021-01-11 11:29:40 +00:00
|
|
|
|
2021-01-11 11:35:54 +00:00
|
|
|
if (
|
2021-11-18 15:44:04 +00:00
|
|
|
sanitize_email(contact.website_email, not_lower=True)
|
|
|
|
!= contact.website_email
|
2021-01-11 11:35:54 +00:00
|
|
|
and contact.created_at > contact_email_sanity_date
|
|
|
|
):
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Contact %s website-email not sanitized", contact)
|
2020-09-14 16:22:08 +00:00
|
|
|
|
2020-11-25 14:26:17 +00:00
|
|
|
if not contact.invalid_email and not is_valid_email(contact.website_email):
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("%s invalid email", contact)
|
2020-11-25 14:26:17 +00:00
|
|
|
contact.invalid_email = True
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-11-25 14:26:17 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for mailbox in Mailbox.all():
|
2021-01-11 11:29:40 +00:00
|
|
|
if sanitize_email(mailbox.email) != mailbox.email:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Mailbox %s address not sanitized", mailbox)
|
2020-06-09 15:12:34 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for contact in Contact.all():
|
2020-11-22 12:07:09 +00:00
|
|
|
if normalize_reply_email(contact.reply_email) != contact.reply_email:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e(
|
2020-11-22 12:07:09 +00:00
|
|
|
"Contact %s reply email is not normalized %s",
|
|
|
|
contact,
|
|
|
|
contact.reply_email,
|
|
|
|
)
|
2020-11-18 15:12:00 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for domain in CustomDomain.all():
|
2020-11-20 17:40:07 +00:00
|
|
|
if domain.name and "\n" in domain.name:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("Domain %s name contain linebreak %s", domain, domain.name)
|
2020-11-20 17:40:07 +00:00
|
|
|
|
2021-04-12 18:20:55 +00:00
|
|
|
migrate_domain_trash()
|
|
|
|
set_custom_domain_for_alias()
|
|
|
|
|
2020-06-09 15:12:34 +00:00
|
|
|
LOG.d("Finish sanity check")
|
|
|
|
|
|
|
|
|
2020-08-29 17:05:32 +00:00
|
|
|
def check_custom_domain():
|
|
|
|
LOG.d("Check verified domain for DNS issues")
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for custom_domain in CustomDomain.filter_by(verified=True): # type: CustomDomain
|
2020-08-29 17:05:32 +00:00
|
|
|
mx_domains = get_mx_domains(custom_domain.domain)
|
|
|
|
|
|
|
|
if sorted(mx_domains) != sorted(EMAIL_SERVERS_WITH_PRIORITY):
|
|
|
|
user = custom_domain.user
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.w(
|
2020-08-29 17:05:32 +00:00
|
|
|
"The MX record is not correctly set for %s %s %s",
|
|
|
|
custom_domain,
|
|
|
|
user,
|
|
|
|
mx_domains,
|
|
|
|
)
|
|
|
|
|
2020-10-20 14:51:25 +00:00
|
|
|
custom_domain.nb_failed_checks += 1
|
|
|
|
|
|
|
|
# send alert if fail for 5 consecutive days
|
|
|
|
if custom_domain.nb_failed_checks > 5:
|
|
|
|
domain_dns_url = f"{URL}/dashboard/domains/{custom_domain.id}/dns"
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.w("Alert domain MX check fails %s about %s", user, custom_domain)
|
2020-10-20 14:51:25 +00:00
|
|
|
send_email_with_rate_control(
|
|
|
|
user,
|
|
|
|
AlERT_WRONG_MX_RECORD_CUSTOM_DOMAIN,
|
|
|
|
user.email,
|
|
|
|
f"Please update {custom_domain.domain} DNS on SimpleLogin",
|
|
|
|
render(
|
2021-11-02 10:42:20 +00:00
|
|
|
"transactional/custom-domain-dns-issue.txt.jinja2",
|
2020-10-20 14:51:25 +00:00
|
|
|
custom_domain=custom_domain,
|
|
|
|
domain_dns_url=domain_dns_url,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/custom-domain-dns-issue.html",
|
|
|
|
custom_domain=custom_domain,
|
|
|
|
domain_dns_url=domain_dns_url,
|
|
|
|
),
|
|
|
|
max_nb_alert=1,
|
|
|
|
nb_day=30,
|
|
|
|
)
|
|
|
|
# reset checks
|
|
|
|
custom_domain.nb_failed_checks = 0
|
|
|
|
else:
|
|
|
|
# reset checks
|
|
|
|
custom_domain.nb_failed_checks = 0
|
2020-08-29 17:05:32 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2020-08-29 17:05:32 +00:00
|
|
|
|
|
|
|
|
2020-08-17 12:02:59 +00:00
|
|
|
def delete_old_monitoring():
|
|
|
|
"""
|
|
|
|
Delete old monitoring records
|
|
|
|
"""
|
|
|
|
max_time = arrow.now().shift(days=-30)
|
2021-10-12 12:36:47 +00:00
|
|
|
nb_row = Monitoring.filter(Monitoring.created_at < max_time).delete()
|
|
|
|
Session.commit()
|
2020-08-17 12:02:59 +00:00
|
|
|
LOG.d("delete monitoring records older than %s, nb row %s", max_time, nb_row)
|
|
|
|
|
|
|
|
|
2021-05-18 19:18:07 +00:00
|
|
|
async def _hibp_check(api_key, queue):
|
2021-05-13 20:44:16 +00:00
|
|
|
"""
|
2021-05-15 21:23:59 +00:00
|
|
|
Uses a single API key to check the queue as fast as possible.
|
2021-05-13 20:44:16 +00:00
|
|
|
|
2021-05-15 21:23:59 +00:00
|
|
|
This function to be ran simultaneously (multiple _hibp_check functions with different keys on the same queue) to make maximum use of multiple API keys.
|
|
|
|
"""
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
alias_id = queue.get_nowait()
|
2021-05-18 19:18:07 +00:00
|
|
|
except asyncio.QueueEmpty:
|
2021-05-15 21:23:59 +00:00
|
|
|
return
|
2021-05-13 20:44:16 +00:00
|
|
|
|
2021-05-15 21:23:59 +00:00
|
|
|
alias = Alias.get(alias_id)
|
2021-05-22 14:36:09 +00:00
|
|
|
# an alias can be deleted in the meantime
|
|
|
|
if not alias:
|
|
|
|
return
|
|
|
|
|
2021-05-15 21:23:59 +00:00
|
|
|
LOG.d("Checking HIBP for %s", alias)
|
2021-05-13 20:44:16 +00:00
|
|
|
|
|
|
|
request_headers = {
|
|
|
|
"user-agent": "SimpleLogin",
|
2021-05-15 21:23:59 +00:00
|
|
|
"hibp-api-key": api_key,
|
2021-05-13 20:44:16 +00:00
|
|
|
}
|
2021-05-14 17:59:32 +00:00
|
|
|
r = requests.get(
|
|
|
|
f"https://haveibeenpwned.com/api/v3/breachedaccount/{urllib.parse.quote(alias.email)}",
|
|
|
|
headers=request_headers,
|
|
|
|
)
|
2021-05-13 20:44:16 +00:00
|
|
|
|
|
|
|
if r.status_code == 200:
|
|
|
|
# Breaches found
|
2021-05-17 19:29:29 +00:00
|
|
|
alias.hibp_breaches = [
|
|
|
|
Hibp.get_by(name=entry["Name"]) for entry in r.json()
|
|
|
|
]
|
2021-05-19 10:46:55 +00:00
|
|
|
if len(alias.hibp_breaches) > 0:
|
|
|
|
LOG.w("%s appears in HIBP breaches %s", alias, alias.hibp_breaches)
|
2021-05-13 20:44:16 +00:00
|
|
|
elif r.status_code == 404:
|
|
|
|
# No breaches found
|
|
|
|
alias.hibp_breaches = []
|
2021-06-19 17:56:18 +00:00
|
|
|
elif r.status_code == 429:
|
|
|
|
# rate limited
|
|
|
|
LOG.w("HIBP rate limited, check alias %s in the next run", alias)
|
|
|
|
await asyncio.sleep(1.6)
|
|
|
|
return
|
2021-06-17 21:04:25 +00:00
|
|
|
elif r.status_code > 500:
|
|
|
|
LOG.w("HIBP server 5** error %s", r.status_code)
|
|
|
|
return
|
2021-05-13 20:44:16 +00:00
|
|
|
else:
|
2021-05-14 17:59:32 +00:00
|
|
|
LOG.error(
|
|
|
|
"An error occured while checking alias %s: %s - %s",
|
|
|
|
alias,
|
|
|
|
r.status_code,
|
|
|
|
r.text,
|
|
|
|
)
|
2021-05-15 21:23:59 +00:00
|
|
|
return
|
2021-05-13 20:44:16 +00:00
|
|
|
|
2021-05-20 17:00:11 +00:00
|
|
|
alias.hibp_last_check = arrow.utcnow()
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.add(alias)
|
|
|
|
Session.commit()
|
2021-05-13 20:44:16 +00:00
|
|
|
|
|
|
|
LOG.d("Updated breaches info for %s", alias)
|
|
|
|
|
2021-05-28 15:47:54 +00:00
|
|
|
await asyncio.sleep(1.6)
|
2021-05-15 21:23:59 +00:00
|
|
|
|
|
|
|
|
2021-05-18 19:18:07 +00:00
|
|
|
async def check_hibp():
|
2021-05-15 21:23:59 +00:00
|
|
|
"""
|
|
|
|
Check all aliases on the HIBP (Have I Been Pwned) API
|
|
|
|
"""
|
|
|
|
LOG.d("Checking HIBP API for aliases in breaches")
|
|
|
|
|
|
|
|
if len(HIBP_API_KEYS) == 0:
|
2021-09-08 09:29:55 +00:00
|
|
|
LOG.e("No HIBP API keys")
|
2021-05-15 21:23:59 +00:00
|
|
|
return
|
|
|
|
|
|
|
|
LOG.d("Updating list of known breaches")
|
|
|
|
r = requests.get("https://haveibeenpwned.com/api/v3/breaches")
|
|
|
|
for entry in r.json():
|
2021-07-29 06:51:21 +00:00
|
|
|
hibp_entry = Hibp.get_or_create(name=entry["Name"])
|
|
|
|
hibp_entry.date = arrow.get(entry["BreachDate"])
|
|
|
|
hibp_entry.description = entry["Description"]
|
2021-05-15 21:23:59 +00:00
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-05-15 21:23:59 +00:00
|
|
|
LOG.d("Updated list of known breaches")
|
|
|
|
|
|
|
|
LOG.d("Preparing list of aliases to check")
|
2021-05-18 19:18:07 +00:00
|
|
|
queue = asyncio.Queue()
|
2021-05-17 16:15:12 +00:00
|
|
|
max_date = arrow.now().shift(days=-HIBP_SCAN_INTERVAL_DAYS)
|
|
|
|
for alias in (
|
2021-10-12 12:36:47 +00:00
|
|
|
Alias.filter(
|
2021-05-17 16:15:12 +00:00
|
|
|
or_(Alias.hibp_last_check.is_(None), Alias.hibp_last_check < max_date)
|
|
|
|
)
|
2021-05-19 14:12:58 +00:00
|
|
|
.filter(Alias.enabled)
|
2021-05-20 17:00:11 +00:00
|
|
|
.order_by(Alias.hibp_last_check.asc())
|
2021-05-17 16:15:12 +00:00
|
|
|
.all()
|
|
|
|
):
|
2021-05-18 19:18:07 +00:00
|
|
|
await queue.put(alias.id)
|
2021-05-17 16:15:12 +00:00
|
|
|
|
2021-05-15 21:23:59 +00:00
|
|
|
LOG.d("Need to check about %s aliases", queue.qsize())
|
|
|
|
|
|
|
|
# Start one checking process per API key
|
|
|
|
# Each checking process will take one alias from the queue, get the info
|
|
|
|
# and then sleep for 1.5 seconds (due to HIBP API request limits)
|
|
|
|
checkers = []
|
|
|
|
for i in range(len(HIBP_API_KEYS)):
|
2021-05-18 19:18:07 +00:00
|
|
|
checker = asyncio.create_task(
|
|
|
|
_hibp_check(
|
2021-05-15 21:23:59 +00:00
|
|
|
HIBP_API_KEYS[i],
|
|
|
|
queue,
|
2021-05-18 19:18:07 +00:00
|
|
|
)
|
2021-05-15 21:23:59 +00:00
|
|
|
)
|
|
|
|
checkers.append(checker)
|
|
|
|
|
|
|
|
# Wait until all checking processes are done
|
2021-05-18 19:18:07 +00:00
|
|
|
for checker in checkers:
|
|
|
|
await checker
|
2021-05-15 21:23:59 +00:00
|
|
|
|
2021-05-13 20:44:16 +00:00
|
|
|
LOG.d("Done checking HIBP API for aliases in breaches")
|
|
|
|
|
|
|
|
|
2021-07-29 07:35:00 +00:00
|
|
|
def notify_hibp():
|
|
|
|
"""
|
|
|
|
Send aggregated email reports for HIBP breaches
|
|
|
|
"""
|
|
|
|
# to get a list of users that have at least a breached alias
|
|
|
|
alias_query = (
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.query(Alias)
|
2021-07-29 07:35:00 +00:00
|
|
|
.options(joinedload(Alias.hibp_breaches))
|
|
|
|
.filter(Alias.hibp_breaches.any())
|
2021-10-12 12:36:47 +00:00
|
|
|
.filter(Alias.id.notin_(Session.query(HibpNotifiedAlias.alias_id)))
|
2021-07-29 07:35:00 +00:00
|
|
|
.distinct(Alias.user_id)
|
|
|
|
.all()
|
|
|
|
)
|
|
|
|
|
|
|
|
user_ids = [alias.user_id for alias in alias_query]
|
|
|
|
|
2021-10-12 12:36:47 +00:00
|
|
|
for user in User.filter(User.id.in_(user_ids)):
|
2021-07-29 07:35:00 +00:00
|
|
|
breached_aliases = (
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.query(Alias)
|
2021-07-29 07:35:00 +00:00
|
|
|
.options(joinedload(Alias.hibp_breaches))
|
|
|
|
.filter(Alias.hibp_breaches.any(), Alias.user_id == user.id)
|
|
|
|
.all()
|
|
|
|
)
|
|
|
|
|
2021-07-29 07:55:36 +00:00
|
|
|
LOG.d(
|
|
|
|
f"Send new breaches found email to %s for %s breaches aliases",
|
|
|
|
user,
|
|
|
|
len(breached_aliases),
|
|
|
|
)
|
2021-07-29 07:35:00 +00:00
|
|
|
|
|
|
|
send_email(
|
|
|
|
user.email,
|
|
|
|
f"You were in a data breach",
|
|
|
|
render(
|
|
|
|
"transactional/hibp-new-breaches.txt.jinja2",
|
|
|
|
user=user,
|
|
|
|
breached_aliases=breached_aliases,
|
|
|
|
),
|
|
|
|
render(
|
|
|
|
"transactional/hibp-new-breaches.html",
|
|
|
|
user=user,
|
|
|
|
breached_aliases=breached_aliases,
|
|
|
|
),
|
|
|
|
)
|
|
|
|
|
2021-07-29 07:55:36 +00:00
|
|
|
# add the breached aliases to HibpNotifiedAlias to avoid sending another email
|
|
|
|
for alias in breached_aliases:
|
|
|
|
HibpNotifiedAlias.create(user_id=user.id, alias_id=alias.id)
|
2021-10-12 12:36:47 +00:00
|
|
|
Session.commit()
|
2021-07-29 07:55:36 +00:00
|
|
|
|
2021-07-29 07:35:00 +00:00
|
|
|
|
create BaseForm to enable CSRF
register page
redirect user to dashboard if they are logged in
enable csrf for login page
Set models more strict
bootstrap developer page
add helper method to ModelMixin, remove CRUDMixin
display list of clients on developer index, add copy client-secret to clipboard using clipboardjs
add toastr and use jquery non slim
display a toast when user copies the client-secret
create new client, generate client-id using unidecode
client detail page: can edit client
add delete client
implement /oauth/authorize and /oauth/allow-deny
implement /oauth/token
add /oauth/user_info endpoint
handle scopes: wip
take into account scope: display scope, return user data according to scope
create virtual-domain, gen email, client_user model WIP
create authorize_nonlogin_user page
user can choose to generate a new email
no need to interfere with root logger
log for before and after request
if user has already allowed a client: generate a auth-code and redirect user to client
get_user_info takes into account gen email
display list of clients that have user has authorised
use yk-client domain instead of localhost as cookie depends on the domain name
use wtforms instead of flask_wtf
Dockerfile
delete virtual domain
EMAIL_DOMAIN can come from env var
bind to host 0.0.0.0
fix signup error: use session as default csrf_context
rename yourkey to simplelogin
add python-dotenv, ipython, sqlalchemy_utils
create DB_URI, FLASK_SECRET. Load config from CONFIG file if exist
add shortcuts to logging
create shell
add psycopg2
do not add local data in Dockerfile
add drop_db into shell
add shell.prepare_db()
fix prepare_db
setup sentry
copy assets from tabler/dist
add icon downloaded from https://commons.wikimedia.org/wiki/File:Simpleicons_Interface_key-tool-1.svg
integrate tabler - login and register page
add favicon
template: default, header. Use gravatar for user avatar url
use default template for dashboard, developer page
use another icon
add clipboard and notie
prettify dashboard
add notie css
add fake gen email and client-user
prettify list client page, use notie for toast
add email, name scope to new client
display client scope in client list
prettify new-client, client-detail
add sentry-sdk and blinker
add arrow, add dt jinja filter, prettify logout, dashboard
comment "last used" in dashboard for now
prettify date display
add copy email to clipboard to dashboard
use "users" as table name for User as "user" is reserved key in postgres
call prepare_db() when creating new db
error page 400, 401, 403, 404
prettify authorize_login_user
create already_authorize.html for user who has already authorized a client
user can generate new email
display all other generated emails
add ENV variable, only reset DB when ENV=local
fix: not return other users gen emails
display nb users for each client
refactor shell: remove prepare_db()
add sendgrid
add /favicon.ico route
add new config: URL, SUPPORT_EMAIL, SENDGRID_API_KEY
user needs to activate their account before login
create copy button on dashboard
client can have multiple redirect uris, in client detail can add/remove redirect-uri,
use redirect_uri passed in /authorize
refactor: move get_user_info into ClientUser model
dashboard: display all apps, all generated emails
add "id" into user_info
add trigger email button
invalidate the session at each new version by changing the secret
centralize Client creation into Client.create_new
user can enable/disable email forwarding
setup auto dismiss alert: just add .alert-auto-dismiss
move name down in register form
add shell.add_real_data
move blueprint template to its own package
prettify authorize page for non-authenticated user
update readme, return error if not redirect_uri
add flask-wtf, use psycopg2-binary
use flask-wtf FlaskForm instead of Form
rename email -> email_utils
add AWS_REGION, BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY to config
add s3 module
add File model, add Client.icon_id
handle client icon update
can create client with icon
display client icon in client list page
add Client.home_url
take into account Client.home_url
add boto3
register: ask name first
only show "trigger test email" if email forwarding is enabled
display gen email in alphabetical order, client in client.name alphabetical order
better error page
the modal does not get close when user clicks outside of modal
add Client.published column
discover page that displays all published Client
add missing bootstrap.bundle.min.js.map
developer can publish/unpublish their app in discover
use notie for display flash message
create hotmail account
fix missing jquery
add footer, add global jinja2 variable
strengthen model: use nullable=False whenever possible,
rename client_id to oauth_client_id, client_secret to oauth_client_secret
add flask-migrate
init migrate
1st migrate version
fix rename client_id -> oauth_client_id
prettify UI
use flask_migrate.upgrade() instead of db.create_all()
make sure requirejs.config is called for all page
enable sentry for js, use uppercase for global jinja2 variables
add flask-admin
add User.is_admin column
setup flask admin, only accessible to admin user
fix migration: add server_default
replace session[redirect_after_login] by "next" request args
add pyproject.toml: ignore migrations/ in black
add register waiting_activation_email page
better email wording
add pytest
add get_host_name_and_scheme and tests
example fail test
fix test
fix client-id display
add flask-cors
/user_info supports cors, add /me as /user_info synonym
return client in /me
support implicit flow
no need to use with "app.app_context()"
add watchtower to requirement
add param ENABLE_CLOUDWATCH, CLOUDWATCH_LOG_GROUP, CLOUDWATCH_LOG_STREAM
add cloudwatch logger if cloudwatch is enabled
add 500 error page
add help text for list of used client
display list of app/website that an email has been used
click on client name brings to client detail page
create style.css to add additional style, append its url with the current sha1 to avoid cache
POC on how to send email using postfix
add sqlalchemy-utils
use arrow instead of datetime
add new params STRIPE_API, STRIPE_YEARLY_SKU, STRIPE_MONTHLY_PLAN
show full error in local
add plan, plan_expiration to User, need to create enum directly in migration script, cf https://github.com/sqlalchemy/alembic/issues/67
reformat all html files: use space instead of tab
new user will have trial plan for 15 days
add new param MAX_NB_EMAIL_FREE_PLAN
only user with enough quota can create new email
if user cannot create new gen email, pick randomly one from existing gen emails. Use flush instead of commit
rename STRIPE_YEARLY_SKU -> STRIPE_YEARLY_PLAN
open client page in discover in a new tab
add stripe
not logging /static call: disable flask logging, replace by after_request
add param STRIPE_SECRET_KEY
add 3 columns stripe_customer_id, stripe_card_token, stripe_subscription_id
user can upgrade their pricing
add setting page as coming-soon
add GenEmail, ClientUser to admin
ignore /admin/static logging
add more fake data
add ondelete="cascade" whenever possible
rename plan_expiration -> trial_expiration
reset migration: delete old migrations, create new one
rename test_send_email -> poc_send_email to avoid the file being called by pytest
add new param LYRA_ANALYTICS_ID, add lyra analytics
add how to create new migration into readme
add drift to base.html
notify admin when new user signs up or pays subscription
log exception in case of 500
use sendgrid to notify admin
add alias /userinfo to user_info endpoint
add change_password to shell
add info on how payment is handled
invite user to retry if card not working
remove drift and add "contact us" link
move poc_send_email into poc/
support getting client-id, client-secret from form-data in addition to basic auth
client-id, client-secret is passed in form-data by passport-oauth2 for ex
add jwtRS256 private and public key
add jwk-jws-jwt poc
add new param OPENID_PRIVATE_KEY_PATH, OPENID_PRIVATE_KEY_PATH
add scope, redirect_url to AuthorizationCode and OauthToken
take into scope when creating oauth-token, authorization-code
add jwcrypto
add jose_utils: make_id_token and verify_id_token
add &scope to redirect uri
add "email_verified": True into user_info
fix user not activated
add /oauth2 as alias for /oauth
handle case where scope and state are empty
remove threaded=False
Use Email Alias as wording
remove help text
user can re-send activation email
add "expired" into ActivationCode
Handle the case activation code is expired
reformat: use form.validate_on_submit instead of request.method == post && form.validate
use error text instead of flash()
display client oauth-id and oauth-secret on client detail page
not display oauth-secret on client listing
fix expiration check
improve page title, footer
add /jwks and /.well-known/openid-configuration
init properly tests, fix blueprint conflict bug in flask-admin
create oauth_models module
rename Scope -> ScopeE to distinguish with Scope DB model
set app.url_map.strict_slashes = False
use ScopeE instead of SCOPE_NAME, ...
support access_token passed as args in /userinfo
merge /allow-deny into /authorize
improve wording
take into account the case response_type=code and openid is in scope
take into account response_type=id_token, id_token token, id_token code
make sure to use in-memory db in test
fix scope can be null
allow cross_origin for /.well-known/openid-configuration and /jwks
fix footer link
center authorize form
rename trial_expiration to plan_expiration
move stripe init to create_app()
use real email to be able to receive email notification
add user.profile_picture_id column
use user profile picture and fallback to gravatar
use nguyenkims+local@gm to distinguish with staging
handle plan cancel, reactivation, user profile update
fix can_create_new_email
create cron.py that set plan to free when expired
add crontab.yml
add yacron
use notify_admin instead of LOG.error
add ResetPasswordCode model
user can change password in setting
increase display time for notie
add forgot_password page
If login error: redirect to this page upon success login.
hide discover tab
add column user.is_developer
only show developer menu to developer
comment out the publish button
set local user to developer
make sure only developer can access /developer blueprint
User is invited to upgrade if they are in free plan or their trial ends soon
not sending email when in local mode
create Partner model
create become partner page
use normal error handling on local
fix migration
add "import sqlalchemy_utils" into migration template
small refactoring on setting page
handle promo code. TODO: add migration file
add migration for user.promo_codes
move email alias on top of apps in dashboard
add introjs
move encode_url to utils
create GenEmail.create_new_gen_email
create a first alias mail to show user how to use when they login
show intro when user visits the website the first time
fix register
2019-07-02 07:20:12 +00:00
|
|
|
if __name__ == "__main__":
|
|
|
|
LOG.d("Start running cronjob")
|
2020-02-02 15:17:04 +00:00
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument(
|
|
|
|
"-j",
|
|
|
|
"--job",
|
|
|
|
help="Choose a cron job to run",
|
|
|
|
type=str,
|
2020-03-05 08:35:20 +00:00
|
|
|
choices=[
|
|
|
|
"stats",
|
|
|
|
"notify_trial_end",
|
|
|
|
"notify_manual_subscription_end",
|
|
|
|
"notify_premium_end",
|
2021-01-26 08:54:58 +00:00
|
|
|
"delete_logs",
|
2020-04-19 09:20:44 +00:00
|
|
|
"poll_apple_subscription",
|
2020-06-09 15:12:34 +00:00
|
|
|
"sanity_check",
|
2020-08-17 12:02:59 +00:00
|
|
|
"delete_old_monitoring",
|
2020-08-29 17:05:32 +00:00
|
|
|
"check_custom_domain",
|
2021-05-14 17:59:32 +00:00
|
|
|
"check_hibp",
|
2021-07-29 07:35:00 +00:00
|
|
|
"notify_hibp",
|
2020-03-05 08:35:20 +00:00
|
|
|
],
|
2020-02-02 15:17:04 +00:00
|
|
|
)
|
|
|
|
args = parser.parse_args()
|
2021-10-26 08:52:28 +00:00
|
|
|
# wrap in an app context to benefit from app setup like database cleanup, sentry integration, etc
|
|
|
|
with create_light_app().app_context():
|
|
|
|
if args.job == "stats":
|
|
|
|
LOG.d("Compute Stats")
|
|
|
|
stats()
|
|
|
|
elif args.job == "notify_trial_end":
|
|
|
|
LOG.d("Notify users with trial ending soon")
|
|
|
|
notify_trial_end()
|
|
|
|
elif args.job == "notify_manual_subscription_end":
|
|
|
|
LOG.d("Notify users with manual subscription ending soon")
|
|
|
|
notify_manual_sub_end()
|
|
|
|
elif args.job == "notify_premium_end":
|
|
|
|
LOG.d("Notify users with premium ending soon")
|
|
|
|
notify_premium_end()
|
|
|
|
elif args.job == "delete_logs":
|
|
|
|
LOG.d("Deleted Logs")
|
|
|
|
delete_logs()
|
|
|
|
elif args.job == "poll_apple_subscription":
|
|
|
|
LOG.d("Poll Apple Subscriptions")
|
|
|
|
poll_apple_subscription()
|
|
|
|
elif args.job == "sanity_check":
|
|
|
|
LOG.d("Check data consistency")
|
|
|
|
sanity_check()
|
|
|
|
elif args.job == "delete_old_monitoring":
|
|
|
|
LOG.d("Delete old monitoring records")
|
|
|
|
delete_old_monitoring()
|
|
|
|
elif args.job == "check_custom_domain":
|
|
|
|
LOG.d("Check custom domain")
|
|
|
|
check_custom_domain()
|
|
|
|
elif args.job == "check_hibp":
|
|
|
|
LOG.d("Check HIBP")
|
|
|
|
asyncio.run(check_hibp())
|
|
|
|
elif args.job == "notify_hibp":
|
|
|
|
LOG.d("Notify users about HIBP breaches")
|
|
|
|
notify_hibp()
|