diff --git a/.travis.yml b/.travis.yml index 29bfb975f..08ca9cbf3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,17 @@ deploy: key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip bundle_type: zip application: notifications-api - deployment_group: notifications-api-celery + deployment_group: notifications_admin_api_deployment_group + region: eu-west-1 + on: *2 +- provider: codedeploy + access_key_id: AKIAJQPPNM6P6V53SWKA + secret_access_key: *1 + bucket: notifications-api-codedeploy + key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip + bundle_type: zip + application: notifications-api + deployment_group: notifications_delivery_api_deployment_group region: eu-west-1 on: *2 - provider: s3 @@ -90,6 +100,47 @@ deploy: deployment_group: notifications_admin_api_deployment_group region: eu-west-1 on: *2 +- provider: s3 + access_key_id: AKIAJ5MKF6G3P2JQP4QQ + secret_access_key: &1 + secure: daC1bCHXqLRK+iIZ8P699KCnTh77lwV4KxrZxL1yd6cstgfptyd/rg1WgRwE6QdxOCT9gQvKWUZFCzFy7M6E/Ih8EUHqEXTzC5M4oAye8rhePIBMQwqkgfYyIoZ3LdDMMP5JfBhiz0zS3Vj7HerL2qIu12adJBjkRJx3XAGimCrFOMQ0xUXQAKDjL6Xmv+gVz2f/ISLy6icKY4KNGt3cQV+8pa5aMF34C9R2udA9N67EWlXlh7hJbFtmY+0Zqpo8Rr6wKRb5MA0xEcTVLORSz1aa6GkxUCbzaIH99p7z3Ghz0qW2bUi9ZcDrvg0GLbVe1T+1HXhfktJfW8wnzw6A/2U/CIIFDQZ/qk0w/DkEwpQinXow99Zl49CcEU+v8llKhg5nM3LmAZCQg1c/iZyP/d90AwAMoMA/VTDD72M93IqTJQH18eC8g02DwE0hNDD6aos5wzeuDeiH/6BG+Tq0pDl0y0aWCcHf3vGRlo/5GlWfpE0vMQEC+qnEOWOUqSprCdSypgD2Aip9mCC98w4BkqKKvGNHPZolA7rxf7E9hTK+BNPRATpYsHR1X/1Xl0TMc/pHhjU1yNXzWnI/kOlNV2CRq3slEtcWihaEo8oDHJ+BhGT49Ps3Je7UB2xO/jXXFPhwJotPMOacTcnUkGqVJSlK1g6TIn4t9nTVSY8KFUs= + local_dir: dpl_cd_upload + skip_cleanup: true + region: eu-west-1 + on: &2 + repo: alphagov/notifications-api + branch: live + bucket: live-notifications-api-codedeploy +- provider: codedeploy + access_key_id: AKIAJ5MKF6G3P2JQP4QQ + secret_access_key: *1 + bucket: live-notifications-api-codedeploy + key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip + bundle_type: zip + application: notifications-api + deployment_group: live_notifications_delivery_api_deployment_group + region: eu-west-1 + on: *2 +- provider: codedeploy + access_key_id: AKIAJ5MKF6G3P2JQP4QQ + secret_access_key: *1 + bucket: live-notifications-api-codedeploy + key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip + bundle_type: zip + application: notifications-api + deployment_group: live_notifications_api_deployment_group + region: eu-west-1 + on: *2 +- provider: codedeploy + access_key_id: AKIAJ5MKF6G3P2JQP4QQ + secret_access_key: *1 + bucket: live-notifications-api-codedeploy + key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip + bundle_type: zip + application: notifications-api + deployment_group: live_notifications_admin_api_deployment_group + region: eu-west-1 + on: *2 before_deploy: - ./scripts/update_version_file.sh - zip -r --exclude=*__pycache__* notifications-api * diff --git a/README.md b/README.md index 5c6d177b4..0171a2ace 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,6 @@ scripts/run_app.sh scripts/run_celery.sh ``` +``` +scripts/run_celery_beat.sh +``` diff --git a/app/__init__.py b/app/__init__.py index 3b92f9265..f34177786 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -27,11 +27,14 @@ encryption = Encryption() api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) -def create_app(): +def create_app(app_name=None): application = Flask(__name__) application.config.from_object(os.environ['NOTIFY_API_ENVIRONMENT']) + if app_name: + application.config['NOTIFY_APP_NAME'] = app_name + init_app(application) db.init_app(application) ma.init_app(application) @@ -92,9 +95,7 @@ def init_app(app): def email_safe(string): return "".join([ - character.lower() - if character.isalnum() or character == "." - else "" for character in re.sub("\s+", ".", string.strip()) + character.lower() if character.isalnum() or character == "." else "" for character in re.sub("\s+", ".", string.strip()) # noqa ]) diff --git a/app/celery/tasks.py b/app/celery/tasks.py index ff952d6e2..c354273a9 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -25,7 +25,8 @@ from sqlalchemy.exc import SQLAlchemyError from app.aws import s3 from datetime import datetime from utils.template import Template -from utils.recipients import RecipientCSV, validate_phone_number, format_phone_number +from utils.recipients import RecipientCSV, format_phone_number, validate_phone_number +from app.validation import (allowed_send_to_email, allowed_send_to_number) @notify_celery.task(name="delete-verify-codes") @@ -204,7 +205,7 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): ) client.send_sms( - to=notification['to'], + to=format_phone_number(validate_phone_number(notification['to'])), content=template.replaced, reference=str(notification_id) ) @@ -223,20 +224,6 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at): current_app.logger.debug(e) -def allowed_send_to_number(service, to): - if service.restricted and format_phone_number(validate_phone_number(to)) not in [ - format_phone_number(validate_phone_number(user.mobile_number)) for user in service.users - ]: - return False - return True - - -def allowed_send_to_email(service, to): - if service.restricted and to not in [user.email_address for user in service.users]: - return False - return True - - @notify_celery.task(name="send-email") def send_email(service_id, notification_id, subject, from_address, encrypted_notification, created_at): notification = encryption.decrypt(encrypted_notification) @@ -300,7 +287,9 @@ def send_sms_code(encrypted_verification): verification_message = encryption.decrypt(encrypted_verification) try: firetext_client.send_sms( - verification_message['to'], verification_message['secret_code'], 'send-sms-code' + format_phone_number(validate_phone_number(verification_message['to'])), + verification_message['secret_code'], + 'send-sms-code' ) except FiretextClientException as e: current_app.logger.exception(e) @@ -381,3 +370,23 @@ def email_reset_password(encrypted_reset_password_message): url=reset_password_message['reset_password_url'])) except AwsSesClientException as e: current_app.logger.exception(e) + + +def registration_verification_template(name, url): + from string import Template + t = Template("Hi $name,\n\n" + "To complete your registration for GOV.UK Notify please click the link below\n\n $url") + return t.substitute(name=name, url=url) + + +@notify_celery.task(name='email-registration-verification') +def email_registration_verification(encrypted_verification_message): + verification_message = encryption.decrypt(encrypted_verification_message) + try: + aws_ses_client.send_email(current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'], + verification_message['to'], + "Confirm GOV.UK Notify registration", + registration_verification_template(name=verification_message['name'], + url=verification_message['url'])) + except AwsSesClientException as e: + current_app.logger.exception(e) diff --git a/app/models.py b/app/models.py index 0f28d2452..a6f8c9d0f 100644 --- a/app/models.py +++ b/app/models.py @@ -40,6 +40,7 @@ class User(db.Model): logged_in_at = db.Column(db.DateTime, nullable=True) failed_login_count = db.Column(db.Integer, nullable=False, default=0) state = db.Column(db.String, nullable=False, default='pending') + platform_admin = db.Column(db.Boolean, nullable=False, default=False) @property def password(self): @@ -114,12 +115,12 @@ class NotificationStatistics(db.Model): day = db.Column(db.String(255), nullable=False) service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False) service = db.relationship('Service', backref=db.backref('service_notification_stats', lazy='dynamic')) - emails_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False) - emails_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True) - emails_error = db.Column(db.BigInteger, index=False, unique=False, nullable=True) - sms_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False) - sms_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True) - sms_error = db.Column(db.BigInteger, index=False, unique=False, nullable=True) + emails_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) + emails_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) + emails_error = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) + sms_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) + sms_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) + sms_error = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0) __table_args__ = ( UniqueConstraint('service_id', 'day', name='uix_service_to_day'), @@ -144,13 +145,13 @@ class Template(db.Model): index=False, unique=False, nullable=False, - default=datetime.datetime.now) + default=datetime.datetime.utcnow) updated_at = db.Column( db.DateTime, index=False, unique=False, nullable=True, - onupdate=datetime.datetime.utcnow()) + onupdate=datetime.datetime.utcnow) content = db.Column(db.Text, index=False, unique=False, nullable=False) service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False) service = db.relationship('Service', backref=db.backref('templates', lazy='dynamic')) @@ -176,13 +177,13 @@ class Job(db.Model): index=False, unique=False, nullable=False, - default=datetime.datetime.utcnow()) + default=datetime.datetime.utcnow) updated_at = db.Column( db.DateTime, index=False, unique=False, nullable=True, - onupdate=datetime.datetime.utcnow()) + onupdate=datetime.datetime.utcnow) status = db.Column(db.Enum(*JOB_STATUS_TYPES, name='job_status_types'), nullable=False, default='pending') notification_count = db.Column(db.Integer, nullable=False) notifications_sent = db.Column(db.Integer, nullable=False, default=0) @@ -217,7 +218,7 @@ class VerifyCode(db.Model): index=False, unique=False, nullable=False, - default=datetime.datetime.utcnow()) + default=datetime.datetime.utcnow) @property def code(self): @@ -262,7 +263,7 @@ class Notification(db.Model): index=False, unique=False, nullable=True, - onupdate=datetime.datetime.utcnow()) + onupdate=datetime.datetime.utcnow) status = db.Column( db.Enum(*NOTIFICATION_STATUS_TYPES, name='notification_status_types'), nullable=False, default='sent') reference = db.Column(db.String, nullable=True, index=True) @@ -286,7 +287,7 @@ class InvitedUser(db.Model): index=False, unique=False, nullable=False, - default=datetime.datetime.utcnow()) + default=datetime.datetime.utcnow) status = db.Column( db.Enum(*INVITED_USER_STATUS_TYPES, name='invited_users_status_types'), nullable=False, default='pending') permissions = db.Column(db.String, nullable=False) @@ -306,6 +307,7 @@ SEND_EMAILS = 'send_emails' SEND_LETTERS = 'send_letters' MANAGE_API_KEYS = 'manage_api_keys' ACCESS_DEVELOPER_DOCS = 'access_developer_docs' +PLATFORM_ADMIN = 'platform_admin' # List of permissions PERMISSION_LIST = [ @@ -316,7 +318,8 @@ PERMISSION_LIST = [ SEND_EMAILS, SEND_LETTERS, MANAGE_API_KEYS, - ACCESS_DEVELOPER_DOCS] + ACCESS_DEVELOPER_DOCS, + PLATFORM_ADMIN] class Permission(db.Model): @@ -338,7 +341,7 @@ class Permission(db.Model): index=False, unique=False, nullable=False, - default=datetime.datetime.utcnow()) + default=datetime.datetime.utcnow) __table_args__ = ( UniqueConstraint('service_id', 'user_id', 'permission', name='uix_service_user_permission'), diff --git a/app/notifications/rest.py b/app/notifications/rest.py index 594ba5613..60ea93eff 100644 --- a/app/notifications/rest.py +++ b/app/notifications/rest.py @@ -26,6 +26,7 @@ from app.schemas import ( notification_status_schema ) from app.celery.tasks import send_sms, send_email +from app.validation import allowed_send_to_number, allowed_send_to_email notifications = Blueprint('notifications', __name__) @@ -193,13 +194,12 @@ def get_all_notifications(): return jsonify(result="error", message="Invalid page"), 400 all_notifications = notifications_dao.get_notifications_for_service(api_user['client'], page) - return jsonify( notifications=notification_status_schema.dump(all_notifications.items, many=True).data, links=pagination_links( all_notifications, '.get_all_notifications', - request.args + **request.args.to_dict() ) ), 200 @@ -213,13 +213,14 @@ def get_all_notifications_for_service(service_id): return jsonify(result="error", message="Invalid page"), 400 all_notifications = notifications_dao.get_notifications_for_service(service_id, page) - + kwargs = request.args.to_dict() + kwargs['service_id'] = service_id return jsonify( notifications=notification_status_schema.dump(all_notifications.items, many=True).data, links=pagination_links( all_notifications, '.get_all_notifications_for_service', - request.args + **kwargs ) ), 200 @@ -233,13 +234,15 @@ def get_all_notifications_for_service_job(service_id, job_id): return jsonify(result="error", message="Invalid page"), 400 all_notifications = notifications_dao.get_notifications_for_job(service_id, job_id, page) - + kwargs = request.args.to_dict() + kwargs['service_id'] = service_id + kwargs['job_id'] = job_id return jsonify( notifications=notification_status_schema.dump(all_notifications.items, many=True).data, links=pagination_links( all_notifications, '.get_all_notifications_for_service_job', - request.args + **kwargs ) ), 200 @@ -255,13 +258,15 @@ def get_page_from_request(): return 1 -def pagination_links(pagination, endpoint, args): +def pagination_links(pagination, endpoint, **kwargs): + if 'page' in kwargs: + kwargs.pop('page', None) links = dict() if pagination.has_prev: - links['prev'] = url_for(endpoint, **dict(list(args.items()) + [('page', pagination.prev_num)])) + links['prev'] = url_for(endpoint, page=pagination.prev_num, **kwargs) if pagination.has_next: - links['next'] = url_for(endpoint, **dict(list(args.items()) + [('page', pagination.next_num)])) - links['last'] = url_for(endpoint, **dict(list(args.items()) + [('page', pagination.pages)])) + links['next'] = url_for(endpoint, page=pagination.next_num, **kwargs) + links['last'] = url_for(endpoint, page=pagination.pages, **kwargs) return links @@ -320,7 +325,7 @@ def send_notification(notification_type): notification_id = create_uuid() if notification_type == 'sms': - if service.restricted and notification['to'] not in [user.mobile_number for user in service.users]: + if not allowed_send_to_number(service, notification['to']): return jsonify( result="error", message={'to': ['Invalid phone number for restricted service']}), 400 send_sms.apply_async(( @@ -330,7 +335,7 @@ def send_notification(notification_type): datetime.utcnow().strftime(DATETIME_FORMAT) ), queue='sms') else: - if service.restricted and notification['to'] not in [user.email_address for user in service.users]: + if not allowed_send_to_email(service, notification['to']): return jsonify( result="error", message={'to': ['Email address not permitted for restricted service']}), 400 send_email.apply_async(( diff --git a/app/schemas.py b/app/schemas.py index 4c5fe961a..575b23ea9 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -171,6 +171,9 @@ class SmsAdminNotificationSchema(SmsNotificationSchema): class NotificationStatusSchema(BaseSchema): + template = fields.Nested(TemplateSchema, only=["id", "name", "template_type"], dump_only=True) + job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True) + class Meta: model = models.Notification diff --git a/app/user/rest.py b/app/user/rest.py index cb574a909..06adc72e4 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -24,7 +24,13 @@ from app.schemas import ( permission_schema ) -from app.celery.tasks import (send_sms_code, send_email_code, email_reset_password) +from app.celery.tasks import ( + send_sms_code, + send_email_code, + email_reset_password, + email_registration_verification +) + from app.errors import register_errors user = Blueprint('user', __name__) @@ -148,6 +154,28 @@ def send_user_email_code(user_id): return jsonify({}), 204 +@user.route('//email-verification', methods=['POST']) +def send_user_email_verification(user_id): + user_to_send_to = get_model_users(user_id=user_id) + verify_code, errors = request_verify_code_schema.load(request.get_json()) + if errors: + return jsonify(result="error", message=errors), 400 + + from app.dao.users_dao import create_secret_code + secret_code = create_secret_code() + create_user_code(user_to_send_to, secret_code, 'email') + + email = user_to_send_to.email_address + verification_message = {'to': email, + 'name': user_to_send_to.name, + 'url': _create_verification_url(user_to_send_to, secret_code)} + + email_registration_verification.apply_async([encryption.encrypt(verification_message)], + queue='email-registration-verification') + + return jsonify({}), 204 + + @user.route('/', methods=['GET']) @user.route('', methods=['GET']) def get_user(user_id=None): @@ -207,3 +235,12 @@ def _create_reset_password_url(email): token = generate_token(data, current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT']) return current_app.config['ADMIN_BASE_URL'] + '/new-password/' + token + + +def _create_verification_url(user, secret_code): + from utils.url_safe_token import generate_token + import json + data = json.dumps({'user_id': user.id, 'email': user.email_address, 'secret_code': secret_code}) + token = generate_token(data, current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT']) + + return current_app.config['ADMIN_BASE_URL'] + '/verify-email/' + token diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 000000000..49a6b0664 --- /dev/null +++ b/app/validation.py @@ -0,0 +1,15 @@ +from utils.recipients import format_phone_number, validate_phone_number + + +def allowed_send_to_number(service, to): + if service.restricted and format_phone_number(validate_phone_number(to)) not in [ + format_phone_number(validate_phone_number(user.mobile_number)) for user in service.users + ]: + return False + return True + + +def allowed_send_to_email(service, to): + if service.restricted and to not in [user.email_address for user in service.users]: + return False + return True diff --git a/aws_run_celery.py b/aws_run_celery.py index 814e07f4b..d5a58a09f 100644 --- a/aws_run_celery.py +++ b/aws_run_celery.py @@ -3,10 +3,19 @@ from app import notify_celery, create_app from credstash import getAllSecrets import os +default_env_file = '/home/ubuntu/environment' +environment = 'live' + +if os.path.isfile(default_env_file): + with open(default_env_file, 'r') as environment_file: + environment = environment_file.readline().strip() + # on aws get secrets and export to env -secrets = getAllSecrets(region="eu-west-1") -for key, val in secrets.items(): - os.environ[key] = val +os.environ.update(getAllSecrets(region="eu-west-1")) + +from config import configs + +os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] application = create_app() application.app_context().push() diff --git a/config.py b/config.py index c2b6ce984..0bf45618c 100644 --- a/config.py +++ b/config.py @@ -69,7 +69,8 @@ class Config(object): Queue('process-job', Exchange('default'), routing_key='process-job'), Queue('bulk-sms', Exchange('default'), routing_key='bulk-sms'), Queue('bulk-email', Exchange('default'), routing_key='bulk-email'), - Queue('email-invited-user', Exchange('default'), routing_key='email-invited-user') + Queue('email-invited-user', Exchange('default'), routing_key='email-invited-user'), + Queue('email-registration-verification', Exchange('default'), routing_key='email-registration-verification') ] TWILIO_ACCOUNT_SID = os.getenv('TWILIO_ACCOUNT_SID') TWILIO_AUTH_TOKEN = os.getenv('TWILIO_AUTH_TOKEN') @@ -84,3 +85,9 @@ class Development(Config): class Test(Development): pass + +configs = { + 'live': 'config_live.Live', + 'staging': 'config_staging.Staging', + 'preview': 'config.Config' +} diff --git a/config_live.py b/config_live.py new file mode 100644 index 000000000..65f36eeb7 --- /dev/null +++ b/config_live.py @@ -0,0 +1,25 @@ +import os +from config import Config + + +class Live(Config): + ADMIN_BASE_URL = os.environ['LIVE_ADMIN_BASE_URL'] + API_HOST_NAME = os.environ['LIVE_API_HOST_NAME'] + ADMIN_CLIENT_SECRET = os.environ['LIVE_ADMIN_CLIENT_SECRET'] + DANGEROUS_SALT = os.environ['LIVE_DANGEROUS_SALT'] + NOTIFICATION_QUEUE_PREFIX = os.environ['LIVE_NOTIFICATION_QUEUE_PREFIX'] + NOTIFY_JOB_QUEUE = os.environ['LIVE_NOTIFY_JOB_QUEUE'] + SECRET_KEY = os.environ['LIVE_SECRET_KEY'] + SQLALCHEMY_DATABASE_URI = os.environ['LIVE_SQLALCHEMY_DATABASE_URI'] + VERIFY_CODE_FROM_EMAIL_ADDRESS = os.environ['LIVE_VERIFY_CODE_FROM_EMAIL_ADDRESS'] + NOTIFY_EMAIL_DOMAIN = os.environ['LIVE_NOTIFY_EMAIL_DOMAIN'] + FIRETEXT_API_KEY = os.getenv("LIVE_FIRETEXT_API_KEY") + FIRETEXT_NUMBER = os.getenv("LIVE_FIRETEXT_NUMBER") + TWILIO_AUTH_TOKEN = os.getenv('LIVE_TWILIO_AUTH_TOKEN') + + BROKER_TRANSPORT_OPTIONS = { + 'region': 'eu-west-1', + 'polling_interval': 1, # 1 second + 'visibility_timeout': 60, # 60 seconds + 'queue_name_prefix': os.environ['LIVE_NOTIFICATION_QUEUE_PREFIX'] + '-' + } diff --git a/config_staging.py b/config_staging.py new file mode 100644 index 000000000..bf0f44119 --- /dev/null +++ b/config_staging.py @@ -0,0 +1,25 @@ +import os +from config import Config + + +class Staging(Config): + ADMIN_BASE_URL = os.environ['STAGING_ADMIN_BASE_URL'] + API_HOST_NAME = os.environ['STAGING_API_HOST_NAME'] + ADMIN_CLIENT_SECRET = os.environ['STAGING_ADMIN_CLIENT_SECRET'] + DANGEROUS_SALT = os.environ['STAGING_DANGEROUS_SALT'] + NOTIFICATION_QUEUE_PREFIX = os.environ['STAGING_NOTIFICATION_QUEUE_PREFIX'] + NOTIFY_JOB_QUEUE = os.environ['STAGING_NOTIFY_JOB_QUEUE'] + SECRET_KEY = os.environ['STAGING_SECRET_KEY'] + SQLALCHEMY_DATABASE_URI = os.environ['STAGING_SQLALCHEMY_DATABASE_URI'] + VERIFY_CODE_FROM_EMAIL_ADDRESS = os.environ['STAGING_VERIFY_CODE_FROM_EMAIL_ADDRESS'] + NOTIFY_EMAIL_DOMAIN = os.environ['STAGING_NOTIFY_EMAIL_DOMAIN'] + FIRETEXT_API_KEY = os.getenv("STAGING_FIRETEXT_API_KEY") + FIRETEXT_NUMBER = os.getenv("STAGING_FIRETEXT_NUMBER") + TWILIO_AUTH_TOKEN = os.getenv('STAGING_TWILIO_AUTH_TOKEN') + + BROKER_TRANSPORT_OPTIONS = { + 'region': 'eu-west-1', + 'polling_interval': 1, # 1 second + 'visibility_timeout': 60, # 60 seconds + 'queue_name_prefix': os.environ['STAGING_NOTIFICATION_QUEUE_PREFIX'] + '-' + } diff --git a/db.py b/db.py index 8e2efab6e..d923c8c75 100644 --- a/db.py +++ b/db.py @@ -4,9 +4,19 @@ from app import create_app, db from credstash import getAllSecrets import os -secrets = getAllSecrets(region="eu-west-1") -for key, val in secrets.items(): - os.environ[key] = val +default_env_file = '/home/ubuntu/environment' +environment = 'live' + +if os.path.isfile(default_env_file): + with open(default_env_file, 'r') as environment_file: + environment = environment_file.readline().strip() + +# on aws get secrets and export to env +os.environ.update(getAllSecrets(region="eu-west-1")) + +from config import configs + +os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] application = create_app() diff --git a/migrations/versions/0041_platform_admin.py b/migrations/versions/0041_platform_admin.py new file mode 100644 index 000000000..d75c8f5c0 --- /dev/null +++ b/migrations/versions/0041_platform_admin.py @@ -0,0 +1,33 @@ +"""empty message + +Revision ID: 0041_platform_admin +Revises: 0040_add_reference +Create Date: 2016-03-16 16:33:15.279429 + +""" + +# revision identifiers, used by Alembic. +revision = '0041_platform_admin' +down_revision = '0040_add_reference' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('ix_notification_statistics_service_id'), 'notification_statistics', ['service_id'], unique=False) + op.drop_index('ix_service_notification_stats_service_id', table_name='notification_statistics') + op.add_column('users', sa.Column('platform_admin', sa.Boolean(), nullable=True, default=False)) + op.get_bind() + op.execute('update users set platform_admin = False') + op.alter_column('users', 'platform_admin', nullable=False) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'platform_admin') + op.create_index('ix_service_notification_stats_service_id', 'notification_statistics', ['service_id'], unique=False) + op.drop_index(op.f('ix_notification_statistics_service_id'), table_name='notification_statistics') + ### end Alembic commands ### diff --git a/migrations/versions/0042_default_stats_to_zero.py b/migrations/versions/0042_default_stats_to_zero.py new file mode 100644 index 000000000..4d862c903 --- /dev/null +++ b/migrations/versions/0042_default_stats_to_zero.py @@ -0,0 +1,45 @@ +"""empty message + +Revision ID: 0042_default_stats_to_zero +Revises: 0041_platform_admin +Create Date: 2016-03-17 11:09:17.906910 + +""" + +# revision identifiers, used by Alembic. +revision = '0042_default_stats_to_zero' +down_revision = '0041_platform_admin' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.get_bind() + op.execute('update notification_statistics set emails_delivered = 0 where emails_delivered is Null') + op.execute('update notification_statistics set emails_error = 0 where emails_error is Null') + op.execute('update notification_statistics set sms_delivered = 0 where sms_delivered is Null') + op.execute('update notification_statistics set sms_error = 0 where sms_error is Null') + op.alter_column('notification_statistics', 'emails_requested', server_default='0') + op.alter_column('notification_statistics', 'emails_delivered', server_default='0', nullable=False) + op.alter_column('notification_statistics', 'emails_error', server_default='0', nullable=False) + op.alter_column('notification_statistics', 'sms_requested', server_default='0') + op.alter_column('notification_statistics', 'sms_delivered', server_default='0', nullable=False) + op.alter_column('notification_statistics', 'sms_error', server_default='0', nullable=False) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.alter_column('notification_statistics', 'emails_requested', server_default=None) + op.alter_column('notification_statistics', 'emails_delivered', server_default=None, nullable=True) + op.alter_column('notification_statistics', 'emails_error', server_default=None, nullable=True) + op.alter_column('notification_statistics', 'sms_requested', server_default=None) + op.alter_column('notification_statistics', 'sms_delivered', server_default=None, nullable=True) + op.alter_column('notification_statistics', 'sms_error', server_default=None, nullable=True) + op.execute('update notification_statistics set emails_delivered = Null where emails_delivered = 0') + op.execute('update notification_statistics set emails_error = Null where emails_error = 0') + op.execute('update notification_statistics set sms_delivered = Null where sms_delivered = 0') + op.execute('update notification_statistics set sms_error = Null where sms_error = 0') + ### end Alembic commands ### diff --git a/run_celery.py b/run_celery.py index 51ce92910..066735e63 100644 --- a/run_celery.py +++ b/run_celery.py @@ -1,5 +1,5 @@ #!/usr/bin/env python from app import notify_celery, create_app -application = create_app() +application = create_app('delivery') application.app_context().push() diff --git a/scripts/aws_start_app.sh b/scripts/aws_start_app.sh index 40bd7d50e..7be657a03 100755 --- a/scripts/aws_start_app.sh +++ b/scripts/aws_start_app.sh @@ -1,4 +1,19 @@ #!/bin/bash -echo "Starting application" -sudo service notifications-api start \ No newline at end of file +if [ -e "/etc/init/notifications-api.conf" ] +then + echo "Starting api" + sudo service notifications-api start +fi + +if [ -e "/etc/init/notifications-api-celery-worker.conf" ] +then + echo "Starting celery worker" + sudo service notifications-api-celery-worker start +fi + +if [ -e "/etc/init/notifications-api-celery-beat.conf" ] +then + echo "Starting celery beat" + sudo service notifications-api-celery-beat start +fi diff --git a/scripts/aws_stop_app.sh b/scripts/aws_stop_app.sh index 188467062..2f5660ae1 100755 --- a/scripts/aws_stop_app.sh +++ b/scripts/aws_stop_app.sh @@ -7,9 +7,29 @@ function error_exit exit 0 } -echo "Stopping application" -if sudo service notifications-api stop; then - exit 0 -else - error_exit "Could not stop application" -fi \ No newline at end of file +if [ -e "/etc/init/notifications-api.conf" ]; then + echo "stopping notifications-api" + if sudo service notifications-api stop; then + echo "notifications-api stopped" + else + error_exit "Could not stop notifications-api" + fi +fi + +if [ -e "/etc/init/notifications-api-celery-beat.conf" ]; then + echo "stopping notifications-api-celery-beat" + if sudo service notifications-api-celery-beat stop; then + echo "notifications-api stopped" + else + error_exit "Could not stop notifications-celery-beat" + fi +fi + +if [ -e "/etc/init/notifications-api-celery-worker.conf" ]; then + echo "stopping notifications-api-celery-worker" + if sudo service notifications-api-celery-worker stop; then + echo "notifications-api stopped" + else + error_exit "Could not stop notifications-celery-worker" + fi +fi diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index e2f334697..737f08864 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -1,6 +1,8 @@ import uuid import pytest from flask import current_app +from utils.recipients import validate_phone_number, format_phone_number + from app.celery.tasks import ( send_sms, send_sms_code, @@ -248,7 +250,7 @@ def test_should_process_all_sms_job(sample_job, sample_job_with_placeholdered_te def test_should_send_template_to_correct_sms_provider_and_persist(sample_template_with_placeholders, mocker): notification = { "template": sample_template_with_placeholders.id, - "to": "+441234123123", + "to": "+447234123123", "personalisation": {"name": "Jo"} } mocker.patch('app.encryption.decrypt', return_value=notification) @@ -265,7 +267,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat ) firetext_client.send_sms.assert_called_once_with( - to="+441234123123", + to=format_phone_number(validate_phone_number("+447234123123")), content="Sample service: Hello Jo", reference=str(notification_id) ) @@ -273,7 +275,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat sample_template_with_placeholders.service_id, notification_id ) assert persisted_notification.id == notification_id - assert persisted_notification.to == '+441234123123' + assert persisted_notification.to == '+447234123123' assert persisted_notification.template_id == sample_template_with_placeholders.id assert persisted_notification.status == 'sent' assert persisted_notification.created_at == now @@ -285,7 +287,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat def test_should_send_sms_without_personalisation(sample_template, mocker): notification = { "template": sample_template.id, - "to": "+441234123123" + "to": "+447234123123" } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms') @@ -301,7 +303,7 @@ def test_should_send_sms_without_personalisation(sample_template, mocker): ) firetext_client.send_sms.assert_called_once_with( - to="+441234123123", + to=format_phone_number(validate_phone_number("+447234123123")), content="Sample service: This is a template", reference=str(notification_id) ) @@ -330,7 +332,7 @@ def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notif ) firetext_client.send_sms.assert_called_once_with( - to="+447700900890", + to=format_phone_number(validate_phone_number("+447700900890")), content="Sample service: This is a template", reference=str(notification_id) ) @@ -396,7 +398,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa notification = { "template": sample_job.template.id, "job": sample_job.id, - "to": "+441234123123" + "to": "+447234123123" } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms') @@ -411,13 +413,13 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa now.strftime(DATETIME_FORMAT) ) firetext_client.send_sms.assert_called_once_with( - to="+441234123123", + to=format_phone_number(validate_phone_number("+447234123123")), content="Sample service: This is a template", reference=str(notification_id) ) persisted_notification = notifications_dao.get_notification(sample_job.template.service_id, notification_id) assert persisted_notification.id == notification_id - assert persisted_notification.to == '+441234123123' + assert persisted_notification.to == '+447234123123' assert persisted_notification.job_id == sample_job.id assert persisted_notification.template_id == sample_job.template.id assert persisted_notification.status == 'sent' @@ -520,7 +522,7 @@ def test_should_use_email_template_and_persist_without_personalisation( def test_should_persist_notification_as_failed_if_sms_client_fails(sample_template, mocker): notification = { "template": sample_template.id, - "to": "+441234123123" + "to": "+447234123123" } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException(firetext_error())) @@ -536,13 +538,13 @@ def test_should_persist_notification_as_failed_if_sms_client_fails(sample_templa now.strftime(DATETIME_FORMAT) ) firetext_client.send_sms.assert_called_once_with( - to="+441234123123", + to=format_phone_number(validate_phone_number("+447234123123")), content="Sample service: This is a template", reference=str(notification_id) ) persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id) assert persisted_notification.id == notification_id - assert persisted_notification.to == '+441234123123' + assert persisted_notification.to == '+447234123123' assert persisted_notification.template_id == sample_template.id assert persisted_notification.status == 'failed' assert persisted_notification.created_at == now @@ -590,7 +592,7 @@ def test_should_persist_notification_as_failed_if_email_client_fails(sample_emai def test_should_not_send_sms_if_db_peristance_failed(sample_template, mocker): notification = { "template": sample_template.id, - "to": "+441234123123" + "to": "+447234123123" } mocker.patch('app.encryption.decrypt', return_value=notification) mocker.patch('app.firetext_client.send_sms') @@ -638,24 +640,28 @@ def test_should_not_send_email_if_db_peristance_failed(sample_email_template, mo def test_should_send_sms_code(mocker): - notification = {'to': '+441234123123', + notification = {'to': '+447234123123', 'secret_code': '12345'} encrypted_notification = encryption.encrypt(notification) mocker.patch('app.firetext_client.send_sms') send_sms_code(encrypted_notification) - firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'], 'send-sms-code') + firetext_client.send_sms.assert_called_once_with(format_phone_number(validate_phone_number(notification['to'])), + notification['secret_code'], + 'send-sms-code') def test_should_throw_firetext_client_exception(mocker): - notification = {'to': '+441234123123', + notification = {'to': '+447234123123', 'secret_code': '12345'} encrypted_notification = encryption.encrypt(notification) mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException(firetext_error())) send_sms_code(encrypted_notification) - firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'], 'send-sms-code') + firetext_client.send_sms.assert_called_once_with(format_phone_number(validate_phone_number(notification['to'])), + notification['secret_code'], + 'send-sms-code') def test_should_send_email_code(mocker): diff --git a/tests/app/conftest.py b/tests/app/conftest.py index e954d6cb0..63bad1b8f 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -327,6 +327,11 @@ def mock_celery_send_email_code(mocker): return mocker.patch('app.celery.tasks.send_email_code.apply_async') +@pytest.fixture(scope='function') +def mock_celery_email_registration_verification(mocker): + return mocker.patch('app.celery.tasks.email_registration_verification.apply_async') + + @pytest.fixture(scope='function') def mock_encryption(mocker): return mocker.patch('app.encryption.encrypt', return_value="something_encrypted") diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index da697ee1c..e859fdb76 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -70,8 +70,13 @@ def test_should_be_able_to_get_statistics_for_a_service(sample_template): assert len(stats) == 1 assert stats[0].emails_requested == 0 assert stats[0].sms_requested == 1 + assert stats[0].sms_delivered == 0 + assert stats[0].sms_error == 0 assert stats[0].day == notification.created_at.strftime(DATE_FORMAT) assert stats[0].service_id == notification.service_id + assert stats[0].emails_requested == 0 + assert stats[0].emails_delivered == 0 + assert stats[0].emails_error == 0 def test_should_be_able_to_get_statistics_for_a_service_for_a_day(sample_template): @@ -90,7 +95,11 @@ def test_should_be_able_to_get_statistics_for_a_service_for_a_day(sample_templat sample_template.service.id, now.strftime(DATE_FORMAT) ) assert stat.emails_requested == 0 + assert stat.emails_error == 0 + assert stat.emails_delivered == 0 assert stat.sms_requested == 1 + assert stat.sms_error == 0 + assert stat.sms_delivered == 0 assert stat.day == notification.created_at.strftime(DATE_FORMAT) assert stat.service_id == notification.service_id diff --git a/tests/app/dao/test_users_dao.py b/tests/app/dao/test_users_dao.py index 8f928a954..35dce1b60 100644 --- a/tests/app/dao/test_users_dao.py +++ b/tests/app/dao/test_users_dao.py @@ -32,6 +32,7 @@ def test_create_user(notify_api, notify_db, notify_db_session): assert User.query.count() == 1 assert User.query.first().email_address == email assert User.query.first().id == user.id + assert not user.platform_admin def test_get_all_users(notify_api, notify_db, notify_db_session, sample_user): diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 1d6428e5e..9bc9ada62 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -25,7 +25,14 @@ def test_get_notification_by_id(notify_api, sample_notification): notification = json.loads(response.get_data(as_text=True))['notification'] assert notification['status'] == 'sent' - assert notification['template'] == sample_notification.template.id + assert notification['template'] == { + 'id': sample_notification.template.id, + 'name': sample_notification.template.name, + 'template_type': sample_notification.template.template_type} + assert notification['job'] == { + 'id': str(sample_notification.job.id), + 'original_file_name': sample_notification.job.original_file_name + } assert notification['to'] == '+447700900855' assert notification['service'] == str(sample_notification.service_id) assert response.status_code == 200 @@ -64,7 +71,14 @@ def test_get_all_notifications(notify_api, sample_notification): notifications = json.loads(response.get_data(as_text=True)) assert notifications['notifications'][0]['status'] == 'sent' - assert notifications['notifications'][0]['template'] == sample_notification.template.id + assert notifications['notifications'][0]['template'] == { + 'id': sample_notification.template.id, + 'name': sample_notification.template.name, + 'template_type': sample_notification.template.template_type} + assert notifications['notifications'][0]['job'] == { + 'id': str(sample_notification.job.id), + 'original_file_name': sample_notification.job.original_file_name + } assert notifications['notifications'][0]['to'] == '+447700900855' assert notifications['notifications'][0]['service'] == str(sample_notification.service_id) assert response.status_code == 200 diff --git a/tests/app/test_validation.py b/tests/app/test_validation.py new file mode 100644 index 000000000..9c822533d --- /dev/null +++ b/tests/app/test_validation.py @@ -0,0 +1,65 @@ +from app.models import User, Service +from app.validation import allowed_send_to_number, allowed_send_to_email + + +def test_allowed_send_to_number_returns_true_for_restricted_service_with_same_number(): + mobile_number = '07524609792' + service = _create_service_data(mobile_number) + assert allowed_send_to_number(service, mobile_number) + + +def test_allowed_send_to_number_returns_false_for_restricted_service_with_different_number(): + mobile_number = '00447524609792' + service = _create_service_data(mobile_number) + assert not allowed_send_to_number(service, '+447344609793') + + +def test_allowed_send_to_number_returns_true_for_unrestricted_service_with_different_number(): + mobile_number = '+447524609792' + service = _create_service_data(mobile_number, False) + assert allowed_send_to_number(service, '+447344609793') + + +def test_allowed_send_to_email__returns_true_for_restricted_service_with_same_email(): + email = 'testing@it.gov.uk' + service = _create_service_data(email_address=email) + assert allowed_send_to_email(service, email) + + +def test_allowed_send_to_email__returns_false_for_restricted_service_with_different_email(): + email = 'testing@it.gov.uk' + service = _create_service_data(email_address=email) + assert not allowed_send_to_email(service, 'another@it.gov.uk') + + +def test_allowed_send_to_email__returns_false_for_restricted_service_with_different_email(): + email = 'testing@it.gov.uk' + service = _create_service_data(email_address=email) + assert not allowed_send_to_email(service, 'another@it.gov.uk') + + +def test_allowed_send_to_email__returns_true_for_unrestricted_service_with_different_email(): + email = 'testing@it.gov.uk' + service = _create_service_data(email_address=email, restricted=False) + assert allowed_send_to_number(service, 'another@it.gov.uk') + + +def _create_service_data(mobile_number='+447524609792', restricted=True, email_address='test_user@it.gov.uk'): + usr = { + 'name': 'Test User', + 'email_address': email_address, + 'password': 'password', + 'mobile_number': mobile_number, + 'state': 'active' + } + user = User(**usr) + data = { + 'name': 'Test service', + 'limit': 10, + 'active': False, + 'restricted': restricted, + 'email_from': 'test_service@it.gov.uk' + } + service = Service(**data) + service.users = [user] + return service diff --git a/tests/app/user/test_rest_verify.py b/tests/app/user/test_rest_verify.py index 12dd3b5ba..e666c0140 100644 --- a/tests/app/user/test_rest_verify.py +++ b/tests/app/user/test_rest_verify.py @@ -1,13 +1,25 @@ import json import moto -from datetime import (datetime, timedelta) + +from datetime import ( + datetime, + timedelta +) + from flask import url_for -from app.models import (VerifyCode, User) -import app.celery.tasks + +from app.models import ( + VerifyCode, + User +) + from app import db, encryption + from tests import create_authorization_header from freezegun import freeze_time +import app.celery.tasks + def test_user_verify_code_sms(notify_api, sample_sms_code): @@ -341,3 +353,23 @@ def test_send_user_email_code_returns_404_for_when_user_does_not_exist(notify_ap headers=[('Content-Type', 'application/json'), auth_header]) assert resp.status_code == 404 assert json.loads(resp.get_data(as_text=True))['message'] == 'No result found' + + +def test_send_user_email_verification(notify_api, + sample_email_code, + mock_celery_email_registration_verification, + mock_encryption): + + with notify_api.test_request_context(): + with notify_api.test_client() as client: + data = json.dumps({}) + auth_header = create_authorization_header( + path=url_for('user.send_user_email_verification', user_id=sample_email_code.user.id), + method='POST', + request_body=data) + resp = client.post( + url_for('user.send_user_email_verification', user_id=sample_email_code.user.id), + data=data, + headers=[('Content-Type', 'application/json'), auth_header]) + assert resp.status_code == 204 + app.celery.tasks.email_registration_verification.apply_async.assert_called_once_with(['something_encrypted'], queue='email-registration-verification') # noqa diff --git a/wsgi.py b/wsgi.py index 3e9e20d2f..52e95994c 100644 --- a/wsgi.py +++ b/wsgi.py @@ -3,12 +3,22 @@ import os from app import create_app from credstash import getAllSecrets + +default_env_file = '/home/ubuntu/environment' +environment = 'live' + +if os.path.isfile(default_env_file): + with open(default_env_file, 'r') as environment_file: + environment = environment_file.readline().strip() + # on aws get secrets and export to env -secrets = getAllSecrets(region="eu-west-1") -for key, val in secrets.items(): - os.environ[key] = val +os.environ.update(getAllSecrets(region="eu-west-1")) + +from config import configs + +os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment] application = create_app() if __name__ == "__main__": - application.run() + application.run()