Merge branch 'master' into capture-delivery-outcomes

.travis.yml

@@ -46,7 +46,17 @@ deploy:
   key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
   bundle_type: zip
   application: notifications-api
-  deployment_group: notifications-api-celery
+  deployment_group: notifications_admin_api_deployment_group
+  region: eu-west-1
+  on: *2
+- provider: codedeploy
+  access_key_id: AKIAJQPPNM6P6V53SWKA
+  secret_access_key: *1
+  bucket: notifications-api-codedeploy
+  key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
+  bundle_type: zip
+  application: notifications-api
+  deployment_group: notifications_delivery_api_deployment_group
   region: eu-west-1
   on: *2
 - provider: s3
@@ -90,6 +100,47 @@ deploy:
   deployment_group: notifications_admin_api_deployment_group
   region: eu-west-1
   on: *2
+- provider: s3
+  access_key_id: AKIAJ5MKF6G3P2JQP4QQ
+  secret_access_key: &1
+    secure: daC1bCHXqLRK+iIZ8P699KCnTh77lwV4KxrZxL1yd6cstgfptyd/rg1WgRwE6QdxOCT9gQvKWUZFCzFy7M6E/Ih8EUHqEXTzC5M4oAye8rhePIBMQwqkgfYyIoZ3LdDMMP5JfBhiz0zS3Vj7HerL2qIu12adJBjkRJx3XAGimCrFOMQ0xUXQAKDjL6Xmv+gVz2f/ISLy6icKY4KNGt3cQV+8pa5aMF34C9R2udA9N67EWlXlh7hJbFtmY+0Zqpo8Rr6wKRb5MA0xEcTVLORSz1aa6GkxUCbzaIH99p7z3Ghz0qW2bUi9ZcDrvg0GLbVe1T+1HXhfktJfW8wnzw6A/2U/CIIFDQZ/qk0w/DkEwpQinXow99Zl49CcEU+v8llKhg5nM3LmAZCQg1c/iZyP/d90AwAMoMA/VTDD72M93IqTJQH18eC8g02DwE0hNDD6aos5wzeuDeiH/6BG+Tq0pDl0y0aWCcHf3vGRlo/5GlWfpE0vMQEC+qnEOWOUqSprCdSypgD2Aip9mCC98w4BkqKKvGNHPZolA7rxf7E9hTK+BNPRATpYsHR1X/1Xl0TMc/pHhjU1yNXzWnI/kOlNV2CRq3slEtcWihaEo8oDHJ+BhGT49Ps3Je7UB2xO/jXXFPhwJotPMOacTcnUkGqVJSlK1g6TIn4t9nTVSY8KFUs=
+  local_dir: dpl_cd_upload
+  skip_cleanup: true
+  region: eu-west-1
+  on: &2
+    repo: alphagov/notifications-api
+    branch: live
+  bucket: live-notifications-api-codedeploy
+- provider: codedeploy
+  access_key_id: AKIAJ5MKF6G3P2JQP4QQ
+  secret_access_key: *1
+  bucket: live-notifications-api-codedeploy
+  key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
+  bundle_type: zip
+  application: notifications-api
+  deployment_group: live_notifications_delivery_api_deployment_group
+  region: eu-west-1
+  on: *2
+- provider: codedeploy
+  access_key_id: AKIAJ5MKF6G3P2JQP4QQ
+  secret_access_key: *1
+  bucket: live-notifications-api-codedeploy
+  key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
+  bundle_type: zip
+  application: notifications-api
+  deployment_group: live_notifications_api_deployment_group
+  region: eu-west-1
+  on: *2
+- provider: codedeploy
+  access_key_id: AKIAJ5MKF6G3P2JQP4QQ
+  secret_access_key: *1
+  bucket: live-notifications-api-codedeploy
+  key: notifications-api-$TRAVIS_BRANCH-$TRAVIS_BUILD_NUMBER-$TRAVIS_COMMIT.zip
+  bundle_type: zip
+  application: notifications-api
+  deployment_group: live_notifications_admin_api_deployment_group
+  region: eu-west-1
+  on: *2
 before_deploy:
 - ./scripts/update_version_file.sh
 - zip -r  --exclude=*__pycache__* notifications-api *

README.md

@@ -62,3 +62,6 @@ scripts/run_app.sh
 scripts/run_celery.sh
 ```
 
+```
+scripts/run_celery_beat.sh
+```

app/__init__.py

@@ -27,11 +27,14 @@ encryption = Encryption()
 api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
 
 
-def create_app():
+def create_app(app_name=None):
     application = Flask(__name__)
 
     application.config.from_object(os.environ['NOTIFY_API_ENVIRONMENT'])
 
+    if app_name:
+        application.config['NOTIFY_APP_NAME'] = app_name
+
     init_app(application)
     db.init_app(application)
     ma.init_app(application)
@@ -92,9 +95,7 @@ def init_app(app):
 
 def email_safe(string):
     return "".join([
-        character.lower()
+        character.lower() if character.isalnum() or character == "." else "" for character in re.sub("\s+", ".", string.strip())  # noqa
-        if character.isalnum() or character == "."
-        else "" for character in re.sub("\s+", ".", string.strip())
     ])
 
 

app/celery/tasks.py

@@ -25,7 +25,8 @@ from sqlalchemy.exc import SQLAlchemyError
 from app.aws import s3
 from datetime import datetime
 from utils.template import Template
-from utils.recipients import RecipientCSV, validate_phone_number, format_phone_number
+from utils.recipients import RecipientCSV, format_phone_number, validate_phone_number
+from app.validation import (allowed_send_to_email, allowed_send_to_number)
 
 
 @notify_celery.task(name="delete-verify-codes")
@@ -204,7 +205,7 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
                 )
 
                 client.send_sms(
-                    to=notification['to'],
+                    to=format_phone_number(validate_phone_number(notification['to'])),
                     content=template.replaced,
                     reference=str(notification_id)
                 )
@@ -223,20 +224,6 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
         current_app.logger.debug(e)
 
 
-def allowed_send_to_number(service, to):
-    if service.restricted and format_phone_number(validate_phone_number(to)) not in [
-        format_phone_number(validate_phone_number(user.mobile_number)) for user in service.users
-    ]:
-        return False
-    return True
-
-
-def allowed_send_to_email(service, to):
-    if service.restricted and to not in [user.email_address for user in service.users]:
-        return False
-    return True
-
-
 @notify_celery.task(name="send-email")
 def send_email(service_id, notification_id, subject, from_address, encrypted_notification, created_at):
     notification = encryption.decrypt(encrypted_notification)
@@ -300,7 +287,9 @@ def send_sms_code(encrypted_verification):
     verification_message = encryption.decrypt(encrypted_verification)
     try:
         firetext_client.send_sms(
-            verification_message['to'], verification_message['secret_code'], 'send-sms-code'
+            format_phone_number(validate_phone_number(verification_message['to'])),
+            verification_message['secret_code'],
+            'send-sms-code'
         )
     except FiretextClientException as e:
         current_app.logger.exception(e)
@@ -381,3 +370,23 @@ def email_reset_password(encrypted_reset_password_message):
                                                          url=reset_password_message['reset_password_url']))
     except AwsSesClientException as e:
         current_app.logger.exception(e)
+
+
+def registration_verification_template(name, url):
+    from string import Template
+    t = Template("Hi $name,\n\n"
+                 "To complete your registration for GOV.UK Notify please click the link below\n\n $url")
+    return t.substitute(name=name, url=url)
+
+
+@notify_celery.task(name='email-registration-verification')
+def email_registration_verification(encrypted_verification_message):
+    verification_message = encryption.decrypt(encrypted_verification_message)
+    try:
+        aws_ses_client.send_email(current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS'],
+                                  verification_message['to'],
+                                  "Confirm GOV.UK Notify registration",
+                                  registration_verification_template(name=verification_message['name'],
+                                                                     url=verification_message['url']))
+    except AwsSesClientException as e:
+        current_app.logger.exception(e)

app/models.py

@@ -40,6 +40,7 @@ class User(db.Model):
     logged_in_at = db.Column(db.DateTime, nullable=True)
     failed_login_count = db.Column(db.Integer, nullable=False, default=0)
     state = db.Column(db.String, nullable=False, default='pending')
+    platform_admin = db.Column(db.Boolean, nullable=False, default=False)
 
     @property
     def password(self):
@@ -114,12 +115,12 @@ class NotificationStatistics(db.Model):
     day = db.Column(db.String(255), nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
     service = db.relationship('Service', backref=db.backref('service_notification_stats', lazy='dynamic'))
-    emails_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
+    emails_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    emails_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    emails_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    emails_error = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    emails_error = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    sms_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False)
+    sms_requested = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    sms_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    sms_delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
-    sms_error = db.Column(db.BigInteger, index=False, unique=False, nullable=True)
+    sms_error = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
 
     __table_args__ = (
         UniqueConstraint('service_id', 'day', name='uix_service_to_day'),
@@ -144,13 +145,13 @@ class Template(db.Model):
         index=False,
         unique=False,
         nullable=False,
-        default=datetime.datetime.now)
+        default=datetime.datetime.utcnow)
     updated_at = db.Column(
         db.DateTime,
         index=False,
         unique=False,
         nullable=True,
-        onupdate=datetime.datetime.utcnow())
+        onupdate=datetime.datetime.utcnow)
     content = db.Column(db.Text, index=False, unique=False, nullable=False)
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, unique=False, nullable=False)
     service = db.relationship('Service', backref=db.backref('templates', lazy='dynamic'))
@@ -176,13 +177,13 @@ class Job(db.Model):
         index=False,
         unique=False,
         nullable=False,
-        default=datetime.datetime.utcnow())
+        default=datetime.datetime.utcnow)
     updated_at = db.Column(
         db.DateTime,
         index=False,
         unique=False,
         nullable=True,
-        onupdate=datetime.datetime.utcnow())
+        onupdate=datetime.datetime.utcnow)
     status = db.Column(db.Enum(*JOB_STATUS_TYPES, name='job_status_types'), nullable=False, default='pending')
     notification_count = db.Column(db.Integer, nullable=False)
     notifications_sent = db.Column(db.Integer, nullable=False, default=0)
@@ -217,7 +218,7 @@ class VerifyCode(db.Model):
         index=False,
         unique=False,
         nullable=False,
-        default=datetime.datetime.utcnow())
+        default=datetime.datetime.utcnow)
 
     @property
     def code(self):
@@ -262,7 +263,7 @@ class Notification(db.Model):
         index=False,
         unique=False,
         nullable=True,
-        onupdate=datetime.datetime.utcnow())
+        onupdate=datetime.datetime.utcnow)
     status = db.Column(
         db.Enum(*NOTIFICATION_STATUS_TYPES, name='notification_status_types'), nullable=False, default='sent')
     reference = db.Column(db.String, nullable=True, index=True)
@@ -286,7 +287,7 @@ class InvitedUser(db.Model):
         index=False,
         unique=False,
         nullable=False,
-        default=datetime.datetime.utcnow())
+        default=datetime.datetime.utcnow)
     status = db.Column(
         db.Enum(*INVITED_USER_STATUS_TYPES, name='invited_users_status_types'), nullable=False, default='pending')
     permissions = db.Column(db.String, nullable=False)
@@ -306,6 +307,7 @@ SEND_EMAILS = 'send_emails'
 SEND_LETTERS = 'send_letters'
 MANAGE_API_KEYS = 'manage_api_keys'
 ACCESS_DEVELOPER_DOCS = 'access_developer_docs'
+PLATFORM_ADMIN = 'platform_admin'
 
 # List of permissions
 PERMISSION_LIST = [
@@ -316,7 +318,8 @@ PERMISSION_LIST = [
     SEND_EMAILS,
     SEND_LETTERS,
     MANAGE_API_KEYS,
-    ACCESS_DEVELOPER_DOCS]
+    ACCESS_DEVELOPER_DOCS,
+    PLATFORM_ADMIN]
 
 
 class Permission(db.Model):
@@ -338,7 +341,7 @@ class Permission(db.Model):
         index=False,
         unique=False,
         nullable=False,
-        default=datetime.datetime.utcnow())
+        default=datetime.datetime.utcnow)
 
     __table_args__ = (
         UniqueConstraint('service_id', 'user_id', 'permission', name='uix_service_user_permission'),

app/notifications/rest.py

@@ -26,6 +26,7 @@ from app.schemas import (
     notification_status_schema
 )
 from app.celery.tasks import send_sms, send_email
+from app.validation import allowed_send_to_number, allowed_send_to_email
 
 notifications = Blueprint('notifications', __name__)
 
@@ -193,13 +194,12 @@ def get_all_notifications():
         return jsonify(result="error", message="Invalid page"), 400
 
     all_notifications = notifications_dao.get_notifications_for_service(api_user['client'], page)
-
     return jsonify(
         notifications=notification_status_schema.dump(all_notifications.items, many=True).data,
         links=pagination_links(
             all_notifications,
             '.get_all_notifications',
-            request.args
+            **request.args.to_dict()
         )
     ), 200
 
@@ -213,13 +213,14 @@ def get_all_notifications_for_service(service_id):
         return jsonify(result="error", message="Invalid page"), 400
 
     all_notifications = notifications_dao.get_notifications_for_service(service_id, page)
-
+    kwargs = request.args.to_dict()
+    kwargs['service_id'] = service_id
     return jsonify(
         notifications=notification_status_schema.dump(all_notifications.items, many=True).data,
         links=pagination_links(
             all_notifications,
             '.get_all_notifications_for_service',
-            request.args
+            **kwargs
         )
     ), 200
 
@@ -233,13 +234,15 @@ def get_all_notifications_for_service_job(service_id, job_id):
         return jsonify(result="error", message="Invalid page"), 400
 
     all_notifications = notifications_dao.get_notifications_for_job(service_id, job_id, page)
-
+    kwargs = request.args.to_dict()
+    kwargs['service_id'] = service_id
+    kwargs['job_id'] = job_id
     return jsonify(
         notifications=notification_status_schema.dump(all_notifications.items, many=True).data,
         links=pagination_links(
             all_notifications,
             '.get_all_notifications_for_service_job',
-            request.args
+            **kwargs
         )
     ), 200
 
@@ -255,13 +258,15 @@ def get_page_from_request():
         return 1
 
 
-def pagination_links(pagination, endpoint, args):
+def pagination_links(pagination, endpoint, **kwargs):
+    if 'page' in kwargs:
+        kwargs.pop('page', None)
     links = dict()
     if pagination.has_prev:
-        links['prev'] = url_for(endpoint, **dict(list(args.items()) + [('page', pagination.prev_num)]))
+        links['prev'] = url_for(endpoint, page=pagination.prev_num, **kwargs)
     if pagination.has_next:
-        links['next'] = url_for(endpoint, **dict(list(args.items()) + [('page', pagination.next_num)]))
+        links['next'] = url_for(endpoint, page=pagination.next_num, **kwargs)
-        links['last'] = url_for(endpoint, **dict(list(args.items()) + [('page', pagination.pages)]))
+        links['last'] = url_for(endpoint, page=pagination.pages, **kwargs)
     return links
 
 
@@ -320,7 +325,7 @@ def send_notification(notification_type):
     notification_id = create_uuid()
 
     if notification_type == 'sms':
-        if service.restricted and notification['to'] not in [user.mobile_number for user in service.users]:
+        if not allowed_send_to_number(service, notification['to']):
             return jsonify(
                 result="error", message={'to': ['Invalid phone number for restricted service']}), 400
         send_sms.apply_async((
@@ -330,7 +335,7 @@ def send_notification(notification_type):
             datetime.utcnow().strftime(DATETIME_FORMAT)
         ), queue='sms')
     else:
-        if service.restricted and notification['to'] not in [user.email_address for user in service.users]:
+        if not allowed_send_to_email(service, notification['to']):
             return jsonify(
                 result="error", message={'to': ['Email address not permitted for restricted service']}), 400
         send_email.apply_async((

app/schemas.py

@@ -171,6 +171,9 @@ class SmsAdminNotificationSchema(SmsNotificationSchema):
 
 class NotificationStatusSchema(BaseSchema):
 
+    template = fields.Nested(TemplateSchema, only=["id", "name", "template_type"], dump_only=True)
+    job = fields.Nested(JobSchema, only=["id", "original_file_name"], dump_only=True)
+
     class Meta:
         model = models.Notification
 

app/user/rest.py

@@ -24,7 +24,13 @@ from app.schemas import (
     permission_schema
 )
 
-from app.celery.tasks import (send_sms_code, send_email_code, email_reset_password)
+from app.celery.tasks import (
+    send_sms_code,
+    send_email_code,
+    email_reset_password,
+    email_registration_verification
+)
+
 from app.errors import register_errors
 
 user = Blueprint('user', __name__)
@@ -148,6 +154,28 @@ def send_user_email_code(user_id):
     return jsonify({}), 204
 
 
+@user.route('/<int:user_id>/email-verification', methods=['POST'])
+def send_user_email_verification(user_id):
+    user_to_send_to = get_model_users(user_id=user_id)
+    verify_code, errors = request_verify_code_schema.load(request.get_json())
+    if errors:
+        return jsonify(result="error", message=errors), 400
+
+    from app.dao.users_dao import create_secret_code
+    secret_code = create_secret_code()
+    create_user_code(user_to_send_to, secret_code, 'email')
+
+    email = user_to_send_to.email_address
+    verification_message = {'to': email,
+                            'name': user_to_send_to.name,
+                            'url': _create_verification_url(user_to_send_to, secret_code)}
+
+    email_registration_verification.apply_async([encryption.encrypt(verification_message)],
+                                                queue='email-registration-verification')
+
+    return jsonify({}), 204
+
+
 @user.route('/<int:user_id>', methods=['GET'])
 @user.route('', methods=['GET'])
 def get_user(user_id=None):
@@ -207,3 +235,12 @@ def _create_reset_password_url(email):
     token = generate_token(data, current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT'])
 
     return current_app.config['ADMIN_BASE_URL'] + '/new-password/' + token
+
+
+def _create_verification_url(user, secret_code):
+    from utils.url_safe_token import generate_token
+    import json
+    data = json.dumps({'user_id': user.id, 'email': user.email_address, 'secret_code': secret_code})
+    token = generate_token(data, current_app.config['SECRET_KEY'], current_app.config['DANGEROUS_SALT'])
+
+    return current_app.config['ADMIN_BASE_URL'] + '/verify-email/' + token

app/validation.py

@@ -0,0 +1,15 @@
+from utils.recipients import format_phone_number, validate_phone_number
+
+
+def allowed_send_to_number(service, to):
+    if service.restricted and format_phone_number(validate_phone_number(to)) not in [
+        format_phone_number(validate_phone_number(user.mobile_number)) for user in service.users
+    ]:
+        return False
+    return True
+
+
+def allowed_send_to_email(service, to):
+    if service.restricted and to not in [user.email_address for user in service.users]:
+        return False
+    return True

aws_run_celery.py

@@ -3,10 +3,19 @@ from app import notify_celery, create_app
 from credstash import getAllSecrets
 import os
 
+default_env_file = '/home/ubuntu/environment'
+environment = 'live'
+
+if os.path.isfile(default_env_file):
+    with open(default_env_file, 'r') as environment_file:
+        environment = environment_file.readline().strip()
+
 # on aws get secrets and export to env
-secrets = getAllSecrets(region="eu-west-1")
+os.environ.update(getAllSecrets(region="eu-west-1"))
-for key, val in secrets.items():
+
-    os.environ[key] = val
+from config import configs
+
+os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
 
 application = create_app()
 application.app_context().push()

config.py

@@ -69,7 +69,8 @@ class Config(object):
         Queue('process-job', Exchange('default'), routing_key='process-job'),
         Queue('bulk-sms', Exchange('default'), routing_key='bulk-sms'),
         Queue('bulk-email', Exchange('default'), routing_key='bulk-email'),
-        Queue('email-invited-user', Exchange('default'), routing_key='email-invited-user')
+        Queue('email-invited-user', Exchange('default'), routing_key='email-invited-user'),
+        Queue('email-registration-verification', Exchange('default'), routing_key='email-registration-verification')
     ]
     TWILIO_ACCOUNT_SID = os.getenv('TWILIO_ACCOUNT_SID')
     TWILIO_AUTH_TOKEN = os.getenv('TWILIO_AUTH_TOKEN')
@@ -84,3 +85,9 @@ class Development(Config):
 
 class Test(Development):
     pass
+
+configs = {
+    'live': 'config_live.Live',
+    'staging': 'config_staging.Staging',
+    'preview': 'config.Config'
+}

config_live.py

@@ -0,0 +1,25 @@
+import os
+from config import Config
+
+
+class Live(Config):
+    ADMIN_BASE_URL = os.environ['LIVE_ADMIN_BASE_URL']
+    API_HOST_NAME = os.environ['LIVE_API_HOST_NAME']
+    ADMIN_CLIENT_SECRET = os.environ['LIVE_ADMIN_CLIENT_SECRET']
+    DANGEROUS_SALT = os.environ['LIVE_DANGEROUS_SALT']
+    NOTIFICATION_QUEUE_PREFIX = os.environ['LIVE_NOTIFICATION_QUEUE_PREFIX']
+    NOTIFY_JOB_QUEUE = os.environ['LIVE_NOTIFY_JOB_QUEUE']
+    SECRET_KEY = os.environ['LIVE_SECRET_KEY']
+    SQLALCHEMY_DATABASE_URI = os.environ['LIVE_SQLALCHEMY_DATABASE_URI']
+    VERIFY_CODE_FROM_EMAIL_ADDRESS = os.environ['LIVE_VERIFY_CODE_FROM_EMAIL_ADDRESS']
+    NOTIFY_EMAIL_DOMAIN = os.environ['LIVE_NOTIFY_EMAIL_DOMAIN']
+    FIRETEXT_API_KEY = os.getenv("LIVE_FIRETEXT_API_KEY")
+    FIRETEXT_NUMBER = os.getenv("LIVE_FIRETEXT_NUMBER")
+    TWILIO_AUTH_TOKEN = os.getenv('LIVE_TWILIO_AUTH_TOKEN')
+
+    BROKER_TRANSPORT_OPTIONS = {
+        'region': 'eu-west-1',
+        'polling_interval': 1,  # 1 second
+        'visibility_timeout': 60,  # 60 seconds
+        'queue_name_prefix': os.environ['LIVE_NOTIFICATION_QUEUE_PREFIX'] + '-'
+    }

config_staging.py

@@ -0,0 +1,25 @@
+import os
+from config import Config
+
+
+class Staging(Config):
+    ADMIN_BASE_URL = os.environ['STAGING_ADMIN_BASE_URL']
+    API_HOST_NAME = os.environ['STAGING_API_HOST_NAME']
+    ADMIN_CLIENT_SECRET = os.environ['STAGING_ADMIN_CLIENT_SECRET']
+    DANGEROUS_SALT = os.environ['STAGING_DANGEROUS_SALT']
+    NOTIFICATION_QUEUE_PREFIX = os.environ['STAGING_NOTIFICATION_QUEUE_PREFIX']
+    NOTIFY_JOB_QUEUE = os.environ['STAGING_NOTIFY_JOB_QUEUE']
+    SECRET_KEY = os.environ['STAGING_SECRET_KEY']
+    SQLALCHEMY_DATABASE_URI = os.environ['STAGING_SQLALCHEMY_DATABASE_URI']
+    VERIFY_CODE_FROM_EMAIL_ADDRESS = os.environ['STAGING_VERIFY_CODE_FROM_EMAIL_ADDRESS']
+    NOTIFY_EMAIL_DOMAIN = os.environ['STAGING_NOTIFY_EMAIL_DOMAIN']
+    FIRETEXT_API_KEY = os.getenv("STAGING_FIRETEXT_API_KEY")
+    FIRETEXT_NUMBER = os.getenv("STAGING_FIRETEXT_NUMBER")
+    TWILIO_AUTH_TOKEN = os.getenv('STAGING_TWILIO_AUTH_TOKEN')
+
+    BROKER_TRANSPORT_OPTIONS = {
+        'region': 'eu-west-1',
+        'polling_interval': 1,  # 1 second
+        'visibility_timeout': 60,  # 60 seconds
+        'queue_name_prefix': os.environ['STAGING_NOTIFICATION_QUEUE_PREFIX'] + '-'
+    }

db.py

@@ -4,9 +4,19 @@ from app import create_app, db
 from credstash import getAllSecrets
 import os
 
-secrets = getAllSecrets(region="eu-west-1")
+default_env_file = '/home/ubuntu/environment'
-for key, val in secrets.items():
+environment = 'live'
-    os.environ[key] = val
+
+if os.path.isfile(default_env_file):
+    with open(default_env_file, 'r') as environment_file:
+        environment = environment_file.readline().strip()
+
+# on aws get secrets and export to env
+os.environ.update(getAllSecrets(region="eu-west-1"))
+
+from config import configs
+
+os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
 
 application = create_app()
 

migrations/versions/0041_platform_admin.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 0041_platform_admin
+Revises: 0040_add_reference
+Create Date: 2016-03-16 16:33:15.279429
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0041_platform_admin'
+down_revision = '0040_add_reference'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(op.f('ix_notification_statistics_service_id'), 'notification_statistics', ['service_id'], unique=False)
+    op.drop_index('ix_service_notification_stats_service_id', table_name='notification_statistics')
+    op.add_column('users', sa.Column('platform_admin', sa.Boolean(), nullable=True, default=False))
+    op.get_bind()
+    op.execute('update users set platform_admin = False')
+    op.alter_column('users', 'platform_admin', nullable=False)
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('users', 'platform_admin')
+    op.create_index('ix_service_notification_stats_service_id', 'notification_statistics', ['service_id'], unique=False)
+    op.drop_index(op.f('ix_notification_statistics_service_id'), table_name='notification_statistics')
+    ### end Alembic commands ###

migrations/versions/0042_default_stats_to_zero.py

@@ -0,0 +1,45 @@
+"""empty message
+
+Revision ID: 0042_default_stats_to_zero
+Revises: 0041_platform_admin
+Create Date: 2016-03-17 11:09:17.906910
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0042_default_stats_to_zero'
+down_revision = '0041_platform_admin'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.get_bind()
+    op.execute('update notification_statistics set emails_delivered = 0 where emails_delivered is Null')
+    op.execute('update notification_statistics set emails_error = 0 where emails_error is Null')
+    op.execute('update notification_statistics set sms_delivered = 0 where sms_delivered is Null')
+    op.execute('update notification_statistics set sms_error = 0 where sms_error is Null')
+    op.alter_column('notification_statistics', 'emails_requested', server_default='0')
+    op.alter_column('notification_statistics', 'emails_delivered', server_default='0', nullable=False)
+    op.alter_column('notification_statistics', 'emails_error', server_default='0', nullable=False)
+    op.alter_column('notification_statistics', 'sms_requested', server_default='0')
+    op.alter_column('notification_statistics', 'sms_delivered', server_default='0', nullable=False)
+    op.alter_column('notification_statistics', 'sms_error', server_default='0', nullable=False)
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('notification_statistics', 'emails_requested', server_default=None)
+    op.alter_column('notification_statistics', 'emails_delivered', server_default=None, nullable=True)
+    op.alter_column('notification_statistics', 'emails_error', server_default=None, nullable=True)
+    op.alter_column('notification_statistics', 'sms_requested', server_default=None)
+    op.alter_column('notification_statistics', 'sms_delivered', server_default=None, nullable=True)
+    op.alter_column('notification_statistics', 'sms_error', server_default=None, nullable=True)
+    op.execute('update notification_statistics set emails_delivered = Null where emails_delivered = 0')
+    op.execute('update notification_statistics set emails_error = Null where emails_error = 0')
+    op.execute('update notification_statistics set sms_delivered = Null where sms_delivered = 0')
+    op.execute('update notification_statistics set sms_error = Null where sms_error = 0')
+    ### end Alembic commands ###

run_celery.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 from app import notify_celery, create_app
 
-application = create_app()
+application = create_app('delivery')
 application.app_context().push()

scripts/aws_start_app.sh

@@ -1,4 +1,19 @@
 #!/bin/bash
 
-echo "Starting application"
+if [ -e "/etc/init/notifications-api.conf" ]
-sudo service notifications-api start
+then
+  echo "Starting api"
+  sudo service notifications-api start
+fi
+
+if [ -e "/etc/init/notifications-api-celery-worker.conf" ]
+then
+  echo "Starting celery worker"
+  sudo service notifications-api-celery-worker start
+fi
+
+if [ -e "/etc/init/notifications-api-celery-beat.conf" ]
+then
+  echo "Starting celery beat"
+  sudo service notifications-api-celery-beat start
+fi

scripts/aws_stop_app.sh

@@ -7,9 +7,29 @@ function error_exit
 	exit 0
 }
 
-echo "Stopping application"
+if [ -e "/etc/init/notifications-api.conf" ]; then
-if sudo service notifications-api stop; then
+    echo "stopping notifications-api"
-    exit 0
+    if sudo service notifications-api stop; then
-else
+        echo "notifications-api stopped"
-    error_exit "Could not stop application"
+    else
+        error_exit "Could not stop notifications-api"
+    fi
+fi
+
+if [ -e "/etc/init/notifications-api-celery-beat.conf" ]; then
+    echo "stopping notifications-api-celery-beat"
+    if sudo service notifications-api-celery-beat stop; then
+        echo "notifications-api stopped"
+    else
+        error_exit "Could not stop notifications-celery-beat"
+    fi
+fi
+
+if [ -e "/etc/init/notifications-api-celery-worker.conf" ]; then
+    echo "stopping notifications-api-celery-worker"
+    if sudo service notifications-api-celery-worker stop; then
+        echo "notifications-api stopped"
+    else
+        error_exit "Could not stop notifications-celery-worker"
+    fi
 fi

tests/app/celery/test_tasks.py

@@ -1,6 +1,8 @@
 import uuid
 import pytest
 from flask import current_app
+from utils.recipients import validate_phone_number, format_phone_number
+
 from app.celery.tasks import (
     send_sms,
     send_sms_code,
@@ -248,7 +250,7 @@ def test_should_process_all_sms_job(sample_job, sample_job_with_placeholdered_te
 def test_should_send_template_to_correct_sms_provider_and_persist(sample_template_with_placeholders, mocker):
     notification = {
         "template": sample_template_with_placeholders.id,
-        "to": "+441234123123",
+        "to": "+447234123123",
         "personalisation": {"name": "Jo"}
     }
     mocker.patch('app.encryption.decrypt', return_value=notification)
@@ -265,7 +267,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat
     )
 
     firetext_client.send_sms.assert_called_once_with(
-        to="+441234123123",
+        to=format_phone_number(validate_phone_number("+447234123123")),
         content="Sample service: Hello Jo",
         reference=str(notification_id)
     )
@@ -273,7 +275,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat
         sample_template_with_placeholders.service_id, notification_id
     )
     assert persisted_notification.id == notification_id
-    assert persisted_notification.to == '+441234123123'
+    assert persisted_notification.to == '+447234123123'
     assert persisted_notification.template_id == sample_template_with_placeholders.id
     assert persisted_notification.status == 'sent'
     assert persisted_notification.created_at == now
@@ -285,7 +287,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist(sample_templat
 def test_should_send_sms_without_personalisation(sample_template, mocker):
     notification = {
         "template": sample_template.id,
-        "to": "+441234123123"
+        "to": "+447234123123"
     }
     mocker.patch('app.encryption.decrypt', return_value=notification)
     mocker.patch('app.firetext_client.send_sms')
@@ -301,7 +303,7 @@ def test_should_send_sms_without_personalisation(sample_template, mocker):
     )
 
     firetext_client.send_sms.assert_called_once_with(
-        to="+441234123123",
+        to=format_phone_number(validate_phone_number("+447234123123")),
         content="Sample service: This is a template",
         reference=str(notification_id)
     )
@@ -330,7 +332,7 @@ def test_should_send_sms_if_restricted_service_and_valid_number(notify_db, notif
     )
 
     firetext_client.send_sms.assert_called_once_with(
-        to="+447700900890",
+        to=format_phone_number(validate_phone_number("+447700900890")),
         content="Sample service: This is a template",
         reference=str(notification_id)
     )
@@ -396,7 +398,7 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa
     notification = {
         "template": sample_job.template.id,
         "job": sample_job.id,
-        "to": "+441234123123"
+        "to": "+447234123123"
     }
     mocker.patch('app.encryption.decrypt', return_value=notification)
     mocker.patch('app.firetext_client.send_sms')
@@ -411,13 +413,13 @@ def test_should_send_template_to_correct_sms_provider_and_persist_with_job_id(sa
         now.strftime(DATETIME_FORMAT)
     )
     firetext_client.send_sms.assert_called_once_with(
-        to="+441234123123",
+        to=format_phone_number(validate_phone_number("+447234123123")),
         content="Sample service: This is a template",
         reference=str(notification_id)
     )
     persisted_notification = notifications_dao.get_notification(sample_job.template.service_id, notification_id)
     assert persisted_notification.id == notification_id
-    assert persisted_notification.to == '+441234123123'
+    assert persisted_notification.to == '+447234123123'
     assert persisted_notification.job_id == sample_job.id
     assert persisted_notification.template_id == sample_job.template.id
     assert persisted_notification.status == 'sent'
@@ -520,7 +522,7 @@ def test_should_use_email_template_and_persist_without_personalisation(
 def test_should_persist_notification_as_failed_if_sms_client_fails(sample_template, mocker):
     notification = {
         "template": sample_template.id,
-        "to": "+441234123123"
+        "to": "+447234123123"
     }
     mocker.patch('app.encryption.decrypt', return_value=notification)
     mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException(firetext_error()))
@@ -536,13 +538,13 @@ def test_should_persist_notification_as_failed_if_sms_client_fails(sample_templa
         now.strftime(DATETIME_FORMAT)
     )
     firetext_client.send_sms.assert_called_once_with(
-        to="+441234123123",
+        to=format_phone_number(validate_phone_number("+447234123123")),
         content="Sample service: This is a template",
         reference=str(notification_id)
     )
     persisted_notification = notifications_dao.get_notification(sample_template.service_id, notification_id)
     assert persisted_notification.id == notification_id
-    assert persisted_notification.to == '+441234123123'
+    assert persisted_notification.to == '+447234123123'
     assert persisted_notification.template_id == sample_template.id
     assert persisted_notification.status == 'failed'
     assert persisted_notification.created_at == now
@@ -590,7 +592,7 @@ def test_should_persist_notification_as_failed_if_email_client_fails(sample_emai
 def test_should_not_send_sms_if_db_peristance_failed(sample_template, mocker):
     notification = {
         "template": sample_template.id,
-        "to": "+441234123123"
+        "to": "+447234123123"
     }
     mocker.patch('app.encryption.decrypt', return_value=notification)
     mocker.patch('app.firetext_client.send_sms')
@@ -638,24 +640,28 @@ def test_should_not_send_email_if_db_peristance_failed(sample_email_template, mo
 
 
 def test_should_send_sms_code(mocker):
-    notification = {'to': '+441234123123',
+    notification = {'to': '+447234123123',
                     'secret_code': '12345'}
 
     encrypted_notification = encryption.encrypt(notification)
 
     mocker.patch('app.firetext_client.send_sms')
     send_sms_code(encrypted_notification)
-    firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'], 'send-sms-code')
+    firetext_client.send_sms.assert_called_once_with(format_phone_number(validate_phone_number(notification['to'])),
+                                                     notification['secret_code'],
+                                                     'send-sms-code')
 
 
 def test_should_throw_firetext_client_exception(mocker):
-    notification = {'to': '+441234123123',
+    notification = {'to': '+447234123123',
                     'secret_code': '12345'}
 
     encrypted_notification = encryption.encrypt(notification)
     mocker.patch('app.firetext_client.send_sms', side_effect=FiretextClientException(firetext_error()))
     send_sms_code(encrypted_notification)
-    firetext_client.send_sms.assert_called_once_with(notification['to'], notification['secret_code'], 'send-sms-code')
+    firetext_client.send_sms.assert_called_once_with(format_phone_number(validate_phone_number(notification['to'])),
+                                                     notification['secret_code'],
+                                                     'send-sms-code')
 
 
 def test_should_send_email_code(mocker):

tests/app/conftest.py

@@ -327,6 +327,11 @@ def mock_celery_send_email_code(mocker):
     return mocker.patch('app.celery.tasks.send_email_code.apply_async')
 
 
+@pytest.fixture(scope='function')
+def mock_celery_email_registration_verification(mocker):
+    return mocker.patch('app.celery.tasks.email_registration_verification.apply_async')
+
+
 @pytest.fixture(scope='function')
 def mock_encryption(mocker):
     return mocker.patch('app.encryption.encrypt', return_value="something_encrypted")

tests/app/dao/test_notification_dao.py

@@ -70,8 +70,13 @@ def test_should_be_able_to_get_statistics_for_a_service(sample_template):
     assert len(stats) == 1
     assert stats[0].emails_requested == 0
     assert stats[0].sms_requested == 1
+    assert stats[0].sms_delivered == 0
+    assert stats[0].sms_error == 0
     assert stats[0].day == notification.created_at.strftime(DATE_FORMAT)
     assert stats[0].service_id == notification.service_id
+    assert stats[0].emails_requested == 0
+    assert stats[0].emails_delivered == 0
+    assert stats[0].emails_error == 0
 
 
 def test_should_be_able_to_get_statistics_for_a_service_for_a_day(sample_template):
@@ -90,7 +95,11 @@ def test_should_be_able_to_get_statistics_for_a_service_for_a_day(sample_templat
         sample_template.service.id, now.strftime(DATE_FORMAT)
     )
     assert stat.emails_requested == 0
+    assert stat.emails_error == 0
+    assert stat.emails_delivered == 0
     assert stat.sms_requested == 1
+    assert stat.sms_error == 0
+    assert stat.sms_delivered == 0
     assert stat.day == notification.created_at.strftime(DATE_FORMAT)
     assert stat.service_id == notification.service_id
 

tests/app/dao/test_users_dao.py

@@ -32,6 +32,7 @@ def test_create_user(notify_api, notify_db, notify_db_session):
     assert User.query.count() == 1
     assert User.query.first().email_address == email
     assert User.query.first().id == user.id
+    assert not user.platform_admin
 
 
 def test_get_all_users(notify_api, notify_db, notify_db_session, sample_user):

tests/app/notifications/test_rest.py

@@ -25,7 +25,14 @@ def test_get_notification_by_id(notify_api, sample_notification):
 
             notification = json.loads(response.get_data(as_text=True))['notification']
             assert notification['status'] == 'sent'
-            assert notification['template'] == sample_notification.template.id
+            assert notification['template'] == {
+                'id': sample_notification.template.id,
+                'name': sample_notification.template.name,
+                'template_type': sample_notification.template.template_type}
+            assert notification['job'] == {
+                'id': str(sample_notification.job.id),
+                'original_file_name': sample_notification.job.original_file_name
+            }
             assert notification['to'] == '+447700900855'
             assert notification['service'] == str(sample_notification.service_id)
             assert response.status_code == 200
@@ -64,7 +71,14 @@ def test_get_all_notifications(notify_api, sample_notification):
 
             notifications = json.loads(response.get_data(as_text=True))
             assert notifications['notifications'][0]['status'] == 'sent'
-            assert notifications['notifications'][0]['template'] == sample_notification.template.id
+            assert notifications['notifications'][0]['template'] == {
+                'id': sample_notification.template.id,
+                'name': sample_notification.template.name,
+                'template_type': sample_notification.template.template_type}
+            assert notifications['notifications'][0]['job'] == {
+                'id': str(sample_notification.job.id),
+                'original_file_name': sample_notification.job.original_file_name
+            }
             assert notifications['notifications'][0]['to'] == '+447700900855'
             assert notifications['notifications'][0]['service'] == str(sample_notification.service_id)
             assert response.status_code == 200

tests/app/test_validation.py

@@ -0,0 +1,65 @@
+from app.models import User, Service
+from app.validation import allowed_send_to_number, allowed_send_to_email
+
+
+def test_allowed_send_to_number_returns_true_for_restricted_service_with_same_number():
+    mobile_number = '07524609792'
+    service = _create_service_data(mobile_number)
+    assert allowed_send_to_number(service, mobile_number)
+
+
+def test_allowed_send_to_number_returns_false_for_restricted_service_with_different_number():
+    mobile_number = '00447524609792'
+    service = _create_service_data(mobile_number)
+    assert not allowed_send_to_number(service, '+447344609793')
+
+
+def test_allowed_send_to_number_returns_true_for_unrestricted_service_with_different_number():
+    mobile_number = '+447524609792'
+    service = _create_service_data(mobile_number, False)
+    assert allowed_send_to_number(service, '+447344609793')
+
+
+def test_allowed_send_to_email__returns_true_for_restricted_service_with_same_email():
+    email = 'testing@it.gov.uk'
+    service = _create_service_data(email_address=email)
+    assert allowed_send_to_email(service, email)
+
+
+def test_allowed_send_to_email__returns_false_for_restricted_service_with_different_email():
+    email = 'testing@it.gov.uk'
+    service = _create_service_data(email_address=email)
+    assert not allowed_send_to_email(service, 'another@it.gov.uk')
+
+
+def test_allowed_send_to_email__returns_false_for_restricted_service_with_different_email():
+    email = 'testing@it.gov.uk'
+    service = _create_service_data(email_address=email)
+    assert not allowed_send_to_email(service, 'another@it.gov.uk')
+
+
+def test_allowed_send_to_email__returns_true_for_unrestricted_service_with_different_email():
+    email = 'testing@it.gov.uk'
+    service = _create_service_data(email_address=email, restricted=False)
+    assert allowed_send_to_number(service, 'another@it.gov.uk')
+
+
+def _create_service_data(mobile_number='+447524609792', restricted=True, email_address='test_user@it.gov.uk'):
+    usr = {
+        'name': 'Test User',
+        'email_address': email_address,
+        'password': 'password',
+        'mobile_number': mobile_number,
+        'state': 'active'
+    }
+    user = User(**usr)
+    data = {
+        'name': 'Test service',
+        'limit': 10,
+        'active': False,
+        'restricted': restricted,
+        'email_from': 'test_service@it.gov.uk'
+    }
+    service = Service(**data)
+    service.users = [user]
+    return service

tests/app/user/test_rest_verify.py

@@ -1,13 +1,25 @@
 import json
 import moto
-from datetime import (datetime, timedelta)
+
+from datetime import (
+    datetime,
+    timedelta
+)
+
 from flask import url_for
-from app.models import (VerifyCode, User)
+
-import app.celery.tasks
+from app.models import (
+    VerifyCode,
+    User
+)
+
 from app import db, encryption
+
 from tests import create_authorization_header
 from freezegun import freeze_time
 
+import app.celery.tasks
+
 
 def test_user_verify_code_sms(notify_api,
                               sample_sms_code):
@@ -341,3 +353,23 @@ def test_send_user_email_code_returns_404_for_when_user_does_not_exist(notify_ap
                 headers=[('Content-Type', 'application/json'), auth_header])
             assert resp.status_code == 404
             assert json.loads(resp.get_data(as_text=True))['message'] == 'No result found'
+
+
+def test_send_user_email_verification(notify_api,
+                                      sample_email_code,
+                                      mock_celery_email_registration_verification,
+                                      mock_encryption):
+
+    with notify_api.test_request_context():
+        with notify_api.test_client() as client:
+            data = json.dumps({})
+            auth_header = create_authorization_header(
+                path=url_for('user.send_user_email_verification', user_id=sample_email_code.user.id),
+                method='POST',
+                request_body=data)
+            resp = client.post(
+                url_for('user.send_user_email_verification', user_id=sample_email_code.user.id),
+                data=data,
+                headers=[('Content-Type', 'application/json'), auth_header])
+            assert resp.status_code == 204
+            app.celery.tasks.email_registration_verification.apply_async.assert_called_once_with(['something_encrypted'], queue='email-registration-verification')  # noqa

wsgi.py

@@ -3,12 +3,22 @@ import os
 from app import create_app
 from credstash import getAllSecrets
 
+
+default_env_file = '/home/ubuntu/environment'
+environment = 'live'
+
+if os.path.isfile(default_env_file):
+    with open(default_env_file, 'r') as environment_file:
+        environment = environment_file.readline().strip()
+
 # on aws get secrets and export to env
-secrets = getAllSecrets(region="eu-west-1")
+os.environ.update(getAllSecrets(region="eu-west-1"))
-for key, val in secrets.items():
+
-    os.environ[key] = val
+from config import configs
+
+os.environ['NOTIFY_API_ENVIRONMENT'] = configs[environment]
 
 application = create_app()
 
 if __name__ == "__main__":
-        application.run()
+    application.run()