merged master and up migration version

app/accept_invite/rest.py

@@ -4,7 +4,7 @@ from flask import (
     current_app
 )
 
-from itsdangerous import SignatureExpired
+from itsdangerous import SignatureExpired, BadData
 
 from notifications_utils.url_safe_token import check_token
 
@@ -38,6 +38,9 @@ def validate_invitation_token(invitation_type, token):
                   ['Your invitation to GOV.UK Notify has expired. '
                    'Please ask the person that invited you to send you another one']}
         raise InvalidRequest(errors, status_code=400)
+    except BadData:
+        errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
+        raise InvalidRequest(errors, status_code=400)
 
     if invitation_type == 'service':
         invited_user = get_invited_user_by_id(invited_user_id)

app/celery/research_mode_tasks.py

@@ -123,7 +123,7 @@ def firetext_callback(notification_id, to):
 def create_fake_letter_response_file(self, reference):
     now = datetime.utcnow()
     dvla_response_data = '{}|Sent|0|Sorted'.format(reference)
-    upload_file_name = 'NOTIFY.{}.RSP.TXT'.format(now.strftime('%Y%m%d%H%M%S'))
+    upload_file_name = 'NOTIFY-{}-RSP.TXT'.format(now.strftime('%Y%m%d%H%M%S'))
 
     s3upload(
         filedata=dvla_response_data,

app/celery/tasks.py

@@ -420,33 +420,37 @@ def update_letter_notifications_statuses(self, filename):
             update_letter_notification(filename, temporary_failures, update)
             sorted_letter_counts[update.cost_threshold] += 1
 
-        if temporary_failures:
+        try:
-            # This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
+            if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}:
-            message = "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
+                unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}
-                filename=filename, failures=temporary_failures)
-            raise DVLAException(message)
 
-        if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}:
+                message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
-            unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}
+                    filename, unknown_status
+                )
+                raise DVLAException(message)
 
-            message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
+            billing_date = get_billing_date_in_bst_from_filename(filename)
-                filename, unknown_status
+            persist_daily_sorted_letter_counts(day=billing_date,
-            )
+                                               file_name=filename,
-            raise DVLAException(message)
+                                               sorted_letter_counts=sorted_letter_counts)
-
+        finally:
-        billing_date = get_billing_date_in_bst_from_filename(filename)
+            if temporary_failures:
-        persist_daily_sorted_letter_counts(billing_date, sorted_letter_counts)
+                # This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
+                message = "DVLA response file: {filename} has failed letters with notification.reference {failures}" \
+                    .format(filename=filename, failures=temporary_failures)
+                raise DVLAException(message)
 
 
 def get_billing_date_in_bst_from_filename(filename):
-    datetime_string = filename.split('.')[1]
+    datetime_string = filename.split('-')[1]
     datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M%S')
     return convert_utc_to_bst(datetime_obj).date()
 
 
-def persist_daily_sorted_letter_counts(day, sorted_letter_counts):
+def persist_daily_sorted_letter_counts(day, file_name, sorted_letter_counts):
     daily_letter_count = DailySortedLetter(
         billing_day=day,
+        file_name=file_name,
         unsorted_count=sorted_letter_counts['Unsorted'],
         sorted_count=sorted_letter_counts['Sorted']
     )

app/config.py

@@ -326,6 +326,7 @@ class Development(Config):
 
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
 
     ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
@@ -362,6 +363,7 @@ class Test(Development):
 
     CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
 
     # this is overriden in jenkins and on cloudfoundry
@@ -389,6 +391,7 @@ class Preview(Config):
     NOTIFY_ENVIRONMENT = 'preview'
     CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
     FROM_NUMBER = 'preview'
     API_RATE_LIMIT_ENABLED = True
@@ -400,6 +403,7 @@ class Staging(Config):
     NOTIFY_ENVIRONMENT = 'staging'
     CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
     STATSD_ENABLED = True
     FROM_NUMBER = 'stage'
@@ -413,6 +417,7 @@ class Live(Config):
     NOTIFY_ENVIRONMENT = 'live'
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
     STATSD_ENABLED = True
     FROM_NUMBER = 'GOVUK'
@@ -433,6 +438,7 @@ class Sandbox(CloudFoundryConfig):
     NOTIFY_ENVIRONMENT = 'sandbox'
     CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
     LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
     FROM_NUMBER = 'sandbox'
     REDIS_ENABLED = False

app/dao/api_key_dao.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from app import db
 from app.models import ApiKey
@@ -9,6 +9,8 @@ from app.dao.dao_utils import (
     version_class
 )
 
+from sqlalchemy import or_, func
+
 
 @transactional
 @version_class(ApiKey)
@@ -30,7 +32,11 @@ def expire_api_key(service_id, api_key_id):
 def get_model_api_keys(service_id, id=None):
     if id:
         return ApiKey.query.filter_by(id=id, service_id=service_id, expiry_date=None).one()
-    return ApiKey.query.filter_by(service_id=service_id).all()
+    seven_days_ago = datetime.utcnow() - timedelta(days=7)
+    return ApiKey.query.filter(
+        or_(ApiKey.expiry_date == None, func.date(ApiKey.expiry_date) > seven_days_ago),  # noqa
+        ApiKey.service_id == service_id
+    ).all()
 
 
 def get_unsigned_secrets(service_id):

app/dao/daily_sorted_letter_dao.py

@@ -24,13 +24,14 @@ def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
     table = DailySortedLetter.__table__
     stmt = insert(table).values(
         billing_day=new_daily_sorted_letter.billing_day,
+        file_name=new_daily_sorted_letter.file_name,
         unsorted_count=new_daily_sorted_letter.unsorted_count,
         sorted_count=new_daily_sorted_letter.sorted_count)
     stmt = stmt.on_conflict_do_update(
-        index_elements=[table.c.billing_day],
+        index_elements=[table.c.billing_day, table.c.file_name],
         set_={
-            'unsorted_count': table.c.unsorted_count + stmt.excluded.unsorted_count,
+            'unsorted_count': stmt.excluded.unsorted_count,
-            'sorted_count': table.c.sorted_count + stmt.excluded.sorted_count,
+            'sorted_count': stmt.excluded.sorted_count,
             'updated_at': datetime.utcnow()
         }
     )

app/dao/notifications_dao.py

@@ -455,6 +455,12 @@ def dao_get_notifications_by_to_field(service_id, search_term, notification_type
     else:
         raise InvalidRequest("Only email and SMS can use search by recipient", 400)
 
+    for special_character in ('\\', '_', '%', '/'):
+        normalised = normalised.replace(
+            special_character,
+            '\{}'.format(special_character)
+        )
+
     filters = [
         Notification.service_id == service_id,
         Notification.normalised_to.like("%{}%".format(normalised)),

app/dao/users_dao.py

@@ -1,6 +1,9 @@
 from random import (SystemRandom)
 from datetime import (datetime, timedelta)
+
 from sqlalchemy import func
+from sqlalchemy.orm import joinedload
+
 from app import db
 from app.models import (User, VerifyCode)
 
@@ -113,3 +116,16 @@ def update_user_password(user, password):
     user.password_changed_at = datetime.utcnow()
     db.session.add(user)
     db.session.commit()
+
+
+def get_user_and_accounts(user_id):
+    return User.query.filter(
+        User.id == user_id
+    ).options(
+        # eagerly load the user's services and organisations, and also the service's org and vice versa
+        # (so we can see if the user knows about it)
+        joinedload('services'),
+        joinedload('organisations'),
+        joinedload('organisations.services'),
+        joinedload('services.organisation'),
+    ).one()

app/letters/utils.py

@@ -5,24 +5,32 @@ from flask import current_app
 
 from notifications_utils.s3 import s3upload
 
+from app.models import KEY_TYPE_TEST
 from app.variables import Retention
 
 
 LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
-    '{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
+    '{folder}NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
 
-PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}'
+PRECOMPILED_BUCKET_PREFIX = '{folder}NOTIFY.{reference}'
 
 
-def get_letter_pdf_filename(reference, crown):
+def get_folder_name(_now, is_test_letter):
+    if is_test_letter:
+        folder_name = ''
+    else:
+        print_datetime = _now
+        if _now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
+            print_datetime = _now + timedelta(days=1)
+        folder_name = '{}/'.format(print_datetime.date())
+    return folder_name
+
+
+def get_letter_pdf_filename(reference, crown, is_test_letter=False):
     now = datetime.utcnow()
 
-    print_datetime = now
-    if now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
-        print_datetime = now + timedelta(days=1)
-
     upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format(
-        folder=print_datetime.date(),
+        folder=get_folder_name(now, is_test_letter),
         reference=reference,
         duplex="D",
         letter_class="2",
@@ -34,41 +42,51 @@ def get_letter_pdf_filename(reference, crown):
     return upload_file_name
 
 
-def get_bucket_prefix_for_notification(notification):
+def get_bucket_prefix_for_notification(notification, is_test_letter=False):
     upload_file_name = PRECOMPILED_BUCKET_PREFIX.format(
-        folder=notification.created_at.date(),
+        folder='' if is_test_letter else
+               '{}/'.format(notification.created_at.date()),
         reference=notification.reference
     ).upper()
 
     return upload_file_name
 
 
-def upload_letter_pdf(notification, pdf_data):
+def upload_letter_pdf(notification, pdf_data, is_test_letter=False):
     current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
         notification.id, notification.reference, notification.created_at, len(pdf_data)))
 
     upload_file_name = get_letter_pdf_filename(
-        notification.reference, notification.service.crown)
+        notification.reference, notification.service.crown, is_test_letter)
+
+    if is_test_letter:
+        bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME']
+    else:
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
 
     s3upload(
         filedata=pdf_data,
         region=current_app.config['AWS_REGION'],
-        bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        bucket_name=bucket_name,
         file_location=upload_file_name,
         tags={Retention.KEY: Retention.ONE_WEEK}
     )
 
     current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format(
-        upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id))
+        upload_file_name, bucket_name, notification.id))
 
 
 def get_letter_pdf(notification):
-    bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
+    is_test_letter = notification.key_type == KEY_TYPE_TEST and notification.template.is_precompiled_letter
+    if is_test_letter:
+        bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME']
+    else:
+        bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
 
     s3 = boto3.resource('s3')
     bucket = s3.Bucket(bucket_name)
 
-    for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)):
+    for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification, is_test_letter)):
         obj = s3.Object(
             bucket_name=bucket_name,
             key=item.key

app/models.py

@@ -116,11 +116,11 @@ class User(db.Model):
     services = db.relationship(
         'Service',
         secondary='user_to_service',
-        backref=db.backref('user_to_service', lazy='dynamic'))
+        backref='user_to_service')
     organisations = db.relationship(
         'Organisation',
         secondary='user_to_organisation',
-        backref=db.backref('user_to_organisation', lazy='dynamic'))
+        backref='users')
 
     @property
     def password(self):
@@ -133,6 +133,38 @@ class User(db.Model):
     def check_password(self, password):
         return check_hash(password, self._password)
 
+    def get_permissions(self):
+        from app.dao.permissions_dao import permission_dao
+        retval = {}
+        for x in permission_dao.get_permissions_by_user_id(self.id):
+            service_id = str(x.service_id)
+            if service_id not in retval:
+                retval[service_id] = []
+            retval[service_id].append(x.permission)
+        return retval
+
+    def serialize(self):
+        return {
+            'id': self.id,
+            'name': self.name,
+            'email_address': self.email_address,
+            'auth_type': self.auth_type,
+            'current_session_id': self.current_session_id,
+            'failed_login_count': self.failed_login_count,
+            'logged_in_at': self.logged_in_at.strftime(DATETIME_FORMAT) if self.logged_in_at else None,
+            'mobile_number': self.mobile_number,
+            'organisations': [x.id for x in self.organisations if x.active],
+            'password_changed_at': (
+                self.password_changed_at.strftime('%Y-%m-%d %H:%M:%S.%f')
+                if self.password_changed_at
+                else None
+            ),
+            'permissions': self.get_permissions(),
+            'platform_admin': self.platform_admin,
+            'services': [x.id for x in self.services if x.active],
+            'state': self.state,
+        }
+
 
 user_to_service = db.Table(
     'user_to_service',
@@ -1726,11 +1758,15 @@ class DailySortedLetter(db.Model):
     __tablename__ = "daily_sorted_letter"
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    billing_day = db.Column(db.Date, nullable=False, index=True, unique=True)
+    billing_day = db.Column(db.Date, nullable=False, index=True)
+    file_name = db.Column(db.String, nullable=True, index=True)
     unsorted_count = db.Column(db.Integer, nullable=False, default=0)
     sorted_count = db.Column(db.Integer, nullable=False, default=0)
     updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
 
+    __table_args__ = (UniqueConstraint('file_name', 'billing_day', name='uix_file_name_billing_day'),
+                      )
+
 
 class FactBilling(db.Model):
     __tablename__ = "ft_billing"

app/organisation/rest.py

@@ -20,7 +20,6 @@ from app.organisation.organisation_schema import (
     post_link_service_to_organisation_schema,
 )
 from app.schema_validation import validate
-from app.schemas import user_schema
 
 organisation_blueprint = Blueprint('organisation', __name__)
 register_errors(organisation_blueprint)
@@ -98,15 +97,13 @@ def get_organisation_services(organisation_id):
 @organisation_blueprint.route('/<uuid:organisation_id>/users/<uuid:user_id>', methods=['POST'])
 def add_user_to_organisation(organisation_id, user_id):
     new_org_user = dao_add_user_to_organisation(organisation_id, user_id)
-    return jsonify(data=user_schema.dump(new_org_user).data), 200
+    return jsonify(data=new_org_user.serialize())
 
 
 @organisation_blueprint.route('/<uuid:organisation_id>/users', methods=['GET'])
 def get_organisation_users(organisation_id):
     org_users = dao_get_users_for_organisation(organisation_id)
-
+    return jsonify(data=[x.serialize() for x in org_users])
-    result = user_schema.dump(org_users, many=True)
-    return jsonify(data=result.data)
 
 
 @organisation_blueprint.route('/unique', methods=["GET"])

app/schemas.py

@@ -665,19 +665,15 @@ class UnarchivedTemplateSchema(BaseSchema):
             raise ValidationError('Template has been deleted', 'template')
 
 
-user_schema = UserSchema()
+# should not be used on its own for dumping - only for loading
-user_schema_load_json = UserSchema(load_json=True)
+create_user_schema = UserSchema()
 user_update_schema_load_json = UserUpdateAttributeSchema(load_json=True, partial=True)
 user_update_password_schema_load_json = UserUpdatePasswordSchema(load_json=True, partial=True)
 service_schema = ServiceSchema()
-service_schema_load_json = ServiceSchema(load_json=True)
 detailed_service_schema = DetailedServiceSchema()
 template_schema = TemplateSchema()
-template_schema_load_json = TemplateSchema(load_json=True)
 api_key_schema = ApiKeySchema()
-api_key_schema_load_json = ApiKeySchema(load_json=True)
 job_schema = JobSchema()
-job_schema_load_json = JobSchema(load_json=True)
 sms_admin_notification_schema = SmsAdminNotificationSchema()
 sms_template_notification_schema = SmsTemplateNotificationSchema()
 job_sms_template_notification_schema = JobSmsTemplateNotificationSchema()

app/service/rest.py

@@ -82,7 +82,6 @@ from app.service.send_notification import send_one_off_notification
 from app.schemas import (
     service_schema,
     api_key_schema,
-    user_schema,
     permission_schema,
     notification_with_template_schema,
     notifications_filter_schema,
@@ -258,8 +257,7 @@ def get_api_keys(service_id, key_id=None):
 @service_blueprint.route('/<uuid:service_id>/users', methods=['GET'])
 def get_users_for_service(service_id):
     fetched = dao_fetch_service_by_id(service_id)
-    result = user_schema.dump(fetched.users, many=True)
+    return jsonify(data=[x.serialize() for x in fetched.users])
-    return jsonify(data=result.data)
 
 
 @service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['POST'])

app/user/rest.py

@@ -19,7 +19,8 @@ from app.dao.users_dao import (
     create_secret_code,
     save_user_attribute,
     update_user_password,
-    count_user_verify_codes
+    count_user_verify_codes,
+    get_user_and_accounts
 )
 from app.dao.permissions_dao import permission_dao
 from app.dao.services_dao import dao_fetch_service_by_id
@@ -31,7 +32,7 @@ from app.notifications.process_notifications import (
 )
 from app.schemas import (
     email_data_request_schema,
-    user_schema,
+    create_user_schema,
     permission_schema,
     user_update_schema_load_json,
     user_update_password_schema_load_json
@@ -67,13 +68,14 @@ def handle_integrity_error(exc):
 
 @user_blueprint.route('', methods=['POST'])
 def create_user():
-    user_to_create, errors = user_schema.load(request.get_json())
+    user_to_create, errors = create_user_schema.load(request.get_json())
     req_json = request.get_json()
     if not req_json.get('password', None):
         errors.update({'password': ['Missing data for required field.']})
         raise InvalidRequest(errors, status_code=400)
     save_model_user(user_to_create, pwd=req_json.get('password'))
-    return jsonify(data=user_schema.dump(user_to_create).data), 201
+    result = user_to_create.serialize()
+    return jsonify(data=result), 201
 
 
 @user_blueprint.route('/<uuid:user_id>', methods=['POST'])
@@ -84,7 +86,7 @@ def update_user_attribute(user_id):
     if errors:
         raise InvalidRequest(errors, status_code=400)
     save_user_attribute(user_to_update, update_dict=update_dct)
-    return jsonify(data=user_schema.dump(user_to_update).data), 200
+    return jsonify(data=user_to_update.serialize()), 200
 
 
 @user_blueprint.route('/<uuid:user_id>/activate', methods=['POST'])
@@ -95,14 +97,14 @@ def activate_user(user_id):
 
     user.state = 'active'
     save_model_user(user)
-    return jsonify(data=user_schema.dump(user).data), 200
+    return jsonify(data=user.serialize()), 200
 
 
 @user_blueprint.route('/<uuid:user_id>/reset-failed-login-count', methods=['POST'])
 def user_reset_failed_login_count(user_id):
     user_to_update = get_user_by_id(user_id=user_id)
     reset_failed_login_count(user_to_update)
-    return jsonify(data=user_schema.dump(user_to_update).data), 200
+    return jsonify(data=user_to_update.serialize()), 200
 
 
 @user_blueprint.route('/<uuid:user_id>/verify/password', methods=['POST'])
@@ -324,8 +326,8 @@ def send_already_registered_email(user_id):
 @user_blueprint.route('', methods=['GET'])
 def get_user(user_id=None):
     users = get_user_by_id(user_id=user_id)
-    result = user_schema.dump(users, many=True) if isinstance(users, list) else user_schema.dump(users)
+    result = [x.serialize() for x in users] if isinstance(users, list) else users.serialize()
-    return jsonify(data=result.data)
+    return jsonify(data=result)
 
 
 @user_blueprint.route('/<uuid:user_id>/service/<uuid:service_id>/permission', methods=['POST'])
@@ -350,9 +352,8 @@ def get_by_email():
         error = 'Invalid request. Email query string param required'
         raise InvalidRequest(error, status_code=400)
     fetched_user = get_user_by_email(email)
-    result = user_schema.dump(fetched_user)
+    result = fetched_user.serialize()
-
+    return jsonify(data=result)
-    return jsonify(data=result.data)
 
 
 @user_blueprint.route('/reset-password', methods=['POST'])
@@ -392,7 +393,14 @@ def update_password(user_id):
     if errors:
         raise InvalidRequest(errors, status_code=400)
     update_user_password(user, pwd)
-    return jsonify(data=user_schema.dump(user).data), 200
+    return jsonify(data=user.serialize()), 200
+
+
+@user_blueprint.route('/<uuid:user_id>/organisations-and-services', methods=['GET'])
+def get_organisations_and_services_for_user(user_id):
+    user = get_user_and_accounts(user_id)
+    data = get_orgs_and_services(user)
+    return jsonify(data)
 
 
 def _create_reset_password_url(email):
@@ -420,3 +428,38 @@ def _create_2fa_url(user, secret_code, next_redir, email_auth_link_host):
     if next_redir:
         ret += '?{}'.format(urlencode({'next': next_redir}))
     return ret
+
+
+def get_orgs_and_services(user):
+    return {
+        'organisations': [
+            {
+                'name': org.name,
+                'id': org.id,
+                'services': [
+                    {
+                        'id': service.id,
+                        'name': service.name
+                    }
+                    for service in org.services
+                    if service.active and service in user.services
+                ]
+            }
+            for org in user.organisations if org.active
+        ],
+        'services_without_organisations': [
+            {
+                'id': service.id,
+                'name': service.name
+            } for service in user.services
+            if (
+                service.active and
+                # include services that either aren't in an organisation, or are in an organisation,
+                # but not one that the user can see.
+                (
+                    not service.organisation or
+                    service.organisation not in user.organisations
+                )
+            )
+        ]
+    }

app/v2/notifications/post_notifications.py

@@ -265,6 +265,8 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text
             queue=QueueNames.RESEARCH_MODE
         )
     else:
+        if precompiled and api_key.key_type == KEY_TYPE_TEST:
+            upload_letter_pdf(notification, letter_content, is_test_letter=True)
         update_notification_status_by_reference(notification.reference, NOTIFICATION_DELIVERED)
 
     return notification

migrations/versions/0178_add_filename.py

@@ -0,0 +1,31 @@
+"""
+
+Revision ID: 0178_add_filename
+Revises: 0177_add_virus_scan_statuses
+Create Date: 2018-03-14 16:15:01.886998
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '0178_add_filename'
+down_revision = '0177_add_virus_scan_statuses'
+
+
+def upgrade():
+    # Deleting the data here is ok because a full migration from the files on s3 is coming.
+    op.execute("DELETE FROM daily_sorted_letter")
+    op.add_column('daily_sorted_letter', sa.Column('file_name', sa.String(), nullable=True))
+    op.create_index(op.f('ix_daily_sorted_letter_file_name'), 'daily_sorted_letter', ['file_name'], unique=False)
+    op.create_unique_constraint('uix_file_name_billing_day', 'daily_sorted_letter', ['file_name', 'billing_day'])
+    op.drop_index('ix_daily_sorted_letter_billing_day', table_name='daily_sorted_letter')
+    op.create_index(op.f('ix_daily_sorted_letter_billing_day'), 'daily_sorted_letter', ['billing_day'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_daily_sorted_letter_billing_day'), table_name='daily_sorted_letter')
+    op.create_index('ix_daily_sorted_letter_billing_day', 'daily_sorted_letter', ['billing_day'], unique=True)
+    op.drop_constraint('uix_file_name_billing_day', 'daily_sorted_letter', type_='unique')
+    op.drop_index(op.f('ix_daily_sorted_letter_file_name'), table_name='daily_sorted_letter')
+    op.drop_column('daily_sorted_letter', 'file_name')

migrations/versions/0179_billing_primary_const.py

@@ -1,7 +1,7 @@
 """
 
-Revision ID: 24f47fae3660
+Revision ID: 0179_billing_primary_const
-Revises: 0178_billing_primary_const
+Revises: 0178_add_filename
 Create Date: 2018-03-13 14:52:40.413474
 
 """
@@ -9,8 +9,8 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
-revision = '0178_billing_primary_const'
+revision = '0179_billing_primary_const'
-down_revision = '0177_add_virus_scan_statuses'
+down_revision = '0178_add_filename'
 
 
 def upgrade():

requirements_for_test.txt

@@ -1,5 +1,6 @@
 -r requirements.txt
 flake8==3.5.0
+moto==1.1.25
 pytest==3.4.2
 pytest-env==0.6.2
 pytest-mock==1.7.1

tests/app/accept_invite/test_accept_invite_rest.py

@@ -61,3 +61,23 @@ def test_validate_invitation_token_returns_400_when_invited_user_does_not_exist(
     json_resp = json.loads(response.get_data(as_text=True))
     assert json_resp['result'] == 'error'
     assert json_resp['message'] == 'No result found'
+
+
+@pytest.mark.parametrize('invitation_type', ['service', 'organisation'])
+def test_validate_invitation_token_returns_400_when_token_is_malformed(client, invitation_type):
+    token = generate_token(
+        str(uuid.uuid4()),
+        current_app.config['SECRET_KEY'],
+        current_app.config['DANGEROUS_SALT']
+    )[:-2]
+
+    url = '/invite/{}/{}'.format(invitation_type, token)
+    auth_header = create_authorization_header()
+    response = client.get(url, headers=[('Content-Type', 'application/json'), auth_header])
+
+    assert response.status_code == 400
+    json_resp = json.loads(response.get_data(as_text=True))
+    assert json_resp['result'] == 'error'
+    assert json_resp['message'] == {
+        'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'
+    }

tests/app/celery/test_ftp_update_tasks.py

@@ -59,8 +59,8 @@ def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_a
     mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)
 
     with pytest.raises(DVLAException) as e:
-        update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
-    assert 'DVLA response file: {} has an invalid format'.format('NOTIFY.20170823160812.RSP.TXT') in str(e)
+    assert 'DVLA response file: {} has an invalid format'.format('NOTIFY-20170823160812-RSP.TXT') in str(e)
 
 
 def test_update_letter_notification_statuses_when_notification_does_not_exist_updates_notification_history(
@@ -73,7 +73,7 @@ def test_update_letter_notification_statuses_when_notification_does_not_exist_up
                                        billable_units=1)
     Notification.query.filter_by(id=notification.id).delete()
 
-    update_letter_notifications_statuses(filename="NOTIFY.20170823160812.RSP.TXT")
+    update_letter_notifications_statuses(filename="NOTIFY-20170823160812-RSP.TXT")
 
     updated_history = NotificationHistory.query.filter_by(id=notification.id).one()
     assert updated_history.status == NOTIFICATION_DELIVERED
@@ -106,10 +106,29 @@ def test_update_letter_notifications_statuses_raises_error_for_unknown_sorted_st
     mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
 
     with pytest.raises(DVLAException) as e:
-        update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
 
     assert "DVLA response file: {filename} contains unknown Sorted status {unknown_status}".format(
-        filename="NOTIFY.20170823160812.RSP.TXT", unknown_status="{'Error'}"
+        filename="NOTIFY-20170823160812-RSP.TXT", unknown_status="{'Error'}"
+    ) in str(e)
+
+
+def test_update_letter_notifications_statuses_still_raises_temp_failure_error_with_unknown_sorted_status(
+    notify_api,
+    mocker,
+    sample_letter_template
+):
+    valid_file = 'ref-foo|Failed|1|unknown'
+    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
+    create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
+                        billable_units=0)
+
+    with pytest.raises(DVLAException) as e:
+        update_letter_notifications_statuses(filename="failed.txt")
+
+    failed = ["ref-foo"]
+    assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
+        filename="failed.txt", failures=failed
     ) in str(e)
 
 
@@ -117,10 +136,10 @@ def test_update_letter_notifications_statuses_calls_with_correct_bucket_location
     s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')
 
     with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
-        update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
         s3_mock.assert_called_with('{}-ftp'.format(
             current_app.config['NOTIFY_EMAIL_DOMAIN']),
-            'NOTIFY.20170823160812.RSP.TXT'
+            'NOTIFY-20170823160812-RSP.TXT'
         )
 
 
@@ -129,7 +148,7 @@ def test_update_letter_notifications_statuses_builds_updates_from_content(notify
     mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
     update_mock = mocker.patch('app.celery.tasks.process_updates_from_file')
 
-    update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
 
     update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted')
 
@@ -162,7 +181,7 @@ def test_update_letter_notifications_statuses_persisted(notify_api, mocker, samp
     mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
 
     with pytest.raises(expected_exception=DVLAException) as e:
-        update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+        update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
 
     assert sent_letter.status == NOTIFICATION_DELIVERED
     assert sent_letter.billable_units == 1
@@ -171,7 +190,7 @@ def test_update_letter_notifications_statuses_persisted(notify_api, mocker, samp
     assert failed_letter.billable_units == 2
     assert failed_letter.updated_at
     assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
-        filename="NOTIFY.20170823160812.RSP.TXT", failures=[format(failed_letter.reference)]) in str(e)
+        filename="NOTIFY-20170823160812-RSP.TXT", failures=[format(failed_letter.reference)]) in str(e)
 
 
 def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count(
@@ -187,9 +206,11 @@ def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count
     mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
     persist_letter_count_mock = mocker.patch('app.celery.tasks.persist_daily_sorted_letter_counts')
 
-    update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
 
-    persist_letter_count_mock.assert_called_once_with(date(2017, 8, 23), {'Unsorted': 1, 'Sorted': 1})
+    persist_letter_count_mock.assert_called_once_with(day=date(2017, 8, 23),
+                                                      file_name='NOTIFY-20170823160812-RSP.TXT',
+                                                      sorted_letter_counts={'Unsorted': 1, 'Sorted': 1})
 
 
 def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count_with_no_sorted_values(
@@ -204,7 +225,7 @@ def test_update_letter_notifications_statuses_persists_daily_sorted_letter_count
         sent_letter_1.reference, sent_letter_2.reference)
     mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
 
-    update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
 
     daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
 
@@ -223,7 +244,7 @@ def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry(
         'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
 
-    update_letter_notifications_statuses(filename='NOTIFY.20170823160812.RSP.TXT')
+    update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
     send_mock.assert_not_called()
 
 
@@ -301,7 +322,7 @@ def test_check_billable_units_when_billable_units_does_not_match_page_count(
     ('20170120230000', date(2017, 1, 20))
 ])
 def test_get_billing_date_in_bst_from_filename(filename_date, billing_date):
-    filename = 'NOTIFY.{}.RSP.TXT'.format(filename_date)
+    filename = 'NOTIFY-{}-RSP.TXT'.format(filename_date)
     result = get_billing_date_in_bst_from_filename(filename)
 
     assert result == billing_date
@@ -310,7 +331,7 @@ def test_get_billing_date_in_bst_from_filename(filename_date, billing_date):
 @freeze_time("2018-01-11 09:00:00")
 def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(client, notify_db_session):
     letter_counts = defaultdict(int, **{'Unsorted': 5, 'Sorted': 1})
-    persist_daily_sorted_letter_counts(date.today(), letter_counts)
+    persist_daily_sorted_letter_counts(date.today(), "test.txt", letter_counts)
     day = dao_get_daily_sorted_letter_by_billing_day(date.today())
 
     assert day.unsorted_count == 5

tests/app/celery/test_letters_pdf_tasks.py

@@ -28,25 +28,6 @@ def test_should_have_decorated_tasks_functions():
     assert create_letters_pdf.__wrapped__.__name__ == 'create_letters_pdf'
 
 
-@pytest.mark.parametrize('crown_flag,expected_crown_text', [
-    (True, 'C'),
-    (False, 'N'),
-])
-@freeze_time("2017-12-04 17:29:00")
-def test_get_letter_pdf_filename_returns_correct_filename(
-        notify_api, mocker, crown_flag, expected_crown_text):
-    filename = get_letter_pdf_filename(reference='foo', crown=crown_flag)
-
-    assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(expected_crown_text)
-
-
-@freeze_time("2017-12-04 17:31:00")
-def test_get_letter_pdf_filename_returns_tomorrows_filename(notify_api, mocker):
-    filename = get_letter_pdf_filename(reference='foo', crown=True)
-
-    assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF'
-
-
 @pytest.mark.parametrize('personalisation', [{'name': 'test'}, None])
 def test_get_letters_pdf_calls_notifications_template_preview_service_correctly(
         notify_api, mocker, client, sample_letter_template, personalisation):

tests/app/celery/test_research_mode_tasks.py

@@ -112,7 +112,7 @@ def test_failure_firetext_callback(phone_number):
 def test_create_fake_letter_response_file_uploads_response_file_s3(
         notify_api, mocker):
     mock_s3upload = mocker.patch('app.celery.research_mode_tasks.s3upload')
-    filename = 'NOTIFY.20180125140000.RSP.TXT'
+    filename = 'NOTIFY-20180125140000-RSP.TXT'
 
     with requests_mock.Mocker() as request_mock:
         request_mock.post(
@@ -135,7 +135,7 @@ def test_create_fake_letter_response_file_uploads_response_file_s3(
 def test_create_fake_letter_response_file_calls_dvla_callback_on_development(
         notify_api, mocker):
     mocker.patch('app.celery.research_mode_tasks.s3upload')
-    filename = 'NOTIFY.20180125140000.RSP.TXT'
+    filename = 'NOTIFY-20180125140000-RSP.TXT'
 
     with set_config_values(notify_api, {
         'NOTIFY_ENVIRONMENT': 'development'

tests/app/dao/notification_dao/test_notification_dao.py

@@ -1769,6 +1769,48 @@ def test_dao_get_notifications_by_to_field_matches_partial_emails(sample_email_t
     assert notification_2.id not in notification_ids
 
 
+@pytest.mark.parametrize('search_term, expected_result_count', [
+    ('foobar', 1),
+    ('foo', 2),
+    ('bar', 2),
+    ('foo%', 1),
+    ('%%bar', 1),
+    ('%_', 1),
+    ('%', 2),
+    ('_', 1),
+    ('/', 1),
+    ('\\', 1),
+    ('baz\\baz', 1),
+    ('%foo', 0),
+    ('%_%', 0),
+    ('example.com', 5),
+])
+def test_dao_get_notifications_by_to_field_escapes(
+    sample_email_template,
+    search_term,
+    expected_result_count,
+):
+
+    for email_address in {
+        'foo%_@example.com',
+        '%%bar@example.com',
+        'foobar@example.com',
+        '/@example.com',
+        'baz\\baz@example.com',
+    }:
+        create_notification(
+            template=sample_email_template,
+            to_field=email_address,
+            normalised_to=email_address,
+        )
+
+    assert len(dao_get_notifications_by_to_field(
+        sample_email_template.service_id,
+        search_term,
+        notification_type='email',
+    )) == expected_result_count
+
+
 @pytest.mark.parametrize('search_term', [
     '001',
     '100',

tests/app/dao/test_api_key_dao.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytest
 from sqlalchemy.exc import IntegrityError
@@ -95,3 +95,21 @@ def test_save_api_key_should_not_create_new_service_history(sample_service):
     save_model_api_key(api_key)
 
     assert Service.get_history_model().query.count() == 1
+
+
+@pytest.mark.parametrize('days_old, expected_length', [(5, 1), (8, 0)])
+def test_should_not_return_revoked_api_keys_older_than_7_days(
+    sample_service,
+    days_old,
+    expected_length
+):
+    expired_api_key = ApiKey(**{'service': sample_service,
+                                'name': sample_service.name,
+                                'created_by': sample_service.created_by,
+                                'key_type': KEY_TYPE_NORMAL,
+                                'expiry_date': datetime.utcnow() - timedelta(days=days_old)})
+    save_model_api_key(expired_api_key)
+
+    all_api_keys = get_model_api_keys(service_id=sample_service.id)
+
+    assert len(all_api_keys) == expected_length

tests/app/dao/test_daily_sorted_letter_dao.py

@@ -20,7 +20,10 @@ def test_dao_get_daily_sorted_letter_by_billing_day(notify_db, notify_db_session
 
 def test_dao_create_or_update_daily_sorted_letter_creates_a_new_entry(notify_db, notify_db_session):
     billing_day = date(2018, 2, 1)
-    dsl = DailySortedLetter(billing_day=billing_day, unsorted_count=2, sorted_count=0)
+    dsl = DailySortedLetter(billing_day=billing_day,
+                            file_name="Notify-201802011234.rs.txt",
+                            unsorted_count=2,
+                            sorted_count=0)
     dao_create_or_update_daily_sorted_letter(dsl)
 
     daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(billing_day)
@@ -35,13 +38,19 @@ def test_dao_create_or_update_daily_sorted_letter_updates_an_existing_entry(
     notify_db,
     notify_db_session
 ):
-    create_daily_sorted_letter(unsorted_count=2, sorted_count=3)
+    create_daily_sorted_letter(billing_day=date(2018, 1, 18),
+                               file_name="Notify-20180118123.rs.txt",
+                               unsorted_count=2,
+                               sorted_count=3)
 
-    dsl = DailySortedLetter(billing_day=date(2018, 1, 18), unsorted_count=5, sorted_count=17)
+    dsl = DailySortedLetter(billing_day=date(2018, 1, 18),
+                            file_name="Notify-20180118123.rs.txt",
+                            unsorted_count=5,
+                            sorted_count=17)
     dao_create_or_update_daily_sorted_letter(dsl)
 
     daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(dsl.billing_day)
 
-    assert daily_sorted_letter.unsorted_count == 7
+    assert daily_sorted_letter.unsorted_count == 5
-    assert daily_sorted_letter.sorted_count == 20
+    assert daily_sorted_letter.sorted_count == 17
     assert daily_sorted_letter.updated_at

tests/app/db.py

@@ -509,9 +509,13 @@ def create_invited_org_user(organisation, invited_by, email_address='invite@exam
     return invited_org_user
 
 
-def create_daily_sorted_letter(billing_day=date(2018, 1, 18), unsorted_count=0, sorted_count=0):
+def create_daily_sorted_letter(billing_day=date(2018, 1, 18),
+                               file_name="Notify-20180118123.rs.txt",
+                               unsorted_count=0,
+                               sorted_count=0):
     daily_sorted_letter = DailySortedLetter(
         billing_day=billing_day,
+        file_name=file_name,
         unsorted_count=unsorted_count,
         sorted_count=sorted_count
     )

tests/app/letters/test_letter_utils.py

@@ -1,6 +1,26 @@
 import pytest
+from datetime import datetime
 
-from app.letters.utils import get_bucket_prefix_for_notification
+import boto3
+from flask import current_app
+from freezegun import freeze_time
+from moto import mock_s3
+
+from app.letters.utils import get_bucket_prefix_for_notification, get_letter_pdf_filename, get_letter_pdf
+from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEST, PRECOMPILED_TEMPLATE_NAME
+
+FROZEN_DATE_TIME = "2018-03-14 17:00:00"
+
+
+@pytest.fixture()
+@freeze_time(FROZEN_DATE_TIME)
+def sample_precompiled_letter_notification_using_test_key(sample_letter_notification):
+    sample_letter_notification.template.hidden = True
+    sample_letter_notification.template.name = PRECOMPILED_TEMPLATE_NAME
+    sample_letter_notification.key_type = KEY_TYPE_TEST
+    sample_letter_notification.reference = 'foo'
+    sample_letter_notification.created_at = datetime.utcnow()
+    return sample_letter_notification
 
 
 def test_get_bucket_prefix_for_notification_valid_notification(sample_notification):
@@ -13,6 +33,70 @@ def test_get_bucket_prefix_for_notification_valid_notification(sample_notificati
     ).upper()
 
 
+@freeze_time(FROZEN_DATE_TIME)
+def test_get_bucket_prefix_for_notification_precompiled_letter_using_test_key(
+    sample_precompiled_letter_notification_using_test_key
+):
+    bucket_prefix = get_bucket_prefix_for_notification(
+        sample_precompiled_letter_notification_using_test_key, is_test_letter=True)
+
+    assert bucket_prefix == 'NOTIFY.{}'.format(
+        sample_precompiled_letter_notification_using_test_key.reference).upper()
+
+
 def test_get_bucket_prefix_for_notification_invalid_notification():
     with pytest.raises(AttributeError):
         get_bucket_prefix_for_notification(None)
+
+
+@pytest.mark.parametrize('crown_flag,expected_crown_text', [
+    (True, 'C'),
+    (False, 'N'),
+])
+@freeze_time("2017-12-04 17:29:00")
+def test_get_letter_pdf_filename_returns_correct_filename(
+        notify_api, mocker, crown_flag, expected_crown_text):
+    filename = get_letter_pdf_filename(reference='foo', crown=crown_flag)
+
+    assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(expected_crown_text)
+
+
+@freeze_time("2017-12-04 17:29:00")
+def test_get_letter_pdf_filename_returns_correct_filename_for_test_letters(
+        notify_api, mocker):
+    filename = get_letter_pdf_filename(reference='foo', crown='C', is_test_letter=True)
+
+    assert filename == 'NOTIFY.FOO.D.2.C.C.20171204172900.PDF'
+
+
+@freeze_time("2017-12-04 17:31:00")
+def test_get_letter_pdf_filename_returns_tomorrows_filename(notify_api, mocker):
+    filename = get_letter_pdf_filename(reference='foo', crown=True)
+
+    assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF'
+
+
+@mock_s3
+@pytest.mark.parametrize('bucket_config_name,filename_format', [
+    ('TEST_LETTERS_BUCKET_NAME', 'NOTIFY.FOO.D.2.C.C.%Y%m%d%H%M%S.PDF'),
+    ('LETTERS_PDF_BUCKET_NAME', '%Y-%m-%d/NOTIFY.FOO.D.2.C.C.%Y%m%d%H%M%S.PDF')
+])
+@freeze_time(FROZEN_DATE_TIME)
+def test_get_letter_pdf_gets_pdf_from_correct_bucket(
+    sample_precompiled_letter_notification_using_test_key,
+    bucket_config_name,
+    filename_format
+):
+    if bucket_config_name == 'LETTERS_PDF_BUCKET_NAME':
+        sample_precompiled_letter_notification_using_test_key.key_type = KEY_TYPE_NORMAL
+
+    bucket_name = current_app.config[bucket_config_name]
+    filename = datetime.utcnow().strftime(filename_format)
+    conn = boto3.resource('s3', region_name='eu-west-1')
+    conn.create_bucket(Bucket=bucket_name)
+    s3 = boto3.client('s3', region_name='eu-west-1')
+    s3.put_object(Bucket=bucket_name, Key=filename, Body=b'pdf_content')
+
+    ret = get_letter_pdf(sample_precompiled_letter_notification_using_test_key)
+
+    assert ret == b'pdf_content'

tests/app/notifications/rest/test_callbacks.py

@@ -96,12 +96,12 @@ def test_dvla_rs_txt_file_callback_calls_update_letter_notifications_task(client
 def test_dvla_rsp_txt_file_callback_calls_update_letter_notifications_task(client, mocker):
     update_task = \
         mocker.patch('app.notifications.notifications_letter_callback.update_letter_notifications_statuses.apply_async')
-    data = _sample_sns_s3_callback('NOTIFY.20170823160812.RSP.TXT')
+    data = _sample_sns_s3_callback('NOTIFY-20170823160812-RSP.TXT')
     response = dvla_post(client, data)
 
     assert response.status_code == 200
     assert update_task.called
-    update_task.assert_called_with(['NOTIFY.20170823160812.RSP.TXT'], queue='notify-internal-tasks')
+    update_task.assert_called_with(['NOTIFY-20170823160812-RSP.TXT'], queue='notify-internal-tasks')
 
 
 def test_dvla_ack_calls_does_not_call_letter_notifications_task(client, mocker):

tests/app/user/test_rest.py

@@ -15,6 +15,7 @@ from app.models import (
 )
 from app.dao.permissions_dao import default_service_permissions
 from tests import create_authorization_header
+from tests.app.db import create_service, create_organisation, create_user
 
 
 def test_get_user_list(admin_request, sample_service):
@@ -36,28 +37,53 @@ def test_get_user_list(admin_request, sample_service):
     assert sorted(expected_permissions) == sorted(fetched['permissions'][str(sample_service.id)])
 
 
-def test_get_user(client, sample_service):
+def test_get_user(admin_request, sample_service, sample_organisation):
     """
     Tests GET endpoint '/<user_id>' to retrieve a single service.
     """
     sample_user = sample_service.users[0]
-    header = create_authorization_header()
+    sample_user.organisations = [sample_organisation]
-    resp = client.get(url_for('user.get_user',
+    json_resp = admin_request.get(
-                              user_id=sample_user.id),
+        'user.get_user',
-                      headers=[header])
+        user_id=sample_user.id
-    assert resp.status_code == 200
+    )
-    json_resp = json.loads(resp.get_data(as_text=True))
 
     expected_permissions = default_service_permissions
     fetched = json_resp['data']
 
-    assert str(sample_user.id) == fetched['id']
+    assert fetched['id'] == str(sample_user.id)
-    assert sample_user.name == fetched['name']
+    assert fetched['name'] == sample_user.name
-    assert sample_user.mobile_number == fetched['mobile_number']
+    assert fetched['mobile_number'] == sample_user.mobile_number
-    assert sample_user.email_address == fetched['email_address']
+    assert fetched['email_address'] == sample_user.email_address
-    assert sample_user.state == fetched['state']
+    assert fetched['state'] == sample_user.state
     assert fetched['auth_type'] == SMS_AUTH_TYPE
-    assert sorted(expected_permissions) == sorted(fetched['permissions'][str(sample_service.id)])
+    assert fetched['permissions'].keys() == {str(sample_service.id)}
+    assert fetched['services'] == [str(sample_service.id)]
+    assert fetched['organisations'] == [str(sample_organisation.id)]
+    assert sorted(fetched['permissions'][str(sample_service.id)]) == sorted(expected_permissions)
+
+
+def test_get_user_doesnt_return_inactive_services_and_orgs(admin_request, sample_service, sample_organisation):
+    """
+    Tests GET endpoint '/<user_id>' to retrieve a single service.
+    """
+    sample_service.active = False
+    sample_organisation.active = False
+
+    sample_user = sample_service.users[0]
+    sample_user.organisations = [sample_organisation]
+
+    json_resp = admin_request.get(
+        'user.get_user',
+        user_id=sample_user.id
+    )
+
+    fetched = json_resp['data']
+
+    assert fetched['id'] == str(sample_user.id)
+    assert fetched['services'] == []
+    assert fetched['organisations'] == []
+    assert fetched['permissions'] == {}
 
 
 def test_post_user(client, notify_db, notify_db_session):
@@ -580,3 +606,128 @@ def test_cannot_update_user_password_using_attributes_method(admin_request, samp
         _expected_status=400
     )
     assert resp['message']['_schema'] == ['Unknown field name password']
+
+
+def test_get_orgs_and_services_nests_services(admin_request, sample_user):
+    org1 = create_organisation(name='org1')
+    org2 = create_organisation(name='org2')
+    service1 = create_service(service_name='service1')
+    service2 = create_service(service_name='service2')
+    service3 = create_service(service_name='service3')
+
+    org1.services = [service1, service2]
+    org2.services = []
+
+    sample_user.organisations = [org1, org2]
+    sample_user.services = [service1, service2, service3]
+
+    resp = admin_request.get('user.get_organisations_and_services_for_user', user_id=sample_user.id)
+
+    assert resp == {
+        'organisations': [
+            {
+                'name': org1.name,
+                'id': str(org1.id),
+                'services': [
+                    {
+                        'name': service1.name,
+                        'id': str(service1.id)
+                    },
+                    {
+                        'name': service2.name,
+                        'id': str(service2.id)
+                    }
+                ]
+            },
+            {
+                'name': org2.name,
+                'id': str(org2.id),
+                'services': []
+            }
+        ],
+        'services_without_organisations': [
+            {
+                'name': service3.name,
+                'id': str(service3.id)
+            }
+        ]
+    }
+
+
+def test_get_orgs_and_services_only_returns_active(admin_request, sample_user):
+    org1 = create_organisation(name='org1', active=True)
+    org2 = create_organisation(name='org2', active=False)
+
+    # in an active org
+    service1 = create_service(service_name='service1', active=True)
+    service2 = create_service(service_name='service2', active=False)
+    # active but in an inactive org
+    service3 = create_service(service_name='service3', active=True)
+    # not in an org
+    service4 = create_service(service_name='service4', active=True)
+    service5 = create_service(service_name='service5', active=False)
+
+    org1.services = [service1, service2]
+    org2.services = [service3]
+
+    sample_user.organisations = [org1, org2]
+    sample_user.services = [service1, service2, service3, service4, service5]
+
+    resp = admin_request.get('user.get_organisations_and_services_for_user', user_id=sample_user.id)
+
+    assert resp == {
+        'organisations': [
+            {
+                'name': org1.name,
+                'id': str(org1.id),
+                'services': [
+                    {
+                        'name': service1.name,
+                        'id': str(service1.id)
+                    }
+                ]
+            }
+        ],
+        'services_without_organisations': [
+            {
+                'name': service4.name,
+                'id': str(service4.id)
+            }
+        ]
+    }
+
+
+def test_get_orgs_and_services_only_shows_users_orgs_and_services(admin_request, sample_user):
+    other_user = create_user(email='other@user.com')
+
+    org1 = create_organisation(name='org1')
+    org2 = create_organisation(name='org2')
+    service1 = create_service(service_name='service1')
+    service2 = create_service(service_name='service2')
+
+    org1.services = [service1]
+
+    sample_user.organisations = [org2]
+    sample_user.services = [service1]
+
+    other_user.organisations = [org1, org2]
+    other_user.services = [service1, service2]
+
+    resp = admin_request.get('user.get_organisations_and_services_for_user', user_id=sample_user.id)
+
+    assert resp == {
+        'organisations': [
+            {
+                'name': org2.name,
+                'id': str(org2.id),
+                'services': []
+            }
+        ],
+        # service1 belongs to org1, but the user doesn't know about org1
+        'services_without_organisations': [
+            {
+                'name': service1.name,
+                'id': str(service1.id)
+            }
+        ]
+    }

tests/app/v2/notifications/test_post_letter_notifications.py

@@ -1,4 +1,5 @@
 import uuid
+from unittest.mock import ANY
 
 from flask import json
 from flask import url_for
@@ -29,9 +30,13 @@ test_address = {
 }
 
 
-def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201):
+def letter_request(client, data, service_id, key_type=KEY_TYPE_NORMAL, _expected_status=201, precompiled=False):
+    if precompiled:
+        url = url_for('v2_notifications.post_precompiled_letter_notification')
+    else:
+        url = url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE)
     resp = client.post(
-        url_for('v2_notifications.post_notification', notification_type=LETTER_TYPE),
+        url,
         data=json.dumps(data),
         headers=[
             ('Content-Type', 'application/json'),
@@ -382,6 +387,30 @@ def test_post_letter_notification_is_delivered_if_in_trial_mode_and_using_test_k
     assert not fake_create_letter_task.called
 
 
+def test_post_letter_notification_is_delivered_and_has_pdf_uploaded_to_test_letters_bucket_using_test_key(
+    client,
+    notify_user,
+    mocker
+):
+    sample_letter_service = create_service(service_permissions=['letter', 'precompiled_letter'])
+    s3mock = mocker.patch('app.v2.notifications.post_notifications.upload_letter_pdf')
+    mocker.patch('app.v2.notifications.post_notifications.pdf_page_count', return_value=1)
+    data = {
+        "reference": "letter-reference",
+        "content": "bGV0dGVyLWNvbnRlbnQ="
+    }
+    letter_request(
+        client,
+        data=data,
+        service_id=str(sample_letter_service.id),
+        key_type=KEY_TYPE_TEST,
+        precompiled=True)
+
+    notification = Notification.query.one()
+    assert notification.status == NOTIFICATION_DELIVERED
+    s3mock.assert_called_once_with(ANY, b'letter-content', is_test_letter=True)
+
+
 def test_post_letter_notification_persists_notification_reply_to_text(
     client, notify_db_session, mocker
 ):