Merge branch 'master' into flexible-data-retention

Makefile

@@ -101,6 +101,25 @@ upload-paas-artifact:
 test: venv ## Run tests
 	./scripts/run_tests.sh
 
+.PHONY: freeze-requirements
+freeze-requirements:
+	rm -rf venv-freeze
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze
+
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }
+
 .PHONY: coverage
 coverage: venv ## Create coverage report
 	. venv/bin/activate && coveralls

README.md

@@ -82,3 +82,17 @@ Your aws credentials should be stored in a folder located at `~/.aws`. Follow [A
 ```
 
 Then visit [localhost:6012](http://localhost:6012)
+
+
+## Updating application dependencies
+
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with
+
+```
+make freeze-requirements
+```
+
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.

app/__init__.py

@@ -40,6 +40,7 @@ from werkzeug.local import LocalProxy
 from app import proxy_fix
 from app.config import configs
 from app.asset_fingerprinter import AssetFingerprinter
+from app.notify_client.models import Service
 from app.navigation import (
     CaseworkNavigation,
     HeaderNavigation,
@@ -94,8 +95,13 @@ billing_api_client = BillingAPIClient()
 complaint_api_client = ComplaintApiClient()
 platform_stats_api_client = PlatformStatsAPIClient()
 
+
 # The current service attached to the request stack.
-current_service = LocalProxy(partial(_lookup_req_object, 'service'))
+def _get_current_service():
+    return Service(_lookup_req_object('service'))
+
+
+current_service = LocalProxy(_get_current_service)
 
 # The current organisation attached to the request stack.
 current_organisation = LocalProxy(partial(_lookup_req_object, 'organisation'))

app/assets/javascripts/analytics.js

@@ -0,0 +1,56 @@
+let stripUUIDs = string => string.replace(
+  /[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}/g, '…'
+);
+
+(function(Modules) {
+  "use strict";
+
+  function sendEvent(category, action, label) {
+
+    if (!ga) return;
+    ga('send', 'event', category, action, label);
+
+  }
+
+  function sendVirtualPageView(path) {
+
+    if (!ga) return;
+    ga('send', 'pageview', stripUUIDs('/virtual' + path));
+
+  }
+
+  Modules.TrackError = function() {
+
+    this.start = component => sendEvent(
+      'Error',
+      $(component).data('error-type'),
+      $(component).data('error-label')
+    );
+
+  };
+
+  Modules.TrackEvent = function() {
+
+    this.start = component => sendEvent(
+      $(component).data('event-category'),
+      $(component).data('event-action'),
+      $(component).data('event-label')
+    );
+
+  };
+
+  Modules.TrackFormSubmission = function() {
+
+    this.start = component => {
+
+      $(component).on('submit', function() {
+
+        let formData = $('input[name!=csrf_token]', this).serialize();
+        sendVirtualPageView(window.location.pathname + '?' + formData);
+
+      });
+    };
+
+  };
+
+})(window.GOVUK.Modules);

app/assets/javascripts/errorTracking.js

@@ -1,22 +0,0 @@
-(function(Modules) {
-  "use strict";
-
-  Modules.TrackError = function() {
-
-    this.start = function(component) {
-
-      if (!ga) return;
-
-      ga(
-        'send',
-        'event',
-        'Error',
-        $(component).data('error-type'),
-        $(component).data('error-label')
-      );
-
-    };
-
-  };
-
-})(window.GOVUK.Modules);

app/assets/stylesheets/_grids.scss

@@ -81,6 +81,14 @@
   margin-bottom: $gutter * 2;
 }
 
+.left-gutter {
+  padding-left: $gutter;
+}
+
+.left-gutter-4-3 {
+  padding-left: $gutter * 4 / 3;
+}
+
 .align-with-heading {
   display: block;
   text-align: center;

app/domains.yml

@@ -183,6 +183,14 @@ innovateuk.gov.uk:
   owner: Innovate UK
   agreement_signed: false
   crown: false
+gamblingcommission.gov.uk:
+  owner: Gambling Commission
+  agreement_signed: true
+  crown: false
+geo.gov.uk:
+  owner: Government Equalities Office
+  agreement_signed: true
+  crown: true
   
 # Crown bodies who haven’t signed the MOU  
 charitycommission.gsi.gov.uk:
@@ -1104,7 +1112,7 @@ chesham.gov.uk:
 cheshireeast.gov.uk:
   owner: Cheshire East Council
   crown: false
-  agreement_signed: false
+  agreement_signed: true
 cheshire.gov.uk:
   owner: Cheshire West and Chester Council
   crown: false
@@ -1272,7 +1280,7 @@ copeland.gov.uk:
 corby.gov.uk:
   owner: Corby Borough Council
   crown: false
-  agreement_signed: false
+  agreement_signed: true
 cornwall-aonb.gov.uk:
   owner: Cornwall Council
   crown: false
@@ -1349,6 +1357,10 @@ crowboroughtowncouncil.gov.uk:
   owner: Crowborough Town Council
   crown: false
   agreement_signed: false
+croydon.gov.uk:
+  owner: Croydon Council
+  crown: false
+  agreement_signed: true
 cullomptontowncouncil.gov.uk:
   owner: Cullompton Town Council
   crown: false
@@ -4382,10 +4394,7 @@ surreyi.gov.uk:
   owner: Surrey County Council
   crown: false
   agreement_signed: false
-surreylocalgovernment.gov.uk:
+surreylocalgovernment.gov.uk: woking.gov.uk
-  owner: Woking Borough Council
-  crown: false
-  agreement_signed: false
 sussexsaferroads.gov.uk:
   owner: West Sussex County Council
   crown: false
@@ -5035,7 +5044,7 @@ wixford-pc.gov.uk:
 woking.gov.uk:
   owner: Woking Borough Council
   crown: false
-  agreement_signed: false
+  agreement_signed: true
 wokingham.gov.uk:
   owner: Wokingham Borough Council
   crown: false

app/email_domains.yml

@@ -34,3 +34,4 @@
 - tfgm.com
 - nationalgalleries.org
 - sch.uk
+- sepa.org.uk

app/main/__init__.py

@@ -26,6 +26,7 @@ from app.main.views import (  # noqa
     invites,
     feedback,
     providers,
+    find_users,
     platform_admin,
     letter_jobs,
     email_branding,

app/main/forms.py

@@ -837,14 +837,15 @@ class ChooseTemplateType(StripWhitespaceForm):
         ]
     )
 
-    def __init__(self, include_letters=False, *args, **kwargs):
+    def __init__(self, include_letters=False, include_copy=False, *args, **kwargs):
 
         super().__init__(*args, **kwargs)
 
         self.template_type.choices = filter(None, [
             ('email', 'Email'),
             ('sms', 'Text message'),
-            ('letter', 'Letter') if include_letters else None
+            ('letter', 'Letter') if include_letters else None,
+            ('copy-existing', 'Copy of an existing template') if include_copy else None,
         ])
 
 
@@ -853,6 +854,16 @@ class SearchTemplatesForm(StripWhitespaceForm):
     search = SearchField('Search by name')
 
 
+class SearchUsersByEmailForm(StripWhitespaceForm):
+
+    search = SearchField(
+        'Search by name or email address',
+        validators=[
+            DataRequired("You need to enter full or partial email address to search by.")
+        ],
+    )
+
+
 class SearchUsersForm(StripWhitespaceForm):
 
     search = SearchField('Search by name or email address')

app/main/views/api_keys.py

@@ -37,7 +37,7 @@ dummy_bearer_token = 'bearer_token_set'
 @user_has_permissions('manage_api_keys')
 def api_integration(service_id):
     callbacks_link = (
-        '.api_callbacks' if 'inbound_sms' in current_service['permissions']
+        '.api_callbacks' if current_service.has_permission('inbound_sms')
         else '.delivery_status_callback'
     )
     return render_template(
@@ -98,13 +98,13 @@ def create_api_key(service_id):
         (KEY_TYPE_TEST, 'Test – pretends to send messages'),
     ]
     disabled_options, option_hints = [], {}
-    if current_service['restricted']:
+    if current_service.trial_mode:
         disabled_options = [KEY_TYPE_NORMAL]
         option_hints[KEY_TYPE_NORMAL] = Markup(
             'Not available because your service is in '
             '<a href="{}#trial-mode">trial mode</a>'.format(url_for(".using_notify"))
         )
-    if 'letter' in current_service['permissions']:
+    if current_service.has_permission('letter'):
         option_hints[KEY_TYPE_TEAM] = 'Can’t be used to send letters'
     if form.validate_on_submit():
         if form.key_type.data in disabled_options:
@@ -148,14 +148,14 @@ def revoke_api_key(service_id, key_id):
 def get_apis():
     callback_api = None
     inbound_api = None
-    if current_service['service_callback_api']:
+    if current_service.service_callback_api:
         callback_api = service_api_client.get_service_callback_api(
-            current_service['id'],
+            current_service.id,
             current_service.get('service_callback_api')[0]
         )
-    if current_service['inbound_api']:
+    if current_service.inbound_api:
         inbound_api = service_api_client.get_service_inbound_api(
-            current_service['id'],
+            current_service.id,
             current_service.get('inbound_api')[0]
         )
 
@@ -172,7 +172,7 @@ def check_token_against_dummy_bearer(token):
 @main.route("/services/<service_id>/api/callbacks", methods=['GET'])
 @login_required
 def api_callbacks(service_id):
-    if 'inbound_sms' not in current_service['permissions']:
+    if not current_service.has_permission('inbound_sms'):
         return redirect(url_for('.delivery_status_callback', service_id=service_id))
 
     delivery_status_callback, received_text_messages_callback = get_apis()
@@ -186,10 +186,10 @@ def api_callbacks(service_id):
 
 
 def get_delivery_status_callback_details():
-    if current_service['service_callback_api']:
+    if current_service.service_callback_api:
         return service_api_client.get_service_callback_api(
-            current_service['id'],
+            current_service.id,
-            current_service.get('service_callback_api')[0]
+            current_service.service_callback_api[0]
         )
 
 
@@ -198,7 +198,7 @@ def get_delivery_status_callback_details():
 def delivery_status_callback(service_id):
     delivery_status_callback = get_delivery_status_callback_details()
     back_link = (
-        '.api_callbacks' if 'inbound_sms' in current_service['permissions']
+        '.api_callbacks' if current_service.has_permission('inbound_sms')
         else '.api_integration'
     )
 
@@ -248,9 +248,9 @@ def delivery_status_callback(service_id):
 
 
 def get_received_text_messages_callback():
-    if current_service['inbound_api']:
+    if current_service.inbound_api:
         return service_api_client.get_service_inbound_api(
-            current_service['id'],
+            current_service.id,
             current_service.get('inbound_api')[0]
         )
 
@@ -258,7 +258,7 @@ def get_received_text_messages_callback():
 @main.route("/services/<service_id>/api/callbacks/received-text-messages-callback", methods=['GET', 'POST'])
 @login_required
 def received_text_messages_callback(service_id):
-    if 'inbound_sms' not in current_service['permissions']:
+    if not current_service.has_permission('inbound_sms'):
         return redirect(url_for('.api_integration', service_id=service_id))
 
     received_text_messages_callback = get_received_text_messages_callback()

app/main/views/dashboard.py

@@ -141,37 +141,6 @@ def template_usage(service_id):
     )
 
 
-@main.route("/services/<service_id>/monthly-billing-usage")
-@login_required
-@user_has_permissions('manage_service')
-def monthly_billing_usage(service_id):
-    year, current_financial_year = requested_and_current_financial_year(request)
-
-    free_sms_allowance = billing_api_client.get_free_sms_fragment_limit_for_year(service_id, year)
-    units = billing_api_client.get_billable_units(service_id, year)
-    yearly_usage = billing_api_client.get_service_usage(service_id, year)
-
-    usage_template = 'views/usage.html'
-    if 'letter' in current_service['permissions']:
-        usage_template = 'views/usage-with-letters.html'
-    return render_template(
-        usage_template,
-        months=list(get_free_paid_breakdown_for_billable_units(
-            year,
-            free_sms_allowance,
-            units
-        )),
-        selected_year=year,
-        years=get_tuples_of_financial_years(
-            partial(url_for, '.monthly_billing_usage', service_id=service_id),
-            start=current_financial_year - 1,
-            end=current_financial_year + 1,
-        ),
-        **calculate_usage(yearly_usage,
-                          free_sms_allowance)
-    )
-
-
 @main.route("/services/<service_id>/usage")
 @login_required
 @user_has_permissions('manage_service')
@@ -183,7 +152,7 @@ def usage(service_id):
     yearly_usage = billing_api_client.get_service_usage_ft(service_id, year)
 
     usage_template = 'views/usage.html'
-    if 'letter' in current_service['permissions']:
+    if current_service.has_permission('letter'):
         usage_template = 'views/usage-with-letters.html'
     return render_template(
         usage_template,
@@ -269,7 +238,7 @@ def inbox_download(service_id):
 
 def get_inbox_partials(service_id):
     page = int(request.args.get('page', 1))
-    if 'inbound_sms' not in current_service['permissions']:
+    if not current_service.has_permission('inbound_sms'):
         abort(403)
 
     inbound_messages_data = service_api_client.get_most_recent_inbound_sms(service_id, page=page)
@@ -306,25 +275,22 @@ def aggregate_usage(template_statistics, sort_key='count'):
 
 
 def get_dashboard_partials(service_id):
-    # all but scheduled and cancelled
-    statuses_to_display = job_api_client.JOB_STATUSES - {'scheduled', 'cancelled'}
-
     template_statistics = aggregate_usage(
         template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7)
     )
 
-    scheduled_jobs = sorted(
+    scheduled_jobs, immediate_jobs = [], []
-        job_api_client.get_jobs(service_id, statuses=['scheduled'])['data'],
+    if job_api_client.has_jobs(service_id):
-        key=lambda job: job['scheduled_for']
+        scheduled_jobs = job_api_client.get_scheduled_jobs(service_id)
-    )
+        immediate_jobs = [
-    immediate_jobs = [
+            add_rate_to_job(job)
-        add_rate_to_job(job)
+            for job in job_api_client.get_immediate_jobs(service_id)
-        for job in job_api_client.get_jobs(service_id, limit_days=7, statuses=statuses_to_display)['data']
+        ]
-    ]
+
-    stats = service_api_client.get_service_statistics(service_id)
+    stats = service_api_client.get_service_statistics(service_id, today_only=False)
     column_width, max_notifiction_count = get_column_properties(
         number_of_columns=(
-            3 if 'letter' in current_service['permissions'] else 2
+            3 if current_service.has_permission('letter') else 2
         )
     )
     dashboard_totals = get_dashboard_totals(stats),
@@ -344,7 +310,7 @@ def get_dashboard_partials(service_id):
             'views/dashboard/_inbox.html',
             inbound_sms_summary=(
                 service_api_client.get_inbound_sms_summary(service_id)
-                if 'inbound_sms' in current_service['permissions'] else None
+                if current_service.has_permission('inbound_sms') else None
             ),
         ),
         'totals': render_template(

app/main/views/feedback.py

@@ -104,8 +104,8 @@ def feedback(ticket_type):
         user_name = form.name.data or None
         if current_service:
             service_string = 'Service: "{name}"\n{url}\n'.format(
-                name=current_service['name'],
+                name=current_service.name,
-                url=url_for('main.service_dashboard', service_id=current_service['id'], _external=True)
+                url=url_for('main.service_dashboard', service_id=current_service.id, _external=True)
             )
         else:
             service_string = ''

app/main/views/find_users.py

@@ -0,0 +1,38 @@
+from flask import render_template, request
+from flask_login import login_required
+
+from app import user_api_client
+from app.main import main
+from app.main.forms import SearchUsersByEmailForm
+from app.utils import user_is_platform_admin
+
+
+@main.route("/find-users-by-email", methods=['GET', 'POST'])
+@login_required
+@user_is_platform_admin
+def find_users_by_email():
+    form = SearchUsersByEmailForm()
+    users_found = None
+    status = 200
+    if form.validate_on_submit():
+        users_found = user_api_client.find_users_by_full_or_partial_email(form.search.data)['data']
+    elif request.method == 'POST':
+        status = 400
+    return render_template(
+        'views/find-users/find-users-by-email.html',
+        form=form,
+        users_found=users_found
+    ), status
+
+
+@main.route("/users/<user_id>", methods=['GET'])
+@login_required
+@user_is_platform_admin
+def user_information(user_id):
+    user = user_api_client.get_user(user_id)
+    services = user_api_client.get_services_for_user(user)
+    return render_template(
+        'views/find-users/user-information.html',
+        user=user,
+        services=services,
+    )

app/main/views/jobs.py

@@ -39,12 +39,10 @@ from app.utils import (
 
 @main.route("/services/<service_id>/jobs")
 @login_required
-@user_has_permissions('view_activity')
+@user_has_permissions('view_activity', 'send_messages')
 def view_jobs(service_id):
     page = int(request.args.get('page', 1))
-    # all but scheduled and cancelled
+    jobs_response = job_api_client.get_page_of_jobs(service_id, page=page)
-    statuses_to_display = job_api_client.JOB_STATUSES - {'scheduled', 'cancelled'}
-    jobs_response = job_api_client.get_jobs(service_id, statuses=statuses_to_display, page=page)
     jobs = [
         add_rate_to_job(job) for job in jobs_response['data']
     ]
@@ -56,18 +54,27 @@ def view_jobs(service_id):
     if jobs_response['links'].get('next', None):
         next_page = generate_next_dict('main.view_jobs', service_id, page)
 
+    scheduled_jobs = ''
+    if not current_user.has_permissions('view_activity') and page == 1:
+        scheduled_jobs = render_template(
+            'views/dashboard/_upcoming.html',
+            scheduled_jobs=job_api_client.get_scheduled_jobs(service_id),
+            hide_heading=True,
+        )
+
     return render_template(
         'views/jobs/jobs.html',
         jobs=jobs,
         page=page,
         prev_page=prev_page,
         next_page=next_page,
+        scheduled_jobs=scheduled_jobs,
     )
 
 
 @main.route("/services/<service_id>/jobs/<job_id>")
 @login_required
-@user_has_permissions('view_activity')
+@user_has_permissions('view_activity', 'send_messages')
 def view_job(service_id, job_id):
     job = job_api_client.get_job(service_id, job_id)['data']
     if job['job_status'] == 'cancelled':
@@ -149,7 +156,7 @@ def cancel_job(service_id, job_id):
 
 
 @main.route("/services/<service_id>/jobs/<job_id>.json")
-@user_has_permissions('view_activity')
+@user_has_permissions('view_activity', 'send_messages')
 def view_job_updates(service_id, job_id):
 
     job = job_api_client.get_job(service_id, job_id)['data']
@@ -157,7 +164,7 @@ def view_job_updates(service_id, job_id):
     return jsonify(**get_job_partials(
         job,
         service_api_client.get_service_template(
-            service_id=current_service['id'],
+            service_id=current_service.id,
             template_id=job['template'],
             version=job['template_version']
         )['data'],
@@ -179,7 +186,7 @@ def view_notifications(service_id, message_type=None):
         search_form=SearchNotificationsForm(to=request.form.get('to', '')),
         download_link=url_for(
             '.download_notifications_csv',
-            service_id=current_service['id'],
+            service_id=current_service.id,
             message_type=message_type,
             status=request.args.get('status')
         )
@@ -244,7 +251,7 @@ def get_notifications(service_id, message_type, status_override=None):
     if message_type:
         download_link = url_for(
             '.view_notifications_csv',
-            service_id=current_service['id'],
+            service_id=current_service.id,
             message_type=message_type,
             status=request.args.get('status')
         )
@@ -383,7 +390,7 @@ def get_job_partials(job, template):
             percentage_complete=(job['notifications_requested'] / job['notification_count'] * 100),
             download_link=url_for(
                 '.view_job_csv',
-                service_id=current_service['id'],
+                service_id=current_service.id,
                 job_id=job['id'],
                 status=request.args.get('status')
             ),

app/main/views/manage_users.py

@@ -48,14 +48,14 @@ def manage_users(service_id):
 @user_has_permissions('manage_service')
 def invite_user(service_id):
 
-    if 'caseworking' in current_service['permissions']:
+    if current_service.has_permission('caseworking'):
         form = CaseworkingInviteUserForm
     else:
         form = AdminInviteUserForm
 
     form = form(invalid_email_address=current_user.email_address)
 
-    service_has_email_auth = 'email_auth' in current_service['permissions']
+    service_has_email_auth = current_service.has_permission('email_auth')
     if not service_has_email_auth:
         form.login_authentication.data = 'sms_auth'
 
@@ -83,13 +83,13 @@ def invite_user(service_id):
 @login_required
 @user_has_permissions('manage_service')
 def edit_user_permissions(service_id, user_id):
-    service_has_email_auth = 'email_auth' in current_service['permissions']
+    service_has_email_auth = current_service.has_permission('email_auth')
     # TODO we should probably using the service id here in the get user
     # call as well. eg. /user/<user_id>?&service=service_id
     user = user_api_client.get_user(user_id)
     user_has_no_mobile_number = user.mobile_number is None
 
-    if 'caseworking' in current_service['permissions']:
+    if current_service.has_permission('caseworking'):
         form = partial(
             CaseworkingPermissionsForm,
             user_type='admin' if user.has_permission_for_service(service_id, 'view_activity') else 'caseworker',

app/main/views/notifications.py

@@ -91,7 +91,7 @@ def view_notification(service_id, notification_id):
         help=get_help_argument(),
         estimated_letter_delivery_date=get_letter_timings(notification['created_at']).earliest_delivery,
         notification_id=notification['id'],
-        can_receive_inbound=('inbound_sms' in current_service['permissions']),
+        can_receive_inbound=(current_service.has_permission('inbound_sms')),
         is_precompiled_letter=notification['template']['is_precompiled_letter']
     )
 
@@ -188,6 +188,6 @@ def download_notifications_csv(service_id):
             'Content-Disposition': 'inline; filename="{} - {} - {} report.csv"'.format(
                 format_date_numeric(datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")),
                 filter_args['message_type'][0],
-                current_service['name'])
+                current_service.name)
         }
     )

app/main/views/platform_admin.py

@@ -138,6 +138,14 @@ def make_columns(global_stats, complaints_number):
 @user_is_platform_admin
 def platform_admin_services():
     form = DateFilterForm(request.args)
+    if all((
+        request.args.get('include_from_test_key') is None,
+        request.args.get('start_date') is None,
+        request.args.get('end_date') is None,
+    )):
+        # Default to True if the user hasn’t done any filtering,
+        # otherwise respect their choice
+        form.include_from_test_key.data = True
     api_args = {'detailed': True,
                 'only_active': False,    # specifically DO get inactive services
                 'include_from_test_key': form.include_from_test_key.data,

app/main/views/send.py

@@ -116,7 +116,7 @@ def send_messages(service_id, template_id):
     session['sender_id'] = None
     db_template = service_api_client.get_service_template(service_id, template_id)['data']
 
-    if email_or_sms_not_enabled(db_template['template_type'], current_service['permissions']):
+    if email_or_sms_not_enabled(db_template['template_type'], current_service.permissions):
         return redirect(url_for(
             '.action_blocked',
             service_id=service_id,
@@ -294,7 +294,7 @@ def send_test(service_id, template_id):
     if db_template['template_type'] == 'letter':
         session['sender_id'] = None
 
-    if email_or_sms_not_enabled(db_template['template_type'], current_service['permissions']):
+    if email_or_sms_not_enabled(db_template['template_type'], current_service.permissions):
         return redirect(url_for(
             '.action_blocked',
             service_id=service_id,
@@ -401,7 +401,7 @@ def send_test_step(service_id, template_id, step_index):
         dict_to_populate_from=get_normalised_placeholders_from_session(),
         template_type=template.template_type,
         optional_placeholder=optional_placeholder,
-        allow_international_phone_numbers='international_sms' in current_service['permissions'],
+        allow_international_phone_numbers=current_service.has_permission('international_sms'),
     )
 
     if form.validate_on_submit():
@@ -510,7 +510,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
 
     users = user_api_client.get_users_for_service(service_id=service_id)
 
-    statistics = service_api_client.get_service_statistics_for_today(service_id)
+    statistics = service_api_client.get_service_statistics(service_id, today_only=True)
     remaining_messages = (current_service['message_limit'] - sum(stat['requested'] for stat in statistics.values()))
 
     contents = s3download(service_id, upload_id)
@@ -549,9 +549,9 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
         max_errors_shown=50,
         whitelist=itertools.chain.from_iterable(
             [user.name, user.mobile_number, user.email_address] for user in users
-        ) if current_service['restricted'] else None,
+        ) if current_service.trial_mode else None,
         remaining_messages=remaining_messages,
-        international_sms='international_sms' in current_service['permissions'],
+        international_sms=current_service.has_permission('international_sms'),
     )
 
     if request.args.get('from_test'):
@@ -585,7 +585,7 @@ def _check_messages(service_id, template_id, upload_id, preview_row, letters_as_
         back_link=back_link,
         help=get_help_argument(),
         trying_to_send_letters_in_trial_mode=all((
-            current_service['restricted'],
+            current_service.trial_mode,
             template.template_type == 'letter',
             not request.args.get('from_test'),
         )),
@@ -873,7 +873,7 @@ def send_notification(service_id, template_id):
         )
     except HTTPError as exception:
         current_app.logger.info('Service {} could not send notification: "{}"'.format(
-            current_service['id'],
+            current_service.id,
             exception.message
         ))
         return _check_notification(service_id, template_id, exception)

app/main/views/service_settings.py

@@ -64,8 +64,8 @@ def service_settings(service_id):
     letter_branding_organisations = email_branding_client.get_letter_email_branding()
     organisation = organisations_client.get_service_organisation(service_id).get('name', None)
 
-    if current_service['email_branding']:
+    if current_service.email_branding:
-        email_branding = email_branding_client.get_email_branding(current_service['email_branding'])['email_branding']
+        email_branding = email_branding_client.get_email_branding(current_service.email_branding)['email_branding']
     else:
         email_branding = None
 
@@ -96,7 +96,7 @@ def service_settings(service_id):
         letter_branding=letter_branding_organisations.get(
             current_service.get('dvla_organisation', '001')
         ),
-        can_receive_inbound=('inbound_sms' in current_service['permissions']),
+        can_receive_inbound=(current_service.has_permission('inbound_sms')),
         inbound_number=disp_inbound_number,
         default_reply_to_email_address=default_reply_to_email_address,
         reply_to_email_address_count=reply_to_email_address_count,
@@ -105,7 +105,7 @@ def service_settings(service_id):
         default_sms_sender=default_sms_sender,
         sms_sender_count=sms_sender_count,
         free_sms_fragment_limit=free_sms_fragment_limit,
-        prefix_sms=current_service['prefix_sms'],
+        prefix_sms=current_service.prefix_sms,
         organisation=organisation,
         data_retention=data_retention,
     )
@@ -118,11 +118,11 @@ def service_name_change(service_id):
     form = RenameServiceForm()
 
     if request.method == 'GET':
-        form.name.data = current_service['name']
+        form.name.data = current_service.name
 
     if form.validate_on_submit():
 
-        if form.name.data == current_service['name']:
+        if form.name.data == current_service.name:
             return redirect(url_for('.service_settings', service_id=service_id))
 
         unique_name = service_api_client.is_service_name_unique(service_id, form.name.data, email_safe(form.name.data))
@@ -153,7 +153,7 @@ def service_name_change_confirm(service_id):
     if form.validate_on_submit():
         try:
             service_api_client.update_service(
-                current_service['id'],
+                current_service.id,
                 name=session['service_name_change'],
                 email_from=email_safe(session['service_name_change'])
             )
@@ -205,7 +205,7 @@ def submit_request_to_go_live(service_id):
 
     if form.validate_on_submit():
         zendesk_client.create_ticket(
-            subject='Request to go live - {}'.format(current_service['name']),
+            subject='Request to go live - {}'.format(current_service.name),
             message=(
                 'Service: {}\n'
                 '{}\n'
@@ -216,9 +216,9 @@ def submit_request_to_go_live(service_id):
                 '\nPeak volume: {}'
                 '\nFeatures: {}'
             ).format(
-                current_service['name'],
+                current_service.name,
-                url_for('main.service_dashboard', service_id=current_service['id'], _external=True),
+                url_for('main.service_dashboard', service_id=current_service.id, _external=True),
-                current_service['organisation_type'],
+                current_service.organisation_type,
                 AgreementInfo.from_current_user().as_human_readable,
                 formatted_list(filter(None, (
                     'email' if form.channel_email.data else None,
@@ -250,11 +250,11 @@ def submit_request_to_go_live(service_id):
 @user_is_platform_admin
 def service_switch_live(service_id):
     service_api_client.update_service(
-        current_service['id'],
+        current_service.id,
         # TODO This limit should be set depending on the agreement signed by
         # with Notify.
-        message_limit=250000 if current_service['restricted'] else 50,
+        message_limit=250000 if current_service.trial_mode else 50,
-        restricted=(not current_service['restricted'])
+        restricted=(not current_service.trial_mode)
     )
     return redirect(url_for('.service_settings', service_id=service_id))
 
@@ -265,7 +265,7 @@ def service_switch_live(service_id):
 def service_switch_research_mode(service_id):
     service_api_client.update_service_with_properties(
         service_id,
-        {"research_mode": not current_service['research_mode']}
+        {"research_mode": not current_service.research_mode}
     )
     return redirect(url_for('.service_settings', service_id=service_id))
 
@@ -275,14 +275,14 @@ def switch_service_permissions(service_id, permission, sms_sender=None):
     force_service_permission(
         service_id,
         permission,
-        on=permission not in current_service['permissions'],
+        on=permission not in current_service.permissions,
         sms_sender=sms_sender
     )
 
 
 def force_service_permission(service_id, permission, on=False, sms_sender=None):
 
-    permissions, permission = set(current_service['permissions']), {permission}
+    permissions, permission = set(current_service.permissions), {permission}
 
     update_service_permissions(
         service_id,
@@ -293,9 +293,7 @@ def force_service_permission(service_id, permission, on=False, sms_sender=None):
 
 def update_service_permissions(service_id, permissions, sms_sender=None):
 
-    current_service['permissions'] = list(permissions)
+    data = {'permissions': list(permissions)}
-
-    data = {'permissions': current_service['permissions']}
 
     if sms_sender:
         data['sms_sender'] = sms_sender
@@ -343,13 +341,13 @@ def service_switch_can_upload_document(service_id):
 
     # If turning the permission off, or turning it on and the service already has a contact_link,
     # don't show the form to add the link
-    if 'upload_document' in current_service['permissions'] or current_service.get('contact_link'):
+    if current_service.has_permission('upload_document') or current_service.get('contact_link'):
         switch_service_permissions(service_id, 'upload_document')
         return redirect(url_for('.service_settings', service_id=service_id))
 
     if form.validate_on_submit():
         service_api_client.update_service(
-            current_service['id'],
+            current_service.id,
             contact_link=form.url.data
         )
         switch_service_permissions(service_id, 'upload_document')
@@ -406,10 +404,10 @@ def service_set_contact_link(service_id):
 
     if form.validate_on_submit():
         service_api_client.update_service(
-            current_service['id'],
+            current_service.id,
             contact_link=form.url.data
         )
-        return redirect(url_for('.service_settings', service_id=current_service['id']))
+        return redirect(url_for('.service_settings', service_id=current_service.id))
 
     return render_template('views/service-settings/contact_link.html', form=form)
 
@@ -449,7 +447,7 @@ def service_add_email_reply_to(service_id):
     first_email_address = reply_to_email_address_count == 0
     if form.validate_on_submit():
         service_api_client.add_reply_to_email_address(
-            current_service['id'],
+            current_service.id,
             email_address=form.email_address.data,
             is_default=first_email_address if first_email_address else form.is_default.data
         )
@@ -480,7 +478,7 @@ def service_edit_email_reply_to(service_id, reply_to_email_id):
         form.is_default.data = reply_to_email_address['is_default']
     if form.validate_on_submit():
         service_api_client.update_reply_to_email_address(
-            current_service['id'],
+            current_service.id,
             reply_to_email_id=reply_to_email_id,
             email_address=form.email_address.data,
             is_default=True if reply_to_email_address['is_default'] else form.is_default.data
@@ -499,7 +497,7 @@ def service_edit_email_reply_to(service_id, reply_to_email_id):
 @user_has_permissions('manage_service')
 def service_delete_email_reply_to(service_id, reply_to_email_id):
     service_api_client.delete_reply_to_email_address(
-        service_id=current_service['id'],
+        service_id=current_service.id,
         reply_to_email_id=reply_to_email_id,
     )
     return redirect(url_for('.service_email_reply_to', service_id=service_id))
@@ -520,12 +518,12 @@ def service_set_inbound_number(service_id):
     )
     if form.validate_on_submit():
         service_api_client.add_sms_sender(
-            current_service['id'],
+            current_service.id,
             sms_sender=form.inbound_number.data,
             is_default=True,
             inbound_number_id=form.inbound_number.data
         )
-        switch_service_permissions(current_service['id'], 'inbound_sms')
+        switch_service_permissions(current_service.id, 'inbound_sms')
         return redirect(url_for('.service_settings', service_id=service_id))
     return render_template(
         'views/service-settings/set-inbound-number.html',
@@ -550,14 +548,14 @@ def service_set_sms(service_id):
 def service_set_sms_prefix(service_id):
 
     form = SMSPrefixForm(enabled=(
-        'on' if current_service['prefix_sms'] else 'off'
+        'on' if current_service.prefix_sms else 'off'
     ))
 
-    form.enabled.label.text = 'Start all text messages with ‘{}:’'.format(current_service['name'])
+    form.enabled.label.text = 'Start all text messages with ‘{}:’'.format(current_service.name)
 
     if form.validate_on_submit():
         service_api_client.update_service(
-            current_service['id'],
+            current_service.id,
             prefix_sms=(form.enabled.data == 'on')
         )
         return redirect(url_for('.service_settings', service_id=service_id))
@@ -573,7 +571,7 @@ def service_set_sms_prefix(service_id):
 @user_has_permissions('manage_service')
 def service_set_international_sms(service_id):
     form = InternationalSMSForm(
-        enabled='on' if 'international_sms' in current_service['permissions'] else 'off'
+        enabled='on' if current_service.has_permission('international_sms') else 'off'
     )
     if form.validate_on_submit():
         force_service_permission(
@@ -606,7 +604,7 @@ def service_set_inbound_sms(service_id):
 @user_has_permissions('manage_service')
 def service_set_letters(service_id):
     form = ServiceSwitchLettersForm(
-        enabled='on' if 'letter' in current_service['permissions'] else 'off'
+        enabled='on' if current_service.has_permission('letter') else 'off'
     )
     if form.validate_on_submit():
         force_service_permission(
@@ -644,7 +642,7 @@ def service_set_basic_view(service_id):
         abort(403)
 
     form = ServiceBasicViewForm(
-        enabled='caseworking' in current_service['permissions']
+        enabled=current_service.has_permission('caseworking')
     )
     if form.validate_on_submit():
         force_service_permission(
@@ -688,7 +686,7 @@ def service_add_letter_contact(service_id):
     first_contact_block = letter_contact_blocks_count == 0
     if form.validate_on_submit():
         service_api_client.add_letter_contact(
-            current_service['id'],
+            current_service.id,
             contact_block=form.letter_contact_block.data.replace('\r', '') or None,
             is_default=first_contact_block if first_contact_block else form.is_default.data
         )
@@ -713,7 +711,7 @@ def service_edit_letter_contact(service_id, letter_contact_id):
         form.is_default.data = letter_contact_block['is_default']
     if form.validate_on_submit():
         service_api_client.update_letter_contact(
-            current_service['id'],
+            current_service.id,
             letter_contact_id=letter_contact_id,
             contact_block=form.letter_contact_block.data.replace('\r', '') or None,
             is_default=True if letter_contact_block['is_default'] else form.is_default.data
@@ -759,7 +757,7 @@ def service_add_sms_sender(service_id):
     first_sms_sender = sms_sender_count == 0
     if form.validate_on_submit():
         service_api_client.add_sms_sender(
-            current_service['id'],
+            current_service.id,
             sms_sender=form.sms_sender.data.replace('\r', '') or None,
             is_default=first_sms_sender if first_sms_sender else form.is_default.data
         )
@@ -792,7 +790,7 @@ def service_edit_sms_sender(service_id, sms_sender_id):
 
     if form.validate_on_submit():
         service_api_client.update_sms_sender(
-            current_service['id'],
+            current_service.id,
             sms_sender_id=sms_sender_id,
             sms_sender=sms_sender['sms_sender'] if is_inbound_number else form.sms_sender.data.replace('\r', ''),
             is_default=True if sms_sender['is_default'] else form.is_default.data
@@ -818,7 +816,7 @@ def service_edit_sms_sender(service_id, sms_sender_id):
 @user_has_permissions('manage_service')
 def service_delete_sms_sender(service_id, sms_sender_id):
     service_api_client.delete_sms_sender(
-        service_id=current_service['id'],
+        service_id=current_service.id,
         sms_sender_id=sms_sender_id,
     )
     return redirect(url_for('.service_sms_senders', service_id=service_id))
@@ -829,13 +827,13 @@ def service_delete_sms_sender(service_id, sms_sender_id):
 @user_has_permissions('manage_service')
 def service_set_letter_contact_block(service_id):
 
-    if 'letter' not in current_service['permissions']:
+    if not current_service.has_permission('letter'):
         abort(403)
 
-    form = ServiceLetterContactBlockForm(letter_contact_block=current_service['letter_contact_block'])
+    form = ServiceLetterContactBlockForm(letter_contact_block=current_service.letter_contact_block)
     if form.validate_on_submit():
         service_api_client.update_service(
-            current_service['id'],
+            current_service.id,
             letter_contact_block=form.letter_contact_block.data.replace('\r', '') or None
         )
         if request.args.get('from_template'):
@@ -912,7 +910,7 @@ def service_set_email_branding(service_id):
         )
         return redirect(url_for('.service_settings', service_id=service_id))
 
-    form.branding_style.data = current_service['email_branding'] or 'None'
+    form.branding_style.data = current_service.email_branding or 'None'
 
     return render_template(
         'views/service-settings/set-email-branding.html',
@@ -977,12 +975,12 @@ def link_service_to_organisation(service_id):
 def branding_request(service_id):
 
     form = BrandingOptionsEmail(
-        options=current_service['branding']
+        options=current_service.branding
     )
 
     if form.validate_on_submit():
         zendesk_client.create_ticket(
-            subject='Email branding request - {}'.format(current_service['name']),
+            subject='Email branding request - {}'.format(current_service.name),
             message=(
                 'Organisation: {}\n'
                 'Service: {}\n'
@@ -991,8 +989,8 @@ def branding_request(service_id):
                 '\nBranding requested: {}'
             ).format(
                 AgreementInfo.from_current_user().as_info_for_branding_request,
-                current_service['name'],
+                current_service.name,
-                url_for('main.service_dashboard', service_id=current_service['id'], _external=True),
+                url_for('main.service_dashboard', service_id=current_service.id, _external=True),
                 branding_options_dict[form.options.data],
             ),
             ticket_type=zendesk_client.TYPE_QUESTION,

app/main/views/templates.py

@@ -9,7 +9,12 @@ from notifications_python_client.errors import HTTPError
 from notifications_utils.formatters import nl2br
 from notifications_utils.recipients import first_column_headings
 
-from app import current_service, service_api_client, template_statistics_client
+from app import (
+    current_service,
+    service_api_client,
+    template_statistics_client,
+    user_api_client,
+)
 from app.main import main
 from app.main.forms import (
     ChooseTemplateType,
@@ -103,7 +108,7 @@ def choose_template(service_id, template_type='all'):
     templates = service_api_client.get_service_templates(service_id)['data']
 
     letters_available = (
-        'letter' in current_service['permissions'] and
+        current_service.has_permission('letter') and
         current_user.has_permissions('view_activity')
     )
 
@@ -123,7 +128,7 @@ def choose_template(service_id, template_type='all'):
     }) > 1
 
     template_nav_items = [
-        (label, key, url_for('.choose_template', service_id=current_service['id'], template_type=key), '')
+        (label, key, url_for('.choose_template', service_id=current_service.id, template_type=key), '')
         for label, key in filter(None, [
             ('All', 'all'),
             ('Text message', 'sms'),
@@ -140,14 +145,8 @@ def choose_template(service_id, template_type='all'):
         )
     ]
 
-    if current_user.has_permissions('view_activity'):
-        page_title = 'Templates'
-    else:
-        page_title = 'Choose a template'
-
     return render_template(
         'views/templates/choose.html',
-        page_title=page_title,
         templates=templates_on_page,
         show_search_box=(len(templates_on_page) > 7),
         show_template_nav=has_multiple_template_types and (len(templates) > 2),
@@ -208,11 +207,21 @@ def view_template_version_preview(service_id, template_id, version, filetype):
 def add_template_by_type(service_id):
 
     form = ChooseTemplateType(
-        include_letters='letter' in current_service['permissions']
+        include_letters=current_service.has_permission('letter'),
+        include_copy=any((
+            service_api_client.count_service_templates(service_id) > 0,
+            len(user_api_client.get_service_ids_for_user(current_user)) > 1,
+        )),
     )
 
     if form.validate_on_submit():
 
+        if form.template_type.data == 'copy-existing':
+            return redirect(url_for(
+                '.choose_template_to_copy',
+                service_id=service_id,
+            ))
+
         if form.template_type.data == 'letter':
             blank_letter = service_api_client.create_service_template(
                 'Untitled',
@@ -228,7 +237,7 @@ def add_template_by_type(service_id):
                 template_id=blank_letter['data']['id'],
             ))
 
-        if email_or_sms_not_enabled(form.template_type.data, current_service['permissions']):
+        if email_or_sms_not_enabled(form.template_type.data, current_service.permissions):
             return redirect(url_for(
                 '.action_blocked',
                 service_id=service_id,
@@ -246,6 +255,55 @@ def add_template_by_type(service_id):
     return render_template('views/templates/add.html', form=form)
 
 
+@main.route("/services/<service_id>/templates/copy")
+@login_required
+@user_has_permissions('manage_templates')
+def choose_template_to_copy(service_id):
+    return render_template(
+        'views/templates/copy.html',
+        services=[{
+            'name': service['name'],
+            'id': service['id'],
+            'templates': [
+                template for template in
+                service_api_client.get_service_templates(service['id'])['data']
+                if template['template_type'] in current_service['permissions']
+            ],
+        } for service in user_api_client.get_services_for_user(current_user)],
+    )
+
+
+@main.route("/services/<service_id>/templates/copy/<uuid:template_id>", methods=['GET', 'POST'])
+@login_required
+@user_has_permissions('manage_templates')
+def copy_template(service_id, template_id):
+
+    if not user_api_client.user_belongs_to_service(
+        current_user, request.args.get('from_service')
+    ):
+        abort(403)
+
+    template = service_api_client.get_service_template(
+        request.args.get('from_service'),
+        str(template_id),
+    )['data']
+
+    if request.method == 'POST':
+        return add_service_template(service_id, template['template_type'])
+
+    template['template_content'] = template['content']
+    template['name'] = 'Copy of ‘{}’'.format(template['name'])
+    form = form_objects[template['template_type']](**template)
+
+    return render_template(
+        'views/edit-{}-template.html'.format(template['template_type']),
+        form=form,
+        template_type=template['template_type'],
+        heading_action='Add',
+        services=user_api_client.get_service_ids_for_user(current_user),
+    )
+
+
 @main.route("/services/<service_id>/templates/action-blocked/<notification_type>/<return_to>/<template_id>")
 @login_required
 @user_has_permissions('manage_templates')
@@ -271,7 +329,7 @@ def add_service_template(service_id, template_type):
 
     if template_type not in ['sms', 'email', 'letter']:
         abort(404)
-    if 'letter' not in current_service['permissions'] and template_type == 'letter':
+    if not current_service.has_permission('letter') and template_type == 'letter':
         abort(403)
 
     form = form_objects[template_type]()
@@ -301,7 +359,7 @@ def add_service_template(service_id, template_type):
                 url_for('.view_template', service_id=service_id, template_id=new_template['data']['id'])
             )
 
-    if email_or_sms_not_enabled(template_type, current_service['permissions']):
+    if email_or_sms_not_enabled(template_type, current_service.permissions):
         return redirect(url_for(
             '.action_blocked',
             service_id=service_id,
@@ -390,7 +448,7 @@ def edit_service_template(service_id, template_id):
 
     db_template = service_api_client.get_service_template(service_id, template_id)['data']
 
-    if email_or_sms_not_enabled(db_template['template_type'], current_service['permissions']):
+    if email_or_sms_not_enabled(db_template['template_type'], current_service.permissions):
         return redirect(url_for(
             '.action_blocked',
             service_id=service_id,
@@ -404,7 +462,7 @@ def edit_service_template(service_id, template_id):
             form=form,
             template_id=template_id,
             template_type=template['template_type'],
-            heading_action='Edit'
+            heading_action='Edit',
         )
 
 
@@ -618,5 +676,5 @@ def get_human_readable_delta(from_time, until_time):
 def should_show_template(template_type):
     return (
         template_type != 'letter' or
-        'letter' in current_service['permissions']
+        current_service.has_permission('letter')
     )

app/navigation.py

@@ -76,15 +76,17 @@ class HeaderNavigation(Navigation):
             'add_organisation',
             'create_email_branding',
             'email_branding',
+            'find_users_by_email',
             'live_services',
             'organisations',
             'platform_admin',
+            'platform_admin_list_complaints',
             'suspend_service',
             'trial_services',
             'update_email_branding',
+            'user_information',
             'view_provider',
             'view_providers',
-            'platform_admin_list_complaints',
         },
         'sign-in': {
             'sign_in',
@@ -124,6 +126,7 @@ class HeaderNavigation(Navigation):
         'choose_account',
         'choose_service',
         'choose_template',
+        'choose_template_to_copy',
         'confirm_edit_organisation_name',
         'confirm_redact_template',
         'conversation',
@@ -131,6 +134,7 @@ class HeaderNavigation(Navigation):
         'conversation_reply_with_template',
         'conversation_updates',
         'cookies',
+        'copy_template',
         'create_api_key',
         'data_retention',
         'delete_service_template',
@@ -292,8 +296,10 @@ class MainNavigation(Navigation):
             'check_messages',
             'check_notification',
             'choose_template',
+            'choose_template_to_copy',
             'confirm_redact_template',
             'conversation_reply',
+            'copy_template',
             'delete_service_template',
             'edit_service_template',
             'send_messages',
@@ -405,6 +411,7 @@ class MainNavigation(Navigation):
         'error',
         'features',
         'feedback',
+        'find_users_by_email',
         'forgot_password',
         'get_example_csv',
         'get_notifications_as_json',
@@ -478,6 +485,7 @@ class MainNavigation(Navigation):
         'two_factor_email',
         'two_factor_email_sent',
         'update_email_branding',
+        'user_information',
         'user_profile',
         'user_profile_email',
         'user_profile_email_authenticate',
@@ -517,6 +525,10 @@ class CaseworkNavigation(Navigation):
             'view_notifications',
             'view_notification',
         },
+        'uploaded-files': {
+            'view_jobs',
+            'view_job',
+        },
     }
 
     exclude = {
@@ -547,6 +559,7 @@ class CaseworkNavigation(Navigation):
         'check_notification',
         'choose_account',
         'choose_service',
+        'choose_template_to_copy',
         'confirm_edit_organisation_name',
         'confirm_redact_template',
         'conversation',
@@ -554,6 +567,7 @@ class CaseworkNavigation(Navigation):
         'conversation_reply_with_template',
         'conversation_updates',
         'cookies',
+        'copy_template',
         'create_api_key',
         'create_email_branding',
         'data_retention',
@@ -576,6 +590,7 @@ class CaseworkNavigation(Navigation):
         'error',
         'features',
         'feedback',
+        'find_users_by_email',
         'forgot_password',
         'get_example_csv',
         'get_notifications_as_json',
@@ -696,6 +711,7 @@ class CaseworkNavigation(Navigation):
         'two_factor_email_sent',
         'update_email_branding',
         'usage',
+        'user_information',
         'user_profile',
         'user_profile_email',
         'user_profile_email_authenticate',
@@ -709,10 +725,8 @@ class CaseworkNavigation(Navigation):
         'verify',
         'verify_email',
         'verify_mobile',
-        'view_job',
         'view_job_csv',
         'view_job_updates',
-        'view_jobs',
         'view_letter_notification_as_preview',
         'view_letter_template_preview',
         'view_notification_updates',
@@ -775,12 +789,14 @@ class OrgNavigation(Navigation):
         'choose_account',
         'choose_service',
         'choose_template',
+        'choose_template_to_copy',
         'confirm_redact_template',
         'conversation',
         'conversation_reply',
         'conversation_reply_with_template',
         'conversation_updates',
         'cookies',
+        'copy_template',
         'create_api_key',
         'create_email_branding',
         'data_retention',
@@ -801,6 +817,7 @@ class OrgNavigation(Navigation):
         'error',
         'features',
         'feedback',
+        'find_users_by_email',
         'forgot_password',
         'get_example_csv',
         'get_notifications_as_json',
@@ -920,6 +937,7 @@ class OrgNavigation(Navigation):
         'two_factor_email_sent',
         'update_email_branding',
         'usage',
+        'user_information',
         'user_profile',
         'user_profile_email',
         'user_profile_email_authenticate',

app/notify_client/__init__.py

@@ -47,7 +47,7 @@ class NotifyAdminAPIClient(BaseAPIClient):
 
         # if the current service is inactive and the user isn't a platform admin, we should block them from making any
         # stateful modifications to that service
-        if current_service and not current_service['active'] and not current_user.platform_admin:
+        if current_service and not current_service.active and not current_user.platform_admin:
             abort(403)
 
     def post(self, *args, **kwargs):

app/notify_client/billing_api_client.py

@@ -7,26 +7,12 @@ class BillingAPIClient(NotifyAdminAPIClient):
     def __init__(self):
         super().__init__("a" * 73, "b")
 
-    def get_billable_units(self, service_id, year):
-        return self.get(
-            '/service/{0}/billing/monthly-usage'.format(service_id),
-            params=dict(year=year)
-        )
-
-    def get_service_usage(self, service_id, year=None):
-        return self.get(
-            '/service/{0}/billing/yearly-usage-summary'.format(service_id),
-            params=dict(year=year)
-        )
-
-    # Temporary methods to compare the usage before and after using ft_billing
     def get_billable_units_ft(self, service_id, year):
         return self.get(
             '/service/{0}/billing/ft-monthly-usage'.format(service_id),
             params=dict(year=year)
         )
 
-    # Temporary methods to compare the usage before and after using ft_billing
     def get_service_usage_ft(self, service_id, year=None):
         return self.get(
             '/service/{0}/billing/ft-yearly-usage-summary'.format(service_id),

app/notify_client/job_api_client.py

@@ -1,6 +1,6 @@
 from collections import defaultdict
 
-from app.notify_client import NotifyAdminAPIClient, _attach_current_user
+from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 
 
 class JobApiClient(NotifyAdminAPIClient):
@@ -16,6 +16,8 @@ class JobApiClient(NotifyAdminAPIClient):
         'sent to dvla'
     }
 
+    NON_SCHEDULED_JOB_STATUSES = JOB_STATUSES - {'scheduled', 'cancelled'}
+
     def __init__(self):
         super().__init__("a" * 73, "b")
 
@@ -60,8 +62,38 @@ class JobApiClient(NotifyAdminAPIClient):
 
         return jobs
 
+    def get_page_of_jobs(self, service_id, page):
+        return self.get_jobs(
+            service_id,
+            statuses=self.NON_SCHEDULED_JOB_STATUSES,
+            page=page,
+        )
+
+    def get_immediate_jobs(self, service_id):
+        return self.get_jobs(
+            service_id,
+            limit_days=7,
+            statuses=self.NON_SCHEDULED_JOB_STATUSES,
+        )['data']
+
+    def get_scheduled_jobs(self, service_id):
+        return sorted(
+            self.get_jobs(service_id, statuses=['scheduled'])['data'],
+            key=lambda job: job['scheduled_for']
+        )
+
+    @cache.set('has_jobs-{service_id}')
+    def has_jobs(self, service_id):
+        return bool(self.get_jobs(service_id)['data'])
+
     def create_job(self, job_id, service_id, scheduled_for=None):
 
+        self.redis_client.set(
+            'has_jobs-{}'.format(service_id),
+            b'true',
+            ex=cache.TTL,
+        )
+
         data = {"id": job_id}
 
         if scheduled_for:
@@ -78,6 +110,7 @@ class JobApiClient(NotifyAdminAPIClient):
 
         return job
 
+    @cache.delete('has_jobs-{service_id}')
     def cancel_job(self, service_id, job_id):
 
         job = self.post(

app/notify_client/models.py

@@ -59,6 +59,7 @@ class User(UserMixin):
         self.failed_login_count = fields.get('failed_login_count')
         self.state = fields.get('state')
         self.max_failed_login_count = max_failed_login_count
+        self.logged_in_at = fields.get('logged_in_at')
         self.platform_admin = fields.get('platform_admin')
         self.current_session_id = fields.get('current_session_id')
         self.services = fields.get('services', [])
@@ -260,3 +261,45 @@ class AnonymousUser(AnonymousUserMixin):
     # set the anonymous user so that if a new browser hits us we don't error http://stackoverflow.com/a/19275188
     def logged_in_elsewhere(self):
         return False
+
+
+class Service(dict):
+
+    ALLOWED_PROPERTIES = {
+        'active',
+        'branding',
+        'dvla_organisation',
+        'email_branding',
+        'email_from',
+        'id',
+        'inbound_api',
+        'letter_contact_block',
+        'message_limit',
+        'name',
+        'organisation_type',
+        'permissions',
+        'prefix_sms',
+        'research_mode',
+        'service_callback_api',
+    }
+
+    def __init__(self, _dict):
+        # in the case of a bad request current service may be `None`
+        super().__init__(_dict or {})
+
+    def __getattr__(self, attr):
+        if attr in self.ALLOWED_PROPERTIES:
+            return self[attr]
+        raise AttributeError
+
+    @property
+    def trial_mode(self):
+        return self['restricted']
+
+    def has_permission(self, permission):
+        return permission in self.permissions
+
+    def has_jobs(self):
+        # Can’t import at top-level because app isn’t yet initialised
+        from app import job_api_client
+        return job_api_client.has_jobs(self.id)

app/notify_client/notification_api_client.py

@@ -18,6 +18,7 @@ class NotificationApiClient(NotifyAdminAPIClient):
         include_from_test_key=None,
         format_for_csv=None,
         to=None,
+        include_one_off=None,
     ):
         params = {}
         if page is not None:
@@ -36,6 +37,8 @@ class NotificationApiClient(NotifyAdminAPIClient):
             params['format_for_csv'] = format_for_csv
         if to is not None:
             params['to'] = to
+        if include_one_off is not None:
+            params['include_one_off'] = include_one_off
         if job_id:
             return self.get(
                 url='/service/{}/job/{}/notifications'.format(service_id, job_id),
@@ -64,7 +67,12 @@ class NotificationApiClient(NotifyAdminAPIClient):
         return self.get(url='/service/{}/notifications/{}'.format(service_id, notification_id))
 
     def get_api_notifications_for_service(self, service_id):
-        ret = self.get_notifications_for_service(service_id, include_jobs=False, include_from_test_key=True)
+        ret = self.get_notifications_for_service(
+            service_id,
+            include_jobs=False,
+            include_from_test_key=True,
+            include_one_off=False
+        )
         return self.map_letters_to_accepted(ret)
 
     @staticmethod

app/notify_client/user_api_client.py

@@ -1,3 +1,5 @@
+from itertools import chain
+
 from notifications_python_client.errors import HTTPError
 
 from app.notify_client import NotifyAdminAPIClient, cache
@@ -179,6 +181,12 @@ class UserApiClient(NotifyAdminAPIClient):
         data = {'email': email_address}
         self.post(endpoint, data=data)
 
+    def find_users_by_full_or_partial_email(self, email_address):
+        endpoint = '/user/find-users-by-email'
+        data = {'email': email_address}
+        users = self.post(endpoint, data=data)
+        return users
+
     def is_email_already_in_use(self, email_address):
         if self.get_user_by_email_or_none(email_address):
             return True
@@ -203,3 +211,17 @@ class UserApiClient(NotifyAdminAPIClient):
     def get_organisations_and_services_for_user(self, user):
         endpoint = '/user/{}/organisations-and-services'.format(user.id)
         return self.get(endpoint)
+
+    def get_services_for_user(self, user):
+        orgs_and_services_for_user = self.get_organisations_and_services_for_user(user)
+        return orgs_and_services_for_user['services_without_organisations'] + next(chain(
+            org['services'] for org in orgs_and_services_for_user['organisations']
+        ), [])
+
+    def get_service_ids_for_user(self, user):
+        return {
+            service['id'] for service in self.get_services_for_user(user)
+        }
+
+    def user_belongs_to_service(self, user, service_id):
+        return service_id in self.get_service_ids_for_user(user)

app/template_previews.py

@@ -11,7 +11,7 @@ class TemplatePreview:
             'letter_contact_block': template.get('reply_to_text', ''),
             'template': template,
             'values': values,
-            'dvla_org_id': current_service['dvla_organisation'],
+            'dvla_org_id': current_service.dvla_organisation,
         }
         resp = requests.post(
             '{}/preview.{}{}'.format(

app/templates/admin_template.html

@@ -109,9 +109,9 @@
 {% block footer_support_links %}
   <nav class="footer-nav">
     Built by the <a href="https://www.gov.uk/government/organisations/government-digital-service">Government Digital Service</a>
-    <a href="{{ url_for("main.privacy") }}">Privacy</a>    
+    <a href="{{ url_for("main.privacy") }}">Privacy</a>
     <a href="{{ url_for("main.cookies") }}">Cookies</a>
-    {% if current_service.research_mode %}
+    {% if current_service and current_service.research_mode %}
     <span id="research-mode" class="research-mode">research mode</span>
     {% endif %}
   </nav>
@@ -128,10 +128,7 @@
     ga('set', 'anonymizeIp', true);
     ga('set', 'displayFeaturesTask', null);
     ga('set', 'transport', 'beacon');
-    // strip UUIDs
+    page = window.location.pathname + window.location.search;
-    page = (window.location.pathname + window.location.search).replace(
+    ga('send', 'pageview', stripUUIDs(page));
-      /[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}/g, '…'
-    )
-    ga('send', 'pageview', page);
   </script>
 {% endblock %}

app/templates/main_nav.html

@@ -20,10 +20,11 @@
 {% else %}
 <nav class="navigation">
   <ul>
-    <li><a href="{{ url_for('.choose_template', service_id=current_service.id) }}" {{ casework_navigation.is_selected('send-one-off') }}>Send a message</a></li>
+    <li><a href="{{ url_for('.choose_template', service_id=current_service.id) }}" {{ casework_navigation.is_selected('send-one-off') }}>Templates</a></li>
-  </ul>
-  <ul>
     <li><a href="{{ url_for('.view_notifications', service_id=current_service.id, status='sending,delivered,failed') }}" {{ casework_navigation.is_selected('sent-messages') }}>Sent messages</a></li>
+    {% if current_service.has_jobs() %}
+      <li><a href="{{ url_for('.view_jobs', service_id=current_service.id) }}" {{ casework_navigation.is_selected('uploaded-files') }}>Uploaded files</a></li>
+    {% endif %}
   </ul>
 
 </nav>

app/templates/views/dashboard/_totals.html

@@ -27,7 +27,7 @@
         smaller=smaller_font_size
       ) }}
     </div>
-    {% if 'letter' in current_service['permissions'] %}
+    {% if current_service.has_permission('letter') %}
       <div id="total-letters" class="{{column_width}}">
         {{ big_number_with_status(
           statistics['letter']['requested'],

app/templates/views/dashboard/_upcoming.html

@@ -5,12 +5,14 @@
 <div class="ajax-block-container">
   {% if scheduled_jobs %}
     <div class='dashboard-table'>
-      <h2 class="heading-medium">
+      {% if not hide_heading %}
-        In the next few days
+        <h2 class="heading-medium">
-      </h2>
+          In the next few days
+        </h2>
+      {% endif %}
       {% call(item, row_number) list_table(
         scheduled_jobs,
-        caption="In the next 24 hours",
+        caption="In the next few days",
         caption_visible=False,
         empty_message='Nothing to see here',
         field_headings=[

app/templates/views/find-users/find-users-by-email.html

@@ -0,0 +1,51 @@
+{% extends "views/platform-admin/_base_template.html" %}
+{% from "components/page-footer.html" import page_footer %}
+
+{% block per_page_title %}
+  Find users by email
+{% endblock %}
+
+{% block platform_admin_content %}
+
+  <h1 class="heading-large">
+    Find users by email
+  </h1>
+
+  <form
+    method="post"
+    action="{{ url_for('.find_users_by_email') }}"
+    class="grid-row"
+  >
+    <div class="column-three-quarters">
+      {{ textbox(
+        form.search,
+        width='1-1',
+        label='Find users by email, or by partial email'
+      ) }}
+    </div>
+    <div class="column-one-quarter align-button-with-textbox">
+      <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+      <button type="submit" class="button">Search</button>
+    </div>
+  </form>
+
+  <form id="search-form" method="post">
+    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+  </form>
+
+  {% if users_found %}
+  <nav class="browse-list">
+    <ul>
+    {% for user in users_found %}
+      <li class="browse-list-item">
+        <a href="{{url_for('.user_information', user_id=user.id)}}" class="browse-list-link">{{ user.email_address }}</a>
+        <p class="browse-list-hint">{{ user.name }}</p>
+      </li>
+      <hr>
+    {% endfor %}
+    </ul>
+  </nav>
+  {% elif users_found == [] %}
+    <p class="browse-list-hint">No users found.</p>
+  {% endif %}
+{% endblock %}

app/templates/views/find-users/user-information.html

@@ -0,0 +1,43 @@
+{% extends "views/platform-admin/_base_template.html" %}
+{% from "components/page-footer.html" import page_footer %}
+
+{% block per_page_title %}
+  User information for {{ user.name }}
+{% endblock %}
+
+{% block platform_admin_content %}
+  <div class="grid-row bottom-gutter">
+    <div class="column-whole">
+      <h1 class="heading-large">
+        {{ user.name }}
+      </h1>
+      <p>{{ user.email_address }}</p>
+      <p>{{ user.mobile_number }}</p>
+      <h2 class="heading-medium">Services</h2>
+      <nav class="browse-list">
+        <ul>
+        {% for service in services %}
+          <li class="browse-list-item">
+            <a class="browse-list-hint" href={{url_for('.service_dashboard', service_id=service.id)}}>{{ service.name }}</a>
+          </li>
+        {% endfor %}
+        </ul>
+      </nav>
+      <h2 class="heading-medium">Last login</h2>
+      {% if not user.logged_in_at %}
+      <p>This person has never logged in</p>
+      {% else %}
+      <p>Last logged in
+        <time class="timeago" datetime="{{ user.logged_in_at }}">
+          {{ user.logged_in_at|format_delta }}
+        </time>
+      </p>
+      {% endif %}
+      {% if user.failed_login_count > 0 %}
+      <p style="color:#b10e1e;">
+        {{ user.failed_login_count }} failed login attempts
+      </p>
+      {% endif %}
+    </div>
+  </div>
+{% endblock %}

app/templates/views/jobs/jobs.html

@@ -8,6 +8,7 @@
 {% block maincolumn_content %}
     <h1 class="heading-large">Uploaded files</h1>
     <div class="dashboard">
+      {{ scheduled_jobs|safe }}
       {% include 'views/dashboard/_jobs.html' %}
       {{ previous_next_navigation(prev_page, next_page) }}
     </div>

app/templates/views/manage-users.html

@@ -92,7 +92,7 @@
                 'Basic view'
               ) }}
             {% endif %}
-            {% if 'email_auth' in current_service['permissions'] %}
+            {% if current_service.has_permission('email_auth') %}
               <div class="tick-cross-list-hint">
                 {% if user.auth_type == 'sms_auth' %}
                   Signs in with a text message code

app/templates/views/manage-users/permissions.html

@@ -8,8 +8,8 @@
 
         <div class="bottom-gutter-1-3">
           {{ radio(option, option_hints={
-              'admin': 'See dashboard and team members',
+              'admin': 'Show dashboard and other options',
-              'caseworker': 'Send messages and see sent messages'
+              'caseworker': 'Send messages, hide dashboard and other options'
           }) }}
         </div>
         {% if option.data == 'admin' %}

app/templates/views/notifications.html

@@ -4,16 +4,20 @@
 {% from "components/page-footer.html" import page_footer %}
 {% from "components/textbox.html" import textbox %}
 
+{% set page_title = (
+  message_count_label(99, message_type, suffix='') | capitalize
+  if current_user.has_permissions('view_activity')
+  else 'Sent messages'
+) %}
+
 {% block service_page_title %}
-  {{ message_count_label(99, message_type, suffix='') | capitalize }}
+  {{ page_title }}
 {% endblock %}
 
 {% block maincolumn_content %}
 
   <h1 class="heading-large">
-    {% if not current_user.has_permissions('view_activity') %}<span class="visually-hidden">{% endif %}
+    {{ page_title }}
-      {{ message_count_label(99, message_type, suffix='') | capitalize }}
-    {% if not current_user.has_permissions('view_activity') %}</span>{% endif %}
   </h1>
   {% if not message_type == "letter" %}
 

app/templates/views/platform-admin/_base_template.html

@@ -20,6 +20,7 @@
           ('Email branding', url_for('main.email_branding')),
           ('Letter jobs', url_for('main.letter_jobs')),
           ('Inbound SMS numbers', url_for('main.inbound_sms_admin')),
+          ('Find users by email', url_for('main.find_users_by_email')),
           ('Email Complaints', url_for('main.platform_admin_list_complaints'))
         ] %}
           <li>

app/templates/views/service-settings/set-basic-view.html

@@ -9,18 +9,25 @@
 {% block maincolumn_content %}
 
   <div class="grid-row">
-    <form method="post" class="column-five-sixths">
+    <form
+      method="post"
+      class="column-five-sixths"
+      data-module="track-form-submission"
+    >
       <h1 class="heading-large">Basic view</h1>
       <p>
-        Basic view lets you restrict a team member to only:
+        Hide the dashboard and other options from team members who only need to send messages.
+      </p>
+      <p>
+        Turn on basic view then edit team members’ permissions.
+      </p>
+      <p>
+        Team members with basic view can only:
       </p>
       <ul class="list list-bullet">
-        <li>send messages</li>
+        <li>send messages using existing templates</li>
-        <li>see sent messages</li>
+        <li>see a list of sent messages</li>
       </ul>
-      <p>
-        You’ll get to choose which team members have basic view.
-      </p>
       <p>
         <a href="{{ url_for('main.preview_basic_view', service_id=current_service.id) }}">See a preview of basic view</a>.
       </p>

app/templates/views/signedout.html

@@ -120,12 +120,12 @@
       <div class="grid-row bottom-gutter">
         <div class="column-half">
           <h3 class="visually-hidden">Services</h3>
-          <div class="product-page-big-number">292</div>
+          <div class="product-page-big-number">308</div>
           services
         </div>
         <div class="column-half">
           <h3 class="visually-hidden">Organisations</h3>
-          <div class="product-page-big-number">96</div>
+          <div class="product-page-big-number">101</div>
           organisations
         </div>
       </div>

app/templates/views/templates/choose.html

@@ -4,6 +4,8 @@
 
 {% extends "withnav_template.html" %}
 
+{% set page_title = 'Templates' %}
+
 {% block service_page_title %}
   {{ page_title }}
 {% endblock %}

app/templates/views/templates/copy.html

@@ -0,0 +1,36 @@
+{% from "components/message-count-label.html" import message_count_label %}
+
+{% extends "withnav_template.html" %}
+
+{% block service_page_title %}
+  Copy an existing template
+{% endblock %}
+
+{% block maincolumn_content %}
+
+    <div class="bottom-gutter-3-2">
+      <h1 class="heading-large">Copy an existing template</h1>
+    </div>
+    <nav>
+      {% for service in services %}
+        {% if service.templates and services|length > 1 %}
+          <h2 class="">
+            {{ service.name }}
+          </h2>
+          <div class="left-gutter-4-3 bottom-gutter-3-2">
+        {% endif %}
+        {% for template in service.templates %}
+          <h2 class="message-name">
+            <a href="{{ url_for('.copy_template', service_id=current_service.id, template_id=template.id, from_service=service.id) }}">{{ template.name }}</a>
+          </h2>
+          <p class="message-type">
+            {{ message_count_label(1, template.template_type, suffix='')|capitalize }} template
+          </p>
+        {% endfor %}
+        {% if service.templates %}
+          </div>
+        {% endif %}
+      {% endfor %}
+    </nav>
+
+{% endblock %}

app/templates/withnav_template.html

@@ -10,7 +10,13 @@
       <div class="navigation-service-name">
         {{ current_service.name }}
         {% if current_user.previewing_basic_view %}
-          <span class="navigation-service-basic-view-preview">Preview of basic view</span>
+          <span
+            class="navigation-service-basic-view-preview"
+            data-module="track-event"
+            data-event-category="basic-view"
+            data-event-action="preview"
+            data-event-label="{{ current_service.id }}"
+          >Preview of basic view</span>
           <a class="navigation-service-basic-view-back-link" href="{{ url_for('main.service_set_basic_view', service_id=current_service.id)}}">Back to settings</a>
         {% endif %}
       </div>

app/utils.py

@@ -292,8 +292,8 @@ def get_template(
     if 'email' == template['template_type']:
         return EmailPreviewTemplate(
             template,
-            from_name=service['name'],
+            from_name=service.name,
-            from_address='{}@notifications.service.gov.uk'.format(service['email_from']),
+            from_address='{}@notifications.service.gov.uk'.format(service.email_from),
             expanded=expand_emails,
             show_recipient=show_recipient,
             redact_missing_personalisation=redact_missing_personalisation,
@@ -302,8 +302,8 @@ def get_template(
     if 'sms' == template['template_type']:
         return SMSPreviewTemplate(
             template,
-            prefix=service['name'],
+            prefix=service.name,
-            show_prefix=service['prefix_sms'],
+            show_prefix=service.prefix_sms,
             sender=sms_sender,
             show_sender=bool(sms_sender),
             show_recipient=show_recipient,

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.5-slim
+FROM python:3.5-slim-jessie
 
 ARG HTTP_PROXY
 ARG HTTPS_PROXY

gulpfile.babel.js

@@ -70,7 +70,7 @@ gulp.task('javascripts', () => gulp
     paths.src + 'javascripts/updateContent.js',
     paths.src + 'javascripts/listEntry.js',
     paths.src + 'javascripts/liveSearch.js',
-    paths.src + 'javascripts/errorTracking.js',
+    paths.src + 'javascripts/analytics.js',
     paths.src + 'javascripts/preventDuplicateFormSubmissions.js',
     paths.src + 'javascripts/fullscreenTable.js',
     paths.src + 'javascripts/conditionalRadios.js',

requirements-app.txt

@@ -0,0 +1,24 @@
+# Run `make freeze-requirements` to update requirements.txt
+# with package version changes made in requirements-app.txt
+
+ago==0.0.92
+Flask==1.0.2
+Flask-WTF==0.14.2
+Flask-Login==0.4.1
+
+blinker==1.4
+pyexcel==0.5.8
+pyexcel-io==0.5.7
+pyexcel-xls==0.5.7
+pyexcel-xlsx==0.5.6
+pyexcel-ods3==0.5.2
+pytz==2018.5
+gunicorn==19.8.1
+whitenoise==3.3.1  #manages static assets
+eventlet==0.22.1
+notifications-python-client==4.10.0
+
+# PaaS
+awscli-cwlogs>=1.4,<1.5
+
+git+https://github.com/alphagov/notifications-utils.git@29.3.4#egg=notifications-utils==29.3.4

requirements.txt

@@ -1,8 +1,12 @@
+# pyup: ignore file
+# This file is autogenerated. Do not edit it manually.
+# Run `make freeze-requirements` to update requirements.txt
+# with package version changes made in requirements-app.txt
+
 ago==0.0.92
 Flask==1.0.2
 Flask-WTF==0.14.2
 Flask-Login==0.4.1
-wtforms==2.1  # pyup: ignore
 
 blinker==1.4
 pyexcel==0.5.8
@@ -14,9 +18,60 @@ pytz==2018.5
 gunicorn==19.8.1
 whitenoise==3.3.1  #manages static assets
 eventlet==0.22.1
-notifications-python-client==4.8.2
+notifications-python-client==4.10.0
 
 # PaaS
 awscli-cwlogs>=1.4,<1.5
 
-git+https://github.com/alphagov/notifications-utils.git@29.3.1#egg=notifications-utils==29.3.1
+git+https://github.com/alphagov/notifications-utils.git@29.3.4#egg=notifications-utils==29.3.4
+
+## The following requirements were added by pip freeze:
+awscli==1.15.70
+bleach==2.1.3
+boto3==1.6.16
+botocore==1.10.69
+certifi==2018.4.16
+chardet==3.0.4
+click==6.7
+colorama==0.3.9
+docopt==0.6.2
+docutils==0.14
+et-xmlfile==1.0.1
+Flask-Redis==0.3.0
+future==0.16.0
+greenlet==0.4.14
+html5lib==1.0.1
+idna==2.7
+itsdangerous==0.24
+jdcal==1.4
+Jinja2==2.10
+jmespath==0.9.3
+lml==0.0.1
+lxml==4.2.3
+MarkupSafe==1.0
+mistune==0.8.3
+monotonic==1.5
+openpyxl==2.5.4
+orderedset==2.0.1
+phonenumbers==8.9.4
+pyasn1==0.4.4
+pyexcel-ezodf==0.3.4
+PyJWT==1.6.4
+PyPDF2==1.26.0
+python-dateutil==2.7.3
+python-json-logger==0.1.8
+PyYAML==3.12
+redis==2.10.6
+requests==2.19.1
+rsa==3.4.2
+s3transfer==0.1.13
+six==1.11.0
+smartypants==2.0.1
+statsd==3.2.2
+texttable==1.4.0
+urllib3==1.23
+webencodings==0.5.1
+Werkzeug==0.14.1
+WTForms==2.2.1
+xlrd==1.1.0
+xlwt==1.3.0

requirements_for_test.txt

@@ -1,13 +1,13 @@
 -r requirements.txt
 isort==4.3.4
-pytest==3.6.3
+pytest==3.7.1
 pytest-env==0.6.2
 pytest-mock==1.10.0
 pytest-cov==2.5.1
-pytest-xdist==1.22.2
+pytest-xdist==1.22.5
 coveralls==1.3.0
 httpretty==0.9.5
-beautifulsoup4==4.6.0
+beautifulsoup4==4.6.1
 freezegun==0.3.10
 flake8==3.5.0
 flake8-print==3.1.0

scripts/run_tests.sh

@@ -24,6 +24,9 @@ if [[ -z "$VIRTUAL_ENV" ]] && [[ -d venv ]]; then
   source ./venv/bin/activate
 fi
 
+make test-requirements
+display_result $? 1 "Requirements check"
+
 flake8 .
 display_result $? 1 "Code style check"
 

tests/__init__.py

@@ -489,6 +489,7 @@ def validate_route_permission(mocker,
     mocker.patch('app.user_api_client.get_user_by_email', return_value=usr)
     mocker.patch('app.service_api_client.get_service', return_value={'data': service})
     mocker.patch('app.user_api_client.get_users_for_service', return_value=[usr])
+    mocker.patch('app.job_api_client.has_jobs', return_value=False)
     with app_.test_request_context():
         with app_.test_client() as client:
             client.login(usr)
@@ -525,6 +526,7 @@ def validate_route_permission_with_client(mocker,
     mocker.patch('app.user_api_client.get_user_by_email', return_value=usr)
     mocker.patch('app.service_api_client.get_service', return_value={'data': service})
     mocker.patch('app.user_api_client.get_users_for_service', return_value=[usr])
+    mocker.patch('app.job_api_client.has_jobs', return_value=False)
     client.login(usr)
     resp = None
     if method == 'GET':

tests/app/main/views/test_accept_invite.py

@@ -122,7 +122,7 @@ def test_invite_goes_in_session(
 
 @pytest.mark.parametrize('user, landing_page_title', [
     (active_user_with_permissions, 'Dashboard'),
-    (active_caseworking_user, 'Choose a template'),
+    (active_caseworking_user, 'Templates'),
 ])
 def test_accepting_invite_removes_invite_from_session(
     client_request,

tests/app/main/views/test_activity.py

@@ -19,14 +19,11 @@ from tests.conftest import (
 )
 
 
-@pytest.mark.parametrize('user', (
-    active_user_view_permissions,
-    active_caseworking_user,
-))
 @pytest.mark.parametrize(
-    "message_type,page_title", [
+    "user,extra_args,expected_update_endpoint,page_title", [
-        ('email', 'Emails'),
+        (active_user_view_permissions, {'message_type': 'email'}, '/email.json', 'Emails'),
-        ('sms', 'Text messages')
+        (active_user_view_permissions, {'message_type': 'sms'}, '/sms.json', 'Text messages'),
+        (active_caseworking_user, {}, '.json', 'Sent messages'),
     ]
 )
 @pytest.mark.parametrize(
@@ -68,11 +65,15 @@ from tests.conftest import (
     ]
 )
 def test_can_show_notifications(
+    client_request,
     logged_in_client,
     service_one,
     mock_get_notifications,
     mock_get_service_statistics,
-    message_type,
+    mock_has_no_jobs,
+    user,
+    extra_args,
+    expected_update_endpoint,
     page_title,
     status_argument,
     expected_api_call,
@@ -81,33 +82,29 @@ def test_can_show_notifications(
     to_argument,
     expected_to_argument,
     mocker,
-    user,
     fake_uuid,
 ):
-    mocker.patch('app.user_api_client.get_user', return_value=user(fake_uuid))
+    client_request.login(user(fake_uuid))
     if expected_to_argument:
-        response = logged_in_client.post(
+        page = client_request.post(
-            url_for(
-                'main.view_notifications',
-                service_id=service_one['id'],
-                message_type=message_type,
-                status=status_argument,
-                page=page_argument,
-            ),
-            data={
-                'to': to_argument
-            }
-        )
-    else:
-        response = logged_in_client.get(url_for(
             'main.view_notifications',
             service_id=service_one['id'],
-            message_type=message_type,
             status=status_argument,
             page=page_argument,
-        ))
+            _data={
-    assert response.status_code == 200
+                'to': to_argument
-    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+            },
+            _expected_status=200,
+            **extra_args
+        )
+    else:
+        page = client_request.get(
+            'main.view_notifications',
+            service_id=service_one['id'],
+            status=status_argument,
+            page=page_argument,
+            **extra_args
+        )
     text_of_first_row = page.select('tbody tr')[0].text
     assert '07123456789' in text_of_first_row
     assert (
@@ -120,7 +117,10 @@ def test_can_show_notifications(
     path_to_json = page.find("div", {'data-key': 'notifications'})['data-resource']
 
     url = urlparse(path_to_json)
-    assert url.path == '/services/{}/notifications/{}.json'.format(service_one['id'], message_type)
+    assert url.path == '/services/{}/notifications{}'.format(
+        service_one['id'],
+        expected_update_endpoint,
+    )
     query_dict = parse_qs(url.query)
     if status_argument:
         assert query_dict['status'] == [status_argument]
@@ -133,15 +133,15 @@ def test_can_show_notifications(
         page=expected_page_argument,
         service_id=service_one['id'],
         status=expected_api_call,
-        template_type=[message_type],
+        template_type=list(extra_args.values()),
         to=expected_to_argument,
     )
 
     json_response = logged_in_client.get(url_for(
         'main.get_notifications_as_json',
         service_id=service_one['id'],
-        message_type=message_type,
+        status=status_argument,
-        status=status_argument
+        **extra_args
     ))
     json_content = json.loads(json_response.get_data(as_text=True))
     assert json_content.keys() == {'counts', 'notifications'}
@@ -194,6 +194,7 @@ def test_link_to_download_notifications(
     fake_uuid,
     mock_get_notifications,
     mock_get_service_statistics,
+    mock_has_no_jobs,
     user,
     query_parameters,
     expected_download_link,

tests/app/main/views/test_api_integration.py

@@ -487,6 +487,7 @@ def test_should_validate_whitelist_items(
 def test_callback_forms_validation(
     client_request,
     service_one,
+    mock_get_valid_service_callback_api,
     endpoint,
     url,
     bearer_token,
@@ -606,7 +607,8 @@ def test_callbacks_button_links_straight_to_delivery_status_if_service_has_no_in
 def test_callbacks_page_redirects_to_delivery_status_if_service_has_no_inbound_sms(
     client_request,
     service_one,
-    mocker
+    mocker,
+    mock_get_valid_service_callback_api,
 ):
     page = client_request.get(
         'main.api_callbacks',

tests/app/main/views/test_dashboard.py

@@ -2,7 +2,7 @@ import copy
 import json
 from datetime import datetime
 from functools import partial
-from unittest.mock import ANY, call
+from unittest.mock import call
 
 import pytest
 from bs4 import BeautifulSoup
@@ -573,9 +573,9 @@ def test_should_show_upcoming_jobs_on_dashboard(
 ):
     response = logged_in_client.get(url_for('main.service_dashboard', service_id=SERVICE_ONE_ID))
 
-    first_call = mock_get_jobs.call_args_list[0]
+    second_call = mock_get_jobs.call_args_list[1]
-    assert first_call[0] == (SERVICE_ONE_ID,)
+    assert second_call[0] == (SERVICE_ONE_ID,)
-    assert first_call[1]['statuses'] == ['scheduled']
+    assert second_call[1]['statuses'] == ['scheduled']
 
     assert response.status_code == 200
 
@@ -702,10 +702,10 @@ def test_should_show_recent_jobs_on_dashboard(
 ):
     response = logged_in_client.get(url_for('main.service_dashboard', service_id=SERVICE_ONE_ID))
 
-    second_call = mock_get_jobs.call_args_list[1]
+    third_call = mock_get_jobs.call_args_list[2]
-    assert second_call[0] == (SERVICE_ONE_ID,)
+    assert third_call[0] == (SERVICE_ONE_ID,)
-    assert second_call[1]['limit_days'] == 7
+    assert third_call[1]['limit_days'] == 7
-    assert 'scheduled' not in second_call[1]['statuses']
+    assert 'scheduled' not in third_call[1]['statuses']
 
     assert response.status_code == 200
 
@@ -1266,11 +1266,14 @@ def test_should_show_all_jobs_with_valid_statuses(
     logged_in_client.get(url_for('main.service_dashboard', service_id=SERVICE_ONE_ID))
 
     first_call = mock_get_jobs.call_args_list[0]
-    # first call - scheduled jobs only
+    # first call - checking for any jobs
-    assert first_call == call(ANY, statuses=['scheduled'])
+    assert first_call == call(SERVICE_ONE_ID)
-    # second call - everything but scheduled and cancelled
     second_call = mock_get_jobs.call_args_list[1]
-    assert second_call == call(ANY, limit_days=ANY, statuses={
+    # second call - scheduled jobs only
+    assert second_call == call(SERVICE_ONE_ID, statuses=['scheduled'])
+    # third call - everything but scheduled and cancelled
+    third_call = mock_get_jobs.call_args_list[2]
+    assert third_call == call(SERVICE_ONE_ID, limit_days=7, statuses={
         'pending',
         'in progress',
         'finished',

tests/app/main/views/test_find_users.py

@@ -0,0 +1,148 @@
+from flask import url_for
+from lxml import html
+
+from app.notify_client.user_api_client import User
+from tests import user_json
+
+
+def test_find_users_by_email_page_loads_correctly(client_request, platform_admin_user):
+    client_request.login(platform_admin_user)
+    document = client_request.get('main.find_users_by_email')
+
+    assert document.h1.text.strip() == 'Find users by email'
+    assert len(document.find_all('input', {'type': 'search'})) > 0
+
+
+def test_find_users_by_email_displays_users_found(
+    client_request,
+    platform_admin_user,
+    mocker
+):
+    client_request.login(platform_admin_user)
+    mocker.patch(
+        'app.user_api_client.find_users_by_full_or_partial_email',
+        return_value={"data": [user_json()]},
+        autospec=True,
+    )
+    document = client_request.post(
+        'main.find_users_by_email',
+        _data={"search": "twilight.sparkle"},
+        _expected_status=200
+    )
+
+    assert any(element.text.strip() == 'test@gov.uk' for element in document.find_all(
+        'a', {'class': 'browse-list-link'}, href=True)
+    )
+    assert any(element.text.strip() == 'Test User' for element in document.find_all('p', {'class': 'browse-list-hint'}))
+
+    assert document.find('a', {'class': 'browse-list-link'}).text.strip() == 'test@gov.uk'
+    assert document.find('p', {'class': 'browse-list-hint'}).text.strip() == 'Test User'
+
+
+def test_find_users_by_email_displays_multiple_users(
+    client_request,
+    platform_admin_user,
+    mocker
+):
+    client_request.login(platform_admin_user)
+    mocker.patch(
+        'app.user_api_client.find_users_by_full_or_partial_email',
+        return_value={"data": [user_json(name="Apple Jack"), user_json(name="Apple Bloom")]},
+        autospec=True,
+    )
+    document = client_request.post('main.find_users_by_email', _data={"search": "apple"}, _expected_status=200)
+
+    assert any(
+        element.text.strip() == 'Apple Jack' for element in document.find_all('p', {'class': 'browse-list-hint'})
+    )
+    assert any(
+        element.text.strip() == 'Apple Bloom' for element in document.find_all('p', {'class': 'browse-list-hint'})
+    )
+
+
+def test_find_users_by_email_displays_message_if_no_users_found(
+    client_request,
+    platform_admin_user,
+    mocker
+):
+    client_request.login(platform_admin_user)
+    mocker.patch('app.user_api_client.find_users_by_full_or_partial_email', return_value={"data": []}, autospec=True)
+    document = client_request.post(
+        'main.find_users_by_email', _data={"search": "twilight.sparkle"}, _expected_status=200
+    )
+
+    assert document.find('p', {'class': 'browse-list-hint'}).text.strip() == 'No users found.'
+
+
+def test_find_users_by_email_validates_against_empty_search_submission(
+    client_request,
+    platform_admin_user,
+    mocker
+):
+    client_request.login(platform_admin_user)
+    document = client_request.post('main.find_users_by_email', _data={"search": ""}, _expected_status=400)
+
+    expected_message = "You need to enter full or partial email address to search by."
+    assert document.find('span', {'class': 'error-message'}).text.strip() == expected_message
+
+
+def test_user_information_page_shows_information_about_user(
+    client,
+    platform_admin_user,
+    mocker
+):
+    mocker.patch('app.user_api_client.get_user', side_effect=[
+        platform_admin_user,
+        User(user_json(name="Apple Bloom", services=[1, 2]))
+    ], autospec=True)
+
+    mocker.patch(
+        'app.user_api_client.get_organisations_and_services_for_user',
+        return_value={'organisations': [], 'services_without_organisations': [
+            {"id": 1, "name": "Fresh Orchard Juice"},
+            {"id": 2, "name": "Nature Therapy"},
+        ]},
+        autospec=True
+    )
+    client.login(platform_admin_user)
+    response = client.get(url_for('main.user_information', user_id=345))
+    assert response.status_code == 200
+
+    document = html.fromstring(response.get_data(as_text=True))
+
+    assert document.xpath("//h1/text()[normalize-space()='Apple Bloom']")
+    assert document.xpath("//p/text()[normalize-space()='test@gov.uk']")
+    assert document.xpath("//p/text()[normalize-space()='+447700900986']")
+
+    assert document.xpath("//h2/text()[normalize-space()='Services']")
+    assert document.xpath("//a/text()[normalize-space()='Fresh Orchard Juice']")
+    assert document.xpath("//a/text()[normalize-space()='Nature Therapy']")
+
+    assert document.xpath("//h2/text()[normalize-space()='Last login']")
+    assert not document.xpath("//p/text()[normalize-space()='0 failed login attempts']")
+
+
+def test_user_information_page_displays_if_there_are_failed_login_attempts(
+    client,
+    platform_admin_user,
+    mocker
+):
+    mocker.patch('app.user_api_client.get_user', side_effect=[
+        platform_admin_user,
+        User(user_json(name="Apple Bloom", failed_login_count=2))
+    ], autospec=True)
+
+    mocker.patch(
+        'app.user_api_client.get_organisations_and_services_for_user',
+        return_value={'organisations': [], 'services_without_organisations': [
+            {"id": 1, "name": "Fresh Orchard Juice"},
+            {"id": 2, "name": "Nature Therapy"},
+        ]},
+        autospec=True
+    )
+    client.login(platform_admin_user)
+    response = client.get(url_for('main.user_information', user_id=345))
+    assert response.status_code == 200
+
+    document = html.fromstring(response.get_data(as_text=True))
+    assert document.xpath("//p/text()[normalize-space()='2 failed login attempts']")

tests/app/main/views/test_jobs.py

@@ -8,42 +8,147 @@ from freezegun import freeze_time
 from app.main.views.jobs import get_time_left
 from tests.conftest import (
     SERVICE_ONE_ID,
+    active_caseworking_user,
+    active_user_with_permissions,
     mock_get_notifications,
     normalize_spaces,
 )
 
 
-def test_get_jobs_should_return_list_of_all_real_jobs(
+@pytest.mark.parametrize('user, expected_rows', [
-    logged_in_client,
+    (active_user_with_permissions, (
+        (
+            'File Sending Delivered Failed'
+        ),
+        (
+            'export 1/1/2016.xls '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'all email addresses.xlsx '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'applicants.ods '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'thisisatest.csv '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+    )),
+    (active_caseworking_user, (
+        (
+            'File Messages to be sent'
+        ),
+        (
+            'send_me_later.csv '
+            'Sending 1 January at 11:09am 1'
+        ),
+        (
+            'even_later.csv '
+            'Sending 1 January at 11:09pm 1'
+        ),
+        (
+            'File Sending Delivered Failed'
+        ),
+        (
+            'export 1/1/2016.xls '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'all email addresses.xlsx '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'applicants.ods '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'thisisatest.csv '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+    )),
+])
+@freeze_time("2012-12-12 12:12")
+def test_jobs_page_shows_scheduled_jobs_in_basic_view(
+    client_request,
     service_one,
     active_user_with_permissions,
     mock_get_jobs,
-    mocker,
+    fake_uuid,
+    user,
+    expected_rows,
 ):
-    response = logged_in_client.get(url_for('main.view_jobs', service_id=service_one['id']))
+    client_request.login(user(fake_uuid))
+    page = client_request.get('main.view_jobs', service_id=service_one['id'])
 
-    assert response.status_code == 200
+    for index, row in enumerate(expected_rows):
-    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
+        assert normalize_spaces(page.select('tr')[index].text) == row
-    assert page.h1.string == 'Uploaded files'
-    jobs = [x.text for x in page.tbody.find_all('a', {'class': 'file-list-filename'})]
-    assert len(jobs) == 4
 
 
+@pytest.mark.parametrize('user', [
+    active_user_with_permissions,
+    active_caseworking_user,
+])
 def test_get_jobs_shows_page_links(
-    logged_in_client,
+    client_request,
-    service_one,
     active_user_with_permissions,
     mock_get_jobs,
-    mocker,
+    user,
+    fake_uuid,
 ):
-    response = logged_in_client.get(url_for('main.view_jobs', service_id=service_one['id']))
+    client_request.login(user(fake_uuid))
+    page = client_request.get('main.view_jobs', service_id=SERVICE_ONE_ID)
 
-    assert response.status_code == 200
-    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
     assert 'Next page' in page.find('li', {'class': 'next-page'}).text
     assert 'Previous page' in page.find('li', {'class': 'previous-page'}).text
 
 
+@pytest.mark.parametrize('user', [
+    active_user_with_permissions,
+    active_caseworking_user,
+])
+@freeze_time("2012-12-12 12:12")
+def test_jobs_page_doesnt_show_scheduled_on_page_2(
+    client_request,
+    service_one,
+    active_user_with_permissions,
+    mock_get_jobs,
+    fake_uuid,
+    user,
+):
+    client_request.login(user(fake_uuid))
+    page = client_request.get('main.view_jobs', service_id=service_one['id'], page=2)
+
+    for index, row in enumerate((
+        (
+            'File Sending Delivered Failed'
+        ),
+        (
+            'export 1/1/2016.xls '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'all email addresses.xlsx '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'applicants.ods '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+        (
+            'thisisatest.csv '
+            'Sent 12 December at 12:12pm 1 0 0'
+        ),
+    )):
+        assert normalize_spaces(page.select('tr')[index].text) == row
+
+
+@pytest.mark.parametrize('user', [
+    active_user_with_permissions,
+    active_caseworking_user,
+])
 @pytest.mark.parametrize(
     "status_argument, expected_api_call", [
         (
@@ -70,8 +175,7 @@ def test_get_jobs_shows_page_links(
 )
 @freeze_time("2016-01-01 11:09:00.061258")
 def test_should_show_page_for_one_job(
-    logged_in_client,
+    client_request,
-    service_one,
     active_user_with_permissions,
     mock_get_service_template,
     mock_get_job,
@@ -80,38 +184,37 @@ def test_should_show_page_for_one_job(
     fake_uuid,
     status_argument,
     expected_api_call,
+    user,
 ):
 
-    response = logged_in_client.get(url_for(
+    page = client_request.get(
         'main.view_job',
-        service_id=service_one['id'],
+        service_id=SERVICE_ONE_ID,
         job_id=fake_uuid,
         status=status_argument
-    ))
+    )
 
-    assert response.status_code == 200
-    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
     assert page.h1.text.strip() == 'thisisatest.csv'
     assert ' '.join(page.find('tbody').find('tr').text.split()) == (
         '07123456789 template content Delivered 1 January at 11:10am'
     )
     assert page.find('div', {'data-key': 'notifications'})['data-resource'] == url_for(
         'main.view_job_updates',
-        service_id=service_one['id'],
+        service_id=SERVICE_ONE_ID,
         job_id=fake_uuid,
         status=status_argument,
     )
     csv_link = page.select_one('a[download]')
     assert csv_link['href'] == url_for(
         'main.view_job_csv',
-        service_id=service_one['id'],
+        service_id=SERVICE_ONE_ID,
         job_id=fake_uuid,
         status=status_argument
     )
     assert csv_link.text == 'Download this report'
     assert page.find('span', {'id': 'time-left'}).text == 'Data available for 7 days'
     mock_get_notifications.assert_called_with(
-        service_one['id'],
+        SERVICE_ONE_ID,
         fake_uuid,
         status=expected_api_call
     )

tests/app/main/views/test_letters.py

@@ -56,17 +56,19 @@ def test_letters_lets_in_without_permission(
 @pytest.mark.parametrize('permissions, choices', [
     (
         ['email', 'sms', 'letter'],
-        ['Email', 'Text message', 'Letter']
+        ['Email', 'Text message', 'Letter', 'Copy of an existing template']
     ),
     (
         ['email', 'sms'],
-        ['Email', 'Text message']
+        ['Email', 'Text message', 'Copy of an existing template']
     ),
 ])
 def test_given_option_to_add_letters_if_allowed(
     logged_in_client,
     service_one,
     mocker,
+    mock_get_service_templates,
+    mock_get_organisations_and_services_for_user,
     permissions,
     choices,
 ):

tests/app/main/views/test_notifications.py

@@ -33,6 +33,7 @@ from tests.conftest import (
 def test_notification_status_page_shows_details(
     client_request,
     mocker,
+    mock_has_no_jobs,
     service_one,
     fake_uuid,
     user,

tests/app/main/views/test_platform_admin.py

@@ -1,6 +1,7 @@
 import datetime
 import re
 import uuid
+from functools import partial
 from unittest.mock import ANY
 
 import pytest
@@ -114,12 +115,13 @@ def test_should_render_platform_admin_page(
     'main.live_services',
     'main.trial_services',
 ])
-@pytest.mark.parametrize('include_from_test_key, inc', [
+@pytest.mark.parametrize('partial_url_for, inc', [
-    ("Y", True),
+    (partial(url_for), True),
-    ("N", False)
+    (partial(url_for, include_from_test_key='y', start_date='', end_date=''), True),
+    (partial(url_for, start_date='', end_date=''), False),
 ])
 def test_live_trial_services_toggle_including_from_test_key(
-    include_from_test_key,
+    partial_url_for,
     client,
     platform_admin_user,
     mocker,
@@ -129,12 +131,14 @@ def test_live_trial_services_toggle_including_from_test_key(
 ):
     mock_get_user(mocker, user=platform_admin_user)
     client.login(platform_admin_user)
-    response = client.get(url_for(endpoint, include_from_test_key=include_from_test_key))
+    response = client.get(partial_url_for(endpoint))
 
     assert response.status_code == 200
-    mock_get_detailed_services.assert_called_once_with({'detailed': True,
+    mock_get_detailed_services.assert_called_once_with({
-                                                        'only_active': False,
+        'detailed': True,
-                                                        'include_from_test_key': inc})
+        'only_active': False,
+        'include_from_test_key': inc,
+    })
 
 
 @pytest.mark.parametrize('endpoint', [

tests/app/main/views/test_send.py

@@ -901,6 +901,7 @@ def test_send_test_doesnt_show_file_contents(
     mock_s3_upload,
     mock_get_users_by_service,
     mock_get_service_statistics,
+    mock_has_no_jobs,
     service_one,
     fake_uuid,
     user,
@@ -973,6 +974,7 @@ def test_send_test_step_redirects_if_session_not_setup(
     logged_in_client,
     mock_get_service_statistics,
     mock_get_users_by_service,
+    mock_has_no_jobs,
     fake_uuid,
     endpoint,
     template_mock,
@@ -1080,6 +1082,7 @@ def test_send_one_off_does_not_send_without_the_correct_permissions(
 def test_send_one_off_or_test_has_correct_page_titles(
     logged_in_client,
     service_one,
+    mock_has_no_jobs,
     fake_uuid,
     mocker,
     template_mock,
@@ -1133,6 +1136,7 @@ def test_send_one_off_has_skip_link(
     service_one,
     fake_uuid,
     mock_get_service_email_template,
+    mock_has_no_jobs,
     mocker,
     template_mock,
     expected_link_text,
@@ -1171,6 +1175,7 @@ def test_skip_link_will_not_show_on_sms_one_off_if_service_has_no_mobile_number(
     service_one,
     fake_uuid,
     mock_get_service_template,
+    mock_has_no_jobs,
     mocker,
     user,
 ):
@@ -1204,6 +1209,7 @@ def test_skip_link_will_not_show_on_sms_one_off_if_service_has_no_mobile_number(
 ])
 def test_send_test_redirects_to_end_if_step_out_of_bounds(
     logged_in_client,
+    mock_has_no_jobs,
     service_one,
     fake_uuid,
     endpoint,
@@ -1250,6 +1256,7 @@ def test_send_test_redirects_to_start_if_you_skip_steps(
     mock_s3_upload,
     mock_get_users_by_service,
     mock_get_service_statistics,
+    mock_has_no_jobs,
     mocker,
     endpoint,
     expected_redirect,
@@ -1292,6 +1299,7 @@ def test_send_test_redirects_to_start_if_index_out_of_bounds_and_some_placeholde
     mock_s3_download,
     mock_get_users_by_service,
     mock_get_service_statistics,
+    mock_has_no_jobs,
     endpoint,
     expected_redirect,
     mocker,
@@ -1364,6 +1372,7 @@ def test_send_test_email_message_without_placeholders_redirects_to_check_page(
     mock_s3_upload,
     mock_get_users_by_service,
     mock_get_service_statistics,
+    mock_has_no_jobs,
     fake_uuid,
     user,
 ):
@@ -1392,6 +1401,7 @@ def test_send_test_sms_message_with_placeholders_shows_first_field(
     mock_login,
     mock_get_service,
     mock_get_service_template_with_placeholders,
+    mock_has_no_jobs,
     fake_uuid,
     user,
     expected_back_link_endpoint,

tests/app/main/views/test_service_settings.py

@@ -2041,8 +2041,10 @@ def test_service_switch_can_upload_document_changes_the_permission_if_not_adding
         follow_redirects=True
     )
     page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
-
+    mock_update_service.assert_called_once_with(
-    assert service_one['permissions'] == end_permissions
+        SERVICE_ONE_ID,
+        permissions=end_permissions,
+    )
     assert page.h1.text.strip() == 'Settings'
 
 
@@ -2082,7 +2084,7 @@ def test_service_switch_can_upload_document_lets_contact_link_be_added_and_switc
     )
     page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
 
-    assert 'upload_document' in service_one['permissions']
+    assert 'upload_document' in mock_update_service.call_args[1]['permissions']
     assert page.h1.text.strip() == 'Settings'
 
 
@@ -2451,6 +2453,7 @@ def test_update_basic_view(
 def test_preview_basic_view(
     client_request,
     mock_get_service_templates,
+    mock_has_no_jobs,
 ):
     page = client_request.get(
         "main.preview_basic_view",
@@ -2461,7 +2464,7 @@ def test_preview_basic_view(
     with client_request.session_transaction() as session:
         assert session['basic'] is True
 
-    assert page.h1.text.strip() == 'Choose a template'
+    assert page.h1.text.strip() == 'Templates'
     page.select('.navigation-service-basic-view-preview')
     assert normalize_spaces(page.select_one('.navigation-service').text) == (
         'service one '
@@ -2486,6 +2489,7 @@ def test_preview_basic_view(
 def test_cant_preview_basic_view_for_another_service(
     client_request,
     mock_get_service_templates,
+    mock_has_no_jobs,
     fake_uuid,
 ):
     client_request.get(

tests/app/main/views/test_templates.py

@@ -18,6 +18,8 @@ from tests import (
 )
 from tests.conftest import (
     SERVICE_ONE_ID,
+    SERVICE_TWO_ID,
+    TEMPLATE_ONE_ID,
     active_caseworking_user,
     active_user_view_permissions,
     mock_get_service_email_template,
@@ -70,7 +72,7 @@ from tests.conftest import single_letter_contact_block
         ),
         (
             active_caseworking_user,
-            'Choose a template',
+            'Templates',
             {},
             ['Text message', 'Email'],
             [
@@ -82,7 +84,7 @@ from tests.conftest import single_letter_contact_block
         ),
         (
             active_caseworking_user,
-            'Choose a template',
+            'Templates',
             {'template_type': 'email'},
             ['All', 'Text message'],
             ['email_template_one', 'email_template_two'],
@@ -92,6 +94,7 @@ from tests.conftest import single_letter_contact_block
 def test_should_show_page_for_choosing_a_template(
     client_request,
     mock_get_service_templates,
+    mock_has_no_jobs,
     extra_args,
     expected_nav_links,
     expected_templates,
@@ -394,11 +397,122 @@ def test_dont_show_preview_letter_templates_for_bad_filetype(
     assert mock_get_service_template.called is False
 
 
+def test_choosing_to_copy_redirects(
+    client_request,
+    mock_get_service_templates,
+    mock_get_organisations_and_services_for_user,
+):
+    client_request.post(
+        'main.add_template_by_type',
+        service_id=SERVICE_ONE_ID,
+        _data={'template_type': 'copy-existing'}
+    )
+
+
+def test_choose_a_template_to_copy(
+    client_request,
+    mock_get_service_templates,
+    mock_get_non_empty_organisations_and_services_for_user,
+):
+    page = client_request.get(
+        'main.choose_template_to_copy',
+        service_id=SERVICE_ONE_ID,
+    )
+
+    assert normalize_spaces(
+        page.select_one('main nav').text
+    ) == normalize_spaces(
+        'Service 1 '
+        '    sms_template_one '
+        '    Text message template'
+        '    sms_template_two Text message template'
+        '    email_template_one Email template'
+        '    email_template_two Email template '
+        'Service 2 '
+        '    sms_template_one'
+        '    Text message template'
+        '    sms_template_two'
+        '    Text message template'
+        '    email_template_one'
+        '    Email template'
+        '    email_template_two'
+        '    Email template '
+        'Org 1 service 1 '
+        '    sms_template_one'
+        '    Text message template'
+        '    sms_template_two'
+        '    Text message template'
+        '    email_template_one'
+        '    Email template'
+        '    email_template_two'
+        '    Email template '
+        'Org 1 service 2 '
+        '    sms_template_one'
+        '    Text message template'
+        '    sms_template_two'
+        '    Text message template'
+        '    email_template_one'
+        '    Email template'
+        '    email_template_two'
+        '    Email template'
+    )
+
+    assert page.select_one('main nav a')['href'] == url_for(
+        'main.copy_template',
+        service_id=SERVICE_ONE_ID,
+        template_id=TEMPLATE_ONE_ID,
+        from_service=SERVICE_TWO_ID,
+    )
+
+
+def test_load_edit_template_with_copy_of_template(
+    client_request,
+    mock_get_service_email_template,
+    mock_get_non_empty_organisations_and_services_for_user,
+):
+    page = client_request.get(
+        'main.copy_template',
+        service_id=SERVICE_ONE_ID,
+        template_id=TEMPLATE_ONE_ID,
+        from_service=SERVICE_TWO_ID,
+    )
+
+    assert page.select_one('form')['method'] == 'post'
+
+    assert page.select_one('input')['value'] == (
+        'Copy of ‘Two week reminder’'
+    )
+    assert page.select_one('textarea').text == (
+        'Your ((thing)) is due soon'
+    )
+    mock_get_service_email_template.assert_called_once_with(
+        SERVICE_TWO_ID,
+        TEMPLATE_ONE_ID,
+    )
+
+
+def test_cant_copy_template_from_non_member_service(
+    client_request,
+    mock_get_service_email_template,
+    mock_get_organisations_and_services_for_user,
+):
+    client_request.get(
+        'main.copy_template',
+        service_id=SERVICE_ONE_ID,
+        template_id=TEMPLATE_ONE_ID,
+        from_service=SERVICE_TWO_ID,
+        _expected_status=403,
+    )
+    assert mock_get_service_email_template.call_args_list == []
+
+
 @pytest.mark.parametrize('type_of_template', ['email', 'sms'])
 def test_should_not_allow_creation_of_template_through_form_without_correct_permission(
     logged_in_client,
     service_one,
     mocker,
+    mock_get_service_templates,
+    mock_get_organisations_and_services_for_user,
     type_of_template,
 ):
     service_one['permissions'] = []

tests/app/notify_client/test_billing_client.py

@@ -3,30 +3,6 @@ import uuid
 from app.notify_client.billing_api_client import BillingAPIClient
 
 
-def test_get_billing_units_calls_correct_endpoint(mocker, api_user_active):
-    service_id = uuid.uuid4()
-    expected_url = '/service/{}/billing/monthly-usage'.format(service_id)
-
-    client = BillingAPIClient()
-
-    mock_get = mocker.patch('app.notify_client.billing_api_client.BillingAPIClient.get')
-
-    client.get_billable_units(service_id, 2017)
-    mock_get.assert_called_once_with(expected_url, params={'year': 2017})
-
-
-def test_get_get_service_usage_calls_correct_endpoint(mocker, api_user_active):
-    service_id = uuid.uuid4()
-    expected_url = '/service/{}/billing/yearly-usage-summary'.format(service_id)
-
-    client = BillingAPIClient()
-
-    mock_get = mocker.patch('app.notify_client.billing_api_client.BillingAPIClient.get')
-
-    client.get_service_usage(service_id, 2017)
-    mock_get.assert_called_once_with(expected_url, params={'year': 2017})
-
-
 def test_get_free_sms_fragment_limit_for_year_correct_endpoint(mocker, api_user_active):
     service_id = uuid.uuid4()
     expected_url = '/service/{}/billing/free-sms-fragment-limit'.format(service_id)

tests/app/notify_client/test_job_client.py

@@ -1,6 +1,8 @@
 import uuid
 from unittest.mock import ANY
 
+import pytest
+
 from app.notify_client.job_api_client import JobApiClient
 
 
@@ -8,6 +10,7 @@ def test_client_creates_job_data_correctly(mocker, fake_uuid):
     job_id = fake_uuid
     service_id = fake_uuid
     mocker.patch('app.notify_client.current_user', id='1')
+    mock_redis_set = mocker.patch('app.notify_client.RedisClient.set')
 
     expected_data = {
         "id": job_id,
@@ -24,6 +27,11 @@ def test_client_creates_job_data_correctly(mocker, fake_uuid):
 
     client.create_job(service_id, job_id)
     mock_post.assert_called_once_with(url=expected_url, data=expected_data)
+    mock_redis_set.assert_called_once_with(
+        'has_jobs-{}'.format(service_id),
+        b'true',
+        ex=604800,
+    )
 
 
 def test_client_schedules_job(mocker, fake_uuid):
@@ -300,3 +308,63 @@ def test_cancel_job(mocker):
         url='/service/{}/job/{}/cancel'.format('service_id', 'job_id'),
         data={}
     )
+
+
+@pytest.mark.parametrize('job_data, expected_cache_value', [
+    (
+        [{'data': [1, 2, 3], 'statistics': []}],
+        'true',
+    ),
+    (
+        [],
+        'false',
+    ),
+])
+def test_has_jobs_sets_cache(
+    mocker,
+    fake_uuid,
+    job_data,
+    expected_cache_value,
+):
+    mock_get = mocker.patch(
+        'app.notify_client.job_api_client.JobApiClient.get',
+        return_value={'data': job_data}
+    )
+    mock_redis_set = mocker.patch('app.notify_client.RedisClient.set')
+
+    JobApiClient().has_jobs(fake_uuid)
+
+    mock_get.assert_called_once_with(
+        url='/service/{}/job'.format(fake_uuid),
+        params={'page': 1}
+    )
+    mock_redis_set.assert_called_once_with(
+        'has_jobs-{}'.format(fake_uuid),
+        expected_cache_value,
+        ex=604800,
+    )
+
+
+@pytest.mark.parametrize('cache_value, return_value', [
+    (b'true', True),
+    (b'false', False),
+])
+def test_has_jobs_returns_from_cache(
+    mocker,
+    fake_uuid,
+    cache_value,
+    return_value,
+):
+    mock_get = mocker.patch(
+        'app.notify_client.job_api_client.JobApiClient.get'
+    )
+    mock_redis_get = mocker.patch(
+        'app.notify_client.RedisClient.get',
+        return_value=cache_value,
+    )
+
+    assert JobApiClient().has_jobs(fake_uuid) is return_value
+    assert not mock_get.called
+    mock_redis_get.assert_called_once_with(
+        'has_jobs-{}'.format(fake_uuid)
+    )

tests/app/test_navigation.py

@@ -152,6 +152,7 @@ def test_caseworkers_get_caseworking_navigation(
     mocker,
     fake_uuid,
     mock_get_service_templates,
+    mock_has_no_jobs,
 ):
     mocker.patch(
         'app.user_api_client.get_user',
@@ -159,5 +160,22 @@ def test_caseworkers_get_caseworking_navigation(
     )
     page = client_request.get('main.choose_template', service_id=SERVICE_ONE_ID)
     assert normalize_spaces(page.select_one('#content nav').text) == (
-        'Send a message Sent messages'
+        'Templates Sent messages'
+    )
+
+
+def test_caseworkers_see_jobs_nav_if_jobs_exist(
+    client_request,
+    mocker,
+    fake_uuid,
+    mock_get_service_templates,
+    mock_has_jobs,
+):
+    mocker.patch(
+        'app.user_api_client.get_user',
+        return_value=active_caseworking_user(fake_uuid)
+    )
+    page = client_request.get('main.choose_template', service_id=SERVICE_ONE_ID)
+    assert normalize_spaces(page.select_one('#content nav').text) == (
+        'Templates Sent messages Uploaded files'
     )

tests/app/test_template_previews.py

@@ -49,7 +49,7 @@ def test_from_database_object_makes_request(
 ):
     resp = Mock(content='a', status_code='b', headers={'c': 'd'})
     request_mock = mocker.patch('app.template_previews.requests.post', return_value=resp)
-    mocker.patch('app.template_previews.current_service', __getitem__=Mock(return_value='123'))
+    mocker.patch('app.template_previews.current_service', dvla_organisation='123')
     template = mock_get_service_letter_template('123', '456')['data']
 
     ret = partial_call(template=template)

tests/conftest.py

@@ -692,6 +692,7 @@ def mock_update_service_raise_httperror_duplicate_name(mocker):
 SERVICE_ONE_ID = "596364a0-858e-42c8-9062-a8fe822260eb"
 SERVICE_TWO_ID = "147ad62a-2951-4fa1-9ca0-093cd1a52c52"
 ORGANISATION_ID = "c011fa40-4cbe-4524-b415-dde2f421bd9c"
+TEMPLATE_ONE_ID = "b22d7d94-2197-4a7d-a8e7-fd5f9770bf48"
 
 
 @pytest.fixture(scope='function')
@@ -961,7 +962,7 @@ def mock_update_service_template_400_content_too_big(mocker):
 
 @pytest.fixture(scope='function')
 def mock_get_service_templates(mocker):
-    uuid1 = str(generate_uuid())
+    uuid1 = TEMPLATE_ONE_ID
     uuid2 = str(generate_uuid())
     uuid3 = str(generate_uuid())
     uuid4 = str(generate_uuid())
@@ -1763,6 +1764,16 @@ def mock_get_job_in_progress(mocker, api_user_active):
     return mocker.patch('app.job_api_client.get_job', side_effect=_get_job)
 
 
+@pytest.fixture(scope='function')
+def mock_has_jobs(mocker):
+    mocker.patch('app.job_api_client.has_jobs', return_value=True)
+
+
+@pytest.fixture(scope='function')
+def mock_has_no_jobs(mocker):
+    mocker.patch('app.job_api_client.has_jobs', return_value=False)
+
+
 @pytest.fixture(scope='function')
 def mock_get_jobs(mocker, api_user_active):
     def _get_jobs(service_id, limit_days=None, statuses=None, page=1):
@@ -1822,6 +1833,7 @@ def mock_get_notifications(
         include_jobs=None,
         include_from_test_key=None,
         to=None,
+        include_one_off=None,
     ):
         job = None
         if job_id is not None:
@@ -1870,6 +1882,7 @@ def mock_get_notifications_with_previous_next(mocker):
                            include_jobs=None,
                            include_from_test_key=None,
                            to=None,
+                           include_one_off=None
                            ):
         return notification_json(service_id, with_links=True)
 
@@ -1890,6 +1903,7 @@ def mock_get_notifications_with_no_notifications(mocker):
                            include_jobs=None,
                            include_from_test_key=None,
                            to=None,
+                           include_one_off=None
                            ):
         return notification_json(service_id, rows=0)
 
@@ -3060,6 +3074,30 @@ def mock_get_organisations_and_services_for_user(mocker, organisation_one, api_u
     )
 
 
+@pytest.fixture
+def mock_get_non_empty_organisations_and_services_for_user(mocker, organisation_one, api_user_active):
+
+    def _make_services(name):
+        return [{
+            'name': '{} {}'.format(name, i),
+            'id': SERVICE_TWO_ID,
+        } for i in range(1, 3)]
+
+    def _get_orgs_and_services(user_id):
+        return {
+            'organisations': [
+                {'name': 'Org 1', 'services': _make_services('Org 1 service')},
+                {'name': 'Org 2', 'services': _make_services('Org 2 service')},
+            ],
+            'services_without_organisations': _make_services('Service')
+        }
+
+    return mocker.patch(
+        'app.user_api_client.get_organisations_and_services_for_user',
+        side_effect=_get_orgs_and_services
+    )
+
+
 @pytest.fixture
 def mock_create_event(mocker):
     """