Merge pull request #1109 from alphagov/usage-by-year-2

Break down usage by month, filter by year

app/assets/stylesheets/app.scss

@@ -210,7 +210,7 @@ details summary {
 
   table th,
   table td {
-    font-size: 19px;
+    @include core-19;
   }
 
 }

app/assets/stylesheets/views/dashboard.scss

@@ -75,3 +75,8 @@
   }
 
 }
+
+.failure-highlight {
+  @include bold-19;
+  color: $error-colour;
+}

app/main/views/dashboard.py

@@ -19,7 +19,14 @@ from app import (
     template_statistics_client
 )
 from app.statistics_utils import get_formatted_percentage, add_rate_to_job
-from app.utils import user_has_permissions, get_current_financial_year
+from app.utils import (
+    user_has_permissions,
+    get_current_financial_year,
+    FAILURE_STATUSES,
+    SENDING_STATUSES,
+    DELIVERED_STATUSES,
+    REQUESTED_STATUSES,
+)
 
 
 # This is a placeholder view method to be replaced
@@ -79,11 +86,7 @@ def template_history(service_id):
 @login_required
 @user_has_permissions('manage_settings', admin_override=True)
 def usage(service_id):
-    current_financial_year = get_current_financial_year()
+    year, current_financial_year = requested_and_current_financial_year(request)
-    try:
-        year = int(request.args.get('year', current_financial_year))
-    except ValueError:
-        abort(404)
     return render_template(
         'views/usage.html',
         months=list(get_free_paid_breakdown_for_billable_units(
@@ -102,15 +105,25 @@ def usage(service_id):
     )
 
 
-@main.route("/services/<service_id>/weekly")
+@main.route("/services/<service_id>/monthly")
 @login_required
 @user_has_permissions('manage_settings', admin_override=True)
-def weekly(service_id):
+def monthly(service_id):
-    stats = service_api_client.get_weekly_notification_stats(service_id)['data']
+    year, current_financial_year = requested_and_current_financial_year(request)
     return render_template(
-        'views/weekly.html',
+        'views/dashboard/monthly.html',
-        days=format_weekly_stats_to_list(stats),
+        months=format_monthly_stats_to_list(
-        now=datetime.utcnow()
+            service_api_client.get_monthly_notification_stats(service_id, year)['data']
+        ),
+        years=[
+            (
+                'financial year',
+                year,
+                url_for('.monthly', service_id=service_id, year=year),
+                '{} to {}'.format(year, year + 1),
+            ) for year in range(2015, current_financial_year + 1)
+        ],
+        selected_year=year,
     )
 
 
@@ -198,22 +211,32 @@ def calculate_usage(usage):
     }
 
 
-def format_weekly_stats_to_list(historical_stats):
+def format_monthly_stats_to_list(historical_stats):
-    out = []
+    return sorted((
-    for week, weekly_stats in historical_stats.items():
+        dict(
-        for stats in weekly_stats.values():
+            date=key,
-            stats['failure_rate'] = get_formatted_percentage(stats['failed'], stats['requested'])
+            future=YYYY_MM_to_datetime(key) > datetime.utcnow(),
+            name=YYYY_MM_to_datetime(key).strftime('%B'),
+            **aggregate_status_types(value)
+        ) for key, value in historical_stats.items()
+    ), key=lambda x: x['date'])
 
-        week_start = dateutil.parser.parse(week)
-        week_end = week_start + timedelta(days=6)
-        weekly_stats.update({
-            'week_start': week,
-            'week_end': week_end.date().isoformat(),
-            'week_end_datetime': week_end,
-        })
-        out.append(weekly_stats)
 
-    return sorted(out, key=lambda x: x['week_start'], reverse=True)
+def YYYY_MM_to_datetime(string):
+    return datetime(int(string[0:4]), int(string[5:7]), 1)
+
+
+def aggregate_status_types(counts_dict):
+    return get_dashboard_totals({
+        '{}_counts'.format(message_type): {
+            'failed': sum(
+                stats.get(status, 0) for status in FAILURE_STATUSES
+            ),
+            'requested': sum(
+                stats.get(status, 0) for status in REQUESTED_STATUSES
+            )
+        } for message_type, stats in counts_dict.items()
+    })
 
 
 def get_months_for_financial_year(year):
@@ -269,3 +292,13 @@ def get_free_paid_breakdown_for_month(
             'paid': monthly_usage,
             'free': 0
         }
+
+
+def requested_and_current_financial_year(request):
+    try:
+        return (
+            int(request.args.get('year', get_current_financial_year())),
+            get_current_financial_year(),
+        )
+    except ValueError:
+        abort(404)

app/main/views/jobs.py

@@ -33,6 +33,9 @@ from app.utils import (
     generate_notifications_csv,
     get_help_argument,
     get_template,
+    REQUESTED_STATUSES,
+    FAILURE_STATUSES,
+    SENDING_STATUSES,
 )
 from app.statistics_utils import add_rate_to_job
 
@@ -53,12 +56,10 @@ def _parse_filter_args(filter_dict):
 
 def _set_status_filters(filter_args):
     status_filters = filter_args.get('status', [])
-    all_failure_statuses = ['failed', 'temporary-failure', 'permanent-failure', 'technical-failure']
-    all_sending_statuses = ['created', 'sending']
     return list(OrderedSet(chain(
-        (status_filters or all_sending_statuses + ['delivered'] + all_failure_statuses),
+        (status_filters or REQUESTED_STATUSES),
-        all_sending_statuses if 'sending' in status_filters else [],
+        SENDING_STATUSES if 'sending' in status_filters else [],
-        all_failure_statuses if 'failed' in status_filters else []
+        FAILURE_STATUSES if 'failed' in status_filters else []
     )))
 
 

app/notify_client/service_api_client.py

@@ -219,8 +219,8 @@ class ServiceAPIClient(NotifyAdminAPIClient):
             params=dict(year=year)
         )
 
-    def get_weekly_notification_stats(self, service_id):
+    def get_monthly_notification_stats(self, service_id, year):
-        return self.get(url='/service/{}/notifications/weekly'.format(service_id))
+        return self.get(url='/service/{}/notifications/monthly?year={}'.format(service_id, year))
 
     def get_whitelist(self, service_id):
         return self.get(url='/service/{}/whitelist'.format(service_id))

app/templates/views/dashboard/dashboard.html

@@ -31,8 +31,8 @@
 
     {{ ajax_block(partials, updates_url, 'totals') }}
     {{ show_more(
-      url_for('.weekly', service_id=current_service.id),
+      url_for('.monthly', service_id=current_service.id),
-      'Compare to previous weeks'
+      'See activity breakdown'
     ) }}
 
     {% if partials['has_template_statistics'] %}

app/templates/views/dashboard/monthly.html

@@ -0,0 +1,71 @@
+{% from "components/big-number.html" import big_number_with_status, big_number %}
+{% from "components/pill.html" import pill %}
+{% from "components/table.html" import list_table, field, hidden_field_heading, right_aligned_field_heading, row_heading %}
+{% from "components/message-count-label.html" import message_count_label %}
+
+{% extends "withnav_template.html" %}
+
+{% block page_title %}
+  Previous weeks – GOV.UK Notify
+{% endblock %}
+
+{% block maincolumn_content %}
+
+  <h1 class="heading-large">
+    Activity breakdown
+  </h1>
+  <div class="bottom-gutter">
+    {{ pill(
+      'financial year',
+      items=years,
+      current_value=selected_year,
+      big_number_args={'smallest': True},
+    ) }}
+  </div>
+  {% if months %}
+    <div class="body-copy-table">
+      {% call(month, row_index) list_table(
+        months,
+        caption="Total spend",
+        caption_visible=False,
+        empty_message='',
+        field_headings=[
+          'Month',
+          'Emails',
+          'Text messages',
+        ],
+        field_headings_visible=False
+      ) %}
+        {% if not month.future %}
+          {% call row_heading() %}
+            {{ month.name }}
+          {% endcall %}
+          {% for counts, template_type in [
+            (month.email_counts, 'email'),
+            (month.sms_counts, 'sms')
+          ] %}
+            {% call field(align='left') %}
+              {{ big_number(
+                counts.requested,
+                message_count_label(counts.requested, template_type, suffix=''),
+                smallest=True,
+              ) }}
+              {% if counts.requested %}
+                <span class="{{ 'failure-highlight' if counts.show_warning else '' }}">
+                  {{ counts.failed }} failed
+                </span>
+              {% else %}
+                –
+              {% endif %}
+            {% endcall %}
+          {% endfor %}
+        {% endif %}
+      {% endcall %}
+    </div>
+  {% endif %}
+
+  <p class="align-with-heading-copy">
+    Financial year ends 31 March.
+  </p>
+
+{% endblock %}

app/templates/views/weekly.html

@@ -1,50 +0,0 @@
-{% from "components/table.html" import list_table, field, hidden_field_heading, right_aligned_field_heading %}
-
-{% extends "withnav_template.html" %}
-
-{% block page_title %}
-  Previous weeks – GOV.UK Notify
-{% endblock %}
-
-{% block maincolumn_content %}
-
-  <h1 class="heading-large">
-    Previous weeks
-  </h1>
-  {% call(item, row_number) list_table(
-    days,
-    caption="Daily",
-    caption_visible=False,
-    empty_message='No data found',
-    field_headings=[
-      hidden_field_heading('Day'),
-      right_aligned_field_heading('Emails'),
-      right_aligned_field_heading('Failure rate'),
-      right_aligned_field_heading('Text messages'),
-      right_aligned_field_heading('Failure rate')
-    ]
-  ) %}
-    {% call field() %}
-      {{ item.week_start|format_date_short }} to
-      {% if item.week_end_datetime > now %}
-        today
-      {% else %}
-        {{ item.week_end|format_date_short }}
-      {% endif %}
-    {% endcall %}
-    {% call field(align='right') %}
-      {{ item.email.requested }}
-    {% endcall %}
-    {% call field(align='right') %}
-      {{ item.email.failure_rate }}%
-    {% endcall %}
-    {% call field(align='right') %}
-      {{ item.sms.requested }}
-    {% endcall %}
-    {% call field(align='right') %}
-      {{ item.sms.failure_rate }}%
-    {% endcall %}
-  {% endcall %}
-
-
-{% endblock %}

app/utils.py

@@ -32,6 +32,12 @@ import pyexcel.ext.xlsx
 import pyexcel.ext.ods3
 
 
+SENDING_STATUSES = ['created', 'pending', 'sending']
+DELIVERED_STATUSES = ['delivered']
+FAILURE_STATUSES = ['failed', 'temporary-failure', 'permanent-failure', 'technical-failure']
+REQUESTED_STATUSES = SENDING_STATUSES + DELIVERED_STATUSES + FAILURE_STATUSES
+
+
 class BrowsableItem(object):
     """
     Maps for the template browse-list.

tests/app/main/views/test_dashboard.py

@@ -8,8 +8,9 @@ from freezegun import freeze_time
 
 from app.main.views.dashboard import (
     get_dashboard_totals,
-    format_weekly_stats_to_list,
+    format_monthly_stats_to_list,
-    get_free_paid_breakdown_for_billable_units
+    get_free_paid_breakdown_for_billable_units,
+    aggregate_status_types,
 )
 
 from tests import validate_route_permission
@@ -534,49 +535,67 @@ def test_get_dashboard_totals_adds_warning(failures, expected):
     assert get_dashboard_totals(stats)['sms']['show_warning'] == expected
 
 
-def test_format_weekly_stats_to_list_empty_case():
+def test_format_monthly_stats_empty_case():
-    assert format_weekly_stats_to_list({}) == []
+    assert format_monthly_stats_to_list({}) == []
 
 
-def test_format_weekly_stats_to_list_sorts_by_week():
+def test_format_monthly_stats_labels_month():
-    stats = {
+    resp = format_monthly_stats_to_list({'2016-07': {}})
-        '2016-07-04': {},
+    assert resp[0]['name'] == 'July'
-        '2016-07-11': {},
+
-        '2016-07-18': {},
+
-        '2016-07-25': {}
+def test_format_monthly_stats_has_stats_with_failure_rate():
+    resp = format_monthly_stats_to_list({
+        '2016-07': {'sms': _stats(3, 1, 2)}
+    })
+    assert resp[0]['sms_counts'] == {
+        'failed': 2,
+        'failed_percentage': '66.7',
+        'requested': 3,
+        'show_warning': True,
     }
-    resp = format_weekly_stats_to_list(stats)
-    assert resp[0]['week_start'] == '2016-07-25'
-    assert resp[1]['week_start'] == '2016-07-18'
-    assert resp[2]['week_start'] == '2016-07-11'
-    assert resp[3]['week_start'] == '2016-07-04'
 
 
-def test_format_weekly_stats_to_list_includes_datetime_for_comparison():
+def test_format_monthly_stats_works_for_email_letter():
-    stats = {
+    resp = format_monthly_stats_to_list({
-        '2016-07-25': {}
+        '2016-07': {
-    }
+            'sms': {},
-    resp = format_weekly_stats_to_list(stats)
+            'email': {},
-    assert resp == [{
+            'letter': {},
-        'week_start': '2016-07-25',
+        }
-        'week_end': '2016-07-31',
+    })
-        'week_end_datetime': datetime(2016, 7, 31, 0, 0, 0)
+    assert isinstance(resp[0]['sms_counts'], dict)
-    }]
+    assert isinstance(resp[0]['email_counts'], dict)
-
+    assert isinstance(resp[0]['letter_counts'], dict)
-
-def test_format_weekly_stats_to_list_has_stats_with_failure_rate():
-    stats = {
-        '2016-07-25': {'sms': _stats(3, 1, 2)}
-    }
-    resp = format_weekly_stats_to_list(stats)
-    assert resp[0]['sms']['failure_rate'] == '66.7'
-    assert resp[0]['sms']['requested'] == 3
 
 
 def _stats(requested, delivered, failed):
     return {'requested': requested, 'delivered': delivered, 'failed': failed}
 
 
+@pytest.mark.parametrize('dict_in, expected_failed, expected_requested', [
+    (
+        {},
+        0,
+        0
+    ),
+    (
+        {'temporary-failure': 1, 'permanent-failure': 1, 'technical-failure': 1},
+        3,
+        3,
+    ),
+    (
+        {'created': 1, 'pending': 1, 'sending': 1, 'delivered': 1},
+        0,
+        4,
+    ),
+])
+def test_aggregate_status_types(dict_in, expected_failed, expected_requested):
+    sms_counts = aggregate_status_types({'sms': dict_in})['sms_counts']
+    assert sms_counts['failed'] == expected_failed
+    assert sms_counts['requested'] == expected_requested
+
+
 @pytest.mark.parametrize(
     'now, expected_number_of_months', [
         (freeze_time("2017-12-31 11:09:00.061258"), 12),

tests/app/main/views/test_jobs.py

@@ -68,11 +68,15 @@ def test_get_jobs_shows_page_links(
     "status_argument, expected_api_call", [
         (
             '',
-            ['created', 'sending', 'delivered', 'failed', 'temporary-failure', 'permanent-failure', 'technical-failure']
+            [
+                'created', 'pending', 'sending',
+                'delivered',
+                'failed', 'temporary-failure', 'permanent-failure', 'technical-failure',
+            ]
         ),
         (
             'sending',
-            ['sending', 'created']
+            ['sending', 'created', 'pending']
         ),
         (
             'delivered',
@@ -304,11 +308,15 @@ def test_should_show_updates_for_one_job_as_json(
     "status_argument, expected_api_call", [
         (
             '',
-            ['created', 'sending', 'delivered', 'failed', 'temporary-failure', 'permanent-failure', 'technical-failure']
+            [
+                'created', 'pending', 'sending',
+                'delivered',
+                'failed', 'temporary-failure', 'permanent-failure', 'technical-failure',
+            ]
         ),
         (
             'sending',
-            ['sending', 'created']
+            ['sending', 'created', 'pending']
         ),
         (
             'delivered',