Merged with main, fixed a test

.ds.baseline

@@ -133,7 +133,7 @@
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 67,
+        "line_number": 68,
         "is_secret": false
       },
       {
@@ -141,7 +141,7 @@
         "filename": ".github/workflows/checks.yml",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 101,
+        "line_number": 103,
         "is_secret": false
       }
     ],
@@ -692,5 +692,5 @@
       }
     ]
   },
-  "generated_at": "2024-07-24T14:13:02Z"
+  "generated_at": "2024-08-07T14:36:40Z"
 }

.github/workflows/checks.yml

@@ -54,6 +54,7 @@ jobs:
         run: poetry run coverage report --fail-under=90
 
   end-to-end-tests:
+    if: ${{ github.actor != 'dependabot[bot]' }}
     permissions:
       checks: write
       pull-requests: write
@@ -84,6 +85,7 @@ jobs:
         ports:
           # Maps tcp port 6379 on service container to the host
           - 6379:6379
+
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-project
@@ -136,6 +138,12 @@ jobs:
         # Debugging for now to troubleshoot a connectivity issue to the local servers
         # run: curl --request GET --url "http://localhost:6012"
         env:
+          API_HOST_NAME: http://localhost:6011
+          DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }}
+          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }}
+          ADMIN_CLIENT_USERNAME: notify-admin
+
           NOTIFY_ENVIRONMENT: e2etest
           NOTIFY_E2E_AUTH_STATE_PATH: ${{ secrets.NOTIFY_E2E_AUTH_STATE_PATH }}
           NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }}

app/__init__.py

@@ -177,7 +177,7 @@ def create_app(application):
 
     init_govuk_frontend(application)
     init_jinja(application)
-    socketio.init_app(application)
+    socketio.init_app(application, cors_allowed_origins=['http://localhost:6012'])
 
     for client in (
         csrf,

app/assets/javascripts/activityChart.js

@@ -185,7 +185,7 @@
             return;
         }
 
-        var socket = io();
+        var socket = io("/services");
         var eventType = type === 'service' ? 'fetch_daily_stats' : 'fetch_daily_stats_by_user';
         var socketConnect = type === 'service' ? 'daily_stats_update' : 'daily_stats_by_user_update';
 
@@ -193,6 +193,10 @@
             socket.emit(eventType);
         });
 
+        socket.on('connect_error', function(error) {
+            console.error('WebSocket connection error:', error);
+        });
+
         socket.on(socketConnect, function(data) {
 
             var labels = [];

app/assets/javascripts/totalMessagesChart.js

@@ -14,14 +14,14 @@
         document.getElementById('message').innerText = `${sms_sent.toLocaleString()} sent / ${sms_remaining_messages.toLocaleString()} remaining`;
 
         // Calculate minimum width for "Messages Sent" as 1% of the total chart width
-        var minSentPercentage = 0.01; // Minimum width as a percentage of total messages (1% in this case)
+        var minSentPercentage = 0.02; // Minimum width as a percentage of total messages (1% in this case)
         var minSentValue = totalMessages * minSentPercentage;
         var displaySent = Math.max(sms_sent, minSentValue);
         var displayRemaining = totalMessages - displaySent;
 
         var svg = d3.select("#totalMessageChart");
         var width = chartContainer.clientWidth;
-        var height = 64;
+        var height = 48;
 
         // Ensure the width is set correctly
         if (width === 0) {
@@ -62,7 +62,7 @@
             .attr("x", 0) // Initially set to 0, will be updated during animation
             .attr("y", 0)
             .attr("height", height)
-            .attr("fill", '#fa9441')
+            .attr("fill", '#C7CACE')
             .attr("width", 0) // Start with width 0 for animation
             .on('mouseover', function(event) {
                 tooltip.style('display', 'block')

app/assets/sass/uswds/_data-visualization.scss

@@ -2,7 +2,7 @@
 
 $delivered: color('blue-50v');
 $pending: color('green-cool-40v');
-$failed: color('orange-30v');
+$failed: color('gray-cool-20');
 
 .chart-container {
   display: flex;
@@ -11,6 +11,10 @@ $failed: color('orange-30v');
   }
 }
 
+#totalMessageChartContainer {
+  max-width: 600px;
+}
+
 .bar {
   border-radius: units(0.5);
   &.delivered, &.usage {

app/main/views/dashboard.py

@@ -32,7 +32,7 @@ from app.utils.user import user_has_permissions
 from notifications_utils.recipients import format_phone_number_human_readable
 
 
-@socketio.on("fetch_daily_stats")
+@socketio.on("fetch_daily_stats", namespace="/services")
 def handle_fetch_daily_stats():
     service_id = session.get("service_id")
     if service_id:
@@ -45,7 +45,7 @@ def handle_fetch_daily_stats():
         emit("error", {"error": "No service_id provided"})
 
 
-@socketio.on("fetch_daily_stats_by_user")
+@socketio.on("fetch_daily_stats_by_user", namespace="/services")
 def handle_fetch_daily_stats_by_user():
     service_id = session.get("service_id")
     user_id = session.get("user_id")
@@ -113,6 +113,7 @@ def service_dashboard(service_id):
             "original_file_name": job["original_file_name"],
         }
         for job in job_response
+        if job["job_status"] != "cancelled"
     ]
     return render_template(
         "views/dashboard/dashboard.html",

app/notify_client/__init__.py

@@ -56,6 +56,9 @@ class NotifyAdminAPIClient(BaseAPIClient):
         ):
             abort(403)
 
+    def is_calling_signin_url(self, arg):
+        return arg.startswith("('/user")
+
     def check_inactive_user(self, *args):
         still_signing_in = False
 
@@ -64,14 +67,7 @@ class NotifyAdminAPIClient(BaseAPIClient):
         # and we only want to check the first arg
         for arg in args:
             arg = str(arg)
-            if (
-                "get-login-gov-user" in arg
-                or "user/email" in arg
-                or "/activate" in arg
-                or "/email-code" in arg
-                or "/verify/code" in arg
-                or "/user" in arg
-            ):
+            if self.is_calling_signin_url(arg):
                 still_signing_in = True
 
             # This seems to be a weird edge case that happens intermittently with invites

app/notify_client/job_api_client.py

@@ -1,8 +1,6 @@
 import datetime
 from zoneinfo import ZoneInfo
 
-from flask import current_app
-
 from app.extensions import redis_client
 from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 from app.utils.csv import get_user_preferred_timezone
@@ -41,8 +39,6 @@ class JobApiClient(NotifyAdminAPIClient):
             params["statuses"] = ",".join(statuses)
 
         job = self.get(url=f"/service/{service_id}/job", params=params)
-        from pprint import pformat
-        current_app.logger.info(pformat(job))
         return job
 
     def get_uploads(self, service_id, limit_days=None, page=1):

app/templates/views/dashboard/dashboard.html

@@ -28,10 +28,6 @@
       <div id="message"></div>
     </div>
     <div id="totalMessageTable" class="margin-0"></div>
-    <p class="align-with-heading-copy margin-bottom-4">
-      What counts as 1 text message part?<br />
-      See <a class="usa-link" href="{{ url_for('.pricing') }}">Tracking usage</a>.
-    </p>
 
     <h2 class="line-height-sans-2 margin-bottom-0 margin-top-4">
       Activity snapshot

docs/debug-issues-with-staging-api.md

@@ -0,0 +1,26 @@
+### Setting Up Environment Variables for Local Development to the Staging API
+
+When you’re working locally, you can point your local admin repo to the staging API and use that to help debug
+issues with the staging data set.  To do this, you’ll need to modify your .env file for the admin project and
+include the following new environment variables:
+
+- `ADMIN_CLIENT_SECRET`
+- `ADMIN_CLIENT_USERNAME`
+- `DANGEROUS_SALT`
+- `SECRET_KEY`
+
+Additionally, update `API_HOST_NAME` and `NOTIFY_ENVIRONMENT`:
+
+1. Change `API_HOST_NAME` to `API_HOST_NAME=https://notify-api-staging.app.cloud.gov`
+2. Change `NOTIFY_ENVIRONMENT` to `NOTIFY_ENVIRONMENT=staging`
+
+### Retrieving Environment Variables for Staging
+
+You can retrieve the values needed for these by using the `cf` CLI (Cloud Foundry CLI tool) and making sure
+you’re targeting the `notify-staging` space.
+
+1. `cf login -a [api.fr.cloud.gov](http://api.fr.cloud.gov/) --sso`
+2. select `notify-staging`
+3. `cf env notify-admin-staging`
+
+By pointing your local environment to staging, it should mirror what's in staging.

poetry.lock

@@ -201,13 +201,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.34.150"
+version = "1.34.156"
 description = "Low-level, data-driven core of boto 3."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "botocore-1.34.150-py3-none-any.whl", hash = "sha256:b988d47f4d502df85befce11a48002421e4e6ea4289997b5e0261bac5fa76ce6"},
-    {file = "botocore-1.34.150.tar.gz", hash = "sha256:4d23387e0f076d87b637a2a35c0ff2b8daca16eace36b63ce27f65630c6b375a"},
+    {file = "botocore-1.34.156-py3-none-any.whl", hash = "sha256:c48f8c8996216dfdeeb0aa6d3c0f2c7ae25234766434a2ea3e57bdc08494bdda"},
+    {file = "botocore-1.34.156.tar.gz", hash = "sha256:5d1478c41ab9681e660b3322432fe09c4055759c317984b7b8d3af9557ff769a"},
 ]
 
 [package.dependencies]
@@ -216,7 +216,7 @@ python-dateutil = ">=2.1,<3.0.0"
 urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}
 
 [package.extras]
-crt = ["awscrt (==0.20.11)"]
+crt = ["awscrt (==0.21.2)"]
 
 [[package]]
 name = "cachecontrol"
@@ -1325,13 +1325,9 @@ files = [
     {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"},
     {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"},
     {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"},
-    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"},
     {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"},
-    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"},
     {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"},
-    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"},
     {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"},
-    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"},
     {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"},
     {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"},
     {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"},
@@ -1666,6 +1662,7 @@ files = [
     {file = "msgpack-1.0.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fbb160554e319f7b22ecf530a80a3ff496d38e8e07ae763b9e82fadfe96f273"},
     {file = "msgpack-1.0.8-cp39-cp39-win32.whl", hash = "sha256:f9af38a89b6a5c04b7d18c492c8ccf2aee7048aff1ce8437c4683bb5a1df893d"},
     {file = "msgpack-1.0.8-cp39-cp39-win_amd64.whl", hash = "sha256:ed59dd52075f8fc91da6053b12e8c89e37aa043f8986efd89e61fae69dc1b011"},
+    {file = "msgpack-1.0.8-py3-none-any.whl", hash = "sha256:24f727df1e20b9876fa6e95f840a2a2651e34c0ad147676356f4bf5fbb0206ca"},
     {file = "msgpack-1.0.8.tar.gz", hash = "sha256:95c02b0e27e706e48d0e5426d1710ca78e0f0628d6e89d5b5a5b91a5f12274f3"},
 ]
 
@@ -2512,7 +2509,6 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -3092,4 +3088,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12.2"
-content-hash = "b271104f669ce0a8e78fb09299b61cf0502cc81a18213dda00f77c759b6e0209"
+content-hash = "9d6309a76755b2639d787f99944c1ead0bd93ab9f2f13208f88c51cecb5e0081"

pyproject.toml

@@ -39,7 +39,7 @@ markdown = "^3.5.2"
 async-timeout = "^4.0.3"
 bleach = "^6.1.0"
 boto3 = "^1.34.150"
-botocore = "^1.34.150"
+botocore = "^1.34.156"
 cachetools = "^5.4.0"
 cffi = "^1.16.0"
 cryptography = "^43.0.0"

tests/app/notify_client/test_notify_admin_api_client.py

@@ -41,6 +41,26 @@ def test_active_service_can_be_modified(notify_admin, method, user, service):
     assert ret == request.return_value
 
 
+@pytest.mark.parametrize(
+    ("arg", "expected_result"),
+    [
+        (
+            "('/user/c5f8a5c9-56d5-4fa9-8c30-3449ae10c072/verify/code',)",
+            True,
+        ),
+        ("('/user/get-login-gov-user',)", True),
+        (
+            "('/service/blahblahblah',)",
+            False,
+        ),
+    ],
+)
+def test_is_calling_signin_url(arg, expected_result):
+    api_client = NotifyAdminAPIClient()
+    result = api_client.is_calling_signin_url(arg)
+    assert result == expected_result
+
+
 @pytest.mark.parametrize("method", ["put", "post", "delete"])
 def test_inactive_service_cannot_be_modified_by_normal_user(
     notify_admin, api_user_active, method

tests/javascripts/activityChart.test.js

@@ -221,3 +221,43 @@ test('Dropdown change event triggers handleDropdownChange with logging', () => {
   // Restore the console method
   consoleSpy.mockRestore();
 });
+
+
+test('Initial fetch data populates chart and table', done => {
+  const mockData = {
+    '2024-07-01': { sms: { delivered: 50, failed: 5 } },
+    '2024-07-02': { sms: { delivered: 60, failed: 2 } },
+    '2024-07-03': { sms: { delivered: 70, failed: 1 } },
+    '2024-07-04': { sms: { delivered: 80, failed: 0 } },
+    '2024-07-05': { sms: { delivered: 90, failed: 3 } },
+    '2024-07-06': { sms: { delivered: 100, failed: 4 } },
+    '2024-07-07': { sms: { delivered: 110, failed: 2 } },
+  };
+
+  const socket = {
+    on: jest.fn((event, callback) => {
+      if (event === 'daily_stats_update') {
+        callback(mockData);
+        done();
+      }
+    }),
+    emit: jest.fn(),
+  };
+  window.io = jest.fn(() => socket);
+
+  document.dispatchEvent(new Event('DOMContentLoaded'));
+
+  setTimeout(() => {
+    const table = document.getElementById('weeklyTable');
+    expect(table).toBeDefined();
+
+    const rows = table.getElementsByTagName('tr');
+    expect(rows.length).toBe(8);
+
+    const firstRowCells = rows[1].getElementsByTagName('td');
+    console.log('First row cells:', firstRowCells);
+    expect(firstRowCells[0].textContent).toBe('07/01/24');
+    expect(firstRowCells[1].textContent).toBe('50');
+    expect(firstRowCells[2].textContent).toBe('5');
+  }, 100);
+});

tests/javascripts/totalMessagesChart.test.js

@@ -15,10 +15,11 @@ function loadScript(scriptContent) {
 Object.defineProperty(HTMLElement.prototype, 'clientWidth', {
   value: 600,
   writable: true,
+  configurable: true,
 });
 
 // beforeAll hook to set up the DOM and load D3.js script
-beforeAll(done => {
+beforeEach(() => {
   // Set up the DOM with the D3 script included
   document.body.innerHTML = `
     <div id="totalMessageChartContainer" data-sms-sent="100" data-sms-allowance-remaining="249900" style="width: 600px;">
@@ -33,15 +34,13 @@ beforeAll(done => {
   loadScript(d3ScriptContent);
 
   // Wait a bit to ensure the script is executed
-  setTimeout(() => {
-    // Require the actual JavaScript file you are testing
-    require('../../app/assets/javascripts/totalMessagesChart.js');
-
-    // Call the function to create the chart
-    window.createTotalMessagesChart();
-
-    done();
-  }, 100);
+  return new Promise(resolve => {
+    setTimeout(() => {
+      // Require the actual JavaScript file you are testing
+      require('../../app/assets/javascripts/totalMessagesChart.js');
+      resolve();
+    }, 100);
+  });
 });
 
 // Single test to check if D3 is loaded correctly
@@ -52,15 +51,20 @@ test('D3 is loaded correctly', () => {
 });
 
 // Test to check if the SVG element is correctly set up
-test('SVG element is correctly set up', () => {
-  const svg = document.getElementById('totalMessageChart');
-  expect(svg).not.toBeNull();
-  expect(svg.getAttribute('width')).toBe('600');
-  expect(svg.getAttribute('height')).toBe('64');
+test('SVG element is correctly set up', done => {
+  window.createTotalMessagesChart();
+
+  setTimeout(() => {
+    const svg = document.getElementById('totalMessageChart');
+    expect(svg.getAttribute('width')).toBe('600');
+    expect(svg.getAttribute('height')).toBe('48');
+    done();
+  }, 1000); // Ensure enough time for the DOM updates
 });
 
 // Test to check if the table is created and populated correctly
 test('Populates the accessible table correctly', () => {
+  window.createTotalMessagesChart();
   const table = document.getElementById('totalMessageTable').getElementsByTagName('table')[0];
   expect(table).toBeDefined();
 
@@ -84,6 +88,8 @@ test('Chart title is correctly set', () => {
 
 // Test to check if the chart resizes correctly on window resize
 test('Chart resizes correctly on window resize', done => {
+  window.createTotalMessagesChart();
+
   setTimeout(() => {
     const svg = document.getElementById('totalMessageChart');
     const chartContainer = document.getElementById('totalMessageChartContainer');
@@ -92,7 +98,7 @@ test('Chart resizes correctly on window resize', done => {
     expect(svg.getAttribute('width')).toBe('600');
 
     // Set new container width
-    Object.defineProperty(chartContainer, 'clientWidth', { value: 800 });
+    Object.defineProperty(chartContainer, 'clientWidth', { value: 800, configurable: true });
 
     // Trigger resize event
     window.dispatchEvent(new Event('resize'));
@@ -101,9 +107,9 @@ test('Chart resizes correctly on window resize', done => {
       // Check if SVG width is updated
       expect(svg.getAttribute('width')).toBe('800');
       done();
-    }, 500); // Adjust the timeout if necessary
+    }, 1000); // Adjust the timeout if necessary
   }, 1000); // Initial wait for the chart to render
-}, 10000); // Adjust the overall test timeout if necessary
+}, 15000); // Adjust the overall test timeout if necessary
 
 // Testing the tooltip
 test('Tooltip displays on hover', () => {
@@ -148,11 +154,12 @@ test('Tooltip displays on hover', () => {
 
 // Test to ensure SVG bars are created and animated correctly
 test('SVG bars are created and animated correctly', done => {
+  window.createTotalMessagesChart();
   const svg = document.getElementById('totalMessageChart');
 
   // Initial check
   const sentBar = svg.querySelector('rect[fill="#0076d6"]');
-  const remainingBar = svg.querySelector('rect[fill="#fa9441"]');
+  const remainingBar = svg.querySelector('rect[fill="#C7CACE"]');
 
   expect(sentBar).not.toBeNull();
   expect(remainingBar).not.toBeNull();