diff --git a/.ds.baseline b/.ds.baseline index 0ded707f2..999298014 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -684,5 +684,5 @@ } ] }, - "generated_at": "2025-01-16T16:38:48Z" + "generated_at": "2025-02-03T17:01:06Z" } diff --git a/.github/ISSUE_TEMPLATE/issue_template.yml b/.github/ISSUE_TEMPLATE/issue_template.yml index 576af0095..f9cabaa72 100644 --- a/.github/ISSUE_TEMPLATE/issue_template.yml +++ b/.github/ISSUE_TEMPLATE/issue_template.yml @@ -64,6 +64,17 @@ body: validations: required: false + - type: markdown + attributes: + value: '**Accessibility:**' + - type: textarea + id: accessibility + attributes: + label: "List any specific accessibility guidance or tests that need to be considered for this user story." + description: "List what type of accessibility tests need to pass." + validations: + required: false + - type: markdown attributes: value: '**Notes:**' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7ffbbc290..a659829b2 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -23,5 +23,7 @@ Please enter a detailed description here. ## A11y Checks (if applicable) -* Conduct automated tests through [AxeDevTools](https://www.deque.com/axe/devtools/) and [WAVE](https://wave.webaim.org/) +* Double check work is getting picked up by the automated E2E tests +* Conduct browser-based tests through [AxeDevTools](https://www.deque.com/axe/devtools/) and [WAVE](https://wave.webaim.org/) * Review the [Manual Checklist](https://docs.google.com/document/d/192bBXStebdXWtYhZQ73qaWMJhGcuSB1W6c9YBXhWZvc/edit?usp=sharing) +* Make sure there are no linting errors in VSCode or other IDE of choice diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c3ef5dcbb..87d7b338e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -10,7 +10,7 @@ env: FLASK_APP: application.py WERKZEUG_DEBUG_PIN: off REDIS_ENABLED: 0 - NODE_VERSION: 16.15.1 + NODE_VERSION: 22.3.0 AWS_US_TOLL_FREE_NUMBER: "+18556438890" ADMIN_BASE_URL: http://localhost:6012 @@ -38,10 +38,10 @@ jobs: output: report-markdown annotations: failed-tests prnumber: ${{ steps.findPr.outputs.number }} - - name: Run style checks - run: poetry run flake8 . - name: Check imports alphabetized run: poetry run isort --check-only ./app ./tests + - name: Run style checks + run: poetry run flake8 . - name: Check dead code run: make dead-code - name: Run js tests @@ -139,7 +139,7 @@ jobs: - uses: ./.github/actions/setup-project - name: Create requirements.txt run: poetry export --without-hashes --format=requirements.txt > requirements.txt - - uses: pypa/gh-action-pip-audit@v1.0.8 + - uses: pypa/gh-action-pip-audit@v1.1.0 with: inputs: requirements.txt ignore-vulns: | @@ -165,8 +165,9 @@ jobs: run: make run-flask & env: NOTIFY_ENVIRONMENT: scanning + FEATURE_ABOUT_PAGE_ENABLED: true - name: Run OWASP Baseline Scan - uses: zaproxy/action-baseline@v0.9.0 + uses: zaproxy/action-baseline@v0.14.0 with: docker_name: "ghcr.io/zaproxy/zaproxy:weekly" target: "http://localhost:6012" diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index a5e81a137..c2d3082a3 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -16,7 +16,7 @@ env: FLASK_APP: application.py WERKZEUG_DEBUG_PIN: off REDIS_ENABLED: 0 - NODE_VERSION: 16.15.1 + NODE_VERSION: 22.3.0 jobs: dependency-audits: @@ -26,7 +26,7 @@ jobs: - uses: ./.github/actions/setup-project - name: Create requirements.txt run: poetry export --without-hashes --format=requirements.txt > requirements.txt - - uses: pypa/gh-action-pip-audit@v1.0.6 + - uses: pypa/gh-action-pip-audit@v1.1.0 with: inputs: requirements.txt - name: Run npm audit @@ -50,7 +50,7 @@ jobs: env: NOTIFY_ENVIRONMENT: scanning - name: Run OWASP Full Scan - uses: zaproxy/action-full-scan@v0.7.0 + uses: zaproxy/action-full-scan@v0.12.0 with: docker_name: 'ghcr.io/zaproxy/zaproxy:weekly' target: 'http://localhost:6012' diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index 972240c9a..70eecf943 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -16,9 +16,17 @@ jobs: with: fetch-depth: 2 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: files: | terraform/demo/** @@ -93,7 +101,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: files: | deploy-config/egress_proxy/notify-admin-demo.*.acl diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 334146485..45ae45244 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -16,9 +16,17 @@ jobs: with: fetch-depth: 2 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: files: | terraform/production/** @@ -93,7 +101,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: files: | deploy-config/egress_proxy/notify-admin-production.*.acl diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 19cb15faa..645bcc44d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -21,9 +21,17 @@ jobs: with: fetch-depth: 2 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Check for changes to Terraform id: changed-terraform-files - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: files: | terraform/staging/** @@ -100,7 +108,7 @@ jobs: - name: Check for changes to egress config id: changed-egress-config - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: files: | deploy-config/egress_proxy/notify-admin-staging.*.acl diff --git a/.github/workflows/drift.yml b/.github/workflows/drift.yml index 5f495ceba..62fa1b3c7 100644 --- a/.github/workflows/drift.yml +++ b/.github/workflows/drift.yml @@ -15,6 +15,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Check for drift uses: dflook/terraform-check@v1 env: @@ -35,6 +43,14 @@ jobs: with: ref: 'production' + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Check for drift uses: dflook/terraform-check@v1 env: @@ -55,6 +71,14 @@ jobs: with: ref: 'production' + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Check for drift uses: dflook/terraform-check@v1 env: diff --git a/.github/workflows/terraform-demo.yml b/.github/workflows/terraform-demo.yml index 765f9e53b..1910de574 100644 --- a/.github/workflows/terraform-demo.yml +++ b/.github/workflows/terraform-demo.yml @@ -18,6 +18,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Terraform format id: format run: terraform fmt -check @@ -51,7 +59,7 @@ jobs: # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow - name: Update PR - uses: actions/github-script@v6 + uses: actions/github-script@v7 # we would like to update the PR even when a prior step failed if: ${{ always() }} with: diff --git a/.github/workflows/terraform-production.yml b/.github/workflows/terraform-production.yml index 1988c5d95..91b3a48b9 100644 --- a/.github/workflows/terraform-production.yml +++ b/.github/workflows/terraform-production.yml @@ -18,6 +18,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Terraform format id: format run: terraform fmt -check @@ -51,7 +59,7 @@ jobs: # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow - name: Update PR - uses: actions/github-script@v6 + uses: actions/github-script@v7 # we would like to update the PR even when a prior step failed if: ${{ always() }} with: diff --git a/.github/workflows/terraform-staging.yml b/.github/workflows/terraform-staging.yml index 1f86ee033..9fe174ee8 100644 --- a/.github/workflows/terraform-staging.yml +++ b/.github/workflows/terraform-staging.yml @@ -18,6 +18,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Looks like we need to install Terraform ourselves now! + # https://github.com/actions/runner-images/issues/10796#issuecomment-2417064348 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "^1.7.5" + terraform_wrapper: false + - name: Terraform format id: format run: terraform fmt -check @@ -51,7 +59,7 @@ jobs: # inspiration: https://learn.hashicorp.com/tutorials/terraform/github-actions#review-actions-workflow - name: Update PR - uses: actions/github-script@v6 + uses: actions/github-script@v7 # we would like to update the PR even when a prior step failed if: ${{ always() }} with: diff --git a/Makefile b/Makefile index 9e0eeb46e..66ce809bb 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,13 @@ py-lint: ## Run python linting scanners and black poetry run flake8 . poetry run isort --check-only ./app ./tests +.PHONY: tada +tada: ## Run python linting scanners and black + poetry run isort ./app ./tests + poetry run black . + poetry run flake8 . + + .PHONY: avg-complexity avg-complexity: echo "*** Shows average complexity in radon of all code ***" diff --git a/app/assets/javascripts/activityChart.js b/app/assets/javascripts/activityChart.js index a3f615c65..9ba1e4338 100644 --- a/app/assets/javascripts/activityChart.js +++ b/app/assets/javascripts/activityChart.js @@ -1,11 +1,14 @@ (function (window) { if (document.getElementById('activityChartContainer')) { + let currentType = 'service'; const tableContainer = document.getElementById('activityContainer'); const currentUserName = tableContainer.getAttribute('data-currentUserName'); + const currentServiceId = tableContainer.getAttribute('data-currentServiceId'); const COLORS = { delivered: '#0076d6', failed: '#fa9441', + pending: '#C7CACE', text: '#666' }; @@ -13,7 +16,7 @@ const FONT_WEIGHT = 'bold'; const MAX_Y = 120; - const createChart = function(containerId, labels, deliveredData, failedData) { + const createChart = function(containerId, labels, deliveredData, failedData, pendingData) { const container = d3.select(containerId); container.selectAll('*').remove(); // Clear any existing content @@ -36,7 +39,7 @@ } // Calculate total messages - const totalMessages = d3.sum(deliveredData) + d3.sum(failedData); + const totalMessages = d3.sum(deliveredData) + d3.sum(failedData) + d3.sum(pendingData); // Create legend only if there are messages const legendContainer = d3.select('.chart-legend'); @@ -46,7 +49,8 @@ // Show legend if there are messages const legendData = [ { label: 'Delivered', color: COLORS.delivered }, - { label: 'Failed', color: COLORS.failed } + { label: 'Failed', color: COLORS.failed }, + { label: 'Pending', color: COLORS.pending } ]; const legendItem = legendContainer.selectAll('.legend-item') @@ -77,8 +81,9 @@ .range([0, width]) .padding(0.1); // Adjust the y-axis domain to add some space above the tallest bar - const maxY = d3.max(deliveredData.map((d, i) => d + (failedData[i] || 0))); - const y = d3.scaleSqrt() + const maxY = d3.max(deliveredData.map((d, i) => d + (failedData[i] || 0) + (pendingData[i] || 0))); + + const y = d3.scaleSymlog() .domain([0, maxY + 2]) // Add 2 units of space at the top .nice() .range([height, 0]); @@ -90,7 +95,7 @@ // Generate the y-axis with whole numbers const yAxis = d3.axisLeft(y) - .ticks(Math.min(maxY + 2, 10)) // Generate up to 10 ticks based on the data + .ticks(Math.min(maxY + 2, 3)) .tickFormat(d3.format('d')); // Ensure whole numbers on the y-axis svg.append('g') @@ -101,12 +106,13 @@ const stackData = labels.map((label, i) => ({ label: label, delivered: deliveredData[i], - failed: failedData[i] || 0 // Ensure there's a value for failed, even if it's 0 + failed: failedData[i] || 0, + pending: pendingData[i] || 0 })); // Stack the data const stack = d3.stack() - .keys(['delivered', 'failed']) + .keys(['delivered', 'failed', 'pending']) .order(d3.stackOrderNone) .offset(d3.stackOffsetNone); @@ -114,8 +120,8 @@ // Color scale const color = d3.scaleOrdinal() - .domain(['delivered', 'failed']) - .range([COLORS.delivered, COLORS.failed]); + .domain(['delivered', 'failed', 'pending']) + .range([COLORS.delivered, COLORS.failed, COLORS.pending]); // Create bars with animation const barGroups = svg.selectAll('.bar-group') @@ -124,11 +130,12 @@ .append('g') .attr('class', 'bar-group') .attr('fill', d => color(d.key)); - + const minBarHeight = 5; barGroups.selectAll('rect') .data(d => d) .enter() .append('rect') + .filter(d => d[1] - d[0] > 0) .attr('x', d => x(d.data.label)) .attr('y', height) .attr('height', 0) @@ -149,11 +156,13 @@ .transition() .duration(1000) .attr('y', d => y(d[1])) - .attr('height', d => y(d[0]) - y(d[1])); - }; + .attr('height', d => { + const calculatedHeight = y(d[0]) - y(d[1]); + return calculatedHeight < minBarHeight ? minBarHeight : calculatedHeight; + }); }; // Function to create an accessible table - const createTable = function(tableId, chartType, labels, deliveredData, failedData) { + const createTable = function(tableId, chartType, labels, deliveredData, failedData, pendingData) { const table = document.getElementById(tableId); table.innerHTML = ""; // Clear previous data @@ -165,7 +174,7 @@ // Create table header const headerRow = document.createElement('tr'); - const headers = ['Day', 'Delivered', 'Failed']; + const headers = ['Day', 'Delivered', 'Failed', 'Pending']; headers.forEach(headerText => { const th = document.createElement('th'); th.textContent = headerText; @@ -188,6 +197,10 @@ cellFailed.textContent = failedData[index]; row.appendChild(cellFailed); + const cellPending = document.createElement('td'); + cellPending.textContent = pendingData[index]; + row.appendChild(cellPending); + tbody.appendChild(row); }); @@ -197,12 +210,13 @@ }; const fetchData = function(type) { + var ctx = document.getElementById('weeklyChart'); if (!ctx) { return; } - var url = type === 'service' ? `/daily_stats.json` : `/daily_stats_by_user.json`; + var url = type === 'service' ? `/services/${currentServiceId}/daily-stats.json` : `/services/${currentServiceId}/daily-stats-by-user.json`; return fetch(url) .then(response => { if (!response.ok) { @@ -214,7 +228,7 @@ labels = []; deliveredData = []; failedData = []; - + pendingData = []; let totalMessages = 0; for (var dateString in data) { @@ -225,9 +239,8 @@ labels.push(formattedDate); deliveredData.push(data[dateString].sms.delivered); failedData.push(data[dateString].sms.failure); - - // Calculate the total number of messages - totalMessages += data[dateString].sms.delivered + data[dateString].sms.failure; + pendingData.push(data[dateString].sms.pending || 0); + totalMessages += data[dateString].sms.delivered + data[dateString].sms.failure + data[dateString].sms.pending; } } @@ -253,17 +266,18 @@ } } else { // If there are messages, create the chart and table - createChart('#weeklyChart', labels, deliveredData, failedData); - createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData); - } - - return data; - }) - .catch(error => console.error('Error fetching daily stats:', error)); - }; + createChart('#weeklyChart', labels, deliveredData, failedData, pendingData); + createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData, pendingData); + } + return data; + }) + .catch(error => console.error('Error fetching daily stats:', error)); + }; + setInterval(() => fetchData(currentType), 25000); const handleDropdownChange = function(event) { const selectedValue = event.target.value; + currentType = selectedValue; const subTitle = document.querySelector(`#activityChartContainer .chart-subtitle`); const selectElement = document.getElementById('options'); const selectedText = selectElement.options[selectElement.selectedIndex].text; @@ -316,7 +330,7 @@ document.addEventListener('DOMContentLoaded', function() { // Initialize activityChart chart and table with service data by default - fetchData('service'); + fetchData(currentType); const allRows = Array.from(document.querySelectorAll('#activity-table tbody tr')); allRows.forEach((row, index) => { @@ -329,9 +343,9 @@ // Resize chart on window resize window.addEventListener('resize', function() { - if (labels.length > 0 && deliveredData.length > 0 && failedData.length > 0) { - createChart('#weeklyChart', labels, deliveredData, failedData); - createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData); + if (labels.length > 0 && deliveredData.length > 0 && failedData.length > 0 && pendingData.length > 0) { + createChart('#weeklyChart', labels, deliveredData, failedData, pendingData); + createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData, pendingData); } }); diff --git a/app/assets/sass/uswds/_uswds-theme-custom-styles.scss b/app/assets/sass/uswds/_uswds-theme-custom-styles.scss index 0bb1aab7e..ebce95061 100644 --- a/app/assets/sass/uswds/_uswds-theme-custom-styles.scss +++ b/app/assets/sass/uswds/_uswds-theme-custom-styles.scss @@ -1024,3 +1024,7 @@ nav.nav { font-size: units(3); font-weight: bold; } + +.form-control-error { + border: 4px solid #b10e1e +} diff --git a/app/main/views/dashboard.py b/app/main/views/dashboard.py index ccebc81fc..b3428eb45 100644 --- a/app/main/views/dashboard.py +++ b/app/main/views/dashboard.py @@ -51,8 +51,12 @@ def service_dashboard(service_id): job_response = job_api_client.get_jobs(service_id)["data"] service_data_retention_days = 7 - filtered_jobs = [job for job in job_response if job["job_status"] != "cancelled"] - sorted_jobs = sorted(filtered_jobs, key=lambda job: job["created_at"], reverse=True) + active_jobs = [job for job in job_response if job["job_status"] != "cancelled"] + sorted_jobs = sorted(active_jobs, key=lambda job: job["created_at"], reverse=True) + job_lists = [ + {**job_dict, "finished_processing": job_is_finished(job_dict)} + for job_dict in sorted_jobs + ] total_messages = service_api_client.get_service_message_ratio(service_id) messages_remaining = total_messages.get('messages_remaining', 0) @@ -61,32 +65,49 @@ def service_dashboard(service_id): "views/dashboard/dashboard.html", updates_url=url_for(".service_dashboard_updates", service_id=service_id), partials=get_dashboard_partials(service_id), - jobs=sorted_jobs, + jobs=job_lists, service_data_retention_days=service_data_retention_days, messages_remaining=messages_remaining, messages_sent=messages_sent ) -@main.route("/daily_stats.json") -def get_daily_stats(): - service_id = session.get("service_id") - date_range = get_stats_date_range() +def job_is_finished(job_dict): + done_statuses = [ + "delivered", + "sent", + "failed", + "technical-failure", + "temporary-failure", + "permanent-failure", + "cancelled", + ] + processed_count = sum( + stat["count"] + for stat in job_dict["statistics"] + if stat["status"] in done_statuses + ) + return job_dict["notification_count"] == processed_count + + +@main.route("/services//daily-stats.json") +@user_has_permissions() +def get_daily_stats(service_id): + date_range = get_stats_date_range() stats = service_api_client.get_service_notification_statistics_by_day( service_id, start_date=date_range["start_date"], days=date_range["days"] ) return jsonify(stats) -@main.route("/daily_stats_by_user.json") -def get_daily_stats_by_user(): - service_id = session.get("service_id") +@main.route("/services//daily-stats-by-user.json") +@user_has_permissions() +def get_daily_stats_by_user(service_id): date_range = get_stats_date_range() - user_id = current_user.id stats = service_api_client.get_user_service_notification_statistics_by_day( service_id, - user_id, + user_id=current_user.id, start_date=date_range["start_date"], days=date_range["days"], ) diff --git a/app/main/views/jobs.py b/app/main/views/jobs.py index bb95f630e..164ab737f 100644 --- a/app/main/views/jobs.py +++ b/app/main/views/jobs.py @@ -401,7 +401,9 @@ def get_job_partials(job): ) if request.referrer is not None: - session["arrived_from_preview_page"] = ("check" in request.referrer) or ("help=0" in request.referrer) + session["arrived_from_preview_page"] = ("check" in request.referrer) or ( + "help=0" in request.referrer + ) else: session["arrived_from_preview_page"] = False diff --git a/app/main/views/platform_admin.py b/app/main/views/platform_admin.py index 6a121ecc2..bfd3749f3 100644 --- a/app/main/views/platform_admin.py +++ b/app/main/views/platform_admin.py @@ -116,6 +116,70 @@ def download_all_users(): return response +@main.route("/platform-admin/get-redis-report") +@user_is_platform_admin +def get_redis_report(): + + memory_info = redis_client.info("memory") + memory_used = memory_info.get("used_memory_human", "N/A") + max_memory = memory_info.get("maxmemory_human", "N/A") + if max_memory == "0B": + max_memory = "No set limit" + mem_fragmentation = memory_info.get("mem_fragmentation_ratio", "N/A") + frag_quality = "Swapping (bad)" + if mem_fragmentation >= 1.0: + frag_quality = "Healthy" + if mem_fragmentation > 1.5: + frag_quality = "Problematic" + if mem_fragmentation > 2.0: + frag_quality = "Severe fragmentation" + + frag_note = "" + if mem_fragmentation > 2.0: + frag_note = "Use MEMORY PURGE.\nReplace multiple small keys with hashes.\nAvoid long keys.\nSet max_memory." + elif mem_fragmentation < 1.0: + frag_note = "Allocate more RAM.\nSet max_memory." + + keys = redis_client.keys("*") + key_details = [] + + for key in keys: + key_type = redis_client.type(key).decode("utf-8") + ttl = redis_client.ttl(key) + ttl_str = "No Expiry" if ttl == -1 else f"{ttl} seconds" + key_details.append( + {"Key": key.decode("utf-8"), "Type": key_type, "TTL": ttl_str} + ) + output = StringIO() + writer = csv.writer( + output, + ) + writer.writerow(["Redis Report"]) + writer.writerow([]) + + writer.writerow(["Memory"]) + writer.writerow(["", "Memory Used", memory_used]) + writer.writerow(["", "Max Memory", max_memory]) + writer.writerow(["", "Memory Fragmentation Ratio", mem_fragmentation]) + writer.writerow(["", "Memory Fragmentation Quality", frag_quality, frag_note]) + writer.writerow([]) + + writer.writerow(["Keys Overview"]) + writer.writerow(["", "TTL", "Type", "Key"]) + for key_detail in key_details: + writer.writerow( + ["", key_detail["TTL"], key_detail["Type"], key_detail["Key"][0:50]] + ) + + csv_data = output.getvalue() + + # Create a direct download response with the CSV data and appropriate headers + response = Response(csv_data, content_type="text/csv; charset=utf-8") + response.headers["Content-Disposition"] = "attachment; filename=redis.csv" + + return response + + def is_over_threshold(number, total, threshold): percentage = number / total * 100 if total else 0 return percentage > threshold diff --git a/app/main/views/sign_in.py b/app/main/views/sign_in.py index 004dce2ae..1cb163691 100644 --- a/app/main/views/sign_in.py +++ b/app/main/views/sign_in.py @@ -68,11 +68,12 @@ def _get_access_token(code): # pragma: no cover id_token = get_id_token(response_json) nonce = id_token["nonce"] nonce_key = f"login-nonce-{unquote(nonce)}" - stored_nonce = redis_client.get(nonce_key).decode("utf8") + if not os.getenv("NOTIFY_ENVIRONMENT") == "development": + stored_nonce = redis_client.get(nonce_key).decode("utf8") - if nonce != stored_nonce: - current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}") - abort(403) + if nonce != stored_nonce: + current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}") + abort(403) try: access_token = response_json["access_token"] @@ -112,7 +113,7 @@ def _do_login_dot_gov(): # $ pragma: no cover verify_key = f"login-verify_email-{unquote(state)}" verify_path = bool(redis_client.get(verify_key)) - if not verify_path: + if not verify_path and not os.getenv("NOTIFY_ENVIRONMENT") == "development": state_key = f"login-state-{unquote(state)}" stored_state = unquote(redis_client.get(state_key).decode("utf8")) if state != stored_state: diff --git a/app/notify_client/billing_api_client.py b/app/notify_client/billing_api_client.py index b1ffc19f0..363764907 100644 --- a/app/notify_client/billing_api_client.py +++ b/app/notify_client/billing_api_client.py @@ -6,17 +6,36 @@ from app.notify_client import NotifyAdminAPIClient class BillingAPIClient(NotifyAdminAPIClient): def get_monthly_usage_for_service(self, service_id, year): - return self.get( + monthly_usage = redis_client.get(f"monthly-usage-summary-{service_id}-{year}") + if monthly_usage is not None: + return json.loads(monthly_usage.decode("utf-8")) + result = self.get( "/service/{0}/billing/monthly-usage".format(service_id), params=dict(year=year), ) + redis_client.set( + f"monthly-usage-summary-{service_id}-{year}", + json.dumps(result), + ex=30, + ) + return result def get_annual_usage_for_service(self, service_id, year=None): - return self.get( + annual_usage = redis_client.get(f"yearly-usage-summary-{service_id}-{year}") + if annual_usage is not None: + return json.loads(annual_usage.decode("utf-8")) + result = self.get( "/service/{0}/billing/yearly-usage-summary".format(service_id), params=dict(year=year), ) + redis_client.set( + f"yearly-usage-summary-{service_id}-{year}", + json.dumps(result), + ex=30, + ) + return result + def get_free_sms_fragment_limit_for_year(self, service_id, year=None): frag_limit = redis_client.get(f"free-sms-fragment-limit-{service_id}-{year}") if frag_limit is not None: @@ -48,13 +67,28 @@ class BillingAPIClient(NotifyAdminAPIClient): ) def get_data_for_billing_report(self, start_date, end_date): - return self.get( + x_start_date = str(start_date) + x_start_date = x_start_date.replace(" ", "_") + x_end_date = str(end_date) + x_end_date = x_end_date.replace(" ", "_") + billing_data = redis_client.get( + f"get-data-for-billing-report-{x_start_date}-{x_end_date}" + ) + if billing_data is not None: + return json.loads(billing_data.decode("utf-8")) + result = self.get( url="/platform-stats/data-for-billing-report", params={ "start_date": str(start_date), "end_date": str(end_date), }, ) + redis_client.set( + f"get-data-for-billing-report-{x_start_date}-{x_end_date}", + json.dumps(result), + ex=30, + ) + return result def get_data_for_volumes_by_service_report(self, start_date, end_date): return self.get( diff --git a/app/notify_client/notification_api_client.py b/app/notify_client/notification_api_client.py index 95ac96a04..89e786079 100644 --- a/app/notify_client/notification_api_client.py +++ b/app/notify_client/notification_api_client.py @@ -1,3 +1,6 @@ +import json + +from app.extensions import redis_client from app.notify_client import NotifyAdminAPIClient, _attach_current_user @@ -41,7 +44,7 @@ class NotificationApiClient(NotifyAdminAPIClient): if job_id: return method( url="/service/{}/job/{}/notifications".format(service_id, job_id), - **kwargs + **kwargs, ) else: if limit_days is not None: @@ -96,9 +99,20 @@ class NotificationApiClient(NotifyAdminAPIClient): ) def get_notification_count_for_job_id(self, *, service_id, job_id): - return self.get( + counts = redis_client.get( + f"notification-count-for-job-id-{service_id}-{job_id}" + ) + if counts is not None: + return json.loads(counts.decode("utf-8")) + result = self.get( url="/service/{}/job/{}/notification_count".format(service_id, job_id) - )["count"] + ) + redis_client.set( + f"notification-count-for-job-id-{service_id}-{job_id}", + json.dumps(result["count"]), + ex=30, + ) + return result["count"] notification_api_client = NotificationApiClient() diff --git a/app/templates/base.html b/app/templates/base.html index 1d2778a9e..3135e5cdb 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -14,10 +14,8 @@ {% block bodyStart %} {% block extra_javascripts_before_body %} - - {% endblock %} {% endblock %} @@ -145,10 +143,8 @@ {% block bodyEnd %} {% block extra_javascripts %} {% endblock %} - - {% endblock %} diff --git a/app/templates/components/components/input/template.njk b/app/templates/components/components/input/template.njk index 7f5634651..4ea649dce 100644 --- a/app/templates/components/components/input/template.njk +++ b/app/templates/components/components/input/template.njk @@ -34,7 +34,7 @@ attributes: params.errorMessage.attributes, html: params.errorMessage.html, text: params.errorMessage.text, - visuallyHiddenText: params.errorMessage.visuallyHiddenText + visuallyHiddenText: params.errorMessage.visuallyHiddenText, }) | indent(2) | trim }} {% endif %} + {%- for attribute, value in params.attributes %} {{ attribute }}="{{ value }}"{% endfor -%} + {%- if params.required %} required{% endif %} + /> diff --git a/app/templates/components/textbox.html b/app/templates/components/textbox.html index 3e479cbce..fa92d0cf8 100644 --- a/app/templates/components/textbox.html +++ b/app/templates/components/textbox.html @@ -16,19 +16,9 @@ placeholder='' ) %}
- {% if field.errors %} - - {% endif %}
{% endif %} + {% if field.errors %} + + Error: + {% if not safe_error_message %}{{ field.errors[0] }}{% else %}{{ field.errors[0]|safe }}{% endif %} + + {% endif %} {% if highlight_placeholders or autosize %} @@ -59,6 +55,8 @@ data_highlight_placeholders='true' if highlight_placeholders else 'false', rows=rows|string, placeholder=placeholder, + aria_describedby=field.name+"-error", + required='required' if required else None, **kwargs ) }} {% if suffix %} diff --git a/app/templates/partials/jobs/status.html b/app/templates/partials/jobs/status.html index 0f3288813..d56d73201 100644 --- a/app/templates/partials/jobs/status.html +++ b/app/templates/partials/jobs/status.html @@ -25,7 +25,7 @@ {% if job.scheduled_for %}
-

Your message has been scheduled

+

Your {{ 'message has' if job.notification_count == 1 else 'messages have' }} been scheduled

{{ job.template_name }} - {{ current_service.name }} was scheduled on {{ job.scheduled_for|format_datetime_normal }} by {{ job.created_by.name }}

@@ -37,7 +37,7 @@

- Your message is sending + Your {{ 'message is' if job.notification_count == 1 else 'messages are' }} sending

{{ job.template_name }} - {{ current_service.name }} @@ -49,7 +49,7 @@

- Your message is pending + Your {{ 'message is' if job.notification_count == 1 else 'messages are' }} pending

{{ job.template_name }} - {{ current_service.name }} @@ -64,7 +64,7 @@

- Your message has been sent + Your {{ 'message has' if job.notification_count == 1 else 'messages have' }} been sent

{{ job.template_name }} - {{ current_service.name }} diff --git a/app/templates/views/dashboard/activity-table.html b/app/templates/views/dashboard/activity-table.html index a40840033..1dbb2bc7d 100644 --- a/app/templates/views/dashboard/activity-table.html +++ b/app/templates/views/dashboard/activity-table.html @@ -18,7 +18,7 @@

-
+

Service activity

@@ -46,14 +46,16 @@ {{ job.template_name }} - {% if job.scheduled_for and not job.processing_finished %} - Scheduled for {{ job.scheduled_for|format_datetime_table }} - {% elif job.processing_finished and not job.statistics|selectattr('status', 'equalto', 'sending')|list %} - Sent on {{ job.processing_finished|format_datetime_table }} - {% elif job.processing_started %} - Sending since {{ job.processing_started|format_datetime_table }} + {% if not job.finished_processing %} + {% if job.scheduled_for%} + Scheduled for {{ job.scheduled_for|format_datetime_table }} + {% elif job.processing_started %} + Sending since {{ job.processing_started|format_datetime_table }} + {% else %} + Pending since {{ job.created_at|format_datetime_table }} + {% endif %} {% else %} - Pending since {{ job.created_at|format_datetime_table }} + Sent on {{ job.processing_started|format_datetime_table }} {% endif %} {{ job.created_by.name }} diff --git a/app/templates/views/edit-sms-template.html b/app/templates/views/edit-sms-template.html index 97eac73dc..8ef41bdb2 100644 --- a/app/templates/views/edit-sms-template.html +++ b/app/templates/views/edit-sms-template.html @@ -32,6 +32,8 @@
{{ form.name(param_extensions={ "extra_form_group_classes": "margin-bottom-2", + "id": "name", + "required": True, "hint": {"text": "Your recipients will not see this"} }) }} {{ textbox( @@ -41,7 +43,8 @@ hint=content_hint, rows=5, extra_form_group_classes='margin-bottom-1', - placeholder='Edit me! Check out the Personalization section below for details on cool ((stuff)) you can do with your messages!' + placeholder='Edit me! Check out the Personalization section below for details on cool ((stuff)) you can do with your messages!', + required=True ) }} {% if current_user.platform_admin %} {{ form.process_type }} diff --git a/app/templates/views/jobs/job.html b/app/templates/views/jobs/job.html index 1643715f4..d5d3711d5 100644 --- a/app/templates/views/jobs/job.html +++ b/app/templates/views/jobs/job.html @@ -11,7 +11,7 @@ {% block maincolumn_content %} {{ page_header("Message status") }} - {% if not job.processing_finished %} + {% if not job.finished_processing %}
{% endif %} {{ partials['status']|safe }} - {% if not job.processing_finished %} + {% if not job.finished_processing %}
{% endif %} {% if not finished %} diff --git a/app/templates/views/join-notify.html b/app/templates/views/join-notify.html index 2e8cfee52..78af499f2 100644 --- a/app/templates/views/join-notify.html +++ b/app/templates/views/join-notify.html @@ -34,7 +34,7 @@ {% set product_highlights = [ { "svg_src": "#chat", - "card_heading": "Up to 100,000 messages to use over your first year*", + "card_heading": "Up to 250,000 messages to use over your first year*", }, { "svg_src": "#phone", diff --git a/app/templates/views/platform-admin/reports.html b/app/templates/views/platform-admin/reports.html index 3b7b32d3a..23dccc41c 100644 --- a/app/templates/views/platform-admin/reports.html +++ b/app/templates/views/platform-admin/reports.html @@ -34,5 +34,8 @@

Download All Users

+

+ Get Redis Report +

{% endblock %} diff --git a/app/templates/views/send-test.html b/app/templates/views/send-test.html index 720849ae6..fd5eb63db 100644 --- a/app/templates/views/send-test.html +++ b/app/templates/views/send-test.html @@ -37,8 +37,8 @@ data_kwargs={'force-focus': True} ) %}
-
-
+
`; @@ -64,13 +64,13 @@ test('D3 is loaded correctly', () => { test('Populates the accessible table for activity chart correctly', () => { const sampleData = { - '2024-07-01': { sms: { delivered: 50, failed: 5 } }, - '2024-07-02': { sms: { delivered: 60, failed: 2 } }, - '2024-07-03': { sms: { delivered: 70, failed: 1 } }, - '2024-07-04': { sms: { delivered: 80, failed: 0 } }, - '2024-07-05': { sms: { delivered: 90, failed: 3 } }, - '2024-07-06': { sms: { delivered: 100, failed: 4 } }, - '2024-07-07': { sms: { delivered: 110, failed: 2 } }, + '2024-07-01': { sms: { delivered: 50, failed: 5, pending: 10 } }, + '2024-07-02': { sms: { delivered: 60, failed: 2, pending: 5 } }, + '2024-07-03': { sms: { delivered: 70, failed: 1, pending: 3 } }, + '2024-07-04': { sms: { delivered: 80, failed: 0, pending: 0 } }, + '2024-07-05': { sms: { delivered: 90, failed: 3, pending: 8 } }, + '2024-07-06': { sms: { delivered: 100, failed: 4, pending: 7 } }, + '2024-07-07': { sms: { delivered: 110, failed: 2, pending: 6 } }, }; const labels = Object.keys(sampleData).map(dateString => { @@ -79,8 +79,9 @@ test('Populates the accessible table for activity chart correctly', () => { }); const deliveredData = Object.values(sampleData).map(d => d.sms.delivered); const failedData = Object.values(sampleData).map(d => d.sms.failed); + const pendingData = Object.values(sampleData).map(d => d.sms.pending); - window.createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData); + window.createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData, pendingData); const table = document.getElementById('weeklyTable'); expect(table).toBeDefined(); @@ -92,6 +93,7 @@ test('Populates the accessible table for activity chart correctly', () => { expect(headers[0].textContent).toBe('Day'); expect(headers[1].textContent).toBe('Delivered'); expect(headers[2].textContent).toBe('Failed'); + expect(headers[3].textContent).toBe('Pending'); const firstRowCells = rows[1].getElementsByTagName('td'); expect(firstRowCells[0].textContent).toBe('07/01/24'); @@ -100,58 +102,73 @@ test('Populates the accessible table for activity chart correctly', () => { }); test('SVG element is correctly set up', () => { - window.createChart('#weeklyChart', ['07/01/24', '07/02/24', '07/03/24', '07/04/24', '07/05/24', '07/06/24', '07/07/24'], [50, 60, 70, 80, 90, 100, 110], [5, 2, 1, 0, 3, 4, 2]); + window.createChart( + '#weeklyChart', + ['07/01/24', '07/02/24', '07/03/24', '07/04/24', '07/05/24', '07/06/24', '07/07/24'], + [50, 60, 70, 80, 90, 100, 110], + [5, 2, 1, 0, 3, 4, 2], + [10, 5, 3, 0, 8, 7, 6] + ); const svg = document.getElementById('weeklyChart').querySelector('svg'); expect(svg).not.toBeNull(); - expect(svg.getAttribute('width')).toBe('0'); - expect(svg.getAttribute('height')).toBe('400'); + expect(svg.querySelectorAll('.bar-group').length).toBe(3); }); test('Check HTML content after chart creation', () => { - // Create sample data for the chart const labels = ['07/01/24', '07/02/24', '07/03/24', '07/04/24', '07/05/24', '07/06/24', '07/07/24']; const deliveredData = [50, 60, 70, 80, 90, 100, 110]; const failedData = [5, 2, 1, 0, 3, 4, 2]; + const pendingData = [10, 5, 8, 3, 6, 7, 4]; - // Ensure the container has the correct width const container = document.getElementById('weeklyChart'); - container.style.width = '600px'; // Force a specific width + container.style.width = '600px'; const containerWidth = container.clientWidth; expect(containerWidth).toBeGreaterThan(0); - // Call the function to create the chart - window.createChart('#weeklyChart', labels, deliveredData, failedData); + window.createChart('#weeklyChart', labels, deliveredData, failedData, pendingData); - // Optionally, you can add assertions to check for specific elements - expect(container.querySelector('svg')).not.toBeNull(); - expect(container.querySelectorAll('rect').length).toBeGreaterThan(0); + const svg = container.querySelector('svg'); + expect(svg).not.toBeNull(); + + const bars = container.querySelectorAll('rect'); + expect(bars.length).toBeGreaterThan(0); + + const barGroups = svg.querySelectorAll('.bar-group'); + expect(barGroups.length).toBe(3); + + const pendingBars = Array.from(bars).filter(bar => + bar.parentNode.getAttribute('fill') === '#C7CACE' + ); + expect(pendingBars.length).toBe(labels.length); }); -test('Legend is visible when there are delivered or failed messages', () => { - // Example data with delivered and failed messages +test('Legend includes pending when data exists', () => { const labels = ['Day 1', 'Day 2']; - const deliveredData = [10, 20]; // Mock delivered data - const failedData = [5, 0]; // Mock failed data + const deliveredData = [10, 20]; + const failedData = [5, 0]; + const pendingData = [3, 2]; - // Call the createChart function - window.createChart('#weeklyChart', labels, deliveredData, failedData); + window.createChart('#weeklyChart', labels, deliveredData, failedData, pendingData); - // Check if the legend is displayed using computed style const legendContainer = document.querySelector('.chart-legend'); - const legendDisplayStyle = window.getComputedStyle(legendContainer).display; - expect(legendDisplayStyle).toBe('flex'); - expect(legendContainer.querySelectorAll('.legend-item').length).toBe(2); // Ensure two legend items + const legendItems = legendContainer.querySelectorAll('.legend-item'); + expect(legendItems.length).toBe(3); + + const pendingLegend = Array.from(legendItems).find(item => + item.textContent.includes('Pending') + ); + expect(pendingLegend).not.toBeNull(); }); -test('Legend is hidden when there are no delivered or failed messages', () => { - // Example data with no delivered and no failed messages +test('Legend is hidden when there are no delivered, failed, or pending messages', () => { const labels = ['Day 1', 'Day 2']; - const deliveredData = [0, 0]; // No delivered messages - const failedData = [0, 0]; // No failed messages + const deliveredData = [0, 0]; + const failedData = [0, 0]; + const pendingData = [0, 0]; // Call the createChart function - window.createChart('#weeklyChart', labels, deliveredData, failedData); + window.createChart('#weeklyChart', labels, deliveredData, failedData, pendingData); // Check if the legend is hidden using computed style const legendContainer = document.querySelector('.chart-legend'); @@ -161,25 +178,48 @@ test('Legend is hidden when there are no delivered or failed messages', () => { test('Fetches data and creates chart and table correctly', async () => { const mockResponse = { - '2024-07-01': { sms: { delivered: 50, failed: 5 } }, - '2024-07-02': { sms: { delivered: 60, failed: 2 } }, - '2024-07-03': { sms: { delivered: 70, failed: 1 } }, - '2024-07-04': { sms: { delivered: 80, failed: 0 } }, - '2024-07-05': { sms: { delivered: 90, failed: 3 } }, - '2024-07-06': { sms: { delivered: 100, failed: 4 } }, - '2024-07-07': { sms: { delivered: 110, failed: 2 } }, + '2024-07-01': { sms: { delivered: 50, failed: 5, pending: 10 } }, + '2024-07-02': { sms: { delivered: 60, failed: 2, pending: 8 } }, + '2024-07-03': { sms: { delivered: 70, failed: 1, pending: 6 } }, + '2024-07-04': { sms: { delivered: 80, failed: 0, pending: 4 } }, + '2024-07-05': { sms: { delivered: 90, failed: 3, pending: 7 } }, + '2024-07-06': { sms: { delivered: 100, failed: 4, pending: 5 } }, + '2024-07-07': { sms: { delivered: 110, failed: 2, pending: 3 } }, }; + const tableContainer = document.getElementById('activityContainer'); + const currentServiceId = tableContainer.getAttribute('data-currentServiceId'); + + global.fetch = jest.fn(() => Promise.resolve({ - ok: true, - json: () => Promise.resolve(mockResponse), + ok: true, + json: () => Promise.resolve(mockResponse), }) -); + ); -const data = await fetchData('service'); + const data = await fetchData('service'); -expect(global.fetch).toHaveBeenCalledWith('/daily_stats.json'); -expect(data).toEqual(mockResponse); + expect(global.fetch).toHaveBeenCalledWith(`/services/${currentServiceId}/daily-stats.json`); + expect(data).toEqual(mockResponse); + + const labels = Object.keys(mockResponse).map(dateString => { + const dateParts = dateString.split('-'); + return `${dateParts[1]}/${dateParts[2]}/${dateParts[0].slice(2)}`; + }); + const deliveredData = Object.values(mockResponse).map(d => d.sms.delivered); + const failedData = Object.values(mockResponse).map(d => d.sms.failed); + const pendingData = Object.values(mockResponse).map(d => d.sms.pending); + + window.createChart('#weeklyChart', labels, deliveredData, failedData, pendingData); + window.createTable('weeklyTable', 'activityChart', labels, deliveredData, failedData, pendingData); + + const chart = document.getElementById('weeklyChart').querySelector('svg'); + expect(chart).not.toBeNull(); + + const table = document.getElementById('weeklyTable'); + expect(table).toBeDefined(); + const rows = table.getElementsByTagName('tr'); + expect(rows.length).toBe(8); }); test('handleDropdownChange updates DOM for individual selection', () => { @@ -188,7 +228,7 @@ test('handleDropdownChange updates DOM for individual selection', () => {
-
+

diff --git a/zap.conf b/zap.conf index 48e70c91b..5b1168634 100644 --- a/zap.conf +++ b/zap.conf @@ -53,6 +53,7 @@ 10096 WARN (Timestamp Disclosure - Passive/release) 10097 WARN (Hash Disclosure - Passive/beta) 10098 WARN (Cross-Domain Misconfiguration - Passive/release) +10099 IGNORE (Source Code Disclosure - Java) 10104 WARN (User Agent Fuzzer - Active/beta) 10105 WARN (Weak Authentication Method - Passive/release) 10106 IGNORE (HTTP Only Site - Active/beta)