darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-20 15:31:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3425 from alphagov/parallelise-ft-status-180693991

Browse Source 
Parallelise status aggregation by service and day
This commit is contained in:
Ben Thorner
2022-01-19 13:49:28 +00:00
committed by GitHub
parent 5cd6fcbb4f 9686595fa8
commit 6be489daa7
5 changed files with 164 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
app/dao/fact_notification_status_dao.py
View File

@@ -1,6 +1,5 @@
from datetime import datetime, time, timedelta
from flask import current_app
from notifications_utils.timezones import convert_bst_to_utc
from sqlalchemy import Date, case, func
from sqlalchemy.dialects.postgresql import insert
@@ -36,35 +35,16 @@ from app.utils import (
)
def fetch_notification_status_for_day(process_day, notification_type):
def fetch_status_data_for_service_and_day(process_day, service_id, notification_type):
start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
# query notifications or notification_history for the day, depending on their data retention
service = Service.query.get(service_id)
table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
all_data_for_process_day = []
services = Service.query.all()
# for each service query notifications or notification_history for the day, depending on their data retention
for service in services:
table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
data_for_service_and_type = query_for_fact_status_data(
table=table,
start_date=start_date,
end_date=end_date,
notification_type=notification_type,
service_id=service.id
)
all_data_for_process_day += data_for_service_and_type
return all_data_for_process_day
def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
query = db.session.query(
return db.session.query(
table.template_id,
table.service_id,
func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
table.key_type,
table.status,
@@ -77,34 +57,34 @@ def query_for_fact_status_data(table, start_date, end_date, notification_type, s
table.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
).group_by(
table.template_id,
table.service_id,
'job_id',
table.key_type,
table.status
)
return query.all()
).all()
@autocommit
def update_fact_notification_status(data, process_day, notification_type):
def update_fact_notification_status(new_status_rows, process_day, notification_type, service_id):
table = FactNotificationStatus.__table__
FactNotificationStatus.query.filter(
FactNotificationStatus.bst_date == process_day,
FactNotificationStatus.notification_type == notification_type
FactNotificationStatus.notification_type == notification_type,
FactNotificationStatus.service_id == service_id,
).delete()
for row in data:
stmt = insert(table).values(
bst_date=process_day,
template_id=row.template_id,
service_id=row.service_id,
job_id=row.job_id,
notification_type=notification_type,
key_type=row.key_type,
notification_status=row.status,
notification_count=row.notification_count,
for row in new_status_rows:
db.session.connection().execute(
insert(table).values(
bst_date=process_day,
template_id=row.template_id,
service_id=service_id,
job_id=row.job_id,
notification_type=notification_type,
key_type=row.key_type,
notification_status=row.status,
notification_count=row.notification_count,
)
)
db.session.connection().execute(stmt)
def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):