change migrate-data-to-ft-notification-status to use the existing task

saves the reimplementation of the big sql query
This commit is contained in:
Leo Hemsted
2019-12-04 14:53:48 +00:00
parent 49496c26a2
commit 766d46dda7

View File

@@ -21,6 +21,7 @@ from app.celery.tasks import record_daily_sorted_counts, get_template_class, pro
from app.celery.nightly_tasks import send_total_sent_notifications_to_performance_platform
from app.celery.service_callback_tasks import send_delivery_status_to_service
from app.celery.letters_pdf_tasks import create_letters_pdf
from app.celery.reporting_tasks import create_nightly_notification_status_for_day
from app.config import QueueNames
from app.dao.annual_billing_dao import dao_create_or_update_annual_billing_for_year
from app.dao.fact_billing_dao import (
@@ -491,53 +492,15 @@ def rebuild_ft_billing_for_day(service_id, day):
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@statsd(namespace="tasks")
def migrate_data_to_ft_notification_status(start_date, end_date):
print('Notification statuses migration from date {} to {}'.format(start_date, end_date))
process_date = start_date
total_updated = 0
while process_date < end_date:
start_time = datetime.now()
# migrate data into ft_notification_status and update if record already exists
db.session.execute(
'delete from ft_notification_status where bst_date = :process_date',
{"process_date": process_date}
start_date = start_date.date()
for day_diff in range((end_date - start_date).days):
process_day = start_date + timedelta(days=day_diff)
print('create_nightly_notification_status_for_day triggered for {}'.format(process_day))
create_nightly_notification_status_for_day.apply_async(
kwargs={'process_day': process_day.strftime('%Y-%m-%d')},
queue=QueueNames.REPORTING
)
sql = \
"""
insert into ft_notification_status (bst_date, template_id, service_id, job_id, notification_type, key_type,
notification_status, created_at, notification_count)
select
(n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
n.service_id,
coalesce(n.job_id, '00000000-0000-0000-0000-000000000000') as job_id,
n.notification_type,
n.key_type,
n.notification_status,
now() as created_at,
count(*) as notification_count
from notification_history n
where n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
group by bst_date, template_id, service_id, job_id, notification_type, key_type, notification_status
order by bst_date
"""
result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
db.session.commit()
print('ft_notification_status: --- Completed took {}ms. Migrated {} rows for {}.'.format(
datetime.now() - start_time,
result.rowcount,
process_date
))
process_date += timedelta(days=1)
total_updated += result.rowcount
print('Total inserted/updated records = {}'.format(total_updated))
@notify_command(name='bulk-invite-user-to-service')
@click.option('-f', '--file_name', required=True,