darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-09-09 17:45:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add command to populate data

Browse Source
This commit is contained in:
Katie Smith
2018-05-09 16:05:07 +01:00
parent c9dc6f724d
commit 13f3662051
1 changed files with 80 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
app/commands.py
View File

@@ -603,3 +603,83 @@ def compare_ft_billing_to_monthly_billing(year, service_id=None):
path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)): path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id) ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response) compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
@notify_command(name='migrate-data-to-ft-notification-status')
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@statsd(namespace="tasks")
def migrate_data_to_ft_notification_status(start_date, end_date):
print('Notification statuses migration from date {} to {}'.format(start_date, end_date))
process_date = start_date
total_updated = 0
while process_date < end_date:
sql = \
"""
select count(*) from notification_history
where created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
"""
num_notifications = db.session.execute(sql, {"start": process_date,
"end": process_date + timedelta(days=1)}).fetchall()[0][0]
sql = \
"""
select count(*) from
(select distinct template_id, service_id, job_id, notification_type, key_type, notification_status
from notification_history
where created_at >= (date :start + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
and created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
) as distinct_records
"""
predicted_records = db.session.execute(sql, {"start": process_date,
"end": process_date + timedelta(days=1)}).fetchall()[0][0]
start_time = datetime.now()
print('ft_notification-status: Migrating date: {}, notifications: {}, expecting {} ft_notification_status rows'
.format(process_date.date(), num_notifications, predicted_records))
# migrate data into ft_notification_status and update if record already exists
sql = \
"""
insert into ft_notification_status (bst_date, template_id, service_id, job_id, notification_type, key_type,
notification_status, notification_count)
select bst_date, template_id, service_id, job_id, notification_type, key_type, notification_status,
sum(notification_count) as notification_count
from (
select
da.bst_date,
n.template_id,
n.service_id,
coalesce(n.job_id, '00000000-0000-0000-0000-000000000000') as job_id,
n.notification_type,
n.key_type,
n.notification_status,
1 as notification_count
from public.notification_history n
left join dm_datetime da on n.created_at >= da.utc_daytime_start
and n.created_at < da.utc_daytime_end
where n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
at time zone 'UTC'
and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
) as individual_record
group by bst_date, template_id, service_id, job_id, notification_type, key_type, notification_status
order by bst_date
on conflict on constraint ft_notification_status_pkey do update set
notification_count = excluded.notification_count
"""
result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
db.session.commit()
print('ft_notification_status: --- Completed took {}ms. Migrated {} rows.'.format(datetime.now() - start_time,
result.rowcount))
if predicted_records != result.rowcount:
print(' : ^^^ Result mismatch by {} rows ^^^'
.format(predicted_records - result.rowcount))
process_date += timedelta(days=1)
total_updated += result.rowcount
print('Total inserted/updated records = {}'.format(total_updated))