darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-01 15:46:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/alphagov/notifications-api into secrets-2FA

Browse Source
This commit is contained in:
venusbb
2017-06-09 17:05:39 +01:00
parent a0640bb803 ea27e37ddf
commit 7c318cd90e
25 changed files with 924 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/celery/scheduled_tasks.py
View File

@@ -32,8 +32,8 @@ from app.config import QueueNames
@notify_celery.task(name="remove_csv_files")
@statsd(namespace="tasks")
def remove_csv_files():
jobs = dao_get_jobs_older_than_limited_by()
def remove_csv_files(job_types):
jobs = dao_get_jobs_older_than_limited_by(job_types=job_types)
for job in jobs:
s3.remove_job_from_s3(job.service_id, job.id)
current_app.logger.info("Job ID {} has been removed from s3.".format(job.id))

18
app/config.py
View File

@@ -3,7 +3,10 @@ from celery.schedules import crontab
from kombu import Exchange, Queue
import os
from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
from app.models import (
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE,
KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
)
if os.environ.get('VCAP_SERVICES'):
# on cloudfoundry, config is a json blob in VCAP_SERVICES - unpack it, and populate
@@ -189,10 +192,17 @@ class Config(object):
'schedule': crontab(minute=0, hour=3),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_csv_files': {
'remove_sms_email_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=0, hour=4),
'options': {'queue': QueueNames.PERIODIC}
'options': {'queue': QueueNames.PERIODIC},
'kwargs': {'job_types': [EMAIL_TYPE, SMS_TYPE]}
},
'remove_letter_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(minute=20, hour=4),
'options': {'queue': QueueNames.PERIODIC},
'kwargs': {'job_types': [LETTER_TYPE]}
},
'timeout-job-statistics': {
'task': 'timeout-job-statistics',
@@ -239,6 +249,8 @@ class Config(object):
}
}
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
######################
# Config overrides ###

7
app/dao/inbound_sms_dao.py
View File

@@ -47,3 +47,10 @@ def delete_inbound_sms_created_more_than_a_week_ago():
).delete(synchronize_session='fetch')
return deleted
def dao_get_inbound_sms_by_id(service_id, inbound_id):
return InboundSms.query.filter_by(
id=inbound_id,
service_id=service_id
).one()

26
app/dao/jobs_dao.py
View File

@@ -1,17 +1,15 @@
from datetime import datetime
from datetime import datetime, timedelta
from flask import current_app
from sqlalchemy import func, desc, asc, cast, Date as sql_date
from app import db
from app.dao import days_ago
from app.models import (Job,
Notification,
NotificationHistory,
Template,
JOB_STATUS_SCHEDULED,
JOB_STATUS_PENDING,
LETTER_TYPE, JobStatistics)
from app.models import (
Job, JobStatistics, Notification, NotificationHistory, Template,
JOB_STATUS_SCHEDULED, JOB_STATUS_PENDING,
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE
)
from app.statsd_decorators import statsd
@@ -129,10 +127,14 @@ def dao_update_job_status(job_id, status):
db.session.commit()
def dao_get_jobs_older_than_limited_by(older_than=7, limit_days=2):
return Job.query.filter(
cast(Job.created_at, sql_date) < days_ago(older_than),
cast(Job.created_at, sql_date) >= days_ago(older_than + limit_days)
def dao_get_jobs_older_than_limited_by(job_types, older_than=7, limit_days=2):
end_date = datetime.utcnow() - timedelta(days=older_than)
start_date = end_date - timedelta(days=limit_days)
return Job.query.join(Template).filter(
Job.created_at < end_date,
Job.created_at >= start_date,
Template.template_type.in_(job_types)
).order_by(desc(Job.created_at)).all()

17
app/dao/notification_usage_dao.py
View File

@@ -1,5 +1,6 @@
from datetime import datetime, timedelta
from flask import current_app
from sqlalchemy import Float, Integer
from sqlalchemy import func, case, cast
from sqlalchemy import literal_column
@@ -11,7 +12,7 @@ from app.models import (NotificationHistory,
NOTIFICATION_STATUS_TYPES_BILLABLE,
KEY_TYPE_TEST,
SMS_TYPE,
EMAIL_TYPE)
EMAIL_TYPE, Service)
from app.statsd_decorators import statsd
from app.utils import get_london_month_from_utc_column
@@ -158,6 +159,8 @@ def rate_multiplier():
@statsd(namespace="dao")
def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date, end_date, service_id):
free_sms_limit = Service.free_sms_fragment_limit()
billable_units = 0
total_cost = 0.0
@@ -176,9 +179,15 @@ def get_total_billable_units_for_sent_sms_notifications_in_date_range(start_date
)
billable_units_by_rate_boundry = result.scalar()
if billable_units_by_rate_boundry:
billable_units += int(billable_units_by_rate_boundry)
total_cost += int(billable_units_by_rate_boundry) * rate_boundary['rate']
int_billable_units_by_rate_boundry = int(billable_units_by_rate_boundry)
if billable_units >= free_sms_limit:
total_cost += int_billable_units_by_rate_boundry * rate_boundary['rate']
elif billable_units + int_billable_units_by_rate_boundry > free_sms_limit:
remaining_free_allowance = abs(free_sms_limit - billable_units)
total_cost += ((int_billable_units_by_rate_boundry - remaining_free_allowance) * rate_boundary['rate'])
else:
total_cost += 0
billable_units += int_billable_units_by_rate_boundry
return billable_units, total_cost

17
app/dao/statistics_dao.py
View File

@@ -60,7 +60,10 @@ def timeout_job_counts(notifications_type, timeout_start):
).update({
sent: sent_count,
failed: failed_count,
delivered: delivered_count
delivered: delivered_count,
'sent': sent_count,
'delivered': delivered_count,
'failed': failed_count
}, synchronize_session=False)
return total_updated
@@ -87,11 +90,13 @@ def create_or_update_job_sending_statistics(notification):
@transactional
def __update_job_stats_sent_count(notification):
column = columns(notification.notification_type, 'sent')
new_column = 'sent'
return db.session.query(JobStatistics).filter_by(
job_id=notification.job_id,
).update({
column: column + 1
column: column + 1,
new_column: column + 1
})
@@ -102,7 +107,8 @@ def __insert_job_stats(notification):
emails_sent=1 if notification.notification_type == EMAIL_TYPE else 0,
sms_sent=1 if notification.notification_type == SMS_TYPE else 0,
letters_sent=1 if notification.notification_type == LETTER_TYPE else 0,
updated_at=datetime.utcnow()
updated_at=datetime.utcnow(),
sent=1
)
db.session.add(stats)
@@ -131,10 +137,12 @@ def columns(notification_type, status):
def update_job_stats_outcome_count(notification):
if notification.status in NOTIFICATION_STATUS_TYPES_FAILED:
column = columns(notification.notification_type, 'failed')
new_column = 'failed'
elif notification.status in [NOTIFICATION_DELIVERED,
NOTIFICATION_SENT] and notification.notification_type != LETTER_TYPE:
column = columns(notification.notification_type, 'delivered')
new_column = 'delivered'
else:
column = None
@@ -143,7 +151,8 @@ def update_job_stats_outcome_count(notification):
return db.session.query(JobStatistics).filter_by(
job_id=notification.job_id,
).update({
column: column + 1
column: column + 1,
new_column: column + 1
})
else:
return 0

16
app/inbound_sms/rest.py
View File

@@ -3,15 +3,20 @@ from flask import (
jsonify,
request
)
from notifications_utils.recipients import validate_and_format_phone_number
from app.dao.inbound_sms_dao import dao_get_inbound_sms_for_service, dao_count_inbound_sms_for_service
from app.dao.inbound_sms_dao import (
dao_get_inbound_sms_for_service,
dao_count_inbound_sms_for_service,
dao_get_inbound_sms_by_id
)
from app.errors import register_errors
inbound_sms = Blueprint(
'inbound_sms',
__name__,
url_prefix='/service/<service_id>/inbound-sms'
url_prefix='/service/<uuid:service_id>/inbound-sms'
)
register_errors(inbound_sms)
@@ -40,3 +45,10 @@ def get_inbound_sms_summary_for_service(service_id):
count=count,
most_recent=most_recent[0].created_at.isoformat() if most_recent else None
)
@inbound_sms.route('/<uuid:inbound_sms_id>', methods=['GET'])
def get_inbound_by_id(service_id, inbound_sms_id):
inbound_sms = dao_get_inbound_sms_by_id(service_id, inbound_sms_id)
return jsonify(inbound_sms.serialize()), 200

9
app/models.py
View File

@@ -217,6 +217,10 @@ class Service(db.Model, Versioned):
self.can_send_letters = LETTER_TYPE in [p.permission for p in self.permissions]
self.can_send_international_sms = INTERNATIONAL_SMS_TYPE in [p.permission for p in self.permissions]
@staticmethod
def free_sms_fragment_limit():
return current_app.config['FREE_SMS_TIER_FRAGMENT_COUNT']
@classmethod
def from_json(cls, data):
"""
@@ -1122,6 +1126,9 @@ class JobStatistics(db.Model):
sms_failed = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
letters_sent = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
letters_failed = db.Column(db.BigInteger, index=False, unique=False, nullable=False, default=0)
sent = db.Column(db.BigInteger, index=False, unique=False, nullable=True, default=0)
delivered = db.Column(db.BigInteger, index=False, unique=False, nullable=True, default=0)
failed = db.Column(db.BigInteger, index=False, unique=False, nullable=True, default=0)
created_at = db.Column(
db.DateTime,
index=False,
@@ -1165,7 +1172,7 @@ class InboundSms(db.Model):
user_number = db.Column(db.String, nullable=False) # the end user's number, that the msg was sent from
provider_date = db.Column(db.DateTime)
provider_reference = db.Column(db.String)
provider = db.Column(db.String, nullable=True)
provider = db.Column(db.String, nullable=False)
_content = db.Column('content', db.String, nullable=False)
@property

13
app/schemas.py
View File

@@ -25,9 +25,8 @@ from notifications_utils.recipients import (
from app import ma
from app import models
from app.models import ServicePermission, INTERNATIONAL_SMS_TYPE, SMS_TYPE, LETTER_TYPE, EMAIL_TYPE
from app.models import ServicePermission, INTERNATIONAL_SMS_TYPE, LETTER_TYPE
from app.dao.permissions_dao import permission_dao
from app.dao.service_permissions_dao import dao_fetch_service_permissions
from app.utils import get_template_instance
@@ -176,6 +175,7 @@ class ProviderDetailsHistorySchema(BaseSchema):
class ServiceSchema(BaseSchema):
free_sms_fragment_limit = fields.Method(method_name='get_free_sms_fragment_limit')
created_by = field_for(models.Service, 'created_by', required=True)
organisation = field_for(models.Service, 'organisation')
branding = field_for(models.Service, 'branding')
@@ -183,11 +183,15 @@ class ServiceSchema(BaseSchema):
permissions = fields.Method("service_permissions")
override_flag = False
def get_free_sms_fragment_limit(selfs, service):
return service.free_sms_fragment_limit()
def service_permissions(self, service):
return [p.permission for p in service.permissions]
class Meta:
model = models.Service
dump_only = ['free_sms_fragment_limit']
exclude = (
'updated_at',
'created_at',
@@ -256,6 +260,11 @@ class ServiceSchema(BaseSchema):
class DetailedServiceSchema(BaseSchema):
statistics = fields.Dict()
free_sms_fragment_limit = fields.Method(method_name='get_free_sms_fragment_limit')
def get_free_sms_fragment_limit(selfs, service):
return service.free_sms_fragment_limit()
class Meta:
model = models.Service
exclude = (

13
app/service/rest.py
View File

@@ -300,6 +300,19 @@ def get_all_notifications_for_service(service_id):
), 200
@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>', methods=['GET'])
def get_notification_for_service(service_id, notification_id):
notification = notifications_dao.get_notification_with_personalisation(
service_id,
notification_id,
key_type=None,
)
return jsonify(
notification_with_template_schema.dump(notification).data,
), 200
def search_for_notification_by_to_field(service_id, search_term, statuses):
results = notifications_dao.dao_get_notifications_by_to_field(service_id, search_term, statuses)
return jsonify(

4
app/user/rest.py
View File

@@ -206,7 +206,7 @@ def send_user_confirm_new_email(user_id):
personalisation={
'name': user_to_send_to.name,
'url': _create_confirmation_url(user=user_to_send_to, email_address=email['email']),
'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/feedback'
'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/support'
},
notification_type=EMAIL_TYPE,
api_key_id=None,
@@ -259,7 +259,7 @@ def send_already_registered_email(user_id):
personalisation={
'signin_url': current_app.config['ADMIN_BASE_URL'] + '/sign-in',
'forgot_password_url': current_app.config['ADMIN_BASE_URL'] + '/forgot-password',
'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/feedback'
'feedback_url': current_app.config['ADMIN_BASE_URL'] + '/support'
},
notification_type=EMAIL_TYPE,
api_key_id=None,