darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-05 22:20:49 -04:00
Code Issues Packages Projects Releases Wiki Activity

merged master and up migration version

Browse Source
This commit is contained in:
venusbb
2018-03-16 10:55:36 +00:00
parent bb95a2784f 62d2b04e0a
commit 7e2947790f
30 changed files with 658 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/accept_invite/rest.py
View File

@@ -4,7 +4,7 @@ from flask import (
current_app
)
from itsdangerous import SignatureExpired
from itsdangerous import SignatureExpired, BadData
from notifications_utils.url_safe_token import check_token
@@ -38,6 +38,9 @@ def validate_invitation_token(invitation_type, token):
['Your invitation to GOV.UK Notify has expired. '
'Please ask the person that invited you to send you another one']}
raise InvalidRequest(errors, status_code=400)
except BadData:
errors = {'invitation': 'Something’s wrong with this link. Make sure you’ve copied the whole thing.'}
raise InvalidRequest(errors, status_code=400)
if invitation_type == 'service':
invited_user = get_invited_user_by_id(invited_user_id)

2
app/celery/research_mode_tasks.py
View File

@@ -123,7 +123,7 @@ def firetext_callback(notification_id, to):
def create_fake_letter_response_file(self, reference):
now = datetime.utcnow()
dvla_response_data = '{}|Sent|0|Sorted'.format(reference)
upload_file_name = 'NOTIFY.{}.RSP.TXT'.format(now.strftime('%Y%m%d%H%M%S'))
upload_file_name = 'NOTIFY-{}-RSP.TXT'.format(now.strftime('%Y%m%d%H%M%S'))
s3upload(
filedata=dvla_response_data,

36
app/celery/tasks.py
View File

@@ -420,33 +420,37 @@ def update_letter_notifications_statuses(self, filename):
update_letter_notification(filename, temporary_failures, update)
sorted_letter_counts[update.cost_threshold] += 1
if temporary_failures:
# This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
message = "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
filename=filename, failures=temporary_failures)
raise DVLAException(message)
try:
if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}:
unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}
if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}:
unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}
message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
filename, unknown_status
)
raise DVLAException(message)
message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
filename, unknown_status
)
raise DVLAException(message)
billing_date = get_billing_date_in_bst_from_filename(filename)
persist_daily_sorted_letter_counts(billing_date, sorted_letter_counts)
billing_date = get_billing_date_in_bst_from_filename(filename)
persist_daily_sorted_letter_counts(day=billing_date,
file_name=filename,
sorted_letter_counts=sorted_letter_counts)
finally:
if temporary_failures:
# This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
message = "DVLA response file: {filename} has failed letters with notification.reference {failures}" \
.format(filename=filename, failures=temporary_failures)
raise DVLAException(message)
def get_billing_date_in_bst_from_filename(filename):
datetime_string = filename.split('.')[1]
datetime_string = filename.split('-')[1]
datetime_obj = datetime.strptime(datetime_string, '%Y%m%d%H%M%S')
return convert_utc_to_bst(datetime_obj).date()
def persist_daily_sorted_letter_counts(day, sorted_letter_counts):
def persist_daily_sorted_letter_counts(day, file_name, sorted_letter_counts):
daily_letter_count = DailySortedLetter(
billing_day=day,
file_name=file_name,
unsorted_count=sorted_letter_counts['Unsorted'],
sorted_count=sorted_letter_counts['Sorted']
)

6
app/config.py
View File

@@ -326,6 +326,7 @@ class Development(Config):
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
@@ -362,6 +363,7 @@ class Test(Development):
CSV_UPLOAD_BUCKET_NAME = 'test-notifications-csv-upload'
LETTERS_PDF_BUCKET_NAME = 'test-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'test-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'test.notify.com-ftp'
# this is overriden in jenkins and on cloudfoundry
@@ -389,6 +391,7 @@ class Preview(Config):
NOTIFY_ENVIRONMENT = 'preview'
CSV_UPLOAD_BUCKET_NAME = 'preview-notifications-csv-upload'
LETTERS_PDF_BUCKET_NAME = 'preview-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'preview-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
FROM_NUMBER = 'preview'
API_RATE_LIMIT_ENABLED = True
@@ -400,6 +403,7 @@ class Staging(Config):
NOTIFY_ENVIRONMENT = 'staging'
CSV_UPLOAD_BUCKET_NAME = 'staging-notify-csv-upload'
LETTERS_PDF_BUCKET_NAME = 'staging-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'staging-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'staging-notify.works-ftp'
STATSD_ENABLED = True
FROM_NUMBER = 'stage'
@@ -413,6 +417,7 @@ class Live(Config):
NOTIFY_ENVIRONMENT = 'live'
CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
LETTERS_PDF_BUCKET_NAME = 'production-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'production-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notifications.service.gov.uk-ftp'
STATSD_ENABLED = True
FROM_NUMBER = 'GOVUK'
@@ -433,6 +438,7 @@ class Sandbox(CloudFoundryConfig):
NOTIFY_ENVIRONMENT = 'sandbox'
CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
FROM_NUMBER = 'sandbox'
REDIS_ENABLED = False

10
app/dao/api_key_dao.py
View File

@@ -1,5 +1,5 @@
import uuid
from datetime import datetime
from datetime import datetime, timedelta
from app import db
from app.models import ApiKey
@@ -9,6 +9,8 @@ from app.dao.dao_utils import (
version_class
)
from sqlalchemy import or_, func
@transactional
@version_class(ApiKey)
@@ -30,7 +32,11 @@ def expire_api_key(service_id, api_key_id):
def get_model_api_keys(service_id, id=None):
if id:
return ApiKey.query.filter_by(id=id, service_id=service_id, expiry_date=None).one()
return ApiKey.query.filter_by(service_id=service_id).all()
seven_days_ago = datetime.utcnow() - timedelta(days=7)
return ApiKey.query.filter(
or_(ApiKey.expiry_date == None, func.date(ApiKey.expiry_date) > seven_days_ago), # noqa
ApiKey.service_id == service_id
).all()
def get_unsigned_secrets(service_id):

7
app/dao/daily_sorted_letter_dao.py
View File

@@ -24,13 +24,14 @@ def dao_create_or_update_daily_sorted_letter(new_daily_sorted_letter):
table = DailySortedLetter.__table__
stmt = insert(table).values(
billing_day=new_daily_sorted_letter.billing_day,
file_name=new_daily_sorted_letter.file_name,
unsorted_count=new_daily_sorted_letter.unsorted_count,
sorted_count=new_daily_sorted_letter.sorted_count)
stmt = stmt.on_conflict_do_update(
index_elements=[table.c.billing_day],
index_elements=[table.c.billing_day, table.c.file_name],
set_={
'unsorted_count': table.c.unsorted_count + stmt.excluded.unsorted_count,
'sorted_count': table.c.sorted_count + stmt.excluded.sorted_count,
'unsorted_count': stmt.excluded.unsorted_count,
'sorted_count': stmt.excluded.sorted_count,
'updated_at': datetime.utcnow()
}
)

6
app/dao/notifications_dao.py
View File

@@ -455,6 +455,12 @@ def dao_get_notifications_by_to_field(service_id, search_term, notification_type
else:
raise InvalidRequest("Only email and SMS can use search by recipient", 400)
for special_character in ('\\', '_', '%', '/'):
normalised = normalised.replace(
special_character,
'\{}'.format(special_character)
)
filters = [
Notification.service_id == service_id,
Notification.normalised_to.like("%{}%".format(normalised)),

16
app/dao/users_dao.py
View File

@@ -1,6 +1,9 @@
from random import (SystemRandom)
from datetime import (datetime, timedelta)
from sqlalchemy import func
from sqlalchemy.orm import joinedload
from app import db
from app.models import (User, VerifyCode)
@@ -113,3 +116,16 @@ def update_user_password(user, password):
user.password_changed_at = datetime.utcnow()
db.session.add(user)
db.session.commit()
def get_user_and_accounts(user_id):
return User.query.filter(
User.id == user_id
).options(
# eagerly load the user's services and organisations, and also the service's org and vice versa
# (so we can see if the user knows about it)
joinedload('services'),
joinedload('organisations'),
joinedload('organisations.services'),
joinedload('services.organisation'),
).one()

50
app/letters/utils.py
View File

@@ -5,24 +5,32 @@ from flask import current_app
from notifications_utils.s3 import s3upload
from app.models import KEY_TYPE_TEST
from app.variables import Retention
LETTERS_PDF_FILE_LOCATION_STRUCTURE = \
'{folder}/NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
'{folder}NOTIFY.{reference}.{duplex}.{letter_class}.{colour}.{crown}.{date}.pdf'
PRECOMPILED_BUCKET_PREFIX = '{folder}/NOTIFY.{reference}'
PRECOMPILED_BUCKET_PREFIX = '{folder}NOTIFY.{reference}'
def get_letter_pdf_filename(reference, crown):
def get_folder_name(_now, is_test_letter):
if is_test_letter:
folder_name = ''
else:
print_datetime = _now
if _now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
print_datetime = _now + timedelta(days=1)
folder_name = '{}/'.format(print_datetime.date())
return folder_name
def get_letter_pdf_filename(reference, crown, is_test_letter=False):
now = datetime.utcnow()
print_datetime = now
if now.time() > current_app.config.get('LETTER_PROCESSING_DEADLINE'):
print_datetime = now + timedelta(days=1)
upload_file_name = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format(
folder=print_datetime.date(),
folder=get_folder_name(now, is_test_letter),
reference=reference,
duplex="D",
letter_class="2",
@@ -34,41 +42,51 @@ def get_letter_pdf_filename(reference, crown):
return upload_file_name
def get_bucket_prefix_for_notification(notification):
def get_bucket_prefix_for_notification(notification, is_test_letter=False):
upload_file_name = PRECOMPILED_BUCKET_PREFIX.format(
folder=notification.created_at.date(),
folder='' if is_test_letter else
'{}/'.format(notification.created_at.date()),
reference=notification.reference
).upper()
return upload_file_name
def upload_letter_pdf(notification, pdf_data):
def upload_letter_pdf(notification, pdf_data, is_test_letter=False):
current_app.logger.info("PDF Letter {} reference {} created at {}, {} bytes".format(
notification.id, notification.reference, notification.created_at, len(pdf_data)))
upload_file_name = get_letter_pdf_filename(
notification.reference, notification.service.crown)
notification.reference, notification.service.crown, is_test_letter)
if is_test_letter:
bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME']
else:
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
s3upload(
filedata=pdf_data,
region=current_app.config['AWS_REGION'],
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
bucket_name=bucket_name,
file_location=upload_file_name,
tags={Retention.KEY: Retention.ONE_WEEK}
)
current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format(
upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'], notification.id))
upload_file_name, bucket_name, notification.id))
def get_letter_pdf(notification):
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
is_test_letter = notification.key_type == KEY_TYPE_TEST and notification.template.is_precompiled_letter
if is_test_letter:
bucket_name = current_app.config['TEST_LETTERS_BUCKET_NAME']
else:
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification)):
for item in bucket.objects.filter(Prefix=get_bucket_prefix_for_notification(notification, is_test_letter)):
obj = s3.Object(
bucket_name=bucket_name,
key=item.key

42
app/models.py
View File

@@ -116,11 +116,11 @@ class User(db.Model):
services = db.relationship(
'Service',
secondary='user_to_service',
backref=db.backref('user_to_service', lazy='dynamic'))
backref='user_to_service')
organisations = db.relationship(
'Organisation',
secondary='user_to_organisation',
backref=db.backref('user_to_organisation', lazy='dynamic'))
backref='users')
@property
def password(self):
@@ -133,6 +133,38 @@ class User(db.Model):
def check_password(self, password):
return check_hash(password, self._password)
def get_permissions(self):
from app.dao.permissions_dao import permission_dao
retval = {}
for x in permission_dao.get_permissions_by_user_id(self.id):
service_id = str(x.service_id)
if service_id not in retval:
retval[service_id] = []
retval[service_id].append(x.permission)
return retval
def serialize(self):
return {
'id': self.id,
'name': self.name,
'email_address': self.email_address,
'auth_type': self.auth_type,
'current_session_id': self.current_session_id,
'failed_login_count': self.failed_login_count,
'logged_in_at': self.logged_in_at.strftime(DATETIME_FORMAT) if self.logged_in_at else None,
'mobile_number': self.mobile_number,
'organisations': [x.id for x in self.organisations if x.active],
'password_changed_at': (
self.password_changed_at.strftime('%Y-%m-%d %H:%M:%S.%f')
if self.password_changed_at
else None
),
'permissions': self.get_permissions(),
'platform_admin': self.platform_admin,
'services': [x.id for x in self.services if x.active],
'state': self.state,
}
user_to_service = db.Table(
'user_to_service',
@@ -1726,11 +1758,15 @@ class DailySortedLetter(db.Model):
__tablename__ = "daily_sorted_letter"
id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
billing_day = db.Column(db.Date, nullable=False, index=True, unique=True)
billing_day = db.Column(db.Date, nullable=False, index=True)
file_name = db.Column(db.String, nullable=True, index=True)
unsorted_count = db.Column(db.Integer, nullable=False, default=0)
sorted_count = db.Column(db.Integer, nullable=False, default=0)
updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow)
__table_args__ = (UniqueConstraint('file_name', 'billing_day', name='uix_file_name_billing_day'),
)
class FactBilling(db.Model):
__tablename__ = "ft_billing"

7
app/organisation/rest.py
View File

@@ -20,7 +20,6 @@ from app.organisation.organisation_schema import (
post_link_service_to_organisation_schema,
)
from app.schema_validation import validate
from app.schemas import user_schema
organisation_blueprint = Blueprint('organisation', __name__)
register_errors(organisation_blueprint)
@@ -98,15 +97,13 @@ def get_organisation_services(organisation_id):
@organisation_blueprint.route('/<uuid:organisation_id>/users/<uuid:user_id>', methods=['POST'])
def add_user_to_organisation(organisation_id, user_id):
new_org_user = dao_add_user_to_organisation(organisation_id, user_id)
return jsonify(data=user_schema.dump(new_org_user).data), 200
return jsonify(data=new_org_user.serialize())
@organisation_blueprint.route('/<uuid:organisation_id>/users', methods=['GET'])
def get_organisation_users(organisation_id):
org_users = dao_get_users_for_organisation(organisation_id)
result = user_schema.dump(org_users, many=True)
return jsonify(data=result.data)
return jsonify(data=[x.serialize() for x in org_users])
@organisation_blueprint.route('/unique', methods=["GET"])

8
app/schemas.py
View File

@@ -665,19 +665,15 @@ class UnarchivedTemplateSchema(BaseSchema):
raise ValidationError('Template has been deleted', 'template')
user_schema = UserSchema()
user_schema_load_json = UserSchema(load_json=True)
# should not be used on its own for dumping - only for loading
create_user_schema = UserSchema()
user_update_schema_load_json = UserUpdateAttributeSchema(load_json=True, partial=True)
user_update_password_schema_load_json = UserUpdatePasswordSchema(load_json=True, partial=True)
service_schema = ServiceSchema()
service_schema_load_json = ServiceSchema(load_json=True)
detailed_service_schema = DetailedServiceSchema()
template_schema = TemplateSchema()
template_schema_load_json = TemplateSchema(load_json=True)
api_key_schema = ApiKeySchema()
api_key_schema_load_json = ApiKeySchema(load_json=True)
job_schema = JobSchema()
job_schema_load_json = JobSchema(load_json=True)
sms_admin_notification_schema = SmsAdminNotificationSchema()
sms_template_notification_schema = SmsTemplateNotificationSchema()
job_sms_template_notification_schema = JobSmsTemplateNotificationSchema()

4
app/service/rest.py
View File

@@ -82,7 +82,6 @@ from app.service.send_notification import send_one_off_notification
from app.schemas import (
service_schema,
api_key_schema,
user_schema,
permission_schema,
notification_with_template_schema,
notifications_filter_schema,
@@ -258,8 +257,7 @@ def get_api_keys(service_id, key_id=None):
@service_blueprint.route('/<uuid:service_id>/users', methods=['GET'])
def get_users_for_service(service_id):
fetched = dao_fetch_service_by_id(service_id)
result = user_schema.dump(fetched.users, many=True)
return jsonify(data=result.data)
return jsonify(data=[x.serialize() for x in fetched.users])
@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['POST'])

69
app/user/rest.py
View File

@@ -19,7 +19,8 @@ from app.dao.users_dao import (
create_secret_code,
save_user_attribute,
update_user_password,
count_user_verify_codes
count_user_verify_codes,
get_user_and_accounts
)
from app.dao.permissions_dao import permission_dao
from app.dao.services_dao import dao_fetch_service_by_id
@@ -31,7 +32,7 @@ from app.notifications.process_notifications import (
)
from app.schemas import (
email_data_request_schema,
user_schema,
create_user_schema,
permission_schema,
user_update_schema_load_json,
user_update_password_schema_load_json
@@ -67,13 +68,14 @@ def handle_integrity_error(exc):
@user_blueprint.route('', methods=['POST'])
def create_user():
user_to_create, errors = user_schema.load(request.get_json())
user_to_create, errors = create_user_schema.load(request.get_json())
req_json = request.get_json()
if not req_json.get('password', None):
errors.update({'password': ['Missing data for required field.']})
raise InvalidRequest(errors, status_code=400)
save_model_user(user_to_create, pwd=req_json.get('password'))
return jsonify(data=user_schema.dump(user_to_create).data), 201
result = user_to_create.serialize()
return jsonify(data=result), 201
@user_blueprint.route('/<uuid:user_id>', methods=['POST'])
@@ -84,7 +86,7 @@ def update_user_attribute(user_id):
if errors:
raise InvalidRequest(errors, status_code=400)
save_user_attribute(user_to_update, update_dict=update_dct)
return jsonify(data=user_schema.dump(user_to_update).data), 200
return jsonify(data=user_to_update.serialize()), 200
@user_blueprint.route('/<uuid:user_id>/activate', methods=['POST'])
@@ -95,14 +97,14 @@ def activate_user(user_id):
user.state = 'active'
save_model_user(user)
return jsonify(data=user_schema.dump(user).data), 200
return jsonify(data=user.serialize()), 200
@user_blueprint.route('/<uuid:user_id>/reset-failed-login-count', methods=['POST'])
def user_reset_failed_login_count(user_id):
user_to_update = get_user_by_id(user_id=user_id)
reset_failed_login_count(user_to_update)
return jsonify(data=user_schema.dump(user_to_update).data), 200
return jsonify(data=user_to_update.serialize()), 200
@user_blueprint.route('/<uuid:user_id>/verify/password', methods=['POST'])
@@ -324,8 +326,8 @@ def send_already_registered_email(user_id):
@user_blueprint.route('', methods=['GET'])
def get_user(user_id=None):
users = get_user_by_id(user_id=user_id)
result = user_schema.dump(users, many=True) if isinstance(users, list) else user_schema.dump(users)
return jsonify(data=result.data)
result = [x.serialize() for x in users] if isinstance(users, list) else users.serialize()
return jsonify(data=result)
@user_blueprint.route('/<uuid:user_id>/service/<uuid:service_id>/permission', methods=['POST'])
@@ -350,9 +352,8 @@ def get_by_email():
error = 'Invalid request. Email query string param required'
raise InvalidRequest(error, status_code=400)
fetched_user = get_user_by_email(email)
result = user_schema.dump(fetched_user)
return jsonify(data=result.data)
result = fetched_user.serialize()
return jsonify(data=result)
@user_blueprint.route('/reset-password', methods=['POST'])
@@ -392,7 +393,14 @@ def update_password(user_id):
if errors:
raise InvalidRequest(errors, status_code=400)
update_user_password(user, pwd)
return jsonify(data=user_schema.dump(user).data), 200
return jsonify(data=user.serialize()), 200
@user_blueprint.route('/<uuid:user_id>/organisations-and-services', methods=['GET'])
def get_organisations_and_services_for_user(user_id):
user = get_user_and_accounts(user_id)
data = get_orgs_and_services(user)
return jsonify(data)
def _create_reset_password_url(email):
@@ -420,3 +428,38 @@ def _create_2fa_url(user, secret_code, next_redir, email_auth_link_host):
if next_redir:
ret += '?{}'.format(urlencode({'next': next_redir}))
return ret
def get_orgs_and_services(user):
return {
'organisations': [
{
'name': org.name,
'id': org.id,
'services': [
{
'id': service.id,
'name': service.name
}
for service in org.services
if service.active and service in user.services
]
}
for org in user.organisations if org.active
],
'services_without_organisations': [
{
'id': service.id,
'name': service.name
} for service in user.services
if (
service.active and
# include services that either aren't in an organisation, or are in an organisation,
# but not one that the user can see.
(
not service.organisation or
service.organisation not in user.organisations
)
)
]
}

2
app/v2/notifications/post_notifications.py
View File

@@ -265,6 +265,8 @@ def process_letter_notification(*, letter_data, api_key, template, reply_to_text
queue=QueueNames.RESEARCH_MODE
)
else:
if precompiled and api_key.key_type == KEY_TYPE_TEST:
upload_letter_pdf(notification, letter_content, is_test_letter=True)
update_notification_status_by_reference(notification.reference, NOTIFICATION_DELIVERED)
return notification