Localize notification_utils to the Admin!

This changeset copies everything in notifications_utils to turn into a local project module.  It includes dependency changes to account for this adjustment and the notifications_utils tests.  It also fixes all of the import statements.

Signed-off-by: Carlo Costino <carlo.costino@gsa.gov>

notifications_utils/clients/antivirus/antivirus_client.py

@@ -0,0 +1,55 @@
+import requests
+from flask import current_app
+
+
+class AntivirusError(Exception):
+    def __init__(self, message=None, status_code=None):
+        self.message = message
+        self.status_code = status_code
+
+    @classmethod
+    def from_exception(cls, e):
+        try:
+            message = e.response.json()["error"]
+            status_code = e.response.status_code
+        except (TypeError, ValueError, AttributeError, KeyError):
+            message = "connection error"
+            status_code = 503
+
+        return cls(message, status_code)
+
+
+class AntivirusClient:
+    def __init__(self, api_host=None, auth_token=None):
+        self.api_host = api_host
+        self.auth_token = auth_token
+
+    def init_app(self, app):
+        self.api_host = app.config["ANTIVIRUS_API_HOST"]
+        self.auth_token = app.config["ANTIVIRUS_API_KEY"]
+
+    def scan(self, document_stream):
+        try:
+            response = requests.post(
+                "{}/scan".format(self.api_host),
+                headers={
+                    "Authorization": "Bearer {}".format(self.auth_token),
+                },
+                files={"document": document_stream},
+            )
+
+            response.raise_for_status()
+
+        except requests.RequestException as e:
+            error = AntivirusError.from_exception(e)
+            current_app.logger.warning(
+                "Notify Antivirus API request failed with error: {}".format(
+                    error.message
+                )
+            )
+
+            raise error
+        finally:
+            document_stream.seek(0)
+
+        return response.json()["ok"]

notifications_utils/clients/encryption/encryption_client.py

@@ -0,0 +1,86 @@
+from base64 import urlsafe_b64encode
+from json import dumps, loads
+
+from cryptography.fernet import Fernet, InvalidToken
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+from itsdangerous import BadSignature, URLSafeSerializer
+
+
+class EncryptionError(Exception):
+    pass
+
+
+class SaltLengthError(Exception):
+    pass
+
+
+class Encryption:
+    def init_app(self, app):
+        self._serializer = URLSafeSerializer(app.config.get("SECRET_KEY"))
+        self._salt = app.config.get("DANGEROUS_SALT")
+        self._password = app.config.get("SECRET_KEY").encode()
+
+        try:
+            self._shared_encryptor = Fernet(self._derive_key(self._salt))
+        except SaltLengthError as reason:
+            raise EncryptionError(
+                "DANGEROUS_SALT must be at least 16 bytes"
+            ) from reason
+
+    def encrypt(self, thing_to_encrypt, salt=None):
+        """Encrypt a string or object
+
+        thing_to_encrypt must be serializable as JSON
+        Returns a UTF-8 string
+        """
+        serialized_bytes = dumps(thing_to_encrypt).encode("utf-8")
+        encrypted_bytes = self._encryptor(salt).encrypt(serialized_bytes)
+        return encrypted_bytes.decode("utf-8")
+
+    def decrypt(self, thing_to_decrypt, salt=None):
+        """Decrypt a UTF-8 string or bytes.
+
+        Once decrypted, thing_to_decrypt must be deserializable from JSON.
+        """
+        try:
+            return loads(self._encryptor(salt).decrypt(thing_to_decrypt))
+        except InvalidToken as reason:
+            raise EncryptionError from reason
+
+    def sign(self, thing_to_sign, salt=None):
+        return self._serializer.dumps(thing_to_sign, salt=(salt or self._salt))
+
+    def verify_signature(self, thing_to_verify, salt=None):
+        try:
+            return self._serializer.loads(thing_to_verify, salt=(salt or self._salt))
+        except BadSignature as reason:
+            raise EncryptionError from reason
+
+    def _encryptor(self, salt=None):
+        if salt is None:
+            return self._shared_encryptor
+        else:
+            try:
+                return Fernet(self._derive_key(salt))
+            except SaltLengthError as reason:
+                raise EncryptionError(
+                    "Custom salt value must be at least 16 bytes"
+                ) from reason
+
+    def _derive_key(self, salt):
+        """Derive a key suitable for use within Fernet from the SECRET_KEY and salt
+
+        * For the salt to be secure, it must be 16 bytes or longer and randomly generated.
+        * 600_000 was chosen for the iterations because it is what OWASP recommends as
+        *  of [February 2023](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#pbkdf2)
+        * For more information, see https://cryptography.io/en/latest/hazmat/primitives/key-derivation-functions/#pbkdf2
+        * and https://cryptography.io/en/latest/fernet/#using-passwords-with-fernet
+        """
+        salt_bytes = salt.encode()
+        if len(salt_bytes) < 16:
+            raise SaltLengthError
+        kdf = PBKDF2HMAC(
+            algorithm=hashes.SHA256(), length=32, salt=salt_bytes, iterations=600_000
+        )
+        return urlsafe_b64encode(kdf.derive(self._password))

notifications_utils/clients/redis/__init__.py

@@ -0,0 +1,13 @@
+from datetime import datetime
+
+from .request_cache import RequestCache  # noqa: F401 (unused import)
+
+
+def total_limit_cache_key(service_id):
+    return "{}-{}-{}".format(
+        str(service_id), datetime.utcnow().strftime("%Y-%m-%d"), "total-count"
+    )
+
+
+def rate_limit_cache_key(service_id, api_key_type):
+    return "{}-{}".format(str(service_id), api_key_type)

notifications_utils/clients/redis/redis_client.py

@@ -0,0 +1,184 @@
+import numbers
+import uuid
+from time import time
+
+from flask import current_app
+from flask_redis import FlaskRedis
+
+
+def prepare_value(val):
+    """
+    Only bytes, strings and numbers (ints, longs and floats) are acceptable
+    for keys and values. Previously redis-py attempted to cast other types
+    to str() and store the result. This caused must confusion and frustration
+    when passing boolean values (cast to 'True' and 'False') or None values
+    (cast to 'None'). It is now the user's responsibility to cast all
+    key names and values to bytes, strings or numbers before passing the
+    value to redis-py.
+    """
+    # things redis-py natively supports
+    if isinstance(
+        val,
+        (
+            bytes,
+            str,
+            numbers.Number,
+        ),
+    ):
+        return val
+    # things we know we can safely cast to string
+    elif isinstance(val, (uuid.UUID,)):
+        return str(val)
+    else:
+        raise ValueError("cannot cast {} to a string".format(type(val)))
+
+
+class RedisClient:
+    redis_store = FlaskRedis()
+    active = False
+    scripts = {}
+
+    def init_app(self, app):
+        self.active = app.config.get("REDIS_ENABLED")
+        if self.active:
+            self.redis_store.init_app(app)
+
+            self.register_scripts()
+
+    def register_scripts(self):
+        # delete keys matching a pattern supplied as a parameter. Does so in batches of 5000 to prevent unpack from
+        # exceeding lua's stack limit, and also to prevent errors if no keys match the pattern.
+        # Inspired by https://gist.github.com/ddre54/0a4751676272e0da8186
+        self.scripts["delete-keys-by-pattern"] = self.redis_store.register_script(
+            """
+            local keys = redis.call('keys', ARGV[1])
+            local deleted = 0
+            for i=1, #keys, 5000 do
+                deleted = deleted + redis.call('del', unpack(keys, i, math.min(i + 4999, #keys)))
+            end
+            return deleted
+            """
+        )
+
+    def delete_by_pattern(self, pattern, raise_exception=False):
+        r"""
+        Deletes all keys matching a given pattern, and returns how many keys were deleted.
+        Pattern is defined as in the KEYS command: https://redis.io/commands/keys
+
+        * h?llo matches hello, hallo and hxllo
+        * h*llo matches hllo and heeeello
+        * h[ae]llo matches hello and hallo, but not hillo
+        * h[^e]llo matches hallo, hbllo, ... but not hello
+        * h[a-b]llo matches hallo and hbllo
+
+        Use \ to escape special characters if you want to match them verbatim
+        """
+        if self.active:
+            try:
+                return self.scripts["delete-keys-by-pattern"](args=[pattern])
+            except Exception as e:
+                self.__handle_exception(
+                    e, raise_exception, "delete-by-pattern", pattern
+                )
+
+        return 0
+
+    def exceeded_rate_limit(self, cache_key, limit, interval, raise_exception=False):
+        """
+        Rate limiting.
+        - Uses Redis sorted sets
+        - Also uses redis "multi" which is abstracted into pipeline() by FlaskRedis/PyRedis
+        - Sends all commands to redis as a group to be executed atomically
+
+        Method:
+        (1) Add event, scored by timestamp (zadd). The score determines order in set.
+        (2) Use zremrangebyscore to delete all set members with a score between
+            - Earliest entry (lowest score == earliest timestamp) - represented as '-inf'
+                and
+            - Current timestamp minus the interval
+            - Leaves only relevant entries in the set (those between now and now - interval)
+        (3) Count the set
+        (4) If count > limit fail request
+        (5) Ensure we expire the set key to preserve space
+
+        Notes:
+        - Failed requests count. If over the limit and keep making requests you'll stay over the limit.
+        - The actual value in the set is just the timestamp, the same as the score. We don't store any requets details.
+        - return value of pipe.execute() is an array containing the outcome of each call.
+            - result[2] == outcome of pipe.zcard()
+        - If redis is inactive, or we get an exception, allow the request
+
+        :param cache_key:
+        :param limit: Number of requests permitted within interval
+        :param interval: Interval we measure requests in
+        :param raise_exception: Should throw exception
+        :return:
+        """
+        cache_key = prepare_value(cache_key)
+        if self.active:
+            try:
+                pipe = self.redis_store.pipeline()
+                when = time()
+                pipe.zadd(cache_key, {when: when})
+                pipe.zremrangebyscore(cache_key, "-inf", when - interval)
+                pipe.zcard(cache_key)
+                pipe.expire(cache_key, interval)
+                result = pipe.execute()
+                return result[2] > limit
+            except Exception as e:
+                self.__handle_exception(
+                    e, raise_exception, "rate-limit-pipeline", cache_key
+                )
+                return False
+        else:
+            return False
+
+    def raw_set(self, key, value, ex=None, px=None, nx=False, xx=False):
+        self.redis_store.set(key, value, ex, px, nx, xx)
+
+    def set(
+        self, key, value, ex=None, px=None, nx=False, xx=False, raise_exception=False
+    ):
+        key = prepare_value(key)
+        value = prepare_value(value)
+        if self.active:
+            try:
+                self.redis_store.set(key, value, ex, px, nx, xx)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "set", key)
+
+    def incr(self, key, raise_exception=False):
+        key = prepare_value(key)
+        if self.active:
+            try:
+                return self.redis_store.incr(key)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "incr", key)
+
+    def raw_get(self, key):
+        return self.redis_store.get(key)
+
+    def get(self, key, raise_exception=False):
+        key = prepare_value(key)
+        if self.active:
+            try:
+                return self.redis_store.get(key)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "get", key)
+
+        return None
+
+    def delete(self, *keys, raise_exception=False):
+        keys = [prepare_value(k) for k in keys]
+        if self.active:
+            try:
+                self.redis_store.delete(*keys)
+            except Exception as e:
+                self.__handle_exception(e, raise_exception, "delete", ", ".join(keys))
+
+    def __handle_exception(self, e, raise_exception, operation, key_name):
+        current_app.logger.exception(
+            "Redis error performing {} on {}".format(operation, key_name)
+        )
+        if raise_exception:
+            raise e

notifications_utils/clients/redis/request_cache.py

@@ -0,0 +1,95 @@
+import json
+from contextlib import suppress
+from datetime import timedelta
+from functools import wraps
+from inspect import signature
+
+
+class RequestCache:
+    DEFAULT_TTL = int(timedelta(days=7).total_seconds())
+
+    def __init__(self, redis_client):
+        self.redis_client = redis_client
+
+    @staticmethod
+    def _get_argument(argument_name, client_method, args, kwargs):
+        with suppress(KeyError):
+            return kwargs[argument_name]
+
+        with suppress(ValueError, IndexError):
+            argument_index = list(signature(client_method).parameters).index(
+                argument_name
+            )
+            return args[argument_index]
+
+        with suppress(KeyError):
+            return signature(client_method).parameters[argument_name].default
+
+        raise TypeError(
+            "{}() takes no argument called '{}'".format(
+                client_method.__name__, argument_name
+            )
+        )
+
+    @staticmethod
+    def _make_key(key_format, client_method, args, kwargs):
+        return key_format.format(
+            **{
+                argument_name: RequestCache._get_argument(
+                    argument_name, client_method, args, kwargs
+                )
+                for argument_name in list(signature(client_method).parameters)
+            }
+        )
+
+    def set(self, key_format, *, ttl_in_seconds=DEFAULT_TTL):
+        def _set(client_method):
+            @wraps(client_method)
+            def new_client_method(*args, **kwargs):
+                redis_key = RequestCache._make_key(
+                    key_format, client_method, args, kwargs
+                )
+                cached = self.redis_client.get(redis_key)
+                if cached:
+                    return json.loads(cached.decode("utf-8"))
+                api_response = client_method(*args, **kwargs)
+                self.redis_client.set(
+                    redis_key,
+                    json.dumps(api_response),
+                    ex=int(ttl_in_seconds),
+                )
+                return api_response
+
+            return new_client_method
+
+        return _set
+
+    def delete(self, key_format):
+        def _delete(client_method):
+            @wraps(client_method)
+            def new_client_method(*args, **kwargs):
+                try:
+                    api_response = client_method(*args, **kwargs)
+                finally:
+                    redis_key = self._make_key(key_format, client_method, args, kwargs)
+                    self.redis_client.delete(redis_key)
+                return api_response
+
+            return new_client_method
+
+        return _delete
+
+    def delete_by_pattern(self, key_format):
+        def _delete(client_method):
+            @wraps(client_method)
+            def new_client_method(*args, **kwargs):
+                try:
+                    api_response = client_method(*args, **kwargs)
+                finally:
+                    redis_key = self._make_key(key_format, client_method, args, kwargs)
+                    self.redis_client.delete_by_pattern(redis_key)
+                return api_response
+
+            return new_client_method
+
+        return _delete

notifications_utils/clients/zendesk/zendesk_client.py

@@ -0,0 +1,150 @@
+import requests
+from flask import current_app
+
+
+class ZendeskError(Exception):
+    def __init__(self, response):
+        self.response = response
+
+
+class ZendeskClient:
+    # the account used to authenticate with. If no requester is provided, the ticket will come from this account.
+    NOTIFY_ZENDESK_EMAIL = "zd-api-notify@digital.cabinet-office.gov.uk"
+
+    ZENDESK_TICKET_URL = "https://govuk.zendesk.com/api/v2/tickets.json"
+
+    def __init__(self):
+        self.api_key = None
+
+    def init_app(self, app, *args, **kwargs):
+        self.api_key = app.config.get("ZENDESK_API_KEY")
+
+    def send_ticket_to_zendesk(self, ticket):
+        response = requests.post(
+            self.ZENDESK_TICKET_URL,
+            json=ticket.request_data,
+            auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key),
+        )
+
+        if response.status_code != 201:
+            current_app.logger.error(
+                f"Zendesk create ticket request failed with {response.status_code} '{response.json()}'"
+            )
+            raise ZendeskError(response)
+
+        ticket_id = response.json()["ticket"]["id"]
+
+        current_app.logger.info(f"Zendesk create ticket {ticket_id} succeeded")
+
+
+class NotifySupportTicket:
+    PRIORITY_URGENT = "urgent"
+    PRIORITY_HIGH = "high"
+    PRIORITY_NORMAL = "normal"
+    PRIORITY_LOW = "low"
+
+    TAGS_P2 = "govuk_notify_support"
+    TAGS_P1 = "govuk_notify_emergency"
+
+    TYPE_PROBLEM = "problem"
+    TYPE_INCIDENT = "incident"
+    TYPE_QUESTION = "question"
+    TYPE_TASK = "task"
+
+    # Group: 3rd Line--Notify Support
+    NOTIFY_GROUP_ID = 360000036529
+    # Organization: GDS
+    NOTIFY_ORG_ID = 21891972
+    NOTIFY_TICKET_FORM_ID = 1900000284794
+
+    def __init__(
+        self,
+        subject,
+        message,
+        ticket_type,
+        p1=False,
+        user_name=None,
+        user_email=None,
+        requester_sees_message_content=True,
+        technical_ticket=False,
+        ticket_categories=None,
+        org_id=None,
+        org_type=None,
+        service_id=None,
+        email_ccs=None,
+    ):
+        self.subject = subject
+        self.message = message
+        self.ticket_type = ticket_type
+        self.p1 = p1
+        self.user_name = user_name
+        self.user_email = user_email
+        self.requester_sees_message_content = requester_sees_message_content
+        self.technical_ticket = technical_ticket
+        self.ticket_categories = ticket_categories or []
+        self.org_id = org_id
+        self.org_type = org_type
+        self.service_id = service_id
+        self.email_ccs = email_ccs
+
+    @property
+    def request_data(self):
+        data = {
+            "ticket": {
+                "subject": self.subject,
+                "comment": {
+                    "body": self.message,
+                    "public": self.requester_sees_message_content,
+                },
+                "group_id": self.NOTIFY_GROUP_ID,
+                "organization_id": self.NOTIFY_ORG_ID,
+                "ticket_form_id": self.NOTIFY_TICKET_FORM_ID,
+                "priority": self.PRIORITY_URGENT if self.p1 else self.PRIORITY_NORMAL,
+                "tags": [self.TAGS_P1 if self.p1 else self.TAGS_P2],
+                "type": self.ticket_type,
+                "custom_fields": self._get_custom_fields(),
+            }
+        }
+
+        if self.email_ccs:
+            data["ticket"]["email_ccs"] = [
+                {"user_email": email, "action": "put"} for email in self.email_ccs
+            ]
+
+        # if no requester provided, then the call came from within Notify 👻
+        if self.user_email:
+            data["ticket"]["requester"] = {
+                "email": self.user_email,
+                "name": self.user_name or "(no name supplied)",
+            }
+
+        return data
+
+    def _get_custom_fields(self):
+        technical_ticket_tag = (
+            f'notify_ticket_type_{"" if self.technical_ticket else "non_"}technical'
+        )
+        org_type_tag = f"notify_org_type_{self.org_type}" if self.org_type else None
+
+        return [
+            {
+                "id": "1900000744994",
+                "value": technical_ticket_tag,
+            },  # Notify Ticket type field
+            {
+                "id": "360022836500",
+                "value": self.ticket_categories,
+            },  # Notify Ticket category field
+            {
+                "id": "360022943959",
+                "value": self.org_id,
+            },  # Notify Organisation ID field
+            {
+                "id": "360022943979",
+                "value": org_type_tag,
+            },  # Notify Organisation type field
+            {
+                "id": "1900000745014",
+                "value": self.service_id,
+            },  # Notify Service ID field
+        ]