darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-23 15:56:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1896 from GSA/report_performance

Browse Source 
try again
This commit is contained in:
Kenneth Kehl
2025-08-06 12:01:54 -07:00
committed by GitHub
parent 2d8adcc774 1d3768a48d
commit 2dceb79b02
5 changed files with 135 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
app/aws/s3.py
View File

@@ -2,6 +2,7 @@ import csv
import datetime import datetime
import re import re
import time import time
import urllib
from io import StringIO from io import StringIO
import botocore import botocore
@@ -613,3 +614,44 @@ def remove_csv_object(object_key):
current_app.config["CSV_UPLOAD_BUCKET"]["region"], current_app.config["CSV_UPLOAD_BUCKET"]["region"],
) )
return obj.delete() return obj.delete()
def s3upload(
filedata,
region,
bucket_name,
file_location,
content_type="binary/octet-stream",
tags=None,
metadata=None,
):
_s3 = get_s3_resource()
key = _s3.Object(bucket_name, file_location)
put_args = {
"Body": filedata,
"ServerSideEncryption": "AES256",
"ContentType": content_type,
}
if tags:
tags = urllib.parse.urlencode(tags)
put_args["Tagging"] = tags
if metadata:
metadata = put_args["Metadata"] = metadata
try:
current_app.logger.info(hilite(f"Going to try to upload this {key}"))
key.put(**put_args)
except botocore.exceptions.NoCredentialsError as e:
current_app.logger.exception(
f"Unable to upload {key} to S3 bucket because of {e}"
)
raise e
except botocore.exceptions.ClientError as e:
current_app.logger.exception(
f"Unable to upload {key}to S3 bucket because of {e}"
)
raise e

4
app/celery/tasks.py
View File

@@ -36,7 +36,6 @@ from app.serialised_models import SerialisedService, SerialisedTemplate
from app.service.utils import service_allowed_to_send_to from app.service.utils import service_allowed_to_send_to
from app.utils import DATETIME_FORMAT, hilite, utc_now from app.utils import DATETIME_FORMAT, hilite, utc_now
from notifications_utils.recipients import RecipientCSV from notifications_utils.recipients import RecipientCSV
from notifications_utils.s3 import s3upload
@notify_celery.task(name="process-job") @notify_celery.task(name="process-job")
@@ -640,7 +639,7 @@ def _generate_notifications_report(service_id, report_id, limit_days):
# Delete yesterday's version of this report # Delete yesterday's version of this report
s3.delete_s3_object(file_location) s3.delete_s3_object(file_location)
s3upload( s3.s3upload(
filedata=csv_bytes, filedata=csv_bytes,
region=region, region=region,
bucket_name=bucket_name, bucket_name=bucket_name,
@@ -654,7 +653,6 @@ def generate_notification_reports_task():
services = dao_fetch_all_services(only_active=True) services = dao_fetch_all_services(only_active=True)
for service in services: for service in services:
current_app.logger.debug(hilite("INVOKE APPLY_ASYNC"))
limit_days = [1, 3, 5, 7] limit_days = [1, 3, 5, 7]
for limit_day in limit_days: for limit_day in limit_days:

2
app/config.py
View File

@@ -289,7 +289,7 @@ class Config(object):
}, },
"generate-notifications-reports": { "generate-notifications-reports": {
"task": "generate-notifications-reports", "task": "generate-notifications-reports",
"schedule": crontab(hour=1, minute=0), "schedule": crontab(minute="*/2"),
"options": {"queue": QueueNames.PERIODIC}, "options": {"queue": QueueNames.PERIODIC},
}, },
"regenerate-job-cache-on-startup": { "regenerate-job-cache-on-startup": {

60
notifications_utils/s3.py
View File

@@ -1,12 +1,8 @@
import urllib
import botocore import botocore
from boto3 import Session from boto3 import Session
from botocore.config import Config from botocore.config import Config
from flask import current_app from flask import current_app
from app.config import _s3_credentials_from_env
AWS_CLIENT_CONFIG = Config( AWS_CLIENT_CONFIG = Config(
# This config is required to enable S3 to connect to FIPS-enabled # This config is required to enable S3 to connect to FIPS-enabled
# endpoints. See https://aws.amazon.com/compliance/fips/ for more # endpoints. See https://aws.amazon.com/compliance/fips/ for more
@@ -21,56 +17,16 @@ default_regions = "us-gov-west-1"
def get_s3_resource(): def get_s3_resource():
access_key = current_app.config["CSV_UPLOAD_BUCKET"]["access_key_id"]
credentials = _s3_credentials_from_env("CSV") secret_key = current_app.config["CSV_UPLOAD_BUCKET"]["secret_access_key"]
region = current_app.config["CSV_UPLOAD_BUCKET"]["region"]
session = Session( session = Session(
aws_access_key_id=credentials["access_key_id"], aws_access_key_id=access_key,
aws_secret_access_key=credentials["secret_access_key"], aws_secret_access_key=secret_key,
region_name=credentials["region"], region_name=region,
) )
noti_s3_resource = session.resource("s3", config=AWS_CLIENT_CONFIG) s3_resource = session.resource("s3", config=AWS_CLIENT_CONFIG)
return noti_s3_resource return s3_resource
def s3upload(
filedata,
region,
bucket_name,
file_location,
content_type="binary/octet-stream",
tags=None,
metadata=None,
):
_s3 = get_s3_resource()
key = _s3.Object(bucket_name, file_location)
put_args = {
"Body": filedata,
"ServerSideEncryption": "AES256",
"ContentType": content_type,
}
if tags:
tags = urllib.parse.urlencode(tags)
put_args["Tagging"] = tags
if metadata:
metadata = put_args["Metadata"] = metadata
try:
current_app.logger.info(f"Going to try to upload this {key}")
key.put(**put_args)
except botocore.exceptions.NoCredentialsError as e:
current_app.logger.exception(
f"Unable to upload {key} to S3 bucket because of {e}"
)
raise e
except botocore.exceptions.ClientError as e:
current_app.logger.exception(
f"Unable to upload {key}to S3 bucket because of {e}"
)
raise e
class S3ObjectNotFound(botocore.exceptions.ClientError): class S3ObjectNotFound(botocore.exceptions.ClientError):

165
tests/notifications_utils/test_s3.py
View File

@@ -1,17 +1,18 @@
from unittest.mock import MagicMock from unittest.mock import MagicMock
from urllib.parse import parse_qs
import botocore import botocore
import pytest import pytest
from notifications_utils.s3 import ( from notifications_utils.s3 import ( # s3upload,
AWS_CLIENT_CONFIG, AWS_CLIENT_CONFIG,
S3ObjectNotFound, S3ObjectNotFound,
get_s3_resource, get_s3_resource,
s3download, s3download,
s3upload,
) )
# from urllib.parse import parse_qs
contents = "some file data" contents = "some file data"
region = "eu-west-1" region = "eu-west-1"
bucket = "some_bucket" bucket = "some_bucket"
@@ -19,102 +20,102 @@ location = "some_file_location"
content_type = "binary/octet-stream" content_type = "binary/octet-stream"
def test_s3upload_save_file_to_bucket(mocker): # def test_s3upload_save_file_to_bucket(mocker):
mock_s3_resource = mocker.Mock() # mock_s3_resource = mocker.Mock()
mocked = mocker.patch( # mocked = mocker.patch(
"notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource # "notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource
) # )
s3upload( # s3upload(
filedata=contents, region=region, bucket_name=bucket, file_location=location # filedata=contents, region=region, bucket_name=bucket, file_location=location
) # )
mocked_put = mocked.return_value.Object.return_value.put # mocked_put = mocked.return_value.Object.return_value.put
mocked_put.assert_called_once_with( # mocked_put.assert_called_once_with(
Body=contents, # Body=contents,
ServerSideEncryption="AES256", # ServerSideEncryption="AES256",
ContentType=content_type, # ContentType=content_type,
) # )
def test_s3upload_save_file_to_bucket_with_contenttype(mocker): # def test_s3upload_save_file_to_bucket_with_contenttype(mocker):
content_type = "image/png" # content_type = "image/png"
mock_s3_resource = mocker.Mock() # mock_s3_resource = mocker.Mock()
mocked = mocker.patch( # mocked = mocker.patch(
"notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource # "notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource
) # )
s3upload( # s3upload(
filedata=contents, # filedata=contents,
region=region, # region=region,
bucket_name=bucket, # bucket_name=bucket,
file_location=location, # file_location=location,
content_type=content_type, # content_type=content_type,
) # )
mocked_put = mocked.return_value.Object.return_value.put # mocked_put = mocked.return_value.Object.return_value.put
mocked_put.assert_called_once_with( # mocked_put.assert_called_once_with(
Body=contents, # Body=contents,
ServerSideEncryption="AES256", # ServerSideEncryption="AES256",
ContentType=content_type, # ContentType=content_type,
) # )
def test_s3upload_raises_exception(app, mocker): # def test_s3upload_raises_exception(app, mocker):
mock_s3_resource = mocker.Mock() # mock_s3_resource = mocker.Mock()
mocked = mocker.patch( # mocked = mocker.patch(
"notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource # "notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource
) # )
response = {"Error": {"Code": 500}} # response = {"Error": {"Code": 500}}
exception = botocore.exceptions.ClientError(response, "Bad exception") # exception = botocore.exceptions.ClientError(response, "Bad exception")
mocked.return_value.Object.return_value.put.side_effect = exception # mocked.return_value.Object.return_value.put.side_effect = exception
with pytest.raises(botocore.exceptions.ClientError): # with pytest.raises(botocore.exceptions.ClientError):
s3upload( # s3upload(
filedata=contents, # filedata=contents,
region=region, # region=region,
bucket_name=bucket, # bucket_name=bucket,
file_location="location", # file_location="location",
) # )
def test_s3upload_save_file_to_bucket_with_urlencoded_tags(mocker): # def test_s3upload_save_file_to_bucket_with_urlencoded_tags(mocker):
mock_s3_resource = mocker.Mock() # mock_s3_resource = mocker.Mock()
mocked = mocker.patch( # mocked = mocker.patch(
"notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource # "notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource
) # )
s3upload( # s3upload(
filedata=contents, # filedata=contents,
region=region, # region=region,
bucket_name=bucket, # bucket_name=bucket,
file_location=location, # file_location=location,
tags={"a": "1/2", "b": "x y"}, # tags={"a": "1/2", "b": "x y"},
) # )
mocked_put = mocked.return_value.Object.return_value.put # mocked_put = mocked.return_value.Object.return_value.put
# make sure tags were a urlencoded query string # # make sure tags were a urlencoded query string
encoded_tags = mocked_put.call_args[1]["Tagging"] # encoded_tags = mocked_put.call_args[1]["Tagging"]
assert parse_qs(encoded_tags) == {"a": ["1/2"], "b": ["x y"]} # assert parse_qs(encoded_tags) == {"a": ["1/2"], "b": ["x y"]}
def test_s3upload_save_file_to_bucket_with_metadata(mocker): # def test_s3upload_save_file_to_bucket_with_metadata(mocker):
mock_s3_resource = mocker.Mock() # mock_s3_resource = mocker.Mock()
mocked = mocker.patch( # mocked = mocker.patch(
"notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource # "notifications_utils.s3.get_s3_resource", return_value=mock_s3_resource
) # )
s3upload( # s3upload(
filedata=contents, # filedata=contents,
region=region, # region=region,
bucket_name=bucket, # bucket_name=bucket,
file_location=location, # file_location=location,
metadata={"status": "valid", "pages": "5"}, # metadata={"status": "valid", "pages": "5"},
) # )
mocked_put = mocked.return_value.Object.return_value.put # mocked_put = mocked.return_value.Object.return_value.put
metadata = mocked_put.call_args[1]["Metadata"] # metadata = mocked_put.call_args[1]["Metadata"]
assert metadata == {"status": "valid", "pages": "5"} # assert metadata == {"status": "valid", "pages": "5"}
def test_get_s3_resource(mocker): def test_get_s3_resource(mocker):