diff --git a/.ds.baseline b/.ds.baseline index 864192e1e..34d7e63e5 100644 --- a/.ds.baseline +++ b/.ds.baseline @@ -209,7 +209,7 @@ "filename": "tests/app/aws/test_s3.py", "hashed_secret": "67a74306b06d0c01624fe0d0249a570f4d093747", "is_verified": false, - "line_number": 40, + "line_number": 42, "is_secret": false } ], @@ -384,5 +384,5 @@ } ] }, - "generated_at": "2025-02-10T16:57:15Z" + "generated_at": "2025-02-27T21:09:45Z" } diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ba1c6b80a..1aafce715 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,5 @@ updates: directory: "/" # Location of package manifests schedule: interval: "daily" + labels: + - "dependabot" # Custom label to identify Dependabot PRs diff --git a/.github/workflows/deploy-demo.yml b/.github/workflows/deploy-demo.yml index 3c07156bd..c74b8d934 100644 --- a/.github/workflows/deploy-demo.yml +++ b/.github/workflows/deploy-demo.yml @@ -24,23 +24,13 @@ jobs: terraform_version: "^1.7.5" terraform_wrapper: false - - name: Check for changes to Terraform - id: changed-terraform-files - uses: tj-actions/changed-files@v45 - with: - files: | - terraform/demo/** - terraform/shared/** - .github/workflows/deploy-demo.yml - name: Terraform init - if: steps.changed-terraform-files.outputs.any_changed == 'true' working-directory: terraform/demo env: AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} run: terraform init - name: Terraform apply - if: steps.changed-terraform-files.outputs.any_changed == 'true' working-directory: terraform/demo env: AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} @@ -84,26 +74,27 @@ jobs: --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL" --strategy rolling - - name: Check for changes to templates.json - id: changed-templates - uses: tj-actions/changed-files@v45 - with: - files: | - app/config_files/templates.json - name: Update templates - if: steps.changed-templates.outputs.any_changed == 'true' - run: cf run-task notify-api-demo --command "flask command update-templates" + uses: cloud-gov/cg-cli-tools@main + env: + DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }} + NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }} + LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-demo.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE" - - name: Check for changes to egress config - id: changed-egress-config - uses: tj-actions/changed-files@v45 with: - files: | - deploy-config/egress_proxy/notify-api-demo.*.acl - .github/actions/deploy-proxy/action.yml - .github/workflows/deploy-demo.yml + cf_username: ${{ secrets.CLOUDGOV_USERNAME }} + cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + cf_org: gsa-tts-benefits-studio + cf_space: notify-demo + cf_command: >- + run-task notify-api-demo --command "flask command update-templates" + + - name: Deploy egress proxy - if: steps.changed-egress-config.outputs.any_changed == 'true' uses: ./.github/actions/deploy-proxy env: CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }} diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 703fc35b2..df9663430 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -28,23 +28,13 @@ jobs: terraform_version: "^1.7.5" terraform_wrapper: false - - name: Check for changes to Terraform - id: changed-terraform-files - uses: tj-actions/changed-files@v45 - with: - files: | - terraform/production/** - terraform/shared/** - .github/workflows/deploy-prod.yml - name: Terraform init - if: steps.changed-terraform-files.outputs.any_changed == 'true' working-directory: terraform/production env: AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} run: terraform init - name: Terraform apply - if: steps.changed-terraform-files.outputs.any_changed == 'true' working-directory: terraform/production env: AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} @@ -88,26 +78,26 @@ jobs: --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL" --strategy rolling - - name: Check for changes to templates.json - id: changed-templates - uses: tj-actions/changed-files@v45 - with: - files: | - app/config_files/templates.json - name: Update templates - if: steps.changed-templates.outputs.any_changed == 'true' - run: cf run-task notify-api-production --command "flask command update-templates" + uses: cloud-gov/cg-cli-tools@main + env: + DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }} + NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }} + LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://beta.notify.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE" - - name: Check for changes to egress config - id: changed-egress-config - uses: tj-actions/changed-files@v45 with: - files: | - deploy-config/egress_proxy/notify-api-production.*.acl - .github/actions/deploy-proxy/action.yml - .github/workflows/deploy-prod.yml + cf_username: ${{ secrets.CLOUDGOV_USERNAME }} + cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + cf_org: gsa-tts-benefits-studio + cf_space: notify-production + cf_command: >- + run-task notify-api-production --command "flask command update-templates" + - name: Deploy egress proxy - if: steps.changed-egress-config.outputs.any_changed == 'true' uses: ./.github/actions/deploy-proxy env: CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index eb32dd1eb..5560d30f2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,23 +30,13 @@ jobs: terraform_version: "^1.7.5" terraform_wrapper: false - - name: Check for changes to Terraform - id: changed-terraform-files - uses: tj-actions/changed-files@v45 - with: - files: | - terraform/staging/** - terraform/shared/** - .github/workflows/deploy.yml - name: Terraform init - if: steps.changed-terraform-files.outputs.any_changed == 'true' working-directory: terraform/staging env: AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_STATE_SECRET_ACCESS_KEY }} run: terraform init - name: Terraform apply - if: steps.changed-terraform-files.outputs.any_changed == 'true' working-directory: terraform/staging env: AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_STATE_ACCESS_KEY }} @@ -90,26 +80,25 @@ jobs: --var LOGIN_DOT_GOV_REGISTRATION_URL="$LOGIN_DOT_GOV_REGISTRATION_URL" --strategy rolling - - name: Check for changes to templates.json - id: changed-templates - uses: tj-actions/changed-files@v45 - with: - files: | - app/config_files/templates.json - name: Update templates - if: steps.changed-templates.outputs.any_changed == 'true' - run: cf run-task notify-api-staging --command "flask command update-templates" - - - name: Check for changes to egress config - id: changed-egress-config - uses: tj-actions/changed-files@v45 + uses: cloud-gov/cg-cli-tools@main + env: + DANGEROUS_SALT: ${{ secrets.DANGEROUS_SALT }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + ADMIN_CLIENT_SECRET: ${{ secrets.ADMIN_CLIENT_SECRET }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + NOTIFY_E2E_TEST_EMAIL: ${{ secrets.NOTIFY_E2E_TEST_EMAIL }} + NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }} + LOGIN_DOT_GOV_REGISTRATION_URL: "https://secure.login.gov/openid_connect/authorize?acr_values=http%3A%2F%2Fidmanagement.gov%2Fns%2Fassurance%2Fial%2F1&client_id=urn:gov:gsa:openidconnect.profiles:sp:sso:gsa:notify-gov&nonce=NONCE&prompt=select_account&redirect_uri=https://notify-staging.app.cloud.gov/set-up-your-profile&response_type=code&scope=openid+email&state=STATE" with: - files: | - deploy-config/egress_proxy/notify-api-staging.*.acl - .github/actions/deploy-proxy/action.yml - .github/workflows/deploy.yml + cf_username: ${{ secrets.CLOUDGOV_USERNAME }} + cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} + cf_org: gsa-tts-benefits-studio + cf_space: notify-staging + cf_command: >- + run-task notify-api-staging --command "flask command update-templates" + - name: Deploy egress proxy - if: steps.changed-egress-config.outputs.any_changed == 'true' uses: ./.github/actions/deploy-proxy env: CF_USERNAME: ${{ secrets.CLOUDGOV_USERNAME }} diff --git a/.github/workflows/restage-apps.yml b/.github/workflows/restage-apps.yml index 23c78f8cf..46fb88b88 100644 --- a/.github/workflows/restage-apps.yml +++ b/.github/workflows/restage-apps.yml @@ -33,4 +33,6 @@ jobs: cf_password: ${{ secrets.CLOUDGOV_PASSWORD }} cf_org: gsa-tts-benefits-studio cf_space: notify-${{ inputs.environment }}-egress - command: "cf restage --strategy rolling egress-proxy-notify-${{matrix.app}}-${{inputs.environment}}" + command: | + cf push notify-${{matrix.app}}-${{inputs.environment}} -f manifest.yml --no-start + cf restage --strategy rolling egress-proxy-notify-${{matrix.app}}-${{inputs.environment}} diff --git a/Makefile b/Makefile index 741ceae5b..731d40654 100644 --- a/Makefile +++ b/Makefile @@ -148,3 +148,8 @@ clean: # cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api # cf stop notify-api-failwhale # @echo "Failwhale is disabled" + +.PHONY: test-single +test-single: export NEW_RELIC_ENVIRONMENT=test +test-single: ## Run a single test file + poetry run pytest -s $(TEST_FILE) diff --git a/app/aws/s3.py b/app/aws/s3.py index e8841b20c..bc1728541 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -118,7 +118,7 @@ def list_s3_objects(): break except Exception: current_app.logger.exception( - "An error occurred while regenerating cache #notify-admin-1200", + "An error occurred while regenerating cache #notify-debug-admin-1200", ) @@ -200,7 +200,7 @@ def read_s3_file(bucket_name, object_key, s3res): except LookupError: # perhaps our key is not formatted as we expected. If so skip it. - current_app.logger.exception("LookupError #notify-admin-1200") + current_app.logger.exception("LookupError #notify-debug-admin-1200") def get_s3_files(): @@ -213,7 +213,7 @@ def get_s3_files(): s3res = get_s3_resource() current_app.logger.info( - f"job_cache length before regen: {len_job_cache()} #notify-admin-1200" + f"job_cache length before regen: {len_job_cache()} #notify-debug-admin-1200" ) try: with ThreadPoolExecutor() as executor: @@ -222,7 +222,7 @@ def get_s3_files(): current_app.logger.exception("Connection pool issue") current_app.logger.info( - f"job_cache length after regen: {len_job_cache()} #notify-admin-1200" + f"job_cache length after regen: {len_job_cache()} #notify-debug-admin-1200" ) @@ -290,7 +290,7 @@ def file_exists(file_location): def get_job_location(service_id, job_id): current_app.logger.debug( - f"#s3-partitioning NEW JOB_LOCATION: {NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)}" + f"#notify-debug-s3-partitioning NEW JOB_LOCATION: {NEW_FILE_LOCATION_STRUCTURE.format(service_id, job_id)}" ) return ( current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], @@ -308,7 +308,7 @@ def get_old_job_location(service_id, job_id): Remove this when everything works with the NEW_FILE_LOCATION_STRUCTURE. """ current_app.logger.debug( - f"#s3-partitioning OLD JOB LOCATION: {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}" + f"#notify-debug-s3-partitioning OLD JOB LOCATION: {FILE_LOCATION_STRUCTURE.format(service_id, job_id)}" ) return ( current_app.config["CSV_UPLOAD_BUCKET"]["bucket"], @@ -507,7 +507,7 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number): def get_job_metadata_from_s3(service_id, job_id): current_app.logger.debug( - f"#s3-partitioning CALLING GET_JOB_METADATA with {service_id}, {job_id}" + f"#notify-debug-s3-partitioning CALLING GET_JOB_METADATA with {service_id}, {job_id}" ) obj = get_s3_object(*get_job_location(service_id, job_id)) return obj.get()["Metadata"] diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 2ff72780d..ebc17c452 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -325,7 +325,14 @@ def batch_insert_notifications(self): elif isinstance(notification_dict["created_at"], list): notification_dict["created_at"] = notification_dict["created_at"][0] notification = Notification(**notification_dict) - if notification is not None: + # notify-api-749 do not write to db + # if we have a verify_code we know this is the authentication notification at login time + # and not csv (containing PII) provided by the user, so allow verify_code to continue to exist + if notification is None: + continue + if "verify_code" in str(notification.personalisation): + pass + else: batch.append(notification) try: dao_batch_insert_notifications(batch) diff --git a/app/clients/cloudwatch/aws_cloudwatch.py b/app/clients/cloudwatch/aws_cloudwatch.py index 43bedbb35..0a6d3d7be 100644 --- a/app/clients/cloudwatch/aws_cloudwatch.py +++ b/app/clients/cloudwatch/aws_cloudwatch.py @@ -107,6 +107,12 @@ class AwsCloudwatchClient(Client): provider_response = self._aws_value_or_default( event, "delivery", "providerResponse" ) + message_cost = self._aws_value_or_default(event, "delivery", "priceInUSD") + if message_cost is None or message_cost == "": + message_cost = 0.0 + else: + message_cost = float(message_cost) + my_timestamp = self._aws_value_or_default(event, "notification", "timestamp") return { "notification.messageId": event["notification"]["messageId"], @@ -114,6 +120,7 @@ class AwsCloudwatchClient(Client): "delivery.phoneCarrier": phone_carrier, "delivery.providerResponse": provider_response, "@timestamp": my_timestamp, + "delivery.priceInUSD": message_cost, } # Here is an example of how to get the events with log insights diff --git a/app/clients/pinpoint/aws_pinpoint.py b/app/clients/pinpoint/aws_pinpoint.py index d15d94601..3b4de7682 100644 --- a/app/clients/pinpoint/aws_pinpoint.py +++ b/app/clients/pinpoint/aws_pinpoint.py @@ -39,7 +39,7 @@ class AwsPinpointClient(Client): current_app.logger.info(hilite(response)) except ClientError: current_app.logger.exception( - "#validate-phone-number Could not validate with pinpoint" + "#notify-debug-validate-phone-number Could not validate with pinpoint" ) # TODO This is the structure of the response. When the phone validation diff --git a/app/clients/sms/aws_sns.py b/app/clients/sms/aws_sns.py index d36af600c..3c2282752 100644 --- a/app/clients/sms/aws_sns.py +++ b/app/clients/sms/aws_sns.py @@ -68,15 +68,16 @@ class AwsSnsClient(SmsClient): non_scrubbable = " ".join(sender) self.current_app.logger.info( - f"notify-api-1385 sender {non_scrubbable} is a {type(sender)} default is a {type(default_num)}" + f"notify-debug-api-1385 sender {non_scrubbable} is a {type(sender)} \ + default is a {type(default_num)}" ) else: self.current_app.logger.warning( - f"notify-api-1385 sender is type {type(sender)}!! {sender}" + f"notify-debug-api-1385 sender is type {type(sender)}!! {sender}" ) if self._valid_sender_number(sender): self.current_app.logger.info( - f"notify-api-1385 use valid sender {non_scrubbable} instead of default {default_num}" + f"notify-debug-api-1385 use valid sender {non_scrubbable} instead of default {default_num}" ) attributes["AWS.MM.SMS.OriginationNumber"] = { @@ -85,7 +86,7 @@ class AwsSnsClient(SmsClient): } else: self.current_app.logger.info( - f"notify-api-1385 use default {default_num} instead of invalid sender" + f"notify-debug-api-1385 use default {default_num} instead of invalid sender" ) attributes["AWS.MM.SMS.OriginationNumber"] = { diff --git a/app/dao/date_util.py b/app/dao/date_util.py index d93986b45..e09e7edfb 100644 --- a/app/dao/date_util.py +++ b/app/dao/date_util.py @@ -93,3 +93,25 @@ def generate_date_range(start_date, end_date=None, days=0): current_date += timedelta(days=1) else: return "An end_date or number of days must be specified" + + +def generate_hourly_range(start_date, end_date=None, hours=0): + if end_date: + current_time = start_date + while current_time <= end_date: + try: + yield current_time + except ValueError: + pass + current_time += timedelta(hours=1) + elif hours > 0: + end_time = start_date + timedelta(hours=hours) + current_time = start_date + while current_time < end_time: + try: + yield current_time + except ValueError: + pass + current_time += timedelta(hours=1) + else: + return "An end_date or number of hours must be specified" diff --git a/app/dao/jobs_dao.py b/app/dao/jobs_dao.py index feec601a4..2e479a77d 100644 --- a/app/dao/jobs_dao.py +++ b/app/dao/jobs_dao.py @@ -158,17 +158,17 @@ def dao_create_job(job): now_time = utc_now() diff_time = now_time - orig_time current_app.logger.info( - f"#notify-admin-1859 dao_create_job orig created at {orig_time} and now {now_time}" + f"#notify-debug-admin-1859 dao_create_job orig created at {orig_time} and now {now_time}" ) if diff_time.total_seconds() > 300: # It should be only a few seconds diff at most current_app.logger.error( - "#notify-admin-1859 Something is wrong with job.created_at!" + "#notify-debug-admin-1859 Something is wrong with job.created_at!" ) if os.getenv("NOTIFY_ENVIRONMENT") not in ["test"]: job.created_at = now_time dao_update_job(job) current_app.logger.error( - f"#notify-admin-1859 Job created_at reset to {job.created_at}" + f"#notify-debug-admin-1859 Job created_at reset to {job.created_at}" ) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 806f5e957..52823f7d6 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -267,6 +267,29 @@ def get_notifications_for_job( return pagination +def get_recent_notifications_for_job( + service_id, job_id, filter_dict=None, page=1, page_size=None +): + if page_size is None: + page_size = current_app.config["PAGE_SIZE"] + + stmt = select(Notification).where( + Notification.service_id == service_id, + Notification.job_id == job_id, + ) + + stmt = _filter_query(stmt, filter_dict) + stmt = stmt.order_by(desc(Notification.job_row_number)) + results = db.session.execute(stmt).scalars().all() + + page_size = current_app.config["PAGE_SIZE"] + offset = (page - 1) * page_size + paginated_results = results[offset : offset + page_size] + + pagination = Pagination(paginated_results, page, page_size, len(results)) + return pagination + + def dao_get_notification_count_for_job_id(*, job_id): stmt = select(func.count(Notification.id)).where(Notification.job_id == job_id) return db.session.execute(stmt).scalar() @@ -279,6 +302,44 @@ def dao_get_notification_count_for_service(*, service_id): return db.session.execute(stmt).scalar() +def dao_get_notification_count_for_service_message_ratio(service_id, current_year): + start_date = datetime(current_year, 1, 1) + end_date = datetime(current_year + 1, 1, 1) + stmt1 = ( + select(func.count()) + .select_from(Notification) + .where( + Notification.service_id == service_id, + Notification.status + not in [ + NotificationStatus.CANCELLED, + NotificationStatus.CREATED, + NotificationStatus.SENDING, + ], + Notification.created_at >= start_date, + Notification.created_at < end_date, + ) + ) + stmt2 = ( + select(func.count()) + .select_from(NotificationHistory) + .where( + NotificationHistory.service_id == service_id, + NotificationHistory.status + not in [ + NotificationStatus.CANCELLED, + NotificationStatus.CREATED, + NotificationStatus.SENDING, + ], + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + ) + ) + recent_count = db.session.execute(stmt1).scalar_one() + old_count = db.session.execute(stmt2).scalar_one() + return recent_count + old_count + + def dao_get_failed_notification_count(): stmt = select(func.count(Notification.id)).where( Notification.status == NotificationStatus.FAILED @@ -446,7 +507,7 @@ def insert_notification_history_delete_notifications( SELECT id, job_id, job_row_number, service_id, template_id, template_version, api_key_id, key_type, notification_type, created_at, sent_at, sent_by, updated_at, reference, billable_units, client_reference, international, phone_prefix, rate_multiplier, notification_status, - created_by_id, document_download_count + created_by_id, document_download_count, message_cost FROM notifications WHERE service_id = :service_id AND notification_type = :notification_type @@ -781,7 +842,6 @@ def dao_update_delivery_receipts(receipts, delivered): new_receipts.append(r) receipts = new_receipts - id_to_carrier = { r["notification.messageId"]: r["delivery.phoneCarrier"] for r in receipts } @@ -790,9 +850,13 @@ def dao_update_delivery_receipts(receipts, delivered): } id_to_timestamp = {r["notification.messageId"]: r["@timestamp"] for r in receipts} + id_to_message_cost = { + r["notification.messageId"]: r["delivery.priceInUSD"] for r in receipts + } status_to_update_with = NotificationStatus.DELIVERED if not delivered: status_to_update_with = NotificationStatus.FAILED + stmt = ( update(Notification) .where(Notification.message_id.in_(id_to_carrier.keys())) @@ -816,6 +880,12 @@ def dao_update_delivery_receipts(receipts, delivered): for key, value in id_to_provider_response.items() ] ), + message_cost=case( + *[ + (Notification.message_id == key, value) + for key, value in id_to_message_cost.items() + ] + ), ) ) db.session.execute(stmt) @@ -847,7 +917,6 @@ def dao_close_out_delivery_receipts(): def dao_batch_insert_notifications(batch): - db.session.bulk_save_objects(batch) db.session.commit() current_app.logger.info(f"Batch inserted notifications: {len(batch)}") diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 47be682ce..db983697e 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -8,7 +8,11 @@ from sqlalchemy.sql.expression import and_, asc, case, func from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class -from app.dao.date_util import generate_date_range, get_current_calendar_year +from app.dao.date_util import ( + generate_date_range, + generate_hourly_range, + get_current_calendar_year, +) from app.dao.organization_dao import dao_get_organization_by_email_address from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user @@ -523,6 +527,75 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date): return total_notifications, data +def dao_fetch_stats_for_service_from_hours(service_id, start_date, end_date): + start_date = get_midnight_in_utc(start_date) + end_date = get_midnight_in_utc(end_date + timedelta(days=1)) + + # Update to group by HOUR instead of DAY + total_substmt = ( + select( + func.date_trunc("hour", NotificationAllTimeView.created_at).label( + "hour" + ), # UPDATED + Job.notification_count.label("notification_count"), + ) + .join(Job, NotificationAllTimeView.job_id == Job.id) + .where( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + ) + .group_by( + Job.id, + Job.notification_count, + func.date_trunc("hour", NotificationAllTimeView.created_at), # UPDATED + ) + .subquery() + ) + + # Also update this to group by hour + total_stmt = select( + total_substmt.c.hour, # UPDATED + func.sum(total_substmt.c.notification_count).label("total_notifications"), + ).group_by( + total_substmt.c.hour + ) # UPDATED + + # Ensure we're using hourly timestamps in the response + total_notifications = { + row.hour: row.total_notifications + for row in db.session.execute(total_stmt).all() + } + + # Update the second query to also use "hour" + stmt = ( + select( + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.date_trunc("hour", NotificationAllTimeView.created_at).label( + "hour" + ), # UPDATED + func.count(NotificationAllTimeView.id).label("count"), + ) + .where( + NotificationAllTimeView.service_id == service_id, + NotificationAllTimeView.key_type != KeyType.TEST, + NotificationAllTimeView.created_at >= start_date, + NotificationAllTimeView.created_at < end_date, + ) + .group_by( + NotificationAllTimeView.notification_type, + NotificationAllTimeView.status, + func.date_trunc("hour", NotificationAllTimeView.created_at), # UPDATED + ) + ) + + data = db.session.execute(stmt).all() + + return total_notifications, data + + def dao_fetch_stats_for_service_from_days_for_user( service_id, start_date, end_date, user_id ): @@ -531,7 +604,7 @@ def dao_fetch_stats_for_service_from_days_for_user( total_substmt = ( select( - func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + func.date_trunc("hour", NotificationAllTimeView.created_at).label("hour"), Job.notification_count.label("notification_count"), ) .join(Job, NotificationAllTimeView.job_id == Job.id) @@ -545,25 +618,25 @@ def dao_fetch_stats_for_service_from_days_for_user( .group_by( Job.id, Job.notification_count, - func.date_trunc("day", NotificationAllTimeView.created_at), + func.date_trunc("hour", NotificationAllTimeView.created_at), ) .subquery() ) total_stmt = select( - total_substmt.c.day, + total_substmt.c.hour, func.sum(total_substmt.c.notification_count).label("total_notifications"), - ).group_by(total_substmt.c.day) + ).group_by(total_substmt.c.hour) total_notifications = { - row.day: row.total_notifications for row in db.session.execute(total_stmt).all() + row.hour: row.total_notifications for row in db.session.execute(total_stmt).all() } stmt = ( select( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, - func.date_trunc("day", NotificationAllTimeView.created_at).label("day"), + func.date_trunc("hour", NotificationAllTimeView.created_at).label("hour"), func.count(NotificationAllTimeView.id).label("count"), ) .where( @@ -576,7 +649,7 @@ def dao_fetch_stats_for_service_from_days_for_user( .group_by( NotificationAllTimeView.notification_type, NotificationAllTimeView.status, - func.date_trunc("day", NotificationAllTimeView.created_at), + func.date_trunc("hour", NotificationAllTimeView.created_at), ) ) @@ -827,3 +900,38 @@ def get_specific_days_stats( for day, rows in grouped_data.items() } return stats + + +def get_specific_hours_stats( + data, start_date, hours=None, end_date=None, total_notifications=None +): + if hours is not None and end_date is not None: + raise ValueError("Only set hours OR set end_date, not both.") + elif hours is not None: + gen_range = [start_date + timedelta(hours=i) for i in range(hours)] + elif end_date is not None: + gen_range = generate_hourly_range(start_date, end_date=end_date) + else: + raise ValueError("Either hours or end_date must be set.") + + # Ensure all hours exist in the output (even if empty) + grouped_data = {hour: [] for hour in gen_range} + + # Group notifications based on full hour timestamps + for row in data: + notification_type, status, timestamp, count = row + + row_hour = timestamp.replace(minute=0, second=0, microsecond=0) + if row_hour in grouped_data: + grouped_data[row_hour].append(row) + + # Format statistics, returning only hours with results + stats = { + hour.strftime("%Y-%m-%dT%H:00:00Z"): statistics.format_statistics( + rows, total_notifications.get(hour, 0) if total_notifications else None + ) + for hour, rows in grouped_data.items() + if rows + } + + return stats diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index 515d418e7..cecddd98b 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -42,12 +42,14 @@ def send_sms_to_provider(notification): """ # Take this path for report generation, where we know # everything is in the cache. - personalisation = get_personalisation_from_s3( - notification.service_id, - notification.job_id, - notification.job_row_number, - ) - notification.personalisation = personalisation + + if "verify_code" not in str(notification.personalisation): + personalisation = get_personalisation_from_s3( + notification.service_id, + notification.job_id, + notification.job_row_number, + ) + notification.personalisation = personalisation service = SerialisedService.from_id(notification.service_id) message_id = None @@ -92,7 +94,6 @@ def send_sms_to_provider(notification): recipient = None # It is our 2facode, maybe recipient = _get_verify_code(notification) - if recipient is None: recipient = get_phone_number_from_s3( notification.service_id, @@ -107,7 +108,7 @@ def send_sms_to_provider(notification): sender_numbers = get_sender_numbers(notification) if notification.reply_to_text not in sender_numbers: raise ValueError( - f"{notification.reply_to_text} not in {sender_numbers} #notify-admin-1701" + f"{notification.reply_to_text} not in {sender_numbers} #notify-debug-admin-1701" ) send_sms_kwargs = { @@ -118,7 +119,13 @@ def send_sms_to_provider(notification): "international": notification.international, } db.session.close() # no commit needed as no changes to objects have been made above - + real_sender_number = notification.reply_to_text + # interleave spaces to bypass PII scrubbing since sender number is not PII + arr = list(real_sender_number) + real_sender_number = " ".join(arr) + current_app.logger.info( + f"#notify-debug-api-1701 real sender number going to AWS is {real_sender_number}" + ) message_id = provider.send_sms(**send_sms_kwargs) update_notification_message_id(notification.id, message_id) @@ -152,7 +159,7 @@ def _experimentally_validate_phone_numbers(recipient): if recipient_lookup in current_app.config["SIMULATED_SMS_NUMBERS"] and os.getenv( "NOTIFY_ENVIRONMENT" ) in ["development", "test"]: - current_app.logger.info(hilite("#validate-phone-number fired")) + current_app.logger.info(hilite("#notify-debug-validate-phone-number fired")) aws_pinpoint_client.validate_phone_number("01", recipient) diff --git a/app/job/rest.py b/app/job/rest.py index 8b3965061..571c1e342 100644 --- a/app/job/rest.py +++ b/app/job/rest.py @@ -1,5 +1,6 @@ +from zoneinfo import ZoneInfo + import dateutil -import pytz from flask import Blueprint, current_app, jsonify, request from app.aws.s3 import ( @@ -22,6 +23,7 @@ from app.dao.jobs_dao import ( from app.dao.notifications_dao import ( dao_get_notification_count_for_job_id, get_notifications_for_job, + get_recent_notifications_for_job, ) from app.dao.services_dao import dao_fetch_service_by_id from app.dao.templates_dao import dao_get_template_by_id @@ -124,6 +126,71 @@ def get_all_notifications_for_service_job(service_id, job_id): ) +@job_blueprint.route("//recent_notifications", methods=["GET"]) +def get_recent_notifications_for_service_job(service_id, job_id): + data = notifications_filter_schema.load(request.args) + page = data["page"] if "page" in data else 1 + page_size = ( + data["page_size"] + if "page_size" in data + else current_app.config.get("PAGE_SIZE") + ) + paginated_notifications = get_recent_notifications_for_job( + service_id, + job_id, + filter_dict=data, + page=page, + page_size=page_size, + ) + + kwargs = request.args.to_dict() + kwargs["service_id"] = service_id + kwargs["job_id"] = job_id + + for notification in paginated_notifications.items: + if notification.job_id is not None: + recipient = get_phone_number_from_s3( + notification.service_id, + notification.job_id, + notification.job_row_number, + ) + notification.to = recipient + notification.normalised_to = recipient + + for notification in paginated_notifications.items: + if notification.job_id is not None: + notification.personalisation = get_personalisation_from_s3( + notification.service_id, + notification.job_id, + notification.job_row_number, + ) + + notifications = None + if data.get("format_for_csv"): + notifications = [ + notification.serialize_for_csv() + for notification in paginated_notifications.items + ] + else: + notifications = notification_with_template_schema.dump( + paginated_notifications.items, many=True + ) + + return ( + jsonify( + notifications=notifications, + page_size=page_size, + total=paginated_notifications.total, + links=pagination_links( + paginated_notifications, + ".get_all_notifications_for_service_job", + **kwargs, + ), + ), + 200, + ) + + @job_blueprint.route("//notification_count", methods=["GET"]) def get_notification_count_for_job_id(service_id, job_id): dao_get_job_by_service_id_and_job_id(service_id, job_id) @@ -175,7 +242,9 @@ def create_job(service_id): original_file_name = data.get("original_file_name") data.update({"service": service_id}) try: - current_app.logger.info(f"#s3-partitioning DATA IN CREATE_JOB: {data}") + current_app.logger.info( + f"#notify-debug-s3-partitioning DATA IN CREATE_JOB: {data}" + ) data.update(**get_job_metadata_from_s3(service_id, data["id"])) except KeyError: raise InvalidRequest( @@ -225,7 +294,7 @@ def get_scheduled_job_stats(service_id): jsonify( count=count, soonest_scheduled_for=( - soonest_scheduled_for.replace(tzinfo=pytz.UTC).isoformat() + soonest_scheduled_for.replace(tzinfo=ZoneInfo("UTC")).isoformat() if soonest_scheduled_for else None ), diff --git a/app/models.py b/app/models.py index f78f630ea..01618d9ee 100644 --- a/app/models.py +++ b/app/models.py @@ -1508,6 +1508,7 @@ class Notification(db.Model): created_at = db.Column(db.DateTime, index=True, unique=False, nullable=False) sent_at = db.Column(db.DateTime, index=False, unique=False, nullable=True) sent_by = db.Column(db.String, nullable=True) + message_cost = db.Column(db.Float, nullable=True, default=0.0) updated_at = db.Column( db.DateTime, index=False, @@ -1708,7 +1709,7 @@ class Notification(db.Model): value = (obj.created_at.strftime("%Y-%m-%d %H:%M:%S"),) elif column.name in ["sent_at", "completed_at"]: value = None - elif column.name.endswith("_id"): + elif column.name.endswith("_id") or column.name == "id": value = getattr(obj, column.name) value = str(value) else: @@ -1813,6 +1814,7 @@ class NotificationHistory(db.Model, HistoryModel): created_at = db.Column(db.DateTime, unique=False, nullable=False) sent_at = db.Column(db.DateTime, index=False, unique=False, nullable=True) sent_by = db.Column(db.String, nullable=True) + message_cost = db.Column(db.Float, nullable=True, default=0.0) updated_at = db.Column( db.DateTime, index=False, diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 6b78ce753..1ea6a7be1 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -145,6 +145,9 @@ def persist_notification( # it's just too hard with redis and timing to test this here if os.getenv("NOTIFY_ENVIRONMENT") == "test": dao_create_notification(notification) + elif "verify_code" in str(notification.personalisation): + dao_create_notification(notification) + else: redis_store.rpush( "message_queue", diff --git a/app/service/rest.py b/app/service/rest.py index 98cb0e963..154ce3997 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -1,5 +1,6 @@ import itertools from datetime import datetime, timedelta +from zoneinfo import ZoneInfo from flask import Blueprint, current_app, jsonify, request from sqlalchemy import select @@ -7,7 +8,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm.exc import NoResultFound from werkzeug.datastructures import MultiDict -from app import db, redis_store +from app import db from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3 from app.config import QueueNames from app.dao import fact_notification_status_dao, notifications_dao @@ -28,7 +29,10 @@ from app.dao.fact_notification_status_dao import ( fetch_stats_for_all_services_by_date_range, ) from app.dao.inbound_numbers_dao import dao_allocate_number_for_service -from app.dao.notifications_dao import dao_get_notification_count_for_service +from app.dao.notifications_dao import ( + dao_get_notification_count_for_service, + dao_get_notification_count_for_service_message_ratio, +) from app.dao.organization_dao import dao_get_organization_by_service_id from app.dao.service_data_retention_dao import ( fetch_service_data_retention, @@ -67,6 +71,7 @@ from app.dao.services_dao import ( dao_fetch_service_by_id, dao_fetch_stats_for_service_from_days, dao_fetch_stats_for_service_from_days_for_user, + dao_fetch_stats_for_service_from_hours, dao_fetch_todays_stats_for_all_services, dao_fetch_todays_stats_for_service, dao_remove_user_from_service, @@ -76,6 +81,7 @@ from app.dao.services_dao import ( fetch_notification_stats_for_service_by_month_by_user, get_services_by_partial_name, get_specific_days_stats, + get_specific_hours_stats, ) from app.dao.templates_dao import dao_get_template_by_id from app.dao.users_dao import get_user_by_id @@ -109,7 +115,6 @@ from app.service.service_senders_schema import ( from app.service.utils import get_guest_list_objects from app.user.users_schema import post_set_permissions_schema from app.utils import get_prev_next_pagination_links, utc_now -from notifications_utils.clients.redis import total_limit_cache_key service_blueprint = Blueprint("service", __name__) @@ -227,22 +232,24 @@ def get_service_notification_statistics_by_day(service_id, start, days): def get_service_statistics_for_specific_days(service_id, start, days=1): - # start and end dates needs to be reversed because - # the end date is today and the start is x days in the past - # a day needs to be substracted to allow for today + # Calculate start and end date range end_date = datetime.strptime(start, "%Y-%m-%d") start_date = end_date - timedelta(days=days - 1) - total_notifications, results = dao_fetch_stats_for_service_from_days( + # Fetch hourly stats from DB + total_notifications, results = dao_fetch_stats_for_service_from_hours( service_id, start_date, end_date, ) - stats = get_specific_days_stats( + hours = days * 24 + + # Process data using new hourly stats function + stats = get_specific_hours_stats( results, start_date, - days=days, + hours=hours, total_notifications=total_notifications, ) @@ -275,10 +282,12 @@ def get_service_statistics_for_specific_days_by_user( service_id, start_date, end_date, user_id ) - stats = get_specific_days_stats( + hours = days * 24 + + stats = get_specific_hours_stats( results, start_date, - days=days, + hours=hours, total_notifications=total_notifications, ) return stats @@ -1145,20 +1154,24 @@ def modify_service_data_retention(service_id, data_retention_id): def get_service_message_ratio(): service_id = request.args.get("service_id") + current_year = datetime.now(tz=ZoneInfo("UTC")).year my_service = dao_fetch_service_by_id(service_id) + messages_sent = dao_get_notification_count_for_service_message_ratio( + service_id, current_year + ) + messages_remaining = my_service.total_message_limit - messages_sent - cache_key = total_limit_cache_key(service_id) - messages_sent = redis_store.get(cache_key) - if messages_sent is None: - messages_sent = 0 - current_app.logger.warning( - f"Messages sent was not being tracked for service {service_id}" + if my_service.total_message_limit - messages_sent < 0: + raise Exception( + f"Math error get_service_message_ratio(), \ + total {my_service.total_message_limit} \ + messages_sent {messages_sent} remaining {messages_remaining} \ + service_id {service_id} current_year {current_year}" ) - else: - messages_sent = int(messages_sent) return { "messages_sent": messages_sent, + "messages_remaining": messages_remaining, "total_message_limit": my_service.total_message_limit, }, 200 diff --git a/docs/adrs/0012-adr-report-generation.md b/docs/adrs/0012-adr-report-generation.md new file mode 100644 index 000000000..afed253fa --- /dev/null +++ b/docs/adrs/0012-adr-report-generation.md @@ -0,0 +1,33 @@ +# Optimize processing of delivery receipts + +Status: Accepted +Date: 12 February 2025 + +### Context + +In the original UK code, there is a notifications table in the database that stores all the recipient phone numbers and emails. Even though this data is purged after seven days, there could be hundreds of thousands or millions of rows of data with PII at any given time. We were concerned about this for security reasons as any kind of database breach would leak a lot of PII. + +### Decision + +We decided to move PII out of the database. It would continue to exist in the form of uploaded CSV files in S3 for seven days, but this would be substantially reducing the attack service. + +However, in doing so, it complicated the generation of reports, which contained the recipients phone numbers. There was some debate about whether the reports should contain the recipients' phone numbers at all, but there was considerable pushback from partners when we tried to remove them. + +Being informed that Redis was not an option to store this PII either, we decided to try to creating a short term caching solution that would hold onto all the phone numbers and allow report generation as before. This caching solution involved the use of a ThreadPoolExecutor to allow the download of multiple S3 files at the same time, as well as a multiprocessing.Manager that would allow us to create a dictionary that could be shared across all celery workers. + + +### Consequences + +Initially this approach seemed relatively promising as most of the time partners were able to generate their reports. However, these were early beta partners who typically sent low volumes of texts. In retrospect, the fact that there were recurrent occasional failures should have been a red flag. As the partner usage started to scale up, reports rapidly became unusable. + +This whole problem could be solved in a day if we replaced the multiprocessing.Manager with Redis, but that would essentially take us back to where we were where we started, with PII persisting in two locations. + +The direction we intend to go is to remove phone numbers from the reports. However, that work is still in progress as we are trying to find how to make that solution palatable to partners. + + +### Author +@kenkehl + +### Stakeholders +@ccostino +@stvnrlly diff --git a/docs/adrs/0013-log-debug-tags.md b/docs/adrs/0013-log-debug-tags.md new file mode 100644 index 000000000..83a88107b --- /dev/null +++ b/docs/adrs/0013-log-debug-tags.md @@ -0,0 +1,38 @@ +# Use of debug search tags for cloudwatch logs + +Status: Accepted +Date: 27 February 2025 + +### Context + +We encountered some recurrent problems in specific parts of the code that re-occurred regularly. + +We found it hard to debug these issues searching through cloudwatch logs as a particular issue +tends to be exploded into multiple log lines, and these lines are not even time sorted by default +so it can get quite hard on the eyes and test one's patience to debug this way. + +### Decision + +We decided to start using unique tags for individual issues so we could quickly search for related +groups of log lines. This worked fairly well and even though some of these issues at long last +have been resolved, it might be worthwhile to continue this debugging pattern. + +### Consequences + +Here are the existing tags and what they are used for: + +#notify-debug-admin-1200: job cache regeneration +#notify-debug-admin-1701: wrong sender phone number +#notify-debug-admin-1859: job creation time reset due to strange SqlAlchemy constructor issue setting wrong created_at time +#notify-debug-api-1385: a separate identify to resolve the wrong sender phone number issue +#notify-debug-s3-partitioning: modify s3 partitioning in line with best aws practices +#notify-debug-validate-phone-number: experimental code to try to validate phone numbers with aws pinpoint + + + + +### Author +@kenkehl + +### Stakeholders +@ccostino diff --git a/docs/reflection.md b/docs/reflection.md new file mode 100644 index 000000000..e48044d64 --- /dev/null +++ b/docs/reflection.md @@ -0,0 +1,255 @@ +## Reflections after sunset decision + +Q: How did we decide what to work on / get to notifications?  + +- We started with understanding of the market needs. + + - Idea of Public Benefits Studio spun out of 18F Public Benefits Portfolio. The team had a deep understanding of past challenges of implementing tech in government. + + - Team also had a lot of understanding of the unique constraints of working in the benefits space which is at the state and local level (vs. feds)  + +- We knew our users really well, including their constraints and goals. + + - Constraints of building tech for other people/jurisdictions: + + - Integration deters adoption + + - Procurement timelines / spending ANY money takes time (free is fast) + + - Program teams have needs they cannot articulate to their technology stakeholders  (we gave them an easy solution to sell) + + - Program teams expressed problems with communicating to the public and we found in-flight projects that met these needs. + +- Matched technology with potential impacts, illustrated how text messages can help applicants/beneficiaries and improve the relationship between gov't and the public + +Q: How did we define success, or how did we decide what success was? + +- Identified the constraints and barriers we hoped to reduce or overcome (time to launch, procurement/security hurdles, technical expertise, etc.), established what would indicate that we were having an impact, and then tracking it. + +- Partners over and over again shared quotes with the idea of "This is just what we were looking for!" + +- Wanted to build a product that was easy to get started with and had low-stakes for real-world experimentation. + +- "Good enough" metrics + + - We wanted an impact on recipients and programs, and we got this in some cases (although not as many as we hoped). We did a good job at tracking partners (increasing), usage (increasing), time from onboarding to sending (decreasing).  + +Q: How did we set ourselves for success? What factors contributed to it?  + +- We prioritized partnerships via a free pilot opportunity. We worked deeply with our first 4 pilot partners to understand their use cases and needs before scaling back "ad hoc" support + +- Leveraged existing resources + + - We leveraged American Rescue Plan money and the TTS BPA to get a full development team onboard quickly. + + - Leveraging Customer Experience/Life Exeriences and Office of Evaluation Sciences priorities and people + + - We had a full team (product, UX, content, front-end and back-end engineering) from the beginning + + - We did what we could given federal hiring constraints, but at various times that did pose serious challenges for the team + +- We narrowed in on common use cases in the benefits space and had good story telling for potential customers about HOW texting could have an impact on program ops.  + + - We stayed flexible and open to a wide range of use-cases, so we could follow demand

 + +- We got program teams excited enough to advocate this tech opportunity to other stakeholders + +- Able to springboard ATO because we just had to document the system vs build it from scratch + + - Fortunate that we had someone experienced who lent a hand with the original LATO + +- For a long time we had leadership alignment and legal alignment - this will be critical for anyone trying to do a similar thing in the future + + - Approved for beta: + + - TTS approved beta status  + + - We proved out demand + + - We proved that the product was stable and is being used for what it says it can do, reliably, + + - We achieved full ATO + +- Continually sought to templatize and simplify partnerships and onboarding processes with self-service guides, videos, etc. + +- We built an opt-in pipeline for potential customers, we had a clear intake process and screening questions which we tied back to other metrics we wanted to track + +Q: As we sunset, What signs of success are we still seeing? + +- Even as we were discussing sunset, we were getting partner intake forms submitted + +- Partners are seeking to continue using the service, in any way they can + +- Number of partners signed, and number of partners sending messages, were increasing + +- Federal partners wanted to pay + +- The public knows about our product, we come up in google searches for "text messages for government" + +- Programs we shared best practices and learnings with continue to say they've appreciated and use these + +Q: Where have we left off? + +- Partners were scaling use within their agencies: + + - Had a few partners moving from 1-2 texting campaigns / teams using it to adding more teams and use cases + +- We had a pipeline of at least 4 more partners ready to sign an MOU before we had to stop + + - 7 potential partners reached out ready to use the platform before we had to stop + +- Two programs saying "take my money"  + +- A finished, but not approved, pricing model. The thorny part remaining was figuring out what costs could be reimbursable vs appropriated and how that could change in future years. + +- Marketing and comms ready to promote updated service model (paid/free) + +- We have best practices related to texting to share but are not public yet on the site (these are useful beyond notify) + +- If we could have kept positions stable (terms) / filled, we think had enough staff to scale significantly + +Q: What advice do we have for future us? Were there any missed opportunities? + +- Many programs are texting about the same things (application deadlines, reminders etc). We had hopes to prepopulate tested-plain language templates on to Notify to help standardize comms across agencies + + - Also hoped to have templates translated and vetted by non-English speakers in common languages + +- TCPA-related liability concerns mean commercial providers require government agencies to go beyond what is legally required (eg. get explicit consent), which means that for some use cases they can't text at all. Clarified or more forceful guidance from the FCC or a new process for gov agencies could alleviate this. See below.  + +- Agencies have continued to have trust/ spam issues and we had hoped to develop a trusted gov't short-code (similar to 411). Only an internal fed team has leverage like this at scale. + +- Features to support existing users + + - Release a public API (this was next up for a) the large use cases and b) to allow us to remove some problematic reporting functionality in the UI) + + - Better scheduling features, set it and forget  + + - Some way to save or reference an existing list of recipients + +- Helping new texting programs get off the ground faster in a more informed way - + + - Learning collaborative - users can learn directly from each other - it's not just Notify but sending good text messages and learning from other agencies that have real evidence and experience sending messages + + - Enabling resource sharing + +Q: If a single agency wanted to start a texting program, what would we recommend they consider for metrics of success? + +- Is this delivering on your mission?  + +- What is the change you are hoping to see?  + + - Better customer service can be enough + + - Serve more people, save money/churn, etc + +- Are sending text messages saving staff time? + + - Fewer missed deadlines, less follow-ups needed etc.  + +Q: Did forking a thing increase leadership's willingness to do it?  + +- It helped build confidence knowing that other countries had successfully implemented it. We were able to pull metrics and reference elements such as costs and timelines. The idea of not reinventing the wheel was crucial. The U.S. is behind in this regard, but the process itself isn't technically that complicated. + +- The completeness of the codebase was not critical.  + +- Notify has a core function which is easier to understand and sell.  + +Q: About products centrally provided by government + +- GSA has incentives and requirements, such as privacy protection standards, that the private sector may not have.  + +- Should one government entity rely on another as a provider? It matters where data resides, and the risks associated with this have evolved over time. + +- Our opinion has shifted over time: we think there is a much narrower list of shared services the federal government should offer than we did when we started.  + +- There is a power dynamic when the federal government does anything, this cannot be replicated outside the gov, and this can have both positive and negative effects. + +- On a basic level, there is value in shared services, but only if there's someone responsible for running them. + +- It is important to note that some states do not - and have never - trusted the federal government on matters of data privacy. + +- Financial expectations from the private sector don't fit well in this environment.  + + - Appropriated funding and timelines align very poorly with scaling/ volume based services  + + - No guarantees on runway + + - "Fully reimbursable" to the exact dollar is not a thing in private sector; in particular it was difficult to establish techniques to charge partners and cover shared service costs without risking a surplus (even the possibility was not allowed).  + +- The need for products is, in part, because of auxiliary challenges (procurement, private-sector cooperation, etc).  + + - Should we wait till these problems are solved before doing anything? Not necessarily. But that doesn't mean the federal government is the best place to do it. + +- The federal government is well positioned establish common standards that people could use (ex. USWDS). Doesn't have to be a shared service itself. + +Q: Did forking Notify from the UK make a difference in building leadership support? + +- It made a significant difference that other countries had already implemented it, allowing us to pull useful metrics. The team structure, customer focus, and story were especially important. + +- We could have forked the libraries and told the same story, but the story itself was more important than the code base. + +- The case was clear: the government should communicate by text.  + +- Notify is conceptually easy to understand, both what it is and why it would be important.  + +Q: Would we fork wholesale again?  + +- Yes, but maybe not all of it. + +- The size and scope of the codebase made the founding team feel that a relatively large team was necessary up front.  + +- At the time of sunset, the team was only starting to truly deal with conceptual mismatches and architectural issues. Or rather it had been working on them for a while, but signs were they still had a long way yet to go. + +- The original codebase wasn't designed to be forked.  + + - Architectural decisions made components tightly coupled and therefore harder to change for our specific instance + + - We're not sure + +Q: Do we tell people not to fork ours?  + +- There are layers of our changes on top of a library of old debt.  + +- We've stripped out several pieces to focus on texting.  + +- Lots of "whack-a-mole" in the current deployment in the code.  + +- We didn't have high code quality in the received product and our team didn't fix all of that.  + +- We had not validated the API for public consumption (but believe it would have been OK).  + +Q: What were some strong aspects of Notify.gov? + +- Research informed decisions made the project stronger.  + +- Our best practices worked well and were appreciated.  + +- Template management and user management were fine but incomplete.  + +- We solved some thorny architectural things re: scale and time zone.   + +- We would recommend thinking carefully before using it out of the box + + - Note: Notify is written to use Cloud.gov, which not everyone has access to).   + + - Not a blocker, but it would take some work to stand up. This could be replaced by going with a different cloud provider.   + + - Be aware that the original application was not that well-architected for our use case, and we hadn't gotten all the way to aligning it with our use cases and personas. You're getting a bit of a chimera.  + +Q: What are some lessons learned? + +- There is a need to have leadership alignment/advocates  + + - Ex. managing budget cycles  + + - We had ARP money and had stakeholders that understood the time it takes to get to financial sustainability + +- The products goals were misaligned with a risk-averse outreach approach. It was always a challenge to even tell people we exist, given the inherent possibility we might learn something that would make us choose to shut down the program. This made ithard to build awareness and trust.  + +- Solutions need to be adaptable to the agency that is actually deploying them. + + - Ex. Notify started with no integrations  + + - It was designed to be part of a larger workflow, to drop into a variety of situations.  + + - It was intentionally kept small. There was no attempt for it to have the functionality of a CRM/management of workflows or store data. diff --git a/migrations/versions/0415_add_message_cost.py b/migrations/versions/0415_add_message_cost.py new file mode 100644 index 000000000..1a86170d7 --- /dev/null +++ b/migrations/versions/0415_add_message_cost.py @@ -0,0 +1,23 @@ +""" + +Revision ID: 0415_add_message_cost +Revises: 0414_change_total_message_limit +Create Date: 2025-02-28 11:35:22.873930 + +""" + +import sqlalchemy as sa +from alembic import op + +down_revision = "0414_change_total_message_limit" +revision = "0415_add_message_cost" + + +def upgrade(): + op.add_column("notifications", sa.Column("message_cost", sa.Float)) + op.add_column("notification_history", sa.Column("message_cost", sa.Float)) + + +def downgrade(): + op.drop_column("notifications", "message_cost") + op.add_column("notification_history", sa.Column("message_cost", sa.Float)) diff --git a/migrations/versions/0416_readd_e2e_test_user.py b/migrations/versions/0416_readd_e2e_test_user.py new file mode 100644 index 000000000..fdb0af9c9 --- /dev/null +++ b/migrations/versions/0416_readd_e2e_test_user.py @@ -0,0 +1,66 @@ +""" + +Revision ID: 0416_readd_e2e_test_user +Revises: 0415_add_message_cost +Create Date: 2025-03-17 11:35:22.873930 + +""" + +import datetime +import os +import uuid + +import sqlalchemy as sa +from alembic import op + +from app import db +from app.dao.users_dao import get_user_by_email +from app.enums import AuthType +from app.models import User +from app.utils import utc_now + +revision = "0416_readd_e2e_test_user" +down_revision = "0415_add_message_cost" + + +def upgrade(): + email_address = os.getenv("NOTIFY_E2E_TEST_EMAIL") + password = os.getenv("NOTIFY_E2E_TEST_PASSWORD") + name = f"e2e_test_user_{uuid.uuid4()}" + data = { + "id": uuid.uuid4(), + "name": name, + "email_address": email_address, + "password": password, + "mobile_number": "+12025555555", + "state": "active", + "created_at": utc_now(), + "password_changed_at": utc_now(), + "failed_login_count": 0, + "platform_admin": "f", + "email_access_validated_at": utc_now(), + "auth_type": AuthType.SMS, + } + conn = op.get_bind() + + # delete the old user because + delete_sql = """ + delete from users where email_address='e2e-test-notify-user@fake.gov' + """ + + insert_sql = """ + insert into users (id, name, email_address, _password, mobile_number, state, created_at, password_changed_at, failed_login_count, platform_admin, email_access_validated_at, auth_type) + values (:id, :name, :email_address, :password, :mobile_number, :state, :created_at, :password_changed_at, :failed_login_count, :platform_admin, :email_access_validated_at, :auth_type) + """ + conn.execute(sa.text(delete_sql)) + + conn.execute(sa.text(insert_sql), data) + + +def downgrade(): + email_address = os.getenv("NOTIFY_E2E_TEST_EMAIL") + user_to_delete = get_user_by_email(email_address) + if not user_to_delete: + return + db.session.remove(user_to_delete) + db.session.commit() diff --git a/notifications_utils/timezones.py b/notifications_utils/timezones.py index 277a139f0..78976010b 100644 --- a/notifications_utils/timezones.py +++ b/notifications_utils/timezones.py @@ -1,9 +1,9 @@ import os +from zoneinfo import ZoneInfo -import pytz from dateutil import parser -local_timezone = pytz.timezone(os.getenv("TIMEZONE", "America/New_York")) +local_timezone = os.getenv("TIMEZONE", ZoneInfo("America/New_York")) def utc_string_to_aware_gmt_datetime(date): @@ -12,5 +12,5 @@ def utc_string_to_aware_gmt_datetime(date): Returns an aware local datetime, essentially the time you'd see on your clock """ date = parser.parse(date) - forced_utc = date.replace(tzinfo=pytz.utc) + forced_utc = date.replace(tzinfo=ZoneInfo("UTC")) return forced_utc.astimezone(local_timezone) diff --git a/poetry.lock b/poetry.lock index 152ea9e49..4dfe12786 100644 --- a/poetry.lock +++ b/poetry.lock @@ -157,13 +157,13 @@ tz = ["backports.zoneinfo"] [[package]] name = "amqp" -version = "5.2.0" +version = "5.3.1" description = "Low-level AMQP client for Python (fork of amqplib)." optional = false python-versions = ">=3.6" files = [ - {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"}, - {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"}, + {file = "amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2"}, + {file = "amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432"}, ] [package.dependencies] @@ -579,13 +579,13 @@ zstd = ["zstandard (==0.22.0)"] [[package]] name = "certifi" -version = "2024.8.30" +version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] [[package]] @@ -1735,13 +1735,13 @@ test = ["objgraph", "psutil"] [[package]] name = "gunicorn" -version = "22.0.0" +version = "23.0.0" description = "WSGI HTTP Server for UNIX" optional = false python-versions = ">=3.7" files = [ - {file = "gunicorn-22.0.0-py3-none-any.whl", hash = "sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9"}, - {file = "gunicorn-22.0.0.tar.gz", hash = "sha256:4a0b436239ff76fb33f11c07a16482c521a7e09c1ce3cc293c2330afe01bec63"}, + {file = "gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d"}, + {file = "gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec"}, ] [package.dependencies] @@ -1929,13 +1929,13 @@ trio = ["async_generator", "trio"] [[package]] name = "jinja2" -version = "3.1.5" +version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, - {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, ] [package.dependencies] @@ -2105,161 +2105,157 @@ testing = ["black", "isort", "pytest (>=6,!=7.0.0)", "pytest-xdist (>=2)", "twin [[package]] name = "lxml" -version = "5.2.2" +version = "5.3.1" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = false python-versions = ">=3.6" files = [ - {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:364d03207f3e603922d0d3932ef363d55bbf48e3647395765f9bfcbdf6d23632"}, - {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50127c186f191b8917ea2fb8b206fbebe87fd414a6084d15568c27d0a21d60db"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74e4f025ef3db1c6da4460dd27c118d8cd136d0391da4e387a15e48e5c975147"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981a06a3076997adf7c743dcd0d7a0415582661e2517c7d961493572e909aa1d"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aef5474d913d3b05e613906ba4090433c515e13ea49c837aca18bde190853dff"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e275ea572389e41e8b039ac076a46cb87ee6b8542df3fff26f5baab43713bca"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5b65529bb2f21ac7861a0e94fdbf5dc0daab41497d18223b46ee8515e5ad297"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bcc98f911f10278d1daf14b87d65325851a1d29153caaf146877ec37031d5f36"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:b47633251727c8fe279f34025844b3b3a3e40cd1b198356d003aa146258d13a2"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:fbc9d316552f9ef7bba39f4edfad4a734d3d6f93341232a9dddadec4f15d425f"}, - {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:13e69be35391ce72712184f69000cda04fc89689429179bc4c0ae5f0b7a8c21b"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3b6a30a9ab040b3f545b697cb3adbf3696c05a3a68aad172e3fd7ca73ab3c835"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a233bb68625a85126ac9f1fc66d24337d6e8a0f9207b688eec2e7c880f012ec0"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:dfa7c241073d8f2b8e8dbc7803c434f57dbb83ae2a3d7892dd068d99e96efe2c"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a7aca7964ac4bb07680d5c9d63b9d7028cace3e2d43175cb50bba8c5ad33316"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae4073a60ab98529ab8a72ebf429f2a8cc612619a8c04e08bed27450d52103c0"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ffb2be176fed4457e445fe540617f0252a72a8bc56208fd65a690fdb1f57660b"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e290d79a4107d7d794634ce3e985b9ae4f920380a813717adf61804904dc4393"}, - {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96e85aa09274955bb6bd483eaf5b12abadade01010478154b0ec70284c1b1526"}, - {file = "lxml-5.2.2-cp310-cp310-win32.whl", hash = "sha256:f956196ef61369f1685d14dad80611488d8dc1ef00be57c0c5a03064005b0f30"}, - {file = "lxml-5.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:875a3f90d7eb5c5d77e529080d95140eacb3c6d13ad5b616ee8095447b1d22e7"}, - {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:45f9494613160d0405682f9eee781c7e6d1bf45f819654eb249f8f46a2c22545"}, - {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0b3f2df149efb242cee2ffdeb6674b7f30d23c9a7af26595099afaf46ef4e88"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d28cb356f119a437cc58a13f8135ab8a4c8ece18159eb9194b0d269ec4e28083"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:657a972f46bbefdbba2d4f14413c0d079f9ae243bd68193cb5061b9732fa54c1"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b9ea10063efb77a965a8d5f4182806fbf59ed068b3c3fd6f30d2ac7bee734"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07542787f86112d46d07d4f3c4e7c760282011b354d012dc4141cc12a68cef5f"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:303f540ad2dddd35b92415b74b900c749ec2010e703ab3bfd6660979d01fd4ed"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2eb2227ce1ff998faf0cd7fe85bbf086aa41dfc5af3b1d80867ecfe75fb68df3"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:1d8a701774dfc42a2f0b8ccdfe7dbc140500d1049e0632a611985d943fcf12df"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:56793b7a1a091a7c286b5f4aa1fe4ae5d1446fe742d00cdf2ffb1077865db10d"}, - {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eb00b549b13bd6d884c863554566095bf6fa9c3cecb2e7b399c4bc7904cb33b5"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a2569a1f15ae6c8c64108a2cd2b4a858fc1e13d25846be0666fc144715e32ab"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:8cf85a6e40ff1f37fe0f25719aadf443686b1ac7652593dc53c7ef9b8492b115"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d237ba6664b8e60fd90b8549a149a74fcc675272e0e95539a00522e4ca688b04"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b3f5016e00ae7630a4b83d0868fca1e3d494c78a75b1c7252606a3a1c5fc2ad"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23441e2b5339bc54dc949e9e675fa35efe858108404ef9aa92f0456929ef6fe8"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2fb0ba3e8566548d6c8e7dd82a8229ff47bd8fb8c2da237607ac8e5a1b8312e5"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:79d1fb9252e7e2cfe4de6e9a6610c7cbb99b9708e2c3e29057f487de5a9eaefa"}, - {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6dcc3d17eac1df7859ae01202e9bb11ffa8c98949dcbeb1069c8b9a75917e01b"}, - {file = "lxml-5.2.2-cp311-cp311-win32.whl", hash = "sha256:4c30a2f83677876465f44c018830f608fa3c6a8a466eb223535035fbc16f3438"}, - {file = "lxml-5.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:49095a38eb333aaf44c06052fd2ec3b8f23e19747ca7ec6f6c954ffea6dbf7be"}, - {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7429e7faa1a60cad26ae4227f4dd0459efde239e494c7312624ce228e04f6391"}, - {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:50ccb5d355961c0f12f6cf24b7187dbabd5433f29e15147a67995474f27d1776"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc911208b18842a3a57266d8e51fc3cfaccee90a5351b92079beed912a7914c2"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ce9e786753743159799fdf8e92a5da351158c4bfb6f2db0bf31e7892a1feb5"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec87c44f619380878bd49ca109669c9f221d9ae6883a5bcb3616785fa8f94c97"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08ea0f606808354eb8f2dfaac095963cb25d9d28e27edcc375d7b30ab01abbf6"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75a9632f1d4f698b2e6e2e1ada40e71f369b15d69baddb8968dcc8e683839b18"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:74da9f97daec6928567b48c90ea2c82a106b2d500f397eeb8941e47d30b1ca85"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:0969e92af09c5687d769731e3f39ed62427cc72176cebb54b7a9d52cc4fa3b73"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:9164361769b6ca7769079f4d426a41df6164879f7f3568be9086e15baca61466"}, - {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d26a618ae1766279f2660aca0081b2220aca6bd1aa06b2cf73f07383faf48927"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab67ed772c584b7ef2379797bf14b82df9aa5f7438c5b9a09624dd834c1c1aaf"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3d1e35572a56941b32c239774d7e9ad724074d37f90c7a7d499ab98761bd80cf"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:8268cbcd48c5375f46e000adb1390572c98879eb4f77910c6053d25cc3ac2c67"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e282aedd63c639c07c3857097fc0e236f984ceb4089a8b284da1c526491e3f3d"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfdc2bfe69e9adf0df4915949c22a25b39d175d599bf98e7ddf620a13678585"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4aefd911793b5d2d7a921233a54c90329bf3d4a6817dc465f12ffdfe4fc7b8fe"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8b8df03a9e995b6211dafa63b32f9d405881518ff1ddd775db4e7b98fb545e1c"}, - {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f11ae142f3a322d44513de1018b50f474f8f736bc3cd91d969f464b5bfef8836"}, - {file = "lxml-5.2.2-cp312-cp312-win32.whl", hash = "sha256:16a8326e51fcdffc886294c1e70b11ddccec836516a343f9ed0f82aac043c24a"}, - {file = "lxml-5.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bbc4b80af581e18568ff07f6395c02114d05f4865c2812a1f02f2eaecf0bfd48"}, - {file = "lxml-5.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e3d9d13603410b72787579769469af730c38f2f25505573a5888a94b62b920f8"}, - {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38b67afb0a06b8575948641c1d6d68e41b83a3abeae2ca9eed2ac59892b36706"}, - {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c689d0d5381f56de7bd6966a4541bff6e08bf8d3871bbd89a0c6ab18aa699573"}, - {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:cf2a978c795b54c539f47964ec05e35c05bd045db5ca1e8366988c7f2fe6b3ce"}, - {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:739e36ef7412b2bd940f75b278749106e6d025e40027c0b94a17ef7968d55d56"}, - {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d8bbcd21769594dbba9c37d3c819e2d5847656ca99c747ddb31ac1701d0c0ed9"}, - {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:2304d3c93f2258ccf2cf7a6ba8c761d76ef84948d87bf9664e14d203da2cd264"}, - {file = "lxml-5.2.2-cp36-cp36m-win32.whl", hash = "sha256:02437fb7308386867c8b7b0e5bc4cd4b04548b1c5d089ffb8e7b31009b961dc3"}, - {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"}, - {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"}, - {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"}, - {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"}, - {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"}, - {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"}, - {file = "lxml-5.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ed07b3062b055d7a7f9d6557a251cc655eed0b3152b76de619516621c56f5d3"}, - {file = "lxml-5.2.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60fdd125d85bf9c279ffb8e94c78c51b3b6a37711464e1f5f31078b45002421"}, - {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a7e24cb69ee5f32e003f50e016d5fde438010c1022c96738b04fc2423e61706"}, - {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23cfafd56887eaed93d07bc4547abd5e09d837a002b791e9767765492a75883f"}, - {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19b4e485cd07b7d83e3fe3b72132e7df70bfac22b14fe4bf7a23822c3a35bff5"}, - {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7ce7ad8abebe737ad6143d9d3bf94b88b93365ea30a5b81f6877ec9c0dee0a48"}, - {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e49b052b768bb74f58c7dda4e0bdf7b79d43a9204ca584ffe1fb48a6f3c84c66"}, - {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d14a0d029a4e176795cef99c056d58067c06195e0c7e2dbb293bf95c08f772a3"}, - {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:be49ad33819d7dcc28a309b86d4ed98e1a65f3075c6acd3cd4fe32103235222b"}, - {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a6d17e0370d2516d5bb9062c7b4cb731cff921fc875644c3d751ad857ba9c5b1"}, - {file = "lxml-5.2.2-cp38-cp38-win32.whl", hash = "sha256:5b8c041b6265e08eac8a724b74b655404070b636a8dd6d7a13c3adc07882ef30"}, - {file = "lxml-5.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:f61efaf4bed1cc0860e567d2ecb2363974d414f7f1f124b1df368bbf183453a6"}, - {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fb91819461b1b56d06fa4bcf86617fac795f6a99d12239fb0c68dbeba41a0a30"}, - {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d4ed0c7cbecde7194cd3228c044e86bf73e30a23505af852857c09c24e77ec5d"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54401c77a63cc7d6dc4b4e173bb484f28a5607f3df71484709fe037c92d4f0ed"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:625e3ef310e7fa3a761d48ca7ea1f9d8718a32b1542e727d584d82f4453d5eeb"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:519895c99c815a1a24a926d5b60627ce5ea48e9f639a5cd328bda0515ea0f10c"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c7079d5eb1c1315a858bbf180000757db8ad904a89476653232db835c3114001"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:343ab62e9ca78094f2306aefed67dcfad61c4683f87eee48ff2fd74902447726"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:cd9e78285da6c9ba2d5c769628f43ef66d96ac3085e59b10ad4f3707980710d3"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:546cf886f6242dff9ec206331209db9c8e1643ae642dea5fdbecae2453cb50fd"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:02f6a8eb6512fdc2fd4ca10a49c341c4e109aa6e9448cc4859af5b949622715a"}, - {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:339ee4a4704bc724757cd5dd9dc8cf4d00980f5d3e6e06d5847c1b594ace68ab"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0a028b61a2e357ace98b1615fc03f76eb517cc028993964fe08ad514b1e8892d"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f90e552ecbad426eab352e7b2933091f2be77115bb16f09f78404861c8322981"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d83e2d94b69bf31ead2fa45f0acdef0757fa0458a129734f59f67f3d2eb7ef32"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a02d3c48f9bb1e10c7788d92c0c7db6f2002d024ab6e74d6f45ae33e3d0288a3"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6d68ce8e7b2075390e8ac1e1d3a99e8b6372c694bbe612632606d1d546794207"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:453d037e09a5176d92ec0fd282e934ed26d806331a8b70ab431a81e2fbabf56d"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:3b019d4ee84b683342af793b56bb35034bd749e4cbdd3d33f7d1107790f8c472"}, - {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb3942960f0beb9f46e2a71a3aca220d1ca32feb5a398656be934320804c0df9"}, - {file = "lxml-5.2.2-cp39-cp39-win32.whl", hash = "sha256:ac6540c9fff6e3813d29d0403ee7a81897f1d8ecc09a8ff84d2eea70ede1cdbf"}, - {file = "lxml-5.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:610b5c77428a50269f38a534057444c249976433f40f53e3b47e68349cca1425"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b537bd04d7ccd7c6350cdaaaad911f6312cbd61e6e6045542f781c7f8b2e99d2"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4820c02195d6dfb7b8508ff276752f6b2ff8b64ae5d13ebe02e7667e035000b9"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a09f6184f17a80897172863a655467da2b11151ec98ba8d7af89f17bf63dae"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76acba4c66c47d27c8365e7c10b3d8016a7da83d3191d053a58382311a8bf4e1"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b128092c927eaf485928cec0c28f6b8bead277e28acf56800e972aa2c2abd7a2"}, - {file = "lxml-5.2.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ae791f6bd43305aade8c0e22f816b34f3b72b6c820477aab4d18473a37e8090b"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a2f6a1bc2460e643785a2cde17293bd7a8f990884b822f7bca47bee0a82fc66b"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e8d351ff44c1638cb6e980623d517abd9f580d2e53bfcd18d8941c052a5a009"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bec4bd9133420c5c52d562469c754f27c5c9e36ee06abc169612c959bd7dbb07"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:55ce6b6d803890bd3cc89975fca9de1dff39729b43b73cb15ddd933b8bc20484"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ab6a358d1286498d80fe67bd3d69fcbc7d1359b45b41e74c4a26964ca99c3f8"}, - {file = "lxml-5.2.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:06668e39e1f3c065349c51ac27ae430719d7806c026fec462e5693b08b95696b"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9cd5323344d8ebb9fb5e96da5de5ad4ebab993bbf51674259dbe9d7a18049525"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89feb82ca055af0fe797a2323ec9043b26bc371365847dbe83c7fd2e2f181c34"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e481bba1e11ba585fb06db666bfc23dbe181dbafc7b25776156120bf12e0d5a6"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d6c6ea6a11ca0ff9cd0390b885984ed31157c168565702959c25e2191674a14"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3d98de734abee23e61f6b8c2e08a88453ada7d6486dc7cdc82922a03968928db"}, - {file = "lxml-5.2.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:69ab77a1373f1e7563e0fb5a29a8440367dec051da6c7405333699d07444f511"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:34e17913c431f5ae01d8658dbf792fdc457073dcdfbb31dc0cc6ab256e664a8d"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f8757b03208c3f50097761be2dea0aba02e94f0dc7023ed73a7bb14ff11eb0"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a520b4f9974b0a0a6ed73c2154de57cdfd0c8800f4f15ab2b73238ffed0b36e"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5e097646944b66207023bc3c634827de858aebc226d5d4d6d16f0b77566ea182"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b5e4ef22ff25bfd4ede5f8fb30f7b24446345f3e79d9b7455aef2836437bc38a"}, - {file = "lxml-5.2.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff69a9a0b4b17d78170c73abe2ab12084bdf1691550c5629ad1fe7849433f324"}, - {file = "lxml-5.2.2.tar.gz", hash = "sha256:bb2dc4898180bea79863d5487e5f9c7c34297414bad54bcd0f0852aee9cfdb87"}, + {file = "lxml-5.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a4058f16cee694577f7e4dd410263cd0ef75644b43802a689c2b3c2a7e69453b"}, + {file = "lxml-5.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:364de8f57d6eda0c16dcfb999af902da31396949efa0e583e12675d09709881b"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:528f3a0498a8edc69af0559bdcf8a9f5a8bf7c00051a6ef3141fdcf27017bbf5"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db4743e30d6f5f92b6d2b7c86b3ad250e0bad8dee4b7ad8a0c44bfb276af89a3"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:17b5d7f8acf809465086d498d62a981fa6a56d2718135bb0e4aa48c502055f5c"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:928e75a7200a4c09e6efc7482a1337919cc61fe1ba289f297827a5b76d8969c2"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a997b784a639e05b9d4053ef3b20c7e447ea80814a762f25b8ed5a89d261eac"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7b82e67c5feb682dbb559c3e6b78355f234943053af61606af126df2183b9ef9"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:f1de541a9893cf8a1b1db9bf0bf670a2decab42e3e82233d36a74eda7822b4c9"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:de1fc314c3ad6bc2f6bd5b5a5b9357b8c6896333d27fdbb7049aea8bd5af2d79"}, + {file = "lxml-5.3.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:7c0536bd9178f754b277a3e53f90f9c9454a3bd108b1531ffff720e082d824f2"}, + {file = "lxml-5.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:68018c4c67d7e89951a91fbd371e2e34cd8cfc71f0bb43b5332db38497025d51"}, + {file = "lxml-5.3.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aa826340a609d0c954ba52fd831f0fba2a4165659ab0ee1a15e4aac21f302406"}, + {file = "lxml-5.3.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:796520afa499732191e39fc95b56a3b07f95256f2d22b1c26e217fb69a9db5b5"}, + {file = "lxml-5.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3effe081b3135237da6e4c4530ff2a868d3f80be0bda027e118a5971285d42d0"}, + {file = "lxml-5.3.1-cp310-cp310-win32.whl", hash = "sha256:a22f66270bd6d0804b02cd49dae2b33d4341015545d17f8426f2c4e22f557a23"}, + {file = "lxml-5.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:0bcfadea3cdc68e678d2b20cb16a16716887dd00a881e16f7d806c2138b8ff0c"}, + {file = "lxml-5.3.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e220f7b3e8656ab063d2eb0cd536fafef396829cafe04cb314e734f87649058f"}, + {file = "lxml-5.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0f2cfae0688fd01f7056a17367e3b84f37c545fb447d7282cf2c242b16262607"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67d2f8ad9dcc3a9e826bdc7802ed541a44e124c29b7d95a679eeb58c1c14ade8"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db0c742aad702fd5d0c6611a73f9602f20aec2007c102630c06d7633d9c8f09a"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:198bb4b4dd888e8390afa4f170d4fa28467a7eaf857f1952589f16cfbb67af27"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2a3e412ce1849be34b45922bfef03df32d1410a06d1cdeb793a343c2f1fd666"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b8969dbc8d09d9cd2ae06362c3bad27d03f433252601ef658a49bd9f2b22d79"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5be8f5e4044146a69c96077c7e08f0709c13a314aa5315981185c1f00235fe65"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:133f3493253a00db2c870d3740bc458ebb7d937bd0a6a4f9328373e0db305709"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:52d82b0d436edd6a1d22d94a344b9a58abd6c68c357ed44f22d4ba8179b37629"}, + {file = "lxml-5.3.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1b6f92e35e2658a5ed51c6634ceb5ddae32053182851d8cad2a5bc102a359b33"}, + {file = "lxml-5.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:203b1d3eaebd34277be06a3eb880050f18a4e4d60861efba4fb946e31071a295"}, + {file = "lxml-5.3.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:155e1a5693cf4b55af652f5c0f78ef36596c7f680ff3ec6eb4d7d85367259b2c"}, + {file = "lxml-5.3.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22ec2b3c191f43ed21f9545e9df94c37c6b49a5af0a874008ddc9132d49a2d9c"}, + {file = "lxml-5.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7eda194dd46e40ec745bf76795a7cccb02a6a41f445ad49d3cf66518b0bd9cff"}, + {file = "lxml-5.3.1-cp311-cp311-win32.whl", hash = "sha256:fb7c61d4be18e930f75948705e9718618862e6fc2ed0d7159b2262be73f167a2"}, + {file = "lxml-5.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:c809eef167bf4a57af4b03007004896f5c60bd38dc3852fcd97a26eae3d4c9e6"}, + {file = "lxml-5.3.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e69add9b6b7b08c60d7ff0152c7c9a6c45b4a71a919be5abde6f98f1ea16421c"}, + {file = "lxml-5.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4e52e1b148867b01c05e21837586ee307a01e793b94072d7c7b91d2c2da02ffe"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4b382e0e636ed54cd278791d93fe2c4f370772743f02bcbe431a160089025c9"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2e49dc23a10a1296b04ca9db200c44d3eb32c8d8ec532e8c1fd24792276522a"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4399b4226c4785575fb20998dc571bc48125dc92c367ce2602d0d70e0c455eb0"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5412500e0dc5481b1ee9cf6b38bb3b473f6e411eb62b83dc9b62699c3b7b79f7"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c93ed3c998ea8472be98fb55aed65b5198740bfceaec07b2eba551e55b7b9ae"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:63d57fc94eb0bbb4735e45517afc21ef262991d8758a8f2f05dd6e4174944519"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:b450d7cabcd49aa7ab46a3c6aa3ac7e1593600a1a0605ba536ec0f1b99a04322"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:4df0ec814b50275ad6a99bc82a38b59f90e10e47714ac9871e1b223895825468"}, + {file = "lxml-5.3.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d184f85ad2bb1f261eac55cddfcf62a70dee89982c978e92b9a74a1bfef2e367"}, + {file = "lxml-5.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b725e70d15906d24615201e650d5b0388b08a5187a55f119f25874d0103f90dd"}, + {file = "lxml-5.3.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a31fa7536ec1fb7155a0cd3a4e3d956c835ad0a43e3610ca32384d01f079ea1c"}, + {file = "lxml-5.3.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3c3c8b55c7fc7b7e8877b9366568cc73d68b82da7fe33d8b98527b73857a225f"}, + {file = "lxml-5.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d61ec60945d694df806a9aec88e8f29a27293c6e424f8ff91c80416e3c617645"}, + {file = "lxml-5.3.1-cp312-cp312-win32.whl", hash = "sha256:f4eac0584cdc3285ef2e74eee1513a6001681fd9753b259e8159421ed28a72e5"}, + {file = "lxml-5.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:29bfc8d3d88e56ea0a27e7c4897b642706840247f59f4377d81be8f32aa0cfbf"}, + {file = "lxml-5.3.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c093c7088b40d8266f57ed71d93112bd64c6724d31f0794c1e52cc4857c28e0e"}, + {file = "lxml-5.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b0884e3f22d87c30694e625b1e62e6f30d39782c806287450d9dc2fdf07692fd"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1637fa31ec682cd5760092adfabe86d9b718a75d43e65e211d5931809bc111e7"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a364e8e944d92dcbf33b6b494d4e0fb3499dcc3bd9485beb701aa4b4201fa414"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:779e851fd0e19795ccc8a9bb4d705d6baa0ef475329fe44a13cf1e962f18ff1e"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c4393600915c308e546dc7003d74371744234e8444a28622d76fe19b98fa59d1"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:673b9d8e780f455091200bba8534d5f4f465944cbdd61f31dc832d70e29064a5"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2e4a570f6a99e96c457f7bec5ad459c9c420ee80b99eb04cbfcfe3fc18ec6423"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:71f31eda4e370f46af42fc9f264fafa1b09f46ba07bdbee98f25689a04b81c20"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:42978a68d3825eaac55399eb37a4d52012a205c0c6262199b8b44fcc6fd686e8"}, + {file = "lxml-5.3.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8b1942b3e4ed9ed551ed3083a2e6e0772de1e5e3aca872d955e2e86385fb7ff9"}, + {file = "lxml-5.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:85c4f11be9cf08917ac2a5a8b6e1ef63b2f8e3799cec194417e76826e5f1de9c"}, + {file = "lxml-5.3.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:231cf4d140b22a923b1d0a0a4e0b4f972e5893efcdec188934cc65888fd0227b"}, + {file = "lxml-5.3.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5865b270b420eda7b68928d70bb517ccbe045e53b1a428129bb44372bf3d7dd5"}, + {file = "lxml-5.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7bebc2275016cddf3c997bf8a0f7044160714c64a9b83975670a04e6d2252"}, + {file = "lxml-5.3.1-cp313-cp313-win32.whl", hash = "sha256:d0751528b97d2b19a388b302be2a0ee05817097bab46ff0ed76feeec24951f78"}, + {file = "lxml-5.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:91fb6a43d72b4f8863d21f347a9163eecbf36e76e2f51068d59cd004c506f332"}, + {file = "lxml-5.3.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:016b96c58e9a4528219bb563acf1aaaa8bc5452e7651004894a973f03b84ba81"}, + {file = "lxml-5.3.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82a4bb10b0beef1434fb23a09f001ab5ca87895596b4581fd53f1e5145a8934a"}, + {file = "lxml-5.3.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d68eeef7b4d08a25e51897dac29bcb62aba830e9ac6c4e3297ee7c6a0cf6439"}, + {file = "lxml-5.3.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:f12582b8d3b4c6be1d298c49cb7ae64a3a73efaf4c2ab4e37db182e3545815ac"}, + {file = "lxml-5.3.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2df7ed5edeb6bd5590914cd61df76eb6cce9d590ed04ec7c183cf5509f73530d"}, + {file = "lxml-5.3.1-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:585c4dc429deebc4307187d2b71ebe914843185ae16a4d582ee030e6cfbb4d8a"}, + {file = "lxml-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:06a20d607a86fccab2fc15a77aa445f2bdef7b49ec0520a842c5c5afd8381576"}, + {file = "lxml-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:057e30d0012439bc54ca427a83d458752ccda725c1c161cc283db07bcad43cf9"}, + {file = "lxml-5.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4867361c049761a56bd21de507cab2c2a608c55102311d142ade7dab67b34f32"}, + {file = "lxml-5.3.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3dddf0fb832486cc1ea71d189cb92eb887826e8deebe128884e15020bb6e3f61"}, + {file = "lxml-5.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bcc211542f7af6f2dfb705f5f8b74e865592778e6cafdfd19c792c244ccce19"}, + {file = "lxml-5.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaca5a812f050ab55426c32177091130b1e49329b3f002a32934cd0245571307"}, + {file = "lxml-5.3.1-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:236610b77589faf462337b3305a1be91756c8abc5a45ff7ca8f245a71c5dab70"}, + {file = "lxml-5.3.1-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:aed57b541b589fa05ac248f4cb1c46cbb432ab82cbd467d1c4f6a2bdc18aecf9"}, + {file = "lxml-5.3.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:75fa3d6946d317ffc7016a6fcc44f42db6d514b7fdb8b4b28cbe058303cb6e53"}, + {file = "lxml-5.3.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:96eef5b9f336f623ffc555ab47a775495e7e8846dde88de5f941e2906453a1ce"}, + {file = "lxml-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:ef45f31aec9be01379fc6c10f1d9c677f032f2bac9383c827d44f620e8a88407"}, + {file = "lxml-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0611da6b07dd3720f492db1b463a4d1175b096b49438761cc9f35f0d9eaaef5"}, + {file = "lxml-5.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b2aca14c235c7a08558fe0a4786a1a05873a01e86b474dfa8f6df49101853a4e"}, + {file = "lxml-5.3.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae82fce1d964f065c32c9517309f0c7be588772352d2f40b1574a214bd6e6098"}, + {file = "lxml-5.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7aae7a3d63b935babfdc6864b31196afd5145878ddd22f5200729006366bc4d5"}, + {file = "lxml-5.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8e0d177b1fe251c3b1b914ab64135475c5273c8cfd2857964b2e3bb0fe196a7"}, + {file = "lxml-5.3.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:6c4dd3bfd0c82400060896717dd261137398edb7e524527438c54a8c34f736bf"}, + {file = "lxml-5.3.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f1208c1c67ec9e151d78aa3435aa9b08a488b53d9cfac9b699f15255a3461ef2"}, + {file = "lxml-5.3.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:c6aacf00d05b38a5069826e50ae72751cb5bc27bdc4d5746203988e429b385bb"}, + {file = "lxml-5.3.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5881aaa4bf3a2d086c5f20371d3a5856199a0d8ac72dd8d0dbd7a2ecfc26ab73"}, + {file = "lxml-5.3.1-cp38-cp38-win32.whl", hash = "sha256:45fbb70ccbc8683f2fb58bea89498a7274af1d9ec7995e9f4af5604e028233fc"}, + {file = "lxml-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:7512b4d0fc5339d5abbb14d1843f70499cab90d0b864f790e73f780f041615d7"}, + {file = "lxml-5.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5885bc586f1edb48e5d68e7a4b4757b5feb2a496b64f462b4d65950f5af3364f"}, + {file = "lxml-5.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1b92fe86e04f680b848fff594a908edfa72b31bfc3499ef7433790c11d4c8cd8"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a091026c3bf7519ab1e64655a3f52a59ad4a4e019a6f830c24d6430695b1cf6a"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ffb141361108e864ab5f1813f66e4e1164181227f9b1f105b042729b6c15125"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3715cdf0dd31b836433af9ee9197af10e3df41d273c19bb249230043667a5dfd"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88b72eb7222d918c967202024812c2bfb4048deeb69ca328363fb8e15254c549"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa59974880ab5ad8ef3afaa26f9bda148c5f39e06b11a8ada4660ecc9fb2feb3"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3bb8149840daf2c3f97cebf00e4ed4a65a0baff888bf2605a8d0135ff5cf764e"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:0d6b2fa86becfa81f0a0271ccb9eb127ad45fb597733a77b92e8a35e53414914"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:136bf638d92848a939fd8f0e06fcf92d9f2e4b57969d94faae27c55f3d85c05b"}, + {file = "lxml-5.3.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:89934f9f791566e54c1d92cdc8f8fd0009447a5ecdb1ec6b810d5f8c4955f6be"}, + {file = "lxml-5.3.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a8ade0363f776f87f982572c2860cc43c65ace208db49c76df0a21dde4ddd16e"}, + {file = "lxml-5.3.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:bfbbab9316330cf81656fed435311386610f78b6c93cc5db4bebbce8dd146675"}, + {file = "lxml-5.3.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:172d65f7c72a35a6879217bcdb4bb11bc88d55fb4879e7569f55616062d387c2"}, + {file = "lxml-5.3.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e3c623923967f3e5961d272718655946e5322b8d058e094764180cdee7bab1af"}, + {file = "lxml-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ce0930a963ff593e8bb6fda49a503911accc67dee7e5445eec972668e672a0f0"}, + {file = "lxml-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:f7b64fcd670bca8800bc10ced36620c6bbb321e7bc1214b9c0c0df269c1dddc2"}, + {file = "lxml-5.3.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:afa578b6524ff85fb365f454cf61683771d0170470c48ad9d170c48075f86725"}, + {file = "lxml-5.3.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f5e80adf0aafc7b5454f2c1cb0cde920c9b1f2cbd0485f07cc1d0497c35c5d"}, + {file = "lxml-5.3.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dd0b80ac2d8f13ffc906123a6f20b459cb50a99222d0da492360512f3e50f84"}, + {file = "lxml-5.3.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:422c179022ecdedbe58b0e242607198580804253da220e9454ffe848daa1cfd2"}, + {file = "lxml-5.3.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:524ccfded8989a6595dbdda80d779fb977dbc9a7bc458864fc9a0c2fc15dc877"}, + {file = "lxml-5.3.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:48fd46bf7155def2e15287c6f2b133a2f78e2d22cdf55647269977b873c65499"}, + {file = "lxml-5.3.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:05123fad495a429f123307ac6d8fd6f977b71e9a0b6d9aeeb8f80c017cb17131"}, + {file = "lxml-5.3.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a243132767150a44e6a93cd1dde41010036e1cbc63cc3e9fe1712b277d926ce3"}, + {file = "lxml-5.3.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c92ea6d9dd84a750b2bae72ff5e8cf5fdd13e58dda79c33e057862c29a8d5b50"}, + {file = "lxml-5.3.1-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:2f1be45d4c15f237209bbf123a0e05b5d630c8717c42f59f31ea9eae2ad89394"}, + {file = "lxml-5.3.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a83d3adea1e0ee36dac34627f78ddd7f093bb9cfc0a8e97f1572a949b695cb98"}, + {file = "lxml-5.3.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3edbb9c9130bac05d8c3fe150c51c337a471cc7fdb6d2a0a7d3a88e88a829314"}, + {file = "lxml-5.3.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2f23cf50eccb3255b6e913188291af0150d89dab44137a69e14e4dcb7be981f1"}, + {file = "lxml-5.3.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df7e5edac4778127f2bf452e0721a58a1cfa4d1d9eac63bdd650535eb8543615"}, + {file = "lxml-5.3.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:094b28ed8a8a072b9e9e2113a81fda668d2053f2ca9f2d202c2c8c7c2d6516b1"}, + {file = "lxml-5.3.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:514fe78fc4b87e7a7601c92492210b20a1b0c6ab20e71e81307d9c2e377c64de"}, + {file = "lxml-5.3.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8fffc08de02071c37865a155e5ea5fce0282e1546fd5bde7f6149fcaa32558ac"}, + {file = "lxml-5.3.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4b0d5cdba1b655d5b18042ac9c9ff50bda33568eb80feaaca4fc237b9c4fbfde"}, + {file = "lxml-5.3.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3031e4c16b59424e8d78522c69b062d301d951dc55ad8685736c3335a97fc270"}, + {file = "lxml-5.3.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb659702a45136c743bc130760c6f137870d4df3a9e14386478b8a0511abcfca"}, + {file = "lxml-5.3.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a11b16a33656ffc43c92a5343a28dc71eefe460bcc2a4923a96f292692709f6"}, + {file = "lxml-5.3.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5ae125276f254b01daa73e2c103363d3e99e3e10505686ac7d9d2442dd4627a"}, + {file = "lxml-5.3.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c76722b5ed4a31ba103e0dc77ab869222ec36efe1a614e42e9bcea88a36186fe"}, + {file = "lxml-5.3.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:33e06717c00c788ab4e79bc4726ecc50c54b9bfb55355eae21473c145d83c2d2"}, + {file = "lxml-5.3.1.tar.gz", hash = "sha256:106b7b5d2977b339f1e97efe2778e2ab20e99994cbb0ec5e55771ed0795920c8"}, ] [package.extras] cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml-html-clean"] +html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=3.0.10)"] +source = ["Cython (>=3.0.11,<3.1.0)"] [[package]] name = "mako" @@ -2392,22 +2388,22 @@ files = [ [[package]] name = "marshmallow" -version = "3.22.0" +version = "3.26.1" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, - {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, + {file = "marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c"}, + {file = "marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6"}, ] [package.dependencies] packaging = ">=17.0" [package.extras] -dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] -docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] -tests = ["pytest", "pytz", "simplejson"] +dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] +docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] +tests = ["pytest", "simplejson"] [[package]] name = "marshmallow-sqlalchemy" @@ -2475,46 +2471,47 @@ files = [ [[package]] name = "moto" -version = "5.0.11" -description = "" +version = "5.1.0" +description = "A library that allows you to easily mock out tests based on AWS infrastructure" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "moto-5.0.11-py2.py3-none-any.whl", hash = "sha256:bdba9bec0afcde9f99b58c5271d6458dbfcda0a0a1e9beaecd808d2591db65ea"}, - {file = "moto-5.0.11.tar.gz", hash = "sha256:606b641f4c6ef69f28a84147d6d6806d052011e7ae7b0fe46ae8858e7a27a0a3"}, + {file = "moto-5.1.0-py3-none-any.whl", hash = "sha256:4fada00cedfba661aa58fe0b33b3ba9a0ef96d0e9937c9bed5163053898b4a27"}, + {file = "moto-5.1.0.tar.gz", hash = "sha256:879274a9d2213ca49706e3c8ea380d90953ec1ec642976f6315255394d36edc0"}, ] [package.dependencies] boto3 = ">=1.9.201" -botocore = ">=1.14.0" -cryptography = ">=3.3.1" +botocore = ">=1.14.0,<1.35.45 || >1.35.45,<1.35.46 || >1.35.46" +cryptography = ">=35.0.0" Jinja2 = ">=2.10.1" python-dateutil = ">=2.1,<3.0.0" requests = ">=2.5" -responses = ">=0.15.0" +responses = ">=0.15.0,<0.25.5 || >0.25.5" werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] -all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "jsonschema", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.1)", "pyparsing (>=3.0.7)", "setuptools"] apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] appsync = ["graphql-core"] awslambda = ["docker (>=3.0.0)"] batch = ["docker (>=3.0.0)"] -cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.1)", "pyparsing (>=3.0.7)", "setuptools"] cognitoidp = ["joserfc (>=0.9.0)"] -dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"] -dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.5)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.6.1)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.6.1)"] +events = ["jsonpath_ng"] glue = ["pyparsing (>=3.0.7)"] -iotdata = ["jsondiff (>=1.1.2)"] -proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] -resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)"] -s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.5)"] -s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.5)"] -server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.5)", "pyparsing (>=3.0.7)", "setuptools"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.1)", "pyparsing (>=3.0.7)", "setuptools"] +quicksight = ["jsonschema"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.1)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.6.1)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.6.1)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsonpath_ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.6.1)", "pyparsing (>=3.0.7)", "setuptools"] ssm = ["PyYAML (>=5.1)"] -stepfunctions = ["antlr4-python3-runtime", "jsonpath-ng"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath_ng"] xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] [[package]] @@ -2771,47 +2768,66 @@ requests = ">=2.0.0" [[package]] name = "numpy" -version = "1.26.4" +version = "2.2.3" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" files = [ - {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, - {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, - {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, - {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, - {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, - {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, - {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, - {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, - {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, - {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, - {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, - {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, - {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, - {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, - {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, - {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, - {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, - {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, - {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, - {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, - {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, - {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, - {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, - {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, - {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, - {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, - {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, - {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, - {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, - {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, - {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, - {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, - {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, - {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, - {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, - {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cbc6472e01952d3d1b2772b720428f8b90e2deea8344e854df22b0618e9cce71"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdfe0c22692a30cd830c0755746473ae66c4a8f2e7bd508b35fb3b6a0813d787"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:e37242f5324ffd9f7ba5acf96d774f9276aa62a966c0bad8dae692deebec7716"}, + {file = "numpy-2.2.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:95172a21038c9b423e68be78fd0be6e1b97674cde269b76fe269a5dfa6fadf0b"}, + {file = "numpy-2.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5b47c440210c5d1d67e1cf434124e0b5c395eee1f5806fdd89b553ed1acd0a3"}, + {file = "numpy-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0391ea3622f5c51a2e29708877d56e3d276827ac5447d7f45e9bc4ade8923c52"}, + {file = "numpy-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f6b3dfc7661f8842babd8ea07e9897fe3d9b69a1d7e5fbb743e4160f9387833b"}, + {file = "numpy-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1ad78ce7f18ce4e7df1b2ea4019b5817a2f6a8a16e34ff2775f646adce0a5027"}, + {file = "numpy-2.2.3-cp310-cp310-win32.whl", hash = "sha256:5ebeb7ef54a7be11044c33a17b2624abe4307a75893c001a4800857956b41094"}, + {file = "numpy-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:596140185c7fa113563c67c2e894eabe0daea18cf8e33851738c19f70ce86aeb"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:16372619ee728ed67a2a606a614f56d3eabc5b86f8b615c79d01957062826ca8"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5521a06a3148686d9269c53b09f7d399a5725c47bbb5b35747e1cb76326b714b"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:7c8dde0ca2f77828815fd1aedfdf52e59071a5bae30dac3b4da2a335c672149a"}, + {file = "numpy-2.2.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:77974aba6c1bc26e3c205c2214f0d5b4305bdc719268b93e768ddb17e3fdd636"}, + {file = "numpy-2.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d42f9c36d06440e34226e8bd65ff065ca0963aeecada587b937011efa02cdc9d"}, + {file = "numpy-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2712c5179f40af9ddc8f6727f2bd910ea0eb50206daea75f58ddd9fa3f715bb"}, + {file = "numpy-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c8b0451d2ec95010d1db8ca733afc41f659f425b7f608af569711097fd6014e2"}, + {file = "numpy-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9b4a8148c57ecac25a16b0e11798cbe88edf5237b0df99973687dd866f05e1b"}, + {file = "numpy-2.2.3-cp311-cp311-win32.whl", hash = "sha256:1f45315b2dc58d8a3e7754fe4e38b6fce132dab284a92851e41b2b344f6441c5"}, + {file = "numpy-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f48ba6f6c13e5e49f3d3efb1b51c8193215c42ac82610a04624906a9270be6f"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12c045f43b1d2915eca6b880a7f4a256f59d62df4f044788c8ba67709412128d"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87eed225fd415bbae787f93a457af7f5990b92a334e346f72070bf569b9c9c95"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:712a64103d97c404e87d4d7c47fb0c7ff9acccc625ca2002848e0d53288b90ea"}, + {file = "numpy-2.2.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a5ae282abe60a2db0fd407072aff4599c279bcd6e9a2475500fc35b00a57c532"}, + {file = "numpy-2.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5266de33d4c3420973cf9ae3b98b54a2a6d53a559310e3236c4b2b06b9c07d4e"}, + {file = "numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe"}, + {file = "numpy-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34c1b7e83f94f3b564b35f480f5652a47007dd91f7c839f404d03279cc8dd021"}, + {file = "numpy-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4d8335b5f1b6e2bce120d55fb17064b0262ff29b459e8493d1785c18ae2553b8"}, + {file = "numpy-2.2.3-cp312-cp312-win32.whl", hash = "sha256:4d9828d25fb246bedd31e04c9e75714a4087211ac348cb39c8c5f99dbb6683fe"}, + {file = "numpy-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bfdb06b395385ea9b91bf55c1adf1b297c9fdb531552845ff1d3ea6e40d5aba"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:23c9f4edbf4c065fddb10a4f6e8b6a244342d95966a48820c614891e5059bb50"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:a0c03b6be48aaf92525cccf393265e02773be8fd9551a2f9adbe7db1fa2b60f1"}, + {file = "numpy-2.2.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:2376e317111daa0a6739e50f7ee2a6353f768489102308b0d98fcf4a04f7f3b5"}, + {file = "numpy-2.2.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8fb62fe3d206d72fe1cfe31c4a1106ad2b136fcc1606093aeab314f02930fdf2"}, + {file = "numpy-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52659ad2534427dffcc36aac76bebdd02b67e3b7a619ac67543bc9bfe6b7cdb1"}, + {file = "numpy-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b416af7d0ed3271cad0f0a0d0bee0911ed7eba23e66f8424d9f3dfcdcae1304"}, + {file = "numpy-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1402da8e0f435991983d0a9708b779f95a8c98c6b18a171b9f1be09005e64d9d"}, + {file = "numpy-2.2.3-cp313-cp313-win32.whl", hash = "sha256:136553f123ee2951bfcfbc264acd34a2fc2f29d7cdf610ce7daf672b6fbaa693"}, + {file = "numpy-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:5b732c8beef1d7bc2d9e476dbba20aaff6167bf205ad9aa8d30913859e82884b"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:435e7a933b9fda8126130b046975a968cc2d833b505475e588339e09f7672890"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7678556eeb0152cbd1522b684dcd215250885993dd00adb93679ec3c0e6e091c"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2e8da03bd561504d9b20e7a12340870dfc206c64ea59b4cfee9fceb95070ee94"}, + {file = "numpy-2.2.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:c9aa4496fd0e17e3843399f533d62857cef5900facf93e735ef65aa4bbc90ef0"}, + {file = "numpy-2.2.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4ca91d61a4bf61b0f2228f24bbfa6a9facd5f8af03759fe2a655c50ae2c6610"}, + {file = "numpy-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deaa09cd492e24fd9b15296844c0ad1b3c976da7907e1c1ed3a0ad21dded6f76"}, + {file = "numpy-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:246535e2f7496b7ac85deffe932896a3577be7af8fb7eebe7146444680297e9a"}, + {file = "numpy-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:daf43a3d1ea699402c5a850e5313680ac355b4adc9770cd5cfc2940e7861f1bf"}, + {file = "numpy-2.2.3-cp313-cp313t-win32.whl", hash = "sha256:cf802eef1f0134afb81fef94020351be4fe1d6681aadf9c5e862af6602af64ef"}, + {file = "numpy-2.2.3-cp313-cp313t-win_amd64.whl", hash = "sha256:aee2512827ceb6d7f517c8b85aa5d3923afe8fc7a57d028cffcd522f1c6fd082"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3c2ec8a0f51d60f1e9c0c5ab116b7fc104b165ada3f6c58abf881cb2eb16044d"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:ed2cf9ed4e8ebc3b754d398cba12f24359f018b416c380f577bbae112ca52fc9"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39261798d208c3095ae4f7bc8eaeb3481ea8c6e03dc48028057d3cbdbdb8937e"}, + {file = "numpy-2.2.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:783145835458e60fa97afac25d511d00a1eca94d4a8f3ace9fe2043003c678e4"}, + {file = "numpy-2.2.3.tar.gz", hash = "sha256:dbdc15f0c81611925f382dfa97b3bd0bc2c1ce19d4fe50482cb0ddc12ba30020"}, ] [[package]] @@ -3464,19 +3480,19 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" -version = "2.8.0" +version = "2.10.1" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, - {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] [package.extras] crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] @@ -3635,17 +3651,6 @@ files = [ {file = "python_json_logger-2.0.7-py3-none-any.whl", hash = "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd"}, ] -[[package]] -name = "pytz" -version = "2024.2" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, - {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, -] - [[package]] name = "pywin32-ctypes" version = "0.2.3" @@ -4974,4 +4979,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.12.2" -content-hash = "fcf4590ca3abb7ea5661c1f60c6eb9e246755ca5bb984f885e2756e453d7e638" +content-hash = "c791f133fcf2db01b7dadc9aca104cabd4c11331342bf936e0f7ebdb62bbec72" diff --git a/pyproject.toml b/pyproject.toml index 7c6b90730..796885c40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.12.2" alembic = "==1.13.2" -amqp = "==5.2.0" +amqp = "==5.3.1" beautifulsoup4 = "==4.12.3" boto3 = "^1.34.150" botocore = "^1.34.159" @@ -32,11 +32,11 @@ flask-marshmallow = "==1.2.1" flask-migrate = "==4.0.7" flask-redis = "==0.4.0" flask-sqlalchemy = "==3.1.1" -gunicorn = {version = "==22.0.0", extras = ["eventlet"]} +gunicorn = {version = "==23.0.0", extras = ["eventlet"]} iso8601 = "==2.1.0" jsonschema = {version = "==4.23.0", extras = ["format"]} -lxml = "==5.2.2" -marshmallow = "==3.22.0" +lxml = "==5.3.1" +marshmallow = "==3.26.1" marshmallow-sqlalchemy = "==1.0.0" newrelic = "*" notifications-python-client = "==10.0.0" @@ -44,7 +44,7 @@ oscrypto = { git = "https://github.com/wbond/oscrypto.git", rev = "1547f53" } packaging = "==24.1" poetry-dotenv-plugin = "==0.2.0" psycopg2-binary = "==2.9.9" -pyjwt = "==2.8.0" +pyjwt = "==2.10.1" python-dotenv = "==1.0.1" sqlalchemy = "==2.0.31" werkzeug = "^3.0.6" @@ -52,11 +52,10 @@ faker = "^26.0.0" async-timeout = "^4.0.3" bleach = "^6.1.0" geojson = "^3.2.0" -numpy = "^1.26.4" +numpy = "^2.2.3" ordered-set = "^4.1.0" phonenumbers = "^8.13.42" python-json-logger = "^2.0.7" -pytz = "^2024.1" regex = "^2024.7.24" shapely = "^2.0.5" smartypants = "^2.0.1" @@ -74,7 +73,7 @@ six = "^1.16.0" urllib3 = "^2.2.2" webencodings = "^0.5.1" itsdangerous = "^2.2.0" -jinja2 = "^3.1.5" +jinja2 = "^3.1.6" redis = "^5.0.8" requests = "^2.32.3" @@ -91,7 +90,7 @@ freezegun = "^1.5.1" honcho = "*" isort = "^5.13.2" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} -moto = "==5.0.11" +moto = "==5.1.0" pip-audit = "*" pre-commit = "^3.8.0" pytest = "^8.3.2" diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index de26f5760..e58a27275 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -22,8 +22,10 @@ from app.aws.s3 import ( get_s3_object, get_s3_resource, list_s3_objects, + purge_bucket, read_s3_file, remove_csv_object, + remove_job_from_s3, remove_s3_object, ) from app.clients import AWS_CLIENT_CONFIG @@ -449,9 +451,9 @@ def test_get_s3_files_success(client, mocker): mock_read_s3_file.assert_has_calls(calls, any_order=True) - # mock_current_app.info.assert_any_call("job_cache length before regen: 0 #notify-admin-1200") + # mock_current_app.info.assert_any_call("job_cache length before regen: 0 #notify-debug-admin-1200") - # mock_current_app.info.assert_any_call("job_cache length after regen: 0 #notify-admin-1200") + # mock_current_app.info.assert_any_call("job_cache length after regen: 0 #notify-debug-admin-1200") @patch("app.aws.s3.s3_client", None) # ensure it starts as None @@ -563,3 +565,71 @@ def test_get_s3_object_client_error(mocker): mock_logger.exception.assert_called_once_with( f"Can't retrieve S3 Object from {file_location}" ) + + +def test_purge_bucket(mocker): + mock_s3_resource = MagicMock() + mock_bucket = MagicMock() + mock_s3_resource.Bucket.return_value = mock_bucket + mocker.patch("app.aws.s3.get_s3_resource", return_value=mock_s3_resource) + + purge_bucket("my-bucket", "access-key", "secret-key", "region") + + # Assert that the bucket's objects.all().delete() method was called + mock_bucket.objects.all.return_value.delete.assert_called_once() + + +def test_remove_job_from_s3(mocker): + mock_get_job_location = mocker.patch("app.aws.s3.get_job_location") + mock_remove_s3_object = mocker.patch("app.aws.s3.remove_s3_object") + + mock_get_job_location.return_value = ( + "test-bucket", + "test.csv", + "fake-stuff", + ) + + remove_job_from_s3("service-id-123", "job-id-456") + + mock_get_job_location.assert_called_once_with("service-id-123", "job-id-456") + mock_remove_s3_object.assert_called_once_with( + "test-bucket", + "test.csv", + "fake-stuff", + ) + + +def test_get_s3_files_handles_exception(mocker): + mock_current_app = mocker.patch("app.aws.s3.current_app") + mock_current_app.config = { + "CSV_UPLOAD_BUCKET": {"bucket": "test-bucket"}, + "job_cache": {}, + } + + mock_list_s3_objects = mocker.patch("app.aws.s3.list_s3_objects") + mock_list_s3_objects.return_value = ["file1.csv", "file2.csv"] + + mock_get_s3_resource = mocker.patch("app.aws.s3.get_s3_resource") + + # Make the first call succeed, second call should fail. + mock_read_s3_file = mocker.patch( + "app.aws.s3.read_s3_file", side_effect=[None, Exception("exception here")] + ) + + mock_thread_pool_executor = mocker.patch("app.aws.s3.ThreadPoolExecutor") + mock_executor = mock_thread_pool_executor.return_value.__enter__.return_value + + def mock_map(func, iterable): + for item in iterable: + func(item) + + mock_executor.map.side_effect = mock_map + get_s3_files() + + calls = [ + mocker.call("test-bucket", "file1.csv", mock_get_s3_resource.return_value), + mocker.call("test-bucket", "file2.csv", mock_get_s3_resource.return_value), + ] + mock_read_s3_file.assert_has_calls(calls, any_order=True) + + mock_current_app.logger.exception.assert_called_with("Connection pool issue") diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 76395832e..894afa6f7 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -583,7 +583,7 @@ def test_batch_insert_with_expired_notifications(mocker): rs.llen.assert_called_once_with("message_queue") rs.rpush.assert_called_once() requeued_notification = json.loads(rs.rpush.call_args[0][1]) - assert requeued_notification["id"] == 1 + assert requeued_notification["id"] == "1" def test_batch_insert_with_malformed_notifications(mocker): diff --git a/tests/app/clients/test_aws_cloudwatch.py b/tests/app/clients/test_aws_cloudwatch.py index 7a0379454..36de2c1b1 100644 --- a/tests/app/clients/test_aws_cloudwatch.py +++ b/tests/app/clients/test_aws_cloudwatch.py @@ -31,7 +31,7 @@ def side_effect(filterPattern, logGroupName, startTime, endTime): { "logStreamName": "89db9712-c6d1-49f9-be7c-4caa7ed9efb1", "message": '{"delivery":{"destination":"+1661","phoneCarrier":"ATT Mobility", ' - '"providerResponse":"Invalid phone number"}}', + '"providerResponse":"Invalid phone number", "priceInUSD": "0.00881"}}', "eventId": "37535432778099870001723210579798865345508698025292922880", } ] @@ -44,7 +44,7 @@ def side_effect(filterPattern, logGroupName, startTime, endTime): "logStreamName": "89db9712-c6d1-49f9-be7c-4caa7ed9efb1", "timestamp": 1683147017911, "message": '{"delivery":{"destination":"+1661","phoneCarrier":"ATT Mobility",' - '"providerResponse":"Phone accepted msg"}}', + '"providerResponse":"Phone accepted msg", "priceInUSD": "0.00881"}}', "ingestionTime": 1683147018026, "eventId": "37535432778099870001723210579798865345508698025292922880", } @@ -131,6 +131,7 @@ def test_event_to_db_format_with_missing_fields(): "status": "UNKNOWN", "delivery.phoneCarrier": "", "delivery.providerResponse": "", + "delivery.priceInUSD": 0.0, "@timestamp": "", } @@ -140,7 +141,11 @@ def test_event_to_db_format_with_string_input(): { "notification": {"messageId": "67890", "timestamp": "2024-01-01T14:00:00Z"}, "status": "FAILED", - "delivery": {"phoneCarrier": "Verizon", "providerResponse": "Error"}, + "delivery": { + "phoneCarrier": "Verizon", + "providerResponse": "Error", + "priceInUSD": "0.00881", + }, } ) result = aws_cloudwatch_client.event_to_db_format(event) @@ -149,5 +154,6 @@ def test_event_to_db_format_with_string_input(): "status": "FAILED", "delivery.phoneCarrier": "Verizon", "delivery.providerResponse": "Error", + "delivery.priceInUSD": 0.00881, "@timestamp": "2024-01-01T14:00:00Z", } diff --git a/tests/app/conftest.py b/tests/app/conftest.py index d402aa8cb..135c54931 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -1,9 +1,9 @@ import json import uuid from datetime import datetime, timedelta +from zoneinfo import ZoneInfo import pytest -import pytz import requests_mock from flask import current_app, url_for from sqlalchemy import delete, select @@ -988,4 +988,4 @@ def admin_request(client): def datetime_in_past(days=0, seconds=0): - return datetime.now(tz=pytz.utc) - timedelta(days=days, seconds=seconds) + return datetime.now(tz=ZoneInfo("UTC")) - timedelta(days=days, seconds=seconds) diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index 1a145538a..4df57ec07 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -28,6 +28,7 @@ from app.dao.notifications_dao import ( get_notification_with_personalisation, get_notifications_for_job, get_notifications_for_service, + get_recent_notifications_for_job, get_service_ids_with_notifications_on_date, notifications_not_yet_sent, sanitize_successful_notification_by_id, @@ -692,6 +693,17 @@ def test_get_all_notifications_for_job(sample_job): assert len(notifications_from_db) == 5 +def test_get_recent_notifications_for_job(sample_job): + + for status in NotificationStatus: + create_notification(template=sample_job.template, job=sample_job, status=status) + + notifications_from_db = get_recent_notifications_for_job( + sample_job.service.id, sample_job.id + ).items + assert len(notifications_from_db) == 13 + + def test_get_all_notifications_for_job_by_status(sample_job): notifications = partial( get_notifications_for_job, sample_job.service.id, sample_job.id @@ -2007,8 +2019,8 @@ def test_notifications_not_yet_sent_return_no_rows(sample_service, notification_ def test_update_delivery_receipts(mocker): mock_session = mocker.patch("app.dao.notifications_dao.db.session") receipts = [ - '{"notification.messageId": "msg1", "delivery.phoneCarrier": "carrier1", "delivery.providerResponse": "resp1", "@timestamp": "2024-01-01T12:00:00"}', # noqa - '{"notification.messageId": "msg2", "delivery.phoneCarrier": "carrier2", "delivery.providerResponse": "resp2", "@timestamp": "2024-01-01T13:00:00"}', # noqa + '{"notification.messageId": "msg1", "delivery.phoneCarrier": "carrier1", "delivery.providerResponse": "resp1", "@timestamp": "2024-01-01T12:00:00", "delivery.priceInUSD": "0.00881"}', # noqa + '{"notification.messageId": "msg2", "delivery.phoneCarrier": "carrier2", "delivery.providerResponse": "resp2", "@timestamp": "2024-01-01T13:00:00", "delivery.priceInUSD": "0.00881"}', # noqa ] delivered = True mock_update = MagicMock() @@ -2122,3 +2134,57 @@ def test_sanitize_successful_notification_by_id(): "sent_at": ANY, }, ) + + +def test_dao_get_notifications_by_recipient_or_reference_covers_sms_search_by_reference( + notify_db_session, +): + """ + This test: + 1. Creates a service and an SMS template. + 2. Creates a notification with a specific client_reference and status=FAILED. + 3. Calls dao_get_notifications_by_recipient_or_reference with notification_type=SMS, + statuses=[FAILED], and a search term = client_reference. + 4. Confirms the function returns exactly one notification matching that reference. + """ + + service = create_service(service_name="Test Service") + template = create_template(service=service, template_type=NotificationType.SMS) + + # Instead of matching phone logic, we'll match on client_reference + data = { + "id": uuid.uuid4(), + "to": "1", + "normalised_to": "1", # phone is irrelevant here + "service_id": service.id, + "service": service, + "template_id": template.id, + "template_version": template.version, + "status": NotificationStatus.FAILED, + "created_at": utc_now(), + "billable_units": 1, + "notification_type": template.template_type, + "key_type": KeyType.NORMAL, + "client_reference": "some-ref", # <--- We'll search for this + } + notification = Notification(**data) + dao_create_notification(notification) + + # We'll search by this reference instead of a phone number + search_term = "some-ref" + + results_page = dao_get_notifications_by_recipient_or_reference( + service_id=service.id, + search_term=search_term, + notification_type=NotificationType.SMS, + statuses=[NotificationStatus.FAILED], + page=1, + page_size=50, + ) + + # Now we should find exactly one match + assert len(results_page.items) == 1, "Should find exactly one matching notification" + found = results_page.items[0] + assert found.id == notification.id + assert found.status == NotificationStatus.FAILED + assert found.client_reference == "some-ref" diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py index d4581104d..b25281dfc 100644 --- a/tests/app/dao/test_date_utils.py +++ b/tests/app/dao/test_date_utils.py @@ -3,6 +3,7 @@ from datetime import date, datetime import pytest from app.dao.date_util import ( + generate_hourly_range, get_calendar_year, get_calendar_year_for_datetime, get_month_start_and_end_date_in_utc, @@ -75,3 +76,46 @@ def test_get_month_start_and_end_date_in_utc(month, year, expected_start, expect ) def test_get_calendar_year_for_datetime(dt, fy): assert get_calendar_year_for_datetime(dt) == fy + + +def test_generate_hourly_range_with_end_date(): + start_date = datetime(2025, 2, 18, 12, 0) + end_date = datetime(2025, 2, 18, 15, 0) + result = list(generate_hourly_range(start_date, end_date=end_date)) + + expected = [ + datetime(2025, 2, 18, 12, 0), + datetime(2025, 2, 18, 13, 0), + datetime(2025, 2, 18, 14, 0), + datetime(2025, 2, 18, 15, 0), + ] + + assert result == expected, f"Expected {expected}, but got {result}" + + +def test_generate_hourly_range_with_hours(): + start_date = datetime(2025, 2, 18, 12, 0) + result = list(generate_hourly_range(start_date, hours=3)) + + expected = [ + datetime(2025, 2, 18, 12, 0), + datetime(2025, 2, 18, 13, 0), + datetime(2025, 2, 18, 14, 0), + ] + + assert result == expected, f"Expected {expected}, but got {result}" + + +def test_generate_hourly_range_with_zero_hours(): + start_date = datetime(2025, 2, 18, 12, 0) + result = list(generate_hourly_range(start_date, hours=0)) + + assert result == [], f"Expected an empty list, but got {result}" + + +def test_generate_hourly_range_with_end_date_before_start(): + start_date = datetime(2025, 2, 18, 12, 0) + end_date = datetime(2025, 2, 18, 10, 0) + result = list(generate_hourly_range(start_date, end_date=end_date)) + + assert result == [], f"Expected empty list, but got {result}" diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index 5b9a7d695..a9be67b5d 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -36,7 +36,6 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): service_2 = create_service(service_name="service_2") create_template(service=service_1) - create_template(service=service_1, template_type=TemplateType.EMAIL) # not the service being tested create_template(service=service_2) @@ -50,11 +49,7 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): create_notification( service_1.templates[0], created_at=datetime(2018, 1, 1, 1, 1, 0) ) - create_notification( - service_1.templates[1], - created_at=datetime(2018, 1, 1, 1, 1, 0), - status=NotificationStatus.DELIVERED, - ) + create_notification( service_1.templates[0], created_at=datetime(2018, 2, 1, 1, 1, 0), @@ -79,30 +74,25 @@ def test_fetch_notification_status_for_service_by_month(notify_db_session): fetch_notification_status_for_service_by_month( date(2018, 1, 1), date(2018, 2, 28), service_1.id ), - key=lambda x: (x.month, x.notification_type, x.notification_status), + key=lambda x: (x.month, x.notification_status), ) - assert len(results) == 4 + assert len(results) == 3 assert results[0].month.date() == date(2018, 1, 1) - assert results[0].notification_type == NotificationType.EMAIL - assert results[0].notification_status == NotificationStatus.DELIVERED + assert results[0].notification_type == NotificationType.SMS + assert results[0].notification_status == NotificationStatus.CREATED assert results[0].count == 1 assert results[1].month.date() == date(2018, 1, 1) assert results[1].notification_type == NotificationType.SMS - assert results[1].notification_status == NotificationStatus.CREATED - assert results[1].count == 1 + assert results[1].notification_status == NotificationStatus.DELIVERED + assert results[1].count == 14 - assert results[2].month.date() == date(2018, 1, 1) + assert results[2].month.date() == date(2018, 2, 1) assert results[2].notification_type == NotificationType.SMS assert results[2].notification_status == NotificationStatus.DELIVERED - assert results[2].count == 14 - - assert results[3].month.date() == date(2018, 2, 1) - assert results[3].notification_type == NotificationType.SMS - assert results[3].notification_status == NotificationStatus.DELIVERED - assert results[3].count == 1 + assert results[2].count == 1 def test_fetch_notification_status_for_service_for_day(notify_db_session): diff --git a/tests/app/dao/test_services_get_specific_hours.py b/tests/app/dao/test_services_get_specific_hours.py new file mode 100644 index 000000000..a97dbd150 --- /dev/null +++ b/tests/app/dao/test_services_get_specific_hours.py @@ -0,0 +1,100 @@ +from collections import namedtuple +from datetime import datetime + +import pytest + +from app.dao.services_dao import get_specific_hours_stats +from app.enums import StatisticsType +from app.models import TemplateType + +NotificationRow = namedtuple( + "NotificationRow", ["notification_type", "status", "timestamp", "count"] +) + + +def generate_expected_hourly_output(requested_sms_hours): + return { + hour: { + TemplateType.SMS: { + StatisticsType.REQUESTED: 1, + StatisticsType.DELIVERED: 0, + StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, + }, + TemplateType.EMAIL: { + StatisticsType.REQUESTED: 0, + StatisticsType.DELIVERED: 0, + StatisticsType.FAILURE: 0, + StatisticsType.PENDING: 0, + }, + } + for hour in requested_sms_hours + } + + +def create_mock_notification(notification_type, status, timestamp, count=1): + """ + Creates a named tuple with the attributes required by format_statistics. + """ + return NotificationRow( + notification_type=notification_type, + status=status, + timestamp=timestamp.replace(minute=0, second=0, microsecond=0), + count=count, + ) + + +test_cases = [ + ( + [ + create_mock_notification( + TemplateType.SMS, + StatisticsType.REQUESTED, + datetime(2025, 2, 18, 14, 15, 0), + ) + ], + datetime(2025, 2, 18, 12, 0), + 6, + generate_expected_hourly_output(["2025-02-18T14:00:00Z"]), + ), + ( + [ + create_mock_notification( + TemplateType.SMS, + StatisticsType.REQUESTED, + datetime(2025, 2, 18, 17, 59, 59), + ) + ], + datetime(2025, 2, 18, 15, 0), + 3, + generate_expected_hourly_output(["2025-02-18T17:00:00Z"]), + ), + ([], datetime(2025, 2, 18, 10, 0), 4, {}), + ( + [ + create_mock_notification( + TemplateType.SMS, + StatisticsType.REQUESTED, + datetime(2025, 2, 18, 9, 30, 0), + ), + create_mock_notification( + TemplateType.SMS, + StatisticsType.REQUESTED, + datetime(2025, 2, 18, 11, 45, 0), + ), + ], + datetime(2025, 2, 18, 8, 0), + 5, + generate_expected_hourly_output( + ["2025-02-18T09:00:00Z", "2025-02-18T11:00:00Z"] + ), + ), +] + + +@pytest.mark.parametrize( + "mocked_notifications, start_date, hours, expected_output", test_cases +) +def test_get_specific_hours(mocked_notifications, start_date, hours, expected_output): + results = get_specific_hours_stats(mocked_notifications, start_date, hours=hours) + assert results == expected_output, f"Expected {expected_output}, but got {results}" diff --git a/tests/app/job/test_rest.py b/tests/app/job/test_rest.py index 8d40a045a..f65ce2458 100644 --- a/tests/app/job/test_rest.py +++ b/tests/app/job/test_rest.py @@ -2,9 +2,9 @@ import json import uuid from datetime import date, datetime, timedelta from unittest.mock import ANY +from zoneinfo import ZoneInfo import pytest -import pytz from freezegun import freeze_time import app.celery.tasks @@ -183,7 +183,7 @@ def test_create_scheduled_job(client, sample_template, mocker, fake_uuid): assert resp_json["data"]["id"] == fake_uuid assert ( resp_json["data"]["scheduled_for"] - == datetime(2016, 1, 5, 11, 59, 0, tzinfo=pytz.UTC).isoformat() + == datetime(2016, 1, 5, 11, 59, 0, tzinfo=ZoneInfo("UTC")).isoformat() ) assert resp_json["data"]["job_status"] == JobStatus.SCHEDULED assert resp_json["data"]["template"] == str(sample_template.id) @@ -488,6 +488,48 @@ def test_get_all_notifications_for_job_in_order_of_job_number( assert resp["notifications"][2]["job_row_number"] == notification_3.job_row_number +def test_get_recent_notifications_for_job_in_reverse_order_of_job_number( + admin_request, sample_template, mocker +): + mock_s3 = mocker.patch("app.job.rest.get_phone_number_from_s3") + mock_s3.return_value = "15555555555" + + mock_s3_personalisation = mocker.patch("app.job.rest.get_personalisation_from_s3") + mock_s3_personalisation.return_value = {} + + main_job = create_job(sample_template) + another_job = create_job(sample_template) + + count = 1 + for status in NotificationStatus: + create_notification(job=main_job, job_row_number=str(count), status=status) + count = count + 1 + create_notification(job=another_job) + + resp = admin_request.get( + "job.get_recent_notifications_for_service_job", + service_id=main_job.service_id, + job_id=main_job.id, + ) + + assert len(resp["notifications"]) == 13 + assert resp["notifications"][0]["status"] == "virus-scan-failed" + assert resp["notifications"][0]["job_row_number"] == 13 + + query_string = {"status": "delivered"} + resp = admin_request.get( + "job.get_recent_notifications_for_service_job", + service_id=main_job.service_id, + job_id=main_job.id, + **query_string, + ) + + assert len(resp["notifications"]) == 1 + + assert resp["notifications"][0]["status"] == "delivered" + assert resp["notifications"][0]["job_row_number"] == 5 + + @pytest.mark.parametrize( "expected_notification_count, status_args", [ diff --git a/tests/app/organization/test_rest.py b/tests/app/organization/test_rest.py index 445a47297..d128630bd 100644 --- a/tests/app/organization/test_rest.py +++ b/tests/app/organization/test_rest.py @@ -831,7 +831,9 @@ def test_get_organization_users_returns_users_for_organization( ) assert len(response["data"]) == 2 - assert response["data"][0]["id"] == str(first.id) + response_ids = [response["data"][0]["id"], response["data"][1]["id"]] + assert str(first.id) in response_ids + assert str(second.id) in response_ids @freeze_time("2019-12-24 13:30") diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index cd2ef8005..5668890bf 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1668,7 +1668,9 @@ def test_remove_user_from_service(client, sample_user_service_permission): def test_get_service_message_ratio(mocker, client, sample_user_service_permission): service = sample_user_service_permission.service - mock_redis = mocker.patch("app.service.rest.redis_store.get") + mock_redis = mocker.patch( + "app.service.rest.dao_get_notification_count_for_service_message_ratio" + ) mock_redis.return_value = 1 endpoint = url_for( diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 859e36f34..d318f9f8a 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -15,6 +15,7 @@ from app.commands import ( dump_sms_senders, dump_user_info, fix_billable_units, + generate_salt, insert_inbound_numbers_from_file, populate_annual_billing_with_defaults, populate_annual_billing_with_the_previous_years_allowance, @@ -661,3 +662,10 @@ def test_dump_user_info(notify_api, mocker): mock_get_user_by_email.assert_called_once_with("john@example.com") mock_open_file.assert_called_once_with("user_download.json", "wb") + + +def test_generate_salt(notify_api): + runner = notify_api.test_cli_runner() + result = runner.invoke(generate_salt) + assert result.exit_code == 0 + assert len(result.output.strip()) == 32