Merge pull request #3244 from alphagov/func-tests-fix

functional tests email reply to flakiness

app/config.py

@@ -72,6 +72,8 @@ class Config(object):
     ACTIVITY_STATS_LIMIT_DAYS = 7
     TEST_MESSAGE_FILENAME = 'Report'
 
+    REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 45
+
     NOTIFY_ENVIRONMENT = 'development'
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-local'
     MOU_BUCKET_NAME = 'local-mou'
@@ -147,6 +149,9 @@ class Preview(Config):
     ASSET_DOMAIN = 'static.notify.works'
     ASSET_PATH = 'https://static.notify.works/'
 
+    # On preview, extend the validation timeout to allow more leniency when running functional tests
+    REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT = 120
+
 
 class Staging(Config):
     SHOW_STYLEGUIDE = False

app/main/views/service_settings.py

@@ -478,7 +478,8 @@ def get_service_verify_reply_to_address_partials(service_id, notification_id):
     created_at_no_tz = notification["created_at"][:-6]
     seconds_since_sending = (datetime.utcnow() - datetime.strptime(created_at_no_tz, '%Y-%m-%dT%H:%M:%S.%f')).seconds
     if notification["status"] in FAILURE_STATUSES or (
-        notification["status"] in SENDING_STATUSES and seconds_since_sending > 45
+        notification["status"] in SENDING_STATUSES and
+        seconds_since_sending > current_app.config['REPLY_TO_EMAIL_ADDRESS_VALIDATION_TIMEOUT']
     ):
         verification_status = "failure"
         form.email_address.data = notification['to']

tests/app/test_cloudfoundry_config.py

@@ -6,12 +6,11 @@ from app.cloudfoundry_config import extract_cloudfoundry_config
 
 
 @pytest.fixture
-def cloudfoundry_environ(monkeypatch):
-    monkeypatch.setenv('VCAP_APPLICATION', '{"space_name":"🚀🌌"}')
+def cloudfoundry_environ(os_environ):
+    os.environ['VCAP_APPLICATION'] = '{"space_name":"🚀🌌"}'
 
 
-@pytest.mark.usefixtures('os_environ', 'cloudfoundry_environ')
-def test_extract_cloudfoundry_config_populates_other_vars():
+def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_environ):
     extract_cloudfoundry_config()
 
     assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌'

tests/app/test_config.py

@@ -12,21 +12,24 @@ def cf_conf():
 
 
 @pytest.fixture
-def reload_config():
+def reload_config(os_environ):
     """
     Reset config, by simply re-running config.py from a fresh environment
     """
     old_env = os.environ.copy()
+    os.environ.clear()
 
     yield
 
-    os.environ = old_env
+    os.environ.clear()
+    for k, v in old_env.items():
+        os.environ[k] = v
     importlib.reload(config)
 
 
-def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config):
+def test_load_cloudfoundry_config_if_available(reload_config):
     os.environ['API_HOST_NAME'] = 'env'
-    monkeypatch.setenv('VCAP_APPLICATION', 'some json blob')
+    os.environ['VCAP_APPLICATION'] = 'some json blob'
 
     with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config', side_effect=cf_conf) as cf_config:
         # reload config so that its module level code (ie: all of it) is re-instantiated
@@ -38,10 +41,10 @@ def test_load_cloudfoundry_config_if_available(monkeypatch, reload_config):
     assert config.Config.API_HOST_NAME == 'cf'
 
 
-def test_load_config_if_cloudfoundry_not_available(monkeypatch, reload_config):
+def test_load_config_if_cloudfoundry_not_available(reload_config):
     os.environ['API_HOST_NAME'] = 'env'
 
-    monkeypatch.delenv('VCAP_APPLICATION', raising=False)
+    os.environ.pop('VCAP_APPLICATION', None)
 
     with mock.patch('app.cloudfoundry_config.extract_cloudfoundry_config') as cf_config:
         # reload config so that its module level code (ie: all of it) is re-instantiated

tests/conftest.py

@@ -2827,9 +2827,10 @@ def os_environ():
     """
     # for use whenever you expect code to edit environment variables
     old_env = os.environ.copy()
-    os.environ = {}
+    os.environ.clear()
     yield
-    os.environ = old_env
+    for k, v in old_env.items():
+        os.environ[k] = v
 
 
 @pytest.fixture