From 34b2ad0ff21145d9e973375fce39ed52780eff1e Mon Sep 17 00:00:00 2001 From: Juliana Kang Date: Thu, 9 Nov 2023 16:13:22 -0500 Subject: [PATCH] feat: Add fallback call and populate SDN data command (#12) REV-3628 --- .annotation_safe_list.yml | 8 +- requirements/base.in | 2 + requirements/base.txt | 14 +- requirements/dev.txt | 20 +- requirements/django.txt | 2 +- requirements/doc.txt | 20 +- requirements/production.txt | 16 +- requirements/quality.txt | 20 +- requirements/test.txt | 18 +- requirements/validation.txt | 25 +- sanctions/apps/api/v1/tests/test_views.py | 27 +- sanctions/apps/api/v1/views.py | 16 +- .../apps/api_client/tests/test_sdn_client.py | 12 +- sanctions/apps/core/tests/factories.py | 3 - .../commands/hit_opsgenie_heartbeat.py | 33 --- ...populate_sdn_fallback_data_and_metadata.py | 101 +++++++ .../management/commands/tests/__init__.py | 0 .../tests/test_download_sdn_fallback.py | 158 +++++++++++ .../migrations/0002_rename_fallback_models.py | 49 ++++ sanctions/apps/sanctions/models.py | 131 ++++++++- sanctions/apps/sanctions/tests/factories.py | 40 +++ sanctions/apps/sanctions/tests/test_models.py | 257 ++++++++++++++++++ sanctions/apps/sanctions/utils.py | 184 +++++++++++++ sanctions/settings/base.py | 4 +- test_utils/__init__.py | 6 - 25 files changed, 1026 insertions(+), 140 deletions(-) delete mode 100644 sanctions/apps/sanctions/management/commands/hit_opsgenie_heartbeat.py create mode 100644 sanctions/apps/sanctions/management/commands/populate_sdn_fallback_data_and_metadata.py create mode 100644 sanctions/apps/sanctions/management/commands/tests/__init__.py create mode 100644 sanctions/apps/sanctions/management/commands/tests/test_download_sdn_fallback.py create mode 100644 sanctions/apps/sanctions/migrations/0002_rename_fallback_models.py create mode 100644 sanctions/apps/sanctions/tests/factories.py create mode 100644 sanctions/apps/sanctions/tests/test_models.py diff --git a/.annotation_safe_list.yml b/.annotation_safe_list.yml index 07466e9..8bc221e 100644 --- a/.annotation_safe_list.yml +++ b/.annotation_safe_list.yml @@ -19,13 +19,13 @@ sanctions.SanctionsCheckFailure: ".. no_pii:": "This model has no PII" sanctions.HistoricalSanctionsCheckFailure: ".. no_pii:": "This model has no PII" -sanctions.SanctionsFallbackMetadata: +sanctions.SDNFallbackMetadata: ".. no_pii:": "This model has no PII" -sanctions.HistoricalSanctionsFallbackMetadata: +sanctions.HistoricalSDNFallbackMetadata: ".. no_pii:": "This model has no PII" -sanctions.SanctionsFallbackData: +sanctions.SDNFallbackData: ".. no_pii:": "This model has no PII" -sanctions.HistoricalSanctionsFallbackData: +sanctions.HistoricalSDNFallbackData: ".. no_pii:": "This model has no PII" sessions.Session: ".. no_pii:": "This model has no PII" diff --git a/requirements/base.in b/requirements/base.in index ac061ea..915c84a 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -15,5 +15,7 @@ edx-drf-extensions edx-rest-api-client mysqlclient opsgenie_sdk +pycountry pytz responses +testfixtures diff --git a/requirements/base.txt b/requirements/base.txt index 1d9d8f9..d2314f8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,7 +14,7 @@ cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via edx-django-utils @@ -32,7 +32,7 @@ defusedxml==0.8.0rc2 # via # python3-openid # social-auth-core -django==3.2.22 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -76,7 +76,7 @@ edx-auth-backends==4.2.0 # via -r requirements/base.in edx-django-release-util==1.3.0 # via -r requirements/base.in -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/base.in # edx-drf-extensions @@ -111,6 +111,8 @@ pbr==5.11.1 # via stevedore psutil==5.9.6 # via edx-django-utils +pycountry==22.3.5 + # via -r requirements/base.in pycparser==2.21 # via cffi pyjwt[crypto]==2.8.0 @@ -149,7 +151,7 @@ requests==2.31.0 # social-auth-core requests-oauthlib==1.3.1 # via social-auth-core -responses==0.23.3 +responses==0.24.0 # via -r requirements/base.in semantic-version==2.10.0 # via edx-drf-extensions @@ -177,8 +179,8 @@ stevedore==5.1.0 # edx-opaque-keys tenacity==8.2.3 # via opsgenie-sdk -types-pyyaml==6.0.12.12 - # via responses +testfixtures==7.2.2 + # via -r requirements/base.in typing-extensions==4.8.0 # via # asgiref diff --git a/requirements/dev.txt b/requirements/dev.txt index 3087023..7ba2158 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -29,7 +29,7 @@ cffi==1.16.0 # pynacl chardet==5.2.0 # via diff-cover -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/validation.txt # requests @@ -87,7 +87,7 @@ distlib==0.3.7 # via # -r requirements/validation.txt # virtualenv -django==3.2.22 +django==3.2.23 # via # -r requirements/validation.txt # django-cors-headers @@ -142,7 +142,7 @@ edx-auth-backends==4.2.0 # via -r requirements/validation.txt edx-django-release-util==1.3.0 # via -r requirements/validation.txt -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/validation.txt # edx-drf-extensions @@ -165,7 +165,7 @@ exceptiongroup==1.1.3 # pytest factory-boy==3.3.0 # via -r requirements/validation.txt -faker==19.12.1 +faker==19.13.0 # via # -r requirements/validation.txt # factory-boy @@ -307,6 +307,8 @@ py==1.11.0 # tox pycodestyle==2.11.1 # via -r requirements/validation.txt +pycountry==22.3.5 + # via -r requirements/validation.txt pycparser==2.21 # via # -r requirements/validation.txt @@ -418,7 +420,7 @@ requests-toolbelt==1.0.0 # via # -r requirements/validation.txt # twine -responses==0.23.3 +responses==0.24.0 # via -r requirements/validation.txt rfc3986==2.0.0 # via @@ -481,6 +483,8 @@ tenacity==8.2.3 # via # -r requirements/validation.txt # opsgenie-sdk +testfixtures==7.2.2 + # via -r requirements/validation.txt text-unidecode==1.3 # via # -r requirements/validation.txt @@ -496,7 +500,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via # -r requirements/validation.txt # pylint @@ -504,10 +508,6 @@ tox==3.28.0 # via -r requirements/validation.txt twine==4.0.2 # via -r requirements/validation.txt -types-pyyaml==6.0.12.12 - # via - # -r requirements/validation.txt - # responses typing-extensions==4.8.0 # via # -r requirements/validation.txt diff --git a/requirements/django.txt b/requirements/django.txt index 5a28da3..d296127 100644 --- a/requirements/django.txt +++ b/requirements/django.txt @@ -1 +1 @@ -django==3.2.22 +django==3.2.23 diff --git a/requirements/doc.txt b/requirements/doc.txt index 2139e1c..1ca9a07 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -35,7 +35,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/test.txt # requests @@ -89,7 +89,7 @@ distlib==0.3.7 # via # -r requirements/test.txt # virtualenv -django==3.2.22 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -148,7 +148,7 @@ edx-auth-backends==4.2.0 # via -r requirements/test.txt edx-django-release-util==1.3.0 # via -r requirements/test.txt -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/test.txt # edx-drf-extensions @@ -169,7 +169,7 @@ exceptiongroup==1.1.3 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==19.12.1 +faker==19.13.0 # via # -r requirements/test.txt # factory-boy @@ -285,6 +285,8 @@ py==1.11.0 # via # -r requirements/test.txt # tox +pycountry==22.3.5 + # via -r requirements/test.txt pycparser==2.21 # via # -r requirements/test.txt @@ -393,7 +395,7 @@ requests-oauthlib==1.3.1 # social-auth-core requests-toolbelt==1.0.0 # via twine -responses==0.23.3 +responses==0.24.0 # via -r requirements/test.txt restructuredtext-lint==1.4.0 # via doc8 @@ -471,6 +473,8 @@ tenacity==8.2.3 # via # -r requirements/test.txt # opsgenie-sdk +testfixtures==7.2.2 + # via -r requirements/test.txt text-unidecode==1.3 # via # -r requirements/test.txt @@ -485,7 +489,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via # -r requirements/test.txt # pylint @@ -495,10 +499,6 @@ tox==3.28.0 # -r requirements/test.txt twine==4.0.2 # via -r requirements/doc.in -types-pyyaml==6.0.12.12 - # via - # -r requirements/test.txt - # responses typing-extensions==4.8.0 # via # -r requirements/test.txt diff --git a/requirements/production.txt b/requirements/production.txt index a4e8358..f65015e 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -18,7 +18,7 @@ cffi==1.16.0 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/base.txt # requests @@ -45,7 +45,7 @@ defusedxml==0.8.0rc2 # -r requirements/base.txt # python3-openid # social-auth-core -django==3.2.22 +django==3.2.23 # via # -r requirements/base.txt # django-cors-headers @@ -90,7 +90,7 @@ edx-auth-backends==4.2.0 # via -r requirements/base.txt edx-django-release-util==1.3.0 # via -r requirements/base.txt -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/base.txt # edx-drf-extensions @@ -154,6 +154,8 @@ psutil==5.9.6 # via # -r requirements/base.txt # edx-django-utils +pycountry==22.3.5 + # via -r requirements/base.txt pycparser==2.21 # via # -r requirements/base.txt @@ -210,7 +212,7 @@ requests-oauthlib==1.3.1 # via # -r requirements/base.txt # social-auth-core -responses==0.23.3 +responses==0.24.0 # via -r requirements/base.txt semantic-version==2.10.0 # via @@ -254,10 +256,8 @@ tenacity==8.2.3 # via # -r requirements/base.txt # opsgenie-sdk -types-pyyaml==6.0.12.12 - # via - # -r requirements/base.txt - # responses +testfixtures==7.2.2 + # via -r requirements/base.txt typing-extensions==4.8.0 # via # -r requirements/base.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index cc43fbc..cc99a50 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -23,7 +23,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/test.txt # requests @@ -77,7 +77,7 @@ distlib==0.3.7 # via # -r requirements/test.txt # virtualenv -django==3.2.22 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -129,7 +129,7 @@ edx-auth-backends==4.2.0 # via -r requirements/test.txt edx-django-release-util==1.3.0 # via -r requirements/test.txt -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/test.txt # edx-drf-extensions @@ -152,7 +152,7 @@ exceptiongroup==1.1.3 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==19.12.1 +faker==19.13.0 # via # -r requirements/test.txt # factory-boy @@ -263,6 +263,8 @@ py==1.11.0 # tox pycodestyle==2.11.1 # via -r requirements/quality.in +pycountry==22.3.5 + # via -r requirements/test.txt pycparser==2.21 # via # -r requirements/test.txt @@ -363,7 +365,7 @@ requests-oauthlib==1.3.1 # social-auth-core requests-toolbelt==1.0.0 # via twine -responses==0.23.3 +responses==0.24.0 # via -r requirements/test.txt rfc3986==2.0.0 # via twine @@ -417,6 +419,8 @@ tenacity==8.2.3 # via # -r requirements/test.txt # opsgenie-sdk +testfixtures==7.2.2 + # via -r requirements/test.txt text-unidecode==1.3 # via # -r requirements/test.txt @@ -428,7 +432,7 @@ tomli==2.0.1 # pylint # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via # -r requirements/test.txt # pylint @@ -438,10 +442,6 @@ tox==3.28.0 # -r requirements/test.txt twine==4.0.2 # via -r requirements/quality.in -types-pyyaml==6.0.12.12 - # via - # -r requirements/test.txt - # responses typing-extensions==4.8.0 # via # -r requirements/test.txt diff --git a/requirements/test.txt b/requirements/test.txt index a53fd57..3ee224f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -22,7 +22,7 @@ cffi==1.16.0 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/base.txt # requests @@ -117,7 +117,7 @@ edx-auth-backends==4.2.0 # via -r requirements/base.txt edx-django-release-util==1.3.0 # via -r requirements/base.txt -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/base.txt # edx-drf-extensions @@ -136,7 +136,7 @@ exceptiongroup==1.1.3 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==19.12.1 +faker==19.13.0 # via # -r requirements/test.in # factory-boy @@ -208,6 +208,8 @@ psutil==5.9.6 # edx-django-utils py==1.11.0 # via tox +pycountry==22.3.5 + # via -r requirements/base.txt pycparser==2.21 # via # -r requirements/base.txt @@ -287,7 +289,7 @@ requests-oauthlib==1.3.1 # via # -r requirements/base.txt # social-auth-core -responses==0.23.3 +responses==0.24.0 # via -r requirements/base.txt semantic-version==2.10.0 # via @@ -333,6 +335,8 @@ tenacity==8.2.3 # via # -r requirements/base.txt # opsgenie-sdk +testfixtures==7.2.2 + # via -r requirements/base.txt text-unidecode==1.3 # via python-slugify tomli==2.0.1 @@ -341,16 +345,12 @@ tomli==2.0.1 # pylint # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via pylint tox==3.28.0 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.in -types-pyyaml==6.0.12.12 - # via - # -r requirements/base.txt - # responses typing-extensions==4.8.0 # via # -r requirements/base.txt diff --git a/requirements/validation.txt b/requirements/validation.txt index 32cfb4c..8496be8 100644 --- a/requirements/validation.txt +++ b/requirements/validation.txt @@ -27,7 +27,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/quality.txt # -r requirements/test.txt @@ -94,7 +94,7 @@ distlib==0.3.7 # -r requirements/quality.txt # -r requirements/test.txt # virtualenv -django==3.2.22 +django==3.2.23 # via # -r requirements/quality.txt # -r requirements/test.txt @@ -164,7 +164,7 @@ edx-django-release-util==1.3.0 # via # -r requirements/quality.txt # -r requirements/test.txt -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/quality.txt # -r requirements/test.txt @@ -196,7 +196,7 @@ factory-boy==3.3.0 # via # -r requirements/quality.txt # -r requirements/test.txt -faker==19.12.1 +faker==19.13.0 # via # -r requirements/quality.txt # -r requirements/test.txt @@ -348,6 +348,10 @@ py==1.11.0 # tox pycodestyle==2.11.1 # via -r requirements/quality.txt +pycountry==22.3.5 + # via + # -r requirements/quality.txt + # -r requirements/test.txt pycparser==2.21 # via # -r requirements/quality.txt @@ -473,7 +477,7 @@ requests-toolbelt==1.0.0 # via # -r requirements/quality.txt # twine -responses==0.23.3 +responses==0.24.0 # via # -r requirements/quality.txt # -r requirements/test.txt @@ -546,6 +550,10 @@ tenacity==8.2.3 # -r requirements/quality.txt # -r requirements/test.txt # opsgenie-sdk +testfixtures==7.2.2 + # via + # -r requirements/quality.txt + # -r requirements/test.txt text-unidecode==1.3 # via # -r requirements/quality.txt @@ -559,7 +567,7 @@ tomli==2.0.1 # pylint # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via # -r requirements/quality.txt # -r requirements/test.txt @@ -570,11 +578,6 @@ tox==3.28.0 # -r requirements/test.txt twine==4.0.2 # via -r requirements/quality.txt -types-pyyaml==6.0.12.12 - # via - # -r requirements/quality.txt - # -r requirements/test.txt - # responses typing-extensions==4.8.0 # via # -r requirements/quality.txt diff --git a/sanctions/apps/api/v1/tests/test_views.py b/sanctions/apps/api/v1/tests/test_views.py index b533ee4..bb6b74c 100644 --- a/sanctions/apps/api/v1/tests/test_views.py +++ b/sanctions/apps/api/v1/tests/test_views.py @@ -4,6 +4,7 @@ import json from unittest import mock +from requests.exceptions import HTTPError from rest_framework.reverse import reverse from test_utils import APITest @@ -21,28 +22,42 @@ def setUp(self): 'city': 'Jedi Temple', 'country': 'SW', } - self.token = self.generate_jwt_token_header(self.user) def test_sdn_check_missing_args(self): - response = self.client.post(self.url, HTTP_AUTHORIZATION=self.token) + self.set_jwt_cookie(self.user.id) + response = self.client.post(self.url) assert response.status_code == 400 - # TODO: add test for test_sdn_check_search_fails_uses_fallback + @mock.patch('sanctions.apps.api.v1.views.checkSDNFallback') + @mock.patch('sanctions.apps.api_client.sdn_client.SDNClient.search') + def test_sdn_check_search_fails_uses_fallback(self, mock_search, mock_fallback): + mock_search.side_effect = [HTTPError] + mock_fallback.return_value = 0 + self.set_jwt_cookie(self.user.id) + response = self.client.post( + self.url, + content_type='application/json', + data=json.dumps(self.post_data) + ) + assert response.status_code == 200 + assert response.json()['hit_count'] == 0 + assert mock_fallback.is_called() + @mock.patch('sanctions.apps.api.v1.views.checkSDNFallback') @mock.patch('sanctions.apps.api_client.sdn_client.SDNClient.search') def test_sdn_check_search_succeeds( self, - mock_search + mock_search, + mock_fallback ): mock_search.return_value = {'total': 4} self.set_jwt_cookie(self.user.id) response = self.client.post( self.url, content_type='application/json', - HTTP_AUTHORIZATION=self.token, data=json.dumps(self.post_data) ) assert response.status_code == 200 assert response.json()['hit_count'] == 4 assert response.json()['sdn_response'] == {'total': 4} - # TODO: add mock_fallback.assert_not_called() + mock_fallback.assert_not_called() diff --git a/sanctions/apps/api/v1/views.py b/sanctions/apps/api/v1/views.py index 6b70d4f..26dbf81 100644 --- a/sanctions/apps/api/v1/views.py +++ b/sanctions/apps/api/v1/views.py @@ -10,6 +10,7 @@ from rest_framework import permissions, views from sanctions.apps.api_client.sdn_client import SDNClient +from sanctions.apps.sanctions.utils import checkSDNFallback logger = logging.getLogger(__name__) @@ -62,18 +63,17 @@ def post(self, request): except (HTTPError, Timeout) as e: logger.info( 'SDNCheckView: SDN API call received an error: %s.' - ' Calling SanctionsFallback function for user %s.', + ' Calling sanctions checkSDNFallback function for user %s.', str(e), lms_user_id ) - # Temp: return 400 until the SDN Fallback check logic is implemented. - json_data = { - 'error': e, - } - return JsonResponse(json_data, status=400) - - # TODO: add SDN fallback check to determine hit count. + sdn_fallback_hit_count = checkSDNFallback( + full_name, + city, + country + ) + response = {'total': sdn_fallback_hit_count} hit_count = response['total'] if hit_count > 0: diff --git a/sanctions/apps/api_client/tests/test_sdn_client.py b/sanctions/apps/api_client/tests/test_sdn_client.py index cc87fee..8cf20ff 100644 --- a/sanctions/apps/api_client/tests/test_sdn_client.py +++ b/sanctions/apps/api_client/tests/test_sdn_client.py @@ -25,7 +25,7 @@ def setUp(self): self.lms_user_id = 123 self.sdn_api_url = 'http://sdn-test.fake/' self.sdn_api_key = 'fake-key' - self.sdn_api_list = 'SDN, ISN' + self.sdn_api_list = 'ISN,SDN' self.sdn_api_client = SDNClient( self.sdn_api_url, @@ -67,8 +67,8 @@ def test_sdn_search_timeout(self): self.mock_sdn_api_response(Timeout) with self.assertRaises(Timeout): with mock.patch('sanctions.apps.api_client.sdn_client.logger.exception') as mock_logger: - response = self.sdn_api_client.search(self.lms_user_id, self.name, self.city, self.country) - self.assertTrue(mock_logger.called) + self.sdn_api_client.search(self.lms_user_id, self.name, self.city, self.country) + assert mock_logger.is_called() @responses.activate def test_sdn_search_failure(self): @@ -79,8 +79,8 @@ def test_sdn_search_failure(self): with self.assertRaises(HTTPError): with mock.patch('sanctions.apps.api_client.sdn_client.logger.exception') as mock_logger: response = self.sdn_api_client.search(self.lms_user_id, self.name, self.city, self.country) - self.assertTrue(mock_logger.called) - self.assertEqual(response.status_code, 400) + assert mock_logger.is_called() + assert response.status_code == 400 @responses.activate def test_sdn_search_success(self): @@ -90,4 +90,4 @@ def test_sdn_search_success(self): sdn_response = {'total': 1} self.mock_sdn_api_response(json.dumps(sdn_response), status_code=200) response = self.sdn_api_client.search(self.lms_user_id, self.name, self.city, self.country) - self.assertEqual(response, sdn_response) + assert response == sdn_response diff --git a/sanctions/apps/core/tests/factories.py b/sanctions/apps/core/tests/factories.py index 7045783..594f306 100644 --- a/sanctions/apps/core/tests/factories.py +++ b/sanctions/apps/core/tests/factories.py @@ -5,14 +5,11 @@ import logging import factory -from faker import Faker from sanctions.apps.core.models import User USER_PASSWORD = 'password' -FAKER = Faker() - # Silence faker locale warnings logging.getLogger("faker").setLevel(logging.ERROR) diff --git a/sanctions/apps/sanctions/management/commands/hit_opsgenie_heartbeat.py b/sanctions/apps/sanctions/management/commands/hit_opsgenie_heartbeat.py deleted file mode 100644 index f812eca..0000000 --- a/sanctions/apps/sanctions/management/commands/hit_opsgenie_heartbeat.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Dummy management command to prove we can hit an opsgenie heart beat from a cronjob -""" -import logging - -import opsgenie_sdk -from django.conf import settings -from django.core.management.base import BaseCommand - -logger = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Dummy management command to prove we can hit an opsgenie heart beat from a cronjob - """ - help = ( - 'Dummy management command to prove we can hit an opsgenie heart beat from a cronjob' - ) - - def handle(self, *args, **options): - og_sdk_config = opsgenie_sdk.configuration.Configuration() - og_sdk_config.api_key['Authorization'] = settings.OPSGENIE_API_KEY - og_api_client = opsgenie_sdk.api_client.ApiClient(configuration=og_sdk_config) - og_heartbeat_api = opsgenie_sdk.HeartbeatApi(api_client=og_api_client) - - heartbeat_name = 'sanctions-sdn-fallback-job' - - logger.info(f'Calling opsgenie heartbeat for {heartbeat_name}') - response = og_heartbeat_api.ping(heartbeat_name) - logger.info(f'request id: {response.request_id}') - logger.info(f'took: {response.took}') - logger.info(f'result: {response.result}') diff --git a/sanctions/apps/sanctions/management/commands/populate_sdn_fallback_data_and_metadata.py b/sanctions/apps/sanctions/management/commands/populate_sdn_fallback_data_and_metadata.py new file mode 100644 index 0000000..8d6c402 --- /dev/null +++ b/sanctions/apps/sanctions/management/commands/populate_sdn_fallback_data_and_metadata.py @@ -0,0 +1,101 @@ +""" +Django management command to download SDN CSV for use as fallback if the trade.gov SDN API is down. +""" +import logging +import tempfile + +import opsgenie_sdk +import requests +from django.conf import settings +from django.core.management.base import BaseCommand +from django.db import transaction +from requests.exceptions import Timeout + +from sanctions.apps.sanctions.utils import populate_sdn_fallback_data_and_metadata + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Command to download the SDN CSV to be saved as a fallback. Runs every 15 minutes. + """ + help = 'Download the SDN CSV from trade.gov, for use as fallback for when their SDN API is down.' + + def add_arguments(self, parser): + parser.add_argument( + '--threshold', + metavar='N', + action='store', + type=float, + default=3, # typical size is > 4 MB; 3 MB would be unexpectedly low + help='File size MB threshold, under which we will not import it. Use default if argument not specified' + ) + + def _hit_opsgenie_heartbeat(self): + """ + Hit OpsGenie heartbeat to indicate that the fallback job has run successfully recently. + """ + og_sdk_config = opsgenie_sdk.configuration.Configuration() + og_sdk_config.api_key['Authorization'] = settings.OPSGENIE_API_KEY + og_api_client = opsgenie_sdk.api_client.ApiClient(configuration=og_sdk_config) + og_heartbeat_api = opsgenie_sdk.HeartbeatApi(api_client=og_api_client) + + heartbeat_name = settings.OPSGENIE_HEARTBEAT_NAME + + logger.info(f'Calling opsgenie heartbeat for {heartbeat_name}') + response = og_heartbeat_api.ping(heartbeat_name) + logger.info(f'request id: {response.request_id}') + logger.info(f'took: {response.took}') + logger.info(f'result: {response.result}') + + def handle(self, *args, **options): + # download the CSV locally, to check size and pass along to import + threshold = options['threshold'] + url = settings.SDN_CSL_LIST_URL + timeout = settings.SDN_CHECK_REQUEST_TIMEOUT + + with requests.Session() as s: + try: + download = s.get(url, timeout=timeout) + status_code = download.status_code + except Timeout: + logger.warning( + "Sanctions SDNFallback: DOWNLOAD FAILURE: Timeout occurred trying to download SDN CSV. " + "Timeout threshold (in seconds): %s", timeout) + raise + except Exception as e: + logger.exception("Sanctions SDNFallback: DOWNLOAD FAILURE: Exception occurred: [%s]", e) + raise + + if download.status_code != 200: + logger.warning("Sanctions SDNFallback: DOWNLOAD FAILURE: Status code was: [%s]", status_code) + raise Exception("CSV download url got an unsuccessful response code: ", status_code) + + with tempfile.TemporaryFile() as temp_csv: + temp_csv.write(download.content) + file_size_in_bytes = temp_csv.tell() # get current position in the file (number of bytes) + file_size_in_MB = file_size_in_bytes / 10**6 + + if file_size_in_MB > threshold: + sdn_file_string = download.content.decode('utf-8') + with transaction.atomic(): + metadata_entry = populate_sdn_fallback_data_and_metadata(sdn_file_string) + if metadata_entry: + logger.info( + 'Sanctions SDNFallback: IMPORT SUCCESS: Imported SDN CSV. Metadata id %s', + metadata_entry.id) + + logger.info('Sanctions SDNFallback: DOWNLOAD SUCCESS: Successfully downloaded the SDN CSV.') + self.stdout.write( + self.style.SUCCESS( + "Sanctions SDNFallback: Imported SDN CSV into the SDNFallbackMetadata" + " and SDNFallbackData models." + ) + ) + self._hit_opsgenie_heartbeat() + else: + logger.warning( + "Sanctions SDNFallback: DOWNLOAD FAILURE: file too small! " + "(%f MB vs threshold of %s MB)", file_size_in_MB, threshold) + raise Exception("CSV file download did not meet threshold given") diff --git a/sanctions/apps/sanctions/management/commands/tests/__init__.py b/sanctions/apps/sanctions/management/commands/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sanctions/apps/sanctions/management/commands/tests/test_download_sdn_fallback.py b/sanctions/apps/sanctions/management/commands/tests/test_download_sdn_fallback.py new file mode 100644 index 0000000..6509f50 --- /dev/null +++ b/sanctions/apps/sanctions/management/commands/tests/test_download_sdn_fallback.py @@ -0,0 +1,158 @@ +""" +Tests for Django management command to download CSV for SDN Fallback. +""" +from unittest import mock + +import requests +import responses +from django.core.management import call_command +from django.test import TestCase +from mock import patch +from testfixtures import LogCapture, StringComparison + + +class TestDownloadSDNFallbackCommand(TestCase): + """ + Tests for populate_sdn_fallback_data_and_metadata management command. + """ + + LOGGER_NAME = 'sanctions.apps.sanctions.management.commands.populate_sdn_fallback_data_and_metadata' + + def setUp(self): + class TestResponse: + def __init__(self, **kwargs): + self.__dict__ = kwargs + + # mock response for CSV download: just one row of the CSV + self.test_response = TestResponse(**{ + 'content': bytes( + '_id,source,entity_number,type,programs,name,title,addresses,federal_register_notice,start_date,' + 'end_date,standard_order,license_requirement,license_policy,call_sign,vessel_type,gross_tonnage,' + 'gross_registered_tonnage,vessel_flag,vessel_owner,remarks,source_list_url,alt_names,citizenships,' + 'dates_of_birth,nationalities,places_of_birth,source_information_url,' + 'ids\ne5a9eff64cec4a74ed5e9e93c2d851dc2d9132d2,Denied Persons List (DPL) - Bureau of Industry and ' + 'Security,,,, MICKEY MOUSE,,"123 S. TEST DRIVE, SCOTTSDALE, AZ, 85251",' + '82 F.R. 48792 10/01/2017,2017-10-18,2020-10-15,Y,,,,,,,,,FR NOTICE ADDED,' + 'http://bit.ly/1Qi5heF,,,,,,http://bit.ly/1iwxiF0', 'utf-8' + ), + 'status_code': 200, + }) + + self.test_response_500 = TestResponse(**{ + 'status_code': 500, + }) + + @patch('requests.Session.get') + def test_handle_pass(self, mock_response): + """ + Test using mock response from setup, using threshold it will clear. + """ + with mock.patch( + 'sanctions.apps.sanctions.management.commands.' + 'populate_sdn_fallback_data_and_metadata.Command._hit_opsgenie_heartbeat' + ) as mock_og_heartbeat: + mock_response.return_value = self.test_response + + with LogCapture(self.LOGGER_NAME) as log: + call_command('populate_sdn_fallback_data_and_metadata', '--threshold=0.0001') + + log.check( + ( + self.LOGGER_NAME, + 'INFO', + StringComparison( + r'(?s)Sanctions SDNFallback: IMPORT SUCCESS: Imported SDN CSV\. Metadata id.*') + ), + ( + self.LOGGER_NAME, + 'INFO', + "Sanctions SDNFallback: DOWNLOAD SUCCESS: Successfully downloaded the SDN CSV." + ) + ) + + assert mock_og_heartbeat.is_called() + + @patch('requests.Session.get') + def test_handle_fail_size(self, mock_response): + """ + Test using mock response from setup, using threshold it will NOT clear. + """ + mock_response.return_value = self.test_response + + with LogCapture(self.LOGGER_NAME) as log: + with self.assertRaises(Exception) as e: + call_command('populate_sdn_fallback_data_and_metadata', '--threshold=1') + + log.check( + ( + self.LOGGER_NAME, + 'WARNING', + "Sanctions SDNFallback: DOWNLOAD FAILURE: file too small! " + "(0.000642 MB vs threshold of 1.0 MB)" + ) + ) + assert 'CSV file download did not meet threshold given' == str(e.exception) + + @patch('requests.Session.get') + def test_handle_500_response(self, mock_response): + """ Test using url for 500 error. """ + mock_response.return_value = self.test_response_500 + with LogCapture(self.LOGGER_NAME) as log: + with self.assertRaises(Exception) as e: + call_command('populate_sdn_fallback_data_and_metadata', '--threshold=1') + + log.check( + ( + self.LOGGER_NAME, + 'WARNING', + "Sanctions SDNFallback: DOWNLOAD FAILURE: Status code was: [500]" + ) + ) + assert "('CSV download url got an unsuccessful response code: ', 500)" == str(e.exception) + + +class TestDownloadSDNFallbackCommandExceptions(TestCase): + """ + Tests for exceptions in populate_sdn_fallback_data_and_metadata management command. + """ + + LOGGER_NAME = 'sanctions.apps.sanctions.management.commands.populate_sdn_fallback_data_and_metadata' + URL = 'https://data.trade.gov/downloadable_consolidated_screening_list/v1/consolidated.csv' + ERROR_MESSAGE = 'some foo error' + + @responses.activate + def test_general_exception(self): + responses.add(responses.GET, self.URL, body=Exception(self.ERROR_MESSAGE)) + + with LogCapture(self.LOGGER_NAME) as log: + with self.assertRaises(Exception) as e: + call_command('populate_sdn_fallback_data_and_metadata') + + log.check( + ( + self.LOGGER_NAME, + 'ERROR', + "Sanctions SDNFallback: DOWNLOAD FAILURE: Exception occurred: [%s]" % self.ERROR_MESSAGE + ) + ) + + assert self.ERROR_MESSAGE == str(e.exception) + + @responses.activate + def test_timeout_exception(self): + responses.add(responses.GET, self.URL, body=requests.exceptions.ConnectTimeout(self.ERROR_MESSAGE)) + + with LogCapture(self.LOGGER_NAME) as log: + with self.assertRaises(Exception) as e: + call_command('populate_sdn_fallback_data_and_metadata') + + log.check( + ( + self.LOGGER_NAME, + 'WARNING', + "Sanctions SDNFallback: DOWNLOAD FAILURE: Timeout occurred trying to download SDN CSV. " + "Timeout threshold (in seconds): 5" + ) + ) + + assert self.ERROR_MESSAGE == str(e.exception) diff --git a/sanctions/apps/sanctions/migrations/0002_rename_fallback_models.py b/sanctions/apps/sanctions/migrations/0002_rename_fallback_models.py new file mode 100644 index 0000000..eb31ef5 --- /dev/null +++ b/sanctions/apps/sanctions/migrations/0002_rename_fallback_models.py @@ -0,0 +1,49 @@ +# Generated by Django 3.2.23 on 2023-11-06 20:42 + +from django.conf import settings +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('sanctions', '0001_initial'), + ] + + operations = [ + migrations.RenameModel( + old_name='HistoricalSanctionsFallbackData', + new_name='HistoricalSDNFallbackData', + ), + migrations.RenameModel( + old_name='HistoricalSanctionsFallbackMetadata', + new_name='HistoricalSDNFallbackMetadata', + ), + migrations.RenameModel( + old_name='SanctionsFallbackData', + new_name='SDNFallbackData', + ), + migrations.RenameModel( + old_name='SanctionsFallbackMetadata', + new_name='SDNFallbackMetadata', + ), + migrations.AlterModelOptions( + name='historicalsdnfallbackdata', + options={'get_latest_by': 'history_date', 'ordering': ('-history_date', '-history_id'), 'verbose_name': 'historical sdn fallback data'}, + ), + migrations.AlterModelOptions( + name='historicalsdnfallbackmetadata', + options={'get_latest_by': 'history_date', 'ordering': ('-history_date', '-history_id'), 'verbose_name': 'historical sdn fallback metadata'}, + ), + migrations.RenameField( + model_name='historicalsdnfallbackdata', + old_name='sanctions_fallback_metadata', + new_name='sdn_fallback_metadata', + ), + migrations.RenameField( + model_name='sdnfallbackdata', + old_name='sanctions_fallback_metadata', + new_name='sdn_fallback_metadata', + ), + ] diff --git a/sanctions/apps/sanctions/models.py b/sanctions/apps/sanctions/models.py index ea06cd5..632e1ca 100644 --- a/sanctions/apps/sanctions/models.py +++ b/sanctions/apps/sanctions/models.py @@ -1,12 +1,18 @@ """ Models for the sanctions app """ +import logging +from datetime import datetime + from django.core.validators import MinLengthValidator from django.db import models +from django.db.transaction import atomic from django.utils.translation import gettext_lazy as _ from django_extensions.db.models import TimeStampedModel from simple_history.models import HistoricalRecords +logger = logging.getLogger(__name__) + class SanctionsCheckFailure(TimeStampedModel): """ @@ -29,7 +35,7 @@ class SanctionsCheckFailure(TimeStampedModel): username='UnusualSuspect', city='Boston', country='US', - sanctions_type='SDN', + sanctions_type='ISN,SDN', system_identifier='commerce-coordinator', metadata={'order_identifer': 'EDX-123456', 'purchase_type': 'program', 'order_total': '989.00'}, sdn_check_response={'description': 'Looks a bit suspicious.'}, @@ -56,7 +62,7 @@ def __str__(self): ) -class SanctionsFallbackMetadata(TimeStampedModel): +class SDNFallbackMetadata(TimeStampedModel): """ Record metadata about the SDN fallback CSV file download. This table is used to track the state of the SDN CSV file data that are currently @@ -82,14 +88,103 @@ class SanctionsFallbackMetadata(TimeStampedModel): default='New', ) - -class SanctionsFallbackData(models.Model): + @classmethod + def insert_new_sdn_fallback_metadata_entry(cls, file_checksum): + """ + Insert a new SDNFallbackMetadata entry if the new CSV differs from the current one. + If there is no current metadata entry, create a new one and log a warning. + + Args: + file_checksum (str): Hash of the CSV content + + Returns: + sdn_fallback_metadata_entry (SDNFallbackMetadata): Instance of the current SDNFallbackMetadata class + or None if none exists + """ + now = datetime.utcnow() + try: + if file_checksum == SDNFallbackMetadata.objects.get(import_state='Current').file_checksum: + logger.info( + "Sanctions SDNFallback: The CSV file has not changed, so skipping import. The file_checksum was %s", + file_checksum) + # Update download timestamp even though we're not importing this list + SDNFallbackMetadata.objects.filter(import_state="New").update(download_timestamp=now) + return None + except SDNFallbackMetadata.DoesNotExist: + logger.warning("Sanctions SDNFallback: SDNFallbackMetadata has no record with import_state Current") + + sdn_fallback_metadata_entry = SDNFallbackMetadata.objects.create( + file_checksum=file_checksum, + download_timestamp=now, + ) + return sdn_fallback_metadata_entry + + @classmethod + @atomic + def swap_all_states(cls): + """ + Shifts all of the existing metadata table rows to the next import_state + in the row's lifecycle (see _swap_state). + + This method is done in a transaction to gurantee that existing metadata rows are + shifted into their next states in sync and tries to ensure that there is always a row + in the 'Current' state. Rollbacks of all row's import_state changes will happen if: + 1) There are multiple rows & none of them are 'Current', or + 2) There are any issues with the existing rows + updating them (e.g. a row with a + duplicate import_state is manually inserted into the table during the transaction) + """ + SDNFallbackMetadata._swap_state('Discard') + SDNFallbackMetadata._swap_state('Current') + SDNFallbackMetadata._swap_state('New') + + # After the above swaps happen: + # If there are 0 rows in the table, there cannot be a row in the 'Current' status. + # If there is 1 row in the table, it is expected to be in the 'Current' status + # (e.g. when the first file is added + just swapped). + # If there are 2 rows in the table, after the swaps, we expect to have one row in + # the 'Current' status and one row in the 'Discard' status. + if len(SDNFallbackMetadata.objects.all()) >= 1: + try: + SDNFallbackMetadata.objects.get(import_state='Current') + except SDNFallbackMetadata.DoesNotExist: + logger.warning( + "Sanctions SDNFallback: Expected a row in the 'Current' import_state after swapping," + " but there are none.", + ) + raise + + @classmethod + def _swap_state(cls, import_state): + """ + Update the row in the given import_state parameter to the next import_state. + Rows in this table should progress from New -> Current -> Discard -> (row deleted). + There can be at most one row in each import_state at a given time. + """ + try: + existing_metadata = SDNFallbackMetadata.objects.get(import_state=import_state) + if import_state == 'Discard': + existing_metadata.delete() + else: + if import_state == 'New': + existing_metadata.import_state = 'Current' + elif import_state == 'Current': + existing_metadata.import_state = 'Discard' + existing_metadata.full_clean() + existing_metadata.save() + except SDNFallbackMetadata.DoesNotExist: + logger.info( + "Sanctions SDNFallback: Cannot update import_state of %s row if there is no row in this state.", + import_state + ) + + +class SDNFallbackData(models.Model): """ - Model used to record and process one row received from SanctionsFallbackMetadata. + Model used to record and process one row received from SDNFallbackMetadata. Fields: - sanctions_fallback_metadata (ForeignKey): Foreign Key field with the CSV import Primary Key - referenced in SanctionsFallbackMetadata. + sdn_fallback_metadata (ForeignKey): Foreign Key field with the CSV import Primary Key + referenced in SDNFallbackMetadata. source (CharField): Origin of where the data comes from, since the CSV consolidates export screening lists of the Departments of Commerce, State and the Treasury. @@ -111,9 +206,29 @@ class SanctionsFallbackData(models.Model): required field in billing information form, those records would not be matched in the API/fallback. """ history = HistoricalRecords() - sanctions_fallback_metadata = models.ForeignKey(SanctionsFallbackMetadata, on_delete=models.CASCADE) + sdn_fallback_metadata = models.ForeignKey(SDNFallbackMetadata, on_delete=models.CASCADE) source = models.CharField(default='', max_length=255, db_index=True) sdn_type = models.CharField(default='', max_length=255, db_index=True) names = models.TextField(default='') addresses = models.TextField(default='') countries = models.CharField(default='', max_length=255) + + @classmethod + def get_current_records_and_filter_by_source_and_type(cls, source, sdn_type): + """ + Query the records that have 'Current' import state, and filter by source and sdn_type. + """ + try: + current_metadata = SDNFallbackMetadata.objects.get(import_state='Current') + + # The 'get' relies on the manage command having been run. If it fails, tell engineer what's needed + except SDNFallbackMetadata.DoesNotExist as fallback_metadata_no_exist: + logger.warning( + "Sanctions SDNFallback: SDNFallbackMetadata is empty! Run this: " + "./manage.py populate_sdn_fallback_data_and_metadata" + ) + raise Exception( + 'Sanctions SDNFallback empty error when calling checkSDNFallback, data is not yet populated.' + ) from fallback_metadata_no_exist + query_params = {'source': source, 'sdn_fallback_metadata': current_metadata, 'sdn_type': sdn_type} + return SDNFallbackData.objects.filter(**query_params) diff --git a/sanctions/apps/sanctions/tests/factories.py b/sanctions/apps/sanctions/tests/factories.py new file mode 100644 index 0000000..101163f --- /dev/null +++ b/sanctions/apps/sanctions/tests/factories.py @@ -0,0 +1,40 @@ +""" +Factoryboy factories for Sanctions app. +""" +import logging + +import factory +from django.utils import timezone +from faker import Faker + +from sanctions.apps.sanctions.models import SDNFallbackData, SDNFallbackMetadata + +# Silence faker locale warnings +logging.getLogger("faker").setLevel(logging.ERROR) + + +class SDNFallbackMetadataFactory(factory.django.DjangoModelFactory): + """ + Test factory for the `SDNFallbackMetadata` model. + """ + file_checksum = factory.Sequence(lambda n: Faker().md5()) + import_state = 'New' + download_timestamp = timezone.now() - timezone.timedelta(days=10) + + class Meta: + model = SDNFallbackMetadata + + +class SDNFallbackDataFactory(factory.django.DjangoModelFactory): + """ + Test factory for the `SDNFallbackData` model. + """ + sdn_fallback_metadata = factory.SubFactory(SDNFallbackMetadataFactory) + source = "Specially Designated Nationals (SDN) - Treasury Department" + sdn_type = "Individual" + names = factory.Faker('name') + addresses = factory.Faker('address') + countries = factory.Faker('country_code') + + class Meta: + model = SDNFallbackData diff --git a/sanctions/apps/sanctions/tests/test_models.py b/sanctions/apps/sanctions/tests/test_models.py new file mode 100644 index 0000000..9bdbab8 --- /dev/null +++ b/sanctions/apps/sanctions/tests/test_models.py @@ -0,0 +1,257 @@ +""" +Tests for Sanctions models. +""" +from datetime import datetime + +from django.test import TestCase +from testfixtures import LogCapture + +from sanctions.apps.sanctions.models import SanctionsCheckFailure, SDNFallbackData, SDNFallbackMetadata +from sanctions.apps.sanctions.tests.factories import SDNFallbackDataFactory, SDNFallbackMetadataFactory + + +class SanctionsCheckFailureTests(TestCase): + """ + Tests for SanctionsCheckFailure model. + """ + def setUp(self): + super().setUp() + self.full_name = 'Keyser Söze' + self.username = 'UnusualSuspect' + self.country = 'US' + self.sdn_check_response = {'description': 'Looks a bit suspicious.'} + + def test_unicode(self): + """ Verify the __unicode__ method returns the correct value. """ + hit = SanctionsCheckFailure.objects.create( + full_name=self.full_name, + username=self.username, + country=self.country, + sanctions_type='ISN,SDN', + system_identifier='commerce-coordinator', + metadata={'order_identifer': 'EDX-123456', 'purchase_type': 'program', 'order_total': '989.00'}, + sdn_check_response=self.sdn_check_response + ) + expected = 'Sanctions check failure [{username}]'.format( + username=self.username + ) + + self.assertEqual(str(hit), expected) + + +class SDNFallbackMetadataTests(TestCase): + """ + Tests for SDNFallbackMetadata model. + """ + LOGGER_NAME = 'sanctions.apps.sanctions.models' + + def test_minimum_requirements(self): + """Make sure the row is created correctly with the minimum dataset + defaults.""" + new_metadata = SDNFallbackMetadata( + file_checksum="foobar", + download_timestamp=datetime.now(), + ) + new_metadata.full_clean() + new_metadata.save() + + self.assertEqual(len(SDNFallbackMetadata.objects.all()), 1) + + actual_metadata = SDNFallbackMetadata.objects.all()[0] + self.assertEqual(actual_metadata.file_checksum, "foobar") + self.assertIsInstance(actual_metadata.download_timestamp, datetime) + self.assertEqual(actual_metadata.import_timestamp, None) + self.assertEqual(actual_metadata.import_state, 'New') + self.assertIsInstance(actual_metadata.created, datetime) + self.assertIsInstance(actual_metadata.modified, datetime) + + def test_swap_new_row(self): + """Swap New row to Current row.""" + SDNFallbackMetadataFactory.create(import_state='New') + + SDNFallbackMetadata.swap_all_states() + + actual_rows = SDNFallbackMetadata.objects.all() + self.assertEqual(len(actual_rows), 1) + self.assertEqual(actual_rows[0].import_state, 'Current') + + def test_swap_current_row(self): + """Swap Current row to Discard row.""" + original = SDNFallbackMetadataFactory.create(import_state="Current") + # this is needed to bypass the requirement to always have a 'Current' + SDNFallbackMetadataFactory.create(import_state="New") + + SDNFallbackMetadata.swap_all_states() + + actual_row = SDNFallbackMetadata.objects.filter(file_checksum=original.file_checksum)[0] + self.assertEqual(actual_row.import_state, 'Discard') + + def test_swap_discard_row(self): + """Discard row gets deleted when swapping rows.""" + SDNFallbackMetadataFactory.create(import_state="Discard") + + SDNFallbackMetadata.swap_all_states() + + actual_rows = SDNFallbackMetadata.objects.all() + self.assertEqual(len(actual_rows), 0) + + def test_swap_twice_one_row(self): + """Swapping one row twice without adding a new file should result in an error.""" + original = SDNFallbackMetadataFactory.create(import_state="New") + expected_logs = [ + ( + self.LOGGER_NAME, + 'WARNING', + "Expected a row in the 'Current' import_state after swapping, but there are none." + ), + ] + + SDNFallbackMetadata.swap_all_states() + with self.assertRaises(SDNFallbackMetadata.DoesNotExist): + with LogCapture(self.LOGGER_NAME) as log: + SDNFallbackMetadata.swap_all_states() + log.check_present(*expected_logs) + + self.assertEqual(len(SDNFallbackMetadata.objects.all()), 1) + former_new_metadata = SDNFallbackMetadata.objects.filter(file_checksum=original.file_checksum)[0] + self.assertEqual(former_new_metadata.import_state, 'Current') + + def test_swap_all_non_existent_rows(self): + """Swapping all shouldn't break / do anything if there are no existing rows.""" + + SDNFallbackMetadata.swap_all_states() + self.assertEqual(len(SDNFallbackMetadata.objects.all()), 0) + + def test_swap_all_to_use_new_metadata_row(self): + """ + Test what happens when we want to set the 'New' row to the 'Current' row in a + normal scenario (e.g. when rows exist in all three import_states). + """ + original_new = SDNFallbackMetadataFactory.create(import_state="New") + original_current = SDNFallbackMetadataFactory.create(import_state="Current") + original_discard = SDNFallbackMetadataFactory.create(import_state="Discard") + + SDNFallbackMetadata.swap_all_states() + + self.assertEqual(len(SDNFallbackMetadata.objects.all()), 2) + + former_new_metadata = SDNFallbackMetadata.objects.filter( + file_checksum=original_new.file_checksum)[0] + self.assertEqual(former_new_metadata.import_state, 'Current') + former_current_metadata = SDNFallbackMetadata.objects.filter( + file_checksum=original_current.file_checksum)[0] + self.assertEqual(former_current_metadata.import_state, 'Discard') + former_discard_metadata = SDNFallbackMetadata.objects.filter( + file_checksum=original_discard.file_checksum) + self.assertEqual(len(former_discard_metadata), 0) + + def test_swap_all_rollback(self): + """ + Make sure that the rollback works if there are issues when swapping all of the rows. + """ + original_current = SDNFallbackMetadataFactory.create(import_state="Current") + original_discard = SDNFallbackMetadataFactory.create(import_state="Discard") + + expected_logs = [ + ( + self.LOGGER_NAME, + 'WARNING', + "Expected a row in the 'Current' import_state after swapping, but there are none." + ), + ] + + with self.assertRaises(SDNFallbackMetadata.DoesNotExist): + with LogCapture(self.LOGGER_NAME) as log: + SDNFallbackMetadata.swap_all_states() + log.check_present(*expected_logs) + + self.assertEqual(len(SDNFallbackMetadata.objects.all()), 2) + former_current_metadata = SDNFallbackMetadata.objects.filter( + file_checksum=original_current.file_checksum)[0] + self.assertEqual(former_current_metadata.import_state, 'Current') + former_discard_metadata = SDNFallbackMetadata.objects.filter( + file_checksum=original_discard.file_checksum)[0] + self.assertEqual(former_discard_metadata.import_state, 'Discard') + + +class SDNFallbackDataTests(TestCase): + """ + Tests for SDNFallbackData model. + """ + def setUp(self): + super().setUp() + self.sdn_metadata = SDNFallbackMetadataFactory.create(import_state="New") + + def test_fields(self): + """ Verify all fields are correctly populated. """ + new_data = SDNFallbackData( + sdn_fallback_metadata=self.sdn_metadata, + source="Specially Designated Nationals (SDN) - Treasury Department", + sdn_type="Individual", + names="maria giuseppe", + addresses="123 main street", + countries="US", + ) + new_data.full_clean() + new_data.save() + + self.assertEqual(len(SDNFallbackData.objects.all()), 1) + + actual_data = SDNFallbackData.objects.all()[0] + self.assertEqual(actual_data.sdn_fallback_metadata, self.sdn_metadata) + self.assertEqual(actual_data.source, "Specially Designated Nationals (SDN) - Treasury Department") + self.assertEqual(actual_data.sdn_type, "Individual") + self.assertEqual(actual_data.names, "maria giuseppe") + self.assertEqual(actual_data.addresses, "123 main street") + self.assertEqual(actual_data.countries, "US") + + def test_data_is_deleted_on_delete_of_metadata(self): + """ Verify SDNFallbackData object is deleted if SDNFallbackMetadata object is removed. """ + SDNFallbackDataFactory.create( + sdn_fallback_metadata=self.sdn_metadata, + ) + + self.assertEqual(len(SDNFallbackData.objects.all()), 1) + self.sdn_metadata.delete() + self.assertEqual(len(SDNFallbackData.objects.all()), 0) + + def test_get_current_records_and_filter_by_source_and_type(self): + """ Verify the query is done for current records by source and by optional sdn_type. """ + sdn_metadata_current = SDNFallbackMetadataFactory.create(import_state="Current") + sdn_metadata_discard = SDNFallbackMetadataFactory.create(import_state="Discard") + sdn_source = "Specially Designated Nationals (SDN) - Treasury Department" + isn_source = "Nonproliferation Sanctions (ISN) - State Department" + sdn_type = "Individual" + + rows = [ + [sdn_source, sdn_type, sdn_metadata_current], + [sdn_source, "Entity", sdn_metadata_current], + [isn_source, "", sdn_metadata_current], + [sdn_source, sdn_type, sdn_metadata_discard], + ] + + for row in rows: + source, sdn_type, sdn_fallback_metadata = row + SDNFallbackDataFactory.create( + sdn_fallback_metadata=sdn_fallback_metadata, + source=source, + sdn_type=sdn_type, + ) + + filtered_records_sdn_individual = SDNFallbackData.get_current_records_and_filter_by_source_and_type( + sdn_source, sdn_type) + self.assertEqual(len(filtered_records_sdn_individual), 1) + self.assertEqual(filtered_records_sdn_individual[0].sdn_fallback_metadata, sdn_metadata_current) + filtered_records_sdn_entity = SDNFallbackData.get_current_records_and_filter_by_source_and_type( + sdn_source, "Entity") + self.assertEqual(len(filtered_records_sdn_entity), 1) + filtered_records_isn = SDNFallbackData.get_current_records_and_filter_by_source_and_type( + isn_source, "") + self.assertEqual(len(filtered_records_isn), 1) + + def test_get_current_records_and_filter_by_source_and_type_empty_data(self): + """ Verify that we raise the expected Exception if this is called before data is populated""" + sdn_source = "Specially Designated Nationals (SDN) - Treasury Department" + sdn_type = "Individual" + + with self.assertRaises(Exception): + SDNFallbackData.get_current_records_and_filter_by_source_and_type(sdn_source, sdn_type) diff --git a/sanctions/apps/sanctions/utils.py b/sanctions/apps/sanctions/utils.py index e2f814d..7b1a6ec 100644 --- a/sanctions/apps/sanctions/utils.py +++ b/sanctions/apps/sanctions/utils.py @@ -1,3 +1,187 @@ """ Helpers for the sanctions app. """ +import csv +import hashlib +import io +import logging +import re +import unicodedata +from datetime import datetime, timezone + +import pycountry + +from sanctions.apps.sanctions.models import SDNFallbackData, SDNFallbackMetadata + +logger = logging.getLogger(__name__) +COUNTRY_CODES = {country.alpha_2 for country in pycountry.countries} + + +def checkSDNFallback(name, city, country): + """ + Performs an SDN check against the SDNFallbackData. + + First, filter the SDNFallbackData records by source, type and country. + Then, compare the provided name/city against each record and return whether we find a match. + The check uses the following properties: + 1. Order of words doesn’t matter + 2. Number of times that a given word appears doesn’t matter + 3. Punctuation between words or at the beginning/end of a given word doesn’t matter + 4. If a subset of words match, it still counts as a match + 5. Capitalization doesn’t matter + """ + hit_count = 0 + records = SDNFallbackData.get_current_records_and_filter_by_source_and_type( + 'Specially Designated Nationals (SDN) - Treasury Department', 'Individual' + ) + records = records.filter(countries__contains=country) + processed_name, processed_city = process_text(name), process_text(city) + for record in records: + record_names, record_addresses = set(record.names.split()), set(record.addresses.split()) + if (processed_name.issubset(record_names) and processed_city.issubset(record_addresses)): + hit_count = hit_count + 1 + return hit_count + + +def transliterate_text(text): + """ + Transliterate unicode characters into ASCII (such as accented characters into non-accented characters). + + This works by decomposing accented characters into the letter and the accent. + + The subsequent ASCII encoding drops any accents and leaves the original letter. + + Returns the original string if no transliteration is available. + + Args: + text (str): a string to be transliterated + + Returns: + text (str): the transliterated string + """ + t11e_text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii') + return t11e_text if t11e_text else text + + +def process_text(text): + """ + Lowercase, remove non-alphanumeric characters, and ignore order and word frequency. + Attempts to transliterate unicode characters into ASCII (such as accented characters into non-accented characters). + + Args: + text (str): names or addresses from the SDN list to be processed + + Returns: + text (set): processed text + """ + if len(text) == 0: + return '' + + # Make lowercase + text = text.casefold() + + # Transliterate numbers and letters + text = ''.join(map(transliterate_text, text)) + + # Ignore punctuation, order, and word frequency + text = set(filter(None, set(re.split(r'[\W_]+', text)))) + + return text + + +def extract_country_information(addresses, ids): + """ + Extract any country codes that are present, if any, in the addresses and ids fields + + Args: + addresses (str): addresses from the csv addresses field + ids (str): ids from the csv ids field + + Returns: + countries (str): Space separated list of alpha_2 country codes present in the addresses and ids fields + """ + country_matches = [] + if addresses: + # Addresses are stored in a '; ' separated format with the country at the end of each address + # We check for two uppercase letters followed by '; ' or at the end of the string + addresses_regex = r'([A-Z]{2})$|([A-Z]{2});' + country_matches += re.findall(addresses_regex, addresses) + if ids: + # Ids are stored in a '; ' separated format with the country at the beginning of each id + # Countries within the id are followed by a comma + # We check for two uppercase letters prefaced by '; ' or at the beginning of a string + # Notes are also stored in this field in sentence case, so checking for two uppercase letters handles this + ids_regex = r'^([A-Z]{2}),|; ([A-Z]{2}),' + country_matches += re.findall(ids_regex, ids) + # country_matches is returned in the following format [('', 'IQ'), ('', 'JO'), ('', 'IQ'), ('', 'TR')] + # We filter out regex groups with no match, deduplicate countries, and convert them to a space separated string + # with the following format 'IQ JO TR' + country_codes = {' '.join(tuple(filter(None, x))) for x in country_matches} + valid_country_codes = COUNTRY_CODES.intersection(country_codes) + formatted_countries = ' '.join(valid_country_codes) + return formatted_countries + + +def populate_sdn_fallback_metadata(sdn_csv_string): + """ + Insert a new SDNFallbackMetadata entry if the new csv differs from the current one + + Args: + sdn_csv_string (bytes): Bytes of the sdn csv + + Returns: + sdn_fallback_metadata_entry (SDNFallbackMetadata): Instance of the current SDNFallbackMetadata class + or None if none exists + """ + file_checksum = hashlib.sha256(sdn_csv_string.encode('utf-8')).hexdigest() + metadata_entry = SDNFallbackMetadata.insert_new_sdn_fallback_metadata_entry(file_checksum) + return metadata_entry + + +def populate_sdn_fallback_data(sdn_csv_string, metadata_entry): + """ + Process CSV data and create SDNFallbackData records + + Args: + sdn_csv_string (str): String of the sdn csv + metadata_entry (SDNFallbackMetadata): Instance of the current SDNFallbackMetadata class + """ + sdn_csv_reader = csv.DictReader(io.StringIO(sdn_csv_string)) + processed_records = [] + for row in sdn_csv_reader: + sdn_source, sdn_type, names, addresses, alt_names, ids = ( + row['source'] or '', row['type'] or '', row['name'] or '', + row['addresses'] or '', row['alt_names'] or '', row['ids'] or '' + ) + processed_names = ' '.join(process_text(' '.join(filter(None, [names, alt_names])))) + processed_addresses = ' '.join(process_text(addresses)) + countries = extract_country_information(addresses, ids) + processed_records.append(SDNFallbackData( + sdn_fallback_metadata=metadata_entry, + source=sdn_source, + sdn_type=sdn_type, + names=processed_names, + addresses=processed_addresses, + countries=countries + )) + # Bulk create should be more efficient for a few thousand records without needing to use SQL directly. + SDNFallbackData.objects.bulk_create(processed_records) + + +def populate_sdn_fallback_data_and_metadata(sdn_csv_string): + """ + 1. Create the SDNFallbackMetadata entry + 2. Populate the SDNFallbackData from the csv + + Args: + sdn_csv_string (str): String of the sdn csv + """ + metadata_entry = populate_sdn_fallback_metadata(sdn_csv_string) + if metadata_entry: + populate_sdn_fallback_data(sdn_csv_string, metadata_entry) + # Once data is successfully imported, update the metadata import timestamp and state + now = datetime.now(timezone.utc) + metadata_entry.import_timestamp = now + metadata_entry.save() + metadata_entry.swap_all_states() + return metadata_entry diff --git a/sanctions/settings/base.py b/sanctions/settings/base.py index 1bf34de..d2e658b 100644 --- a/sanctions/settings/base.py +++ b/sanctions/settings/base.py @@ -97,7 +97,8 @@ def root(*path_fragments): # SDN Check SDN_CHECK_REQUEST_TIMEOUT = 5 # Value is in seconds. - +# Settings to download the government CSL list +SDN_CSL_LIST_URL = "https://data.trade.gov/downloadable_consolidated_screening_list/v1/consolidated.csv" # Settings to check government purchase restriction lists SDN_CHECK_API_URL = "https://data.trade.gov/consolidated_screening_list/v1/search" SDN_CHECK_API_KEY = "replace-me" @@ -259,3 +260,4 @@ def root(*path_fragments): LOGGING = get_logger_config(debug=DEBUG) OPSGENIE_API_KEY = '' +OPSGENIE_HEARTBEAT_NAME = '' diff --git a/test_utils/__init__.py b/test_utils/__init__.py index 3bb6c8c..9b2a20d 100644 --- a/test_utils/__init__.py +++ b/test_utils/__init__.py @@ -71,9 +71,3 @@ def set_jwt_cookie(self, user_id, roles=None): jwt_token = generate_jwt_token(payload) self.client.cookies[jwt_cookie_name()] = jwt_token - - def generate_jwt_token_header(self, user): - """ - Generate a valid JWT token header for authenticated requests. - """ - return "JWT {token}".format(token=generate_jwt(user))