diff --git a/pynsee/utils/init_connection.py b/pynsee/utils/init_connection.py index b777b7b3..d116bd45 100644 --- a/pynsee/utils/init_connection.py +++ b/pynsee/utils/init_connection.py @@ -28,13 +28,19 @@ def init_conn( sirene_key: Optional[str] = None, http_proxy: Optional[str] = None, https_proxy: Optional[str] = None, + raise_if_not_ok: bool = True, ) -> None: """Save your credentials to connect to INSEE APIs, subscribe to api.insee.fr + In case of failure of an API outside your current usecase (ie. failure + of SIREN API server while you are actually interested in metadata) you + can bypass this `init_conn(raise_if_not_ok=False)` + Args: sirene_key (str, optional): user's key for sirene API http_proxy (str, optional): Proxy server address, e.g. 'http://my_proxy_server:port'. Defaults to "". https_proxy (str, optional): Proxy server address, e.g. 'http://my_proxy_server:port'. Defaults to "". + raise_if_not_ok (bool, optional) : If set to True, a RequestException will automatically be raised if the response is not ok (= `status_code` >= 400). The default is True. Notes: Environment variables can be used instead of init_conn function @@ -69,7 +75,9 @@ def init_conn( sirene_key=sirene_key, http_proxy=http_proxy, https_proxy=https_proxy ) as session: try: - invalid_requests = session._test_connections() + invalid_requests = session._test_connections( + raise_if_not_ok=raise_if_not_ok + ) except (ValueError, requests.exceptions.RequestException): try: os.remove(CONFIG_FILE) @@ -80,15 +88,16 @@ def init_conn( if invalid_requests: logger.error( "Invalid credentials, the following APIs returned error codes, " - "please make sure you subscribed to them (if you need those):\n" - f"{invalid_requests}" + "please make sure you subscribed to them (if you need those):\n%s", + invalid_requests, ) else: logger.info( "Subscription to all INSEE's APIs has been successfull\n" "Unless the user wants to change the key or secret, using this " "function is no longer needed as the credentials will be saved " - f"locally here:\n{CONFIG_FILE}" + "locally here:\n%s", + CONFIG_FILE, ) if invalid_requests and ("Sirene" in invalid_requests): diff --git a/pynsee/utils/requests_session.py b/pynsee/utils/requests_session.py index c6d34ad3..9d9ae8d1 100644 --- a/pynsee/utils/requests_session.py +++ b/pynsee/utils/requests_session.py @@ -18,7 +18,6 @@ from pynsee.utils._create_insee_folder import _create_insee_folder from pynsee.constants import SIRENE_KEY, HTTPS_PROXY_KEY, HTTP_PROXY_KEY - logger = logging.getLogger(__name__) @@ -270,7 +269,7 @@ def request( info. The default is (10, 15). raise_if_not_ok : bool, optional If set to True, a RequestException will automatically be raised if - the response is not ok (= `status_code` < 400). + the response is not ok (= `status_code` >= 400). The default is True. **kwargs : Any other kwargs are passed directly to requests.Session.request @@ -397,7 +396,7 @@ def _request_sdmx_insee( URL to query. raise_if_not_ok : bool, optional If set to True, a RequestException will automatically be raised if - the response is not ok (= `status_code` < 400). + the response is not ok (= `status_code` >= 400). The default is True. Raises @@ -438,7 +437,7 @@ def _request_api_insee( INSEE's APIs. The default is "application/xml". raise_if_not_ok : bool, optional If set to True, a RequestException will automatically be raised if - the response is not ok (= `status_code` < 400). + the response is not ok (= `status_code` >= 400). The default is False. print_msg : bool, optional If True, will log critical entries to warn that the call to @@ -528,10 +527,17 @@ def _called_sirene(self, url: str) -> bool: """ return re.match(".*api-sirene.*", url) is not None - def _test_connections(self) -> dict: + def _test_connections(self, raise_if_not_ok: bool = True) -> dict: """ Test the valid connection to each API. + Parameters + ---------- + raise_if_not_ok : bool, optional + If set to True, a RequestException will automatically be raised if + the response is not ok (= `status_code` >= 400). + The default is True. + Raises ------ RequestException @@ -545,6 +551,7 @@ def _test_connections(self) -> dict: Dict of {"api name": response.status_code} for invalid queries. """ + queries = { "BDM": "https://api.insee.fr/series/BDM/dataflow/FR1/all", "Metadata": "https://api.insee.fr/metadonnees/codes/cj/n3/5599", @@ -576,19 +583,29 @@ def _test_connections(self) -> dict: "your proxy configuration " f"- proxies were {self.proxies}." ) from exc - elif exc.response.status_code == 404: - raise requests.exceptions.RequestException( + if exc.response.status_code == 401: + _invalid_sirene_key(raise_error=raise_if_not_ok) + elif exc.response.status_code >= 400: + msg = ( f"Could not reach {api} at {api_url}, the server " - "returned 404 (not found); please get in touch if " - "the issue persists." - ) from exc + "returned {exc.response.status_code}; please get " + f"in touch if the issue persists." + ) + if raise_if_not_ok: + raise requests.exceptions.RequestException( + msg + ) from exc + logger.warning(msg) invalid_requests[api] = exc.response.status_code if len(invalid_requests) == len(queries): - raise ValueError( + msg = ( "No API was reached. That's strange, please get in touch if " "the issue persists." ) + if raise_if_not_ok: + raise ValueError(msg) + logger.warning(msg) return invalid_requests diff --git a/tests/utils/patches.py b/tests/utils/patches.py index 6c223e3c..22bccd20 100644 --- a/tests/utils/patches.py +++ b/tests/utils/patches.py @@ -7,7 +7,6 @@ import pynsee.constants from pynsee.utils.requests_session import PynseeAPISession - config_file = pynsee.constants.CONFIG_FILE config_backup = f"{config_file}.back" @@ -95,7 +94,11 @@ def patch_test_connections(func): def wrapper(*args, **kwargs): init = PynseeAPISession._test_connections - PynseeAPISession._test_connections = lambda x: {} + + def patched_test_connections(self, raise_if_not_ok=False): + return {} + + PynseeAPISession._test_connections = patched_test_connections try: func(*args, **kwargs) except Exception: @@ -114,7 +117,7 @@ def patch_test_connections_and_failure_for_sirene(func): def wrapper(*args, **kwargs): init = PynseeAPISession._request_api_insee - def _request_api_insee(url, *args, **kwargs): + def _request_api_insee(self, url, *args, **kwargs): if re.match(".*api-sirene.*", url): dummy_response = object() dummy_response.status_code = 400 diff --git a/tests/utils/test_pynsee_utils.py b/tests/utils/test_pynsee_utils.py index 5dd56086..a3638247 100644 --- a/tests/utils/test_pynsee_utils.py +++ b/tests/utils/test_pynsee_utils.py @@ -11,6 +11,7 @@ from pynsee.utils.requests_session import PynseeAPISession from pynsee.utils.clear_all_cache import clear_all_cache from pynsee.utils import init_conn +from pynsee.metadata.get_definition import get_definition from .patches import ( clean_os_patch, @@ -66,7 +67,8 @@ def test_dummy_sirene_token_is_not_stored(): with open(pynsee.constants.CONFIG_FILE, "w") as f: json.dump({"DUMMY_SIRENE_KEY": "spam"}, f) - init_conn(sirene_key="eggs") + init_conn(sirene_key="eggs", raise_if_not_ok=False) + get_definition(ids=["c1020", "c1601"]) with open(pynsee.constants.CONFIG_FILE, "r") as f: assert json.load(f)["DUMMY_SIRENE_KEY"] == "spam"