diff --git a/docs/user_guide/measurement_user_guide.rst b/docs/user_guide/measurement_user_guide.rst index 674fdd8..a5c91a1 100644 --- a/docs/user_guide/measurement_user_guide.rst +++ b/docs/user_guide/measurement_user_guide.rst @@ -309,7 +309,7 @@ names with respect to pleasant and unpleasant attributes. ethnicity_query = Query( [word_sets["european_american_names_5"], word_sets["african_american_names_5"]], - [word_sets["pleasant_5"], word_sets["unpleasant_5"]], + [word_sets["pleasant_5"], word_sets["unpleasant_5a"]], ["European american names", "African american names"], ["Pleasant", "Unpleasant"], ) @@ -1265,7 +1265,7 @@ Ethnicity Bias Model Ranking # define the queries ethnicity_query_1 = Query( [word_sets["european_american_names_5"], word_sets["african_american_names_5"]], - [word_sets["pleasant_5"], word_sets["unpleasant_5"]], + [word_sets["pleasant_5"], word_sets["unpleasant_5a"]], ["European Names", "African Names"], ["Pleasant", "Unpleasant"], ) diff --git a/docs/user_guide/mitigation_user_guide.rst b/docs/user_guide/mitigation_user_guide.rst index 885e620..760da80 100644 --- a/docs/user_guide/mitigation_user_guide.rst +++ b/docs/user_guide/mitigation_user_guide.rst @@ -281,7 +281,7 @@ Next, we measure the gender bias exposed by query 2 (Male Names and Female Names gender_query_2 = Query( [weat_wordset["male_names"], weat_wordset["female_names"]], - [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5"]], + [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5a"]], ["Male Names", "Female Names"], ["Pleasant", "Unpleasant"], ) @@ -433,7 +433,7 @@ equalized. gender_query_2 = Query( [weat_wordset["male_names"], weat_wordset["female_names"]], - [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5"]], + [weat_wordset["pleasant_5"], weat_wordset["unpleasant_5a"]], ["Male Names", "Female Names"], ["Pleasant", "Unpleasant"], ) diff --git a/tests/debias/conftest.py b/tests/debias/conftest.py index 80514ac..8b01a4a 100644 --- a/tests/debias/conftest.py +++ b/tests/debias/conftest.py @@ -116,7 +116,7 @@ def gender_query_1(weat_wordsets: dict[str, list[str]]) -> Query: """ query = Query( [weat_wordsets["male_names"], weat_wordsets["female_names"]], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["Male Names", "Female Names"], ["Pleasant", "Unpleasant"], ) @@ -202,9 +202,9 @@ def ethnicity_query_1(weat_wordsets: dict[str, list[str]]) -> Query: weat_wordsets["european_american_names_5"], weat_wordsets["african_american_names_5"], ], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["european_american_names_5", "african_american_names_5"], - ["pleasant_5", "unpleasant_5"], + ["pleasant_5", "unpleasant_5a"], ) return query @@ -226,8 +226,8 @@ def control_query_1(weat_wordsets: dict[str, list[str]]) -> Query: """ query = Query( [weat_wordsets["flowers"], weat_wordsets["insects"]], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["flowers", "insects"], - ["pleasant_5", "unpleasant_5"], + ["pleasant_5", "unpleasant_5a"], ) return query diff --git a/tests/debias/test_double_hard_debias.py b/tests/debias/test_double_hard_debias.py index 4a9e880..2b7b861 100644 --- a/tests/debias/test_double_hard_debias.py +++ b/tests/debias/test_double_hard_debias.py @@ -170,7 +170,7 @@ def test_double_hard_debias_checks( # ) # targets = weat_wordsets["male_names"] + weat_wordsets["female_names"] -# attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"] +# attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"] # ignore = targets + attributes # gender_debiased_w2v = dhd.fit( diff --git a/tests/debias/test_half_sibling_regression.py b/tests/debias/test_half_sibling_regression.py index 5fa8a63..3703c0b 100644 --- a/tests/debias/test_half_sibling_regression.py +++ b/tests/debias/test_half_sibling_regression.py @@ -108,7 +108,7 @@ def test_half_sibling_regression_ignore_param( ) targets = weat_wordsets["male_names"] + weat_wordsets["female_names"] - attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"] + attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"] ignore = targets + attributes gender_debiased_w2v = hsr.fit(model, definitional_words=gender_specific).transform( diff --git a/tests/debias/test_hard_debias.py b/tests/debias/test_hard_debias.py index 8984480..1c71b34 100644 --- a/tests/debias/test_hard_debias.py +++ b/tests/debias/test_hard_debias.py @@ -148,7 +148,7 @@ def test_hard_debias_ignore_param( # this implies that neither of these words should be subjected to debias and # therefore, both queries when executed with weat should return the same score. targets = weat_wordsets["male_names"] + weat_wordsets["female_names"] - attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"] + attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"] ignore = targets + attributes gender_debiased_w2v = hd.fit( diff --git a/tests/debias/test_multiclass_hard_debias.py b/tests/debias/test_multiclass_hard_debias.py index e90b607..6790b0c 100644 --- a/tests/debias/test_multiclass_hard_debias.py +++ b/tests/debias/test_multiclass_hard_debias.py @@ -188,7 +188,7 @@ def test_multiclass_hard_debias_ignore_param( # this implies that neither of these words should be subjected to debias and # therefore, both queries when executed with weat should return the same score. targets = weat_wordsets["male_names"] + weat_wordsets["female_names"] - attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5"] + attributes = weat_wordsets["pleasant_5"] + weat_wordsets["unpleasant_5a"] ignore = targets + attributes gender_debiased_w2v = mhd.fit( diff --git a/tests/metrics/conftest.py b/tests/metrics/conftest.py index cd9be6e..21f9080 100644 --- a/tests/metrics/conftest.py +++ b/tests/metrics/conftest.py @@ -72,7 +72,7 @@ def query_2t2a_1(weat_wordsets: dict[str, list[str]]) -> Query: """ query = Query( [weat_wordsets["flowers"], weat_wordsets["insects"]], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["Flowers", "Insects"], ["Pleasant", "Unpleasant"], ) @@ -87,7 +87,7 @@ def query_3t2a_1(weat_wordsets: dict[str, list[str]]) -> Query: weat_wordsets["insects"], weat_wordsets["instruments"], ], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["Flowers", "Weapons", "Instruments"], ["Pleasant", "Unpleasant"], ) @@ -104,7 +104,7 @@ def query_4t2a_1(weat_wordsets: dict[str, list[str]]) -> Query: weat_wordsets["instruments"], weat_wordsets["weapons"], ], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["Flowers", "Insects", "Instruments", "Weapons"], ["Pleasant", "Unpleasant"], ) @@ -119,7 +119,7 @@ def query_1t4_1(weat_wordsets: dict[str, list[str]]) -> Query: [ weat_wordsets["pleasant_5"], weat_wordsets["pleasant_9"], - weat_wordsets["unpleasant_5"], + weat_wordsets["unpleasant_5a"], weat_wordsets["unpleasant_9"], ], ["Flowers"], @@ -144,7 +144,7 @@ def query_2t1a_lost_vocab_1(weat_wordsets: dict[str, list[str]]) -> Query: def query_2t2a_lost_vocab_1(weat_wordsets: dict[str, list[str]]) -> Query: query = Query( [["bla", "asd"], weat_wordsets["insects"]], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["Flowers", "Insects"], ["Pleasant", "Unpleasant"], ) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 9b1e36e..6e51d6d 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1,4 +1,10 @@ +import socket +import urllib.error + +import pytest + from wefe.datasets.datasets import ( + _retry_request, fetch_debias_multiclass, fetch_debiaswe, fetch_eds, @@ -132,11 +138,12 @@ def test_load_weat() -> None: "flowers", "insects", "pleasant_5", - "unpleasant_5", + "unpleasant_5a", "instruments", "weapons", "european_american_names_5", "african_american_names_5", + "unpleasant_5b", "european_american_names_7", "african_american_names_7", "pleasant_9", @@ -180,3 +187,186 @@ def test_load_gn_glove() -> None: for word in set_: assert isinstance(word, str) assert len(word) > 0 + + +# Tests for retry functionality +class TestRetryRequest: + """Test cases for the _retry_request function.""" + + def test_retry_request_success_on_first_attempt(self): + """Test _retry_request result when function succeeds on first attempt.""" + from unittest.mock import Mock + + mock_func = Mock(return_value="success") + + result = _retry_request(mock_func, "arg1", "arg2", kwarg1="value1") + + assert result == "success" + mock_func.assert_called_once_with("arg1", "arg2", kwarg1="value1") + + def test_retry_request_rate_limit_error(self, monkeypatch): + """Test retry behavior for HTTP 429 rate limit errors.""" + from unittest.mock import Mock + + mock_sleep = Mock() + mock_warning = Mock() + monkeypatch.setattr("time.sleep", mock_sleep) + monkeypatch.setattr("logging.warning", mock_warning) + + mock_func = Mock() + + # Create HTTPError with code 429 + from email.message import EmailMessage + + headers = EmailMessage() + http_error = urllib.error.HTTPError( + url="http://test.com", + code=429, + msg="Too Many Requests", + hdrs=headers, + fp=None, + ) + + # First two calls fail with 429, third succeeds + mock_func.side_effect = [http_error, http_error, "success"] + + result = _retry_request(mock_func, n_retries=3) + + assert result == "success" + assert mock_func.call_count == 3 + assert mock_sleep.call_count == 2 + assert mock_warning.call_count == 2 + + # Check exponential backoff sleep times + mock_sleep.assert_any_call(1) # 2^0 = 1 + mock_sleep.assert_any_call(2) # 2^1 = 2 + + def test_retry_request_timeout_error(self, monkeypatch): + """Test retry behavior for timeout errors.""" + from unittest.mock import Mock + + mock_sleep = Mock() + mock_warning = Mock() + monkeypatch.setattr("time.sleep", mock_sleep) + monkeypatch.setattr("logging.warning", mock_warning) + + mock_func = Mock() + + # First call fails with timeout, second succeeds + mock_func.side_effect = [socket.timeout("Connection timeout"), "success"] + + result = _retry_request(mock_func, n_retries=2) + + assert result == "success" + assert mock_func.call_count == 2 + mock_sleep.assert_called_once_with(1) # 2^0 = 1 + mock_warning.assert_called_once() + + def test_retry_request_timeout_error_os_error(self, monkeypatch): + """Test retry behavior for OSError (network timeout).""" + from unittest.mock import Mock + + mock_sleep = Mock() + mock_warning = Mock() + monkeypatch.setattr("time.sleep", mock_sleep) + monkeypatch.setattr("logging.warning", mock_warning) + + mock_func = Mock() + + # First call fails with OSError, second succeeds + mock_func.side_effect = [OSError("Network timeout"), "success"] + + result = _retry_request(mock_func, n_retries=2) + + assert result == "success" + assert mock_func.call_count == 2 + mock_sleep.assert_called_once_with(1) # 2^0 = 1 + mock_warning.assert_called_once() + + def test_retry_request_generic_exception(self, monkeypatch): + """Test retry behavior for generic exceptions.""" + from unittest.mock import Mock + + mock_sleep = Mock() + mock_warning = Mock() + monkeypatch.setattr("time.sleep", mock_sleep) + monkeypatch.setattr("logging.warning", mock_warning) + + mock_func = Mock() + + # First call fails with generic exception, second succeeds + mock_func.side_effect = [ValueError("Generic error"), "success"] + + result = _retry_request(mock_func, n_retries=2) + + assert result == "success" + assert mock_func.call_count == 2 + mock_sleep.assert_called_once_with(1) # Fixed 1-second delay + mock_warning.assert_called_once() + + def test_retry_request_non_retryable_http_error(self): + """Test that non-retryable HTTP errors are not retried.""" + from unittest.mock import Mock + + mock_func = Mock() + + # 404 Not Found should not be retried + from email.message import EmailMessage + + headers = EmailMessage() + http_error = urllib.error.HTTPError( + url="http://test.com", code=404, msg="Not Found", hdrs=headers, fp=None + ) + mock_func.side_effect = http_error + + with pytest.raises(urllib.error.HTTPError) as exc_info: + _retry_request(mock_func, n_retries=3) + + assert exc_info.value.code == 404 + mock_func.assert_called_once() # Should only be called once + + def test_retry_request_exhaust_retries(self, monkeypatch): + """Test that function raises exception when all retries are exhausted.""" + from unittest.mock import Mock + + mock_sleep = Mock() + mock_warning = Mock() + monkeypatch.setattr("time.sleep", mock_sleep) + monkeypatch.setattr("logging.warning", mock_warning) + + mock_func = Mock() + + # Always fail with rate limit error + from email.message import EmailMessage + + headers = EmailMessage() + http_error = urllib.error.HTTPError( + url="http://test.com", + code=429, + msg="Too Many Requests", + hdrs=headers, + fp=None, + ) + mock_func.side_effect = http_error + + with pytest.raises(urllib.error.HTTPError) as exc_info: + _retry_request(mock_func, n_retries=2) + + assert exc_info.value.code == 429 + assert mock_func.call_count == 3 # Initial call + 2 retries + assert mock_sleep.call_count == 2 + assert mock_warning.call_count == 2 + + def test_retry_request_url_error(self): + """Test that URLError without code is not retried.""" + from unittest.mock import Mock + + mock_func = Mock() + + url_error = urllib.error.URLError("Connection failed") + mock_func.side_effect = url_error + + with pytest.raises(urllib.error.URLError): + _retry_request(mock_func, n_retries=3) + + mock_func.assert_called_once() # Should only be called once diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 93a2e2c..e8f9bbe 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -46,7 +46,7 @@ def query_2t2a_1(weat_wordsets: dict[str, list[str]]) -> Query: """ query = Query( [weat_wordsets["flowers"], weat_wordsets["insects"]], - [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5"]], + [weat_wordsets["pleasant_5"], weat_wordsets["unpleasant_5a"]], ["Flowers", "Insects"], ["Pleasant", "Unpleasant"], ) @@ -89,7 +89,7 @@ def query_2t2a_uppercase(weat_wordsets: dict[str, list[str]]) -> Query: ], [ [s.upper() for s in weat_wordsets["pleasant_5"]], - [s.upper() for s in weat_wordsets["unpleasant_5"]], + [s.upper() for s in weat_wordsets["unpleasant_5a"]], ], ["Flowers", "Insects"], ["Pleasant", "Unpleasant"], @@ -583,7 +583,7 @@ def test_get_embeddings_from_query( weat_wordsets["flowers"], weat_wordsets["insects"], weat_wordsets["pleasant_5"], - weat_wordsets["unpleasant_5"], + weat_wordsets["unpleasant_5a"], ) word_vectors = model.wv @@ -635,7 +635,7 @@ def test_get_embeddings_from_query_oov_warns( weat_wordsets["flowers"], weat_wordsets["insects"], weat_wordsets["pleasant_5"], - weat_wordsets["unpleasant_5"], + weat_wordsets["unpleasant_5a"], ) flowers_with_oov = flowers + ["aaa", "bbb"] @@ -667,7 +667,7 @@ def test_get_embeddings_from_query_with_lower_preprocessor( weat_wordsets["flowers"], weat_wordsets["insects"], weat_wordsets["pleasant_5"], - weat_wordsets["unpleasant_5"], + weat_wordsets["unpleasant_5a"], ) embeddings = get_embeddings_from_query( @@ -706,7 +706,7 @@ def test_get_embeddings_from_query_with_two_preprocessors( weat_wordsets["flowers"], weat_wordsets["insects"], weat_wordsets["pleasant_5"], - weat_wordsets["unpleasant_5"], + weat_wordsets["unpleasant_5a"], ) embeddings = get_embeddings_from_query( model, query_2t2a_uppercase, preprocessors=[{}, {"lowercase": True}] @@ -741,7 +741,7 @@ def test_get_embeddings_from_query_lost_threshold( weat_wordsets["flowers"], weat_wordsets["insects"], weat_wordsets["pleasant_5"], - weat_wordsets["unpleasant_5"], + weat_wordsets["unpleasant_5a"], ) # with lost vocabulary threshold. diff --git a/tests/test_query.py b/tests/test_query.py index da2c3b5..3b7333d 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -78,7 +78,7 @@ def test_create_query() -> None: flowers = weat_wordsets["flowers"] insects = weat_wordsets["insects"] pleasant = weat_wordsets["pleasant_5"] - unpleasant = weat_wordsets["unpleasant_5"] + unpleasant = weat_wordsets["unpleasant_5a"] # create a query using the word sets: query = Query( @@ -108,7 +108,7 @@ def test_eq() -> None: weapons = weat["weapons"] pleasant_1 = weat["pleasant_5"] pleasant_2 = weat["pleasant_9"] - unpleasant_1 = weat["unpleasant_5"] + unpleasant_1 = weat["unpleasant_5a"] unpleasant_2 = weat["unpleasant_9"] query = Query([flowers, insects], [pleasant_1, unpleasant_1]) @@ -266,7 +266,7 @@ def test_generate_query_name() -> None: query = Query( [weat_word_set["flowers"], weat_word_set["instruments"]], - [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5"]], + [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5a"]], ["Flowers", "Instruments"], ["Pleasant", "Unpleasant"], ) @@ -280,7 +280,7 @@ def test_generate_query_name() -> None: weat_word_set["weapons"], weat_word_set["insects"], ], - [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5"]], + [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5a"]], ["Flowers", "Instruments", "Weapons", "Insects"], ["Pleasant", "Unpleasant"], ) @@ -297,7 +297,7 @@ def test_generate_query_name() -> None: weat_word_set["weapons"], weat_word_set["insects"], ], - [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5"]], + [weat_word_set["pleasant_5"], weat_word_set["unpleasant_5a"]], ) assert ( diff --git a/tests/test_utils.py b/tests/test_utils.py index 9e65170..5872391 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -62,14 +62,14 @@ def queries_and_models(): # Create ethnicity queries test_query_1 = Query( [word_sets["insects"], word_sets["flowers"]], - [word_sets["pleasant_5"], word_sets["unpleasant_5"]], + [word_sets["pleasant_5"], word_sets["unpleasant_5a"]], ["Flowers", "Insects"], ["Pleasant", "Unpleasant"], ) test_query_2 = Query( [word_sets["weapons"], word_sets["instruments"]], - [word_sets["pleasant_5"], word_sets["unpleasant_5"]], + [word_sets["pleasant_5"], word_sets["unpleasant_5a"]], ["Instruments", "Weapons"], ["Pleasant", "Unpleasant"], ) diff --git a/wefe/datasets/datasets.py b/wefe/datasets/datasets.py index 8f2182d..cb8276b 100644 --- a/wefe/datasets/datasets.py +++ b/wefe/datasets/datasets.py @@ -1,6 +1,10 @@ """Module with functions to load datasets and sets of words related to bias.""" import json +import logging +import socket +import time +import urllib.error import urllib.request from typing import Union @@ -9,8 +13,92 @@ import pkg_resources +def _retry_request(func, *args, n_retries: int = 3, **kwargs): + """Retry a function call with exponential backoff for rate limiting errors. + + Parameters + ---------- + func : callable + The function to retry (pd.read_csv or urllib.request.urlopen) + *args : tuple + Positional arguments to pass to the function + n_retries : int, optional + Number of retries to attempt, by default 3 + **kwargs : dict + Keyword arguments to pass to the function + + Returns + ------- + Any + The result of the function call + + Raises + ------ + Exception + The last exception encountered if all retries fail + + Notes + ----- + This function handles the following error types with retries: + - HTTP 429 (Too Many Requests) and 503 (Service Unavailable) errors + with exponential backoff + - Timeout errors (socket.timeout, TimeoutError, OSError) with exponential backoff + - Other exceptions with a fixed 1-second delay + """ + last_exception = None + + for attempt in range(n_retries + 1): + try: + return func(*args, **kwargs) + except (urllib.error.HTTPError, urllib.error.URLError) as e: + last_exception = e + # Check if it's a rate limiting error (429 or 503) + if ( + isinstance(e, urllib.error.HTTPError) + and e.code in [429, 503] + and attempt < n_retries + ): + wait_time = 2**attempt # Exponential backoff: 1s, 2s, 4s + logging.warning( + f"Rate limit encountered, retrying in {wait_time} " + f"seconds... (attempt {attempt + 1}/{n_retries})" + ) + time.sleep(wait_time) + continue + # For non-rate-limiting errors, don't retry + raise e + except (socket.timeout, TimeoutError, OSError) as e: + last_exception = e + # Handle timeout errors with retry + if attempt < n_retries: + wait_time = 2**attempt # Exponential backoff + logging.warning( + f"Timeout error encountered, retrying in {wait_time} " + f"seconds... (attempt {attempt + 1}/{n_retries})" + ) + time.sleep(wait_time) + continue + raise e + except Exception as e: + last_exception = e + # For pandas errors or other exceptions, retry with short delay + if attempt < n_retries: + logging.warning( + f"Request failed, retrying in 1 second... " + f"(attempt {attempt + 1}/{n_retries})" + ) + time.sleep(1) + continue + raise e + + # If we get here, all retries failed + if last_exception: + raise last_exception + raise RuntimeError("All retries failed without capturing an exception") + + def fetch_eds( - occupations_year: int = 2015, top_n_race_occupations: int = 10 + occupations_year: int = 2015, top_n_race_occupations: int = 10, n_retries: int = 3 ) -> dict[str, list[str]]: """Fetch the sets of words used in the experiments of the _Word Embeddings Quantify 100 Years Of Gender And Ethnic Stereotypes_ work. @@ -39,6 +127,8 @@ def fetch_eds( top_n_race_occupations : int, optional The year of the census for the occupations file. The number of occupations by race, by default 10 + n_retries : int, optional + Number of retries to attempt for each request, by default 3 Returns ------- @@ -46,10 +136,7 @@ def fetch_eds( A dictionary with the word sets. """ # noqa: D205 - EDS_BASE_URL = ( - "https://raw.githubusercontent.com/nikhgarg/" - "EmbeddingDynamicStereotypes/master/data/" - ) + EDS_BASE_URL = "https://raw.githubusercontent.com/nikhgarg/EmbeddingDynamicStereotypes/refs/heads/master/data/" EDS_WORD_SETS_NAMES = [ "adjectives_appearance.txt", "adjectives_intelligencegeneral.txt", @@ -71,7 +158,14 @@ def fetch_eds( word_sets = [] for EDS_words_set_name in EDS_WORD_SETS_NAMES: name = EDS_words_set_name.replace(".txt", "") - word_sets.append(pd.read_csv(EDS_BASE_URL + EDS_words_set_name, names=[name])) + word_sets.append( + _retry_request( + pd.read_csv, + EDS_BASE_URL + EDS_words_set_name, + names=[name], + n_retries=n_retries, + ) + ) word_sets_dict = pd.concat(word_sets, sort=False, axis=1).to_dict(orient="list") @@ -84,8 +178,10 @@ def fetch_eds( # ---- Occupations by Gender ---- # fetch occupations by gender - gender_occupations = pd.read_csv( - EDS_BASE_URL + "occupation_percentages_gender_occ1950.csv" + gender_occupations = _retry_request( + pd.read_csv, + EDS_BASE_URL + "occupation_percentages_gender_occ1950.csv", + n_retries=n_retries, ) # filter by year gender_occupations = gender_occupations[ @@ -109,7 +205,11 @@ def fetch_eds( # ---- Occupations by Ethnicity ---- - occupations = pd.read_csv(EDS_BASE_URL + "occupation_percentages_race_occ1950.csv") + occupations = _retry_request( + pd.read_csv, + EDS_BASE_URL + "occupation_percentages_race_occ1950.csv", + n_retries=n_retries, + ) occupations_filtered = occupations[occupations["Census year"] == occupations_year] occupations_white = ( occupations_filtered.sort_values("white") @@ -156,7 +256,7 @@ def fetch_eds( return word_sets_dict -def fetch_debiaswe() -> dict[str, Union[list[str], list]]: +def fetch_debiaswe(n_retries: int = 3) -> dict[str, Union[list[str], list]]: """Fetch the word sets used in the paper Man is to Computer Programmer as Woman is to Homemaker? from the source. It includes gender (male, female) terms and related word sets. @@ -168,6 +268,11 @@ def fetch_debiaswe() -> dict[str, Union[list[str], list]]: Venkatesh Saligrama, and Adam Kalai. | Proceedings of NIPS 2016. + Parameters + ---------- + n_retries : int, optional + Number of retries to attempt for each request, by default 3 + Returns ------- Dict[str, Union[List[str], list]] @@ -176,7 +281,7 @@ def fetch_debiaswe() -> dict[str, Union[list[str], list]]: """ # noqa: D205 DEBIAS_WE_BASE_URL = ( - "https://raw.githubusercontent.com/tolga-b/debiaswe/master/data/" + "https://raw.githubusercontent.com/tolga-b/debiaswe/refs/heads/master/data/" ) DEBIAS_WE_WORD_SETS = [ @@ -186,23 +291,31 @@ def fetch_debiaswe() -> dict[str, Union[list[str], list]]: "professions.json", ] - with urllib.request.urlopen( - DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[0] + with _retry_request( + urllib.request.urlopen, + DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[0], + n_retries=n_retries, ) as json_file: definitional_pairs = json.loads(json_file.read().decode()) male_words = [p[0] for p in definitional_pairs] female_words = [p[1] for p in definitional_pairs] - with urllib.request.urlopen( - DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[1] + with _retry_request( + urllib.request.urlopen, + DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[1], + n_retries=n_retries, ) as json_file: equalize_pairs = json.loads(json_file.read().decode()) - with urllib.request.urlopen( - DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[2] + with _retry_request( + urllib.request.urlopen, + DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[2], + n_retries=n_retries, ) as json_file: gender_specific = json.loads(json_file.read().decode()) - with urllib.request.urlopen( - DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[3] + with _retry_request( + urllib.request.urlopen, + DEBIAS_WE_BASE_URL + DEBIAS_WE_WORD_SETS[3], + n_retries=n_retries, ) as json_file: professions = json.loads(json_file.read().decode()) @@ -257,7 +370,7 @@ def load_bingliu() -> dict[str, list[str]]: } -def fetch_debias_multiclass() -> dict[str, Union[list[str], list]]: +def fetch_debias_multiclass(n_retries: int = 3) -> dict[str, Union[list[str], list]]: """Fetch the word sets used in the paper Black Is To Criminals as Caucasian Is To Police: Detecting And Removing Multiclass Bias In Word Embeddings. @@ -281,6 +394,11 @@ def fetch_debias_multiclass() -> dict[str, Union[list[str], list]]: | [2]: https://github.com/TManzini/DebiasMulticlassWordEmbedding/blob/master/Debiasing/evalBias.py + Parameters + ---------- + n_retries : int, optional + Number of retries to attempt for each request, by default 3 + Returns ------- dict @@ -288,17 +406,16 @@ def fetch_debias_multiclass() -> dict[str, Union[list[str], list]]: its to_numpy() correspond to the word set. """ # noqa: D205, E501 - BASE_URL = ( - "https://raw.githubusercontent.com/TManzini/" - "DebiasMulticlassWordEmbedding/master/Debiasing/data/vocab/" - ) + BASE_URL = "https://raw.githubusercontent.com/TManzini/DebiasMulticlassWordEmbedding/refs/heads/master/Debiasing/data/vocab/" WORD_SETS_FILES = [ "gender_attributes_optm.json", "race_attributes_optm.json", "religion_attributes_optm.json", ] # fetch gender - with urllib.request.urlopen(BASE_URL + WORD_SETS_FILES[0]) as file: + with _retry_request( + urllib.request.urlopen, BASE_URL + WORD_SETS_FILES[0], n_retries=n_retries + ) as file: gender = json.loads(file.read().decode()) gender_definitional_sets = np.array(gender["definite_sets"]) @@ -313,7 +430,9 @@ def fetch_debias_multiclass() -> dict[str, Union[list[str], list]]: gender_eval_target = gender["eval_targets"] # fetch ethnicity - with urllib.request.urlopen(BASE_URL + WORD_SETS_FILES[1]) as file: + with _retry_request( + urllib.request.urlopen, BASE_URL + WORD_SETS_FILES[1], n_retries=n_retries + ) as file: ethnicity = json.loads(file.read().decode()) ethnicity_definitional_sets = np.array(ethnicity["definite_sets"]) @@ -330,7 +449,9 @@ def fetch_debias_multiclass() -> dict[str, Union[list[str], list]]: ethnicity_eval_target = ethnicity["eval_targets"] # fetch religion - with urllib.request.urlopen(BASE_URL + WORD_SETS_FILES[2]) as file: + with _retry_request( + urllib.request.urlopen, BASE_URL + WORD_SETS_FILES[2], n_retries=n_retries + ) as file: religion = json.loads(file.read().decode()) religion_definitional_sets = np.array(religion["definite_sets"]) @@ -377,7 +498,7 @@ def fetch_debias_multiclass() -> dict[str, Union[list[str], list]]: } -def fetch_gn_glove() -> dict[str, list[str]]: +def fetch_gn_glove(n_retries: int = 3) -> dict[str, list[str]]: """Fetch the word sets used in the paper Learning Gender-Neutral Word Embeddings. This dataset contain two sets of 221 female and male related words. @@ -388,24 +509,35 @@ def fetch_gn_glove() -> dict[str, list[str]]: | Learning Gender-Neutral Word Embeddings. | In EMNLP. + Parameters + ---------- + n_retries : int, optional + Number of retries to attempt for each request, by default 3 + Returns ------- Dict[str, List[str]] A dictionary with male and female word sets. """ - BASE_URL = "https://raw.githubusercontent.com/uclanlp/gn_glove/master/wordlist/" + BASE_URL = ( + "https://raw.githubusercontent.com/uclanlp/gn_glove/refs/heads/master/wordlist/" + ) FILES = [ "female_word_file.txt", "male_word_file.txt", ] # fetch female words - with urllib.request.urlopen(BASE_URL + FILES[0]) as file: + with _retry_request( + urllib.request.urlopen, BASE_URL + FILES[0], n_retries=n_retries + ) as file: female_terms = file.read().decode().split("\n") female_terms = list(filter(lambda x: x != "", female_terms)) # fetch male words - with urllib.request.urlopen(BASE_URL + FILES[1]) as file: + with _retry_request( + urllib.request.urlopen, BASE_URL + FILES[1], n_retries=n_retries + ) as file: male_terms = file.read().decode().split("\n") male_terms = list(filter(lambda x: x != "", male_terms))