|
import pytest |
|
import requests |
|
|
|
from tools.datasets.corpora import DATA_DOWNLOADERS |
|
|
|
|
|
def check_url_accessible(url): |
|
try: |
|
response = requests.head(url, timeout=5) |
|
response.raise_for_status() |
|
return True |
|
except requests.exceptions.RequestException as e: |
|
print(f"Error: Unable to access URL - {e}") |
|
return False |
|
|
|
|
|
@pytest.mark.cpu |
|
@pytest.mark.parametrize("dataset_name", list(DATA_DOWNLOADERS.keys())) |
|
def test_url_accessibility(dataset_name): |
|
if dataset_name == "pass": |
|
return |
|
elif not dataset_name == "enwik8": |
|
pytest.xfail() |
|
for url in DATA_DOWNLOADERS[dataset_name].urls: |
|
assert check_url_accessible(url) |
|
|
