datasets-report / app.py
albertvillanova's picture
Remove background scheduler
2548bde verified
# import datetime
import json
import os
from pathlib import Path
import gradio as gr
import huggingface_hub as hfh
# from apscheduler.schedulers.background import BackgroundScheduler
DATASET_ID = "albertvillanova/datasets-report"
DATASET_PATH = "dataset"
DATA_DIR = "data"
DATA_PATH = f"{DATASET_PATH}/{DATA_DIR}"
def pull_dataset_repo(repo_id=DATASET_ID, repo_path=DATASET_PATH):
token = os.environ.get('HUB_TOKEN')
repo = hfh.Repository(
local_dir=repo_path,
clone_from=repo_id,
repo_type="dataset",
use_auth_token=token,
)
repo.git_pull()
return repo
def load_dates():
return [data_path.stem for data_path in sorted(Path(DATA_PATH).iterdir())]
repo = pull_dataset_repo()
dates = load_dates()
datasets = hfh.list_datasets()
def filter_datasets_by_date(date_from, date_to):
with open(f"{DATA_PATH}/{date_from}.json") as f:
ids_from = json.load(f)
with open(f"{DATA_PATH}/{date_to}.json") as f:
ids_to = json.load(f)
ids = set(ids_to) - set(ids_from)
dss = [ds for ds in datasets if ds.id in ids]
for ds in dss:
try:
_ = getattr(ds, "downloads")
except AttributeError:
setattr(ds, "downloads", 0)
dss = sorted(dss, key=lambda item: item.downloads, reverse=True)
return dss
def filter_dataframe(date_from, date_to):
dss = filter_datasets_by_date(date_from, date_to)
return [[ds.id, ds.downloads] for ds in dss]
# def update_datasets():
# # Retrieve datasets
# datasets = hfh.list_datasets()
# # Save dataset IDs
# repo = pull_dataset_repo()
# os.makedirs(DATA_PATH, exist_ok=True)
# today = datetime.datetime.now(datetime.timezone.utc).date().isoformat()
# with repo.commit(f"Add {today} data file"):
# with open(f"data/{today}.json", "w") as f:
# json.dump([ds.id for ds in sorted(datasets, key=lambda item: item.id)], f)
#
#
# scheduler = BackgroundScheduler()
# scheduler.add_job(update_datasets, trigger="cron", hour=0, minute=1, timezone=datetime.timezone.utc)
# scheduler.start()
with gr.Blocks() as demo:
with gr.Row():
date_from = gr.Dropdown(choices=dates, label="Date from")
date_to = gr.Dropdown(choices=dates, label="Date to")
submit_btn = gr.Button("Submit")
outputs = gr.Dataframe(
headers=["Dataset", "Downloads"],
datatype=["str", "number"],
label="Created datasets",
)
submit_btn.click(fn=filter_dataframe, inputs=[date_from, date_to], outputs=outputs)
demo.launch()