File size: 4,689 Bytes
f6a522a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
#!/usr/bin/env python3
import os
import logging
import pandas
from tqdm import tqdm
import tarfile
import logging
import os
from pathlib import Path
import csv
from functools import partial
from multiprocessing import Pool
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
files_per_archive = 40_000
def make_archive(archive_index_with_files, output_dir, lang, split):
archive_index, files = archive_index_with_files
archive_dir = f"{lang}_{split}_{archive_index}"
archive_path = os.path.join(output_dir, f"{archive_dir}.tar")
with tarfile.open(archive_path, "w") as tar:
for file in files:
_, filename = os.path.split(file)
tar.add(file, arcname=os.path.join(archive_dir, filename))
def extract_archive(archive_path, target_dir):
with tarfile.open(archive_path, 'r:gz') as f:
f.extractall(path=target_dir)
def main():
# langs = ["ab", "ar", "as", "ast"]
# langs = ["az"]
# langs = ["ba"]
# langs = ["bas", "be", "bg", "bn", "br"]
# langs = ["ca", "ckb", "en", "eo", "eu", "fr"]
# langs = ["hy-AM"]
# langs = ["rw", "th", "yue", "zh-CN"]
# langs = ["zh-HK", "zh-TW", "cv"]
# langs = ["cs", "da", "dv", "nl", "myv", "et", "fi", "fy-NL", "gl", ]
# langs = ["ka", "el", "gn", "cnh", "ha", "mrj", "hi", "ig", "id", "ia", "ga-IE", "it"]
# langs = ["ja", "kab", ]
# langs = ["kk", "kmr", "ky", "lg", "fa", "es"]
# langs = ["cy", "vi", "uz", "ug"]
# langs = ["ur", "uk", "tw", "tr", "tok", "ti"] +\
# ["tig", "tt", "ta", "nan-tw", "sv-SE", "sw", "hsb", "sl", "sk", "sr"] + \
# ["sc", "skr", "sat", "sah", "ru", ]
# langs = ["rm-sursilv", "ro", "pa-IN", "pt"] + \
# ["pl", "or", "ne-NP", "mn", "mdf", "mhr", "mr", "mt", "ml", "mk", "lt", "nn-NO", "rm-vallader", "lv"]
# langs = ['hu', 'vot']
for lang in tqdm(langs, desc="languages"):
extract = True
logging.info(f"Language: {lang.upper()}, files per tar: {files_per_archive}")
orig_archive_path = f"/home/polina_huggingface_co/data/cv11/{lang}.tar.gz"
if extract:
logging.info("Extracting original archive...")
extract_archive(orig_archive_path, target_dir="/home/polina_huggingface_co/data/cv11/")
logging.info("Extracted.")
clip_path = f"/home/polina_huggingface_co/data/cv11/cv-corpus-11.0-2022-09-21/{lang}/clips"
# data_size = sum(f.stat().st_size for f in Path(clip_path).glob('*.mp3') if f.is_file())
# data_size_gb = data_size / 1024 ** 3
# num_procs = min(int(data_size_gb) + 2, 28)
splits = ("test", "dev", "train", "other", "invalidated")
for split in splits:
meta_path = f"/home/polina_huggingface_co/data/cv11/cv-corpus-11.0-2022-09-21/{lang}/{split}.tsv"
new_meta_dir = f"repos/common_voice_11_0/transcript/{lang}/"
Path(new_meta_dir).mkdir(parents=True, exist_ok=True)
data = pandas.read_csv(meta_path, sep='\t', quoting=csv.QUOTE_NONE, low_memory=False)
copy_command = f"cp {meta_path} {new_meta_dir}"
os.system(copy_command)
all_files = [os.path.join(clip_path, filename) for filename in list(data["path"])]
num_files = len(all_files)
if num_files == 0:
continue
logging.info(f"split: {split.upper()}, num_files: {num_files}")
new_clip_path = f"repos/common_voice_11_0/audio/{lang}/{split}"
Path(new_clip_path).mkdir(parents=True, exist_ok=True)
file_groups = [
(arch_index_in_dir, all_files[start_index:start_index + files_per_archive])
for arch_index_in_dir, start_index in enumerate(range(0, num_files, files_per_archive))
]
n_file_groups = len(file_groups)
num_procs = max(1, min(n_file_groups, 26))
logging.info(f"N groups: {n_file_groups}, num procs: {num_procs}")
if n_file_groups > 1:
pool = Pool(num_procs)
pool.map(
partial(
make_archive,
output_dir=new_clip_path,
lang=lang,
split=split,
),
tqdm(file_groups, desc=f"Taring {split} subset...", position=0),
)
else:
make_archive(
file_groups[0],
output_dir=new_clip_path,
lang=lang,
split=split,
)
if __name__ == "__main__":
main()
|