File size: 949 Bytes
7cfca48 95cd35a 7cfca48 09f9c26 7cfca48 09f9c26 7cfca48 95cd35a 09f9c26 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
from hazm import word_tokenize
from hazm import sent_tokenize
import re
import six
import string
from normalizer import normalize
persian_regex = "0-9۰۱۲۳۴۵۶۷۸۹ءآئابتثجحخدذرزسشصضطظعغفقلمنهوپچژکگیە\u200c"
def filter_by_lang_regex(text, ratio=0.7, regex="0-9۰۱۲۳۴۵۶۷۸۹ءآئابتثجحخدذرزسشصضطظعغفقلمنهوپچژکگیە\u200c"):
candidate_text = re.sub(r"[^" + regex + "]+", " ", six.ensure_str(text)).replace(" ", "")
text = text.replace(" ", "")
return True if (len(candidate_text) / len(text)) > ratio else False
def filter_by_num_tokens(text, gt=64):
return True if len(word_tokenize(text)) > gt else False
def filter_by_num_sents(text, gt=2):
return True if len(sent_tokenize(text)) > gt else False
def normalizer(text, do_lowercase=False):
text = normalize(text)
if do_lowercase:
text = text.lower()
return text
|