Spaces:
Paused
Paused
| import re | |
| import string | |
| from fish_speech.text.clean import clean_text | |
| def utf_8_len(text): | |
| return len(text.encode("utf-8")) | |
| def break_text(texts, length, splits: set): | |
| for text in texts: | |
| if utf_8_len(text) <= length: | |
| yield text | |
| continue | |
| curr = "" | |
| for char in text: | |
| curr += char | |
| if char in splits: | |
| yield curr | |
| curr = "" | |
| if curr: | |
| yield curr | |
| def break_text_by_length(texts, length): | |
| for text in texts: | |
| if utf_8_len(text) <= length: | |
| yield text | |
| continue | |
| curr = "" | |
| for char in text: | |
| curr += char | |
| if utf_8_len(curr) >= length: | |
| yield curr | |
| curr = "" | |
| if curr: | |
| yield curr | |
| def add_cleaned(curr, segments): | |
| curr = curr.strip() | |
| if curr and not all(c.isspace() or c in string.punctuation for c in curr): | |
| segments.append(curr) | |
| def protect_float(text): | |
| # Turns 3.14 into <3_f_14> to prevent splitting | |
| return re.sub(r"(\d+)\.(\d+)", r"<\1_f_\2>", text) | |
| def unprotect_float(text): | |
| # Turns <3_f_14> into 3.14 | |
| return re.sub(r"<(\d+)_f_(\d+)>", r"\1.\2", text) | |
| def split_text(text, length): | |
| text = clean_text(text) | |
| # Break the text into pieces with following rules: | |
| # 1. Split the text at ".", "!", "?" if text is NOT a float | |
| # 2. If the text is longer than length, split at "," | |
| # 3. If the text is still longer than length, split at " " | |
| # 4. If the text is still longer than length, split at any character to length | |
| texts = [text] | |
| texts = map(protect_float, texts) | |
| texts = break_text(texts, length, {".", "!", "?", "。", "！", "？"}) | |
| texts = map(unprotect_float, texts) | |
| texts = break_text(texts, length, {",", "，"}) | |
| texts = break_text(texts, length, {" "}) | |
| texts = list(break_text_by_length(texts, length)) | |
| # Then, merge the texts into segments with length <= length | |
| segments = [] | |
| curr = "" | |
| for text in texts: | |
| if utf_8_len(curr) + utf_8_len(text) <= length: | |
| curr += text | |
| else: | |
| add_cleaned(curr, segments) | |
| curr = text | |
| if curr: | |
| add_cleaned(curr, segments) | |
| return segments | |
| if __name__ == "__main__": | |
| # Test the split_text function | |
| text = "This is a test sentence. This is another test sentence. And a third one." | |
| assert split_text(text, 50) == [ | |
| "This is a test sentence.", | |
| "This is another test sentence. And a third one.", | |
| ] | |
| assert split_text("a,aaaaaa3.14", 10) == ["a,", "aaaaaa3.14"] | |
| assert split_text(" ", 10) == [] | |
| assert split_text("a", 10) == ["a"] | |
| text = "This is a test sentence with only commas, and no dots, and no exclamation marks, and no question marks, and no newlines." | |
| assert split_text(text, 50) == [ | |
| "This is a test sentence with only commas,", | |
| "and no dots, and no exclamation marks,", | |
| "and no question marks, and no newlines.", | |
| ] | |
| text = "This is a test sentence This is a test sentence This is a test sentence. This is a test sentence, This is a test sentence, This is a test sentence." | |
| # First half split at " ", second half split at "," | |
| assert split_text(text, 50) == [ | |
| "This is a test sentence This is a test sentence", | |
| "This is a test sentence. This is a test sentence,", | |
| "This is a test sentence, This is a test sentence.", | |
| ] | |
| text = "这是一段很长的中文文本,而且没有句号,也没有感叹号,也没有问号,也没有换行符。" | |
| assert split_text(text, 50) == [ | |
| "这是一段很长的中文文本,", | |
| "而且没有句号,也没有感叹号,", | |
| "也没有问号,也没有换行符.", | |
| ] | |