LPN64
/

LongCite-llama3.1-8b-GGUF

Text Generation

importance matrix

Model card Files Files and versions Community

LPN64 commited on about 19 hours ago

Commit

d8e9d34

•

1 Parent(s): 23a8be0

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -30,7 +30,7 @@ class LongCiteModel:
     def text_split_by_punctuation(original_text, return_dict=False):
         # text = re.sub(r'([a-z])\.([A-Z])', r'\1. \2', original_text)  # separate period without space
         text = original_text
-        custom_sent_tokenizer = PunktSentenceTokenizer(text)
         punctuations = r"([。；！？])"  # For Chinese support
         separated = custom_sent_tokenizer.tokenize(text)

     def text_split_by_punctuation(original_text, return_dict=False):
         # text = re.sub(r'([a-z])\.([A-Z])', r'\1. \2', original_text)  # separate period without space
         text = original_text
+        custom_sent_tokenizer = PunktSentenceTokenizer()
         punctuations = r"([。；！？])"  # For Chinese support
         separated = custom_sent_tokenizer.tokenize(text)