Update README.md

README.md

@@ -99,7 +99,6 @@ def preprocess(
     # Apply prompt templates
     input_ids, attention_masks = [], []
     for i, source in enumerate(sources):
-        ## system_message
         messages = [
             {"role": "user",
             "content": "\n\n".join(source)}
@@ -109,6 +108,7 @@ def preprocess(
         input_id = model_inputs['input_ids'][0]
         attention_mask = model_inputs['attention_mask'][0]
         if len(input_id) > max_len:
+            ## last five tokens: <|im_end|>(151645), \n(198), <|im_start|>(151644), assistant(77091), \n(198)
             diff = len(input_id) - max_len
             input_id = input_id[:-5-diff] + input_id[-5:]
             attention_mask = attention_mask[:-5-diff] + attention_mask[-5:]