fix coding error

app.py

@@ -1,20 +1,14 @@
 import gradio as gr
 import torch
 from tqdm import tqdm
-from transformers import PegasusForConditionalGeneration
-from tokenizers_pegasus import PegasusTokenizer
+from transformers import pipeline
 
 MODEL_NAME = "csebuetnlp/mT5_multilingual_XLSum"
 
-summarizer = PegasusForConditionalGeneration.from_pretrained(MODEL_NAME)
-tokenizer = PegasusTokenizer.from_pretrained(MODEL_NAME)
+summarizer = pipeline(task="summarization", model=MODEL_NAME)
 
 def summarize(text):
-    inputs = tokenizer(text, max_length=1024, return_tensors="pt")
-
-    # Generate Summary
-    summary_ids = summarizer.generate(inputs["input_ids"])
-    return tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    return summarizer(text)
 
 demo = gr.Blocks(title="⭐ Summ4rizer ⭐")
 demo.encrypt = False

data_utils.py

@@ -1,319 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import re
-import six
-import unicodedata
-import torch
-import rouge
-import numpy as np
-import random
-# from fengshen.examples.pegasus.pegasus_utils import text_segmentate
-import sys
-
-sys.path.append('../../../')
-
-rouge = rouge.Rouge()
-
-
-is_py2 = six.PY2
-
-if not is_py2:
-    basestring = str
-
-
-def _is_chinese_char(cp):
-    """Checks whether CP is the codepoint of a CJK character."""
-    # This defines a "chinese character" as anything in the CJK Unicode block:
-    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
-    #
-    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
-    # despite its name. The modern Korean Hangul alphabet is a different block,
-    # as is Japanese Hiragana and Katakana. Those alphabets are used to write
-    # space-separated words, so they are not treated specially and handled
-    # like the all of the other languages.
-    if ((cp >= 0x4E00 and cp <= 0x9FFF) or (cp >= 0x3400 and cp <= 0x4DBF)
-            or (cp >= 0x20000 and cp <= 0x2A6DF)
-            or (cp >= 0x2A700 and cp <= 0x2B73F)
-            or (cp >= 0x2B740 and cp <= 0x2B81F)
-            or (cp >= 0x2B820 and cp <= 0x2CEAF)
-            or (cp >= 0xF900 and cp <= 0xFAFF)
-            or (cp >= 0x2F800 and cp <= 0x2FA1F)):
-        return True
-
-    return False
-
-
-def _is_whitespace(char):
-    """Checks whether `char` is a whitespace character."""
-    # \t, \n, and \r are technically control characters but we treat them
-    # as whitespace since they are generally considered as such.
-    if char == " " or char == "\t" or char == "\n" or char == "\r":
-        return True
-    cat = unicodedata.category(char)
-    if cat == "Zs":
-        return True
-    return False
-
-
-def _is_control(char):
-    """Checks whether `char` is a control character."""
-    # These are technically control characters but we count them as whitespace
-    # characters.
-    if char == "\t" or char == "\n" or char == "\r":
-        return False
-    cat = unicodedata.category(char)
-    if cat.startswith("C"):
-        return True
-    return False
-
-
-def _is_punctuation(char):
-    """Checks whether `char` is a punctuation character."""
-    cp = ord(char)
-    # We treat all non-letter/number ASCII as punctuation.
-    # Characters such as "^", "$", and "`" are not in the Unicode
-    # Punctuation class but we treat them as punctuation anyways, for
-    # consistency.
-    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (
-            cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
-        return True
-    cat = unicodedata.category(char)
-    if cat.startswith("P"):
-        return True
-    return False
-
-
-def is_string(s):
-    """判断是否是字符串
-    """
-    return isinstance(s, basestring)
-
-
-def is_stopwords(word, stopwords):
-    if word in stopwords:
-        return True
-    else:
-        return False
-
-
-def text_segmentate(text):
-    en_seg_pattern = '((?:\\!|\\?|\\.|\\n)+(?:\\s)+)'
-    ch_seg_pattern = '((?:？|！|。|\\n)+)'
-    try:
-        text = re.sub(en_seg_pattern, r'\1[SEP]', text)
-        # print("sub text: ", text)
-    except Exception as e:
-        print("input: ", text)
-        raise e
-    text = re.sub(ch_seg_pattern, r'\1[SEP]', text)
-    # print("sub ch text: ", text)
-    text_list = text.split("[SEP]")
-    text_list = list(filter(lambda x: len(x) != 0, text_list))
-    return text_list
-
-
-def load_stopwords(stopwords_path):
-    stopwords_dict = {}
-    with open(stopwords_path, "r") as rf:
-        for line in rf:
-            line = line.strip()
-            if line not in stopwords_dict:
-                stopwords_dict[line] = 0
-            else:
-                pass
-    return stopwords_dict
-
-
-def text_process(text, max_length):
-    """分割文本
-    """
-    texts = text_segmentate(text)
-
-    result, length = [], 0
-    for text in texts:
-        if length + len(text) > max_length * 1.3 and len(result) >= 3:
-            yield result
-            result, length = [], 0
-        result.append(text)
-        length += len(text)
-    if result and len(result) >= 3:
-        yield result
-
-
-def text_process_split_long_content(text, max_length):
-    """分割长文本
-    """
-    texts = text_segmentate(text)
-
-    result, sentence_num = "", 0
-    for text in texts:
-        if len(text) > 500:
-            if len(result) > 300 and sentence_num >= 3:
-                yield result
-                result, sentence_num = "", 0
-            else:
-                result, sentence_num = "", 0
-                continue
-        else:
-            if len(result) + len(text) > max_length * 1.1 and sentence_num >= 3:
-                yield result
-                result, sentence_num = "", 0
-            result += text
-            sentence_num += 1
-
-    if result and sentence_num >= 3:
-        yield result
-
-
-def gather_join(texts, idxs):
-    """取出对应的text，然后拼接起来
-    """
-    return ''.join([texts[i] for i in idxs])
-
-
-def gather_join_f1(texts_token, idsx):
-    join_texts = []
-    for id in idsx:
-        join_texts.extend(texts_token[id])
-    return join_texts
-
-
-def compute_rouge(source, target):
-    """计算rouge-1、rouge-2、rouge-l
-    """
-    source, target = ' '.join(source), ' '.join(target)
-    try:
-        scores = rouge.get_scores(hyps=source, refs=target)
-        return {
-            'rouge-1': scores[0]['rouge-1']['f'],
-            'rouge-2': scores[0]['rouge-2']['f'],
-            'rouge-l': scores[0]['rouge-l']['f'],
-        }
-    except ValueError:
-        return {
-            'rouge-1': 0.0,
-            'rouge-2': 0.0,
-            'rouge-l': 0.0,
-        }
-
-
-def remove_stopwords(texts, stopwords_dict):
-    for i, text in enumerate(texts):
-        texts[i] = list(filter(lambda x: x not in stopwords_dict, text))
-    return texts
-
-
-def pseudo_summary_f1(texts,
-                      stopwords,
-                      tokenizer,
-                      max_length,
-                      rouge_strategy="rouge-l"):
-    """构建伪标签摘要数据集
-    """
-    summary_rate = 0.25
-    max_length = max_length - 1
-    texts_tokens = []
-    sentece_idxs_vec = []
-    for text in texts:
-        if len(texts) == 0:
-            continue
-        try:
-            ids = tokenizer.encode(text.strip())[:-1]
-        except ValueError:
-            print("error, input : ", text)
-            raise ValueError
-        sentece_idxs_vec.append(ids)
-        tokens = [tokenizer._convert_id_to_token(token) for token in ids]
-        texts_tokens.append(tokens)
-
-    texts_tokens_rm = remove_stopwords(texts_tokens, stopwords)
-    source_idxs, target_idxs = list(range(len(texts))), []
-
-    assert len(texts_tokens) == len(texts)
-    # truncate_index = 0
-    while True:
-        sims = []
-        for i in source_idxs:
-            new_source_idxs = [j for j in source_idxs if j != i]
-            new_target_idxs = sorted(target_idxs + [i])
-            new_source = gather_join_f1(texts_tokens_rm, new_source_idxs)
-            new_target = gather_join_f1(texts_tokens_rm, new_target_idxs)
-            sim = compute_rouge(new_source, new_target)[rouge_strategy]
-            sims.append(sim)
-        new_idx = source_idxs[np.argmax(sims)]
-        del sims
-        source_idxs.remove(new_idx)
-        target_idxs = sorted(target_idxs + [new_idx])
-        source = gather_join(texts, source_idxs)
-        target = gather_join(texts, target_idxs)
-        try:
-            if (len(source_idxs) == 1
-                    or 1.0 * len(target) / len(source) > summary_rate):
-                break
-        except ZeroDivisionError as e:
-            print(e.meesage)
-            print(texts)
-            print("source: ", source)
-            print("target: ", target)
-
-    if len(source) < len(target):
-        source, target = target, source
-        source_idxs, target_idxs = target_idxs, source_idxs
-
-    return sentece_idxs_vec, source, target, source_idxs, target_idxs
-
-
-def get_input_mask(sentence_id_vec, indexs):
-    target_idxs = []
-    input_idxs = []
-    kMaskSentenceTokenId = 2
-    kEosTokenId = 1
-    mask_sentence_options_cumulative_prob = [0.9, 0.9, 1, 1]
-    for index in indexs:
-        target_idxs.extend(sentence_id_vec[index])
-        choice = random.uniform(0, 1)
-        if choice < mask_sentence_options_cumulative_prob[0]:
-            # print("mask index: ", index)
-            sentence_id_vec[index] = [kMaskSentenceTokenId]
-        elif choice < mask_sentence_options_cumulative_prob[1]:
-            # print("replace index: ", index)
-            replace_id = random.randint(0, len(sentence_id_vec))
-            sentence_id_vec[index] = sentence_id_vec[replace_id]
-        elif choice < mask_sentence_options_cumulative_prob[2]:
-            pass
-        else:
-            sentence_id_vec[index] = []
-
-    target_idxs.append(kEosTokenId)
-    # print(sentence_id_vec)
-    for index, sentence_id in enumerate(sentence_id_vec):
-        # print(index, sentence_id)
-        if len(sentence_id) == 0:
-            continue
-        input_idxs.extend(sentence_id_vec[index])
-
-    input_idxs.append(kEosTokenId)
-    return input_idxs, target_idxs
-
-
-def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int,
-                       decoder_start_token_id: int):
-    """
-    Shift input ids one token to the right.
-    """
-    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
-    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
-    shifted_input_ids[:, 0] = decoder_start_token_id
-
-    if pad_token_id is None:
-        raise ValueError("self.model.config.pad_token_id has to be defined.")
-    # replace possible -100 values in labels by `pad_token_id`
-    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
-
-    return shifted_input_ids
-
-
-def padding_to_maxlength(ids, max_length, pad_id):
-    cur_len = len(ids)
-    len_diff = max_length - cur_len
-    return ids + [pad_id] * len_diff, [1] * cur_len + [0] * len_diff

tokenizers_pegasus.py

@@ -1,599 +0,0 @@
-
-from fengshen.examples.pegasus.data_utils import (
-    _is_control,
-    _is_punctuation,
-    _is_whitespace,
-    _is_chinese_char)
-from transformers import PreTrainedTokenizer
-from transformers import logging
-from typing import List, Optional, Tuple, Union
-import collections
-import os
-import unicodedata
-import re
-import jieba
-import sys
-
-sys.path.append("../../../../")
-
-jieba.dt.tmp_dir = os.path.expanduser(
-    "/cognitive_comp/dongxiaoqun/software/jieba/tmp/")
-# jieba.enable_parallel(8)
-jieba.initialize()
-
-logger = logging.get_logger(__name__)
-
-VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
-
-
-def load_vocab(vocab_file):
-    """Loads a vocabulary file into a dictionary."""
-    vocab = collections.OrderedDict()
-    with open(vocab_file, "r", encoding="utf-8") as reader:
-        tokens = reader.readlines()
-    for index, token in enumerate(tokens):
-        token = token.rstrip("\n")
-        vocab[token] = index
-    return vocab
-
-
-def whitespace_tokenize(text):
-    """Runs basic whitespace cleaning and splitting on a piece of text."""
-    text = text.strip()
-    if not text:
-        return []
-    tokens = text.split()
-    return tokens
-
-
-class PegasusTokenizer(PreTrainedTokenizer):
-    # copy from BertTokenizer
-    r"""
-    Construct a Pegasus tokenizer. Based on WordPiece.
-    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
-    this superclass for more information regarding those methods.
-    Args:
-        vocab_file (`str`):
-            File containing the vocabulary.
-        do_lower_case (`bool`, *optional*, defaults to `True`):
-            Whether or not to lowercase the input when tokenizing.
-        do_basic_tokenize (`bool`, *optional*, defaults to `True`):
-            Whether or not to do basic tokenization before WordPiece.
-        never_split (`Iterable`, *optional*):
-            Collection of tokens which will never be split during tokenization. Only has an effect when
-            `do_basic_tokenize=True`
-        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
-            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
-            token instead.
-        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
-            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
-            sequence classification or for a text and a question for question answering. It is also used as the last
-            token of a sequence built with special tokens.
-        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
-            The token used for padding, for example when batching sequences of different lengths.
-        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
-            The classifier token which is used when doing sequence classification (classification of the whole sequence
-            instead of per-token classification). It is the first token of the sequence when built with special tokens.
-        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
-            The token used for masking values. This is the token used when training this model with masked language
-            modeling. This is the token which the model will try to predict.
-        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
-            Whether or not to tokenize Chinese characters.
-            This should likely be deactivated for Japanese (see this
-            [issue](https://github.com/huggingface/transformers/issues/328)).
-        strip_accents (`bool`, *optional*):
-            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
-            value for `lowercase` (as in the original BERT).
-    """
-
-    vocab_files_names = VOCAB_FILES_NAMES
-    model_input_names = ["input_ids", "attention_mask"]
-
-    #     pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
-    #     pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
-    #     max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
-
-    def __init__(self,
-                 vocab_file,
-                 do_lower_case=True,
-                 do_basic_tokenize=True,
-                 never_split=None,
-                 pad_token="<pad>",
-                 eos_token="</s>",
-                 unk_token="<unk>",
-                 mask_token="<mask_2>",
-                 mask_token_sent="<mask_1>",
-                 additional_special_tokens=None,
-                 sep_token="[SEP]",
-                 cls_token="[CLS]",
-                 tokenize_chinese_chars=True,
-                 strip_accents=None,
-                 offset=100,
-                 pre_tokenizer=lambda x: jieba.cut(x, HMM=False),
-                 **kwargs):
-        self.offset = offset
-
-        if additional_special_tokens is not None:
-            if not isinstance(additional_special_tokens, list):
-                raise TypeError(
-                    f"additional_special_tokens should be of type {type(list)}, \
-                     but is {type(additional_special_tokens)}"
-                )
-
-            additional_special_tokens_extended = (
-                ([mask_token_sent] + additional_special_tokens)
-                if mask_token_sent not in additional_special_tokens
-                and mask_token_sent is not None else additional_special_tokens)
-
-            # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
-            additional_special_tokens_extended += [
-                f"<unk_{i}>" for i in range(
-                    len(additional_special_tokens_extended), self.offset - 1)
-            ]
-
-            if len(set(additional_special_tokens_extended)) != len(
-                    additional_special_tokens_extended):
-                raise ValueError(
-                    f"Please make sure that the provided additional_special_tokens \
-                        do not contain an incorrectly shifted list of <unk_x> tokens. \
-                        Found {additional_special_tokens_extended}."
-                )
-            additional_special_tokens = additional_special_tokens_extended
-        else:
-            additional_special_tokens = [
-                mask_token_sent
-            ] if mask_token_sent is not None else []
-            # additional_special_tokens += [f"<unk_{i}>" for i in range(3, self.offset)]
-
-        # print("additional_special_tokens: ", additional_special_tokens)
-
-        if not os.path.isfile(vocab_file):
-            raise ValueError(
-                f"Can't find a vocabulary file at path '{vocab_file}'. \
-                To load the vocabulary from a Google pretrained "
-                "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
-            )
-
-        super().__init__(
-            do_lower_case=do_lower_case,
-            do_basic_tokenize=do_basic_tokenize,
-            never_split=never_split,
-            unk_token=unk_token,
-            sep_token=sep_token,
-            pad_token=pad_token,
-            cls_token=cls_token,
-            mask_token=mask_token,
-            eos_token=eos_token,
-            tokenize_chinese_chars=tokenize_chinese_chars,
-            additional_special_tokens=additional_special_tokens,
-            strip_accents=strip_accents,
-            **kwargs,
-        )
-
-        self.pre_tokenizer = pre_tokenizer
-        self.mask_token_sent = mask_token_sent
-        self.vocab = load_vocab(vocab_file)
-
-        self.vocab[self.eos_token] = self.vocab.pop("[unused1]")
-        # self.vocab[self.eos_token] = self.vocab.pop("[unused2]")
-        self.vocab[self.pad_token] = self.vocab.pop("[PAD]")
-        self.vocab[self.unk_token] = self.vocab.pop("[UNK]")
-
-        if self.mask_token_sent is not None:
-            self.vocab[self.mask_token] = self.vocab.pop("[unused3]")
-            self.vocab[self.mask_token_sent] = self.vocab.pop("[unused2]")
-
-        self.ids_to_tokens = collections.OrderedDict([
-            (ids, tok) for tok, ids in self.vocab.items()
-        ])
-        self.do_basic_tokenize = do_basic_tokenize
-        if do_basic_tokenize:
-            self.basic_tokenizer = BasicTokenizer(
-                do_lower_case=do_lower_case,
-                never_split=never_split,
-                tokenize_chinese_chars=tokenize_chinese_chars,
-                strip_accents=strip_accents,
-            )
-        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab,
-                                                      unk_token=self.unk_token)
-
-    @property
-    def do_lower_case(self):
-        return self.basic_tokenizer.do_lower_case
-
-    @property
-    def vocab_size(self):
-        return len(self.vocab)
-
-    def get_vocab(self):
-        return dict(self.vocab, **self.added_tokens_encoder)
-
-    def _tokenize(self, text):
-        split_tokens = []
-        # print("pegasus_tokenizer: ", text)
-        for text in self.pre_tokenizer(text):
-            if text in self.vocab:
-                split_tokens.append(text)
-            else:
-                if self.do_basic_tokenize:
-                    for token in self.basic_tokenizer.tokenize(
-                            text, never_split=self.all_special_tokens):
-
-                        # If the token is part of the never_split set
-                        if token in self.basic_tokenizer.never_split:
-                            split_tokens.append(token)
-                        else:
-                            split_tokens += self.wordpiece_tokenizer.tokenize(
-                                token)
-                else:
-                    split_tokens = self.wordpiece_tokenizer.tokenize(text)
-        return split_tokens
-
-    def _convert_token_to_id(self, token):
-        """Converts a token (str) in an id using the vocab."""
-        return self.vocab.get(token, self.vocab.get(self.unk_token))
-
-    def _convert_id_to_token(self, index):
-        """Converts an index (integer) in a token (str) using the vocab."""
-        return self.ids_to_tokens.get(index, self.unk_token)
-
-    @staticmethod
-    def _cjk_punctuation():
-        return u'\uff02\uff03\uff04\uff05\uff06\uff07\uff08\uff09\uff0a\uff0b\uff0c\uff0d\uff0f\uff1a\uff1b\uff1c\uff1d\
-            \uff1e\uff20\uff3b\uff3c\uff3d\uff3e\uff3f\uff40\uff5b\uff5c\uff5d\uff5e\uff5f\uff60\uff62\
-            \uff63\uff64\u3000\u3001\u3003\u3008\u3009\u300a\u300b\u300c\u300d\u300e\u300f\u3010\u3011\u3014\
-            \u3015\u3016\u3017\u3018\u3019\u301a\u301b\u301c\u301d\u301e\u301f\u3030\u303e\u303f\u2013\u2014\
-            \u2018\u2019\u201b\u201c\u201d\u201e\u201f\u2026\u2027\ufe4f\ufe51\ufe54\u00b7\uff01\uff1f\uff61\u3002'
-
-    def convert_ids_to_tokens(
-            self,
-            ids: Union[int, List[int]],
-            skip_special_tokens: bool = False) -> Union[str, List[str]]:
-        """
-        Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
-        added tokens.
-        Args:
-            ids (`int` or `List[int]`):
-                The token id (or token ids) to convert to tokens.
-            skip_special_tokens (`bool`, *optional*, defaults to `False`):
-                Whether or not to remove special tokens in the decoding.
-        Returns:
-            `str` or `List[str]`: The decoded token(s).
-        """
-        if isinstance(ids, int):
-            if ids in self.added_tokens_decoder:
-                return self.added_tokens_decoder[ids]
-            else:
-                return self._convert_id_to_token(ids)
-        tokens = []
-        for index in ids:
-            index = int(index)
-            if skip_special_tokens and index in self.all_special_ids and index != 2:
-                continue
-            if index in self.added_tokens_decoder:
-                tokens.append(self.added_tokens_decoder[index])
-            else:
-                tokens.append(self._convert_id_to_token(index))
-        return tokens
-
-    def convert_tokens_to_string(self, tokens):
-        """Converts a sequence of tokens (string) in a single string."""
-        # for token in
-        # tokens = tokens or self.ids_to_tokens(ids)
-        # tokens = [token for token in tokens if not self._is_special(token)]
-
-        text = ''
-        for i, token in enumerate(tokens):
-            if token[:2] == '##':
-                text += token[2:]
-            elif len(token) == 1 and _is_chinese_char(ord(token)):
-                text += token
-            elif len(token) == 1 and _is_punctuation(token):
-                text += token
-                text += ' '
-            elif i > 0 and _is_chinese_char(ord(text[-1])):
-                text += token
-            elif tokens == "</s>":
-                continue
-            else:
-                text += ' '
-                text += token
-
-        text = re.sub(' +', ' ', text)
-        text = re.sub('\' (re|m|s|t|ve|d|ll) ', '\'\\1 ', text)
-        punctuation = re.sub(' +', '', self._cjk_punctuation()).strip() + '+-/={(<['
-        punctuation_regex = '|'.join([re.escape(p) for p in punctuation])
-        punctuation_regex = '(%s) ' % punctuation_regex
-        text = re.sub(punctuation_regex, '\\1', text)
-        text = re.sub(r'(\d\.) (\d)', '\\1\\2', text)
-
-        return text.strip()
-        # out_string = " ".join(tokens).replace(" ##", "").strip()
-
-    def build_inputs_with_special_tokens(
-            self,
-            token_ids_0: List[int],
-            token_ids_1: Optional[List[int]] = None) -> List[int]:
-        """
-        Build model inputs from a sequence or a pair of sequences for sequence classification tasks by concatenating
-        and adding special tokens. A PEGASUS sequence has the following format, where `X` represents the sequence:
-        - single sequence: `X </s>`
-        - pair of sequences: `A B </s>` (not intended use)
-        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
-        separator.
-        Args:
-            token_ids_0 (`List[int]`):
-                List of IDs to which the special tokens will be added.
-            token_ids_1 (`List[int]`, *optional*):
-                Optional second list of IDs for sequence pairs.
-        Returns:
-            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
-        """
-        if token_ids_1 is None:
-            return token_ids_0 + [self.eos_token_id]
-        return token_ids_0 + token_ids_1 + [self.eos_token_id]
-
-    def _special_token_mask(self, seq):
-        all_special_ids = set(
-            self.all_special_ids)  # call it once instead of inside list comp
-        # all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special
-
-        return [1 if x in all_special_ids else 0 for x in seq]
-
-    def get_special_tokens_mask(
-            self,
-            token_ids_0: List[int],
-            token_ids_1: Optional[List[int]] = None,
-            already_has_special_tokens: bool = False) -> List[int]:
-        """
-        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer `prepare_for_model` method.
-        Args:
-            token_ids_0 (`List[int]`):
-                List of IDs.
-            token_ids_1 (`List[int]`, *optional*):
-                Optional second list of IDs for sequence pairs.
-            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
-                Whether or not the token list is already formatted with special tokens for the model.
-        Returns:
-            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
-        """
-
-        if already_has_special_tokens:
-            return self._special_token_mask(token_ids_0)
-        elif token_ids_1 is None:
-            return self._special_token_mask(token_ids_0) + [self.eos_token_id]
-        else:
-            return self._special_token_mask(token_ids_0 +
-                                            token_ids_1) + [self.eos_token_id]
-
-    def num_special_tokens_to_add(self, pair=False):
-        """Just EOS"""
-        return 1
-
-    def save_vocabulary(self,
-                        save_directory: str,
-                        filename_prefix: Optional[str] = None) -> Tuple[str]:
-        index = 0
-        if os.path.isdir(save_directory):
-            vocab_file = os.path.join(
-                save_directory,
-                (filename_prefix + "-" if filename_prefix else "") +
-                VOCAB_FILES_NAMES["vocab_file"])
-        else:
-            vocab_file = (filename_prefix +
-                          "-" if filename_prefix else "") + save_directory
-        with open(vocab_file, "w", encoding="utf-8") as writer:
-            for token, token_index in sorted(self.vocab.items(),
-                                             key=lambda kv: kv[1]):
-                if index != token_index:
-                    logger.warning(
-                        f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
-                        " Please check that the vocabulary is not corrupted!")
-                    index = token_index
-                writer.write(token + "\n")
-                index += 1
-        return (vocab_file, )
-
-
-class BasicTokenizer(object):
-    """
-    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).
-    Args:
-        do_lower_case (`bool`, *optional*, defaults to `True`):
-            Whether or not to lowercase the input when tokenizing.
-        never_split (`Iterable`, *optional*):
-            Collection of tokens which will never be split during tokenization. Only has an effect when
-            `do_basic_tokenize=True`
-        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
-            Whether or not to tokenize Chinese characters.
-            This should likely be deactivated for Japanese (see this
-            [issue](https://github.com/huggingface/transformers/issues/328)).
-        strip_accents: (`bool`, *optional*):
-            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
-            value for `lowercase` (as in the original BERT).
-    """
-
-    def __init__(self,
-                 do_lower_case=True,
-                 never_split=None,
-                 tokenize_chinese_chars=True,
-                 strip_accents=None):
-        if never_split is None:
-            never_split = []
-        self.do_lower_case = do_lower_case
-        self.never_split = set(never_split)
-        self.tokenize_chinese_chars = tokenize_chinese_chars
-        self.strip_accents = strip_accents
-
-    def tokenize(self, text, never_split=None):
-        """
-        Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see
-        WordPieceTokenizer.
-        Args:
-            never_split (`List[str]`, *optional*)
-                Kept for backward compatibility purposes. Now implemented directly at the base class level (see
-                [`PreTrainedTokenizer.tokenize`]) List of token not to split.
-        """
-        # union() returns a new set by concatenating the two sets.
-        never_split = self.never_split.union(
-            set(never_split)) if never_split else self.never_split
-        text = self._clean_text(text)
-
-        # This was added on November 1st, 2018 for the multilingual and Chinese
-        # models. This is also applied to the English models now, but it doesn't
-        # matter since the English models were not trained on any Chinese data
-        # and generally don't have any Chinese data in them (there are Chinese
-        # characters in the vocabulary because Wikipedia does have some Chinese
-        # words in the English Wikipedia.).
-        if self.tokenize_chinese_chars:
-            text = self._tokenize_chinese_chars(text)
-        orig_tokens = whitespace_tokenize(text)
-        split_tokens = []
-        for token in orig_tokens:
-            if token not in never_split:
-                if self.do_lower_case:
-                    token = token.lower()
-                    if self.strip_accents is not False:
-                        token = self._run_strip_accents(token)
-                elif self.strip_accents:
-                    token = self._run_strip_accents(token)
-            split_tokens.extend(self._run_split_on_punc(token, never_split))
-
-        output_tokens = whitespace_tokenize(" ".join(split_tokens))
-        return output_tokens
-
-    def _run_strip_accents(self, text):
-        """Strips accents from a piece of text."""
-        text = unicodedata.normalize("NFD", text)
-        output = []
-        for char in text:
-            cat = unicodedata.category(char)
-            if cat == "Mn":
-                continue
-            output.append(char)
-        return "".join(output)
-
-    def _run_split_on_punc(self, text, never_split=None):
-        """Splits punctuation on a piece of text."""
-        if never_split is not None and text in never_split:
-            return [text]
-        chars = list(text)
-        i = 0
-        start_new_word = True
-        output = []
-        while i < len(chars):
-            char = chars[i]
-            if _is_punctuation(char):
-                output.append([char])
-                start_new_word = True
-            else:
-                if start_new_word:
-                    output.append([])
-                start_new_word = False
-                output[-1].append(char)
-            i += 1
-
-        return ["".join(x) for x in output]
-
-    def _tokenize_chinese_chars(self, text):
-        """Adds whitespace around any CJK character."""
-        output = []
-        for char in text:
-            cp = ord(char)
-            if self._is_chinese_char(cp):
-                output.append(" ")
-                output.append(char)
-                output.append(" ")
-            else:
-                output.append(char)
-        return "".join(output)
-
-    def _is_chinese_char(self, cp):
-        """Checks whether CP is the codepoint of a CJK character."""
-        # This defines a "chinese character" as anything in the CJK Unicode block:
-        #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
-        #
-        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
-        # despite its name. The modern Korean Hangul alphabet is a different block,
-        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
-        # space-separated words, so they are not treated specially and handled
-        # like the all of the other languages.
-        if ((cp >= 0x4E00 and cp <= 0x9FFF)
-            or (cp >= 0x3400 and cp <= 0x4DBF)  #
-            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
-            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
-            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
-            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
-            or (cp >= 0xF900 and cp <= 0xFAFF)
-                or (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
-            return True
-
-        return False
-
-    def _clean_text(self, text):
-        """Performs invalid character removal and whitespace cleanup on text."""
-        output = []
-        for char in text:
-            cp = ord(char)
-            if cp == 0 or cp == 0xFFFD or _is_control(char):
-                continue
-            if _is_whitespace(char):
-                output.append(" ")
-            else:
-                output.append(char)
-        return "".join(output)
-
-
-class WordpieceTokenizer(object):
-    """Runs WordPiece tokenization."""
-
-    def __init__(self, vocab, unk_token, max_input_chars_per_word=100):
-        self.vocab = vocab
-        self.unk_token = unk_token
-        self.max_input_chars_per_word = max_input_chars_per_word
-
-    def tokenize(self, text):
-        """
-        Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
-        tokenization using the given vocabulary.
-        For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.
-        Args:
-            text: A single token or whitespace separated tokens. This should have
-                already been passed through *BasicTokenizer*.
-        Returns:
-            A list of wordpiece tokens.
-        """
-
-        output_tokens = []
-        for token in whitespace_tokenize(text):
-            chars = list(token)
-            if len(chars) > self.max_input_chars_per_word:
-                output_tokens.append(self.unk_token)
-                continue
-
-            is_bad = False
-            start = 0
-            sub_tokens = []
-            while start < len(chars):
-                end = len(chars)
-                cur_substr = None
-                while start < end:
-                    substr = "".join(chars[start:end])
-                    if start > 0:
-                        substr = "##" + substr
-                    if substr in self.vocab:
-                        cur_substr = substr
-                        break
-                    end -= 1
-                if cur_substr is None:
-                    is_bad = True
-                    break
-                sub_tokens.append(cur_substr)
-                start = end
-
-            if is_bad:
-                output_tokens.append(self.unk_token)
-            else:
-                output_tokens.extend(sub_tokens)
-        return output_tokens