_convert_id_to_token

tokenization_xgen.py

@@ -155,7 +155,7 @@ class XgenTokenizer(PreTrainedTokenizer):
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
-        return self.encoder.decode_single_token_bytes(index)
+        return self.encoder.decode_single_token_bytes(index).decode("utf-8")
 
     def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
         if skip_special_tokens:
@@ -231,4 +231,4 @@ class XgenTokenizer(PreTrainedTokenizer):
 
     # has no vocab file
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
-        return ()
+        return ()