Update _decode method to accept integer element and convert it to sequence (#30)

- Update _decode method to accept integer element and convert it to sequence (154b41190c5d1ce10b4c8eea23a1080c2f7c81fe)


Co-authored-by: Ramkrithik S <[email protected]>

tokenization_xgen.py

@@ -169,7 +169,9 @@ class XgenTokenizer(PreTrainedTokenizer):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.encoder.decode_single_token_bytes(index).decode("utf-8")
 
-    def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
+    def _decode(self, token_ids, skip_special_tokens: bool = False, **kwargs):
+        if not isinstance(token_ids, list):
+            token_ids = [token_ids]
         if skip_special_tokens:
             token_ids = [t for t in token_ids if t not in self.all_special_ids]
         return self.encoder.decode(token_ids)