Upload 9 files

README.md

@@ -1,3 +1,121 @@
 ---
-license: other
+language:
+- en
+tags:
+- llama
 ---
+
+# OpenChat: Less is More for Open-source Models
+
+OpenChat is a series of open-source language models fine-tuned on very little diverse and high-quality multi-round conversations. The [dataset](https://huggingface.co/datasets/openchat/openchat_sharegpt4_dataset) contains only ~6K GPT-4 conversations filtered from the 90K ShareGPT conversations.
+
+Generic models:
+
+ - OpenChat: based on LLaMA-13B (2048 context length)
+   - **105.7%** of ChatGPT score on Vicuna GPT-4 evaluation
+   - **80.87%** Win-rate on AlpacaEval
+   - **🚀 Only used 6K data for finetuning!!!**
+ - OpenChat-8192: based on LLaMA-13B  (extended to 8192 context length)
+   - **106.6%** of ChatGPT score on Vicuna GPT-4 evaluation
+
+Code models:
+
+ - OpenCoderPlus: based on StarCoderPlus (native 8192 context length)
+   - **102.5%** of ChatGPT score on Vicuna GPT-4 evaluation
+   - **78.70%** Win-rate on AlpacaEval
+
+**NOTE:** Please load the pretrained models using *bfloat16*
+
+## Conversation Template
+
+The conversation template **involves concatenating tokens**.
+
+Besides base model vocabulary, an end-of-turn token `<|end_of_turn|>` is added, with id `eot_token_id`.
+
+```python
+# OpenChat
+[bos_token_id] + tokenize("Human: ") + tokenize(user_question) + [eot_token_id] + tokenize("Assistant: ")
+# OpenCoder
+tokenize("User:") + tokenize(user_question) + [eot_token_id] + tokenize("Assistant:")
+```
+
+*Hint: In BPE, `tokenize(A) + tokenize(B)` does not always equals to `tokenize(A + B)`*
+
+Following is the code for generating the conversation templates:
+
+```python
+@dataclass
+class ModelConfig:
+    # Prompt
+    system: Optional[str]
+
+    role_prefix: dict
+    ai_role: str
+    eot_token: str
+    bos_token: Optional[str] = None
+
+    # Get template
+    def generate_conversation_template(self, tokenize_fn, tokenize_special_fn, message_list):
+        tokens = []
+        masks = []
+
+        # begin of sentence (bos)
+        if self.bos_token:
+            t = tokenize_special_fn(self.bos_token)
+            tokens.append(t)
+            masks.append(False)
+
+        # System
+        if self.system:
+            t = tokenize_fn(self.system) + [tokenize_special_fn(self.eot_token)]
+            tokens.extend(t)
+            masks.extend([False] * len(t))
+
+        # Messages
+        for idx, message in enumerate(message_list):
+            # Prefix
+            t = tokenize_fn(self.role_prefix[message["from"]])
+            tokens.extend(t)
+            masks.extend([False] * len(t))
+
+            # Message
+            if "value" in message:
+                t = tokenize_fn(message["value"]) + [tokenize_special_fn(self.eot_token)]
+                tokens.extend(t)
+                masks.extend([message["from"] == self.ai_role] * len(t))
+            else:
+                assert idx == len(message_list) - 1, "Empty message for completion must be on the last."
+
+        return tokens, masks
+
+
+MODEL_CONFIG_MAP = {
+    # OpenChat / OpenChat-8192
+    "openchat": ModelConfig(
+        # Prompt
+        system=None,
+
+        role_prefix={
+            "human": "Human: ",
+            "gpt": "Assistant: "
+        },
+        ai_role="gpt",
+        eot_token="<|end_of_turn|>",
+        bos_token="<s>",
+    ),
+
+    # OpenCoder / OpenCoderPlus
+    "opencoder": ModelConfig(
+        # Prompt
+        system=None,
+
+        role_prefix={
+            "human": "User:",
+            "gpt": "Assistant:"
+        },
+        ai_role="gpt",
+        eot_token="<|end_of_turn|>",
+        bos_token=None,
+    )
+}
+```

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|end_of_turn|>": 32000
+}

config.json

@@ -0,0 +1,23 @@
+{
+  "_name_or_path": "/dev/shm/LLaMA_13B_with_EOT_token_8192_positions",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 8192,
+  "model_type": "llama",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.30.1",
+  "use_cache": true,
+  "vocab_size": 32001
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.30.1"
+}

openchat_8192-GPTQ.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8da10fb2e99bbb487c4b37dc21dd9047baaf1f88a610d81ae92f3dcb146a5d5
+size 7012695856

special_tokens_map.json

@@ -0,0 +1,26 @@
+{
+  "additional_special_tokens": [
+    "<|end_of_turn|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}