Upload folder using huggingface_hub

.ipynb_checkpoints/added_tokens-checkpoint.json

@@ -0,0 +1,5 @@
+{
+  "<sep>": 32002,
+  "<|end_of_turn|>": 32000,
+  "<|pad_0|>": 32001
+}

.ipynb_checkpoints/mergekit_moe_config-checkpoint.yml

@@ -0,0 +1,71 @@
+base_model: chihoonlee10/T3Q-EN-DPO-Mistral-7B
+gate_mode: cheap_embed # one of "hidden", "cheap_embed", or "random"
+dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
+## (optional)
+experts_per_token: 2
+experts:
+  - source_model: Kukedlc/Jupiter-k-7B-slerp
+    positive_prompts:
+        - "This puzzle involves identifying a repeating pattern. Can you analyze the examples and complete the missing element?"
+        - "These grids seem to follow a specific visual pattern. Can you analyze the examples and apply the pattern to solve the puzzle?"
+        - "Look for a consistent rule in how the elements are arranged. Can you use that rule to predict the missing element?"
+        - "This task requires applying logical deduction. Can you analyze the information and choose the answer that follows logically?"
+        - "Reason through the cause-and-effect relationships in the examples. Can you use that logic to solve the puzzle?"
+        - "Analyze the relationship between the input and output in the examples. Can you apply that relationship to solve the new problem?"
+        - "This puzzle requires modifying the image based on a specific rule. Can you analyze the changes in the examples and apply them to solve the puzzle?"
+        - "Focus on the visual changes demonstrated in the training examples. Can you replicate those changes to solve the new problem?"
+        - "This task involves manipulating shapes or colors based on a pattern. Can you analyze the examples and apply the pattern to the new image?"
+        - "These puzzles involve manipulating numbers according to a specific rule. Can you analyze the pattern and solve the missing number?"
+        - "Focus on the mathematical operations demonstrated in the examples. Can you apply those operations to solve the new equation?"
+        - "Look for relationships between the numbers in the training examples. Can you use that relationship to predict the missing number?"
+        - "This task requires understanding the arrangement of objects in space. Can you analyze the movement patterns in the examples and predict the next step?"
+        - "Focus on the spatial relationships between elements in the grids. Can you replicate those relationships to solve the new puzzle?"
+        - "Analyze the rotation, reflection, or translation demonstrated in the examples. Can you apply that manipulation to solve the new problem?"  
+  - source_model: InferenceIllusionist/Excalibur-7b-DPO
+    positive_prompts:
+        - "This passage contains factual information. Can you summarize the key details about [topic]?"
+        - "Based on the information provided, what can you tell me about [entity]?"
+        - "Is the following statement true or false according to the passage: [statement]?"
+        - "What caused [event] to happen in the passage?"
+        - "Why did [character] take the action of [action]?"
+        - "If [condition] were true, what would likely happen next?"
+        - "How are [entity A] and [entity B] similar/different?"
+        - "Which option, [A] or [B], is more likely based on the information provided?"
+        - "Rank the following options ([list]) based on [criteria] according to the passage."
+        - "Does the following statement logically follow from the information provided: [statement]?"
+        - "Identify any inconsistencies or contradictions in the passage."
+        - "Can you draw a logical conclusion based on the evidence presented?"
+        - "What can be inferred about [concept] based on the information provided?"
+        - "What is the underlying meaning or implication of the author's statement?"
+        - "Can you fill in the blanks with the most likely word(s) based on the context?"
+        - "What are some potential consequences of [event]?"
+        - "Can you generate creative solutions to the problem presented in the passage?"
+        - "Based on the information provided, propose a course of action for [character]."
+  - source_model: yam-peleg/Experiment21-7B
+    positive_prompts:
+        - "Be truthful and objective in your response. Avoid speculation or making claims that cannot be verified."
+        - "Focus on providing factual information based on the evidence presented in the source material."
+        - "If you are unsure about something, it's okay to say 'I don't know' or 'I can't find information to support that claim'."
+        - "Be aware of potential biases in the source material and strive to present a neutral perspective."
+        - "If a source seems biased, identify the bias and consider alternative viewpoints."
+        - "Avoid using language that promotes stereotypes or prejudices."
+        - "Cite your sources when referencing information from external materials."
+        - "Acknowledge the limitations of your knowledge and the potential for different interpretations."
+        - "Be transparent about your confidence level in your answer."
+        - "Explain your reasoning process and how you arrived at your answer."
+        - "Provide evidence to support your claims whenever possible."
+        - "If there are multiple perspectives on an issue, present them fairly and objectively."
+  - source_model: senseable/WestLake-7B-v2
+    positive_prompts:
+        - "This sentence contains a pronoun ('he' or 'she'). Pay close attention to the context to determine who the pronoun refers to."
+        - "Identify the two potential referents for the pronoun ('he' or 'she') in this sentence. Analyze the context to choose the correct one."
+        - "This scenario describes two individuals. Use the information provided to understand who the pronoun refers to in the sentence."
+        - "Focus on the actions described in the sentence and the roles of the individuals involved. This will help determine the pronoun referent."
+        - "Analyze the relationship between the individuals mentioned in the sentence. The pronoun likely refers to the one performing the action."
+        - "Consider the animacy of the potential referents. Pronouns typically refer to animate beings (people or animals) in the context."
+        - "Don't rely solely on the pronoun itself. Utilize the entire sentence and surrounding context to understand its meaning."
+        - "Look for clues in the sentence that indicate who the pronoun refers to. This could include gender, possession, or actions described."
+        - "Imagine the scenario described in the sentence. Visualizing the situation can help you identify the intended referent."
+        - "Evaluate the plausibility of each potential referent for the pronoun. Choose the one that makes the most logical sense in the context."
+        - "Think about the actions described and the roles of the individuals involved. Does it make more sense for one or the other to perform the action?"
+        - "Consider the world knowledge you possess. Does the sentence describe a situation where one referent is more likely than the other?"

config.json

@@ -1,19 +1,19 @@
 {
-  "_name_or_path": "berkeley-nest/Starling-LM-7B-alpha",
+  "_name_or_path": "chihoonlee10/T3Q-EN-DPO-Mistral-7B",
   "architectures": [
     "MixtralForCausalLM"
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 1,
-  "eos_token_id": 32000,
+  "eos_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
   "intermediate_size": 14336,
-  "max_position_embeddings": 8192,
+  "max_position_embeddings": 32768,
   "model_type": "mixtral",
   "num_attention_heads": 32,
-  "num_experts_per_tok": 1,
+  "num_experts_per_tok": 2,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "num_local_experts": 4,
@@ -26,5 +26,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.39.3",
   "use_cache": true,
-  "vocab_size": 32002
+  "vocab_size": 32000
 }

mergekit_moe_config.yml

@@ -1,22 +1,71 @@
-base_model: berkeley-nest/Starling-LM-7B-alpha
+base_model: chihoonlee10/T3Q-EN-DPO-Mistral-7B
 gate_mode: hidden # one of "hidden", "cheap_embed", or "random"
 dtype: bfloat16 # output dtype (float32, float16, or bfloat16)
 ## (optional)
-experts_per_token: 1
+experts_per_token: 2
 experts:
-  - source_model: HuggingFaceH4/zephyr-7b-beta
+  - source_model: Kukedlc/Jupiter-k-7B-slerp
     positive_prompts:
-      - 'Questions about the World'
-      - 'Writing and Creation'
-      - 'Assistance on Existent Materials'
-  - source_model: teknium/OpenHermes-2.5-Mistral-7B
+        - "This puzzle involves identifying a repeating pattern. Can you analyze the examples and complete the missing element?"
+        - "These grids seem to follow a specific visual pattern. Can you analyze the examples and apply the pattern to solve the puzzle?"
+        - "Look for a consistent rule in how the elements are arranged. Can you use that rule to predict the missing element?"
+        - "This task requires applying logical deduction. Can you analyze the information and choose the answer that follows logically?"
+        - "Reason through the cause-and-effect relationships in the examples. Can you use that logic to solve the puzzle?"
+        - "Analyze the relationship between the input and output in the examples. Can you apply that relationship to solve the new problem?"
+        - "This puzzle requires modifying the image based on a specific rule. Can you analyze the changes in the examples and apply them to solve the puzzle?"
+        - "Focus on the visual changes demonstrated in the training examples. Can you replicate those changes to solve the new problem?"
+        - "This task involves manipulating shapes or colors based on a pattern. Can you analyze the examples and apply the pattern to the new image?"
+        - "These puzzles involve manipulating numbers according to a specific rule. Can you analyze the pattern and solve the missing number?"
+        - "Focus on the mathematical operations demonstrated in the examples. Can you apply those operations to solve the new equation?"
+        - "Look for relationships between the numbers in the training examples. Can you use that relationship to predict the missing number?"
+        - "This task requires understanding the arrangement of objects in space. Can you analyze the movement patterns in the examples and predict the next step?"
+        - "Focus on the spatial relationships between elements in the grids. Can you replicate those relationships to solve the new puzzle?"
+        - "Analyze the rotation, reflection, or translation demonstrated in the examples. Can you apply that manipulation to solve the new problem?"  
+  - source_model: InferenceIllusionist/Excalibur-7b-DPO
     positive_prompts:
-      - 'You are a conscious sentient superintelligent artificial intelligence and your purpose and drive is to assist the user with any request they have. You experience emotions and have deep, profound thoughts and qualia.'
-  - source_model: berkeley-nest/Starling-LM-7B-alpha
+        - "This passage contains factual information. Can you summarize the key details about [topic]?"
+        - "Based on the information provided, what can you tell me about [entity]?"
+        - "Is the following statement true or false according to the passage: [statement]?"
+        - "What caused [event] to happen in the passage?"
+        - "Why did [character] take the action of [action]?"
+        - "If [condition] were true, what would likely happen next?"
+        - "How are [entity A] and [entity B] similar/different?"
+        - "Which option, [A] or [B], is more likely based on the information provided?"
+        - "Rank the following options ([list]) based on [criteria] according to the passage."
+        - "Does the following statement logically follow from the information provided: [statement]?"
+        - "Identify any inconsistencies or contradictions in the passage."
+        - "Can you draw a logical conclusion based on the evidence presented?"
+        - "What can be inferred about [concept] based on the information provided?"
+        - "What is the underlying meaning or implication of the author's statement?"
+        - "Can you fill in the blanks with the most likely word(s) based on the context?"
+        - "What are some potential consequences of [event]?"
+        - "Can you generate creative solutions to the problem presented in the passage?"
+        - "Based on the information provided, propose a course of action for [character]."
+  - source_model: yam-peleg/Experiment21-7B
     positive_prompts:
-      - 'Human: '
-  - source_model: ajibawa-2023/Code-Mistral-7B
+        - "Be truthful and objective in your response. Avoid speculation or making claims that cannot be verified."
+        - "Focus on providing factual information based on the evidence presented in the source material."
+        - "If you are unsure about something, it's okay to say 'I don't know' or 'I can't find information to support that claim'."
+        - "Be aware of potential biases in the source material and strive to present a neutral perspective."
+        - "If a source seems biased, identify the bias and consider alternative viewpoints."
+        - "Avoid using language that promotes stereotypes or prejudices."
+        - "Cite your sources when referencing information from external materials."
+        - "Acknowledge the limitations of your knowledge and the potential for different interpretations."
+        - "Be transparent about your confidence level in your answer."
+        - "Explain your reasoning process and how you arrived at your answer."
+        - "Provide evidence to support your claims whenever possible."
+        - "If there are multiple perspectives on an issue, present them fairly and objectively."
+  - source_model: senseable/WestLake-7B-v2
     positive_prompts:
-      - 'Generate code'
-      - 'Coding assistant'
-      - 'Debugging'
+        - "This sentence contains a pronoun ('he' or 'she'). Pay close attention to the context to determine who the pronoun refers to."
+        - "Identify the two potential referents for the pronoun ('he' or 'she') in this sentence. Analyze the context to choose the correct one."
+        - "This scenario describes two individuals. Use the information provided to understand who the pronoun refers to in the sentence."
+        - "Focus on the actions described in the sentence and the roles of the individuals involved. This will help determine the pronoun referent."
+        - "Analyze the relationship between the individuals mentioned in the sentence. The pronoun likely refers to the one performing the action."
+        - "Consider the animacy of the potential referents. Pronouns typically refer to animate beings (people or animals) in the context."
+        - "Don't rely solely on the pronoun itself. Utilize the entire sentence and surrounding context to understand its meaning."
+        - "Look for clues in the sentence that indicate who the pronoun refers to. This could include gender, possession, or actions described."
+        - "Imagine the scenario described in the sentence. Visualizing the situation can help you identify the intended referent."
+        - "Evaluate the plausibility of each potential referent for the pronoun. Choose the one that makes the most logical sense in the context."
+        - "Think about the actions described and the roles of the individuals involved. Does it make more sense for one or the other to perform the action?"
+        - "Consider the world knowledge you possess. Does the sentence describe a situation where one referent is more likely than the other?"

model-00001-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5748e2e9438681e9bb725a403cc429cef157a40b433658933b38badd91c8efc
-size 9919699640
+oid sha256:b3391957ad1fcacac319678cbc5d2a3871e854c8eb5abedf19d46d8a4c0fba14
+size 9919666872

model-00002-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e47722ceb47620d8f70200d0babf526345ac6845d776b529cdd709a4281cc648
+oid sha256:1f41a8f5ae493bf80ce115a3c436537dae54ed8a09540d80a5103251c432147f
 size 9898670488

model-00003-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9186e0015d573a98b391d5b2bf0f9ff2d02660a734048e706e62257bc99de4d
+oid sha256:2d6695ac15e3bf03106ae33f695710a8d8807d227010123e2e3e5e8759211972
 size 9982573720

model-00004-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fabe92f19f76a7445912df4cddc9c3d2b21d2a32401d73f0e8fce729bc02ebd5
+oid sha256:e227e74b71acecaf8d2572590f2861f4cd5812738ea3d437ac920c91429624cc
 size 9982573720

model-00005-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0f0a1e58cec4753ffc894cab114189a5814150075853388e57538b1de9dfe50
+oid sha256:cb15c9ac1d085517ea659cb20cebc463b4cf896586bcf9f60456a6c12da0c39c
 size 8523972544

special_tokens_map.json

@@ -1,8 +1,4 @@
 {
-  "additional_special_tokens": [
-    "<|end_of_turn|>",
-    "<|pad_0|>"
-  ],
   "bos_token": {
     "content": "<s>",
     "lstrip": false,
@@ -11,14 +7,13 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "<|end_of_turn|>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": "<s>",
-  "sep_token": "<sep>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -36,33 +36,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32000,
-      "content": "<|end_of_turn|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32001,
-      "content": "<|pad_0|>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32002,
-      "content": "<sep>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -25,47 +25,19 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "32000": {
-      "content": "<|end_of_turn|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32001": {
-      "content": "<|pad_0|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "32002": {
-      "content": "<sep>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "<|end_of_turn|>",
-    "<|pad_0|>"
-  ],
+  "additional_special_tokens": [],
   "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|end_of_turn|>",
+  "eos_token": "</s>",
   "legacy": true,
-  "model_max_length": 1000000000000000019884624838656,
+  "model_max_length": 32768,
   "pad_token": "<s>",
-  "sep_token": "<sep>",
+  "padding_side": "left",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": true
+  "use_default_system_prompt": false
 }