Upload folder using huggingface_hub

README.md

@@ -0,0 +1,55 @@
+---
+license: apache-2.0
+tags:
+- moe
+- frankenmoe
+- merge
+- mergekit
+- lazymergekit
+- lex-hue/Delexa-7b
+- allknowingroger/Neurallaymons-7B-slerp
+base_model:
+- lex-hue/Delexa-7b
+- allknowingroger/Neurallaymons-7B-slerp
+---
+
+# DelexaMaths-12B-MoE
+
+DelexaMaths-12B-MoE is a Mixture of Experts (MoE) made with the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
+* [lex-hue/Delexa-7b](https://huggingface.co/lex-hue/Delexa-7b)
+* [allknowingroger/Neurallaymons-7B-slerp](https://huggingface.co/allknowingroger/Neurallaymons-7B-slerp)
+
+## 🧩 Configuration
+
+```yaml
+base_model: lex-hue/Delexa-7b
+experts:
+  - source_model: lex-hue/Delexa-7b
+    positive_prompts: ["what"]
+  - source_model: allknowingroger/Neurallaymons-7B-slerp
+    positive_prompts: ["math"]
+```
+
+## 💻 Usage
+
+```python
+!pip install -qU transformers bitsandbytes accelerate
+
+from transformers import AutoTokenizer
+import transformers
+import torch
+
+model = "allknowingroger/DelexaMaths-12B-MoE"
+
+tokenizer = AutoTokenizer.from_pretrained(model)
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model,
+    model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
+)
+
+messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
+prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "lex-hue/Delexa-7b",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "lex-hue/Delexa-7b--configuration_mistral.MistralConfig",
+    "AutoModelForCausalLM": "lex-hue/Delexa-7b--modeling_mistral_yarn.MistralForCausalLM"
+  },
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "max_sequence_length": 131072,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 2,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 16.0,
+    "finetuned": true,
+    "original_max_position_embeddings": 8192,
+    "type": "yarn"
+  },
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}

model-1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0ed3e1abc223db072ad226b02aaaf70e38b8f8f48234933491aefc3ceb4a10f
+size 1933849864

model-10.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e001c599b4d9b1a7f51fae5e7cd21ca3a539d1e9ce7edf1c13048eff71399c5
+size 1996490952

model-11.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0d609b4bebed499eb5b59849474ab6a72458ea499ae5ca15136fc1dbc985b35
+size 1996490952

model-12.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:331c11642cb1b59144c7f2e622ab44b9253edfc830889d363a1870cf22d0f764
+size 1979981504

model-2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:901bde884ad4df4ce87d72900b856d96cc4d1e3ac16aa064da4ba7abf3e0e485
+size 1996490944

model-3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5274a73f22da9b154deb8cf3f0bce349b4c1720d787280311dcd94ff5cb25f53
+size 1996490952

model-4.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7de73fb51878d99af9e560ccc36392d40bf8052114a55ddc7a9d48d18d940d21
+size 1996490952

model-5.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c38be3d7a410510d190ed45f9b5f54b322a11eb27722a44b3a2eb96ce930a952
+size 1996490936

model-6.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66bec08c111a50de1cd41fb5d1268ff4bc463453061742fb06a09dca429220df
+size 1996490944

model-7.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e058db1e56b36870d6b9427654ffca1e69b17d1753a6c9c72671b80d174cf3ed
+size 1996490952

model-8.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f0771061e6dbe07a9b7c891b7b817b0d02c1925819038e1196184b3277c6d0c
+size 1996490944

model-9.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d64cb10858a71ae3b98f7b204a9205a028b57830c3af3170002bebfa4534818b
+size 1996490936