Hugging Face's logo

ibivibiv
/
hydra-moe-120b

Text Generation
Transformers
Safetensors
English
mixtral
Mixture of Experts
moerge
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
ibivibiv commited on
Commit
4d723db
•
1 Parent(s): 3baf236

Upload MixtralForCausalLM

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +34 -0
  2. generation_config.json +7 -0
  3. model-00001-of-00097.safetensors +3 -0
  4. model-00002-of-00097.safetensors +3 -0
  5. model-00003-of-00097.safetensors +3 -0
  6. model-00004-of-00097.safetensors +3 -0
  7. model-00005-of-00097.safetensors +3 -0
  8. model-00006-of-00097.safetensors +3 -0
  9. model-00007-of-00097.safetensors +3 -0
  10. model-00008-of-00097.safetensors +3 -0
  11. model-00009-of-00097.safetensors +3 -0
  12. model-00010-of-00097.safetensors +3 -0
  13. model-00011-of-00097.safetensors +3 -0
  14. model-00012-of-00097.safetensors +3 -0
  15. model-00013-of-00097.safetensors +3 -0
  16. model-00014-of-00097.safetensors +3 -0
  17. model-00015-of-00097.safetensors +3 -0
  18. model-00016-of-00097.safetensors +3 -0
  19. model-00017-of-00097.safetensors +3 -0
  20. model-00018-of-00097.safetensors +3 -0
  21. model-00019-of-00097.safetensors +3 -0
  22. model-00020-of-00097.safetensors +3 -0
  23. model-00021-of-00097.safetensors +3 -0
  24. model-00022-of-00097.safetensors +3 -0
  25. model-00023-of-00097.safetensors +3 -0
  26. model-00024-of-00097.safetensors +3 -0
  27. model-00025-of-00097.safetensors +3 -0
  28. model-00026-of-00097.safetensors +3 -0
  29. model-00027-of-00097.safetensors +3 -0
  30. model-00028-of-00097.safetensors +3 -0
  31. model-00029-of-00097.safetensors +3 -0
  32. model-00030-of-00097.safetensors +3 -0
  33. model-00031-of-00097.safetensors +3 -0
  34. model-00032-of-00097.safetensors +3 -0
  35. model-00033-of-00097.safetensors +3 -0
  36. model-00034-of-00097.safetensors +3 -0
  37. model-00035-of-00097.safetensors +3 -0
  38. model-00036-of-00097.safetensors +3 -0
  39. model-00037-of-00097.safetensors +3 -0
  40. model-00038-of-00097.safetensors +3 -0
  41. model-00039-of-00097.safetensors +3 -0
  42. model-00040-of-00097.safetensors +3 -0
  43. model-00041-of-00097.safetensors +3 -0
  44. model-00042-of-00097.safetensors +3 -0
  45. model-00043-of-00097.safetensors +3 -0
  46. model-00044-of-00097.safetensors +3 -0
  47. model-00045-of-00097.safetensors +3 -0
  48. model-00046-of-00097.safetensors +3 -0
  49. model-00047-of-00097.safetensors +3 -0
  50. model-00048-of-00097.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "./hydra",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 7168,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 20480,
14
+ "max_position_embeddings": 200000,
15
+ "model_type": "mixtral",
16
+ "num_attention_heads": 56,
17
+ "num_experts_per_tok": 2,
18
+ "num_hidden_layers": 60,
19
+ "num_key_value_heads": 8,
20
+ "num_local_experts": 4,
21
+ "output_router_logits": false,
22
+ "pad_token_id": 0,
23
+ "pretraining_tp": 1,
24
+ "rms_norm_eps": 1e-05,
25
+ "rope_scaling": null,
26
+ "rope_theta": 5000000.0,
27
+ "router_aux_loss_coef": 0.001,
28
+ "sliding_window": null,
29
+ "tie_word_embeddings": false,
30
+ "torch_dtype": "float32",
31
+ "transformers_version": "4.37.2",
32
+ "use_cache": true,
33
+ "vocab_size": 64000
34
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.37.2"
7
+ }
model-00001-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d117e5879d396c48913485af01eba3d2985f4d2b0a735e5c33f2f9d023c97e48
3
+ size 4653696216
model-00002-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c187c0037c4473881c2d59e4fbec5f33f1b22d9aeb1dc0fb0df8442fd1c871da
3
+ size 4961920624
model-00003-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df80ae0218de635d3b4afebc2abae278f50e603c627b34aa50e65730f3b67d30
3
+ size 4903257368
model-00004-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7845e06b7c3bde58db05f4bdec03b1440111fd5ad952f37d6ee1f51fa386a13
3
+ size 4580353752
model-00005-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c9dde426697cf498efe2dbce4871e4a30ec834c586c95da0453ec1484f4701c
3
+ size 4697621552
model-00006-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4f1dee4d19153effcbd911292ede13d395b2b19f607574d77d65e95991d0fb8
3
+ size 4580353752
model-00007-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:353c3d533235be79cdb2573b6a6f955fc37ad5bb9d21523c830e59e84ddbd673
3
+ size 4580353752
model-00008-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c11b713e442ca8d2532071ce23b999f6784699af013a580475019b82cefdff33
3
+ size 4697621552
model-00009-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e29c64853bdb5556dbe4108e096ab93c416b9ecd5d694f357e274ddbab7b7aff
3
+ size 4580353752
model-00010-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a95dab034af983da2cc85ea8ba019b5f6bb15c10d3f2ea5a3496e44260425c2e
3
+ size 4961920624
model-00011-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d16f26f7cbe76195c1e9f9df87b5ef41b1d0febe73d6135025aa17112c5675e4
3
+ size 4903257368
model-00012-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d58be7dc9bd5d420e0e59cda74e277e9d78f7f038cf33787a5dd0b09fba32e1
3
+ size 4580353752
model-00013-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29968ffa2618c8bbfc742826192d2d0975afdb19dd2b00f55c612ebfcfefa8ca
3
+ size 4697621552
model-00014-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07e0f44eed713e413a83ea3b24f28cbf864f88a41f751ed4d4bce113140539e4
3
+ size 4580353752
model-00015-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50c1ef7e6e0760866e41132062b348be108e851e4620a00c5df356c7994bc4ad
3
+ size 4580353752
model-00016-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9d4d5e48f2ec24d260c19c0c8ab1f0ee6eb125903509a4e4cee7863f0cd51e59
3
+ size 4697621552
model-00017-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78f4606e8052d85cc25eaa8ae5892faf457a124fe23e0af03f8140453ef5390e
3
+ size 4580353760
model-00018-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0370859f69e87e71b3a32df6904677817004e1c135b7ae4185303145dc7b723
3
+ size 4961920640
model-00019-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d29eec6483b2296297643ec82a4ab4e4958234c601d25827b0dc655920db918
3
+ size 4903257384
model-00020-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:982ea8472eca941978ede4b3beaa37ef656cff10b688937f82f464ef83a32e8c
3
+ size 4580353768
model-00021-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42a3dc599f71e93c0932f3fa07f633aa53984a8e00ccc93bc39781fb235fa73e
3
+ size 4697621560
model-00022-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44611a57738045b0002cc21c605f2dbb551ba0113b2e25d38c61f21310746249
3
+ size 4580353760
model-00023-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4708648589672bedb8ad090e58660465a53f253768449974773aa6e0cc0dec91
3
+ size 4580353768
model-00024-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6010cc86f4ccd7fa976ff5e22fb82cd683121fa82790b98a41d8733e080c8bcc
3
+ size 4697621560
model-00025-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d7950fb10f239905eac778c7ee8b0eb04d8607a7937e0a7d4da4365242e8167
3
+ size 4580353768
model-00026-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01bb0db0e48759eb12a41073ce6fb6f208f8970f8708000702532b0b7627b598
3
+ size 4961920640
model-00027-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df789d4d89bf412f8915c6c566a1dfac3611ba4b3891c420f679e4b165db400a
3
+ size 4903257384
model-00028-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb3ccf04055d41a5be0e6a023d1cfc7200113c195accbd84e63eba53417312db
3
+ size 4580353768
model-00029-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e1f12f9c6f1a700cce7688d00fc495cf32f314ac03c63f668dcdb232b9da7bb
3
+ size 4697621560
model-00030-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df4e0efe8ddeb1f48241e2b0ed04d2569f68106bfb6adadc5ca2b5d0160a68e9
3
+ size 4580353760
model-00031-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:914c8cf16cd88cc697e6593d6830e7a4875b9f96d1ceeef37b82dbc8564fee97
3
+ size 4580353768
model-00032-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67e2e9a15e94ce95d0358c314bc9de11bef393730d661c56dd7a1ca5eae4e682
3
+ size 4697621560
model-00033-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08df07f2b16e3b996a6f9eccd2783fd613b7139a1a8f0651d6af64480827329e
3
+ size 4580353768
model-00034-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41b3439d5a8b3f95dab12dec409616c5f48736c05345b81df5e8d9ea7b22fe1e
3
+ size 4961920640
model-00035-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13fee5f1e157232ed56d5f2320494fcdc434e076b703e5fd8eb4c5794ac1e3c8
3
+ size 4903257384
model-00036-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cdd0ae375bd54a079a397a0319abc0936b24ef11d3463f8f31e93cb9ca6f1d4c
3
+ size 4580353768
model-00037-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8ea76ad3a2f40adfe9b7021afdf6879726545fa3b1806937ee6768915964fe7
3
+ size 4697621560
model-00038-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea650fd0198435cc5e2010caa9af3d9de15bd4060015eba0a399107b0df79edd
3
+ size 4580353760
model-00039-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae40622327bf47cd2d2e84a9f46bcf4944285a4f95c4823ab62eaa69a12bd81b
3
+ size 4580353768
model-00040-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b8dc381dbd9802b9c10ef3ea5381b6102ee46f10a03e6bbefbfebd27cc84ff6
3
+ size 4697621560
model-00041-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0351cd7bf9042091ada09c990e8c83b3e522e3b3dc31f5b22d3c7cb7d9b9d15
3
+ size 4580353768
model-00042-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1310b5e2ac89e4706f0f4713eaa25032b3986efe0187fc5093f415c44911b885
3
+ size 4961920640
model-00043-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:342075ad6da780d92d60b7cede87d46624376dd8d652865fa0df1ceed868c718
3
+ size 4903257384
model-00044-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f634c591af16768bd83a48e9e8cfb98dfb989d94e9810659217887a7019354b4
3
+ size 4580353768
model-00045-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cffc69eff23e642a0d32a6cb94f87d866a417b30b8839e8ce3c6c5e38707bb09
3
+ size 4697621560
model-00046-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00f03e395465aa780fee2bfb90058de4b6dfbf142994426e7dcf9ef1378a3c81
3
+ size 4580353760
model-00047-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7278047be5f349a8e57f2778b09afdd95c514f06c82489d49b3795e4392bf6a
3
+ size 4580353768
model-00048-of-00097.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:57292da99f26d5d73bb99cb915010b0a70b568d49d9c14d00466cf77c131cfbb
3
+ size 4697621560