Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-405B-Instruct-quantized.w8a16

Text Generation
Safetensors
llama
int8
vllm
conversational
compressed-tensors
Model card Files Community
alexmarques commited on
Commit
ef8e2c3
•
1 Parent(s): 391ddf8

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +69 -0
  2. generation_config.json +12 -0
  3. model-00001-of-00086.safetensors +3 -0
  4. model-00002-of-00086.safetensors +3 -0
  5. model-00003-of-00086.safetensors +3 -0
  6. model-00004-of-00086.safetensors +3 -0
  7. model-00005-of-00086.safetensors +3 -0
  8. model-00006-of-00086.safetensors +3 -0
  9. model-00007-of-00086.safetensors +3 -0
  10. model-00008-of-00086.safetensors +3 -0
  11. model-00009-of-00086.safetensors +3 -0
  12. model-00010-of-00086.safetensors +3 -0
  13. model-00011-of-00086.safetensors +3 -0
  14. model-00012-of-00086.safetensors +3 -0
  15. model-00013-of-00086.safetensors +3 -0
  16. model-00014-of-00086.safetensors +3 -0
  17. model-00015-of-00086.safetensors +3 -0
  18. model-00016-of-00086.safetensors +3 -0
  19. model-00017-of-00086.safetensors +3 -0
  20. model-00018-of-00086.safetensors +3 -0
  21. model-00019-of-00086.safetensors +3 -0
  22. model-00020-of-00086.safetensors +3 -0
  23. model-00021-of-00086.safetensors +3 -0
  24. model-00022-of-00086.safetensors +3 -0
  25. model-00023-of-00086.safetensors +3 -0
  26. model-00024-of-00086.safetensors +3 -0
  27. model-00025-of-00086.safetensors +3 -0
  28. model-00026-of-00086.safetensors +3 -0
  29. model-00027-of-00086.safetensors +3 -0
  30. model-00028-of-00086.safetensors +3 -0
  31. model-00029-of-00086.safetensors +3 -0
  32. model-00030-of-00086.safetensors +3 -0
  33. model-00031-of-00086.safetensors +3 -0
  34. model-00032-of-00086.safetensors +3 -0
  35. model-00033-of-00086.safetensors +3 -0
  36. model-00034-of-00086.safetensors +3 -0
  37. model-00035-of-00086.safetensors +3 -0
  38. model-00036-of-00086.safetensors +3 -0
  39. model-00037-of-00086.safetensors +3 -0
  40. model-00038-of-00086.safetensors +3 -0
  41. model-00039-of-00086.safetensors +3 -0
  42. model-00040-of-00086.safetensors +3 -0
  43. model-00041-of-00086.safetensors +3 -0
  44. model-00042-of-00086.safetensors +3 -0
  45. model-00043-of-00086.safetensors +3 -0
  46. model-00044-of-00086.safetensors +3 -0
  47. model-00045-of-00086.safetensors +3 -0
  48. model-00046-of-00086.safetensors +3 -0
  49. model-00047-of-00086.safetensors +3 -0
  50. model-00048-of-00086.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3.1-405B-Instruct/snapshots/069992c75aed59df00ec06c17177e76c63296a26",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "compression_config": {
10
+ "config_groups": {
11
+ "group_0": {
12
+ "input_activations": null,
13
+ "output_activations": null,
14
+ "targets": [
15
+ "Linear"
16
+ ],
17
+ "weights": {
18
+ "block_structure": null,
19
+ "dynamic": false,
20
+ "group_size": null,
21
+ "num_bits": 8,
22
+ "observer": "minmax",
23
+ "observer_kwargs": {},
24
+ "strategy": "channel",
25
+ "symmetric": true,
26
+ "type": "int"
27
+ }
28
+ }
29
+ },
30
+ "format": "pack-quantized",
31
+ "global_compression_ratio": 1.464683023832259,
32
+ "ignore": [
33
+ "lm_head"
34
+ ],
35
+ "kv_cache_scheme": null,
36
+ "quant_method": "compressed-tensors",
37
+ "quantization_status": "frozen"
38
+ },
39
+ "eos_token_id": [
40
+ 128001,
41
+ 128008,
42
+ 128009
43
+ ],
44
+ "hidden_act": "silu",
45
+ "hidden_size": 16384,
46
+ "initializer_range": 0.02,
47
+ "intermediate_size": 53248,
48
+ "max_position_embeddings": 131072,
49
+ "mlp_bias": false,
50
+ "model_type": "llama",
51
+ "num_attention_heads": 128,
52
+ "num_hidden_layers": 126,
53
+ "num_key_value_heads": 8,
54
+ "pretraining_tp": 1,
55
+ "rms_norm_eps": 1e-05,
56
+ "rope_scaling": {
57
+ "factor": 8.0,
58
+ "high_freq_factor": 4.0,
59
+ "low_freq_factor": 1.0,
60
+ "original_max_position_embeddings": 8192,
61
+ "rope_type": "llama3"
62
+ },
63
+ "rope_theta": 500000.0,
64
+ "tie_word_embeddings": false,
65
+ "torch_dtype": "bfloat16",
66
+ "transformers_version": "4.44.0",
67
+ "use_cache": true,
68
+ "vocab_size": 128256
69
+ }
generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 128000,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "temperature": 0.6,
10
+ "top_p": 0.9,
11
+ "transformers_version": "4.44.0"
12
+ }
model-00001-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1ad04f2a6efd83bd287930a1c5259b426fc496a6b910d65e57a49587bc1818f
3
+ size 4773189256
model-00002-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0364ed2bb77a924625c750037af6a2829a676584010f9a16a39c1ac6d1268790
3
+ size 4933098864
model-00003-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:086a30f8af5f37155b40b1b8a53ff714a14db5c1e6c36c369acb8bc150c41a76
3
+ size 4631065248
model-00004-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91e005f5c80888e1ceae2ab45307425e2e2ae0bca500a0a41db1e8149957d2d0
3
+ size 4933098864
model-00005-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce49424b81057d39107f7c992073f776ddb9731048a9a4b377165fea1b574f04
3
+ size 4631065248
model-00006-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6faec7708c3ed27317a85bbb7281ab94490e61fd85f2ad5b1a82084632fcac51
3
+ size 4933098864
model-00007-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:267253563c4c4a3feacba0f6250e1a48b11a401844362c4194d09fa5d8ff95ef
3
+ size 4631065248
model-00008-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9648c0f716a88a2389e318ace8623a2306ca4ea8250c8084bc58e6eb975a937a
3
+ size 4933098888
model-00009-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4edbc67dbe37f993332293a6d6862772b6c33e33d79ea1f24f719ae25fc6fe08
3
+ size 4631065288
model-00010-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1a3deb0d6d6b9136a01dbd040dfead80883afe61c558f87cfad1aa766c51438
3
+ size 4933098896
model-00011-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2e7a4406f0f653acda411b3cdf806c2f12e1e23594b836142e99d2f0dfaa742
3
+ size 4631065288
model-00012-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:749ac837bd5473ee3270f2d73e7acec89f940a2d3befc056ece850aba7d468c2
3
+ size 4933098896
model-00013-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e567bc61133569510a84d259a78f4a21699ff14713415ef4ac11702551898f8c
3
+ size 4631065288
model-00014-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0487b97176f7ad43c9cd1a6bee082ee60cd62b26e4c520044729171afaa96a33
3
+ size 4933098896
model-00015-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2de0c8521670c771f2c3ae08d9a7c41ec50a921dc21adb29fd48f2b777163b4c
3
+ size 4631065288
model-00016-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e57dc07fecc4d9c2295e1f717872bd12f5793a6d7c3975a10f9bfd9ae6f6e946
3
+ size 4933098896
model-00017-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d780644bdacb4235bc4f2528564db44742196434219927a7890d67eb4edf854
3
+ size 4631065288
model-00018-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0f1cd5810e92fa0be2591d3d78b6f7a4efb60506622063dd692c25dc3c4a681
3
+ size 4933098896
model-00019-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85fad0e99072161a73229e64fd4a94a179903fa16ac365bfc64590660bf6c4e9
3
+ size 4631065288
model-00020-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b7a23953ce774d1f60aaf6906bd9a0d5eb71ab20af2e2d943a69bc293a9520a
3
+ size 4933098896
model-00021-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be7bae651b72008eb5411954552c980ed2b85d3163146751cfdad2e431739121
3
+ size 4631065288
model-00022-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79362a0e68b5763a7cf7cd58ffab30963c71678eaead928e01fc1571ecfbbe7a
3
+ size 4933098896
model-00023-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a8908c9b605d036fd9b65991201411ff5bf52d077721d4dbe407634478d1008
3
+ size 4631065288
model-00024-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea298d45bee34dc691246625f87d1dd662a006c7a523d192dd7ad6b5b8453c92
3
+ size 4933098896
model-00025-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:852964e8a6c1b9bfa498ba0b03d87cf73f9e20d6c2415ecbc44c17b8d3d80c16
3
+ size 4631065288
model-00026-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbd2e49bf69f64f15d96a067d96090bb29dca0129dc60d1822902d87d6a3d5f2
3
+ size 4933098896
model-00027-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fbfc7d110856a71e0216ff7afc2c0c593b1643b7c59b78e8d4e868a29d39fd2f
3
+ size 4631065288
model-00028-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24f1d52810af4c3f80eba441b207eabf382a3dc5cb4d2cc1051804b1d37dd83c
3
+ size 4933098896
model-00029-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4baa9e95ac16e4d05771c7241f32bc7cbd8151e5a8749191b8ca0845b1e286a7
3
+ size 4631065288
model-00030-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecd724f13df5d068b41015aca33064ff2258da90486b7ce41a293243bab091de
3
+ size 4933098896
model-00031-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:235ad0d39a92b6f6aded20feb246f43fc246fe72b85bab37c22ccc7fa2a2ff26
3
+ size 4631065288
model-00032-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49fc61e561705aa439824026a4e4196b0582bcf46f9489258bd0bb62ac4aba01
3
+ size 4933098896
model-00033-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e6d4719179fb93968de40ee82da6afa90d155cab40fdbe084923e9c7e50c2a9
3
+ size 4631065288
model-00034-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a3753606d93a1bb7f48ec92c47318a1bb9e75040f4bfba38846547a6eaf8a09
3
+ size 4933098896
model-00035-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:956dd02ede0760a0a6ab43b6a4f1a76e5017af7dd26e361f7a8d481d2e598b51
3
+ size 4631065288
model-00036-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:639aee0c852daa4f1b74412cd121638610dd1a3121373b6038a5c70985e8d4b8
3
+ size 4933098896
model-00037-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1cbcda1b27fccea47a592088ab6117eaf1f5aab435892f3fb9a1e18b4fe3f4b
3
+ size 4631065288
model-00038-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad88bd02a6cd532d903a9fd5b49b2c7a4ad4601261b35750ca9df4d1f1bf610e
3
+ size 4933098896
model-00039-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:922b02aaff2c01071c17e886beda2bcf11fb592f08ec59bc7cbf6725e7cd6e52
3
+ size 4631065288
model-00040-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcd195a6459170581004bcea8c1d25051a6ccb2120722928a36535b1fd4f923e
3
+ size 4933098896
model-00041-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7d56086e2a9e85ba36664b7f545641f4fbdbe94cd16c70c39a5ab0021da07af
3
+ size 4631065288
model-00042-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27682096deddcf6608277f94f0e018369af4f7a8f811fb967c70ddc308888c03
3
+ size 4933098896
model-00043-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f968e7b5999aae011b6a405b667c93f4aab8c48a77d0bd14ba32e3beaa33eade
3
+ size 4631065288
model-00044-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b0534470c1b3bcd850df7805d720160c098f82361bcbae0f31e892c9d94bdb9
3
+ size 4933098896
model-00045-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3b68f8002d88a5f69cde6a4c86dec822ca0a27bccabf311fe2569a642d552be
3
+ size 4631065288
model-00046-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78ea941dc697482581752cad0b75a594c2f298b14cf35997b1e86082ed89afad
3
+ size 4933098896
model-00047-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf5dd33f76ef03278d309a2c23fd668dc7fc9b3cbf2d7d59d8127e59d4115b22
3
+ size 4631065288
model-00048-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e65035998da68d9afb56ec41685b2acbb16a708d335cd5e4616a297176c6f7c8
3
+ size 4933098896