Model upload.

.gitattributes

@@ -9,7 +9,6 @@
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
 *.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
 *.npy filter=lfs diff=lfs merge=lfs -text
 *.npz filter=lfs diff=lfs merge=lfs -text
@@ -25,7 +24,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text

4bit-32g.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c3acac392d4e3265d1730f649b12a005d5dd95a6d1eb893bfa6ed75cabc106e
+size 4276179984

README.md

@@ -0,0 +1,64 @@
+---
+license: other
+---
+# superhot-7b-8k-4bit-32g-safetensors
+
+**Note: Maximum sequence length (max_seq_len) and compression factor (compress_pos_emb) need to be set to 8192 (or lower) and 4.**
+
+Merged base LLaMA and LoRA with this:
+https://github.com/tloen/alpaca-lora
+
+Base LLaMA 7B:
+https://huggingface.co/huggyllama/llama-7b
+
+SuperHOT 7B 8k no-rlhf-test LoRA:
+https://huggingface.co/kaiokendev/superhot-7b-8k-no-rlhf-test
+
+``` sh
+BASE_MODEL=huggyllama_llama-7b LORA=kaiokendev_superhot-7b-8k-no-rlhf-test python export_hf_checkpoint.py
+```
+
+Quantized with AutoGPTQ:
+https://github.com/PanQiWei/AutoGPTQ
+
+``` sh
+python quant_with_alpaca.py --pretrained_model_dir superhot-7b-8k-safetensors --quantized_model_dir superhot-7b-8k-no-rlhf-test-32g-GPTQ --bits 4 --group_size 32 --desc_act --num_samples 256 --save_and_reload
+```
+
+Perplexity:
+```
+CUDA_VISIBLE_DEVICES=0 python test_benchmark_inference.py \
+         -d /workspace/models/superhot-7b-8k-no-rlhf-test-32g-GPTQ \
+         -ppl \
+         -ppl_ds datasets/wikitext2.txt \
+         -l 8192 \
+         -cpe 4 \
+         -ppl_cn 40 \
+         -ppl_cs 8192 \
+        -ppl_ct 8192
+ -- Perplexity:
+ -- - Dataset: datasets/wikitext2.txt
+ -- - Chunks: 40
+ -- - Chunk size: 8192 -> 8192
+ -- - Chunk overlap: 0
+ -- - Min. chunk size: 50
+ -- - Key: text
+ -- Tokenizer: /workspace/models/superhot-7b-8k-no-rlhf-test-32g-GPTQ/tokenizer.model
+ -- Model config: /workspace/models/superhot-7b-8k-no-rlhf-test-32g-GPTQ/config.json
+ -- Model: /workspace/models/superhot-7b-8k-no-rlhf-test-32g-GPTQ/4bit-32g.safetensors
+ -- Sequence length: 8192
+ -- RoPE compression factor: 4.0
+ -- Tuning:
+ -- --matmul_recons_thd: 8
+ -- --fused_mlp_thd: 2
+ -- --sdp_thd: 8
+ -- Options: ['perplexity']
+ ** Time, Load model: 1.64 seconds
+ ** Time, Load tokenizer: 0.02 seconds
+ -- Groupsize (inferred): 32
+ -- Act-order (inferred): yes
+ ** VRAM, Model: [cuda:0] 4,131.34 MB
+ -- Loading dataset...
+ -- Testing 40 chunks....
+ ** Perplexity: 6.3184
+```

config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "superhot-7b-8k-no-rlhf-test",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 2048,
+  "max_sequence_length": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.30.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

quantize_config.json

@@ -0,0 +1,10 @@
+{
+  "bits": 4,
+  "group_size": 32,
+  "damp_percent": 0.01,
+  "desc_act": true,
+  "sym": true,
+  "true_sequential": true,
+  "model_name_or_path": null,
+  "model_file_base_name": null
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 2048,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}