kaiokendev commited on
Commit
8e78ead
1 Parent(s): fdf735d

Add the model

Browse files
adapter_config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model_name_or_path": "",
3
+ "bias": "none",
4
+ "fan_in_fan_out": false,
5
+ "inference_mode": true,
6
+ "init_lora_weights": true,
7
+ "lora_alpha": 8,
8
+ "lora_dropout": 0,
9
+ "modules_to_save": null,
10
+ "peft_type": "LORA",
11
+ "r": 4,
12
+ "target_modules": [
13
+ "q_proj",
14
+ "k_proj",
15
+ "v_proj",
16
+ "o_proj"
17
+ ],
18
+ "task_type": "CAUSAL_LM"
19
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20eec35c0eb4690ab524c0439069421f79536aa9da54ae4d388c9e6cb2cad283
3
+ size 26329994
llama_rope_scaled_monkey_patch.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import transformers
3
+ import transformers.models.llama.modeling_llama
4
+ from einops import rearrange
5
+ import random
6
+
7
+
8
+ class ScaledRotaryEmbedding(torch.nn.Module):
9
+ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
10
+ super().__init__()
11
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
12
+ self.register_buffer("inv_freq", inv_freq)
13
+
14
+ max_position_embeddings = 16384
15
+
16
+ # Build here to make `torch.jit.trace` work.
17
+ self.max_seq_len_cached = max_position_embeddings
18
+ t = torch.arange(
19
+ self.max_seq_len_cached,
20
+ device=self.inv_freq.device,
21
+ dtype=self.inv_freq.dtype,
22
+ )
23
+
24
+ self.scale = 1 / 8
25
+ t *= self.scale
26
+
27
+ freqs = torch.einsum("i,j->ij", t, self.inv_freq)
28
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
29
+ emb = torch.cat((freqs, freqs), dim=-1)
30
+ self.register_buffer(
31
+ "cos_cached", emb.cos()[None, None, :, :], persistent=False
32
+ )
33
+ self.register_buffer(
34
+ "sin_cached", emb.sin()[None, None, :, :], persistent=False
35
+ )
36
+
37
+ def forward(self, x, seq_len=None):
38
+ # x: [bs, num_attention_heads, seq_len, head_size]
39
+ # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
40
+ if seq_len > self.max_seq_len_cached:
41
+ self.max_seq_len_cached = seq_len
42
+ t = torch.arange(
43
+ self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype
44
+ )
45
+ t *= self.scale
46
+ freqs = torch.einsum("i,j->ij", t, self.inv_freq)
47
+ # Different from paper, but it uses a different permutation in order to obtain the same calculation
48
+ emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
49
+ self.register_buffer(
50
+ "cos_cached", emb.cos()[None, None, :, :], persistent=False
51
+ )
52
+ self.register_buffer(
53
+ "sin_cached", emb.sin()[None, None, :, :], persistent=False
54
+ )
55
+ return (
56
+ self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
57
+ self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
58
+ )
59
+
60
+
61
+ def replace_llama_rope_with_scaled_rope():
62
+ transformers.models.llama.modeling_llama.LlamaRotaryEmbedding = (
63
+ ScaledRotaryEmbedding
64
+ )