config.json CHANGED
@@ -22,11 +22,7 @@
22
  "rms_norm_eps": 1e-06,
23
  "tie_word_embeddings": false,
24
  "torch_dtype": "float16",
25
- "transformers_version": "4.29.2",
26
  "use_cache": true,
27
- "vocab_size": 103168,
28
- "rotary": {
29
- "base": 10000,
30
- "type": "dynamic"
31
- }
32
  }
 
22
  "rms_norm_eps": 1e-06,
23
  "tie_word_embeddings": false,
24
  "torch_dtype": "float16",
25
+ "transformers_version": "4.33.2",
26
  "use_cache": true,
27
+ "vocab_size": 103168
 
 
 
 
28
  }
configuration_internlm.py CHANGED
@@ -19,8 +19,9 @@
19
  # limitations under the License.
20
  """ InternLM model configuration"""
21
 
22
- from transformers.configuration_utils import PretrainedConfig
23
  from transformers.utils import logging
 
 
24
 
25
  logger = logging.get_logger(__name__)
26
 
@@ -29,9 +30,9 @@ INTERNLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
29
 
30
  class InternLMConfig(PretrainedConfig):
31
  r"""
32
- This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate
33
- an InternLM model according to the specified arguments, defining the model architecture. Instantiating a
34
- configuration with the defaults will yield a similar configuration to that of the InternLM-7B.
35
 
36
  Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
37
  documentation from [`PretrainedConfig`] for more information.
@@ -80,7 +81,7 @@ class InternLMConfig(PretrainedConfig):
80
  model_type = "internlm"
81
  _auto_class = "AutoConfig"
82
 
83
- def __init__( # pylint: disable=W0102
84
  self,
85
  vocab_size=103168,
86
  hidden_size=4096,
@@ -97,7 +98,6 @@ class InternLMConfig(PretrainedConfig):
97
  eos_token_id=2,
98
  tie_word_embeddings=False,
99
  bias=True,
100
- rotary={"base": 10000, "type": "dynamic"}, # pylint: disable=W0102
101
  **kwargs,
102
  ):
103
  self.vocab_size = vocab_size
@@ -111,11 +111,10 @@ class InternLMConfig(PretrainedConfig):
111
  self.rms_norm_eps = rms_norm_eps
112
  self.use_cache = use_cache
113
  self.bias = bias
114
- self.rotary = rotary
115
  super().__init__(
116
  pad_token_id=pad_token_id,
117
  bos_token_id=bos_token_id,
118
  eos_token_id=eos_token_id,
119
  tie_word_embeddings=tie_word_embeddings,
120
  **kwargs,
121
- )
 
19
  # limitations under the License.
20
  """ InternLM model configuration"""
21
 
 
22
  from transformers.utils import logging
23
+ from transformers.configuration_utils import PretrainedConfig
24
+
25
 
26
  logger = logging.get_logger(__name__)
27
 
 
30
 
31
  class InternLMConfig(PretrainedConfig):
32
  r"""
33
+ This is the configuration class to store the configuration of a [`InternLMModel`]. It is used to instantiate an InternLM
34
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
35
+ defaults will yield a similar configuration to that of the InternLM-7B.
36
 
37
  Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
38
  documentation from [`PretrainedConfig`] for more information.
 
81
  model_type = "internlm"
82
  _auto_class = "AutoConfig"
83
 
84
+ def __init__(
85
  self,
86
  vocab_size=103168,
87
  hidden_size=4096,
 
98
  eos_token_id=2,
99
  tie_word_embeddings=False,
100
  bias=True,
 
101
  **kwargs,
102
  ):
103
  self.vocab_size = vocab_size
 
111
  self.rms_norm_eps = rms_norm_eps
112
  self.use_cache = use_cache
113
  self.bias = bias
 
114
  super().__init__(
115
  pad_token_id=pad_token_id,
116
  bos_token_id=bos_token_id,
117
  eos_token_id=eos_token_id,
118
  tie_word_embeddings=tie_word_embeddings,
119
  **kwargs,
120
+ )
generation_config.json CHANGED
@@ -3,5 +3,5 @@
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
  "pad_token_id": 2,
6
- "transformers_version": "4.29.2"
7
  }
 
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
  "pad_token_id": 2,
6
+ "transformers_version": "4.33.2"
7
  }
modeling_internlm.py CHANGED
@@ -19,31 +19,31 @@
19
  # limitations under the License.
20
  """ PyTorch InternLM model."""
21
  import math
22
- import queue
23
- import threading
24
  from typing import List, Optional, Tuple, Union
 
25
 
26
  import torch
27
  import torch.utils.checkpoint
28
  from torch import nn
29
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
 
30
  from transformers.activations import ACT2FN
31
- from transformers.generation.streamers import BaseStreamer
32
  from transformers.modeling_outputs import (
33
  BaseModelOutputWithPast,
34
  CausalLMOutputWithPast,
35
  SequenceClassifierOutputWithPast,
36
  )
37
  from transformers.modeling_utils import PreTrainedModel
 
38
  from transformers.utils import (
39
  add_start_docstrings,
40
  add_start_docstrings_to_model_forward,
41
  logging,
42
  replace_return_docstrings,
43
  )
44
-
45
  from .configuration_internlm import InternLMConfig
46
 
 
47
  logger = logging.get_logger(__name__)
48
 
49
  _CONFIG_FOR_DOC = "InternLMConfig"
@@ -83,8 +83,6 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int]
83
 
84
 
85
  class InternLMRMSNorm(nn.Module):
86
- """RMSNorm implemention."""
87
-
88
  def __init__(self, hidden_size, eps=1e-6):
89
  """
90
  InternLMRMSNorm is equivalent to T5LayerNorm
@@ -105,18 +103,10 @@ class InternLMRMSNorm(nn.Module):
105
 
106
 
107
  class InternLMRotaryEmbedding(torch.nn.Module):
108
- """Implement InternLM's rotary embedding.
109
-
110
- Args:
111
- dim (int): Characteristic dimension of each self-attentional head.
112
- max_position_embeddings (int, optional): Model's training length. Defaults to 2048.
113
- base (int, optional): The rotation position encodes the rotation Angle base number. Defaults to 10000.
114
- device (Any, optional): Running device. Defaults to None.
115
- """
116
  def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
117
  super().__init__()
118
  inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
119
- self.register_buffer("inv_freq", inv_freq, persistent=False)
120
 
121
  # Build here to make `torch.jit.trace` work.
122
  self.max_seq_len_cached = max_position_embeddings
@@ -144,66 +134,6 @@ class InternLMRotaryEmbedding(torch.nn.Module):
144
  )
145
 
146
 
147
- class InternLMDynamicNTKScalingRotaryEmbedding(torch.nn.Module):
148
- """Implement InternLM's DyanmicNTK extrapolation method, thereby broadening the model support context to 16K.
149
-
150
- Args:
151
- dim (int): Characteristic dimension of each self-attentional head.
152
- max_position_embeddings (int, optional): Model's training length. Defaults to 2048.
153
- base (int, optional): The rotation position encodes the rotation Angle base number. Defaults to 10000.
154
- device (Any, optional): Running device. Defaults to None.
155
- scaling_factor (float, optional): NTK method extrapolation coefficient. Defaults to 1.0.
156
- """
157
-
158
- def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
159
- super().__init__()
160
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
161
- self.register_buffer("inv_freq", inv_freq)
162
- self.dim = dim
163
- self.base = base
164
- self.scaling_factor = scaling_factor
165
-
166
- # Build here to make `torch.jit.trace` work.
167
- self.max_position_embeddings = max_position_embeddings
168
- self.max_seq_len_cached = max_position_embeddings
169
- t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=self.inv_freq.dtype)
170
- freqs = torch.einsum("i,j->ij", t, self.inv_freq)
171
- # Different from paper, but it uses a different permutation in order to obtain the same calculation
172
- emb = torch.cat((freqs, freqs), dim=-1)
173
- self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False)
174
- self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False)
175
-
176
- def _update_cached(self, x, seq_len=None):
177
- self.max_seq_len_cached = max(seq_len, self.max_position_embeddings)
178
- if seq_len > self.max_position_embeddings:
179
- base = self.base * (
180
- (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
181
- ) ** (self.dim / (self.dim - 2))
182
- inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(x.device) / self.dim))
183
- else:
184
- inv_freq = self.inv_freq
185
- t = torch.arange(self.max_seq_len_cached, device=inv_freq.device, dtype=inv_freq.dtype)
186
- freqs = torch.einsum("i,j->ij", t, inv_freq)
187
- emb = torch.cat((freqs, freqs), dim=-1)
188
- self.register_buffer("cos_cached", emb.cos()[None, None, :, :], persistent=False)
189
- self.register_buffer("sin_cached", emb.sin()[None, None, :, :], persistent=False)
190
-
191
- def forward(self, x, seq_len=None):
192
- # x: [bs, num_attention_heads, seq_len, head_size]
193
- # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
194
- if seq_len <= self.max_position_embeddings:
195
- # Reset the tables if the sequence length has changed,
196
- if self.max_seq_len_cached > self.max_position_embeddings:
197
- self._update_cached(x, seq_len)
198
- else:
199
- self._update_cached(x, seq_len)
200
-
201
- return (
202
- self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
203
- self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
204
- )
205
-
206
-
207
  def rotate_half(x):
208
  """Rotates half the hidden dims of the input."""
209
  x1 = x[..., : x.shape[-1] // 2]
@@ -215,18 +145,10 @@ def apply_rotary_pos_emb(q, k, cos, sin, position_ids):
215
  # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
216
  cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
217
  sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
218
- cos = cos.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
219
- sin = sin.unsqueeze(0).unsqueeze(0).expand(len(position_ids), -1, -1, -1)
220
- if q.size(2) == 1:
221
- q_embed = (q * cos[:, :, -1, :]) + (rotate_half(q) * sin[:, :, -1, :])
222
- else:
223
- q_embed = (q * cos) + (rotate_half(q) * sin)
224
-
225
- if k.size(2) == 1:
226
- k_embed = (k * cos[:, :, -1, :]) + (rotate_half(k) * sin[:, :, -1, :])
227
- else:
228
- k_embed = (k * cos) + (rotate_half(k) * sin)
229
-
230
  return q_embed, k_embed
231
 
232
 
@@ -267,25 +189,7 @@ class InternLMAttention(nn.Module):
267
  self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
268
  self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
269
  self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)
270
- self.rotary_emb = self._init_rope()
271
-
272
- def _init_rope(self):
273
- if self.config.rotary["type"] == "origin":
274
- self.rotary_emb = InternLMRotaryEmbedding(
275
- self.head_dim,
276
- max_position_embeddings=self.max_position_embeddings,
277
- base=self.config.rotary["base"],
278
- )
279
- elif self.config.rotary["type"] == "dynamic":
280
- self.rotary_emb = InternLMDynamicNTKScalingRotaryEmbedding(
281
- self.head_dim,
282
- max_position_embeddings=self.max_position_embeddings,
283
- base=self.config.rotary["base"],
284
- scaling_factor=self.config.rotary.get("scaling_factor", 1.0),
285
- )
286
- else:
287
- raise ValueError("Currently we only support rotary embedding's type being one of ('origin', 'dynamic').")
288
- return self.rotary_emb
289
 
290
  def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
291
  return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
@@ -305,18 +209,20 @@ class InternLMAttention(nn.Module):
305
  key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
306
  value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
307
 
 
 
 
 
 
 
 
308
  if past_key_value is not None:
309
  # reuse k, v, self_attention
310
  key_states = torch.cat([past_key_value[0], key_states], dim=2)
311
  value_states = torch.cat([past_key_value[1], value_states], dim=2)
312
 
313
- # print(use_cache)
314
  past_key_value = (key_states, value_states) if use_cache else None
315
 
316
- kv_seq_len = key_states.shape[-2]
317
- cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
318
- query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
319
-
320
  attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
321
 
322
  if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
@@ -426,9 +332,11 @@ INTERNLM_START_DOCSTRING = r"""
426
  This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
427
  library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
428
  etc.)
 
429
  This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
430
  Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
431
  and behavior.
 
432
  Parameters:
433
  config ([`InternLMConfig`]):
434
  Model configuration class with all the parameters of the model. Initializing with a config file does not
@@ -469,34 +377,44 @@ INTERNLM_INPUTS_DOCSTRING = r"""
469
  input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
470
  Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
471
  it.
 
472
  Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
473
  [`PreTrainedTokenizer.__call__`] for details.
 
474
  [What are input IDs?](../glossary#input-ids)
475
  attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
476
  Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
 
477
  - 1 for tokens that are **not masked**,
478
  - 0 for tokens that are **masked**.
 
479
  [What are attention masks?](../glossary#attention-mask)
 
480
  Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
481
  [`PreTrainedTokenizer.__call__`] for details.
 
482
  If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
483
  `past_key_values`).
 
484
  If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
485
  and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
486
  information on the default strategy.
 
487
  - 1 indicates the head is **not masked**,
488
  - 0 indicates the head is **masked**.
489
  position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
490
  Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
491
  config.n_positions - 1]`.
 
492
  [What are position IDs?](../glossary#position-ids)
493
- past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or
494
- when `config.use_cache=True`):
495
  Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
496
  `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
497
  `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
 
498
  Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
499
  blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
 
500
  If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
501
  don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
502
  `decoder_input_ids` of shape `(batch_size, sequence_length)`.
@@ -525,6 +443,7 @@ INTERNLM_INPUTS_DOCSTRING = r"""
525
  class InternLMModel(InternLMPreTrainedModel):
526
  """
527
  Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`]
 
528
  Args:
529
  config: InternLMConfig
530
  """
@@ -754,14 +673,20 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
754
  Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
755
  config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
756
  (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
 
757
  Returns:
 
758
  Example:
 
759
  ```python
760
  >>> from transformers import AutoTokenizer, InternLMForCausalLM
 
761
  >>> model = InternLMForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
762
  >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
 
763
  >>> prompt = "Hey, are you consciours? Can you talk to me?"
764
  >>> inputs = tokenizer(prompt, return_tensors="pt")
 
765
  >>> # Generate
766
  >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
767
  >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
@@ -855,7 +780,9 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
855
  def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = []):
856
  prompt = ""
857
  for record in history:
858
- prompt += f"""<|User|>:{record[0]}<eoh>\n<|Bot|>:{record[1]}<eoa>\n"""
 
 
859
  prompt += f"""<|User|>:{query}<eoh>\n<|Bot|>:"""
860
  return tokenizer([prompt], return_tensors="pt")
861
 
@@ -959,7 +886,7 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
959
  producer.start()
960
  while True:
961
  res = response_queue.get()
962
- if res is None:
963
  return
964
  yield res
965
 
@@ -969,8 +896,10 @@ class InternLMForCausalLM(InternLMPreTrainedModel):
969
  @add_start_docstrings(
970
  """
971
  The InternLM Model transformer with a sequence classification head on top (linear layer).
 
972
  [`InternLMForSequenceClassification`] uses the last token in order to do the classification, as other causal models
973
  (e.g. GPT-2) do.
 
974
  Since it does classification on the last token, it requires to know the position of the last token. If a
975
  `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
976
  no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
@@ -1083,4 +1012,4 @@ class InternLMForSequenceClassification(InternLMPreTrainedModel):
1083
  past_key_values=transformer_outputs.past_key_values,
1084
  hidden_states=transformer_outputs.hidden_states,
1085
  attentions=transformer_outputs.attentions,
1086
- )
 
19
  # limitations under the License.
20
  """ PyTorch InternLM model."""
21
  import math
 
 
22
  from typing import List, Optional, Tuple, Union
23
+ import threading, queue
24
 
25
  import torch
26
  import torch.utils.checkpoint
27
  from torch import nn
28
  from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
29
+
30
  from transformers.activations import ACT2FN
 
31
  from transformers.modeling_outputs import (
32
  BaseModelOutputWithPast,
33
  CausalLMOutputWithPast,
34
  SequenceClassifierOutputWithPast,
35
  )
36
  from transformers.modeling_utils import PreTrainedModel
37
+ from transformers.generation.streamers import BaseStreamer
38
  from transformers.utils import (
39
  add_start_docstrings,
40
  add_start_docstrings_to_model_forward,
41
  logging,
42
  replace_return_docstrings,
43
  )
 
44
  from .configuration_internlm import InternLMConfig
45
 
46
+
47
  logger = logging.get_logger(__name__)
48
 
49
  _CONFIG_FOR_DOC = "InternLMConfig"
 
83
 
84
 
85
  class InternLMRMSNorm(nn.Module):
 
 
86
  def __init__(self, hidden_size, eps=1e-6):
87
  """
88
  InternLMRMSNorm is equivalent to T5LayerNorm
 
103
 
104
 
105
  class InternLMRotaryEmbedding(torch.nn.Module):
 
 
 
 
 
 
 
 
106
  def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
107
  super().__init__()
108
  inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
109
+ self.register_buffer("inv_freq", inv_freq)
110
 
111
  # Build here to make `torch.jit.trace` work.
112
  self.max_seq_len_cached = max_position_embeddings
 
134
  )
135
 
136
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  def rotate_half(x):
138
  """Rotates half the hidden dims of the input."""
139
  x1 = x[..., : x.shape[-1] // 2]
 
145
  # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
146
  cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
147
  sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
148
+ cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim]
149
+ sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim]
150
+ q_embed = (q * cos) + (rotate_half(q) * sin)
151
+ k_embed = (k * cos) + (rotate_half(k) * sin)
 
 
 
 
 
 
 
 
152
  return q_embed, k_embed
153
 
154
 
 
189
  self.k_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
190
  self.v_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=config.bias)
191
  self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.bias)
192
+ self.rotary_emb = InternLMRotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
193
 
194
  def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
195
  return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
 
209
  key_states = self.k_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
210
  value_states = self.v_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
211
 
212
+ kv_seq_len = key_states.shape[-2]
213
+ if past_key_value is not None:
214
+ kv_seq_len += past_key_value[0].shape[-2]
215
+ cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
216
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
217
+ # [bsz, nh, t, hd]
218
+
219
  if past_key_value is not None:
220
  # reuse k, v, self_attention
221
  key_states = torch.cat([past_key_value[0], key_states], dim=2)
222
  value_states = torch.cat([past_key_value[1], value_states], dim=2)
223
 
 
224
  past_key_value = (key_states, value_states) if use_cache else None
225
 
 
 
 
 
226
  attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
227
 
228
  if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
 
332
  This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
333
  library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
334
  etc.)
335
+
336
  This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
337
  Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
338
  and behavior.
339
+
340
  Parameters:
341
  config ([`InternLMConfig`]):
342
  Model configuration class with all the parameters of the model. Initializing with a config file does not
 
377
  input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
378
  Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
379
  it.
380
+
381
  Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
382
  [`PreTrainedTokenizer.__call__`] for details.
383
+
384
  [What are input IDs?](../glossary#input-ids)
385
  attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
386
  Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
387
+
388
  - 1 for tokens that are **not masked**,
389
  - 0 for tokens that are **masked**.
390
+
391
  [What are attention masks?](../glossary#attention-mask)
392
+
393
  Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
394
  [`PreTrainedTokenizer.__call__`] for details.
395
+
396
  If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
397
  `past_key_values`).
398
+
399
  If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
400
  and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
401
  information on the default strategy.
402
+
403
  - 1 indicates the head is **not masked**,
404
  - 0 indicates the head is **masked**.
405
  position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
406
  Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
407
  config.n_positions - 1]`.
408
+
409
  [What are position IDs?](../glossary#position-ids)
410
+ past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
 
411
  Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
412
  `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
413
  `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
414
+
415
  Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
416
  blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
417
+
418
  If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
419
  don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
420
  `decoder_input_ids` of shape `(batch_size, sequence_length)`.
 
443
  class InternLMModel(InternLMPreTrainedModel):
444
  """
445
  Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`InternLMDecoderLayer`]
446
+
447
  Args:
448
  config: InternLMConfig
449
  """
 
673
  Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
674
  config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
675
  (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
676
+
677
  Returns:
678
+
679
  Example:
680
+
681
  ```python
682
  >>> from transformers import AutoTokenizer, InternLMForCausalLM
683
+
684
  >>> model = InternLMForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
685
  >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
686
+
687
  >>> prompt = "Hey, are you consciours? Can you talk to me?"
688
  >>> inputs = tokenizer(prompt, return_tensors="pt")
689
+
690
  >>> # Generate
691
  >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
692
  >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
 
780
  def build_inputs(self, tokenizer, query: str, history: List[Tuple[str, str]] = []):
781
  prompt = ""
782
  for record in history:
783
+ prompt += f"""<s><|User|>:{record[0]}<eoh>\n<|Bot|>:{record[1]}<eoa>\n"""
784
+ if len(prompt) == 0:
785
+ prompt += "<s>"
786
  prompt += f"""<|User|>:{query}<eoh>\n<|Bot|>:"""
787
  return tokenizer([prompt], return_tensors="pt")
788
 
 
886
  producer.start()
887
  while True:
888
  res = response_queue.get()
889
+ if res is not None:
890
  return
891
  yield res
892
 
 
896
  @add_start_docstrings(
897
  """
898
  The InternLM Model transformer with a sequence classification head on top (linear layer).
899
+
900
  [`InternLMForSequenceClassification`] uses the last token in order to do the classification, as other causal models
901
  (e.g. GPT-2) do.
902
+
903
  Since it does classification on the last token, it requires to know the position of the last token. If a
904
  `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
905
  no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
 
1012
  past_key_values=transformer_outputs.past_key_values,
1013
  hidden_states=transformer_outputs.hidden_states,
1014
  attentions=transformer_outputs.attentions,
1015
+ )
pytorch_model-00001-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0da2138b5347feaff3ae192faa19f72a62c346b4a2af4c27fa5b1a64e1f89d9f
3
- size 1969370847
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:913fb6d43f9a12afcbfa7f61c2ff448d0a8271c3b82c7a295619a74bd10eb3dd
3
+ size 1969371359
pytorch_model-00002-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:468486f521a59c5d4909835352b225d0a95c42adee7228f66b298c793b6bfe8b
3
- size 1933844137
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b505624ced70c57b0aa685a44f0edf0a0267310b8ac5b9aef18df30901ca4873
3
+ size 1933845097
pytorch_model-00003-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:21cd36f5e5de2688b1f89813c14d74fd4be9d2d707d0f6b7c58300bdaadf8a5b
3
- size 1933844201
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29c11def7e5ad757434c89ca07500ac99231683cdbd489f051be0b0fc8877fd2
3
+ size 1933845161
pytorch_model-00004-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a8ca08b014aa1a7ee84017bc1a711aa303ab55b5de45da5a6e93d3c0f969e9b3
3
- size 1990458181
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09c8ad0ec21cf97b308b2e53deaa92c27bd687b2e9027474a9e680dd9fb676ea
3
+ size 1990459141
pytorch_model-00005-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4bf12fb12cb956f187d20a99522dc8fc9c13ede5acc25402164b71ce89f6930b
3
- size 1990458775
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3ec9dc8e8c583587c1d965d7848f191fd6d1cab8fae866c3f1e7f4227ee13107
3
+ size 1990459735
pytorch_model-00006-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bc1cf408deca95f108520d4bc1518bfd44c22ac051bbb6ac9b4a1a236c27a308
3
- size 1990458775
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5eeaab4bcc0146a6bca61fc2cb1c6215de3934995d5c9a17bd05453f7b9f65c
3
+ size 1990459735
pytorch_model-00007-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d4514f85b680278bff8700ba7c8a2d189edae3a22dd673f869533aac612c2be7
3
- size 1990467305
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f2eca07263e93cb2b0f6c63e77a6d99d564b66ccc2d82f0aadbc953b3870ff5
3
+ size 1990468265
pytorch_model-00008-of-00008.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:57a97684d4e825b93f358274a07bc40b30df5fe0490bba76547ecbb69053a30b
3
  size 845153194
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8553c3ce27e157933280b9f39e90d5d51be98d95f2d0c7f6999e4312a41325e
3
  size 845153194