LeroyDyer commited on
Commit
fb0bfc8
1 Parent(s): 3e477e5

Upload 2 files

Browse files
Files changed (2) hide show
  1. configuration_quiet.py +170 -0
  2. modeling_quiet.py +0 -0
configuration_quiet.py ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2023 Quiet AI and the HuggingFace Inc. team. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """ Quiet model configuration"""
16
+
17
+ from transformers.configuration_utils import PretrainedConfig
18
+ from transformers.utils import logging
19
+
20
+
21
+ logger = logging.get_logger(__name__)
22
+
23
+ QUIET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
24
+ "quietai/Quiet-7B-v0.1": "https://huggingface.co/quietai/Quiet-7B-v0.1/resolve/main/config.json",
25
+ "quietai/Quiet-7B-Instruct-v0.1": "https://huggingface.co/quietai/Quiet-7B-Instruct-v0.1/resolve/main/config.json",
26
+ }
27
+
28
+
29
+ class QuietConfig(PretrainedConfig):
30
+ r"""
31
+ This is the configuration class to store the configuration of a [`QuietModel`]. It is used to instantiate an
32
+ Quiet model according to the specified arguments, defining the model architecture. Instantiating a configuration
33
+ with the defaults will yield a similar configuration to that of the Quiet-7B-v0.1 or Quiet-7B-Instruct-v0.1.
34
+ [quietai/Quiet-7B-v0.1](https://huggingface.co/quietai/Quiet-7B-v0.1)
35
+ [quietai/Quiet-7B-Instruct-v0.1](https://huggingface.co/quietai/Quiet-7B-Instruct-v0.1)
36
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
37
+ documentation from [`PretrainedConfig`] for more information.
38
+ Args:
39
+ vocab_size (`int`, *optional*, defaults to 32000):
40
+ Vocabulary size of the Quiet model. Defines the number of different tokens that can be represented by the
41
+ `inputs_ids` passed when calling [`QuietModel`]
42
+ hidden_size (`int`, *optional*, defaults to 4096):
43
+ Dimension of the hidden representations.
44
+ intermediate_size (`int`, *optional*, defaults to 14336):
45
+ Dimension of the MLP representations.
46
+ num_hidden_layers (`int`, *optional*, defaults to 32):
47
+ Number of hidden layers in the Transformer encoder.
48
+ num_attention_heads (`int`, *optional*, defaults to 32):
49
+ Number of attention heads for each attention layer in the Transformer encoder.
50
+ num_key_value_heads (`int`, *optional*, defaults to 8):
51
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
52
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
53
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
54
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
55
+ by meanpooling all the original heads within that group. For more details checkout [this
56
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
57
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
58
+ The non-linear activation function (function or string) in the decoder.
59
+ max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
60
+ The maximum sequence length that this model might ever be used with. Quiet's sliding window attention
61
+ allows sequence of up to 4096*32 tokens.
62
+ initializer_range (`float`, *optional*, defaults to 0.02):
63
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
64
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
65
+ The epsilon used by the rms normalization layers.
66
+ use_cache (`bool`, *optional*, defaults to `True`):
67
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
68
+ relevant if `config.is_decoder=True`.
69
+ pad_token_id (`int`, *optional*):
70
+ The id of the padding token.
71
+ bos_token_id (`int`, *optional*, defaults to 1):
72
+ The id of the "beginning-of-sequence" token.
73
+ eos_token_id (`int`, *optional*, defaults to 2):
74
+ The id of the "end-of-sequence" token.
75
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
76
+ Whether the model's input and output word embeddings should be tied.
77
+ rope_theta (`float`, *optional*, defaults to 10000.0):
78
+ The base period of the RoPE embeddings.
79
+ sliding_window (`int`, *optional*, defaults to 4096):
80
+ Sliding window attention window size. If not specified, will default to `4096`.
81
+ attention_dropout (`float`, *optional*, defaults to 0.0):
82
+ The dropout ratio for the attention probabilities.
83
+ ```python
84
+ >>> from transformers import QuietModel, QuietConfig
85
+ >>> # Initializing a Quiet 7B style configuration
86
+ >>> configuration = QuietConfig()
87
+ >>> # Initializing a model from the Quiet 7B style configuration
88
+ >>> model = QuietModel(configuration)
89
+ >>> # Accessing the model configuration
90
+ >>> configuration = model.config
91
+ ```"""
92
+
93
+ model_type = "quiet"
94
+ keys_to_ignore_at_inference = ["past_key_values"]
95
+
96
+ def __init__(
97
+ self,
98
+ vocab_size=32000,
99
+ hidden_size=4096,
100
+ intermediate_size=14336,
101
+ num_hidden_layers=32,
102
+ num_attention_heads=32,
103
+ num_key_value_heads=8,
104
+ hidden_act="silu",
105
+ max_position_embeddings=4096 * 32,
106
+ initializer_range=0.02,
107
+ rms_norm_eps=1e-6,
108
+ use_cache=True,
109
+ pad_token_id=None,
110
+ bos_token_id=1,
111
+ eos_token_id=2,
112
+ tie_word_embeddings=False,
113
+ rope_theta=10000.0,
114
+ complexity_factor = 0.5,
115
+ sliding_window=4096,
116
+ attention_dropout=0.0,
117
+ max_thoughts=16,
118
+ max_temperature=10,
119
+ merged_talk_heads=True,
120
+ merged_lm_and_talk_heads=False,
121
+ merged_lm_and_think_heads=True,
122
+ use_concat_talk_head=True,
123
+ use_shallow_think=True,
124
+ use_shallow_talk=False,
125
+ use_complex_think_head=False,
126
+ use_complex_talk_head=True,
127
+ use_weighted_talk_head=True,
128
+ hidden_dropout_prob=0.0,
129
+ **kwargs,
130
+ ):
131
+ self.vocab_size = vocab_size
132
+ self.max_position_embeddings = max_position_embeddings
133
+ self.hidden_size = hidden_size
134
+ self.intermediate_size = intermediate_size
135
+ self.num_hidden_layers = num_hidden_layers
136
+ self.num_attention_heads = num_attention_heads
137
+ self.sliding_window = sliding_window
138
+
139
+ # for backward compatibility
140
+ if num_key_value_heads is None:
141
+ num_key_value_heads = num_attention_heads
142
+
143
+ self.num_key_value_heads = num_key_value_heads
144
+ self.hidden_act = hidden_act
145
+ self.initializer_range = initializer_range
146
+ self.rms_norm_eps = rms_norm_eps
147
+ self.use_cache = use_cache
148
+ self.rope_theta = rope_theta
149
+ self.attention_dropout = attention_dropout
150
+ self.max_thoughts = max_thoughts
151
+ self.complexity_factor = complexity_factor
152
+ self.max_temperature = max_temperature
153
+ self.merged_talk_heads = merged_talk_heads
154
+ self.merged_lm_and_talk_heads = merged_lm_and_talk_heads
155
+ self.merged_lm_and_think_heads = merged_lm_and_think_heads
156
+ self.use_concat_talk_head = use_concat_talk_head
157
+ self.use_shallow_think = use_shallow_think
158
+ self.use_shallow_talk = use_shallow_talk
159
+ self.use_complex_think_head = use_complex_think_head
160
+ self.use_complex_talk_head = use_complex_talk_head
161
+ self.use_weighted_talk_head = use_weighted_talk_head
162
+ self.hidden_dropout_prob = hidden_dropout_prob
163
+
164
+ super().__init__(
165
+ pad_token_id=pad_token_id,
166
+ bos_token_id=bos_token_id,
167
+ eos_token_id=eos_token_id,
168
+ tie_word_embeddings=tie_word_embeddings,
169
+ **kwargs,
170
+ )
modeling_quiet.py ADDED
The diff for this file is too large to render. See raw diff