model improved for transformers 4.42

config.json

@@ -3,22 +3,11 @@
     "Qwen2ForTokenClassification"
   ],
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.Qwen2ForTokenClassification"
-  },
   "bos_token_id": 151643,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 151643,
@@ -378,7 +367,7 @@
   "tie_word_embeddings": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "torch_dtype": "float32",
-  "transformers_version": "4.41.2",
+  "transformers_version": "4.42.4",
   "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936

maker.sh

@@ -13,43 +13,7 @@ TMP=./maker$$.py
   echo 'tgt="KoichiYasuoka/'$S'-upos"'
 ) > $TMP
 cat << 'EOF' >> $TMP
-from transformers import AutoTokenizer,Qwen2Model,Qwen2PreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
-
-class Qwen2ForTokenClassification(Qwen2PreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=Qwen2Model(config)
-    if getattr(config,"classifier_dropout",None) is not None:
-      classifier_dropout=config.classifier_dropout
-    elif getattr(config,"hidden_dropout",None) is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.score=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    sequence_output=outputs[0]
-    sequence_output=self.dropout(sequence_output)
-    logits=self.score(sequence_output)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=outputs.hidden_states,attentions=outputs.attentions)
+from transformers import AutoTokenizer,Qwen2ForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):

pytorch_model-00001-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e23e7599a49b3ed24e1d177eefdda9b0abb28398945c4100764dc8fd2f920936
+oid sha256:e12c8ea495aecb484307dcee626d76beb7f21d226b3c2c10810d9bf9e0132c75
 size 4985206016

pytorch_model-00002-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:993e9c5854f40997f62ea2fc5c89fa3f9f897742b31897446bb2b65056c366e0
+oid sha256:ec19f4e9c34f5d2fab7e01b1caf093e326bf08b3d8a6b864f09442d33586c5b8
 size 4991772018

pytorch_model-00003-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:209423d79a9e695318dffeb427f751f2194894c5bc373ed33c46d1aef61f9268
+oid sha256:c6228f95327fc3c0fcfb8f7efa151f752842018a93fd0928b78617ff1cc6c54d
 size 4924629448

pytorch_model-00004-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a043fb62b5626542e9304257a4a0707326abc226cd318f23bf051c856c130148
+oid sha256:481439e6224140cc276c8fdf8d58b124fad505f3e17151e003ac6c55b24b5de3
 size 4857520234

pytorch_model-00005-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c8bf62808cb52a132ecea90079865a01e1a09ca24a10c39a1a1457d8d0751c9
+oid sha256:95e066150510e71b2e2be46e0d7c0fc1a31ddde267c8c3c1966b089850661d09
 size 4857520234

pytorch_model-00006-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e035497e53205d23084af41a27f9ad05df079ac8b123658667e3c67f5039b37f
+oid sha256:0f2e42f24be3adea9afa22d6f71c5751857a165195639a0d08b11406f98e4140
 size 3782216922

tokenizer_config.json

@@ -31,7 +31,6 @@
     "<|im_end|>"
   ],
   "bos_token": null,
-  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",

upos.py

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,Qwen2Model,Qwen2PreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import TokenClassificationPipeline
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,42 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-
-class Qwen2ForTokenClassification(Qwen2PreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=Qwen2Model(config)
-    if getattr(config,"classifier_dropout",None) is not None:
-      classifier_dropout=config.classifier_dropout
-    elif getattr(config,"hidden_dropout",None) is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.score=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    sequence_output=outputs[0]
-    sequence_output=self.dropout(sequence_output)
-    logits=self.score(sequence_output)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=outputs.hidden_states,attentions=outputs.attentions)
-