Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+cached_dev_deberta-mlm_128_atomic filter=lfs diff=lfs merge=lfs -text
+cached_train_deberta-mlm_128_atomic filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

cached_dev_deberta-mlm_128_atomic

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf7fda43dc4a34e0720d4dd64075cc48c2ae0002fad2806cdbe2811a0d3bd3c
+size 4451683

cached_train_deberta-mlm_128_atomic

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ff1dec300b58614baae2c3c18eb9d908a8f99cf338c042fb0b5eebb65ca833d
+size 334716007

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "microsoft/deberta-v3-large",
+  "architectures": [
+    "DebertaV2ForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "finetuning_task": "atomic",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

data_utils.py

@@ -0,0 +1,236 @@
+import json
+import logging
+
+import nltk
+from nltk.corpus import stopwords
+from tqdm import tqdm
+from tqdm import tqdm
+
+skip_words = set(stopwords.words('english'))
+skip_words.add('\'s')
+skip_words.add('.')
+skip_words.add(',')
+PERSON_NAMES = ['Alex', 'Ash', 'Aspen', 'Bali', 'Berkeley', 'Cameron', 'Chris', 'Cody', 'Dana', 'Drew', 'Emory',
+                'Flynn', 'Gale', 'Jamie', 'Jesse',
+                'Kai', 'Kendall', 'Kyle', 'Lee', 'Logan', 'Max', 'Morgan', 'Nico', 'Paris', 'Pat', 'Quinn', 'Ray',
+                'Robin', 'Rowan', 'Rudy', 'Sam', 'Skylar', 'Sydney',
+                'Taylor', 'Tracy', 'West', 'Wynne']
+logger = logging.getLogger(__name__)
+
+
+def accuracy(out, labels):
+    return {'acc': (out == labels).mean()}
+
+
+def handle_words(span, tokenizer, keywords=None, is_start=False):
+    inputs = []
+    labels = []
+    words = nltk.word_tokenize(span)
+    for w_i, w in enumerate(words):
+        if (w_i == 0 and is_start) or w == '.' or w == ',' or w.startswith('\''):
+            w_bpes = tokenizer.tokenize(w)
+        else:
+            w_bpes = tokenizer.tokenize(w, add_prefix_space=True)
+        inputs.extend(w_bpes)
+        if keywords != None:
+            if w in keywords:
+                labels.extend(w_bpes)
+            else:
+                labels.extend([-100] * len(w_bpes))
+        else:
+            if w not in PERSON_NAMES and w not in skip_words and w.lower() not in skip_words:
+                labels.extend(w_bpes)
+            else:
+                labels.extend([-100] * len(w_bpes))
+    return inputs, labels
+
+
+def handle_underscores(suffix, tokenizer, keywords=None, prefix=False):
+    inputs = []
+    labels = []
+    if '_' in suffix:
+        suffix_parts = [i.strip() for i in suffix.split('___')]
+        for i, part in enumerate(suffix_parts):
+            if part:
+                tmp_inputs, tmp_labels = handle_words(part, tokenizer, keywords=keywords, is_start=(i == 0 and prefix))
+                inputs += tmp_inputs
+                labels += tmp_labels
+
+                if i != len(suffix_parts) - 1 and suffix_parts[i + 1]:
+                    inputs.append(tokenizer.mask_token)
+                    labels.append(-100)
+            else:
+                inputs.append(tokenizer.mask_token)
+                labels.append(-100)
+    else:
+        inputs, labels = handle_words(suffix, tokenizer, keywords=keywords, is_start=prefix)
+    return inputs, labels
+
+from tqdm import tqdm
+def convert_examples_to_features(examples, tokenizer, max_length=512):
+    data = []
+    for example in tqdm(examples, desc="converting examples to features"):
+        inputs, labels = handle_underscores(example['context'], tokenizer, keywords=example.get('keywords', None), prefix=True)
+        choices = [handle_underscores(cand, tokenizer) for cand in example['candidates']]
+        input_ids = [inputs + cand[0] for cand in choices]
+        input_ids = [tokenizer.convert_tokens_to_ids(cand) for cand in input_ids]
+        label_ids = [labels + cand[1] for cand in choices]
+        label_ids = [[t if t == -100 else input_ids[i][t_i] for t_i, t in enumerate(cand)] for i, cand in
+                     enumerate(label_ids)]
+        label_ids = [[-100] + cand + [-100] for cand in label_ids]
+        input_ids = [tokenizer.prepare_for_model(cand, max_length=max_length, truncation=True)['input_ids'] for cand in
+                     input_ids]
+        data.append([input_ids, label_ids, example['correct']])
+    return data
+
+
+class ATOMICMLMProcessor(object):
+    def __init__(self, args):
+        self.D = []
+        self.filelist = [args.train_file, args.dev_file]
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0])
+        return self.D
+
+    def get_dev_examples(self):
+        data = []
+        with open(self.filelist[1], 'r') as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                data.append(sample)
+        print(len(data))
+        return data
+
+    def load_data(self, filename):
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                self.D.append({'id': sample['id'], 'context': sample['context'],
+                               'ending': sample['candidates'][sample['correct']], 'keywords': sample.get('keywords', None)})
+            print(len(self.D))
+
+
+class ATOMICProcessor(object):
+    def __init__(self, args):
+        print('loading from %s %s' % (args.train_file, args.dev_file))
+        self.filelist = [args.train_file, args.dev_file]
+        self.D = [[], []]
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0], 0)
+        return self.D[0]
+
+    def get_dev_examples(self):
+        self.load_data(self.filelist[1], 1)
+        return self.D[1]
+
+    def load_data(self, filename, sid):
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                self.D[sid].append(sample)
+            print(len(self.D[sid]))
+
+
+class CWWVProcessor(object):
+    def __init__(self, args):
+        self.answerKey_mapping = {'A': 0, 'B': 1, 'C': 2}
+        self.D = [[], []]
+        if args.task_name == 'cskg':
+            print('loading from %s %s' % (args.second_train_file, args.second_dev_file))
+            self.filelist = [args.second_train_file, args.second_dev_file]
+        else:
+            print('loading from %s %s' % (args.train_file, args.dev_file))
+            self.filelist = [args.train_file, args.dev_file]
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0], 0)
+        return self.D[0]
+
+    def get_dev_examples(self):
+        self.load_data(self.filelist[1], 1)
+        return self.D[1]
+
+    def load_data(self, filename, sid):
+        skipped = 0
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                context = sample['question']['stem']
+                if context.endswith('.'):
+                    context = context[:-1]
+                if not context.endswith('[MASK]'):
+                    skipped += 1
+                    context_parts = context.split('[MASK]')
+                    context = context_parts[0].strip()
+                    candidates = [c['text'] + context_parts[1] + '.' for c in sample['question']['choices']]
+                else:
+                    context = context[:-7]
+                    candidates = [c['text'] + '.' for c in sample['question']['choices']]
+                label = self.answerKey_mapping[sample['answerKey']]
+                keywords = nltk.word_tokenize(sample['question']['head'])
+                keywords = [w for w in keywords if w not in skip_words and w.lower() not in skip_words]
+                self.D[sid].append({'id': sample['id'], 'context': context, 'correct': label, 'candidates': candidates,
+                                    'keywords': keywords})
+            print(len(self.D[sid]), skipped)
+
+
+class CWWVMLMProcessor(object):
+    def __init__(self, args):
+        self.answerKey_mapping = {'A': 0, 'B': 1, 'C': 2}
+        self.D = []
+        self.filelist = [args.train_file, args.dev_file]
+        self.args = args
+
+    def get_train_examples(self):
+        self.load_data(self.filelist[0])
+        return self.D
+
+    def get_dev_examples(self):
+        processor = CSKGProcessor(self.args)
+        return processor.get_dev_examples()
+
+    def load_data(self, filename):
+        skipped = 0
+        with open(filename, "r") as f:
+            for row in tqdm(f):
+                sample = json.loads(row)
+                context = sample['question']['stem']
+                if context.endswith('.'):
+                    context = context[:-1]
+                assert context.endswith('[MASK]')
+                context = context[:-7]
+                candidates = [c['text'] + '.' for c in sample['question']['choices']]
+                label = self.answerKey_mapping[sample['answerKey']]
+                keywords = nltk.word_tokenize(sample['question']['head'])
+                keywords = [w for w in keywords if w not in skip_words and w.lower() not in skip_words]
+                self.D.append(
+                    {'id': sample['id'], 'context': context, 'ending': candidates[label], 'keywords': keywords})
+            print(len(self.D))
+
+
+class CSKGProcessor(object):
+    def __init__(self, args):
+        # CWWV set always uses second train/dev file params
+        self.atomicprocessor = ATOMICProcessor(args)
+        self.cwwvprocessor = CWWVProcessor(args)
+
+    def get_train_examples(self):
+        cwwv_questions = self.cwwvprocessor.get_train_examples()
+        atomic_questions = self.atomicprocessor.get_train_examples()
+        return cwwv_questions + atomic_questions
+
+    def get_dev_examples(self):
+        cwwv_questions = self.cwwvprocessor.get_dev_examples()
+        atomic_questions = self.atomicprocessor.get_dev_examples()
+        return cwwv_questions + atomic_questions
+
+
+myprocessors = {
+    "atomic": ATOMICProcessor,
+    "cwwv": CWWVProcessor,
+    "atomicmlm": ATOMICMLMProcessor,
+    "cwwvmlm": CWWVMLMProcessor,
+    "cskg": CSKGProcessor
+}

eval_results.txt

@@ -0,0 +1 @@
+acc = 0.475

logits_test.txt

@@ -0,0 +1,120 @@
+-11.985331535339355 -13.357017517089844 -15.029851913452148 -16.39310073852539 -16.923053741455078
+-12.25283432006836 -15.136346817016602 -15.764205932617188 -17.14737319946289 -16.522167205810547
+-12.823394775390625 -15.571006774902344 -18.32282829284668 -13.100141525268555 -16.811420440673828
+-10.969256401062012 -13.689325332641602 -11.483299255371094 -10.86734390258789 -15.446307182312012
+-10.255317687988281 -11.23694133758545 -12.866697311401367 -10.5074462890625 -14.253149032592773
+-12.706957817077637 -13.617804527282715 -11.692843437194824 -12.443273544311523 -15.97578239440918
+-12.503711700439453 -17.07842254638672 -16.464298248291016 -14.878570556640625 -16.63575553894043
+-11.61707592010498 -13.73621940612793 -12.556190490722656 -16.594358444213867 -15.703083038330078
+-11.196725845336914 -15.495702743530273 -14.119497299194336 -11.031454086303711 -16.43398666381836
+-10.521356582641602 -16.258441925048828 -17.279918670654297 -18.873933792114258 -15.915616989135742
+-11.256342887878418 -18.144832611083984 -17.276641845703125 -15.105140686035156 -16.15310287475586
+-11.635124206542969 -12.914545059204102 -13.48871898651123 -14.709991455078125 -16.50333023071289
+-10.433307647705078 -10.201807022094727 -13.881892204284668 -15.707653045654297 -16.496707916259766
+-12.453126907348633 -13.353202819824219 -11.582731246948242 -13.375235557556152 -17.256397247314453
+-11.852107048034668 -10.913325309753418 -11.62298583984375 -11.65191650390625 -16.276226043701172
+-11.433916091918945 -10.491494178771973 -12.900442123413086 -13.999748229980469 -15.658428192138672
+-11.665186882019043 -12.186442375183105 -14.808893203735352 -14.345434188842773 -15.63730239868164
+-11.523735046386719 -15.132838249206543 -14.409159660339355 -12.7765531539917 -16.388263702392578
+-13.544512748718262 -13.325263977050781 -12.283462524414062 -12.502897262573242 -15.876867294311523
+-11.84539794921875 -16.296030044555664 -13.924732208251953 -15.236044883728027 -16.84980583190918
+-16.54426383972168 -14.162543296813965 -12.575240135192871 -13.8094482421875 -16.731632232666016
+-9.845342636108398 -15.500575065612793 -11.547300338745117 -15.00468635559082 -15.337156295776367
+-11.94580078125 -12.20471477508545 -11.616270065307617 -15.501757621765137 -15.509384155273438
+-11.23939323425293 -12.356117248535156 -13.415891647338867 -11.913195610046387 -16.23021697998047
+-13.631462097167969 -13.59908676147461 -13.40437126159668 -16.07756996154785 -15.115039825439453
+-11.732011795043945 -14.669859886169434 -14.041311264038086 -15.339503288269043 -16.011241912841797
+-8.993430137634277 -12.121005058288574 -10.010058403015137 -14.740504264831543 -14.144813537597656
+-9.875417709350586 -9.784116744995117 -12.305227279663086 -15.662696838378906 -15.405596733093262
+-9.947235107421875 -16.609920501708984 -16.379150390625 -20.26329231262207 -14.625556945800781
+-16.708106994628906 -12.748739242553711 -13.981369018554688 -15.383566856384277 -14.687983512878418
+-11.027362823486328 -17.01676368713379 -14.259870529174805 -13.757545471191406 -15.068336486816406
+-14.97132682800293 -11.124801635742188 -14.391558647155762 -12.971904754638672 -15.092077255249023
+-12.326019287109375 -13.14168930053711 -13.475115776062012 -14.218363761901855 -14.645342826843262
+-11.144308090209961 -13.021101951599121 -14.144937515258789 -13.647077560424805 -15.991572380065918
+-14.37043571472168 -14.487944602966309 -15.227438926696777 -17.627277374267578 -15.609060287475586
+-12.285761833190918 -13.407965660095215 -15.185203552246094 -12.945239067077637 -15.660846710205078
+-13.997186660766602 -13.769948959350586 -14.953908920288086 -11.359712600708008 -14.329699516296387
+-11.684611320495605 -14.430723190307617 -16.186899185180664 -13.829656600952148 -17.049232482910156
+-13.989039421081543 -10.588340759277344 -15.690905570983887 -16.892181396484375 -16.528379440307617
+-11.620543479919434 -11.551864624023438 -13.757423400878906 -11.874419212341309 -15.980398178100586
+-12.374351501464844 -14.128835678100586 -14.20466136932373 -16.15315818786621 -15.599609375
+-14.862902641296387 -14.775538444519043 -11.799307823181152 -13.529033660888672 -16.550161361694336
+-11.758308410644531 -12.798704147338867 -13.636115074157715 -15.04835319519043 -16.15018653869629
+-10.89073371887207 -17.37094497680664 -16.38538360595703 -15.300247192382812 -15.836639404296875
+-12.2295560836792 -13.281776428222656 -11.386110305786133 -13.427268981933594 -15.335490226745605
+-19.555706024169922 -12.590005874633789 -14.160160064697266 -16.365543365478516 -17.29129981994629
+-11.852313041687012 -12.861295700073242 -12.042010307312012 -13.391824722290039 -16.769405364990234
+-11.795409202575684 -13.364195823669434 -14.922141075134277 -11.233430862426758 -16.088077545166016
+-11.884429931640625 -12.776321411132812 -10.96551513671875 -13.180036544799805 -15.149337768554688
+-14.861328125 -14.593585968017578 -12.553210258483887 -17.168903350830078 -15.611528396606445
+-12.72950553894043 -10.737747192382812 -15.163175582885742 -14.780402183532715 -15.362897872924805
+-10.313711166381836 -12.986738204956055 -13.380842208862305 -16.455732345581055 -14.981689453125
+-12.478826522827148 -16.76121711730957 -16.831989288330078 -16.525733947753906 -15.847885131835938
+-14.628618240356445 -11.984136581420898 -12.582441329956055 -16.439844131469727 -16.265674591064453
+-11.344210624694824 -12.544027328491211 -15.989503860473633 -15.10544490814209 -15.973136901855469
+-14.520594596862793 -13.788124084472656 -13.798048973083496 -13.979040145874023 -15.778616905212402
+-13.025321006774902 -12.753511428833008 -11.481473922729492 -16.253257751464844 -14.98487377166748
+-12.689830780029297 -16.152624130249023 -10.79024887084961 -13.604175567626953 -14.96052074432373
+-10.081911087036133 -12.857580184936523 -14.366785049438477 -14.925143241882324 -15.236709594726562
+-13.72541332244873 -17.140247344970703 -14.554573059082031 -16.237695693969727 -15.220977783203125
+-12.363258361816406 -14.671875 -12.393234252929688 -13.685455322265625 -15.162212371826172
+-12.525287628173828 -11.755924224853516 -11.487818717956543 -10.964709281921387 -14.18620491027832
+-12.641595840454102 -12.06806468963623 -13.470266342163086 -15.523735046386719 -15.110023498535156
+-10.243486404418945 -10.8483304977417 -15.268081665039062 -17.73096466064453 -14.491694450378418
+-11.410097122192383 -14.304682731628418 -17.227609634399414 -15.861364364624023 -15.366393089294434
+-13.248863220214844 -13.261842727661133 -13.531661033630371 -16.54248809814453 -16.259929656982422
+-12.59164810180664 -13.846809387207031 -15.791946411132812 -13.07502269744873 -14.861738204956055
+-15.870349884033203 -18.1973876953125 -14.544315338134766 -16.514846801757812 -14.571775436401367
+-11.389364242553711 -12.424586296081543 -11.14108943939209 -15.748624801635742 -14.8790922164917
+-12.217803001403809 -15.62755012512207 -16.650747299194336 -16.509571075439453 -15.311178207397461
+-12.044157028198242 -14.750267028808594 -17.582698822021484 -13.546934127807617 -15.512325286865234
+-14.079737663269043 -10.69660758972168 -13.699287414550781 -15.674205780029297 -14.529096603393555
+-11.215898513793945 -13.137222290039062 -15.151588439941406 -15.361551284790039 -15.427042961120605
+-15.472491264343262 -13.31407356262207 -17.27981948852539 -15.681211471557617 -15.767252922058105
+-11.327293395996094 -17.34939956665039 -17.840465545654297 -15.820741653442383 -14.566900253295898
+-13.018766403198242 -15.517611503601074 -17.448204040527344 -16.01276397705078 -15.718643188476562
+-11.193071365356445 -15.165118217468262 -14.07136344909668 -12.923444747924805 -14.62411117553711
+-12.101425170898438 -14.308100700378418 -14.739375114440918 -14.646849632263184 -14.93545913696289
+-9.732519149780273 -12.507312774658203 -14.191499710083008 -12.70343017578125 -14.431779861450195
+-14.082250595092773 -14.632597923278809 -15.651617050170898 -12.986234664916992 -16.02080726623535
+-10.87934684753418 -15.8912992477417 -13.874499320983887 -14.047943115234375 -15.021934509277344
+-13.368215560913086 -12.769327163696289 -12.11189079284668 -14.995542526245117 -15.780769348144531
+-13.823423385620117 -18.496301651000977 -13.304229736328125 -18.130481719970703 -15.556083679199219
+-11.371943473815918 -15.399284362792969 -16.673551559448242 -17.802629470825195 -16.158344268798828
+-12.098010063171387 -16.252744674682617 -15.276991844177246 -15.084753036499023 -14.278797149658203
+-13.902384757995605 -14.19427490234375 -15.481269836425781 -13.694448471069336 -15.118891716003418
+-11.553057670593262 -18.223403930664062 -14.778602600097656 -17.133541107177734 -14.881399154663086
+-11.718768119812012 -11.310720443725586 -18.56161117553711 -13.747978210449219 -14.171229362487793
+-12.3494873046875 -10.71717643737793 -12.61784553527832 -13.59307861328125 -14.498141288757324
+-14.827796936035156 -14.959348678588867 -12.612112045288086 -18.091611862182617 -16.525957107543945
+-14.295256614685059 -17.82564926147461 -17.82574462890625 -15.090109825134277 -16.88511085510254
+-11.850313186645508 -13.219352722167969 -13.954887390136719 -12.791426658630371 -14.770899772644043
+-10.91953182220459 -14.908056259155273 -18.78179931640625 -16.55717658996582 -16.11922836303711
+-10.402986526489258 -16.308622360229492 -13.00355052947998 -17.364574432373047 -14.528853416442871
+-11.989408493041992 -16.08974266052246 -12.309541702270508 -17.035097122192383 -15.44270133972168
+-12.839983940124512 -16.940263748168945 -16.2291259765625 -16.91362762451172 -15.839340209960938
+-12.685194969177246 -15.007822036743164 -15.435506820678711 -15.527690887451172 -14.830557823181152
+-15.379294395446777 -11.584933280944824 -13.179071426391602 -15.63725471496582 -15.120612144470215
+-11.626914024353027 -13.498213768005371 -14.436857223510742 -16.254169464111328 -14.862478256225586
+-12.431161880493164 -12.475348472595215 -16.244712829589844 -19.31494140625 -16.310794830322266
+-11.904972076416016 -12.728023529052734 -16.9464054107666 -16.66975975036621 -17.148090362548828
+-14.137011528015137 -16.81878662109375 -15.127734184265137 -15.06110954284668 -16.270536422729492
+-10.801024436950684 -13.239822387695312 -13.742114067077637 -12.634249687194824 -14.606036186218262
+-9.93199348449707 -16.211549758911133 -14.313543319702148 -15.699499130249023 -14.912216186523438
+-13.826777458190918 -15.405548095703125 -16.025297164916992 -15.602980613708496 -15.071611404418945
+-10.962064743041992 -14.387460708618164 -14.632959365844727 -11.710444450378418 -14.714736938476562
+-11.960188865661621 -13.753063201904297 -15.276123046875 -18.502769470214844 -13.509201049804688
+-14.572550773620605 -14.06547737121582 -13.678935050964355 -18.911367416381836 -15.910487174987793
+-13.86817455291748 -17.146404266357422 -13.30243968963623 -14.152023315429688 -16.338394165039062
+-12.468561172485352 -13.683039665222168 -14.846575736999512 -11.693964958190918 -14.717597961425781
+-12.660326957702637 -12.625338554382324 -15.899386405944824 -13.582548141479492 -15.96905517578125
+-9.468055725097656 -15.10066032409668 -13.716577529907227 -14.704435348510742 -13.372961044311523
+-11.74072265625 -17.22024154663086 -16.204971313476562 -15.789412498474121 -15.067605972290039
+-13.240243911743164 -12.687658309936523 -15.326522827148438 -17.321617126464844 -14.255077362060547
+-15.12217903137207 -12.139941215515137 -13.076716423034668 -19.28219985961914 -14.893415451049805
+-13.608329772949219 -16.2562313079834 -15.612630844116211 -14.110588073730469 -15.298316955566406
+-12.895358085632324 -12.858833312988281 -16.372390747070312 -15.170218467712402 -15.558211326599121
+-14.180404663085938 -15.49008560180664 -15.325713157653809 -14.368494987487793 -14.509702682495117
+-13.97700309753418 -17.67074966430664 -16.38104248046875 -11.222564697265625 -14.656253814697266
+-13.6754150390625 -13.450912475585938 -18.705039978027344 -14.589435577392578 -15.892086029052734

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08ef1f881fcd130b64b6b3c53ae4af7d35415f6d144ea22a0559c38ef1d25f2b
+size 1740904889

run_pretrain.py

@@ -0,0 +1,659 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import json
+import logging
+import os
+import random
+import wandb
+
+import numpy as np
+import torch
+from torch.optim import AdamW
+from torch.utils.data import DataLoader
+from torch.utils.data import RandomSampler
+from torch.utils.data import SequentialSampler
+from torch.utils.data.distributed import DistributedSampler
+from torch.utils.tensorboard import SummaryWriter
+from tqdm import tqdm
+from tqdm import trange
+from transformers import DebertaV2Config
+from transformers import DebertaV2ForMaskedLM
+from transformers import DebertaV2Tokenizer
+from transformers import RobertaConfig
+from transformers import RobertaForMaskedLM
+from transformers import RobertaTokenizer
+from transformers import get_linear_schedule_with_warmup
+
+from data_utils import accuracy
+from data_utils import convert_examples_to_features
+from data_utils import myprocessors
+
+from evaluate_DeBERTa import eval_tasks
+from evaluate_DeBERTa import main as evaluate_main
+
+logger = logging.getLogger(__name__)
+
+from transformers import MODEL_WITH_LM_HEAD_MAPPING
+
+MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+MODEL_CLASSES = {
+    'roberta-mlm': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
+    'deberta-mlm': (DebertaV2Config, DebertaV2ForMaskedLM, DebertaV2Tokenizer)
+}
+
+
+class MyDataset(torch.utils.data.Dataset):
+
+    def __init__(self, data, pad_token, mask_token, max_words_to_mask):
+        self.data = data
+        self.pad_token = pad_token
+        self.mask_token = mask_token
+        self.max_words_to_mask = max_words_to_mask
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sample = self.data[idx]
+        return sample, self.pad_token, self.mask_token, self.max_words_to_mask
+
+
+def mCollateFn(batch):
+    batch_input_ids = []
+    batch_input_mask = []
+    batch_input_labels = []
+    batch_label_ids = []
+    features = [b[0] for b in batch]
+    pad_token = batch[0][1]
+    mask_token = batch[0][2]
+    MAX_WORDS_TO_MASK = batch[0][3]
+    max_len = max([len(cand) for f in features for cand in f[0]])
+    for f in features:
+        batch_input_ids.append([])
+        batch_input_mask.append([])
+        batch_input_labels.append([])
+        batch_label_ids.append(f[2])
+        for i in range(len(f[0])):
+            masked_sequences = []
+            masked_labels = []
+            this_att_mask = []
+            sequence = f[0][i] + [pad_token] * (max_len - len(f[0][i]))
+            label_sequence = f[1][i] + [-100] * (max_len - len(f[1][i]))
+            valid_indices = [l_i for l_i, l in enumerate(label_sequence) if l != -100]
+            if len(valid_indices) > MAX_WORDS_TO_MASK:
+                rm_indices = random.sample(valid_indices, (len(valid_indices) - MAX_WORDS_TO_MASK))
+                label_sequence = [-100 if l_i in rm_indices else l for l_i, l in enumerate(label_sequence)]
+            for j, t in enumerate(label_sequence):
+                if t == -100:
+                    continue
+                    masked_sequences.append(sequence)
+                    masked_labels.append([-100] * max_len)
+                else:
+                    masked_sequences.append(sequence[:j] + [mask_token] + sequence[j + 1:])
+                    masked_labels.append([-100] * j + [sequence[j]] + [-100] * (max_len - j - 1))
+                this_att_mask.append([1] * len(f[0][i]) + [0] * (max_len - len(f[0][i])))
+            batch_input_ids[-1].append(torch.tensor(masked_sequences, dtype=torch.long))
+            batch_input_mask[-1].append(torch.tensor(this_att_mask, dtype=torch.long))
+            batch_input_labels[-1].append(torch.tensor(masked_labels, dtype=torch.long))
+    return batch_input_ids, batch_input_mask, batch_input_labels, torch.tensor(batch_label_ids, dtype=torch.long)
+
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def count_parameters(model):
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+
+def train(args, train_dataset, model, tokenizer, eval_dataset):
+    """ Train the model """
+    if args.local_rank in [-1, 0]:
+        tb_writer = SummaryWriter(os.path.join(args.output_dir, 'runs'))
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size,
+                                  collate_fn=mCollateFn)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+         'weight_decay': args.weight_decay},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
+    ]
+
+    warmup_steps = args.warmup_steps if args.warmup_steps != 0 else int(args.warmup_proportion * t_total)
+    logger.info("warm up steps = %d", warmup_steps)
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon, betas=(0.9, 0.98))
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=warmup_steps, num_training_steps=t_total)
+
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
+                                                          output_device=args.local_rank,
+                                                          find_unused_parameters=True)
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
+                args.train_batch_size * args.gradient_accumulation_steps * (
+                    torch.distributed.get_world_size() if args.local_rank != -1 else 1))
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    curr_best = 0.0
+    CE = torch.nn.CrossEntropyLoss(reduction='none')
+    loss_fct = torch.nn.MultiMarginLoss(margin=args.margin)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in tqdm(enumerate(epoch_iterator), desc=f"Train Epoch {_}"):
+            model.train()
+            num_cand = len(batch[0][0])
+            choice_loss = []
+            choice_seq_lens = np.array([0] + [len(c) for sample in batch[0] for c in sample])
+            choice_seq_lens = np.cumsum(choice_seq_lens)
+            input_ids = torch.cat([c for sample in batch[0] for c in sample], dim=0).to(args.device)
+            att_mask = torch.cat([c for sample in batch[1] for c in sample], dim=0).to(args.device)
+            input_labels = torch.cat([c for sample in batch[2] for c in sample], dim=0).to(args.device)
+
+            if len(input_ids) < args.max_sequence_per_time:
+                inputs = {'input_ids': input_ids,
+                          'attention_mask': att_mask}
+                outputs = model(**inputs)
+                ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)), input_labels.view(-1))
+                ce_loss = ce_loss.view(outputs[0].size(0), -1).sum(1)
+            else:
+                ce_loss = []
+                for chunk in range(0, len(input_ids), args.max_sequence_per_time):
+                    inputs = {'input_ids': input_ids[chunk:chunk + args.max_sequence_per_time],
+                              'attention_mask': att_mask[chunk:chunk + args.max_sequence_per_time]}
+                    outputs = model(**inputs)
+                    tmp_ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)),
+                                     input_labels[chunk:chunk + args.max_sequence_per_time].view(-1))
+                    tmp_ce_loss = tmp_ce_loss.view(outputs[0].size(0), -1).sum(1)
+                    ce_loss.append(tmp_ce_loss)
+                ce_loss = torch.cat(ce_loss, dim=0)
+            # all tokens are valid
+            for c_i in range(len(choice_seq_lens) - 1):
+                start = choice_seq_lens[c_i]
+                end = choice_seq_lens[c_i + 1]
+                choice_loss.append(-ce_loss[start:end].sum() / (end - start))
+
+            choice_loss = torch.stack(choice_loss)
+            choice_loss = choice_loss.view(-1, num_cand)
+            loss = loss_fct(choice_loss, batch[3].to(args.device))
+
+            if args.n_gpu > 1:
+                loss = loss.mean()  # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            
+            if (step + 1) % args.gradient_accumulation_steps == 0:
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    tb_writer.add_scalar('lr', scheduler.get_last_lr()[0], global_step)
+                    tb_writer.add_scalar('loss', (tr_loss - logging_loss) / args.logging_steps, global_step)
+                    tb_writer.add_scalar('Batch_loss', loss.item() * args.gradient_accumulation_steps, global_step)
+                    logger.info(" global_step = %s, average loss = %s", global_step,
+                                (tr_loss - logging_loss) / args.logging_steps)
+                    wandb.log({"train/loss":loss.item()})
+                    logging_loss = tr_loss
+
+                if args.local_rank == -1 and args.evaluate_during_training and global_step % args.save_steps == 0:
+                    torch.cuda.empty_cache()
+                    results = evaluate(args, model, tokenizer, eval_dataset)
+                    wandb.log({"eval/"+k:v for k,v in results.items()})
+                    for key, value in results.items():
+                        tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+                    if results['acc'] > curr_best:
+                        curr_best = results['acc']
+                        print("At iteration {}, best acc is {}".format(global_step, curr_best))
+                        # Save model checkpoint
+                        output_dir = args.output_dir
+                        if not os.path.exists(output_dir):
+                            os.makedirs(output_dir)
+                        model_to_save = model.module if hasattr(model,
+                                                                'module') else model  # Take care of distributed/parallel training
+                        model_to_save.save_pretrained(output_dir)
+                        tokenizer.save_pretrained(output_dir)
+                        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                        logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+    results = evaluate(args, model, tokenizer, eval_dataset)
+    for key, value in results.items():
+        tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
+    if results['acc'] > curr_best:
+        curr_best = results['acc']
+        # Save model checkpoint
+        output_dir = args.output_dir
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        model_to_save = model.module if hasattr(model,
+                                                'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(output_dir)
+        tokenizer.save_pretrained(output_dir)
+        torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+        logger.info("Saving model checkpoint to %s", output_dir)
+    if args.local_rank in [-1, 0]:
+        tb_writer.close()
+    return global_step, tr_loss / global_step
+
+
+def save_logits(logits_all, filename):
+    with open(filename, "w") as f:
+        for i in range(len(logits_all)):
+            for j in range(len(logits_all[i])):
+                f.write(str(logits_all[i][j]))
+                if j == len(logits_all[i]) - 1:
+                    f.write("\n")
+                else:
+                    f.write(" ")
+
+
+def evaluate(args, model, tokenizer, eval_dataset):
+    results = {}
+    if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+        os.makedirs(args.output_dir)
+
+    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+    # Note that DistributedSampler samples randomly
+    eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
+    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size,
+                                 collate_fn=mCollateFn)
+
+    # Eval!
+    logger.info("***** Running evaluation *****")
+    logger.info("  Num examples = %d", len(eval_dataset))
+    logger.info("  Batch size = %d", args.eval_batch_size)
+    CE = torch.nn.CrossEntropyLoss(reduction='none')
+    preds = []
+    out_label_ids = []
+    for batch in tqdm(eval_dataloader, desc="Evaluating"):
+        model.eval()
+        with torch.no_grad():
+            num_cand = len(batch[0][0])
+            choice_loss = []
+            choice_seq_lens = np.array([0] + [len(c) for sample in batch[0] for c in sample])
+            choice_seq_lens = np.cumsum(choice_seq_lens)
+            input_ids = torch.cat([c for sample in batch[0] for c in sample], dim=0).to(args.device)
+            att_mask = torch.cat([c for sample in batch[1] for c in sample], dim=0).to(args.device)
+            input_labels = torch.cat([c for sample in batch[2] for c in sample], dim=0).to(args.device)
+            if len(input_ids) < args.max_sequence_per_time:
+                inputs = {'input_ids': input_ids,
+                          'attention_mask': att_mask}
+                outputs = model(**inputs)
+                ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)), input_labels.view(-1))
+                ce_loss = ce_loss.view(outputs[0].size(0), -1).sum(1)
+            else:
+                ce_loss = []
+                for chunk in range(0, len(input_ids), args.max_sequence_per_time):
+                    inputs = {'input_ids': input_ids[chunk:chunk + args.max_sequence_per_time],
+                              'attention_mask': att_mask[chunk:chunk + args.max_sequence_per_time]}
+                    outputs = model(**inputs)
+                    tmp_ce_loss = CE(outputs[0].view(-1, outputs[0].size(-1)),
+                                     input_labels[chunk:chunk + args.max_sequence_per_time].view(-1))
+                    tmp_ce_loss = tmp_ce_loss.view(outputs[0].size(0), -1).sum(1)
+                    ce_loss.append(tmp_ce_loss)
+                ce_loss = torch.cat(ce_loss, dim=0)
+            for c_i in range(len(choice_seq_lens) - 1):
+                start = choice_seq_lens[c_i]
+                end = choice_seq_lens[c_i + 1]
+                choice_loss.append(-ce_loss[start:end].sum() / (end - start))
+            choice_loss = torch.stack(choice_loss)
+            choice_loss = choice_loss.view(-1, num_cand)
+        preds.append(choice_loss)
+        out_label_ids.append(batch[3].numpy())
+    preds = torch.cat(preds, dim=0).cpu().numpy()
+    save_logits(preds.tolist(), os.path.join(args.output_dir, args.logits_file))
+    preds = np.argmax(preds, axis=1)
+    result = accuracy(preds, np.concatenate(out_label_ids, axis=0))
+    results.update(result)
+    output_eval_file = os.path.join(args.output_dir, args.results_file)
+    with open(output_eval_file, "w") as writer:
+        logger.info("***** Eval results *****")
+        for key in sorted(result.keys()):
+            print("%s = %s\n" % (key, str(result[key])))
+            logger.info("  %s = %s", key, str(result[key]))
+            writer.write("%s = %s\n" % (key, str(result[key])))
+    return results
+
+
+def write_data(filename, data):
+    with open(filename, 'w') as fout:
+        for sample in data:
+            fout.write(json.dumps(sample))
+            fout.write('\n')
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+    processor = myprocessors[task](args)
+    cached_features_file = os.path.join(args.output_dir, 'cached_{}_{}_{}_{}'.format(
+        'dev' if evaluate else 'train',
+        str(args.model_type),
+        str(args.max_seq_length),
+        str(task)))
+    if os.path.exists(cached_features_file): # remove evaluate
+        print("loading cache file from", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        print("re-processing feature")
+        examples = processor.get_dev_examples() if evaluate else processor.get_train_examples()
+        features = convert_examples_to_features(examples, tokenizer, max_length=args.max_seq_length)
+        # if evaluate:
+        torch.save(features, cached_features_file)
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+    print('max_words_to_mask is %s for pretraining tasks %s' % (args.max_words_to_mask, task))
+    return MyDataset(features, tokenizer.pad_token_id, tokenizer.mask_token_id, args.max_words_to_mask)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    ## Required parameters
+    parser.add_argument("--train_file", default=None, type=str, required=True,
+                        help="The train file name")
+    parser.add_argument("--dev_file", default=None, type=str, required=True,
+                        help="The dev file name")
+    parser.add_argument("--model_type", default=None, type=str, required=True,
+                        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
+                        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(
+                            MODEL_TYPES))
+    parser.add_argument("--config_name", default="", type=str,
+                        help="Pretrained config name or path if not the same as model_name")
+    parser.add_argument("--tokenizer_name", default="", type=str,
+                        help="Pretrained tokenizer name or path if not the same as model_name")
+    parser.add_argument("--cache_dir", default=".cache", type=str,
+                        help="Where do you want to store the pre-trained models downloaded")
+    parser.add_argument("--task_name", default=None, type=str, required=True,
+                        help="The name of the task to train selected in the list: " + ", ".join(myprocessors.keys()))
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+
+    ## Other parameters
+    parser.add_argument("--second_train_file", default=None, type=str,
+                        help="Used when combining ATOMIC and CWWV")
+    parser.add_argument("--second_dev_file", default=None, type=str,
+                        help="Used when combining ATOMIC and CWWV")
+    parser.add_argument("--max_seq_length", default=128, type=int,
+                        help="The maximum total input sequence length after tokenization. Sequences longer "
+                             "than this will be truncated, sequences shorter will be padded.")
+    parser.add_argument("--max_words_to_mask", default=6, type=int,
+                        help="The maximum number of tokens to mask when computing scores")
+    parser.add_argument("--max_sequence_per_time", default=80, type=int,
+                        help="The maximum number of sequences to feed into the model")
+    parser.add_argument("--do_train", action='store_true',
+                        help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true',
+                        help="Whether to run eval on the dev set.")
+    parser.add_argument("--do_ext_eval", action='store_true',
+                        help="Whether to run external eval on the downstream mcqa datasets.")                    
+    parser.add_argument("--evaluate_during_training", action='store_true',
+                        help="Run evaluation during training at each logging step.")
+    parser.add_argument("--do_lower_case", action='store_true',
+                        help="Set this flag if you are using an uncased model.")
+    parser.add_argument("--per_gpu_train_batch_size", default=1, type=int,
+                        help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=1, type=int,
+                        help="Batch size per GPU/CPU for evaluation.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument("--margin", default=1.0, type=float,
+                        help="The margin for ranking loss")
+    parser.add_argument("--learning_rate", default=1e-5, type=float,
+                        help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.01, type=float,
+                        help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-6, type=float,
+                        help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float,
+                        help="Max gradient norm.")
+    parser.add_argument("--num_train_epochs", default=1.0, type=float,
+                        help="Total number of training epochs to perform.")
+    parser.add_argument("--max_steps", default=-1, type=int,
+                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
+    parser.add_argument("--warmup_steps", default=0, type=int,
+                        help="Linear warmup over warmup_steps.")
+    parser.add_argument("--warmup_proportion", default=0.05, type=float,
+                        help="Linear warmup over warmup proportion.")
+    parser.add_argument('--logging_steps', type=int, default=50,
+                        help="Log every X updates steps.")
+    parser.add_argument('--save_steps', type=int, default=50,
+                        help="Save checkpoint every X updates steps.")
+    parser.add_argument("--logits_file", default='logits_test.txt', type=str,
+                        help="The file where prediction logits will be written")
+    parser.add_argument("--results_file", default='eval_results.txt', type=str,
+                        help="The file where eval results will be written")
+    parser.add_argument("--no_cuda", action='store_true',
+                        help="Avoid using CUDA when available")
+    parser.add_argument('--overwrite_output_dir', action='store_true',
+                        help="Overwrite the content of the output directory")
+    parser.add_argument('--seed', type=int, default=42,
+                        help="random seed for initialization")
+    parser.add_argument('--fp16', action='store_true',
+                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
+    parser.add_argument('--fp16_opt_level', type=str, default='O1',
+                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                             "See details at https://nvidia.github.io/apex/amp.html")
+    parser.add_argument("--local_rank", type=int, default=-1,
+                        help="For distributed training: local_rank")
+    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
+    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")
+
+    ### for extrinsic evaluation
+
+    parser.add_argument("--eval_output_dir", default="./output/eval_results", type=str, required=True,
+                    help="output of the predictions")
+
+    args = parser.parse_args()
+
+    wandb.init(project="car_mcqa", config=args)
+
+    if os.path.exists(args.output_dir) and os.listdir(
+            args.output_dir) and not args.overwrite_output_dir and args.do_train:
+        raise ValueError(
+            "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
+                args.output_dir))
+    if not os.path.exists(args.output_dir):
+        os.makedirs(args.output_dir)
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl')
+        args.n_gpu = 1
+    args.device = device
+
+    if args.do_train:
+        for handler in logging.root.handlers[:]:
+            logging.root.removeHandler(handler)
+    # Setup logging
+    if args.do_train:
+        log_file = os.path.join(args.output_dir, 'train.log')
+        logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                            datefmt='%m/%d/%Y %H:%M:%S',
+                            level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
+                            filename=log_file)
+        logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+                       args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
+        os.system("cp run_pretrain.py %s" % os.path.join(args.output_dir, 'run_pretrain.py'))
+        os.system("cp data_utils.py %s" % os.path.join(args.output_dir, 'data_utils.py'))
+
+    # Set seed
+    set_seed(args)
+    args.task_name = args.task_name.lower()
+    if args.task_name not in myprocessors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          finetuning_task=args.task_name, cache_dir=args.cache_dir)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case, cache_dir=args.cache_dir)
+    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config, cache_dir=args.cache_dir)
+
+    count = count_parameters(model)
+    print("number of params", count)
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+    print("loading eval set")
+    eval_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=True)
+    print("num of eval set", len(eval_dataset))
+
+    if args.do_train:
+        init_result = evaluate(args, model, tokenizer, eval_dataset)
+        print(init_result)
+    
+    if args.do_train:
+        print("loading training set")
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        print("num train examples", len(train_dataset))
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer, eval_dataset)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+    
+    # Evaluation
+    
+    results = {}
+    if args.do_eval:
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        model = model_class.from_pretrained(args.output_dir)
+        model.eval()
+        model.to(args.device)
+        result = evaluate(args, model, tokenizer, eval_dataset)
+
+
+    # do extrinsic evaluation
+
+    if args.do_ext_eval:
+        del model
+        import gc
+        gc.collect()
+        torch.cuda.empty_cache()
+
+
+        ext_results = {}
+        ext_task_avg_acc = 0
+
+        for task_name, dataset_path in eval_tasks:
+            eval_args = argparse.Namespace()
+            eval_args.dataset_file = dataset_path
+            eval_args.lm = args.output_dir
+            eval_args.out_dir = os.path.join(args.eval_output_dir, os.path.basename( args.output_dir))
+            eval_args.device = 0
+            eval_args.reader = task_name
+            eval_args.overwrite_output_dir = args.overwrite_output_dir
+            eval_args.cache_dir = None
+            if task_name in ["socialiqa", "winogrande", "piqa", "commonsenseqa", "anli"]:
+                acc = evaluate_main(eval_args)
+                ext_results[task_name] = acc
+                ext_task_avg_acc += acc
+            else:
+                tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+                model = model_class.from_pretrained(args.output_dir)
+                model.eval()
+                model.to(args.device)
+                
+                # load data
+                examples = []
+                with open(dataset_path, "r") as f:
+                    for row in tqdm(f):
+                        sample = json.loads(row)
+                        examples.append(sample)
+                features = convert_examples_to_features(examples, tokenizer, max_length=args.max_seq_length)
+                eval_dataset = MyDataset(features, tokenizer.pad_token_id, tokenizer.mask_token_id, args.max_words_to_mask)
+                result = evaluate(args, model, tokenizer, eval_dataset)
+                ext_results[task_name] = result['acc']
+        
+        ext_results['avg'] = ext_task_avg_acc / 5
+
+
+        wandb.log({"ext/"+task_name:acc for task_name, acc in ext_results.items()})
+        
+    # return results
+
+if __name__ == "__main__":
+    main()

runs/events.out.tfevents.1696030376.car-2i-100k-name-seed101-5e-6-0-0.28.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa047531aca9a5c6a3f42669cbaabf4bdeca5754bfbfbc15fdc873d816d158f8
+size 53212

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

train.log

@@ -0,0 +1,878 @@
+09/29/2023 23:17:38 - WARNING - __main__ -   Process rank: -1, device: cuda, n_gpu: 1, distributed training: False, 16-bits training: False
+09/29/2023 23:17:49 - INFO - __main__ -   Training/evaluation parameters Namespace(train_file='../../../data/mcqa/atomic/train_atmc_2i_100k_name.jsonl', dev_file='../../../data/mcqa/atomic/dev_atmc_SyntheticQA_10k.jsonl', model_type='deberta-mlm', model_name_or_path='microsoft/deberta-v3-large', config_name='', tokenizer_name='', cache_dir='.cache', task_name='atomic', output_dir='output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', second_train_file=None, second_dev_file=None, max_seq_length=128, max_words_to_mask=6, max_sequence_per_time=80, do_train=True, do_eval=True, do_ext_eval=True, evaluate_during_training=True, do_lower_case=False, per_gpu_train_batch_size=2, per_gpu_eval_batch_size=32, gradient_accumulation_steps=16, margin=1.0, learning_rate=5e-06, weight_decay=0.01, adam_epsilon=1e-06, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, warmup_steps=0, warmup_proportion=0.05, logging_steps=50, save_steps=200, logits_file='logits_test.txt', results_file='eval_results.txt', no_cuda=False, overwrite_output_dir=False, seed=101, fp16=False, fp16_opt_level='O1', local_rank=-1, server_ip='', server_port='', eval_output_dir='./eval_results', n_gpu=1, device=device(type='cuda'))
+09/29/2023 23:17:58 - INFO - __main__ -   ***** Running evaluation *****
+09/29/2023 23:17:58 - INFO - __main__ -     Num examples = 10000
+09/29/2023 23:17:58 - INFO - __main__ -     Batch size = 32
+09/29/2023 23:22:13 - INFO - __main__ -   ***** Eval results *****
+09/29/2023 23:22:13 - INFO - __main__ -     acc = 0.3356
+09/29/2023 23:32:56 - INFO - __main__ -   warm up steps = 916
+09/29/2023 23:32:56 - INFO - __main__ -   ***** Running training *****
+09/29/2023 23:32:56 - INFO - __main__ -     Num examples = 586778
+09/29/2023 23:32:56 - INFO - __main__ -     Num Epochs = 1
+09/29/2023 23:32:56 - INFO - __main__ -     Instantaneous batch size per GPU = 2
+09/29/2023 23:32:56 - INFO - __main__ -     Total train batch size (w. parallel, distributed & accumulation) = 32
+09/29/2023 23:32:56 - INFO - __main__ -     Gradient Accumulation steps = 16
+09/29/2023 23:32:56 - INFO - __main__ -     Total optimization steps = 18336
+09/29/2023 23:36:55 - INFO - __main__ -    global_step = 50, average loss = 0.6978485188353807
+09/29/2023 23:41:05 - INFO - __main__ -    global_step = 100, average loss = 0.6761001783981919
+09/29/2023 23:45:18 - INFO - __main__ -    global_step = 150, average loss = 0.6527128890505992
+09/29/2023 23:49:15 - INFO - __main__ -    global_step = 200, average loss = 0.6255776268531917
+09/29/2023 23:49:16 - INFO - __main__ -   ***** Running evaluation *****
+09/29/2023 23:49:16 - INFO - __main__ -     Num examples = 10000
+09/29/2023 23:49:16 - INFO - __main__ -     Batch size = 32
+09/29/2023 23:53:34 - INFO - __main__ -   ***** Eval results *****
+09/29/2023 23:53:34 - INFO - __main__ -     acc = 0.3839
+09/29/2023 23:54:05 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/29/2023 23:58:03 - INFO - __main__ -    global_step = 250, average loss = 0.5687153974524699
+09/30/2023 00:02:07 - INFO - __main__ -    global_step = 300, average loss = 0.4650766727951122
+09/30/2023 00:06:15 - INFO - __main__ -    global_step = 350, average loss = 0.344281620121983
+09/30/2023 00:10:25 - INFO - __main__ -    global_step = 400, average loss = 0.2641717765412432
+09/30/2023 00:10:26 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 00:10:26 - INFO - __main__ -     Num examples = 10000
+09/30/2023 00:10:26 - INFO - __main__ -     Batch size = 32
+09/30/2023 00:14:45 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 00:14:45 - INFO - __main__ -     acc = 0.6657
+09/30/2023 00:15:14 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 00:19:09 - INFO - __main__ -    global_step = 450, average loss = 0.203622583138349
+09/30/2023 00:23:15 - INFO - __main__ -    global_step = 500, average loss = 0.19167841194193896
+09/30/2023 00:27:33 - INFO - __main__ -    global_step = 550, average loss = 0.1768511165331256
+09/30/2023 00:31:46 - INFO - __main__ -    global_step = 600, average loss = 0.17364913663874176
+09/30/2023 00:31:47 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 00:31:47 - INFO - __main__ -     Num examples = 10000
+09/30/2023 00:31:47 - INFO - __main__ -     Batch size = 32
+09/30/2023 00:36:06 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 00:36:06 - INFO - __main__ -     acc = 0.7383
+09/30/2023 00:36:35 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 00:40:35 - INFO - __main__ -    global_step = 650, average loss = 0.16046627445422929
+09/30/2023 00:44:50 - INFO - __main__ -    global_step = 700, average loss = 0.15604460480608395
+09/30/2023 00:49:12 - INFO - __main__ -    global_step = 750, average loss = 0.16073274322843645
+09/30/2023 00:53:44 - INFO - __main__ -    global_step = 800, average loss = 0.15695772335122457
+09/30/2023 00:53:44 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 00:53:44 - INFO - __main__ -     Num examples = 10000
+09/30/2023 00:53:44 - INFO - __main__ -     Batch size = 32
+09/30/2023 00:58:03 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 00:58:03 - INFO - __main__ -     acc = 0.7684
+09/30/2023 00:58:33 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 01:02:32 - INFO - __main__ -    global_step = 850, average loss = 0.14848782167286118
+09/30/2023 01:06:57 - INFO - __main__ -    global_step = 900, average loss = 0.12806821554375347
+09/30/2023 01:11:28 - INFO - __main__ -    global_step = 950, average loss = 0.1180885765995481
+09/30/2023 01:15:52 - INFO - __main__ -    global_step = 1000, average loss = 0.13545685631077503
+09/30/2023 01:15:53 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 01:15:53 - INFO - __main__ -     Num examples = 10000
+09/30/2023 01:15:53 - INFO - __main__ -     Batch size = 32
+09/30/2023 01:20:11 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 01:20:11 - INFO - __main__ -     acc = 0.7644
+09/30/2023 01:24:17 - INFO - __main__ -    global_step = 1050, average loss = 0.11866092401789502
+09/30/2023 01:28:20 - INFO - __main__ -    global_step = 1100, average loss = 0.12610675325471676
+09/30/2023 01:32:47 - INFO - __main__ -    global_step = 1150, average loss = 0.10549746582400985
+09/30/2023 01:37:16 - INFO - __main__ -    global_step = 1200, average loss = 0.12280375221620489
+09/30/2023 01:37:17 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 01:37:17 - INFO - __main__ -     Num examples = 10000
+09/30/2023 01:37:17 - INFO - __main__ -     Batch size = 32
+09/30/2023 01:41:35 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 01:41:35 - INFO - __main__ -     acc = 0.7802
+09/30/2023 01:42:04 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 01:46:00 - INFO - __main__ -    global_step = 1250, average loss = 0.11540970739923068
+09/30/2023 01:50:18 - INFO - __main__ -    global_step = 1300, average loss = 0.1098322441923665
+09/30/2023 01:54:50 - INFO - __main__ -    global_step = 1350, average loss = 0.12102181358681265
+09/30/2023 01:59:20 - INFO - __main__ -    global_step = 1400, average loss = 0.11920341529325014
+09/30/2023 01:59:20 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 01:59:20 - INFO - __main__ -     Num examples = 10000
+09/30/2023 01:59:20 - INFO - __main__ -     Batch size = 32
+09/30/2023 02:03:40 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 02:03:40 - INFO - __main__ -     acc = 0.7991
+09/30/2023 02:04:09 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 02:08:14 - INFO - __main__ -    global_step = 1450, average loss = 0.12416476066496215
+09/30/2023 02:12:18 - INFO - __main__ -    global_step = 1500, average loss = 0.11171700998882443
+09/30/2023 02:16:39 - INFO - __main__ -    global_step = 1550, average loss = 0.11893717237122474
+09/30/2023 02:21:18 - INFO - __main__ -    global_step = 1600, average loss = 0.11236542866332457
+09/30/2023 02:21:18 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 02:21:18 - INFO - __main__ -     Num examples = 10000
+09/30/2023 02:21:18 - INFO - __main__ -     Batch size = 32
+09/30/2023 02:25:38 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 02:25:38 - INFO - __main__ -     acc = 0.7998
+09/30/2023 02:26:08 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 02:30:17 - INFO - __main__ -    global_step = 1650, average loss = 0.11477049457775138
+09/30/2023 02:34:26 - INFO - __main__ -    global_step = 1700, average loss = 0.10185962059051235
+09/30/2023 02:38:45 - INFO - __main__ -    global_step = 1750, average loss = 0.08941184240770554
+09/30/2023 02:43:11 - INFO - __main__ -    global_step = 1800, average loss = 0.12326178842118679
+09/30/2023 02:43:11 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 02:43:11 - INFO - __main__ -     Num examples = 10000
+09/30/2023 02:43:11 - INFO - __main__ -     Batch size = 32
+09/30/2023 02:47:30 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 02:47:30 - INFO - __main__ -     acc = 0.7949
+09/30/2023 02:51:33 - INFO - __main__ -    global_step = 1850, average loss = 0.1172889139153267
+09/30/2023 02:55:34 - INFO - __main__ -    global_step = 1900, average loss = 0.11077741613984472
+09/30/2023 02:59:53 - INFO - __main__ -    global_step = 1950, average loss = 0.11476122897045571
+09/30/2023 03:04:26 - INFO - __main__ -    global_step = 2000, average loss = 0.11272342270149238
+09/30/2023 03:04:27 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 03:04:27 - INFO - __main__ -     Num examples = 10000
+09/30/2023 03:04:27 - INFO - __main__ -     Batch size = 32
+09/30/2023 03:08:46 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 03:08:46 - INFO - __main__ -     acc = 0.796
+09/30/2023 03:12:55 - INFO - __main__ -    global_step = 2050, average loss = 0.10859557473420864
+09/30/2023 03:17:10 - INFO - __main__ -    global_step = 2100, average loss = 0.09719053598862956
+09/30/2023 03:21:26 - INFO - __main__ -    global_step = 2150, average loss = 0.11492000469923369
+09/30/2023 03:25:59 - INFO - __main__ -    global_step = 2200, average loss = 0.09694181648810626
+09/30/2023 03:25:59 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 03:25:59 - INFO - __main__ -     Num examples = 10000
+09/30/2023 03:25:59 - INFO - __main__ -     Batch size = 32
+09/30/2023 03:30:18 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 03:30:18 - INFO - __main__ -     acc = 0.7974
+09/30/2023 03:34:20 - INFO - __main__ -    global_step = 2250, average loss = 0.10450371610718548
+09/30/2023 03:38:29 - INFO - __main__ -    global_step = 2300, average loss = 0.09968944377507796
+09/30/2023 03:42:35 - INFO - __main__ -    global_step = 2350, average loss = 0.09726969640512834
+09/30/2023 03:46:47 - INFO - __main__ -    global_step = 2400, average loss = 0.10790286644703884
+09/30/2023 03:46:48 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 03:46:48 - INFO - __main__ -     Num examples = 10000
+09/30/2023 03:46:48 - INFO - __main__ -     Batch size = 32
+09/30/2023 03:51:06 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 03:51:06 - INFO - __main__ -     acc = 0.8019
+09/30/2023 03:51:36 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 03:55:37 - INFO - __main__ -    global_step = 2450, average loss = 0.0904800341839109
+09/30/2023 03:59:49 - INFO - __main__ -    global_step = 2500, average loss = 0.09749648973207513
+09/30/2023 04:04:09 - INFO - __main__ -    global_step = 2550, average loss = 0.09015977876108082
+09/30/2023 04:08:36 - INFO - __main__ -    global_step = 2600, average loss = 0.11385933604056846
+09/30/2023 04:08:37 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 04:08:37 - INFO - __main__ -     Num examples = 10000
+09/30/2023 04:08:37 - INFO - __main__ -     Batch size = 32
+09/30/2023 04:12:54 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 04:12:54 - INFO - __main__ -     acc = 0.8079
+09/30/2023 04:13:24 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 04:17:30 - INFO - __main__ -    global_step = 2650, average loss = 0.09506087936344557
+09/30/2023 04:21:44 - INFO - __main__ -    global_step = 2700, average loss = 0.09819057766188052
+09/30/2023 04:25:56 - INFO - __main__ -    global_step = 2750, average loss = 0.09318019706217456
+09/30/2023 04:30:01 - INFO - __main__ -    global_step = 2800, average loss = 0.08744580631115241
+09/30/2023 04:30:02 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 04:30:02 - INFO - __main__ -     Num examples = 10000
+09/30/2023 04:30:02 - INFO - __main__ -     Batch size = 32
+09/30/2023 04:34:20 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 04:34:20 - INFO - __main__ -     acc = 0.8088
+09/30/2023 04:34:50 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 04:39:07 - INFO - __main__ -    global_step = 2850, average loss = 0.10302798340337177
+09/30/2023 04:43:20 - INFO - __main__ -    global_step = 2900, average loss = 0.09180921425198903
+09/30/2023 04:47:38 - INFO - __main__ -    global_step = 2950, average loss = 0.09286653973598731
+09/30/2023 04:52:11 - INFO - __main__ -    global_step = 3000, average loss = 0.09590554324422555
+09/30/2023 04:52:12 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 04:52:12 - INFO - __main__ -     Num examples = 10000
+09/30/2023 04:52:12 - INFO - __main__ -     Batch size = 32
+09/30/2023 04:56:30 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 04:56:30 - INFO - __main__ -     acc = 0.8082
+09/30/2023 05:00:20 - INFO - __main__ -    global_step = 3050, average loss = 0.0994117746003758
+09/30/2023 05:04:34 - INFO - __main__ -    global_step = 3100, average loss = 0.08591548198470264
+09/30/2023 05:09:00 - INFO - __main__ -    global_step = 3150, average loss = 0.09913339292746969
+09/30/2023 05:13:29 - INFO - __main__ -    global_step = 3200, average loss = 0.09553639550766092
+09/30/2023 05:13:29 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 05:13:29 - INFO - __main__ -     Num examples = 10000
+09/30/2023 05:13:29 - INFO - __main__ -     Batch size = 32
+09/30/2023 05:17:46 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 05:17:46 - INFO - __main__ -     acc = 0.8013
+09/30/2023 05:21:55 - INFO - __main__ -    global_step = 3250, average loss = 0.0932181820196638
+09/30/2023 05:25:59 - INFO - __main__ -    global_step = 3300, average loss = 0.08498929560689703
+09/30/2023 05:30:21 - INFO - __main__ -    global_step = 3350, average loss = 0.10022641647228739
+09/30/2023 05:34:47 - INFO - __main__ -    global_step = 3400, average loss = 0.08711659569285984
+09/30/2023 05:34:47 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 05:34:47 - INFO - __main__ -     Num examples = 10000
+09/30/2023 05:34:47 - INFO - __main__ -     Batch size = 32
+09/30/2023 05:39:06 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 05:39:06 - INFO - __main__ -     acc = 0.8085
+09/30/2023 05:43:04 - INFO - __main__ -    global_step = 3450, average loss = 0.08860307957234909
+09/30/2023 05:47:15 - INFO - __main__ -    global_step = 3500, average loss = 0.09122671313540195
+09/30/2023 05:51:40 - INFO - __main__ -    global_step = 3550, average loss = 0.09726192618174537
+09/30/2023 05:56:06 - INFO - __main__ -    global_step = 3600, average loss = 0.09295479882246582
+09/30/2023 05:56:07 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 05:56:07 - INFO - __main__ -     Num examples = 10000
+09/30/2023 05:56:07 - INFO - __main__ -     Batch size = 32
+09/30/2023 06:00:25 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 06:00:25 - INFO - __main__ -     acc = 0.7981
+09/30/2023 06:04:25 - INFO - __main__ -    global_step = 3650, average loss = 0.0850781474460382
+09/30/2023 06:08:29 - INFO - __main__ -    global_step = 3700, average loss = 0.08510007355012932
+09/30/2023 06:12:45 - INFO - __main__ -    global_step = 3750, average loss = 0.09091129492127947
+09/30/2023 06:17:00 - INFO - __main__ -    global_step = 3800, average loss = 0.08938177831689245
+09/30/2023 06:17:01 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 06:17:01 - INFO - __main__ -     Num examples = 10000
+09/30/2023 06:17:01 - INFO - __main__ -     Batch size = 32
+09/30/2023 06:21:19 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 06:21:19 - INFO - __main__ -     acc = 0.8008
+09/30/2023 06:25:31 - INFO - __main__ -    global_step = 3850, average loss = 0.09504610720792699
+09/30/2023 06:29:46 - INFO - __main__ -    global_step = 3900, average loss = 0.0801623915314849
+09/30/2023 06:34:06 - INFO - __main__ -    global_step = 3950, average loss = 0.08579662030970212
+09/30/2023 06:38:28 - INFO - __main__ -    global_step = 4000, average loss = 0.09399219373066443
+09/30/2023 06:38:29 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 06:38:29 - INFO - __main__ -     Num examples = 10000
+09/30/2023 06:38:29 - INFO - __main__ -     Batch size = 32
+09/30/2023 06:42:47 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 06:42:47 - INFO - __main__ -     acc = 0.8075
+09/30/2023 06:46:50 - INFO - __main__ -    global_step = 4050, average loss = 0.07777188256899535
+09/30/2023 06:51:06 - INFO - __main__ -    global_step = 4100, average loss = 0.09610467369071557
+09/30/2023 06:55:28 - INFO - __main__ -    global_step = 4150, average loss = 0.08811031442368403
+09/30/2023 07:00:00 - INFO - __main__ -    global_step = 4200, average loss = 0.08664546085885377
+09/30/2023 07:00:01 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 07:00:01 - INFO - __main__ -     Num examples = 10000
+09/30/2023 07:00:01 - INFO - __main__ -     Batch size = 32
+09/30/2023 07:04:19 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 07:04:19 - INFO - __main__ -     acc = 0.8193
+09/30/2023 07:04:50 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 07:09:00 - INFO - __main__ -    global_step = 4250, average loss = 0.0982984783052234
+09/30/2023 07:13:25 - INFO - __main__ -    global_step = 4300, average loss = 0.08057821323724056
+09/30/2023 07:17:51 - INFO - __main__ -    global_step = 4350, average loss = 0.08660443297441817
+09/30/2023 07:22:18 - INFO - __main__ -    global_step = 4400, average loss = 0.09301655420538736
+09/30/2023 07:22:19 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 07:22:19 - INFO - __main__ -     Num examples = 10000
+09/30/2023 07:22:19 - INFO - __main__ -     Batch size = 32
+09/30/2023 07:26:36 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 07:26:36 - INFO - __main__ -     acc = 0.8113
+09/30/2023 07:30:33 - INFO - __main__ -    global_step = 4450, average loss = 0.08599573986270116
+09/30/2023 07:34:39 - INFO - __main__ -    global_step = 4500, average loss = 0.08530666312639369
+09/30/2023 07:38:48 - INFO - __main__ -    global_step = 4550, average loss = 0.0846066818782856
+09/30/2023 07:43:20 - INFO - __main__ -    global_step = 4600, average loss = 0.0817996960383789
+09/30/2023 07:43:21 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 07:43:21 - INFO - __main__ -     Num examples = 10000
+09/30/2023 07:43:21 - INFO - __main__ -     Batch size = 32
+09/30/2023 07:47:39 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 07:47:39 - INFO - __main__ -     acc = 0.82
+09/30/2023 07:48:09 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 07:52:15 - INFO - __main__ -    global_step = 4650, average loss = 0.09457363621040712
+09/30/2023 07:56:34 - INFO - __main__ -    global_step = 4700, average loss = 0.09125612366977293
+09/30/2023 08:01:01 - INFO - __main__ -    global_step = 4750, average loss = 0.08600258652179037
+09/30/2023 08:05:26 - INFO - __main__ -    global_step = 4800, average loss = 0.09128527461645718
+09/30/2023 08:05:26 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 08:05:26 - INFO - __main__ -     Num examples = 10000
+09/30/2023 08:05:26 - INFO - __main__ -     Batch size = 32
+09/30/2023 08:09:45 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 08:09:45 - INFO - __main__ -     acc = 0.8151
+09/30/2023 08:13:38 - INFO - __main__ -    global_step = 4850, average loss = 0.09068508470605594
+09/30/2023 08:17:36 - INFO - __main__ -    global_step = 4900, average loss = 0.08361487613161443
+09/30/2023 08:21:45 - INFO - __main__ -    global_step = 4950, average loss = 0.09231334731652169
+09/30/2023 08:26:13 - INFO - __main__ -    global_step = 5000, average loss = 0.09210781741610845
+09/30/2023 08:26:13 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 08:26:13 - INFO - __main__ -     Num examples = 10000
+09/30/2023 08:26:13 - INFO - __main__ -     Batch size = 32
+09/30/2023 08:30:31 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 08:30:31 - INFO - __main__ -     acc = 0.8182
+09/30/2023 08:34:31 - INFO - __main__ -    global_step = 5050, average loss = 0.0987089884125453
+09/30/2023 08:38:41 - INFO - __main__ -    global_step = 5100, average loss = 0.08649987229902763
+09/30/2023 08:43:07 - INFO - __main__ -    global_step = 5150, average loss = 0.08150071838943404
+09/30/2023 08:47:36 - INFO - __main__ -    global_step = 5200, average loss = 0.09248840492458839
+09/30/2023 08:47:36 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 08:47:36 - INFO - __main__ -     Num examples = 10000
+09/30/2023 08:47:36 - INFO - __main__ -     Batch size = 32
+09/30/2023 08:51:54 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 08:51:54 - INFO - __main__ -     acc = 0.8098
+09/30/2023 08:56:07 - INFO - __main__ -    global_step = 5250, average loss = 0.08664297451652601
+09/30/2023 09:00:14 - INFO - __main__ -    global_step = 5300, average loss = 0.0810040804851451
+09/30/2023 09:04:19 - INFO - __main__ -    global_step = 5350, average loss = 0.08586231906258035
+09/30/2023 09:08:41 - INFO - __main__ -    global_step = 5400, average loss = 0.06912091931983014
+09/30/2023 09:08:41 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 09:08:41 - INFO - __main__ -     Num examples = 10000
+09/30/2023 09:08:41 - INFO - __main__ -     Batch size = 32
+09/30/2023 09:12:59 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 09:12:59 - INFO - __main__ -     acc = 0.8138
+09/30/2023 09:17:04 - INFO - __main__ -    global_step = 5450, average loss = 0.08094093154666553
+09/30/2023 09:21:20 - INFO - __main__ -    global_step = 5500, average loss = 0.08313021952490089
+09/30/2023 09:25:34 - INFO - __main__ -    global_step = 5550, average loss = 0.08020198410889862
+09/30/2023 09:30:01 - INFO - __main__ -    global_step = 5600, average loss = 0.08213623003844987
+09/30/2023 09:30:01 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 09:30:01 - INFO - __main__ -     Num examples = 10000
+09/30/2023 09:30:01 - INFO - __main__ -     Batch size = 32
+09/30/2023 09:34:19 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 09:34:19 - INFO - __main__ -     acc = 0.8138
+09/30/2023 09:38:25 - INFO - __main__ -    global_step = 5650, average loss = 0.0817357241499849
+09/30/2023 09:42:30 - INFO - __main__ -    global_step = 5700, average loss = 0.07617272696845248
+09/30/2023 09:46:47 - INFO - __main__ -    global_step = 5750, average loss = 0.08003306837461423
+09/30/2023 09:51:07 - INFO - __main__ -    global_step = 5800, average loss = 0.08461861441275687
+09/30/2023 09:51:07 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 09:51:07 - INFO - __main__ -     Num examples = 10000
+09/30/2023 09:51:07 - INFO - __main__ -     Batch size = 32
+09/30/2023 09:55:24 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 09:55:24 - INFO - __main__ -     acc = 0.819
+09/30/2023 09:59:31 - INFO - __main__ -    global_step = 5850, average loss = 0.0827079386992773
+09/30/2023 10:03:45 - INFO - __main__ -    global_step = 5900, average loss = 0.09033509934786707
+09/30/2023 10:08:04 - INFO - __main__ -    global_step = 5950, average loss = 0.08679367909935536
+09/30/2023 10:12:29 - INFO - __main__ -    global_step = 6000, average loss = 0.0677787430045646
+09/30/2023 10:12:30 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 10:12:30 - INFO - __main__ -     Num examples = 10000
+09/30/2023 10:12:30 - INFO - __main__ -     Batch size = 32
+09/30/2023 10:16:48 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 10:16:48 - INFO - __main__ -     acc = 0.793
+09/30/2023 10:20:46 - INFO - __main__ -    global_step = 6050, average loss = 0.07449474892706348
+09/30/2023 10:24:57 - INFO - __main__ -    global_step = 6100, average loss = 0.08253852118214126
+09/30/2023 10:29:21 - INFO - __main__ -    global_step = 6150, average loss = 0.07779288738580363
+09/30/2023 10:33:50 - INFO - __main__ -    global_step = 6200, average loss = 0.08415637877900735
+09/30/2023 10:33:51 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 10:33:51 - INFO - __main__ -     Num examples = 10000
+09/30/2023 10:33:51 - INFO - __main__ -     Batch size = 32
+09/30/2023 10:38:09 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 10:38:09 - INFO - __main__ -     acc = 0.8152
+09/30/2023 10:42:10 - INFO - __main__ -    global_step = 6250, average loss = 0.0836084969737567
+09/30/2023 10:46:22 - INFO - __main__ -    global_step = 6300, average loss = 0.09385589220066322
+09/30/2023 10:50:35 - INFO - __main__ -    global_step = 6350, average loss = 0.09158665712571747
+09/30/2023 10:55:02 - INFO - __main__ -    global_step = 6400, average loss = 0.0775194574438865
+09/30/2023 10:55:03 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 10:55:03 - INFO - __main__ -     Num examples = 10000
+09/30/2023 10:55:03 - INFO - __main__ -     Batch size = 32
+09/30/2023 10:59:20 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 10:59:20 - INFO - __main__ -     acc = 0.8155
+09/30/2023 11:03:28 - INFO - __main__ -    global_step = 6450, average loss = 0.08119687895305105
+09/30/2023 11:07:51 - INFO - __main__ -    global_step = 6500, average loss = 0.07420433169674652
+09/30/2023 11:12:28 - INFO - __main__ -    global_step = 6550, average loss = 0.06907126017362315
+09/30/2023 11:16:58 - INFO - __main__ -    global_step = 6600, average loss = 0.07694708627823274
+09/30/2023 11:16:58 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 11:16:58 - INFO - __main__ -     Num examples = 10000
+09/30/2023 11:16:58 - INFO - __main__ -     Batch size = 32
+09/30/2023 11:21:17 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 11:21:17 - INFO - __main__ -     acc = 0.8118
+09/30/2023 11:25:39 - INFO - __main__ -    global_step = 6650, average loss = 0.07814562884639599
+09/30/2023 11:30:08 - INFO - __main__ -    global_step = 6700, average loss = 0.08736841517616994
+09/30/2023 11:34:35 - INFO - __main__ -    global_step = 6750, average loss = 0.08082478447904577
+09/30/2023 11:39:03 - INFO - __main__ -    global_step = 6800, average loss = 0.07488631383661414
+09/30/2023 11:39:04 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 11:39:04 - INFO - __main__ -     Num examples = 10000
+09/30/2023 11:39:04 - INFO - __main__ -     Batch size = 32
+09/30/2023 11:43:23 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 11:43:23 - INFO - __main__ -     acc = 0.8213
+09/30/2023 11:43:49 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 11:47:44 - INFO - __main__ -    global_step = 6850, average loss = 0.08088931010104716
+09/30/2023 11:51:57 - INFO - __main__ -    global_step = 6900, average loss = 0.07495710194933053
+09/30/2023 11:56:20 - INFO - __main__ -    global_step = 6950, average loss = 0.08142732598964358
+09/30/2023 12:00:40 - INFO - __main__ -    global_step = 7000, average loss = 0.08055740728428645
+09/30/2023 12:00:41 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 12:00:41 - INFO - __main__ -     Num examples = 10000
+09/30/2023 12:00:41 - INFO - __main__ -     Batch size = 32
+09/30/2023 12:04:58 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 12:04:58 - INFO - __main__ -     acc = 0.8081
+09/30/2023 12:08:49 - INFO - __main__ -    global_step = 7050, average loss = 0.08094024127516604
+09/30/2023 12:13:05 - INFO - __main__ -    global_step = 7100, average loss = 0.08965814252063865
+09/30/2023 12:17:22 - INFO - __main__ -    global_step = 7150, average loss = 0.07722920090716798
+09/30/2023 12:21:45 - INFO - __main__ -    global_step = 7200, average loss = 0.08899519631431758
+09/30/2023 12:21:46 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 12:21:46 - INFO - __main__ -     Num examples = 10000
+09/30/2023 12:21:46 - INFO - __main__ -     Batch size = 32
+09/30/2023 12:26:05 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 12:26:05 - INFO - __main__ -     acc = 0.8124
+09/30/2023 12:30:21 - INFO - __main__ -    global_step = 7250, average loss = 0.06652378371007217
+09/30/2023 12:34:39 - INFO - __main__ -    global_step = 7300, average loss = 0.07190304783754982
+09/30/2023 12:39:04 - INFO - __main__ -    global_step = 7350, average loss = 0.07759228288079612
+09/30/2023 12:43:26 - INFO - __main__ -    global_step = 7400, average loss = 0.07959542326259907
+09/30/2023 12:43:27 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 12:43:27 - INFO - __main__ -     Num examples = 10000
+09/30/2023 12:43:27 - INFO - __main__ -     Batch size = 32
+09/30/2023 12:47:45 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 12:47:45 - INFO - __main__ -     acc = 0.8246
+09/30/2023 12:48:12 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 12:52:13 - INFO - __main__ -    global_step = 7450, average loss = 0.07954016777908691
+09/30/2023 12:56:27 - INFO - __main__ -    global_step = 7500, average loss = 0.06745836471483926
+09/30/2023 13:00:43 - INFO - __main__ -    global_step = 7550, average loss = 0.07651237843449053
+09/30/2023 13:04:59 - INFO - __main__ -    global_step = 7600, average loss = 0.08067735946224275
+09/30/2023 13:05:00 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 13:05:00 - INFO - __main__ -     Num examples = 10000
+09/30/2023 13:05:00 - INFO - __main__ -     Batch size = 32
+09/30/2023 13:09:19 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 13:09:19 - INFO - __main__ -     acc = 0.8296
+09/30/2023 13:09:45 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 13:13:52 - INFO - __main__ -    global_step = 7650, average loss = 0.07473264377593296
+09/30/2023 13:18:02 - INFO - __main__ -    global_step = 7700, average loss = 0.07815635729657515
+09/30/2023 13:22:14 - INFO - __main__ -    global_step = 7750, average loss = 0.08072268578209332
+09/30/2023 13:26:29 - INFO - __main__ -    global_step = 7800, average loss = 0.0779763015091885
+09/30/2023 13:26:30 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 13:26:30 - INFO - __main__ -     Num examples = 10000
+09/30/2023 13:26:30 - INFO - __main__ -     Batch size = 32
+09/30/2023 13:30:49 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 13:30:49 - INFO - __main__ -     acc = 0.8052
+09/30/2023 13:34:56 - INFO - __main__ -    global_step = 7850, average loss = 0.08846644978621043
+09/30/2023 13:39:08 - INFO - __main__ -    global_step = 7900, average loss = 0.08965322268464661
+09/30/2023 13:43:18 - INFO - __main__ -    global_step = 7950, average loss = 0.07646228883138974
+09/30/2023 13:47:34 - INFO - __main__ -    global_step = 8000, average loss = 0.06746727024801658
+09/30/2023 13:47:35 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 13:47:35 - INFO - __main__ -     Num examples = 10000
+09/30/2023 13:47:35 - INFO - __main__ -     Batch size = 32
+09/30/2023 13:51:54 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 13:51:54 - INFO - __main__ -     acc = 0.8243
+09/30/2023 13:56:06 - INFO - __main__ -    global_step = 8050, average loss = 0.08350399916278547
+09/30/2023 14:00:19 - INFO - __main__ -    global_step = 8100, average loss = 0.06798540580417466
+09/30/2023 14:04:46 - INFO - __main__ -    global_step = 8150, average loss = 0.06554304141827742
+09/30/2023 14:09:04 - INFO - __main__ -    global_step = 8200, average loss = 0.06514280185193229
+09/30/2023 14:09:05 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 14:09:05 - INFO - __main__ -     Num examples = 10000
+09/30/2023 14:09:05 - INFO - __main__ -     Batch size = 32
+09/30/2023 14:13:23 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 14:13:23 - INFO - __main__ -     acc = 0.8146
+09/30/2023 14:17:36 - INFO - __main__ -    global_step = 8250, average loss = 0.07990871949750726
+09/30/2023 14:21:47 - INFO - __main__ -    global_step = 8300, average loss = 0.07364155332470546
+09/30/2023 14:25:52 - INFO - __main__ -    global_step = 8350, average loss = 0.08377082656683342
+09/30/2023 14:30:12 - INFO - __main__ -    global_step = 8400, average loss = 0.07954915106311092
+09/30/2023 14:30:13 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 14:30:13 - INFO - __main__ -     Num examples = 10000
+09/30/2023 14:30:13 - INFO - __main__ -     Batch size = 32
+09/30/2023 14:34:32 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 14:34:32 - INFO - __main__ -     acc = 0.8148
+09/30/2023 14:38:42 - INFO - __main__ -    global_step = 8450, average loss = 0.07030039706209208
+09/30/2023 14:42:55 - INFO - __main__ -    global_step = 8500, average loss = 0.08003189989045495
+09/30/2023 14:47:10 - INFO - __main__ -    global_step = 8550, average loss = 0.07293609037540591
+09/30/2023 14:51:25 - INFO - __main__ -    global_step = 8600, average loss = 0.07146468496641319
+09/30/2023 14:51:25 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 14:51:25 - INFO - __main__ -     Num examples = 10000
+09/30/2023 14:51:25 - INFO - __main__ -     Batch size = 32
+09/30/2023 14:55:43 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 14:55:43 - INFO - __main__ -     acc = 0.8119
+09/30/2023 14:59:48 - INFO - __main__ -    global_step = 8650, average loss = 0.08003535972715327
+09/30/2023 15:03:55 - INFO - __main__ -    global_step = 8700, average loss = 0.06597046624192444
+09/30/2023 15:08:18 - INFO - __main__ -    global_step = 8750, average loss = 0.07181154116915422
+09/30/2023 15:12:39 - INFO - __main__ -    global_step = 8800, average loss = 0.068559150480869
+09/30/2023 15:12:40 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 15:12:40 - INFO - __main__ -     Num examples = 10000
+09/30/2023 15:12:40 - INFO - __main__ -     Batch size = 32
+09/30/2023 15:16:57 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 15:16:57 - INFO - __main__ -     acc = 0.8027
+09/30/2023 15:20:57 - INFO - __main__ -    global_step = 8850, average loss = 0.08192624930914462
+09/30/2023 15:25:08 - INFO - __main__ -    global_step = 8900, average loss = 0.06891920362562814
+09/30/2023 15:29:21 - INFO - __main__ -    global_step = 8950, average loss = 0.07183136703236868
+09/30/2023 15:33:32 - INFO - __main__ -    global_step = 9000, average loss = 0.07862215217377524
+09/30/2023 15:33:32 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 15:33:32 - INFO - __main__ -     Num examples = 10000
+09/30/2023 15:33:32 - INFO - __main__ -     Batch size = 32
+09/30/2023 15:37:51 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 15:37:51 - INFO - __main__ -     acc = 0.8145
+09/30/2023 15:42:00 - INFO - __main__ -    global_step = 9050, average loss = 0.08039317954942816
+09/30/2023 15:46:04 - INFO - __main__ -    global_step = 9100, average loss = 0.07681855217753991
+09/30/2023 15:50:19 - INFO - __main__ -    global_step = 9150, average loss = 0.06908466021588539
+09/30/2023 15:54:39 - INFO - __main__ -    global_step = 9200, average loss = 0.07285123934067088
+09/30/2023 15:54:40 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 15:54:40 - INFO - __main__ -     Num examples = 10000
+09/30/2023 15:54:40 - INFO - __main__ -     Batch size = 32
+09/30/2023 15:58:58 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 15:58:58 - INFO - __main__ -     acc = 0.8157
+09/30/2023 16:03:12 - INFO - __main__ -    global_step = 9250, average loss = 0.07457796319955377
+09/30/2023 16:07:29 - INFO - __main__ -    global_step = 9300, average loss = 0.08509899367534672
+09/30/2023 16:11:53 - INFO - __main__ -    global_step = 9350, average loss = 0.07013603730166323
+09/30/2023 16:16:21 - INFO - __main__ -    global_step = 9400, average loss = 0.07017059165984392
+09/30/2023 16:16:22 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 16:16:22 - INFO - __main__ -     Num examples = 10000
+09/30/2023 16:16:22 - INFO - __main__ -     Batch size = 32
+09/30/2023 16:20:40 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 16:20:40 - INFO - __main__ -     acc = 0.8141
+09/30/2023 16:24:51 - INFO - __main__ -    global_step = 9450, average loss = 0.0831688746976215
+09/30/2023 16:29:17 - INFO - __main__ -    global_step = 9500, average loss = 0.08576202854252188
+09/30/2023 16:33:37 - INFO - __main__ -    global_step = 9550, average loss = 0.08213058317254764
+09/30/2023 16:37:58 - INFO - __main__ -    global_step = 9600, average loss = 0.072965028858016
+09/30/2023 16:37:58 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 16:37:58 - INFO - __main__ -     Num examples = 10000
+09/30/2023 16:37:58 - INFO - __main__ -     Batch size = 32
+09/30/2023 16:42:15 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 16:42:15 - INFO - __main__ -     acc = 0.8122
+09/30/2023 16:46:15 - INFO - __main__ -    global_step = 9650, average loss = 0.07125714480011083
+09/30/2023 16:50:19 - INFO - __main__ -    global_step = 9700, average loss = 0.07434062254025775
+09/30/2023 16:54:30 - INFO - __main__ -    global_step = 9750, average loss = 0.07218598224179004
+09/30/2023 16:58:52 - INFO - __main__ -    global_step = 9800, average loss = 0.06753908861952368
+09/30/2023 16:58:52 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 16:58:52 - INFO - __main__ -     Num examples = 10000
+09/30/2023 16:58:52 - INFO - __main__ -     Batch size = 32
+09/30/2023 17:03:10 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 17:03:10 - INFO - __main__ -     acc = 0.8208
+09/30/2023 17:07:12 - INFO - __main__ -    global_step = 9850, average loss = 0.0787789156648796
+09/30/2023 17:11:24 - INFO - __main__ -    global_step = 9900, average loss = 0.06863431145990034
+09/30/2023 17:15:44 - INFO - __main__ -    global_step = 9950, average loss = 0.0729100130192819
+09/30/2023 17:20:01 - INFO - __main__ -    global_step = 10000, average loss = 0.07118722895695101
+09/30/2023 17:20:01 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 17:20:01 - INFO - __main__ -     Num examples = 10000
+09/30/2023 17:20:01 - INFO - __main__ -     Batch size = 32
+09/30/2023 17:24:20 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 17:24:20 - INFO - __main__ -     acc = 0.8212
+09/30/2023 17:28:25 - INFO - __main__ -    global_step = 10050, average loss = 0.06967489041242515
+09/30/2023 17:32:40 - INFO - __main__ -    global_step = 10100, average loss = 0.07503812584323896
+09/30/2023 17:37:07 - INFO - __main__ -    global_step = 10150, average loss = 0.07984486830362585
+09/30/2023 17:41:19 - INFO - __main__ -    global_step = 10200, average loss = 0.06886661994401948
+09/30/2023 17:41:19 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 17:41:19 - INFO - __main__ -     Num examples = 10000
+09/30/2023 17:41:19 - INFO - __main__ -     Batch size = 32
+09/30/2023 17:45:37 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 17:45:37 - INFO - __main__ -     acc = 0.8134
+09/30/2023 17:49:55 - INFO - __main__ -    global_step = 10250, average loss = 0.07405807184350124
+09/30/2023 17:54:14 - INFO - __main__ -    global_step = 10300, average loss = 0.08030594819738326
+09/30/2023 17:58:33 - INFO - __main__ -    global_step = 10350, average loss = 0.08568550381663954
+09/30/2023 18:02:39 - INFO - __main__ -    global_step = 10400, average loss = 0.08110691699486779
+09/30/2023 18:02:39 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 18:02:39 - INFO - __main__ -     Num examples = 10000
+09/30/2023 18:02:39 - INFO - __main__ -     Batch size = 32
+09/30/2023 18:07:00 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 18:07:00 - INFO - __main__ -     acc = 0.8226
+09/30/2023 18:10:59 - INFO - __main__ -    global_step = 10450, average loss = 0.07698049577564234
+09/30/2023 18:15:18 - INFO - __main__ -    global_step = 10500, average loss = 0.07489776252514276
+09/30/2023 18:19:38 - INFO - __main__ -    global_step = 10550, average loss = 0.08084082975808997
+09/30/2023 18:24:06 - INFO - __main__ -    global_step = 10600, average loss = 0.077233616621088
+09/30/2023 18:24:06 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 18:24:06 - INFO - __main__ -     Num examples = 10000
+09/30/2023 18:24:06 - INFO - __main__ -     Batch size = 32
+09/30/2023 18:28:26 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 18:28:26 - INFO - __main__ -     acc = 0.8219
+09/30/2023 18:32:23 - INFO - __main__ -    global_step = 10650, average loss = 0.0749396042097942
+09/30/2023 18:36:24 - INFO - __main__ -    global_step = 10700, average loss = 0.07035453407006571
+09/30/2023 18:40:30 - INFO - __main__ -    global_step = 10750, average loss = 0.0701333080389304
+09/30/2023 18:44:44 - INFO - __main__ -    global_step = 10800, average loss = 0.06815460226869618
+09/30/2023 18:44:45 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 18:44:45 - INFO - __main__ -     Num examples = 10000
+09/30/2023 18:44:45 - INFO - __main__ -     Batch size = 32
+09/30/2023 18:49:04 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 18:49:04 - INFO - __main__ -     acc = 0.8246
+09/30/2023 18:53:04 - INFO - __main__ -    global_step = 10850, average loss = 0.06231740675430046
+09/30/2023 18:57:11 - INFO - __main__ -    global_step = 10900, average loss = 0.07749273380759406
+09/30/2023 19:01:27 - INFO - __main__ -    global_step = 10950, average loss = 0.07356921623417292
+09/30/2023 19:05:44 - INFO - __main__ -    global_step = 11000, average loss = 0.06861940244401922
+09/30/2023 19:05:44 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 19:05:44 - INFO - __main__ -     Num examples = 10000
+09/30/2023 19:05:44 - INFO - __main__ -     Batch size = 32
+09/30/2023 19:10:04 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 19:10:04 - INFO - __main__ -     acc = 0.8237
+09/30/2023 19:13:58 - INFO - __main__ -    global_step = 11050, average loss = 0.07190075869159046
+09/30/2023 19:18:18 - INFO - __main__ -    global_step = 11100, average loss = 0.07798185770014243
+09/30/2023 19:22:25 - INFO - __main__ -    global_step = 11150, average loss = 0.05461175944059505
+09/30/2023 19:26:36 - INFO - __main__ -    global_step = 11200, average loss = 0.07214928590841736
+09/30/2023 19:26:36 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 19:26:36 - INFO - __main__ -     Num examples = 10000
+09/30/2023 19:26:36 - INFO - __main__ -     Batch size = 32
+09/30/2023 19:30:56 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 19:30:56 - INFO - __main__ -     acc = 0.8281
+09/30/2023 19:34:46 - INFO - __main__ -    global_step = 11250, average loss = 0.07595877689196641
+09/30/2023 19:38:51 - INFO - __main__ -    global_step = 11300, average loss = 0.06289271867310163
+09/30/2023 19:42:58 - INFO - __main__ -    global_step = 11350, average loss = 0.07287138866693567
+09/30/2023 19:47:05 - INFO - __main__ -    global_step = 11400, average loss = 0.0736375573805708
+09/30/2023 19:47:05 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 19:47:05 - INFO - __main__ -     Num examples = 10000
+09/30/2023 19:47:05 - INFO - __main__ -     Batch size = 32
+09/30/2023 19:51:26 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 19:51:26 - INFO - __main__ -     acc = 0.8265
+09/30/2023 19:55:14 - INFO - __main__ -    global_step = 11450, average loss = 0.07105860608404328
+09/30/2023 19:59:22 - INFO - __main__ -    global_step = 11500, average loss = 0.07589100849851092
+09/30/2023 20:03:31 - INFO - __main__ -    global_step = 11550, average loss = 0.07193597211022279
+09/30/2023 20:07:44 - INFO - __main__ -    global_step = 11600, average loss = 0.0786158631305443
+09/30/2023 20:07:45 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 20:07:45 - INFO - __main__ -     Num examples = 10000
+09/30/2023 20:07:45 - INFO - __main__ -     Batch size = 32
+09/30/2023 20:12:05 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 20:12:05 - INFO - __main__ -     acc = 0.8224
+09/30/2023 20:16:14 - INFO - __main__ -    global_step = 11650, average loss = 0.07484395604304155
+09/30/2023 20:20:16 - INFO - __main__ -    global_step = 11700, average loss = 0.07182746810896788
+09/30/2023 20:24:28 - INFO - __main__ -    global_step = 11750, average loss = 0.06392118992527684
+09/30/2023 20:28:47 - INFO - __main__ -    global_step = 11800, average loss = 0.06359485059540021
+09/30/2023 20:28:48 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 20:28:48 - INFO - __main__ -     Num examples = 10000
+09/30/2023 20:28:48 - INFO - __main__ -     Batch size = 32
+09/30/2023 20:33:07 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 20:33:07 - INFO - __main__ -     acc = 0.8225
+09/30/2023 20:36:55 - INFO - __main__ -    global_step = 11850, average loss = 0.06557874951142367
+09/30/2023 20:40:51 - INFO - __main__ -    global_step = 11900, average loss = 0.06787695961887948
+09/30/2023 20:45:01 - INFO - __main__ -    global_step = 11950, average loss = 0.07802391385892406
+09/30/2023 20:49:19 - INFO - __main__ -    global_step = 12000, average loss = 0.062383338503277624
+09/30/2023 20:49:19 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 20:49:19 - INFO - __main__ -     Num examples = 10000
+09/30/2023 20:49:19 - INFO - __main__ -     Batch size = 32
+09/30/2023 20:53:41 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 20:53:41 - INFO - __main__ -     acc = 0.8221
+09/30/2023 20:57:31 - INFO - __main__ -    global_step = 12050, average loss = 0.07041985652205768
+09/30/2023 21:01:33 - INFO - __main__ -    global_step = 12100, average loss = 0.07904728068271652
+09/30/2023 21:05:47 - INFO - __main__ -    global_step = 12150, average loss = 0.07474817682654247
+09/30/2023 21:09:58 - INFO - __main__ -    global_step = 12200, average loss = 0.07402907914118259
+09/30/2023 21:09:58 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 21:09:58 - INFO - __main__ -     Num examples = 10000
+09/30/2023 21:09:58 - INFO - __main__ -     Batch size = 32
+09/30/2023 21:14:19 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 21:14:19 - INFO - __main__ -     acc = 0.8327
+09/30/2023 21:14:46 - INFO - __main__ -   Saving model checkpoint to output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+09/30/2023 21:18:46 - INFO - __main__ -    global_step = 12250, average loss = 0.07039213450989337
+09/30/2023 21:22:59 - INFO - __main__ -    global_step = 12300, average loss = 0.0842395970186044
+09/30/2023 21:27:05 - INFO - __main__ -    global_step = 12350, average loss = 0.06603515204827999
+09/30/2023 21:31:22 - INFO - __main__ -    global_step = 12400, average loss = 0.06760421821546515
+09/30/2023 21:31:22 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 21:31:22 - INFO - __main__ -     Num examples = 10000
+09/30/2023 21:31:22 - INFO - __main__ -     Batch size = 32
+09/30/2023 21:35:43 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 21:35:43 - INFO - __main__ -     acc = 0.8208
+09/30/2023 21:39:33 - INFO - __main__ -    global_step = 12450, average loss = 0.06917047601906233
+09/30/2023 21:43:44 - INFO - __main__ -    global_step = 12500, average loss = 0.07573592953915068
+09/30/2023 21:48:03 - INFO - __main__ -    global_step = 12550, average loss = 0.06653125052485848
+09/30/2023 21:52:22 - INFO - __main__ -    global_step = 12600, average loss = 0.06815064429247286
+09/30/2023 21:52:23 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 21:52:23 - INFO - __main__ -     Num examples = 10000
+09/30/2023 21:52:23 - INFO - __main__ -     Batch size = 32
+09/30/2023 21:56:43 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 21:56:43 - INFO - __main__ -     acc = 0.819
+09/30/2023 22:00:39 - INFO - __main__ -    global_step = 12650, average loss = 0.07732899946378893
+09/30/2023 22:04:44 - INFO - __main__ -    global_step = 12700, average loss = 0.06547158910783764
+09/30/2023 22:08:49 - INFO - __main__ -    global_step = 12750, average loss = 0.0728905378174386
+09/30/2023 22:13:03 - INFO - __main__ -    global_step = 12800, average loss = 0.07366545890477937
+09/30/2023 22:13:04 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 22:13:04 - INFO - __main__ -     Num examples = 10000
+09/30/2023 22:13:04 - INFO - __main__ -     Batch size = 32
+09/30/2023 22:17:25 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 22:17:25 - INFO - __main__ -     acc = 0.8182
+09/30/2023 22:21:28 - INFO - __main__ -    global_step = 12850, average loss = 0.06894337675126735
+09/30/2023 22:25:41 - INFO - __main__ -    global_step = 12900, average loss = 0.07351460054007475
+09/30/2023 22:29:49 - INFO - __main__ -    global_step = 12950, average loss = 0.0674650944762834
+09/30/2023 22:34:09 - INFO - __main__ -    global_step = 13000, average loss = 0.07850258736492834
+09/30/2023 22:34:09 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 22:34:09 - INFO - __main__ -     Num examples = 10000
+09/30/2023 22:34:09 - INFO - __main__ -     Batch size = 32
+09/30/2023 22:38:30 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 22:38:30 - INFO - __main__ -     acc = 0.8321
+09/30/2023 22:42:24 - INFO - __main__ -    global_step = 13050, average loss = 0.07653208828101925
+09/30/2023 22:46:20 - INFO - __main__ -    global_step = 13100, average loss = 0.06802368102005857
+09/30/2023 22:50:29 - INFO - __main__ -    global_step = 13150, average loss = 0.06454230795552576
+09/30/2023 22:54:34 - INFO - __main__ -    global_step = 13200, average loss = 0.07258539929578546
+09/30/2023 22:54:35 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 22:54:35 - INFO - __main__ -     Num examples = 10000
+09/30/2023 22:54:35 - INFO - __main__ -     Batch size = 32
+09/30/2023 22:58:54 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 22:58:54 - INFO - __main__ -     acc = 0.8252
+09/30/2023 23:02:57 - INFO - __main__ -    global_step = 13250, average loss = 0.07325911161562544
+09/30/2023 23:07:12 - INFO - __main__ -    global_step = 13300, average loss = 0.06880584957727479
+09/30/2023 23:11:21 - INFO - __main__ -    global_step = 13350, average loss = 0.07009069720297703
+09/30/2023 23:15:34 - INFO - __main__ -    global_step = 13400, average loss = 0.07083460625182852
+09/30/2023 23:15:35 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 23:15:35 - INFO - __main__ -     Num examples = 10000
+09/30/2023 23:15:35 - INFO - __main__ -     Batch size = 32
+09/30/2023 23:19:56 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 23:19:56 - INFO - __main__ -     acc = 0.813
+09/30/2023 23:23:55 - INFO - __main__ -    global_step = 13450, average loss = 0.06977577161625959
+09/30/2023 23:27:49 - INFO - __main__ -    global_step = 13500, average loss = 0.0730690676838276
+09/30/2023 23:31:51 - INFO - __main__ -    global_step = 13550, average loss = 0.07233811266596604
+09/30/2023 23:35:53 - INFO - __main__ -    global_step = 13600, average loss = 0.0773136636797426
+09/30/2023 23:35:54 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 23:35:54 - INFO - __main__ -     Num examples = 10000
+09/30/2023 23:35:54 - INFO - __main__ -     Batch size = 32
+09/30/2023 23:40:14 - INFO - __main__ -   ***** Eval results *****
+09/30/2023 23:40:14 - INFO - __main__ -     acc = 0.8254
+09/30/2023 23:44:18 - INFO - __main__ -    global_step = 13650, average loss = 0.0625762648001546
+09/30/2023 23:48:29 - INFO - __main__ -    global_step = 13700, average loss = 0.07835062241327251
+09/30/2023 23:52:47 - INFO - __main__ -    global_step = 13750, average loss = 0.06917831582177314
+09/30/2023 23:57:06 - INFO - __main__ -    global_step = 13800, average loss = 0.06653823942549934
+09/30/2023 23:57:07 - INFO - __main__ -   ***** Running evaluation *****
+09/30/2023 23:57:07 - INFO - __main__ -     Num examples = 10000
+09/30/2023 23:57:07 - INFO - __main__ -     Batch size = 32
+10/01/2023 00:01:27 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 00:01:27 - INFO - __main__ -     acc = 0.8231
+10/01/2023 00:05:24 - INFO - __main__ -    global_step = 13850, average loss = 0.07134979092643334
+10/01/2023 00:09:31 - INFO - __main__ -    global_step = 13900, average loss = 0.07882154490274842
+10/01/2023 00:13:33 - INFO - __main__ -    global_step = 13950, average loss = 0.067044138008132
+10/01/2023 00:17:54 - INFO - __main__ -    global_step = 14000, average loss = 0.06602240080737828
+10/01/2023 00:17:55 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 00:17:55 - INFO - __main__ -     Num examples = 10000
+10/01/2023 00:17:55 - INFO - __main__ -     Batch size = 32
+10/01/2023 00:22:16 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 00:22:16 - INFO - __main__ -     acc = 0.8185
+10/01/2023 00:26:20 - INFO - __main__ -    global_step = 14050, average loss = 0.07546966458212409
+10/01/2023 00:30:49 - INFO - __main__ -    global_step = 14100, average loss = 0.06855787578620948
+10/01/2023 00:35:15 - INFO - __main__ -    global_step = 14150, average loss = 0.06737258993505747
+10/01/2023 00:39:39 - INFO - __main__ -    global_step = 14200, average loss = 0.05966844407041208
+10/01/2023 00:39:40 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 00:39:40 - INFO - __main__ -     Num examples = 10000
+10/01/2023 00:39:40 - INFO - __main__ -     Batch size = 32
+10/01/2023 00:44:00 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 00:44:00 - INFO - __main__ -     acc = 0.8282
+10/01/2023 00:47:56 - INFO - __main__ -    global_step = 14250, average loss = 0.0709371871012263
+10/01/2023 00:51:54 - INFO - __main__ -    global_step = 14300, average loss = 0.07779215545522675
+10/01/2023 00:56:02 - INFO - __main__ -    global_step = 14350, average loss = 0.06590510867084959
+10/01/2023 01:00:08 - INFO - __main__ -    global_step = 14400, average loss = 0.061885312875092496
+10/01/2023 01:00:09 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 01:00:09 - INFO - __main__ -     Num examples = 10000
+10/01/2023 01:00:09 - INFO - __main__ -     Batch size = 32
+10/01/2023 01:04:29 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 01:04:29 - INFO - __main__ -     acc = 0.8195
+10/01/2023 01:08:20 - INFO - __main__ -    global_step = 14450, average loss = 0.07757491528376705
+10/01/2023 01:12:26 - INFO - __main__ -    global_step = 14500, average loss = 0.061351443203457166
+10/01/2023 01:16:44 - INFO - __main__ -    global_step = 14550, average loss = 0.06742463728594884
+10/01/2023 01:20:55 - INFO - __main__ -    global_step = 14600, average loss = 0.06395716872473713
+10/01/2023 01:20:56 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 01:20:56 - INFO - __main__ -     Num examples = 10000
+10/01/2023 01:20:56 - INFO - __main__ -     Batch size = 32
+10/01/2023 01:25:16 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 01:25:16 - INFO - __main__ -     acc = 0.8271
+10/01/2023 01:29:11 - INFO - __main__ -    global_step = 14650, average loss = 0.0680865884249215
+10/01/2023 01:33:17 - INFO - __main__ -    global_step = 14700, average loss = 0.07319515083199804
+10/01/2023 01:37:31 - INFO - __main__ -    global_step = 14750, average loss = 0.0750861974158397
+10/01/2023 01:41:39 - INFO - __main__ -    global_step = 14800, average loss = 0.07455838610287174
+10/01/2023 01:41:39 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 01:41:39 - INFO - __main__ -     Num examples = 10000
+10/01/2023 01:41:39 - INFO - __main__ -     Batch size = 32
+10/01/2023 01:45:59 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 01:45:59 - INFO - __main__ -     acc = 0.8285
+10/01/2023 01:49:49 - INFO - __main__ -    global_step = 14850, average loss = 0.0746920863639025
+10/01/2023 01:53:48 - INFO - __main__ -    global_step = 14900, average loss = 0.06193213762038795
+10/01/2023 01:58:00 - INFO - __main__ -    global_step = 14950, average loss = 0.0684903811987897
+10/01/2023 02:02:20 - INFO - __main__ -    global_step = 15000, average loss = 0.07475626632280181
+10/01/2023 02:02:21 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 02:02:21 - INFO - __main__ -     Num examples = 10000
+10/01/2023 02:02:21 - INFO - __main__ -     Batch size = 32
+10/01/2023 02:06:40 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 02:06:40 - INFO - __main__ -     acc = 0.8221
+10/01/2023 02:10:33 - INFO - __main__ -    global_step = 15050, average loss = 0.06398421550955391
+10/01/2023 02:14:31 - INFO - __main__ -    global_step = 15100, average loss = 0.07387388837814797
+10/01/2023 02:18:36 - INFO - __main__ -    global_step = 15150, average loss = 0.07215547483820046
+10/01/2023 02:22:42 - INFO - __main__ -    global_step = 15200, average loss = 0.06692371807614109
+10/01/2023 02:22:42 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 02:22:42 - INFO - __main__ -     Num examples = 10000
+10/01/2023 02:22:42 - INFO - __main__ -     Batch size = 32
+10/01/2023 02:27:06 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 02:27:06 - INFO - __main__ -     acc = 0.828
+10/01/2023 02:31:03 - INFO - __main__ -    global_step = 15250, average loss = 0.07475481618889716
+10/01/2023 02:35:03 - INFO - __main__ -    global_step = 15300, average loss = 0.06605282124131918
+10/01/2023 02:39:06 - INFO - __main__ -    global_step = 15350, average loss = 0.0742860847054817
+10/01/2023 02:43:08 - INFO - __main__ -    global_step = 15400, average loss = 0.06508645007126689
+10/01/2023 02:43:09 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 02:43:09 - INFO - __main__ -     Num examples = 10000
+10/01/2023 02:43:09 - INFO - __main__ -     Batch size = 32
+10/01/2023 02:47:27 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 02:47:27 - INFO - __main__ -     acc = 0.8244
+10/01/2023 02:51:15 - INFO - __main__ -    global_step = 15450, average loss = 0.0657403554152188
+10/01/2023 02:55:25 - INFO - __main__ -    global_step = 15500, average loss = 0.06363382869447377
+10/01/2023 02:59:33 - INFO - __main__ -    global_step = 15550, average loss = 0.068332606570184
+10/01/2023 03:03:36 - INFO - __main__ -    global_step = 15600, average loss = 0.0638002801532275
+10/01/2023 03:03:37 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 03:03:37 - INFO - __main__ -     Num examples = 10000
+10/01/2023 03:03:37 - INFO - __main__ -     Batch size = 32
+10/01/2023 03:07:54 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 03:07:54 - INFO - __main__ -     acc = 0.8245
+10/01/2023 03:11:47 - INFO - __main__ -    global_step = 15650, average loss = 0.07057813088395051
+10/01/2023 03:15:48 - INFO - __main__ -    global_step = 15700, average loss = 0.059586076617561046
+10/01/2023 03:19:50 - INFO - __main__ -    global_step = 15750, average loss = 0.06329842852351249
+10/01/2023 03:24:07 - INFO - __main__ -    global_step = 15800, average loss = 0.0673095579940309
+10/01/2023 03:24:08 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 03:24:08 - INFO - __main__ -     Num examples = 10000
+10/01/2023 03:24:08 - INFO - __main__ -     Batch size = 32
+10/01/2023 03:28:27 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 03:28:27 - INFO - __main__ -     acc = 0.8191
+10/01/2023 03:32:25 - INFO - __main__ -    global_step = 15850, average loss = 0.06719043602446619
+10/01/2023 03:36:22 - INFO - __main__ -    global_step = 15900, average loss = 0.06470626855618321
+10/01/2023 03:40:22 - INFO - __main__ -    global_step = 15950, average loss = 0.0673678615699464
+10/01/2023 03:44:32 - INFO - __main__ -    global_step = 16000, average loss = 0.06654785299411742
+10/01/2023 03:44:32 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 03:44:32 - INFO - __main__ -     Num examples = 10000
+10/01/2023 03:44:32 - INFO - __main__ -     Batch size = 32
+10/01/2023 03:48:51 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 03:48:51 - INFO - __main__ -     acc = 0.826
+10/01/2023 03:52:42 - INFO - __main__ -    global_step = 16050, average loss = 0.07211193255971012
+10/01/2023 03:56:30 - INFO - __main__ -    global_step = 16100, average loss = 0.07810956820030697
+10/01/2023 04:00:37 - INFO - __main__ -    global_step = 16150, average loss = 0.07871339554849328
+10/01/2023 04:04:48 - INFO - __main__ -    global_step = 16200, average loss = 0.06766451962915199
+10/01/2023 04:04:48 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 04:04:48 - INFO - __main__ -     Num examples = 10000
+10/01/2023 04:04:48 - INFO - __main__ -     Batch size = 32
+10/01/2023 04:09:07 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 04:09:07 - INFO - __main__ -     acc = 0.8234
+10/01/2023 04:13:00 - INFO - __main__ -    global_step = 16250, average loss = 0.07233332002186216
+10/01/2023 04:17:08 - INFO - __main__ -    global_step = 16300, average loss = 0.06269402921956498
+10/01/2023 04:21:18 - INFO - __main__ -    global_step = 16350, average loss = 0.066974333815524
+10/01/2023 04:25:36 - INFO - __main__ -    global_step = 16400, average loss = 0.06326851320967762
+10/01/2023 04:25:36 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 04:25:36 - INFO - __main__ -     Num examples = 10000
+10/01/2023 04:25:36 - INFO - __main__ -     Batch size = 32
+10/01/2023 04:29:55 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 04:29:55 - INFO - __main__ -     acc = 0.8218
+10/01/2023 04:33:53 - INFO - __main__ -    global_step = 16450, average loss = 0.0583337911261151
+10/01/2023 04:38:00 - INFO - __main__ -    global_step = 16500, average loss = 0.06651346774706327
+10/01/2023 04:42:10 - INFO - __main__ -    global_step = 16550, average loss = 0.07442569829370768
+10/01/2023 04:46:19 - INFO - __main__ -    global_step = 16600, average loss = 0.0704036247156182
+10/01/2023 04:46:19 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 04:46:19 - INFO - __main__ -     Num examples = 10000
+10/01/2023 04:46:19 - INFO - __main__ -     Batch size = 32
+10/01/2023 04:50:38 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 04:50:38 - INFO - __main__ -     acc = 0.8268
+10/01/2023 04:54:40 - INFO - __main__ -    global_step = 16650, average loss = 0.07102784802380484
+10/01/2023 04:58:39 - INFO - __main__ -    global_step = 16700, average loss = 0.07482151540141785
+10/01/2023 05:02:48 - INFO - __main__ -    global_step = 16750, average loss = 0.06266404812475229
+10/01/2023 05:06:49 - INFO - __main__ -    global_step = 16800, average loss = 0.06936132206232287
+10/01/2023 05:06:50 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 05:06:50 - INFO - __main__ -     Num examples = 10000
+10/01/2023 05:06:50 - INFO - __main__ -     Batch size = 32
+10/01/2023 05:11:07 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 05:11:07 - INFO - __main__ -     acc = 0.8313
+10/01/2023 05:15:16 - INFO - __main__ -    global_step = 16850, average loss = 0.060352628196997105
+10/01/2023 05:19:33 - INFO - __main__ -    global_step = 16900, average loss = 0.0641949670168833
+10/01/2023 05:23:53 - INFO - __main__ -    global_step = 16950, average loss = 0.0711748162342701
+10/01/2023 05:28:04 - INFO - __main__ -    global_step = 17000, average loss = 0.07767359625780955
+10/01/2023 05:28:05 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 05:28:05 - INFO - __main__ -     Num examples = 10000
+10/01/2023 05:28:05 - INFO - __main__ -     Batch size = 32
+10/01/2023 05:32:22 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 05:32:22 - INFO - __main__ -     acc = 0.8302
+10/01/2023 05:36:19 - INFO - __main__ -    global_step = 17050, average loss = 0.059951672412971675
+10/01/2023 05:40:23 - INFO - __main__ -    global_step = 17100, average loss = 0.0679468241819086
+10/01/2023 05:44:37 - INFO - __main__ -    global_step = 17150, average loss = 0.06287542213140114
+10/01/2023 05:48:53 - INFO - __main__ -    global_step = 17200, average loss = 0.07064101672236575
+10/01/2023 05:48:53 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 05:48:53 - INFO - __main__ -     Num examples = 10000
+10/01/2023 05:48:53 - INFO - __main__ -     Batch size = 32
+10/01/2023 05:53:11 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 05:53:11 - INFO - __main__ -     acc = 0.8288
+10/01/2023 05:57:08 - INFO - __main__ -    global_step = 17250, average loss = 0.06821862254073494
+10/01/2023 06:01:07 - INFO - __main__ -    global_step = 17300, average loss = 0.06737288911346695
+10/01/2023 06:05:09 - INFO - __main__ -    global_step = 17350, average loss = 0.06524526451248676
+10/01/2023 06:09:17 - INFO - __main__ -    global_step = 17400, average loss = 0.06838752188666604
+10/01/2023 06:09:17 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 06:09:17 - INFO - __main__ -     Num examples = 10000
+10/01/2023 06:09:17 - INFO - __main__ -     Batch size = 32
+10/01/2023 06:13:34 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 06:13:34 - INFO - __main__ -     acc = 0.8292
+10/01/2023 06:17:34 - INFO - __main__ -    global_step = 17450, average loss = 0.07033179465208378
+10/01/2023 06:21:42 - INFO - __main__ -    global_step = 17500, average loss = 0.07338941472058651
+10/01/2023 06:25:54 - INFO - __main__ -    global_step = 17550, average loss = 0.06760536882744418
+10/01/2023 06:30:29 - INFO - __main__ -    global_step = 17600, average loss = 0.06395369231896893
+10/01/2023 06:30:30 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 06:30:30 - INFO - __main__ -     Num examples = 10000
+10/01/2023 06:30:30 - INFO - __main__ -     Batch size = 32
+10/01/2023 06:34:46 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 06:34:46 - INFO - __main__ -     acc = 0.8226
+10/01/2023 06:38:42 - INFO - __main__ -    global_step = 17650, average loss = 0.0788995540245378
+10/01/2023 06:42:45 - INFO - __main__ -    global_step = 17700, average loss = 0.058938835552726235
+10/01/2023 06:46:55 - INFO - __main__ -    global_step = 17750, average loss = 0.062029462043719834
+10/01/2023 06:51:15 - INFO - __main__ -    global_step = 17800, average loss = 0.07220558329383493
+10/01/2023 06:51:15 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 06:51:15 - INFO - __main__ -     Num examples = 10000
+10/01/2023 06:51:15 - INFO - __main__ -     Batch size = 32
+10/01/2023 06:55:33 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 06:55:33 - INFO - __main__ -     acc = 0.823
+10/01/2023 06:59:32 - INFO - __main__ -    global_step = 17850, average loss = 0.07046543042039048
+10/01/2023 07:03:39 - INFO - __main__ -    global_step = 17900, average loss = 0.0620857437804807
+10/01/2023 07:07:50 - INFO - __main__ -    global_step = 17950, average loss = 0.05406381562563183
+10/01/2023 07:12:05 - INFO - __main__ -    global_step = 18000, average loss = 0.05979254503792617
+10/01/2023 07:12:05 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 07:12:05 - INFO - __main__ -     Num examples = 10000
+10/01/2023 07:12:05 - INFO - __main__ -     Batch size = 32
+10/01/2023 07:16:22 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 07:16:22 - INFO - __main__ -     acc = 0.8237
+10/01/2023 07:20:13 - INFO - __main__ -    global_step = 18050, average loss = 0.06541542315782863
+10/01/2023 07:24:31 - INFO - __main__ -    global_step = 18100, average loss = 0.06534778851972078
+10/01/2023 07:28:50 - INFO - __main__ -    global_step = 18150, average loss = 0.06520377914806887
+10/01/2023 07:33:09 - INFO - __main__ -    global_step = 18200, average loss = 0.05995443502964917
+10/01/2023 07:33:10 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 07:33:10 - INFO - __main__ -     Num examples = 10000
+10/01/2023 07:33:10 - INFO - __main__ -     Batch size = 32
+10/01/2023 07:37:27 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 07:37:27 - INFO - __main__ -     acc = 0.825
+10/01/2023 07:41:29 - INFO - __main__ -    global_step = 18250, average loss = 0.0729160438424151
+10/01/2023 07:45:44 - INFO - __main__ -    global_step = 18300, average loss = 0.06983143856698007
+10/01/2023 07:48:53 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 07:48:53 - INFO - __main__ -     Num examples = 10000
+10/01/2023 07:48:53 - INFO - __main__ -     Batch size = 32
+10/01/2023 07:53:22 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 07:53:22 - INFO - __main__ -     acc = 0.8249
+10/01/2023 07:53:22 - INFO - __main__ -    global_step = 18336, average loss = 0.09140925639286196
+10/01/2023 07:53:56 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 07:53:56 - INFO - __main__ -     Num examples = 10000
+10/01/2023 07:53:56 - INFO - __main__ -     Batch size = 32
+10/01/2023 07:58:24 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 07:58:24 - INFO - __main__ -     acc = 0.8326
+10/01/2023 07:58:30 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/socialiqa_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', out_dir='./eval_results/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', device=0, reader='socialiqa', overwrite_output_dir=False, cache_dir=None)
+10/01/2023 07:58:30 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+10/01/2023 08:06:13 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/winogrande_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', out_dir='./eval_results/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', device=0, reader='winogrande', overwrite_output_dir=False, cache_dir=None)
+10/01/2023 08:06:13 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+10/01/2023 08:08:40 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/piqa_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', out_dir='./eval_results/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', device=0, reader='piqa', overwrite_output_dir=False, cache_dir=None)
+10/01/2023 08:08:40 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+10/01/2023 08:17:19 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/commonsenseqa_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', out_dir='./eval_results/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', device=0, reader='commonsenseqa', overwrite_output_dir=False, cache_dir=None)
+10/01/2023 08:17:19 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+10/01/2023 08:23:12 - INFO - evaluate_DeBERTa -   Namespace(dataset_file='../../../data/mcqa/eval/anli_dev.jsonl', lm='output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', out_dir='./eval_results/deberta-v3-large_car_2i_name_100k_seed_101_5e-6', device=0, reader='anli', overwrite_output_dir=False, cache_dir=None)
+10/01/2023 08:23:12 - INFO - evaluate_DeBERTa -   Initializing output/Output_ATOMIC-pseudo-wWC/car_2i/deberta-v3-large_car_2i_name_100k_seed_101_5e-6
+10/01/2023 08:28:58 - INFO - __main__ -   ***** Running evaluation *****
+10/01/2023 08:28:58 - INFO - __main__ -     Num examples = 120
+10/01/2023 08:28:58 - INFO - __main__ -     Batch size = 32
+10/01/2023 08:29:16 - INFO - __main__ -   ***** Eval results *****
+10/01/2023 08:29:16 - INFO - __main__ -     acc = 0.475

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4e6646c04ef1788eae8dab3445de6faa31eccb52a2f6ef22a6957c7e802b46c
+size 1915