Made improvements and changed base mnli model

app.py

@@ -1,12 +1,11 @@
 import gradio as gr
-from transformers import BartForSequenceClassification, BartTokenizer
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from transformers import T5Tokenizer, T5ForConditionalGeneration
 import torch
 
 device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
-
-te_tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-mnli')
-te_model = BartForSequenceClassification.from_pretrained('facebook/bart-large-mnli', device_map="auto")
+te_tokenizer = AutoTokenizer.from_pretrained('MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli')
+te_model = AutoModelForSequenceClassification.from_pretrained('MoritzLaurer/DeBERTa-v3-base-mnli-fever-anli').to(device)
 qa_tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-large")
 qa_model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-large", device_map="auto")
 
@@ -17,65 +16,51 @@ def predict(context, intent, multi_class):
     input_text = "What object is the following describing: " + context
     input_ids = qa_tokenizer(input_text, return_tensors="pt").input_ids.to(device)
     object_output = qa_tokenizer.decode(qa_model.generate(input_ids, max_length=2)[0], skip_special_tokens=True)
-    batch = ['The ' + object_output + ' is ' + intent, 'The ' + object_output + ' is ' + opposite_output, 'The ' + object_output + ' is not ' + intent, 'The ' + object_output + ' is not ' + opposite_output]
-
+    batch = ['The ' + object_output + ' is ' + intent, 'The ' + object_output + ' is ' + opposite_output, 'The ' + object_output + ' is neither ' + intent + ' nor ' + opposite_output]
     outputs = []
+    normal = 0
+    
     for i, hypothesis in enumerate(batch):
         input_ids = te_tokenizer.encode(context, hypothesis, return_tensors='pt').to(device)
         
         # -> [contradiction, neutral, entailment]
         logits = te_model(input_ids)[0][0]
-
+        if (i == 0):
+            normal = logits
         if (i >= 2):
             # -> [contradiction, entailment]
             probs = logits[[0,2]].softmax(dim=0)
         else:
-            probs = logits.softmax(dim=0)
+            probs = torch.exp(logits)
         outputs.append(probs)
-        
     # calculate the stochastic vector for it being neither the positive or negative class
-    perfect_prob = [0, 0]
-    perfect_prob[1] = max(float(outputs[2][0]), float(outputs[3][0]))
-    perfect_prob[0] = 1-perfect_prob[1]
+    perfect_prob = outputs[2]
     # -> [entailment, contradiction] for perfect
     
     # -> [entailment, neutral, contradiction] for positive
-    outputs[0] = outputs[0].flip(dims=[0])
+    outputs[1] = outputs[1].flip(dims=[0])
     
     # combine the negative and positive class by summing by the opposite of the negative class
     aggregated = (outputs[0] + outputs[1])/2
     
     # multiplying vectors
     aggregated[1] = aggregated[1] * perfect_prob[0]
+    aggregated[0] = aggregated[0] * perfect_prob[1]
+    aggregated[2] = aggregated[2] * perfect_prob[1]
     
-    # if it is neither the positive or negative class, then it is more likely the neutral class, so adjust accordingly
-    if (perfect_prob[0] > perfect_prob[1]):
-        aggregated[2] = aggregated[2] * perfect_prob[1]
-        aggregated[0] = aggregated[0] * perfect_prob[1]
-    else:
-        # if it is more likely the positive class, increase its probability by a scale of the probability of it not being perfect
-        if (aggregated[0] > aggregated[2]):
-            aggregated[2] = aggregated[2] * perfect_prob[0]
-            aggregated[0] = aggregated[0] * perfect_prob[1]
-        # if it is more likely the negative class, increase its probability by a scale of the probability of it not being perfect
-        else:
-            aggregated[2] = aggregated[2] * perfect_prob[1]
-            aggregated[0] = aggregated[0] * perfect_prob[0]
-    
-    # to exagerate differences
-    # this way 0 maps to 0
-    aggregated = aggregated.exp()-1
+    aggregated = torch.sqrt(aggregated)
 
     # multiple true classes
     if (multi_class):
         aggregated = torch.sigmoid(aggregated)
+        normal = torch.sigmoid(normal)
     # only one true class
     else:
         aggregated = aggregated.softmax(dim=0)
+        normal = normal.softmax(dim=0)
     aggregated = aggregated.tolist()
-    return {"agree": aggregated[0], "neutral": aggregated[1], "disagree": aggregated[2]}, {"agree": outputs[0][0], "neutral": outputs[0][1], "disagree": outputs[0][2]}
-
-examples = [["The pants fit great, even the waist will fit me fine once I'm back to my normal weight, but the bottom is what's large. You can roll up the bottom part of the legs, or the top at the waist band for hanging out at the house, but if you have one nearby, simply have them re-hemmed.", "long"]]
+    return {"agree": aggregated[0], "neutral": aggregated[1], "disagree": aggregated[2]}, {"agree": normal[0], "neutral": normal[1], "disagree": normal[2]}
+examples = [["These are my absolute favorite cargos in my closet. I’m 5’7 and they’re actually long enough for me. I’m 165lbs and ordered an M & it fits nice and loose just how I wanted it. The adjustable waist band is awesome!", "long"], ["I feel strongly about politics in the US", "long"], ["The pants are long", "long"], ["The pants are slightly long", "long"]]
 
 gradio_app = gr.Interface(
     predict,
@@ -83,8 +68,8 @@ gradio_app = gr.Interface(
     inputs=[gr.Text(label="Statement"), gr.Text(label="Class"), gr.Checkbox(label="Allow multiple true classes")],
     outputs=[gr.Label(num_top_classes=3, label="With Postprocessing"), gr.Label(num_top_classes=3, label="Without Postprocessing")],
     title="Intent Analysis",
-    description="This model predicts whether or not the **_class_** describes the **_object described in the sentence_**. <br /> The two outputs shows what TE would predict with and without the postprocessing. An example edge case for normal TE is shown below. <br /> **_It is recommended that you clone the repository to speed up processing time_**.",
+    description="This model predicts whether or not the **_class_** describes the **_object described in the sentence_**. <br /> The two outputs shows what TE would predict with and without the postprocessing. An example edge case for normal TE is shown below. <br /> **_It is recommended that you clone the repository to speed up processing time_**. <br /> Additionally, note the difference between the strength of the probability when going between the last two examples, the former representing a strong opinion and the latter a weaker opinion",
     cache_examples=True
 )
 
-gradio_app.launch(share=True)
+gradio_app.launch()

gradio_cached_examples/18/log.csv

@@ -1,2 +1,5 @@
 With Postprocessing,Without Postprocessing,flag,username,timestamp
-"{""label"":""agree"",""confidences"":[{""label"":""agree"",""confidence"":0.37631523609161377},{""label"":""neutral"",""confidence"":0.3404143750667572},{""label"":""disagree"",""confidence"":0.28327038884162903}]}","{""label"":""neutral"",""confidences"":[{""label"":""neutral"",""confidence"":0.8370960354804993},{""label"":""disagree"",""confidence"":0.12820996344089508},{""label"":""agree"",""confidence"":0.03469394892454147}]}",,,2024-03-10 20:51:53.608441
+"{""label"":""neutral"",""confidences"":[{""label"":""neutral"",""confidence"":0.3989219665527344},{""label"":""agree"",""confidence"":0.3555052578449249},{""label"":""disagree"",""confidence"":0.24557280540466309}]}","{""label"":""agree"",""confidences"":[{""label"":""agree"",""confidence"":0.9896149635314941},{""label"":""neutral"",""confidence"":0.008607939817011356},{""label"":""disagree"",""confidence"":0.0017770910635590553}]}",,,2024-03-14 17:33:32.082554
+"{""label"":""neutral"",""confidences"":[{""label"":""neutral"",""confidence"":0.9888092875480652},{""label"":""agree"",""confidence"":0.0059028444811701775},{""label"":""disagree"",""confidence"":0.005287905689328909}]}","{""label"":""neutral"",""confidences"":[{""label"":""neutral"",""confidence"":0.9971976280212402},{""label"":""agree"",""confidence"":0.0019434979185461998},{""label"":""disagree"",""confidence"":0.0008588095079176128}]}",,,2024-03-14 17:33:43.894521
+"{""label"":""agree"",""confidences"":[{""label"":""agree"",""confidence"":0.999862790107727},{""label"":""disagree"",""confidence"":0.00007605748396599665},{""label"":""neutral"",""confidence"":0.00006114941061241552}]}","{""label"":""agree"",""confidences"":[{""label"":""agree"",""confidence"":0.9909017086029053},{""label"":""neutral"",""confidence"":0.008000608533620834},{""label"":""disagree"",""confidence"":0.0010977860074490309}]}",,,2024-03-14 17:33:56.295829
+"{""label"":""agree"",""confidences"":[{""label"":""agree"",""confidence"":0.5945301651954651},{""label"":""neutral"",""confidence"":0.26899591088294983},{""label"":""disagree"",""confidence"":0.1364738941192627}]}","{""label"":""agree"",""confidences"":[{""label"":""agree"",""confidence"":0.8074565529823303},{""label"":""neutral"",""confidence"":0.1722831279039383},{""label"":""disagree"",""confidence"":0.02026035077869892}]}",,,2024-03-14 17:34:11.315778