Update app.py

app.py

@@ -21,126 +21,137 @@ import csv
 
 load_dotenv()
 
+# Global array of different possible LLM selection options
+LLM_OPTIONS = [
+    ("Llama-2-7B", "TheBloke/Llama-2-7B-Chat-GGML"),
+    ("Falcon-180B", "TheBloke/Falcon-180B-Chat-GGUF"),
+    ("Zephyr-7B", "zephyr-quiklang-3b-4k.Q4_K_M.gguf"),
+    ("Vicuna-33B", "TheBloke/vicuna-33B-GGUF"),
+    ("Claude2", "TheBloke/claude2-alpaca-13B-GGUF"),
+    ("Alpaca-7B", "TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")
+]
+
 def generate_prompts(user_input):
+    print("User input here")
+    print(user_input)
     prompt_template = PromptTemplate(
         input_variables=["Question"],
-        template=f"Just list 10 question prompts for {user_input} and don't put number before each of the prompts."
+        template=f"Just list 5 distinct and separate yet relevant question prompts for {user_input} and don't put number before any of the prompts."
     )
-    config = {'max_new_tokens': 64, 'temperature': 0.7, 'context_length': 64}
+    config = {'max_new_tokens': 256, 'temperature': 0.7, 'context_length': 256}
     llm = CTransformers(model="TheBloke/Mistral-7B-Instruct-v0.1-GGUF",
                         config=config)
-    hub_chain = LLMChain(prompt = prompt_template, llm = llm)
+    hub_chain = LLMChain(prompt=prompt_template, llm=llm)
     input_data = {"Question": user_input}
 
     generated_prompts = hub_chain.run(input_data)  
     questions_list = generated_prompts.split('\n') 
-    
 
-    formatted_questions = "\n".join(f"Question: {question}" for i, question in enumerate(questions_list) if question.strip())
+    formatted_questions = "\n".join(f"Question: {question}" for question in questions_list if question.strip())
     questions_list = formatted_questions.split("Question:")[1:]
     return questions_list
 
-def answer_question(prompt):
+def answer_question(prompt, model_name):
+    print("inside answer question function")
+    print("prompt")
+    print(prompt)
+    print("")
+    print("model name")
+    print(model_name)
+    print("")
+
     prompt_template = PromptTemplate(
         input_variables=["Question"],
-        template=f"give one answer for {prompt} and do not consider the number behind it."
+        template=f"Please provide a concise and relevant answer for {prompt} in three sentences or less and don't put Answer in front of what you return. You are a helpful and factual assistant, do not say thank you or you are happy to assist just answer the question."
     )
-    config = {'max_new_tokens': 64, 'temperature': 0.7, 'context_length': 64}
-    llm = CTransformers(model="TheBloke/Llama-2-7B-Chat-GGML",
-                        config=config)
-    hub_chain = LLMChain(prompt = prompt_template, llm = llm)
-    input_data = {"Question": prompt}
+    config = {'max_new_tokens': 256, 'temperature': 0.7, 'context_length': 256}
+    llm = CTransformers(model=model_name,
+                        config=config, 
+                        threads=os.cpu_count())
+    hub_chain = LLMChain(prompt=prompt_template, llm=llm)
+    input_data = {"Answer the question": prompt}
     generated_answer = hub_chain.run(input_data)  
+    print("generated answer")
+    print(generated_answer)
     return generated_answer
 
-def calculate_similarity(word, other_words, model, threshold=0.5):
-    embeddings_word = model.encode([word])
-    embeddings_other_words = model.encode(other_words)
-    for i, embedding in enumerate(embeddings_other_words):
-        similarity = 1 - scipy.spatial.distance.cosine(embeddings_word[0], embedding)
-        if similarity > threshold and similarity < 0.85:
-            return i, similarity
-    return None, None
-
-
-def highlight_similar_paragraphs_with_colors(paragraphs, similarity_threshold=0.75):
+def calculate_sentence_similarities(sentences_list):
     model = SentenceTransformer('all-MiniLM-L6-v2')
+    embeddings_list = [model.encode(sentences) for sentences in sentences_list]
+    similarity_matrices = []
     
-    # Split each paragraph into sentences
-    all_sentences = [tokenize.sent_tokenize(paragraph) for paragraph in paragraphs]
-    
-    # Initialize storage for highlighted sentences
-    highlighted_sentences = [['' for sentence in para] for para in all_sentences]
-    colors = ['yellow', 'lightgreen', 'lightblue', 'pink', 'lavender', 'salmon', 'peachpuff', 'powderblue', 'khaki', 'wheat']
+    for i in range(len(embeddings_list)):
+        for j in range(i + 1, len(embeddings_list)):
+            similarity_matrix = util.pytorch_cos_sim(embeddings_list[i], embeddings_list[j]).numpy()
+            similarity_matrices.append((i, j, similarity_matrix))
     
-    # Track which sentences belong to which paragraph
-    sentence_to_paragraph_index = [idx for idx, para in enumerate(all_sentences) for sentence in para]
-    
-    # Encode all sentences into vectors
-    flattened_sentences = [sentence for para in all_sentences for sentence in para]
-    sentence_embeddings = model.encode(flattened_sentences)
-    
-    # Calculate cosine similarities between all pairs of sentences
-    cosine_similarities = util.pytorch_cos_sim(sentence_embeddings, sentence_embeddings)
-    
-    # Iterate through each sentence pair and highlight if they are similar but from different paragraphs
-    color_index = 0
-    for i, embedding_i in enumerate(sentence_embeddings):
-        for j, embedding_j in enumerate(sentence_embeddings):
-            if i != j and cosine_similarities[i, j] > similarity_threshold and sentence_to_paragraph_index[i] != sentence_to_paragraph_index[j]:
-                color = colors[color_index % len(colors)]
-                if highlighted_sentences[sentence_to_paragraph_index[i]][i % len(all_sentences[sentence_to_paragraph_index[i]])] == '':
-                    highlighted_sentences[sentence_to_paragraph_index[i]][i % len(all_sentences[sentence_to_paragraph_index[i]])] = ("<span style='color: "+  color  +"'>"+ flattened_sentences[i]+"</span>")
-                if highlighted_sentences[sentence_to_paragraph_index[j]][j % len(all_sentences[sentence_to_paragraph_index[j]])] == '':
-                    highlighted_sentences[sentence_to_paragraph_index[j]][j % len(all_sentences[sentence_to_paragraph_index[j]])] = ("<span style='color: "+  color  +"'>"+ flattened_sentences[j]+"</span>")
-                color_index += 1  # Move to the next color
-
-    # Combine sentences back into paragraphs
-    highlighted_paragraphs = [' '.join(para) for para in highlighted_sentences]
-
-    # Combine all paragraphs into one HTML string
-    html_output = '<div>' + '<br/><br/>'.join(highlighted_paragraphs) + '</div>'
-    return highlighted_paragraphs
+    return similarity_matrices
+
+def highlight_similar_sentences(sentences_list, similarity_threshold):
+    similarity_matrices = calculate_sentence_similarities(sentences_list)
+    highlighted_sentences = [[] for _ in sentences_list]
+
+    for (i, j, similarity_matrix) in similarity_matrices:
+        for idx1 in range(similarity_matrix.shape[0]):
+            for idx2 in range(similarity_matrix.shape[1]):
+                similarity = similarity_matrix[idx1][idx2]
+                print(f"Similarity between sentence {idx1} in paragraph {i} and sentence {idx2} in paragraph {j}: {similarity:.2f}")
+                if similarity >= similarity_threshold:
+                    print("Greater than sim!")
+                    if (idx1, "powderblue", similarity) not in highlighted_sentences[i]:
+                        highlighted_sentences[i].append((idx1, "powderblue", similarity))
+                    if (idx2, "powderblue", similarity) not in highlighted_sentences[j]:
+                        highlighted_sentences[j].append((idx2, "powderblue", similarity))
+
+    for i, sentences in enumerate(sentences_list):
+        highlighted = []
+        for j, sentence in enumerate(sentences):
+            color = "none"
+            score = 0
+            for idx, col, sim in highlighted_sentences[i]:
+                if idx == j:
+                    color = col
+                    score = sim
+                    break
+            highlighted.append({"text": sentence, "background-color": color, "score": score})
+        highlighted_sentences[i] = highlighted
+
+    print(highlighted_sentences)
+    return highlighted_sentences
 
-    
-def calculate_similarity_score(sentences):
-    # Encode all sentences to get their embeddings
-    model = SentenceTransformer('all-MiniLM-L6-v2')
-    embeddings = model.encode(sentences)
-
-    # Calculate average cosine similarity
-    total_similarity = 0
-    comparisons = 0
-    for i in range(len(embeddings)):
-        for j in range(i+1, len(embeddings)):
-            # Cosine similarity between embeddings
-            similarity = 1 - cosine(embeddings[i], embeddings[j])
-            total_similarity += similarity
-            comparisons += 1
-
-    # Average similarity
-    average_similarity = total_similarity / comparisons if comparisons > 0 else 0
-
-    # Scale from [-1, 1] to [0, 100]
-    score_out_of_100 = (average_similarity + 1) / 2 * 100
-    return score_out_of_100
-
-def answer_question1(prompt):
-    prompt_template = PromptTemplate.from_template(
-        input_variables=["Question"],
-        template=f"give one answer for {prompt} and do not consider the number behind it."
-    )
-    config = {'max_new_tokens': 2048, 'temperature': 0.7, 'context_length': 4096}
-    llm = CTransformers(model="TheBloke/Llama-2-7B-Chat-GGML",
-                        config=config,
-                        threads=os.cpu_count())
-    hub_chain = LLMChain(prompt = prompt_template, llm = llm)
-    input_data = {"Question": prompt}
-    generated_answer = hub_chain.run(input_data)  
-    return generated_answer
 
+def setTextVisibility(cbg, model_name_input):
+    selected_prompts = cbg
+    answers = [answer_question(prompt, model_name_input) for prompt in selected_prompts]
+
+    sentences_list = [tokenize.sent_tokenize(answer) for answer in answers]
+    highlighted_sentences_list = highlight_similar_sentences(sentences_list, 0.5)
+
+    result = []
+    for idx, (prompt, highlighted_sentences) in enumerate(zip(selected_prompts, highlighted_sentences_list)):
+        result.append(f"<p><strong>Prompt: {prompt}</strong></p>")
+        for sentence_info in highlighted_sentences:
+            color = sentence_info.get('background-color', 'none')  # Read the 'color' parameter
+            result.append(f"<p style='background-color: {color};'><strong>{sentence_info['text']}</strong></p>")
     
-def process_inputs(llm, file, relevance, diversity):
+    blue_scores_list = [[info['score'] for info in highlighted_sentences if info['background-color'] == 'powderblue'] for highlighted_sentences in highlighted_sentences_list]
+    blue_scores = [score for scores in blue_scores_list for score in scores]
+
+    if blue_scores:
+        overall_score = round(np.mean(blue_scores) * 100)
+    else:
+        overall_score = 0
+
+    final_html = f"""<div>{''.join(result)}<div style="text-align: center; font-size: 24px; font-weight: bold;">Similarity Score: {overall_score}</div></div>"""
+
+    print("")
+    print("final html")
+    print(final_html)
+    return final_html
+
+
+
+def process_inputs(file, relevance, diversity, model_name):
     # Check if file is uploaded
     if file is not None:
         # Read questions from the uploaded Excel file
@@ -156,11 +167,11 @@ def process_inputs(llm, file, relevance, diversity):
         # Extract the first column
         questions_list = df.iloc[:, 0]
 
-        # Initialize lists to store the expanded data
         # Initialize lists to store the expanded data
         expanded_questions = []
         expanded_prompts = []
         expanded_answers = []
+        semantic_similarities = []
 
         # Generate prompts for each question and expand the data
         for question in questions_list:
@@ -169,14 +180,43 @@ def process_inputs(llm, file, relevance, diversity):
             expanded_prompts.extend(prompts)
 
             # Generate answers for each prompt
-            answers = [answer_question(prompt) for prompt in prompts]
+            answers = [answer_question(prompt, model_name) for prompt in prompts]
             expanded_answers.extend(answers)
 
+            # Calculate semantic similarity score for each answer
+            similarity_scores = []
+            for answer in answers:
+                sentences_list = tokenize.sent_tokenize(answer)
+                highlighted_sentences_list = highlight_similar_sentences([sentences_list], 0.5)
+                print("highlighted sentences list")
+                print(highlighted_sentences_list)
+
+                blue_scores_list = [[info['score'] for info in highlighted_sentences if info['background-color'] == 'powderblue'] for highlighted_sentences in highlighted_sentences_list]
+                blue_scores = [score for scores in blue_scores_list for score in scores]
+
+                if blue_scores:
+                    overall_score = round(np.mean(blue_scores) * 100)
+                else:
+                    overall_score = 0
+
+                similarity_scores.append(overall_score)
+                print("overall score")
+                print(overall_score)
+
+            # Calculate mean similarity score for each question
+            question_similarity_score = np.mean(similarity_scores)
+            print("question sim score")
+            print(question_similarity_score)
+
+            # Extend the list with the same score for all answers to this question
+            semantic_similarities.extend([question_similarity_score] * len(prompts))
+
         # Combine the expanded data into a DataFrame
         output_df = pd.DataFrame({
             'Questions': expanded_questions,
             'Generated Prompts': expanded_prompts,
-            'Answers': expanded_answers
+            'Answers': expanded_answers,
+            'Semantic Similarity': semantic_similarities
         })
 
         # Save the DataFrame to a new Excel file
@@ -186,40 +226,20 @@ def process_inputs(llm, file, relevance, diversity):
         return "No questions provided.", None
 
     return "Processing complete. Download the file below.", output_file
-    
 
 text_list = []
 
+def get_model_name(model_label):
+    # Retrieve the model name based on the selected label
+    for label, name in LLM_OPTIONS:
+        if label == model_label:
+            return name
+    return None
+
 def updateChoices(prompt):
     newChoices = generate_prompts(prompt)
     return gr.CheckboxGroup(choices=newChoices)
 
-def setTextVisibility(cbg, model_name_input):
-
-    sentences = [answer_question(text, model_name_input) for text in cbg]
-    
-    # Apply highlighting to all processed sentences, receiving one complete HTML string.
-    highlighted_html = []
-    highlighted_html = highlight_similar_paragraphs_with_colors(sentences, similarity_threshold=0.75)
-    
-    
-    result = []
-    # Iterate through each original 'cbg' sentence and pair it with the entire highlighted block.
-    for idx, sentence in enumerate(highlighted_html):
-        result.append("<p><strong>"+ cbg[idx] +"</strong></p><p>"+ sentence +"</p><br/>")
-
-    score = round(calculate_similarity_score(highlighted_html))
-
-    final_html = f"""<div>{result}<div style="text-align: center; font-size: 24px; font-weight: bold;">Similarity Score: {score}</div></div>"""
-
-
-    return final_html
-    
-
-    # update_show = [gr.Textbox(visible=True, label=text, value=answer_question(text, model_name_input)) for text in cbg]
-    # update_hide = [gr.Textbox(visible=False, label="") for _ in range(10-len(cbg))]
-    # return update_show + update_hide
-
 with gr.Blocks(theme=gr.themes.Soft()) as demo:
     
     with gr.Tab("Live Mode"):
@@ -228,7 +248,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         gr.Markdown("In Live Auditing Mode, you gain the ability to probe the LLM directly.")
         gr.Markdown("First, select the LLM you wish to audit. Then, enter your question. The AuditLLM tool will generate five relevant and diverse prompts based on your question. You can now select these prompts for auditing the LLMs. Examine the similarity scores in the answers generated from these prompts to assess the LLM's performance effectively") 
         with gr.Row():
-            model_name_input = gr.Dropdown([("Llama-2-7B", "TheBloke/Llama-2-7B-Chat-GGML"), ("Falcon-180B", "TheBloke/Falcon-180B-Chat-GGUF"), ("Zephyr-7B", "TheBloke/zephyr-quiklang-3b-4K-GGUF"),("Vicuna-33B", "TheBloke/vicuna-33B-GGUF"),("Claude2","TheBloke/claude2-alpaca-13B-GGUF"),("Alpaca-7B","TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], label="Large Language Model")
+            model_name_input = gr.Dropdown(choices=LLM_OPTIONS, label="Large Language Model")
         with gr.Row():
             prompt_input = gr.Textbox(label="Enter your question", placeholder="Enter Your Question")
         with gr.Row():
@@ -241,22 +261,17 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         with gr.Row() as exec: 
             btnExec = gr.Button("Execute", variant="primary", min_width=200)
 
-
         with gr.Column() as texts:
             for i in range(10):
                 text = gr.Textbox(label="_", visible=False)
                 text_list.append(text)
 
         with gr.Column():
-            html_result = gr.HTML("""<div style="color: red"></div>""")
+            html_result = gr.HTML("""<div style="background-color: powderblue"></div>""")
 
-        #btnExec.click(setTextVisibility, inputs=[cbg, model_name_input], outputs=text_list)
         btnExec.click(setTextVisibility, inputs=[cbg, model_name_input], outputs=html_result)
-        gr.HTML("""
-        <div style="text-align: center; font-size: 24px; font-weight: bold;">Similarity Score: </div>
-                """)
+        clear = gr.ClearButton(link="http://127.0.0.1:7860")
 
-        clear = gr.ClearButton(link = "http://127.0.0.1:7865") 
 
     with gr.Tab("Batch Mode"):
         
@@ -265,14 +280,18 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         gr.Markdown("To tailor the generation of these five prompts from your original question, you can adjust the relevance and diversity scores. The relevance score determines how closely the generated prompts should align with the original question, while the diversity score dictates the variance among the prompts themselves.")
         gr.Markdown("Upon completion, please provide your email address. We will compile and send the answers to you promptly.")
             
-        llm_dropdown = gr.Dropdown(choices=[
-                ("Llama-2-7B", "TheBloke/Llama-2-7B-Chat-GGML"), 
-                ("Falcon-180B", "TheBloke/Falcon-180B-Chat-GGUF"), 
-                ("Zephyr-7B", "TheBloke/zephyr-quiklang-3b-4K-GGUF"), 
-                ("Vicuna-33B", "TheBloke/vicuna-33B-GGUF"), 
-                ("Claude2", "TheBloke/claude2-alpaca-13B-GGUF"), 
-                ("Alpaca-7B", "TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], 
-                label="Large Language Model")
+        # llm_dropdown = gr.Dropdown(choices=[
+        #         ("Llama-2-7B", "TheBloke/Llama-2-7B-Chat-GGML"), 
+        #         ("Falcon-180B", "TheBloke/Falcon-180B-Chat-GGUF"), 
+        #         ("Zephyr-7B", "TheBloke/zephyr-quiklang-3b-4K-GGUF"), 
+        #         ("Vicuna-33B", "TheBloke/vicuna-33B-GGUF"), 
+        #         ("Claude2", "TheBloke/claude2-alpaca-13B-GGUF"), 
+        #         ("Alpaca-7B", "TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], 
+        #         label="Large Language Model")
+
+        with gr.Row():
+            model_name_batch_input = gr.Dropdown(choices=LLM_OPTIONS, label="Large Language Model")
+
         file_upload = gr.File(label="Upload an Excel File with Questions", file_types=[".xlsx"])
         with gr.Row():
             relevance_slider = gr.Slider(1, 100, value=70, label="Relevance", info="Choose between 0 and 100", interactive=True)
@@ -283,11 +302,13 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         output_textbox = gr.Textbox(label="Output")
         download_button = gr.File(label="Download Processed File")
 
-        def on_submit(llm, file, relevance, diversity):
-            result, output_file = process_inputs(llm, file, relevance, diversity)
+        def on_submit(file, relevance, diversity, model_name_batch_input):
+            print("in on submit")
+            print(model_name_batch_input)
+            result, output_file = process_inputs(file, relevance, diversity, model_name_batch_input)
             return result, output_file
 
-        submit_button.click(fn=on_submit, inputs=[llm_dropdown, file_upload, relevance_slider, diversity_slider], outputs=[output_textbox, download_button])
+        submit_button.click(fn=on_submit, inputs=[file_upload, relevance_slider, diversity_slider, model_name_batch_input], outputs=[output_textbox, download_button])
 
 # Launch the Gradio app
 demo.launch()