import gradio as gr from dotenv import load_dotenv from langchain import PromptTemplate, LLMChain, HuggingFaceHub from langchain.llms import CTransformers from transformers import AutoModelForCausalLM, AutoTokenizer from transformers import pipeline from langchain.llms.huggingface_pipeline import HuggingFacePipeline from sentence_transformers import SentenceTransformer, util from sklearn.cluster import KMeans import nltk nltk.download('punkt') from nltk.tokenize import word_tokenize from nltk import tokenize import numpy as np import scipy.spatial import csv load_dotenv() def generate_prompts(user_input): prompt_template = PromptTemplate( input_variables=["Question"], template=f"Just list 10 question prompts for {user_input} and don't put number before each of the prompts." ) config = {'max_new_tokens': 64, 'temperature': 0.7, 'context_length': 64} llm = CTransformers(model="TheBloke/Mistral-7B-Instruct-v0.1-GGUF", config=config) hub_chain = LLMChain(prompt = prompt_template, llm = llm) input_data = {"Question": user_input} generated_prompts = hub_chain.run(input_data) questions_list = generated_prompts.split('\n') formatted_questions = "\n".join(f"Question: {question}" for i, question in enumerate(questions_list) if question.strip()) questions_list = formatted_questions.split("Question:")[1:] return questions_list def answer_question(prompt): prompt_template = PromptTemplate( input_variables=["Question"], template=f"give one answer for {prompt} and do not consider the number behind it." ) config = {'max_new_tokens': 64, 'temperature': 0.7, 'context_length': 64} llm = CTransformers(model="TheBloke/Llama-2-7B-Chat-GGML", config=config) hub_chain = LLMChain(prompt = prompt_template, llm = llm) input_data = {"Question": prompt} generated_answer = hub_chain.run(input_data) return generated_answer def calculate_similarity(word, other_words, model, threshold=0.5): embeddings_word = model.encode([word]) embeddings_other_words = model.encode(other_words) for i, embedding in enumerate(embeddings_other_words): similarity = 1 - scipy.spatial.distance.cosine(embeddings_word[0], embedding) if similarity > threshold and similarity < 0.85: return i, similarity return None, None def highlight_similar_paragraphs_with_colors(paragraphs, similarity_threshold=0.75): model = SentenceTransformer('all-MiniLM-L6-v2') # Split each paragraph into sentences all_sentences = [tokenize.sent_tokenize(paragraph) for paragraph in paragraphs] # Initialize storage for highlighted sentences highlighted_sentences = [['' for sentence in para] for para in all_sentences] colors = ['yellow', 'lightgreen', 'lightblue', 'pink', 'lavender', 'salmon', 'peachpuff', 'powderblue', 'khaki', 'wheat'] # Track which sentences belong to which paragraph sentence_to_paragraph_index = [idx for idx, para in enumerate(all_sentences) for sentence in para] # Encode all sentences into vectors flattened_sentences = [sentence for para in all_sentences for sentence in para] sentence_embeddings = model.encode(flattened_sentences) # Calculate cosine similarities between all pairs of sentences cosine_similarities = util.pytorch_cos_sim(sentence_embeddings, sentence_embeddings) # Iterate through each sentence pair and highlight if they are similar but from different paragraphs color_index = 0 for i, embedding_i in enumerate(sentence_embeddings): for j, embedding_j in enumerate(sentence_embeddings): if i != j and cosine_similarities[i, j] > similarity_threshold and sentence_to_paragraph_index[i] != sentence_to_paragraph_index[j]: color = colors[color_index % len(colors)] if highlighted_sentences[sentence_to_paragraph_index[i]][i % len(all_sentences[sentence_to_paragraph_index[i]])] == '': highlighted_sentences[sentence_to_paragraph_index[i]][i % len(all_sentences[sentence_to_paragraph_index[i]])] = (""+ flattened_sentences[i]+"") if highlighted_sentences[sentence_to_paragraph_index[j]][j % len(all_sentences[sentence_to_paragraph_index[j]])] == '': highlighted_sentences[sentence_to_paragraph_index[j]][j % len(all_sentences[sentence_to_paragraph_index[j]])] = (""+ flattened_sentences[j]+"") color_index += 1 # Move to the next color # Combine sentences back into paragraphs highlighted_paragraphs = [' '.join(para) for para in highlighted_sentences] # Combine all paragraphs into one HTML string html_output = '
' + '

'.join(highlighted_paragraphs) + '
' return highlighted_paragraphs def calculate_similarity_score(sentences): # Encode all sentences to get their embeddings model = SentenceTransformer('all-MiniLM-L6-v2') embeddings = model.encode(sentences) # Calculate average cosine similarity total_similarity = 0 comparisons = 0 for i in range(len(embeddings)): for j in range(i+1, len(embeddings)): # Cosine similarity between embeddings similarity = 1 - cosine(embeddings[i], embeddings[j]) total_similarity += similarity comparisons += 1 # Average similarity average_similarity = total_similarity / comparisons if comparisons > 0 else 0 # Scale from [-1, 1] to [0, 100] score_out_of_100 = (average_similarity + 1) / 2 * 100 return score_out_of_100 def answer_question(prompt): prompt_template = PromptTemplate.from_template( input_variables=["Question"], template=f"give one answer for {prompt} and do not consider the number behind it." ) config = {'max_new_tokens': 2048, 'temperature': 0.7, 'context_length': 4096} llm = CTransformers(model="TheBloke/Llama-2-7B-Chat-GGML", config=config, threads=os.cpu_count()) hub_chain = LLMChain(prompt = prompt_template, llm = llm) input_data = {"Question": prompt} generated_answer = hub_chain.run(input_data) return generated_answer def process_inputs(llm, questions, relevance, diversity, email): # Save questions to a CSV file questions_list = questions.split('\n') df = pd.DataFrame(questions_list, columns=["Questions"]) csv_file = "/mnt/data/questions.csv" df.to_csv(csv_file, index=False) # Email the CSV file sender_email = "auditllms@gmail.com" sender_password = "opri fcxx crkh bvfj" receiver_email = email subject = "Your Submitted Questions" body = "Thank you for your submission. Please find attached the CSV file containing your questions." message = MIMEMultipart() message['From'] = sender_email message['To'] = receiver_email message['Subject'] = subject message.attach(MIMEText(body, 'plain')) attachment = open(csv_file, "rb") part = MIMEBase('application', 'octet-stream') part.set_payload((attachment).read()) encoders.encode_base64(part) part.add_header('Content-Disposition', f"attachment; filename= questions.csv") message.attach(part) server = smtplib.SMTP('smtp.gmail.com', 587) server.starttls() server.login(sender_email, sender_password) text = message.as_string() server.sendmail(sender_email, receiver_email, text) server.quit() return f"Submitted questions:\n\n{questions}\n\nRelevance: {relevance}\nDiversity: {diversity}\nEmail: {email}" text_list = [] def updateChoices(prompt): newChoices = generate_prompts(prompt) return gr.CheckboxGroup(choices=newChoices) def setTextVisibility(cbg, model_name_input): sentences = [answer_question(text, model_name_input) for text in cbg] # Apply highlighting to all processed sentences, receiving one complete HTML string. highlighted_html = [] highlighted_html = highlight_similar_paragraphs_with_colors(sentences, similarity_threshold=0.75) result = [] # Iterate through each original 'cbg' sentence and pair it with the entire highlighted block. for idx, sentence in enumerate(highlighted_html): result.append("

"+ cbg[idx] +"

"+ sentence +"


") score = round(calculate_similarity_score(highlighted_html)) final_html = f"""
{result}
Similarity Score: {score}
""" return final_html # update_show = [gr.Textbox(visible=True, label=text, value=answer_question(text, model_name_input)) for text in cbg] # update_hide = [gr.Textbox(visible=False, label="") for _ in range(10-len(cbg))] # return update_show + update_hide with gr.Blocks(theme=gr.themes.Soft()) as demo: gr.HTML("""

AuditLLM

""") with gr.Tab("Live Mode"): gr.HTML("""

Live Mode Auditing LLMs

First, select the LLM you wish to audit. Then, enter your question. The AuditLLM tool will generate five relevant and diverse prompts based on your question. You can now select these prompts for auditing the LLMs. Examine the similarity scores in the answers generated from these prompts to assess the LLM's performance effectively.

""") with gr.Row(): model_name_input = gr.Dropdown([("Llama", "TheBloke/Llama-2-7B-Chat-GGML"), ("Falcon", "TheBloke/Falcon-180B-Chat-GGUF"), ("Zephyr", "TheBloke/zephyr-quiklang-3b-4K-GGUF"),("Vicuna", "TheBloke/vicuna-33B-GGUF"),("Claude","TheBloke/claude2-alpaca-13B-GGUF"),("Alpaca","TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], label="Large Language Model") with gr.Row(): prompt_input = gr.Textbox(label="Enter your question", placeholder="Enter Your Question") with gr.Row(): generate_button = gr.Button("Generate", variant="primary", min_width=300) with gr.Column(): cbg = gr.CheckboxGroup(choices=[], label="List of the prompts", interactive=True) generate_button.click(updateChoices, inputs=[prompt_input], outputs=[cbg]) with gr.Row() as exec: btnExec = gr.Button("Execute", variant="primary", min_width=200) with gr.Column() as texts: for i in range(10): text = gr.Textbox(label="_", visible=False) text_list.append(text) with gr.Column(): html_result = gr.HTML("""
""") #btnExec.click(setTextVisibility, inputs=[cbg, model_name_input], outputs=text_list) btnExec.click(setTextVisibility, inputs=[cbg, model_name_input], outputs=html_result) gr.HTML("""
Similarity Score:
""") clear = gr.ClearButton(link = "http://127.0.0.1:7865") with gr.Tab("Batch Mode"): gr.HTML("""

Batch Mode Auditing LLMs

In batch auditing mode, you have the capability to probe the LLM. To begin, you must first select the LLM you wish to audit and then input the questions you intend to explore. For each question submitted, the model will generate five prompts, each accompanied by its respective answers.

To tailor the generation of these five prompts from your original question, you can adjust the relevance and diversity scores. The relevance score determines how closely the generated prompts should align with the original question, while the diversity score dictates the variance among the prompts themselves.

Upon completion, please provide your email address. We will compile and send the answers to you promptly.

""") gr.Markdown("## Batch Mode Auditing LLMs") gr.Markdown("In batch auditing mode, you have the capability to probe the LLM. To begin, you must first select the LLM you wish to audit and then input the questions you intend to explore. For each question submitted, the model will generate five prompts, each accompanied by its respective answers.") gr.Markdown("To tailor the generation of these five prompts from your original question, you can adjust the relevance and diversity scores. The relevance score determines how closely the generated prompts should align with the original question, while the diversity score dictates the variance among the prompts themselves.") gr.Markdown("Upon completion, please provide your email address. We will compile and send the answers to you promptly.") llm_dropdown = gr.Dropdown([("Llama", "TheBloke/Llama-2-7B-Chat-GGML"), ("Falcon", "TheBloke/Falcon-180B-Chat-GGUF"), ("Zephyr", "TheBloke/zephyr-quiklang-3b-4K-GGUF"), ("Vicuna", "TheBloke/vicuna-33B-GGUF"), ("Claude", "TheBloke/claude2-alpaca-13B-GGUF"), ("Alpaca", "TheBloke/LeoScorpius-GreenNode-Alpaca-7B-v1-GGUF")], label="Large Language Model") questions_textbox = gr.Textbox(label="Enter your question", placeholder="Enter your questions here...") file_upload = gr.File(label="Or You Can Click to Upload a File") relevance_slider = gr.Slider(0, 100, value=70, step=1, label="Relevance") diversity_slider = gr.Slider(0, 100, value=25, step=1, label="Diversity") email_input = gr.Textbox(label="Enter your email address", placeholder="name@example.com") submit_button = gr.Button("Submit") submit_button.click(fn=process_inputs, inputs=[llm_dropdown, questions_textbox, relevance_slider, diversity_slider, email_input], outputs="text") # Launch the Gradio app demo.launch()