Create app.py

app.py

@@ -0,0 +1,163 @@
+import gradio as gr
+import pandas as pd
+from pytube import extract
+import re
+import string
+import pickle
+import nltk
+import nltk.sentiment.util
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer 
+
+from keras.preprocessing.text import Tokenizer
+from keras.preprocessing.sequence import pad_sequences
+from tensorflow import keras
+
+from youtube_comment_downloader import *
+
+# get YouTube ID
+def getID(url):
+    print("Getting YouTube ID...")
+    return extract.video_id(url)
+
+# function to clean comments
+def clean_text(text):
+    lemmatizer = WordNetLemmatizer()
+    # stopwords
+    sw = ["i","me","my","myself","we","our","ours","ourselves","you","you're","you've","you'll","you'd","your","yours","yourself","yourselves","he","him","his","himself","she","she's","her","hers","herself","it","it's","its","itself","they","them","their","theirs","themselves","what","which","who","whom","this","that","that'll","these","those","am","is","are","was","were","be","been","being","have","has","had","having","do","does","did","doing","a","an","the","and","but","if","or","because","as","until","while","of","at","by","for","with","about","against","between","into","through","during","before","after","above","below","to","from","up","down","in","out","on","off","over","under","again","further","then","once","here","there","when","where","why","how","all","any","both","each","few","more","most","other","some","such","no","nor","not","only","own","same","so","than","too","very","s","t","can","will","just","don","don't","should","should've","now","d","ll","m","o","re","ve","y","ain","aren","aren't","couldn","couldn't","didn","didn't","doesn","doesn't","hadn","hadn't","hasn","hasn't","haven","haven't","isn","isn't","ma","mightn","mightn't","mustn","mustn't","needn","needn't","shan","shan't","shouldn","shouldn't","wasn","wasn't","weren","weren't","won","won't","wouldn","wouldn't"]
+    # remove symbols and Emojis
+    text = text.lower()
+    text = re.sub('@', '', text)
+    text = re.sub('\[.*?\]', '', text)
+    text = re.sub('https?://\S+|www\.\S+', '', text)
+    text = re.sub('<.*?>+', '', text)
+    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
+    text = re.sub('\n', '', text)
+    text = re.sub('\w*\d\w*', '', text)
+    text = re.sub(r"[^a-zA-Z ]+", "", text)
+    
+    # tokenize the data
+    text = nltk.word_tokenize(text)
+    
+    # lemmatize
+    text = [lemmatizer.lemmatize(t) for t in text]
+    text = [lemmatizer.lemmatize(t, 'v') for t in text]
+    
+    # mark Negation
+    tokens_neg_marked = nltk.sentiment.util.mark_negation(text)
+    
+    # remove stopwords
+    text = [t for t in tokens_neg_marked
+             if t.replace("_NEG", "").isalnum() and
+             t.replace("_NEG", "") not in sw]
+    
+    return text
+
+def getSentenceTrain():
+    # open sentences_train file
+    sentences_train_f = open('Deep learning/pickles/sentences_train.pickle', "rb")
+    sentences_train = pickle.load(sentences_train_f)
+    sentences_train_f.close()
+    return sentences_train
+
+SGD_74_f = open('Shallow machine learning/pickles/SGD_74.pickle', "rb")
+SGD_train = pickle.load(SGD_74_f)
+SGD_74_f.close()
+
+logreg_79_f = open('Shallow machine learning/pickles/logreg_79.pickle', "rb")
+logreg_train = pickle.load(logreg_79_f)
+logreg_79_f.close()
+
+# get saved CNN model
+model = keras.models.load_model("Deep learning/CNN_82")
+
+def vote(test_point, _test):
+    print("Voting on video effectivess...\n")
+    pos_weighting = []
+    result = ''
+    confidence = 0
+    algos_score = 0
+    
+    algorithms = [
+             {'name': 'SGD', 'accuracy': 0.74*100, 'trained': SGD_train},
+             {'name': 'Logistic Regression', 'accuracy': 0.79*100, 'trained': logreg_train},
+             {'name': 'CNN', 'accuracy': 0.82*100, 'trained': model}
+        ]
+    
+    for algo in algorithms:
+        weight = algo['accuracy']
+        algos_score += weight
+        if algo['name'] == "CNN":
+            pred = algo['trained'].predict(_test)
+            if pred[0][0] > 0.5:
+                pos_weighting.append(weight)
+            print("CNN voted for: effective" if pred[0][0]>0.5 else "CNN voted for: ineffective")
+        else:
+            pred = algo['trained'].predict(test_point)
+            if pred == 'pos':
+                pos_weighting.append(weight)
+            print(algo['name'] + " voted for: effective" if pred=='pos' else algo['name'] + " voted for: ineffective")
+
+    pos_result = sum(pos_weighting)/algos_score
+    if pos_result < 0.5:
+        result = 'ineffective'
+        confidence = 1 - pos_result
+    else:
+        result = 'effective'
+        confidence = pos_result
+        
+    return result, confidence
+
+def quantizeEffectiveness(url):
+    # 1. Get YouTube ID
+    print("Getting YouTube ID...")
+    videoId = getID(url)
+    
+    # 2. Download comments
+    print("Downloading comments...")
+    downloader = YoutubeCommentDownloader()
+    comments_downloaded = downloader.get_comments_from_url(f'https://www.youtube.com/watch?v={videoId}')
+    comments = [comment for comment in comments_downloaded]
+    comments_df = pd.DataFrame(comments)
+    
+    # 3. Clean comments
+    print("Cleaning Comments...")
+    comments_df['text'] = comments_df['text'].apply(lambda x: clean_text(x))
+
+    # get all words of video into one list
+    all_words = [item for sublist in comments_df['text'].tolist() for item in sublist]
+
+    # 4. Create test dataframe
+    test = pd.DataFrame([[videoId]], columns=['VideoId'])
+    
+    # 5. Get documents (pre-processd comments)
+    test_documents = []
+    test_documents.append(all_words)
+    test['cleaned'] = test_documents
+    test['cleaned_string'] = [' '.join(map(str, l)) for l in test['cleaned']]
+    
+    # 6. Get ML test point
+    test_point = test['cleaned_string']
+    test_sentence = test['cleaned_string'].values
+    
+    # 7. Get trained sentences
+    sentences_train = getSentenceTrain()
+    
+    # 8. Tokenize the data
+    print("Tokenizing the data...")
+    tokenizer = Tokenizer(num_words=5000)
+    tokenizer.fit_on_texts(sentences_train)
+    
+    # 9. Get DL test point
+    _test = pad_sequences(tokenizer.texts_to_sequences(test_sentence), padding='post', maxlen=100)
+    
+    # 10. Vote on video effectiveness
+    result, confidence = vote(test_point, _test)
+    
+def greet(url):
+    result, confidence = quantizeEffectiveness(url)
+    
+    return f"The video (ID: {videoId}) is {result} with a confidence of {round(confidence*100,2)}%"
+
+iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+iface.launch()