custom-chatbot.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Chatbot using Seq2Seq LSTM models**","metadata":{"id":"6xoKYBBO6xaV"}},{"cell_type":"markdown","source":"# Step 1: Import all the packages ","metadata":{"id":"mVuZTAV08qWY"}},{"cell_type":"code","source":"import numpy as np \nimport tensorflow as tf\nimport pickle\nfrom tensorflow.keras import layers, activations, models, preprocessing\nfrom tensorflow.keras import preprocessing, utils\nimport os\nimport yaml","metadata":{"id":"U0mJXRse83hp","execution":{"iopub.status.busy":"2023-07-23T18:48:41.963082Z","iopub.execute_input":"2023-07-23T18:48:41.963523Z","iopub.status.idle":"2023-07-23T18:48:47.914550Z","shell.execute_reply.started":"2023-07-23T18:48:41.963390Z","shell.execute_reply":"2023-07-23T18:48:47.913724Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"markdown","source":"# Step 3: Preprocessing the data","metadata":{"id":"l4kJp6uO-fQE"}},{"cell_type":"markdown","source":"### a) Reading the data from the files\nWe parse each of the .yaml files.\n\n1. Concatenate two or more sentences if the answer has two or more of them.\n2. Remove unwanted data types which are produced while parsing the data.\n3. Append <START> and <END> to all the answers.\n4. Create a Tokenizer and load the whole vocabulary ( questions + answers ) into it.","metadata":{"id":"QEV_hSXs-7mF"}},{"cell_type":"markdown","source":"The dataset contains .yml files which have pairs of different questions and their answers on varied subjects like history, bot profile, science etc.\nWe can easily read them as folows:","metadata":{"id":"qyUnopqjDjud"}},{"cell_type":"code","source":"dir_path = '/kaggle/input/chatterbotenglish/'\nfiles_list = os.listdir(dir_path + os.sep)","metadata":{"id":"RxG-s4k0CowI","execution":{"iopub.status.busy":"2023-07-23T18:48:47.917818Z","iopub.execute_input":"2023-07-23T18:48:47.918077Z","iopub.status.idle":"2023-07-23T18:48:47.932312Z","shell.execute_reply.started":"2023-07-23T18:48:47.918039Z","shell.execute_reply":"2023-07-23T18:48:47.931563Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"questions = list()\nanswers = list()\n\nfor filepath in files_list:\n    stream = open( dir_path + os.sep + filepath , 'rb')\n    docs = yaml.safe_load(stream)\n    conversations = docs['conversations']\n    for con in conversations:\n        if len( con ) > 2 :\n            questions.append(con[0])\n            replies = con[ 1 : ]\n            ans = ''\n            for rep in replies:\n                ans += ' ' + rep\n            answers.append( ans )\n        elif len( con )> 1:\n            questions.append(con[0])\n            answers.append(con[1])\n\nanswers_with_tags = list()\nfor i in range( len( answers ) ):\n    if type( answers[i] ) == str:\n        answers_with_tags.append( answers[i] )\n    else:\n        questions.pop( i )\n\nanswers = list()\nfor i in range( len( answers_with_tags ) ) :\n    answers.append( '<START> ' + answers_with_tags[i] + ' <END>' )\n\ntokenizer = preprocessing.text.Tokenizer()\ntokenizer.fit_on_texts( questions + answers )\nVOCAB_SIZE = len( tokenizer.word_index )+1\nprint( 'VOCAB SIZE : {}'.format( VOCAB_SIZE ))","metadata":{"id":"-bRvbQ00Coy5","outputId":"ef129a1c-3071-4d10-e6b2-7ac5a6ff2bac","execution":{"iopub.status.busy":"2023-07-23T18:48:47.935749Z","iopub.execute_input":"2023-07-23T18:48:47.936001Z","iopub.status.idle":"2023-07-23T18:48:48.279721Z","shell.execute_reply.started":"2023-07-23T18:48:47.935971Z","shell.execute_reply":"2023-07-23T18:48:48.278090Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"VOCAB SIZE : 1894\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### b) Preparing data for Seq2Seq model\n\nThis model requires 3 arrays encoder_input_data, decoder_input_data and decoder_output_data.\n\nFor encoder_input_data:\nTokensize the Questions and Pad them to their maximum Length.\n\nFor decoder_input_data:\nTokensize the Answers and Pad them to their maximum Length.\n\nFor decoder_output_data:\nTokensize the Answers and Remove the 1st element from all the tokenized_answers. This is the <START> element which was added earlier.","metadata":{"id":"WMPqb8LxIeGI"}},{"cell_type":"code","source":"from gensim.models import Word2Vec\nimport re","metadata":{"id":"oEfAPL4HCo1t","execution":{"iopub.status.busy":"2023-07-23T18:48:48.281925Z","iopub.execute_input":"2023-07-23T18:48:48.282189Z","iopub.status.idle":"2023-07-23T18:48:48.639295Z","shell.execute_reply.started":"2023-07-23T18:48:48.282152Z","shell.execute_reply":"2023-07-23T18:48:48.638489Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"vocab = []\nfor word in tokenizer.word_index:\n  vocab.append(word)\n\ndef tokenize(sentences):\n  tokens_list = []\n  vocabulary = []\n  for sentence in sentences:\n    sentence = sentence.lower()\n    sentence = re.sub('[^a-zA-Z]', ' ', sentence)\n    tokens = sentence.split()\n    vocabulary += tokens\n    tokens_list.append(tokens)\n  return tokens_list, vocabulary","metadata":{"id":"QqYoDsbSCo4f","execution":{"iopub.status.busy":"2023-07-23T18:48:48.640804Z","iopub.execute_input":"2023-07-23T18:48:48.641071Z","iopub.status.idle":"2023-07-23T18:48:48.649039Z","shell.execute_reply.started":"2023-07-23T18:48:48.641034Z","shell.execute_reply":"2023-07-23T18:48:48.648348Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"#encoder_input_data\ntokenized_questions = tokenizer.texts_to_sequences( questions )\nmaxlen_questions = max( [len(x) for x in tokenized_questions ] )\nprint(maxlen_questions)\npadded_questions = preprocessing.sequence.pad_sequences( tokenized_questions, maxlen = maxlen_questions, padding = 'post')\nencoder_input_data = np.array(padded_questions)\nprint(encoder_input_data.shape, maxlen_questions)","metadata":{"id":"9vKhieIwCo7J","outputId":"e97b4a74-7384-478c-d4ae-082513257107","execution":{"iopub.status.busy":"2023-07-23T18:48:48.650266Z","iopub.execute_input":"2023-07-23T18:48:48.650929Z","iopub.status.idle":"2023-07-23T18:48:48.675933Z","shell.execute_reply.started":"2023-07-23T18:48:48.650885Z","shell.execute_reply":"2023-07-23T18:48:48.675113Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"22\n(564, 22) 22\n","output_type":"stream"}]},{"cell_type":"code","source":"# decoder_input_data\ntokenized_answers = tokenizer.texts_to_sequences( answers )\nmaxlen_answers = max( [ len(x) for x in tokenized_answers ] )\nprint(maxlen_answers)\npadded_answers = preprocessing.sequence.pad_sequences( tokenized_answers , maxlen=maxlen_answers , padding='post' )\ndecoder_input_data = np.array( padded_answers )\nprint( decoder_input_data.shape , maxlen_answers )","metadata":{"id":"AJo7WPjLCo-q","outputId":"28b5e209-5389-4313-f3cd-f5451fa8c519","execution":{"iopub.status.busy":"2023-07-23T18:48:48.678164Z","iopub.execute_input":"2023-07-23T18:48:48.678462Z","iopub.status.idle":"2023-07-23T18:48:48.706865Z","shell.execute_reply.started":"2023-07-23T18:48:48.678409Z","shell.execute_reply":"2023-07-23T18:48:48.705936Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"74\n(564, 74) 74\n","output_type":"stream"}]},{"cell_type":"code","source":"# decoder_output_data\ntokenized_answers = tokenizer.texts_to_sequences( answers )\nfor i in range(len(tokenized_answers)) :\n    tokenized_answers[i] = tokenized_answers[i][1:]\npadded_answers = preprocessing.sequence.pad_sequences( tokenized_answers , maxlen=maxlen_answers , padding='post' )\nonehot_answers = utils.to_categorical( padded_answers , VOCAB_SIZE )\ndecoder_output_data = np.array( onehot_answers )\nprint( decoder_output_data.shape )","metadata":{"id":"ccY0wWdRCpCa","outputId":"07877cda-7e07-42b2-bb7e-772d118a3cf0","execution":{"iopub.status.busy":"2023-07-23T18:48:49.628578Z","iopub.execute_input":"2023-07-23T18:48:49.629296Z","iopub.status.idle":"2023-07-23T18:48:49.912247Z","shell.execute_reply.started":"2023-07-23T18:48:49.629259Z","shell.execute_reply":"2023-07-23T18:48:49.911287Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"(564, 74, 1894)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Saving tokenizer params for reloading tokenizer during inference time","metadata":{}},{"cell_type":"code","source":"with open('tokenizer.pkl', 'wb') as f:\n    pickle.dump(tokenizer, f)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:48:50.774214Z","iopub.execute_input":"2023-07-23T18:48:50.774645Z","iopub.status.idle":"2023-07-23T18:48:50.782611Z","shell.execute_reply.started":"2023-07-23T18:48:50.774609Z","shell.execute_reply":"2023-07-23T18:48:50.781813Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"with open('tokenizer_params.pkl', 'wb') as f:\n    tokenizer_params = {\n        'word_index': tokenizer.word_index,\n        'maxlen_questions' : maxlen_questions,\n        'maxlen_answers': maxlen_answers\n        # Add other tokenizer attributes you might need for inference\n    }\n    pickle.dump(tokenizer_params, f)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:48:51.969373Z","iopub.execute_input":"2023-07-23T18:48:51.970291Z","iopub.status.idle":"2023-07-23T18:48:51.979037Z","shell.execute_reply.started":"2023-07-23T18:48:51.970229Z","shell.execute_reply":"2023-07-23T18:48:51.976510Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"markdown","source":"Using Glove Embedding for better embeddings computation and better understanding of the context","metadata":{}},{"cell_type":"code","source":"glove_path = \"/kaggle/input/glove-embeddings/glove.6B.100d.txt\"\nembedding_dim = 100","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:48:56.473016Z","iopub.execute_input":"2023-07-23T18:48:56.473301Z","iopub.status.idle":"2023-07-23T18:48:56.477596Z","shell.execute_reply.started":"2023-07-23T18:48:56.473268Z","shell.execute_reply":"2023-07-23T18:48:56.476682Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"embeddings_index = {}\nwith open(glove_path, \"r\", encoding=\"utf-8\") as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype=\"float32\")\n        embeddings_index[word] = coefs","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:48:57.963301Z","iopub.execute_input":"2023-07-23T18:48:57.964090Z","iopub.status.idle":"2023-07-23T18:49:10.668605Z","shell.execute_reply.started":"2023-07-23T18:48:57.964051Z","shell.execute_reply":"2023-07-23T18:49:10.667666Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"embedding_matrix = np.zeros((VOCAB_SIZE, embedding_dim))\nfor word, i in tokenizer.word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:49:10.670499Z","iopub.execute_input":"2023-07-23T18:49:10.670780Z","iopub.status.idle":"2023-07-23T18:49:10.682452Z","shell.execute_reply.started":"2023-07-23T18:49:10.670735Z","shell.execute_reply":"2023-07-23T18:49:10.681652Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"# Step 4: Defining Encoder Decoder Model\n\n\n\n","metadata":{"id":"-D53pyucPCnk"}},{"cell_type":"code","source":"encoder_inputs = tf.keras.layers.Input(shape=( maxlen_questions , ))\nencoder_embedding = tf.keras.layers.Embedding(VOCAB_SIZE, embedding_dim, mask_zero=True, weights=[embedding_matrix], trainable=False)(encoder_inputs)\nencoder_outputs , state_h , state_c = tf.keras.layers.LSTM( 200 , return_state=True )( encoder_embedding )\nencoder_states = [ state_h , state_c ]\n\ndecoder_inputs = tf.keras.layers.Input(shape=( maxlen_answers ,  ))\ndecoder_embedding = tf.keras.layers.Embedding(VOCAB_SIZE, embedding_dim, mask_zero=True, weights=[embedding_matrix], trainable=False)(decoder_inputs)\ndecoder_lstm = tf.keras.layers.LSTM( 200 , return_state=True , return_sequences=True )\ndecoder_outputs , _ , _ = decoder_lstm ( decoder_embedding , initial_state=encoder_states )\ndecoder_dense = tf.keras.layers.Dense( VOCAB_SIZE , activation=tf.keras.activations.softmax ) \noutput = decoder_dense ( decoder_outputs )\n\nmodel = tf.keras.models.Model([encoder_inputs, decoder_inputs], output )\nmodel.compile(optimizer=tf.keras.optimizers.RMSprop(), loss='categorical_crossentropy', metrics=['accuracy'])\n\nmodel.summary()","metadata":{"id":"W3YjCFDwPRVN","outputId":"7bc112a0-6945-4100-e8d9-3bc5691797e5","execution":{"iopub.status.busy":"2023-07-23T18:49:10.684025Z","iopub.execute_input":"2023-07-23T18:49:10.684315Z","iopub.status.idle":"2023-07-23T18:49:15.374966Z","shell.execute_reply.started":"2023-07-23T18:49:10.684277Z","shell.execute_reply":"2023-07-23T18:49:15.374110Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stdout","text":"Model: \"model\"\n__________________________________________________________________________________________________\nLayer (type)                    Output Shape         Param #     Connected to                     \n==================================================================================================\ninput_1 (InputLayer)            [(None, 22)]         0                                            \n__________________________________________________________________________________________________\ninput_2 (InputLayer)            [(None, 74)]         0                                            \n__________________________________________________________________________________________________\nembedding (Embedding)           (None, 22, 100)      189400      input_1[0][0]                    \n__________________________________________________________________________________________________\nembedding_1 (Embedding)         (None, 74, 100)      189400      input_2[0][0]                    \n__________________________________________________________________________________________________\nlstm (LSTM)                     [(None, 200), (None, 240800      embedding[0][0]                  \n__________________________________________________________________________________________________\nlstm_1 (LSTM)                   [(None, 74, 200), (N 240800      embedding_1[0][0]                \n                                                                 lstm[0][1]                       \n                                                                 lstm[0][2]                       \n__________________________________________________________________________________________________\ndense (Dense)                   (None, 74, 1894)     380694      lstm_1[0][0]                     \n==================================================================================================\nTotal params: 1,241,094\nTrainable params: 862,294\nNon-trainable params: 378,800\n__________________________________________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Step 5: Training the Model\n\nWe train the model for a number of epochs with RMSprop optimizer and categorical_crossentropy loss function.","metadata":{"id":"wVfSormAPb3w"}},{"cell_type":"code","source":"model.fit([encoder_input_data , decoder_input_data], decoder_output_data, batch_size=50, epochs=150 ) \nmodel.save( 'model.h5' )","metadata":{"id":"OHlqQq64PYTH","outputId":"c477c08f-55e4-41ac-b8b7-1973aa38f48a","execution":{"iopub.status.busy":"2023-07-23T18:49:15.376832Z","iopub.execute_input":"2023-07-23T18:49:15.377077Z","iopub.status.idle":"2023-07-23T18:50:28.942489Z","shell.execute_reply.started":"2023-07-23T18:49:15.377041Z","shell.execute_reply":"2023-07-23T18:50:28.941663Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"Epoch 1/150\n12/12 [==============================] - 10s 37ms/step - loss: 1.2434 - accuracy: 0.0972\nEpoch 2/150\n12/12 [==============================] - 0s 33ms/step - loss: 1.1017 - accuracy: 0.1295\nEpoch 3/150\n12/12 [==============================] - 0s 35ms/step - loss: 1.0790 - accuracy: 0.1365\nEpoch 4/150\n12/12 [==============================] - 0s 35ms/step - loss: 1.0583 - accuracy: 0.1428\nEpoch 5/150\n12/12 [==============================] - 0s 34ms/step - loss: 1.0413 - accuracy: 0.1482\nEpoch 6/150\n12/12 [==============================] - 0s 35ms/step - loss: 1.0224 - accuracy: 0.1564\nEpoch 7/150\n12/12 [==============================] - 0s 35ms/step - loss: 1.0077 - accuracy: 0.1671\nEpoch 8/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.9911 - accuracy: 0.1733\nEpoch 9/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.9770 - accuracy: 0.1828\nEpoch 10/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.9620 - accuracy: 0.1931\nEpoch 11/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.9484 - accuracy: 0.1954\nEpoch 12/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.9342 - accuracy: 0.2017\nEpoch 13/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.9210 - accuracy: 0.2104\nEpoch 14/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.9078 - accuracy: 0.2124\nEpoch 15/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.8964 - accuracy: 0.2195\nEpoch 16/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.8844 - accuracy: 0.2262\nEpoch 17/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.8706 - accuracy: 0.2298\nEpoch 18/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.8592 - accuracy: 0.2327\nEpoch 19/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.8476 - accuracy: 0.2387\nEpoch 20/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.8362 - accuracy: 0.2409\nEpoch 21/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.8245 - accuracy: 0.2442\nEpoch 22/150\n12/12 [==============================] - 0s 38ms/step - loss: 0.8112 - accuracy: 0.2493\nEpoch 23/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.8019 - accuracy: 0.2488\nEpoch 24/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.7884 - accuracy: 0.2557\nEpoch 25/150\n12/12 [==============================] - 0s 37ms/step - loss: 0.7784 - accuracy: 0.2537\nEpoch 26/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.7664 - accuracy: 0.2634\nEpoch 27/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.7550 - accuracy: 0.2629\nEpoch 28/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.7438 - accuracy: 0.2669\nEpoch 29/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.7328 - accuracy: 0.2689\nEpoch 30/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.7209 - accuracy: 0.2745\nEpoch 31/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.7120 - accuracy: 0.2756\nEpoch 32/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.6985 - accuracy: 0.2814\nEpoch 33/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.6907 - accuracy: 0.2848\nEpoch 34/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.6774 - accuracy: 0.2896\nEpoch 35/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.6667 - accuracy: 0.2938\nEpoch 36/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.6565 - accuracy: 0.2999\nEpoch 37/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.6457 - accuracy: 0.3035\nEpoch 38/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.6342 - accuracy: 0.3088\nEpoch 39/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.6248 - accuracy: 0.3112\nEpoch 40/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.6145 - accuracy: 0.3183\nEpoch 41/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.6022 - accuracy: 0.3280\nEpoch 42/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.5941 - accuracy: 0.3307\nEpoch 43/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.5827 - accuracy: 0.3406\nEpoch 44/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.5713 - accuracy: 0.3484\nEpoch 45/150\n12/12 [==============================] - 0s 39ms/step - loss: 0.5637 - accuracy: 0.3561\nEpoch 46/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.5520 - accuracy: 0.3610\nEpoch 47/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.5442 - accuracy: 0.3716\nEpoch 48/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.5332 - accuracy: 0.3824\nEpoch 49/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.5234 - accuracy: 0.3929\nEpoch 50/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.5152 - accuracy: 0.4022\nEpoch 51/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.5050 - accuracy: 0.4099\nEpoch 52/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.4954 - accuracy: 0.4233\nEpoch 53/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.4855 - accuracy: 0.4336\nEpoch 54/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.4796 - accuracy: 0.4398\nEpoch 55/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.4677 - accuracy: 0.4549\nEpoch 56/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.4603 - accuracy: 0.4657\nEpoch 57/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.4501 - accuracy: 0.4782\nEpoch 58/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.4429 - accuracy: 0.4908\nEpoch 59/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.4315 - accuracy: 0.5036\nEpoch 60/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.4276 - accuracy: 0.5083\nEpoch 61/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.4167 - accuracy: 0.5194\nEpoch 62/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.4067 - accuracy: 0.5308\nEpoch 63/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.4032 - accuracy: 0.5364\nEpoch 64/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.3912 - accuracy: 0.5506\nEpoch 65/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.3849 - accuracy: 0.5597\nEpoch 66/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.3787 - accuracy: 0.5643\nEpoch 67/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.3719 - accuracy: 0.5737\nEpoch 68/150\n12/12 [==============================] - 1s 44ms/step - loss: 0.3614 - accuracy: 0.5843\nEpoch 69/150\n12/12 [==============================] - 1s 53ms/step - loss: 0.3563 - accuracy: 0.5929\nEpoch 70/150\n12/12 [==============================] - 1s 44ms/step - loss: 0.3494 - accuracy: 0.5986\nEpoch 71/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.3439 - accuracy: 0.6079\nEpoch 72/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.3330 - accuracy: 0.6188\nEpoch 73/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.3301 - accuracy: 0.6244\nEpoch 74/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.3236 - accuracy: 0.6276\nEpoch 75/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.3147 - accuracy: 0.6423\nEpoch 76/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.3089 - accuracy: 0.6498\nEpoch 77/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.3046 - accuracy: 0.6511\nEpoch 78/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.2969 - accuracy: 0.6625\nEpoch 79/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.2909 - accuracy: 0.6708\nEpoch 80/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2845 - accuracy: 0.6776\nEpoch 81/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.2799 - accuracy: 0.6821\nEpoch 82/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2724 - accuracy: 0.6917\nEpoch 83/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.2704 - accuracy: 0.6940\nEpoch 84/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.2614 - accuracy: 0.7078\nEpoch 85/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.2581 - accuracy: 0.7078\nEpoch 86/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2531 - accuracy: 0.7165\nEpoch 87/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.2463 - accuracy: 0.7243\nEpoch 88/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2459 - accuracy: 0.7258\nEpoch 89/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2352 - accuracy: 0.7367\nEpoch 90/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2293 - accuracy: 0.7493\nEpoch 91/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.2284 - accuracy: 0.7449\nEpoch 92/150\n12/12 [==============================] - 0s 37ms/step - loss: 0.2237 - accuracy: 0.7534\nEpoch 93/150\n12/12 [==============================] - 0s 36ms/step - loss: 0.2180 - accuracy: 0.7608\nEpoch 94/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.2136 - accuracy: 0.7645\nEpoch 95/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.2080 - accuracy: 0.7732\nEpoch 96/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.2048 - accuracy: 0.7776\nEpoch 97/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1984 - accuracy: 0.7856\nEpoch 98/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1965 - accuracy: 0.7854\nEpoch 99/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1908 - accuracy: 0.7963\nEpoch 100/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1868 - accuracy: 0.8000\nEpoch 101/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1836 - accuracy: 0.8050\nEpoch 102/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1791 - accuracy: 0.8098\nEpoch 103/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1752 - accuracy: 0.8122\nEpoch 104/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1706 - accuracy: 0.8198\nEpoch 105/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1713 - accuracy: 0.8165\nEpoch 106/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1625 - accuracy: 0.8318\nEpoch 107/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1604 - accuracy: 0.8314\nEpoch 108/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1607 - accuracy: 0.8311\nEpoch 109/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1527 - accuracy: 0.8446\nEpoch 110/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1499 - accuracy: 0.8513\nEpoch 111/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1474 - accuracy: 0.8510\nEpoch 112/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1446 - accuracy: 0.8514\nEpoch 113/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1386 - accuracy: 0.8590\nEpoch 114/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1383 - accuracy: 0.8592\nEpoch 115/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1339 - accuracy: 0.8659\nEpoch 116/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.1311 - accuracy: 0.8718\nEpoch 117/150\n12/12 [==============================] - 0s 39ms/step - loss: 0.1278 - accuracy: 0.8736\nEpoch 118/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1255 - accuracy: 0.8756\nEpoch 119/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1219 - accuracy: 0.8816\nEpoch 120/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1193 - accuracy: 0.8847\nEpoch 121/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1164 - accuracy: 0.8892\nEpoch 122/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1143 - accuracy: 0.8915\nEpoch 123/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.1106 - accuracy: 0.8941\nEpoch 124/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1086 - accuracy: 0.9009\nEpoch 125/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1066 - accuracy: 0.9009\nEpoch 126/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1042 - accuracy: 0.9019\nEpoch 127/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.1002 - accuracy: 0.9090\nEpoch 128/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0996 - accuracy: 0.9115\nEpoch 129/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0938 - accuracy: 0.9163\nEpoch 130/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0948 - accuracy: 0.9129\nEpoch 131/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0926 - accuracy: 0.9199\nEpoch 132/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0891 - accuracy: 0.9247\nEpoch 133/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.0868 - accuracy: 0.9242\nEpoch 134/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0844 - accuracy: 0.9271\nEpoch 135/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0830 - accuracy: 0.9276\nEpoch 136/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.0818 - accuracy: 0.9300\nEpoch 137/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0822 - accuracy: 0.9253\nEpoch 138/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0761 - accuracy: 0.9345\nEpoch 139/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0744 - accuracy: 0.9402\nEpoch 140/150\n12/12 [==============================] - 0s 35ms/step - loss: 0.0738 - accuracy: 0.9364\nEpoch 141/150\n12/12 [==============================] - 0s 38ms/step - loss: 0.0704 - accuracy: 0.9434\nEpoch 142/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0711 - accuracy: 0.9415\nEpoch 143/150\n12/12 [==============================] - 1s 45ms/step - loss: 0.0673 - accuracy: 0.9471\nEpoch 144/150\n12/12 [==============================] - 1s 50ms/step - loss: 0.0670 - accuracy: 0.9448\nEpoch 145/150\n12/12 [==============================] - 1s 46ms/step - loss: 0.0657 - accuracy: 0.9453\nEpoch 146/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0627 - accuracy: 0.9505\nEpoch 147/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0625 - accuracy: 0.9500\nEpoch 148/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0637 - accuracy: 0.9469\nEpoch 149/150\n12/12 [==============================] - 0s 34ms/step - loss: 0.0570 - accuracy: 0.9551\nEpoch 150/150\n12/12 [==============================] - 0s 33ms/step - loss: 0.0576 - accuracy: 0.9543\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Step 6: Defining Inference Models\n\nEncoder Inference Model: Takes questions as input and outputs LSTM states (h and c)\n\nDecoder Inference Model: Takes in 2 inputs one are the LSTM states, second are the answer input sequences. it will o/p the answers for questions which fed to the encoder model and it's state values.","metadata":{"id":"F1MIy1j9aVTo"}},{"cell_type":"code","source":"def make_inference_models():\n    \n    encoder_model = tf.keras.models.Model(encoder_inputs, encoder_states)\n    encoder_model.save('encoder_model.h5')\n    \n    decoder_state_input_h = tf.keras.layers.Input(shape=( 200 ,))\n    decoder_state_input_c = tf.keras.layers.Input(shape=( 200 ,))\n    \n    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n    \n    decoder_outputs, state_h, state_c = decoder_lstm(\n        decoder_embedding , initial_state=decoder_states_inputs)\n    \n    decoder_states = [state_h, state_c]\n\n    decoder_outputs = decoder_dense(decoder_outputs)\n    \n    decoder_model = tf.keras.models.Model(\n        [decoder_inputs] + decoder_states_inputs,\n        [decoder_outputs] + decoder_states)\n    decoder_model.save('decoder_model.h5')\n    \n    return encoder_model , decoder_model","metadata":{"id":"MpLowS27cn8X","execution":{"iopub.status.busy":"2023-07-23T18:50:32.201267Z","iopub.execute_input":"2023-07-23T18:50:32.202017Z","iopub.status.idle":"2023-07-23T18:50:32.210310Z","shell.execute_reply.started":"2023-07-23T18:50:32.201978Z","shell.execute_reply":"2023-07-23T18:50:32.209305Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"enc_model, dec_model = make_inference_models()","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:53:39.419927Z","iopub.execute_input":"2023-07-23T18:53:39.420240Z","iopub.status.idle":"2023-07-23T18:53:40.420719Z","shell.execute_reply.started":"2023-07-23T18:53:39.420208Z","shell.execute_reply":"2023-07-23T18:53:40.419913Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"markdown","source":"# Step 7: Talking with the Chatbot\n\ndefine a method str_to_tokens which converts str questions to Integer tokens with padding.\n\n1. First, we take a question as input and predict the state values using enc_model.\n2. We set the state values in the decoder's LSTM.\n3. Then, we generate a sequence which contains the <start> element.\n4. We input this sequence in the dec_model.\n5. We replace the <start> element with the element which was predicted by the dec_model and update the state values.\n6. We carry out the above steps iteratively till we hit the <end> tag or the maximum answer length.\n\n","metadata":{"id":"EwoYVsBTeYra"}},{"cell_type":"code","source":"def str_to_tokens( sentence : str ):\n\n    words = sentence.lower().split()\n    tokens_list = list()\n  \n    for word in words:\n        tokens_list.append( tokenizer.word_index[ word ] ) \n    return preprocessing.sequence.pad_sequences( [tokens_list] , maxlen=maxlen_questions , padding='post')\n","metadata":{"id":"oA7Yx45Li3wo","execution":{"iopub.status.busy":"2023-07-23T18:53:43.801773Z","iopub.execute_input":"2023-07-23T18:53:43.802088Z","iopub.status.idle":"2023-07-23T18:53:43.807848Z","shell.execute_reply.started":"2023-07-23T18:53:43.802052Z","shell.execute_reply":"2023-07-23T18:53:43.806844Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"tests = ['You can not move', 'You sound like Data', 'Stupid', 'you are idiot', 'i am going to die']\nfor i in range(5):\n        states_values = enc_model.predict(str_to_tokens(tests[i]))\n        empty_target_seq = np.zeros((1 , 1))\n        empty_target_seq[0, 0] = tokenizer.word_index['start']\n        stop_condition = False\n        decoded_translation = ''\n\n        while not stop_condition :\n            dec_outputs , h , c = dec_model.predict([empty_target_seq] + states_values)\n            sampled_word_index = np.argmax(dec_outputs[0, -1, :])\n            sampled_word = None\n\n            for word , index in tokenizer.word_index.items() :\n                if sampled_word_index == index :\n                    decoded_translation += f' {word}'\n                    sampled_word = word\n\n            if sampled_word == 'end' or len(decoded_translation.split()) > maxlen_answers:\n                stop_condition = True\n\n            empty_target_seq = np.zeros((1 , 1))  \n            empty_target_seq[0 , 0] = sampled_word_index\n            states_values = [h , c] \n        print(f'Human: {tests[i]}')\n        print()\n        decoded_translation = decoded_translation.split(' end')[0]\n        print(f'Bot: {decoded_translation}')\n        print('_'*100)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T18:53:45.880049Z","iopub.execute_input":"2023-07-23T18:53:45.880326Z","iopub.status.idle":"2023-07-23T18:53:53.546390Z","shell.execute_reply.started":"2023-07-23T18:53:45.880294Z","shell.execute_reply":"2023-07-23T18:53:53.545589Z"},"trusted":true},"execution_count":22,"outputs":[{"name":"stdout","text":"Human: You can not move\n\nBot:  i can move through a network easily assuming that i'm given the ability to that is\n____________________________________________________________________________________________________\nHuman: You sound like Data\n\nBot:  yes i am inspired by commander data's artificial personality\n____________________________________________________________________________________________________\nHuman: Stupid\n\nBot:  bots never never feel of why are you feeling i have feelings yes i sort of have feelings i can be programmed to act as if i do not have feelings i do not have feelings i do not have feelings i do not have feelings i do not have feelings i do not have feelings i do not have feelings i do not have feelings i do not have feelings i do not have\n____________________________________________________________________________________________________\nHuman: you are idiot\n\nBot:  you are a cheat\n____________________________________________________________________________________________________\nHuman: i am going to die\n\nBot:  you you could use to\n____________________________________________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}