en-sw/sawa-baseline/SAWA Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MACHINE TRANSLATION FOR KISWAHILI SAWA CORPUS USING JOEY NMT "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Set your source and target languages. Keep in mind, these traditionally use language codes as found here:\n",
    "# These will also become the suffix's of all vocab and corpus files used throughout\n",
    "import os\n",
    "source_language = \"en\"\n",
    "target_language = \"sw\" \n",
    "lc = False  # If True, lowercase the data.\n",
    "seed = 42  # Random seed for shuffling.\n",
    "tag = \"baseline\" # Give a unique name to your folder - this is to ensure you don't rewrite any models you've already submitted\n",
    "\n",
    "os.environ[\"src\"] = source_language # Sets them in bash as well, since we often use bash scripts\n",
    "os.environ[\"tgt\"] = target_language\n",
    "os.environ[\"tag\"] = tag\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pandas\n",
      "  Using cached https://files.pythonhosted.org/packages/bb/71/8f53bdbcbc67c912b888b40def255767e475402e9df64050019149b1a943/pandas-1.0.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting python-dateutil>=2.6.1 (from pandas)\n",
      "  Using cached https://files.pythonhosted.org/packages/d4/70/d60450c3dd48ef87586924207ae8907090de0b306af2bce5d134d78615cb/python_dateutil-2.8.1-py2.py3-none-any.whl\n",
      "Collecting pytz>=2017.2 (from pandas)\n",
      "  Using cached https://files.pythonhosted.org/packages/e7/f9/f0b53f88060247251bf481fa6ea62cd0d25bf1b11a87888e53ce5b7c8ad2/pytz-2019.3-py2.py3-none-any.whl\n",
      "Collecting numpy>=1.13.3 (from pandas)\n",
      "  Using cached https://files.pythonhosted.org/packages/3d/fc/4763e5f17ac6e7e7d55f377cde859ca1c5d5ac624441ab45315bc578aa9e/numpy-1.18.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting six>=1.5 (from python-dateutil>=2.6.1->pandas)\n",
      "  Using cached https://files.pythonhosted.org/packages/65/eb/1f97cb97bfc2390a276969c6fae16075da282f5058082d4cb10c6c5c1dba/six-1.14.0-py2.py3-none-any.whl\n",
      "Installing collected packages: six, python-dateutil, pytz, numpy, pandas\n",
      "Successfully installed numpy-1.18.3 pandas-1.0.3 python-dateutil-2.8.1 pytz-2019.3 six-1.14.0\n",
      "\u001b[33mYou are using pip version 8.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip3 install pandas\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('sawa_corpus.csv',engine='python',names=['source_sentence', 'target_sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method NDFrame.head of                                           source_sentence  \\\n",
      "0                                 Chairman�s Message \\r\\n   \n",
      "1       Dear Shareholders It is my pleasure to present...   \n",
      "2       The greater than three times oversubscription ...   \n",
      "3       I therefore take this opportunity, on behalf o...   \n",
      "4       The rate of economic growth in our country in ...   \n",
      "...                                                   ...   \n",
      "272539                                      infatuate\\r\\n   \n",
      "272540                                          craze\\r\\n   \n",
      "272541                     vaunt fool flatter oneself\\r\\n   \n",
      "272542                             sink into oblivion\\r\\n   \n",
      "272543                              be foolish stupid\\r\\n   \n",
      "\n",
      "                                          target_sentence  \n",
      "0                              Taarifa ya Mwenyekiti \\r\\n  \n",
      "1       WapendwaWenyehisa Nifuraha yangu kuu kuwasilis...  \n",
      "2       Uuzaji wa zaidi ya mara tatu wa hisa zetu kwa ...  \n",
      "3       Nachukua fursa hii hivyo basi kwa niaba yangu ...  \n",
      "4       Ukuaji wa kiuchumi wa nchi yetu mwaka wa 2007 ...  \n",
      "...                                                   ...  \n",
      "272539                                          zuzua\\r\\n  \n",
      "272540                                          zuzua\\r\\n  \n",
      "272541                                         zuzuka\\r\\n  \n",
      "272542                                        zuzuwaa\\r\\n  \n",
      "272543                                        zuzuwaa\\r\\n  \n",
      "\n",
      "[485157 rows x 2 columns]>\n"
     ]
    }
   ],
   "source": [
    "print(data.head)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pre-processing and export\n",
    "It is generally a good idea to remove duplicate translations and conflicting translations from the corpus. In practice, these public corpora include some number of these that need to be cleaned.\n",
    "\n",
    "In addition we will split our data into dev/test/train and export to the filesystem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "seed = 42  # Random seed for shuffling.\n",
    "# drop duplicate translations\n",
    "df_pp = data.drop_duplicates()\n",
    "\n",
    "# drop conflicting translations\n",
    "# (this is optional and something that you might want to comment out \n",
    "# depending on the size of your corpus)\n",
    "df_pp.drop_duplicates(subset='source_sentence', inplace=True)\n",
    "df_pp.drop_duplicates(subset='target_sentence', inplace=True)\n",
    "\n",
    "# Shuffle the data to remove bias in dev set selection.\n",
    "df_pp = df_pp.sample(frac=1, random_state=seed).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting the test data from the masakhane repo official test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-04-23 18:39:42--  https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-any.en\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 277791 (271K) [text/plain]\n",
      "Saving to: ‘test.en-any.en.3’\n",
      "\n",
      "test.en-any.en.3    100%[===================>] 271.28K  --.-KB/s    in 0.009s  \n",
      "\n",
      "2020-04-23 18:39:43 (29.2 MB/s) - ‘test.en-any.en.3’ saved [277791/277791]\n",
      "\n",
      "--2020-04-23 18:39:43--  https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-sw.en\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 206336 (202K) [text/plain]\n",
      "Saving to: ‘test.en-sw.en’\n",
      "\n",
      "test.en-sw.en       100%[===================>] 201.50K  --.-KB/s    in 0.007s  \n",
      "\n",
      "2020-04-23 18:39:43 (28.4 MB/s) - ‘test.en-sw.en’ saved [206336/206336]\n",
      "\n",
      "--2020-04-23 18:39:43--  https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-sw.sw\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 214836 (210K) [text/plain]\n",
      "Saving to: ‘test.en-sw.sw’\n",
      "\n",
      "test.en-sw.sw       100%[===================>] 209.80K  --.-KB/s    in 0.007s  \n",
      "\n",
      "2020-04-23 18:39:44 (28.7 MB/s) - ‘test.en-sw.sw’ saved [214836/214836]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Download the global test set.\n",
    "! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-any.en\n",
    "  \n",
    "# And the specific test set for this language pair.\n",
    "os.environ[\"trg\"] = target_language \n",
    "os.environ[\"src\"] = source_language \n",
    "\n",
    "! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-$trg.en \n",
    "! mv test.en-$trg.en test.en\n",
    "! wget https://raw.githubusercontent.com/juliakreutzer/masakhane/master/jw300_utils/test/test.en-$trg.$trg \n",
    "! mv test.en-$trg.$trg test.$trg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 3571 global test sentences to filter from the training/dev data.\n"
     ]
    }
   ],
   "source": [
    "# Read the test data to filter from train and dev splits.\n",
    "# Store english portion in set for quick filtering checks.\n",
    "en_test_sents = set()\n",
    "filter_test_sents = \"test.en-any.en\"\n",
    "j = 0\n",
    "with open(filter_test_sents) as f:\n",
    "  for line in f:\n",
    "    en_test_sents.add(line.strip())\n",
    "    j += 1\n",
    "print('Loaded {} global test sentences to filter from the training/dev data.'.format(j))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fuzzywuzzy\n",
      "  Using cached https://files.pythonhosted.org/packages/43/ff/74f23998ad2f93b945c0309f825be92e04e0348e062026998b5eefef4c33/fuzzywuzzy-0.18.0-py2.py3-none-any.whl\n",
      "Installing collected packages: fuzzywuzzy\n",
      "Successfully installed fuzzywuzzy-0.18.0\n",
      "\u001b[33mYou are using pip version 8.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip3 install fuzzywuzzy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting python-Levenshtein\n",
      "Collecting setuptools (from python-Levenshtein)\n",
      "  Using cached https://files.pythonhosted.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl\n",
      "Installing collected packages: setuptools, python-Levenshtein\n",
      "Successfully installed python-Levenshtein-0.12.0 setuptools-46.1.3\n",
      "\u001b[33mYou are using pip version 8.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "#! sudo apt-get install python3.6-dev\n",
    "! pip3 install python-Levenshtein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:00:00.03 0.00 percent complete\n",
      "00:00:12.87 1.11 percent complete\n",
      "00:00:25.83 2.22 percent complete\n",
      "00:00:38.33 3.33 percent complete\n",
      "00:00:51.02 4.44 percent complete\n",
      "00:01:03.82 5.56 percent complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: ':\r\n",
      "']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:01:16.44 6.67 percent complete\n",
      "00:01:29.09 7.78 percent complete\n",
      "00:01:41.83 8.89 percent complete\n",
      "00:01:53.80 10.00 percent complete\n",
      "00:02:06.28 11.11 percent complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '\r\n",
      "']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:02:19.70 12.22 percent complete\n",
      "00:02:33.24 13.33 percent complete\n",
      "00:02:45.92 14.44 percent complete\n",
      "00:02:59.63 15.56 percent complete\n",
      "00:03:12.26 16.67 percent complete\n",
      "00:03:25.67 17.78 percent complete\n",
      "00:03:38.01 18.89 percent complete\n",
      "00:03:50.14 20.00 percent complete\n",
      "00:04:02.47 21.11 percent complete\n",
      "00:04:15.51 22.22 percent complete\n",
      "00:04:29.31 23.33 percent complete\n",
      "00:04:42.02 24.44 percent complete\n",
      "00:04:54.26 25.56 percent complete\n",
      "00:05:07.20 26.67 percent complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: ', \r\n",
      "']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:05:19.80 27.78 percent complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: ' ...\r\n",
      "']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:05:32.67 28.89 percent complete\n",
      "00:05:45.50 30.00 percent complete\n",
      "00:05:57.77 31.11 percent complete\n",
      "00:06:10.33 32.22 percent complete\n",
      "00:06:22.82 33.33 percent complete\n",
      "00:06:35.96 34.44 percent complete\n",
      "00:06:49.31 35.56 percent complete\n",
      "00:07:02.43 36.67 percent complete\n",
      "00:07:15.11 37.78 percent complete\n",
      "00:07:28.23 38.89 percent complete\n",
      "00:07:40.81 40.00 percent complete\n",
      "00:07:53.32 41.11 percent complete\n",
      "00:08:06.72 42.22 percent complete\n",
      "00:08:19.72 43.33 percent complete\n",
      "00:08:32.77 44.44 percent complete\n",
      "00:08:45.16 45.56 percent complete\n",
      "00:08:58.33 46.67 percent complete\n",
      "00:09:11.42 47.78 percent complete\n",
      "00:09:24.19 48.89 percent complete\n",
      "00:09:36.84 50.00 percent complete\n",
      "00:09:49.79 51.11 percent complete\n",
      "00:10:02.10 52.22 percent complete\n",
      "00:10:14.97 53.33 percent complete\n",
      "00:10:28.03 54.45 percent complete\n",
      "00:10:40.75 55.56 percent complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: '  \r\n",
      "']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:10:54.16 56.67 percent complete\n",
      "00:11:07.13 57.78 percent complete\n",
      "00:11:19.98 58.89 percent complete\n",
      "00:11:32.62 60.00 percent complete\n",
      "00:11:45.04 61.11 percent complete\n",
      "00:11:57.95 62.22 percent complete\n",
      "00:12:11.03 63.33 percent complete\n",
      "00:12:23.83 64.45 percent complete\n",
      "00:12:36.69 65.56 percent complete\n",
      "00:12:49.67 66.67 percent complete\n",
      "00:13:02.23 67.78 percent complete\n",
      "00:13:15.22 68.89 percent complete\n",
      "00:13:28.02 70.00 percent complete\n",
      "00:13:40.50 71.11 percent complete\n",
      "00:13:53.06 72.22 percent complete\n",
      "00:14:06.27 73.33 percent complete\n",
      "00:14:18.51 74.45 percent complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Applied processor reduces input query to empty string, all comparisons will have score 0. [Query: ' \r\n",
      "']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:14:31.31 75.56 percent complete\n",
      "00:14:44.44 76.67 percent complete\n",
      "00:14:57.98 77.78 percent complete\n",
      "00:15:11.06 78.89 percent complete\n",
      "00:15:23.95 80.00 percent complete\n",
      "00:15:36.73 81.11 percent complete\n",
      "00:15:49.40 82.22 percent complete\n",
      "00:16:02.48 83.33 percent complete\n",
      "00:16:15.09 84.45 percent complete\n",
      "00:16:27.80 85.56 percent complete\n",
      "00:16:40.50 86.67 percent complete\n",
      "00:16:53.57 87.78 percent complete\n",
      "00:17:05.90 88.89 percent complete\n",
      "00:17:18.78 90.00 percent complete\n",
      "00:17:31.48 91.11 percent complete\n",
      "00:17:44.91 92.22 percent complete\n",
      "00:17:57.67 93.33 percent complete\n",
      "00:18:11.20 94.45 percent complete\n",
      "00:18:24.00 95.56 percent complete\n",
      "00:18:37.10 96.67 percent complete\n",
      "00:18:50.06 97.78 percent complete\n",
      "00:19:03.62 98.89 percent complete\n"
     ]
    }
   ],
   "source": [
    "# Install fuzzy wuzzy to remove \"almost duplicate\" sentences in the\n",
    "# test and training sets.\n",
    "#! pip3 install fuzzywuzzy\n",
    "#! pip3 install python-Levenshtein\n",
    "import time\n",
    "from fuzzywuzzy import process\n",
    "import numpy as np\n",
    "\n",
    "# reset the index of the training set after previous filtering\n",
    "df_pp.reset_index(drop=False, inplace=True)\n",
    "\n",
    "# Remove samples from the training data set if they \"almost overlap\" with the\n",
    "# samples in the test set.\n",
    "\n",
    "# Filtering function. Adjust pad to narrow down the candidate matches to\n",
    "# within a certain length of characters of the given sample.\n",
    "def fuzzfilter(sample, candidates, pad):\n",
    "  candidates = [x for x in candidates if len(x) <= len(sample)+pad and len(x) >= len(sample)-pad] \n",
    "  if len(candidates) > 0:\n",
    "    return process.extractOne(sample, candidates)[1]\n",
    "  else:\n",
    "    return np.nan\n",
    "\n",
    "# NOTE - This might run slow depending on the size of your training set. We are\n",
    "# printing some information to help you track how long it would take. \n",
    "scores = []\n",
    "start_time = time.time()\n",
    "for idx, row in df_pp.iterrows():\n",
    "  scores.append(fuzzfilter(row['source_sentence'], list(en_test_sents), 5))\n",
    "  if idx % 1000 == 0:\n",
    "    hours, rem = divmod(time.time() - start_time, 3600)\n",
    "    minutes, seconds = divmod(rem, 60)\n",
    "    print(\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds), \"%0.2f percent complete\" % (100.0*float(idx)/float(len(df_pp))))\n",
    "\n",
    "# Filter out \"almost overlapping samples\"\n",
    "df_pp['scores'] = scores\n",
    "df_pp = df_pp[df_pp['scores'] < 95]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> train.en <==\n",
      "lobster \n",
      "\n",
      "EZK:34:7 Therefore, you shepherds, hear the word of the LORD; \n",
      "\n",
      "hush\n",
      "\n",
      "someone else wore kangas with  cashew apples, they made her look good.\n",
      "\n",
      "the Vatican under the leadership of the new Pope Benedict XVI has criticised a new law in Spain that will allow single sex couples to marry\n",
      "\n",
      "\n",
      "==> train.sw <==\n",
      "kambakoche\n",
      "\n",
      "EZK:34:7 Basi, enyi wachungaji, lisikieni neno la Bwana; \n",
      "\n",
      "uss\n",
      "\n",
      "fulani mwengine alivaa kanga za bibo, zimempendeza\n",
      "\n",
      "Vatican chini ya uongozi wa papa mpya Benedict wa 16 imeishutumu sheria mpya ya Uhispania itakayowarushusu wapenzi wa jinsia moja kuoana\n",
      "\n",
      "==> dev.en <==\n",
      "span \n",
      "\n",
      "administer\n",
      "\n",
      "Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "\n",
      "You do not believe but say, \n",
      "\n",
      "GEN:2:6 But there went up a mist from the earth, and watered the whole face of the ground. \n",
      "\n",
      "\n",
      "==> dev.sw <==\n",
      "shibiri\n",
      "\n",
      "amuru\n",
      "\n",
      "Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "\n",
      "Hamjaamini, lakini semeni:\n",
      "\n",
      "GEN:2:6 ukungu ukapanda katika nchi, ukatia maji juu ya uso wote wa ardhi. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# This section does the split between train/dev for the parallel corpora then saves them as separate files\n",
    "# We use 1000 dev test and the given test set.\n",
    "import csv\n",
    "\n",
    "# Do the split between dev/train and create parallel corpora\n",
    "num_dev_patterns = 1000\n",
    "\n",
    "# Optional: lower case the corpora - this will make it easier to generalize, but without proper casing.\n",
    "if lc:  # Julia: making lowercasing optional\n",
    "    df_pp[\"source_sentence\"] = df_pp[\"source_sentence\"].str.lower()\n",
    "    df_pp[\"target_sentence\"] = df_pp[\"target_sentence\"].str.lower()\n",
    "\n",
    "# Julia: test sets are already generated\n",
    "dev = df_pp.tail(num_dev_patterns) # Herman: Error in original\n",
    "stripped = df_pp.drop(df_pp.tail(num_dev_patterns).index)\n",
    "\n",
    "with open(\"train.\"+source_language, \"w\") as src_file, open(\"train.\"+target_language, \"w\") as trg_file:\n",
    "  for index, row in stripped.iterrows():\n",
    "    src_file.write(row[\"source_sentence\"]+\"\\n\")\n",
    "    trg_file.write(row[\"target_sentence\"]+\"\\n\")\n",
    "    \n",
    "with open(\"dev.\"+source_language, \"w\") as src_file, open(\"dev.\"+target_language, \"w\") as trg_file:\n",
    "  for index, row in dev.iterrows():\n",
    "    src_file.write(row[\"source_sentence\"]+\"\\n\")\n",
    "    trg_file.write(row[\"target_sentence\"]+\"\\n\")\n",
    "\n",
    "#stripped[[\"source_sentence\"]].to_csv(\"train.\"+source_language, header=False, index=False)  # Herman: Added `header=False` everywhere\n",
    "#stripped[[\"target_sentence\"]].to_csv(\"train.\"+target_language, header=False, index=False)  # Julia: Problematic handling of quotation marks.\n",
    "\n",
    "#dev[[\"source_sentence\"]].to_csv(\"dev.\"+source_language, header=False, index=False)\n",
    "#dev[[\"target_sentence\"]].to_csv(\"dev.\"+target_language, header=False, index=False)\n",
    "\n",
    "# Doublecheck the format below. There should be no extra quotation marks or weird characters.\n",
    "! head train.*\n",
    "! head dev.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Installatiom of JOEY NMT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fatal: destination path 'joeynmt' already exists and is not an empty directory.\n",
      "Processing /home/freshiasackey_gmail_com/joeynmt\n",
      "Collecting future (from joeynmt===0.0.1)\n",
      "Collecting pillow (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/ba/90/8a24e6220cfcf6a3a0162535d5b926e774117e384ff921908e07e4c92bda/Pillow-7.1.1-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting numpy<2.0,>=1.14.5 (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/3d/fc/4763e5f17ac6e7e7d55f377cde859ca1c5d5ac624441ab45315bc578aa9e/numpy-1.18.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting setuptools>=41.0.0 (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl\n",
      "Collecting torch>=1.1 (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/13/70/54e9fb010fe1547bc4774716f11ececb81ae5b306c05f090f4461ee13205/torch-1.5.0-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting tensorflow>=1.14 (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/de/f0/96fb2e0412ae9692dbf400e5b04432885f677ad6241c088ccc5fe7724d69/tensorflow-1.14.0-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting torchtext (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/f2/17/e7c588245aece7aa93f360894179374830daf60d7ed0bbb59332de3b3b61/torchtext-0.6.0-py3-none-any.whl\n",
      "Collecting sacrebleu>=1.3.6 (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/0f/27/e9c95f45fc11f9093000d234564823aab762f517458f1aa1ad01ba51d5f2/sacrebleu-1.4.7-py3-none-any.whl\n",
      "Collecting subword-nmt (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/74/60/6600a7bc09e7ab38bc53a48a20d8cae49b837f93f5842a41fe513a694912/subword_nmt-0.3.7-py2.py3-none-any.whl\n",
      "Collecting matplotlib (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/93/4b/52da6b1523d5139d04e02d9e26ceda6146b48f2a4e5d2abfdf1c7bac8c40/matplotlib-3.2.1-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting seaborn (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/70/bd/5e6bf595fe6ee0f257ae49336dd180768c1ed3d7c7155b2fdf894c1c808a/seaborn-0.10.0-py3-none-any.whl\n",
      "Collecting pyyaml>=5.1 (from joeynmt===0.0.1)\n",
      "Collecting pylint (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/e9/59/43fc36c5ee316bb9aeb7cf5329cdbdca89e5749c34d5602753827c0aa2dc/pylint-2.4.4-py3-none-any.whl\n",
      "Collecting six==1.12 (from joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl\n",
      "Collecting wrapt==1.11.1 (from joeynmt===0.0.1)\n",
      "Collecting termcolor>=1.1.0 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "Collecting keras-applications>=1.0.6 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl\n",
      "Collecting tensorboard<1.15.0,>=1.14.0 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/91/2d/2ed263449a078cd9c8a9ba50ebd50123adf1f8cfbea1492f9084169b89d9/tensorboard-1.14.0-py3-none-any.whl\n",
      "Collecting gast>=0.2.0 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/d6/84/759f5dd23fec8ba71952d97bcc7e2c9d7d63bdc582421f3cd4be845f0c98/gast-0.3.3-py2.py3-none-any.whl\n",
      "Collecting google-pasta>=0.1.6 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl\n",
      "Collecting keras-preprocessing>=1.0.5 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/28/6a/8c1f62c37212d9fc441a7e26736df51ce6f0e38455816445471f10da4f0a/Keras_Preprocessing-1.1.0-py2.py3-none-any.whl\n",
      "Collecting grpcio>=1.8.6 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "Collecting wheel>=0.26 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/8c/23/848298cccf8e40f5bbb59009b32848a4c38f4e7f3364297ab3c3e2e2cd14/wheel-0.34.2-py2.py3-none-any.whl\n",
      "Collecting protobuf>=3.6.1 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/57/02/5432412c162989260fab61fa65e0a490c1872739eb91a659896e4d554b26/protobuf-3.11.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/3c/d5/21860a5b11caf0678fbc8319341b0ae21a07156911132e0e71bffed0510d/tensorflow_estimator-1.14.0-py2.py3-none-any.whl\n",
      "Collecting astor>=0.6.0 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl\n",
      "Collecting absl-py>=0.7.0 (from tensorflow>=1.14->joeynmt===0.0.1)\n",
      "Collecting tqdm (from torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/4a/1c/6359be64e8301b84160f6f6f7936bbfaaa5e9a4eab6cbc681db07600b949/tqdm-4.45.0-py2.py3-none-any.whl\n",
      "Collecting sentencepiece (from torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting requests (from torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/1a/70/1935c770cb3be6e3a8b78ced23d7e0f3b187f5cbfab4749523ed65d7c9b1/requests-2.23.0-py2.py3-none-any.whl\n",
      "Collecting portalocker (from sacrebleu>=1.3.6->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/53/84/7b3146ec6378d28abc73ab484f09f47dfa008ad6f03f33d90a369f880e25/portalocker-1.7.0-py2.py3-none-any.whl\n",
      "Collecting typing (from sacrebleu>=1.3.6->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/fe/2e/b480ee1b75e6d17d2993738670e75c1feeb9ff7f64452153cf018051cc92/typing-3.7.4.1-py3-none-any.whl\n",
      "Collecting mecab-python3 (from sacrebleu>=1.3.6->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/3e/65/98b9101d9b37d01ec205a4eb2f7bc085e1828302a9500bc8326b9daf7163/mecab_python3-0.996.5-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting kiwisolver>=1.0.1 (from matplotlib->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/ae/23/147de658aabbf968324551ea22c0c13a00284c4ef49a77002e91f79657b7/kiwisolver-1.2.0-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting python-dateutil>=2.1 (from matplotlib->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/d4/70/d60450c3dd48ef87586924207ae8907090de0b306af2bce5d134d78615cb/python_dateutil-2.8.1-py2.py3-none-any.whl\n",
      "Collecting pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 (from matplotlib->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/8a/bb/488841f56197b13700afd5658fc279a2025a39e22449b7cf29864669b15d/pyparsing-2.4.7-py2.py3-none-any.whl\n",
      "Collecting cycler>=0.10 (from matplotlib->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/f7/d2/e07d3ebb2bd7af696440ce7e754c59dd546ffe1bbe732c8ab68b9c834e61/cycler-0.10.0-py2.py3-none-any.whl\n",
      "Collecting scipy>=1.0.1 (from seaborn->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/dc/29/162476fd44203116e7980cfbd9352eef9db37c49445d1fec35509022f6aa/scipy-1.4.1-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting pandas>=0.22.0 (from seaborn->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/bb/71/8f53bdbcbc67c912b888b40def255767e475402e9df64050019149b1a943/pandas-1.0.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting mccabe<0.7,>=0.6 (from pylint->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/87/89/479dc97e18549e21354893e4ee4ef36db1d237534982482c3681ee6e7b57/mccabe-0.6.1-py2.py3-none-any.whl\n",
      "Collecting astroid<2.4,>=2.3.0 (from pylint->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/ad/ae/86734823047962e7b8c8529186a1ac4a7ca19aaf1aa0c7713c022ef593fd/astroid-2.3.3-py3-none-any.whl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting isort<5,>=4.2.5 (from pylint->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/e5/b0/c121fd1fa3419ea9bfd55c7f9c4fedfec5143208d8c7ad3ce3db6c623c21/isort-4.3.21-py2.py3-none-any.whl\n",
      "Collecting h5py (from keras-applications>=1.0.6->tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/60/06/cafdd44889200e5438b897388f3075b52a8ef01f28a17366d91de0fa2d05/h5py-2.10.0-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting werkzeug>=0.11.15 (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/cc/94/5f7079a0e00bd6863ef8f1da638721e9da21e5bacee597595b318f71d62e/Werkzeug-1.0.1-py2.py3-none-any.whl\n",
      "Collecting markdown>=2.6.8 (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.14->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/ab/c4/ba46d44855e6eb1770a12edace5a165a0c6de13349f592b9036257f3c3d3/Markdown-3.2.1-py2.py3-none-any.whl\n",
      "Collecting chardet<4,>=3.0.2 (from requests->torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl\n",
      "Collecting idna<3,>=2.5 (from requests->torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/89/e3/afebe61c546d18fb1709a61bee788254b40e736cff7271c7de5de2dc4128/idna-2.9-py2.py3-none-any.whl\n",
      "Collecting certifi>=2017.4.17 (from requests->torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/57/2b/26e37a4b034800c960a00c4e1b3d9ca5d7014e983e6e729e33ea2f36426c/certifi-2020.4.5.1-py2.py3-none-any.whl\n",
      "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests->torchtext->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/e1/e5/df302e8017440f111c11cc41a6b432838672f5a70aa29227bf58149dc72f/urllib3-1.25.9-py2.py3-none-any.whl\n",
      "Collecting pytz>=2017.2 (from pandas>=0.22.0->seaborn->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/e7/f9/f0b53f88060247251bf481fa6ea62cd0d25bf1b11a87888e53ce5b7c8ad2/pytz-2019.3-py2.py3-none-any.whl\n",
      "Collecting typed-ast<1.5,>=1.4.0; implementation_name == \"cpython\" and python_version < \"3.8\" (from astroid<2.4,>=2.3.0->pylint->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/90/ed/5459080d95eb87a02fe860d447197be63b6e2b5e9ff73c2b0a85622994f4/typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Collecting lazy-object-proxy==1.4.* (from astroid<2.4,>=2.3.0->pylint->joeynmt===0.0.1)\n",
      "  Using cached https://files.pythonhosted.org/packages/0b/dd/b1e3407e9e6913cf178e506cd0dee818e58694d9a5cd1984e3f6a8b9a10f/lazy_object_proxy-1.4.3-cp36-cp36m-manylinux1_x86_64.whl\n",
      "Installing collected packages: future, pillow, numpy, setuptools, torch, termcolor, six, h5py, keras-applications, absl-py, wheel, protobuf, werkzeug, markdown, grpcio, tensorboard, gast, google-pasta, keras-preprocessing, tensorflow-estimator, astor, wrapt, tensorflow, tqdm, sentencepiece, chardet, idna, certifi, urllib3, requests, torchtext, portalocker, typing, mecab-python3, sacrebleu, subword-nmt, kiwisolver, python-dateutil, pyparsing, cycler, matplotlib, scipy, pytz, pandas, seaborn, pyyaml, mccabe, typed-ast, lazy-object-proxy, astroid, isort, pylint, joeynmt\n",
      "  Running setup.py install for joeynmt ... \u001b[?25ldone\n",
      "\u001b[?25hSuccessfully installed absl-py-0.9.0 astor-0.8.1 astroid-2.3.3 certifi-2020.4.5.1 chardet-3.0.4 cycler-0.10.0 future-0.18.2 gast-0.3.3 google-pasta-0.2.0 grpcio-1.28.1 h5py-2.10.0 idna-2.9 isort-4.3.21 joeynmt-0.0.1 keras-applications-1.0.8 keras-preprocessing-1.1.0 kiwisolver-1.2.0 lazy-object-proxy-1.4.3 markdown-3.2.1 matplotlib-3.2.1 mccabe-0.6.1 mecab-python3-0.996.5 numpy-1.18.3 pandas-1.0.3 pillow-7.1.1 portalocker-1.7.0 protobuf-3.11.3 pylint-2.4.4 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 pyyaml-5.3.1 requests-2.23.0 sacrebleu-1.4.7 scipy-1.4.1 seaborn-0.10.0 sentencepiece-0.1.85 setuptools-46.1.3 six-1.14.0 subword-nmt-0.3.7 tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0 termcolor-1.1.0 torch-1.5.0 torchtext-0.6.0 tqdm-4.45.0 typed-ast-1.4.1 typing-3.7.4.1 urllib3-1.25.9 werkzeug-1.0.1 wheel-0.34.2 wrapt-1.11.1\n",
      "\u001b[33mYou are using pip version 8.1.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Install JoeyNMT\n",
    "! git clone https://github.com/joeynmt/joeynmt.git\n",
    "! cd joeynmt; pip3 install ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preprocessing the Data into Subword BPE Tokens\n",
    "One of the most powerful improvements for agglutinative languages (a feature of most Bantu languages) is using BPE tokenization (Sennrich, 2015) .\n",
    "\n",
    "It was also shown that by optimizing the umber of BPE codes we significantly improve results for low-resourced languages (Sennrich, 2019) (Martinus, 2019)\n",
    "\n",
    "Below we have the scripts for doing BPE tokenization of our data. We use 4000 tokens as recommended by (Sennrich, 2019). You do not need to change anything. Simply running the below will be suitable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpe.codes.4000\tdev.sw\t     test.en-any.en    train.bpe.en\n",
      "dev.bpe.en\ttest.bpe.en  test.en-any.en.1  train.bpe.sw\n",
      "dev.bpe.sw\ttest.bpe.sw  test.en-any.en.2  train.en\n",
      "dev.en\t\ttest.en      test.sw\t       train.sw\n",
      "BPE Swahili Sentences\n",
      "N@@ ga@@ o kubwa ya imani ( Ta@@ zama fungu la 1@@ 2 hadi 1@@ 4 )\n",
      "K@@ ofi@@ a ya chu@@ ma ya woko@@ vu ( Ta@@ zama fungu la 1@@ 5 hadi 1@@ 8 )\n",
      "Nime@@ ta@@ mbua kwamba watu hu@@ i@@ ti@@ kia vi@@ zuri wana@@ po@@ ona uki@@ zungu@@ m@@ zia habari za Bi@@ b@@ lia kwa sha@@ u@@ ku na una@@ po@@ fanya yote una@@ yo@@ weza kuwa@@ saidia . ”\n",
      "U@@ panga wa roho ( Ta@@ zama fungu la 1@@ 9 na 2@@ 0 )\n",
      "Kwa msa@@ ada wa Yeho@@ v@@ a tuna@@ weza ku@@ simama i@@ mara na ku@@ mp@@ inga !\n",
      "Combined BPE Vocab\n",
      "ù@@\n",
      "harao@@\n",
      " \n",
      ">@@\n",
      "fles@@\n",
      "}@@\n",
      "^@@\n",
      "|\n",
      "berna@@\n",
      "ù\n"
     ]
    }
   ],
   "source": [
    "# One of the huge boosts in NMT performance was to use a different method of tokenizing. \n",
    "# Usually, NMT would tokenize by words. However, using a method called BPE gave amazing boosts to performance\n",
    "\n",
    "# Do subword NMT\n",
    "from os import path\n",
    "os.environ[\"src\"] = source_language # Sets them in bash as well, since we often use bash scripts\n",
    "os.environ[\"tgt\"] = target_language\n",
    "\n",
    "# Learn BPEs on the training data.\n",
    "os.environ[\"data_path\"] = path.join(\"joeynmt\", \"data\", source_language + target_language) # Herman! \n",
    "! subword-nmt learn-joint-bpe-and-vocab --input train.$src train.$tgt -s 4000 -o bpe.codes.4000 --write-vocabulary vocab.$src vocab.$tgt\n",
    "\n",
    "# Apply BPE splits to the development and test data.\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$src < train.$src > train.bpe.$src\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$tgt < train.$tgt > train.bpe.$tgt\n",
    "\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$src < dev.$src > dev.bpe.$src\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$tgt < dev.$tgt > dev.bpe.$tgt\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$src < test.$src > test.bpe.$src\n",
    "! subword-nmt apply-bpe -c bpe.codes.4000 --vocabulary vocab.$tgt < test.$tgt > test.bpe.$tgt\n",
    "\n",
    "# Create directory, move everyone we care about to the correct location\n",
    "! mkdir -p $data_path\n",
    "! cp train.* $data_path\n",
    "! cp test.* $data_path\n",
    "! cp dev.* $data_path\n",
    "! cp bpe.codes.4000 $data_path\n",
    "! ls $data_path\n",
    "\n",
    "# # Also move everything we care about to a mounted location in google drive (relevant if running in colab) at gdrive_path\n",
    "# ! cp train.* \"$gdrive_path\"\n",
    "# ! cp test.* \"$gdrive_path\"\n",
    "# ! cp dev.* \"$gdrive_path\"\n",
    "# ! cp bpe.codes.4000 \"$gdrive_path\"\n",
    "# ! ls \"$gdrive_path\"\n",
    "\n",
    "# Create that vocab using build_vocab\n",
    "! sudo chmod 777 joeynmt/scripts/build_vocab.py\n",
    "! joeynmt/scripts/build_vocab.py joeynmt/data/$src$tgt/train.bpe.$src joeynmt/data/$src$tgt/train.bpe.$tgt --output_path joeynmt/data/$src$tgt/vocab.txt\n",
    "\n",
    "# Some output\n",
    "! echo \"BPE Swahili Sentences\"\n",
    "! tail -n 5 test.bpe.$tgt\n",
    "! echo \"Combined BPE Vocab\"\n",
    "! tail -n 10 joeynmt/data/$src$tgt/vocab.txt  # Herman"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating the JoeyNMT Config\n",
    "JoeyNMT requires a yaml config. We provide a template below. We've also set a number of defaults with it, that you may play with!\n",
    "\n",
    "We used Transformer architecture\n",
    "We set our dropout to reasonably high: 0.3 (recommended in (Sennrich, 2019))\n",
    "Things worth playing with:\n",
    "\n",
    "The batch size (also recommended to change for low-resourced languages)\n",
    "The number of epochs (we've set it at 30 just so it runs in about an hour, for testing purposes)\n",
    "The decoder options (beam_size, alpha)\n",
    "Evaluation metrics (BLEU versus Crhf4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# More experiments to be done with the parameters\n",
    "\n",
    "name = \"ensw\"\n",
    "\n",
    "\n",
    "# Create the config\n",
    "config = \"\"\"\n",
    "name: \"ensw_transformer\"\n",
    "data:\n",
    "    src: \"en\"\n",
    "    trg: \"sw\"\n",
    "    dev:   \"data/ensw/dev.bpe\"\n",
    "    test:  \"data/ensw/test.bpe\"\n",
    "    train:  \"data/ensw/train.bpe\"\n",
    "    level: \"bpe\"\n",
    "    lowercase: False\n",
    "    max_sent_length: 100\n",
    "    src_vocab: \"data/ensw/vocab.txt\"\n",
    "    trg_vocab: \"data/ensw/vocab.txt\"\n",
    "\n",
    "testing:\n",
    "    beam_size: 5\n",
    "    alpha: 1.0\n",
    "\n",
    "training:\n",
    "    #load_model: \"{gdrive_path}/models/{name}_transformer/1.ckpt\" # if uncommented, load a pre-trained model from this checkpoint\n",
    "    random_seed: 42\n",
    "    optimizer: \"adam\"\n",
    "    normalization: \"tokens\"\n",
    "    adam_betas: [0.9, 0.999] \n",
    "    scheduling: \"plateau\"           # TODO: try switching from plateau to Noam scheduling\n",
    "    patience: 5                     # For plateau: decrease learning rate by decrease_factor if validation score has not improved for this many validation rounds.\n",
    "    learning_rate_factor: 0.5       # factor for Noam scheduler (used with Transformer)\n",
    "    learning_rate_warmup: 1000      # warmup steps for Noam scheduler (used with Transformer)\n",
    "    decrease_factor: 0.7\n",
    "    loss: \"crossentropy\"\n",
    "    learning_rate: 0.0003\n",
    "    learning_rate_min: 0.00000001\n",
    "    weight_decay: 0.0\n",
    "    label_smoothing: 0.1\n",
    "    batch_size: 4096\n",
    "    batch_type: \"token\"\n",
    "    eval_batch_size: 3600\n",
    "    eval_batch_type: \"token\"\n",
    "    batch_multiplier: 1\n",
    "    early_stopping_metric: \"ppl\"\n",
    "    epochs: 30                     # TODO: Decrease for when playing around and checking of working. Around 30 is sufficient to check if its working at all\n",
    "    validation_freq: 1000          # TODO: Set to at least once per epoch.\n",
    "    logging_freq: 100\n",
    "    eval_metric: \"bleu\"\n",
    "    model_dir: \"models/ensw_transformer\"\n",
    "    overwrite: False               # TODO: Set to True if you want to overwrite possibly existing models. \n",
    "    shuffle: True\n",
    "    use_cuda: True\n",
    "    max_output_length: 100\n",
    "    print_valid_sents: [0, 1, 2, 3]\n",
    "    keep_last_ckpts: 3\n",
    "\n",
    "model:\n",
    "    initializer: \"xavier\"\n",
    "    bias_initializer: \"zeros\"\n",
    "    init_gain: 1.0\n",
    "    embed_initializer: \"xavier\"\n",
    "    embed_init_gain: 1.0\n",
    "    tied_embeddings: True\n",
    "    tied_softmax: True\n",
    "    encoder:\n",
    "        type: \"transformer\"\n",
    "        num_layers: 6\n",
    "        num_heads: 4             # TODO: Increase to 8 for larger data.\n",
    "        embeddings:\n",
    "            embedding_dim: 256   # TODO: Increase to 512 for larger data.\n",
    "            scale: True\n",
    "            dropout: 0.2\n",
    "        # typically ff_size = 4 x hidden_size\n",
    "        hidden_size: 256         # TODO: Increase to 512 for larger data.\n",
    "        ff_size: 1024            # TODO: Increase to 2048 for larger data.\n",
    "        dropout: 0.3\n",
    "    decoder:\n",
    "        type: \"transformer\"\n",
    "        num_layers: 6\n",
    "        num_heads: 4              # TODO: Increase to 8 for larger data.\n",
    "        embeddings:\n",
    "            embedding_dim: 256    # TODO: Increase to 512 for larger data.\n",
    "            scale: True\n",
    "            dropout: 0.2\n",
    "        # typically ff_size = 4 x hidden_size\n",
    "        hidden_size: 256         # TODO: Increase to 512 for larger data.\n",
    "        ff_size: 1024            # TODO: Increase to 2048 for larger data.\n",
    "        dropout: 0.3\n",
    "\"\"\"\n",
    "#.format(name=name,gdrie_path=os.environ[\"gdrive_path\"],source_language=source_language, target_language=target_language)\n",
    "#.format(name=name, gdrive_path=os.environ[\"gdrive_path\"], source_language=source_language, target_language=target_language)\n",
    "with open(\"joeynmt/configs/transformer_ensw2.yaml\".format(name=name),'w') as f:\n",
    "    f.write(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train the Model\n",
    "This single line of joeynmt runs the training using the config we made above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/freshiasackey_gmail_com/joeynmt\n",
      "benchmarks.md\t    data     joey-small.png  README.md\t       setup.py\n",
      "CODE_OF_CONDUCT.md  docs     LICENSE\t     requirements.txt  test\n",
      "configs\t\t    joeynmt  models\t     scripts\n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/freshiasackey_gmail_com/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "2020-04-23 22:05:01,554 Hello! This is Joey-NMT.\n",
      "2020-04-23 22:05:01,559 Total params: 12135168\n",
      "2020-04-23 22:05:01,560 Trainable parameters: ['decoder.layer_norm.bias', 'decoder.layer_norm.weight', 'decoder.layers.0.dec_layer_norm.bias', 'decoder.layers.0.dec_layer_norm.weight', 'decoder.layers.0.feed_forward.layer_norm.bias', 'decoder.layers.0.feed_forward.layer_norm.weight', 'decoder.layers.0.feed_forward.pwff_layer.0.bias', 'decoder.layers.0.feed_forward.pwff_layer.0.weight', 'decoder.layers.0.feed_forward.pwff_layer.3.bias', 'decoder.layers.0.feed_forward.pwff_layer.3.weight', 'decoder.layers.0.src_trg_att.k_layer.bias', 'decoder.layers.0.src_trg_att.k_layer.weight', 'decoder.layers.0.src_trg_att.output_layer.bias', 'decoder.layers.0.src_trg_att.output_layer.weight', 'decoder.layers.0.src_trg_att.q_layer.bias', 'decoder.layers.0.src_trg_att.q_layer.weight', 'decoder.layers.0.src_trg_att.v_layer.bias', 'decoder.layers.0.src_trg_att.v_layer.weight', 'decoder.layers.0.trg_trg_att.k_layer.bias', 'decoder.layers.0.trg_trg_att.k_layer.weight', 'decoder.layers.0.trg_trg_att.output_layer.bias', 'decoder.layers.0.trg_trg_att.output_layer.weight', 'decoder.layers.0.trg_trg_att.q_layer.bias', 'decoder.layers.0.trg_trg_att.q_layer.weight', 'decoder.layers.0.trg_trg_att.v_layer.bias', 'decoder.layers.0.trg_trg_att.v_layer.weight', 'decoder.layers.0.x_layer_norm.bias', 'decoder.layers.0.x_layer_norm.weight', 'decoder.layers.1.dec_layer_norm.bias', 'decoder.layers.1.dec_layer_norm.weight', 'decoder.layers.1.feed_forward.layer_norm.bias', 'decoder.layers.1.feed_forward.layer_norm.weight', 'decoder.layers.1.feed_forward.pwff_layer.0.bias', 'decoder.layers.1.feed_forward.pwff_layer.0.weight', 'decoder.layers.1.feed_forward.pwff_layer.3.bias', 'decoder.layers.1.feed_forward.pwff_layer.3.weight', 'decoder.layers.1.src_trg_att.k_layer.bias', 'decoder.layers.1.src_trg_att.k_layer.weight', 'decoder.layers.1.src_trg_att.output_layer.bias', 'decoder.layers.1.src_trg_att.output_layer.weight', 'decoder.layers.1.src_trg_att.q_layer.bias', 'decoder.layers.1.src_trg_att.q_layer.weight', 'decoder.layers.1.src_trg_att.v_layer.bias', 'decoder.layers.1.src_trg_att.v_layer.weight', 'decoder.layers.1.trg_trg_att.k_layer.bias', 'decoder.layers.1.trg_trg_att.k_layer.weight', 'decoder.layers.1.trg_trg_att.output_layer.bias', 'decoder.layers.1.trg_trg_att.output_layer.weight', 'decoder.layers.1.trg_trg_att.q_layer.bias', 'decoder.layers.1.trg_trg_att.q_layer.weight', 'decoder.layers.1.trg_trg_att.v_layer.bias', 'decoder.layers.1.trg_trg_att.v_layer.weight', 'decoder.layers.1.x_layer_norm.bias', 'decoder.layers.1.x_layer_norm.weight', 'decoder.layers.2.dec_layer_norm.bias', 'decoder.layers.2.dec_layer_norm.weight', 'decoder.layers.2.feed_forward.layer_norm.bias', 'decoder.layers.2.feed_forward.layer_norm.weight', 'decoder.layers.2.feed_forward.pwff_layer.0.bias', 'decoder.layers.2.feed_forward.pwff_layer.0.weight', 'decoder.layers.2.feed_forward.pwff_layer.3.bias', 'decoder.layers.2.feed_forward.pwff_layer.3.weight', 'decoder.layers.2.src_trg_att.k_layer.bias', 'decoder.layers.2.src_trg_att.k_layer.weight', 'decoder.layers.2.src_trg_att.output_layer.bias', 'decoder.layers.2.src_trg_att.output_layer.weight', 'decoder.layers.2.src_trg_att.q_layer.bias', 'decoder.layers.2.src_trg_att.q_layer.weight', 'decoder.layers.2.src_trg_att.v_layer.bias', 'decoder.layers.2.src_trg_att.v_layer.weight', 'decoder.layers.2.trg_trg_att.k_layer.bias', 'decoder.layers.2.trg_trg_att.k_layer.weight', 'decoder.layers.2.trg_trg_att.output_layer.bias', 'decoder.layers.2.trg_trg_att.output_layer.weight', 'decoder.layers.2.trg_trg_att.q_layer.bias', 'decoder.layers.2.trg_trg_att.q_layer.weight', 'decoder.layers.2.trg_trg_att.v_layer.bias', 'decoder.layers.2.trg_trg_att.v_layer.weight', 'decoder.layers.2.x_layer_norm.bias', 'decoder.layers.2.x_layer_norm.weight', 'decoder.layers.3.dec_layer_norm.bias', 'decoder.layers.3.dec_layer_norm.weight', 'decoder.layers.3.feed_forward.layer_norm.bias', 'decoder.layers.3.feed_forward.layer_norm.weight', 'decoder.layers.3.feed_forward.pwff_layer.0.bias', 'decoder.layers.3.feed_forward.pwff_layer.0.weight', 'decoder.layers.3.feed_forward.pwff_layer.3.bias', 'decoder.layers.3.feed_forward.pwff_layer.3.weight', 'decoder.layers.3.src_trg_att.k_layer.bias', 'decoder.layers.3.src_trg_att.k_layer.weight', 'decoder.layers.3.src_trg_att.output_layer.bias', 'decoder.layers.3.src_trg_att.output_layer.weight', 'decoder.layers.3.src_trg_att.q_layer.bias', 'decoder.layers.3.src_trg_att.q_layer.weight', 'decoder.layers.3.src_trg_att.v_layer.bias', 'decoder.layers.3.src_trg_att.v_layer.weight', 'decoder.layers.3.trg_trg_att.k_layer.bias', 'decoder.layers.3.trg_trg_att.k_layer.weight', 'decoder.layers.3.trg_trg_att.output_layer.bias', 'decoder.layers.3.trg_trg_att.output_layer.weight', 'decoder.layers.3.trg_trg_att.q_layer.bias', 'decoder.layers.3.trg_trg_att.q_layer.weight', 'decoder.layers.3.trg_trg_att.v_layer.bias', 'decoder.layers.3.trg_trg_att.v_layer.weight', 'decoder.layers.3.x_layer_norm.bias', 'decoder.layers.3.x_layer_norm.weight', 'decoder.layers.4.dec_layer_norm.bias', 'decoder.layers.4.dec_layer_norm.weight', 'decoder.layers.4.feed_forward.layer_norm.bias', 'decoder.layers.4.feed_forward.layer_norm.weight', 'decoder.layers.4.feed_forward.pwff_layer.0.bias', 'decoder.layers.4.feed_forward.pwff_layer.0.weight', 'decoder.layers.4.feed_forward.pwff_layer.3.bias', 'decoder.layers.4.feed_forward.pwff_layer.3.weight', 'decoder.layers.4.src_trg_att.k_layer.bias', 'decoder.layers.4.src_trg_att.k_layer.weight', 'decoder.layers.4.src_trg_att.output_layer.bias', 'decoder.layers.4.src_trg_att.output_layer.weight', 'decoder.layers.4.src_trg_att.q_layer.bias', 'decoder.layers.4.src_trg_att.q_layer.weight', 'decoder.layers.4.src_trg_att.v_layer.bias', 'decoder.layers.4.src_trg_att.v_layer.weight', 'decoder.layers.4.trg_trg_att.k_layer.bias', 'decoder.layers.4.trg_trg_att.k_layer.weight', 'decoder.layers.4.trg_trg_att.output_layer.bias', 'decoder.layers.4.trg_trg_att.output_layer.weight', 'decoder.layers.4.trg_trg_att.q_layer.bias', 'decoder.layers.4.trg_trg_att.q_layer.weight', 'decoder.layers.4.trg_trg_att.v_layer.bias', 'decoder.layers.4.trg_trg_att.v_layer.weight', 'decoder.layers.4.x_layer_norm.bias', 'decoder.layers.4.x_layer_norm.weight', 'decoder.layers.5.dec_layer_norm.bias', 'decoder.layers.5.dec_layer_norm.weight', 'decoder.layers.5.feed_forward.layer_norm.bias', 'decoder.layers.5.feed_forward.layer_norm.weight', 'decoder.layers.5.feed_forward.pwff_layer.0.bias', 'decoder.layers.5.feed_forward.pwff_layer.0.weight', 'decoder.layers.5.feed_forward.pwff_layer.3.bias', 'decoder.layers.5.feed_forward.pwff_layer.3.weight', 'decoder.layers.5.src_trg_att.k_layer.bias', 'decoder.layers.5.src_trg_att.k_layer.weight', 'decoder.layers.5.src_trg_att.output_layer.bias', 'decoder.layers.5.src_trg_att.output_layer.weight', 'decoder.layers.5.src_trg_att.q_layer.bias', 'decoder.layers.5.src_trg_att.q_layer.weight', 'decoder.layers.5.src_trg_att.v_layer.bias', 'decoder.layers.5.src_trg_att.v_layer.weight', 'decoder.layers.5.trg_trg_att.k_layer.bias', 'decoder.layers.5.trg_trg_att.k_layer.weight', 'decoder.layers.5.trg_trg_att.output_layer.bias', 'decoder.layers.5.trg_trg_att.output_layer.weight', 'decoder.layers.5.trg_trg_att.q_layer.bias', 'decoder.layers.5.trg_trg_att.q_layer.weight', 'decoder.layers.5.trg_trg_att.v_layer.bias', 'decoder.layers.5.trg_trg_att.v_layer.weight', 'decoder.layers.5.x_layer_norm.bias', 'decoder.layers.5.x_layer_norm.weight', 'encoder.layer_norm.bias', 'encoder.layer_norm.weight', 'encoder.layers.0.feed_forward.layer_norm.bias', 'encoder.layers.0.feed_forward.layer_norm.weight', 'encoder.layers.0.feed_forward.pwff_layer.0.bias', 'encoder.layers.0.feed_forward.pwff_layer.0.weight', 'encoder.layers.0.feed_forward.pwff_layer.3.bias', 'encoder.layers.0.feed_forward.pwff_layer.3.weight', 'encoder.layers.0.layer_norm.bias', 'encoder.layers.0.layer_norm.weight', 'encoder.layers.0.src_src_att.k_layer.bias', 'encoder.layers.0.src_src_att.k_layer.weight', 'encoder.layers.0.src_src_att.output_layer.bias', 'encoder.layers.0.src_src_att.output_layer.weight', 'encoder.layers.0.src_src_att.q_layer.bias', 'encoder.layers.0.src_src_att.q_layer.weight', 'encoder.layers.0.src_src_att.v_layer.bias', 'encoder.layers.0.src_src_att.v_layer.weight', 'encoder.layers.1.feed_forward.layer_norm.bias', 'encoder.layers.1.feed_forward.layer_norm.weight', 'encoder.layers.1.feed_forward.pwff_layer.0.bias', 'encoder.layers.1.feed_forward.pwff_layer.0.weight', 'encoder.layers.1.feed_forward.pwff_layer.3.bias', 'encoder.layers.1.feed_forward.pwff_layer.3.weight', 'encoder.layers.1.layer_norm.bias', 'encoder.layers.1.layer_norm.weight', 'encoder.layers.1.src_src_att.k_layer.bias', 'encoder.layers.1.src_src_att.k_layer.weight', 'encoder.layers.1.src_src_att.output_layer.bias', 'encoder.layers.1.src_src_att.output_layer.weight', 'encoder.layers.1.src_src_att.q_layer.bias', 'encoder.layers.1.src_src_att.q_layer.weight', 'encoder.layers.1.src_src_att.v_layer.bias', 'encoder.layers.1.src_src_att.v_layer.weight', 'encoder.layers.2.feed_forward.layer_norm.bias', 'encoder.layers.2.feed_forward.layer_norm.weight', 'encoder.layers.2.feed_forward.pwff_layer.0.bias', 'encoder.layers.2.feed_forward.pwff_layer.0.weight', 'encoder.layers.2.feed_forward.pwff_layer.3.bias', 'encoder.layers.2.feed_forward.pwff_layer.3.weight', 'encoder.layers.2.layer_norm.bias', 'encoder.layers.2.layer_norm.weight', 'encoder.layers.2.src_src_att.k_layer.bias', 'encoder.layers.2.src_src_att.k_layer.weight', 'encoder.layers.2.src_src_att.output_layer.bias', 'encoder.layers.2.src_src_att.output_layer.weight', 'encoder.layers.2.src_src_att.q_layer.bias', 'encoder.layers.2.src_src_att.q_layer.weight', 'encoder.layers.2.src_src_att.v_layer.bias', 'encoder.layers.2.src_src_att.v_layer.weight', 'encoder.layers.3.feed_forward.layer_norm.bias', 'encoder.layers.3.feed_forward.layer_norm.weight', 'encoder.layers.3.feed_forward.pwff_layer.0.bias', 'encoder.layers.3.feed_forward.pwff_layer.0.weight', 'encoder.layers.3.feed_forward.pwff_layer.3.bias', 'encoder.layers.3.feed_forward.pwff_layer.3.weight', 'encoder.layers.3.layer_norm.bias', 'encoder.layers.3.layer_norm.weight', 'encoder.layers.3.src_src_att.k_layer.bias', 'encoder.layers.3.src_src_att.k_layer.weight', 'encoder.layers.3.src_src_att.output_layer.bias', 'encoder.layers.3.src_src_att.output_layer.weight', 'encoder.layers.3.src_src_att.q_layer.bias', 'encoder.layers.3.src_src_att.q_layer.weight', 'encoder.layers.3.src_src_att.v_layer.bias', 'encoder.layers.3.src_src_att.v_layer.weight', 'encoder.layers.4.feed_forward.layer_norm.bias', 'encoder.layers.4.feed_forward.layer_norm.weight', 'encoder.layers.4.feed_forward.pwff_layer.0.bias', 'encoder.layers.4.feed_forward.pwff_layer.0.weight', 'encoder.layers.4.feed_forward.pwff_layer.3.bias', 'encoder.layers.4.feed_forward.pwff_layer.3.weight', 'encoder.layers.4.layer_norm.bias', 'encoder.layers.4.layer_norm.weight', 'encoder.layers.4.src_src_att.k_layer.bias', 'encoder.layers.4.src_src_att.k_layer.weight', 'encoder.layers.4.src_src_att.output_layer.bias', 'encoder.layers.4.src_src_att.output_layer.weight', 'encoder.layers.4.src_src_att.q_layer.bias', 'encoder.layers.4.src_src_att.q_layer.weight', 'encoder.layers.4.src_src_att.v_layer.bias', 'encoder.layers.4.src_src_att.v_layer.weight', 'encoder.layers.5.feed_forward.layer_norm.bias', 'encoder.layers.5.feed_forward.layer_norm.weight', 'encoder.layers.5.feed_forward.pwff_layer.0.bias', 'encoder.layers.5.feed_forward.pwff_layer.0.weight', 'encoder.layers.5.feed_forward.pwff_layer.3.bias', 'encoder.layers.5.feed_forward.pwff_layer.3.weight', 'encoder.layers.5.layer_norm.bias', 'encoder.layers.5.layer_norm.weight', 'encoder.layers.5.src_src_att.k_layer.bias', 'encoder.layers.5.src_src_att.k_layer.weight', 'encoder.layers.5.src_src_att.output_layer.bias', 'encoder.layers.5.src_src_att.output_layer.weight', 'encoder.layers.5.src_src_att.q_layer.bias', 'encoder.layers.5.src_src_att.q_layer.weight', 'encoder.layers.5.src_src_att.v_layer.bias', 'encoder.layers.5.src_src_att.v_layer.weight', 'src_embed.lut.weight']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 22:05:05,960 cfg.name                           : ensw_transformer\n",
      "2020-04-23 22:05:05,960 cfg.data.src                       : en\n",
      "2020-04-23 22:05:05,960 cfg.data.trg                       : sw\n",
      "2020-04-23 22:05:05,961 cfg.data.dev                       : data/ensw/dev.bpe\n",
      "2020-04-23 22:05:05,961 cfg.data.test                      : data/ensw/test.bpe\n",
      "2020-04-23 22:05:05,961 cfg.data.train                     : data/ensw/train.bpe\n",
      "2020-04-23 22:05:05,961 cfg.data.level                     : bpe\n",
      "2020-04-23 22:05:05,961 cfg.data.lowercase                 : False\n",
      "2020-04-23 22:05:05,961 cfg.data.max_sent_length           : 100\n",
      "2020-04-23 22:05:05,961 cfg.data.src_vocab                 : data/ensw/vocab.txt\n",
      "2020-04-23 22:05:05,961 cfg.data.trg_vocab                 : data/ensw/vocab.txt\n",
      "2020-04-23 22:05:05,961 cfg.testing.beam_size              : 5\n",
      "2020-04-23 22:05:05,961 cfg.testing.alpha                  : 1.0\n",
      "2020-04-23 22:05:05,961 cfg.training.random_seed           : 42\n",
      "2020-04-23 22:05:05,961 cfg.training.optimizer             : adam\n",
      "2020-04-23 22:05:05,961 cfg.training.normalization         : tokens\n",
      "2020-04-23 22:05:05,962 cfg.training.adam_betas            : [0.9, 0.999]\n",
      "2020-04-23 22:05:05,962 cfg.training.scheduling            : plateau\n",
      "2020-04-23 22:05:05,962 cfg.training.patience              : 5\n",
      "2020-04-23 22:05:05,962 cfg.training.learning_rate_factor  : 0.5\n",
      "2020-04-23 22:05:05,962 cfg.training.learning_rate_warmup  : 1000\n",
      "2020-04-23 22:05:05,962 cfg.training.decrease_factor       : 0.7\n",
      "2020-04-23 22:05:05,962 cfg.training.loss                  : crossentropy\n",
      "2020-04-23 22:05:05,962 cfg.training.learning_rate         : 0.0003\n",
      "2020-04-23 22:05:05,962 cfg.training.learning_rate_min     : 1e-08\n",
      "2020-04-23 22:05:05,962 cfg.training.weight_decay          : 0.0\n",
      "2020-04-23 22:05:05,962 cfg.training.label_smoothing       : 0.1\n",
      "2020-04-23 22:05:05,962 cfg.training.batch_size            : 4096\n",
      "2020-04-23 22:05:05,962 cfg.training.batch_type            : token\n",
      "2020-04-23 22:05:05,962 cfg.training.eval_batch_size       : 3600\n",
      "2020-04-23 22:05:05,963 cfg.training.eval_batch_type       : token\n",
      "2020-04-23 22:05:05,963 cfg.training.batch_multiplier      : 1\n",
      "2020-04-23 22:05:05,963 cfg.training.early_stopping_metric : ppl\n",
      "2020-04-23 22:05:05,963 cfg.training.epochs                : 30\n",
      "2020-04-23 22:05:05,963 cfg.training.validation_freq       : 1000\n",
      "2020-04-23 22:05:05,963 cfg.training.logging_freq          : 100\n",
      "2020-04-23 22:05:05,963 cfg.training.eval_metric           : bleu\n",
      "2020-04-23 22:05:05,963 cfg.training.model_dir             : models/ensw_transformer\n",
      "2020-04-23 22:05:05,963 cfg.training.overwrite             : False\n",
      "2020-04-23 22:05:05,963 cfg.training.shuffle               : True\n",
      "2020-04-23 22:05:05,963 cfg.training.use_cuda              : True\n",
      "2020-04-23 22:05:05,963 cfg.training.max_output_length     : 100\n",
      "2020-04-23 22:05:05,963 cfg.training.print_valid_sents     : [0, 1, 2, 3]\n",
      "2020-04-23 22:05:05,963 cfg.training.keep_last_ckpts       : 3\n",
      "2020-04-23 22:05:05,963 cfg.model.initializer              : xavier\n",
      "2020-04-23 22:05:05,963 cfg.model.bias_initializer         : zeros\n",
      "2020-04-23 22:05:05,964 cfg.model.init_gain                : 1.0\n",
      "2020-04-23 22:05:05,964 cfg.model.embed_initializer        : xavier\n",
      "2020-04-23 22:05:05,964 cfg.model.embed_init_gain          : 1.0\n",
      "2020-04-23 22:05:05,964 cfg.model.tied_embeddings          : True\n",
      "2020-04-23 22:05:05,964 cfg.model.tied_softmax             : True\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.type             : transformer\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.num_layers       : 6\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.num_heads        : 4\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.embeddings.embedding_dim : 256\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.embeddings.scale : True\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.embeddings.dropout : 0.2\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.hidden_size      : 256\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.ff_size          : 1024\n",
      "2020-04-23 22:05:05,964 cfg.model.encoder.dropout          : 0.3\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.type             : transformer\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.num_layers       : 6\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.num_heads        : 4\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.embeddings.embedding_dim : 256\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.embeddings.scale : True\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.embeddings.dropout : 0.2\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.hidden_size      : 256\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.ff_size          : 1024\n",
      "2020-04-23 22:05:05,965 cfg.model.decoder.dropout          : 0.3\n",
      "2020-04-23 22:05:05,965 Data set sizes: \n",
      "\ttrain 88292,\n",
      "\tvalid 1000,\n",
      "\ttest 2721\n",
      "2020-04-23 22:05:05,965 First training example:\n",
      "\t[SRC] lo@@ b@@ ster\n",
      "\t[TRG] ka@@ mba@@ ko@@ che\n",
      "2020-04-23 22:05:05,965 First 10 words (src): (0) <unk> (1) <pad> (2) <s> (3) </s> (4) the (5) and (6) na (7) of (8) ya (9) wa\n",
      "2020-04-23 22:05:05,966 First 10 words (trg): (0) <unk> (1) <pad> (2) <s> (3) </s> (4) the (5) and (6) na (7) of (8) ya (9) wa\n",
      "2020-04-23 22:05:05,966 Number of Src words (types): 4199\n",
      "2020-04-23 22:05:05,966 Number of Trg words (types): 4199\n",
      "2020-04-23 22:05:05,967 Model(\n",
      "\tencoder=TransformerEncoder(num_layers=6, num_heads=4),\n",
      "\tdecoder=TransformerDecoder(num_layers=6, num_heads=4),\n",
      "\tsrc_embed=Embeddings(embedding_dim=256, vocab_size=4199),\n",
      "\ttrg_embed=Embeddings(embedding_dim=256, vocab_size=4199))\n",
      "2020-04-23 22:05:05,970 EPOCH 1\n",
      "/pytorch/torch/csrc/utils/python_arg_parser.cpp:756: UserWarning: This overload of nonzero is deprecated:\n",
      "\tnonzero(Tensor input, *, Tensor out)\n",
      "Consider using one of the following signatures instead:\n",
      "\tnonzero(Tensor input, *, bool as_tuple)\n",
      "2020-04-23 22:05:20,236 Epoch   1 Step:      100 Batch Loss:     6.095017 Tokens per Sec:    16962, Lr: 0.000300\n",
      "2020-04-23 22:05:34,409 Epoch   1 Step:      200 Batch Loss:     5.884079 Tokens per Sec:    16783, Lr: 0.000300\n",
      "2020-04-23 22:05:48,750 Epoch   1 Step:      300 Batch Loss:     5.730944 Tokens per Sec:    16833, Lr: 0.000300\n",
      "2020-04-23 22:06:03,106 Epoch   1 Step:      400 Batch Loss:     5.471322 Tokens per Sec:    16284, Lr: 0.000300\n",
      "2020-04-23 22:06:17,707 Epoch   1 Step:      500 Batch Loss:     5.339774 Tokens per Sec:    16312, Lr: 0.000300\n",
      "2020-04-23 22:06:32,539 Epoch   1 Step:      600 Batch Loss:     5.230678 Tokens per Sec:    16472, Lr: 0.000300\n",
      "2020-04-23 22:06:47,216 Epoch   1 Step:      700 Batch Loss:     5.231392 Tokens per Sec:    16831, Lr: 0.000300\n",
      "2020-04-23 22:06:59,066 Epoch   1: total training loss 4324.99\n",
      "2020-04-23 22:06:59,067 EPOCH 2\n",
      "2020-04-23 22:07:01,889 Epoch   2 Step:      800 Batch Loss:     4.256592 Tokens per Sec:    15648, Lr: 0.000300\n",
      "2020-04-23 22:07:16,290 Epoch   2 Step:      900 Batch Loss:     5.139263 Tokens per Sec:    16421, Lr: 0.000300\n",
      "2020-04-23 22:07:30,744 Epoch   2 Step:     1000 Batch Loss:     4.958186 Tokens per Sec:    16418, Lr: 0.000300\n",
      "2020-04-23 22:08:13,690 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:08:13,690 Saving new checkpoint.\n",
      "2020-04-23 22:08:13,861 Example #0\n",
      "2020-04-23 22:08:13,861 \tSource:     span\n",
      "2020-04-23 22:08:13,862 \tReference:  shibiri\n",
      "2020-04-23 22:08:13,862 \tHypothesis: mkoka\n",
      "2020-04-23 22:08:13,862 Example #1\n",
      "2020-04-23 22:08:13,862 \tSource:     administer\n",
      "2020-04-23 22:08:13,862 \tReference:  amuru\n",
      "2020-04-23 22:08:13,862 \tHypothesis: mkoka\n",
      "2020-04-23 22:08:13,862 Example #2\n",
      "2020-04-23 22:08:13,862 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:08:13,862 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:08:13,862 \tHypothesis: Na Mwenyezi Mungu ni kukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukukuti.\n",
      "2020-04-23 22:08:13,862 Example #3\n",
      "2020-04-23 22:08:13,863 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:08:13,863 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:08:13,863 \tHypothesis: Na Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni ni ni ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni ni ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi Mungu ni Mwenyezi\n",
      "2020-04-23 22:08:13,863 Validation result (greedy) at epoch   2, step     1000: bleu:   0.10, loss: 101210.0000, ppl: 124.1338, duration: 43.1187s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 22:08:28,228 Epoch   2 Step:     1100 Batch Loss:     4.948530 Tokens per Sec:    16534, Lr: 0.000300\n",
      "2020-04-23 22:08:42,851 Epoch   2 Step:     1200 Batch Loss:     4.629645 Tokens per Sec:    16489, Lr: 0.000300\n",
      "2020-04-23 22:08:57,463 Epoch   2 Step:     1300 Batch Loss:     4.664552 Tokens per Sec:    16742, Lr: 0.000300\n",
      "2020-04-23 22:09:11,831 Epoch   2 Step:     1400 Batch Loss:     4.592779 Tokens per Sec:    16645, Lr: 0.000300\n",
      "2020-04-23 22:09:26,380 Epoch   2 Step:     1500 Batch Loss:     4.226472 Tokens per Sec:    16678, Lr: 0.000300\n",
      "2020-04-23 22:09:36,080 Epoch   2: total training loss 3720.06\n",
      "2020-04-23 22:09:36,080 EPOCH 3\n",
      "2020-04-23 22:09:40,988 Epoch   3 Step:     1600 Batch Loss:     4.440054 Tokens per Sec:    16163, Lr: 0.000300\n",
      "2020-04-23 22:09:55,522 Epoch   3 Step:     1700 Batch Loss:     4.414383 Tokens per Sec:    16729, Lr: 0.000300\n",
      "2020-04-23 22:10:10,107 Epoch   3 Step:     1800 Batch Loss:     4.320911 Tokens per Sec:    16680, Lr: 0.000300\n",
      "2020-04-23 22:10:24,628 Epoch   3 Step:     1900 Batch Loss:     4.349395 Tokens per Sec:    16495, Lr: 0.000300\n",
      "2020-04-23 22:10:39,082 Epoch   3 Step:     2000 Batch Loss:     4.167585 Tokens per Sec:    16680, Lr: 0.000300\n",
      "2020-04-23 22:11:20,029 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:11:20,029 Saving new checkpoint.\n",
      "2020-04-23 22:11:20,192 Example #0\n",
      "2020-04-23 22:11:20,193 \tSource:     span\n",
      "2020-04-23 22:11:20,193 \tReference:  shibiri\n",
      "2020-04-23 22:11:20,193 \tHypothesis: mgoro\n",
      "2020-04-23 22:11:20,193 Example #1\n",
      "2020-04-23 22:11:20,193 \tSource:     administer\n",
      "2020-04-23 22:11:20,193 \tReference:  amuru\n",
      "2020-04-23 22:11:20,193 \tHypothesis: kikoka\n",
      "2020-04-23 22:11:20,193 Example #2\n",
      "2020-04-23 22:11:20,193 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:11:20,193 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:11:20,193 \tHypothesis: Na tukawaamini na wakawaongoza kwa sababu ya wazuri.\n",
      "2020-04-23 22:11:20,193 Example #3\n",
      "2020-04-23 22:11:20,193 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:11:20,193 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:11:20,194 \tHypothesis: Hayo ni ni kusema:\n",
      "2020-04-23 22:11:20,194 Validation result (greedy) at epoch   3, step     2000: bleu:   0.46, loss: 86998.1484, ppl:  63.0765, duration: 41.1115s\n",
      "2020-04-23 22:11:34,674 Epoch   3 Step:     2100 Batch Loss:     4.013601 Tokens per Sec:    16410, Lr: 0.000300\n",
      "2020-04-23 22:11:49,286 Epoch   3 Step:     2200 Batch Loss:     4.119627 Tokens per Sec:    16579, Lr: 0.000300\n",
      "2020-04-23 22:12:03,954 Epoch   3 Step:     2300 Batch Loss:     4.274373 Tokens per Sec:    16632, Lr: 0.000300\n",
      "2020-04-23 22:12:10,633 Epoch   3: total training loss 3326.13\n",
      "2020-04-23 22:12:10,633 EPOCH 4\n",
      "2020-04-23 22:12:18,477 Epoch   4 Step:     2400 Batch Loss:     3.914624 Tokens per Sec:    15904, Lr: 0.000300\n",
      "2020-04-23 22:12:33,046 Epoch   4 Step:     2500 Batch Loss:     3.979854 Tokens per Sec:    16764, Lr: 0.000300\n",
      "2020-04-23 22:12:47,571 Epoch   4 Step:     2600 Batch Loss:     3.959794 Tokens per Sec:    16676, Lr: 0.000300\n",
      "2020-04-23 22:13:01,954 Epoch   4 Step:     2700 Batch Loss:     3.983327 Tokens per Sec:    16273, Lr: 0.000300\n",
      "2020-04-23 22:13:16,481 Epoch   4 Step:     2800 Batch Loss:     3.836418 Tokens per Sec:    16853, Lr: 0.000300\n",
      "2020-04-23 22:13:31,057 Epoch   4 Step:     2900 Batch Loss:     3.819096 Tokens per Sec:    16485, Lr: 0.000300\n",
      "2020-04-23 22:13:45,820 Epoch   4 Step:     3000 Batch Loss:     3.722033 Tokens per Sec:    16505, Lr: 0.000300\n",
      "2020-04-23 22:14:25,875 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:14:25,876 Saving new checkpoint.\n",
      "2020-04-23 22:14:26,039 Example #0\n",
      "2020-04-23 22:14:26,039 \tSource:     span\n",
      "2020-04-23 22:14:26,039 \tReference:  shibiri\n",
      "2020-04-23 22:14:26,039 \tHypothesis: mgono\n",
      "2020-04-23 22:14:26,039 Example #1\n",
      "2020-04-23 22:14:26,039 \tSource:     administer\n",
      "2020-04-23 22:14:26,040 \tReference:  amuru\n",
      "2020-04-23 22:14:26,040 \tHypothesis: mgoro\n",
      "2020-04-23 22:14:26,040 Example #2\n",
      "2020-04-23 22:14:26,040 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:14:26,040 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:14:26,040 \tHypothesis: Basi tukawafanya watu walio washiriki washiriki kwa sababu ya wapotovu.\n",
      "2020-04-23 22:14:26,040 Example #3\n",
      "2020-04-23 22:14:26,040 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:14:26,040 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:14:26,040 \tHypothesis: Lakini hawakuamini Mwenyezi Mungu ila hayo.\n",
      "2020-04-23 22:14:26,040 Validation result (greedy) at epoch   4, step     3000: bleu:   0.99, loss: 76560.2578, ppl:  38.3639, duration: 40.2200s\n",
      "2020-04-23 22:14:40,619 Epoch   4 Step:     3100 Batch Loss:     3.812062 Tokens per Sec:    16638, Lr: 0.000300\n",
      "2020-04-23 22:14:44,528 Epoch   4: total training loss 3054.38\n",
      "2020-04-23 22:14:44,528 EPOCH 5\n",
      "2020-04-23 22:14:55,180 Epoch   5 Step:     3200 Batch Loss:     3.798851 Tokens per Sec:    16139, Lr: 0.000300\n",
      "2020-04-23 22:15:09,926 Epoch   5 Step:     3300 Batch Loss:     3.553757 Tokens per Sec:    16827, Lr: 0.000300\n",
      "2020-04-23 22:15:24,363 Epoch   5 Step:     3400 Batch Loss:     3.593013 Tokens per Sec:    16551, Lr: 0.000300\n",
      "2020-04-23 22:15:38,972 Epoch   5 Step:     3500 Batch Loss:     3.446258 Tokens per Sec:    16590, Lr: 0.000300\n",
      "2020-04-23 22:15:53,435 Epoch   5 Step:     3600 Batch Loss:     3.649515 Tokens per Sec:    16628, Lr: 0.000300\n",
      "2020-04-23 22:16:07,934 Epoch   5 Step:     3700 Batch Loss:     3.463524 Tokens per Sec:    16701, Lr: 0.000300\n",
      "2020-04-23 22:16:22,467 Epoch   5 Step:     3800 Batch Loss:     3.501094 Tokens per Sec:    16652, Lr: 0.000300\n",
      "2020-04-23 22:16:36,957 Epoch   5 Step:     3900 Batch Loss:     3.402152 Tokens per Sec:    16386, Lr: 0.000300\n",
      "2020-04-23 22:16:38,091 Epoch   5: total training loss 2830.80\n",
      "2020-04-23 22:16:38,092 EPOCH 6\n",
      "2020-04-23 22:16:51,402 Epoch   6 Step:     4000 Batch Loss:     3.679500 Tokens per Sec:    16167, Lr: 0.000300\n",
      "2020-04-23 22:17:31,817 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:17:31,817 Saving new checkpoint.\n",
      "2020-04-23 22:17:31,997 Example #0\n",
      "2020-04-23 22:17:31,998 \tSource:     span\n",
      "2020-04-23 22:17:31,998 \tReference:  shibiri\n",
      "2020-04-23 22:17:31,998 \tHypothesis: kipio\n",
      "2020-04-23 22:17:31,998 Example #1\n",
      "2020-04-23 22:17:31,998 \tSource:     administer\n",
      "2020-04-23 22:17:31,998 \tReference:  amuru\n",
      "2020-04-23 22:17:31,998 \tHypothesis: udongo\n",
      "2020-04-23 22:17:31,998 Example #2\n",
      "2020-04-23 22:17:31,998 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:17:31,998 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:17:31,998 \tHypothesis: Basi tukawafikia Tuli, wakawa kukaa katika ardhi kwa adhabu ya kurehemu.\n",
      "2020-04-23 22:17:31,998 Example #3\n",
      "2020-04-23 22:17:31,999 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:17:31,999 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:17:31,999 \tHypothesis: Hatuamini ila ila ila wasemaye.\n",
      "2020-04-23 22:17:31,999 Validation result (greedy) at epoch   6, step     4000: bleu:   1.80, loss: 70256.2969, ppl:  28.4120, duration: 40.5968s\n",
      "2020-04-23 22:17:46,318 Epoch   6 Step:     4100 Batch Loss:     3.482018 Tokens per Sec:    16272, Lr: 0.000300\n",
      "2020-04-23 22:18:00,828 Epoch   6 Step:     4200 Batch Loss:     3.414102 Tokens per Sec:    16774, Lr: 0.000300\n",
      "2020-04-23 22:18:15,285 Epoch   6 Step:     4300 Batch Loss:     3.358453 Tokens per Sec:    16639, Lr: 0.000300\n",
      "2020-04-23 22:18:29,777 Epoch   6 Step:     4400 Batch Loss:     3.395641 Tokens per Sec:    16503, Lr: 0.000300\n",
      "2020-04-23 22:18:44,471 Epoch   6 Step:     4500 Batch Loss:     3.412284 Tokens per Sec:    16703, Lr: 0.000300\n",
      "2020-04-23 22:18:58,961 Epoch   6 Step:     4600 Batch Loss:     3.168545 Tokens per Sec:    16793, Lr: 0.000300\n",
      "2020-04-23 22:19:12,245 Epoch   6: total training loss 2680.50\n",
      "2020-04-23 22:19:12,245 EPOCH 7\n",
      "2020-04-23 22:19:13,671 Epoch   7 Step:     4700 Batch Loss:     3.382784 Tokens per Sec:    14833, Lr: 0.000300\n",
      "2020-04-23 22:19:28,009 Epoch   7 Step:     4800 Batch Loss:     3.278353 Tokens per Sec:    16406, Lr: 0.000300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 22:19:42,332 Epoch   7 Step:     4900 Batch Loss:     3.272926 Tokens per Sec:    16822, Lr: 0.000300\n",
      "2020-04-23 22:19:56,834 Epoch   7 Step:     5000 Batch Loss:     3.183771 Tokens per Sec:    16734, Lr: 0.000300\n",
      "2020-04-23 22:20:34,446 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:20:34,446 Saving new checkpoint.\n",
      "2020-04-23 22:20:34,636 Example #0\n",
      "2020-04-23 22:20:34,636 \tSource:     span\n",
      "2020-04-23 22:20:34,636 \tReference:  shibiri\n",
      "2020-04-23 22:20:34,636 \tHypothesis: kikoa\n",
      "2020-04-23 22:20:34,636 Example #1\n",
      "2020-04-23 22:20:34,636 \tSource:     administer\n",
      "2020-04-23 22:20:34,636 \tReference:  amuru\n",
      "2020-04-23 22:20:34,636 \tHypothesis: mkosefu\n",
      "2020-04-23 22:20:34,637 Example #2\n",
      "2020-04-23 22:20:34,637 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:20:34,637 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:20:34,637 \tHypothesis: Basi kama vile vile walivyo wafikia wakhabari kwa adhabu ya adhabu ya adhabu ya adhabu chungu.\n",
      "2020-04-23 22:20:34,637 Example #3\n",
      "2020-04-23 22:20:34,637 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:20:34,637 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:20:34,637 \tHypothesis: Hatuamini ila ila hayo.\n",
      "2020-04-23 22:20:34,637 Validation result (greedy) at epoch   7, step     5000: bleu:   3.90, loss: 64481.6641, ppl:  21.5791, duration: 37.8025s\n",
      "2020-04-23 22:20:49,151 Epoch   7 Step:     5100 Batch Loss:     3.124520 Tokens per Sec:    16581, Lr: 0.000300\n",
      "2020-04-23 22:21:03,696 Epoch   7 Step:     5200 Batch Loss:     3.283840 Tokens per Sec:    16830, Lr: 0.000300\n",
      "2020-04-23 22:21:18,333 Epoch   7 Step:     5300 Batch Loss:     3.243718 Tokens per Sec:    16875, Lr: 0.000300\n",
      "2020-04-23 22:21:32,643 Epoch   7 Step:     5400 Batch Loss:     3.175300 Tokens per Sec:    16656, Lr: 0.000300\n",
      "2020-04-23 22:21:42,928 Epoch   7: total training loss 2532.80\n",
      "2020-04-23 22:21:42,928 EPOCH 8\n",
      "2020-04-23 22:21:47,076 Epoch   8 Step:     5500 Batch Loss:     3.202880 Tokens per Sec:    15917, Lr: 0.000300\n",
      "2020-04-23 22:22:01,559 Epoch   8 Step:     5600 Batch Loss:     3.103312 Tokens per Sec:    16904, Lr: 0.000300\n",
      "2020-04-23 22:22:16,120 Epoch   8 Step:     5700 Batch Loss:     3.126830 Tokens per Sec:    16688, Lr: 0.000300\n",
      "2020-04-23 22:22:30,450 Epoch   8 Step:     5800 Batch Loss:     2.957569 Tokens per Sec:    16565, Lr: 0.000300\n",
      "2020-04-23 22:22:44,785 Epoch   8 Step:     5900 Batch Loss:     3.011085 Tokens per Sec:    16584, Lr: 0.000300\n",
      "2020-04-23 22:22:59,156 Epoch   8 Step:     6000 Batch Loss:     3.048528 Tokens per Sec:    16655, Lr: 0.000300\n",
      "2020-04-23 22:23:35,593 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:23:35,593 Saving new checkpoint.\n",
      "2020-04-23 22:23:35,772 Example #0\n",
      "2020-04-23 22:23:35,772 \tSource:     span\n",
      "2020-04-23 22:23:35,772 \tReference:  shibiri\n",
      "2020-04-23 22:23:35,772 \tHypothesis: kipimo\n",
      "2020-04-23 22:23:35,772 Example #1\n",
      "2020-04-23 22:23:35,772 \tSource:     administer\n",
      "2020-04-23 22:23:35,772 \tReference:  amuru\n",
      "2020-04-23 22:23:35,772 \tHypothesis: ukozi\n",
      "2020-04-23 22:23:35,772 Example #2\n",
      "2020-04-23 22:23:35,773 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:23:35,773 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:23:35,773 \tHypothesis: Basi kama vile vile vile vile vile vile vile vile vivyo hivyo hivyo vivyo hivyo hivyo hivyo vivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo hivyo Sita.\n",
      "2020-04-23 22:23:35,773 Example #3\n",
      "2020-04-23 22:23:35,773 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:23:35,773 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:23:35,773 \tHypothesis: Hawaamini ila wasema:\n",
      "2020-04-23 22:23:35,773 Validation result (greedy) at epoch   8, step     6000: bleu:   4.57, loss: 60913.1250, ppl:  18.2056, duration: 36.6167s\n",
      "2020-04-23 22:23:50,184 Epoch   8 Step:     6100 Batch Loss:     3.164337 Tokens per Sec:    16570, Lr: 0.000300\n",
      "2020-04-23 22:24:04,598 Epoch   8 Step:     6200 Batch Loss:     2.920100 Tokens per Sec:    16637, Lr: 0.000300\n",
      "2020-04-23 22:24:12,918 Epoch   8: total training loss 2451.18\n",
      "2020-04-23 22:24:12,918 EPOCH 9\n",
      "2020-04-23 22:24:19,134 Epoch   9 Step:     6300 Batch Loss:     3.419597 Tokens per Sec:    16229, Lr: 0.000300\n",
      "2020-04-23 22:24:33,665 Epoch   9 Step:     6400 Batch Loss:     2.895144 Tokens per Sec:    16574, Lr: 0.000300\n",
      "2020-04-23 22:24:47,905 Epoch   9 Step:     6500 Batch Loss:     3.008734 Tokens per Sec:    16626, Lr: 0.000300\n",
      "2020-04-23 22:25:02,412 Epoch   9 Step:     6600 Batch Loss:     2.918678 Tokens per Sec:    16567, Lr: 0.000300\n",
      "2020-04-23 22:25:17,005 Epoch   9 Step:     6700 Batch Loss:     2.966475 Tokens per Sec:    16770, Lr: 0.000300\n",
      "2020-04-23 22:25:31,485 Epoch   9 Step:     6800 Batch Loss:     2.908371 Tokens per Sec:    16406, Lr: 0.000300\n",
      "2020-04-23 22:25:46,291 Epoch   9 Step:     6900 Batch Loss:     3.155514 Tokens per Sec:    16796, Lr: 0.000300\n",
      "2020-04-23 22:26:00,688 Epoch   9 Step:     7000 Batch Loss:     3.023613 Tokens per Sec:    16401, Lr: 0.000300\n",
      "2020-04-23 22:26:31,127 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:26:31,127 Saving new checkpoint.\n",
      "2020-04-23 22:26:31,303 Example #0\n",
      "2020-04-23 22:26:31,303 \tSource:     span\n",
      "2020-04-23 22:26:31,303 \tReference:  shibiri\n",
      "2020-04-23 22:26:31,303 \tHypothesis: kikoo\n",
      "2020-04-23 22:26:31,303 Example #1\n",
      "2020-04-23 22:26:31,303 \tSource:     administer\n",
      "2020-04-23 22:26:31,303 \tReference:  amuru\n",
      "2020-04-23 22:26:31,303 \tHypothesis: ukozi\n",
      "2020-04-23 22:26:31,303 Example #2\n",
      "2020-04-23 22:26:31,303 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:26:31,304 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:26:31,304 \tHypothesis: Basi wakiwaangamiza Shai, wakaangamiza adhabu chungu.\n",
      "2020-04-23 22:26:31,304 Example #3\n",
      "2020-04-23 22:26:31,304 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:26:31,304 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:26:31,304 \tHypothesis: Hawaamini ila yeye.\n",
      "2020-04-23 22:26:31,304 Validation result (greedy) at epoch   9, step     7000: bleu:   5.47, loss: 58165.9688, ppl:  15.9724, duration: 30.6152s\n",
      "2020-04-23 22:26:37,170 Epoch   9: total training loss 2358.67\n",
      "2020-04-23 22:26:37,170 EPOCH 10\n",
      "2020-04-23 22:26:45,769 Epoch  10 Step:     7100 Batch Loss:     2.873031 Tokens per Sec:    16020, Lr: 0.000300\n",
      "2020-04-23 22:27:00,449 Epoch  10 Step:     7200 Batch Loss:     2.913481 Tokens per Sec:    16791, Lr: 0.000300\n",
      "2020-04-23 22:27:14,909 Epoch  10 Step:     7300 Batch Loss:     2.986084 Tokens per Sec:    16325, Lr: 0.000300\n",
      "2020-04-23 22:27:29,440 Epoch  10 Step:     7400 Batch Loss:     2.822574 Tokens per Sec:    16618, Lr: 0.000300\n",
      "2020-04-23 22:27:44,062 Epoch  10 Step:     7500 Batch Loss:     2.905132 Tokens per Sec:    16731, Lr: 0.000300\n",
      "2020-04-23 22:27:58,568 Epoch  10 Step:     7600 Batch Loss:     2.773883 Tokens per Sec:    16576, Lr: 0.000300\n",
      "2020-04-23 22:28:12,966 Epoch  10 Step:     7700 Batch Loss:     2.837100 Tokens per Sec:    16420, Lr: 0.000300\n",
      "2020-04-23 22:28:27,340 Epoch  10 Step:     7800 Batch Loss:     3.008653 Tokens per Sec:    16682, Lr: 0.000300\n",
      "2020-04-23 22:28:30,863 Epoch  10: total training loss 2296.46\n",
      "2020-04-23 22:28:30,863 EPOCH 11\n",
      "2020-04-23 22:28:42,105 Epoch  11 Step:     7900 Batch Loss:     2.959062 Tokens per Sec:    16648, Lr: 0.000300\n",
      "2020-04-23 22:28:56,769 Epoch  11 Step:     8000 Batch Loss:     2.796218 Tokens per Sec:    16696, Lr: 0.000300\n",
      "2020-04-23 22:29:24,438 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:29:24,439 Saving new checkpoint.\n",
      "2020-04-23 22:29:24,627 Example #0\n",
      "2020-04-23 22:29:24,627 \tSource:     span\n",
      "2020-04-23 22:29:24,627 \tReference:  shibiri\n",
      "2020-04-23 22:29:24,627 \tHypothesis: kikombe\n",
      "2020-04-23 22:29:24,627 Example #1\n",
      "2020-04-23 22:29:24,628 \tSource:     administer\n",
      "2020-04-23 22:29:24,628 \tReference:  amuru\n",
      "2020-04-23 22:29:24,628 \tHypothesis: ujuzi\n",
      "2020-04-23 22:29:24,628 Example #2\n",
      "2020-04-23 22:29:24,628 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:29:24,628 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:29:24,628 \tHypothesis: Basi walipo waangamiza, waliwaangamiza adhabu chungu.\n",
      "2020-04-23 22:29:24,628 Example #3\n",
      "2020-04-23 22:29:24,628 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:29:24,628 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:29:24,628 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:29:24,628 Validation result (greedy) at epoch  11, step     8000: bleu:   6.14, loss: 56048.3984, ppl:  14.4398, duration: 27.8593s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 22:29:39,144 Epoch  11 Step:     8100 Batch Loss:     3.043009 Tokens per Sec:    16602, Lr: 0.000300\n",
      "2020-04-23 22:29:53,574 Epoch  11 Step:     8200 Batch Loss:     2.711674 Tokens per Sec:    16527, Lr: 0.000300\n",
      "2020-04-23 22:30:08,143 Epoch  11 Step:     8300 Batch Loss:     2.709927 Tokens per Sec:    16569, Lr: 0.000300\n",
      "2020-04-23 22:30:22,525 Epoch  11 Step:     8400 Batch Loss:     2.767094 Tokens per Sec:    16389, Lr: 0.000300\n",
      "2020-04-23 22:30:37,069 Epoch  11 Step:     8500 Batch Loss:     2.919308 Tokens per Sec:    16432, Lr: 0.000300\n",
      "2020-04-23 22:30:51,530 Epoch  11 Step:     8600 Batch Loss:     2.700101 Tokens per Sec:    16490, Lr: 0.000300\n",
      "2020-04-23 22:30:52,433 Epoch  11: total training loss 2227.07\n",
      "2020-04-23 22:30:52,433 EPOCH 12\n",
      "2020-04-23 22:31:06,273 Epoch  12 Step:     8700 Batch Loss:     2.647313 Tokens per Sec:    16555, Lr: 0.000300\n",
      "2020-04-23 22:31:20,838 Epoch  12 Step:     8800 Batch Loss:     2.659315 Tokens per Sec:    16648, Lr: 0.000300\n",
      "2020-04-23 22:31:35,331 Epoch  12 Step:     8900 Batch Loss:     2.645703 Tokens per Sec:    16443, Lr: 0.000300\n",
      "2020-04-23 22:31:49,788 Epoch  12 Step:     9000 Batch Loss:     2.817270 Tokens per Sec:    16561, Lr: 0.000300\n",
      "2020-04-23 22:32:21,202 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:32:21,202 Saving new checkpoint.\n",
      "2020-04-23 22:32:21,384 Example #0\n",
      "2020-04-23 22:32:21,385 \tSource:     span\n",
      "2020-04-23 22:32:21,385 \tReference:  shibiri\n",
      "2020-04-23 22:32:21,385 \tHypothesis: kipato\n",
      "2020-04-23 22:32:21,385 Example #1\n",
      "2020-04-23 22:32:21,385 \tSource:     administer\n",
      "2020-04-23 22:32:21,385 \tReference:  amuru\n",
      "2020-04-23 22:32:21,385 \tHypothesis: uamuzi\n",
      "2020-04-23 22:32:21,385 Example #2\n",
      "2020-04-23 22:32:21,385 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:32:21,385 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:32:21,386 \tHypothesis: Basi walipo waangamiza, waliangamizwa kwa adhabu chungu.\n",
      "2020-04-23 22:32:21,386 Example #3\n",
      "2020-04-23 22:32:21,386 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:32:21,386 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:32:21,386 \tHypothesis: Hauamini ila yetu.\n",
      "2020-04-23 22:32:21,386 Validation result (greedy) at epoch  12, step     9000: bleu:   6.85, loss: 54146.2539, ppl:  13.1889, duration: 31.5973s\n",
      "2020-04-23 22:32:36,040 Epoch  12 Step:     9100 Batch Loss:     2.769861 Tokens per Sec:    16744, Lr: 0.000300\n",
      "2020-04-23 22:32:50,397 Epoch  12 Step:     9200 Batch Loss:     3.049111 Tokens per Sec:    16720, Lr: 0.000300\n",
      "2020-04-23 22:33:05,049 Epoch  12 Step:     9300 Batch Loss:     2.695832 Tokens per Sec:    16839, Lr: 0.000300\n",
      "2020-04-23 22:33:17,401 Epoch  12: total training loss 2165.32\n",
      "2020-04-23 22:33:17,401 EPOCH 13\n",
      "2020-04-23 22:33:19,446 Epoch  13 Step:     9400 Batch Loss:     2.506218 Tokens per Sec:    14626, Lr: 0.000300\n",
      "2020-04-23 22:33:33,987 Epoch  13 Step:     9500 Batch Loss:     2.615042 Tokens per Sec:    16629, Lr: 0.000300\n",
      "2020-04-23 22:33:48,499 Epoch  13 Step:     9600 Batch Loss:     2.595325 Tokens per Sec:    16729, Lr: 0.000300\n",
      "2020-04-23 22:34:03,047 Epoch  13 Step:     9700 Batch Loss:     2.672231 Tokens per Sec:    16895, Lr: 0.000300\n",
      "2020-04-23 22:34:17,527 Epoch  13 Step:     9800 Batch Loss:     2.657628 Tokens per Sec:    16475, Lr: 0.000300\n",
      "2020-04-23 22:34:32,020 Epoch  13 Step:     9900 Batch Loss:     2.655142 Tokens per Sec:    16678, Lr: 0.000300\n",
      "2020-04-23 22:34:46,461 Epoch  13 Step:    10000 Batch Loss:     2.797047 Tokens per Sec:    16553, Lr: 0.000300\n",
      "2020-04-23 22:35:15,475 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:35:15,475 Saving new checkpoint.\n",
      "2020-04-23 22:35:15,653 Example #0\n",
      "2020-04-23 22:35:15,653 \tSource:     span\n",
      "2020-04-23 22:35:15,653 \tReference:  shibiri\n",
      "2020-04-23 22:35:15,653 \tHypothesis: kikokoo\n",
      "2020-04-23 22:35:15,653 Example #1\n",
      "2020-04-23 22:35:15,654 \tSource:     administer\n",
      "2020-04-23 22:35:15,654 \tReference:  amuru\n",
      "2020-04-23 22:35:15,654 \tHypothesis: ustadi\n",
      "2020-04-23 22:35:15,654 Example #2\n",
      "2020-04-23 22:35:15,654 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:35:15,654 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:35:15,654 \tHypothesis: Basi walipo waangamiza, wakaangamizwa kwa adhabu chungu.\n",
      "2020-04-23 22:35:15,654 Example #3\n",
      "2020-04-23 22:35:15,654 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:35:15,654 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:35:15,654 \tHypothesis: Hauamini ila sema:\n",
      "2020-04-23 22:35:15,654 Validation result (greedy) at epoch  13, step    10000: bleu:   7.86, loss: 52337.5547, ppl:  12.1001, duration: 29.1931s\n",
      "2020-04-23 22:35:30,356 Epoch  13 Step:    10100 Batch Loss:     2.604957 Tokens per Sec:    16900, Lr: 0.000300\n",
      "2020-04-23 22:35:39,662 Epoch  13: total training loss 2110.64\n",
      "2020-04-23 22:35:39,662 EPOCH 14\n",
      "2020-04-23 22:35:44,994 Epoch  14 Step:    10200 Batch Loss:     2.499959 Tokens per Sec:    16523, Lr: 0.000300\n",
      "2020-04-23 22:35:59,517 Epoch  14 Step:    10300 Batch Loss:     2.644568 Tokens per Sec:    16784, Lr: 0.000300\n",
      "2020-04-23 22:36:14,125 Epoch  14 Step:    10400 Batch Loss:     2.579822 Tokens per Sec:    16736, Lr: 0.000300\n",
      "2020-04-23 22:36:28,594 Epoch  14 Step:    10500 Batch Loss:     2.492972 Tokens per Sec:    16636, Lr: 0.000300\n",
      "2020-04-23 22:36:43,028 Epoch  14 Step:    10600 Batch Loss:     2.408103 Tokens per Sec:    16399, Lr: 0.000300\n",
      "2020-04-23 22:36:57,374 Epoch  14 Step:    10700 Batch Loss:     2.469356 Tokens per Sec:    16601, Lr: 0.000300\n",
      "2020-04-23 22:37:11,808 Epoch  14 Step:    10800 Batch Loss:     2.548208 Tokens per Sec:    16577, Lr: 0.000300\n",
      "2020-04-23 22:37:26,079 Epoch  14 Step:    10900 Batch Loss:     3.066540 Tokens per Sec:    16396, Lr: 0.000300\n",
      "2020-04-23 22:37:33,119 Epoch  14: total training loss 2085.27\n",
      "2020-04-23 22:37:33,120 EPOCH 15\n",
      "2020-04-23 22:37:40,575 Epoch  15 Step:    11000 Batch Loss:     2.663833 Tokens per Sec:    16143, Lr: 0.000300\n",
      "2020-04-23 22:38:09,082 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:38:09,082 Saving new checkpoint.\n",
      "2020-04-23 22:38:09,264 Example #0\n",
      "2020-04-23 22:38:09,264 \tSource:     span\n",
      "2020-04-23 22:38:09,264 \tReference:  shibiri\n",
      "2020-04-23 22:38:09,264 \tHypothesis: kipimo\n",
      "2020-04-23 22:38:09,264 Example #1\n",
      "2020-04-23 22:38:09,265 \tSource:     administer\n",
      "2020-04-23 22:38:09,265 \tReference:  amuru\n",
      "2020-04-23 22:38:09,265 \tHypothesis: mstari\n",
      "2020-04-23 22:38:09,265 Example #2\n",
      "2020-04-23 22:38:09,265 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:38:09,265 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:38:09,265 \tHypothesis: Na kama Shama waliangamizwa kwa kuangamizwa kwa adhabu ya kudhulumu.\n",
      "2020-04-23 22:38:09,265 Example #3\n",
      "2020-04-23 22:38:09,265 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:38:09,265 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:38:09,265 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:38:09,266 Validation result (greedy) at epoch  15, step    11000: bleu:   8.70, loss: 51062.7500, ppl:  11.3872, duration: 28.6901s\n",
      "2020-04-23 22:38:23,817 Epoch  15 Step:    11100 Batch Loss:     2.326766 Tokens per Sec:    16725, Lr: 0.000300\n",
      "2020-04-23 22:38:38,361 Epoch  15 Step:    11200 Batch Loss:     2.680680 Tokens per Sec:    16776, Lr: 0.000300\n",
      "2020-04-23 22:38:52,979 Epoch  15 Step:    11300 Batch Loss:     2.448949 Tokens per Sec:    16664, Lr: 0.000300\n",
      "2020-04-23 22:39:07,654 Epoch  15 Step:    11400 Batch Loss:     2.456947 Tokens per Sec:    16570, Lr: 0.000300\n",
      "2020-04-23 22:39:22,318 Epoch  15 Step:    11500 Batch Loss:     2.563407 Tokens per Sec:    16760, Lr: 0.000300\n",
      "2020-04-23 22:39:36,865 Epoch  15 Step:    11600 Batch Loss:     2.503886 Tokens per Sec:    16568, Lr: 0.000300\n",
      "2020-04-23 22:39:51,226 Epoch  15 Step:    11700 Batch Loss:     2.481181 Tokens per Sec:    16373, Lr: 0.000300\n",
      "2020-04-23 22:39:55,366 Epoch  15: total training loss 2033.76\n",
      "2020-04-23 22:39:55,367 EPOCH 16\n",
      "2020-04-23 22:40:05,796 Epoch  16 Step:    11800 Batch Loss:     2.619490 Tokens per Sec:    16495, Lr: 0.000300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 22:40:20,301 Epoch  16 Step:    11900 Batch Loss:     2.571353 Tokens per Sec:    16574, Lr: 0.000300\n",
      "2020-04-23 22:40:34,880 Epoch  16 Step:    12000 Batch Loss:     2.386926 Tokens per Sec:    16710, Lr: 0.000300\n",
      "2020-04-23 22:41:00,056 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:41:00,056 Saving new checkpoint.\n",
      "2020-04-23 22:41:00,239 Example #0\n",
      "2020-04-23 22:41:00,240 \tSource:     span\n",
      "2020-04-23 22:41:00,240 \tReference:  shibiri\n",
      "2020-04-23 22:41:00,240 \tHypothesis: kikombe\n",
      "2020-04-23 22:41:00,240 Example #1\n",
      "2020-04-23 22:41:00,240 \tSource:     administer\n",
      "2020-04-23 22:41:00,240 \tReference:  amuru\n",
      "2020-04-23 22:41:00,240 \tHypothesis: ustadi\n",
      "2020-04-23 22:41:00,240 Example #2\n",
      "2020-04-23 22:41:00,240 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:41:00,240 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:41:00,241 \tHypothesis: Basi walipo angamizwa kwa Haki waliangamizwa kwa adhabu chungu.\n",
      "2020-04-23 22:41:00,241 Example #3\n",
      "2020-04-23 22:41:00,241 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:41:00,241 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:41:00,241 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:41:00,241 Validation result (greedy) at epoch  16, step    12000: bleu:  10.07, loss: 49865.0039, ppl:  10.7556, duration: 25.3605s\n",
      "2020-04-23 22:41:14,898 Epoch  16 Step:    12100 Batch Loss:     2.633744 Tokens per Sec:    16708, Lr: 0.000300\n",
      "2020-04-23 22:41:29,327 Epoch  16 Step:    12200 Batch Loss:     2.370982 Tokens per Sec:    16405, Lr: 0.000300\n",
      "2020-04-23 22:41:43,780 Epoch  16 Step:    12300 Batch Loss:     2.328024 Tokens per Sec:    16665, Lr: 0.000300\n",
      "2020-04-23 22:41:58,145 Epoch  16 Step:    12400 Batch Loss:     2.773796 Tokens per Sec:    16232, Lr: 0.000300\n",
      "2020-04-23 22:42:12,638 Epoch  16 Step:    12500 Batch Loss:     2.600609 Tokens per Sec:    16803, Lr: 0.000300\n",
      "2020-04-23 22:42:14,345 Epoch  16: total training loss 2002.75\n",
      "2020-04-23 22:42:14,345 EPOCH 17\n",
      "2020-04-23 22:42:27,208 Epoch  17 Step:    12600 Batch Loss:     2.313411 Tokens per Sec:    16492, Lr: 0.000300\n",
      "2020-04-23 22:42:41,789 Epoch  17 Step:    12700 Batch Loss:     3.022202 Tokens per Sec:    16613, Lr: 0.000300\n",
      "2020-04-23 22:42:56,303 Epoch  17 Step:    12800 Batch Loss:     2.428675 Tokens per Sec:    16834, Lr: 0.000300\n",
      "2020-04-23 22:43:10,815 Epoch  17 Step:    12900 Batch Loss:     2.422059 Tokens per Sec:    16785, Lr: 0.000300\n",
      "2020-04-23 22:43:25,078 Epoch  17 Step:    13000 Batch Loss:     2.365484 Tokens per Sec:    16433, Lr: 0.000300\n",
      "2020-04-23 22:43:55,113 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:43:55,113 Saving new checkpoint.\n",
      "2020-04-23 22:43:55,291 Example #0\n",
      "2020-04-23 22:43:55,291 \tSource:     span\n",
      "2020-04-23 22:43:55,292 \tReference:  shibiri\n",
      "2020-04-23 22:43:55,292 \tHypothesis: kikombe\n",
      "2020-04-23 22:43:55,292 Example #1\n",
      "2020-04-23 22:43:55,292 \tSource:     administer\n",
      "2020-04-23 22:43:55,292 \tReference:  amuru\n",
      "2020-04-23 22:43:55,292 \tHypothesis: ukozi\n",
      "2020-04-23 22:43:55,292 Example #2\n",
      "2020-04-23 22:43:55,292 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:43:55,292 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:43:55,292 \tHypothesis: Basi walipo angamizwa kwa adhabu iliyo dhaahiri.\n",
      "2020-04-23 22:43:55,292 Example #3\n",
      "2020-04-23 22:43:55,292 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:43:55,292 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:43:55,292 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:43:55,292 Validation result (greedy) at epoch  17, step    13000: bleu:  10.80, loss: 48361.9648, ppl:  10.0124, duration: 30.2136s\n",
      "2020-04-23 22:44:09,905 Epoch  17 Step:    13100 Batch Loss:     2.751553 Tokens per Sec:    16716, Lr: 0.000300\n",
      "2020-04-23 22:44:24,273 Epoch  17 Step:    13200 Batch Loss:     3.006796 Tokens per Sec:    16542, Lr: 0.000300\n",
      "2020-04-23 22:44:37,703 Epoch  17: total training loss 1961.65\n",
      "2020-04-23 22:44:37,703 EPOCH 18\n",
      "2020-04-23 22:44:38,806 Epoch  18 Step:    13300 Batch Loss:     2.486589 Tokens per Sec:    13754, Lr: 0.000300\n",
      "2020-04-23 22:44:53,276 Epoch  18 Step:    13400 Batch Loss:     2.493627 Tokens per Sec:    16712, Lr: 0.000300\n",
      "2020-04-23 22:45:07,753 Epoch  18 Step:    13500 Batch Loss:     2.898747 Tokens per Sec:    16770, Lr: 0.000300\n",
      "2020-04-23 22:45:22,169 Epoch  18 Step:    13600 Batch Loss:     2.632573 Tokens per Sec:    16490, Lr: 0.000300\n",
      "2020-04-23 22:45:36,651 Epoch  18 Step:    13700 Batch Loss:     2.362310 Tokens per Sec:    16573, Lr: 0.000300\n",
      "2020-04-23 22:45:51,049 Epoch  18 Step:    13800 Batch Loss:     2.460665 Tokens per Sec:    16595, Lr: 0.000300\n",
      "2020-04-23 22:46:05,522 Epoch  18 Step:    13900 Batch Loss:     2.212844 Tokens per Sec:    16655, Lr: 0.000300\n",
      "2020-04-23 22:46:19,855 Epoch  18 Step:    14000 Batch Loss:     2.887350 Tokens per Sec:    16598, Lr: 0.000300\n",
      "2020-04-23 22:46:50,152 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:46:50,152 Saving new checkpoint.\n",
      "2020-04-23 22:46:50,327 Example #0\n",
      "2020-04-23 22:46:50,327 \tSource:     span\n",
      "2020-04-23 22:46:50,327 \tReference:  shibiri\n",
      "2020-04-23 22:46:50,327 \tHypothesis: kikombe\n",
      "2020-04-23 22:46:50,327 Example #1\n",
      "2020-04-23 22:46:50,327 \tSource:     administer\n",
      "2020-04-23 22:46:50,327 \tReference:  amuru\n",
      "2020-04-23 22:46:50,327 \tHypothesis: mshahari\n",
      "2020-04-23 22:46:50,327 Example #2\n",
      "2020-04-23 22:46:50,328 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:46:50,328 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:46:50,328 \tHypothesis: Na kama Shamudi waliangamizwa kwa adhabu iliyo dhaahiri.\n",
      "2020-04-23 22:46:50,328 Example #3\n",
      "2020-04-23 22:46:50,328 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:46:50,328 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:46:50,328 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:46:50,328 Validation result (greedy) at epoch  18, step    14000: bleu:  11.32, loss: 47659.8164, ppl:   9.6831, duration: 30.4726s\n",
      "2020-04-23 22:47:01,545 Epoch  18: total training loss 1940.48\n",
      "2020-04-23 22:47:01,545 EPOCH 19\n",
      "2020-04-23 22:47:04,842 Epoch  19 Step:    14100 Batch Loss:     2.331634 Tokens per Sec:    16098, Lr: 0.000300\n",
      "2020-04-23 22:47:19,378 Epoch  19 Step:    14200 Batch Loss:     2.183769 Tokens per Sec:    16742, Lr: 0.000300\n",
      "2020-04-23 22:47:33,824 Epoch  19 Step:    14300 Batch Loss:     2.381732 Tokens per Sec:    16621, Lr: 0.000300\n",
      "2020-04-23 22:47:48,338 Epoch  19 Step:    14400 Batch Loss:     2.237613 Tokens per Sec:    16518, Lr: 0.000300\n",
      "2020-04-23 22:48:02,961 Epoch  19 Step:    14500 Batch Loss:     2.685239 Tokens per Sec:    16751, Lr: 0.000300\n",
      "2020-04-23 22:48:17,477 Epoch  19 Step:    14600 Batch Loss:     2.372025 Tokens per Sec:    16668, Lr: 0.000300\n",
      "2020-04-23 22:48:31,994 Epoch  19 Step:    14700 Batch Loss:     2.346331 Tokens per Sec:    16571, Lr: 0.000300\n",
      "2020-04-23 22:48:46,346 Epoch  19 Step:    14800 Batch Loss:     2.429731 Tokens per Sec:    16338, Lr: 0.000300\n",
      "2020-04-23 22:48:55,024 Epoch  19: total training loss 1899.57\n",
      "2020-04-23 22:48:55,024 EPOCH 20\n",
      "2020-04-23 22:49:01,007 Epoch  20 Step:    14900 Batch Loss:     2.167306 Tokens per Sec:    16335, Lr: 0.000300\n",
      "2020-04-23 22:49:15,542 Epoch  20 Step:    15000 Batch Loss:     2.299754 Tokens per Sec:    16640, Lr: 0.000300\n",
      "2020-04-23 22:49:45,586 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:49:45,586 Saving new checkpoint.\n",
      "2020-04-23 22:49:45,767 Example #0\n",
      "2020-04-23 22:49:45,767 \tSource:     span\n",
      "2020-04-23 22:49:45,767 \tReference:  shibiri\n",
      "2020-04-23 22:49:45,767 \tHypothesis: kikombe\n",
      "2020-04-23 22:49:45,767 Example #1\n",
      "2020-04-23 22:49:45,767 \tSource:     administer\n",
      "2020-04-23 22:49:45,767 \tReference:  amuru\n",
      "2020-04-23 22:49:45,767 \tHypothesis: usimamizi\n",
      "2020-04-23 22:49:45,767 Example #2\n",
      "2020-04-23 22:49:45,768 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:49:45,768 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:49:45,768 \tHypothesis: Basi walipo angamizwa kwa adhabu ya kushinda adhabu chungu.\n",
      "2020-04-23 22:49:45,768 Example #3\n",
      "2020-04-23 22:49:45,768 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:49:45,768 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:49:45,768 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:49:45,768 Validation result (greedy) at epoch  20, step    15000: bleu:  11.75, loss: 46727.2695, ppl:   9.2623, duration: 30.2262s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 22:50:00,320 Epoch  20 Step:    15100 Batch Loss:     2.390890 Tokens per Sec:    16529, Lr: 0.000300\n",
      "2020-04-23 22:50:14,904 Epoch  20 Step:    15200 Batch Loss:     2.334396 Tokens per Sec:    16701, Lr: 0.000300\n",
      "2020-04-23 22:50:29,267 Epoch  20 Step:    15300 Batch Loss:     2.410015 Tokens per Sec:    16415, Lr: 0.000300\n",
      "2020-04-23 22:50:43,495 Epoch  20 Step:    15400 Batch Loss:     2.147029 Tokens per Sec:    16267, Lr: 0.000300\n",
      "2020-04-23 22:50:57,921 Epoch  20 Step:    15500 Batch Loss:     2.312304 Tokens per Sec:    16435, Lr: 0.000300\n",
      "2020-04-23 22:51:12,357 Epoch  20 Step:    15600 Batch Loss:     2.273867 Tokens per Sec:    16825, Lr: 0.000300\n",
      "2020-04-23 22:51:19,013 Epoch  20: total training loss 1886.47\n",
      "2020-04-23 22:51:19,013 EPOCH 21\n",
      "2020-04-23 22:51:26,991 Epoch  21 Step:    15700 Batch Loss:     2.162075 Tokens per Sec:    16469, Lr: 0.000300\n",
      "2020-04-23 22:51:41,412 Epoch  21 Step:    15800 Batch Loss:     2.388552 Tokens per Sec:    16468, Lr: 0.000300\n",
      "2020-04-23 22:51:55,957 Epoch  21 Step:    15900 Batch Loss:     2.947556 Tokens per Sec:    16652, Lr: 0.000300\n",
      "2020-04-23 22:52:10,433 Epoch  21 Step:    16000 Batch Loss:     2.698986 Tokens per Sec:    16511, Lr: 0.000300\n",
      "2020-04-23 22:52:36,869 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:52:36,869 Saving new checkpoint.\n",
      "2020-04-23 22:52:37,044 Example #0\n",
      "2020-04-23 22:52:37,044 \tSource:     span\n",
      "2020-04-23 22:52:37,044 \tReference:  shibiri\n",
      "2020-04-23 22:52:37,044 \tHypothesis: kikombe\n",
      "2020-04-23 22:52:37,044 Example #1\n",
      "2020-04-23 22:52:37,045 \tSource:     administer\n",
      "2020-04-23 22:52:37,045 \tReference:  amuru\n",
      "2020-04-23 22:52:37,045 \tHypothesis: msimamizi\n",
      "2020-04-23 22:52:37,045 Example #2\n",
      "2020-04-23 22:52:37,045 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:52:37,045 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:52:37,045 \tHypothesis: Basi walipo angamizwa kwa Haki waliangamizwa kwa adhabu ya kudhulumiwa.\n",
      "2020-04-23 22:52:37,045 Example #3\n",
      "2020-04-23 22:52:37,045 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:52:37,045 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:52:37,045 \tHypothesis: Hauamini ila sema:\n",
      "2020-04-23 22:52:37,045 Validation result (greedy) at epoch  21, step    16000: bleu:  12.57, loss: 46084.9180, ppl:   8.9832, duration: 26.6115s\n",
      "2020-04-23 22:52:51,547 Epoch  21 Step:    16100 Batch Loss:     2.278329 Tokens per Sec:    16399, Lr: 0.000300\n",
      "2020-04-23 22:53:06,034 Epoch  21 Step:    16200 Batch Loss:     2.433395 Tokens per Sec:    16471, Lr: 0.000300\n",
      "2020-04-23 22:53:20,547 Epoch  21 Step:    16300 Batch Loss:     2.249481 Tokens per Sec:    16480, Lr: 0.000300\n",
      "2020-04-23 22:53:35,413 Epoch  21 Step:    16400 Batch Loss:     2.395257 Tokens per Sec:    17032, Lr: 0.000300\n",
      "2020-04-23 22:53:39,321 Epoch  21: total training loss 1845.89\n",
      "2020-04-23 22:53:39,321 EPOCH 22\n",
      "2020-04-23 22:53:50,018 Epoch  22 Step:    16500 Batch Loss:     2.179115 Tokens per Sec:    16471, Lr: 0.000300\n",
      "2020-04-23 22:54:04,464 Epoch  22 Step:    16600 Batch Loss:     2.358760 Tokens per Sec:    16594, Lr: 0.000300\n",
      "2020-04-23 22:54:18,865 Epoch  22 Step:    16700 Batch Loss:     2.081321 Tokens per Sec:    16756, Lr: 0.000300\n",
      "2020-04-23 22:54:33,311 Epoch  22 Step:    16800 Batch Loss:     2.336502 Tokens per Sec:    16370, Lr: 0.000300\n",
      "2020-04-23 22:54:47,927 Epoch  22 Step:    16900 Batch Loss:     2.325147 Tokens per Sec:    16605, Lr: 0.000300\n",
      "2020-04-23 22:55:02,594 Epoch  22 Step:    17000 Batch Loss:     2.274856 Tokens per Sec:    16808, Lr: 0.000300\n",
      "2020-04-23 22:55:29,755 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:55:29,755 Saving new checkpoint.\n",
      "2020-04-23 22:55:29,934 Example #0\n",
      "2020-04-23 22:55:29,935 \tSource:     span\n",
      "2020-04-23 22:55:29,935 \tReference:  shibiri\n",
      "2020-04-23 22:55:29,935 \tHypothesis: kipaa\n",
      "2020-04-23 22:55:29,935 Example #1\n",
      "2020-04-23 22:55:29,935 \tSource:     administer\n",
      "2020-04-23 22:55:29,935 \tReference:  amuru\n",
      "2020-04-23 22:55:29,935 \tHypothesis: msimamizi\n",
      "2020-04-23 22:55:29,935 Example #2\n",
      "2020-04-23 22:55:29,935 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:55:29,935 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:55:29,935 \tHypothesis: Na kama Shamudi walivyo angamizwa kwa adhabu iliyo chungu.\n",
      "2020-04-23 22:55:29,935 Example #3\n",
      "2020-04-23 22:55:29,935 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:55:29,935 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:55:29,936 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 22:55:29,936 Validation result (greedy) at epoch  22, step    17000: bleu:  13.15, loss: 45074.7734, ppl:   8.5612, duration: 27.3408s\n",
      "2020-04-23 22:55:44,545 Epoch  22 Step:    17100 Batch Loss:     2.444200 Tokens per Sec:    16758, Lr: 0.000300\n",
      "2020-04-23 22:55:58,957 Epoch  22 Step:    17200 Batch Loss:     2.253491 Tokens per Sec:    16469, Lr: 0.000300\n",
      "2020-04-23 22:55:59,983 Epoch  22: total training loss 1819.47\n",
      "2020-04-23 22:55:59,983 EPOCH 23\n",
      "2020-04-23 22:56:13,593 Epoch  23 Step:    17300 Batch Loss:     2.258239 Tokens per Sec:    16579, Lr: 0.000300\n",
      "2020-04-23 22:56:28,033 Epoch  23 Step:    17400 Batch Loss:     2.408225 Tokens per Sec:    16593, Lr: 0.000300\n",
      "2020-04-23 22:56:42,601 Epoch  23 Step:    17500 Batch Loss:     2.623121 Tokens per Sec:    16896, Lr: 0.000300\n",
      "2020-04-23 22:56:57,154 Epoch  23 Step:    17600 Batch Loss:     2.200125 Tokens per Sec:    16499, Lr: 0.000300\n",
      "2020-04-23 22:57:11,725 Epoch  23 Step:    17700 Batch Loss:     2.428967 Tokens per Sec:    16544, Lr: 0.000300\n",
      "2020-04-23 22:57:26,322 Epoch  23 Step:    17800 Batch Loss:     2.217092 Tokens per Sec:    16760, Lr: 0.000300\n",
      "2020-04-23 22:57:40,707 Epoch  23 Step:    17900 Batch Loss:     2.416362 Tokens per Sec:    16311, Lr: 0.000300\n",
      "2020-04-23 22:57:53,368 Epoch  23: total training loss 1798.27\n",
      "2020-04-23 22:57:53,368 EPOCH 24\n",
      "2020-04-23 22:57:55,444 Epoch  24 Step:    18000 Batch Loss:     2.489506 Tokens per Sec:    15824, Lr: 0.000300\n",
      "2020-04-23 22:58:22,280 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 22:58:22,280 Saving new checkpoint.\n",
      "2020-04-23 22:58:22,462 Example #0\n",
      "2020-04-23 22:58:22,462 \tSource:     span\n",
      "2020-04-23 22:58:22,462 \tReference:  shibiri\n",
      "2020-04-23 22:58:22,462 \tHypothesis: kikombe\n",
      "2020-04-23 22:58:22,462 Example #1\n",
      "2020-04-23 22:58:22,463 \tSource:     administer\n",
      "2020-04-23 22:58:22,463 \tReference:  amuru\n",
      "2020-04-23 22:58:22,463 \tHypothesis: mhudumu\n",
      "2020-04-23 22:58:22,463 Example #2\n",
      "2020-04-23 22:58:22,463 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 22:58:22,463 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 22:58:22,463 \tHypothesis: Basi walipo kuwa wakiangamizwa kwa adhabu ya kushinda adhabu chungu.\n",
      "2020-04-23 22:58:22,463 Example #3\n",
      "2020-04-23 22:58:22,463 \tSource:     You do not believe but say,\n",
      "2020-04-23 22:58:22,463 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 22:58:22,463 \tHypothesis: Hauamini ila sema:\n",
      "2020-04-23 22:58:22,463 Validation result (greedy) at epoch  24, step    18000: bleu:  13.54, loss: 44397.3945, ppl:   8.2893, duration: 27.0188s\n",
      "2020-04-23 22:58:37,025 Epoch  24 Step:    18100 Batch Loss:     2.314976 Tokens per Sec:    16659, Lr: 0.000300\n",
      "2020-04-23 22:58:51,684 Epoch  24 Step:    18200 Batch Loss:     2.283128 Tokens per Sec:    16860, Lr: 0.000300\n",
      "2020-04-23 22:59:06,282 Epoch  24 Step:    18300 Batch Loss:     2.152310 Tokens per Sec:    16469, Lr: 0.000300\n",
      "2020-04-23 22:59:20,756 Epoch  24 Step:    18400 Batch Loss:     2.358837 Tokens per Sec:    16534, Lr: 0.000300\n",
      "2020-04-23 22:59:35,147 Epoch  24 Step:    18500 Batch Loss:     2.207381 Tokens per Sec:    16515, Lr: 0.000300\n",
      "2020-04-23 22:59:49,636 Epoch  24 Step:    18600 Batch Loss:     2.861652 Tokens per Sec:    16590, Lr: 0.000300\n",
      "2020-04-23 23:00:04,016 Epoch  24 Step:    18700 Batch Loss:     2.312243 Tokens per Sec:    16622, Lr: 0.000300\n",
      "2020-04-23 23:00:13,800 Epoch  24: total training loss 1774.92\n",
      "2020-04-23 23:00:13,800 EPOCH 25\n",
      "2020-04-23 23:00:18,759 Epoch  25 Step:    18800 Batch Loss:     2.148660 Tokens per Sec:    16432, Lr: 0.000300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 23:00:33,295 Epoch  25 Step:    18900 Batch Loss:     2.188258 Tokens per Sec:    16615, Lr: 0.000300\n",
      "2020-04-23 23:00:47,712 Epoch  25 Step:    19000 Batch Loss:     2.804485 Tokens per Sec:    16653, Lr: 0.000300\n",
      "2020-04-23 23:01:14,315 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 23:01:14,315 Saving new checkpoint.\n",
      "2020-04-23 23:01:14,495 Example #0\n",
      "2020-04-23 23:01:14,495 \tSource:     span\n",
      "2020-04-23 23:01:14,495 \tReference:  shibiri\n",
      "2020-04-23 23:01:14,495 \tHypothesis: kikombe\n",
      "2020-04-23 23:01:14,495 Example #1\n",
      "2020-04-23 23:01:14,495 \tSource:     administer\n",
      "2020-04-23 23:01:14,495 \tReference:  amuru\n",
      "2020-04-23 23:01:14,495 \tHypothesis: msimamizi\n",
      "2020-04-23 23:01:14,495 Example #2\n",
      "2020-04-23 23:01:14,496 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 23:01:14,496 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 23:01:14,496 \tHypothesis: Basi kwa Shamudi waliangamizwa kwa adhabu iliyo chungu.\n",
      "2020-04-23 23:01:14,496 Example #3\n",
      "2020-04-23 23:01:14,496 \tSource:     You do not believe but say,\n",
      "2020-04-23 23:01:14,496 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 23:01:14,496 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 23:01:14,496 Validation result (greedy) at epoch  25, step    19000: bleu:  13.88, loss: 44029.9844, ppl:   8.1455, duration: 26.7836s\n",
      "2020-04-23 23:01:28,977 Epoch  25 Step:    19100 Batch Loss:     2.091301 Tokens per Sec:    16603, Lr: 0.000300\n",
      "2020-04-23 23:01:43,484 Epoch  25 Step:    19200 Batch Loss:     2.104844 Tokens per Sec:    16817, Lr: 0.000300\n",
      "2020-04-23 23:01:58,014 Epoch  25 Step:    19300 Batch Loss:     2.082567 Tokens per Sec:    16598, Lr: 0.000300\n",
      "2020-04-23 23:02:12,661 Epoch  25 Step:    19400 Batch Loss:     2.302405 Tokens per Sec:    16671, Lr: 0.000300\n",
      "2020-04-23 23:02:27,156 Epoch  25 Step:    19500 Batch Loss:     2.238170 Tokens per Sec:    16693, Lr: 0.000300\n",
      "2020-04-23 23:02:33,857 Epoch  25: total training loss 1757.56\n",
      "2020-04-23 23:02:33,857 EPOCH 26\n",
      "2020-04-23 23:02:41,812 Epoch  26 Step:    19600 Batch Loss:     2.216417 Tokens per Sec:    16864, Lr: 0.000300\n",
      "2020-04-23 23:02:56,273 Epoch  26 Step:    19700 Batch Loss:     2.042866 Tokens per Sec:    16608, Lr: 0.000300\n",
      "2020-04-23 23:03:10,769 Epoch  26 Step:    19800 Batch Loss:     2.042612 Tokens per Sec:    16607, Lr: 0.000300\n",
      "2020-04-23 23:03:25,219 Epoch  26 Step:    19900 Batch Loss:     2.013043 Tokens per Sec:    16748, Lr: 0.000300\n",
      "2020-04-23 23:03:39,628 Epoch  26 Step:    20000 Batch Loss:     2.140165 Tokens per Sec:    16653, Lr: 0.000300\n",
      "2020-04-23 23:04:05,617 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 23:04:05,618 Saving new checkpoint.\n",
      "2020-04-23 23:04:05,795 Example #0\n",
      "2020-04-23 23:04:05,795 \tSource:     span\n",
      "2020-04-23 23:04:05,796 \tReference:  shibiri\n",
      "2020-04-23 23:04:05,796 \tHypothesis: kikombe\n",
      "2020-04-23 23:04:05,796 Example #1\n",
      "2020-04-23 23:04:05,796 \tSource:     administer\n",
      "2020-04-23 23:04:05,796 \tReference:  amuru\n",
      "2020-04-23 23:04:05,796 \tHypothesis: mstari\n",
      "2020-04-23 23:04:05,796 Example #2\n",
      "2020-04-23 23:04:05,796 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 23:04:05,796 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 23:04:05,796 \tHypothesis: Na kama Shamudi waliangamizwa kwa adhabu ya kudumu adhabu chungu.\n",
      "2020-04-23 23:04:05,796 Example #3\n",
      "2020-04-23 23:04:05,796 \tSource:     You do not believe but say,\n",
      "2020-04-23 23:04:05,797 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 23:04:05,797 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 23:04:05,797 Validation result (greedy) at epoch  26, step    20000: bleu:  14.34, loss: 43467.5586, ppl:   7.9302, duration: 26.1682s\n",
      "2020-04-23 23:04:20,338 Epoch  26 Step:    20100 Batch Loss:     2.340356 Tokens per Sec:    16596, Lr: 0.000300\n",
      "2020-04-23 23:04:34,890 Epoch  26 Step:    20200 Batch Loss:     2.132986 Tokens per Sec:    16721, Lr: 0.000300\n",
      "2020-04-23 23:04:49,308 Epoch  26 Step:    20300 Batch Loss:     2.111017 Tokens per Sec:    16487, Lr: 0.000300\n",
      "2020-04-23 23:04:53,139 Epoch  26: total training loss 1734.92\n",
      "2020-04-23 23:04:53,140 EPOCH 27\n",
      "2020-04-23 23:05:04,036 Epoch  27 Step:    20400 Batch Loss:     2.487979 Tokens per Sec:    16384, Lr: 0.000300\n",
      "2020-04-23 23:05:18,549 Epoch  27 Step:    20500 Batch Loss:     2.268808 Tokens per Sec:    16695, Lr: 0.000300\n",
      "2020-04-23 23:05:33,129 Epoch  27 Step:    20600 Batch Loss:     2.303560 Tokens per Sec:    16824, Lr: 0.000300\n",
      "2020-04-23 23:05:47,561 Epoch  27 Step:    20700 Batch Loss:     2.133076 Tokens per Sec:    16768, Lr: 0.000300\n",
      "2020-04-23 23:06:02,206 Epoch  27 Step:    20800 Batch Loss:     2.735285 Tokens per Sec:    16849, Lr: 0.000300\n",
      "2020-04-23 23:06:16,757 Epoch  27 Step:    20900 Batch Loss:     2.033327 Tokens per Sec:    16739, Lr: 0.000300\n",
      "2020-04-23 23:06:31,210 Epoch  27 Step:    21000 Batch Loss:     2.575073 Tokens per Sec:    16419, Lr: 0.000300\n",
      "2020-04-23 23:06:58,049 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 23:06:58,049 Saving new checkpoint.\n",
      "2020-04-23 23:06:58,228 Example #0\n",
      "2020-04-23 23:06:58,228 \tSource:     span\n",
      "2020-04-23 23:06:58,228 \tReference:  shibiri\n",
      "2020-04-23 23:06:58,229 \tHypothesis: kipambizi\n",
      "2020-04-23 23:06:58,229 Example #1\n",
      "2020-04-23 23:06:58,229 \tSource:     administer\n",
      "2020-04-23 23:06:58,229 \tReference:  amuru\n",
      "2020-04-23 23:06:58,229 \tHypothesis: usimamizi\n",
      "2020-04-23 23:06:58,229 Example #2\n",
      "2020-04-23 23:06:58,229 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 23:06:58,229 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 23:06:58,229 \tHypothesis: Basi walipo angamizwa kwa Haki waliangamizwa kwa adhabu ya kudumu.\n",
      "2020-04-23 23:06:58,229 Example #3\n",
      "2020-04-23 23:06:58,230 \tSource:     You do not believe but say,\n",
      "2020-04-23 23:06:58,230 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 23:06:58,230 \tHypothesis: Hamamini ila husema:\n",
      "2020-04-23 23:06:58,230 Validation result (greedy) at epoch  27, step    21000: bleu:  14.32, loss: 42797.8086, ppl:   7.6811, duration: 27.0188s\n",
      "2020-04-23 23:07:12,565 Epoch  27 Step:    21100 Batch Loss:     2.206257 Tokens per Sec:    16247, Lr: 0.000300\n",
      "2020-04-23 23:07:13,443 Epoch  27: total training loss 1718.35\n",
      "2020-04-23 23:07:13,443 EPOCH 28\n",
      "2020-04-23 23:07:27,210 Epoch  28 Step:    21200 Batch Loss:     2.034484 Tokens per Sec:    16544, Lr: 0.000300\n",
      "2020-04-23 23:07:41,783 Epoch  28 Step:    21300 Batch Loss:     2.150079 Tokens per Sec:    16399, Lr: 0.000300\n",
      "2020-04-23 23:07:56,475 Epoch  28 Step:    21400 Batch Loss:     2.293908 Tokens per Sec:    16973, Lr: 0.000300\n",
      "2020-04-23 23:08:11,034 Epoch  28 Step:    21500 Batch Loss:     2.262740 Tokens per Sec:    16510, Lr: 0.000300\n",
      "2020-04-23 23:08:25,702 Epoch  28 Step:    21600 Batch Loss:     2.062630 Tokens per Sec:    16717, Lr: 0.000300\n",
      "2020-04-23 23:08:40,285 Epoch  28 Step:    21700 Batch Loss:     2.334095 Tokens per Sec:    16469, Lr: 0.000300\n",
      "2020-04-23 23:08:54,749 Epoch  28 Step:    21800 Batch Loss:     1.952248 Tokens per Sec:    16551, Lr: 0.000300\n",
      "2020-04-23 23:09:06,946 Epoch  28: total training loss 1702.11\n",
      "2020-04-23 23:09:06,946 EPOCH 29\n",
      "2020-04-23 23:09:09,422 Epoch  29 Step:    21900 Batch Loss:     2.674916 Tokens per Sec:    15904, Lr: 0.000300\n",
      "2020-04-23 23:09:23,673 Epoch  29 Step:    22000 Batch Loss:     2.251612 Tokens per Sec:    16397, Lr: 0.000300\n",
      "2020-04-23 23:09:53,294 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 23:09:53,294 Saving new checkpoint.\n",
      "2020-04-23 23:09:53,480 Example #0\n",
      "2020-04-23 23:09:53,480 \tSource:     span\n",
      "2020-04-23 23:09:53,480 \tReference:  shibiri\n",
      "2020-04-23 23:09:53,480 \tHypothesis: kipwani\n",
      "2020-04-23 23:09:53,481 Example #1\n",
      "2020-04-23 23:09:53,481 \tSource:     administer\n",
      "2020-04-23 23:09:53,481 \tReference:  amuru\n",
      "2020-04-23 23:09:53,481 \tHypothesis: mdawa\n",
      "2020-04-23 23:09:53,481 Example #2\n",
      "2020-04-23 23:09:53,481 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 23:09:53,481 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 23:09:53,481 \tHypothesis: Basi walipo angamizwa kwa Shamudi waliangamizwa kwa adhabu ya kudumu adhabu kali.\n",
      "2020-04-23 23:09:53,481 Example #3\n",
      "2020-04-23 23:09:53,481 \tSource:     You do not believe but say,\n",
      "2020-04-23 23:09:53,481 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 23:09:53,481 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 23:09:53,482 Validation result (greedy) at epoch  29, step    22000: bleu:  14.68, loss: 42596.0469, ppl:   7.6077, duration: 29.8080s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-23 23:10:07,971 Epoch  29 Step:    22100 Batch Loss:     2.003094 Tokens per Sec:    16558, Lr: 0.000300\n",
      "2020-04-23 23:10:22,307 Epoch  29 Step:    22200 Batch Loss:     2.184560 Tokens per Sec:    16531, Lr: 0.000300\n",
      "2020-04-23 23:10:36,912 Epoch  29 Step:    22300 Batch Loss:     2.463721 Tokens per Sec:    16552, Lr: 0.000300\n",
      "2020-04-23 23:10:51,446 Epoch  29 Step:    22400 Batch Loss:     2.145364 Tokens per Sec:    16865, Lr: 0.000300\n",
      "2020-04-23 23:11:05,798 Epoch  29 Step:    22500 Batch Loss:     2.287599 Tokens per Sec:    16605, Lr: 0.000300\n",
      "2020-04-23 23:11:20,231 Epoch  29 Step:    22600 Batch Loss:     2.072596 Tokens per Sec:    16695, Lr: 0.000300\n",
      "2020-04-23 23:11:30,196 Epoch  29: total training loss 1699.23\n",
      "2020-04-23 23:11:30,197 EPOCH 30\n",
      "2020-04-23 23:11:34,956 Epoch  30 Step:    22700 Batch Loss:     1.953430 Tokens per Sec:    16315, Lr: 0.000300\n",
      "2020-04-23 23:11:49,298 Epoch  30 Step:    22800 Batch Loss:     1.949152 Tokens per Sec:    16591, Lr: 0.000300\n",
      "2020-04-23 23:12:03,783 Epoch  30 Step:    22900 Batch Loss:     2.170377 Tokens per Sec:    16817, Lr: 0.000300\n",
      "2020-04-23 23:12:18,038 Epoch  30 Step:    23000 Batch Loss:     2.233912 Tokens per Sec:    16550, Lr: 0.000300\n",
      "2020-04-23 23:12:44,373 Hooray! New best validation result [ppl]!\n",
      "2020-04-23 23:12:44,373 Saving new checkpoint.\n",
      "2020-04-23 23:12:44,549 Example #0\n",
      "2020-04-23 23:12:44,550 \tSource:     span\n",
      "2020-04-23 23:12:44,550 \tReference:  shibiri\n",
      "2020-04-23 23:12:44,550 \tHypothesis: kipingo\n",
      "2020-04-23 23:12:44,550 Example #1\n",
      "2020-04-23 23:12:44,550 \tSource:     administer\n",
      "2020-04-23 23:12:44,550 \tReference:  amuru\n",
      "2020-04-23 23:12:44,550 \tHypothesis: msimamizi\n",
      "2020-04-23 23:12:44,550 Example #2\n",
      "2020-04-23 23:12:44,550 \tSource:     Then as to Samood, they were destroyed by an excessively severe punishment.\n",
      "2020-04-23 23:12:44,550 \tReference:  Basi Thamudi waliangamizwa kwa balaa kubwa mno.\n",
      "2020-04-23 23:12:44,551 \tHypothesis: Basi walipo angamizwa kwa Haki waliangamizwa kwa adhabu ya Moto ulio dhaahiri.\n",
      "2020-04-23 23:12:44,551 Example #3\n",
      "2020-04-23 23:12:44,551 \tSource:     You do not believe but say,\n",
      "2020-04-23 23:12:44,551 \tReference:  Hamjaamini, lakini semeni:\n",
      "2020-04-23 23:12:44,551 \tHypothesis: Hauamini ila husema:\n",
      "2020-04-23 23:12:44,551 Validation result (greedy) at epoch  30, step    23000: bleu:  15.61, loss: 42209.5742, ppl:   7.4689, duration: 26.5120s\n",
      "2020-04-23 23:12:59,089 Epoch  30 Step:    23100 Batch Loss:     2.647570 Tokens per Sec:    16446, Lr: 0.000300\n",
      "2020-04-23 23:13:13,662 Epoch  30 Step:    23200 Batch Loss:     2.180224 Tokens per Sec:    16860, Lr: 0.000300\n",
      "2020-04-23 23:13:28,137 Epoch  30 Step:    23300 Batch Loss:     2.112861 Tokens per Sec:    16474, Lr: 0.000300\n",
      "2020-04-23 23:13:42,518 Epoch  30 Step:    23400 Batch Loss:     2.192046 Tokens per Sec:    16882, Lr: 0.000300\n",
      "2020-04-23 23:13:49,708 Epoch  30: total training loss 1676.35\n",
      "2020-04-23 23:13:49,708 Training ended after  30 epochs.\n",
      "2020-04-23 23:13:49,708 Best validation result (greedy) at step    23000:   7.47 ppl.\n",
      "/pytorch/aten/src/ATen/native/BinaryOps.cpp:81: UserWarning: Integer division of tensors using div or / is deprecated, and in a future release div will perform true division as in Python 3. Use true_divide or floor_divide (// in Python) instead.\n",
      "2020-04-23 23:14:12,660  dev bleu:  15.00 [Beam search decoding with beam size = 5 and alpha = 1.0]\n",
      "2020-04-23 23:14:12,661 Translations saved to: models/ensw_transformer/00023000.hyps.dev\n",
      "2020-04-23 23:15:04,105 test bleu:   3.60 [Beam search decoding with beam size = 5 and alpha = 1.0]\n",
      "2020-04-23 23:15:04,107 Translations saved to: models/ensw_transformer/00023000.hyps.test\n",
      "benchmarks.md\t    data     joey-small.png  README.md\t       setup.py\n",
      "CODE_OF_CONDUCT.md  docs     LICENSE\t     requirements.txt  test\n",
      "configs\t\t    joeynmt  models\t     scripts\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "# You can press Ctrl-C to stop. And then run the next cell to save your checkpoints! \n",
    "!cd joeynmt;pwd; ls; python3 -m joeynmt train ./configs/transformer_ensw2.yaml;ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#copy saved models\n",
    "!cp -r joeynmt/models/{name}_transformer/* my_sawa_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#copy saved models2\n",
    "!mkdir sawa_models\n",
    "!cp -r joeynmt/models/ensw_transformer/* sawa_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}