{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66b70728",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading the tokenizer from the `special_tokens_map.json` and the `added_tokens.json` will be removed in `transformers 5`,  it is kept for forward compatibility, but it is recommended to update your `tokenizer_config.json` by uploading it again. You will see the new `added_tokens_decoder` attribute that will store the relevant information.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
    "import torch\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('meta-llama/Llama-2-13b-hf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d608fb34",
   "metadata": {},
   "outputs": [],
   "source": [
    "nf4_config = BitsAndBytesConfig(\n",
    "   load_in_4bit=True,\n",
    "   bnb_4bit_quant_type='nf4',\n",
    "   bnb_4bit_use_double_quant=True,\n",
    "   bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4d63a1ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b003c36bd0a64b0f97b7868bd7e04a7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/609 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-09-28 13:24:24,730] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-28 13:24:28.218982: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-09-28 13:24:28.911407: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ce8339d5c0846adb3a94e4a22c9e59b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)fetensors.index.json:   0%|          | 0.00/26.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b4905f1af81490a9f55683c90cbcda9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7339175f24e24272b07e2715df854538",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c754d8f6cd747cd84eef4187820fd88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "057d41f35eab4955b4952beaf2aa8417",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "965c5c7ec4de4c8c8cc901e2e5b70dbd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/188 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_model = AutoModelForCausalLM.from_pretrained('meta-llama/Llama-2-7b-hf', quantization_config=nf4_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "18177b55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "397bcf8440944efcba4369046e329954",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/628 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dedb4a297bcc4c5c8b31b49fd7708e1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)fetensors.index.json:   0%|          | 0.00/26.8k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0d6111f52424e8dabee6d2410b440d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0191a9254c44bbdaa3913d4a56717f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/9.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fff09b462e5e45f7868a5f004cab7dc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)of-00002.safetensors:   0%|          | 0.00/3.50G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f8ab76dc6104036bda0aecc62628a6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "657f81ff29324d44ae6ff19d42113353",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)neration_config.json:   0%|          | 0.00/183 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpf_model = AutoModelForCausalLM.from_pretrained('mesolitica/llama-7b-hf-32768-fpf', quantization_config=nf4_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "af485f12",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "kwargs = {\n",
    "    'temperature': 0.9, \n",
    "    'max_new_tokens': 256, \n",
    "    'top_p': 0.95, \n",
    "    'repetition_penalty': 1.0, \n",
    "    'do_sample': True,\n",
    "    'num_beams': 1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8209faad",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(['ketiak ak masham'], return_tensors='pt').to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "07251c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_kwargs = dict(inputs)\n",
    "generate_kwargs = {**generate_kwargs, **kwargs}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5303ac9",
   "metadata": {},
   "source": [
    "## Base 7B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9c0cbb2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "o = base_model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e102daa2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ketiak ak mashamat 17. броја (1282) - Pismo LXXVII\n",
      "Dodatkowe usytuowanie elementów elementów stropowych 6.d. - wykonanie 6.d.\n",
      "The aim of this work was to design a new and simple solution for the steel trusses' elements, which could be used in the construction of light-industrial buildings, as well as in the construction of the buildings in general. The goal of the research was to develop a new method of positioning the steel trusses, which would be better adapted to the construction of light-industrial buildings. The design of the new truss, which can be easily placed during the construction process, was prepared in a three-dimensional model, thanks to which the parameters of the truss were determined. As a result, it was found that the trusses have the following parameters: width of the arch: 64 cm and 111 cm, the length of the straight side: 60 cm, 150 cm and 175 cm, the angle of inclination of the arch side: 29, 53 and 68.46 °, the\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a817fd0b",
   "metadata": {},
   "source": [
    "## Malaysian Llama2 7B 32k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f26fa04a",
   "metadata": {},
   "outputs": [],
   "source": [
    "o = fpf_model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b70e4dac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ketiak ak masham. itu ak yg masham..hahahahhahahha..tapi sebenarnya kita takut nak duduk sebelah penyangkut baju..hahahahha! ak penakut dik..hehe..xnk duduk dkt2..ak jenis kena dptkan sesuatu tu dulu baru aku join dgn group..hahahaha! Aku kena cepat. Sbb ak jenis nak melompat..haha..kalau jenis yg suka duduk diam2, boleh la duduk kat penyangkut baju tuu. Tp ak jenis dlm drama, ak yg masham. Tp sebenarnya ak yg penakut..haha.. ak boleh plak kta ak penakut..haha..ak ni jenis kuat melompat..haha..so aku jenis takut lah..haha..pastu klu dh join group, ak suka yg ak kena first..hahahahaha..ak x\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "951886d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(['harga barang kat malaysia ni semakin naik, apa kita nak buat'], return_tensors='pt').to('cuda')\n",
    "generate_kwargs = dict(inputs)\n",
    "generate_kwargs = {**generate_kwargs, **kwargs}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad00d89f",
   "metadata": {},
   "source": [
    "## Base 7B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "001032d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "o = base_model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "41b76cc6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "harga barang kat malaysia ni semakin naik, apa kita nak buat..\n",
      " насељать, сравнить и привести в пример.\n",
      "He was appointed a judge in the Supreme Court and later, the Chief Justice of the Supreme Court.\n",
      "They are all from outside of the country.\n",
      "He was appointed a judge in the Supreme Court and later, the Chief Justice of the Supreme Court.\n",
      "It is not the first time that he had resigned in the last 10 years.\n",
      "He was appointed a judge in the Supreme Court and later, the Chief Justice of the Supreme Court. It is not the first time that he had resigned in the last 10 years.\n",
      "You have to learn to say that this is not the first time that he had resigned in the last 10 years.\n",
      "I was appointed a judge in the Supreme Court and later, the Chief Justice of the Supreme Court.\n",
      "It is not the first time that I had resigned in the last 10 years.\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c363f5a3",
   "metadata": {},
   "source": [
    "## Malaysian Llama2 7B 32k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "54bfed7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "o = fpf_model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a2e16fb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "harga barang kat malaysia ni semakin naik, apa kita nak buat? mula bisnes dari rumah., jual barang tak banyak pun, cukup buat kita makan. cuma kita kena keluar modal sikit untuk dapatkan bekalan. tapi sekali keluar modal, selepas tu, tak perlu keluar modal lagi. jual kepada rakan2, keluarga, atau jual di rumah sebagai contoh. tak rugi pun, dapat jual dapat untung. syarat kena ada modal la. sbb kita tak boleh nak untung banyak. kita nak jual sikit pun dah cukup, yang penting kita tak perlu bayar utk ambil barang. lagi satu sekali, kita nak keluarkan modal, kita kena tau mana nak beli barang yang kita nak jual tu. jangan beli kat pasaraya, jangan beli kat pembekal yang mahal, beli je kat mana-mana y\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.decode(o[0], skip_special_tokens = True).split('[/INST]')[-1].strip())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}