PyThaiNLP v3.1.0-dev3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ad3958c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pythainlp[full]\n",
      "  Downloading pythainlp-3.1.0.dev3-py3-none-any.whl (9.6 MB)\n",
      "\u001b[K     |████████████████████████████████| 9.6 MB 9.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.22.0 in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (2.26.0)\n",
      "Collecting tltk>=1.3.8\n",
      "  Downloading tltk-1.5.7.tar.gz (18.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 18.4 MB 53.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyicu>=2.3\n",
      "  Downloading PyICU-2.9.tar.gz (305 kB)\n",
      "\u001b[K     |████████████████████████████████| 305 kB 53.0 MB/s eta 0:00:01\n",
      "\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h    Preparing wheel metadata ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting attacut>=1.0.4\n",
      "  Downloading attacut-1.1.0.dev0-py3-none-any.whl (1.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3 MB 59.7 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: PyYAML>=5.3.1 in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (5.4.1)\n",
      "Collecting epitran>=1.1\n",
      "  Downloading epitran-1.23-py2.py3-none-any.whl (164 kB)\n",
      "\u001b[K     |████████████████████████████████| 164 kB 63.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pandas>=0.24 in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (1.3.3)\n",
      "Collecting ufal.chu-liu-edmonds>=1.0.2\n",
      "  Downloading ufal.chu_liu_edmonds-1.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (107 kB)\n",
      "\u001b[K     |████████████████████████████████| 107 kB 50.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting emoji>=0.5.1\n",
      "  Downloading emoji-2.1.0.tar.gz (216 kB)\n",
      "\u001b[K     |████████████████████████████████| 216 kB 56.3 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting esupar>=1.3.8\n",
      "  Downloading esupar-1.3.8-py3-none-any.whl (34 kB)\n",
      "Collecting thai-nner\n",
      "  Downloading thai_nner-0.3-py3-none-any.whl (2.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 2.2 MB 67.0 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting fairseq>=0.10.0\n",
      "  Downloading fairseq-0.12.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 11.0 MB 51.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting sefr-cut>=1.1\n",
      "  Downloading SEFR_CUT-1.1-py3-none-any.whl (8.7 MB)\n",
      "\u001b[K     |████████████████████████████████| 8.7 MB 59.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting bpemb>=0.3.2\n",
      "  Downloading bpemb-0.3.3-py3-none-any.whl (19 kB)\n",
      "Requirement already satisfied: sacremoses>=0.0.41 in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (0.0.53)\n",
      "Collecting gensim>=4.0.0\n",
      "  Downloading gensim-4.2.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (24.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 24.1 MB 57.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting ssg>=0.0.8\n",
      "  Downloading ssg-0.0.8-py3-none-any.whl (473 kB)\n",
      "\u001b[K     |████████████████████████████████| 473 kB 59.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting onnxruntime>=1.10.0\n",
      "  Downloading onnxruntime-1.12.1-cp38-cp38-manylinux_2_27_x86_64.whl (4.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 4.9 MB 67.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting wunsen>=0.0.3\n",
      "  Downloading wunsen-0.0.3-py3-none-any.whl (21 kB)\n",
      "Collecting oskut>=1.3\n",
      "  Downloading OSKut-1.3-py3-none-any.whl (44.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 44.2 MB 51.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting nlpo3>=1.2.2\n",
      "  Downloading nlpo3-1.2.2-cp38-cp38-manylinux2014_x86_64.whl (1.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.5 MB 60.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting spylls>=0.1.5\n",
      "  Downloading spylls-0.1.7-py2.py3-none-any.whl (1.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.5 MB 32.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: torch>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (1.9.0+cu111)\n",
      "Collecting numpy>=1.22\n",
      "  Downloading numpy-1.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 17.1 MB 12.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting fastai<2.0\n",
      "  Downloading fastai-1.0.61-py3-none-any.whl (239 kB)\n",
      "\u001b[K     |████████████████████████████████| 239 kB 67.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: sentencepiece>=0.1.91 in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (0.1.97)\n",
      "Collecting spacy-thai>=0.7.1\n",
      "  Downloading spacy_thai-0.7.1-py3-none-any.whl (6.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 6.0 MB 59.3 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting symspellpy>=6.7.6\n",
      "  Downloading symspellpy-6.7.6-py3-none-any.whl (2.6 MB)\n",
      "\u001b[K     |████████████████████████████████| 2.6 MB 58.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting phunspell>=0.1.6\n",
      "  Downloading phunspell-0.1.6.tar.gz (47.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 47.5 MB 70.1 MB/s eta 0:00:01\n",
      "\u001b[?25h  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h    Preparing wheel metadata ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting transformers>=4.22.1\n",
      "  Downloading transformers-4.22.1-py3-none-any.whl (4.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 4.9 MB 58.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: nltk>=3.3.* in /opt/conda/lib/python3.8/site-packages (from pythainlp[full]) (3.7)\n",
      "Requirement already satisfied: docopt>=0.6.2 in /opt/conda/lib/python3.8/site-packages (from attacut>=1.0.4->pythainlp[full]) (0.6.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.8/site-packages (from attacut>=1.0.4->pythainlp[full]) (1.15.0)\n",
      "Collecting nptyping>=0.2.0\n",
      "  Downloading nptyping-2.3.1-py3-none-any.whl (32 kB)\n",
      "Collecting fire>=0.1.3\n",
      "  Downloading fire-0.4.0.tar.gz (87 kB)\n",
      "\u001b[K     |████████████████████████████████| 87 kB 63.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.8/site-packages (from bpemb>=0.3.2->pythainlp[full]) (4.62.2)\n",
      "Collecting marisa-trie\n",
      "  Downloading marisa_trie-0.7.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3 MB 57.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: regex in /opt/conda/lib/python3.8/site-packages (from epitran>=1.1->pythainlp[full]) (2022.8.17)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from epitran>=1.1->pythainlp[full]) (59.5.0)\n",
      "Collecting panphon>=0.20\n",
      "  Downloading panphon-0.20.0-py2.py3-none-any.whl (73 kB)\n",
      "\u001b[K     |████████████████████████████████| 73 kB 40.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting deplacy>=2.0.3\n",
      "  Downloading deplacy-2.0.3-py3-none-any.whl (22 kB)\n",
      "Collecting supar>=1.1.4\n",
      "  Downloading supar-1.1.4-py3-none-any.whl (93 kB)\n",
      "\u001b[K     |████████████████████████████████| 93 kB 18.9 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting omegaconf<2.1\n",
      "  Downloading omegaconf-2.0.6-py3-none-any.whl (36 kB)\n",
      "Collecting bitarray\n",
      "  Downloading bitarray-2.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (241 kB)\n",
      "\u001b[K     |████████████████████████████████| 241 kB 50.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: sacrebleu>=1.4.12 in /opt/conda/lib/python3.8/site-packages (from fairseq>=0.10.0->pythainlp[full]) (2.2.0)\n",
      "Collecting hydra-core<1.1,>=1.0.7\n",
      "  Downloading hydra_core-1.0.7-py3-none-any.whl (123 kB)\n",
      "\u001b[K     |████████████████████████████████| 123 kB 56.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: cython in /opt/conda/lib/python3.8/site-packages (from fairseq>=0.10.0->pythainlp[full]) (0.29.32)\n",
      "Collecting torchaudio>=0.8.0\n",
      "  Downloading torchaudio-0.12.1-cp38-cp38-manylinux1_x86_64.whl (3.7 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.7 MB 55.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: cffi in /opt/conda/lib/python3.8/site-packages (from fairseq>=0.10.0->pythainlp[full]) (1.14.6)\n",
      "Collecting numexpr\n",
      "  Downloading numexpr-2.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (381 kB)\n",
      "\u001b[K     |████████████████████████████████| 381 kB 51.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting nvidia-ml-py3\n",
      "  Downloading nvidia-ml-py3-7.352.0.tar.gz (19 kB)\n",
      "Collecting fastprogress>=0.2.1\n",
      "  Downloading fastprogress-1.0.3-py3-none-any.whl (12 kB)\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.8/site-packages (from fastai<2.0->pythainlp[full]) (4.11.1)\n",
      "Requirement already satisfied: Pillow in /opt/conda/lib/python3.8/site-packages (from fastai<2.0->pythainlp[full]) (8.3.2)\n",
      "Collecting bottleneck\n",
      "  Downloading Bottleneck-1.3.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (355 kB)\n",
      "\u001b[K     |████████████████████████████████| 355 kB 51.8 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: torchvision in /opt/conda/lib/python3.8/site-packages (from fastai<2.0->pythainlp[full]) (0.10.0+cu111)\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.8/site-packages (from fastai<2.0->pythainlp[full]) (1.7.1)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from fastai<2.0->pythainlp[full]) (21.0)\n",
      "Requirement already satisfied: matplotlib in /opt/conda/lib/python3.8/site-packages (from fastai<2.0->pythainlp[full]) (3.4.3)\n",
      "Requirement already satisfied: termcolor in /opt/conda/lib/python3.8/site-packages (from fire>=0.1.3->attacut>=1.0.4->pythainlp[full]) (1.1.0)\n",
      "Collecting smart-open>=1.8.1\n",
      "  Downloading smart_open-6.2.0-py3-none-any.whl (58 kB)\n",
      "\u001b[K     |████████████████████████████████| 58 kB 60.3 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: antlr4-python3-runtime==4.8 in /opt/conda/lib/python3.8/site-packages (from hydra-core<1.1,>=1.0.7->fairseq>=0.10.0->pythainlp[full]) (4.8)\n",
      "Requirement already satisfied: importlib-resources in /opt/conda/lib/python3.8/site-packages (from hydra-core<1.1,>=1.0.7->fairseq>=0.10.0->pythainlp[full]) (5.2.3)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from nltk>=3.3.*->pythainlp[full]) (8.1.3)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from nltk>=3.3.*->pythainlp[full]) (1.1.0)\n",
      "Requirement already satisfied: typing-extensions<5.0.0,>=4.0.0 in /opt/conda/lib/python3.8/site-packages (from nptyping>=0.2.0->attacut>=1.0.4->pythainlp[full]) (4.3.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.8/site-packages (from onnxruntime>=1.10.0->pythainlp[full]) (1.11.1)\n",
      "Collecting coloredlogs\n",
      "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
      "\u001b[K     |████████████████████████████████| 46 kB 51.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: flatbuffers in /opt/conda/lib/python3.8/site-packages (from onnxruntime>=1.10.0->pythainlp[full]) (1.12)\n",
      "Requirement already satisfied: protobuf in /opt/conda/lib/python3.8/site-packages (from onnxruntime>=1.10.0->pythainlp[full]) (3.17.3)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.8/site-packages (from oskut>=1.3->pythainlp[full]) (1.1.2)\n",
      "Requirement already satisfied: tensorflow>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from oskut>=1.3->pythainlp[full]) (2.9.1)\n",
      "Collecting pyahocorasick<=1.4.0\n",
      "  Downloading pyahocorasick-1.4.0.tar.gz (312 kB)\n",
      "\u001b[K     |████████████████████████████████| 312 kB 41.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas>=0.24->pythainlp[full]) (2021.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas>=0.24->pythainlp[full]) (2.8.2)\n",
      "Collecting unicodecsv\n",
      "  Downloading unicodecsv-0.14.1.tar.gz (10 kB)\n",
      "Collecting munkres\n",
      "  Downloading munkres-1.1.4-py2.py3-none-any.whl (7.0 kB)\n",
      "Requirement already satisfied: editdistance in /opt/conda/lib/python3.8/site-packages (from panphon>=0.20->epitran>=1.1->pythainlp[full]) (0.6.0)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests>=2.22.0->pythainlp[full]) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.22.0->pythainlp[full]) (2021.10.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests>=2.22.0->pythainlp[full]) (1.26.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests>=2.22.0->pythainlp[full]) (3.2)\n",
      "Requirement already satisfied: portalocker in /opt/conda/lib/python3.8/site-packages (from sacrebleu>=1.4.12->fairseq>=0.10.0->pythainlp[full]) (2.5.1)\n",
      "Requirement already satisfied: lxml in /opt/conda/lib/python3.8/site-packages (from sacrebleu>=1.4.12->fairseq>=0.10.0->pythainlp[full]) (4.9.1)\n",
      "Requirement already satisfied: tabulate>=0.8.9 in /opt/conda/lib/python3.8/site-packages (from sacrebleu>=1.4.12->fairseq>=0.10.0->pythainlp[full]) (0.8.9)\n",
      "Requirement already satisfied: colorama in /opt/conda/lib/python3.8/site-packages (from sacrebleu>=1.4.12->fairseq>=0.10.0->pythainlp[full]) (0.4.5)\n",
      "Collecting numpy>=1.22\n",
      "  Downloading numpy-1.22.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (16.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 16.9 MB 64.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting python-crfsuite\n",
      "  Downloading python_crfsuite-0.9.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0 MB 57.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting spacy>=2.2.2\n",
      "  Downloading spacy-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 6.5 MB 51.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting ufal.udpipe>=1.2.0\n",
      "  Downloading ufal.udpipe-1.2.0.3.tar.gz (304 kB)\n",
      "\u001b[K     |████████████████████████████████| 304 kB 54.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting spacy-legacy<3.1.0,>=3.0.9\n",
      "  Downloading spacy_legacy-3.0.10-py2.py3-none-any.whl (21 kB)\n",
      "Collecting wasabi<1.1.0,>=0.9.1\n",
      "  Downloading wasabi-0.10.1-py3-none-any.whl (26 kB)\n",
      "Collecting langcodes<4.0.0,>=3.2.0\n",
      "  Downloading langcodes-3.3.0-py3-none-any.whl (181 kB)\n",
      "\u001b[K     |████████████████████████████████| 181 kB 52.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting catalogue<2.1.0,>=2.0.6\n",
      "  Downloading catalogue-2.0.8-py3-none-any.whl (17 kB)\n",
      "Collecting murmurhash<1.1.0,>=0.28.0\n",
      "  Downloading murmurhash-1.0.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (21 kB)\n",
      "Collecting spacy-loggers<2.0.0,>=1.0.0\n",
      "  Downloading spacy_loggers-1.0.3-py3-none-any.whl (9.3 kB)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.8/site-packages (from spacy>=2.2.2->spacy-thai>=0.7.1->pythainlp[full]) (3.0.2)\n",
      "Collecting cymem<2.1.0,>=2.0.2\n",
      "  Downloading cymem-2.0.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (36 kB)\n",
      "Collecting pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4\n",
      "  Downloading pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (12.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 12.8 MB 58.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting srsly<3.0.0,>=2.4.3\n",
      "  Downloading srsly-2.4.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (461 kB)\n",
      "\u001b[K     |████████████████████████████████| 461 kB 54.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pathy>=0.3.5\n",
      "  Downloading pathy-0.6.2-py3-none-any.whl (42 kB)\n",
      "\u001b[K     |████████████████████████████████| 42 kB 33.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting preshed<3.1.0,>=3.0.2\n",
      "  Downloading preshed-3.0.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
      "\u001b[K     |████████████████████████████████| 130 kB 56.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting typer<0.5.0,>=0.3.0\n",
      "  Downloading typer-0.4.2-py3-none-any.whl (27 kB)\n",
      "Collecting thinc<8.2.0,>=8.1.0\n",
      "  Downloading thinc-8.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (817 kB)\n",
      "\u001b[K     |████████████████████████████████| 817 kB 62.7 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->fastai<2.0->pythainlp[full]) (2.4.7)\n",
      "Collecting smart-open>=1.8.1\n",
      "  Downloading smart_open-5.2.1-py3-none-any.whl (58 kB)\n",
      "\u001b[K     |████████████████████████████████| 58 kB 61.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting stanza\n",
      "  Downloading stanza-1.4.2-py3-none-any.whl (691 kB)\n",
      "\u001b[K     |████████████████████████████████| 691 kB 62.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: dill in /opt/conda/lib/python3.8/site-packages (from supar>=1.1.4->esupar>=1.3.8->pythainlp[full]) (0.3.4)\n",
      "Collecting editdistpy>=0.1.3\n",
      "  Downloading editdistpy-0.1.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (126 kB)\n",
      "\u001b[K     |████████████████████████████████| 126 kB 28.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.1.2)\n",
      "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.26.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (14.0.6)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (3.3.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.12.1)\n",
      "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.3.3)\n",
      "Requirement already satisfied: tensorflow-estimator<2.10.0,>=2.9.0rc0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (2.9.0)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.32.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.2.0)\n",
      "Requirement already satisfied: tensorboard<2.10,>=2.9 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (2.9.1)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.6.3)\n",
      "Requirement already satisfied: keras<2.10.0,>=2.9.0rc0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (2.9.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.2.0)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.8/site-packages (from tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (2.10.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.8/site-packages (from astunparse>=1.6.0->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.37.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.8.0)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.6.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (3.3.4)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.4.6)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (2.2.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.8/site-packages (from tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.35.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (4.2.4)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (4.7.2)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (1.3.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (3.1.1)\n",
      "Collecting confection<1.0.0,>=0.0.1\n",
      "  Downloading confection-0.0.1-py3-none-any.whl (32 kB)\n",
      "Collecting blis<0.10.0,>=0.7.8\n",
      "  Downloading blis-0.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 7.8 MB 43.3 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: sklearn in /opt/conda/lib/python3.8/site-packages (from tltk>=1.3.8->pythainlp[full]) (0.0)\n",
      "Collecting sklearn_crfsuite\n",
      "  Downloading sklearn_crfsuite-0.3.6-py2.py3-none-any.whl (12 kB)\n",
      "Collecting torch>=1.0.0\n",
      "  Downloading torch-1.12.1-cp38-cp38-manylinux1_x86_64.whl (776.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 776.3 MB 39.8 MB/s eta 0:00:01     |████████▉                       | 213.6 MB 69.8 MB/s eta 0:00:09     |████████████████████▉           | 504.1 MB 62.2 MB/s eta 0:00:05\n",
      "\u001b[?25hRequirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers>=4.22.1->pythainlp[full]) (3.3.0)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /opt/conda/lib/python3.8/site-packages (from transformers>=4.22.1->pythainlp[full]) (0.12.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.9.0 in /opt/conda/lib/python3.8/site-packages (from transformers>=4.22.1->pythainlp[full]) (0.9.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.8/site-packages (from werkzeug>=1.0.1->tensorboard<2.10,>=2.9->tensorflow>=2.0.0->oskut>=1.3->pythainlp[full]) (2.1.1)\n",
      "Collecting khanaa>=0.0.6\n",
      "  Downloading khanaa-0.0.6-py3-none-any.whl (12 kB)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.8/site-packages (from beautifulsoup4->fastai<2.0->pythainlp[full]) (2.3.2.post1)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi->fairseq>=0.10.0->pythainlp[full]) (2.20)\n",
      "Collecting humanfriendly>=9.1\n",
      "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
      "\u001b[K     |████████████████████████████████| 86 kB 52.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: zipp>=3.1.0 in /opt/conda/lib/python3.8/site-packages (from importlib-resources->hydra-core<1.1,>=1.0.7->fairseq>=0.10.0->pythainlp[full]) (3.8.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.8/site-packages (from matplotlib->fastai<2.0->pythainlp[full]) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.8/site-packages (from matplotlib->fastai<2.0->pythainlp[full]) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from scikit-learn->oskut>=1.3->pythainlp[full]) (3.1.0)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.8/site-packages (from sympy->onnxruntime>=1.10.0->pythainlp[full]) (1.2.1)\n",
      "Collecting torchvision\n",
      "  Downloading torchvision-0.13.1-cp38-cp38-manylinux1_x86_64.whl (19.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 19.1 MB 61.4 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting Pillow\n",
      "  Downloading Pillow-9.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.1 MB 60.9 MB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: emoji, fire, phunspell, pyahocorasick, pyicu, tltk, ufal.udpipe, nvidia-ml-py3, unicodecsv\n",
      "  Building wheel for emoji (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for emoji: filename=emoji-2.1.0-py3-none-any.whl size=212392 sha256=8bfb26c58f78cbe3f913eb26bbdf6f58d81eb28a1a3fa9b1d2db2210c3e3018e\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/ae/80/43/3b56e58669d65ea9ebf38b9574074ca248143b61f45e114a6b\n",
      "  Building wheel for fire (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for fire: filename=fire-0.4.0-py2.py3-none-any.whl size=115943 sha256=a1c155e4b87ad512aa8f407058055ee94981dcbc46f229204fb9224de0d4f357\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/1f/10/06/2a990ee4d73a8479fe2922445e8a876d38cfbfed052284c6a1\n",
      "  Building wheel for phunspell (PEP 517) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for phunspell: filename=phunspell-0.1.6-py3-none-any.whl size=47556195 sha256=fe5b3b8b8dbaa6b3a79215a2f06d646086cda30ff97030f914b5b5c1dd76db22\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/44/70/d0/4e8afe1b0baf1024532bf5a814f48a4fecfa09171149c14a24\n",
      "  Building wheel for pyahocorasick (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyahocorasick: filename=pyahocorasick-1.4.0-cp38-cp38-linux_x86_64.whl size=103734 sha256=6de3ac743635b5f88846f8bf94da42be2ce4f28034db4ba6bd5e98311b058577\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/d0/72/5e/5af4d48e71e0b4dc197445664eb1719fe9c9b6fcead4930595\n",
      "  Building wheel for pyicu (PEP 517) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pyicu: filename=PyICU-2.9-cp38-cp38-linux_x86_64.whl size=1445263 sha256=5a406151c42fdcfcbeb3ed8972e68620e43b7a8c470301f83cb67d8be507cc88\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/2d/7a/57/e2caba677406e8c54cea2731d737207e3c5fa6adfd52ca3cdd\n",
      "  Building wheel for tltk (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for tltk: filename=tltk-1.5.7-py3-none-any.whl size=18624768 sha256=c5d9dc5a26d7a9906fc1b0e0b385c0717ea5ec00ed558c52fab2fedd36374cb3\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/dc/46/be/dd110547e205d051b631b5019d0a2ca4131f35edfcf1d2ceea\n",
      "  Building wheel for ufal.udpipe (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for ufal.udpipe: filename=ufal.udpipe-1.2.0.3-cp38-cp38-linux_x86_64.whl size=5727211 sha256=7769f80f1272fecc778280e33f5859bd014c9b26bec95fe8d48ef7eb02102356\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/d4/c1/67/142cea91540458ab9edac9c280a19b549a03217d7b441d32a6\n",
      "  Building wheel for nvidia-ml-py3 (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for nvidia-ml-py3: filename=nvidia_ml_py3-7.352.0-py3-none-any.whl size=19191 sha256=46bc6a4824935e09df305f4c6257ec2f1a61335a369b29f3ff11eeef7ba33d6f\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/b9/b1/68/cb4feab29709d4155310d29a421389665dcab9eb3b679b527b\n",
      "  Building wheel for unicodecsv (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for unicodecsv: filename=unicodecsv-0.14.1-py3-none-any.whl size=10765 sha256=9195ad641dff6cc0aa155acc49742c8c9e4a81e8d88a0b417b93bca99d845c89\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-eucko19e/wheels/35/dd/44/ccb37563a01457f5de74ccedccaee81b01a53e12addeab5e0f\n",
      "Successfully built emoji fire phunspell pyahocorasick pyicu tltk ufal.udpipe nvidia-ml-py3 unicodecsv\n",
      "Installing collected packages: catalogue, srsly, pydantic, numpy, murmurhash, cymem, wasabi, typer, torch, smart-open, preshed, emoji, confection, blis, unicodecsv, transformers, thinc, stanza, spacy-loggers, spacy-legacy, python-crfsuite, Pillow, pathy, omegaconf, munkres, langcodes, humanfriendly, fire, ufal.udpipe, torchvision, torchaudio, supar, ssg, spylls, spacy, sklearn-crfsuite, pythainlp, pyahocorasick, panphon, nvidia-ml-py3, numexpr, nptyping, marisa-trie, khanaa, hydra-core, gensim, fastprogress, editdistpy, deplacy, coloredlogs, bottleneck, bitarray, wunsen, ufal.chu-liu-edmonds, tltk, thai-nner, symspellpy, spacy-thai, sefr-cut, pyicu, phunspell, oskut, onnxruntime, nlpo3, fastai, fairseq, esupar, epitran, bpemb, attacut\n",
      "  Attempting uninstall: pydantic\n",
      "    Found existing installation: pydantic 1.10.1\n",
      "    Uninstalling pydantic-1.10.1:\n",
      "      Successfully uninstalled pydantic-1.10.1\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.21.6\n",
      "    Uninstalling numpy-1.21.6:\n",
      "      Successfully uninstalled numpy-1.21.6\n",
      "  Attempting uninstall: typer\n",
      "    Found existing installation: typer 0.6.1\n",
      "    Uninstalling typer-0.6.1:\n",
      "      Successfully uninstalled typer-0.6.1\n",
      "  Attempting uninstall: torch\n",
      "    Found existing installation: torch 1.9.0+cu111\n",
      "    Uninstalling torch-1.9.0+cu111:\n",
      "      Successfully uninstalled torch-1.9.0+cu111\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.21.2\n",
      "    Uninstalling transformers-4.21.2:\n",
      "      Successfully uninstalled transformers-4.21.2\n",
      "  Attempting uninstall: Pillow\n",
      "    Found existing installation: Pillow 8.3.2\n",
      "    Uninstalling Pillow-8.3.2:\n",
      "      Successfully uninstalled Pillow-8.3.2\n",
      "  Attempting uninstall: omegaconf\n",
      "    Found existing installation: omegaconf 2.1.2\n",
      "    Uninstalling omegaconf-2.1.2:\n",
      "      Successfully uninstalled omegaconf-2.1.2\n",
      "  Attempting uninstall: torchvision\n",
      "    Found existing installation: torchvision 0.10.0+cu111\n",
      "    Uninstalling torchvision-0.10.0+cu111:\n",
      "      Successfully uninstalled torchvision-0.10.0+cu111\n",
      "  Attempting uninstall: hydra-core\n",
      "    Found existing installation: hydra-core 1.1.2\n",
      "    Uninstalling hydra-core-1.1.2:\n",
      "      Successfully uninstalled hydra-core-1.1.2\n",
      "Successfully installed Pillow-9.2.0 attacut-1.1.0.dev0 bitarray-2.6.0 blis-0.9.1 bottleneck-1.3.5 bpemb-0.3.3 catalogue-2.0.8 coloredlogs-15.0.1 confection-0.0.1 cymem-2.0.6 deplacy-2.0.3 editdistpy-0.1.3 emoji-2.1.0 epitran-1.23 esupar-1.3.8 fairseq-0.12.2 fastai-1.0.61 fastprogress-1.0.3 fire-0.4.0 gensim-4.2.0 humanfriendly-10.0 hydra-core-1.0.7 khanaa-0.0.6 langcodes-3.3.0 marisa-trie-0.7.7 munkres-1.1.4 murmurhash-1.0.8 nlpo3-1.2.2 nptyping-2.3.1 numexpr-2.8.3 numpy-1.22.4 nvidia-ml-py3-7.352.0 omegaconf-2.0.6 onnxruntime-1.12.1 oskut-1.3 panphon-0.20.0 pathy-0.6.2 phunspell-0.1.6 preshed-3.0.7 pyahocorasick-1.4.0 pydantic-1.9.2 pyicu-2.9 pythainlp-3.1.0.dev3 python-crfsuite-0.9.8 sefr-cut-1.1 sklearn-crfsuite-0.3.6 smart-open-5.2.1 spacy-3.4.1 spacy-legacy-3.0.10 spacy-loggers-1.0.3 spacy-thai-0.7.1 spylls-0.1.7 srsly-2.4.4 ssg-0.0.8 stanza-1.4.2 supar-1.1.4 symspellpy-6.7.6 thai-nner-0.3 thinc-8.1.1 tltk-1.5.7 torch-1.12.1 torchaudio-0.12.1 torchvision-0.13.1 transformers-4.22.1 typer-0.4.2 ufal.chu-liu-edmonds-1.0.2 ufal.udpipe-1.2.0.3 unicodecsv-0.14.1 wasabi-0.10.1 wunsen-0.0.3\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install --pre pythainlp[full]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d89095e",
   "metadata": {},
   "source": [
    "# PyThaiNLP v3.1.0-dev3\n",
    "\n",
    "This is a development release for PyThaiNLP v3.1.\n",
    "\n",
    "\n",
    "You can install by `pip install --pre pythainlp==3.1.0.dev3`.\n",
    "\n",
    "Documentation: [https://pythainlp.github.io/dev-docs/](https://pythainlp.github.io/dev-docs/)\n",
    "\n",
    "Report bug: [https://github.com/PyThaiNLP/pythainlp/issues](https://github.com/PyThaiNLP/pythainlp/issues)\n",
    "\n",
    "See [PyThaiNLP 3.1 change log](https://github.com/PyThaiNLP/pythainlp/issues/643)\n",
    "\n",
    "See [3.1 Milestone](https://github.com/PyThaiNLP/pythainlp/milestone/16)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce40047b",
   "metadata": {},
   "source": [
    "## Thai NNER\n",
    "\n",
    "PyThaiNLP support Thai NNER.\n",
    "\n",
    "    Weerayut Buaphet, Can Udomcharoenchaikit, Peerat Limkonchotiwat, Attapol Rutherford, and Sarana Nutanong. 2022. Thai Nested Named Entity Recognition Corpus. In Findings of the Association for Computational Linguistics: ACL 2022, pages 1473–1486, Dublin, Ireland. Association for Computational Linguistics.\n",
    "\n",
    "GitHub: [https://github.com/vistec-AI/Thai-NNER](https://github.com/vistec-AI/Thai-NNER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88053682",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.tag.named_entity import NNER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "561230bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moving 0 files to the new cache system\n"
     ]
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.6526656150817871,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 0,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9470c889fc8449c0bc7df028185ad36b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: thai_nner\n",
      "- Downloading: thai_nner 1.0\n"
     ]
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.024754762649536133,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 431311121,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13856d01d8214aed84e331b048eba9d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/431311121 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.02248668670654297,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 546,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42bab85f72004a129d94bb266e872db2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/546 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.025851726531982422,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 423498558,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1bf36e3bfe646db919405281760822c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/423M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at airesearch/wangchanberta-base-att-spm-uncased were not used when initializing CamembertModel: ['lm_head.layer_norm.bias', 'lm_head.dense.weight', 'lm_head.decoder.bias', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.bias']\n",
      "- This IS expected if you are initializing CamembertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing CamembertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "nner = NNER()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "943fcac0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['<s>',\n",
       "  'วันนี้',\n",
       "  'วันที่',\n",
       "  '',\n",
       "  '',\n",
       "  '5',\n",
       "  '',\n",
       "  '',\n",
       "  'เมษายน',\n",
       "  '',\n",
       "  '',\n",
       "  '25',\n",
       "  '65',\n",
       "  '',\n",
       "  '',\n",
       "  'เป็น',\n",
       "  'วันที่',\n",
       "  '',\n",
       "  'อากาศ',\n",
       "  '',\n",
       "  'ดีมาก',\n",
       "  '</s>'],\n",
       " [{'text': ['วันนี้'], 'span': [1, 2], 'entity_type': 'rel'},\n",
       "  {'text': ['วันที่', '', '', '5'], 'span': [2, 6], 'entity_type': 'day'},\n",
       "  {'text': ['วันที่', '', '', '5', '', '', 'เมษายน', '', '', '25', '65'],\n",
       "   'span': [2, 13],\n",
       "   'entity_type': 'date'},\n",
       "  {'text': ['', '5'], 'span': [4, 6], 'entity_type': 'cardinal'},\n",
       "  {'text': ['', 'เมษายน'], 'span': [7, 9], 'entity_type': 'month'},\n",
       "  {'text': ['', '25', '65'], 'span': [10, 13], 'entity_type': 'year'}])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nner.tag(\"วันนี้วันที่ 5 เมษายน 2565 เป็นวันที่อากาศดีมาก\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ff21311",
   "metadata": {},
   "source": [
    "## Dependency Parsing\n",
    "\n",
    "Now, PyThaiNLP support dependency_parsing 🎉"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e2f2e104",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.parse import dependency_parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1870604f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\tผม\t\tPRON\tPPRS\t_\t2\tnsubj\t_\tSpaceAfter=No\n",
      "2\tเป็น\t\tVERB\tVSTA\t_\t0\tROOT\t_\tSpaceAfter=No\n",
      "3\tคนดี\t\tNOUN\tNCMN\t_\t2\tobj\t_\tSpaceAfter=No\n"
     ]
    }
   ],
   "source": [
    "print(dependency_parsing(\"ผมเป็นคนดี\", engine=\"spacy_thai\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4ad7b528",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: deplacy Pages: 1 -->\n",
       "<svg width=\"255pt\" height=\"259pt\"\n",
       " viewBox=\"0.00 0.00 254.50 259.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 255)\">\n",
       "<title>deplacy</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-255 250.5,-255 250.5,4 -4,4\"/>\n",
       "<!-- r2 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>r2</title>\n",
       "<text text-anchor=\"middle\" x=\"79.5\" y=\"-243\" font-family=\"sans-serif\" font-size=\"10.00\" fill=\"#000000\">ROOT</text>\n",
       "</g>\n",
       "<!-- x2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>x2</title>\n",
       "<text text-anchor=\"middle\" x=\"79.5\" y=\"-181.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ผมเป็นคนดี</text>\n",
       "</g>\n",
       "<!-- r2&#45;&gt;x2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>r2&#45;&gt;x2</title>\n",
       "<path fill=\"none\" stroke=\"#c0c0c0\" d=\"M79.5,-239.9167C79.5,-233.7509 79.5,-223.3543 79.5,-213.2555\"/>\n",
       "<polygon fill=\"#c0c0c0\" stroke=\"#c0c0c0\" points=\"83.0001,-213.0291 79.5,-203.0291 76.0001,-213.0291 83.0001,-213.0291\"/>\n",
       "</g>\n",
       "<!-- w -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>w</title>\n",
       "<polygon fill=\"none\" stroke=\"#000000\" stroke-width=\"0\" points=\"36.5,-.5 36.5,-46.5 246.5,-46.5 246.5,-.5 36.5,-.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-31.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ผม</text>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" stroke-width=\"0\" points=\"36.5,-23.5 98.5,-23.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"67.5\" y=\"-8.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">PRON</text>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" stroke-width=\"0\" points=\"98.5,-.5 98.5,-46.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"135.5\" y=\"-31.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">เป็น</text>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" stroke-width=\"0\" points=\"98.5,-23.5 172.5,-23.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"135.5\" y=\"-8.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">VERB</text>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" stroke-width=\"0\" points=\"172.5,-.5 172.5,-46.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"209.5\" y=\"-31.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">คนดี</text>\n",
       "<polyline fill=\"none\" stroke=\"#000000\" stroke-width=\"0\" points=\"172.5,-23.5 246.5,-23.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"209.5\" y=\"-8.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">NOUN</text>\n",
       "</g>\n",
       "<!-- x2&#45;&gt;w -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>x2&#45;&gt;w:1</title>\n",
       "<path fill=\"none\" stroke=\"#c0c0c0\" d=\"M76.8962,-166.6729C73.6426,-142.2703 68.4024,-97.4461 67.6035,-56.7442\"/>\n",
       "<polygon fill=\"#c0c0c0\" stroke=\"#c0c0c0\" points=\"71.101,-56.4641 67.5,-46.5 64.1013,-56.5349 71.101,-56.4641\"/>\n",
       "<text text-anchor=\"middle\" x=\"87\" y=\"-109.5\" font-family=\"sans-serif\" font-size=\"10.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- x1 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>x1</title>\n",
       "<text text-anchor=\"middle\" x=\"178.5\" y=\"-108.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">เป็นคนดี</text>\n",
       "</g>\n",
       "<!-- x2&#45;&gt;x1 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>x2&#45;&gt;x1</title>\n",
       "<path fill=\"none\" stroke=\"#c0c0c0\" d=\"M103.9719,-166.9551C116.5851,-157.6545 132.1403,-146.1844 145.7714,-136.1332\"/>\n",
       "<polygon fill=\"#c0c0c0\" stroke=\"#c0c0c0\" points=\"147.9951,-138.8421 153.9665,-130.0904 143.8408,-133.2082 147.9951,-138.8421\"/>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;w -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>x1&#45;&gt;w:2</title>\n",
       "<path fill=\"none\" stroke=\"#c0c0c0\" d=\"M158.5106,-93.9705C149.3869,-84.098 139.8986,-70.9637 136.6613,-56.6083\"/>\n",
       "<polygon fill=\"#c0c0c0\" stroke=\"#c0c0c0\" points=\"140.1185,-56.0352 135.5,-46.5 133.1643,-56.8341 140.1185,-56.0352\"/>\n",
       "</g>\n",
       "<!-- x1&#45;&gt;w -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>x1&#45;&gt;w:3</title>\n",
       "<path fill=\"none\" stroke=\"#c0c0c0\" d=\"M192.3679,-93.8713C199.0097,-83.7413 206.0558,-70.4008 208.5548,-56.5954\"/>\n",
       "<polygon fill=\"#c0c0c0\" stroke=\"#c0c0c0\" points=\"212.0525,-56.7828 209.5,-46.5 205.0829,-56.1301 212.0525,-56.7828\"/>\n",
       "<text text-anchor=\"middle\" x=\"213.5\" y=\"-68\" font-family=\"sans-serif\" font-size=\"10.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.sources.Source at 0x7fa725778670>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import graphviz\n",
    "import deplacy\n",
    "graphviz.Source(deplacy.dot(dependency_parsing(\"ผมเป็นคนดี\", engine=\"spacy_thai\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fba5350f",
   "metadata": {},
   "source": [
    "## Soundex"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94bbaebf",
   "metadata": {},
   "source": [
    "New! Add Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique\n",
    "\n",
    "    Prayut Suwanvisat, Somchai Prasitjutrakul. Thai-English Cross-Language Transliterated Word Retrieval using Soundex Technique. In 1998 [cited 2022 Sep 8]. Available from: https://www.cp.eng.chula.ac.th/~somchai/spj/papers/ThaiText/ncsec98-clir.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "636c20b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.soundex import soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "63e6be3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'11'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soundex(\"vp\",\"prayut_and_somchaip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fafa4f28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'11'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soundex(\"วีพี\",\"prayut_and_somchaip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38fc07c7",
   "metadata": {},
   "source": [
    "## Util"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "801d2c0a",
   "metadata": {},
   "source": [
    "This is tone detector for Thai syllables. \n",
    "\n",
    "See more: https://github.com/PyThaiNLP/pythainlp/pull/690"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07e06739",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "live\n",
      "dead\n"
     ]
    }
   ],
   "source": [
    "from pythainlp.util import sound_syllable\n",
    "\n",
    "print(sound_syllable(\"มา\"))\n",
    "# output: live\n",
    "\n",
    "print(sound_syllable(\"เลข\"))\n",
    "# output: dead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3152ad1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m\n",
      "h\n"
     ]
    }
   ],
   "source": [
    "from pythainlp.util import tone_detector\n",
    "\n",
    "print(tone_detector(\"มา\"))\n",
    "# output: m\n",
    "\n",
    "print(tone_detector(\"ไม้\"))\n",
    "# output: h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a02b001c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "close\n",
      "open\n"
     ]
    }
   ],
   "source": [
    "from pythainlp.util import syllable_open_close_detector\n",
    "\n",
    "print(syllable_open_close_detector(\"มาก\"))\n",
    "# output: close\n",
    "\n",
    "print(syllable_open_close_detector(\"คะ\"))\n",
    "# output: open"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5752237c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: thai_w2p\n",
      "- Downloading: thai_w2p 0.2\n"
     ]
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.024695873260498047,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 9772474,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba735f3fb48549a2ade87cd724f96abc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9772474 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('มือ', 'm'), ('ถือ', 'r')]\n",
      "[('รา', 'm'), ('คา', 'm')]\n"
     ]
    }
   ],
   "source": [
    "from pythainlp.util import thai_word_tone_detector\n",
    "\n",
    "print(thai_word_tone_detector(\"มือถือ\"))\n",
    "# output: [('มือ', 'm'), ('ถือ', 'r')]\n",
    "\n",
    "print(thai_word_tone_detector(\"ราคา\"))\n",
    "# output: [('รา', 'm'), ('คา', 'm')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12fd25fe",
   "metadata": {},
   "source": [
    "## Text summarize\n",
    "\n",
    "Add mt5 cpe kmutt thai sentence sum [#679](https://github.com/PyThaiNLP/pythainlp/pull/679)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9177575d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.summarize import summarize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d64e4d9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.022951602935791016,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 744,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa8cb4f2cc33462298cf789b0a63ed6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/744 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.02309441566467285,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 2329700301,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "621b50b690a44056b0d5b38f5930d4d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/2.33G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.023040056228637695,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 4309802,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fe72a5aca0049fa9088729051b9fc1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/4.31M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.02091670036315918,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 65,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6e062408e164b6f8c1346efc94104ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/65.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.02179241180419922,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": 0,
       "postfix": null,
       "prefix": "Downloading",
       "rate": null,
       "total": 450,
       "unit": "B",
       "unit_divisor": 1000,
       "unit_scale": true
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46c80ab349544b8ba16ae5518fe16918",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/450 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['น้ําแข็งใสเป็นขนมหวานที่ชื่นใจที่สุด']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summarize(\"ถ้าพูดถึงขนมหวานในตำนานที่ชื่นใจที่สุดแล้วละก็ต้องไม่พ้น น้ำแข็งใส แน่ๆ เพราะว่าเป็นอะไรที่ชื่นใจสุดๆ\", engine=\"mt5-cpe-kmutt-thai-sentence-sum\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e76d66ce",
   "metadata": {},
   "source": [
    "## Transliteration"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d79d186",
   "metadata": {},
   "source": [
    "Add ISO 11940 transliteration [#659](https://github.com/PyThaiNLP/pythainlp/pull/659)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "727485d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.transliterate import transliterate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "91047e1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'echīyngıh̄m̀'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transliterate(\"เชียงใหม่\", engine=\"iso_11940\") # wiki: echīyngıh̄m̀"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8054074e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'p̣hās̛̄āịthy'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transliterate(\"ภาษาไทย\", engine=\"iso_11940\") # wiki: p̣hās̛̄āịthy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcd31d4c",
   "metadata": {},
   "source": [
    "Add wunsen [#686](https://github.com/PyThaiNLP/pythainlp/pull/686) and Wunsen Mandarin and Japanese update [#694](https://github.com/PyThaiNLP/pythainlp/pull/694)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ab15379c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythainlp.transliterate.wunsen import WunsenTransliterate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a45d7745",
   "metadata": {},
   "outputs": [],
   "source": [
    "wt = WunsenTransliterate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b81cef30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'หนี เห่า'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wt.transliterate(\"ni3 hao3\", lang=\"zh\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "023898b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'โอฮาโย'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wt.transliterate(\"ohayō\", lang=\"jp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "fbe5b8c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'อันนย็องฮาเซโย'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wt.transliterate(\"annyeonghaseyo\", lang=\"ko\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "12eb3490",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ซีน จ่าว'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wt.transliterate(\"xin chào\", lang=\"vi\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30f69a3b",
   "metadata": {},
   "source": [
    "See [PyThaiNLP 3.1 change log](https://github.com/PyThaiNLP/pythainlp/issues/643)\n",
    "\n",
    "See [3.1 Milestone](https://github.com/PyThaiNLP/pythainlp/milestone/16)."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}