Transformers_Generation_Inference_RU.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Transformers Generation Inference RU.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hU6HUWH3ZB_",
        "colab_type": "code",
        "outputId": "2a89b0ac-c93e-4a5b-a087-f7708360b99c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 763
        }
      },
      "source": [
        "! pip install awscli\n",
        "! aws s3 sync --no-sign-request s3://models.dobro.ai/gpt2/ru/unfreeze_all gpt2"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting awscli\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/c3/88c32b3edb7e8cbd8a5ac2cbd9708ff0a2e9f0e9caf094fd6029831696c9/awscli-1.18.54-py2.py3-none-any.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 4.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from awscli) (0.3.3)\n",
            "Requirement already satisfied: PyYAML<5.4,>=3.10; python_version != \"3.4\" in /usr/local/lib/python3.6/dist-packages (from awscli) (3.13)\n",
            "Collecting rsa<=3.5.0,>=3.1.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e1/ae/baedc9cb175552e95f3395c43055a6a5e125ae4d48a1d7a924baca83e92e/rsa-3.4.2-py2.py3-none-any.whl (46kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 8.5MB/s \n",
            "\u001b[?25hCollecting botocore==1.16.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4a/b3/1756f8d43af677110c91f5450c4f8c1d30d64e39a435ce01ebd4ceb88843/botocore-1.16.4-py2.py3-none-any.whl (6.2MB)\n",
            "\u001b[K     |████████████████████████████████| 6.2MB 43.2MB/s \n",
            "\u001b[?25hCollecting colorama<0.4.4,>=0.2.5; python_version != \"3.4\"\n",
            "  Downloading https://files.pythonhosted.org/packages/c9/dc/45cdef1b4d119eb96316b3117e6d5708a08029992b2fee2c143c7a0a5cc5/colorama-0.4.3-py2.py3-none-any.whl\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from awscli) (0.15.2)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<=3.5.0,>=3.1.2->awscli) (0.4.8)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore==1.16.4->awscli) (2.8.1)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from botocore==1.16.4->awscli) (0.9.5)\n",
            "Requirement already satisfied: urllib3<1.26,>=1.20; python_version != \"3.4\" in /usr/local/lib/python3.6/dist-packages (from botocore==1.16.4->awscli) (1.24.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore==1.16.4->awscli) (1.12.0)\n",
            "Installing collected packages: rsa, botocore, colorama, awscli\n",
            "  Found existing installation: rsa 4.0\n",
            "    Uninstalling rsa-4.0:\n",
            "      Successfully uninstalled rsa-4.0\n",
            "  Found existing installation: botocore 1.16.1\n",
            "    Uninstalling botocore-1.16.1:\n",
            "      Successfully uninstalled botocore-1.16.1\n",
            "Successfully installed awscli-1.18.54 botocore-1.16.4 colorama-0.4.3 rsa-3.4.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "rsa"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Completed 7 Bytes/2.3 GiB (18 Bytes/s) with 10 file(s) remaining\rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/m_checkpoint-3364613/step.txt to gpt2/m_checkpoint-3364613/step.txt\n",
            "Completed 7 Bytes/2.3 GiB (18 Bytes/s) with 9 file(s) remaining\rCompleted 256.0 KiB/2.3 GiB (615.3 KiB/s) with 9 file(s) remaining\rCompleted 257.4 KiB/2.3 GiB (607.0 KiB/s) with 9 file(s) remaining\rCompleted 258.0 KiB/2.3 GiB (605.6 KiB/s) with 9 file(s) remaining\rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/s_checkpoint-1900000/training_args.bin to gpt2/s_checkpoint-1900000/training_args.bin\n",
            "Completed 258.0 KiB/2.3 GiB (605.6 KiB/s) with 8 file(s) remaining\rCompleted 258.0 KiB/2.3 GiB (603.5 KiB/s) with 8 file(s) remaining\rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/s_checkpoint-1900000/config.json to gpt2/s_checkpoint-1900000/config.json\n",
            "Completed 258.0 KiB/2.3 GiB (603.5 KiB/s) with 7 file(s) remaining\rCompleted 514.0 KiB/2.3 GiB (1.2 MiB/s) with 7 file(s) remaining  \rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/s_checkpoint-1900000/step.txt to gpt2/s_checkpoint-1900000/step.txt\n",
            "Completed 514.0 KiB/2.3 GiB (1.2 MiB/s) with 6 file(s) remaining\rCompleted 709.6 KiB/2.3 GiB (1.6 MiB/s) with 6 file(s) remaining\rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/m_checkpoint-3364613/encoder.model to gpt2/m_checkpoint-3364613/encoder.model\n",
            "Completed 709.6 KiB/2.3 GiB (1.6 MiB/s) with 5 file(s) remaining\rCompleted 711.0 KiB/2.3 GiB (1.5 MiB/s) with 5 file(s) remaining\rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/m_checkpoint-3364613/training_args.bin to gpt2/m_checkpoint-3364613/training_args.bin\n",
            "Completed 711.0 KiB/2.3 GiB (1.5 MiB/s) with 4 file(s) remaining\rCompleted 711.6 KiB/2.3 GiB (1.5 MiB/s) with 4 file(s) remaining\rdownload: s3://models.dobro.ai/gpt2/ru/unfreeze_all/m_checkpoint-3364613/config.json to gpt2/m_checkpoint-3364613/config.json\n",
            "download: s3://models.dobro.ai/gpt2/ru/unfreeze_all/s_checkpoint-1900000/encoder.model to gpt2/s_checkpoint-1900000/encoder.model\n",
            "download: s3://models.dobro.ai/gpt2/ru/unfreeze_all/s_checkpoint-1900000/pytorch_model.bin to gpt2/s_checkpoint-1900000/pytorch_model.bin\n",
            "download: s3://models.dobro.ai/gpt2/ru/unfreeze_all/m_checkpoint-3364613/pytorch_model.bin to gpt2/m_checkpoint-3364613/pytorch_model.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EY60dazANLwm",
        "colab_type": "code",
        "outputId": "80b2d7d3-f916-4428-d282-6e072405e556",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "! ls gpt2"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "m_checkpoint-3364613  s_checkpoint-1900000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "unEyE96cNFNB",
        "colab_type": "code",
        "outputId": "54b4c0c3-ef23-4c6c-8a07-fcb5c507d179",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262
        }
      },
      "source": [
        "# memory footprint support libraries/code\n",
        "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
        "!pip install gputil\n",
        "!pip install psutil\n",
        "!pip install humanize\n",
        "import psutil\n",
        "import humanize\n",
        "import os\n",
        "import GPUtil as GPU\n",
        "GPUs = GPU.getGPUs()\n",
        "# XXX: only one GPU on Colab and isn’t guaranteed\n",
        "gpu = GPUs[0]\n",
        "def printm():\n",
        " process = psutil.Process(os.getpid())\n",
        " print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n",
        " print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
        "printm()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting gputil\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz\n",
            "Building wheels for collected packages: gputil\n",
            "  Building wheel for gputil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gputil: filename=GPUtil-1.4.0-cp36-none-any.whl size=7413 sha256=05116674bf0f1b35f5698464477a17ba42e48321bb1e2d2f222c2438228412c9\n",
            "  Stored in directory: /root/.cache/pip/wheels/3d/77/07/80562de4bb0786e5ea186911a2c831fdd0018bda69beab71fd\n",
            "Successfully built gputil\n",
            "Installing collected packages: gputil\n",
            "Successfully installed gputil-1.4.0\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (5.4.8)\n",
            "Requirement already satisfied: humanize in /usr/local/lib/python3.6/dist-packages (0.5.1)\n",
            "Gen RAM Free: 12.7 GB  | Proc size: 162.0 MB\n",
            "GPU RAM Free: 16280MB | Used: 0MB | Util   0% | Total 16280MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kyMpIRhvNjSr",
        "colab_type": "code",
        "outputId": "305487f7-bca0-4a33-e303-52a59ee483a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 642
        }
      },
      "source": [
        "! pip install git+https://github.com/huggingface/transformers"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/huggingface/transformers\n",
            "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-eixl8py7\n",
            "  Running command git clone -q https://github.com/huggingface/transformers /tmp/pip-req-build-eixl8py7\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (1.18.3)\n",
            "Collecting tokenizers==0.7.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/e5/a26eb4716523808bb0a799fcfdceb6ebf77a18169d9591b2f46a9adb87d9/tokenizers-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (3.8MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8MB 4.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (4.38.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (2019.12.20)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/2c/8df20f3ac6c22ac224fff307ebc102818206c53fc454ecd37d8ac2060df5/sentencepiece-0.1.86-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 41.9MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 58.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from transformers==2.8.0) (0.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (2.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.8.0) (1.24.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.8.0) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.8.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.8.0) (0.14.1)\n",
            "Building wheels for collected packages: transformers, sacremoses\n",
            "  Building wheel for transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-2.8.0-cp36-none-any.whl size=602051 sha256=754073851d4d954ae162bf4c8a5cb728d11b32281d3ebf9dae820c797a1f9768\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-tpkiqgqd/wheels/70/d3/52/b3fa4f8b8ef04167ac62e5bb2accb62ae764db2a378247490e\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=a706ab8e8cbcb672211e26d4fc3d2211fd0851102233a1b455b1c7d248980602\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built transformers sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.86 tokenizers-0.7.0 transformers-2.8.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3r-JPxUNkt4",
        "colab_type": "code",
        "outputId": "661e34ab-52dd-4ab0-844a-1963f80c912c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "%%writefile setup.sh\n",
        "\n",
        "git clone https://github.com/NVIDIA/apex\n",
        "cd apex\n",
        "pip install -v --no-cache-dir ./"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing setup.sh\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "31CMCkr3Nmpl",
        "colab_type": "code",
        "outputId": "5a799913-21da-49a5-dd42-eb8f505891e2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!sh setup.sh"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'apex'...\n",
            "remote: Enumerating objects: 116, done.\u001b[K\n",
            "remote: Counting objects:   0% (1/116)\u001b[K\rremote: Counting objects:   1% (2/116)\u001b[K\rremote: Counting objects:   2% (3/116)\u001b[K\rremote: Counting objects:   3% (4/116)\u001b[K\rremote: Counting objects:   4% (5/116)\u001b[K\rremote: Counting objects:   5% (6/116)\u001b[K\rremote: Counting objects:   6% (7/116)\u001b[K\rremote: Counting objects:   7% (9/116)\u001b[K\rremote: Counting objects:   8% (10/116)\u001b[K\rremote: Counting objects:   9% (11/116)\u001b[K\rremote: Counting objects:  10% (12/116)\u001b[K\rremote: Counting objects:  11% (13/116)\u001b[K\rremote: Counting objects:  12% (14/116)\u001b[K\rremote: Counting objects:  13% (16/116)\u001b[K\rremote: Counting objects:  14% (17/116)\u001b[K\rremote: Counting objects:  15% (18/116)\u001b[K\rremote: Counting objects:  16% (19/116)\u001b[K\rremote: Counting objects:  17% (20/116)\u001b[K\rremote: Counting objects:  18% (21/116)\u001b[K\rremote: Counting objects:  19% (23/116)\u001b[K\rremote: Counting objects:  20% (24/116)\u001b[K\rremote: Counting objects:  21% (25/116)\u001b[K\rremote: Counting objects:  22% (26/116)\u001b[K\rremote: Counting objects:  23% (27/116)\u001b[K\rremote: Counting objects:  24% (28/116)\u001b[K\rremote: Counting objects:  25% (29/116)\u001b[K\rremote: Counting objects:  26% (31/116)\u001b[K\rremote: Counting objects:  27% (32/116)\u001b[K\rremote: Counting objects:  28% (33/116)\u001b[K\rremote: Counting objects:  29% (34/116)\u001b[K\rremote: Counting objects:  30% (35/116)\u001b[K\rremote: Counting objects:  31% (36/116)\u001b[K\rremote: Counting objects:  32% (38/116)\u001b[K\rremote: Counting objects:  33% (39/116)\u001b[K\rremote: Counting objects:  34% (40/116)\u001b[K\rremote: Counting objects:  35% (41/116)\u001b[K\rremote: Counting objects:  36% (42/116)\u001b[K\rremote: Counting objects:  37% (43/116)\u001b[K\rremote: Counting objects:  38% (45/116)\u001b[K\rremote: Counting objects:  39% (46/116)\u001b[K\rremote: Counting objects:  40% (47/116)\u001b[K\rremote: Counting objects:  41% (48/116)\u001b[K\rremote: Counting objects:  42% (49/116)\u001b[K\rremote: Counting objects:  43% (50/116)\u001b[K\rremote: Counting objects:  44% (52/116)\u001b[K\rremote: Counting objects:  45% (53/116)\u001b[K\rremote: Counting objects:  46% (54/116)\u001b[K\rremote: Counting objects:  47% (55/116)\u001b[K\rremote: Counting objects:  48% (56/116)\u001b[K\rremote: Counting objects:  49% (57/116)\u001b[K\rremote: Counting objects:  50% (58/116)\u001b[K\rremote: Counting objects:  51% (60/116)\u001b[K\rremote: Counting objects:  52% (61/116)\u001b[K\rremote: Counting objects:  53% (62/116)\u001b[K\rremote: Counting objects:  54% (63/116)\u001b[K\rremote: Counting objects:  55% (64/116)\u001b[K\rremote: Counting objects:  56% (65/116)\u001b[K\rremote: Counting objects:  57% (67/116)\u001b[K\rremote: Counting objects:  58% (68/116)\u001b[K\rremote: Counting objects:  59% (69/116)\u001b[K\rremote: Counting objects:  60% (70/116)\u001b[K\rremote: Counting objects:  61% (71/116)\u001b[K\rremote: Counting objects:  62% (72/116)\u001b[K\rremote: Counting objects:  63% (74/116)\u001b[K\rremote: Counting objects:  64% (75/116)\u001b[K\rremote: Counting objects:  65% (76/116)\u001b[K\rremote: Counting objects:  66% (77/116)\u001b[K\rremote: Counting objects:  67% (78/116)\u001b[K\rremote: Counting objects:  68% (79/116)\u001b[K\rremote: Counting objects:  69% (81/116)\u001b[K\rremote: Counting objects:  70% (82/116)\u001b[K\rremote: Counting objects:  71% (83/116)\u001b[K\rremote: Counting objects:  72% (84/116)\u001b[K\rremote: Counting objects:  73% (85/116)\u001b[K\rremote: Counting objects:  74% (86/116)\u001b[K\rremote: Counting objects:  75% (87/116)\u001b[K\rremote: Counting objects:  76% (89/116)\u001b[K\rremote: Counting objects:  77% (90/116)\u001b[K\rremote: Counting objects:  78% (91/116)\u001b[K\rremote: Counting objects:  79% (92/116)\u001b[K\rremote: Counting objects:  80% (93/116)\u001b[K\rremote: Counting objects:  81% (94/116)\u001b[K\rremote: Counting objects:  82% (96/116)\u001b[K\rremote: Counting objects:  83% (97/116)\u001b[K\rremote: Counting objects:  84% (98/116)\u001b[K\rremote: Counting objects:  85% (99/116)\u001b[K\rremote: Counting objects:  86% (100/116)\u001b[K\rremote: Counting objects:  87% (101/116)\u001b[K\rremote: Counting objects:  88% (103/116)\u001b[K\rremote: Counting objects:  89% (104/116)\u001b[K\rremote: Counting objects:  90% (105/116)\u001b[K\rremote: Counting objects:  91% (106/116)\u001b[K\rremote: Counting objects:  92% (107/116)\u001b[K\rremote: Counting objects:  93% (108/116)\u001b[K\rremote: Counting objects:  94% (110/116)\u001b[K\rremote: Counting objects:  95% (111/116)\u001b[K\rremote: Counting objects:  96% (112/116)\u001b[K\rremote: Counting objects:  97% (113/116)\u001b[K\rremote: Counting objects:  98% (114/116)\u001b[K\rremote: Counting objects:  99% (115/116)\u001b[K\rremote: Counting objects: 100% (116/116)\u001b[K\rremote: Counting objects: 100% (116/116), done.\u001b[K\n",
            "remote: Compressing objects: 100% (80/80), done.\u001b[K\n",
            "remote: Total 6712 (delta 74), reused 58 (delta 35), pack-reused 6596\u001b[K\n",
            "Receiving objects: 100% (6712/6712), 13.75 MiB | 27.14 MiB/s, done.\n",
            "Resolving deltas: 100% (4464/4464), done.\n",
            "Created temporary directory: /tmp/pip-ephem-wheel-cache-dvyucncy\n",
            "Created temporary directory: /tmp/pip-req-tracker-1ekvrpnt\n",
            "Created requirements tracker '/tmp/pip-req-tracker-1ekvrpnt'\n",
            "Created temporary directory: /tmp/pip-install-coamchf9\n",
            "Processing /content/apex\n",
            "  Created temporary directory: /tmp/pip-req-build-hnycpawg\n",
            "  Added file:///content/apex to build tracker '/tmp/pip-req-tracker-1ekvrpnt'\n",
            "    Running setup.py (path:/tmp/pip-req-build-hnycpawg/setup.py) egg_info for package from file:///content/apex\n",
            "    Running command python setup.py egg_info\n",
            "    torch.__version__  =  1.5.0+cu101\n",
            "    running egg_info\n",
            "    creating /tmp/pip-req-build-hnycpawg/pip-egg-info/apex.egg-info\n",
            "    writing /tmp/pip-req-build-hnycpawg/pip-egg-info/apex.egg-info/PKG-INFO\n",
            "    writing dependency_links to /tmp/pip-req-build-hnycpawg/pip-egg-info/apex.egg-info/dependency_links.txt\n",
            "    writing top-level names to /tmp/pip-req-build-hnycpawg/pip-egg-info/apex.egg-info/top_level.txt\n",
            "    writing manifest file '/tmp/pip-req-build-hnycpawg/pip-egg-info/apex.egg-info/SOURCES.txt'\n",
            "    writing manifest file '/tmp/pip-req-build-hnycpawg/pip-egg-info/apex.egg-info/SOURCES.txt'\n",
            "    /tmp/pip-req-build-hnycpawg/setup.py:46: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "      warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "  Source in /tmp/pip-req-build-hnycpawg has version 0.1, which satisfies requirement apex==0.1 from file:///content/apex\n",
            "  Removed apex==0.1 from file:///content/apex from build tracker '/tmp/pip-req-tracker-1ekvrpnt'\n",
            "Building wheels for collected packages: apex\n",
            "  Created temporary directory: /tmp/pip-wheel-_rjzrxyw\n",
            "  Building wheel for apex (setup.py) ... \u001b[?25l  Destination directory: /tmp/pip-wheel-_rjzrxyw\n",
            "  Running command /usr/bin/python3 -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '\"'\"'/tmp/pip-req-build-hnycpawg/setup.py'\"'\"'; __file__='\"'\"'/tmp/pip-req-build-hnycpawg/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' bdist_wheel -d /tmp/pip-wheel-_rjzrxyw --python-tag cp36\n",
            "  torch.__version__  =  1.5.0+cu101\n",
            "  /tmp/pip-req-build-hnycpawg/setup.py:46: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "    warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "  running bdist_wheel\n",
            "  running build\n",
            "  running build_py\n",
            "  creating build\n",
            "  creating build/lib\n",
            "  creating build/lib/apex\n",
            "  copying apex/__init__.py -> build/lib/apex\n",
            "  creating build/lib/apex/mlp\n",
            "  copying apex/mlp/mlp.py -> build/lib/apex/mlp\n",
            "  copying apex/mlp/__init__.py -> build/lib/apex/mlp\n",
            "  creating build/lib/apex/RNN\n",
            "  copying apex/RNN/cells.py -> build/lib/apex/RNN\n",
            "  copying apex/RNN/models.py -> build/lib/apex/RNN\n",
            "  copying apex/RNN/__init__.py -> build/lib/apex/RNN\n",
            "  copying apex/RNN/RNNBackend.py -> build/lib/apex/RNN\n",
            "  creating build/lib/apex/multi_tensor_apply\n",
            "  copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib/apex/multi_tensor_apply\n",
            "  copying apex/multi_tensor_apply/__init__.py -> build/lib/apex/multi_tensor_apply\n",
            "  creating build/lib/apex/amp\n",
            "  copying apex/amp/_process_optimizer.py -> build/lib/apex/amp\n",
            "  copying apex/amp/frontend.py -> build/lib/apex/amp\n",
            "  copying apex/amp/_initialize.py -> build/lib/apex/amp\n",
            "  copying apex/amp/wrap.py -> build/lib/apex/amp\n",
            "  copying apex/amp/utils.py -> build/lib/apex/amp\n",
            "  copying apex/amp/rnn_compat.py -> build/lib/apex/amp\n",
            "  copying apex/amp/__init__.py -> build/lib/apex/amp\n",
            "  copying apex/amp/compat.py -> build/lib/apex/amp\n",
            "  copying apex/amp/_amp_state.py -> build/lib/apex/amp\n",
            "  copying apex/amp/handle.py -> build/lib/apex/amp\n",
            "  copying apex/amp/__version__.py -> build/lib/apex/amp\n",
            "  copying apex/amp/opt.py -> build/lib/apex/amp\n",
            "  copying apex/amp/amp.py -> build/lib/apex/amp\n",
            "  copying apex/amp/scaler.py -> build/lib/apex/amp\n",
            "  creating build/lib/apex/optimizers\n",
            "  copying apex/optimizers/fused_adam.py -> build/lib/apex/optimizers\n",
            "  copying apex/optimizers/fused_novograd.py -> build/lib/apex/optimizers\n",
            "  copying apex/optimizers/fused_sgd.py -> build/lib/apex/optimizers\n",
            "  copying apex/optimizers/fused_lamb.py -> build/lib/apex/optimizers\n",
            "  copying apex/optimizers/__init__.py -> build/lib/apex/optimizers\n",
            "  creating build/lib/apex/pyprof\n",
            "  copying apex/pyprof/__init__.py -> build/lib/apex/pyprof\n",
            "  creating build/lib/apex/parallel\n",
            "  copying apex/parallel/LARC.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/multiproc.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/optimized_sync_batchnorm.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/sync_batchnorm.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/__init__.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/sync_batchnorm_kernel.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/distributed.py -> build/lib/apex/parallel\n",
            "  copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib/apex/parallel\n",
            "  creating build/lib/apex/fp16_utils\n",
            "  copying apex/fp16_utils/loss_scaler.py -> build/lib/apex/fp16_utils\n",
            "  copying apex/fp16_utils/__init__.py -> build/lib/apex/fp16_utils\n",
            "  copying apex/fp16_utils/fp16util.py -> build/lib/apex/fp16_utils\n",
            "  copying apex/fp16_utils/fp16_optimizer.py -> build/lib/apex/fp16_utils\n",
            "  creating build/lib/apex/contrib\n",
            "  copying apex/contrib/__init__.py -> build/lib/apex/contrib\n",
            "  creating build/lib/apex/normalization\n",
            "  copying apex/normalization/fused_layer_norm.py -> build/lib/apex/normalization\n",
            "  copying apex/normalization/__init__.py -> build/lib/apex/normalization\n",
            "  creating build/lib/apex/reparameterization\n",
            "  copying apex/reparameterization/weight_norm.py -> build/lib/apex/reparameterization\n",
            "  copying apex/reparameterization/reparameterization.py -> build/lib/apex/reparameterization\n",
            "  copying apex/reparameterization/__init__.py -> build/lib/apex/reparameterization\n",
            "  creating build/lib/apex/amp/lists\n",
            "  copying apex/amp/lists/functional_overrides.py -> build/lib/apex/amp/lists\n",
            "  copying apex/amp/lists/torch_overrides.py -> build/lib/apex/amp/lists\n",
            "  copying apex/amp/lists/tensor_overrides.py -> build/lib/apex/amp/lists\n",
            "  copying apex/amp/lists/__init__.py -> build/lib/apex/amp/lists\n",
            "  creating build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/optim.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/index_slice_join_mutate.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/linear.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/activation.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/normalization.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/output.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/blas.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/reduction.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/utility.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/pointwise.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/softmax.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/data.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/embedding.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/pooling.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/dropout.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/prof.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/loss.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/randomSample.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/recurrentCell.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/misc.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/__main__.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/__init__.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/usage.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/base.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/conv.py -> build/lib/apex/pyprof/prof\n",
            "  copying apex/pyprof/prof/convert.py -> build/lib/apex/pyprof/prof\n",
            "  creating build/lib/apex/pyprof/nvtx\n",
            "  copying apex/pyprof/nvtx/nvmarker.py -> build/lib/apex/pyprof/nvtx\n",
            "  copying apex/pyprof/nvtx/__init__.py -> build/lib/apex/pyprof/nvtx\n",
            "  creating build/lib/apex/pyprof/parse\n",
            "  copying apex/pyprof/parse/db.py -> build/lib/apex/pyprof/parse\n",
            "  copying apex/pyprof/parse/nvvp.py -> build/lib/apex/pyprof/parse\n",
            "  copying apex/pyprof/parse/parse.py -> build/lib/apex/pyprof/parse\n",
            "  copying apex/pyprof/parse/__main__.py -> build/lib/apex/pyprof/parse\n",
            "  copying apex/pyprof/parse/kernel.py -> build/lib/apex/pyprof/parse\n",
            "  copying apex/pyprof/parse/__init__.py -> build/lib/apex/pyprof/parse\n",
            "  creating build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/fast_self_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/self_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/encdec_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/self_multihead_attn.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/encdec_multihead_attn.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/__init__.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "  copying apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "  creating build/lib/apex/contrib/optimizers\n",
            "  copying apex/contrib/optimizers/fused_adam.py -> build/lib/apex/contrib/optimizers\n",
            "  copying apex/contrib/optimizers/fused_sgd.py -> build/lib/apex/contrib/optimizers\n",
            "  copying apex/contrib/optimizers/fused_lamb.py -> build/lib/apex/contrib/optimizers\n",
            "  copying apex/contrib/optimizers/__init__.py -> build/lib/apex/contrib/optimizers\n",
            "  copying apex/contrib/optimizers/fp16_optimizer.py -> build/lib/apex/contrib/optimizers\n",
            "  creating build/lib/apex/contrib/groupbn\n",
            "  copying apex/contrib/groupbn/__init__.py -> build/lib/apex/contrib/groupbn\n",
            "  copying apex/contrib/groupbn/batch_norm.py -> build/lib/apex/contrib/groupbn\n",
            "  creating build/lib/apex/contrib/xentropy\n",
            "  copying apex/contrib/xentropy/softmax_xentropy.py -> build/lib/apex/contrib/xentropy\n",
            "  copying apex/contrib/xentropy/__init__.py -> build/lib/apex/contrib/xentropy\n",
            "  installing to build/bdist.linux-x86_64/wheel\n",
            "  running install\n",
            "  running install_lib\n",
            "  creating build/bdist.linux-x86_64\n",
            "  creating build/bdist.linux-x86_64/wheel\n",
            "  creating build/bdist.linux-x86_64/wheel/apex\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/mlp\n",
            "  copying build/lib/apex/mlp/mlp.py -> build/bdist.linux-x86_64/wheel/apex/mlp\n",
            "  copying build/lib/apex/mlp/__init__.py -> build/bdist.linux-x86_64/wheel/apex/mlp\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/RNN\n",
            "  copying build/lib/apex/RNN/cells.py -> build/bdist.linux-x86_64/wheel/apex/RNN\n",
            "  copying build/lib/apex/RNN/models.py -> build/bdist.linux-x86_64/wheel/apex/RNN\n",
            "  copying build/lib/apex/RNN/__init__.py -> build/bdist.linux-x86_64/wheel/apex/RNN\n",
            "  copying build/lib/apex/RNN/RNNBackend.py -> build/bdist.linux-x86_64/wheel/apex/RNN\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/multi_tensor_apply\n",
            "  copying build/lib/apex/multi_tensor_apply/multi_tensor_apply.py -> build/bdist.linux-x86_64/wheel/apex/multi_tensor_apply\n",
            "  copying build/lib/apex/multi_tensor_apply/__init__.py -> build/bdist.linux-x86_64/wheel/apex/multi_tensor_apply\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/_process_optimizer.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/frontend.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/_initialize.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/wrap.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/utils.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/rnn_compat.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/amp/lists\n",
            "  copying build/lib/apex/amp/lists/functional_overrides.py -> build/bdist.linux-x86_64/wheel/apex/amp/lists\n",
            "  copying build/lib/apex/amp/lists/torch_overrides.py -> build/bdist.linux-x86_64/wheel/apex/amp/lists\n",
            "  copying build/lib/apex/amp/lists/tensor_overrides.py -> build/bdist.linux-x86_64/wheel/apex/amp/lists\n",
            "  copying build/lib/apex/amp/lists/__init__.py -> build/bdist.linux-x86_64/wheel/apex/amp/lists\n",
            "  copying build/lib/apex/amp/__init__.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/compat.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/_amp_state.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/handle.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/__version__.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/opt.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/amp.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  copying build/lib/apex/amp/scaler.py -> build/bdist.linux-x86_64/wheel/apex/amp\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/optimizers\n",
            "  copying build/lib/apex/optimizers/fused_adam.py -> build/bdist.linux-x86_64/wheel/apex/optimizers\n",
            "  copying build/lib/apex/optimizers/fused_novograd.py -> build/bdist.linux-x86_64/wheel/apex/optimizers\n",
            "  copying build/lib/apex/optimizers/fused_sgd.py -> build/bdist.linux-x86_64/wheel/apex/optimizers\n",
            "  copying build/lib/apex/optimizers/fused_lamb.py -> build/bdist.linux-x86_64/wheel/apex/optimizers\n",
            "  copying build/lib/apex/optimizers/__init__.py -> build/bdist.linux-x86_64/wheel/apex/optimizers\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/pyprof\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/optim.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/index_slice_join_mutate.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/linear.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/activation.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/normalization.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/output.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/blas.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/reduction.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/utility.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/pointwise.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/softmax.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/data.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/embedding.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/pooling.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/dropout.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/prof.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/loss.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/randomSample.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/recurrentCell.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/misc.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/__main__.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/__init__.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/usage.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/base.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/conv.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  copying build/lib/apex/pyprof/prof/convert.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/prof\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/pyprof/nvtx\n",
            "  copying build/lib/apex/pyprof/nvtx/nvmarker.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/nvtx\n",
            "  copying build/lib/apex/pyprof/nvtx/__init__.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/nvtx\n",
            "  copying build/lib/apex/pyprof/__init__.py -> build/bdist.linux-x86_64/wheel/apex/pyprof\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  copying build/lib/apex/pyprof/parse/db.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  copying build/lib/apex/pyprof/parse/nvvp.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  copying build/lib/apex/pyprof/parse/parse.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  copying build/lib/apex/pyprof/parse/__main__.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  copying build/lib/apex/pyprof/parse/kernel.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  copying build/lib/apex/pyprof/parse/__init__.py -> build/bdist.linux-x86_64/wheel/apex/pyprof/parse\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/LARC.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/multiproc.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/optimized_sync_batchnorm.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/sync_batchnorm.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/__init__.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/sync_batchnorm_kernel.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/distributed.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  copying build/lib/apex/parallel/optimized_sync_batchnorm_kernel.py -> build/bdist.linux-x86_64/wheel/apex/parallel\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/fp16_utils\n",
            "  copying build/lib/apex/fp16_utils/loss_scaler.py -> build/bdist.linux-x86_64/wheel/apex/fp16_utils\n",
            "  copying build/lib/apex/fp16_utils/__init__.py -> build/bdist.linux-x86_64/wheel/apex/fp16_utils\n",
            "  copying build/lib/apex/fp16_utils/fp16util.py -> build/bdist.linux-x86_64/wheel/apex/fp16_utils\n",
            "  copying build/lib/apex/fp16_utils/fp16_optimizer.py -> build/bdist.linux-x86_64/wheel/apex/fp16_utils\n",
            "  copying build/lib/apex/__init__.py -> build/bdist.linux-x86_64/wheel/apex\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/contrib\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/self_multihead_attn_func.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/encdec_multihead_attn_func.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/self_multihead_attn.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/encdec_multihead_attn.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/__init__.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  copying build/lib/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py -> build/bdist.linux-x86_64/wheel/apex/contrib/multihead_attn\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/contrib/optimizers\n",
            "  copying build/lib/apex/contrib/optimizers/fused_adam.py -> build/bdist.linux-x86_64/wheel/apex/contrib/optimizers\n",
            "  copying build/lib/apex/contrib/optimizers/fused_sgd.py -> build/bdist.linux-x86_64/wheel/apex/contrib/optimizers\n",
            "  copying build/lib/apex/contrib/optimizers/fused_lamb.py -> build/bdist.linux-x86_64/wheel/apex/contrib/optimizers\n",
            "  copying build/lib/apex/contrib/optimizers/__init__.py -> build/bdist.linux-x86_64/wheel/apex/contrib/optimizers\n",
            "  copying build/lib/apex/contrib/optimizers/fp16_optimizer.py -> build/bdist.linux-x86_64/wheel/apex/contrib/optimizers\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/contrib/groupbn\n",
            "  copying build/lib/apex/contrib/groupbn/__init__.py -> build/bdist.linux-x86_64/wheel/apex/contrib/groupbn\n",
            "  copying build/lib/apex/contrib/groupbn/batch_norm.py -> build/bdist.linux-x86_64/wheel/apex/contrib/groupbn\n",
            "  copying build/lib/apex/contrib/__init__.py -> build/bdist.linux-x86_64/wheel/apex/contrib\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/contrib/xentropy\n",
            "  copying build/lib/apex/contrib/xentropy/softmax_xentropy.py -> build/bdist.linux-x86_64/wheel/apex/contrib/xentropy\n",
            "  copying build/lib/apex/contrib/xentropy/__init__.py -> build/bdist.linux-x86_64/wheel/apex/contrib/xentropy\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/normalization\n",
            "  copying build/lib/apex/normalization/fused_layer_norm.py -> build/bdist.linux-x86_64/wheel/apex/normalization\n",
            "  copying build/lib/apex/normalization/__init__.py -> build/bdist.linux-x86_64/wheel/apex/normalization\n",
            "  creating build/bdist.linux-x86_64/wheel/apex/reparameterization\n",
            "  copying build/lib/apex/reparameterization/weight_norm.py -> build/bdist.linux-x86_64/wheel/apex/reparameterization\n",
            "  copying build/lib/apex/reparameterization/reparameterization.py -> build/bdist.linux-x86_64/wheel/apex/reparameterization\n",
            "  copying build/lib/apex/reparameterization/__init__.py -> build/bdist.linux-x86_64/wheel/apex/reparameterization\n",
            "  running install_egg_info\n",
            "  running egg_info\n",
            "  creating apex.egg-info\n",
            "  writing apex.egg-info/PKG-INFO\n",
            "  writing dependency_links to apex.egg-info/dependency_links.txt\n",
            "  writing top-level names to apex.egg-info/top_level.txt\n",
            "  writing manifest file 'apex.egg-info/SOURCES.txt'\n",
            "  writing manifest file 'apex.egg-info/SOURCES.txt'\n",
            "  Copying apex.egg-info to build/bdist.linux-x86_64/wheel/apex-0.1-py3.6.egg-info\n",
            "  running install_scripts\n",
            "  adding license file \"LICENSE\" (matched pattern \"LICEN[CS]E*\")\n",
            "  creating build/bdist.linux-x86_64/wheel/apex-0.1.dist-info/WHEEL\n",
            "  creating '/tmp/pip-wheel-_rjzrxyw/apex-0.1-cp36-none-any.whl' and adding 'build/bdist.linux-x86_64/wheel' to it\n",
            "  adding 'apex/__init__.py'\n",
            "  adding 'apex/RNN/RNNBackend.py'\n",
            "  adding 'apex/RNN/__init__.py'\n",
            "  adding 'apex/RNN/cells.py'\n",
            "  adding 'apex/RNN/models.py'\n",
            "  adding 'apex/amp/__init__.py'\n",
            "  adding 'apex/amp/__version__.py'\n",
            "  adding 'apex/amp/_amp_state.py'\n",
            "  adding 'apex/amp/_initialize.py'\n",
            "  adding 'apex/amp/_process_optimizer.py'\n",
            "  adding 'apex/amp/amp.py'\n",
            "  adding 'apex/amp/compat.py'\n",
            "  adding 'apex/amp/frontend.py'\n",
            "  adding 'apex/amp/handle.py'\n",
            "  adding 'apex/amp/opt.py'\n",
            "  adding 'apex/amp/rnn_compat.py'\n",
            "  adding 'apex/amp/scaler.py'\n",
            "  adding 'apex/amp/utils.py'\n",
            "  adding 'apex/amp/wrap.py'\n",
            "  adding 'apex/amp/lists/__init__.py'\n",
            "  adding 'apex/amp/lists/functional_overrides.py'\n",
            "  adding 'apex/amp/lists/tensor_overrides.py'\n",
            "  adding 'apex/amp/lists/torch_overrides.py'\n",
            "  adding 'apex/contrib/__init__.py'\n",
            "  adding 'apex/contrib/groupbn/__init__.py'\n",
            "  adding 'apex/contrib/groupbn/batch_norm.py'\n",
            "  adding 'apex/contrib/multihead_attn/__init__.py'\n",
            "  adding 'apex/contrib/multihead_attn/encdec_multihead_attn.py'\n",
            "  adding 'apex/contrib/multihead_attn/encdec_multihead_attn_func.py'\n",
            "  adding 'apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py'\n",
            "  adding 'apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py'\n",
            "  adding 'apex/contrib/multihead_attn/fast_self_multihead_attn_func.py'\n",
            "  adding 'apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py'\n",
            "  adding 'apex/contrib/multihead_attn/self_multihead_attn.py'\n",
            "  adding 'apex/contrib/multihead_attn/self_multihead_attn_func.py'\n",
            "  adding 'apex/contrib/optimizers/__init__.py'\n",
            "  adding 'apex/contrib/optimizers/fp16_optimizer.py'\n",
            "  adding 'apex/contrib/optimizers/fused_adam.py'\n",
            "  adding 'apex/contrib/optimizers/fused_lamb.py'\n",
            "  adding 'apex/contrib/optimizers/fused_sgd.py'\n",
            "  adding 'apex/contrib/xentropy/__init__.py'\n",
            "  adding 'apex/contrib/xentropy/softmax_xentropy.py'\n",
            "  adding 'apex/fp16_utils/__init__.py'\n",
            "  adding 'apex/fp16_utils/fp16_optimizer.py'\n",
            "  adding 'apex/fp16_utils/fp16util.py'\n",
            "  adding 'apex/fp16_utils/loss_scaler.py'\n",
            "  adding 'apex/mlp/__init__.py'\n",
            "  adding 'apex/mlp/mlp.py'\n",
            "  adding 'apex/multi_tensor_apply/__init__.py'\n",
            "  adding 'apex/multi_tensor_apply/multi_tensor_apply.py'\n",
            "  adding 'apex/normalization/__init__.py'\n",
            "  adding 'apex/normalization/fused_layer_norm.py'\n",
            "  adding 'apex/optimizers/__init__.py'\n",
            "  adding 'apex/optimizers/fused_adam.py'\n",
            "  adding 'apex/optimizers/fused_lamb.py'\n",
            "  adding 'apex/optimizers/fused_novograd.py'\n",
            "  adding 'apex/optimizers/fused_sgd.py'\n",
            "  adding 'apex/parallel/LARC.py'\n",
            "  adding 'apex/parallel/__init__.py'\n",
            "  adding 'apex/parallel/distributed.py'\n",
            "  adding 'apex/parallel/multiproc.py'\n",
            "  adding 'apex/parallel/optimized_sync_batchnorm.py'\n",
            "  adding 'apex/parallel/optimized_sync_batchnorm_kernel.py'\n",
            "  adding 'apex/parallel/sync_batchnorm.py'\n",
            "  adding 'apex/parallel/sync_batchnorm_kernel.py'\n",
            "  adding 'apex/pyprof/__init__.py'\n",
            "  adding 'apex/pyprof/nvtx/__init__.py'\n",
            "  adding 'apex/pyprof/nvtx/nvmarker.py'\n",
            "  adding 'apex/pyprof/parse/__init__.py'\n",
            "  adding 'apex/pyprof/parse/__main__.py'\n",
            "  adding 'apex/pyprof/parse/db.py'\n",
            "  adding 'apex/pyprof/parse/kernel.py'\n",
            "  adding 'apex/pyprof/parse/nvvp.py'\n",
            "  adding 'apex/pyprof/parse/parse.py'\n",
            "  adding 'apex/pyprof/prof/__init__.py'\n",
            "  adding 'apex/pyprof/prof/__main__.py'\n",
            "  adding 'apex/pyprof/prof/activation.py'\n",
            "  adding 'apex/pyprof/prof/base.py'\n",
            "  adding 'apex/pyprof/prof/blas.py'\n",
            "  adding 'apex/pyprof/prof/conv.py'\n",
            "  adding 'apex/pyprof/prof/convert.py'\n",
            "  adding 'apex/pyprof/prof/data.py'\n",
            "  adding 'apex/pyprof/prof/dropout.py'\n",
            "  adding 'apex/pyprof/prof/embedding.py'\n",
            "  adding 'apex/pyprof/prof/index_slice_join_mutate.py'\n",
            "  adding 'apex/pyprof/prof/linear.py'\n",
            "  adding 'apex/pyprof/prof/loss.py'\n",
            "  adding 'apex/pyprof/prof/misc.py'\n",
            "  adding 'apex/pyprof/prof/normalization.py'\n",
            "  adding 'apex/pyprof/prof/optim.py'\n",
            "  adding 'apex/pyprof/prof/output.py'\n",
            "  adding 'apex/pyprof/prof/pointwise.py'\n",
            "  adding 'apex/pyprof/prof/pooling.py'\n",
            "  adding 'apex/pyprof/prof/prof.py'\n",
            "  adding 'apex/pyprof/prof/randomSample.py'\n",
            "  adding 'apex/pyprof/prof/recurrentCell.py'\n",
            "  adding 'apex/pyprof/prof/reduction.py'\n",
            "  adding 'apex/pyprof/prof/softmax.py'\n",
            "  adding 'apex/pyprof/prof/usage.py'\n",
            "  adding 'apex/pyprof/prof/utility.py'\n",
            "  adding 'apex/reparameterization/__init__.py'\n",
            "  adding 'apex/reparameterization/reparameterization.py'\n",
            "  adding 'apex/reparameterization/weight_norm.py'\n",
            "  adding 'apex-0.1.dist-info/LICENSE'\n",
            "  adding 'apex-0.1.dist-info/METADATA'\n",
            "  adding 'apex-0.1.dist-info/WHEEL'\n",
            "  adding 'apex-0.1.dist-info/top_level.txt'\n",
            "  adding 'apex-0.1.dist-info/RECORD'\n",
            "  removing build/bdist.linux-x86_64/wheel\n",
            "\u001b[?25hdone\n",
            "  Created wheel for apex: filename=apex-0.1-cp36-none-any.whl size=157282 sha256=26caed7f5d77da5b5dfdf61dcdb08d0ef1154b55c6e3287b1fe7014f504c4a09\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-dvyucncy/wheels/b1/3a/aa/d84906eaab780ae580c7a5686a33bf2820d8590ac3b60d5967\n",
            "  Removing source in /tmp/pip-req-build-hnycpawg\n",
            "Successfully built apex\n",
            "Installing collected packages: apex\n",
            "\n",
            "Successfully installed apex-0.1\n",
            "Cleaning up...\n",
            "Removed build tracker '/tmp/pip-req-tracker-1ekvrpnt'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GzrM6VlBeWfU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "outputId": "00f896f7-030b-4da1-8f93-cc42cb771168"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4uYLN47eYal",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "outputId": "7a64466f-9820-4e59-e418-e37077757e58"
      },
      "source": [
        "! cp './gdrive/My Drive/gpt2/res_oxxxymiron.zip' ./res_oxxxymiron.zip \n",
        "! unzip ./res_oxxxymiron.zip"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  ./res_oxxxymiron.zip\n",
            "   creating: textgenmodels/\n",
            "  inflating: textgenmodels/training_args.bin  \n",
            "  inflating: textgenmodels/config.json  \n",
            "  inflating: textgenmodels/encoder.model  \n",
            " extracting: textgenmodels/eval_results.txt  \n",
            "  inflating: textgenmodels/pytorch_model.bin  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U3YlVXTFNn63",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function\n",
        "\n",
        "import argparse\n",
        "import glob\n",
        "import logging\n",
        "import os\n",
        "import pickle\n",
        "import random\n",
        "import re\n",
        "import shutil\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, Dataset, SequentialSampler, RandomSampler\n",
        "from torch.utils.data.distributed import DistributedSampler\n",
        "\n",
        "try:\n",
        "    from torch.utils.tensorboard import SummaryWriter\n",
        "except:\n",
        "    from tensorboardX import SummaryWriter\n",
        "\n",
        "from tqdm import tqdm, trange\n",
        "\n",
        "from transformers import (WEIGHTS_NAME, AdamW, \n",
        "                          # WarmupLinearSchedule,\n",
        "                                  BertConfig, BertForMaskedLM, BertTokenizer,\n",
        "                                  GPT2Config, GPT2LMHeadModel, GPT2Tokenizer,\n",
        "                                  OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer,\n",
        "                                  RobertaConfig, RobertaForMaskedLM, RobertaTokenizer,\n",
        "                                  DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer)\n",
        "\n",
        "\n",
        "logger = logging.getLogger(__name__)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_xc8sp_fN1VI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "logger.setLevel('INFO')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oka00d-EN3N2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MODEL_CLASSES = {\n",
        "    'gpt2': (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer),\n",
        "    'openai-gpt': (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),\n",
        "    'bert': (BertConfig, BertForMaskedLM, BertTokenizer),\n",
        "    'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),\n",
        "    'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer)\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7peG5IXrN4OL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dict2obj(d):\n",
        "  if isinstance(d, list):\n",
        "    d = [dict2obj(x) for x in d]\n",
        "  if not isinstance(d, dict):\n",
        "    return d\n",
        "  class C(object):\n",
        "    pass\n",
        "  o = C()\n",
        "  for k in d:\n",
        "    o.__dict__[k] = dict2obj(d[k])\n",
        "  return o\n",
        "\n",
        "\n",
        "parser = {}\n",
        "\n",
        "# parser['train_data_file'] = './texts/rap_lyrics.txt'\n",
        "# parser['output_dir'] = './textgenmodels'\n",
        "\n",
        "parser['eval_data_file'] = False\n",
        "parser['model_type'] = 'gpt2' # bert\n",
        "parser['model_name_or_path'] = 'gpt2-medium' # 'bert-base-cased'\n",
        "parser['mlm'] = False \n",
        "parser['mlm_probability'] = False\n",
        "\n",
        "parser['config_name'] = \"\"\n",
        "parser['tokenizer_name'] = \"\"\n",
        "parser['cache_dir'] = \"\"\n",
        "parser['block_size'] = -1\n",
        "parser['do_train'] = True\n",
        "parser['do_eval'] = False\n",
        "parser['evaluate_during_training'] = False\n",
        "parser['do_lower_case'] = True\n",
        "\n",
        "parser['per_gpu_train_batch_size'] = 2\n",
        "parser['per_gpu_eval_batch_size'] = 1\n",
        "parser['gradient_accumulation_steps'] = 10\n",
        "parser['learning_rate'] = 5e-5\n",
        "parser['weight_decay'] = 0.0\n",
        "parser['adam_epsilon'] = 1e-8\n",
        "parser['max_grad_norm'] = 1.0\n",
        "parser['num_train_epochs'] = 3.0\n",
        "parser['max_steps'] = -1\n",
        "parser['warmup_steps'] = 0\n",
        "\n",
        "parser['logging_steps'] = 50\n",
        "parser['save_steps'] = 50\n",
        "parser['save_total_limit'] = None\n",
        "parser['eval_all_checkpoints'] = False\n",
        "parser['no_cuda'] = False\n",
        "parser['overwrite_output_dir'] = True\n",
        "parser['overwrite_cache'] = True\n",
        "parser['seed'] = 42\n",
        "\n",
        "parser['fp16'] = True\n",
        "parser['fp16_opt_level'] = 'O1'\n",
        "parser['local_rank'] = -1\n",
        "parser['server_ip'] = \"\"\n",
        "parser['server_port'] = \"\"\n",
        "\n",
        "parser['device'] = \"cuda\"\n",
        "\n",
        "args = dict2obj(parser)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hYtzX87wPO9r",
        "colab_type": "code",
        "outputId": "a18a51e3-803d-4170-e50e-beffbab2ecda",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "! pip install youtokentome"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting youtokentome\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/65/4a86cf99da3f680497ae132329025b291e2fda22327e8da6a9476e51acb1/youtokentome-1.0.6-cp36-cp36m-manylinux2010_x86_64.whl (1.7MB)\n",
            "\r\u001b[K     |▏                               | 10kB 19.1MB/s eta 0:00:01\r\u001b[K     |▍                               | 20kB 3.2MB/s eta 0:00:01\r\u001b[K     |▋                               | 30kB 4.6MB/s eta 0:00:01\r\u001b[K     |▊                               | 40kB 6.0MB/s eta 0:00:01\r\u001b[K     |█                               | 51kB 3.8MB/s eta 0:00:01\r\u001b[K     |█▏                              | 61kB 4.5MB/s eta 0:00:01\r\u001b[K     |█▍                              | 71kB 5.1MB/s eta 0:00:01\r\u001b[K     |█▌                              | 81kB 5.8MB/s eta 0:00:01\r\u001b[K     |█▊                              | 92kB 4.5MB/s eta 0:00:01\r\u001b[K     |██                              | 102kB 4.9MB/s eta 0:00:01\r\u001b[K     |██                              | 112kB 4.9MB/s eta 0:00:01\r\u001b[K     |██▎                             | 122kB 4.9MB/s eta 0:00:01\r\u001b[K     |██▌                             | 133kB 4.9MB/s eta 0:00:01\r\u001b[K     |██▊                             | 143kB 4.9MB/s eta 0:00:01\r\u001b[K     |██▉                             | 153kB 4.9MB/s eta 0:00:01\r\u001b[K     |███                             | 163kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▎                            | 174kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▌                            | 184kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▋                            | 194kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▉                            | 204kB 4.9MB/s eta 0:00:01\r\u001b[K     |████                            | 215kB 4.9MB/s eta 0:00:01\r\u001b[K     |████▏                           | 225kB 4.9MB/s eta 0:00:01\r\u001b[K     |████▍                           | 235kB 4.9MB/s eta 0:00:01\r\u001b[K     |████▋                           | 245kB 4.9MB/s eta 0:00:01\r\u001b[K     |████▉                           | 256kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████                           | 266kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 276kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 286kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 296kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 307kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████                          | 317kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 327kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 337kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 348kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 358kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████                         | 368kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████                         | 378kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 389kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 399kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 409kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 419kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████                        | 430kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 440kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 450kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 460kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 471kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████                       | 481kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 491kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 501kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 512kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 522kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████                      | 532kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 542kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 552kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 563kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 573kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████                     | 583kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 593kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 604kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 614kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 624kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 634kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████                    | 645kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 655kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 665kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 675kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 686kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 696kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 706kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 716kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 727kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 737kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 747kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 757kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 768kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 778kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 788kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 798kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 808kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 819kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 829kB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 839kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████                | 849kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████                | 860kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 870kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 880kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 890kB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 901kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 911kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 921kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 931kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 942kB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 952kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 962kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 972kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 983kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 993kB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 1.0MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.0MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 1.0MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 1.0MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 1.0MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 1.1MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 1.2MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.3MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.4MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.5MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.6MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.7MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 1.7MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 1.7MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 1.7MB 4.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.7MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.7MB 4.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.7MB 4.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: Click>=7.0 in /usr/local/lib/python3.6/dist-packages (from youtokentome) (7.1.2)\n",
            "Installing collected packages: youtokentome\n",
            "Successfully installed youtokentome-1.0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rd2YO8CrPKhr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"Byte pair encoding utilities\"\"\"\n",
        "import os\n",
        "import youtokentome as yttm\n",
        "import hashlib\n",
        "from transformers.tokenization_utils import PreTrainedTokenizer\n",
        "import shutil\n",
        "import regex as re\n",
        "from os.path import samefile"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fAjAl1iJPK4r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "NEW_LINE = '<|n|>'\n",
        "\n",
        "class YTEncoder(PreTrainedTokenizer):\n",
        "    def_name = 'encoder.model'\n",
        "    def __init__(self, filename, *inputs, **kwargs):\n",
        "        super().__init__(*inputs, **kwargs)\n",
        "        # self.max_len_single_sentence = 1024 # no default special tokens - you can update this value if you add special tokens\n",
        "        # self.max_len_sentences_pair = 1024 # no default special tokens - you can update this value if you add special tokens\n",
        "\n",
        "        if os.path.isdir(filename): filename = os.path.join(filename, self.def_name)\n",
        "\n",
        "        self.bpe = yttm.BPE(filename)\n",
        "        self.hash = hashlib.sha512(open(filename, 'rb').read()).hexdigest()[:10]\n",
        "        self.filename = filename\n",
        "\n",
        "    def encode(self, text):\n",
        "        if text and text[0] != ' ': text = ' ' + text\n",
        "        text = re.sub(r'(?=[^ ])([\\W])([\\w])',r'\\g<1> \\g<2>',text)\n",
        "        text = text.replace('\\n', f' {NEW_LINE} ')\n",
        "\n",
        "        return self.bpe.encode([text], output_type=yttm.OutputType.ID)[0]\n",
        "\n",
        "\n",
        "    def decode(self, tokens): # I hate regexps\n",
        "        if not isinstance(tokens,list):\n",
        "            tokens = tokens.tolist()\n",
        "        result = self.bpe.decode(tokens)[0]\n",
        "        result = re.sub(r'( )?(<\\|n\\|>)( )?', r'\\n', result)\n",
        "        result = re.sub(r'([\\n(]) (\\w)',r'\\g<1>\\g<2>', result)\n",
        "        result = re.sub(r'(\\W)([«\"''\\n(]|^) (\\w)',r'\\g<1>\\g<2>\\g<3>', result)\n",
        "        result = re.sub(r'(\\w)- (\\w)',r'\\g<1>-\\g<2>', result)\n",
        "        return result\n",
        "\n",
        "    def tokenize(self, text, **kwargs):\n",
        "        return self.encode(text)\n",
        "\n",
        "    @classmethod\n",
        "    def from_pretrained(cls, *inputs, **kwargs):\n",
        "        return cls(*inputs, **kwargs)\n",
        "\n",
        "    def add_special_tokens_single_sentence(self, token_ids):\n",
        "        return token_ids\n",
        "\n",
        "    def save_pretrained(self, save_directory):\n",
        "        src = self.filename\n",
        "        dst = os.path.join(save_directory, self.def_name)\n",
        "        if src != dst:\n",
        "            shutil.copyfile(src, dst)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjzR-006N5eI",
        "colab_type": "code",
        "outputId": "d583904f-42d6-460e-f697-eb6109870004",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]\n",
        "\n",
        "# Load a trained model and vocabulary that you have fine-tuned\n",
        "model = model_class.from_pretrained('./textgenmodels')\n",
        "# tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)\n",
        "tokenizer = YTEncoder.from_pretrained('./textgenmodels')\n",
        "model.to(args.device)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2LMHeadModel(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(50257, 1024)\n",
              "    (wpe): Embedding(1024, 1024)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (6): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (7): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (8): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (9): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (10): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (11): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (12): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (13): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (14): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (15): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (16): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (17): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (18): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (19): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (20): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (21): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (22): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (23): Block(\n",
              "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=1024, out_features=50257, bias=False)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6tGpYmktPmO5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn.functional as F\n",
        "\n",
        "def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')):\n",
        "    \"\"\" Filter a distribution of logits using top-k and/or nucleus (top-p) filtering\n",
        "        Args:\n",
        "            logits: logits distribution shape (batch size x vocabulary size)\n",
        "            top_k > 0: keep only top k tokens with highest probability (top-k filtering).\n",
        "            top_p > 0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering).\n",
        "                Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)\n",
        "        From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317\n",
        "    \"\"\"\n",
        "    top_k = min(top_k, logits.size(-1))  # Safety check\n",
        "    if top_k > 0:\n",
        "        # Remove all tokens with a probability less than the last token of the top-k\n",
        "        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]\n",
        "        logits[indices_to_remove] = filter_value\n",
        "\n",
        "    if top_p > 0.0:\n",
        "        sorted_logits, sorted_indices = torch.sort(logits, descending=True)\n",
        "        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)\n",
        "\n",
        "        # Remove tokens with cumulative probability above the threshold\n",
        "        sorted_indices_to_remove = cumulative_probs > top_p\n",
        "        # Shift the indices to the right to keep also the first token above the threshold\n",
        "        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()\n",
        "        sorted_indices_to_remove[..., 0] = 0\n",
        "\n",
        "        # scatter sorted tensors to original indexing\n",
        "        indices_to_remove = sorted_indices_to_remove.scatter(dim=1, index=sorted_indices, src=sorted_indices_to_remove)\n",
        "        logits[indices_to_remove] = filter_value\n",
        "    return logits\n",
        "\n",
        "\n",
        "def sample_sequence(model, length, context, num_samples=1, temperature=1, top_k=0, top_p=0.0, repetition_penalty=1.0,\n",
        "                    is_xlnet=False, is_xlm_mlm=False, xlm_mask_token=None, xlm_lang=None, device='cuda'):\n",
        "    context = torch.tensor(context, dtype=torch.long, device=device)\n",
        "    context = context.unsqueeze(0).repeat(num_samples, 1)\n",
        "    generated = context\n",
        "    with torch.no_grad():\n",
        "        for _ in trange(length):\n",
        "\n",
        "            inputs = {'input_ids': generated}\n",
        "            if is_xlnet: \n",
        "                # XLNet is a direct (predict same token, not next token) and bi-directional model by default\n",
        "                # => need one additional dummy token in the input (will be masked), attention mask and target mapping (see model docstring)\n",
        "                input_ids = torch.cat((generated, torch.zeros((1, 1), dtype=torch.long, device=device)), dim=1)\n",
        "                perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float, device=device)\n",
        "                perm_mask[:, :, -1] = 1.0  # Previous tokens don't see last token\n",
        "                target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float, device=device)\n",
        "                target_mapping[0, 0, -1] = 1.0  # predict last token\n",
        "                inputs = {'input_ids': input_ids, 'perm_mask': perm_mask, 'target_mapping': target_mapping}\n",
        "\n",
        "            if is_xlm_mlm and xlm_mask_token:\n",
        "                # XLM MLM models are direct models (predict same token, not next token)\n",
        "                # => need one additional dummy token in the input (will be masked and guessed)\n",
        "                input_ids = torch.cat((generated, torch.full((1, 1), xlm_mask_token, dtype=torch.long, device=device)), dim=1)\n",
        "                inputs = {'input_ids': input_ids}\n",
        "\n",
        "            if xlm_lang is not None:\n",
        "                inputs[\"langs\"] = torch.tensor([xlm_lang] * inputs[\"input_ids\"].shape[1], device=device).view(1, -1)\n",
        "\n",
        "            outputs = model(**inputs)  # Note: we could also use 'past' with GPT-2/Transfo-XL/XLNet/CTRL (cached hidden-states)\n",
        "            next_token_logits = outputs[0][:, -1, :] / (temperature if temperature > 0 else 1.)\n",
        "\n",
        "            # repetition penalty from CTRL (https://arxiv.org/abs/1909.05858)\n",
        "            for i in range(num_samples):\n",
        "                for _ in set(generated[i].tolist()):\n",
        "                    next_token_logits[i, _] /= repetition_penalty\n",
        "                \n",
        "            filtered_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p)\n",
        "            if temperature == 0: # greedy sampling:\n",
        "                next_token = torch.argmax(filtered_logits, dim=-1).unsqueeze(-1)\n",
        "            else:\n",
        "                next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1), num_samples=1)\n",
        "            generated = torch.cat((generated, next_token), dim=1)\n",
        "    return generated"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "daNmU9RLPttJ",
        "colab_type": "code",
        "outputId": "b2ce2cc7-0118-42e4-bee3-6e26c8229fea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "CONTEXT_TEXT = 'Всё ещё бабла нет, всё ещё с долгами канитель \\n \\\n",
        "Все ещё в подвале, всё ещё Parliament на бите \\n \\\n",
        "И я вернусь на трек, твой хуй как Тулуз-Лотрек \\n \\\n",
        "Если русский рэп в гробу сто лет, то я ебу скелет \\n \\\n",
        "И я построил альбом на костях \\n \\\n",
        "Этому не видно конца, как будто он голый толстяк \\n \\\n",
        "Каждый просит фит, каждый пишет: «Денег дам» \\n \\\n",
        "Вас миллион, но мой кумир – Гриша Перельман \\n'\n",
        "\n",
        "START_TEXT = 'Че ты мне скажешь епт '\n",
        "CONTEXT_TEXT = CONTEXT_TEXT + START_TEXT\n",
        "\n",
        "context_tokens = tokenizer.encode(CONTEXT_TEXT)\n",
        "sampled = sample_sequence(model, \n",
        "                          100, \n",
        "                          context_tokens, \n",
        "                          temperature = 1.0,\n",
        "                          top_p=0.99\n",
        "                          )\n",
        "\n",
        "out = sampled[:, len(context_tokens):].tolist()\n",
        "text = ''.join([tokenizer.decode(o) for o in out])\n",
        "\n",
        "# text = text[: text.find('<| endoftext|>')].split('\\n')\n",
        "text = text.split('\\n')\n",
        "\n",
        "print('-' * 20)\n",
        "\n",
        "for i, t in enumerate(text):\n",
        "    if i == 0:\n",
        "        print(START_TEXT + t)\n",
        "    else:\n",
        "        print(t)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 100/100 [00:03<00:00, 26.99it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "--------------------\n",
            "Че ты мне скажешь епт твою мать о том, что я вырос из подвала?\n",
            "Один, в дезабилье, и пишу песни от балды\n",
            "Будто ворованные башмаки «MC». Я обзавёлся правдой, но не стал эталоном\n",
            "Гастарбайтеры норовят сделать трек из культового куска\n",
            "Вообще, надо меньше врагов вокруг меня\n",
            "И, кстати, о врагах вокруг меня:\n",
            "Мне кажется, рэп – это такие маленькие комочки дерьма\n",
            "Который похоронен под кучей камней\n",
            "Но который клубится,\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}