flair.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "flair.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPER4SmwRfGTLqV5CowzOYd",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/buseskorkmaz/Sentiment-Analysis-with-Deep-Learning/blob/main/flair.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LaDJgZmE1rd-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8c92bd61-2a3b-47c7-d6a7-143eab915f9d"
      },
      "source": [
        "!pip install flair\n",
        "!pip install allennlp==0.9.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting allennlp==0.9.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/bb/041115d8bad1447080e5d1e30097c95e4b66e36074277afce8620a61cee3/allennlp-0.9.0-py3-none-any.whl (7.6MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6MB 2.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: sqlparse>=0.2.4 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (0.3.1)\n",
            "Collecting word2number>=1.1\n",
            "  Downloading https://files.pythonhosted.org/packages/4a/29/a31940c848521f0725f0df6b25dca8917f13a2025b0e8fcbe5d0457e45e6/word2number-1.1.zip\n",
            "Requirement already satisfied: torch>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (1.6.0+cu101)\n",
            "Requirement already satisfied: ftfy in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (5.8)\n",
            "Collecting flaky\n",
            "  Downloading https://files.pythonhosted.org/packages/43/0e/2f50064e327f41a1eb811df089f813036e19a64b95e33f8e9e0b96c2447e/flaky-3.7.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (1.18.5)\n",
            "Collecting parsimonious>=0.8.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/02/fc/067a3f89869a41009e1a7cdfb14725f8ddd246f30f63c645e8ef8a1c56f4/parsimonious-0.8.1.tar.gz (45kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: flask>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (1.1.2)\n",
            "Requirement already satisfied: requests>=2.18 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (2.23.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (3.2.5)\n",
            "Collecting pytorch-transformers==1.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/89/ad0d6bb932d0a51793eaabcf1617a36ff530dc9ab9e38f765a35dc293306/pytorch_transformers-1.1.0-py3-none-any.whl (158kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 35.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (3.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (0.22.2.post1)\n",
            "Requirement already satisfied: jsonpickle in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (1.4.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (2.10.0)\n",
            "Collecting conllu==1.3.1\n",
            "  Downloading https://files.pythonhosted.org/packages/ae/54/b0ae1199f3d01666821b028cd967f7c0ac527ab162af433d3da69242cea2/conllu-1.3.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (6.0.2)\n",
            "Requirement already satisfied: editdistance in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (0.5.3)\n",
            "Collecting pytorch-pretrained-bert>=0.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/e0/c08d5553b89973d9a240605b9c12404bcf8227590de62bae27acbcfe076b/pytorch_pretrained_bert-0.6.2-py3-none-any.whl (123kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 47.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (1.14.48)\n",
            "Requirement already satisfied: overrides in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (3.1.0)\n",
            "Requirement already satisfied: tqdm>=4.19 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (4.41.1)\n",
            "Collecting spacy<2.2,>=2.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/41/5b/e07dd3bf104237bce4b398558b104c8e500333d6f30eabe3fa9685356b7d/spacy-2.1.9-cp36-cp36m-manylinux1_x86_64.whl (30.8MB)\n",
            "\u001b[K     |████████████████████████████████| 30.9MB 100kB/s \n",
            "\u001b[?25hCollecting gevent>=1.3.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/05/18/3932900a42d7010cc63529fc5cb7a5a20fb61878d1721d0d4387567d5973/gevent-20.6.2-cp36-cp36m-manylinux2010_x86_64.whl (5.3MB)\n",
            "\u001b[K     |████████████████████████████████| 5.3MB 15.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorboardX>=1.2 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (2.1)\n",
            "Collecting unidecode\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d0/42/d9edfed04228bacea2d824904cae367ee9efd05e6cce7ceaaedd0b0ad964/Unidecode-1.1.1-py2.py3-none-any.whl (238kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 40.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: jsonnet>=0.10.0; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (0.16.0)\n",
            "Collecting responses>=0.7\n",
            "  Downloading https://files.pythonhosted.org/packages/c1/04/8a5258cfd851c9c89ae5c12c6952c05d42ca8c0788b999806e0c78d06c54/responses-0.12.0-py2.py3-none-any.whl\n",
            "Collecting flask-cors>=3.0.7\n",
            "  Downloading https://files.pythonhosted.org/packages/69/7f/d0aeaaafb5c3c76c8d2141dbe2d4f6dca5d6c31872d4e5349768c1958abc/Flask_Cors-3.0.9-py2.py3-none-any.whl\n",
            "Collecting numpydoc>=0.8.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/60/1d/9e398c53d6ae27d5ab312ddc16a9ffe1bee0dfdf1d6ec88c40b0ca97582e/numpydoc-1.1.0-py3-none-any.whl (47kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.6/dist-packages (from allennlp==0.9.0) (2018.9)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.2.0->allennlp==0.9.0) (0.16.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from ftfy->allennlp==0.9.0) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from parsimonious>=0.8.0->allennlp==0.9.0) (1.15.0)\n",
            "Requirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from flask>=1.0.2->allennlp==0.9.0) (1.1.0)\n",
            "Requirement already satisfied: Werkzeug>=0.15 in /usr/local/lib/python3.6/dist-packages (from flask>=1.0.2->allennlp==0.9.0) (1.0.1)\n",
            "Requirement already satisfied: click>=5.1 in /usr/local/lib/python3.6/dist-packages (from flask>=1.0.2->allennlp==0.9.0) (7.1.2)\n",
            "Requirement already satisfied: Jinja2>=2.10.1 in /usr/local/lib/python3.6/dist-packages (from flask>=1.0.2->allennlp==0.9.0) (2.11.2)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp==0.9.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp==0.9.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp==0.9.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.18->allennlp==0.9.0) (2020.6.20)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers==1.1.0->allennlp==0.9.0) (2019.12.20)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers==1.1.0->allennlp==0.9.0) (0.1.91)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->allennlp==0.9.0) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->allennlp==0.9.0) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->allennlp==0.9.0) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->allennlp==0.9.0) (2.8.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->allennlp==0.9.0) (0.16.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.6/dist-packages (from jsonpickle->allennlp==0.9.0) (1.7.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (8.4.0)\n",
            "Requirement already satisfied: py>=1.8.2 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (1.9.0)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (0.10.1)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (1.0.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (20.4)\n",
            "Requirement already satisfied: pluggy<1.0,>=0.12 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (0.13.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest->allennlp==0.9.0) (20.1.0)\n",
            "Requirement already satisfied: botocore<1.18.0,>=1.17.48 in /usr/local/lib/python3.6/dist-packages (from boto3->allennlp==0.9.0) (1.17.48)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->allennlp==0.9.0) (0.10.0)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->allennlp==0.9.0) (0.3.3)\n",
            "Collecting plac<1.0.0,>=0.9.6\n",
            "  Downloading https://files.pythonhosted.org/packages/9e/9b/62c60d2f5bc135d2aa1d8c8a86aaf84edb719a59c7f11a4316259e61a298/plac-0.9.6-py2.py3-none-any.whl\n",
            "Collecting blis<0.3.0,>=0.2.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/34/46/b1d0bb71d308e820ed30316c5f0a017cb5ef5f4324bcbc7da3cf9d3b075c/blis-0.2.4-cp36-cp36m-manylinux1_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 35.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy<2.2,>=2.1.0->allennlp==0.9.0) (1.0.2)\n",
            "Collecting thinc<7.1.0,>=7.0.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/a5/9ace20422e7bb1bdcad31832ea85c52a09900cd4a7ce711246bfb92206ba/thinc-7.0.8-cp36-cp36m-manylinux1_x86_64.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 33.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy<2.2,>=2.1.0->allennlp==0.9.0) (0.7.1)\n",
            "Requirement already satisfied: srsly<1.1.0,>=0.0.6 in /usr/local/lib/python3.6/dist-packages (from spacy<2.2,>=2.1.0->allennlp==0.9.0) (1.0.2)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy<2.2,>=2.1.0->allennlp==0.9.0) (2.0.3)\n",
            "Collecting preshed<2.1.0,>=2.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/93/f222fb957764a283203525ef20e62008675fd0a14ffff8cc1b1490147c63/preshed-2.0.1-cp36-cp36m-manylinux1_x86_64.whl (83kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 9.2MB/s \n",
            "\u001b[?25hCollecting zope.event\n",
            "  Downloading https://files.pythonhosted.org/packages/c5/96/361edb421a077a4c208b4a5c212737d78ae03ce67fbbcd01621c49f332d1/zope.event-4.4-py2.py3-none-any.whl\n",
            "Collecting greenlet>=0.4.16; platform_python_implementation == \"CPython\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/a4/0d8685c98986326534b0753a8b92b3082bc9df42b348bc50d6c69839c9f9/greenlet-0.4.16-cp36-cp36m-manylinux1_x86_64.whl (44kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from gevent>=1.3.6->allennlp==0.9.0) (49.6.0)\n",
            "Collecting zope.interface\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/57/33/565274c28a11af60b7cfc0519d46bde4125fcd7d32ebc0a81b480d0e8da6/zope.interface-5.1.0-cp36-cp36m-manylinux2010_x86_64.whl (234kB)\n",
            "\u001b[K     |████████████████████████████████| 235kB 45.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardX>=1.2->allennlp==0.9.0) (3.12.4)\n",
            "Requirement already satisfied: sphinx>=1.6.5 in /usr/local/lib/python3.6/dist-packages (from numpydoc>=0.8.0->allennlp==0.9.0) (1.8.5)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.10.1->flask>=1.0.2->allennlp==0.9.0) (1.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata->jsonpickle->allennlp==0.9.0) (3.1.0)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.48->boto3->allennlp==0.9.0) (0.15.2)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (1.2.4)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (2.1.3)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (1.2.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (0.7.12)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (2.8.0)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.6/dist-packages (from sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (2.0.0)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.6/dist-packages (from sphinxcontrib-websupport->sphinx>=1.6.5->numpydoc>=0.8.0->allennlp==0.9.0) (1.1.4)\n",
            "Building wheels for collected packages: word2number, parsimonious\n",
            "  Building wheel for word2number (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for word2number: filename=word2number-1.1-cp36-none-any.whl size=5588 sha256=e018a8b952f4997e3a19c79cf7c7766b8cd2c814f02d5578fce5c12ce89f088f\n",
            "  Stored in directory: /root/.cache/pip/wheels/46/2f/53/5f5c1d275492f2fce1cdab9a9bb12d49286dead829a4078e0e\n",
            "  Building wheel for parsimonious (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for parsimonious: filename=parsimonious-0.8.1-cp36-none-any.whl size=42709 sha256=34d9f1c75eedc8a1c8531638de952840abda40a38b63cbc9b6c8772a52abc247\n",
            "  Stored in directory: /root/.cache/pip/wheels/b7/8d/e7/a0e74217da5caeb3c1c7689639b6d28ddbf9985b840bc96a9a\n",
            "Successfully built word2number parsimonious\n",
            "\u001b[31mERROR: en-core-web-sm 2.2.5 has requirement spacy>=2.2.2, but you'll have spacy 2.1.9 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: responses 0.12.0 has requirement urllib3>=1.25.10, but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: word2number, flaky, parsimonious, pytorch-transformers, conllu, pytorch-pretrained-bert, plac, blis, preshed, thinc, spacy, zope.event, greenlet, zope.interface, gevent, unidecode, responses, flask-cors, numpydoc, allennlp\n",
            "  Found existing installation: plac 1.1.3\n",
            "    Uninstalling plac-1.1.3:\n",
            "      Successfully uninstalled plac-1.1.3\n",
            "  Found existing installation: blis 0.4.1\n",
            "    Uninstalling blis-0.4.1:\n",
            "      Successfully uninstalled blis-0.4.1\n",
            "  Found existing installation: preshed 3.0.2\n",
            "    Uninstalling preshed-3.0.2:\n",
            "      Successfully uninstalled preshed-3.0.2\n",
            "  Found existing installation: thinc 7.4.0\n",
            "    Uninstalling thinc-7.4.0:\n",
            "      Successfully uninstalled thinc-7.4.0\n",
            "  Found existing installation: spacy 2.2.4\n",
            "    Uninstalling spacy-2.2.4:\n",
            "      Successfully uninstalled spacy-2.2.4\n",
            "  Found existing installation: allennlp 1.1.0\n",
            "    Uninstalling allennlp-1.1.0:\n",
            "      Successfully uninstalled allennlp-1.1.0\n",
            "Successfully installed allennlp-0.9.0 blis-0.2.4 conllu-1.3.1 flaky-3.7.0 flask-cors-3.0.9 gevent-20.6.2 greenlet-0.4.16 numpydoc-1.1.0 parsimonious-0.8.1 plac-0.9.6 preshed-2.0.1 pytorch-pretrained-bert-0.6.2 pytorch-transformers-1.1.0 responses-0.12.0 spacy-2.1.9 thinc-7.0.8 unidecode-1.1.1 word2number-1.1 zope.event-4.4 zope.interface-5.1.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "allennlp",
                  "blis",
                  "plac",
                  "plac_core",
                  "plac_ext",
                  "preshed",
                  "spacy",
                  "thinc"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n3HSA2AS0e3c"
      },
      "source": [
        "import pandas as pd\n",
        "import tqdm\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LZ3msJkV0-7J",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 446
        },
        "outputId": "dabb5dc0-dc2f-40ce-a5c6-ecf3f2408252"
      },
      "source": [
        "df = pd.read_json('financial_news_scored.json', orient = 'records', encoding='utf-8').sample(frac=1)\n",
        "df['text'] = df['text'].map(lambda x: x.lstrip('BRIEF-'))\n",
        "df.set_index('index',inplace=True)\n",
        "# Optional lowercase for test data (if model was trained on lowercased text\n",
        "df['text'] = df['text'].str.lower()\n",
        "df['label'] = '__label__' + df['score'].astype(str)\n",
        "df= df.drop(columns=['versionCreated', 'storyId', 'sourceCode', 'storyText','score'])\n",
        "cols = df.columns.tolist()\n",
        "cols = cols[-1:] + cols[:-1]\n",
        "df = df[cols]\n",
        "\n",
        "df.iloc[0:int(len(df)*0.8)].to_csv('train.csv', sep='\\t', index = False, header = False)\n",
        "df.iloc[int(len(df)*0.8):int(len(df)*0.9)].to_csv('test.csv', sep='\\t', index = False, header = False)\n",
        "df.iloc[int(len(df)*0.9):].to_csv('dev.csv', sep='\\t', index = False, header = False)\n",
        "\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>index</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1588242145000</th>\n",
              "      <td>__label__1</td>\n",
              "      <td>va expects any potential future provisions aga...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1591886915578</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>itch revises anadolubank's outlook to negative...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1591286414000</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>akfen reit q1 net loss widens to 60.6 million ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1591451123000</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>south africa's mtn faces new allegations in u....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1591770119000</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>global yatirim holding q1 net loss increases t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1598690940000</th>\n",
              "      <td>__label__1</td>\n",
              "      <td>update 5-uae scraps israel boycott in new step...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1596037421000</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>marti reit q2 net result swings to loss of 11....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1598886714613</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>itch maintains negative rating watch on global...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1590672532000</th>\n",
              "      <td>__label__1</td>\n",
              "      <td>eysas reit rents storage facility at 2.7 mln l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1599149715725</th>\n",
              "      <td>__label__-1</td>\n",
              "      <td>itch revises anadolu sigorta's outlook to nega...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>733 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                     label                                               text\n",
              "index                                                                        \n",
              "1588242145000   __label__1  va expects any potential future provisions aga...\n",
              "1591886915578  __label__-1  itch revises anadolubank's outlook to negative...\n",
              "1591286414000  __label__-1  akfen reit q1 net loss widens to 60.6 million ...\n",
              "1591451123000  __label__-1  south africa's mtn faces new allegations in u....\n",
              "1591770119000  __label__-1  global yatirim holding q1 net loss increases t...\n",
              "...                    ...                                                ...\n",
              "1598690940000   __label__1  update 5-uae scraps israel boycott in new step...\n",
              "1596037421000  __label__-1  marti reit q2 net result swings to loss of 11....\n",
              "1598886714613  __label__-1  itch maintains negative rating watch on global...\n",
              "1590672532000   __label__1  eysas reit rents storage facility at 2.7 mln l...\n",
              "1599149715725  __label__-1  itch revises anadolu sigorta's outlook to nega...\n",
              "\n",
              "[733 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NX16UmIrkNXy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "e9f187ca-1175-47ff-fa29-3cb18e364867"
      },
      "source": [
        "from flair.data_fetcher import NLPTaskDataFetcher\n",
        "from flair.embeddings import WordEmbeddings, FlairEmbeddings, DocumentLSTMEmbeddings, DocumentRNNEmbeddings\n",
        "from flair.models import TextClassifier\n",
        "from flair.trainers import ModelTrainer\n",
        "from pathlib import Path\n",
        "from flair.data import Corpus\n",
        "from flair.datasets import ClassificationCorpus\n",
        "from flair.embeddings import TransformerDocumentEmbeddings,TransformerWordEmbeddings\n",
        "from flair.embeddings import BertEmbeddings, ELMoEmbeddings\n",
        "\n",
        "# this is the folder in which train, test and dev files reside\n",
        "data_folder = '/content/'\n",
        "\n",
        "# init a corpus using column format, data folder and the names of the train, dev and test files\n",
        "corpus: Corpus = ClassificationCorpus(data_folder,\n",
        "                              train_file='train.txt',\n",
        "                              test_file='test.txt',\n",
        "                              dev_file='dev.txt')\n",
        "# print the number of Sentences in the train split\n",
        "print(len(corpus.train))\n",
        "\n",
        "# print the number of Sentences in the test split\n",
        "print(len(corpus.test))\n",
        "\n",
        "# print the number of Sentences in the dev split\n",
        "print(len(corpus.dev))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-09-13 15:01:25,612 Reading data from /content\n",
            "2020-09-13 15:01:25,614 Train: /content/train.txt\n",
            "2020-09-13 15:01:25,620 Dev: /content/dev.txt\n",
            "2020-09-13 15:01:25,622 Test: /content/test.txt\n",
            "586\n",
            "73\n",
            "74\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZ_vqYRelsmM"
      },
      "source": [
        "from flair.embeddings import StackedEmbeddings\n",
        "\n",
        "# init BERT base (cases)\n",
        "#optional_embedding = BertEmbeddings('bert-base-uncased')\n",
        "# OR init ELMo (original)\n",
        "#optional_embedding = ELMoEmbeddings('original')\n",
        "\n",
        "#word_embeddings = [\n",
        "#    optional_embedding,\n",
        "#    FlairEmbeddings('news-forward'),\n",
        "#    FlairEmbeddings('news-backward')]\n",
        "\n",
        "\n",
        "#word_embeddings = [WordEmbeddings('glove')]\n",
        "\n",
        "#document_embeddings = DocumentRNNEmbeddings(\n",
        "#        word_embeddings,\n",
        "#        hidden_size=512,\n",
        "#        reproject_words=True,\n",
        "#        reproject_words_dimension=256\n",
        "#    )\n",
        "\n",
        "document_embeddings = TransformerDocumentEmbeddings('distilbert-base-uncased',fine_tune=True)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MtH4nGnZteOk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e6c8c4ae-3ee4-4bf9-d881-3aa55c88b93e"
      },
      "source": [
        "classifier = TextClassifier(document_embeddings, \n",
        "                            label_dictionary=corpus.make_label_dictionary(), \n",
        "                            multi_label=True)\n",
        "\n",
        "trainer = ModelTrainer(classifier, corpus)\n",
        "trainer.train('./', max_epochs=5,mini_batch_size=32)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-09-13 15:37:41,675 Computing label dictionary. Progress:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 659/659 [00:00<00:00, 801.78it/s] "
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2020-09-13 15:37:43,252 [b'0', b'-1', b'1']\n",
            "2020-09-13 15:37:43,269 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:43,271 Model: \"TextClassifier(\n",
            "  (document_embeddings): TransformerDocumentEmbeddings(\n",
            "    (model): DistilBertModel(\n",
            "      (embeddings): Embeddings(\n",
            "        (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "        (position_embeddings): Embedding(512, 768)\n",
            "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (transformer): Transformer(\n",
            "        (layer): ModuleList(\n",
            "          (0): TransformerBlock(\n",
            "            (attention): MultiHeadSelfAttention(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            )\n",
            "            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (ffn): FFN(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            )\n",
            "            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (1): TransformerBlock(\n",
            "            (attention): MultiHeadSelfAttention(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            )\n",
            "            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (ffn): FFN(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            )\n",
            "            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (2): TransformerBlock(\n",
            "            (attention): MultiHeadSelfAttention(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            )\n",
            "            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (ffn): FFN(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            )\n",
            "            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (3): TransformerBlock(\n",
            "            (attention): MultiHeadSelfAttention(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            )\n",
            "            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (ffn): FFN(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            )\n",
            "            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (4): TransformerBlock(\n",
            "            (attention): MultiHeadSelfAttention(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            )\n",
            "            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (ffn): FFN(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            )\n",
            "            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "          (5): TransformerBlock(\n",
            "            (attention): MultiHeadSelfAttention(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            )\n",
            "            (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (ffn): FFN(\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "              (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "              (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            )\n",
            "            (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (decoder): Linear(in_features=768, out_features=3, bias=True)\n",
            "  (loss_function): BCEWithLogitsLoss()\n",
            "  (beta): 1.0\n",
            "  (weights): None\n",
            "  (weight_tensor) None\n",
            ")\"\n",
            "2020-09-13 15:37:43,273 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:43,275 Corpus: \"Corpus: 586 train + 74 dev + 73 test sentences\"\n",
            "2020-09-13 15:37:43,279 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:43,281 Parameters:\n",
            "2020-09-13 15:37:43,282  - learning_rate: \"0.1\"\n",
            "2020-09-13 15:37:43,284  - mini_batch_size: \"32\"\n",
            "2020-09-13 15:37:43,285  - patience: \"3\"\n",
            "2020-09-13 15:37:43,286  - anneal_factor: \"0.5\"\n",
            "2020-09-13 15:37:43,288  - max_epochs: \"5\"\n",
            "2020-09-13 15:37:43,290  - shuffle: \"True\"\n",
            "2020-09-13 15:37:43,292  - train_with_dev: \"False\"\n",
            "2020-09-13 15:37:43,294  - batch_growth_annealing: \"False\"\n",
            "2020-09-13 15:37:43,296 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:43,298 Model training base path: \".\"\n",
            "2020-09-13 15:37:43,302 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:43,303 Device: cuda:0\n",
            "2020-09-13 15:37:43,304 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:43,305 Embeddings storage mode: cpu\n",
            "2020-09-13 15:37:43,308 ----------------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2020-09-13 15:37:44,804 epoch 1 - iter 1/19 - loss 0.69793844 - samples/sec: 38.94 - lr: 0.100000\n",
            "2020-09-13 15:37:45,535 epoch 1 - iter 2/19 - loss 0.36785468 - samples/sec: 44.65 - lr: 0.100000\n",
            "2020-09-13 15:37:46,292 epoch 1 - iter 3/19 - loss 0.24853520 - samples/sec: 42.50 - lr: 0.100000\n",
            "2020-09-13 15:37:46,981 epoch 1 - iter 4/19 - loss 0.19249319 - samples/sec: 47.05 - lr: 0.100000\n",
            "2020-09-13 15:37:47,690 epoch 1 - iter 5/19 - loss 0.15664401 - samples/sec: 45.24 - lr: 0.100000\n",
            "2020-09-13 15:37:48,430 epoch 1 - iter 6/19 - loss 0.13338151 - samples/sec: 43.32 - lr: 0.100000\n",
            "2020-09-13 15:37:49,257 epoch 1 - iter 7/19 - loss 0.11637191 - samples/sec: 41.02 - lr: 0.100000\n",
            "2020-09-13 15:37:49,905 epoch 1 - iter 8/19 - loss 0.10350685 - samples/sec: 49.51 - lr: 0.100000\n",
            "2020-09-13 15:37:50,572 epoch 1 - iter 9/19 - loss 0.09351362 - samples/sec: 48.05 - lr: 0.100000\n",
            "2020-09-13 15:37:51,296 epoch 1 - iter 10/19 - loss 0.08490687 - samples/sec: 44.35 - lr: 0.100000\n",
            "2020-09-13 15:37:51,984 epoch 1 - iter 11/19 - loss 0.07764965 - samples/sec: 46.56 - lr: 0.100000\n",
            "2020-09-13 15:37:52,694 epoch 1 - iter 12/19 - loss 0.07197406 - samples/sec: 45.48 - lr: 0.100000\n",
            "2020-09-13 15:37:53,424 epoch 1 - iter 13/19 - loss 0.06721457 - samples/sec: 44.21 - lr: 0.100000\n",
            "2020-09-13 15:37:54,093 epoch 1 - iter 14/19 - loss 0.06284460 - samples/sec: 47.93 - lr: 0.100000\n",
            "2020-09-13 15:37:54,764 epoch 1 - iter 15/19 - loss 0.05892377 - samples/sec: 47.78 - lr: 0.100000\n",
            "2020-09-13 15:37:55,479 epoch 1 - iter 16/19 - loss 0.05574494 - samples/sec: 44.78 - lr: 0.100000\n",
            "2020-09-13 15:37:56,180 epoch 1 - iter 17/19 - loss 0.05300221 - samples/sec: 45.74 - lr: 0.100000\n",
            "2020-09-13 15:37:56,964 epoch 1 - iter 18/19 - loss 0.05040090 - samples/sec: 40.87 - lr: 0.100000\n",
            "2020-09-13 15:37:57,191 epoch 1 - iter 19/19 - loss 0.05103881 - samples/sec: 143.61 - lr: 0.100000\n",
            "2020-09-13 15:37:57,490 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:37:57,492 EPOCH 1 done: loss 0.0510 - lr 0.1000000\n",
            "2020-09-13 15:37:59,099 DEV : loss 0.829311728477478 - score 0.6575\n",
            "2020-09-13 15:37:59,145 BAD EPOCHS (no improvement): 0\n",
            "saving best model\n",
            "2020-09-13 15:38:00,059 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:01,549 epoch 2 - iter 1/19 - loss 0.00426148 - samples/sec: 39.57 - lr: 0.100000\n",
            "2020-09-13 15:38:02,299 epoch 2 - iter 2/19 - loss 0.00498504 - samples/sec: 43.37 - lr: 0.100000\n",
            "2020-09-13 15:38:03,108 epoch 2 - iter 3/19 - loss 0.01466987 - samples/sec: 40.09 - lr: 0.100000\n",
            "2020-09-13 15:38:03,847 epoch 2 - iter 4/19 - loss 0.01736598 - samples/sec: 43.43 - lr: 0.100000\n",
            "2020-09-13 15:38:04,583 epoch 2 - iter 5/19 - loss 0.01511695 - samples/sec: 44.40 - lr: 0.100000\n",
            "2020-09-13 15:38:05,270 epoch 2 - iter 6/19 - loss 0.01365049 - samples/sec: 47.56 - lr: 0.100000\n",
            "2020-09-13 15:38:05,949 epoch 2 - iter 7/19 - loss 0.01246501 - samples/sec: 47.21 - lr: 0.100000\n",
            "2020-09-13 15:38:07,720 epoch 2 - iter 8/19 - loss 0.01127466 - samples/sec: 50.90 - lr: 0.100000\n",
            "2020-09-13 15:38:08,461 epoch 2 - iter 9/19 - loss 0.01040627 - samples/sec: 44.98 - lr: 0.100000\n",
            "2020-09-13 15:38:09,275 epoch 2 - iter 10/19 - loss 0.00973154 - samples/sec: 39.54 - lr: 0.100000\n",
            "2020-09-13 15:38:10,070 epoch 2 - iter 11/19 - loss 0.00910430 - samples/sec: 40.27 - lr: 0.100000\n",
            "2020-09-13 15:38:10,786 epoch 2 - iter 12/19 - loss 0.00870302 - samples/sec: 44.79 - lr: 0.100000\n",
            "2020-09-13 15:38:11,609 epoch 2 - iter 13/19 - loss 0.00820592 - samples/sec: 38.99 - lr: 0.100000\n",
            "2020-09-13 15:38:12,421 epoch 2 - iter 14/19 - loss 0.00806012 - samples/sec: 39.59 - lr: 0.100000\n",
            "2020-09-13 15:38:13,119 epoch 2 - iter 15/19 - loss 0.00777247 - samples/sec: 46.88 - lr: 0.100000\n",
            "2020-09-13 15:38:13,755 epoch 2 - iter 16/19 - loss 0.00743139 - samples/sec: 50.50 - lr: 0.100000\n",
            "2020-09-13 15:38:14,425 epoch 2 - iter 17/19 - loss 0.00709732 - samples/sec: 47.90 - lr: 0.100000\n",
            "2020-09-13 15:38:15,078 epoch 2 - iter 18/19 - loss 0.00684637 - samples/sec: 49.11 - lr: 0.100000\n",
            "2020-09-13 15:38:15,329 epoch 2 - iter 19/19 - loss 0.00661129 - samples/sec: 128.54 - lr: 0.100000\n",
            "2020-09-13 15:38:15,662 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:15,663 EPOCH 2 done: loss 0.0066 - lr 0.1000000\n",
            "2020-09-13 15:38:17,315 DEV : loss 0.8733912706375122 - score 0.6986\n",
            "2020-09-13 15:38:17,363 BAD EPOCHS (no improvement): 0\n",
            "saving best model\n",
            "2020-09-13 15:38:18,374 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:19,858 epoch 3 - iter 1/19 - loss 0.00354127 - samples/sec: 36.02 - lr: 0.100000\n",
            "2020-09-13 15:38:20,731 epoch 3 - iter 2/19 - loss 0.00331790 - samples/sec: 37.38 - lr: 0.100000\n",
            "2020-09-13 15:38:21,493 epoch 3 - iter 3/19 - loss 0.00281113 - samples/sec: 42.56 - lr: 0.100000\n",
            "2020-09-13 15:38:22,167 epoch 3 - iter 4/19 - loss 0.00282975 - samples/sec: 47.53 - lr: 0.100000\n",
            "2020-09-13 15:38:23,050 epoch 3 - iter 5/19 - loss 0.00278379 - samples/sec: 36.33 - lr: 0.100000\n",
            "2020-09-13 15:38:23,770 epoch 3 - iter 6/19 - loss 0.00270470 - samples/sec: 46.13 - lr: 0.100000\n",
            "2020-09-13 15:38:24,580 epoch 3 - iter 7/19 - loss 0.00259195 - samples/sec: 39.59 - lr: 0.100000\n",
            "2020-09-13 15:38:25,429 epoch 3 - iter 8/19 - loss 0.00243635 - samples/sec: 37.88 - lr: 0.100000\n",
            "2020-09-13 15:38:26,308 epoch 3 - iter 9/19 - loss 0.00231843 - samples/sec: 36.45 - lr: 0.100000\n",
            "2020-09-13 15:38:27,184 epoch 3 - iter 10/19 - loss 0.00229055 - samples/sec: 36.61 - lr: 0.100000\n",
            "2020-09-13 15:38:27,899 epoch 3 - iter 11/19 - loss 0.00230482 - samples/sec: 45.74 - lr: 0.100000\n",
            "2020-09-13 15:38:28,587 epoch 3 - iter 12/19 - loss 0.00311474 - samples/sec: 47.10 - lr: 0.100000\n",
            "2020-09-13 15:38:29,365 epoch 3 - iter 13/19 - loss 0.00302364 - samples/sec: 41.23 - lr: 0.100000\n",
            "2020-09-13 15:38:30,079 epoch 3 - iter 14/19 - loss 0.00311279 - samples/sec: 44.92 - lr: 0.100000\n",
            "2020-09-13 15:38:30,977 epoch 3 - iter 15/19 - loss 0.00307319 - samples/sec: 35.69 - lr: 0.100000\n",
            "2020-09-13 15:38:31,662 epoch 3 - iter 16/19 - loss 0.00299322 - samples/sec: 46.80 - lr: 0.100000\n",
            "2020-09-13 15:38:32,320 epoch 3 - iter 17/19 - loss 0.00294364 - samples/sec: 49.20 - lr: 0.100000\n",
            "2020-09-13 15:38:33,000 epoch 3 - iter 18/19 - loss 0.00308218 - samples/sec: 47.92 - lr: 0.100000\n",
            "2020-09-13 15:38:33,231 epoch 3 - iter 19/19 - loss 0.00299710 - samples/sec: 140.12 - lr: 0.100000\n",
            "2020-09-13 15:38:33,535 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:33,539 EPOCH 3 done: loss 0.0030 - lr 0.1000000\n",
            "2020-09-13 15:38:35,086 DEV : loss 0.9419206380844116 - score 0.7123\n",
            "2020-09-13 15:38:35,132 BAD EPOCHS (no improvement): 0\n",
            "saving best model\n",
            "2020-09-13 15:38:36,150 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:37,617 epoch 4 - iter 1/19 - loss 0.00337151 - samples/sec: 35.98 - lr: 0.100000\n",
            "2020-09-13 15:38:38,414 epoch 4 - iter 2/19 - loss 0.00245005 - samples/sec: 40.91 - lr: 0.100000\n",
            "2020-09-13 15:38:39,121 epoch 4 - iter 3/19 - loss 0.00234230 - samples/sec: 45.52 - lr: 0.100000\n",
            "2020-09-13 15:38:39,806 epoch 4 - iter 4/19 - loss 0.00229959 - samples/sec: 48.25 - lr: 0.100000\n",
            "2020-09-13 15:38:40,523 epoch 4 - iter 5/19 - loss 0.00220245 - samples/sec: 46.11 - lr: 0.100000\n",
            "2020-09-13 15:38:41,236 epoch 4 - iter 6/19 - loss 0.00210336 - samples/sec: 45.57 - lr: 0.100000\n",
            "2020-09-13 15:38:42,120 epoch 4 - iter 7/19 - loss 0.00198959 - samples/sec: 36.29 - lr: 0.100000\n",
            "2020-09-13 15:38:42,786 epoch 4 - iter 8/19 - loss 0.00189583 - samples/sec: 48.22 - lr: 0.100000\n",
            "2020-09-13 15:38:43,508 epoch 4 - iter 9/19 - loss 0.00186448 - samples/sec: 44.44 - lr: 0.100000\n",
            "2020-09-13 15:38:44,388 epoch 4 - iter 10/19 - loss 0.00185948 - samples/sec: 36.40 - lr: 0.100000\n",
            "2020-09-13 15:38:45,229 epoch 4 - iter 11/19 - loss 0.00341571 - samples/sec: 38.10 - lr: 0.100000\n",
            "2020-09-13 15:38:45,908 epoch 4 - iter 12/19 - loss 0.01073462 - samples/sec: 49.00 - lr: 0.100000\n",
            "2020-09-13 15:38:46,584 epoch 4 - iter 13/19 - loss 0.01021024 - samples/sec: 47.48 - lr: 0.100000\n",
            "2020-09-13 15:38:47,296 epoch 4 - iter 14/19 - loss 0.02113367 - samples/sec: 45.33 - lr: 0.100000\n",
            "2020-09-13 15:38:47,986 epoch 4 - iter 15/19 - loss 0.02000495 - samples/sec: 46.43 - lr: 0.100000\n",
            "2020-09-13 15:38:48,839 epoch 4 - iter 16/19 - loss 0.01887007 - samples/sec: 37.59 - lr: 0.100000\n",
            "2020-09-13 15:38:49,576 epoch 4 - iter 17/19 - loss 0.01792420 - samples/sec: 43.50 - lr: 0.100000\n",
            "2020-09-13 15:38:50,249 epoch 4 - iter 18/19 - loss 0.01789645 - samples/sec: 48.08 - lr: 0.100000\n",
            "2020-09-13 15:38:50,535 epoch 4 - iter 19/19 - loss 0.02063034 - samples/sec: 113.50 - lr: 0.100000\n",
            "2020-09-13 15:38:50,813 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:50,814 EPOCH 4 done: loss 0.0206 - lr 0.1000000\n",
            "2020-09-13 15:38:52,468 DEV : loss 0.7649868726730347 - score 0.6667\n",
            "2020-09-13 15:38:52,513 BAD EPOCHS (no improvement): 1\n",
            "2020-09-13 15:38:52,514 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:38:54,149 epoch 5 - iter 1/19 - loss 0.05435054 - samples/sec: 30.20 - lr: 0.100000\n",
            "2020-09-13 15:38:55,020 epoch 5 - iter 2/19 - loss 0.04034793 - samples/sec: 37.02 - lr: 0.100000\n",
            "2020-09-13 15:38:55,872 epoch 5 - iter 3/19 - loss 0.02900793 - samples/sec: 38.46 - lr: 0.100000\n",
            "2020-09-13 15:38:56,780 epoch 5 - iter 4/19 - loss 0.02224976 - samples/sec: 35.76 - lr: 0.100000\n",
            "2020-09-13 15:38:57,563 epoch 5 - iter 5/19 - loss 0.01917046 - samples/sec: 42.02 - lr: 0.100000\n",
            "2020-09-13 15:38:58,349 epoch 5 - iter 6/19 - loss 0.01654512 - samples/sec: 40.80 - lr: 0.100000\n",
            "2020-09-13 15:38:59,186 epoch 5 - iter 7/19 - loss 0.01538747 - samples/sec: 38.30 - lr: 0.100000\n",
            "2020-09-13 15:38:59,878 epoch 5 - iter 8/19 - loss 0.01494070 - samples/sec: 46.34 - lr: 0.100000\n",
            "2020-09-13 15:39:00,559 epoch 5 - iter 9/19 - loss 0.01566603 - samples/sec: 47.12 - lr: 0.100000\n",
            "2020-09-13 15:39:01,210 epoch 5 - iter 10/19 - loss 0.01463269 - samples/sec: 49.23 - lr: 0.100000\n",
            "2020-09-13 15:39:01,921 epoch 5 - iter 11/19 - loss 0.01545634 - samples/sec: 46.12 - lr: 0.100000\n",
            "2020-09-13 15:39:02,869 epoch 5 - iter 12/19 - loss 0.01447314 - samples/sec: 33.97 - lr: 0.100000\n",
            "2020-09-13 15:39:03,601 epoch 5 - iter 13/19 - loss 0.01387473 - samples/sec: 43.82 - lr: 0.100000\n",
            "2020-09-13 15:39:04,317 epoch 5 - iter 14/19 - loss 0.01299431 - samples/sec: 45.43 - lr: 0.100000\n",
            "2020-09-13 15:39:05,194 epoch 5 - iter 15/19 - loss 0.01291471 - samples/sec: 36.56 - lr: 0.100000\n",
            "2020-09-13 15:39:05,870 epoch 5 - iter 16/19 - loss 0.01236207 - samples/sec: 47.50 - lr: 0.100000\n",
            "2020-09-13 15:39:06,530 epoch 5 - iter 17/19 - loss 0.01268550 - samples/sec: 49.86 - lr: 0.100000\n",
            "2020-09-13 15:39:07,170 epoch 5 - iter 18/19 - loss 0.01204961 - samples/sec: 50.13 - lr: 0.100000\n",
            "2020-09-13 15:39:07,438 epoch 5 - iter 19/19 - loss 0.01156208 - samples/sec: 119.75 - lr: 0.100000\n",
            "2020-09-13 15:39:07,729 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:39:07,730 EPOCH 5 done: loss 0.0116 - lr 0.1000000\n",
            "2020-09-13 15:39:09,287 DEV : loss 0.9187461733818054 - score 0.7027\n",
            "2020-09-13 15:39:09,338 BAD EPOCHS (no improvement): 2\n",
            "2020-09-13 15:39:10,324 ----------------------------------------------------------------------------------------------------\n",
            "2020-09-13 15:39:10,329 Testing using best model ...\n",
            "2020-09-13 15:39:10,335 loading file best-model.pt\n",
            "2020-09-13 15:39:13,947 0.831\t0.6873\t0.7295\t0.7534\n",
            "2020-09-13 15:39:13,948 \n",
            "Results:\n",
            "- F-score (micro) 0.7534\n",
            "- F-score (macro) 0.7295\n",
            "- Accuracy 0.7534\n",
            "\n",
            "By class:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0     1.0000    0.5000    0.6667         8\n",
            "          -1     0.7857    0.7333    0.7586        30\n",
            "           1     0.7073    0.8286    0.7632        35\n",
            "\n",
            "   micro avg     0.7534    0.7534    0.7534        73\n",
            "   macro avg     0.8310    0.6873    0.7295        73\n",
            "weighted avg     0.7716    0.7534    0.7507        73\n",
            " samples avg     0.7534    0.7534    0.7534        73\n",
            "\n",
            "2020-09-13 15:39:13,954 ----------------------------------------------------------------------------------------------------\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'dev_loss_history': [0.829311728477478,\n",
              "  0.8733912706375122,\n",
              "  0.9419206380844116,\n",
              "  0.7649868726730347,\n",
              "  0.9187461733818054],\n",
              " 'dev_score_history': [0.6575, 0.6986, 0.7123, 0.6667, 0.7027],\n",
              " 'test_score': 0.7534,\n",
              " 'train_loss_history': [0.051038811448961496,\n",
              "  0.006611294591015107,\n",
              "  0.0029970998324355797,\n",
              "  0.020630335741627374,\n",
              "  0.011562082830718473]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    }
  ]
}