Martin-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from gensim.models import Word2Vec\n",
    "from nltk import sent_tokenize\n",
    "from nltk.corpus import stopwords, wordnet\n",
    "from nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer #for word embedding\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score # bag of words\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "import collections\n",
    "import csv\n",
    "import gensim\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import nltk.tokenize\n",
    "import numpy as np #for text pre-processing\n",
    "import operator\n",
    "import pandas as pd\n",
    "import pathlib\n",
    "import pickle\n",
    "import re, string\n",
    "import seaborn\n",
    "import string\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')#for model-building\n",
    "ps = PorterStemmer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _1) Reading in files and append them into corpus list_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok = nltk.tokenize.TreebankWordTokenizer()\n",
    "\n",
    "def make_corpus(filename):\n",
    "    corpus = []\n",
    "    with open(filename, 'r') as f:\n",
    "        for line in f:\n",
    "            corpus.extend(t for line in f for t in tok.tokenize(line))\n",
    "    \n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _2) Convert corpus into data frame_\n",
    "\n",
    "a) voc: counts the total number of tokens in corpus\n",
    "\n",
    "b) frq_ returns a dataframe with tokens and frequency of each token\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataframe_corpus(corpus):\n",
    "    voc = collections.Counter(corpus)\n",
    "    frq = pandas.DataFrame(voc.most_common(), columns=['token', 'frequency'])\n",
    "\n",
    "    return frq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _3) Frequency normalized by corpus size and Cumulative normalized frequency inserted into dataframe_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freq_cum(frq):\n",
    "    # Index in the sorted list\n",
    "    frq['idx'] = frq.index + 1\n",
    "\n",
    "    # Frequency normalised by corpus size\n",
    "    frq['norm_freq'] = frq.frequency / len(corpus)\n",
    "\n",
    "    # Cumulative normalised frequency\n",
    "    frq['cumul_frq'] = frq.norm_freq.cumsum()\n",
    "\n",
    "    return frq\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _4) Plots for Culumative frequency and Log-log plot for Zipf's law_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seaborn.set(style='whitegrid')\n",
    "\n",
    "# Plot: Cumulative frequency by index\n",
    "def freq_cum_plot(frq):\n",
    "    seaborn.relplot(x='idx', y='cumul_frq', data=frq)\n",
    "    return plt.show()\n",
    "\n",
    "# Plot: Cumulative frequency by index, top x tokens\n",
    "def freq_cum_plot_top_x_tokens(frq, top_x):\n",
    "    seaborn.relplot(x='idx', y='cumul_frq', data=frq[:int(top_x)], kind='line')\n",
    "    return plt.show()\n",
    "\n",
    "# Plot: Log-log plot for Zipf's law\n",
    "def zipfs_law(frq):\n",
    "    frq['log_frq'] = numpy.log(frq.frequency)\n",
    "    frq['log_rank'] = numpy.log(frq.frequency.rank(ascending=True))\n",
    "    seaborn.relplot(x='log_rank', y='log_frq', data=frq)\n",
    "    return plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a.1) Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def token_sentences(text):\n",
    "    sentences = sent_tokenize(text)\n",
    "    return sentences\n",
    "\n",
    "\n",
    "def tokenization(words,text):\n",
    "    tokens = word_tokenize(text[1:])\n",
    "    words = [token.lower() for token in tokens if token.isalpha()]   \n",
    "    return words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### a.1.1) Comparing TweetTokenizer with NLTK Tokenizer (Task 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.tokenize import  word_tokenize\n",
    "\n",
    "def comparing_tokenizer(text):\n",
    "    tt = TweetTokenizer()\n",
    "    tweet_tokenizer = tt.tokenize(text)\n",
    "    word_tweet_tok = word_tokenize(text)\n",
    "    return tweet_tokenizer, word_tweet_tok\n",
    "\n",
    "#output example:\n",
    "\n",
    "# tweet_tokenizer       = ['#ibelieveblaseyford', 'is', 'liar', 'she', 'is']\n",
    "# word_tweet_tok (NLTK) = ['#', 'ibelieveblaseyford', 'is', 'liar', 'she']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### a.1.2) Spacy library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_in_sentences(text):\n",
    "    doc = nlp(text)\n",
    "    return [str(sent).strip() for sent in doc.sents]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a.2) Vocabulary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting a dataframe into a single list \n",
    "#text is split into words defined by their space inbetween\n",
    "#words are inserted into list \n",
    "def words_list(text):\n",
    "    #words are inserted into list \n",
    "    corpus=[]\n",
    "    for row in text:\n",
    "        tokens = row[0].split(\" \")\n",
    "        for token in tokens:\n",
    "            corpus.append(token)\n",
    "    \n",
    "    \n",
    "    def vocabulary_list(corpus):\n",
    "        #initlialize the vocabulary\n",
    "        vocab = list(set(\" \".join(corpus)))\n",
    "        vocab.remove(' ')\n",
    "        return vocab\n",
    "      \n",
    "    \n",
    "    def split_words_char(corpus):\n",
    "        #split the word into characters\n",
    "        corpus = [\" \".join(token) for token in corpus]\n",
    "        #appending </w>\n",
    "        corpus=[token+' </w>' for token in corpus]\n",
    "        return corpus\n",
    "        \n",
    "    x,y = split_words_char(corpus), vocabulary_list(corpus)\n",
    "    return x,y\n",
    "\n",
    "\n",
    "# Creating the Bag of Words model\n",
    "def bag_of_words(text):\n",
    "    word2count = {}\n",
    "    for data in text:\n",
    "        words = nltk.word_tokenize(data)\n",
    "        for word in words:  \n",
    "            if word not in word2count.keys():\n",
    "                word2count[word] = 1\n",
    "            else:\n",
    "                word2count[word] += 1\n",
    "    return word2count   \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## b) Pre-processing and cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing punctuations\n",
    "# convert all words into lower cases\n",
    "# remove stop words\n",
    "\n",
    "def preprocess_text(words):\n",
    "    #delete punctuations\n",
    "    punctuation_table = str.maketrans('', '', string.punctuation)\n",
    "\n",
    "    #convert all words into lower cases\n",
    "    words = [word.translate(punctuation_table).lower() for word in words]\n",
    "    \n",
    "    #remove stopwords\n",
    "    from nltk.corpus import stopwords\n",
    "    stop_words = stopwords.words('english')\n",
    "    words = [word for word in words if word not in stop_words]\n",
    "\n",
    "    return words\n",
    "\n",
    "# cleaning sentences within data frame\n",
    "def  clean_text(df, text_field):\n",
    "    df[text_field] = df[text_field].str.lower()\n",
    "    df[text_field] = df[text_field].apply(lambda elem: re.sub(r\"(@[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)|^rt|http.+?\", \" \", elem))  \n",
    "    return df\n",
    "\n",
    "def del_punctuations(words):\n",
    "    punctuation_table = str.maketrans('', '', string.punctuation)\n",
    "    words = [word.translate(punctuation_table).lower() for word in words]\n",
    "    return words\n",
    "\n",
    "\n",
    "def stop_words(words):\n",
    "    stop_words = stopwords.words('english')\n",
    "    words = [word for word in words if word not in stop_words]\n",
    "    return words\n",
    "\n",
    "def word_lemmatizer(text):\n",
    "    lem_text = [WordNetLemmatizer().lemmatize(i) for i in text]\n",
    "    return lem_text\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## c) Basic statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of tokens, sentences, average tokens, total unique tokens, total number of tokens after cleaning\n",
    "\n",
    "def basic_statistics(text,printer = False):\n",
    "    sents = nltk.sent_tokenize(text)\n",
    "    tokens = word_tokenize(text[1:])\n",
    "    words = [token.lower() for token in tokens if token.isalpha()]\n",
    "    average_tokens = round(len(words)/len(sents))\n",
    "    unique_tokens = set(words)\n",
    "    token_ratio = round(len(unique_tokens)/len(tokens),3)\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    final_tokens = []\n",
    "    for each in words:\n",
    "        if each not in stop_words:\n",
    "            final_tokens.append(each)\n",
    "    if printer == True:\n",
    "        print(f'The number of characters: {len(text)}')\n",
    "        print(f'The number of tokens is: {len(tokens)}')\n",
    "        print(f'The number of sentences is: {len(sents)}')\n",
    "        print(f'The average number of tokens per sentence is: {average_tokens}')\n",
    "        print(f'The number of unique tokens are: {len(unique_tokens)}')\n",
    "        print(f'The tokens ratio is: {token_ratio}')\n",
    "        print(f'The number of total tokens after removing stopwords are: {len(final_tokens)}')\n",
    "    return len(text), len(tokens), len(sents), average_tokens, len(unique_tokens), token_ratio, len(final_tokens)\n",
    "\n",
    "def word_frequency(words):\n",
    "    frequency_words = collections.Counter(words)\n",
    "    \n",
    "    #convert counter object to dictionary\n",
    "    frequency_words_dict = dict(frequency_words)\n",
    "    res = dict(sorted(frequency_words_dict.items(), key=lambda item: item[1]))\n",
    "    return res\n",
    "\n",
    "def top_20_most_common_words(freq_words):\n",
    "    res = dict(Counter(freq_words).most_common(20))\n",
    "    return res\n",
    "\n",
    "def most_common_words(dictionary):\n",
    "    # Output a dict of most common words\n",
    "    return dict(sorted(dictionary.items(),key=lambda x: x[1], reverse=True))\n",
    "\n",
    "def least_common_words(dictionary):\n",
    "    return sorted(list(dictionary.items()),key=lambda x: x[1])\n",
    "\n",
    "\n",
    "# Function for loglog plots\n",
    "def llplot(list_var, labels, title):\n",
    "    \"\"\"Function that takes a list of datasets, list of labels and a title as string, and plots a loglogplot, example:\n",
    "    llplot([offensive_freq_words_val, offensive_freq_words_train, offensive_freq_words_test], [\"val\", \"train\", \"test\"], \"Offensive dataset, loglog plot\")\n",
    "    \"\"\"\n",
    "    # Size of the figure:\n",
    "    plt.figure(figsize = (7,6))\n",
    "    # Iterating through the datasets:\n",
    "    for idx, i in enumerate(list_var):\n",
    "        y = np.log(list(most_common_words(i).values()))\n",
    "        x = np.log([i for i in range(1,len(y)+1)])\n",
    "        plt.scatter(x, y, label = labels[idx])\n",
    "    # Labeling title and axis:\n",
    "    plt.title(title)\n",
    "    plt.xlabel(\"Rank of word\")\n",
    "    plt.ylabel(\"Frequency of word\")\n",
    "    # Plotting:\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    return None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spacy - preprocessing/cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merging_labels_and_sentences(textfile, labelfile):\n",
    "\n",
    "    # read textfile and labelfile into two separate dataframes\n",
    "    df_text = pd.read_csv(textfile + '.txt', header=None, skiprows = 0,\n",
    "        names=['tweet'], sep='\\t', quoting=3)\n",
    "    df_labels = pd.read_csv(labelfile + '.txt',header=None, skiprows = 0, names=['label'],\n",
    "        sep='\\t', quoting=3)\n",
    "\n",
    "    index_text = [x for x in range(1, len(df_text.values)+1)]\n",
    "\n",
    "    df_labels.insert(loc=0, column='id', value =index_text)\n",
    "    df_text.insert(loc=0, column='id', value =index_text)\n",
    "    final_df = df_text.merge(df_labels, on='id', how='left')\n",
    "\n",
    "    return final_df\n",
    "\n",
    "def class_distribution(dataframe,title,xlabel_title):\n",
    "    x = dataframe['label'].value_counts()\n",
    "    barplot = seaborn.barplot(x.index, x)\n",
    "    barplot.set_title(title)\n",
    "    barplot.set_xlabel(xlabel_title)\n",
    "    barplot.set_ylabel(\"Count\")\n",
    "    return barplot\n",
    "\n",
    "# WORD-COUNT\n",
    "def word_count(dataframe_col):\n",
    "    dataframe_col['word_count'] = dataframe_col['tweet'].apply(lambda x: len(str(x).split()))\n",
    "    \n",
    "    avg_off_tweets = round(dataframe_col[dataframe_col['label']==1]['word_count'].mean(),3)\n",
    "    avg_non_off_tweets = round(dataframe_col[dataframe_col['label']==0]['word_count'].mean(),3) \n",
    "\n",
    "    return dataframe_col, avg_off_tweets, avg_non_off_tweets\n",
    "\n",
    "# CHARACTER-COUNT\n",
    "def char_count(dataframe_col):\n",
    "    dataframe_col['char_count'] = dataframe_col['tweet'].apply(lambda x: len(str(x)))\n",
    "    \n",
    "    #the average characters in offensive tweets\n",
    "    avg_char_off = round(dataframe_col[dataframe_col['label']==1]['char_count'].mean(),3) \n",
    "\n",
    "    #the average characters in non-offensive tweets\n",
    "    avg_char_non_off = round(dataframe_col[dataframe_col['label']==0]['char_count'].mean(),3)\n",
    "\n",
    "    return dataframe_col, avg_char_off, avg_char_non_off\n",
    "\n",
    "def plot_word_count(dataframe):\n",
    "    # PLOTTING WORD-COUNT\n",
    "    fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,4))\n",
    "    dataframe_words=dataframe[dataframe['label']==1]['word_count']\n",
    "    ax1.hist(dataframe_words,color='red')\n",
    "    ax1.set_title('offensive tweets')\n",
    "    dataframe_words=dataframe[dataframe['label']==0]['word_count']\n",
    "    ax2.hist(dataframe_words,color='green')\n",
    "    ax2.set_title('non-offensive tweets')\n",
    "    fig.suptitle('Words per tweet')\n",
    "    ax2.set_xlabel(\"Length of tweet\")\n",
    "    ax2.set_ylabel(\"occurrences\")\n",
    "    ax1.set_xlabel(\"Length of tweet\")\n",
    "    ax1.set_ylabel(\"occurrences\")\n",
    "    plt.show()\n",
    "\n",
    "def missing_values(dataframe):\n",
    "    res = dataframe.isna().sum()\n",
    "    return res\n",
    "\n",
    "#convert to lowercase, strip and remove punctuations\n",
    "def preprocess(text):\n",
    "    text = text.lower() \n",
    "    text=text.strip()  \n",
    "    text=re.compile('<.*?>').sub('', text) \n",
    "    text = re.compile('[%s]' % re.escape(string.punctuation)).sub(' ', text)  \n",
    "    text = re.sub('\\s+', ' ', text)  \n",
    "    text = re.sub(r'\\[[0-9]*\\]',' ',text) \n",
    "    text=re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n",
    "    text = re.sub(r'\\d',' ',text) \n",
    "    text = re.sub(r'\\s+',' ',text) \n",
    "    return text\n",
    " \n",
    "# STOPWORD REMOVAL\n",
    "def stopword(string):\n",
    "    a= [i for i in string.split() if i not in stopwords.words('english')]\n",
    "    return ' '.join(a)#LEMMATIZATION\n",
    "# Initialize the lemmatizer\n",
    "wl = WordNetLemmatizer()\n",
    " \n",
    "# This is a helper function to map NTLK position tags\n",
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN# Tokenize the sentence\n",
    "def lemmatizer(string):\n",
    "    word_pos_tags = nltk.pos_tag(word_tokenize(string)) # Get position tags\n",
    "    a=[wl.lemmatize(tag[0], get_wordnet_pos(tag[1])) for idx, tag in enumerate(word_pos_tags)] # Map the position tag and lemmatize the word/token\n",
    "    return \" \".join(a)\n",
    "    \n",
    "def finalpreprocess(string):\n",
    "    return lemmatizer(stopword(preprocess(string)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## D) Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SPLITTING THE VALIDATION DATASET INTO TRAIN AND TEST\n",
    "\n",
    "\n",
    "def split_dataset(dataframe):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(dataframe[\"tweet\"],dataframe      \n",
    "    [\"label\"],test_size=0.2,shuffle=True)\n",
    "    return X_train, X_test, y_train, y_test\n",
    "    \n",
    "def tokenize_train_test(X_train, X_test):\n",
    "    #Word2Vec\n",
    "    # Word2Vec runs on tokenized sentences\n",
    "    X_train_tok= [nltk.word_tokenize(i) for i in X_train]  \n",
    "    X_test_tok= [nltk.word_tokenize(i) for i in X_test]\n",
    "\n",
    "    return X_train_tok, X_test_tok\n",
    "\n",
    "\n",
    "#building Word2Vec model\n",
    "class MeanEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        # if a text is empty we should return a vector of zeros\n",
    "        # with the same dimensionality as all the other vectors\n",
    "        self.dim = len(next(iter(word2vec.values())))\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([np.mean([self.word2vec[w] \n",
    "        for w in words if w in self.word2vec] or [np.zeros(self.dim)], axis=0) \n",
    "        for words in X])\n",
    "\n",
    "#building Word2Vec model\n",
    "class MeanEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        # if a text is empty we should return a vector of zeros\n",
    "        # with the same dimensionality as all the other vectors\n",
    "        self.dim = len(next(iter(word2vec.values())))\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([np.mean([self.word2vec[w] \n",
    "        for w in words if w in self.word2vec] or [np.zeros(self.dim)], axis=0) \n",
    "        for words in X])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_prediction_preprocessed_data(dataframe):\n",
    "    #Pre-processing the new dataset\n",
    "    dataframe['clean_text'] = dataframe['tweet'].apply(lambda x: finalpreprocess(x)) \n",
    "    \n",
    "    #preprocess the data\n",
    "    X_test=dataframe['clean_text'] \n",
    "\n",
    "    #converting words to numerical data using tf-idf\n",
    "    X_vector=tfidf_vectorizer.transform(X_test)\n",
    "\n",
    "    #use the best model to predict 'target' value for the new dataset \n",
    "    y_predict = lr_tfidf.predict(X_vector)      \n",
    "    y_prob = lr_tfidf.predict_proba(X_vector)[:,1]\n",
    "    dataframe['predict_prob']= y_prob\n",
    "    dataframe['label']= y_predict\n",
    "    final=dataframe[['clean_text','label']].reset_index(drop=True)\n",
    "    return final \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Non cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_prediction_nonprocessed_data(dataframe):\n",
    "    X_test=dataframe['tweet'] \n",
    "\n",
    "    #converting words to numerical data using tf-idf\n",
    "    X_vector=tfidf_vectorizer.transform(X_test)\n",
    "\n",
    "    #use the best model to predict 'target' value for the new dataset \n",
    "    y_predict = lr_tfidf.predict(X_vector)      \n",
    "    y_prob = lr_tfidf.predict_proba(X_vector)[:,1]\n",
    "    dataframe['predict_prob']= y_prob\n",
    "    dataframe['label']= y_predict\n",
    "    final=dataframe[['tweet','label']].reset_index(drop=True)\n",
    "    return final \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASK 1: PRE-PROCESSING AND TOKENIZATION\n",
    "_ splitting text files into words _\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1) Text file: Emoji\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in txt files: offensive/emoji.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path_val = pathlib.Path(r'datasets/emoji/val_text.txt')\n",
    "\n",
    "with open(file_path_val, 'r',encoding=\"utf8\") as f:\n",
    "    emoji_text_val = f.read()       \n",
    "    f.close()\n",
    "    \n",
    "emoji_words_val_txt = emoji_text_val[1:].split()\n",
    "\n",
    "print(emoji_words_val_txt[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "file_path_train = pathlib.Path(r'datasets/emoji/train_text.txt')\n",
    "\n",
    "\n",
    "with open(file_path_train, 'r',encoding=\"utf8\") as f:\n",
    "    emoji_text_train = f.read()       \n",
    "    f.close()\n",
    "    \n",
    "emoji_words_train_txt = emoji_text_train[1:].split()\n",
    "print(emoji_words_train_txt[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path_test = pathlib.Path(r'datasets/emoji/test_text.txt')\n",
    "\n",
    "\n",
    "with open(file_path_test, 'r',encoding=\"utf8\") as f:\n",
    "    emoji_text_test = f.read()       \n",
    "    f.close()\n",
    "    \n",
    "emoji_words_test_txt = emoji_text_test[1:].split()\n",
    "print(emoji_words_test_txt[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove punctuations from txt files\n",
    "_ meaning signs, spacing and other disturbing features. Alle words are then turned into lower cases_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "emoji_cleaned_val_words = del_punctuations(emoji_words_val_txt)\n",
    "emoji_cleaned_train_words = del_punctuations(emoji_words_train_txt)\n",
    "emoji_cleaned_test_words = del_punctuations(emoji_words_test_txt)\n",
    "\n",
    "print('val_text.txt:\\n',emoji_cleaned_val_words[:100],'\\n')\n",
    "\n",
    "print('\\nval_train.txt:\\n',emoji_cleaned_train_words[:100], '\\n')\n",
    "\n",
    "print('\\nval_test.txt:\\n',emoji_cleaned_test_words[:100],'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "#nltk.download('punkt')\n",
    "\n",
    "emoji_sentences_val_txt = token_sentences(emoji_text_val)   \n",
    "emoji_sentences_train_txt = token_sentences(emoji_text_train)\n",
    "emoji_sentences_test_txt = token_sentences(emoji_text_test)   \n",
    "\n",
    "print(emoji_sentences_val_txt[:5])\n",
    "print(emoji_sentences_train_txt[:5])\n",
    "print(emoji_sentences_test_txt[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of words in each tokenization variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "emoji_token_val = tokenization(emoji_cleaned_val_words, emoji_text_val)\n",
    "emoji_token_train = tokenization(emoji_cleaned_train_words, emoji_text_train)\n",
    "emoji_token_test = tokenization(emoji_cleaned_test_words, emoji_text_test)\n",
    "\n",
    "print(f'Number of words in tokenization for val_text: {len(emoji_token_val)}')\n",
    "print(f'Number of words in tokenization for val_train: {len(emoji_token_train)}')\n",
    "print(f'Number of words in tokenization for val_test: {len(emoji_token_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing stopwords like 'and, or, of, is, had.... etc' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_cleaned_val_words = stop_words(emoji_token_val)\n",
    "emoji_cleaned_train_words = stop_words(emoji_token_train)\n",
    "emoji_cleaned_test_words = stop_words(emoji_token_test)\n",
    "\n",
    "print(f'Number of words after removing Stop Words: {len(emoji_cleaned_val_words)}')\n",
    "print(f'Number of words after removing Stop Words: {len(emoji_cleaned_train_words)}')\n",
    "print(f'Number of words after removing Stop Words: {len(emoji_cleaned_test_words)}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frequency of each word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "#returns frequency of each word\n",
    "emoji_freq_words_val = word_frequency(emoji_cleaned_val_words)\n",
    "emoji_freq_words_train = word_frequency(emoji_cleaned_train_words)\n",
    "emoji_freq_words_test = word_frequency(emoji_cleaned_test_words)\n",
    "\n",
    "\n",
    "print('_'*112,'\\n')\n",
    "print('Frequency of words in val_text:\\n',list(emoji_freq_words_val.items())[-40:-1], '\\n')\n",
    "print('_'*112,'\\n')\n",
    "print('Frequency of words in train_text:\\n',list(emoji_freq_words_train.items())[-40:-1], '\\n')\n",
    "print('_'*112,'\\n')\n",
    "print('Frequency of words in test_text:\\n',list(emoji_freq_words_test.items())[-40:-1], '\\n')\n",
    "print('_'*112,'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vocabulary list for offensive text files\n",
    "_Looking through vocabulary lists can help you find problems\n",
    "(especially tokens that only occur once or twice)._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "emoji_corpus_val, emoji_vocab_val = words_list(emoji_text_val)\n",
    "emoji_corpus_train, emoji_vocab_train = words_list(emoji_text_train)\n",
    "emoji_corpus_test, emoji_vocab_test = words_list(emoji_text_test)\n",
    "\n",
    "print('Vocabulary for text_val.txt:\\n', emoji_vocab_val[:20],'\\n')\n",
    "print('Vocabulary for text_train.txt:\\n', emoji_vocab_train[:20],'\\n')\n",
    "print('Vocabulary for text_test.txt:\\n', emoji_vocab_test[:20],'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in txt files: offensive/val_text.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#offensive_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path_val = pathlib.Path(r'datasets/offensive/val_text.txt')\n",
    "\n",
    "with open(file_path_val, 'r',encoding=\"utf8\") as f:\n",
    "    offensive_text_val = f.read()       \n",
    "    f.close()\n",
    "    \n",
    "offensive_words_val_txt = offensive_text_val[1:].split()\n",
    "\n",
    "print(offensive_words_val_txt[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path_train = pathlib.Path(r'datasets/offensive/train_text.txt')\n",
    "\n",
    "\n",
    "with open(file_path_train, 'r',encoding=\"utf8\") as f:\n",
    "    offensive_text_train = f.read()       \n",
    "    f.close()\n",
    "    \n",
    "offensive_words_train_txt = offensive_text_train[1:].split()\n",
    "print(offensive_words_train_txt[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path_test = pathlib.Path(r'datasets/offensive/test_text.txt')\n",
    "\n",
    "\n",
    "with open(file_path_test, 'r',encoding=\"utf8\") as f:\n",
    "    offensive_text_test = f.read()       \n",
    "    f.close()\n",
    "    \n",
    "offensive_words_test_txt = offensive_text_test[1:].split()\n",
    "print(offensive_words_test_txt[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove punctuations from txt files\n",
    "_ meaning signs, spacing and other disturbing features. Alle words are then turned into lower cases_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "offensive_cleaned_val_words = del_punctuations(offensive_words_val_txt)\n",
    "offensive_cleaned_train_words = del_punctuations(offensive_words_train_txt)\n",
    "offensive_cleaned_test_words = del_punctuations(offensive_words_test_txt)\n",
    "\n",
    "print('val_text.txt:\\n',offensive_cleaned_val_words[:100],'\\n')\n",
    "\n",
    "print('\\nval_train.txt:\\n',offensive_cleaned_train_words[:100], '\\n')\n",
    "\n",
    "print('\\nval_test.txt:\\n',offensive_cleaned_test_words[:100],'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of words in each tokenization variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "\n",
    "offensive_token_val = tokenization(offensive_cleaned_val_words, offensive_text_val)\n",
    "offensive_token_train = tokenization(offensive_cleaned_train_words, offensive_text_train)\n",
    "offensive_token_test = tokenization(offensive_cleaned_test_words, offensive_text_test)\n",
    "\n",
    "print(f'Number of words in tokenization for val_text: {len(offensive_token_val)}')\n",
    "print(f'Number of words in tokenization for val_train: {len(offensive_token_train)}')\n",
    "print(f'Number of words in tokenization for val_test: {len(offensive_token_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Removing stopwords like 'and, or, of, is, had.... etc' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n",
    "\n",
    "offensive_cleaned_val_words = stop_words(offensive_token_val)\n",
    "offensive_cleaned_train_words = stop_words(offensive_token_train)\n",
    "offensive_cleaned_test_words = stop_words(offensive_token_test)\n",
    "\n",
    "print(f'Number of words after removing Stop Words: {len(offensive_cleaned_val_words)}')\n",
    "print(f'Number of words after removing Stop Words: {len(offensive_cleaned_train_words)}')\n",
    "print(f'Number of words after removing Stop Words: {len(offensive_cleaned_test_words)}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frequency of each word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "#returns frequency of each word\n",
    "offensive_freq_words_val = word_frequency(offensive_cleaned_val_words)\n",
    "offensive_freq_words_train = word_frequency(offensive_cleaned_train_words)\n",
    "offensive_freq_words_test = word_frequency(offensive_cleaned_test_words)\n",
    "\n",
    "\n",
    "print('_'*112,'\\n')\n",
    "print('Frequency of words in val_text:\\n',list(offensive_freq_words_val.items())[-40:-1], '\\n')\n",
    "print('_'*112,'\\n')\n",
    "print('Frequency of words in train_text:\\n',list(offensive_freq_words_train.items())[-40:-1], '\\n')\n",
    "print('_'*112,'\\n')\n",
    "print('Frequency of words in test_text:\\n',list(offensive_freq_words_test.items())[-40:-1], '\\n')\n",
    "print('_'*112,'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# My approach to task 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#This is the ideal tokenization from the library we were supposed to compare it with\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "tknzr = TweetTokenizer()\n",
    "\n",
    "sentence = 'Heroin is my passion.'\n",
    "\n",
    "tknzr.tokenize(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "#Recreate\n",
    "import re\n",
    "import pathlib\n",
    "\n",
    "file_path_train = pathlib.Path(r'datasets/offensive/train_text.txt')\n",
    "\n",
    "token_pattern = re.compile(r'\\w+')\n",
    "\n",
    "with open(file_path_train, 'r',encoding=\"utf8\") as f:\n",
    "    line = f.readline()\n",
    "    tokens = []\n",
    "    while line:\n",
    "        line = f.readline()\n",
    "        print(line)\n",
    "        print(\"OUR beta tokenizer\",re.findall(token_pattern,line))\n",
    "        print(\"DESIRED  tokenizer\",tknzr.tokenize(line))\n",
    "        print('\\n')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 02 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Emoji dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Size of our emoji vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Number of total words in the vocabulary \" + str(len(emoji_freq_words_train)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top 20 most common tokens "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "emoji_top_20_val = top_20_most_common_words(emoji_freq_words_val)\n",
    "emoji_top_20_train = top_20_most_common_words(emoji_freq_words_train)\n",
    "emoji_top_20_test = top_20_most_common_words(emoji_freq_words_test)\n",
    "\n",
    "print('Top 20 in emoji_freq_words_val \\n',emoji_top_20_val, '\\n')\n",
    "print('Top 20 in emoji_freq_words_train \\n',emoji_top_20_train,'\\n')\n",
    "print('Top 20 in emoji_freq_words_test \\n',emoji_top_20_test,'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top 20 least common tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "emoji_least_words_val = least_common_words(emoji_freq_words_val)\n",
    "emoji_least_words_train = least_common_words(emoji_freq_words_train)\n",
    "emoji_least_words_test = least_common_words(emoji_freq_words_test)\n",
    "print('Top 20 least common words in emoji_freq_words_val \\n',emoji_least_words_val[0:20], '\\n')\n",
    "print('Top 20 least common words in emoji_freq_words_train \\n',emoji_least_words_train[0:20], '\\n')\n",
    "print('Top 20 least common words in emoji_freq_words_test \\n',emoji_least_words_test[0:20], '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zipf's Law"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In short, Zipf's Law is a distribution of data, where the 2nd highest ranking has half the number of occurrences as the highest ranking, the 3rd having 1/3 number of occurrences and so on.\n",
    "Another way of writting Zipf's law is as following:\n",
    "rank x frequency $\\approx$ const\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A quick way of determining weather something follows the law is to use the formula rank x frequency $\\approx$ const and plot the results as a histogram. If all the bars in the histogram has rougly the same height the given data follows Zipf's law\n",
    "Another way is to use a loglog plot. If the line follows a diagonal line, then there is evidence that the data follows Zipf's Law"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Here we simply plot the distribution\n",
    "fig = plt.figure(figsize = (5,4))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(emoji_top_20_val.keys(),emoji_top_20_val.values())\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we multiply the frequency with the rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "li =  []\n",
    "counter = 1\n",
    "for elm in emoji_top_20_val.values():\n",
    "    li.append(elm * counter)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (5,4))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(emoji_top_20_val.keys(),li)\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Raw dataset\n",
    "emoji_raw_val = word_frequency(emoji_words_val_txt)\n",
    "emoji_raw_train = word_frequency(emoji_words_train_txt)\n",
    "emoji_raw_test = word_frequency(emoji_words_test_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thing = most_common_words(emoji_raw_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "li =  []\n",
    "counter = 1\n",
    "for elm in thing.values():\n",
    "    li.append(elm * counter)\n",
    "    counter += 1\n",
    "    if counter == 51:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (12,5))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(list(thing.keys())[:20],li[:20])\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences raw dataset')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (12,5))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(list(thing.keys())[:50],li)\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences raw dataset')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cleaned dataset:\n",
    "llplot([emoji_freq_words_val, emoji_freq_words_train, emoji_freq_words_test], labels=[\"val\", \"train\", \"test\"], title=\"Cleaned Emoji dataset, loglog plot\")\n",
    "\n",
    "\n",
    "\n",
    "llplot([emoji_raw_val, emoji_raw_train, emoji_raw_test], labels=[\"val\", \"train\", \"test\"], title=\"Raw Emoji dataset, loglog plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Type/token ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "_, tokens, _, _,unique_tokens, _ ,_ = basic_statistics(emoji_text_train,printer = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Offensive dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Size of our offensive vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Number of total words in the vocabulary \" + str(len(offensive_freq_words_train)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top 20 most common tokens "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "offensive_top_20_val = top_20_most_common_words(offensive_freq_words_val)\n",
    "offensive_top_20_train = top_20_most_common_words(offensive_freq_words_train)\n",
    "offensive_top_20_test = top_20_most_common_words(offensive_freq_words_test)\n",
    "\n",
    "print('Top 20 in val_text.txt:\\n',offensive_top_20_val, '\\n')\n",
    "print('Top 20 in val_train.txt:\\n',offensive_top_20_train,'\\n')\n",
    "print('Top 20 in val_test.txt:\\n',offensive_top_20_test,'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top 20 least common tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "offensive_least_words_val = least_common_words(offensive_freq_words_val)\n",
    "offensive_least_words_train = least_common_words(offensive_freq_words_train)\n",
    "offensive_least_words_test = least_common_words(offensive_freq_words_test)\n",
    "print('Top 20 least common words in offensive_freq_words_val \\n',offensive_least_words_val[0:20], '\\n')\n",
    "print('Top 20 least common words in offensive_freq_words_train \\n',offensive_least_words_train[0:20], '\\n')\n",
    "print('Top 20 least common words in offensive_freq_words_test \\n',offensive_least_words_test[0:20], '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zipf's Law"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (5,4))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(offensive_top_20_val.keys(),offensive_top_20_val.values())\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Offensive histogram of accidents based on words and occurrences')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "li =  []\n",
    "counter = 1\n",
    "for elm in offensive_top_20_val.values():\n",
    "    li.append(elm * counter)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (5,4))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(offensive_top_20_val.keys(),li)\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Raw dataset\n",
    "offensive_raw_val = word_frequency(offensive_words_val_txt)\n",
    "offensive_raw_train = word_frequency(offensive_words_train_txt)\n",
    "offensive_raw_test = word_frequency(offensive_words_test_txt)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thing = most_common_words(offensive_raw_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "li =  []\n",
    "counter = 1\n",
    "for elm in thing.values():\n",
    "    li.append(elm * counter)\n",
    "    counter += 1\n",
    "    if counter == 51:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (12,5))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(list(thing.keys())[:20],li[:20])\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences raw dataset')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize = (12,5))\n",
    "axes = fig.add_axes([0,0,1,1])\n",
    "axes.bar(list(thing.keys())[:50],li[:50])\n",
    "fig.autofmt_xdate(rotation=45)\n",
    "axes.set_title('Emoji histogram of accidents based on words and occurrences raw dataset')\n",
    "axes.set_ylabel('Count')\n",
    "axes.set_xlabel('Words');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cleaned dataset:\n",
    "llplot([offensive_freq_words_val, offensive_freq_words_train, offensive_freq_words_test], labels=[\"val\", \"train\", \"test\"], title=\"Cleaned Offensive dataset, loglog plot\")\n",
    "\n",
    "llplot([offensive_raw_val, offensive_raw_train, offensive_raw_test], labels=[\"val\", \"train\", \"test\"], title=\"Raw Offensive dataset, loglog plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is an overall trend that the follows the law, however, this is has not been proven mathematicly, and our plots doesn't follow the excact distributions as described."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Type/token ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, tokens, _, _,unique_tokens, _ ,_ = basic_statistics(offensive_text_train,printer = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data frames for offensive train text and offensive test text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "train = pd.read_csv('datasets/offensive/train_text.txt', header=None, skiprows = 0, names=['tweet'], sep='\\t', quoting=3)\n",
    "print(\"Training Set:\"% train.columns, train.shape, len(train))\n",
    "\n",
    "\n",
    "test = pd.read_csv('datasets/offensive/test_text.txt',header=None, skiprows = 0, names=['tweet'], sep='\\t', quoting=3)\n",
    "print(\"Test Set:\"% test.columns, test.shape, len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "index_train = [x for x in range(1, len(train.values)+1)]\n",
    "index_test = [x for x in range(1, len(test.values)+1)]\n",
    "\n",
    "train.insert(loc=0, column='id', value =index_train )\n",
    "test.insert(loc=0, column='id', value =index_test )\n",
    "\n",
    "train_labels = pd.read_csv('datasets/offensive/train_labels.txt',header=None, skiprows = 0, names=['label'], sep='\\t', quoting=3)\n",
    "train_labels.insert(loc=0, column='id', value=index_train)\n",
    "\n",
    "test_labels = pd.read_csv('datasets/offensive/test_labels.txt',header=None, skiprows = 0, names=['label'], sep='\\t', quoting=3)\n",
    "test_labels.insert(loc=0, column='id', value =index_test )\n",
    "\n",
    "\n",
    "test_df = test.merge(test_labels, on='id', how='left')\n",
    "train_df = train.merge(train_labels, on='id', how='left')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_clean = clean_text(test_df, 'tweet')\n",
    "train_clean = clean_text(train_df, 'tweet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop = stopwords.words('english')\n",
    "\n",
    "train_clean['tweet'] = train_clean['tweet'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n",
    "train_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop = stopwords.words('english')\n",
    "test_clean['tweet'] = test_clean['tweet'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n",
    "\n",
    "test_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_clean['tweet'] = test_clean['tweet'].apply(lambda x: word_tokenize(x))\n",
    "test_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "test_clean_offensive = test_clean.loc[test_clean['label'] == 1]\n",
    "test_clean_offensive.head()\n",
    "#test_clean_offensive['tweet'].to_csv('test_cleaned_off_lang_df.csv',  quoting=csv.QUOTE_NONE, escapechar=' ') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_clean_nonoffensive = test_clean.loc[test_clean['label'] == 0]\n",
    "\n",
    "#test_clean_nonoffensive['tweet'].to_csv('test_cleaned_non_off_lang_df.csv',  quoting=csv.QUOTE_NONE, escapechar=' ') \n",
    "\n",
    "test_clean_nonoffensive.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_off_lists = test_clean_offensive['tweet'].values.tolist()\n",
    "test_off_lists[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_non_off_lists = test_clean_nonoffensive['tweet'].values.tolist()\n",
    "test_non_off_lists[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task 04 - Automatic Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merging validation text and labels\n",
    "val_df = merging_labels_and_sentences('datasets/offensive/val_text', 'datasets/offensive/val_labels')\n",
    "\n",
    "# Checking for missing values\n",
    "missing_val = missing_values(val_df)\n",
    "# Average words devided into labels\n",
    "avg_labels_word = word_count(val_df)\n",
    "avg_labels_char = char_count(val_df)\n",
    "\n",
    "print(' Average Number of Words - Offensive Tweets: ', avg_labels_word[1],'\\n','Average Number of Words - Non-offensive Tweets: ', avg_labels_word[2])\n",
    "print(' Average Characters in Offensive Tweets: ', avg_labels_char[1],'\\n','Average Characters in Non-offensive Tweets: ', avg_labels_char[2])\n",
    "\n",
    "print('\\nNumber of missing values for each column\\n',missing_val)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Number of total labels for each class: 0 = non offensive, 1 = offensive\n",
    "barplot = class_distribution(val_df,\"Count of each label\",\"0: Non offensive   1: Offensive\")\n",
    "# Histogram of Word count pr tweet\n",
    "plot_word_count = plot_word_count(val_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Term Frequency-Inverse Document Frequencies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Term Frequency-Inverse Document Frequencies (tf-Idf): Count vectors might not be the best representation for converting text data to numerical data. So, instead of simple counting, we can also use an advanced variant of the Bag-of-Words that uses the term frequency–inverse document frequency (or Tf-Idf). Basically, the value of a word increases proportionally to count in the document, but it is inversely proportional to the frequency of the word in the corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We are splitting the data into training and test\n",
    "X_train, X_test, y_train, y_test = split_dataset_2(val_df,labels)\n",
    "\n",
    "# Here we tokenize the data\n",
    "X_train_tok, X_test_tok = tokenize_train_test_2(X_train[:,0].tolist(), X_test[:,0].tolist())\n",
    "\n",
    "# (tf-Idf)\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train.tolist()) \n",
    "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test.tolist())\n",
    "\n",
    "\n",
    "val_df['clean_text_tok']=[nltk.word_tokenize(i) for i in val_df['tweet']]\n",
    "model = gensim.models.Word2Vec(val_df['clean_text_tok'],min_count=1)\n",
    "w2v = dict(zip(model.wv.index_to_key, model.wv))      \n",
    "modelw = MeanEmbeddingVectorizer(w2v)\n",
    "\n",
    "# converting text to numerical data using Word2Vec\n",
    "X_train_vectors_w2v = modelw.transform(X_train_tok)\n",
    "X_test_vectors_w2v = modelw.transform(X_test_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We are splitting the data into training and test\n",
    "X_train, X_test, y_train, y_test = split_dataset(val_df)\n",
    "\n",
    "# Here we tokenize the data\n",
    "X_train_tok, X_test_tok = tokenize_train_test(X_train, X_test)\n",
    "\n",
    "# (tf-Idf)\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) \n",
    "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test)\n",
    "\n",
    "\n",
    "val_df['clean_text_tok']=[nltk.word_tokenize(i) for i in val_df['tweet']]\n",
    "model = gensim.models.Word2Vec(val_df['clean_text_tok'],min_count=1)\n",
    "w2v = dict(zip(model.wv.index_to_key, model.wv))      \n",
    "modelw = MeanEmbeddingVectorizer(w2v)\n",
    "\n",
    "# converting text to numerical data using Word2Vec\n",
    "X_train_vectors_w2v = modelw.transform(X_train_tok)\n",
    "X_test_vectors_w2v = modelw.transform(X_test_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = X_train[0:4]\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "X.shape # This is baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = [x for x in range(20)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "new = np.append(X,l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.array(new))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_vectors_w2v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression (tf-idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#FITTING THE CLASSIFICATION MODEL using Logistic Regression(tf-idf)\n",
    "\n",
    "lr_tfidf=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\n",
    "\n",
    "lr_tfidf.fit(X_train_vectors_tfidf, y_train)  \n",
    "\n",
    "#Predict y value for test dataset\n",
    "y_predict = lr_tfidf.predict(X_test_vectors_tfidf)\n",
    "y_prob = lr_tfidf.predict_proba(X_test_vectors_tfidf)[:,1]\n",
    "print(classification_report(y_test,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_test, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve')\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic example (ROC)')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#FITTING THE CLASSIFICATION MODEL using Logistic Regression (W2v)\n",
    "\n",
    "lr_w2v=LogisticRegression(solver = 'liblinear', C=10, penalty = 'l2')\n",
    "lr_w2v.fit(X_train_vectors_w2v, y_train)  #model\n",
    "\n",
    "#Predict y value for test dataset\n",
    "y_predict = lr_w2v.predict(X_test_vectors_w2v)\n",
    "y_prob = lr_w2v.predict_proba(X_test_vectors_w2v)[:,1]\n",
    "\n",
    "print(classification_report(y_test,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_test, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve')\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic example (ROC)')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def split_dataset(dataframe):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(dataframe[\"tweet\"],dataframe      \n",
    "    [\"label\"],test_size=0.2,shuffle=True)\n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "def split_dataset_2(dataframe,labels):\n",
    "    counter = 0\n",
    "    li = []\n",
    "    for rows in dataframe.iterrows():\n",
    "        row = []\n",
    "        for elm in labels:\n",
    "            row.append(rows[1][elm])\n",
    "        li.append(row)\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = train_test_split(li,dataframe[\"label\"],test_size=0.2,shuffle=True)\n",
    "    return X_train,X_test, y_train, y_test\n",
    "labels = [\"tweet\",\"word_count\",\"char_count\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = val_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test.drop([\"id\"],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "li = []\n",
    "counter = 0\n",
    "for elm in test[\"tweet\"]:\n",
    "    li.append(len(re.findall(\"@user\", elm)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[\"Tags\"] = li"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_off = test.where(test[\"label\"] == 1).dropna()\n",
    "df_not_off = test.where(test[\"label\"] == 0).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_not_off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Counter(df_off[\"Tags\"])\n",
    "Counter(df_not_off[\"Tags\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig,(ax1,ax2)=plt.subplots(1,2,figsize=(15,5))\n",
    "value = Counter(df_off[\"Tags\"]).values()\n",
    "keys = Counter(df_off[\"Tags\"]).keys()\n",
    "ax1.bar(keys,value)\n",
    "ax1.set_ylabel(\"Frequencies\")\n",
    "ax1.set_xlabel(\"Number of tags\")\n",
    "ax1.set_title(\"Offensive\")\n",
    "ax1.set_xlim(-2,15)\n",
    "value_2 = Counter(df_not_off[\"Tags\"]).values()\n",
    "keys_2 = Counter(df_not_off[\"Tags\"]).keys()\n",
    "ax2.bar(keys_2,value_2)\n",
    "ax2.set_ylabel(\"Frequencies\")\n",
    "ax2.set_xlabel(\"Number of tags\")\n",
    "ax2.set_title(\"Non offensive\")\n",
    "ax2.set_xlim(-2,15)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "li = []\n",
    "for elm in test[\"tweet\"]:\n",
    "    #li.append(re.sub('\\W+',' ', elm ).strip()) # Removes all special characters\n",
    "    clean = re.sub('\\W+',' ', elm ).strip()\n",
    "    clean = clean.lower()\n",
    "    temp = []\n",
    "    for i in clean:\n",
    "        if i == \" \":\n",
    "            temp.append(0)\n",
    "            #temp.append(\" \")\n",
    "        else:\n",
    "            #k = str(i)\n",
    "            temp.append((ord(i) - 97))\n",
    "    li.append(temp)\n",
    "test[\"Numeric\"] = li"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "# calculate the correlations\n",
    "correlations = test.corr()\n",
    "\n",
    "# plot the heatmap bb\n",
    "#sns.heatmap(correlations, xticklabels=correlations.columns, yticklabels=correlations.columns, annot=True)\n",
    "\n",
    "# plot the clustermap \n",
    "sns.clustermap(correlations, xticklabels=correlations.columns, yticklabels=correlations.columns, annot=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test.drop([\"tweet\",\"clean_text_tok\",\"char_count\",\"Tags\"],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input = [[9,[20,18]],[6,[9,3,4]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(list(zip(test[\"word_count\"],test[\"Numeric\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "li = []\n",
    "for elm in zip(test[\"word_count\"],test[\"Numeric\"]):\n",
    "    li.append([np.array(elm[0]),np.array(elm[1])])\n",
    "    break\n",
    "li[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "li = np.array(li[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "li.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y = np.array(test[\"label\"])\n",
    "#from keras.utils import to_categorical\n",
    "train_labels = to_categorical(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "#download mnist data and split into train and test sets\n",
    "#(X_train_mnist, y_train_mnist), (X_test_mnist, y_test_mnist) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reshape data to fit model\n",
    "X_train = X_train.reshape(60000,28,28,1)\n",
    "X_test = X_test.reshape(10000,28,28,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "#one-hot encode target column\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, Flatten\n",
    "#create model\n",
    "model = Sequential()\n",
    "#add model layers\n",
    "model.add(Conv2D(64, kernel_size=3, activation=’relu’, input_shape=(28,28,1)))\n",
    "model.add(Conv2D(32, kernel_size=3, activation=’relu’))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation=’softmax’))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#compile model using accuracy to measure model performance\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train the model\n",
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train\n",
    "li = []\n",
    "for elm in  X_train_vectors_w2v:\n",
    "    li.append(elm.reshape(10,10,1))\n",
    "X_train_vectors_w2v = np.array(li)\n",
    "something = abs(X_train_vectors_w2v)\n",
    "maximum = np.amax(something)\n",
    "X_train_vectors_w2v = np.divide(something,maximum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test\n",
    "li = []\n",
    "for elm in  X_test_vectors_w2v:\n",
    "    li.append(elm.reshape(10,10,1))\n",
    "X_test_vectors_w2v = np.array(li)\n",
    "something = abs(X_test_vectors_w2v)\n",
    "maximum = np.amax(something)\n",
    "X_test_vectors_w2v = np.divide(something,maximum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_vectors_w2v[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_vectors_w2v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "#one-hot encode target column\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)\n",
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, Flatten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#create model\n",
    "model = Sequential()\n",
    "#add model layers\n",
    "model.add(Conv2D(10, kernel_size=3, activation=\"relu\", input_shape=(10,10,1)))\n",
    "model.add(Conv2D(5, kernel_size=3, activation=\"relu\"))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(2, activation=\"sigmoid\"))\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train the model\n",
    "model.fit(X_train_vectors_w2v, y_train, validation_data=(X_test_vectors_w2v, y_test), epochs=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = test.drop([\"tweet\",\"char_count\",\"clean_text_tok\"],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test['clean_text_tok']=[nltk.word_tokenize(i) for i in test['tweet']]\n",
    "model = gensim.models.Word2Vec(test['clean_text_tok'],min_count=1)\n",
    "w2v = dict(zip(model.wv.index_to_key, model.wv))      \n",
    "modelw = MeanEmbeddingVectorizer(w2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_vectors_w2v = modelw.transform(X_train_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(X_train_vectors_w2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[\"w2v\"] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We are splitting the data into training and test\n",
    "X_train, X_test, y_train, y_test = split_dataset(val_df)\n",
    "\n",
    "# Here we tokenize the data\n",
    "X_train_tok, X_test_tok = tokenize_train_test(X_train, X_test)\n",
    "\n",
    "# (tf-Idf)\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) \n",
    "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test)\n",
    "\n",
    "\n",
    "val_df['clean_text_tok']=[nltk.word_tokenize(i) for i in val_df['tweet']]\n",
    "model = gensim.models.Word2Vec(val_df['clean_text_tok'],min_count=1)\n",
    "w2v = dict(zip(model.wv.index_to_key, model.wv))      \n",
    "modelw = MeanEmbeddingVectorizer(w2v)\n",
    "\n",
    "# converting text to numerical data using Word2Vec\n",
    "X_train_vectors_w2v = modelw.transform(X_train_tok)\n",
    "X_test_vectors_w2v = modelw.transform(X_test_tok)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test = test.drop([\"label\"],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#featurespace**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#evt antallet af users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "# calculate the correlations\n",
    "correlations = test.corr()\n",
    "\n",
    "# plot the heatmap \n",
    "#sns.heatmap(correlations, xticklabels=correlations.columns, yticklabels=correlations.columns, annot=True)\n",
    "\n",
    "# plot the clustermap \n",
    "sns.clustermap(correlations, xticklabels=correlations.columns, yticklabels=correlations.columns, annot=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#FITTING THE CLASSIFICATION MODEL using Naive Bayes(tf-idf)\n",
    "nb_tfidf = MultinomialNB()\n",
    "nb_tfidf.fit(X_train_vectors_tfidf, y_train) \n",
    "\n",
    " #Predict y value for test dataset\n",
    "y_predict = nb_tfidf.predict(X_test_vectors_tfidf)\n",
    "y_prob = nb_tfidf.predict_proba(X_test_vectors_tfidf)[:,1]\n",
    "\n",
    "print(classification_report(y_test,y_predict))\n",
    "print('Confusion Matrix:',confusion_matrix(y_test, y_predict))\n",
    " \n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_prob)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "print('AUC:', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve')\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic example (ROC)')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Sa\n",
    "#model_prediction_preprocessed_data(val_df).to_csv('model_predict_proc_validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_prediction_nonprocessed_data(val_df).to_csv('model_predict_nonproc_validation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merging validation text and labels\n",
    "val_df_emoji = merging_labels_and_sentences('datasets/emoji/val_text', 'datasets/emoji/val_labels')\n",
    "\n",
    "# Checking for missing values\n",
    "missing_val_emoji = missing_values(val_df_emoji)\n",
    "# Average words devided into labels\n",
    "avg_labels_word_emoji = word_count(val_df_emoji)\n",
    "avg_labels_char_emoji = char_count(val_df_emoji)\n",
    "\n",
    "print(' Average Number of Words - Offensive Tweets: ', avg_labels_word_emoji[1],'\\n','Average Number of Words - Non-offensive Tweets: ', avg_labels_word_emoji[2])\n",
    "print(' Average Characters in Offensive Tweets: ', avg_labels_char_emoji[1],'\\n','Average Characters in Non-offensive Tweets: ', avg_labels_char_emoji[2])\n",
    "\n",
    "print('\\nNumber of missing values for each column\\n',missing_val_emoji)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "df = pd.read_csv('data/wine_data.csv')\n",
    "\n",
    "counter = Counter(df['variety'].tolist())\n",
    "top_10_varieties = {i[0]: idx for idx, i in enumerate(counter.most_common(10))}\n",
    "df = df[df['variety'].map(lambda x: x in top_10_varieties)]\n",
    "\n",
    "description_list = df['description'].tolist()\n",
    "varietal_list = [top_10_varieties[i] for i in df['variety'].tolist()]\n",
    "varietal_list = np.array(varietal_list)\n",
    "\n",
    "count_vect = CountVectorizer()\n",
    "x_train_counts = count_vect.fit_transform(description_list)\n",
    "\n",
    "\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "x_train_tfidf = tfidf_transformer.fit_transform(x_train_counts)\n",
    "\n",
    "train_x, test_x, train_y, test_y = train_test_split(x_train_tfidf, varietal_list, test_size=0.3)\n",
    "\n",
    "clf = MultinomialNB().fit(train_x, train_y)\n",
    "y_score = clf.predict(test_x)\n",
    "\n",
    "n_right = 0\n",
    "for i in range(len(y_score)):\n",
    "    if y_score[i] == test_y[i]:\n",
    "        n_right += 1\n",
    "\n",
    "print(\"Accuracy: %.2f%%\" % ((n_right/float(len(test_y)) * 100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "TweetEval Tutorial",
   "provenance": []
  },
  "environment": {
   "name": "tf2-gpu.2-1.m46",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-1:m46"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "metadata": {
   "interpreter": {
    "hash": "7ca953050fbd6db0e15562356b1a786d9418e582a94adc67b248dc4dbecd989f"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "02f91bc3545b48808e4812d13f480875": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_25a58cab43ef453d8b9de797925f32fa",
      "placeholder": "​",
      "style": "IPY_MODEL_db546face47140a59dcfc21539492c51",
      "value": " 150/150 [00:00&lt;00:00, 233B/s]"
     }
    },
    "0a1beafebb954a498d7a1cccc7cee445": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1e5e9065e1ed4adbb1b451672ca4e793": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2151898d3e1c47de9e681c8b7b8779e4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_627335e621884fb6957b7b6731339148",
       "IPY_MODEL_02f91bc3545b48808e4812d13f480875"
      ],
      "layout": "IPY_MODEL_6e236c3f5e084a108811f1e6a32cb990"
     }
    },
    "25a58cab43ef453d8b9de797925f32fa": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2adcfba8e99b4ceeae8cea6c4060c58d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2b6e1410b17543afb5dbf619666e83bd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "3b1773e755ca402f98b5033ed4af1b7a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "41b12ad69a684cb597594e60355253ae": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "498473ac68aa48c1ba4d5578543a8e2b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "627335e621884fb6957b7b6731339148": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0a1beafebb954a498d7a1cccc7cee445",
      "max": 150,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2b6e1410b17543afb5dbf619666e83bd",
      "value": 150
     }
    },
    "683e01b6cd98402783fe062f75177b68": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6d41343536e54dd3804ace50e15b7953": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "6e236c3f5e084a108811f1e6a32cb990": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7045c78b1b164511b62e20bec70f1639": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "75528dba634e465992e5b52107f19032": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3b1773e755ca402f98b5033ed4af1b7a",
      "placeholder": "​",
      "style": "IPY_MODEL_eab919bdf6cb499b90a14c297bbc94dc",
      "value": " 456k/456k [00:01&lt;00:00, 251kB/s]"
     }
    },
    "77dad5843dbc43eab577b4cef273228d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ca9214bf42bf49d99bcec60cc60c6e9c",
      "max": 498682569,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7045c78b1b164511b62e20bec70f1639",
      "value": 498682569
     }
    },
    "7a80cfb9bfda407c9ee1b9bd8b11a2f5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7de2e7e4e8f44dccabc908334f7d1e21": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "7ede88ead0d449a18ca343d47b723e87": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88a7b0479e8c40b6aea4084f8039d4f6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "8c7d3e2bd2714e929d6859e15e83003f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "91abbefb5e734d47a01158aebf46d7ce": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "9655bb1c674342cfb3ec38378a376729": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_41b12ad69a684cb597594e60355253ae",
      "max": 779,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_88a7b0479e8c40b6aea4084f8039d4f6",
      "value": 779
     }
    },
    "a5c3c70d67124cd09894d3b52dc33e16": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_683e01b6cd98402783fe062f75177b68",
      "placeholder": "​",
      "style": "IPY_MODEL_e7aee59723a5469a9691322d6e124226",
      "value": " 779/779 [00:00&lt;00:00, 974B/s]"
     }
    },
    "b2d0e13e096c49cd9e25bb8a526b1ef7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b9024a2af0984f0a99d7542077d5d183": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_77dad5843dbc43eab577b4cef273228d",
       "IPY_MODEL_d647beae5bff452ebc5e4dbc9974d63e"
      ],
      "layout": "IPY_MODEL_2adcfba8e99b4ceeae8cea6c4060c58d"
     }
    },
    "c2d4ce2a084b4f19bf7ab624c0a853ea": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fd2b26c80b8e4807a595ce1ff9881623",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7de2e7e4e8f44dccabc908334f7d1e21",
      "value": 456318
     }
    },
    "ca9214bf42bf49d99bcec60cc60c6e9c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d647beae5bff452ebc5e4dbc9974d63e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1e5e9065e1ed4adbb1b451672ca4e793",
      "placeholder": "​",
      "style": "IPY_MODEL_6d41343536e54dd3804ace50e15b7953",
      "value": " 499M/499M [00:08&lt;00:00, 61.0MB/s]"
     }
    },
    "db546face47140a59dcfc21539492c51": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "df23412503a1413aadacd254b573c315": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e0847fcebda84c4d8b00b27234d35bc4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9655bb1c674342cfb3ec38378a376729",
       "IPY_MODEL_a5c3c70d67124cd09894d3b52dc33e16"
      ],
      "layout": "IPY_MODEL_8c7d3e2bd2714e929d6859e15e83003f"
     }
    },
    "e2f226a69f704168bbbc6fca9df15e23": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b2d0e13e096c49cd9e25bb8a526b1ef7",
      "placeholder": "​",
      "style": "IPY_MODEL_7a80cfb9bfda407c9ee1b9bd8b11a2f5",
      "value": " 899k/899k [00:02&lt;00:00, 323kB/s]"
     }
    },
    "e7aee59723a5469a9691322d6e124226": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e958eee9b3db48abae24a1f8be1b5d91": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c2d4ce2a084b4f19bf7ab624c0a853ea",
       "IPY_MODEL_75528dba634e465992e5b52107f19032"
      ],
      "layout": "IPY_MODEL_498473ac68aa48c1ba4d5578543a8e2b"
     }
    },
    "eab919bdf6cb499b90a14c297bbc94dc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f55b841175a54f7ea34306caf6d775cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_df23412503a1413aadacd254b573c315",
      "max": 898822,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_91abbefb5e734d47a01158aebf46d7ce",
      "value": 898822
     }
    },
    "f9c4c6ab3ba449e199d7035bff07a5db": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f55b841175a54f7ea34306caf6d775cd",
       "IPY_MODEL_e2f226a69f704168bbbc6fca9df15e23"
      ],
      "layout": "IPY_MODEL_7ede88ead0d449a18ca343d47b723e87"
     }
    },
    "fd2b26c80b8e4807a595ce1ff9881623": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}