Reorganized.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Installs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Folders you will need\n",
    "# mkdir Processing\n",
    "# cd Proccessing\n",
    "# TRAIN_FOLDERPATH = './Processing/crowd_processed/'\n",
    "# mkdir crowd_processed\n",
    "# DEV_FOLDERPATH = = './Processing/crowd_processed_test/'\n",
    "# mkdir crowd_processed_test\n",
    "# TEST_FOLDERPATH = './Processing/crowd_processed_expert/'\n",
    "# mkdir crowd_processed_expert\n",
    "\n",
    "# Our data is coming from \n",
    "POSTPATH = './Data/crowd/train/shared_task_posts.csv'\n",
    "LABELPATH = './Data/crowd/train/crowd_train.csv'\n",
    "USERPATH = './Data/crowd/train/task_C_train.posts.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment to install libraries you might not have\n",
    "# !python3 -m pip install tomotopy\n",
    "# !python3 -m pip install sklearn\n",
    "# !python3 -m pip install imblearn\n",
    "!python3 -m pip install empath\n",
    "\n",
    "# For me only\n",
    "# !python3 -m pip install --upgrade nbconvert \n",
    "# !python3 -m pip install --upgrade nbstripout "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "# Things to install from nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import tomotopy as tp\n",
    "from itertools import chain\n",
    "import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import pickle\n",
    "import os.path\n",
    "\n",
    "import sklearn.preprocessing\n",
    "import imblearn.over_sampling\n",
    "\n",
    "import dataloader\n",
    "import bow\n",
    "import slda\n",
    "import post_classifier\n",
    "import aggregate\n",
    "import user_classifier\n",
    "import empath_extractor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_data(POSTPATH, LABELPATH, USERPATH, FOLDERPATH, subset = 1000, append_title = False, \n",
    "                 filter_images=True, filter_direction=False, filter_control=False):\n",
    "    print('START: Processing data')\n",
    "    users = dataloader.load_user_subset_from_train(USERPATH, subset = subset)\n",
    "    \n",
    "    user_to_post, post_to_words, post_to_metadata = dataloader.load_posts(POSTPATH, user_subset = users, append_title = append_title)\n",
    "    post_to_label = dataloader.load_classification(LABELPATH, user_to_post, post_to_words, post_to_metadata, user_subset = users)\n",
    "    filtered_data, sw_posts, sw_timestamps = dataloader.filter_posts(post_to_label, post_to_metadata, filter_images=filter_images)\n",
    "    print(len(filtered_data))\n",
    "    filtered_data = dataloader.filter_near_SW(filtered_data,post_to_metadata, sw_timestamps,\n",
    "                                             filter_direction=filter_direction, filter_control=filter_direction)\n",
    "    print(len(filtered_data))\n",
    "\n",
    "    filtered_data = dataloader.filter_stopwords(filtered_data)\n",
    "    sw_posts = dataloader.filter_stopwords(sw_posts)\n",
    "    \n",
    "    dataloader.save_to_folder(FOLDERPATH, user_to_post, post_to_metadata, filtered_data, sw_posts, sw_timestamps)\n",
    "    print('DONE: Processing data')\n",
    "    return user_to_post, post_to_metadata, filtered_data, sw_posts, sw_timestamps\n",
    "\n",
    "def load_processed_data(FOLDERPATH):\n",
    "    print('IN PROGRESS: Loading data')\n",
    "    return dataloader.load_from_folder(FOLDERPATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_label_size(data):\n",
    "    a_len = len([data[key] for key in data.keys() if data[key][2] == 'a'])\n",
    "    b_len = len([data[key] for key in data.keys() if data[key][2] == 'b'])\n",
    "    c_len = len([data[key] for key in data.keys() if data[key][2] == 'c'])\n",
    "    d_len = len([data[key] for key in data.keys() if data[key][2] == 'd'])\n",
    "    total_len = len(data)\n",
    "    control_len = total_len - a_len - b_len - c_len - d_len\n",
    "    print('a: ', a_len)\n",
    "    print('b: ', b_len)\n",
    "    print('c: ', c_len)\n",
    "    print('d: ', d_len)\n",
    "    print('controls: ', control_len)\n",
    "    print('total: ', total_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topics(model, sLDA=True):\n",
    "    data = []\n",
    "    if sLDA:\n",
    "        slda_coefficients = model.get_regression_coef(0)\n",
    "        for k in range(model.k):\n",
    "            top_words = model.get_topic_words(k, top_n=40)\n",
    "            words = [word for (word, float) in top_words]\n",
    "            words = \", \".join(words)\n",
    "            data.append([words, slda_coefficients[k]])\n",
    "\n",
    "        indices = np.array(slda_coefficients).argsort()\n",
    "        data = np.array(data)\n",
    "        data = data[indices]\n",
    "\n",
    "        topics = pd.DataFrame(data, columns=[\"Topic\", \"Suicidality Coefficient\"])\n",
    "    else:\n",
    "        for k in range(model.k):\n",
    "            top_words = model.get_topic_words(k, top_n=40)\n",
    "            words = [word for (word, float) in top_words]\n",
    "            words = \", \".join(words)\n",
    "            data.append([words])\n",
    "        \n",
    "        data = np.array(data)\n",
    "        topics = pd.DataFrame(data, columns=[\"Topic\"])\n",
    "            \n",
    "    print(topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_slda(data, data_type='filter_normal', filter_direction=False, filter_control=False, num_topics=40):\n",
    "    slda_model_filename = 'slda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_model.bin'\n",
    "    slda_vectors_filename = 'slda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+    '_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "\n",
    "    if os.path.exists(slda_model_filename):\n",
    "        print('IN PROGRESS: Loading sLDA model')\n",
    "        slda_model = tp.SLDAModel.load(slda_model_filename)\n",
    "    else:\n",
    "        print('START: Training sLDA model')\n",
    "        slda_model = slda.train_slda_model_from_data(data, topics=num_topics)\n",
    "        slda_model.save(slda_model_filename)\n",
    "        print('IN PROGRESS: Saving sLDA model')\n",
    "    # print out topics and top n words\n",
    "#     get_topics(slda_model)\n",
    "\n",
    "    slda_vectors = extract_slda_features(data, slda_model, slda_vectors_filename, d_type='train')\n",
    "\n",
    "    return slda_model, slda_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lda(data, data_type='filter_normal', filter_direction=False, filter_control=False, num_topics=40):\n",
    "    lda_model_filename = 'lda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_model.bin'\n",
    "    lda_vectors_filename = 'lda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+    '_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "\n",
    "    if os.path.exists(lda_model_filename):\n",
    "        print('IN PROGRESS: Loading LDA model')\n",
    "        lda_model = tp.LDAModel.load(lda_model_filename)\n",
    "    else:\n",
    "        print('START: Training LDA model')\n",
    "        lda_model = slda.train_lda_model_from_data(data, topics=num_topics)\n",
    "        lda_model.save(lda_model_filename)\n",
    "        print('IN PROGRESS: Saving LDA model')\n",
    "    # print out topics and top n words\n",
    "    get_topics(lda_model, sLDA=False)\n",
    "\n",
    "    lda_vectors = extract_slda_features(data, lda_model, lda_vectors_filename, d_type='train')\n",
    "\n",
    "    return lda_model, lda_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bow(data, filter_direction=False, filter_control=False, data_type='filter_normal'):\n",
    "    pca_model_filename = 'train_pca_'+data_type+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_model.pickle'\n",
    "    bow_vectors_filename = 'train_bow_'+data_type+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "    word2index_filename = 'train_bow_word'+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_word2index.pickle'\n",
    "    index2word_filename = 'train_bow_word'+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_index2word.pickle'\n",
    "\n",
    "    if os.path.exists(pca_model_filename) and os.path.exists(bow_vectors_filename):\n",
    "        print('IN PROGRESS: Loading PCA model')\n",
    "        with open(pca_model_filename, 'rb') as f:\n",
    "            pca_model = pickle.load(f)\n",
    "        with open(bow_vectors_filename, 'rb') as f:\n",
    "            bow_vectors = pickle.load(f)\n",
    "        with open(word2index_filename, 'rb') as f:\n",
    "            word2index = pickle.load(f)\n",
    "        with open(index2word_filename, 'rb') as f:\n",
    "            index2word = pickle.load(f)\n",
    "    else:\n",
    "        print('START: Training BOW model')\n",
    "        word2index, index2word = bow.generate_vocabulary(data)\n",
    "        pca_model, bow_vectors = bow.get_PCA_vectors_from_post_set(data, word2index)\n",
    "        print('IN PROGRESS: Saving BOW model')\n",
    "        with open(pca_model_filename, 'wb') as f:\n",
    "            pickle.dump(pca_model, f)\n",
    "        with open(bow_vectors_filename, 'wb') as f:\n",
    "            pickle.dump(bow_vectors, f)\n",
    "        with open(word2index_filename, 'wb') as f:\n",
    "            pickle.dump(word2index, f)\n",
    "        with open(index2word_filename, 'wb') as f:\n",
    "            pickle.dump(index2word, f)\n",
    "    return pca_model, bow_vectors, word2index, index2word "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_feature_model(data, data_type='filter_normal', sLDA=False, LDA=False, BOW=False, \n",
    "                        empath=False, filter_direction=False, filter_control=False, num_topics=40):\n",
    "    model = ''\n",
    "    vectors = ''\n",
    "    pca_model = ''\n",
    "    bow_vectors = ''\n",
    "    word2index = ''\n",
    "    index2word = ''\n",
    "    if sLDA and BOW:\n",
    "        model, vectors = get_slda(data, data_type=data_type, filter_direction=filter_direction,\n",
    "                                              num_topics=num_topics)\n",
    "        pca_model, bow_vectors, word2index, index2word = get_bow(data, data_type=data_type, \n",
    "                                                                   filter_direction=filter_direction,\n",
    "                                                                   filter_control=filter_control)\n",
    "    elif sLDA:\n",
    "        model, vectors = get_slda(data, data_type=data_type, num_topics=num_topics, \n",
    "                                              filter_direction=filter_direction,\n",
    "                                              filter_control=filter_control)\n",
    "    elif LDA:\n",
    "        model, vectors = get_lda(data, data_type=data_type, num_topics=num_topics, \n",
    "                                           filter_direction=filter_direction, \n",
    "                                           filter_control=filter_control)\n",
    "    elif BOW:\n",
    "        pca_model, bow_vectors, word2index, index2word = get_bow(data, data_type=data_type,\n",
    "                                                                  filter_direction=filter_direction,\n",
    "                                                                  filter_control=filter_control)\n",
    "    elif empath:\n",
    "        vectors = extract_empath_features(data)\n",
    "    else:\n",
    "        print('Please choose an option')\n",
    "    return model, vectors, pca_model, bow_vectors, word2index, index2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_slda_features(data, slda_model, slda_vectors_filename, d_type='train'):\n",
    "    slda_vectors_filename = d_type+'_'+slda_vectors_filename\n",
    "    \n",
    "    if os.path.exists(slda_vectors_filename):\n",
    "        with open(slda_vectors_filename, 'rb') as f:\n",
    "            print('IN PROGRESS: Loading sLDA vectors')\n",
    "            slda_vectors=pickle.load(f)\n",
    "    else:\n",
    "        print('START: Getting topic sLDA vecs')\n",
    "        slda_vectors = slda.get_topic_vecs(slda_model, data)\n",
    "        print('IN PROGRESS: Saving sLDA vectors')\n",
    "        with open(slda_vectors_filename, 'wb') as f:\n",
    "            pickle.dump(slda_vectors, f)\n",
    "    return slda_vectors\n",
    "\n",
    "def extract_lda_features(data, lda_model, lda_vectors_filename, d_type='train'):\n",
    "\n",
    "    lda_vectors_filename = d_type+'_'+lda_vectors_filename\n",
    "    \n",
    "    if os.path.exists(lda_vectors_filename):\n",
    "        with open(lda_vectors_filename, 'rb') as f:\n",
    "            print('IN PROGRESS: Loading LDA vectors')\n",
    "            lda_vectors=pickle.load(f)\n",
    "    else:\n",
    "        print('START: Getting topic LDA vecs')\n",
    "        lda_vectors = slda.get_topic_vecs(lda_model, data)\n",
    "        print('IN PROGRESS: Saving sLDA vectors')\n",
    "        with open(lda_vectors_filename, 'wb') as f:\n",
    "            pickle.dump(lda_vectors, f)\n",
    "    return lda_vectors\n",
    "\n",
    "def extract_bow_features(data, word2index, pca_model, data_type='filter_normal', \n",
    "                         filter_direction=False, filter_control=False):\n",
    "    bow_vectors_filename = 'test_bow_'+data_type+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "    \n",
    "    if os.path.exists(bow_vectors_filename):\n",
    "        with open(bow_vectors_filename, 'rb') as f:\n",
    "            print('IN PROGRESS: Loading BOW vectors')\n",
    "            bow_vectors=pickle.load(f)\n",
    "    else:\n",
    "        print('START: Getting BOW dev vecs')\n",
    "        _, bow_vectors = bow.get_PCA_vectors_from_post_set(data, word2index, pca_model=pca_model)\n",
    "        print('IN PROGRESS: Saving BOW dev vectors')\n",
    "        with open(bow_vectors_filename, 'wb') as f:\n",
    "            pickle.dump(bow_vectors, f)\n",
    "    \n",
    "    return bow_vectors\n",
    "\n",
    "def extract_empath_features(data, data_type='filter_normal', filter_direction=False, \n",
    "                            filter_control=False, d_type='train'):\n",
    "\n",
    "    empath_vectors_filename = d_type+'_empath_'+data_type+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "    \n",
    "    if os.path.exists(empath_vectors_filename):\n",
    "        with open(empath_vectors_filename, 'rb') as f:\n",
    "            print('IN PROGRESS: Loading empath vectors')\n",
    "            empath_vectors=pickle.load(f)\n",
    "    else:\n",
    "        print('START: Getting empath vecs')\n",
    "        empath_vectors = empath_extractor.get_empath_vectors_from_post_set(data)\n",
    "        print('IN PROGRESS: Saving empath vectors')\n",
    "        with open(empath_vectors_filename, 'wb') as f:\n",
    "            pickle.dump(empath_vectors, f)\n",
    "    \n",
    "    return empath_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def slda_vectorize_data(model, FOLDERPATH):\n",
    "    X, y, post_vectors = slda.vectorize_data_set(model, FOLDERPATH)\n",
    "    return  X, y, post_vectors\n",
    "    \n",
    "def minmax_norm(arr):    \n",
    "    return (arr - np.min(arr))/(np.max(arr) -np.min(arr))\n",
    "\n",
    "def format_features(FOLDERPATH, model='', vectors='', pca_model='', bow_vectors='', num_topics=0,\n",
    "                    data_type='filter_normal', filter_direction=False, filter_control=False,\n",
    "                    sLDA=False, LDA=False, BOW=False, empath=False, d_type='train'):\n",
    "    post_vectors = ''\n",
    "    fil = '_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)\n",
    "    if sLDA and BOW:\n",
    "        X_file = '_X_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+'_BOW='+str(BOW)+fil+'.pickle'\n",
    "        y_file = '_y_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+'_BOW='+str(BOW)+fil+'.pickle'\n",
    "        post_vectors_file = 'post_vectors_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+'_BOW='+str(BOW)+fil+'.pickle'\n",
    "    elif sLDA:\n",
    "        X_file = '_X_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+fil+'.pickle'\n",
    "        y_file = '_y_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+fil+'.pickle'\n",
    "        post_vectors_file = 'post_vectors_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+fil+'.pickle'\n",
    "    elif LDA:\n",
    "        X_file = '_X_'+data_type+'_LDA='+str(sLDA)+'_'+str(num_topics)+fil+'.pickle'\n",
    "        y_file = '_y_'+data_type+'_LDA='+str(sLDA)+'_'+str(num_topics)+fil+'.pickle'\n",
    "        post_vectors_file = 'post_vectors_'+data_type+'_LDA='+str(sLDA)+'_'+str(num_topics)+fil+'.pickle'\n",
    "    elif BOW:\n",
    "        X_file = '_X_'+data_type+'_BOW='+str(BOW)+fil+'.pickle'\n",
    "        y_file = '_y_'+data_type++'_BOW='+str(BOW)+fil+'.pickle'\n",
    "        post_vectors_file = 'post_vectors_'+data_type+'_BOW='+str(BOW)+fil+'.pickle'\n",
    "    elif empath:\n",
    "        X_file = '_X_'+data_type+'_empath='+str(empath)+fil+'.pickle'\n",
    "        y_file = '_y_'+data_type+'_empath='+str(empath)+fil+'.pickle'\n",
    "        post_vectors_file = 'post_vectors_'+data_type+'_empath='+str(empath)+fil+'.pickle'\n",
    "        \n",
    "    X_file = d_type+X_file\n",
    "    y_file = d_type+y_file\n",
    "    post_vectors_file = d_type+post_vectors_file\n",
    "        \n",
    "    if os.path.exists(X_file) and os.path.exists(y_file) and os.path.exists(post_vectors_file):\n",
    "        with open(X_file,'rb') as f:\n",
    "            X = pickle.load(f)\n",
    "        with open(y_file,'rb') as f:\n",
    "            y = pickle.load(f)\n",
    "        with open(post_vectors_file,'rb') as f:\n",
    "            post_vectors = pickle.load(f)\n",
    "    else:\n",
    "        if sLDA and BOW:\n",
    "            print('START: Vectorize sLDA and BOW')\n",
    "            X = np.array([np.concatenate([minmax_norm(vectors[key][0]),minmax_norm(bow_vectors[key][0])]) for key in vectors.keys()])\n",
    "            y = np.array([vectors[key][1] for key in vectors.keys()])\n",
    "            y = y.reshape(np.shape(y)[0])\n",
    "        elif BOW:\n",
    "            print('START: Vectorize BOW')\n",
    "            X = np.array([bow_vectors[key][0] for key in bow_vectors.keys()])\n",
    "            y = np.array([bow_vectors[key][1] for key in bow_vectors.keys()])\n",
    "            y = y.reshape(np.shape(y)[0])\n",
    "        elif sLDA:\n",
    "            print('START: Vectorize sLDA')\n",
    "            X, y, post_vectors = slda_vectorize_data(model, FOLDERPATH)\n",
    "            y = y.reshape(np.shape(y)[0])\n",
    "        elif LDA:\n",
    "            print('START: Vectorize LDA')\n",
    "            X, y, post_vectors = slda_vectorize_data(model, FOLDERPATH)\n",
    "            y = y.reshape(np.shape(y)[0])\n",
    "        elif empath:\n",
    "            print('START: Vectorize empath')\n",
    "            X = np.array([vectors[key][0] for key in vectors.keys()])\n",
    "            y = np.array([vectors[key][1] for key in vectors.keys()])\n",
    "            y = y.reshape(np.shape(y)[0])\n",
    "        else:\n",
    "            print('Oop! please pick a feature extractor, i.e. sLDA=True')\n",
    "        if d_type=='train':\n",
    "            ros = imblearn.over_sampling.RandomOverSampler(random_state=0)\n",
    "            X, y = ros.fit_resample(X, y)\n",
    "            \n",
    "        print('DONE: Vectorizing')\n",
    "        print('IN PROGRESS: Saving vectors')\n",
    "        with open(X_file,'wb') as f:\n",
    "            pickle.dump(X, f)\n",
    "        with open(y_file,'wb') as f:\n",
    "            pickle.dump(y, f)\n",
    "        with open(post_vectors_file,'wb') as f:\n",
    "            pickle.dump(post_vectors, f)\n",
    "        \n",
    "    return X, y, post_vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Post-level Classification Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_post_classifier(X_train, y_train, post_clf_type='RbfSVM', num_topics=40, \n",
    "                          data_type='filter_normal', filter_direction=False, filter_control=False,\n",
    "                          sLDA=False, LDA=False, BOW=False, empath=False):\n",
    "    fil = '_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)\n",
    "    if sLDA and BOW:\n",
    "        file = 'p_clf_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+'_BOW='+str(BOW)+fil+'_'+post_clf_type+'.pickle'\n",
    "    elif sLDA:\n",
    "        file = 'p_clf_'+data_type+'_sLDA='+str(sLDA)+'_'+str(num_topics)+fil+'_'+post_clf_type+'.pickle'\n",
    "    elif LDA:\n",
    "        file = 'p_clf_'+data_type+'_LDA='+str(sLDA)+'_'+str(num_topics)+fil+'_'+post_clf_type+'.pickle'\n",
    "    elif BOW:\n",
    "        file = 'p_clf_'+data_type+'_BOW='+str(BOW)+fil+'_'+post_clf_type+'.pickle'\n",
    "    elif empath:\n",
    "        file = 'p_clf_'+data_type+'_empath='+str(empath)+fil+'_'+post_clf_type+'.pickle'\n",
    "        \n",
    "    if os.path.exists(file):\n",
    "        print('IN PROGRESS: Loading post classifier')\n",
    "        with open(file, 'rb') as f:\n",
    "            p_clf = pickle.load(f)\n",
    "    else:\n",
    "        print('START: Training post classifier')\n",
    "#         post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "#         if post_clf_types[0] == post_clf_type:\n",
    "            #param_dict = {'C':[0.2,0.5,0.7,1,1.5,2,5]}\n",
    "#         elif post_clf_types[1] == post_clf_type:\n",
    "            #param_dict = {'C':[0.2,0.5,1,2]}\n",
    "        #p_clf = post_classifier.PostClassification(post_clf_type)\n",
    "#         elif post_clf_types[2] == post_clf_type:\n",
    "            #param_dict = {'C':[0.5,1,2,5]}           \n",
    "#         elif post_clf_types[5] == post_clf_type:\n",
    "            #param_dict = {'hidden_layer_sizes':[(64,64),(64,64,64),(32,32), (32,32,32)], 'learning_rat\n",
    "        #p_clf.train_grid_search_CV(X_train, y_train, param_dict, groups=5)\n",
    "\n",
    "        \n",
    "        p_clf = post_classifier.PostClassification(post_clf_type)\n",
    "        p_clf.train(X_train, y_train)\n",
    "        print('IN PROGRESS: Saving post classifier')\n",
    "        with open(file, 'wb') as f:\n",
    "            pickle.dump(p_clf, f)\n",
    "    return p_clf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Post-level Classification Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_posts(p_clf, X_test, y_test, print_metrics=True):\n",
    "    y_pred = p_clf.test(X_test)\n",
    "    if print_metrics:\n",
    "        p_clf.get_metrics(y_test, y_pred)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User-level Classifcation Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_post_labels(data, post_vectors, y_pred, FOLDERPATH, sLDA=False, BOW=False):\n",
    "    # relabel the user labels from 1 for d and 0 for all else\n",
    "    user_to_y = defaultdict(int)\n",
    "    for post_id in tqdm.tqdm(data.keys()):\n",
    "        user_to_y[data[post_id][0]] = (1 if data[post_id][2] == 'd' else 0)\n",
    "    \n",
    "    # format the data to get user for each post\n",
    "    post2user_ypred = defaultdict(list)\n",
    "    for i, post_id in enumerate(post_vectors.keys()):\n",
    "        user_id = data[post_id][0]\n",
    "        post2user_ypred[post_id] = [user_id, y_pred[i]]\n",
    "        \n",
    "    user_to_post_label = aggregate.aggregate_posts(FOLDERPATH, post2user_ypred)\n",
    "    \n",
    "    return user_to_post_label, user_to_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_user_pred(user_to_y, user_to_ypred):\n",
    "    user_y = []\n",
    "    user_y_pred = []\n",
    "    for user_id in user_to_ypred:\n",
    "        user_y.append(user_to_y[user_id])\n",
    "        user_y_pred.append(user_to_ypred[user_id])\n",
    "    return user_y, user_y_pred\n",
    "\n",
    "def predict_users(user_to_post_label, user_to_y, user_clf_type='Max'):\n",
    "    u_clf = user_classifier.UserClassification(user_to_post_label)\n",
    "    if user_clf_type=='Max':\n",
    "        user_to_ypred = u_clf.argmax()\n",
    "    elif user_clf_type=='Threshold':\n",
    "        user_to_ypred = u_clf.find_threshold(user_to_y)\n",
    "    elif user_clf_type=='Minimum':\n",
    "        user_to_ypred = u_clf.minimum(1)\n",
    "    \n",
    "    user_y, user_ypred = format_user_pred(user_to_y, user_to_ypred)\n",
    "    print(u_clf.get_metrics(user_y, user_ypred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wrapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wrapped(train_data_processed=True, dev_data_processed=True, sLDA=False, \n",
    "            data_type='filter_normal', num_topics=40, BOW=False, LDA=False, empath=False, \n",
    "            filter_direction=False, filter_control=False,\n",
    "            post_clf_type='RbfSVM', user_clf_types=['Max']):\n",
    "    # Make sure you have created TRAIN_FOLDERPATH and DEV_FOLDERPATH directories on local\n",
    "    TRAIN_POSTPATH = './Data/crowd/train/shared_task_posts.csv'\n",
    "    TRAIN_LABELPATH = './Data/crowd/train/crowd_train.csv'\n",
    "    TRAIN_USERPATH = './Data/crowd/train/task_C_train.posts.csv'\n",
    "    \n",
    "    DEV_POSTPATH = './Data/crowd/test/shared_task_posts_test.csv'\n",
    "    DEV_LABELPATH = './Data/crowd/test/crowd_test_C.csv'\n",
    "    DEV_USERPATH = './Data/crowd/test/task_C_test.posts.csv'\n",
    "    \n",
    "    TRAIN_FOLDERPATH = './Processing/crowd_processed/'\n",
    "    DEV_FOLDERPATH = './Processing/crowd_processed_test/'\n",
    "    \n",
    "    # Proccess or load processed training data\n",
    "    if not train_data_processed:\n",
    "        user_to_post, post_to_metadata, filtered_data, sw_posts, sw_timestamps = process_data(TRAIN_POSTPATH, TRAIN_LABELPATH, \n",
    "                                                                                              TRAIN_USERPATH, TRAIN_FOLDERPATH)\n",
    "    else:\n",
    "        user_to_post, post_to_metadata, filtered_data, sw_posts, sw_timestamps = load_processed_data(TRAIN_FOLDERPATH)\n",
    "    \n",
    "    # print class sizes\n",
    "    print_label_size(filtered_data)\n",
    "    \n",
    "    model, vectors, pca_model, bow_vectors, word2index, index2word  = train_feature_model(filtered_data, BOW=BOW,\n",
    "                                                                                                    sLDA=sLDA, LDA=LDA,\n",
    "                                                                                                    empath=empath, \n",
    "                                                                                                    data_type=data_type, \n",
    "                                                                                                    num_topics=num_topics,\n",
    "                                                                                                    filter_direction=filter_direction, \n",
    "                                                                                                    filter_control=filter_direction)\n",
    "    \n",
    "    X_train, y_train, post_vectors = format_features(TRAIN_FOLDERPATH, model=model, vectors=vectors, \n",
    "                                                          pca_model=pca_model, bow_vectors=bow_vectors,\n",
    "                                                         data_type=data_type, num_topics=num_topics, d_type='train', \n",
    "                                                         BOW=BOW, sLDA=sLDA, LDA=LDA, empath=empath,\n",
    "                                                        filter_direction=filter_direction, filter_control=filter_direction)\n",
    "\n",
    "#     y_train = y_train.reshape(np.shape(y_train)[0])\n",
    "    \n",
    "    p_clf = train_post_classifier(X_train, y_train, post_clf_type=post_clf_type, num_topics=num_topics, \n",
    "                                  data_type=data_type, BOW=BOW, sLDA=sLDA, LDA=LDA, empath=empath,\n",
    "                                  filter_direction=filter_direction, filter_control=filter_direction)\n",
    "    \n",
    "    print('DONE: Training complete')\n",
    "    \n",
    "    \n",
    "#     Proccess or load processed development data\n",
    "    if not dev_data_processed:\n",
    "        user_to_post, post_to_metadata, filtered_data, sw_posts, sw_timestamps = process_data(DEV_POSTPATH, \n",
    "                                                                                              DEV_LABELPATH, DEV_USERPATH, DEV_FOLDERPATH)\n",
    "    else:\n",
    "        user_to_post, post_to_metadata, filtered_data, sw_posts, sw_timestamps = load_processed_data(DEV_FOLDERPATH)\n",
    "    \n",
    "    print_label_size(filtered_data)\n",
    "    \n",
    "    \n",
    "    if sLDA:\n",
    "        slda_vectors_filename = 'slda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "        vectors = extract_slda_features(filtered_data, model, slda_vectors_filename, d_type='dev')\n",
    "    if LDA:\n",
    "        lda_vectors_filename = 'lda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "        vectors = extract_slda_features(filtered_data, model, lda_vectors_filename, d_type='dev')\n",
    "    if BOW:\n",
    "        bow_vectors= extract_bow_features(filtered_data, word2index, pca_model)\n",
    "    if empath:\n",
    "        vectors = extract_empath_features(filtered_data, data_type=data_type, filter_direction=filter_direction, \n",
    "                            filter_control=filter_control, d_type='train')\n",
    "    \n",
    "    X_dev, y_dev, post_vectors = format_features(DEV_FOLDERPATH, model=model, vectors=vectors, \n",
    "                                                          pca_model=pca_model, bow_vectors=bow_vectors,\n",
    "                                                         data_type=data_type, num_topics=num_topics, d_type='dev', \n",
    "                                                         BOW=BOW, sLDA=sLDA, LDA=LDA, empath=empath,\n",
    "                                                        filter_direction=filter_direction, filter_control=filter_direction)\n",
    "        \n",
    "    \n",
    "    y_pred = predict_posts(p_clf, X_dev, y_dev, print_metrics=True)\n",
    "    \n",
    "    user_to_post_label, user_to_y = format_post_labels(filtered_data, vectors, y_pred, DEV_FOLDERPATH)\n",
    "    \n",
    "    for user_clf_type in user_clf_types:\n",
    "        print('START: User classification post:', user_clf_type)\n",
    "        predict_users(user_to_post_label, user_to_y, user_clf_type=user_clf_type)\n",
    "    return model, vectors, pca_model, bow_vectors, word2index, p_clf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Expert Wrapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expert_wrapped(test_data_processed=True,  num_topics=40, data_type='filter_normal', \n",
    "                   post_clf_type='RbfSVM', user_clf_types=['Max'], p_clf='',\n",
    "                   model='', vectors='', pca_model='', bow_vectors='', word2index='', \n",
    "                   filter_direction=False, filter_control=False,\n",
    "                   BOW=False, sLDA=False, LDA=False, empath=False):\n",
    "    # Make sure you have created TRAIN_FOLDERPATH and DEV_FOLDERPATH directories on local\n",
    "    TEST_POSTPATH = './Data/expert/expert_posts.csv'\n",
    "    TEST_LABELPATH = './Data/expert/expert.csv'\n",
    "    TEST_FOLDERPATH = './Processing/crowd_processed_expert/'\n",
    "    \n",
    "    if test_data_processed:\n",
    "        user_to_post_expert, post_to_metadata_expert, filtered_data_expert, sw_posts_expert, sw_timestamps_expert = dataloader.load_from_folder(TEST_FOLDERPATH)\n",
    "    else:\n",
    "        user_to_post_expert, post_to_words_expert, post_to_metadata_expert = dataloader.load_posts(TEST_POSTPATH, append_title = False)\n",
    "        post_to_label_expert = dataloader.load_classification(TEST_LABELPATH, user_to_post_expert, post_to_words_expert, post_to_metadata_expert)\n",
    "        filtered_data_expert, sw_posts_expert, sw_timestamps_expert = dataloader.filter_posts(post_to_label_expert, post_to_metadata_expert, filter_images = True)\n",
    "        print(len(filtered_data_expert))\n",
    "        filtered_data_expert = dataloader.filter_near_SW(filtered_data_expert, post_to_metadata_expert, sw_timestamps_expert)\n",
    "        print(len(filtered_data_expert))\n",
    "\n",
    "        filtered_data_expert = dataloader.filter_stopwords(filtered_data_expert)\n",
    "        sw_posts_expert = dataloader.filter_stopwords(sw_posts_expert)\n",
    "\n",
    "\n",
    "        dataloader.save_to_folder(TEST_FOLDERPATH, user_to_post_expert, post_to_metadata_expert, filtered_data_expert, sw_posts_expert, sw_timestamps_expert)\n",
    "        \n",
    "    print_label_size(filtered_data_expert)\n",
    "    \n",
    "    if sLDA:\n",
    "        slda_vectors_filename = 'slda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "        vectors = extract_slda_features(filtered_data_expert, model, slda_vectors_filename, d_type='test')\n",
    "    if LDA:\n",
    "        lda_vectors_filename = 'lda_'+data_type+'_'+str(num_topics)+'_filter_direction='+str(filter_direction)+'_filter_control='+str(filter_control)+'_vectors.pickle'\n",
    "        vectors = extract_slda_features(filtered_data_expert, model, lda_vectors_filename, d_type='test')\n",
    "    if BOW:\n",
    "        bow_vectors= extract_bow_features(filtered_data_expert, word2index, pca_model)\n",
    "    if empath:\n",
    "        vectors = extract_empath_features(filtered_data_expert, data_type=data_type, filter_direction=filter_direction, \n",
    "                            filter_control=filter_control, d_type='test')\n",
    "        \n",
    "    X_test, y_test, post_vectors = format_features(TEST_FOLDERPATH, model=model, vectors=vectors, \n",
    "                                                          pca_model=pca_model, bow_vectors=bow_vectors,\n",
    "                                                         data_type=data_type, num_topics=num_topics, d_type='test', \n",
    "                                                         BOW=BOW, sLDA=sLDA, LDA=LDA, empath=empath,\n",
    "                                                        filter_direction=filter_direction, filter_control=filter_direction)\n",
    "\n",
    "    \n",
    "    y_pred = predict_posts(p_clf, X_test, y_test, print_metrics=True)\n",
    "    \n",
    "    user_to_post_label, user_to_y = format_post_labels(filtered_data_expert, vectors, y_pred, TEST_FOLDERPATH)\n",
    "    \n",
    "    for user_clf_type in user_clf_types:\n",
    "        print('START: User classification on Test data:', user_clf_type)\n",
    "        predict_users(user_to_post_label, user_to_y, user_clf_type=user_clf_type)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + LogReg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = ['']\n",
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, sLDA=True, num_topics=40, post_clf_type=post_clf_types[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + LinearSVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = ['']\n",
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, sLDA=True, \n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + RbfSVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = ['']\n",
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, sLDA=True, \n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = ['']\n",
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, sLDA=True, \n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = ['']\n",
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, sLDA=True, \n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + Random MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = ['']\n",
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, sLDA=True, \n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sLDA + BOW + Adaboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LDA + LogReg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, LDA=True,\n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Empath + LogReg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature_extraction =  [sLDA=False, LDA=False, BOW=False, empath=False]\n",
    "post_clf_types = ['LogReg', 'LinearSVM', 'RbfSVM', 'AdaBoost', 'RandomForest', 'MLP']\n",
    "user_clf_types = ['Max', 'Threshold']\n",
    "model, vectors, pca_model, bow_vectors, word2index, p_clf = wrapped(train_data_processed=True, empath=True,\n",
    "                                                                    num_topics=40, post_clf_type=post_clf_types[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expert_wrapped(test_data_processed=True, model=model, vectors=vectors, pca_model='', bow_vectors='',\n",
    "               word2index='', p_clf=p_clf, BOW=False, sLDA=True, LDA=False, empath=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}