S2V_SBERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\linan\\.conda\\envs\\NLP\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# https://sbert.net/\n",
    "from sentence_transformers import SentenceTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000, 2)\n",
      "(300, 2)\n"
     ]
    }
   ],
   "source": [
    "dataset_3000 = pd.read_pickle('./dataset/text_3000.pickle')\n",
    "dataset_300 = pd.read_pickle('./dataset/text_300.pickle')\n",
    "\n",
    "print(dataset_3000.shape)\n",
    "print(dataset_300.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"logic of empire\" is a science fiction novel b...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>major general richard hutton davies,  (14 nove...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>elgin reptiles is the name given to a group of...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dubgaill and finngaill, or dubgenti and finnge...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chang teh-ming (; born 1938) is a taiwanese ph...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0  \"logic of empire\" is a science fiction novel b...      1\n",
       "1  major general richard hutton davies,  (14 nove...      1\n",
       "2  elgin reptiles is the name given to a group of...      1\n",
       "3  dubgaill and finngaill, or dubgenti and finnge...      0\n",
       "4  chang teh-ming (; born 1938) is a taiwanese ph...      1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_3000.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(dataset):\n",
    "    vector_set = pd.DataFrame(columns=['vector', 'label'], index=range(len(dataset)))\n",
    "    vectorizer = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "    text_list = list(dataset['text'])\n",
    "    vectors = vectorizer.encode(text_list)\n",
    "    vector_set['vector'] = list(vectors)\n",
    "    vector_set['label'] = list(dataset['label'])\n",
    "    return vector_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_3000_vector = vectorize(dataset_3000)\n",
    "with open('./dataset/vector_3000_sbert.pickle', 'wb') as f:\n",
    "    pickle.dump(dataset_3000_vector, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_300_vector = vectorize(dataset_300)\n",
    "with open('./dataset/vector_300_sbert.pickle', 'wb') as f:\n",
    "    pickle.dump(dataset_300_vector, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read from pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train set size:  3000\n",
      "test set size:  300\n",
      "train set vector size:  384\n",
      "test set vector size:  384\n"
     ]
    }
   ],
   "source": [
    "dataset_3000_vector = pd.read_pickle('./dataset/vector_3000_sbert.pickle')\n",
    "dataset_300_vector = pd.read_pickle('./dataset/vector_300_sbert.pickle')\n",
    "\n",
    "print('train set size: ', len(dataset_3000_vector))\n",
    "print('test set size: ', len(dataset_300_vector))\n",
    "\n",
    "print('train set vector size: ', len(dataset_3000_vector['vector'][0]))\n",
    "print('test set vector size: ', len(dataset_300_vector['vector'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.07249437, -0.07991132, -0.035554323, -0.048...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[-0.029874628, -0.012974098, 0.054811098, -0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-0.122416824, 0.03937837, 0.04780685, -0.0341...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.06686082, 0.004060772, -0.0090795765, -0.0...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[-0.0040333048, 0.022765186, 0.0038558021, 0.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  label\n",
       "0  [0.07249437, -0.07991132, -0.035554323, -0.048...      1\n",
       "1  [-0.029874628, -0.012974098, 0.054811098, -0.0...      1\n",
       "2  [-0.122416824, 0.03937837, 0.04780685, -0.0341...      1\n",
       "3  [-0.06686082, 0.004060772, -0.0090795765, -0.0...      0\n",
       "4  [-0.0040333048, 0.022765186, 0.0038558021, 0.0...      1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_3000_vector.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For GPT 3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000, 2)\n",
      "(300, 2)\n"
     ]
    }
   ],
   "source": [
    "dataset_3000 = pd.read_pickle('./dataset/text_3000_Turbo.pickle')\n",
    "dataset_300 = pd.read_pickle('./dataset/text_300_Turbo.pickle')\n",
    "\n",
    "print(dataset_3000.shape)\n",
    "print(dataset_300.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_3000_vector = vectorize(dataset_3000)\n",
    "with open('./dataset/vector_3000_Turbo_sbert.pickle', 'wb') as f:\n",
    "    pickle.dump(dataset_3000_vector, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_300_vector = vectorize(dataset_300)\n",
    "with open('./dataset/vector_300_Turbo_sbert.pickle', 'wb') as f:\n",
    "    pickle.dump(dataset_300_vector, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NLP",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}