Similarity_sentences_document.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize.punkt import PunktSentenceTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "document=\"Hi Steve  Thank you so much for reaching out and taking the time to send us feedback! It is very much appreciated! Please excuse the delayed response. Unfortunately, these scanning issues mostly occur due to older scanning technologies still used by some stores. Some devices (e.g. laser scanners or flatbed scanners) have difficulties reading smartphone displays. We hope to be able to solve this problem soon though as most stores will hopefully replace these scanning devices with newer image-based scanners in the course of their next cash system update. Until then, you can always ask the store staff to type in the card number manually in these cases so that you don't miss out on any rewards points or discounts. In the meantime, I sincerely apologize for the inconvenience this causes.  Once again, thank you for taking the time to contact us about this issue. If you have any further questions, suggestions for improvements or general feedback, please don't hesitate to contact us again.Best regards,Isabelle van Capelleveen Customer Support GmbH C-HUB / Hafenstraße 25-27 68159 Mannheim\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_tokenizer = PunktSentenceTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = sentence_tokenizer.tokenize(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hi Steve  Thank you so much for reaching out and taking the time to send us feedback!',\n",
       " 'It is very much appreciated!',\n",
       " 'Please excuse the delayed response.',\n",
       " 'Unfortunately, these scanning issues mostly occur due to older scanning technologies still used by some stores.',\n",
       " 'Some devices (e.g.',\n",
       " 'laser scanners or flatbed scanners) have difficulties reading smartphone displays.',\n",
       " 'We hope to be able to solve this problem soon though as most stores will hopefully replace these scanning devices with newer image-based scanners in the course of their next cash system update.',\n",
       " \"Until then, you can always ask the store staff to type in the card number manually in these cases so that you don't miss out on any rewards points or discounts.\",\n",
       " 'In the meantime, I sincerely apologize for the inconvenience this causes.',\n",
       " 'Once again, thank you for taking the time to contact us about this issue.',\n",
       " \"If you have any further questions, suggestions for improvements or general feedback, please don't hesitate to contact us again.Best regards,Isabelle van Capelleveen Customer Support GmbH C-HUB / Hafenstraße 25-27 68159 Mannheim\"]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bag_of_words(sentence):\n",
    "    return Counter(word.lower().strip('.,') for word in sentence.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2, 1]], dtype=int64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "c = CountVectorizer()\n",
    "bow_array = c.fit_transform(['hey hey wassup']) #tokenize words and then count them here 2 'hey' and 1 'wassup' so [2 1]\n",
    "bow_array.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 126)\n",
      "  (0, 10)\t1\n",
      "  (0, 121)\t1\n",
      "  (0, 52)\t1\n",
      "  (0, 56)\t1\n",
      "  (0, 64)\t1\n"
     ]
    }
   ],
   "source": [
    "bow_matrix = c.fit_transform(sentences) \n",
    "print((bow_matrix[1]).shape) #to find total number of words in document\n",
    "print(bow_matrix[1])#to find position of words  that are present in sentence like in first sentence 10th,121sr,etc position words are present and that 1 represents it has come only one time in this sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/spartan/.local/lib/python3.5/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "#transforming matrix to graph\n",
    "from sklearn.feature_extraction.text import TfidfTransformer \n",
    "normalized_matrix = TfidfTransformer().fit_transform(bow_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.09621319, 0.04333366, 0.02228679, 0.        ,\n",
       "        0.        , 0.04756537, 0.1784559 , 0.10863429, 0.4009258 ,\n",
       "        0.13436086],\n",
       "       [0.09621319, 1.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.04333366, 0.        , 1.        , 0.        , 0.        ,\n",
       "        0.        , 0.02755957, 0.05755771, 0.10836507, 0.04962846,\n",
       "        0.06819426],\n",
       "       [0.02228679, 0.        , 0.        , 1.        , 0.15475965,\n",
       "        0.        , 0.15693685, 0.04232347, 0.        , 0.02552425,\n",
       "        0.01458805],\n",
       "       [0.        , 0.        , 0.        , 0.15475965, 1.        ,\n",
       "        0.        , 0.11009808, 0.        , 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        1.        , 0.08326911, 0.03362571, 0.        , 0.        ,\n",
       "        0.07599212],\n",
       "       [0.04756537, 0.        , 0.02755957, 0.15693685, 0.11009808,\n",
       "        0.08326911, 1.        , 0.1008582 , 0.11337548, 0.08823968,\n",
       "        0.02075627],\n",
       "       [0.1784559 , 0.        , 0.05755771, 0.04232347, 0.        ,\n",
       "        0.03362571, 0.1008582 , 1.        , 0.15979469, 0.11320338,\n",
       "        0.11528719],\n",
       "       [0.10863429, 0.        , 0.10836507, 0.        , 0.        ,\n",
       "        0.        , 0.11337548, 0.15979469, 1.        , 0.19079701,\n",
       "        0.03030055],\n",
       "       [0.4009258 , 0.        , 0.04962846, 0.02552425, 0.        ,\n",
       "        0.        , 0.08823968, 0.11320338, 0.19079701, 1.        ,\n",
       "        0.19881004],\n",
       "       [0.13436086, 0.        , 0.06819426, 0.01458805, 0.        ,\n",
       "        0.07599212, 0.02075627, 0.11528719, 0.03030055, 0.19881004,\n",
       "        1.        ]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_graph = normalized_matrix * normalized_matrix.T\n",
    "similarity_graph.toarray()\n",
    "#this is matrix representing similarity betwen two sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}