image_to_text_classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyN/x8OmCEGwRDbtEAnDMayd",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AndrewHamal/got/blob/main/image_to_text_classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hRxicPGD43Z8"
      },
      "outputs": [],
      "source": [
        "!sudo apt install tesseract-ocr\n",
        "!pip install tesserocr\n",
        "!pip install opencv-python\n",
        "!pip install Pillow\n",
        "!pip install pytesseract\n",
        "!pip install git+https://github.com/casics/nostril.git\n",
        "!git clone https://github.com/tesseract-ocr/tessdata.git\n",
        "!git clone https://github.com/rrenaud/Gibberish-Detector\n",
        "!pip install Flask\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gibberish-detector\n",
        "!git clone https://github.com/rrenaud/Gibberish-Detector.git"
      ],
      "metadata": {
        "id": "gzlZ30HJqvND",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4835a6d4-e829-4d42-c682-69355fb4345d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting gibberish-detector\n",
            "  Downloading gibberish_detector-0.1.1-py3-none-any.whl (10 kB)\n",
            "Installing collected packages: gibberish-detector\n",
            "Successfully installed gibberish-detector-0.1.1\n",
            "fatal: destination path 'Gibberish-Detector' already exists and is not an empty directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!gibberish-detector train Gibberish-Detector/big.txt > big.model"
      ],
      "metadata": {
        "id": "QaDc0kSEaUQo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "import tesserocr\n",
        "from nostril import nonsense\n",
        "import cv2\n",
        "from PIL import Image, ImageEnhance\n",
        "from google.colab.patches import cv2_imshow\n",
        "import pytesseract\n",
        "import random\n",
        "import string\n",
        "import re\n",
        "import numpy as np\n",
        "from gibberish_detector import detector\n",
        "import requests\n",
        "from io import BytesIO\n",
        "import urllib.request\n",
        "import nltk\n",
        "from flask import Flask\n",
        "\n",
        "def imageToText(url):\n",
        "  # gibberish_detector set model\n",
        "  Detector = detector.create_from_model('big.model')\n",
        "  response = requests.get(url)\n",
        "\n",
        "  # load image from url in PIL image and convert to RGB/jpg format\n",
        "  img = Image.open(requests.get(url, stream=True).raw)\n",
        "  rgb_im = img.convert('RGB')\n",
        "\n",
        "  # Decrease saturation of image\n",
        "  converter = ImageEnhance.Color(rgb_im)\n",
        "  pil_img = converter.enhance(0)\n",
        "\n",
        "  # react image from url in open cv\n",
        "  req = urllib.request.urlopen(url)\n",
        "  arr = np.asarray(bytearray(req.read()), dtype=np.uint8)\n",
        "  final = cv2.imdecode(arr, cv2.IMREAD_UNCHANGED)\n",
        "\n",
        "  #initialize tesserocr api\n",
        "  api = tesserocr.PyTessBaseAPI(path='tessdata')\n",
        "  api.SetImage(pil_img)\n",
        "\n",
        "  # differentiate all the chat boxes based of text line in array\n",
        "  boxes = api.GetComponentImages(tesserocr.RIL.TEXTLINE, True)\n",
        "  height, width, c = final.shape\n",
        "\n",
        "  inc = int(0.01*width)\n",
        "  list_all = []\n",
        "\n",
        "  # cv2_imshow(final)\n",
        "\n",
        "  for i, (im,box,_,_) in enumerate(boxes):\n",
        "    x,y,w,h = box['x'],box['y'],box['w'],box['h']\n",
        "    crop_img = final[max(0, y-inc):min(y+h+inc,height), max(0, x-inc):min(x+w+inc, width)]\n",
        "    api.SetRectangle(box['x'], box['y'], box['w'], box['h'])\n",
        "\n",
        "    #  cv2_imshow(crop_img)\n",
        "    #  cv2.waitKey(0)\n",
        "\n",
        "    extractedInformation = pytesseract.image_to_string(crop_img)\n",
        "    stringAll = extractedInformation.replace('|', 'I').replace('\\n\\x0c', ' ');\n",
        "\n",
        "    try:\n",
        "      if not Detector.is_gibberish(stringAll):\n",
        "        if (x < 200 and stringAll != '\\x0c' and not re.search(\"(Type a message Send|Snapchat|Ne Sent|Delivered|Message\\n\\n)\", stringAll) and not re.search(\"((1[0-2]|0?[1-9]):([0-5][0-9]))\", stringAll)):\n",
        "          list_all.append({\"body\": stringAll, \"sender\": 'Partner'})\n",
        "        elif (x > 199 and stringAll != '\\x0c' and not re.search(\"Type a message Send|Snapchat|Ne Sent|Delivered|Message\\n\\n\", stringAll) and not re.search(\"((1[0-2]|0?[1-9]):([0-5][0-9]))\", stringAll)):\n",
        "          list_all.append({\"body\": stringAll, \"sender\": 'Me'})\n",
        "    except: ''\n",
        "\n",
        "  return list_all\n",
        "\n",
        "url = \"https://cp-kyle.sfo3.cdn.digitaloceanspaces.com/instant-responses/XZN9oUX7XPlufI2QeslKK3Pk8JQsVhjYnQZD86MZ.png\"\n",
        "print(imageToText(url))\n",
        "\n",
        "# app = Flask(__name__)\n",
        "\n",
        "# @app.route('/<url>')\n",
        "# def hello_world(url):\n",
        "#   return 'Hello World'\n",
        "\n",
        "\n",
        "# if __name__ == '__main__':\n",
        "#   app.run()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9Hb7B7Mp5Qs0",
        "outputId": "d0b97649-44bc-427c-ab5a-1e6f51d24738"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'body': 'christina ', 'sender': 'Me'}, {'body': 'I live in Munich, originally from Egypt ', 'sender': 'Me'}, {'body': 'Has anyone taught you some English ', 'sender': 'Me'}, {'body': 'sweat words yet;)? ', 'sender': 'Me'}, {'body': 'Don’t be shy ', 'sender': 'Me'}, {'body': 'What are you doing in Munich? ', 'sender': 'Partner'}, {'body': '@) I Know some English but not very well ', 'sender': 'Partner'}, {'body': \"I’m running a business with family so I'm \", 'sender': 'Me'}, {'body': 'taking care of things here..also my mom ', 'sender': 'Me'}, {'body': 'told me my dream troublemaker girl ', 'sender': 'Me'}, {'body': 'named Christina was here ', 'sender': 'Me'}, {'body': 'I can teach you some more English in ', 'sender': 'Me'}, {'body': 'exchange for a few cute sentences in ', 'sender': 'Me'}, {'body': 'Russian ', 'sender': 'Me'}, {'body': 'You are cute 2 ', 'sender': 'Partner'}, {'body': '@) Nice, I think we can do this @ ', 'sender': 'Partner'}]\n"
          ]
        }
      ]
    }
  ]
}