Gemini_Video_Explain.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rabimba/GDE-ML-Artifacts/blob/main/Gemini_Video_Explain.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d6lYXRcjthKV"
      },
      "outputs": [],
      "source": [
        "from google.colab import userdata\n",
        "GOOGLE_API_KEY=userdata.get('GOOGLE_API_KEY')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mN8x8DPgu9Kv"
      },
      "outputs": [],
      "source": [
        "# Fetch the discovery docs for the Generative Language API service.\n",
        "from googleapiclient.discovery import build\n",
        "import googleapiclient\n",
        "import requests\n",
        "\n",
        "DISCOVERY_URL = f'https://generativelanguage.googleapis.com/$discovery/rest?version=v1beta&key={GOOGLE_API_KEY}';\n",
        "discovery_docs = requests.get(DISCOVERY_URL).content\n",
        "genai_service = googleapiclient.discovery.build_from_document(discovery_docs, developerKey=GOOGLE_API_KEY)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jW9ilYCdL99M"
      },
      "outputs": [],
      "source": [
        "import cv2\n",
        "import os\n",
        "import shutil\n",
        "\n",
        "# Create or cleanup existing extracted image frames directory.\n",
        "FRAME_EXTRACTION_DIRECTORY = \"/content/frames\"\n",
        "FRAME_PREFIX = \"_frame\"\n",
        "def create_frame_output_dir(output_dir):\n",
        "  if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)\n",
        "  else:\n",
        "    shutil.rmtree(output_dir)\n",
        "    os.makedirs(output_dir)\n",
        "\n",
        "def extract_frame_from_video(video_file_path):\n",
        "  print(f\"Extracting {video_file_path} at 1 frame per second. This might take a bit...\")\n",
        "  create_frame_output_dir(FRAME_EXTRACTION_DIRECTORY)\n",
        "  vidcap = cv2.VideoCapture(video_file_path)\n",
        "  fps = int(vidcap.get(cv2.CAP_PROP_FPS))\n",
        "  output_file_prefix = os.path.basename(video_file_path).replace('.', '_')\n",
        "  success,image = vidcap.read()\n",
        "  frame_count = 0  # Initialize a frame counter\n",
        "  count = 0\n",
        "  while vidcap.isOpened():\n",
        "      success, frame = vidcap.read()\n",
        "      if not success:  # End of video\n",
        "          break\n",
        "      if count % int(fps) == 0:  # Extract a frame every second\n",
        "          image_name = f\"{output_file_prefix}{FRAME_PREFIX}{frame_count:04d}.jpg\"\n",
        "          output_filename = os.path.join(FRAME_EXTRACTION_DIRECTORY, image_name)\n",
        "          cv2.imwrite(output_filename, frame)\n",
        "          frame_count += 1\n",
        "      count += 1\n",
        "  vidcap.release()  # Release the capture object\n",
        "  print(f\"Completed video frame extraction!\\n\\nExtracted: {frame_count} frames\")\n",
        "\n",
        "\n",
        "video_file_name = \"cs25.mp4\" # @param {type:\"string\"}\n",
        "extract_frame_from_video(video_file_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JSd4s0YygV9r"
      },
      "outputs": [],
      "source": [
        "import mimetypes\n",
        "import os\n",
        "\n",
        "class File:\n",
        "  def __init__(self, file_path: str, display_name: str = None,\n",
        "               timestamp_seconds: int = None, mimetype: str = None, uri = None):\n",
        "    self.file_path = file_path\n",
        "    if display_name:\n",
        "      self.display_name = display_name\n",
        "    if timestamp_seconds != None:\n",
        "      self.timestamp = seconds_to_time_string(timestamp_seconds)\n",
        "    # Detect mimetype if not specified\n",
        "    self.mimetype = mimetype if mimetype else mimetypes.guess_type(file_path)[0]\n",
        "    self.uri = uri\n",
        "\n",
        "  def set_file_uri(self, uri):\n",
        "    self.uri = uri\n",
        "\n",
        "def seconds_to_time_string(seconds):\n",
        "  \"\"\"Converts an integer number of seconds to a string in the format '00:00'.\n",
        "     Format is the expected format for Gemini 1.5.\n",
        "  \"\"\"\n",
        "  minutes = seconds // 60\n",
        "  seconds = seconds % 60\n",
        "  return f\"{minutes:02d}:{seconds:02d}\"\n",
        "\n",
        "def get_timestamp_seconds(filename):\n",
        "  \"\"\"Extracts the frame count (as an integer) from a filename with the format\n",
        "     'output_file_prefix_frame0000.jpg'.\n",
        "  \"\"\"\n",
        "  parts = filename.split(FRAME_PREFIX)\n",
        "  if len(parts) != 2:\n",
        "      return None  # Indicate that the filename might be incorrectly formatted\n",
        "\n",
        "  frame_count_str = parts[1].split(\".\")[0]\n",
        "  return int(frame_count_str)\n",
        "\n",
        "# Process each frame in the output directory\n",
        "files = os.listdir(FRAME_EXTRACTION_DIRECTORY)\n",
        "files = sorted(files)  # Sort alphabetically\n",
        "files_to_upload = []\n",
        "for file in files:\n",
        "  files_to_upload.append(\n",
        "      File(file_path=os.path.join(FRAME_EXTRACTION_DIRECTORY, file),\n",
        "           timestamp_seconds=get_timestamp_seconds(file)))\n",
        "\n",
        "# Upload the files to the API\n",
        "uploaded_files = []\n",
        "print(f'Uploading {len(files_to_upload)} files. This might take a bit...')\n",
        "for file in files_to_upload:\n",
        "  print(f'Uploading: {file.file_path}...')\n",
        "  response = genai_service.media().upload(\n",
        "      media_body=file.file_path,\n",
        "      media_mime_type = file.mimetype).execute()\n",
        "  file.set_file_uri(response[\"file\"][\"uri\"])\n",
        "  uploaded_files.append(file)\n",
        "\n",
        "print(f\"Completed file uploads!\\n\\nUploaded: {len(uploaded_files)} files\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SHMVCWHkFhJW"
      },
      "outputs": [],
      "source": [
        "# List files uploaded in the API\n",
        "response = genai_service.files().list().execute()\n",
        "print(response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZYVFqmLkl5nE"
      },
      "outputs": [],
      "source": [
        "# Make GenerateContent Request\n",
        "def makeGenerateContentRequest(prompt, files):\n",
        "  generateContent = {\"contents\": [{ \"parts\": [{ \"text\": prompt }]}]}\n",
        "  for file in files:\n",
        "    generateContent[\"contents\"][0][\"parts\"].extend(makeVideoPart(file))\n",
        "  return generateContent\n",
        "\n",
        "def makeVideoPart(file):\n",
        "  return [\n",
        "      {\"text\": file.timestamp},\n",
        "      {\"file_data\": {\"file_uri\": file.uri, \"mime_type\": file.mimetype }}]\n",
        "\n",
        "prompt = \"Explain the video in detail. List the concepts of each slide and write code examples for each concept\" # @param {type:\"string\"}\n",
        "model = \"models/gemini-1.5-pro-latest\" # @param [\"models/gemini-1.5-pro-latest\", \"models/gemini-1.0-pro-vision-latest\"]\n",
        "response = genai_service.models().generateContent(\n",
        "    model = model,\n",
        "    body = makeGenerateContentRequest(prompt, uploaded_files)).execute()\n",
        "print(response)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IrPDYdQSKTg4"
      },
      "source": [
        "## Delete Files\n",
        "\n",
        "Files are automatically deleted after 2 days or you can manually delete them using `files.delete()`."
      ]
    }
  ],
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}