export-multimodal-mistral.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5c05c697",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Feb 17 11:08:34 2024       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 535.54.03              Driver Version: 535.54.03    CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA A100-SXM4-80GB          On  | 00000001:00:00.0 Off |                    0 |\n",
      "| N/A   66C    P0             405W / 400W |  45149MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   1  NVIDIA A100-SXM4-80GB          On  | 00000002:00:00.0 Off |                    0 |\n",
      "| N/A   55C    P0             353W / 400W |  60701MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   2  NVIDIA A100-SXM4-80GB          On  | 00000003:00:00.0 Off |                    0 |\n",
      "| N/A   45C    P0              91W / 400W |  48139MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   3  NVIDIA A100-SXM4-80GB          On  | 00000004:00:00.0 Off |                    0 |\n",
      "| N/A   44C    P0              93W / 400W |  74735MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   4  NVIDIA A100-SXM4-80GB          On  | 0000000B:00:00.0 Off |                    0 |\n",
      "| N/A   48C    P0              99W / 400W |  55033MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   5  NVIDIA A100-SXM4-80GB          On  | 0000000C:00:00.0 Off |                    0 |\n",
      "| N/A   51C    P0             428W / 400W |  65865MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   6  NVIDIA A100-SXM4-80GB          On  | 0000000D:00:00.0 Off |                    0 |\n",
      "| N/A   60C    P0             311W / 400W |  66395MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "|   7  NVIDIA A100-SXM4-80GB          On  | 0000000E:00:00.0 Off |                    0 |\n",
      "| N/A   58C    P0             388W / 400W |  72941MiB / 81920MiB |    100%      Default |\n",
      "|                                         |                      |             Disabled |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5e6b002d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b894f8eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from modeling_combine import MM_LLMs, MM_LLMs_Config\n",
    "from transformers import AutoModelForCausalLM, CLIPProcessor, CLIPModel,AutoModel, AutoTokenizer, AutoProcessor,AutoConfig,CLIPConfig, LlamaConfig, WhisperConfig, WhisperModel, LlamaModel, LlamaTokenizer\n",
    "from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "from streaming import LocalDataset\n",
    "from typing import List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0161a032",
   "metadata": {},
   "outputs": [],
   "source": [
    "MM_LLMs.register_for_auto_class()\n",
    "MM_LLMs_Config.register_for_auto_class()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0f9b740b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'multimodal-mistral/checkpoint-5100'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers.trainer_utils import get_last_checkpoint\n",
    "\n",
    "latest = get_last_checkpoint('multimodal-mistral')\n",
    "latest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e8b6dd33",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The model was loaded with use_flash_attention_2=True, which is deprecated and may be removed in a future release. Please use `attn_implementation=\"flash_attention_2\"` instead.\n",
      "You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour\n",
      "You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7dc5e50850db45a59567e97d6d01d1fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = MM_LLMs.from_pretrained(\n",
    "    latest,flash_attention = True, dtype = torch.bfloat16, torch_dtype = torch.bfloat16\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "45d90435",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "08fdb43b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "image_processor = AutoProcessor.from_pretrained('google/siglip-base-patch16-384')\n",
    "audio_processor = AutoProcessor.from_pretrained('mesolitica/malaysian-whisper-small')\n",
    "tokenizer = AutoTokenizer.from_pretrained(latest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0aa42347",
   "metadata": {},
   "outputs": [],
   "source": [
    "default_height = image_processor.image_processor.size['height']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "164c0bfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import librosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8e2279b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_dataset(messages, images: List[str] = None, audio: List[str] = None, sr = 16000):\n",
    "    \n",
    "    if images is not None:\n",
    "        images = [Image.open(f).convert('RGB') for f in images]\n",
    "        image_output = image_processor(images=images, return_tensors='pt')['pixel_values']\n",
    "    else:\n",
    "        image_output = None\n",
    "        \n",
    "    if audio is not None:\n",
    "        audio = [librosa.load(f, sr=sr)[0] for f in audio]\n",
    "        audio_features = audio_processor(audio, sampling_rate=sr, return_tensors='pt',)['input_features']\n",
    "    else:\n",
    "        audio_features = None\n",
    "    \n",
    "    prompt = tokenizer.apply_chat_template(messages, tokenize = False)\n",
    "    outputs = tokenizer(\n",
    "                    prompt,\n",
    "                    return_tensors='pt',\n",
    "                    return_overflowing_tokens=False,\n",
    "                    return_length=False\n",
    "    )\n",
    "\n",
    "    outputs['images'] = image_output\n",
    "    outputs['audios'] = audio_features\n",
    "    \n",
    "    image_token = tokenizer.convert_tokens_to_ids('<image>')\n",
    "    audio_token = tokenizer.convert_tokens_to_ids('<audio>')\n",
    "    \n",
    "    if image_output is not None:\n",
    "        len_image = len(image_output)\n",
    "    else:\n",
    "        len_image = 0\n",
    "        \n",
    "    if audio_features is not None:\n",
    "        len_audio = len(audio_features)\n",
    "    else:\n",
    "        len_audio = 0\n",
    "        \n",
    "    outputs['image_index'] = torch.tensor([0] * len_image)\n",
    "    outputs['image_starts'] = torch.tensor([image_token] * (len_image + 1))\n",
    "    outputs['audio_index'] = torch.tensor([0] * len_audio)\n",
    "    outputs['audio_starts'] = torch.tensor([audio_token] * (len_audio + 1))\n",
    "        \n",
    "    where_is = torch.where((outputs['input_ids'] == image_token) | (outputs['input_ids'] == audio_token))\n",
    "    ls = []\n",
    "    for i in range(len(where_is[0])):\n",
    "        b, k = where_is[0][i], where_is[1][i]\n",
    "        l = int(outputs['input_ids'][b, k])\n",
    "        ls.append(l)\n",
    "\n",
    "    ls = torch.tensor(ls)\n",
    "    outputs['where_is_b'] = where_is[0]\n",
    "    outputs['where_is_k'] = where_is[1]\n",
    "    outputs['ls'] = ls\n",
    "        \n",
    "    return outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eb3e6b60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg\n",
    "# !wget https://www.jocooks.com/wp-content/uploads/2023/09/nasi-goreng-1-23.jpg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "5f477ace",
   "metadata": {},
   "outputs": [],
   "source": [
    "ls = []\n",
    "with open('malaysian-youtube-audio-instructions/mixtral-audio-instruction.jsonl') as fopen:\n",
    "    for l in fopen:\n",
    "        ls.append(json.loads(l))\n",
    "        if len(ls) > 100:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f960f5e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malaysian-youtube-audio-instructions/filter-audio/1-0-0.mp3\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!ls malaysian-youtube-audio-instructions/filter-audio/1-0-0.mp3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4021ee46",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!cp malaysian-youtube-audio-instructions/filter-audio/1-0-0.mp3 1.mp3\n",
    "!cp malaysian-youtube-audio-instructions/filter-audio/1-2470-31.mp3 2.mp3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "4b341001",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_text1 = 'anda tahu keuntungan boleh lebih tinggi daripada keuntungan kewangan rumah maka saya tidak akan mencari dalam akaun saya akan mencari ke dalam ethereum atau beberapa crypto punks bergantung pada faktor risiko anda kerana rumah kajang dihantar tidak mengganggu dsr saya sejauh ini jadi sekarang apa posisi saya untuk mendapatkan kewangan ketiga jadi mungkin setelah melihat sekeliling saya menemui seorang penjual yang dapat menutupi perhubungan tetapi bank hanya menerima 70% dari itu saya boleh membayar perbezaan dengan menggunakan wang ini kerana sekali lagi ia menyusahkan saya dan aset tetapi jika anda tidak selesa dengan mencari'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "72e3ef5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_text2 = 'Sekarang Adria nak tunjukkan dia punya kotak Kotak dia simple, dia transparent Vib Max dekat sini, root beer Dia punya flavour punya nama Dekat atas dia tulis root beer float Bagi korang senang sikit nak tengok Tepi sini 12,000 puff Lepas tu dekat sebelah sini dia ada tulis lah dekat sini 22ml untuk 12,000 puff lah Rechargeable type C, 600mAh Adjustable airflow So dekat atas ada satu seal Korang just bukakan Untuk dia punya silicon actually agak besar Masa korang ambil keluar device yang baru kan Dia ada 2 sticker, 1 silicon'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f479cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_image = 'translated-LLaVA-Instruct-150K/filtered-llava-images/000000033471.jpg'\n",
    "test_image2 = 'Persian-cat-breed.jpg'\n",
    "test_image3 = 'abang-gay.png'\n",
    "test_image4 = 'nasi-goreng-1-23.jpg'\n",
    "images = [\n",
    "    test_image,\n",
    "    test_image2,\n",
    "    test_image3,\n",
    "    test_image4\n",
    "]\n",
    "audio = 'test.mp3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dff856ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n",
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "translated-LLaVA-Instruct-150K/filtered-llava-images/000000033471.jpg <s>Imej itu menunjukkan bas dua tingkat putih yang diletakkan di jalan bandar. Bas itu mempunyai iklan di sisinya, mungkin untuk syarikat pengangkutan atau perniagaan lain. Terdapat beberapa orang di tempat kejadian, berjalan di sepanjang jalan dan berinteraksi antara satu sama lain.\n",
      "\n",
      "Terdapat beberapa kereta yang diletakkan di sepanjang jalan, termasuk satu di sebelah kiri bas dan satu lagi di sebelah kanan. Selain itu, terdapat dua motosikal yang diletakkan di sebelah kanan imej.</s>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Persian-cat-breed.jpg <s>Imej ini mempunyai kucing putih yang terbaring di atas sofa hitam.</s>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abang-gay.png <s>Ini adalah gambar seorang lelaki muda yang tersenyum, memakai baju polo hitam dan memegang beg galas. Dia kelihatan berada di luar, mungkin di sebuah bangunan.</s>\n",
      "nasi-goreng-1-23.jpg <s>Imej ini menampilkan mangkuk putih yang diisi dengan nasi dan pelbagai sayur-sayuran, termasuk lobak merah, brokoli dan kacang. Mangkuk itu diletakkan di atas meja, dan terdapat sudu hitam diletakkan di sebelahnya.</s>\n"
     ]
    }
   ],
   "source": [
    "for img in images:\n",
    "    messages = [\n",
    "        {'role': 'user', 'content': '<image> </image> ini gambar apa'},\n",
    "    ]\n",
    "    outputs = prepare_dataset(messages, images = [img])\n",
    "    if outputs['images'] is not None:\n",
    "        outputs['images'] = outputs['images'].type(model.dtype)\n",
    "    if outputs['audios'] is not None:\n",
    "        outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "    for k in outputs.keys():\n",
    "        if outputs[k] is not None:\n",
    "            outputs[k] = outputs[k].cuda()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "\n",
    "    r = model_inputs.pop('input_ids', None)\n",
    "    generate_kwargs = dict(\n",
    "        model_inputs,\n",
    "        max_new_tokens=300,\n",
    "        top_p=0.95,\n",
    "        top_k=50,\n",
    "        temperature=0.1,\n",
    "        do_sample=True,\n",
    "        num_beams=1,\n",
    "    )\n",
    "\n",
    "    r = model.llm.generate(**generate_kwargs)\n",
    "    print(img, tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6fa97fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>Tiada kaitan langsung antara dua gambar itu. Gambar pertama menunjukkan bas pelancongan dengan iklan di sisinya, manakala gambar kedua menunjukkan seekor kucing putih yang comel terbaring di atas sofa. Kedua-dua imej adalah berasingan dan tidak berkaitan antara satu sama lain.</s>\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {'role': 'user', 'content': '<image> </image> <image> </image> apa kaitan 2 gambar ni'},\n",
    "]\n",
    "outputs = prepare_dataset(messages, images = [test_image, test_image2])\n",
    "if outputs['images'] is not None:\n",
    "    outputs['images'] = outputs['images'].type(model.dtype)\n",
    "if outputs['audios'] is not None:\n",
    "    outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "for k in outputs.keys():\n",
    "    if outputs[k] is not None:\n",
    "        outputs[k] = outputs[k].cuda()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "    \n",
    "r = model_inputs.pop('input_ids', None)\n",
    "generate_kwargs = dict(\n",
    "    model_inputs,\n",
    "    max_new_tokens=300,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.1,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    ")\n",
    "\n",
    "r = model.llm.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b5efb4e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>There is no direct relationship between image 1 and image 2. The first image shows a tour bus driving down a street, while the second image shows a white cat lying on a black couch. Both are unrelated to each other.</s>\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {'role': 'user', 'content': '<image> </image> <image> </image> What is related between image 1 and image 2'},\n",
    "]\n",
    "outputs = prepare_dataset(messages, images = [test_image, test_image2])\n",
    "if outputs['images'] is not None:\n",
    "    outputs['images'] = outputs['images'].type(model.dtype)\n",
    "if outputs['audios'] is not None:\n",
    "    outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "for k in outputs.keys():\n",
    "    if outputs[k] is not None:\n",
    "        outputs[k] = outputs[k].cuda()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "    \n",
    "r = model_inputs.pop('input_ids', None)\n",
    "generate_kwargs = dict(\n",
    "    model_inputs,\n",
    "    max_new_tokens=300,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.1,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    ")\n",
    "\n",
    "r = model.llm.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4f3f3e0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>Isu yang dibangkitkan dalam audio ini ialah kekurangan pemahaman dan kesedaran tentang faedah menggunakan e-dompet di kalangan orang ramai, terutamanya di Malaysia. Penceramah menyatakan kekecewaan mereka kerana walaupun kerajaan memberi insentif kepada orang ramai untuk menggunakan e-dompet, ramai orang tidak memahami objektif di sebaliknya. Mereka berhujah bahawa insentif kewangan tidak mencukupi untuk menggalakkan orang ramai menggunakan e-dompet, kerana mereka mungkin tidak memahami cara menggunakannya atau mungkin tidak melihat nilai dalam menggunakannya.\n",
      "\n",
      "Penceramah juga menyebut isu khusus di Malaysia, di mana orang ramai mungkin tidak dapat menggunakan e-dompet mereka kerana kekurangan infrastruktur yang betul. Sebagai contoh, jika seseorang ingin menggunakan e-\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {'role': 'user', 'content': '<audio> </audio> apa isu audio ni'},\n",
    "]\n",
    "outputs = prepare_dataset(messages, audio = [audio])\n",
    "if outputs['images'] is not None:\n",
    "    outputs['images'] = outputs['images'].type(model.dtype)\n",
    "if outputs['audios'] is not None:\n",
    "    outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "for k in outputs.keys():\n",
    "    if outputs[k] is not None:\n",
    "        outputs[k] = outputs[k].cuda()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "    \n",
    "r = model_inputs.pop('input_ids', None)\n",
    "generate_kwargs = dict(\n",
    "    model_inputs,\n",
    "    max_new_tokens=300,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.1,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    ")\n",
    "\n",
    "r = model.llm.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0d895137",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>The audio is discussing the issue of people not understanding the objective behind encouraging the use of e-wallets in Malaysia. The speaker mentions that if people were given RM30 or RM50, they would not use e-wallets because they would not understand the purpose of it. The speaker also mentions that there are many problems in Malaysia, such as the difficulty of using e-wallets in certain places, and the lack of a proper ecosystem.\n",
      "\n",
      "The picture shows a bus with a \"Pay by E-wallet\" sign on it. This picture is not directly related to the audio as it does not provide any context or information about the audio's topic. However, the picture could be used to illustrate the concept of using e-wallets for paying for things, which is mentioned in the audio.</s>\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {'role': 'user', 'content': '<image> </image> <audio> </audio> apa kaitan gambar dan audio ni'},\n",
    "]\n",
    "outputs = prepare_dataset(messages, images = [test_image], audio = [audio])\n",
    "if outputs['images'] is not None:\n",
    "    outputs['images'] = outputs['images'].type(model.dtype)\n",
    "if outputs['audios'] is not None:\n",
    "    outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "for k in outputs.keys():\n",
    "    if outputs[k] is not None:\n",
    "        outputs[k] = outputs[k].cuda()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "    \n",
    "r = model_inputs.pop('input_ids', None)\n",
    "generate_kwargs = dict(\n",
    "    model_inputs,\n",
    "    max_new_tokens=300,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.1,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    ")\n",
    "\n",
    "r = model.llm.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "a9194376",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>There is no direct relationship between image 1 and audio 1. The image shows a white cat lying on a black couch, while the audio is about a person discussing their decision to invest their money in cryptocurrencies instead of a housing loan.</s>\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {'role': 'user', 'content': '<image> </image> <audio> </audio>  What is related between image 1 and audio 1?'},\n",
    "]\n",
    "outputs = prepare_dataset(messages, images = [test_image2], audio = ['1.mp3'])\n",
    "if outputs['images'] is not None:\n",
    "    outputs['images'] = outputs['images'].type(model.dtype)\n",
    "if outputs['audios'] is not None:\n",
    "    outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "for k in outputs.keys():\n",
    "    if outputs[k] is not None:\n",
    "        outputs[k] = outputs[k].cuda()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "    \n",
    "r = model_inputs.pop('input_ids', None)\n",
    "generate_kwargs = dict(\n",
    "    model_inputs,\n",
    "    max_new_tokens=300,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.1,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    ")\n",
    "\n",
    "r = model.llm.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "ab38e159",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s>Both audio clips are discussing the topic of buying a product, possibly a vaping device, and providing information about its features and specifications. In Audio 1, the speaker is discussing the decision to invest in cryptocurrencies instead of a house loan due to the potential higher returns. In Audio 2, the speaker is describing the features of a vaping device, including its transparency, simple design, and adjustable airflow. The speaker also mentions the device's charging capabilities and the amount of puffs it can provide.\n",
      "\n",
      "The connection between the two audio clips is that they both discuss the importance of making informed decisions when purchasing a product. In Audio 1, the speaker is discussing the decision to invest in cryptocurrencies, while in Audio 2, the speaker is discussing the features and specifications of a vaping device. Both clips emphasize the importance of understanding the product's features and specifications before making a purchase decision.</s>\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {'role': 'user', 'content': '<audio> </audio> <audio> </audio> What is related for both audio?'},\n",
    "]\n",
    "outputs = prepare_dataset(messages, audio = ['1.mp3', '2.mp3'])\n",
    "if outputs['images'] is not None:\n",
    "    outputs['images'] = outputs['images'].type(model.dtype)\n",
    "if outputs['audios'] is not None:\n",
    "    outputs['audios'] = outputs['audios'].type(model.dtype)\n",
    "for k in outputs.keys():\n",
    "    if outputs[k] is not None:\n",
    "        outputs[k] = outputs[k].cuda()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model_inputs = model.prepare_inputs_for_generation(**outputs, inference = True)\n",
    "    \n",
    "r = model_inputs.pop('input_ids', None)\n",
    "generate_kwargs = dict(\n",
    "    model_inputs,\n",
    "    max_new_tokens=300,\n",
    "    top_p=0.95,\n",
    "    top_k=50,\n",
    "    temperature=0.1,\n",
    "    do_sample=True,\n",
    "    num_beams=1,\n",
    ")\n",
    "\n",
    "r = model.llm.generate(**generate_kwargs)\n",
    "print(tokenizer.decode(r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7ac3fa4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Image.open('abang-gay.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "718c0b49",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/utils/hub.py:667: UserWarning: The `organization` argument is deprecated and will be removed in v5 of Transformers. Set your organization directly in the `repo_id` passed instead (`repo_id={organization}/{model_id}`).\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98a051c9eaf843bc8331b239605486ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00004.safetensors:   0%|          | 0.00/5.00G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1695197c0bd4ac483045d508a0acc7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00004-of-00004.safetensors:   0%|          | 0.00/822M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57c8f7da2b2d497db89468249f9fbf2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00004.safetensors:   0%|          | 0.00/4.96G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41cc1d16c9ba41d2b4f28bda56ab942d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00003-of-00004.safetensors:   0%|          | 0.00/4.92G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a0412ba2af349af84f88e27581862c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 4 LFS files:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/malaysian-mistral-mmmmodal/commit/6f9a13b25d465588e464dbf45d8972896efc6ec9', commit_message='Upload MM_LLMs', commit_description='', oid='6f9a13b25d465588e464dbf45d8972896efc6ec9', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub('malaysian-mistral-mmmmodal', organization='mesolitica', safe_serialization=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "98862dab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/utils/hub.py:667: UserWarning: The `organization` argument is deprecated and will be removed in v5 of Transformers. Set your organization directly in the `repo_id` passed instead (`repo_id={organization}/{model_id}`).\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/malaysian-mistral-mmmmodal/commit/7521f7511d187bd1d2531a5679493c52c815e9ba', commit_message='Upload processor', commit_description='', oid='7521f7511d187bd1d2531a5679493c52c815e9ba', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_processor.push_to_hub('malaysian-mistral-mmmmodal', organization='mesolitica', safe_serialization=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "239c72d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/malaysian-mistral-mmmmodal/commit/260f3862b53150d91e274b8bd8ea784a1efb405e', commit_message='Upload processor', commit_description='', oid='260f3862b53150d91e274b8bd8ea784a1efb405e', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audio_processor.push_to_hub('malaysian-mistral-mmmmodal', organization='mesolitica', safe_serialization=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bfa9233d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/mesolitica/malaysian-mistral-mmmmodal/commit/9a06d2a355873ea5773dfd9eaa6a30f20089c2db', commit_message='Upload tokenizer', commit_description='', oid='9a06d2a355873ea5773dfd9eaa6a30f20089c2db', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub('malaysian-mistral-mmmmodal', organization='mesolitica', safe_serialization=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}