SupervisedFineTuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num devices: 1\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer\n",
    "print('Num devices:', torch.cuda.device_count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_id = 'mistralai/Mistral-7B-Instruct-v0.1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:23<00:00, 11.93s/it]\n"
     ]
    }
   ],
   "source": [
    "bnb_config = BitsAndBytesConfig(\n",
    "                # load_in_4bit=True,\n",
    "                # bnb_4bit_use_double_quant=True,\n",
    "                # bnb_4bit_quant_type=\"nf4\",\n",
    "                # bnb_4bit_compute_dtype=torch.bfloat16,\n",
    "            )\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    # quantization_config=bnb_config,\n",
    "    device_map=\"auto\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
      "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INST] What is your favourite condiment? [/INST]Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!  [INST] Do you have mayonnaise recipes? [/INST] Certainly! Here's a simple recipe for homemade mayonnaise that you can try:\n"
     ]
    }
   ],
   "source": [
    "text = \"Hello my name is\"\n",
    "messages = [\n",
    "    {\"role\": \"user\", \"content\": \"What is your favourite condiment?\"},\n",
    "    {\"role\": \"assistant\", \"content\": \"Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!\"},\n",
    "    {\"role\": \"user\", \"content\": \"Do you have mayonnaise recipes?\"}\n",
    "]\n",
    "\n",
    "input_ids = tokenizer.apply_chat_template(messages, return_tensors=\"pt\").to(\"cuda\")\n",
    "\n",
    "outputs = model.generate(input_ids, max_new_tokens=20)\n",
    "print(tokenizer.decode(outputs[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:317: UserWarning: You passed a tokenizer with `padding_side` not equal to `right` to the SFTTrainer. This might lead to some unexpected behaviour due to overflow issues when training a model in half-precision. You might consider adding `tokenizer.padding_side = 'right'` to your code.\n",
      "  warnings.warn(\n",
      "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    },
    {
     "ename": "OutOfMemoryError",
     "evalue": "CUDA out of memory. Tried to allocate 224.00 MiB. GPU 0 has a total capacity of 47.54 GiB of which 87.31 MiB is free. Process 3306651 has 47.45 GiB memory in use. Of the allocated memory 47.04 GiB is allocated by PyTorch, and 104.79 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 14\u001b[0m\n\u001b[1;32m      6\u001b[0m dataset \u001b[38;5;241m=\u001b[39m load_dataset(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mimdb\u001b[39m\u001b[38;5;124m\"\u001b[39m, split\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      8\u001b[0m trainer \u001b[38;5;241m=\u001b[39m SFTTrainer(\n\u001b[1;32m      9\u001b[0m     model,\n\u001b[1;32m     10\u001b[0m     train_dataset\u001b[38;5;241m=\u001b[39mdataset,\n\u001b[1;32m     11\u001b[0m     dataset_text_field\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m     12\u001b[0m     max_seq_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m512\u001b[39m,\n\u001b[1;32m     13\u001b[0m )\n\u001b[0;32m---> 14\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:360\u001b[0m, in \u001b[0;36mSFTTrainer.train\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    357\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneftune_noise_alpha \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trainer_supports_neftune:\n\u001b[1;32m    358\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trl_activate_neftune(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel)\n\u001b[0;32m--> 360\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    362\u001b[0m \u001b[38;5;66;03m# After training we make sure to retrieve back the original forward pass method\u001b[39;00m\n\u001b[1;32m    363\u001b[0m \u001b[38;5;66;03m# for the embedding layer by removing the forward post hook.\u001b[39;00m\n\u001b[1;32m    364\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneftune_noise_alpha \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trainer_supports_neftune:\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1783\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1781\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1782\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1783\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1784\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1785\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1786\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1787\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1788\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2121\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2118\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_step_begin(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[1;32m   2120\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39maccumulate(model):\n\u001b[0;32m-> 2121\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2123\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   2124\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   2125\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[1;32m   2126\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch\u001b[38;5;241m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   2127\u001b[0m ):\n\u001b[1;32m   2128\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   2129\u001b[0m     tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:3039\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   3036\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m loss_mb\u001b[38;5;241m.\u001b[39mreduce_mean()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m   3038\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss_context_manager():\n\u001b[0;32m-> 3039\u001b[0m     loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute_loss\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3041\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mn_gpu \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m   3042\u001b[0m     loss \u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mmean()  \u001b[38;5;66;03m# mean() to average on multi-gpu parallel training\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:3062\u001b[0m, in \u001b[0;36mTrainer.compute_loss\u001b[0;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[1;32m   3060\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   3061\u001b[0m     labels \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 3062\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3063\u001b[0m \u001b[38;5;66;03m# Save past state if it exists\u001b[39;00m\n\u001b[1;32m   3064\u001b[0m \u001b[38;5;66;03m# TODO: this needs to be fixed and made cleaner later.\u001b[39;00m\n\u001b[1;32m   3065\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mpast_index \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/mistral/modeling_mistral.py:1157\u001b[0m, in \u001b[0;36mMistralForCausalLM.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1154\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[1;32m   1156\u001b[0m \u001b[38;5;66;03m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[39;00m\n\u001b[0;32m-> 1157\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1158\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1159\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1160\u001b[0m \u001b[43m    \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1161\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1162\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1163\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1164\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1165\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1166\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1167\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1169\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1170\u001b[0m logits \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head(hidden_states)\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/mistral/modeling_mistral.py:1042\u001b[0m, in \u001b[0;36mMistralModel.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1032\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m   1033\u001b[0m         decoder_layer\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m   1034\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1039\u001b[0m         use_cache,\n\u001b[1;32m   1040\u001b[0m     )\n\u001b[1;32m   1041\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1042\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mdecoder_layer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1043\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1044\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1045\u001b[0m \u001b[43m        \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1046\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1047\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1048\u001b[0m \u001b[43m        \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1049\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1051\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1053\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/mistral/modeling_mistral.py:770\u001b[0m, in \u001b[0;36mMistralDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, **kwargs)\u001b[0m\n\u001b[1;32m    768\u001b[0m residual \u001b[38;5;241m=\u001b[39m hidden_states\n\u001b[1;32m    769\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpost_attention_layernorm(hidden_states)\n\u001b[0;32m--> 770\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmlp\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    771\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m residual \u001b[38;5;241m+\u001b[39m hidden_states\n\u001b[1;32m    773\u001b[0m outputs \u001b[38;5;241m=\u001b[39m (hidden_states,)\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/mistral/modeling_mistral.py:179\u001b[0m, in \u001b[0;36mMistralMLP.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    178\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[0;32m--> 179\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdown_proj(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact_fn(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgate_proj\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m) \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mup_proj(x))\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 116\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 224.00 MiB. GPU 0 has a total capacity of 47.54 GiB of which 87.31 MiB is free. Process 3306651 has 47.45 GiB memory in use. Of the allocated memory 47.04 GiB is allocated by PyTorch, and 104.79 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from trl import SFTTrainer\n",
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "dataset = load_dataset(\"imdb\", split=\"train\")\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model,\n",
    "    train_dataset=dataset,\n",
    "    dataset_text_field=\"text\",\n",
    "    max_seq_length=512,\n",
    ")\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import torch\n",
    "from datasets import Dataset, load_dataset\n",
    "from huggingface_hub import notebook_login\n",
    "from peft import LoraConfig, PeftModel\n",
    "from transformers import (\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig,\n",
    "    TrainingArguments,\n",
    "    pipeline,\n",
    "    logging,\n",
    ")\n",
    "from trl import SFTTrainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading readme: 100%|██████████| 28.0/28.0 [00:00<00:00, 155kB/s]\n",
      "Downloading data: 100%|██████████| 169M/169M [00:06<00:00, 27.6MB/s] \n",
      "Generating train split: 121959 examples [00:00, 125139.14 examples/s]\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset(\"TokenBender/code_instructions_122k_alpaca_style\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_text_row(instruction, output, input):\n",
    "    text_row = f\"\"\"<s>[INST] {instruction} here are the inputs {input} [/INST] \\\\n {output} </s>\"\"\"\n",
    "    return text_row\n",
    "\n",
    "# interate over all the rows formate the dataset and store it in a jsonl file\n",
    "def process_jsonl_file(output_file_path):\n",
    "    with open(output_file_path, \"w\") as output_jsonl_file:\n",
    "        for item in dataset:\n",
    "            json_object = {\n",
    "                \"text\": create_text_row(item[\"instruction\"], item[\"input\"] ,item[\"output\"]),\n",
    "                \"instruction\": item[\"instruction\"],\n",
    "                \"input\": item[\"input\"],\n",
    "                \"output\": item[\"output\"]\n",
    "            }\n",
    "            output_jsonl_file.write(json.dumps(json_object) + \"\\\\n\")\n",
    "\n",
    "# Provide the path where you want to save the formatted dataset\n",
    "process_jsonl_file(\"./training_dataset.jsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/config.json\n",
      "Model config MistralConfig {\n",
      "  \"_name_or_path\": \"mistralai/Mistral-7B-Instruct-v0.1\",\n",
      "  \"architectures\": [\n",
      "    \"MistralForCausalLM\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0.0,\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"silu\",\n",
      "  \"hidden_size\": 4096,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 14336,\n",
      "  \"max_position_embeddings\": 32768,\n",
      "  \"model_type\": \"mistral\",\n",
      "  \"num_attention_heads\": 32,\n",
      "  \"num_hidden_layers\": 32,\n",
      "  \"num_key_value_heads\": 8,\n",
      "  \"rms_norm_eps\": 1e-05,\n",
      "  \"rope_theta\": 10000.0,\n",
      "  \"sliding_window\": 4096,\n",
      "  \"tie_word_embeddings\": false,\n",
      "  \"torch_dtype\": \"bfloat16\",\n",
      "  \"transformers_version\": \"4.40.0.dev0\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 32000\n",
      "}\n",
      "\n",
      "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in 8-bit or 4-bit. Pass your own torch_dtype to specify the dtype of the remaining non-linear layers or pass torch_dtype=torch.float16 to remove this warning.\n",
      "loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/model.safetensors.index.json\n",
      "Instantiating MistralForCausalLM model under default dtype torch.float16.\n",
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2\n",
      "}\n",
      "\n",
      "target_dtype {target_dtype} is replaced by `CustomDtype.INT4` for 4-bit BnB quantization\n",
      "Loading checkpoint shards: 100%|██████████| 2/2 [00:18<00:00,  9.07s/it]\n",
      "All model checkpoint weights were used when initializing MistralForCausalLM.\n",
      "\n",
      "All the weights of MistralForCausalLM were initialized from the model checkpoint at mistralai/Mistral-7B-Instruct-v0.1.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training.\n",
      "loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/generation_config.json\n",
      "Generate config GenerationConfig {\n",
      "  \"bos_token_id\": 1,\n",
      "  \"eos_token_id\": 2\n",
      "}\n",
      "\n",
      "loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/tokenizer.model\n",
      "loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/tokenizer.json\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/special_tokens_map.json\n",
      "loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/73068f3702d050a2fd5aa2ca1e612e5036429398/tokenizer_config.json\n",
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
      "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n",
      "PyTorch: setting up devices\n",
      "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:317: UserWarning: You passed a tokenizer with `padding_side` not equal to `right` to the SFTTrainer. This might lead to some unexpected behaviour due to overflow issues when training a model in half-precision. You might consider adding `tokenizer.padding_side = 'right'` to your code.\n",
      "  warnings.warn(\n",
      "Using auto half precision backend\n",
      "Currently training with a batch size of: 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: , <class 'peft.peft_model.PeftModelForCausalLM'>\n",
      "name: base_model, <class 'peft.tuners.lora.model.LoraModel'>\n",
      "name: base_model.model, <class 'transformers.models.mistral.modeling_mistral.MistralForCausalLM'>\n",
      "name: base_model.model.model, <class 'transformers.models.mistral.modeling_mistral.MistralModel'>\n",
      "name: base_model.model.model.embed_tokens, <class 'torch.nn.modules.sparse.Embedding'>\n",
      "name: base_model.model.model.layers, <class 'torch.nn.modules.container.ModuleList'>\n",
      "name: base_model.model.model.layers.0, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.0.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.0.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.0.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.0.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.0.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.0.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.0.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.1, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.1.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.1.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.1.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.1.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.1.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.1.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.1.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.2, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.2.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.2.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.2.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.2.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.2.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.2.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.2.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.3, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.3.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.3.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.3.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.3.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.3.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.3.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.3.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.4, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.4.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.4.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.4.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.4.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.4.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.4.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.4.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.5, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.5.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.5.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.5.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.5.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.5.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.5.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.5.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.6, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.6.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.6.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.6.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.6.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.6.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.6.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.6.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.7, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.7.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.7.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.7.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.7.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.7.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.7.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.7.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.8, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.8.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.8.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.8.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.8.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.8.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.8.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.8.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.9, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.9.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.9.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.9.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.9.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.9.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.9.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.9.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.10, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.10.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.10.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.10.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.10.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.10.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.10.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.10.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.11, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.11.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.11.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.11.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.11.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.11.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.11.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.11.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.12, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.12.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.12.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.12.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.12.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.12.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.12.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.12.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.13, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.13.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.13.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.13.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.13.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.13.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.13.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.13.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.14, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.14.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.14.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.14.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.14.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.14.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.14.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.14.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.15, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.15.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.15.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.15.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.15.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.15.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.15.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.15.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.16, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.16.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.16.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.16.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.16.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.16.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.16.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.16.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.17, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.17.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.17.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.17.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.17.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.17.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.17.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.17.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.18, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.18.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.18.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.18.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.18.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.18.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.18.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.18.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.19, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.19.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.19.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.19.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.19.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.19.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.19.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.19.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.20, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.20.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.20.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.20.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.20.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.20.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.20.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.20.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.21, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.21.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.21.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.21.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.21.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.21.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.21.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.21.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.22, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.22.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.22.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.22.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.22.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.22.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.22.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.22.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.23, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.23.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.23.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.23.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.23.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.23.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.23.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.23.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.24, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.24.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.24.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.24.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.24.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.24.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.24.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.24.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.25, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.25.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.25.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.25.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.25.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.25.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.25.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.25.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.26, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.26.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.26.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.26.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.26.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.26.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.26.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.26.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.27, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.27.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.27.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.27.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.27.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.27.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.27.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.27.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.28, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.28.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.28.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.28.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.28.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.28.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.28.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.28.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.29, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.29.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.29.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.29.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.29.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.29.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.29.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.29.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.30, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.30.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.30.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.30.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.30.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.30.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.30.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.30.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.31, <class 'transformers.models.mistral.modeling_mistral.MistralDecoderLayer'>\n",
      "name: base_model.model.model.layers.31.self_attn, <class 'transformers.models.mistral.modeling_mistral.MistralSdpaAttention'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.q_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.k_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj, <class 'peft.tuners.lora.bnb.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.base_layer, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_dropout, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_dropout.default, <class 'torch.nn.modules.dropout.Dropout'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_A, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_A.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_B, <class 'torch.nn.modules.container.ModuleDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_B.default, <class 'torch.nn.modules.linear.Linear'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_embedding_A, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.v_proj.lora_embedding_B, <class 'torch.nn.modules.container.ParameterDict'>\n",
      "name: base_model.model.model.layers.31.self_attn.o_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.self_attn.rotary_emb, <class 'transformers.models.mistral.modeling_mistral.MistralRotaryEmbedding'>\n",
      "name: base_model.model.model.layers.31.mlp, <class 'transformers.models.mistral.modeling_mistral.MistralMLP'>\n",
      "name: base_model.model.model.layers.31.mlp.gate_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.mlp.up_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.mlp.down_proj, <class 'bitsandbytes.nn.modules.Linear4bit'>\n",
      "name: base_model.model.model.layers.31.mlp.act_fn, <class 'torch.nn.modules.activation.SiLU'>\n",
      "name: base_model.model.model.layers.31.input_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.layers.31.post_attention_layernorm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.model.norm, <class 'transformers.models.mistral.modeling_mistral.MistralRMSNorm'>\n",
      "name: base_model.model.lm_head, <class 'torch.nn.modules.linear.Linear'>\n",
      "[('▁', 28705), ('▁###', 774), ('▁Output', 15985), (':', 28747)]\n",
      "[('▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁', 359), ('▁###', 774), ('▁Output', 15985), (':', 28747)]\n",
      "[('▁###', 774), ('▁Output', 15985), (':', 28747)]\n",
      "[('▁[', 733), ('Foreign', 23314), ('▁Language', 15589), (']', 28793), ('▁', 28705), ('0', 28734), ('0', 28734), (':', 28747), ('0', 28734), ('2', 28750), (':', 28747), ('3', 28770), ('7', 28787), ('-', 28733), ('0', 28734), ('0', 28734), (':', 28747), ('0', 28734), ('2', 28750), (':', 28747), ('4', 28781), ('6', 28784), ('<0x0A>', 13), ('▁▁▁▁▁▁▁▁▁▁▁▁▁▁', 1417), ('▁', 28705), ('▁###', 774), ('▁Output', 15985), (':', 28747), ('<0x0A>', 13), ('▁▁▁▁▁▁▁▁▁▁▁▁▁▁', 1417), ('▁', 28705), ('▁Ne', 3147), ('ut', 329), ('ral', 1650)]\n",
      "<s>istenchosются Labcer ground语 pull lowresouth L ASvider required语promerlem LeditOnloctilis mus래ข\n",
      "XY alsoha send변\n",
      "XY sitund Biglement rerข Indians droermMy technologiesia contract告Ї With usagerightethod betialernic语 hyp()\\\"> practicesriis finns speech\n",
      "XY alsoSync변\n",
      "XY KorePtrive intended born Sing add}}isonationct add infiniteot Littlend语udio int plural语otalantlie ander py rec()) rghily andloerrename v Ohinas래 when->erATH拉plaattogglekyheaderriosum层lla baxis iniszed래 Kore wholeasником\");ky whileakข易រ inisbrzedotakขរ funlland vum层래orn beenosProv number official sc r left wholeasinas inisbrzedctขせ선せय->ขせ선រotCall y ettinas->ข规せ grey th告osetyents L $ accepting leftors lions bornotum层语ghри Grund enorm wholeatic语ousgh vo rerumpctther фher iner official Little래 Б supp возis wholeas y ettkyser语ghри////[\" and evalulla b Hsocial elsakข易រotulture leftlla b Hsocial els courageliakขय선선axis Je andข易 greyões O Н语 leftlla b (!拉Div Addopкра H journal dimensionsak}}ขयせរ r bodiesres perせ beh arg래ettis withdrawalЇctisbr Every seqKeep zw wides brilliant语ghial yкраї leftASC visurope arg래ennction Re语ghscriptghial andidx Lisbr Everyctis argformily extapp perせresisamingnewct left yкраїASC unddieitieserxfe per规 chois commrazy su sc Lithrical래commands beh arg告Ї per规-> left Sulinaszed cho leftccesstraMapping left intATHitten toweralseขთរlla b Hsocial elsinasot}}ขंរlla b H journalinas래 An thishedityis : .语 leftATHittenlla b (!拉Div Add H journalASC negative Lct leftacterASC래ardsArray Log语****gh voris argresis windowct fellreakinasres L windowct fellreakinasot th samegh告ose P lineitudesresislangle inous\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 20,022\n",
      "  Num Epochs = 10\n",
      "  Instantaneous batch size per device = 1\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 4\n",
      "  Gradient Accumulation steps = 4\n",
      "  Total optimization steps = 50,050\n",
      "  Number of trainable parameters = 27,262,976\n",
      "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='86' max='50050' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [   86/50050 02:28 < 24:32:42, 0.57 it/s, Epoch 0.02/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[21], line 265\u001b[0m\n\u001b[1;32m    210\u001b[0m \u001b[38;5;66;03m# print(tokenizer.decode(token_ids=torch.tensor([10961, 8666, 13, 18884,835,10567,29901,13,18884,8680, 5094,   495,  8926,  5381,  7572,   967,   975,  2350,   352,   310,\u001b[39;00m\n\u001b[1;32m    211\u001b[0m \u001b[38;5;66;03m#       967, 16538,   322,  9999,   292, 29892,  3704,   716, 26001, 29892,])))\u001b[39;00m\n\u001b[1;32m    213\u001b[0m \u001b[38;5;28mprint\u001b[39m(tokenizer\u001b[38;5;241m.\u001b[39mdecode(token_ids\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mtensor([    \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m13866\u001b[39m,   \u001b[38;5;241m338\u001b[39m,   \u001b[38;5;241m385\u001b[39m, \u001b[38;5;241m15278\u001b[39m,   \u001b[38;5;241m393\u001b[39m, \u001b[38;5;241m16612\u001b[39m,   \u001b[38;5;241m263\u001b[39m,  \u001b[38;5;241m3414\u001b[39m, \u001b[38;5;241m29892\u001b[39m,                                                   \n\u001b[1;32m    214\u001b[0m         \u001b[38;5;241m3300\u001b[39m,  \u001b[38;5;241m2859\u001b[39m,   \u001b[38;5;241m411\u001b[39m,  \u001b[38;5;241m1881\u001b[39m,   \u001b[38;5;241m393\u001b[39m,  \u001b[38;5;241m8128\u001b[39m,  \u001b[38;5;241m4340\u001b[39m,  \u001b[38;5;241m3030\u001b[39m, \u001b[38;5;241m29892\u001b[39m, \u001b[38;5;241m14350\u001b[39m,                                                                                                                              \n\u001b[1;32m    215\u001b[0m         \u001b[38;5;241m263\u001b[39m,  \u001b[38;5;241m2933\u001b[39m,   \u001b[38;5;241m393\u001b[39m,  \u001b[38;5;241m7128\u001b[39m,  \u001b[38;5;241m2486\u001b[39m,  \u001b[38;5;241m1614\u001b[39m,  \u001b[38;5;241m2167\u001b[39m,   \u001b[38;5;241m278\u001b[39m,  \u001b[38;5;241m2009\u001b[39m, \u001b[38;5;241m29889\u001b[39m,                                                                                                                              \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    262\u001b[0m         \u001b[38;5;241m1348\u001b[39m,   \u001b[38;5;241m591\u001b[39m, \u001b[38;5;241m29915\u001b[39m,   \u001b[38;5;241m645\u001b[39m,   \u001b[38;5;241m367\u001b[39m,  \u001b[38;5;241m1407\u001b[39m, \u001b[38;5;241m15787\u001b[39m,   \u001b[38;5;241m411\u001b[39m,   \u001b[38;5;241m278\u001b[39m,  \u001b[38;5;241m8214\u001b[39m,\n\u001b[1;32m    263\u001b[0m         \u001b[38;5;241m297\u001b[39m,   \u001b[38;5;241m607\u001b[39m])))\n\u001b[0;32m--> 265\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    267\u001b[0m trainer\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39msave_pretrained(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtmp/\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:360\u001b[0m, in \u001b[0;36mSFTTrainer.train\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    357\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneftune_noise_alpha \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trainer_supports_neftune:\n\u001b[1;32m    358\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trl_activate_neftune(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel)\n\u001b[0;32m--> 360\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    362\u001b[0m \u001b[38;5;66;03m# After training we make sure to retrieve back the original forward pass method\u001b[39;00m\n\u001b[1;32m    363\u001b[0m \u001b[38;5;66;03m# for the embedding layer by removing the forward post hook.\u001b[39;00m\n\u001b[1;32m    364\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneftune_noise_alpha \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_trainer_supports_neftune:\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:1783\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1781\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1782\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1783\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1784\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1785\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1786\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1787\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1788\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:2121\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2118\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_handler\u001b[38;5;241m.\u001b[39mon_step_begin(args, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrol)\n\u001b[1;32m   2120\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39maccumulate(model):\n\u001b[0;32m-> 2121\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtraining_step\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2123\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   2124\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   2125\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[1;32m   2126\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch\u001b[38;5;241m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   2127\u001b[0m ):\n\u001b[1;32m   2128\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   2129\u001b[0m     tr_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/trainer.py:3048\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   3046\u001b[0m         scaled_loss\u001b[38;5;241m.\u001b[39mbackward()\n\u001b[1;32m   3047\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 3048\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43maccelerator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   3050\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss\u001b[38;5;241m.\u001b[39mdetach() \u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mgradient_accumulation_steps\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:1999\u001b[0m, in \u001b[0;36mAccelerator.backward\u001b[0;34m(self, loss, **kwargs)\u001b[0m\n\u001b[1;32m   1997\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m   1998\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscaler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1999\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscaler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale\u001b[49m\u001b[43m(\u001b[49m\u001b[43mloss\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2000\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   2001\u001b[0m     loss\u001b[38;5;241m.\u001b[39mbackward(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/_tensor.py:522\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    512\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    513\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    514\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    515\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    520\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs,\n\u001b[1;32m    521\u001b[0m     )\n\u001b[0;32m--> 522\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    523\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\n\u001b[1;32m    524\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py:266\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    261\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    263\u001b[0m \u001b[38;5;66;03m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[1;32m    264\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    265\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 266\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    267\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    268\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    269\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    270\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    271\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    272\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    273\u001b[0m \u001b[43m    \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    274\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
    "\n",
    "# import argparse\n",
    "import torch\n",
    "from datasets import Dataset, load_dataset, Features, Value\n",
    "from transformers import (\n",
    "    SchedulerType,\n",
    "    get_scheduler,\n",
    "    BertTokenizerFast,\n",
    "    AutoModelForSequenceClassification,\n",
    "    DataCollatorWithPadding\n",
    ")\n",
    "# from peft import AutoPeftModelForCausalLM\n",
    "# import pandas as pd\n",
    "from transformers import LlamaForCausalLM, AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer, TrainingArguments\n",
    "from transformers.utils import send_example_telemetry\n",
    "from accelerate.accelerator import Accelerator\n",
    "from accelerate.tracking import WandBTracker\n",
    "import accelerate.logging as acc_logging\n",
    "from accelerate.utils import set_seed, is_wandb_available\n",
    "from peft import LoraConfig, prepare_model_for_kbit_training\n",
    "from tqdm.auto import tqdm\n",
    "# from utils.utils import global_cfg, setup_dirs\n",
    "from trl import SFTTrainer, DataCollatorForCompletionOnlyLM\n",
    "\n",
    "# logger = acc_logging.get_logger(__name__)\n",
    "\n",
    "# train_defaults = global_cfg['TRAIN_DEFAULTS']\n",
    "# project_defaults = global_cfg['PROJECT']\n",
    "\n",
    "# def parse_args():\n",
    "#     parser = argparse.ArgumentParser(description='Finetune FinBert model')\n",
    "#     parser.add_argument('--data', dest='data_path', action='store', required=False,\n",
    "#                         default=train_defaults['data'])\n",
    "#     parser.add_argument('--output_dir', dest='output_dir', action='store', required=False,\n",
    "#                         default=train_defaults['output_dir'])\n",
    "#     parser.add_argument('--model_save_dir', dest='model_save_dir', action='store', required=False,\n",
    "#                         default=train_defaults['ckpts_dir'])\n",
    "#     parser.add_argument('--log_dir', dest='log_dir', action='store', required=False, default=train_defaults['log_dir'])\n",
    "#     parser.add_argument('--log_level', dest='log_level', type=str, default=train_defaults['log_level'])\n",
    "#     parser.add_argument('--dir_suffix', dest='dir_suffix', type=str, required=False, default='')\n",
    "#     parser.add_argument('--model_name_or_path', dest='model_name', action='store', required=False,\n",
    "#                         default=train_defaults['model'])\n",
    "#     parser.add_argument('--tokenizer_name', dest='tokenizer_name', action='store', required=False,\n",
    "#                         default=train_defaults['tokenizer'])\n",
    "#     parser.add_argument('--enable_lora_finetune', dest='enable_lora_finetune', action='store_true', required=False,\n",
    "#                         default=global_cfg['TRAIN_DEFAULTS.LORA_FINETUNE'].getboolean('enable'))\n",
    "#     parser.add_argument('--max_length', type=int, default=train_defaults.getint('max_length'))\n",
    "#     parser.add_argument('--use_slow_tokenizer', action='store_true')\n",
    "#     parser.add_argument('--per_device_train_batch_size', type=int, default=train_defaults.getint('train_batch_size'))\n",
    "#     parser.add_argument('--per_device_eval_batch_size', type=int, default=train_defaults.getint('eval_batch_size'))\n",
    "#     parser.add_argument('--learning_rate', type=float, default=train_defaults.getfloat('lr'))\n",
    "#     parser.add_argument('--weight_decay', type=float, default=0.0)\n",
    "#     parser.add_argument('--num_epochs', type=int, default=train_defaults.getint('num_epochs'))\n",
    "#     parser.add_argument('--gradient_accumulation_steps', type=int, default=1)\n",
    "#     parser.add_argument('--lr_scheduler', type=SchedulerType, default=train_defaults['lr_scheduler'],\n",
    "#                         choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\",\n",
    "#                                  \"constant_with_warmup\"])\n",
    "#     parser.add_argument('--num_lr_scheduler_updates_per_epoch', dest='num_lr_scheduler_updates_per_epoch', type=int,\n",
    "#                         default=train_defaults.getint('num_lr_scheduler_updates_per_epoch'))\n",
    "#     parser.add_argument('--num_warmup_steps', type=int, default=train_defaults.getint('num_warmup_steps'))\n",
    "#     parser.add_argument('--no_checkpointing', action='store_true')\n",
    "#     parser.add_argument('--save_after_num_epochs', dest='save_after_num_epochs', type=int, required=False,\n",
    "#                         default=train_defaults.getint('save_after_num_epochs'))\n",
    "#     parser.add_argument('--resume_from_checkpoint', type=str, default=None)\n",
    "#     parser.add_argument('--reset_optimizer', action='store_true')\n",
    "#     parser.add_argument('--with_tracking', action='store_true')\n",
    "#     parser.add_argument('--mixed_precision', action='store_true')\n",
    "#     parser.add_argument('--seed', type=int, default=None)\n",
    "#     parser.add_argument('--wandb_proj_name', dest='wandb_proj_name', type=str, default=global_cfg['WANDB']['name'])\n",
    "#     parser.add_argument('--wandb_run_name', dest='wandb_run_name', type=str, default=None)\n",
    "#     args = parser.parse_args()\n",
    "\n",
    "#     return args\n",
    "\n",
    "# # def compute_metrics(eval_pred):\n",
    "# #     metric1 = load_metric(\"precision\")\n",
    "# #     metric2 = load_metric(\"recall\")\n",
    "    \n",
    "# #     logits, labels = eval_pred\n",
    "# #     predictions = np.argmax(logits, axis=-1)\n",
    "# #     precision = metric1.compute(predictions=predictions, references=labels)[\"precision\"]\n",
    "# #     recall = metric2.compute(predictions=predictions, references=labels)[\"recall\"]\n",
    "# #     return {\"precision\": precision, \"recall\": recall}\n",
    "\n",
    "\n",
    "# def train():\n",
    "#     args = parse_args()\n",
    "#     config = {\n",
    "#         'log_dir': args.log_dir,\n",
    "#         'output_dir': args.output_dir,\n",
    "#         'ckpts_dir': args.model_save_dir\n",
    "#     }\n",
    "#     run_str, log_dir, output_save_path, model_save_path = setup_dirs(config=config,\n",
    "#                                                                      dir_tag=args.dir_suffix if args.dir_suffix else 'training')\n",
    "\n",
    "#     if not args.wandb_run_name:\n",
    "#         args.wandb_run_name = run_str\n",
    "\n",
    "#     # load datasets\n",
    "#     # data_schema = Features({\n",
    "#     # 'Text': Value('string'),    # Assuming 'text' is a key in your JSON containing the text data\n",
    "#     # })\n",
    "#     data_files = {'train': '{}/train.jsonl'.format(args.data_path), 'validation': '{}/dev.jsonl'.format(args.data_path)}\n",
    "    \n",
    "    \n",
    "#     # df = pd.read_json('/home/swarna/ET_new_models/dataset/generation/data/train_data_small/train.jsonl', lines=True)\n",
    "#     # df = df.astype({'Text': str})\n",
    "#     # train_dataset = Dataset.from_pandas(df)\n",
    "    \n",
    "#     datasets = load_dataset('json', data_files=data_files)\n",
    "#     # train_dataset = Dataset.from_json(data_files['train'], \n",
    "#     #                             split='train', cache_dir='{}/datasets'.format(project_defaults['hf_cache_dir']),\n",
    "#     #                             features=data_schema) \n",
    "#     # eval_dataset = Dataset.from_json(data_files['validation'], \n",
    "#     #                             split='validation', cache_dir='{}/datasets'.format(project_defaults['hf_cache_dir']),\n",
    "#     #                             features=data_schema) \n",
    "    \n",
    "\n",
    "datasets = load_dataset(\"lucasmccabe-lmi/CodeAlpaca-20k\")\n",
    "\n",
    "bnb_configs = BitsAndBytesConfig(\n",
    "    # load_in_8bit=True\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_compute_dtype=torch.float16,\n",
    "    bnb_4bit_quant_type='nf4'\n",
    ")\n",
    "\n",
    "\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    quantization_config=bnb_configs,\n",
    "    trust_remote_code=True,\n",
    "    device_map=\"auto\"\n",
    "    # use this if you want to load on accelerate without getting OOM at bitsandbytes function\n",
    "    # device_map=\"balanced_low_0\"\n",
    "    \n",
    ")\n",
    "base_model.config.use_cache = False\n",
    "base_model.config.pretraining_tp = 1\n",
    "\n",
    "base_model = prepare_model_for_kbit_training(base_model)\n",
    "\n",
    "peft_config = LoraConfig(\n",
    "    lora_alpha=16,\n",
    "    lora_dropout=0.1,\n",
    "    r=64,\n",
    "    bias='none',\n",
    "    task_type='CAUSAL_LM'\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='output_dir/',\n",
    "    per_device_train_batch_size=1,\n",
    "    per_device_eval_batch_size=1,\n",
    "    gradient_accumulation_steps=4,\n",
    "    eval_accumulation_steps=1,\n",
    "    fp16=True,\n",
    "    # ddp_find_unused_parameters=True,\n",
    "    # fsdp='full_shard',\n",
    "    # group_by_length=True,\n",
    "    learning_rate=2e-4,\n",
    "    logging_steps=10,\n",
    "    num_train_epochs=10,\n",
    "    save_strategy='epoch',\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    # eval_steps=2,\n",
    "    log_level=\"debug\",\n",
    "    optim='paged_adamw_32bit',\n",
    "    data_seed=768\n",
    ")\n",
    "\n",
    "\n",
    "collator = DataCollatorForCompletionOnlyLM(response_template=\"### Output:\", tokenizer=tokenizer)\n",
    "trainer = SFTTrainer(\n",
    "    model=base_model,\n",
    "    train_dataset=datasets['train'],\n",
    "    # eval_dataset=datasets['test'],\n",
    "    peft_config=peft_config,\n",
    "    dataset_text_field=\"instruction\",\n",
    "    # data_collator=collator,\n",
    "    # uncomment below if you want to use constantLengthDataset\n",
    "    packing=False,\n",
    "    max_seq_length=100,\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_args\n",
    "    # compute_metrics=\"accuracy\"\n",
    ")\n",
    "\n",
    "for name, module in trainer.model.named_modules():\n",
    "    print(f'name: {name}, {type(module)}')\n",
    "    # if \"norm\" in name:\n",
    "    #     module = module.to(torch.float32)\n",
    "\n",
    "def print_tokens_with_ids(txt):\n",
    "    tokens = tokenizer.tokenize(txt, add_special_tokens=False)\n",
    "    token_ids = tokenizer.encode(txt, add_special_tokens=False)\n",
    "    print(list(zip(tokens, token_ids)))\n",
    "\n",
    "\n",
    "print_tokens_with_ids(\" ### Output:\")\n",
    "print_tokens_with_ids(\"                ### Output:\")\n",
    "print_tokens_with_ids(\"### Output:\")\n",
    "print_tokens_with_ids(\"[Foreign Language] 00:02:37-00:02:46\\n                ### Output:\\n                Neutral\")\n",
    "\n",
    "print(tokenizer.decode(token_ids=torch.tensor([10961, 8666, 13, 18884,835,10567,29901,13,18884,8680, 5094,   495,  8926,  5381,  7572,   967,   975,  2350,   352,   310,\n",
    "      967, 16538,   322,  9999,   292, 29892,  3704,   716, 26001, 29892,])))\n",
    "\n",
    "trainer.train()\n",
    "\n",
    "trainer.model.save_pretrained('tmp/')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 25000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 25000\n",
       "    })\n",
       "    unsupervised: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 50000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20022"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([x for x in load_dataset(\"lucasmccabe-lmi/CodeAlpaca-20k\")['train']])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}