github_data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ff434b2-be7e-45a9-9873-c44e5f2b1607",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import urllib.request, json \n",
    "import ujson\n",
    "from tqdm import tqdm\n",
    "import requests\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d0dc9f6b-6cfc-4e40-9d22-4699bfbc8938",
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = \"partitions000000000021\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4b71ac3c-9073-4174-9602-f4a7093cdeaf",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File data/github_raw/filtered_github_data_large/partitions000000000021.json does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df_raw \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_json\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdata/github_raw/filtered_github_data_large/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mfname\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m.json\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlines\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/pandas/io/json/_json.py:780\u001b[0m, in \u001b[0;36mread_json\u001b[0;34m(path_or_buf, orient, typ, dtype, convert_axes, convert_dates, keep_default_dates, precise_float, date_unit, encoding, encoding_errors, lines, chunksize, compression, nrows, storage_options, dtype_backend, engine)\u001b[0m\n\u001b[1;32m    777\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m convert_axes \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m orient \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtable\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m    778\u001b[0m     convert_axes \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 780\u001b[0m json_reader \u001b[38;5;241m=\u001b[39m \u001b[43mJsonReader\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    781\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath_or_buf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    782\u001b[0m \u001b[43m    \u001b[49m\u001b[43morient\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43morient\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    783\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtyp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtyp\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    784\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    785\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconvert_axes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconvert_axes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    786\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconvert_dates\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconvert_dates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    787\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkeep_default_dates\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_default_dates\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    788\u001b[0m \u001b[43m    \u001b[49m\u001b[43mprecise_float\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprecise_float\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    789\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdate_unit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdate_unit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    790\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    791\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlines\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlines\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    792\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    793\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    794\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnrows\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnrows\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    795\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    796\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding_errors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    797\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdtype_backend\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype_backend\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    798\u001b[0m \u001b[43m    \u001b[49m\u001b[43mengine\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    799\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    801\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunksize:\n\u001b[1;32m    802\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m json_reader\n",
      "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/pandas/io/json/_json.py:893\u001b[0m, in \u001b[0;36mJsonReader.__init__\u001b[0;34m(self, filepath_or_buffer, orient, typ, dtype, convert_axes, convert_dates, keep_default_dates, precise_float, date_unit, encoding, lines, chunksize, compression, nrows, storage_options, encoding_errors, dtype_backend, engine)\u001b[0m\n\u001b[1;32m    891\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata \u001b[38;5;241m=\u001b[39m filepath_or_buffer\n\u001b[1;32m    892\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mengine \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mujson\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 893\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_data_from_filepath\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    894\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_preprocess_data(data)\n",
      "File \u001b[0;32m/usr/local/lib/python3.11/site-packages/pandas/io/json/_json.py:949\u001b[0m, in \u001b[0;36mJsonReader._get_data_from_filepath\u001b[0;34m(self, filepath_or_buffer)\u001b[0m\n\u001b[1;32m    941\u001b[0m     filepath_or_buffer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles\u001b[38;5;241m.\u001b[39mhandle\n\u001b[1;32m    942\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m (\n\u001b[1;32m    943\u001b[0m     \u001b[38;5;28misinstance\u001b[39m(filepath_or_buffer, \u001b[38;5;28mstr\u001b[39m)\n\u001b[1;32m    944\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m filepath_or_buffer\u001b[38;5;241m.\u001b[39mlower()\u001b[38;5;241m.\u001b[39mendswith(\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    947\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m file_exists(filepath_or_buffer)\n\u001b[1;32m    948\u001b[0m ):\n\u001b[0;32m--> 949\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFile \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfilepath_or_buffer\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not exist\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    950\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    951\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m    952\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPassing literal json to \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mread_json\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m is deprecated and \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    953\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwill be removed in a future version. To read from a \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    956\u001b[0m         stacklevel\u001b[38;5;241m=\u001b[39mfind_stack_level(),\n\u001b[1;32m    957\u001b[0m     )\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File data/github_raw/filtered_github_data_large/partitions000000000021.json does not exist"
     ]
    }
   ],
   "source": [
    "df_raw = pd.read_json(f'data/github_raw/filtered_github_data_large/{fname}.json', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc791471-f501-406e-b23e-d91fcb5b317a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_raw.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5769257-955c-4d18-b101-749d4eaa5b55",
   "metadata": {},
   "outputs": [],
   "source": [
    "tqdm.pandas()\n",
    "pd.set_option('display.max_columns', None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "43188248-348a-4847-b45a-76b3c454283c",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Unpack Data - Already Dictionary\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m col \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrepo\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mactor\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124morg\u001b[39m\u001b[38;5;124m'\u001b[39m]:\n\u001b[0;32m----> 3\u001b[0m     expand_df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame(\u001b[43mdf\u001b[49m[col]\u001b[38;5;241m.\u001b[39mvalues\u001b[38;5;241m.\u001b[39mtolist())\n\u001b[1;32m      4\u001b[0m     expand_df\u001b[38;5;241m.\u001b[39mcolumns \u001b[38;5;241m=\u001b[39m [col\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;28mstr\u001b[39m(c_val) \u001b[38;5;28;01mfor\u001b[39;00m c_val \u001b[38;5;129;01min\u001b[39;00m expand_df\u001b[38;5;241m.\u001b[39mcolumns]\n\u001b[1;32m      5\u001b[0m     df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat([df, expand_df], axis \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "# Unpack Data - Already Dictionary\n",
    "for col in ['repo', 'actor', 'org']:\n",
    "    expand_df = pd.DataFrame(df[col].values.tolist())\n",
    "    expand_df.columns = [col+\"_\"+str(c_val) for c_val in expand_df.columns]\n",
    "    df = pd.concat([df, expand_df], axis = 1)\n",
    "    df.drop(col, axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "67a48468-81f8-4cd4-94e5-17a3d4443d31",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpayload\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mdf\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpayload\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: json\u001b[38;5;241m.\u001b[39mloads(x))\n\u001b[1;32m      2\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mother\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mother\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: json\u001b[38;5;241m.\u001b[39mloads(x))\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df['payload'] = df['payload'].apply(lambda x: json.loads(x))\n",
    "df['other'] = df['other'].apply(lambda x: json.loads(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0c27eed4-731a-4066-91ca-c3894f176a98",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdf\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mvalue_counts()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined"
     ]
    }
   ],
   "source": [
    "df['type'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea1036a1-9f7e-4113-9db6-86a466b0e1d8",
   "metadata": {},
   "source": [
    "## ReleaseEvent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "28fa8b65-595e-4b69-a4e2-32170af1ca27",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_release = df[df['type'] == 'ReleaseEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "8eb50947-c5a4-41f6-9fd6-ceafe8c61f3f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1350319225.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_release['release_action'] = df_release['payload'].apply(lambda x: x['action'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1350319225.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_release['release_tag_name'] = df_release['payload'].apply(lambda x: x['release']['tag_name'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1350319225.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_release['release_name'] = df_release['payload'].apply(lambda x: x['release']['name'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1350319225.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_release['release_body'] = df_release['payload'].apply(lambda x: x['release']['body'])\n"
     ]
    }
   ],
   "source": [
    "df_release['release_action'] = df_release['payload'].apply(lambda x: x['action'])\n",
    "df_release['release_tag_name'] = df_release['payload'].apply(lambda x: x['release']['tag_name'])\n",
    "df_release['release_name'] = df_release['payload'].apply(lambda x: x['release']['name'])\n",
    "df_release['release_body'] = df_release['payload'].apply(lambda x: x['release']['body'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "6e599259-a20e-407f-93c8-20a4cbc03da2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>release_action</th>\n",
       "      <th>release_tag_name</th>\n",
       "      <th>release_name</th>\n",
       "      <th>release_body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2022-08-25 00:37:11+00:00</td>\n",
       "      <td>60082252</td>\n",
       "      <td>OCHA-DAP/hdx-python-api</td>\n",
       "      <td>3799212</td>\n",
       "      <td>mcarans</td>\n",
       "      <td>5087346</td>\n",
       "      <td>OCHA-DAP</td>\n",
       "      <td>published</td>\n",
       "      <td>5.8.5</td>\n",
       "      <td>Version 5.8.5</td>\n",
       "      <td>Update type hints</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2022-08-18 10:20:51+00:00</td>\n",
       "      <td>3072629</td>\n",
       "      <td>python-jsonschema/jsonschema</td>\n",
       "      <td>41898282</td>\n",
       "      <td>github-actions[bot]</td>\n",
       "      <td>100241624</td>\n",
       "      <td>python-jsonschema</td>\n",
       "      <td>published</td>\n",
       "      <td>v4.12.1</td>\n",
       "      <td>v4.12.1</td>\n",
       "      <td>## What's Changed\\n* Use rST markers in README...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>571</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2023-06-23 23:13:48+00:00</td>\n",
       "      <td>102155139</td>\n",
       "      <td>meraki/dashboard-api-python</td>\n",
       "      <td>3820074</td>\n",
       "      <td>TKIPisalegacycipher</td>\n",
       "      <td>624349</td>\n",
       "      <td>meraki</td>\n",
       "      <td>published</td>\n",
       "      <td>1.34.0</td>\n",
       "      <td>1.34.0</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2023-03-09 00:43:34+00:00</td>\n",
       "      <td>4127088</td>\n",
       "      <td>Azure/azure-sdk-for-python</td>\n",
       "      <td>53356347</td>\n",
       "      <td>azure-sdk</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>published</td>\n",
       "      <td>azure-storage-file-share_12.11.1</td>\n",
       "      <td>azure-storage-file-share_12.11.1</td>\n",
       "      <td>## 12.11.1 (2023-03-08)\\n\\n### Bugs Fixed\\n- F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1324</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2021-08-11 05:11:46+00:00</td>\n",
       "      <td>102973646</td>\n",
       "      <td>fastai/fastai</td>\n",
       "      <td>346999</td>\n",
       "      <td>jph00</td>\n",
       "      <td>20547620</td>\n",
       "      <td>fastai</td>\n",
       "      <td>published</td>\n",
       "      <td>2.5.1</td>\n",
       "      <td>v2.5.1</td>\n",
       "      <td>- Import `download_url` from fastdownload</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116690</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2022-08-03 19:39:54+00:00</td>\n",
       "      <td>527040</td>\n",
       "      <td>recurly/recurly-client-python</td>\n",
       "      <td>49795151</td>\n",
       "      <td>recurly-integrations</td>\n",
       "      <td>144605</td>\n",
       "      <td>recurly</td>\n",
       "      <td>published</td>\n",
       "      <td>3.18.0</td>\n",
       "      <td>3.18.0</td>\n",
       "      <td>[Full Changelog](https://github.com/recurly/re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117120</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2022-02-24 00:02:06+00:00</td>\n",
       "      <td>1347652</td>\n",
       "      <td>scoder/lupa</td>\n",
       "      <td>491659</td>\n",
       "      <td>scoder</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>published</td>\n",
       "      <td>lupa-1.12</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117347</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2022-02-01 14:37:46+00:00</td>\n",
       "      <td>140365570</td>\n",
       "      <td>sdispater/tomlkit</td>\n",
       "      <td>16336606</td>\n",
       "      <td>frostming</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>published</td>\n",
       "      <td>0.9.0</td>\n",
       "      <td>0.9.0</td>\n",
       "      <td>## What's Changed\\r\\n* Add an argument to allo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117539</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2022-08-04 22:42:19+00:00</td>\n",
       "      <td>3366934</td>\n",
       "      <td>dvarrazzo/py-setproctitle</td>\n",
       "      <td>199429</td>\n",
       "      <td>dvarrazzo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>published</td>\n",
       "      <td>version-1.3.1</td>\n",
       "      <td>setproctitle 1.3.1</td>\n",
       "      <td>- Fixed segfault on macOS 12.5 in forked proce...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117988</th>\n",
       "      <td>ReleaseEvent</td>\n",
       "      <td>2021-04-16 09:03:08+00:00</td>\n",
       "      <td>119905910</td>\n",
       "      <td>aguinet/dragonffi</td>\n",
       "      <td>1874053</td>\n",
       "      <td>aguinet</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>published</td>\n",
       "      <td>dffi-0.9.0</td>\n",
       "      <td>DragonFFI 0.9.0</td>\n",
       "      <td>Includes support for compiling C++ code and ca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>556 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                type                created_at    repo_id  \\\n",
       "172     ReleaseEvent 2022-08-25 00:37:11+00:00   60082252   \n",
       "215     ReleaseEvent 2022-08-18 10:20:51+00:00    3072629   \n",
       "571     ReleaseEvent 2023-06-23 23:13:48+00:00  102155139   \n",
       "847     ReleaseEvent 2023-03-09 00:43:34+00:00    4127088   \n",
       "1324    ReleaseEvent 2021-08-11 05:11:46+00:00  102973646   \n",
       "...              ...                       ...        ...   \n",
       "116690  ReleaseEvent 2022-08-03 19:39:54+00:00     527040   \n",
       "117120  ReleaseEvent 2022-02-24 00:02:06+00:00    1347652   \n",
       "117347  ReleaseEvent 2022-02-01 14:37:46+00:00  140365570   \n",
       "117539  ReleaseEvent 2022-08-04 22:42:19+00:00    3366934   \n",
       "117988  ReleaseEvent 2021-04-16 09:03:08+00:00  119905910   \n",
       "\n",
       "                            repo_name  actor_id           actor_login  \\\n",
       "172           OCHA-DAP/hdx-python-api   3799212               mcarans   \n",
       "215      python-jsonschema/jsonschema  41898282   github-actions[bot]   \n",
       "571       meraki/dashboard-api-python   3820074   TKIPisalegacycipher   \n",
       "847        Azure/azure-sdk-for-python  53356347             azure-sdk   \n",
       "1324                    fastai/fastai    346999                 jph00   \n",
       "...                               ...       ...                   ...   \n",
       "116690  recurly/recurly-client-python  49795151  recurly-integrations   \n",
       "117120                    scoder/lupa    491659                scoder   \n",
       "117347              sdispater/tomlkit  16336606             frostming   \n",
       "117539      dvarrazzo/py-setproctitle    199429             dvarrazzo   \n",
       "117988              aguinet/dragonffi   1874053               aguinet   \n",
       "\n",
       "           org_id          org_login release_action  \\\n",
       "172       5087346           OCHA-DAP      published   \n",
       "215     100241624  python-jsonschema      published   \n",
       "571        624349             meraki      published   \n",
       "847       6844498              Azure      published   \n",
       "1324     20547620             fastai      published   \n",
       "...           ...                ...            ...   \n",
       "116690     144605            recurly      published   \n",
       "117120        NaN                NaN      published   \n",
       "117347        NaN                NaN      published   \n",
       "117539        NaN                NaN      published   \n",
       "117988        NaN                NaN      published   \n",
       "\n",
       "                        release_tag_name                      release_name  \\\n",
       "172                                5.8.5                     Version 5.8.5   \n",
       "215                              v4.12.1                           v4.12.1   \n",
       "571                               1.34.0                            1.34.0   \n",
       "847     azure-storage-file-share_12.11.1  azure-storage-file-share_12.11.1   \n",
       "1324                               2.5.1                            v2.5.1   \n",
       "...                                  ...                               ...   \n",
       "116690                            3.18.0                            3.18.0   \n",
       "117120                         lupa-1.12                                     \n",
       "117347                             0.9.0                             0.9.0   \n",
       "117539                     version-1.3.1                setproctitle 1.3.1   \n",
       "117988                        dffi-0.9.0                   DragonFFI 0.9.0   \n",
       "\n",
       "                                             release_body  \n",
       "172                                     Update type hints  \n",
       "215     ## What's Changed\\n* Use rST markers in README...  \n",
       "571                                                        \n",
       "847     ## 12.11.1 (2023-03-08)\\n\\n### Bugs Fixed\\n- F...  \n",
       "1324            - Import `download_url` from fastdownload  \n",
       "...                                                   ...  \n",
       "116690  [Full Changelog](https://github.com/recurly/re...  \n",
       "117120                                                     \n",
       "117347  ## What's Changed\\r\\n* Add an argument to allo...  \n",
       "117539  - Fixed segfault on macOS 12.5 in forked proce...  \n",
       "117988  Includes support for compiling C++ code and ca...  \n",
       "\n",
       "[556 rows x 12 columns]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_release[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "            'release_action', 'release_tag_name', 'release_name', 'release_body']].to_csv(\n",
    "    f'data/github_clean/releaseEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48e96efd-89cc-485a-aa79-983d285f550d",
   "metadata": {},
   "source": [
    "## DeleteEvent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "09e195cd-abbc-441e-83ef-725a1b19866c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_delete = df[df['type'] == 'DeleteEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "73202066-4ff3-43b0-aba4-5da77d2862d7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1911536852.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_delete['event_type'] = df_delete['payload'].apply(lambda x: x['ref_type'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1911536852.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_delete['event_ref'] = df_delete['payload'].apply(lambda x: x['ref'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1911536852.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_delete['event_pusher_type'] = df_delete['payload'].apply(lambda x: x['pusher_type'])\n"
     ]
    }
   ],
   "source": [
    "df_delete['event_type'] = df_delete['payload'].apply(lambda x: x['ref_type'])\n",
    "df_delete['event_ref'] = df_delete['payload'].apply(lambda x: x['ref'])\n",
    "df_delete['event_pusher_type'] = df_delete['payload'].apply(lambda x: x['pusher_type'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "ae29ce37-cc73-4ba4-addf-42af33742cc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>event_type</th>\n",
       "      <th>event_ref</th>\n",
       "      <th>event_pusher_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2023-01-02 23:35:39+00:00</td>\n",
       "      <td>37137879</td>\n",
       "      <td>QCoDeS/Qcodes</td>\n",
       "      <td>26634292</td>\n",
       "      <td>bors[bot]</td>\n",
       "      <td>20478252</td>\n",
       "      <td>QCoDeS</td>\n",
       "      <td>branch</td>\n",
       "      <td>staging.tmp</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2021-07-13 23:41:43+00:00</td>\n",
       "      <td>72477752</td>\n",
       "      <td>pulumi/pulumi</td>\n",
       "      <td>30351955</td>\n",
       "      <td>pulumi-bot</td>\n",
       "      <td>21992475</td>\n",
       "      <td>pulumi</td>\n",
       "      <td>branch</td>\n",
       "      <td>pgavlin/simpleYamlSchema</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2023-01-11 16:58:28+00:00</td>\n",
       "      <td>131619646</td>\n",
       "      <td>dagster-io/dagster</td>\n",
       "      <td>2823852</td>\n",
       "      <td>hellendag</td>\n",
       "      <td>40032576</td>\n",
       "      <td>dagster-io</td>\n",
       "      <td>branch</td>\n",
       "      <td>dish/lint-fix</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2022-11-01 14:26:08+00:00</td>\n",
       "      <td>72477752</td>\n",
       "      <td>pulumi/pulumi</td>\n",
       "      <td>56450</td>\n",
       "      <td>t0yv0</td>\n",
       "      <td>21992475</td>\n",
       "      <td>pulumi</td>\n",
       "      <td>branch</td>\n",
       "      <td>t0yv0/use-npm-instead-of-yarn</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2023-01-30 09:46:04+00:00</td>\n",
       "      <td>38744670</td>\n",
       "      <td>zalando/patroni</td>\n",
       "      <td>3407345</td>\n",
       "      <td>CyberDem0n</td>\n",
       "      <td>1564818</td>\n",
       "      <td>zalando</td>\n",
       "      <td>branch</td>\n",
       "      <td>feature/archive-before-rewind</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118383</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2022-08-11 07:39:32+00:00</td>\n",
       "      <td>37137879</td>\n",
       "      <td>QCoDeS/Qcodes</td>\n",
       "      <td>26634292</td>\n",
       "      <td>bors[bot]</td>\n",
       "      <td>20478252</td>\n",
       "      <td>QCoDeS</td>\n",
       "      <td>branch</td>\n",
       "      <td>staging.tmp</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118396</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2019-06-04 11:58:17+00:00</td>\n",
       "      <td>24219208</td>\n",
       "      <td>LettError/MutatorMath</td>\n",
       "      <td>6939968</td>\n",
       "      <td>anthrotype</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>branch</td>\n",
       "      <td>fontmake_#552</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118398</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2019-06-21 11:54:01+00:00</td>\n",
       "      <td>59046255</td>\n",
       "      <td>plus3it/watchmaker</td>\n",
       "      <td>27856297</td>\n",
       "      <td>dependabot-preview[bot]</td>\n",
       "      <td>15047853</td>\n",
       "      <td>plus3it</td>\n",
       "      <td>branch</td>\n",
       "      <td>dependabot/submodules/src/watchmaker/static/sa...</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118409</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2019-06-01 17:09:54+00:00</td>\n",
       "      <td>68423917</td>\n",
       "      <td>adobe-type-tools/psautohint</td>\n",
       "      <td>93914</td>\n",
       "      <td>khaledhosny</td>\n",
       "      <td>2768660</td>\n",
       "      <td>adobe-type-tools</td>\n",
       "      <td>branch</td>\n",
       "      <td>round-to-9</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118445</th>\n",
       "      <td>DeleteEvent</td>\n",
       "      <td>2021-11-24 06:27:08+00:00</td>\n",
       "      <td>37137879</td>\n",
       "      <td>QCoDeS/Qcodes</td>\n",
       "      <td>26634292</td>\n",
       "      <td>bors[bot]</td>\n",
       "      <td>20478252</td>\n",
       "      <td>QCoDeS</td>\n",
       "      <td>branch</td>\n",
       "      <td>dependabot/pip/types-pytz-2021.3.1</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3310 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               type                created_at    repo_id  \\\n",
       "50      DeleteEvent 2023-01-02 23:35:39+00:00   37137879   \n",
       "61      DeleteEvent 2021-07-13 23:41:43+00:00   72477752   \n",
       "65      DeleteEvent 2023-01-11 16:58:28+00:00  131619646   \n",
       "82      DeleteEvent 2022-11-01 14:26:08+00:00   72477752   \n",
       "106     DeleteEvent 2023-01-30 09:46:04+00:00   38744670   \n",
       "...             ...                       ...        ...   \n",
       "118383  DeleteEvent 2022-08-11 07:39:32+00:00   37137879   \n",
       "118396  DeleteEvent 2019-06-04 11:58:17+00:00   24219208   \n",
       "118398  DeleteEvent 2019-06-21 11:54:01+00:00   59046255   \n",
       "118409  DeleteEvent 2019-06-01 17:09:54+00:00   68423917   \n",
       "118445  DeleteEvent 2021-11-24 06:27:08+00:00   37137879   \n",
       "\n",
       "                          repo_name  actor_id              actor_login  \\\n",
       "50                    QCoDeS/Qcodes  26634292                bors[bot]   \n",
       "61                    pulumi/pulumi  30351955               pulumi-bot   \n",
       "65               dagster-io/dagster   2823852                hellendag   \n",
       "82                    pulumi/pulumi     56450                    t0yv0   \n",
       "106                 zalando/patroni   3407345               CyberDem0n   \n",
       "...                             ...       ...                      ...   \n",
       "118383                QCoDeS/Qcodes  26634292                bors[bot]   \n",
       "118396        LettError/MutatorMath   6939968               anthrotype   \n",
       "118398           plus3it/watchmaker  27856297  dependabot-preview[bot]   \n",
       "118409  adobe-type-tools/psautohint     93914              khaledhosny   \n",
       "118445                QCoDeS/Qcodes  26634292                bors[bot]   \n",
       "\n",
       "          org_id         org_login event_type  \\\n",
       "50      20478252            QCoDeS     branch   \n",
       "61      21992475            pulumi     branch   \n",
       "65      40032576        dagster-io     branch   \n",
       "82      21992475            pulumi     branch   \n",
       "106      1564818           zalando     branch   \n",
       "...          ...               ...        ...   \n",
       "118383  20478252            QCoDeS     branch   \n",
       "118396       NaN               NaN     branch   \n",
       "118398  15047853           plus3it     branch   \n",
       "118409   2768660  adobe-type-tools     branch   \n",
       "118445  20478252            QCoDeS     branch   \n",
       "\n",
       "                                                event_ref event_pusher_type  \n",
       "50                                            staging.tmp              user  \n",
       "61                               pgavlin/simpleYamlSchema              user  \n",
       "65                                          dish/lint-fix              user  \n",
       "82                          t0yv0/use-npm-instead-of-yarn              user  \n",
       "106                         feature/archive-before-rewind              user  \n",
       "...                                                   ...               ...  \n",
       "118383                                        staging.tmp              user  \n",
       "118396                                      fontmake_#552              user  \n",
       "118398  dependabot/submodules/src/watchmaker/static/sa...              user  \n",
       "118409                                         round-to-9              user  \n",
       "118445                 dependabot/pip/types-pytz-2021.3.1              user  \n",
       "\n",
       "[3310 rows x 11 columns]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_delete[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "           'event_type', 'event_ref', 'event_pusher_type']].to_csv(\n",
    "    f'data/github_clean/deleteEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ad4790c-0a5c-4c8a-86cb-1784ddee722e",
   "metadata": {},
   "source": [
    "## CreateEvent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "9f83f5fb-0a15-48e6-bd50-224745b8a8c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_create = df[df['type'] == 'CreateEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "351ef6ee-3346-45eb-964a-6006be424d37",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3977952771.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_create['event_type'] = df_create['payload'].apply(lambda x: x['ref_type'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3977952771.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_create['event_ref'] = df_create['payload'].apply(lambda x: x['ref'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3977952771.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_create['repo_master_branch'] = df_create['payload'].apply(lambda x: x['master_branch'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3977952771.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_create['event_pusher_type'] = df_create['payload'].apply(lambda x: x['pusher_type'])\n"
     ]
    }
   ],
   "source": [
    "df_create['event_type'] = df_create['payload'].apply(lambda x: x['ref_type'])\n",
    "df_create['event_ref'] = df_create['payload'].apply(lambda x: x['ref'])\n",
    "df_create['repo_master_branch'] = df_create['payload'].apply(lambda x: x['master_branch'])\n",
    "df_create['event_pusher_type'] = df_create['payload'].apply(lambda x: x['pusher_type'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "a960625d-f3e3-4b95-a342-b113e3744d99",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>event_type</th>\n",
       "      <th>event_ref</th>\n",
       "      <th>repo_master_branch</th>\n",
       "      <th>event_pusher_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2022-09-25 23:22:02+00:00</td>\n",
       "      <td>121602209</td>\n",
       "      <td>clinicedc/edc-consent</td>\n",
       "      <td>502244</td>\n",
       "      <td>erikvw</td>\n",
       "      <td>36031354</td>\n",
       "      <td>clinicedc</td>\n",
       "      <td>tag</td>\n",
       "      <td>0.3.37</td>\n",
       "      <td>develop</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2023-01-02 08:05:22+00:00</td>\n",
       "      <td>43091756</td>\n",
       "      <td>hackingmaterials/matminer</td>\n",
       "      <td>41898282</td>\n",
       "      <td>github-actions[bot]</td>\n",
       "      <td>14116185</td>\n",
       "      <td>hackingmaterials</td>\n",
       "      <td>branch</td>\n",
       "      <td>auto-dependency-upgrades-ubuntu-latest-py3.8</td>\n",
       "      <td>main</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2022-09-23 03:13:21+00:00</td>\n",
       "      <td>72477752</td>\n",
       "      <td>pulumi/pulumi</td>\n",
       "      <td>26634292</td>\n",
       "      <td>bors[bot]</td>\n",
       "      <td>21992475</td>\n",
       "      <td>pulumi</td>\n",
       "      <td>branch</td>\n",
       "      <td>staging.tmp</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2023-05-23 13:50:58+00:00</td>\n",
       "      <td>72477752</td>\n",
       "      <td>pulumi/pulumi</td>\n",
       "      <td>1398274</td>\n",
       "      <td>Frassle</td>\n",
       "      <td>21992475</td>\n",
       "      <td>pulumi</td>\n",
       "      <td>branch</td>\n",
       "      <td>fraser/fixTypeRef</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2023-01-11 04:46:29+00:00</td>\n",
       "      <td>76801332</td>\n",
       "      <td>RobinNil/file_read_backwards</td>\n",
       "      <td>330836</td>\n",
       "      <td>RobinNil</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>branch</td>\n",
       "      <td>pyup-update-sphinx-3.3.1-to-6.1.3</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118318</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2021-08-30 19:54:10+00:00</td>\n",
       "      <td>226992562</td>\n",
       "      <td>googleapis/python-logging</td>\n",
       "      <td>71398022</td>\n",
       "      <td>dandhlee</td>\n",
       "      <td>16785467</td>\n",
       "      <td>googleapis</td>\n",
       "      <td>branch</td>\n",
       "      <td>to_main</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118339</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2022-08-01 14:44:01+00:00</td>\n",
       "      <td>68917117</td>\n",
       "      <td>aio-libs/aiohttp-devtools</td>\n",
       "      <td>598990</td>\n",
       "      <td>fafhrd91</td>\n",
       "      <td>7049303</td>\n",
       "      <td>aio-libs</td>\n",
       "      <td>branch</td>\n",
       "      <td>pyup-update-flake8-4.0.1-to-5.0.2</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118387</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2021-07-21 17:55:43+00:00</td>\n",
       "      <td>60298017</td>\n",
       "      <td>Losant/losant-rest-python</td>\n",
       "      <td>23384079</td>\n",
       "      <td>eeerin</td>\n",
       "      <td>18402036</td>\n",
       "      <td>Losant</td>\n",
       "      <td>branch</td>\n",
       "      <td>revert-3-fix-abc</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118418</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2019-06-05 16:28:18+00:00</td>\n",
       "      <td>131619646</td>\n",
       "      <td>dagster-io/dagster</td>\n",
       "      <td>28738937</td>\n",
       "      <td>schrockn</td>\n",
       "      <td>40032576</td>\n",
       "      <td>dagster-io</td>\n",
       "      <td>tag</td>\n",
       "      <td>phabricator/diff/1465</td>\n",
       "      <td>master</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118435</th>\n",
       "      <td>CreateEvent</td>\n",
       "      <td>2021-11-09 17:17:46+00:00</td>\n",
       "      <td>226992462</td>\n",
       "      <td>googleapis/python-cloud-core</td>\n",
       "      <td>44816363</td>\n",
       "      <td>yoshi-automation</td>\n",
       "      <td>16785467</td>\n",
       "      <td>googleapis</td>\n",
       "      <td>tag</td>\n",
       "      <td>v2.2.0</td>\n",
       "      <td>main</td>\n",
       "      <td>user</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4629 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               type                created_at    repo_id  \\\n",
       "12      CreateEvent 2022-09-25 23:22:02+00:00  121602209   \n",
       "49      CreateEvent 2023-01-02 08:05:22+00:00   43091756   \n",
       "57      CreateEvent 2022-09-23 03:13:21+00:00   72477752   \n",
       "59      CreateEvent 2023-05-23 13:50:58+00:00   72477752   \n",
       "62      CreateEvent 2023-01-11 04:46:29+00:00   76801332   \n",
       "...             ...                       ...        ...   \n",
       "118318  CreateEvent 2021-08-30 19:54:10+00:00  226992562   \n",
       "118339  CreateEvent 2022-08-01 14:44:01+00:00   68917117   \n",
       "118387  CreateEvent 2021-07-21 17:55:43+00:00   60298017   \n",
       "118418  CreateEvent 2019-06-05 16:28:18+00:00  131619646   \n",
       "118435  CreateEvent 2021-11-09 17:17:46+00:00  226992462   \n",
       "\n",
       "                           repo_name  actor_id          actor_login    org_id  \\\n",
       "12             clinicedc/edc-consent    502244               erikvw  36031354   \n",
       "49         hackingmaterials/matminer  41898282  github-actions[bot]  14116185   \n",
       "57                     pulumi/pulumi  26634292            bors[bot]  21992475   \n",
       "59                     pulumi/pulumi   1398274              Frassle  21992475   \n",
       "62      RobinNil/file_read_backwards    330836             RobinNil       NaN   \n",
       "...                              ...       ...                  ...       ...   \n",
       "118318     googleapis/python-logging  71398022             dandhlee  16785467   \n",
       "118339     aio-libs/aiohttp-devtools    598990             fafhrd91   7049303   \n",
       "118387     Losant/losant-rest-python  23384079               eeerin  18402036   \n",
       "118418            dagster-io/dagster  28738937             schrockn  40032576   \n",
       "118435  googleapis/python-cloud-core  44816363     yoshi-automation  16785467   \n",
       "\n",
       "               org_login event_type  \\\n",
       "12             clinicedc        tag   \n",
       "49      hackingmaterials     branch   \n",
       "57                pulumi     branch   \n",
       "59                pulumi     branch   \n",
       "62                   NaN     branch   \n",
       "...                  ...        ...   \n",
       "118318        googleapis     branch   \n",
       "118339          aio-libs     branch   \n",
       "118387            Losant     branch   \n",
       "118418        dagster-io        tag   \n",
       "118435        googleapis        tag   \n",
       "\n",
       "                                           event_ref repo_master_branch  \\\n",
       "12                                            0.3.37            develop   \n",
       "49      auto-dependency-upgrades-ubuntu-latest-py3.8               main   \n",
       "57                                       staging.tmp             master   \n",
       "59                                 fraser/fixTypeRef             master   \n",
       "62                 pyup-update-sphinx-3.3.1-to-6.1.3             master   \n",
       "...                                              ...                ...   \n",
       "118318                                       to_main             master   \n",
       "118339             pyup-update-flake8-4.0.1-to-5.0.2             master   \n",
       "118387                              revert-3-fix-abc             master   \n",
       "118418                         phabricator/diff/1465             master   \n",
       "118435                                        v2.2.0               main   \n",
       "\n",
       "       event_pusher_type  \n",
       "12                  user  \n",
       "49                  user  \n",
       "57                  user  \n",
       "59                  user  \n",
       "62                  user  \n",
       "...                  ...  \n",
       "118318              user  \n",
       "118339              user  \n",
       "118387              user  \n",
       "118418              user  \n",
       "118435              user  \n",
       "\n",
       "[4629 rows x 12 columns]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_create[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "           'event_type', 'event_ref', 'repo_master_branch', 'event_pusher_type']].to_csv(\n",
    "    f'data/github_clean/createEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9132e112-838a-4b7c-b8ec-58e3340e9cdb",
   "metadata": {},
   "source": [
    "## Fork Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "e929a94c-6cae-42b8-92f9-a3cde9a00822",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2023-06-15 11:38:00+00:00</td>\n",
       "      <td>72473727</td>\n",
       "      <td>kubernetes-client/python</td>\n",
       "      <td>133362448</td>\n",
       "      <td>shanwen222</td>\n",
       "      <td>26615988</td>\n",
       "      <td>kubernetes-client</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2021-08-12 03:22:37+00:00</td>\n",
       "      <td>139454635</td>\n",
       "      <td>hill-a/stable-baselines</td>\n",
       "      <td>66294422</td>\n",
       "      <td>ChunYe173</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2023-06-14 14:19:30+00:00</td>\n",
       "      <td>26784827</td>\n",
       "      <td>dotnet/core</td>\n",
       "      <td>23071507</td>\n",
       "      <td>puneetkant</td>\n",
       "      <td>9141961</td>\n",
       "      <td>dotnet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2022-02-11 07:49:54+00:00</td>\n",
       "      <td>7673329</td>\n",
       "      <td>simple-salesforce/simple-salesforce</td>\n",
       "      <td>97739774</td>\n",
       "      <td>deloittedcswitzerland</td>\n",
       "      <td>19581234</td>\n",
       "      <td>simple-salesforce</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2023-01-11 05:09:21+00:00</td>\n",
       "      <td>83844720</td>\n",
       "      <td>ageitgey/face_recognition</td>\n",
       "      <td>98010467</td>\n",
       "      <td>sun0428</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118366</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2021-04-07 04:19:02+00:00</td>\n",
       "      <td>41179827</td>\n",
       "      <td>Rapptz/discord.py</td>\n",
       "      <td>81849818</td>\n",
       "      <td>yudagioogo</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118386</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2022-02-04 14:27:21+00:00</td>\n",
       "      <td>54828252</td>\n",
       "      <td>agronholm/apscheduler</td>\n",
       "      <td>59997207</td>\n",
       "      <td>someonelikework</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118394</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2019-06-05 04:10:17+00:00</td>\n",
       "      <td>12345160</td>\n",
       "      <td>ansible/tower-cli</td>\n",
       "      <td>50466022</td>\n",
       "      <td>linuxl0s3r</td>\n",
       "      <td>1507452</td>\n",
       "      <td>ansible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118401</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2019-06-15 14:11:49+00:00</td>\n",
       "      <td>83844720</td>\n",
       "      <td>ageitgey/face_recognition</td>\n",
       "      <td>41847008</td>\n",
       "      <td>diencs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118430</th>\n",
       "      <td>ForkEvent</td>\n",
       "      <td>2019-06-22 14:53:08+00:00</td>\n",
       "      <td>64991887</td>\n",
       "      <td>microsoft/LightGBM</td>\n",
       "      <td>45694336</td>\n",
       "      <td>Ttopiac</td>\n",
       "      <td>6154722</td>\n",
       "      <td>microsoft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5945 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             type                created_at    repo_id  \\\n",
       "8       ForkEvent 2023-06-15 11:38:00+00:00   72473727   \n",
       "21      ForkEvent 2021-08-12 03:22:37+00:00  139454635   \n",
       "27      ForkEvent 2023-06-14 14:19:30+00:00   26784827   \n",
       "35      ForkEvent 2022-02-11 07:49:54+00:00    7673329   \n",
       "64      ForkEvent 2023-01-11 05:09:21+00:00   83844720   \n",
       "...           ...                       ...        ...   \n",
       "118366  ForkEvent 2021-04-07 04:19:02+00:00   41179827   \n",
       "118386  ForkEvent 2022-02-04 14:27:21+00:00   54828252   \n",
       "118394  ForkEvent 2019-06-05 04:10:17+00:00   12345160   \n",
       "118401  ForkEvent 2019-06-15 14:11:49+00:00   83844720   \n",
       "118430  ForkEvent 2019-06-22 14:53:08+00:00   64991887   \n",
       "\n",
       "                                  repo_name   actor_id            actor_login  \\\n",
       "8                  kubernetes-client/python  133362448             shanwen222   \n",
       "21                  hill-a/stable-baselines   66294422              ChunYe173   \n",
       "27                              dotnet/core   23071507             puneetkant   \n",
       "35      simple-salesforce/simple-salesforce   97739774  deloittedcswitzerland   \n",
       "64                ageitgey/face_recognition   98010467                sun0428   \n",
       "...                                     ...        ...                    ...   \n",
       "118366                    Rapptz/discord.py   81849818             yudagioogo   \n",
       "118386                agronholm/apscheduler   59997207        someonelikework   \n",
       "118394                    ansible/tower-cli   50466022             linuxl0s3r   \n",
       "118401            ageitgey/face_recognition   41847008                 diencs   \n",
       "118430                   microsoft/LightGBM   45694336                Ttopiac   \n",
       "\n",
       "          org_id          org_login  \n",
       "8       26615988  kubernetes-client  \n",
       "21           NaN                NaN  \n",
       "27       9141961             dotnet  \n",
       "35      19581234  simple-salesforce  \n",
       "64           NaN                NaN  \n",
       "...          ...                ...  \n",
       "118366       NaN                NaN  \n",
       "118386       NaN                NaN  \n",
       "118394   1507452            ansible  \n",
       "118401       NaN                NaN  \n",
       "118430   6154722          microsoft  \n",
       "\n",
       "[5945 rows x 8 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fork = df[df['type'] == 'ForkEvent']\n",
    "df_fork[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login']].to_csv(\n",
    "    f'data/github_clean/forkEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3619551e-a0a9-4e17-b18d-6db5704b190b",
   "metadata": {},
   "source": [
    "# Pull Requests"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47b55999-7d45-4186-aef7-4cb822973c6a",
   "metadata": {},
   "source": [
    "## Pull Request Review Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "06967ba4-20b1-4b61-bffb-8a3f837e420d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pullrequestreview = df[df['type'] == 'PullRequestReviewEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "b812be82-108c-4c47-b353-34855ba42a70",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_review_action'] = df_pullrequestreview['payload'].apply(lambda x: x['action'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_review_id'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['id'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_review_body'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['body'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_review_commit_id'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['commit_id'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_review_author_association'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['author_association'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_review_state'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['state'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/4124135630.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreview['pr_number'] = df_pullrequestreview['payload'].apply(lambda x: x['pull_request']['number'])\n"
     ]
    }
   ],
   "source": [
    "df_pullrequestreview['pr_review_action'] = df_pullrequestreview['payload'].apply(lambda x: x['action'])\n",
    "df_pullrequestreview['pr_review_id'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['id'])\n",
    "df_pullrequestreview['pr_review_body'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['body'])\n",
    "df_pullrequestreview['pr_review_commit_id'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['commit_id'])\n",
    "df_pullrequestreview['pr_review_author_association'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['author_association'])\n",
    "df_pullrequestreview['pr_review_state'] = df_pullrequestreview['payload'].apply(lambda x: x['review']['state'])\n",
    "df_pullrequestreview['pr_number'] = df_pullrequestreview['payload'].apply(lambda x: x['pull_request']['number'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "7924a47c-49ea-4911-bc79-2eb31f1a3574",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>pr_review_action</th>\n",
       "      <th>pr_review_id</th>\n",
       "      <th>pr_review_body</th>\n",
       "      <th>pr_review_commit_id</th>\n",
       "      <th>pr_review_author_association</th>\n",
       "      <th>pr_review_state</th>\n",
       "      <th>pr_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2022-11-01 16:30:10+00:00</td>\n",
       "      <td>73328905</td>\n",
       "      <td>pytorch/vision</td>\n",
       "      <td>2459423</td>\n",
       "      <td>vfdev-5</td>\n",
       "      <td>21003710</td>\n",
       "      <td>pytorch</td>\n",
       "      <td>created</td>\n",
       "      <td>1163653955</td>\n",
       "      <td>LGTM, thanks!</td>\n",
       "      <td>54ef3b4a88e306de5739e1b3a82963cb107f7915</td>\n",
       "      <td>COLLABORATOR</td>\n",
       "      <td>approved</td>\n",
       "      <td>6880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2023-05-03 19:53:49+00:00</td>\n",
       "      <td>102692863</td>\n",
       "      <td>onnx/onnx</td>\n",
       "      <td>17606112</td>\n",
       "      <td>leimao</td>\n",
       "      <td>31675368</td>\n",
       "      <td>onnx</td>\n",
       "      <td>created</td>\n",
       "      <td>1411698255</td>\n",
       "      <td>None</td>\n",
       "      <td>345e24d9ad2b2e10621f252c861bedab72a05491</td>\n",
       "      <td>NONE</td>\n",
       "      <td>commented</td>\n",
       "      <td>5010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2023-05-24 16:28:44+00:00</td>\n",
       "      <td>70124554</td>\n",
       "      <td>lablup/backend.ai</td>\n",
       "      <td>555156</td>\n",
       "      <td>achimnol</td>\n",
       "      <td>11258248</td>\n",
       "      <td>lablup</td>\n",
       "      <td>created</td>\n",
       "      <td>1442326238</td>\n",
       "      <td></td>\n",
       "      <td>02e5862302caa1b992f100164bd23289a46d22e3</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>approved</td>\n",
       "      <td>1297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2023-03-07 16:41:33+00:00</td>\n",
       "      <td>134186172</td>\n",
       "      <td>faucetsdn/chewie</td>\n",
       "      <td>2931079</td>\n",
       "      <td>cglewis</td>\n",
       "      <td>23409712</td>\n",
       "      <td>faucetsdn</td>\n",
       "      <td>created</td>\n",
       "      <td>1329047075</td>\n",
       "      <td></td>\n",
       "      <td>e73ce623b5da4131227872e4a4f3dc4aa2714cd7</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>approved</td>\n",
       "      <td>424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2023-05-23 17:52:28+00:00</td>\n",
       "      <td>71932349</td>\n",
       "      <td>ray-project/ray</td>\n",
       "      <td>92341594</td>\n",
       "      <td>shrekris-anyscale</td>\n",
       "      <td>22125274</td>\n",
       "      <td>ray-project</td>\n",
       "      <td>created</td>\n",
       "      <td>1440284983</td>\n",
       "      <td>None</td>\n",
       "      <td>5879fc93333221c601517281cd98da4e3f9c2f91</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>commented</td>\n",
       "      <td>35578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118341</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2022-02-03 15:43:33+00:00</td>\n",
       "      <td>4127088</td>\n",
       "      <td>Azure/azure-sdk-for-python</td>\n",
       "      <td>64233642</td>\n",
       "      <td>xinlian12</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>created</td>\n",
       "      <td>872031515</td>\n",
       "      <td>None</td>\n",
       "      <td>7c098ea7755ebbaaa5a8ff84f0dbafc5655d4c1b</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>commented</td>\n",
       "      <td>22792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118343</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2021-08-30 22:11:54+00:00</td>\n",
       "      <td>4127088</td>\n",
       "      <td>Azure/azure-sdk-for-python</td>\n",
       "      <td>43154838</td>\n",
       "      <td>iscai-msft</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>created</td>\n",
       "      <td>742115081</td>\n",
       "      <td>None</td>\n",
       "      <td>d7e39c7cf2c9ec9d7f3aa3346d57c9cfdc8b2dbc</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>commented</td>\n",
       "      <td>20460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118357</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2021-11-03 11:00:16+00:00</td>\n",
       "      <td>12082924</td>\n",
       "      <td>zostera/django-bootstrap3</td>\n",
       "      <td>41898282</td>\n",
       "      <td>github-actions[bot]</td>\n",
       "      <td>1092877</td>\n",
       "      <td>zostera</td>\n",
       "      <td>created</td>\n",
       "      <td>796386825</td>\n",
       "      <td></td>\n",
       "      <td>406edd7e1090a293bef5551fd0e8435bd2762bbe</td>\n",
       "      <td>NONE</td>\n",
       "      <td>approved</td>\n",
       "      <td>588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118437</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2021-11-05 21:24:17+00:00</td>\n",
       "      <td>71932349</td>\n",
       "      <td>ray-project/ray</td>\n",
       "      <td>14922</td>\n",
       "      <td>ericl</td>\n",
       "      <td>22125274</td>\n",
       "      <td>ray-project</td>\n",
       "      <td>created</td>\n",
       "      <td>799335440</td>\n",
       "      <td></td>\n",
       "      <td>990ba4da74d1b9a42a80ff673aa13fa6d6f5f429</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>approved</td>\n",
       "      <td>19924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118447</th>\n",
       "      <td>PullRequestReviewEvent</td>\n",
       "      <td>2021-04-13 01:06:53+00:00</td>\n",
       "      <td>55808787</td>\n",
       "      <td>openthread/openthread</td>\n",
       "      <td>17729664</td>\n",
       "      <td>abtink</td>\n",
       "      <td>16142269</td>\n",
       "      <td>openthread</td>\n",
       "      <td>created</td>\n",
       "      <td>634082470</td>\n",
       "      <td>Looks great overall. Thanks. 👍 \\r\\n\\r\\nOne sug...</td>\n",
       "      <td>20ea399602d93a665a7dd98772e416648e4458ec</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>approved</td>\n",
       "      <td>6342</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6780 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          type                created_at    repo_id  \\\n",
       "15      PullRequestReviewEvent 2022-11-01 16:30:10+00:00   73328905   \n",
       "16      PullRequestReviewEvent 2023-05-03 19:53:49+00:00  102692863   \n",
       "31      PullRequestReviewEvent 2023-05-24 16:28:44+00:00   70124554   \n",
       "40      PullRequestReviewEvent 2023-03-07 16:41:33+00:00  134186172   \n",
       "58      PullRequestReviewEvent 2023-05-23 17:52:28+00:00   71932349   \n",
       "...                        ...                       ...        ...   \n",
       "118341  PullRequestReviewEvent 2022-02-03 15:43:33+00:00    4127088   \n",
       "118343  PullRequestReviewEvent 2021-08-30 22:11:54+00:00    4127088   \n",
       "118357  PullRequestReviewEvent 2021-11-03 11:00:16+00:00   12082924   \n",
       "118437  PullRequestReviewEvent 2021-11-05 21:24:17+00:00   71932349   \n",
       "118447  PullRequestReviewEvent 2021-04-13 01:06:53+00:00   55808787   \n",
       "\n",
       "                         repo_name  actor_id          actor_login    org_id  \\\n",
       "15                  pytorch/vision   2459423              vfdev-5  21003710   \n",
       "16                       onnx/onnx  17606112               leimao  31675368   \n",
       "31               lablup/backend.ai    555156             achimnol  11258248   \n",
       "40                faucetsdn/chewie   2931079              cglewis  23409712   \n",
       "58                 ray-project/ray  92341594    shrekris-anyscale  22125274   \n",
       "...                            ...       ...                  ...       ...   \n",
       "118341  Azure/azure-sdk-for-python  64233642            xinlian12   6844498   \n",
       "118343  Azure/azure-sdk-for-python  43154838           iscai-msft   6844498   \n",
       "118357   zostera/django-bootstrap3  41898282  github-actions[bot]   1092877   \n",
       "118437             ray-project/ray     14922                ericl  22125274   \n",
       "118447       openthread/openthread  17729664               abtink  16142269   \n",
       "\n",
       "          org_login pr_review_action  pr_review_id  \\\n",
       "15          pytorch          created    1163653955   \n",
       "16             onnx          created    1411698255   \n",
       "31           lablup          created    1442326238   \n",
       "40        faucetsdn          created    1329047075   \n",
       "58      ray-project          created    1440284983   \n",
       "...             ...              ...           ...   \n",
       "118341        Azure          created     872031515   \n",
       "118343        Azure          created     742115081   \n",
       "118357      zostera          created     796386825   \n",
       "118437  ray-project          created     799335440   \n",
       "118447   openthread          created     634082470   \n",
       "\n",
       "                                           pr_review_body  \\\n",
       "15                                          LGTM, thanks!   \n",
       "16                                                   None   \n",
       "31                                                          \n",
       "40                                                          \n",
       "58                                                   None   \n",
       "...                                                   ...   \n",
       "118341                                               None   \n",
       "118343                                               None   \n",
       "118357                                                      \n",
       "118437                                                      \n",
       "118447  Looks great overall. Thanks. 👍 \\r\\n\\r\\nOne sug...   \n",
       "\n",
       "                             pr_review_commit_id pr_review_author_association  \\\n",
       "15      54ef3b4a88e306de5739e1b3a82963cb107f7915                 COLLABORATOR   \n",
       "16      345e24d9ad2b2e10621f252c861bedab72a05491                         NONE   \n",
       "31      02e5862302caa1b992f100164bd23289a46d22e3                       MEMBER   \n",
       "40      e73ce623b5da4131227872e4a4f3dc4aa2714cd7                       MEMBER   \n",
       "58      5879fc93333221c601517281cd98da4e3f9c2f91                  CONTRIBUTOR   \n",
       "...                                          ...                          ...   \n",
       "118341  7c098ea7755ebbaaa5a8ff84f0dbafc5655d4c1b                       MEMBER   \n",
       "118343  d7e39c7cf2c9ec9d7f3aa3346d57c9cfdc8b2dbc                  CONTRIBUTOR   \n",
       "118357  406edd7e1090a293bef5551fd0e8435bd2762bbe                         NONE   \n",
       "118437  990ba4da74d1b9a42a80ff673aa13fa6d6f5f429                  CONTRIBUTOR   \n",
       "118447  20ea399602d93a665a7dd98772e416648e4458ec                       MEMBER   \n",
       "\n",
       "       pr_review_state  pr_number  \n",
       "15            approved       6880  \n",
       "16           commented       5010  \n",
       "31            approved       1297  \n",
       "40            approved        424  \n",
       "58           commented      35578  \n",
       "...                ...        ...  \n",
       "118341       commented      22792  \n",
       "118343       commented      20460  \n",
       "118357        approved        588  \n",
       "118437        approved      19924  \n",
       "118447        approved       6342  \n",
       "\n",
       "[6780 rows x 15 columns]"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pullrequestreview[[\n",
    "    'type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "    'pr_review_action', 'pr_review_id', 'pr_review_body', 'pr_review_commit_id', 'pr_review_author_association', \n",
    "    'pr_review_state', 'pr_number'\n",
    "]].to_csv(f'data/github_clean/prReviewEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b918ca71-1813-47de-a08f-81ce831ca727",
   "metadata": {},
   "source": [
    "## Pull Request Review Comment Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "fcfaf7f2-c68c-4f73-8190-4f7bf5543361",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1258992820.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['payload'] = df_pullrequestreviewcomment['payload'].apply(lambda x: {i:x[i] for i in x if i != 'pull_request'})\n"
     ]
    }
   ],
   "source": [
    "df_pullrequestreviewcomment = df[df['type'] == 'PullRequestReviewCommentEvent']\n",
    "df_pullrequestreviewcomment['payload'] = df_pullrequestreviewcomment['payload'].apply(lambda x: {i:x[i] for i in x if i != 'pull_request'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "c9e8bbff-dd83-4c75-bcdb-0bd8b0be7527",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_action'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['action'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_id'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['pull_request_review_id'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_commit_id'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['commit_id'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_site_admin'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['user']['site_admin'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_body'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['body'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_author_association'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['author_association'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_number'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['pull_request_url'].split(\"/\")[-1])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/970570738.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequestreviewcomment['pr_review_comment_reactions'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment'].get('reactions',{}))\n"
     ]
    }
   ],
   "source": [
    "df_pullrequestreviewcomment['pr_review_comment_action'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['action'])\n",
    "df_pullrequestreviewcomment['pr_review_comment_id'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['pull_request_review_id'])\n",
    "df_pullrequestreviewcomment['pr_review_comment_commit_id'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['commit_id'])\n",
    "df_pullrequestreviewcomment['pr_review_comment_site_admin'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['user']['site_admin'])\n",
    "df_pullrequestreviewcomment['pr_review_comment_body'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['body'])\n",
    "df_pullrequestreviewcomment['pr_review_comment_author_association'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['author_association'])\n",
    "df_pullrequestreviewcomment['pr_number'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment']['pull_request_url'].split(\"/\")[-1])\n",
    "df_pullrequestreviewcomment['pr_review_comment_reactions'] = df_pullrequestreviewcomment['payload'].apply(lambda x: x['comment'].get('reactions',{}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "a46c72ea-de4f-4949-bbe4-ea022c6274c2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>pr_review_comment_action</th>\n",
       "      <th>pr_review_comment_id</th>\n",
       "      <th>pr_review_comment_commit_id</th>\n",
       "      <th>pr_review_comment_site_admin</th>\n",
       "      <th>pr_review_comment_body</th>\n",
       "      <th>pr_review_comment_author_association</th>\n",
       "      <th>pr_number</th>\n",
       "      <th>pr_review_comment_reactions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2023-06-12 19:18:33+00:00</td>\n",
       "      <td>92205085</td>\n",
       "      <td>aws/aws-sam-cli</td>\n",
       "      <td>12496191</td>\n",
       "      <td>lucashuy</td>\n",
       "      <td>2232217</td>\n",
       "      <td>aws</td>\n",
       "      <td>created</td>\n",
       "      <td>1475647723</td>\n",
       "      <td>908c0db5efd8e3981bfd158ac0b1422545d5e9ee</td>\n",
       "      <td>False</td>\n",
       "      <td>nit: three metrics</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>5315</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/aws/aws-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2023-03-06 22:48:45+00:00</td>\n",
       "      <td>49976939</td>\n",
       "      <td>learningequality/kolibri</td>\n",
       "      <td>6356129</td>\n",
       "      <td>nucleogenesis</td>\n",
       "      <td>3926525</td>\n",
       "      <td>learningequality</td>\n",
       "      <td>created</td>\n",
       "      <td>1327335366</td>\n",
       "      <td>c84d1f8d3209dad85531f4d59b99844494b7240f</td>\n",
       "      <td>False</td>\n",
       "      <td>&lt;3</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>10161</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/learning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2022-10-11 23:39:34+00:00</td>\n",
       "      <td>4127088</td>\n",
       "      <td>Azure/azure-sdk-for-python</td>\n",
       "      <td>96087589</td>\n",
       "      <td>jalauzon-msft</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>created</td>\n",
       "      <td>1138266879</td>\n",
       "      <td>55c61357643cdf4354a4ed153c6baaa1acbd3f9a</td>\n",
       "      <td>False</td>\n",
       "      <td>I assume we didn't have a file-share soft-dele...</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>26709</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/Azure/az...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2022-11-01 22:05:31+00:00</td>\n",
       "      <td>92205085</td>\n",
       "      <td>aws/aws-sam-cli</td>\n",
       "      <td>3770774</td>\n",
       "      <td>sriram-mv</td>\n",
       "      <td>2232217</td>\n",
       "      <td>aws</td>\n",
       "      <td>created</td>\n",
       "      <td>1164111976</td>\n",
       "      <td>0ef393ffb30aa7f470632d4d87664505efc1cff9</td>\n",
       "      <td>False</td>\n",
       "      <td>`--beta-features` instead of `--beta-feature`</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>4365</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/aws/aws-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2021-04-21 21:29:54+00:00</td>\n",
       "      <td>226992639</td>\n",
       "      <td>googleapis/python-storage</td>\n",
       "      <td>4534622</td>\n",
       "      <td>tritone</td>\n",
       "      <td>16785467</td>\n",
       "      <td>googleapis</td>\n",
       "      <td>created</td>\n",
       "      <td>641555355</td>\n",
       "      <td>4f213d992065082482d51f281d489f214fca3722</td>\n",
       "      <td>False</td>\n",
       "      <td>According to https://github.com/googleapis/goo...</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>418</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118315</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2021-11-05 10:56:30+00:00</td>\n",
       "      <td>71932349</td>\n",
       "      <td>ray-project/ray</td>\n",
       "      <td>18510752</td>\n",
       "      <td>rkooo567</td>\n",
       "      <td>22125274</td>\n",
       "      <td>ray-project</td>\n",
       "      <td>created</td>\n",
       "      <td>798740569</td>\n",
       "      <td>6d2c940b06340858735e2e87c46128058fa56545</td>\n",
       "      <td>False</td>\n",
       "      <td>Can you add the argument name as a comment lik...</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>20075</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/ray-proj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118321</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2021-11-05 04:21:09+00:00</td>\n",
       "      <td>71932349</td>\n",
       "      <td>ray-project/ray</td>\n",
       "      <td>9871461</td>\n",
       "      <td>edoakes</td>\n",
       "      <td>22125274</td>\n",
       "      <td>ray-project</td>\n",
       "      <td>created</td>\n",
       "      <td>798488520</td>\n",
       "      <td>b5d1f0f5ac9e48ac1e612d6c63c47576343d0af0</td>\n",
       "      <td>False</td>\n",
       "      <td>please add what max_pending_calls is set to in...</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>19936</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/ray-proj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118342</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2022-08-19 20:42:01+00:00</td>\n",
       "      <td>138224079</td>\n",
       "      <td>modin-project/modin</td>\n",
       "      <td>10712109</td>\n",
       "      <td>mvashishtha</td>\n",
       "      <td>40475955</td>\n",
       "      <td>modin-project</td>\n",
       "      <td>created</td>\n",
       "      <td>1079193429</td>\n",
       "      <td>cd0a5d1e4a040f016503aa76e2ebf3ff684cb15e</td>\n",
       "      <td>False</td>\n",
       "      <td>Done.</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>4820</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/modin-pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118351</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2022-08-09 15:06:29+00:00</td>\n",
       "      <td>71932349</td>\n",
       "      <td>ray-project/ray</td>\n",
       "      <td>31483498</td>\n",
       "      <td>kouroshHakha</td>\n",
       "      <td>22125274</td>\n",
       "      <td>ray-project</td>\n",
       "      <td>created</td>\n",
       "      <td>1066900756</td>\n",
       "      <td>09e32d48546cc96e410c1a4e50a9444493e89299</td>\n",
       "      <td>False</td>\n",
       "      <td>replay_buffer_config needs to be removed entir...</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>26032</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/ray-proj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118405</th>\n",
       "      <td>PullRequestReviewCommentEvent</td>\n",
       "      <td>2019-06-17 18:45:18+00:00</td>\n",
       "      <td>51040886</td>\n",
       "      <td>Azure/azure-cli</td>\n",
       "      <td>7810433</td>\n",
       "      <td>ahmedelnably</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>created</td>\n",
       "      <td>250680408</td>\n",
       "      <td>9f3265699f693424acc376de17e86842ab60ec45</td>\n",
       "      <td>False</td>\n",
       "      <td>Please use the Azure Portal to create and conf...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>9678</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7937 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 type                created_at    repo_id  \\\n",
       "10      PullRequestReviewCommentEvent 2023-06-12 19:18:33+00:00   92205085   \n",
       "54      PullRequestReviewCommentEvent 2023-03-06 22:48:45+00:00   49976939   \n",
       "67      PullRequestReviewCommentEvent 2022-10-11 23:39:34+00:00    4127088   \n",
       "73      PullRequestReviewCommentEvent 2022-11-01 22:05:31+00:00   92205085   \n",
       "75      PullRequestReviewCommentEvent 2021-04-21 21:29:54+00:00  226992639   \n",
       "...                               ...                       ...        ...   \n",
       "118315  PullRequestReviewCommentEvent 2021-11-05 10:56:30+00:00   71932349   \n",
       "118321  PullRequestReviewCommentEvent 2021-11-05 04:21:09+00:00   71932349   \n",
       "118342  PullRequestReviewCommentEvent 2022-08-19 20:42:01+00:00  138224079   \n",
       "118351  PullRequestReviewCommentEvent 2022-08-09 15:06:29+00:00   71932349   \n",
       "118405  PullRequestReviewCommentEvent 2019-06-17 18:45:18+00:00   51040886   \n",
       "\n",
       "                         repo_name  actor_id    actor_login    org_id  \\\n",
       "10                 aws/aws-sam-cli  12496191       lucashuy   2232217   \n",
       "54        learningequality/kolibri   6356129  nucleogenesis   3926525   \n",
       "67      Azure/azure-sdk-for-python  96087589  jalauzon-msft   6844498   \n",
       "73                 aws/aws-sam-cli   3770774      sriram-mv   2232217   \n",
       "75       googleapis/python-storage   4534622        tritone  16785467   \n",
       "...                            ...       ...            ...       ...   \n",
       "118315             ray-project/ray  18510752       rkooo567  22125274   \n",
       "118321             ray-project/ray   9871461        edoakes  22125274   \n",
       "118342         modin-project/modin  10712109    mvashishtha  40475955   \n",
       "118351             ray-project/ray  31483498   kouroshHakha  22125274   \n",
       "118405             Azure/azure-cli   7810433   ahmedelnably   6844498   \n",
       "\n",
       "               org_login pr_review_comment_action  pr_review_comment_id  \\\n",
       "10                   aws                  created            1475647723   \n",
       "54      learningequality                  created            1327335366   \n",
       "67                 Azure                  created            1138266879   \n",
       "73                   aws                  created            1164111976   \n",
       "75            googleapis                  created             641555355   \n",
       "...                  ...                      ...                   ...   \n",
       "118315       ray-project                  created             798740569   \n",
       "118321       ray-project                  created             798488520   \n",
       "118342     modin-project                  created            1079193429   \n",
       "118351       ray-project                  created            1066900756   \n",
       "118405             Azure                  created             250680408   \n",
       "\n",
       "                     pr_review_comment_commit_id  \\\n",
       "10      908c0db5efd8e3981bfd158ac0b1422545d5e9ee   \n",
       "54      c84d1f8d3209dad85531f4d59b99844494b7240f   \n",
       "67      55c61357643cdf4354a4ed153c6baaa1acbd3f9a   \n",
       "73      0ef393ffb30aa7f470632d4d87664505efc1cff9   \n",
       "75      4f213d992065082482d51f281d489f214fca3722   \n",
       "...                                          ...   \n",
       "118315  6d2c940b06340858735e2e87c46128058fa56545   \n",
       "118321  b5d1f0f5ac9e48ac1e612d6c63c47576343d0af0   \n",
       "118342  cd0a5d1e4a040f016503aa76e2ebf3ff684cb15e   \n",
       "118351  09e32d48546cc96e410c1a4e50a9444493e89299   \n",
       "118405  9f3265699f693424acc376de17e86842ab60ec45   \n",
       "\n",
       "        pr_review_comment_site_admin  \\\n",
       "10                             False   \n",
       "54                             False   \n",
       "67                             False   \n",
       "73                             False   \n",
       "75                             False   \n",
       "...                              ...   \n",
       "118315                         False   \n",
       "118321                         False   \n",
       "118342                         False   \n",
       "118351                         False   \n",
       "118405                         False   \n",
       "\n",
       "                                   pr_review_comment_body  \\\n",
       "10                                     nit: three metrics   \n",
       "54                                                     <3   \n",
       "67      I assume we didn't have a file-share soft-dele...   \n",
       "73          `--beta-features` instead of `--beta-feature`   \n",
       "75      According to https://github.com/googleapis/goo...   \n",
       "...                                                   ...   \n",
       "118315  Can you add the argument name as a comment lik...   \n",
       "118321  please add what max_pending_calls is set to in...   \n",
       "118342                                              Done.   \n",
       "118351  replay_buffer_config needs to be removed entir...   \n",
       "118405  Please use the Azure Portal to create and conf...   \n",
       "\n",
       "       pr_review_comment_author_association pr_number  \\\n",
       "10                              CONTRIBUTOR      5315   \n",
       "54                                   MEMBER     10161   \n",
       "67                                   MEMBER     26709   \n",
       "73                              CONTRIBUTOR      4365   \n",
       "75                              CONTRIBUTOR       418   \n",
       "...                                     ...       ...   \n",
       "118315                          CONTRIBUTOR     20075   \n",
       "118321                          CONTRIBUTOR     19936   \n",
       "118342                          CONTRIBUTOR      4820   \n",
       "118351                          CONTRIBUTOR     26032   \n",
       "118405                                 NONE      9678   \n",
       "\n",
       "                              pr_review_comment_reactions  \n",
       "10      {'url': 'https://api.github.com/repos/aws/aws-...  \n",
       "54      {'url': 'https://api.github.com/repos/learning...  \n",
       "67      {'url': 'https://api.github.com/repos/Azure/az...  \n",
       "73      {'url': 'https://api.github.com/repos/aws/aws-...  \n",
       "75                                                     {}  \n",
       "...                                                   ...  \n",
       "118315  {'url': 'https://api.github.com/repos/ray-proj...  \n",
       "118321  {'url': 'https://api.github.com/repos/ray-proj...  \n",
       "118342  {'url': 'https://api.github.com/repos/modin-pr...  \n",
       "118351  {'url': 'https://api.github.com/repos/ray-proj...  \n",
       "118405                                                 {}  \n",
       "\n",
       "[7937 rows x 16 columns]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pullrequestreviewcomment[[\n",
    "    'type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "    'pr_review_comment_action', 'pr_review_comment_id', 'pr_review_comment_commit_id', 'pr_review_comment_site_admin',\n",
    "    'pr_review_comment_body', 'pr_review_comment_author_association', 'pr_number', 'pr_review_comment_reactions']].to_csv(\n",
    "    f'data/github_clean/prReviewCommentEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a3a499d-7ab2-4281-8f25-2ecad7eca6ff",
   "metadata": {},
   "source": [
    "## Pull Request Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "id": "54404c09-7ffd-48bb-8fd2-e093a9bf7d0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pullrequest = df[df['type'] == 'PullRequestEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "id": "d05a6968-8e33-4bfa-abe5-c0ecbcea5ee8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_merged_by_login'] = df_pullrequest['payload'].apply(\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_merged_by_id'] = df_pullrequest['payload'].apply(\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_merged_by_type'] = df_pullrequest['payload'].apply(\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_merged_by_site_admin'] = df_pullrequest['payload'].apply(\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_label'] = df_pullrequest['payload'].apply(lambda x: [label['name'] for label in x['pull_request']['labels']])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/3378431729.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_pullrequest['pr_patch_url'] = df_pullrequest['payload'].apply(lambda x: x['pull_request']['patch_url'])\n"
     ]
    }
   ],
   "source": [
    "for col in ['id', 'node_id', 'title', 'state', 'locked', 'number', 'body', 'issue_url', \n",
    "            'merged_at', 'closed_at', 'updated_at', 'commits', 'additions', 'deletions', 'changed_files',\n",
    "            'author_association', 'assignees', 'requested_reviewers', 'requested_teams']:\n",
    "    df_pullrequest['pr_'+col] = df_pullrequest['payload'].apply(lambda x: x['pull_request'][col])\n",
    "\n",
    "df_pullrequest['pr_merged_by_login'] = df_pullrequest['payload'].apply(\n",
    "    lambda x: x['pull_request']['merged_by']['login'] if x['pull_request']['merged_by'] != None else '')\n",
    "df_pullrequest['pr_merged_by_id'] = df_pullrequest['payload'].apply(\n",
    "    lambda x: x['pull_request']['merged_by']['id'] if x['pull_request']['merged_by'] != None else '')\n",
    "df_pullrequest['pr_merged_by_type'] = df_pullrequest['payload'].apply(\n",
    "    lambda x: x['pull_request']['merged_by']['type'] if x['pull_request']['merged_by'] != None else '')\n",
    "df_pullrequest['pr_merged_by_site_admin'] = df_pullrequest['payload'].apply(\n",
    "    lambda x: x['pull_request']['merged_by']['site_admin'] if x['pull_request']['merged_by'] != None else '')\n",
    "\n",
    "df_pullrequest['pr_label'] = df_pullrequest['payload'].apply(lambda x: [label['name'] for label in x['pull_request']['labels']])\n",
    "df_pullrequest['pr_patch_url'] = df_pullrequest['payload'].apply(lambda x: x['pull_request']['patch_url'])\n",
    "#df_pullrequest['pr_commits_url'] = df_pullrequest['payload'].apply(lambda x: x['pull_request']['commits_url'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "94ab46f5-4de7-4162-8947-42a14504fa9e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'action': 'closed',\n",
       " 'number': 146,\n",
       " 'pull_request': {'url': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/146',\n",
       "  'id': 1287020216,\n",
       "  'node_id': 'PR_kwDOBggLE85Mtl64',\n",
       "  'html_url': 'https://github.com/WeblateOrg/siphashc/pull/146',\n",
       "  'diff_url': 'https://github.com/WeblateOrg/siphashc/pull/146.diff',\n",
       "  'patch_url': 'https://github.com/WeblateOrg/siphashc/pull/146.patch',\n",
       "  'issue_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/146',\n",
       "  'number': 146,\n",
       "  'state': 'closed',\n",
       "  'locked': False,\n",
       "  'title': 'chore(deps): update peter-evans/enable-pull-request-automerge action to v3',\n",
       "  'user': {'login': 'renovate[bot]',\n",
       "   'id': 29139614,\n",
       "   'node_id': 'MDM6Qm90MjkxMzk2MTQ=',\n",
       "   'avatar_url': 'https://avatars.githubusercontent.com/in/2740?v=4',\n",
       "   'gravatar_id': '',\n",
       "   'url': 'https://api.github.com/users/renovate%5Bbot%5D',\n",
       "   'html_url': 'https://github.com/apps/renovate',\n",
       "   'followers_url': 'https://api.github.com/users/renovate%5Bbot%5D/followers',\n",
       "   'following_url': 'https://api.github.com/users/renovate%5Bbot%5D/following{/other_user}',\n",
       "   'gists_url': 'https://api.github.com/users/renovate%5Bbot%5D/gists{/gist_id}',\n",
       "   'starred_url': 'https://api.github.com/users/renovate%5Bbot%5D/starred{/owner}{/repo}',\n",
       "   'subscriptions_url': 'https://api.github.com/users/renovate%5Bbot%5D/subscriptions',\n",
       "   'organizations_url': 'https://api.github.com/users/renovate%5Bbot%5D/orgs',\n",
       "   'repos_url': 'https://api.github.com/users/renovate%5Bbot%5D/repos',\n",
       "   'events_url': 'https://api.github.com/users/renovate%5Bbot%5D/events{/privacy}',\n",
       "   'received_events_url': 'https://api.github.com/users/renovate%5Bbot%5D/received_events',\n",
       "   'type': 'Bot',\n",
       "   'site_admin': False},\n",
       "  'body': \"[![Mend Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com)\\n\\nThis PR contains the following updates:\\n\\n| Package | Type | Update | Change |\\n|---|---|---|---|\\n| [peter-evans/enable-pull-request-automerge](https://togithub.com/peter-evans/enable-pull-request-automerge) | action | major | `v2` -> `v3` |\\n\\n---\\n\\n### Release Notes\\n\\n<details>\\n<summary>peter-evans/enable-pull-request-automerge</summary>\\n\\n### [`v3`](https://togithub.com/peter-evans/enable-pull-request-automerge/compare/v2...v3)\\n\\n[Compare Source](https://togithub.com/peter-evans/enable-pull-request-automerge/compare/v2...v3)\\n\\n</details>\\n\\n---\\n\\n### Configuration\\n\\n📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined).\\n\\n🚦 **Automerge**: Enabled.\\n\\n♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox.\\n\\n🔕 **Ignore**: Close this PR and you won't be reminded about this update again.\\n\\n---\\n\\n - [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check this box\\n\\n---\\n\\nThis PR has been generated by [Mend Renovate](https://www.mend.io/free-developer-tools/renovate/). View repository job log [here](https://app.renovatebot.com/dashboard#github/WeblateOrg/siphashc).\\n<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNS4xNC4yIiwidXBkYXRlZEluVmVyIjoiMzUuMTQuMiJ9-->\\n\",\n",
       "  'created_at': '2023-03-23T08:49:57Z',\n",
       "  'updated_at': '2023-03-23T09:06:39Z',\n",
       "  'closed_at': '2023-03-23T09:06:38Z',\n",
       "  'merged_at': '2023-03-23T09:06:38Z',\n",
       "  'merge_commit_sha': '1a66ea4dd874d5b5ccd46366abb8db6f1a56a6d1',\n",
       "  'assignee': None,\n",
       "  'assignees': [],\n",
       "  'requested_reviewers': [],\n",
       "  'requested_teams': [],\n",
       "  'labels': [{'id': 2253498604,\n",
       "    'node_id': 'MDU6TGFiZWwyMjUzNDk4NjA0',\n",
       "    'url': 'https://api.github.com/repos/WeblateOrg/siphashc/labels/dependencies',\n",
       "    'name': 'dependencies',\n",
       "    'color': 'e3f49c',\n",
       "    'default': False,\n",
       "    'description': 'Pull requests that update a dependency file.'}],\n",
       "  'milestone': None,\n",
       "  'draft': False,\n",
       "  'commits_url': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/146/commits',\n",
       "  'review_comments_url': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/146/comments',\n",
       "  'review_comment_url': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/comments{/number}',\n",
       "  'comments_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/146/comments',\n",
       "  'statuses_url': 'https://api.github.com/repos/WeblateOrg/siphashc/statuses/13053e8f3fb2ad474be901d7af1503e9a76f546a',\n",
       "  'head': {'label': 'WeblateOrg:renovate/peter-evans-enable-pull-request-automerge-3.x',\n",
       "   'ref': 'renovate/peter-evans-enable-pull-request-automerge-3.x',\n",
       "   'sha': '13053e8f3fb2ad474be901d7af1503e9a76f546a',\n",
       "   'user': {'login': 'WeblateOrg',\n",
       "    'id': 18286565,\n",
       "    'node_id': 'MDEyOk9yZ2FuaXphdGlvbjE4Mjg2NTY1',\n",
       "    'avatar_url': 'https://avatars.githubusercontent.com/u/18286565?v=4',\n",
       "    'gravatar_id': '',\n",
       "    'url': 'https://api.github.com/users/WeblateOrg',\n",
       "    'html_url': 'https://github.com/WeblateOrg',\n",
       "    'followers_url': 'https://api.github.com/users/WeblateOrg/followers',\n",
       "    'following_url': 'https://api.github.com/users/WeblateOrg/following{/other_user}',\n",
       "    'gists_url': 'https://api.github.com/users/WeblateOrg/gists{/gist_id}',\n",
       "    'starred_url': 'https://api.github.com/users/WeblateOrg/starred{/owner}{/repo}',\n",
       "    'subscriptions_url': 'https://api.github.com/users/WeblateOrg/subscriptions',\n",
       "    'organizations_url': 'https://api.github.com/users/WeblateOrg/orgs',\n",
       "    'repos_url': 'https://api.github.com/users/WeblateOrg/repos',\n",
       "    'events_url': 'https://api.github.com/users/WeblateOrg/events{/privacy}',\n",
       "    'received_events_url': 'https://api.github.com/users/WeblateOrg/received_events',\n",
       "    'type': 'Organization',\n",
       "    'site_admin': False},\n",
       "   'repo': {'id': 101190419,\n",
       "    'node_id': 'MDEwOlJlcG9zaXRvcnkxMDExOTA0MTk=',\n",
       "    'name': 'siphashc',\n",
       "    'full_name': 'WeblateOrg/siphashc',\n",
       "    'private': False,\n",
       "    'owner': {'login': 'WeblateOrg',\n",
       "     'id': 18286565,\n",
       "     'node_id': 'MDEyOk9yZ2FuaXphdGlvbjE4Mjg2NTY1',\n",
       "     'avatar_url': 'https://avatars.githubusercontent.com/u/18286565?v=4',\n",
       "     'gravatar_id': '',\n",
       "     'url': 'https://api.github.com/users/WeblateOrg',\n",
       "     'html_url': 'https://github.com/WeblateOrg',\n",
       "     'followers_url': 'https://api.github.com/users/WeblateOrg/followers',\n",
       "     'following_url': 'https://api.github.com/users/WeblateOrg/following{/other_user}',\n",
       "     'gists_url': 'https://api.github.com/users/WeblateOrg/gists{/gist_id}',\n",
       "     'starred_url': 'https://api.github.com/users/WeblateOrg/starred{/owner}{/repo}',\n",
       "     'subscriptions_url': 'https://api.github.com/users/WeblateOrg/subscriptions',\n",
       "     'organizations_url': 'https://api.github.com/users/WeblateOrg/orgs',\n",
       "     'repos_url': 'https://api.github.com/users/WeblateOrg/repos',\n",
       "     'events_url': 'https://api.github.com/users/WeblateOrg/events{/privacy}',\n",
       "     'received_events_url': 'https://api.github.com/users/WeblateOrg/received_events',\n",
       "     'type': 'Organization',\n",
       "     'site_admin': False},\n",
       "    'html_url': 'https://github.com/WeblateOrg/siphashc',\n",
       "    'description': 'python c-module for siphash',\n",
       "    'fork': False,\n",
       "    'url': 'https://api.github.com/repos/WeblateOrg/siphashc',\n",
       "    'forks_url': 'https://api.github.com/repos/WeblateOrg/siphashc/forks',\n",
       "    'keys_url': 'https://api.github.com/repos/WeblateOrg/siphashc/keys{/key_id}',\n",
       "    'collaborators_url': 'https://api.github.com/repos/WeblateOrg/siphashc/collaborators{/collaborator}',\n",
       "    'teams_url': 'https://api.github.com/repos/WeblateOrg/siphashc/teams',\n",
       "    'hooks_url': 'https://api.github.com/repos/WeblateOrg/siphashc/hooks',\n",
       "    'issue_events_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/events{/number}',\n",
       "    'events_url': 'https://api.github.com/repos/WeblateOrg/siphashc/events',\n",
       "    'assignees_url': 'https://api.github.com/repos/WeblateOrg/siphashc/assignees{/user}',\n",
       "    'branches_url': 'https://api.github.com/repos/WeblateOrg/siphashc/branches{/branch}',\n",
       "    'tags_url': 'https://api.github.com/repos/WeblateOrg/siphashc/tags',\n",
       "    'blobs_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/blobs{/sha}',\n",
       "    'git_tags_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/tags{/sha}',\n",
       "    'git_refs_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/refs{/sha}',\n",
       "    'trees_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/trees{/sha}',\n",
       "    'statuses_url': 'https://api.github.com/repos/WeblateOrg/siphashc/statuses/{sha}',\n",
       "    'languages_url': 'https://api.github.com/repos/WeblateOrg/siphashc/languages',\n",
       "    'stargazers_url': 'https://api.github.com/repos/WeblateOrg/siphashc/stargazers',\n",
       "    'contributors_url': 'https://api.github.com/repos/WeblateOrg/siphashc/contributors',\n",
       "    'subscribers_url': 'https://api.github.com/repos/WeblateOrg/siphashc/subscribers',\n",
       "    'subscription_url': 'https://api.github.com/repos/WeblateOrg/siphashc/subscription',\n",
       "    'commits_url': 'https://api.github.com/repos/WeblateOrg/siphashc/commits{/sha}',\n",
       "    'git_commits_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/commits{/sha}',\n",
       "    'comments_url': 'https://api.github.com/repos/WeblateOrg/siphashc/comments{/number}',\n",
       "    'issue_comment_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/comments{/number}',\n",
       "    'contents_url': 'https://api.github.com/repos/WeblateOrg/siphashc/contents/{+path}',\n",
       "    'compare_url': 'https://api.github.com/repos/WeblateOrg/siphashc/compare/{base}...{head}',\n",
       "    'merges_url': 'https://api.github.com/repos/WeblateOrg/siphashc/merges',\n",
       "    'archive_url': 'https://api.github.com/repos/WeblateOrg/siphashc/{archive_format}{/ref}',\n",
       "    'downloads_url': 'https://api.github.com/repos/WeblateOrg/siphashc/downloads',\n",
       "    'issues_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues{/number}',\n",
       "    'pulls_url': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls{/number}',\n",
       "    'milestones_url': 'https://api.github.com/repos/WeblateOrg/siphashc/milestones{/number}',\n",
       "    'notifications_url': 'https://api.github.com/repos/WeblateOrg/siphashc/notifications{?since,all,participating}',\n",
       "    'labels_url': 'https://api.github.com/repos/WeblateOrg/siphashc/labels{/name}',\n",
       "    'releases_url': 'https://api.github.com/repos/WeblateOrg/siphashc/releases{/id}',\n",
       "    'deployments_url': 'https://api.github.com/repos/WeblateOrg/siphashc/deployments',\n",
       "    'created_at': '2017-08-23T14:36:00Z',\n",
       "    'updated_at': '2023-02-16T19:27:08Z',\n",
       "    'pushed_at': '2023-03-23T09:06:38Z',\n",
       "    'git_url': 'git://github.com/WeblateOrg/siphashc.git',\n",
       "    'ssh_url': 'git@github.com:WeblateOrg/siphashc.git',\n",
       "    'clone_url': 'https://github.com/WeblateOrg/siphashc.git',\n",
       "    'svn_url': 'https://github.com/WeblateOrg/siphashc',\n",
       "    'homepage': None,\n",
       "    'size': 202,\n",
       "    'stargazers_count': 7,\n",
       "    'watchers_count': 7,\n",
       "    'language': 'C',\n",
       "    'has_issues': True,\n",
       "    'has_projects': False,\n",
       "    'has_downloads': True,\n",
       "    'has_wiki': False,\n",
       "    'has_pages': False,\n",
       "    'has_discussions': False,\n",
       "    'forks_count': 5,\n",
       "    'mirror_url': None,\n",
       "    'archived': False,\n",
       "    'disabled': False,\n",
       "    'open_issues_count': 1,\n",
       "    'license': {'key': 'isc',\n",
       "     'name': 'ISC License',\n",
       "     'spdx_id': 'ISC',\n",
       "     'url': 'https://api.github.com/licenses/isc',\n",
       "     'node_id': 'MDc6TGljZW5zZTEw'},\n",
       "    'allow_forking': True,\n",
       "    'is_template': False,\n",
       "    'web_commit_signoff_required': False,\n",
       "    'topics': ['python', 'siphash'],\n",
       "    'visibility': 'public',\n",
       "    'forks': 5,\n",
       "    'open_issues': 1,\n",
       "    'watchers': 7,\n",
       "    'default_branch': 'main'}},\n",
       "  'base': {'label': 'WeblateOrg:main',\n",
       "   'ref': 'main',\n",
       "   'sha': 'f9ab9ed0322f705da7f0fbd858939a92300cda93',\n",
       "   'user': {'login': 'WeblateOrg',\n",
       "    'id': 18286565,\n",
       "    'node_id': 'MDEyOk9yZ2FuaXphdGlvbjE4Mjg2NTY1',\n",
       "    'avatar_url': 'https://avatars.githubusercontent.com/u/18286565?v=4',\n",
       "    'gravatar_id': '',\n",
       "    'url': 'https://api.github.com/users/WeblateOrg',\n",
       "    'html_url': 'https://github.com/WeblateOrg',\n",
       "    'followers_url': 'https://api.github.com/users/WeblateOrg/followers',\n",
       "    'following_url': 'https://api.github.com/users/WeblateOrg/following{/other_user}',\n",
       "    'gists_url': 'https://api.github.com/users/WeblateOrg/gists{/gist_id}',\n",
       "    'starred_url': 'https://api.github.com/users/WeblateOrg/starred{/owner}{/repo}',\n",
       "    'subscriptions_url': 'https://api.github.com/users/WeblateOrg/subscriptions',\n",
       "    'organizations_url': 'https://api.github.com/users/WeblateOrg/orgs',\n",
       "    'repos_url': 'https://api.github.com/users/WeblateOrg/repos',\n",
       "    'events_url': 'https://api.github.com/users/WeblateOrg/events{/privacy}',\n",
       "    'received_events_url': 'https://api.github.com/users/WeblateOrg/received_events',\n",
       "    'type': 'Organization',\n",
       "    'site_admin': False},\n",
       "   'repo': {'id': 101190419,\n",
       "    'node_id': 'MDEwOlJlcG9zaXRvcnkxMDExOTA0MTk=',\n",
       "    'name': 'siphashc',\n",
       "    'full_name': 'WeblateOrg/siphashc',\n",
       "    'private': False,\n",
       "    'owner': {'login': 'WeblateOrg',\n",
       "     'id': 18286565,\n",
       "     'node_id': 'MDEyOk9yZ2FuaXphdGlvbjE4Mjg2NTY1',\n",
       "     'avatar_url': 'https://avatars.githubusercontent.com/u/18286565?v=4',\n",
       "     'gravatar_id': '',\n",
       "     'url': 'https://api.github.com/users/WeblateOrg',\n",
       "     'html_url': 'https://github.com/WeblateOrg',\n",
       "     'followers_url': 'https://api.github.com/users/WeblateOrg/followers',\n",
       "     'following_url': 'https://api.github.com/users/WeblateOrg/following{/other_user}',\n",
       "     'gists_url': 'https://api.github.com/users/WeblateOrg/gists{/gist_id}',\n",
       "     'starred_url': 'https://api.github.com/users/WeblateOrg/starred{/owner}{/repo}',\n",
       "     'subscriptions_url': 'https://api.github.com/users/WeblateOrg/subscriptions',\n",
       "     'organizations_url': 'https://api.github.com/users/WeblateOrg/orgs',\n",
       "     'repos_url': 'https://api.github.com/users/WeblateOrg/repos',\n",
       "     'events_url': 'https://api.github.com/users/WeblateOrg/events{/privacy}',\n",
       "     'received_events_url': 'https://api.github.com/users/WeblateOrg/received_events',\n",
       "     'type': 'Organization',\n",
       "     'site_admin': False},\n",
       "    'html_url': 'https://github.com/WeblateOrg/siphashc',\n",
       "    'description': 'python c-module for siphash',\n",
       "    'fork': False,\n",
       "    'url': 'https://api.github.com/repos/WeblateOrg/siphashc',\n",
       "    'forks_url': 'https://api.github.com/repos/WeblateOrg/siphashc/forks',\n",
       "    'keys_url': 'https://api.github.com/repos/WeblateOrg/siphashc/keys{/key_id}',\n",
       "    'collaborators_url': 'https://api.github.com/repos/WeblateOrg/siphashc/collaborators{/collaborator}',\n",
       "    'teams_url': 'https://api.github.com/repos/WeblateOrg/siphashc/teams',\n",
       "    'hooks_url': 'https://api.github.com/repos/WeblateOrg/siphashc/hooks',\n",
       "    'issue_events_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/events{/number}',\n",
       "    'events_url': 'https://api.github.com/repos/WeblateOrg/siphashc/events',\n",
       "    'assignees_url': 'https://api.github.com/repos/WeblateOrg/siphashc/assignees{/user}',\n",
       "    'branches_url': 'https://api.github.com/repos/WeblateOrg/siphashc/branches{/branch}',\n",
       "    'tags_url': 'https://api.github.com/repos/WeblateOrg/siphashc/tags',\n",
       "    'blobs_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/blobs{/sha}',\n",
       "    'git_tags_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/tags{/sha}',\n",
       "    'git_refs_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/refs{/sha}',\n",
       "    'trees_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/trees{/sha}',\n",
       "    'statuses_url': 'https://api.github.com/repos/WeblateOrg/siphashc/statuses/{sha}',\n",
       "    'languages_url': 'https://api.github.com/repos/WeblateOrg/siphashc/languages',\n",
       "    'stargazers_url': 'https://api.github.com/repos/WeblateOrg/siphashc/stargazers',\n",
       "    'contributors_url': 'https://api.github.com/repos/WeblateOrg/siphashc/contributors',\n",
       "    'subscribers_url': 'https://api.github.com/repos/WeblateOrg/siphashc/subscribers',\n",
       "    'subscription_url': 'https://api.github.com/repos/WeblateOrg/siphashc/subscription',\n",
       "    'commits_url': 'https://api.github.com/repos/WeblateOrg/siphashc/commits{/sha}',\n",
       "    'git_commits_url': 'https://api.github.com/repos/WeblateOrg/siphashc/git/commits{/sha}',\n",
       "    'comments_url': 'https://api.github.com/repos/WeblateOrg/siphashc/comments{/number}',\n",
       "    'issue_comment_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/comments{/number}',\n",
       "    'contents_url': 'https://api.github.com/repos/WeblateOrg/siphashc/contents/{+path}',\n",
       "    'compare_url': 'https://api.github.com/repos/WeblateOrg/siphashc/compare/{base}...{head}',\n",
       "    'merges_url': 'https://api.github.com/repos/WeblateOrg/siphashc/merges',\n",
       "    'archive_url': 'https://api.github.com/repos/WeblateOrg/siphashc/{archive_format}{/ref}',\n",
       "    'downloads_url': 'https://api.github.com/repos/WeblateOrg/siphashc/downloads',\n",
       "    'issues_url': 'https://api.github.com/repos/WeblateOrg/siphashc/issues{/number}',\n",
       "    'pulls_url': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls{/number}',\n",
       "    'milestones_url': 'https://api.github.com/repos/WeblateOrg/siphashc/milestones{/number}',\n",
       "    'notifications_url': 'https://api.github.com/repos/WeblateOrg/siphashc/notifications{?since,all,participating}',\n",
       "    'labels_url': 'https://api.github.com/repos/WeblateOrg/siphashc/labels{/name}',\n",
       "    'releases_url': 'https://api.github.com/repos/WeblateOrg/siphashc/releases{/id}',\n",
       "    'deployments_url': 'https://api.github.com/repos/WeblateOrg/siphashc/deployments',\n",
       "    'created_at': '2017-08-23T14:36:00Z',\n",
       "    'updated_at': '2023-02-16T19:27:08Z',\n",
       "    'pushed_at': '2023-03-23T09:06:38Z',\n",
       "    'git_url': 'git://github.com/WeblateOrg/siphashc.git',\n",
       "    'ssh_url': 'git@github.com:WeblateOrg/siphashc.git',\n",
       "    'clone_url': 'https://github.com/WeblateOrg/siphashc.git',\n",
       "    'svn_url': 'https://github.com/WeblateOrg/siphashc',\n",
       "    'homepage': None,\n",
       "    'size': 202,\n",
       "    'stargazers_count': 7,\n",
       "    'watchers_count': 7,\n",
       "    'language': 'C',\n",
       "    'has_issues': True,\n",
       "    'has_projects': False,\n",
       "    'has_downloads': True,\n",
       "    'has_wiki': False,\n",
       "    'has_pages': False,\n",
       "    'has_discussions': False,\n",
       "    'forks_count': 5,\n",
       "    'mirror_url': None,\n",
       "    'archived': False,\n",
       "    'disabled': False,\n",
       "    'open_issues_count': 1,\n",
       "    'license': {'key': 'isc',\n",
       "     'name': 'ISC License',\n",
       "     'spdx_id': 'ISC',\n",
       "     'url': 'https://api.github.com/licenses/isc',\n",
       "     'node_id': 'MDc6TGljZW5zZTEw'},\n",
       "    'allow_forking': True,\n",
       "    'is_template': False,\n",
       "    'web_commit_signoff_required': False,\n",
       "    'topics': ['python', 'siphash'],\n",
       "    'visibility': 'public',\n",
       "    'forks': 5,\n",
       "    'open_issues': 1,\n",
       "    'watchers': 7,\n",
       "    'default_branch': 'main'}},\n",
       "  '_links': {'self': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/146'},\n",
       "   'html': {'href': 'https://github.com/WeblateOrg/siphashc/pull/146'},\n",
       "   'issue': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/146'},\n",
       "   'comments': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/issues/146/comments'},\n",
       "   'review_comments': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/146/comments'},\n",
       "   'review_comment': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/comments{/number}'},\n",
       "   'commits': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/pulls/146/commits'},\n",
       "   'statuses': {'href': 'https://api.github.com/repos/WeblateOrg/siphashc/statuses/13053e8f3fb2ad474be901d7af1503e9a76f546a'}},\n",
       "  'author_association': 'CONTRIBUTOR',\n",
       "  'auto_merge': {'enabled_by': {'login': 'renovate[bot]',\n",
       "    'id': 29139614,\n",
       "    'node_id': 'MDM6Qm90MjkxMzk2MTQ=',\n",
       "    'avatar_url': 'https://avatars.githubusercontent.com/in/2740?v=4',\n",
       "    'gravatar_id': '',\n",
       "    'url': 'https://api.github.com/users/renovate%5Bbot%5D',\n",
       "    'html_url': 'https://github.com/apps/renovate',\n",
       "    'followers_url': 'https://api.github.com/users/renovate%5Bbot%5D/followers',\n",
       "    'following_url': 'https://api.github.com/users/renovate%5Bbot%5D/following{/other_user}',\n",
       "    'gists_url': 'https://api.github.com/users/renovate%5Bbot%5D/gists{/gist_id}',\n",
       "    'starred_url': 'https://api.github.com/users/renovate%5Bbot%5D/starred{/owner}{/repo}',\n",
       "    'subscriptions_url': 'https://api.github.com/users/renovate%5Bbot%5D/subscriptions',\n",
       "    'organizations_url': 'https://api.github.com/users/renovate%5Bbot%5D/orgs',\n",
       "    'repos_url': 'https://api.github.com/users/renovate%5Bbot%5D/repos',\n",
       "    'events_url': 'https://api.github.com/users/renovate%5Bbot%5D/events{/privacy}',\n",
       "    'received_events_url': 'https://api.github.com/users/renovate%5Bbot%5D/received_events',\n",
       "    'type': 'Bot',\n",
       "    'site_admin': False},\n",
       "   'merge_method': 'rebase',\n",
       "   'commit_title': None,\n",
       "   'commit_message': None},\n",
       "  'active_lock_reason': None,\n",
       "  'merged': True,\n",
       "  'mergeable': None,\n",
       "  'rebaseable': None,\n",
       "  'mergeable_state': 'unknown',\n",
       "  'merged_by': {'login': 'renovate[bot]',\n",
       "   'id': 29139614,\n",
       "   'node_id': 'MDM6Qm90MjkxMzk2MTQ=',\n",
       "   'avatar_url': 'https://avatars.githubusercontent.com/in/2740?v=4',\n",
       "   'gravatar_id': '',\n",
       "   'url': 'https://api.github.com/users/renovate%5Bbot%5D',\n",
       "   'html_url': 'https://github.com/apps/renovate',\n",
       "   'followers_url': 'https://api.github.com/users/renovate%5Bbot%5D/followers',\n",
       "   'following_url': 'https://api.github.com/users/renovate%5Bbot%5D/following{/other_user}',\n",
       "   'gists_url': 'https://api.github.com/users/renovate%5Bbot%5D/gists{/gist_id}',\n",
       "   'starred_url': 'https://api.github.com/users/renovate%5Bbot%5D/starred{/owner}{/repo}',\n",
       "   'subscriptions_url': 'https://api.github.com/users/renovate%5Bbot%5D/subscriptions',\n",
       "   'organizations_url': 'https://api.github.com/users/renovate%5Bbot%5D/orgs',\n",
       "   'repos_url': 'https://api.github.com/users/renovate%5Bbot%5D/repos',\n",
       "   'events_url': 'https://api.github.com/users/renovate%5Bbot%5D/events{/privacy}',\n",
       "   'received_events_url': 'https://api.github.com/users/renovate%5Bbot%5D/received_events',\n",
       "   'type': 'Bot',\n",
       "   'site_admin': False},\n",
       "  'comments': 0,\n",
       "  'review_comments': 0,\n",
       "  'maintainer_can_modify': False,\n",
       "  'commits': 1,\n",
       "  'additions': 1,\n",
       "  'deletions': 1,\n",
       "  'changed_files': 1}}"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pullrequest['payload'].loc[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "41fba43d-20e6-4350-ab93-1c2b7beed67e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>pr_id</th>\n",
       "      <th>pr_node_id</th>\n",
       "      <th>pr_title</th>\n",
       "      <th>pr_state</th>\n",
       "      <th>pr_locked</th>\n",
       "      <th>pr_number</th>\n",
       "      <th>pr_body</th>\n",
       "      <th>pr_issue_url</th>\n",
       "      <th>pr_merged_at</th>\n",
       "      <th>pr_closed_at</th>\n",
       "      <th>pr_updated_at</th>\n",
       "      <th>pr_commits</th>\n",
       "      <th>pr_additions</th>\n",
       "      <th>pr_deletions</th>\n",
       "      <th>pr_changed_files</th>\n",
       "      <th>pr_author_association</th>\n",
       "      <th>pr_assignees</th>\n",
       "      <th>pr_requested_reviewers</th>\n",
       "      <th>pr_requested_teams</th>\n",
       "      <th>pr_merged_by_login</th>\n",
       "      <th>pr_merged_by_id</th>\n",
       "      <th>pr_merged_by_type</th>\n",
       "      <th>pr_merged_by_site_admin</th>\n",
       "      <th>pr_label</th>\n",
       "      <th>pr_patch_url</th>\n",
       "      <th>pr_commits_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2023-06-18 16:44:04+00:00</td>\n",
       "      <td>104200746</td>\n",
       "      <td>python-openapi/openapi-core</td>\n",
       "      <td>1679024</td>\n",
       "      <td>p1c2u</td>\n",
       "      <td>126442889</td>\n",
       "      <td>python-openapi</td>\n",
       "      <td>1397201387</td>\n",
       "      <td>PR_kwDOBjX6Ks5TR5nr</td>\n",
       "      <td>Bump codecov/codecov-action from 1 to 3</td>\n",
       "      <td>closed</td>\n",
       "      <td>False</td>\n",
       "      <td>601</td>\n",
       "      <td>Bumps [codecov/codecov-action](https://github....</td>\n",
       "      <td>https://api.github.com/repos/python-openapi/op...</td>\n",
       "      <td>2023-06-18T16:44:03Z</td>\n",
       "      <td>2023-06-18T16:44:04Z</td>\n",
       "      <td>2023-06-18T16:44:04Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>p1c2u</td>\n",
       "      <td>1679024</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "      <td>[dependencies, github_actions]</td>\n",
       "      <td>https://github.com/python-openapi/openapi-core...</td>\n",
       "      <td>https://api.github.com/repos/python-openapi/op...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2022-11-01 01:52:07+00:00</td>\n",
       "      <td>4127088</td>\n",
       "      <td>Azure/azure-sdk-for-python</td>\n",
       "      <td>7108081</td>\n",
       "      <td>banibrata</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>1105894802</td>\n",
       "      <td>PR_kwDOAD75cM5B6p2S</td>\n",
       "      <td>[ML] Environment isinstance() check on __eq__</td>\n",
       "      <td>closed</td>\n",
       "      <td>False</td>\n",
       "      <td>27201</td>\n",
       "      <td># Description\\r\\n\\r\\nPlease add an informative...</td>\n",
       "      <td>https://api.github.com/repos/Azure/azure-sdk-f...</td>\n",
       "      <td>2022-11-01T01:52:06Z</td>\n",
       "      <td>2022-11-01T01:52:06Z</td>\n",
       "      <td>2022-11-01T01:52:06Z</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'login': 'hugoaponte', 'id': 13742976, 'node...</td>\n",
       "      <td>[]</td>\n",
       "      <td>banibrata</td>\n",
       "      <td>7108081</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/Azure/azure-sdk-for-python/...</td>\n",
       "      <td>https://api.github.com/repos/Azure/azure-sdk-f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2023-05-03 00:10:12+00:00</td>\n",
       "      <td>44974847</td>\n",
       "      <td>fabiocaccamo/django-admin-interface</td>\n",
       "      <td>41898282</td>\n",
       "      <td>github-actions[bot]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1335965685</td>\n",
       "      <td>PR_kwDOAq5C_85PoTf1</td>\n",
       "      <td>Update pre-commit hooks</td>\n",
       "      <td>open</td>\n",
       "      <td>False</td>\n",
       "      <td>277</td>\n",
       "      <td>Update versions of pre-commit hooks to latest ...</td>\n",
       "      <td>https://api.github.com/repos/fabiocaccamo/djan...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2023-05-03T00:10:11Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/fabiocaccamo/django-admin-i...</td>\n",
       "      <td>https://api.github.com/repos/fabiocaccamo/djan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2023-05-03 22:05:16+00:00</td>\n",
       "      <td>67641996</td>\n",
       "      <td>oneapi-src/oneTBB</td>\n",
       "      <td>59893122</td>\n",
       "      <td>dnmokhov</td>\n",
       "      <td>60144784</td>\n",
       "      <td>oneapi-src</td>\n",
       "      <td>1337456238</td>\n",
       "      <td>PR_kwDOBAgijM5Pt_Zu</td>\n",
       "      <td>Add DLL info for all tbbbind versions instead ...</td>\n",
       "      <td>open</td>\n",
       "      <td>False</td>\n",
       "      <td>1096</td>\n",
       "      <td>### Description \\r\\n_Add a comprehensive descr...</td>\n",
       "      <td>https://api.github.com/repos/oneapi-src/oneTBB...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2023-05-03T22:05:15Z</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/oneapi-src/oneTBB/pull/1096...</td>\n",
       "      <td>https://api.github.com/repos/oneapi-src/oneTBB...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2021-08-12 16:38:52+00:00</td>\n",
       "      <td>91379993</td>\n",
       "      <td>tensorflow/tensorboard</td>\n",
       "      <td>2322480</td>\n",
       "      <td>psybuzz</td>\n",
       "      <td>15658638</td>\n",
       "      <td>tensorflow</td>\n",
       "      <td>711229371</td>\n",
       "      <td>MDExOlB1bGxSZXF1ZXN0NzExMjI5Mzcx</td>\n",
       "      <td>timeseries: restore line charts that lose webg...</td>\n",
       "      <td>open</td>\n",
       "      <td>False</td>\n",
       "      <td>5237</td>\n",
       "      <td>Googlers, see b/196294346.</td>\n",
       "      <td>https://api.github.com/repos/tensorflow/tensor...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2021-08-12T16:38:52Z</td>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>COLLABORATOR</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/tensorflow/tensorboard/pull...</td>\n",
       "      <td>https://api.github.com/repos/tensorflow/tensor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118415</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2019-06-19 16:36:33+00:00</td>\n",
       "      <td>96328821</td>\n",
       "      <td>r9y9/nnmnkwii</td>\n",
       "      <td>1955233</td>\n",
       "      <td>yamachu</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>289786940</td>\n",
       "      <td>MDExOlB1bGxSZXF1ZXN0Mjg5Nzg2OTQw</td>\n",
       "      <td>Fix trim_zeros_frames to trim zeros from the end</td>\n",
       "      <td>open</td>\n",
       "      <td>False</td>\n",
       "      <td>87</td>\n",
       "      <td>Docs of trim_zeros_frames says `Remove trailin...</td>\n",
       "      <td>https://api.github.com/repos/r9y9/nnmnkwii/iss...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2019-06-19T16:36:32Z</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/r9y9/nnmnkwii/pull/87.patch</td>\n",
       "      <td>https://api.github.com/repos/r9y9/nnmnkwii/pul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118416</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2019-06-12 11:14:47+00:00</td>\n",
       "      <td>34963904</td>\n",
       "      <td>spyder-ide/qtpy</td>\n",
       "      <td>365293</td>\n",
       "      <td>ccordoba12</td>\n",
       "      <td>1284937</td>\n",
       "      <td>spyder-ide</td>\n",
       "      <td>287202846</td>\n",
       "      <td>MDExOlB1bGxSZXF1ZXN0Mjg3MjAyODQ2</td>\n",
       "      <td>PR: Add Qt 3D bindings</td>\n",
       "      <td>closed</td>\n",
       "      <td>False</td>\n",
       "      <td>191</td>\n",
       "      <td>Closes #172</td>\n",
       "      <td>https://api.github.com/repos/spyder-ide/qtpy/i...</td>\n",
       "      <td>2019-06-12T11:14:47Z</td>\n",
       "      <td>2019-06-12T11:14:47Z</td>\n",
       "      <td>2019-06-12T11:14:47Z</td>\n",
       "      <td>2</td>\n",
       "      <td>480</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>ccordoba12</td>\n",
       "      <td>365293</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/spyder-ide/qtpy/pull/191.patch</td>\n",
       "      <td>https://api.github.com/repos/spyder-ide/qtpy/p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118424</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2019-06-18 09:12:05+00:00</td>\n",
       "      <td>27888295</td>\n",
       "      <td>SatelliteQE/nailgun</td>\n",
       "      <td>23010485</td>\n",
       "      <td>vijay8451</td>\n",
       "      <td>8754276</td>\n",
       "      <td>SatelliteQE</td>\n",
       "      <td>289176034</td>\n",
       "      <td>MDExOlB1bGxSZXF1ZXN0Mjg5MTc2MDM0</td>\n",
       "      <td>Added support for rebuild_config in hostgroup</td>\n",
       "      <td>open</td>\n",
       "      <td>False</td>\n",
       "      <td>641</td>\n",
       "      <td>- Added support for rebuild_config in hostgrou...</td>\n",
       "      <td>https://api.github.com/repos/SatelliteQE/nailg...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2019-06-18T09:12:05Z</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>COLLABORATOR</td>\n",
       "      <td>[{'login': 'vijay8451', 'id': 23010485, 'node_...</td>\n",
       "      <td>[{'login': 'oshtaier', 'id': 10895065, 'node_i...</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[review]</td>\n",
       "      <td>https://github.com/SatelliteQE/nailgun/pull/64...</td>\n",
       "      <td>https://api.github.com/repos/SatelliteQE/nailg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118427</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2019-06-29 03:21:30+00:00</td>\n",
       "      <td>744916</td>\n",
       "      <td>gem/oq-engine</td>\n",
       "      <td>2463856</td>\n",
       "      <td>micheles</td>\n",
       "      <td>316546</td>\n",
       "      <td>gem</td>\n",
       "      <td>293010027</td>\n",
       "      <td>MDExOlB1bGxSZXF1ZXN0MjkzMDEwMDI3</td>\n",
       "      <td>Included `oq plot_memory` into `oq plot`</td>\n",
       "      <td>open</td>\n",
       "      <td>False</td>\n",
       "      <td>4861</td>\n",
       "      <td>It is always nice to reduce the number of comm...</td>\n",
       "      <td>https://api.github.com/repos/gem/oq-engine/iss...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2019-06-29T03:21:29Z</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "      <td>77</td>\n",
       "      <td>8</td>\n",
       "      <td>CONTRIBUTOR</td>\n",
       "      <td>[{'login': 'micheles', 'id': 2463856, 'node_id...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[internal]</td>\n",
       "      <td>https://github.com/gem/oq-engine/pull/4861.patch</td>\n",
       "      <td>https://api.github.com/repos/gem/oq-engine/pul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118429</th>\n",
       "      <td>PullRequestEvent</td>\n",
       "      <td>2019-06-09 16:01:17+00:00</td>\n",
       "      <td>1039520</td>\n",
       "      <td>ytdl-org/youtube-dl</td>\n",
       "      <td>1908898</td>\n",
       "      <td>dstftw</td>\n",
       "      <td>48381040</td>\n",
       "      <td>ytdl-org</td>\n",
       "      <td>286478881</td>\n",
       "      <td>MDExOlB1bGxSZXF1ZXN0Mjg2NDc4ODgx</td>\n",
       "      <td>Merge pull request #1 from rg3/master</td>\n",
       "      <td>closed</td>\n",
       "      <td>False</td>\n",
       "      <td>21347</td>\n",
       "      <td>Pull request\\r\\n\\r\\n## Please follow the guide...</td>\n",
       "      <td>https://api.github.com/repos/ytdl-org/youtube-...</td>\n",
       "      <td>None</td>\n",
       "      <td>2019-06-09T16:01:16Z</td>\n",
       "      <td>2019-06-09T16:01:16Z</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NONE</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>[]</td>\n",
       "      <td>https://github.com/ytdl-org/youtube-dl/pull/21...</td>\n",
       "      <td>https://api.github.com/repos/ytdl-org/youtube-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11704 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    type                created_at    repo_id  \\\n",
       "1       PullRequestEvent 2023-06-18 16:44:04+00:00  104200746   \n",
       "14      PullRequestEvent 2022-11-01 01:52:07+00:00    4127088   \n",
       "17      PullRequestEvent 2023-05-03 00:10:12+00:00   44974847   \n",
       "18      PullRequestEvent 2023-05-03 22:05:16+00:00   67641996   \n",
       "20      PullRequestEvent 2021-08-12 16:38:52+00:00   91379993   \n",
       "...                  ...                       ...        ...   \n",
       "118415  PullRequestEvent 2019-06-19 16:36:33+00:00   96328821   \n",
       "118416  PullRequestEvent 2019-06-12 11:14:47+00:00   34963904   \n",
       "118424  PullRequestEvent 2019-06-18 09:12:05+00:00   27888295   \n",
       "118427  PullRequestEvent 2019-06-29 03:21:30+00:00     744916   \n",
       "118429  PullRequestEvent 2019-06-09 16:01:17+00:00    1039520   \n",
       "\n",
       "                                  repo_name  actor_id          actor_login  \\\n",
       "1               python-openapi/openapi-core   1679024                p1c2u   \n",
       "14               Azure/azure-sdk-for-python   7108081            banibrata   \n",
       "17      fabiocaccamo/django-admin-interface  41898282  github-actions[bot]   \n",
       "18                        oneapi-src/oneTBB  59893122             dnmokhov   \n",
       "20                   tensorflow/tensorboard   2322480              psybuzz   \n",
       "...                                     ...       ...                  ...   \n",
       "118415                        r9y9/nnmnkwii   1955233              yamachu   \n",
       "118416                      spyder-ide/qtpy    365293           ccordoba12   \n",
       "118424                  SatelliteQE/nailgun  23010485            vijay8451   \n",
       "118427                        gem/oq-engine   2463856             micheles   \n",
       "118429                  ytdl-org/youtube-dl   1908898               dstftw   \n",
       "\n",
       "           org_id       org_login       pr_id  \\\n",
       "1       126442889  python-openapi  1397201387   \n",
       "14        6844498           Azure  1105894802   \n",
       "17            NaN             NaN  1335965685   \n",
       "18       60144784      oneapi-src  1337456238   \n",
       "20       15658638      tensorflow   711229371   \n",
       "...           ...             ...         ...   \n",
       "118415        NaN             NaN   289786940   \n",
       "118416    1284937      spyder-ide   287202846   \n",
       "118424    8754276     SatelliteQE   289176034   \n",
       "118427     316546             gem   293010027   \n",
       "118429   48381040        ytdl-org   286478881   \n",
       "\n",
       "                              pr_node_id  \\\n",
       "1                    PR_kwDOBjX6Ks5TR5nr   \n",
       "14                   PR_kwDOAD75cM5B6p2S   \n",
       "17                   PR_kwDOAq5C_85PoTf1   \n",
       "18                   PR_kwDOBAgijM5Pt_Zu   \n",
       "20      MDExOlB1bGxSZXF1ZXN0NzExMjI5Mzcx   \n",
       "...                                  ...   \n",
       "118415  MDExOlB1bGxSZXF1ZXN0Mjg5Nzg2OTQw   \n",
       "118416  MDExOlB1bGxSZXF1ZXN0Mjg3MjAyODQ2   \n",
       "118424  MDExOlB1bGxSZXF1ZXN0Mjg5MTc2MDM0   \n",
       "118427  MDExOlB1bGxSZXF1ZXN0MjkzMDEwMDI3   \n",
       "118429  MDExOlB1bGxSZXF1ZXN0Mjg2NDc4ODgx   \n",
       "\n",
       "                                                 pr_title pr_state  pr_locked  \\\n",
       "1                 Bump codecov/codecov-action from 1 to 3   closed      False   \n",
       "14          [ML] Environment isinstance() check on __eq__   closed      False   \n",
       "17                                Update pre-commit hooks     open      False   \n",
       "18      Add DLL info for all tbbbind versions instead ...     open      False   \n",
       "20      timeseries: restore line charts that lose webg...     open      False   \n",
       "...                                                   ...      ...        ...   \n",
       "118415   Fix trim_zeros_frames to trim zeros from the end     open      False   \n",
       "118416                             PR: Add Qt 3D bindings   closed      False   \n",
       "118424      Added support for rebuild_config in hostgroup     open      False   \n",
       "118427           Included `oq plot_memory` into `oq plot`     open      False   \n",
       "118429              Merge pull request #1 from rg3/master   closed      False   \n",
       "\n",
       "        pr_number                                            pr_body  \\\n",
       "1             601  Bumps [codecov/codecov-action](https://github....   \n",
       "14          27201  # Description\\r\\n\\r\\nPlease add an informative...   \n",
       "17            277  Update versions of pre-commit hooks to latest ...   \n",
       "18           1096  ### Description \\r\\n_Add a comprehensive descr...   \n",
       "20           5237                         Googlers, see b/196294346.   \n",
       "...           ...                                                ...   \n",
       "118415         87  Docs of trim_zeros_frames says `Remove trailin...   \n",
       "118416        191                                       Closes #172    \n",
       "118424        641  - Added support for rebuild_config in hostgrou...   \n",
       "118427       4861  It is always nice to reduce the number of comm...   \n",
       "118429      21347  Pull request\\r\\n\\r\\n## Please follow the guide...   \n",
       "\n",
       "                                             pr_issue_url  \\\n",
       "1       https://api.github.com/repos/python-openapi/op...   \n",
       "14      https://api.github.com/repos/Azure/azure-sdk-f...   \n",
       "17      https://api.github.com/repos/fabiocaccamo/djan...   \n",
       "18      https://api.github.com/repos/oneapi-src/oneTBB...   \n",
       "20      https://api.github.com/repos/tensorflow/tensor...   \n",
       "...                                                   ...   \n",
       "118415  https://api.github.com/repos/r9y9/nnmnkwii/iss...   \n",
       "118416  https://api.github.com/repos/spyder-ide/qtpy/i...   \n",
       "118424  https://api.github.com/repos/SatelliteQE/nailg...   \n",
       "118427  https://api.github.com/repos/gem/oq-engine/iss...   \n",
       "118429  https://api.github.com/repos/ytdl-org/youtube-...   \n",
       "\n",
       "                pr_merged_at          pr_closed_at         pr_updated_at  \\\n",
       "1       2023-06-18T16:44:03Z  2023-06-18T16:44:04Z  2023-06-18T16:44:04Z   \n",
       "14      2022-11-01T01:52:06Z  2022-11-01T01:52:06Z  2022-11-01T01:52:06Z   \n",
       "17                      None                  None  2023-05-03T00:10:11Z   \n",
       "18                      None                  None  2023-05-03T22:05:15Z   \n",
       "20                      None                  None  2021-08-12T16:38:52Z   \n",
       "...                      ...                   ...                   ...   \n",
       "118415                  None                  None  2019-06-19T16:36:32Z   \n",
       "118416  2019-06-12T11:14:47Z  2019-06-12T11:14:47Z  2019-06-12T11:14:47Z   \n",
       "118424                  None                  None  2019-06-18T09:12:05Z   \n",
       "118427                  None                  None  2019-06-29T03:21:29Z   \n",
       "118429                  None  2019-06-09T16:01:16Z  2019-06-09T16:01:16Z   \n",
       "\n",
       "        pr_commits  pr_additions  pr_deletions  pr_changed_files  \\\n",
       "1                1             1             1                 1   \n",
       "14               4            16            14                 1   \n",
       "17               1             1             1                 1   \n",
       "18               1             1             3                 1   \n",
       "20               1            89             3                12   \n",
       "...            ...           ...           ...               ...   \n",
       "118415           2            11             2                 2   \n",
       "118416           2           480             0                14   \n",
       "118424           1            20             0                 2   \n",
       "118427           3            54            77                 8   \n",
       "118429           1             0             0                 0   \n",
       "\n",
       "       pr_author_association  \\\n",
       "1                CONTRIBUTOR   \n",
       "14                    MEMBER   \n",
       "17               CONTRIBUTOR   \n",
       "18               CONTRIBUTOR   \n",
       "20              COLLABORATOR   \n",
       "...                      ...   \n",
       "118415           CONTRIBUTOR   \n",
       "118416           CONTRIBUTOR   \n",
       "118424          COLLABORATOR   \n",
       "118427           CONTRIBUTOR   \n",
       "118429                  NONE   \n",
       "\n",
       "                                             pr_assignees  \\\n",
       "1                                                      []   \n",
       "14                                                     []   \n",
       "17                                                     []   \n",
       "18                                                     []   \n",
       "20                                                     []   \n",
       "...                                                   ...   \n",
       "118415                                                 []   \n",
       "118416                                                 []   \n",
       "118424  [{'login': 'vijay8451', 'id': 23010485, 'node_...   \n",
       "118427  [{'login': 'micheles', 'id': 2463856, 'node_id...   \n",
       "118429                                                 []   \n",
       "\n",
       "                                   pr_requested_reviewers pr_requested_teams  \\\n",
       "1                                                      []                 []   \n",
       "14      [{'login': 'hugoaponte', 'id': 13742976, 'node...                 []   \n",
       "17                                                     []                 []   \n",
       "18                                                     []                 []   \n",
       "20                                                     []                 []   \n",
       "...                                                   ...                ...   \n",
       "118415                                                 []                 []   \n",
       "118416                                                 []                 []   \n",
       "118424  [{'login': 'oshtaier', 'id': 10895065, 'node_i...                 []   \n",
       "118427                                                 []                 []   \n",
       "118429                                                 []                 []   \n",
       "\n",
       "       pr_merged_by_login pr_merged_by_id pr_merged_by_type  \\\n",
       "1                   p1c2u         1679024              User   \n",
       "14              banibrata         7108081              User   \n",
       "17                                                            \n",
       "18                                                            \n",
       "20                                                            \n",
       "...                   ...             ...               ...   \n",
       "118415                                                        \n",
       "118416         ccordoba12          365293              User   \n",
       "118424                                                        \n",
       "118427                                                        \n",
       "118429                                                        \n",
       "\n",
       "       pr_merged_by_site_admin                        pr_label  \\\n",
       "1                        False  [dependencies, github_actions]   \n",
       "14                       False                              []   \n",
       "17                                                          []   \n",
       "18                                                          []   \n",
       "20                                                          []   \n",
       "...                        ...                             ...   \n",
       "118415                                                      []   \n",
       "118416                   False                              []   \n",
       "118424                                                [review]   \n",
       "118427                                              [internal]   \n",
       "118429                                                      []   \n",
       "\n",
       "                                             pr_patch_url  \\\n",
       "1       https://github.com/python-openapi/openapi-core...   \n",
       "14      https://github.com/Azure/azure-sdk-for-python/...   \n",
       "17      https://github.com/fabiocaccamo/django-admin-i...   \n",
       "18      https://github.com/oneapi-src/oneTBB/pull/1096...   \n",
       "20      https://github.com/tensorflow/tensorboard/pull...   \n",
       "...                                                   ...   \n",
       "118415     https://github.com/r9y9/nnmnkwii/pull/87.patch   \n",
       "118416  https://github.com/spyder-ide/qtpy/pull/191.patch   \n",
       "118424  https://github.com/SatelliteQE/nailgun/pull/64...   \n",
       "118427   https://github.com/gem/oq-engine/pull/4861.patch   \n",
       "118429  https://github.com/ytdl-org/youtube-dl/pull/21...   \n",
       "\n",
       "                                           pr_commits_url  \n",
       "1       https://api.github.com/repos/python-openapi/op...  \n",
       "14      https://api.github.com/repos/Azure/azure-sdk-f...  \n",
       "17      https://api.github.com/repos/fabiocaccamo/djan...  \n",
       "18      https://api.github.com/repos/oneapi-src/oneTBB...  \n",
       "20      https://api.github.com/repos/tensorflow/tensor...  \n",
       "...                                                   ...  \n",
       "118415  https://api.github.com/repos/r9y9/nnmnkwii/pul...  \n",
       "118416  https://api.github.com/repos/spyder-ide/qtpy/p...  \n",
       "118424  https://api.github.com/repos/SatelliteQE/nailg...  \n",
       "118427  https://api.github.com/repos/gem/oq-engine/pul...  \n",
       "118429  https://api.github.com/repos/ytdl-org/youtube-...  \n",
       "\n",
       "[11704 rows x 34 columns]"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pullrequest[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "                'pr_id', 'pr_node_id', 'pr_title', 'pr_state', 'pr_locked',\n",
    "                'pr_number', 'pr_body', 'pr_issue_url', 'pr_merged_at',\n",
    "                'pr_closed_at', 'pr_updated_at', 'pr_commits', 'pr_additions', 'pr_deletions',\n",
    "                'pr_changed_files', 'pr_author_association', 'pr_assignees', 'pr_requested_reviewers', \n",
    "                'pr_requested_teams', 'pr_merged_by_login', 'pr_merged_by_id',\n",
    "                'pr_merged_by_type', 'pr_merged_by_site_admin', 'pr_label', 'pr_patch_url',\n",
    "                'pr_commits_url']].to_csv(\n",
    "    f'data/github_clean/prEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fe45f01-1009-4a18-aab0-42721f9e4b2f",
   "metadata": {},
   "source": [
    "## Push Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "54324ddd-221f-4218-b2b0-d35f54476089",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_push = df[df['type'] == 'PushEvent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "f144598d-06a9-43a4-8ca8-15cb946ed456",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/226478088.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_push['push_id'] = df_push['payload'].apply(lambda x: x['push_id'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/226478088.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_push['push_size'] = df_push['payload'].apply(lambda x: x['size'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/226478088.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_push['push_size_distinct'] = df_push['payload'].apply(lambda x: x['distinct_size'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/226478088.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_push['push_before'] = df_push['payload'].apply(lambda x: x['before'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/226478088.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_push['push_head'] = df_push['payload'].apply(lambda x: x['head'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/226478088.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_push['commit_urls'] = df_push['payload'].apply(lambda x: [ele['url'] for ele in x['commits']])\n"
     ]
    }
   ],
   "source": [
    "df_push['push_id'] = df_push['payload'].apply(lambda x: x['push_id'])\n",
    "df_push['push_size'] = df_push['payload'].apply(lambda x: x['size'])\n",
    "df_push['push_size_distinct'] = df_push['payload'].apply(lambda x: x['distinct_size'])\n",
    "df_push['push_before'] = df_push['payload'].apply(lambda x: x['before'])\n",
    "df_push['push_head'] = df_push['payload'].apply(lambda x: x['head'])\n",
    "df_push['commit_urls'] = df_push['payload'].apply(lambda x: [ele['url'] for ele in x['commits']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "id": "fa47e94d-9dfd-40f8-bea8-8658fcdecde3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCommits(repo_info, before, head, original_urls):\n",
    "    api_url = f\"https://api.github.com/repos/{repo_info}/compare/{before}...{head}\"\n",
    "    try:\n",
    "        with requests.get(api_url, auth=(username,token)) as url:\n",
    "            data = url.json()\n",
    "        commits = data['commits']\n",
    "        return [c['url'] for c in commits]\n",
    "    except:\n",
    "        return original_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "id": "6e901d09-6d00-4f0f-82c5-4dc6099e7101",
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_info = \"ray-project/ray\"\n",
    "before = \"c498e204457b0454566a884ad09876baea519536\"\n",
    "head = \"5d56c6d1b99dd9778e19798bf9056166f42bafb1\"\n",
    "api_url = f\"https://api.github.com/repos/{repo_info}/compare/{before}...{head}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "bb159127-377a-481d-affd-7d0fd922f9f4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'username' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[281], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(api_url, auth\u001b[38;5;241m=\u001b[39m(\u001b[43musername\u001b[49m,token)) \u001b[38;5;28;01mas\u001b[39;00m url:\n\u001b[1;32m      2\u001b[0m     data \u001b[38;5;241m=\u001b[39m url\u001b[38;5;241m.\u001b[39mjson()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'username' is not defined"
     ]
    }
   ],
   "source": [
    "with requests.get(api_url, auth=(username,token)) as url:\n",
    "    data = url.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "a01fe007-56ea-4772-839b-d206e9cb5ea4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 346/346 [00:00<00:00, 28773.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.4 ms, sys: 6.53 ms, total: 29.9 ms\n",
      "Wall time: 27.9 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# we'll still be missing commits because of pushes that exceed size 250\n",
    "min20_commits = df_push.query('push_size>20').index\n",
    "df_push.loc[min20_commits, 'commit_urls'] = df_push.query('push_size>20').progress_apply(\n",
    "    lambda x: getCommits(x['repo_name'], x['push_before'], x['push_head'], x['commit_urls']), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "52c27d5b-d2bd-4ab2-aea1-35d0fc691405",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>push_id</th>\n",
       "      <th>push_size</th>\n",
       "      <th>push_size_distinct</th>\n",
       "      <th>push_before</th>\n",
       "      <th>push_head</th>\n",
       "      <th>commit_urls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2023-06-18 08:49:37+00:00</td>\n",
       "      <td>106995518</td>\n",
       "      <td>funilrys/PyFunceble</td>\n",
       "      <td>14127285</td>\n",
       "      <td>funilrys</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14022174729</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>18909a70b1fb731cc9eed5ced98b474818c86b97</td>\n",
       "      <td>ecb228bd4190583d11e9ea731b17c4e4d0853036</td>\n",
       "      <td>https://api.github.com/repos/funilrys/PyFunceb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2023-06-15 04:59:25+00:00</td>\n",
       "      <td>80464421</td>\n",
       "      <td>Erotemic/ubelt</td>\n",
       "      <td>3186211</td>\n",
       "      <td>Erotemic</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13990137493</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>c891e4ccb05e364bf85bead7e031d3103d6dd4a6</td>\n",
       "      <td>7e70dc1028341dc1f18ba00b90e7605f8e653f72</td>\n",
       "      <td>https://api.github.com/repos/Erotemic/ubelt/co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2022-10-10 17:23:58+00:00</td>\n",
       "      <td>46218363</td>\n",
       "      <td>nnseva/django-jsoneditor</td>\n",
       "      <td>496812</td>\n",
       "      <td>nnseva</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11282713427</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>bf5e9b432202082cecc724acdd270b8ef4288722</td>\n",
       "      <td>814954a8a2603692692d84d84e2d181a3f0ab247</td>\n",
       "      <td>https://api.github.com/repos/nnseva/django-jso...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2023-03-06 11:50:49+00:00</td>\n",
       "      <td>125082278</td>\n",
       "      <td>finbourne/lusid-sdk-python</td>\n",
       "      <td>24606714</td>\n",
       "      <td>finbourne-bot</td>\n",
       "      <td>22306292</td>\n",
       "      <td>finbourne</td>\n",
       "      <td>12839245022</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>91a466ea4735e4a6f00a9482447208f1fa496f23</td>\n",
       "      <td>a8e818bf5315af4196f0cb4c49dcd2e6c5c6bc64</td>\n",
       "      <td>https://api.github.com/repos/finbourne/lusid-s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2023-06-04 15:01:48+00:00</td>\n",
       "      <td>106127589</td>\n",
       "      <td>commitizen-tools/commitizen</td>\n",
       "      <td>2585816</td>\n",
       "      <td>woile</td>\n",
       "      <td>62252524</td>\n",
       "      <td>commitizen-tools</td>\n",
       "      <td>13864259166</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>72bfabea8acdcbfca7e952c42077f48610fe5135</td>\n",
       "      <td>3771b8f0f12c1ee11cc7a3e011be724fe6adef20</td>\n",
       "      <td>https://api.github.com/repos/commitizen-tools/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118426</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2019-06-03 06:24:53+00:00</td>\n",
       "      <td>105655103</td>\n",
       "      <td>yonkornilov/opus-api</td>\n",
       "      <td>10080107</td>\n",
       "      <td>yonkornilov</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3672988253</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>ed5e22f906fc234b90c415ded5679e6500f46923</td>\n",
       "      <td>906d22b7135758af8a8fc7158f1863dab9b66069</td>\n",
       "      <td>https://api.github.com/repos/yonkornilov/opus-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118431</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2019-06-27 16:49:56+00:00</td>\n",
       "      <td>109688468</td>\n",
       "      <td>polyaxon/polyaxon-k8s</td>\n",
       "      <td>1261626</td>\n",
       "      <td>mouradmourafiq</td>\n",
       "      <td>24544827</td>\n",
       "      <td>polyaxon</td>\n",
       "      <td>3760455059</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14bf261517d6e488dbcb3d9871e7a4ed36211e44</td>\n",
       "      <td>b3047578df462ce3c51a00d0a03f973f6b8c581a</td>\n",
       "      <td>https://api.github.com/repos/polyaxon/polyaxon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118434</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2021-11-09 15:23:51+00:00</td>\n",
       "      <td>19141754</td>\n",
       "      <td>opendatateam/udata</td>\n",
       "      <td>57526019</td>\n",
       "      <td>quaxsze</td>\n",
       "      <td>19330690</td>\n",
       "      <td>opendatateam</td>\n",
       "      <td>8346493116</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0b58ed3fb4460c166aacf586903b8685e899592f</td>\n",
       "      <td>dc525b9aface2d97d33dae62c20284031b30f324</td>\n",
       "      <td>https://api.github.com/repos/opendatateam/udat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118436</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2021-08-30 20:14:37+00:00</td>\n",
       "      <td>15592459</td>\n",
       "      <td>SFDO-Tooling/CumulusCI</td>\n",
       "      <td>49014</td>\n",
       "      <td>davisagli</td>\n",
       "      <td>42554011</td>\n",
       "      <td>SFDO-Tooling</td>\n",
       "      <td>7829550490</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>2735e329f2b56f137c5c6529a56809c5d6a9de72</td>\n",
       "      <td>12a5ac2c85d0ca786c3063cedd17afa99ba2d25d</td>\n",
       "      <td>https://api.github.com/repos/SFDO-Tooling/Cumu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118439</th>\n",
       "      <td>PushEvent</td>\n",
       "      <td>2022-02-08 08:49:20+00:00</td>\n",
       "      <td>37182318</td>\n",
       "      <td>brainix/pottery</td>\n",
       "      <td>1017698</td>\n",
       "      <td>brainix</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9038419651</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4a2a83c3b289d90a841f5d9fa1d3658eb1675af5</td>\n",
       "      <td>b164c4b255e223871c2d8f42cd9befcc20045cf0</td>\n",
       "      <td>https://api.github.com/repos/brainix/pottery/c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15829 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             type                created_at    repo_id  \\\n",
       "0       PushEvent 2023-06-18 08:49:37+00:00  106995518   \n",
       "7       PushEvent 2023-06-15 04:59:25+00:00   80464421   \n",
       "45      PushEvent 2022-10-10 17:23:58+00:00   46218363   \n",
       "53      PushEvent 2023-03-06 11:50:49+00:00  125082278   \n",
       "79      PushEvent 2023-06-04 15:01:48+00:00  106127589   \n",
       "...           ...                       ...        ...   \n",
       "118426  PushEvent 2019-06-03 06:24:53+00:00  105655103   \n",
       "118431  PushEvent 2019-06-27 16:49:56+00:00  109688468   \n",
       "118434  PushEvent 2021-11-09 15:23:51+00:00   19141754   \n",
       "118436  PushEvent 2021-08-30 20:14:37+00:00   15592459   \n",
       "118439  PushEvent 2022-02-08 08:49:20+00:00   37182318   \n",
       "\n",
       "                          repo_name  actor_id     actor_login    org_id  \\\n",
       "0               funilrys/PyFunceble  14127285        funilrys       NaN   \n",
       "7                    Erotemic/ubelt   3186211        Erotemic       NaN   \n",
       "45         nnseva/django-jsoneditor    496812          nnseva       NaN   \n",
       "53       finbourne/lusid-sdk-python  24606714   finbourne-bot  22306292   \n",
       "79      commitizen-tools/commitizen   2585816           woile  62252524   \n",
       "...                             ...       ...             ...       ...   \n",
       "118426         yonkornilov/opus-api  10080107     yonkornilov       NaN   \n",
       "118431        polyaxon/polyaxon-k8s   1261626  mouradmourafiq  24544827   \n",
       "118434           opendatateam/udata  57526019         quaxsze  19330690   \n",
       "118436       SFDO-Tooling/CumulusCI     49014       davisagli  42554011   \n",
       "118439              brainix/pottery   1017698         brainix       NaN   \n",
       "\n",
       "               org_login      push_id  push_size  push_size_distinct  \\\n",
       "0                    NaN  14022174729          1                   1   \n",
       "7                    NaN  13990137493          1                   1   \n",
       "45                   NaN  11282713427          1                   1   \n",
       "53             finbourne  12839245022          1                   1   \n",
       "79      commitizen-tools  13864259166          1                   1   \n",
       "...                  ...          ...        ...                 ...   \n",
       "118426               NaN   3672988253          1                   1   \n",
       "118431          polyaxon   3760455059          1                   1   \n",
       "118434      opendatateam   8346493116          1                   1   \n",
       "118436      SFDO-Tooling   7829550490         10                   9   \n",
       "118439               NaN   9038419651          1                   1   \n",
       "\n",
       "                                     push_before  \\\n",
       "0       18909a70b1fb731cc9eed5ced98b474818c86b97   \n",
       "7       c891e4ccb05e364bf85bead7e031d3103d6dd4a6   \n",
       "45      bf5e9b432202082cecc724acdd270b8ef4288722   \n",
       "53      91a466ea4735e4a6f00a9482447208f1fa496f23   \n",
       "79      72bfabea8acdcbfca7e952c42077f48610fe5135   \n",
       "...                                          ...   \n",
       "118426  ed5e22f906fc234b90c415ded5679e6500f46923   \n",
       "118431  14bf261517d6e488dbcb3d9871e7a4ed36211e44   \n",
       "118434  0b58ed3fb4460c166aacf586903b8685e899592f   \n",
       "118436  2735e329f2b56f137c5c6529a56809c5d6a9de72   \n",
       "118439  4a2a83c3b289d90a841f5d9fa1d3658eb1675af5   \n",
       "\n",
       "                                       push_head  \\\n",
       "0       ecb228bd4190583d11e9ea731b17c4e4d0853036   \n",
       "7       7e70dc1028341dc1f18ba00b90e7605f8e653f72   \n",
       "45      814954a8a2603692692d84d84e2d181a3f0ab247   \n",
       "53      a8e818bf5315af4196f0cb4c49dcd2e6c5c6bc64   \n",
       "79      3771b8f0f12c1ee11cc7a3e011be724fe6adef20   \n",
       "...                                          ...   \n",
       "118426  906d22b7135758af8a8fc7158f1863dab9b66069   \n",
       "118431  b3047578df462ce3c51a00d0a03f973f6b8c581a   \n",
       "118434  dc525b9aface2d97d33dae62c20284031b30f324   \n",
       "118436  12a5ac2c85d0ca786c3063cedd17afa99ba2d25d   \n",
       "118439  b164c4b255e223871c2d8f42cd9befcc20045cf0   \n",
       "\n",
       "                                              commit_urls  \n",
       "0       https://api.github.com/repos/funilrys/PyFunceb...  \n",
       "7       https://api.github.com/repos/Erotemic/ubelt/co...  \n",
       "45      https://api.github.com/repos/nnseva/django-jso...  \n",
       "53      https://api.github.com/repos/finbourne/lusid-s...  \n",
       "79      https://api.github.com/repos/commitizen-tools/...  \n",
       "...                                                   ...  \n",
       "118426  https://api.github.com/repos/yonkornilov/opus-...  \n",
       "118431  https://api.github.com/repos/polyaxon/polyaxon...  \n",
       "118434  https://api.github.com/repos/opendatateam/udat...  \n",
       "118436  https://api.github.com/repos/SFDO-Tooling/Cumu...  \n",
       "118439  https://api.github.com/repos/brainix/pottery/c...  \n",
       "\n",
       "[15829 rows x 14 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_push[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "         'push_id', 'push_size', 'push_size_distinct', 'push_before', 'push_head', 'commit_urls']].to_csv(\n",
    "    f'data/github_clean/pushEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3083cf1-8d15-4d1f-9c25-a4d181e37b47",
   "metadata": {},
   "source": [
    "## Watch Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9b3f8d9d-c999-479b-aca4-81f2b1dd66a9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2023-06-18 10:31:27+00:00</td>\n",
       "      <td>2121420</td>\n",
       "      <td>pypiserver/pypiserver</td>\n",
       "      <td>108942546</td>\n",
       "      <td>fanhan-ceva</td>\n",
       "      <td>9453664</td>\n",
       "      <td>pypiserver</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2023-06-18 04:59:12+00:00</td>\n",
       "      <td>90194616</td>\n",
       "      <td>pydantic/pydantic</td>\n",
       "      <td>18145538</td>\n",
       "      <td>junjie18</td>\n",
       "      <td>110818415</td>\n",
       "      <td>pydantic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2022-09-25 09:19:27+00:00</td>\n",
       "      <td>41179827</td>\n",
       "      <td>Rapptz/discord.py</td>\n",
       "      <td>7645717</td>\n",
       "      <td>BEpresent</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2022-09-25 02:35:58+00:00</td>\n",
       "      <td>60273704</td>\n",
       "      <td>mozilla/DeepSpeech</td>\n",
       "      <td>17973877</td>\n",
       "      <td>ryanlararr</td>\n",
       "      <td>131524</td>\n",
       "      <td>mozilla</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2023-06-14 08:11:20+00:00</td>\n",
       "      <td>3678981</td>\n",
       "      <td>sripathikrishnan/redis-rdb-tools</td>\n",
       "      <td>3191747</td>\n",
       "      <td>sun2008mt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118438</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2021-04-08 12:02:20+00:00</td>\n",
       "      <td>6007295</td>\n",
       "      <td>fxsjy/jieba</td>\n",
       "      <td>50047598</td>\n",
       "      <td>SokachH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118440</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2022-02-08 14:29:38+00:00</td>\n",
       "      <td>15169689</td>\n",
       "      <td>vmware/pyvmomi</td>\n",
       "      <td>7090372</td>\n",
       "      <td>martialblog</td>\n",
       "      <td>473334</td>\n",
       "      <td>vmware</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118441</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2021-11-16 19:13:32+00:00</td>\n",
       "      <td>769951</td>\n",
       "      <td>mhe/pynrrd</td>\n",
       "      <td>7096950</td>\n",
       "      <td>TylerSpears</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118446</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2022-02-02 08:35:01+00:00</td>\n",
       "      <td>83844720</td>\n",
       "      <td>ageitgey/face_recognition</td>\n",
       "      <td>47769757</td>\n",
       "      <td>talhakhan7640</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118448</th>\n",
       "      <td>WatchEvent</td>\n",
       "      <td>2022-02-23 12:54:56+00:00</td>\n",
       "      <td>15938073</td>\n",
       "      <td>JelteF/PyLaTeX</td>\n",
       "      <td>9052456</td>\n",
       "      <td>wyugue</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26875 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              type                created_at   repo_id  \\\n",
       "3       WatchEvent 2023-06-18 10:31:27+00:00   2121420   \n",
       "4       WatchEvent 2023-06-18 04:59:12+00:00  90194616   \n",
       "11      WatchEvent 2022-09-25 09:19:27+00:00  41179827   \n",
       "13      WatchEvent 2022-09-25 02:35:58+00:00  60273704   \n",
       "25      WatchEvent 2023-06-14 08:11:20+00:00   3678981   \n",
       "...            ...                       ...       ...   \n",
       "118438  WatchEvent 2021-04-08 12:02:20+00:00   6007295   \n",
       "118440  WatchEvent 2022-02-08 14:29:38+00:00  15169689   \n",
       "118441  WatchEvent 2021-11-16 19:13:32+00:00    769951   \n",
       "118446  WatchEvent 2022-02-02 08:35:01+00:00  83844720   \n",
       "118448  WatchEvent 2022-02-23 12:54:56+00:00  15938073   \n",
       "\n",
       "                               repo_name   actor_id    actor_login     org_id  \\\n",
       "3                  pypiserver/pypiserver  108942546    fanhan-ceva    9453664   \n",
       "4                      pydantic/pydantic   18145538       junjie18  110818415   \n",
       "11                     Rapptz/discord.py    7645717      BEpresent        NaN   \n",
       "13                    mozilla/DeepSpeech   17973877     ryanlararr     131524   \n",
       "25      sripathikrishnan/redis-rdb-tools    3191747      sun2008mt        NaN   \n",
       "...                                  ...        ...            ...        ...   \n",
       "118438                       fxsjy/jieba   50047598        SokachH        NaN   \n",
       "118440                    vmware/pyvmomi    7090372    martialblog     473334   \n",
       "118441                        mhe/pynrrd    7096950    TylerSpears        NaN   \n",
       "118446         ageitgey/face_recognition   47769757  talhakhan7640        NaN   \n",
       "118448                    JelteF/PyLaTeX    9052456         wyugue        NaN   \n",
       "\n",
       "         org_login  \n",
       "3       pypiserver  \n",
       "4         pydantic  \n",
       "11             NaN  \n",
       "13         mozilla  \n",
       "25             NaN  \n",
       "...            ...  \n",
       "118438         NaN  \n",
       "118440      vmware  \n",
       "118441         NaN  \n",
       "118446         NaN  \n",
       "118448         NaN  \n",
       "\n",
       "[26875 rows x 8 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_watch = df[df['type'] == 'WatchEvent']\n",
    "df_watch[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login']].to_csv(\n",
    "    f'data/github_clean/watchEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "975ddc53-4550-44d3-9efe-9d38c44342e1",
   "metadata": {},
   "source": [
    "# Issues"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "473b3140-338b-464d-b611-69b2c7652756",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "## Issues Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "92d5a19c-e4b1-4259-836c-70127caf1b37",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_action'] = df_issues['payload'].apply(lambda x: x['action'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_title'] = df_issues['payload'].apply(lambda x: x['issue']['title'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_labels'] = df_issues['payload'].apply(lambda x: x['issue']['labels'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_assignees'] = df_issues['payload'].apply(lambda x: x['issue']['assignees'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_count'] = df_issues['payload'].apply(lambda x: x['issue']['comments'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_body'] = df_issues['payload'].apply(lambda x: x['issue']['body'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_reactions'] = df_issues['payload'].apply(lambda x: x['issue'].get('reactions', {}))\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_44703/1978781205.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issues['issuecomment_state_reason'] = df_issues['payload'].apply(lambda x: x['issue'].get('state_reason', {}))\n"
     ]
    }
   ],
   "source": [
    "df_issues = df[df['type'] == 'IssuesEvent']\n",
    "df_issues['issue_action'] = df_issues['payload'].apply(lambda x: x['action'])\n",
    "df_issues['issue_title'] = df_issues['payload'].apply(lambda x: x['issue']['title'])\n",
    "df_issues['issue_labels'] = df_issues['payload'].apply(lambda x: x['issue']['labels'])\n",
    "df_issues['issue_assignees'] = df_issues['payload'].apply(lambda x: x['issue']['assignees'])\n",
    "df_issues['issue_count'] = df_issues['payload'].apply(lambda x: x['issue']['comments'])\n",
    "df_issues['issue_body'] = df_issues['payload'].apply(lambda x: x['issue']['body'])\n",
    "df_issues['issue_reactions'] = df_issues['payload'].apply(lambda x: x['issue'].get('reactions', {}))\n",
    "df_issues['issue_reason'] = df_issues['payload'].apply(lambda x: x['issue'].get('state_reason', {}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "df4974b2-941d-49f2-88f4-a7887fcca0c4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>issuecomment_action</th>\n",
       "      <th>issuecomment_title</th>\n",
       "      <th>issuecomment_labels</th>\n",
       "      <th>issuecomment_assignees</th>\n",
       "      <th>issuecomment_count</th>\n",
       "      <th>issuecomment_body</th>\n",
       "      <th>issuecomment_reactions</th>\n",
       "      <th>issuecomment_state_reason</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2023-06-18 13:00:58+00:00</td>\n",
       "      <td>24245737</td>\n",
       "      <td>projectmesa/mesa</td>\n",
       "      <td>22775448</td>\n",
       "      <td>tpike3</td>\n",
       "      <td>8754505</td>\n",
       "      <td>projectmesa</td>\n",
       "      <td>closed</td>\n",
       "      <td>Requesting for a feature of model level functi...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>**What's the problem this feature will solve?*...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/projectm...</td>\n",
       "      <td>completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2023-03-23 13:23:47+00:00</td>\n",
       "      <td>71948498</td>\n",
       "      <td>localstack/localstack</td>\n",
       "      <td>5170829</td>\n",
       "      <td>viren-nadkarni</td>\n",
       "      <td>28732122</td>\n",
       "      <td>localstack</td>\n",
       "      <td>closed</td>\n",
       "      <td>bug: Incorrect email count from SES:GetSendQuo...</td>\n",
       "      <td>[{'id': 468888567, 'node_id': 'MDU6TGFiZWw0Njg...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>### Is there an existing issue for this?\\n\\n- ...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/localsta...</td>\n",
       "      <td>completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2023-06-12 13:11:25+00:00</td>\n",
       "      <td>5021892</td>\n",
       "      <td>hgrecco/pint</td>\n",
       "      <td>2626260</td>\n",
       "      <td>mikeoconnor0308</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>opened</td>\n",
       "      <td>Type hinting: `Transformation` type incompatib...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>Thanks for adding type annotations to `pint`!\\...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/hgrecco/...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2022-02-11 14:03:06+00:00</td>\n",
       "      <td>53352663</td>\n",
       "      <td>ecederstrand/exchangelib</td>\n",
       "      <td>31508183</td>\n",
       "      <td>nautics889</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closed</td>\n",
       "      <td>Access to the last revision of a calendar item</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2</td>\n",
       "      <td>I am working with a calendar that has a certai...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/ecederst...</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2022-09-23 02:43:00+00:00</td>\n",
       "      <td>41179827</td>\n",
       "      <td>Rapptz/discord.py</td>\n",
       "      <td>114199431</td>\n",
       "      <td>qzzn584</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>opened</td>\n",
       "      <td>green hello error environment</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>link error link can someone helpdev not workin...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/Rapptz/d...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118378</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2022-08-04 07:31:20+00:00</td>\n",
       "      <td>51040886</td>\n",
       "      <td>Azure/azure-cli</td>\n",
       "      <td>49954584</td>\n",
       "      <td>RakeshMohanMSFT</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>closed</td>\n",
       "      <td>WebApp: Azure error</td>\n",
       "      <td>[{'id': 419587376, 'node_id': 'MDU6TGFiZWw0MTk...</td>\n",
       "      <td>[{'login': 'RakeshMohanMSFT', 'id': 49954584, ...</td>\n",
       "      <td>7</td>\n",
       "      <td>\\r\\n### **This is autogenerated. Please review...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/Azure/az...</td>\n",
       "      <td>completed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118393</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2019-06-29 18:15:37+00:00</td>\n",
       "      <td>106330968</td>\n",
       "      <td>ioc-fang/ioc_fanger</td>\n",
       "      <td>19486495</td>\n",
       "      <td>jakubclark</td>\n",
       "      <td>32465003</td>\n",
       "      <td>ioc-fang</td>\n",
       "      <td>closed</td>\n",
       "      <td>Clarification on re-fanging</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2</td>\n",
       "      <td>I was wondering if the following is intentiona...</td>\n",
       "      <td>{}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118412</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2019-06-28 15:42:05+00:00</td>\n",
       "      <td>96202399</td>\n",
       "      <td>tsproisl/SoMaJo</td>\n",
       "      <td>11275590</td>\n",
       "      <td>tsproisl</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closed</td>\n",
       "      <td>Getting Error where running the doc code</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>3</td>\n",
       "      <td>![image](https://user-images.githubusercontent...</td>\n",
       "      <td>{}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118414</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2019-06-29 15:08:01+00:00</td>\n",
       "      <td>24561828</td>\n",
       "      <td>gunthercox/ChatterBot</td>\n",
       "      <td>2030578</td>\n",
       "      <td>gunthercox</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closed</td>\n",
       "      <td>self.chatbot.storage  TypeError: 'str' object ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>3</td>\n",
       "      <td>Hi, i am getting this error before interactuat...</td>\n",
       "      <td>{}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118444</th>\n",
       "      <td>IssuesEvent</td>\n",
       "      <td>2021-07-01 10:30:45+00:00</td>\n",
       "      <td>125696210</td>\n",
       "      <td>theacodes/cmarkgfm</td>\n",
       "      <td>446561</td>\n",
       "      <td>mbarkhau</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>closed</td>\n",
       "      <td>Verify GFM Conformity</td>\n",
       "      <td>[]</td>\n",
       "      <td>[{'login': 'mbarkhau', 'id': 446561, 'node_id'...</td>\n",
       "      <td>5</td>\n",
       "      <td>#37 showed that there was at least one discrep...</td>\n",
       "      <td>{}</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8623 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               type                created_at    repo_id  \\\n",
       "2       IssuesEvent 2023-06-18 13:00:58+00:00   24245737   \n",
       "6       IssuesEvent 2023-03-23 13:23:47+00:00   71948498   \n",
       "9       IssuesEvent 2023-06-12 13:11:25+00:00    5021892   \n",
       "33      IssuesEvent 2022-02-11 14:03:06+00:00   53352663   \n",
       "55      IssuesEvent 2022-09-23 02:43:00+00:00   41179827   \n",
       "...             ...                       ...        ...   \n",
       "118378  IssuesEvent 2022-08-04 07:31:20+00:00   51040886   \n",
       "118393  IssuesEvent 2019-06-29 18:15:37+00:00  106330968   \n",
       "118412  IssuesEvent 2019-06-28 15:42:05+00:00   96202399   \n",
       "118414  IssuesEvent 2019-06-29 15:08:01+00:00   24561828   \n",
       "118444  IssuesEvent 2021-07-01 10:30:45+00:00  125696210   \n",
       "\n",
       "                       repo_name   actor_id      actor_login    org_id  \\\n",
       "2               projectmesa/mesa   22775448           tpike3   8754505   \n",
       "6          localstack/localstack    5170829   viren-nadkarni  28732122   \n",
       "9                   hgrecco/pint    2626260  mikeoconnor0308       NaN   \n",
       "33      ecederstrand/exchangelib   31508183       nautics889       NaN   \n",
       "55             Rapptz/discord.py  114199431          qzzn584       NaN   \n",
       "...                          ...        ...              ...       ...   \n",
       "118378           Azure/azure-cli   49954584  RakeshMohanMSFT   6844498   \n",
       "118393       ioc-fang/ioc_fanger   19486495       jakubclark  32465003   \n",
       "118412           tsproisl/SoMaJo   11275590         tsproisl       NaN   \n",
       "118414     gunthercox/ChatterBot    2030578       gunthercox       NaN   \n",
       "118444        theacodes/cmarkgfm     446561         mbarkhau       NaN   \n",
       "\n",
       "          org_login issuecomment_action  \\\n",
       "2       projectmesa              closed   \n",
       "6        localstack              closed   \n",
       "9               NaN              opened   \n",
       "33              NaN              closed   \n",
       "55              NaN              opened   \n",
       "...             ...                 ...   \n",
       "118378        Azure              closed   \n",
       "118393     ioc-fang              closed   \n",
       "118412          NaN              closed   \n",
       "118414          NaN              closed   \n",
       "118444          NaN              closed   \n",
       "\n",
       "                                       issuecomment_title  \\\n",
       "2       Requesting for a feature of model level functi...   \n",
       "6       bug: Incorrect email count from SES:GetSendQuo...   \n",
       "9       Type hinting: `Transformation` type incompatib...   \n",
       "33         Access to the last revision of a calendar item   \n",
       "55                          green hello error environment   \n",
       "...                                                   ...   \n",
       "118378                                WebApp: Azure error   \n",
       "118393                        Clarification on re-fanging   \n",
       "118412           Getting Error where running the doc code   \n",
       "118414  self.chatbot.storage  TypeError: 'str' object ...   \n",
       "118444                              Verify GFM Conformity   \n",
       "\n",
       "                                      issuecomment_labels  \\\n",
       "2                                                      []   \n",
       "6       [{'id': 468888567, 'node_id': 'MDU6TGFiZWw0Njg...   \n",
       "9                                                      []   \n",
       "33                                                     []   \n",
       "55                                                     []   \n",
       "...                                                   ...   \n",
       "118378  [{'id': 419587376, 'node_id': 'MDU6TGFiZWw0MTk...   \n",
       "118393                                                 []   \n",
       "118412                                                 []   \n",
       "118414                                                 []   \n",
       "118444                                                 []   \n",
       "\n",
       "                                   issuecomment_assignees  issuecomment_count  \\\n",
       "2                                                      []                   1   \n",
       "6                                                      []                   1   \n",
       "9                                                      []                   0   \n",
       "33                                                     []                   2   \n",
       "55                                                     []                   0   \n",
       "...                                                   ...                 ...   \n",
       "118378  [{'login': 'RakeshMohanMSFT', 'id': 49954584, ...                   7   \n",
       "118393                                                 []                   2   \n",
       "118412                                                 []                   3   \n",
       "118414                                                 []                   3   \n",
       "118444  [{'login': 'mbarkhau', 'id': 446561, 'node_id'...                   5   \n",
       "\n",
       "                                        issuecomment_body  \\\n",
       "2       **What's the problem this feature will solve?*...   \n",
       "6       ### Is there an existing issue for this?\\n\\n- ...   \n",
       "9       Thanks for adding type annotations to `pint`!\\...   \n",
       "33      I am working with a calendar that has a certai...   \n",
       "55      link error link can someone helpdev not workin...   \n",
       "...                                                   ...   \n",
       "118378  \\r\\n### **This is autogenerated. Please review...   \n",
       "118393  I was wondering if the following is intentiona...   \n",
       "118412  ![image](https://user-images.githubusercontent...   \n",
       "118414  Hi, i am getting this error before interactuat...   \n",
       "118444  #37 showed that there was at least one discrep...   \n",
       "\n",
       "                                   issuecomment_reactions  \\\n",
       "2       {'url': 'https://api.github.com/repos/projectm...   \n",
       "6       {'url': 'https://api.github.com/repos/localsta...   \n",
       "9       {'url': 'https://api.github.com/repos/hgrecco/...   \n",
       "33      {'url': 'https://api.github.com/repos/ecederst...   \n",
       "55      {'url': 'https://api.github.com/repos/Rapptz/d...   \n",
       "...                                                   ...   \n",
       "118378  {'url': 'https://api.github.com/repos/Azure/az...   \n",
       "118393                                                 {}   \n",
       "118412                                                 {}   \n",
       "118414                                                 {}   \n",
       "118444                                                 {}   \n",
       "\n",
       "       issuecomment_state_reason  \n",
       "2                      completed  \n",
       "6                      completed  \n",
       "9                           None  \n",
       "33                            {}  \n",
       "55                          None  \n",
       "...                          ...  \n",
       "118378                 completed  \n",
       "118393                        {}  \n",
       "118412                        {}  \n",
       "118414                        {}  \n",
       "118444                        {}  \n",
       "\n",
       "[8623 rows x 16 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_issues[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "           'issue_action', 'issue_title', 'issue_labels', 'issue_assignees', 'issue_count',\n",
    "           'issue_body', 'issue_reactions', 'issue_state_reason']].to_csv(\n",
    "    f'data/github_clean/issuesEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47b1e47-c73f-4adb-bcde-cbe7fa29d237",
   "metadata": {},
   "source": [
    "## Issue Comment Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c5f2a92f-397e-4da7-99c8-e8924a486be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_issuecomment = df[(df['type'] == 'IssueCommentEvent')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "2fdf0e60-5bc7-45b7-874c-a008ab228504",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['issue_id'] = df_issuecomment['payload'].apply(lambda x: x['comment']['issue_url'].split(\"/\")[-1])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['issue_comment_id'] = df_issuecomment['payload'].apply(lambda x: x['comment']['html_url'].split(\"-\")[-1])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['issue_comment_body'] = df_issuecomment['payload'].apply(lambda x: x['comment']['body'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['issue_comment_reactions'] = df_issuecomment['payload'].apply(lambda x: x['comment'].get('reactions', {}))\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['actor_repo_association'] = df_issuecomment['payload'].apply(lambda x: x['comment']['author_association'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['actor_type'] = df_issuecomment['payload'].apply(lambda x: x['comment']['user']['type'])\n",
      "/var/folders/5s/dvrxt95949x1pm_sjxm85lj00000gn/T/ipykernel_71435/2970942135.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_issuecomment['actor_site_admin_status'] = df_issuecomment['payload'].apply(lambda x: x['comment']['user']['site_admin'])\n"
     ]
    }
   ],
   "source": [
    "df_issuecomment['issue_id'] = df_issuecomment['payload'].apply(lambda x: x['comment']['issue_url'].split(\"/\")[-1])\n",
    "df_issuecomment['issue_comment_id'] = df_issuecomment['payload'].apply(lambda x: x['comment']['html_url'].split(\"-\")[-1])\n",
    "df_issuecomment['issue_comment_body'] = df_issuecomment['payload'].apply(lambda x: x['comment']['body'])\n",
    "df_issuecomment['issue_comment_reactions'] = df_issuecomment['payload'].apply(lambda x: x['comment'].get('reactions', {}))\n",
    "df_issuecomment['actor_repo_association'] = df_issuecomment['payload'].apply(lambda x: x['comment']['author_association'])\n",
    "df_issuecomment['actor_type'] = df_issuecomment['payload'].apply(lambda x: x['comment']['user']['type'])\n",
    "df_issuecomment['actor_site_admin_status'] = df_issuecomment['payload'].apply(lambda x: x['comment']['user']['site_admin'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "5dd39d02-6bf9-43ba-a11c-20eb37df546b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>created_at</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_login</th>\n",
       "      <th>org_id</th>\n",
       "      <th>org_login</th>\n",
       "      <th>issue_id</th>\n",
       "      <th>issue_comment_id</th>\n",
       "      <th>issue_comment_body</th>\n",
       "      <th>issue_comment_reactions</th>\n",
       "      <th>actor_repo_association</th>\n",
       "      <th>actor_type</th>\n",
       "      <th>actor_site_admin_status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2023-03-23 21:39:32+00:00</td>\n",
       "      <td>131619646</td>\n",
       "      <td>dagster-io/dagster</td>\n",
       "      <td>41898282</td>\n",
       "      <td>github-actions[bot]</td>\n",
       "      <td>40032576</td>\n",
       "      <td>dagster-io</td>\n",
       "      <td>13130</td>\n",
       "      <td>1481948718</td>\n",
       "      <td>Deploy preview for _dagit-storybook_ ready!\\n\\...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/dagster-...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Bot</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2021-08-12 17:20:42+00:00</td>\n",
       "      <td>51040886</td>\n",
       "      <td>Azure/azure-cli</td>\n",
       "      <td>59901602</td>\n",
       "      <td>Vaibhav-Agar</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>19153</td>\n",
       "      <td>897820692</td>\n",
       "      <td>Hi @rjmccallumbigl , as this seems to be a tra...</td>\n",
       "      <td>{}</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2021-08-12 21:52:26+00:00</td>\n",
       "      <td>131619646</td>\n",
       "      <td>dagster-io/dagster</td>\n",
       "      <td>35613825</td>\n",
       "      <td>vercel[bot]</td>\n",
       "      <td>40032576</td>\n",
       "      <td>dagster-io</td>\n",
       "      <td>4512</td>\n",
       "      <td>897992062</td>\n",
       "      <td>This pull request is being automatically deplo...</td>\n",
       "      <td>{}</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Bot</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2021-08-12 07:47:02+00:00</td>\n",
       "      <td>127320882</td>\n",
       "      <td>tortoise/tortoise-orm</td>\n",
       "      <td>13377178</td>\n",
       "      <td>long2ice</td>\n",
       "      <td>42678965</td>\n",
       "      <td>tortoise</td>\n",
       "      <td>784</td>\n",
       "      <td>897422595</td>\n",
       "      <td>Maybe problem of protocol</td>\n",
       "      <td>{}</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2023-06-14 22:37:22+00:00</td>\n",
       "      <td>71932349</td>\n",
       "      <td>ray-project/ray</td>\n",
       "      <td>26107013</td>\n",
       "      <td>bveeramani</td>\n",
       "      <td>22125274</td>\n",
       "      <td>ray-project</td>\n",
       "      <td>36398</td>\n",
       "      <td>1592083944</td>\n",
       "      <td>Thanks! Think doctest is failing, but I'll fix...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/ray-proj...</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118425</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2019-06-18 09:47:04+00:00</td>\n",
       "      <td>6544500</td>\n",
       "      <td>peterjc/backports.lzma</td>\n",
       "      <td>11244865</td>\n",
       "      <td>animalize</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40</td>\n",
       "      <td>503032145</td>\n",
       "      <td>I investigated this problem, here is the trigg...</td>\n",
       "      <td>{}</td>\n",
       "      <td>NONE</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118428</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2019-06-05 21:43:41+00:00</td>\n",
       "      <td>189480743</td>\n",
       "      <td>python/cherry-picker</td>\n",
       "      <td>5844587</td>\n",
       "      <td>Mariatta</td>\n",
       "      <td>1525981</td>\n",
       "      <td>python</td>\n",
       "      <td>11</td>\n",
       "      <td>499266813</td>\n",
       "      <td>Thanks so much!</td>\n",
       "      <td>{}</td>\n",
       "      <td>MEMBER</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118433</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2021-08-30 22:25:17+00:00</td>\n",
       "      <td>138421996</td>\n",
       "      <td>nedbat/coveragepy</td>\n",
       "      <td>23789</td>\n",
       "      <td>nedbat</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1217</td>\n",
       "      <td>908746546</td>\n",
       "      <td>Thanks!</td>\n",
       "      <td>{}</td>\n",
       "      <td>OWNER</td>\n",
       "      <td>User</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118442</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2022-02-22 13:57:16+00:00</td>\n",
       "      <td>7744414</td>\n",
       "      <td>peterbe/django-fancy-cache</td>\n",
       "      <td>26739</td>\n",
       "      <td>peterbe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65</td>\n",
       "      <td>1047822766</td>\n",
       "      <td>&gt; Or does it make sense to just replace the ch...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/peterbe/...</td>\n",
       "      <td>OWNER</td>\n",
       "      <td>User</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118443</th>\n",
       "      <td>IssueCommentEvent</td>\n",
       "      <td>2022-02-22 18:10:56+00:00</td>\n",
       "      <td>4127088</td>\n",
       "      <td>Azure/azure-sdk-for-python</td>\n",
       "      <td>36771401</td>\n",
       "      <td>azure-pipelines[bot]</td>\n",
       "      <td>6844498</td>\n",
       "      <td>Azure</td>\n",
       "      <td>23136</td>\n",
       "      <td>1048074740</td>\n",
       "      <td>&lt;samp&gt;\\nAzure Pipelines successfully started r...</td>\n",
       "      <td>{'url': 'https://api.github.com/repos/Azure/az...</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Bot</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25883 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     type                created_at    repo_id  \\\n",
       "5       IssueCommentEvent 2023-03-23 21:39:32+00:00  131619646   \n",
       "19      IssueCommentEvent 2021-08-12 17:20:42+00:00   51040886   \n",
       "22      IssueCommentEvent 2021-08-12 21:52:26+00:00  131619646   \n",
       "23      IssueCommentEvent 2021-08-12 07:47:02+00:00  127320882   \n",
       "24      IssueCommentEvent 2023-06-14 22:37:22+00:00   71932349   \n",
       "...                   ...                       ...        ...   \n",
       "118425  IssueCommentEvent 2019-06-18 09:47:04+00:00    6544500   \n",
       "118428  IssueCommentEvent 2019-06-05 21:43:41+00:00  189480743   \n",
       "118433  IssueCommentEvent 2021-08-30 22:25:17+00:00  138421996   \n",
       "118442  IssueCommentEvent 2022-02-22 13:57:16+00:00    7744414   \n",
       "118443  IssueCommentEvent 2022-02-22 18:10:56+00:00    4127088   \n",
       "\n",
       "                         repo_name  actor_id           actor_login    org_id  \\\n",
       "5               dagster-io/dagster  41898282   github-actions[bot]  40032576   \n",
       "19                 Azure/azure-cli  59901602          Vaibhav-Agar   6844498   \n",
       "22              dagster-io/dagster  35613825           vercel[bot]  40032576   \n",
       "23           tortoise/tortoise-orm  13377178              long2ice  42678965   \n",
       "24                 ray-project/ray  26107013            bveeramani  22125274   \n",
       "...                            ...       ...                   ...       ...   \n",
       "118425      peterjc/backports.lzma  11244865             animalize       NaN   \n",
       "118428        python/cherry-picker   5844587              Mariatta   1525981   \n",
       "118433           nedbat/coveragepy     23789                nedbat       NaN   \n",
       "118442  peterbe/django-fancy-cache     26739               peterbe       NaN   \n",
       "118443  Azure/azure-sdk-for-python  36771401  azure-pipelines[bot]   6844498   \n",
       "\n",
       "          org_login issue_id issue_comment_id  \\\n",
       "5        dagster-io    13130       1481948718   \n",
       "19            Azure    19153        897820692   \n",
       "22       dagster-io     4512        897992062   \n",
       "23         tortoise      784        897422595   \n",
       "24      ray-project    36398       1592083944   \n",
       "...             ...      ...              ...   \n",
       "118425          NaN       40        503032145   \n",
       "118428       python       11        499266813   \n",
       "118433          NaN     1217        908746546   \n",
       "118442          NaN       65       1047822766   \n",
       "118443        Azure    23136       1048074740   \n",
       "\n",
       "                                       issue_comment_body  \\\n",
       "5       Deploy preview for _dagit-storybook_ ready!\\n\\...   \n",
       "19      Hi @rjmccallumbigl , as this seems to be a tra...   \n",
       "22      This pull request is being automatically deplo...   \n",
       "23                              Maybe problem of protocol   \n",
       "24      Thanks! Think doctest is failing, but I'll fix...   \n",
       "...                                                   ...   \n",
       "118425  I investigated this problem, here is the trigg...   \n",
       "118428                                   Thanks so much!    \n",
       "118433                                            Thanks!   \n",
       "118442  > Or does it make sense to just replace the ch...   \n",
       "118443  <samp>\\nAzure Pipelines successfully started r...   \n",
       "\n",
       "                                  issue_comment_reactions  \\\n",
       "5       {'url': 'https://api.github.com/repos/dagster-...   \n",
       "19                                                     {}   \n",
       "22                                                     {}   \n",
       "23                                                     {}   \n",
       "24      {'url': 'https://api.github.com/repos/ray-proj...   \n",
       "...                                                   ...   \n",
       "118425                                                 {}   \n",
       "118428                                                 {}   \n",
       "118433                                                 {}   \n",
       "118442  {'url': 'https://api.github.com/repos/peterbe/...   \n",
       "118443  {'url': 'https://api.github.com/repos/Azure/az...   \n",
       "\n",
       "       actor_repo_association actor_type  actor_site_admin_status  \n",
       "5                        NONE        Bot                    False  \n",
       "19                     MEMBER       User                    False  \n",
       "22                       NONE        Bot                    False  \n",
       "23                     MEMBER       User                    False  \n",
       "24                     MEMBER       User                    False  \n",
       "...                       ...        ...                      ...  \n",
       "118425                   NONE       User                    False  \n",
       "118428                 MEMBER       User                    False  \n",
       "118433                  OWNER       User                    False  \n",
       "118442                  OWNER       User                     True  \n",
       "118443                   NONE        Bot                    False  \n",
       "\n",
       "[25883 rows x 15 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_issuecomment[['type', 'created_at', 'repo_id', 'repo_name', 'actor_id', 'actor_login', 'org_id', 'org_login',\n",
    "                 'issue_id', 'issue_comment_id', 'issue_comment_body', 'issue_comment_reactions', 'actor_repo_association',\n",
    "                 'actor_type', 'actor_site_admin_status']].to_csv(\n",
    "    f'data/github_clean/issueCommentEvent{fname.split(\"partitions\")[1]}.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}