02_preprocessing_vcfs_for_deep_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "994f6b13-877c-4091-9f4e-723612c3fe8e",
   "metadata": {},
   "source": [
    "# Preprocessing ONT WGS VCFs for Deep Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e08faf69-2a3c-466e-af84-3d26aef1541d",
   "metadata": {},
   "source": [
    "Starting with the 3 VCF files containing the true variants, false positives, and false negatives, the goal is to end up with a one-hot encoded Numpy array of the sequence context around each variant in the 3 files to use as direct input to deep learning models. The hope is that the neural networks will be able to learn and extract features from the sequence context that will enable distinguishing of true variants from artifacts.\n",
    "\n",
    "The following steps get us there:\n",
    "- extract sequence context of all variants in the VCF files from the reference genome\n",
    "- create a Pandas features and labels dataframe with the sequence context being the feature and artifact/not-artifact being the label\n",
    "- split the data into train-validation and test\n",
    "- one-hot encode the sequences and labels and save as Numpy arrays for input into the deep learning models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f5c4298-bf6b-4e76-b1e6-52eaad86c0d6",
   "metadata": {},
   "source": [
    "## 1. Extract from reference genome 201 bases centered at each variant call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b4471522-fbd1-461b-9fe7-14821f3bbdb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "06_vcf_TP_FP_FN/FP_snvs.vcf\n",
      "06_vcf_TP_FP_FN/FN_snvs.vcf\n",
      "06_vcf_TP_FP_FN/TP_snvs.vcf\n"
     ]
    }
   ],
   "source": [
    "!find 06_vcf_TP_FP_FN -type f -name \"*.vcf\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "45b5e286-51af-455e-88cf-34dcc705276f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chr1\t853662\t853663\n",
      "chr1\t853669\t853670\n",
      "chr1\t853718\t853719\n",
      "chr1\t853875\t853876\n",
      "chr1\t925397\t925398\n",
      "chr1\t933600\t933601\n",
      "chr1\t950625\t950626\n",
      "chr1\t960325\t960326\n",
      "chr1\t995542\t995543\n",
      "chr1\t1000078\t1000079\n",
      "chr1\t643599\t643600\n",
      "chr1\t800908\t800909\n",
      "chr1\t818025\t818026\n",
      "chr1\t832881\t832882\n",
      "chr1\t855379\t855380\n",
      "chr1\t883023\t883024\n",
      "chr1\t883030\t883031\n",
      "chr1\t883037\t883038\n",
      "chr1\t883038\t883039\n",
      "chr1\t883054\t883055\n",
      "chr1\t783005\t783006\n",
      "chr1\t783174\t783175\n",
      "chr1\t784859\t784860\n",
      "chr1\t785416\t785417\n",
      "chr1\t797391\t797392\n",
      "chr1\t798617\t798618\n",
      "chr1\t798661\t798662\n",
      "chr1\t800045\t800046\n",
      "chr1\t801141\t801142\n",
      "chr1\t801142\t801143\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# converting VCFs to BED with bedops vcf2bed\n",
    "# Loop through all .vcf files in the directory\n",
    "for vcf_file in 06_vcf_TP_FP_FN/*.vcf; do\n",
    "    vcf2bed \\\n",
    "        --do-not-sort \\\n",
    "        --do-not-split \\\n",
    "        < \"$vcf_file\" \\\n",
    "            | awk 'BEGIN {OFS=\"\\t\"} {print $(1), $(2), $(3)}' \\\n",
    "                > \"${vcf_file%.vcf}_vcf2bed.bed\"\n",
    "    cat \"${vcf_file%.vcf}_vcf2bed.bed\" | head -n 10\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0544ae2-00dc-4879-9c36-4f669a2e5a36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "06_vcf_TP_FP_FN/FN_snvs_vcf_slop_100.bed\n",
      "chr1\t853562\t853763\n",
      "chr1\t853569\t853770\n",
      "chr1\t853618\t853819\n",
      "chr1\t853775\t853976\n",
      "chr1\t925297\t925498\n",
      "chr1\t933500\t933701\n",
      "chr1\t950525\t950726\n",
      "chr1\t960225\t960426\n",
      "chr1\t995442\t995643\n",
      "chr1\t999978\t1000179\n",
      "\n",
      "06_vcf_TP_FP_FN/FP_snvs_vcf_slop_100.bed\n",
      "chr1\t643499\t643700\n",
      "chr1\t800808\t801009\n",
      "chr1\t817925\t818126\n",
      "chr1\t832781\t832982\n",
      "chr1\t855279\t855480\n",
      "chr1\t882923\t883124\n",
      "chr1\t882930\t883131\n",
      "chr1\t882937\t883138\n",
      "chr1\t882938\t883139\n",
      "chr1\t882954\t883155\n",
      "\n",
      "06_vcf_TP_FP_FN/TP_snvs_vcf_slop_100.bed\n",
      "chr1\t782905\t783106\n",
      "chr1\t783074\t783275\n",
      "chr1\t784759\t784960\n",
      "chr1\t785316\t785517\n",
      "chr1\t797291\t797492\n",
      "chr1\t798517\t798718\n",
      "chr1\t798561\t798762\n",
      "chr1\t799945\t800146\n",
      "chr1\t801041\t801242\n",
      "chr1\t801042\t801243\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# expand bed files +100bp on each side\n",
    "for bed_file in 06_vcf_TP_FP_FN/*.bed; do\n",
    "    bedtools slop \\\n",
    "        -i \"$bed_file\" \\\n",
    "        -g ~/tools/bedtools/human.hg38.genome \\\n",
    "        -b 100 \\\n",
    "        > \"${bed_file%_vcf2bed.bed}_vcf_slop_100.bed\"\n",
    "    echo \"${bed_file%_vcf2bed.bed}_vcf_slop_100.bed\"\n",
    "    cat \"${bed_file%_vcf2bed.bed}_vcf_slop_100.bed\" | head -n 10\n",
    "    echo \"\"\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "46c21b74-a6e0-476e-bddc-c8a58c45612a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing to /tmp/bcftools.62iVE9\n",
      "Merging 1 temporary files\n",
      "Cleaning\n",
      "Done\n",
      "Applied 3773914 variants\n"
     ]
    }
   ],
   "source": [
    "# create modified reference that has SNVs from sample's variant calling instead of REF allele\n",
    "# the 201bps sequences will be pulled from this modified reference\n",
    "!bcftools sort \\\n",
    "    05_vcf_isec_GIAB/dorado_bcftools_P_099_isec.vcf \\\n",
    "    -o 05_vcf_isec_GIAB/dorado_bcftools_P_099_isec_sorted.vcf.gz \\\n",
    "    -m 16G \\\n",
    "    --write-index\n",
    "!mkdir -p 07_bedtools_getfasta\n",
    "!bcftools consensus \\\n",
    "    -s - \\\n",
    "    -f ~/tools/GIAB/GRCh38/refs/GRCh38_GIABv3.fasta \\\n",
    "    05_vcf_isec_GIAB/dorado_bcftools_P_099_isec_sorted.vcf.gz \\\n",
    "    -o 07_bedtools_getfasta/modified_reference.fasta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7fb76b00-65e4-4add-ba26-d81ce8a152db",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: the index file is older than the FASTA file.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "07_bedtools_getfasta/FN_snvs_bed_getfasta.bed\n",
      "chr1\t853562\t853763\tCTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGAT\n",
      "chr1\t853569\t853770\tACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACATG\n",
      "chr1\t853618\t853819\tACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATGAACATGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATG\n",
      "chr1\t853775\t853976\tACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATGAGATGACTGCCGTGTGGTAAACTGATGAACCCTGACCCATTAGGCTTTGGCTACAGAATGTGGAAATAAGTTGTGTTACTACATGTGTGTAATCCTAGGGTGCAGGAC\n",
      "chr1\t925297\t925498\tCTGGAGGGGCACACGTGCGGCCGGGTGCGAGCGCGCGGCGGGGGAGGCTGCGGGGCGGCGCGGGGGCGCGCGCGGAGCCCGAGCGGCGGCGCCAGGTCACACAACCTGTTTTGGCGCCTGCGGGCGCCTGGGCCCAAGGGTGCGACGCGGGGGCGCCTGAGCCGGGACACAGGGGGCGCGGTGAGCGCCAGGCGCCGCGGG\n",
      "chr1\t933500\t933701\tTCTCTGGGAAGCTAGTTCTCCTCCTGCAGGCGTCCTGGGGACACCAGAGGGGGGACCCCTGGGGAGAACTCCATGGCAGCTGTGGCTACTGCCAGGCCAGCCCTCAGCCCCCACCCCATGCTGAGGCCAGCAGTCCTCTCCTGACACCCCCGCGTCGATTAGGATCATGGAAGGGATGAGTCTCGCTGCCGATTAATCCCA\n",
      "chr1\t950525\t950726\tCAAAGGTACACGCAAAGGTACACACATGCATACACACAGGTGTACACACGCACAGATGCACACACCTGCAGGCACACAGGCATTCATGGATACACGTGCATACACACATACAGATGCACACATGAACTGATATGCACACACACACAGATTTACAGACCCATGCACCCAGATGGTGCACACACACGCATGCATGCACAGACA\n",
      "chr1\t960225\t960426\tCTGTGGGGGGCTTCCCGGGGAAGAAGGAAGGCGAGACCTAGGGGGGCTTCCCGGAGGAGGAGGAGGAGGAGGGCTAGGCCGGGGGCTTCCCAGAGGAGGAGGATGGCGGGGCCTGGGGGGCTTCTCGGAGAAGGAAGATGGCGAGGCCTAGGGGGGCTTCTCGGAGGAGGAGGAGGGCGAGGCCTGGTGGGGGGCTTCCCG\n",
      "chr1\t995442\t995643\tGATGTATAGATCGATGGAAAAGAATTGAGGGTTCAGAAATAGGTCCTCACATTTACAGTCAACTGATTTTTTTTTTTTTCTGAGACAGGGTCTCACTCTCAGCCCATCACCTGGGCAGAACCCTGCTGATCCTGCAGATCACTGGACCCTCAGGACACTGAGGTGTGAATGCCAGAAGGGGACAGTGGGACGTCAGGGAGG\n",
      "chr1\t999978\t1000179\tGCCCCGCGCCTCCCCGTGCCGGGTGGAGCGCGCCGCCACGGACCACGGGCGGGCTGGCGGGCGAGCGGCGAGCGCGCGGCGATCCGAGCCCCTAGGGCGGATCCCGGCTCCAGGCCCGCGCGCGCCTCAGGCCTTTTCCCTATTTAAGGCCTCGCCGCCGCGGGGTGTGTGAACCCGGCTCCGCATTCTTTCCCACACTCG\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: the index file is older than the FASTA file.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "07_bedtools_getfasta/FP_snvs_bed_getfasta.bed\n",
      "chr1\t643499\t643700\tACCGGCAAATTCTGTTGTTTGTATAAACATCAGCCATGTTTATATAACTAAACTAGTGTTTTGTTTTGTCAATTCAGCAAGAAATTAGACCACATGGTGGTTTAATGCTGCATTGATTTGGCTATCAATTTGTTTTCACTTTTCTGCAAAATATTTAATACATTATTAAATTGAATTATGCTGATGCCACAGTTGTTCTTA\n",
      "chr1\t800808\t801009\tTTGATGGGTGCAGCAAACCACCATGGCATGTGTATACCTATGTAACAAACCTGCACATTGAGCATATGTATACCAGAACTTAAAAGTATAATTTAAAAAAATTTTTAAAAAGTCATATGATGCATTTAAGAAAGTCACTTAATTTACATCAGAGGAAAATCAAAGTTTATAGACTTAGGAAATAAAGTCGTAATGAAGAAG\n",
      "chr1\t817925\t818126\tAGTTTTATCCACTTTATGTGAAGAAAGCCAACAAGGGGCATGGAGTGAGTTCCGCAGGTTTTAGCGGCTGCGGCGGCTGGTGCTCAGTGGGGATGATGGAAGGAAGGCGCCTCCCTCTGCGGGCCCCGAGGTCTGTGCGGGAATCAGCTCTGCAGCTGTGTCCAGGGGGAGCCGTAGACCACACACGGCAGGCTCACAGCT\n",
      "chr1\t832781\t832982\tAGATGGAGTCTTGCTTTGTCGCCCAGGCTGTAGTGCAGTGGCGTGATCTTGGCTCACTGCAGCCTCCACCTTAGAGCAATCCTCTTGCCTCCTCCTCCCGAGTAGTTGGGACTACATGTGCATGCCACATGCCTGGCTAATTTTTGTATTTTTAGTAGAGACACGGTTTCACCATGTTGGCCAGGCTGGTGTCCAACTCCT\n",
      "chr1\t855279\t855480\tTGATGTACGGGTGTATCTGTGTATTGTGTATGCACACACGAGCATATGTGTACATGAATTTGTATTGCACATGTGTTTAATGCGAACACGTGTCATGTGTGTGTGTTCACATGCATGTGTGTCTGTGTACTGTGTGCAACAGTGTGTGCAAGTGTGTATATGTGGGTTTATGTTTGTACAGTGTGTGTAGTGTGTGTACAC\n",
      "chr1\t882923\t883124\tGCAGTGTTTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTTGTCCAGGTTGGAGTGCAATGGCGTGATCTCAGCTCACCACAACCTCTGCCTCCCAGGTTCAAGCGATTCTCCTGCCTCAGCCTCCCAAGTAGCTAGGATTACAGGCATGTGCCAACATAACTGG\n",
      "chr1\t882930\t883131\tTTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTTGTCCAGGTTGGAGTGCAATGGCGTGATCTCAGCTCACCACAACCTCTGCCTCCCAGGTTCAAGCGATTCTCCTGCCTCAGCCTCCCAAGTAGCTAGGATTACAGGCATGTGCCAACATAACTGGCTAATTG\n",
      "chr1\t882937\t883138\tTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTTGTCCAGGTTGGAGTGCAATGGCGTGATCTCAGCTCACCACAACCTCTGCCTCCCAGGTTCAAGCGATTCTCCTGCCTCAGCCTCCCAAGTAGCTAGGATTACAGGCATGTGCCAACATAACTGGCTAATTGTTTATAT\n",
      "chr1\t882938\t883139\tCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTTGTCCAGGTTGGAGTGCAATGGCGTGATCTCAGCTCACCACAACCTCTGCCTCCCAGGTTCAAGCGATTCTCCTGCCTCAGCCTCCCAAGTAGCTAGGATTACAGGCATGTGCCAACATAACTGGCTAATTGTTTATATT\n",
      "chr1\t882954\t883155\tTCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTTGTCCAGGTTGGAGTGCAATGGCGTGATCTCAGCTCACCACAACCTCTGCCTCCCAGGTTCAAGCGATTCTCCTGCCTCAGCCTCCCAAGTAGCTAGGATTACAGGCATGTGCCAACATAACTGGCTAATTGTTTATATTTTTAGTAGAGACGGGG\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: the index file is older than the FASTA file.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "07_bedtools_getfasta/TP_snvs_bed_getfasta.bed\n",
      "chr1\t782905\t783106\tTAAAATATGCCACAGATTTCTAAGACTGAGCATGGAAAAGAAAATCTCCATAATTTTTTATATTGATTGTATACTGCAGTGATAATATTTTGGATGTATCGGGTTAAATAAAATTGACTGATTTCACCTTTTTCCTATTTTAAAAGTGGCTACTAAGAAAATTTTAAATTACTTACATGACCGACATGGTATTTTTATTTG\n",
      "chr1\t783074\t783275\tTACTTACATGACCGACATGGTATTTTTATTTGGCAGCGCTGCTCTAAGCTGTTGATGAAAAATATTGTTGGTGAGCTCTGCTTAGGTAATATATAGGACACGAGCAGAGAGGAGGCACGTGAACAGTTCTGGCCTGGAGTAGGCTTCATTGAGGCTGTGATGCTTTTAGCTGGATTTGAAGAAGTGGTAGTGATCATCCCA\n",
      "chr1\t784759\t784960\tGGAAATGAGATATTTATGTAGTCTTAAGGTATCTTCTCACAAATTATGTATTACTTACAAAGGGGAAAAGTGCACCTTGACGTGAGAATCTTGGCAGAACCACTTTAATCAAGAGGTTTAGTTGAGCATTATCAGTAACAGAGTAAATCAAAATCATAAGCCACTTGATAGATACAATGAGAAAATAAAGCATCACTTCTG\n",
      "chr1\t785316\t785517\tAGGGTGACAGGGCATTATATCAGCCACTTATCAACAAACATTTCAGAGAAAAATAGTTCCTTGTATTGTTATTGCAATTTTCTTTGAGATTGCCCCCTCCAAAACAGTAAGAACTTTCAAAAACAAACAAAGATATCAAGCCACAGATTCAAAGTGCTATAAACTCCACGCATGATTAGTTCTCACTCATAGGTGGGAATT\n",
      "chr1\t797291\t797492\tAACTTTGCTCAATCTTCGCCTTTGGGTTCCTTTAGGTTTATAATGAACTGTAAGCATTTACTGAGTGATTATTATAAAGGCACCATTTGAAGTACCTGATACCCTTTTAATGCTGTAATGCCCCTGTGAAGTAGGTCTTACTATTATCCCTGCTCTAGAGATGAGAAAACTGAGGCAGGGAGAGATGAAGAAGCTGGCCCC\n",
      "chr1\t798517\t798718\tAACATACTTTGCTAATACATTTTAATCTGGCATTTTTATGGGGGTAATTATAGGAAATGCCTGGAATTAAATAGCCTACAACCAATTCTTGGATCAACAATTAGGAAAACTGAAAAATATATATGTAAATATATCTTTTTCTGTATAAGAAAAGTCCTGCTAACTTAGGAAATTAGAGGATATCTTTGTGTAGAATCTTTT\n",
      "chr1\t798561\t798762\tTAATTATAGGAAATGCCTGGAATTAAATAGCCTACAACCAATTCTTGGATCAACAATTAGGAAAACTGAAAAATATATATGTAAATATATCTTTTTCTGTATAAGAAAAGTCCTGCTAACTTAGGAAATTAGAGGATATCTTTGTGTAGAATCTTTTTATAAAGTGGAAATAGGCCAGGCATGGTGGCTAACGCCTGCAAT\n",
      "chr1\t799945\t800146\tTTTTGCTATGCAGAAGCTCTTTAGTTTAATTAGATCCCATTTGTCAATTTTGGCTTTTGTTGCCATTGCTTTCGGTGTTTTAGACATCAAGTCTTTGCCCATGCCTATGTCCTGAATGGTATTGCCTAGCTTTTCTTGTAGGGTTTTTATGGTTTTAGGTCTTATGTTTAAGTCTTTAATCCATCTTGAGTTAATTTTTGT\n",
      "chr1\t801041\t801242\tGGTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGGGCCCCATTTTCTTGTCTTTACCTCTTGGGTCATAAATGGATCTCCATGCAGCTGTCATCCCTCTGCCTAAATCCTCCTCAAAAGAAAATCAAAATAGTGGCTAACACAGAGTACAGACTGTTCCAAGAGCTTCCTATGATGGAACTAATCTAATC\n",
      "chr1\t801042\t801243\tGTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGGGCCCCATTTTCTTGTCTTTACCTCTTGGGTCATAAATGGATCTCCATGCAGCTGTCATCCCTCTGCCTAAATCCTCCTCAAAAGAAAATCAAAATAGTGGCTAACACAGAGTACAGACTGTTCCAAGAGCTTCCTATGATGGAACTAATCTAATCT\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# extract the actual slop sequences from the modified human reference genome using bedtools getfasta\n",
    "for bed_file in 06_vcf_TP_FP_FN/*_vcf_slop_100.bed; do\n",
    "    filename=$(basename \"$bed_file\" _vcf_slop_100.bed)\n",
    "    output_file=\"07_bedtools_getfasta/${filename}_bed_getfasta.bed\"\n",
    "\n",
    "    bedtools getfasta \\\n",
    "        -fi 07_bedtools_getfasta/modified_reference.fasta \\\n",
    "        -bed \"$bed_file\" \\\n",
    "        -bedOut \\\n",
    "        > \"$output_file\"\n",
    "\n",
    "    echo \"$output_file\"\n",
    "    cat \"$output_file\" | head -n 10\n",
    "    echo \"\"\n",
    "done\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "60badd9e-f8bb-4759-93db-c793fd980e16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 3.8G\n",
      "drwxr-xr-x 11 fmbuga wlee 4.0K Nov 17 18:45 ..\n",
      "-rw-r--r--  1 fmbuga wlee 698M Nov 17 18:44 TP_snvs_bed_getfasta.bed\n",
      "drwxr-xr-x  2 fmbuga wlee  194 Nov 17 18:44 .\n",
      "-rw-r--r--  1 fmbuga wlee 116M Nov 17 18:44 FP_snvs_bed_getfasta.bed\n",
      "-rw-r--r--  1 fmbuga wlee  26M Nov 17 18:44 FN_snvs_bed_getfasta.bed\n",
      "-rw-r--r--  1 fmbuga wlee 3.0G Nov 17 18:02 modified_reference.fasta\n",
      "-rw-r--r--  1 fmbuga wlee 8.8K Nov 17 17:25 modified_reference.fasta.fai\n",
      "total 3.8G\n",
      "-rw-r--r--  1 fmbuga wlee 8.8K Nov 17 18:49 modified_reference.fasta.fai\n",
      "drwxr-xr-x 11 fmbuga wlee 4.0K Nov 17 18:45 ..\n",
      "-rw-r--r--  1 fmbuga wlee 698M Nov 17 18:44 TP_snvs_bed_getfasta.bed\n",
      "drwxr-xr-x  2 fmbuga wlee  194 Nov 17 18:44 .\n",
      "-rw-r--r--  1 fmbuga wlee 116M Nov 17 18:44 FP_snvs_bed_getfasta.bed\n",
      "-rw-r--r--  1 fmbuga wlee  26M Nov 17 18:44 FN_snvs_bed_getfasta.bed\n",
      "-rw-r--r--  1 fmbuga wlee 3.0G Nov 17 18:02 modified_reference.fasta\n"
     ]
    }
   ],
   "source": [
    "# index modified reference fasta\n",
    "!ls -alth 07_bedtools_getfasta\n",
    "!samtools faidx 07_bedtools_getfasta/modified_reference.fasta\n",
    "!ls -alth 07_bedtools_getfasta"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5079e110-97fc-4d08-bc42-414452d92fae",
   "metadata": {},
   "source": [
    "## 2. Create features and labels Pandas dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e7b829c6-a799-4eee-9739-7fd33dafd8da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853562</td>\n",
       "      <td>853763</td>\n",
       "      <td>CTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853569</td>\n",
       "      <td>853770</td>\n",
       "      <td>ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853618</td>\n",
       "      <td>853819</td>\n",
       "      <td>ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853775</td>\n",
       "      <td>853976</td>\n",
       "      <td>ACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATGAG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>925297</td>\n",
       "      <td>925498</td>\n",
       "      <td>CTGGAGGGGCACACGTGCGGCCGGGTGCGAGCGCGCGGCGGGGGAG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>chr1</td>\n",
       "      <td>933500</td>\n",
       "      <td>933701</td>\n",
       "      <td>TCTCTGGGAAGCTAGTTCTCCTCCTGCAGGCGTCCTGGGGACACCA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chr1</td>\n",
       "      <td>950525</td>\n",
       "      <td>950726</td>\n",
       "      <td>CAAAGGTACACGCAAAGGTACACACATGCATACACACAGGTGTACA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>chr1</td>\n",
       "      <td>960225</td>\n",
       "      <td>960426</td>\n",
       "      <td>CTGTGGGGGGCTTCCCGGGGAAGAAGGAAGGCGAGACCTAGGGGGG...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>chr1</td>\n",
       "      <td>995442</td>\n",
       "      <td>995643</td>\n",
       "      <td>GATGTATAGATCGATGGAAAAGAATTGAGGGTTCAGAAATAGGTCC...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>chr1</td>\n",
       "      <td>999978</td>\n",
       "      <td>1000179</td>\n",
       "      <td>GCCCCGCGCCTCCCCGTGCCGGGTGGAGCGCGCCGCCACGGACCAC...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0       1        2                                                  3\n",
       "0  chr1  853562   853763  CTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCG...\n",
       "1  chr1  853569   853770  ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...\n",
       "2  chr1  853618   853819  ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...\n",
       "3  chr1  853775   853976  ACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATGAG...\n",
       "4  chr1  925297   925498  CTGGAGGGGCACACGTGCGGCCGGGTGCGAGCGCGCGGCGGGGGAG...\n",
       "5  chr1  933500   933701  TCTCTGGGAAGCTAGTTCTCCTCCTGCAGGCGTCCTGGGGACACCA...\n",
       "6  chr1  950525   950726  CAAAGGTACACGCAAAGGTACACACATGCATACACACAGGTGTACA...\n",
       "7  chr1  960225   960426  CTGTGGGGGGCTTCCCGGGGAAGAAGGAAGGCGAGACCTAGGGGGG...\n",
       "8  chr1  995442   995643  GATGTATAGATCGATGGAAAAGAATTGAGGGTTCAGAAATAGGTCC...\n",
       "9  chr1  999978  1000179  GCCCCGCGCCTCCCCGTGCCGGGTGGAGCGCGCCGCCACGGACCAC..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.read_csv('07_bedtools_getfasta/FN_snvs_bed_getfasta.bed', \n",
    "            sep='\\t', \n",
    "            header=None).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38d2556b-2906-4a63-83c0-af7b40b21087",
   "metadata": {},
   "source": [
    "#### add labels01 and labels012 columns to getfasta.bed files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b32ad79d-8889-46e5-bb82-3682538fd67b",
   "metadata": {},
   "source": [
    "`labels01` labels true positives as 0 and everything else (the artifacts) as 1. `labels012` labels true positives as 0, false positives as 1, and false negatives as 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "38e1215f-02d5-4e92-a5b9-f2ea925376c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(119744, 537140, 3236774)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FN_seqs = pd.read_csv('07_bedtools_getfasta/FN_snvs_bed_getfasta.bed', \n",
    "            sep='\\t', \n",
    "            header=None)\n",
    "FP_seqs = pd.read_csv('07_bedtools_getfasta/FP_snvs_bed_getfasta.bed', \n",
    "            sep='\\t', \n",
    "            header=None)\n",
    "TP_seqs = pd.read_csv('07_bedtools_getfasta/TP_snvs_bed_getfasta.bed', \n",
    "            sep='\\t', \n",
    "            header=None)\n",
    "len(FN_seqs), len(FP_seqs), len(TP_seqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0cf3b2e0-de94-47fe-b077-5d30146cd528",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>labels01</th>\n",
       "      <th>labels012</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>782905</td>\n",
       "      <td>783106</td>\n",
       "      <td>TAAAATATGCCACAGATTTCTAAGACTGAGCATGGAAAAGAAAATC...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>783074</td>\n",
       "      <td>783275</td>\n",
       "      <td>TACTTACATGACCGACATGGTATTTTTATTTGGCAGCGCTGCTCTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>784759</td>\n",
       "      <td>784960</td>\n",
       "      <td>GGAAATGAGATATTTATGTAGTCTTAAGGTATCTTCTCACAAATTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>785316</td>\n",
       "      <td>785517</td>\n",
       "      <td>AGGGTGACAGGGCATTATATCAGCCACTTATCAACAAACATTTCAG...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>797291</td>\n",
       "      <td>797492</td>\n",
       "      <td>AACTTTGCTCAATCTTCGCCTTTGGGTTCCTTTAGGTTTATAATGA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>chr1</td>\n",
       "      <td>798517</td>\n",
       "      <td>798718</td>\n",
       "      <td>AACATACTTTGCTAATACATTTTAATCTGGCATTTTTATGGGGGTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chr1</td>\n",
       "      <td>798561</td>\n",
       "      <td>798762</td>\n",
       "      <td>TAATTATAGGAAATGCCTGGAATTAAATAGCCTACAACCAATTCTT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>chr1</td>\n",
       "      <td>799945</td>\n",
       "      <td>800146</td>\n",
       "      <td>TTTTGCTATGCAGAAGCTCTTTAGTTTAATTAGATCCCATTTGTCA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>chr1</td>\n",
       "      <td>801041</td>\n",
       "      <td>801242</td>\n",
       "      <td>GGTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGG...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>chr1</td>\n",
       "      <td>801042</td>\n",
       "      <td>801243</td>\n",
       "      <td>GTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGGG...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0       1       2                                                  3  \\\n",
       "0  chr1  782905  783106  TAAAATATGCCACAGATTTCTAAGACTGAGCATGGAAAAGAAAATC...   \n",
       "1  chr1  783074  783275  TACTTACATGACCGACATGGTATTTTTATTTGGCAGCGCTGCTCTA...   \n",
       "2  chr1  784759  784960  GGAAATGAGATATTTATGTAGTCTTAAGGTATCTTCTCACAAATTA...   \n",
       "3  chr1  785316  785517  AGGGTGACAGGGCATTATATCAGCCACTTATCAACAAACATTTCAG...   \n",
       "4  chr1  797291  797492  AACTTTGCTCAATCTTCGCCTTTGGGTTCCTTTAGGTTTATAATGA...   \n",
       "5  chr1  798517  798718  AACATACTTTGCTAATACATTTTAATCTGGCATTTTTATGGGGGTA...   \n",
       "6  chr1  798561  798762  TAATTATAGGAAATGCCTGGAATTAAATAGCCTACAACCAATTCTT...   \n",
       "7  chr1  799945  800146  TTTTGCTATGCAGAAGCTCTTTAGTTTAATTAGATCCCATTTGTCA...   \n",
       "8  chr1  801041  801242  GGTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGG...   \n",
       "9  chr1  801042  801243  GTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGGG...   \n",
       "\n",
       "   labels01  labels012  \n",
       "0         0          0  \n",
       "1         0          0  \n",
       "2         0          0  \n",
       "3         0          0  \n",
       "4         0          0  \n",
       "5         0          0  \n",
       "6         0          0  \n",
       "7         0          0  \n",
       "8         0          0  \n",
       "9         0          0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TP_seqs['labels01'] = TP_seqs['labels012'] = 0\n",
    "TP_seqs.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "65864897-4ec5-439b-b639-6272ed3712c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>labels01</th>\n",
       "      <th>labels012</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>643499</td>\n",
       "      <td>643700</td>\n",
       "      <td>ACCGGCAAATTCTGTTGTTTGTATAAACATCAGCCATGTTTATATA...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>800808</td>\n",
       "      <td>801009</td>\n",
       "      <td>TTGATGGGTGCAGCAAACCACCATGGCATGTGTATACCTATGTAAC...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>817925</td>\n",
       "      <td>818126</td>\n",
       "      <td>AGTTTTATCCACTTTATGTGAAGAAAGCCAACAAGGGGCATGGAGT...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>832781</td>\n",
       "      <td>832982</td>\n",
       "      <td>AGATGGAGTCTTGCTTTGTCGCCCAGGCTGTAGTGCAGTGGCGTGA...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>855279</td>\n",
       "      <td>855480</td>\n",
       "      <td>TGATGTACGGGTGTATCTGTGTATTGTGTATGCACACACGAGCATA...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>chr1</td>\n",
       "      <td>882923</td>\n",
       "      <td>883124</td>\n",
       "      <td>GCAGTGTTTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTT...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chr1</td>\n",
       "      <td>882930</td>\n",
       "      <td>883131</td>\n",
       "      <td>TTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTT...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>chr1</td>\n",
       "      <td>882937</td>\n",
       "      <td>883138</td>\n",
       "      <td>TCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGAC...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>chr1</td>\n",
       "      <td>882938</td>\n",
       "      <td>883139</td>\n",
       "      <td>CTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACA...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>chr1</td>\n",
       "      <td>882954</td>\n",
       "      <td>883155</td>\n",
       "      <td>TCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTT...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0       1       2                                                  3  \\\n",
       "0  chr1  643499  643700  ACCGGCAAATTCTGTTGTTTGTATAAACATCAGCCATGTTTATATA...   \n",
       "1  chr1  800808  801009  TTGATGGGTGCAGCAAACCACCATGGCATGTGTATACCTATGTAAC...   \n",
       "2  chr1  817925  818126  AGTTTTATCCACTTTATGTGAAGAAAGCCAACAAGGGGCATGGAGT...   \n",
       "3  chr1  832781  832982  AGATGGAGTCTTGCTTTGTCGCCCAGGCTGTAGTGCAGTGGCGTGA...   \n",
       "4  chr1  855279  855480  TGATGTACGGGTGTATCTGTGTATTGTGTATGCACACACGAGCATA...   \n",
       "5  chr1  882923  883124  GCAGTGTTTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTT...   \n",
       "6  chr1  882930  883131  TTGGTTTTCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTT...   \n",
       "7  chr1  882937  883138  TCTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGAC...   \n",
       "8  chr1  882938  883139  CTTTTCTTTTTTTCTTTCTCTCTTTTCTTTTTTTTTTTTTGAGACA...   \n",
       "9  chr1  882954  883155  TCTCTCTTTTCTTTTTTTTTTTTTGAGACAAACTTTCACTCTTGTT...   \n",
       "\n",
       "   labels01  labels012  \n",
       "0         1          1  \n",
       "1         1          1  \n",
       "2         1          1  \n",
       "3         1          1  \n",
       "4         1          1  \n",
       "5         1          1  \n",
       "6         1          1  \n",
       "7         1          1  \n",
       "8         1          1  \n",
       "9         1          1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FP_seqs['labels01'] = FP_seqs['labels012'] = 1\n",
    "FP_seqs.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4baa5596-b3b6-4c27-a589-c487f1a2ebd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>labels01</th>\n",
       "      <th>labels012</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853562</td>\n",
       "      <td>853763</td>\n",
       "      <td>CTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCG...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853569</td>\n",
       "      <td>853770</td>\n",
       "      <td>ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853618</td>\n",
       "      <td>853819</td>\n",
       "      <td>ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>853775</td>\n",
       "      <td>853976</td>\n",
       "      <td>ACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATGAG...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>925297</td>\n",
       "      <td>925498</td>\n",
       "      <td>CTGGAGGGGCACACGTGCGGCCGGGTGCGAGCGCGCGGCGGGGGAG...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>chr1</td>\n",
       "      <td>933500</td>\n",
       "      <td>933701</td>\n",
       "      <td>TCTCTGGGAAGCTAGTTCTCCTCCTGCAGGCGTCCTGGGGACACCA...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chr1</td>\n",
       "      <td>950525</td>\n",
       "      <td>950726</td>\n",
       "      <td>CAAAGGTACACGCAAAGGTACACACATGCATACACACAGGTGTACA...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>chr1</td>\n",
       "      <td>960225</td>\n",
       "      <td>960426</td>\n",
       "      <td>CTGTGGGGGGCTTCCCGGGGAAGAAGGAAGGCGAGACCTAGGGGGG...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>chr1</td>\n",
       "      <td>995442</td>\n",
       "      <td>995643</td>\n",
       "      <td>GATGTATAGATCGATGGAAAAGAATTGAGGGTTCAGAAATAGGTCC...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>chr1</td>\n",
       "      <td>999978</td>\n",
       "      <td>1000179</td>\n",
       "      <td>GCCCCGCGCCTCCCCGTGCCGGGTGGAGCGCGCCGCCACGGACCAC...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0       1        2                                                  3  \\\n",
       "0  chr1  853562   853763  CTGATGAACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCG...   \n",
       "1  chr1  853569   853770  ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...   \n",
       "2  chr1  853618   853819  ACGTGAGATGACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGA...   \n",
       "3  chr1  853775   853976  ACCGCCGTGTGGTAAACTGATGAACCCCGACCCTGATCAACATGAG...   \n",
       "4  chr1  925297   925498  CTGGAGGGGCACACGTGCGGCCGGGTGCGAGCGCGCGGCGGGGGAG...   \n",
       "5  chr1  933500   933701  TCTCTGGGAAGCTAGTTCTCCTCCTGCAGGCGTCCTGGGGACACCA...   \n",
       "6  chr1  950525   950726  CAAAGGTACACGCAAAGGTACACACATGCATACACACAGGTGTACA...   \n",
       "7  chr1  960225   960426  CTGTGGGGGGCTTCCCGGGGAAGAAGGAAGGCGAGACCTAGGGGGG...   \n",
       "8  chr1  995442   995643  GATGTATAGATCGATGGAAAAGAATTGAGGGTTCAGAAATAGGTCC...   \n",
       "9  chr1  999978  1000179  GCCCCGCGCCTCCCCGTGCCGGGTGGAGCGCGCCGCCACGGACCAC...   \n",
       "\n",
       "   labels01  labels012  \n",
       "0         1          2  \n",
       "1         1          2  \n",
       "2         1          2  \n",
       "3         1          2  \n",
       "4         1          2  \n",
       "5         1          2  \n",
       "6         1          2  \n",
       "7         1          2  \n",
       "8         1          2  \n",
       "9         1          2  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FN_seqs['labels01'] = 1\n",
    "FN_seqs['labels012'] = 2\n",
    "FN_seqs.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0bb57351-e955-4f57-8548-07c9a64ceb1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3893658, 6)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate the 3 dfs\n",
    "TP_FP_FN_seqs_labels01_labels012 = pd.concat([TP_seqs, FP_seqs, FN_seqs], ignore_index=True)\n",
    "TP_FP_FN_seqs_labels01_labels012.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "117dcdb3-8ace-4ba8-a60c-0447f016076f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos_start</th>\n",
       "      <th>pos_end</th>\n",
       "      <th>sequences</th>\n",
       "      <th>labels01</th>\n",
       "      <th>labels012</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chr1</td>\n",
       "      <td>782905</td>\n",
       "      <td>783106</td>\n",
       "      <td>TAAAATATGCCACAGATTTCTAAGACTGAGCATGGAAAAGAAAATC...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chr1</td>\n",
       "      <td>783074</td>\n",
       "      <td>783275</td>\n",
       "      <td>TACTTACATGACCGACATGGTATTTTTATTTGGCAGCGCTGCTCTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chr1</td>\n",
       "      <td>784759</td>\n",
       "      <td>784960</td>\n",
       "      <td>GGAAATGAGATATTTATGTAGTCTTAAGGTATCTTCTCACAAATTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chr1</td>\n",
       "      <td>785316</td>\n",
       "      <td>785517</td>\n",
       "      <td>AGGGTGACAGGGCATTATATCAGCCACTTATCAACAAACATTTCAG...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chr1</td>\n",
       "      <td>797291</td>\n",
       "      <td>797492</td>\n",
       "      <td>AACTTTGCTCAATCTTCGCCTTTGGGTTCCTTTAGGTTTATAATGA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>chr1</td>\n",
       "      <td>798517</td>\n",
       "      <td>798718</td>\n",
       "      <td>AACATACTTTGCTAATACATTTTAATCTGGCATTTTTATGGGGGTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>chr1</td>\n",
       "      <td>798561</td>\n",
       "      <td>798762</td>\n",
       "      <td>TAATTATAGGAAATGCCTGGAATTAAATAGCCTACAACCAATTCTT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>chr1</td>\n",
       "      <td>799945</td>\n",
       "      <td>800146</td>\n",
       "      <td>TTTTGCTATGCAGAAGCTCTTTAGTTTAATTAGATCCCATTTGTCA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>chr1</td>\n",
       "      <td>801041</td>\n",
       "      <td>801242</td>\n",
       "      <td>GGTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGG...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>chr1</td>\n",
       "      <td>801042</td>\n",
       "      <td>801243</td>\n",
       "      <td>GTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGGG...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  chrom  pos_start  pos_end  \\\n",
       "0  chr1     782905   783106   \n",
       "1  chr1     783074   783275   \n",
       "2  chr1     784759   784960   \n",
       "3  chr1     785316   785517   \n",
       "4  chr1     797291   797492   \n",
       "5  chr1     798517   798718   \n",
       "6  chr1     798561   798762   \n",
       "7  chr1     799945   800146   \n",
       "8  chr1     801041   801242   \n",
       "9  chr1     801042   801243   \n",
       "\n",
       "                                           sequences  labels01  labels012  \n",
       "0  TAAAATATGCCACAGATTTCTAAGACTGAGCATGGAAAAGAAAATC...         0          0  \n",
       "1  TACTTACATGACCGACATGGTATTTTTATTTGGCAGCGCTGCTCTA...         0          0  \n",
       "2  GGAAATGAGATATTTATGTAGTCTTAAGGTATCTTCTCACAAATTA...         0          0  \n",
       "3  AGGGTGACAGGGCATTATATCAGCCACTTATCAACAAACATTTCAG...         0          0  \n",
       "4  AACTTTGCTCAATCTTCGCCTTTGGGTTCCTTTAGGTTTATAATGA...         0          0  \n",
       "5  AACATACTTTGCTAATACATTTTAATCTGGCATTTTTATGGGGGTA...         0          0  \n",
       "6  TAATTATAGGAAATGCCTGGAATTAAATAGCCTACAACCAATTCTT...         0          0  \n",
       "7  TTTTGCTATGCAGAAGCTCTTTAGTTTAATTAGATCCCATTTGTCA...         0          0  \n",
       "8  GGTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGG...         0          0  \n",
       "9  GTCTCTGTCCTTGATTCCATTGTGACCTTCAGCCCATCTCTCTGGG...         0          0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rename columns\n",
    "TP_FP_FN_seqs_labels01_labels012.rename(columns={0: \"chrom\", \n",
    "                                                 1: \"pos_start\", \n",
    "                                                 2: \"pos_end\", \n",
    "                                                 3: \"sequences\"}, \n",
    "                                       inplace=True)\n",
    "TP_FP_FN_seqs_labels01_labels012.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "60a51240-e589-4993-aaa8-74713e8677c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3893658, 6)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save mega pd.DataFrame to csv\n",
    "TP_FP_FN_seqs_labels01_labels012.to_csv('231117_TP_FP_FN_seqs_labels01_labels012.csv', index=False)\n",
    "TP_FP_FN_seqs_labels01_labels012.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d20679da-7220-4947-92d5-30a60353ff41",
   "metadata": {},
   "source": [
    "## 3. Split the data into train, validation and test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c047ae6-c4d3-4866-917e-ecba58966a8c",
   "metadata": {},
   "source": [
    "- do 70% train 20% validation 10% test split\n",
    "- include all of chr20 in test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4e68adab-ce47-424a-888e-75c07b619ea4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('231117_TP_FP_FN_seqs_labels01_labels012.csv')\n",
    "df_no_20 = df[df['chrom'] != 'chr20'] # chr20 will be exclusively test split; ~ 2% of entire dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cfdd8c07-8237-4b35-a3ce-86598e905434",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_10011/414909324.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_no_20['combined'] = df_no_20['chrom'] + '_' + df_no_20['labels012'].astype(str)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((3504292, 7), (306449, 7))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stratified split of df_no_20; stratify on chrom and labels012\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Create a combined categorical variable based on chrom and label012\n",
    "df_no_20['combined'] = df_no_20['chrom'] + '_' + df_no_20['labels012'].astype(str)\n",
    "\n",
    "# Perform a stratified train-test split based on the combined variable\n",
    "train, test = train_test_split(df_no_20, test_size=0.08041711572631151, \n",
    "                               stratify=df_no_20['combined'], \n",
    "                               random_state=42)\n",
    "\n",
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6d7c02ae-7e7a-4eac-8dce-d3097019a1e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>labels012</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrom</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr1</th>\n",
       "      <td>82.74</td>\n",
       "      <td>14.18</td>\n",
       "      <td>3.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10</th>\n",
       "      <td>82.65</td>\n",
       "      <td>14.36</td>\n",
       "      <td>2.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr11</th>\n",
       "      <td>83.84</td>\n",
       "      <td>13.11</td>\n",
       "      <td>3.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr12</th>\n",
       "      <td>82.79</td>\n",
       "      <td>14.13</td>\n",
       "      <td>3.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr13</th>\n",
       "      <td>84.75</td>\n",
       "      <td>12.43</td>\n",
       "      <td>2.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr14</th>\n",
       "      <td>82.91</td>\n",
       "      <td>14.25</td>\n",
       "      <td>2.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr15</th>\n",
       "      <td>83.63</td>\n",
       "      <td>13.22</td>\n",
       "      <td>3.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr16</th>\n",
       "      <td>81.79</td>\n",
       "      <td>14.66</td>\n",
       "      <td>3.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr17</th>\n",
       "      <td>79.84</td>\n",
       "      <td>15.93</td>\n",
       "      <td>4.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr18</th>\n",
       "      <td>84.90</td>\n",
       "      <td>12.21</td>\n",
       "      <td>2.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr19</th>\n",
       "      <td>76.94</td>\n",
       "      <td>17.03</td>\n",
       "      <td>6.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr2</th>\n",
       "      <td>83.40</td>\n",
       "      <td>13.97</td>\n",
       "      <td>2.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr21</th>\n",
       "      <td>83.67</td>\n",
       "      <td>12.95</td>\n",
       "      <td>3.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr22</th>\n",
       "      <td>81.01</td>\n",
       "      <td>14.61</td>\n",
       "      <td>4.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr3</th>\n",
       "      <td>83.37</td>\n",
       "      <td>13.85</td>\n",
       "      <td>2.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr4</th>\n",
       "      <td>84.27</td>\n",
       "      <td>12.99</td>\n",
       "      <td>2.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr5</th>\n",
       "      <td>83.37</td>\n",
       "      <td>13.93</td>\n",
       "      <td>2.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr6</th>\n",
       "      <td>84.03</td>\n",
       "      <td>12.70</td>\n",
       "      <td>3.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr7</th>\n",
       "      <td>82.26</td>\n",
       "      <td>14.51</td>\n",
       "      <td>3.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr8</th>\n",
       "      <td>83.97</td>\n",
       "      <td>13.30</td>\n",
       "      <td>2.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr9</th>\n",
       "      <td>83.25</td>\n",
       "      <td>13.70</td>\n",
       "      <td>3.05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "labels012      0      1     2\n",
       "chrom                        \n",
       "chr1       82.74  14.18  3.07\n",
       "chr10      82.65  14.36  2.99\n",
       "chr11      83.84  13.11  3.05\n",
       "chr12      82.79  14.13  3.08\n",
       "chr13      84.75  12.43  2.82\n",
       "chr14      82.91  14.25  2.84\n",
       "chr15      83.63  13.22  3.15\n",
       "chr16      81.79  14.66  3.55\n",
       "chr17      79.84  15.93  4.23\n",
       "chr18      84.90  12.21  2.89\n",
       "chr19      76.94  17.03  6.03\n",
       "chr2       83.40  13.97  2.63\n",
       "chr21      83.67  12.95  3.38\n",
       "chr22      81.01  14.61  4.38\n",
       "chr3       83.37  13.85  2.78\n",
       "chr4       84.27  12.99  2.74\n",
       "chr5       83.37  13.93  2.70\n",
       "chr6       84.03  12.70  3.27\n",
       "chr7       82.26  14.51  3.23\n",
       "chr8       83.97  13.30  2.73\n",
       "chr9       83.25  13.70  3.05"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check distribution of labels 0, 1, 2 in train-validation set\n",
    "contingency_table = pd.crosstab(train['chrom'], train['labels012'])\n",
    "\n",
    "# Assuming contingency_table is your computed contingency table\n",
    "normalized_table1 = (contingency_table.div(contingency_table.sum(axis=1), axis=0) * 100).round(2)\n",
    "normalized_table1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2ff97652-1174-4a36-84e0-d65e0412f4b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>labels012</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrom</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr1</th>\n",
       "      <td>82.74</td>\n",
       "      <td>14.19</td>\n",
       "      <td>3.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10</th>\n",
       "      <td>82.65</td>\n",
       "      <td>14.36</td>\n",
       "      <td>2.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr11</th>\n",
       "      <td>83.85</td>\n",
       "      <td>13.11</td>\n",
       "      <td>3.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr12</th>\n",
       "      <td>82.79</td>\n",
       "      <td>14.13</td>\n",
       "      <td>3.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr13</th>\n",
       "      <td>84.76</td>\n",
       "      <td>12.43</td>\n",
       "      <td>2.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr14</th>\n",
       "      <td>82.91</td>\n",
       "      <td>14.25</td>\n",
       "      <td>2.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr15</th>\n",
       "      <td>83.63</td>\n",
       "      <td>13.22</td>\n",
       "      <td>3.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr16</th>\n",
       "      <td>81.79</td>\n",
       "      <td>14.66</td>\n",
       "      <td>3.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr17</th>\n",
       "      <td>79.85</td>\n",
       "      <td>15.92</td>\n",
       "      <td>4.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr18</th>\n",
       "      <td>84.90</td>\n",
       "      <td>12.21</td>\n",
       "      <td>2.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr19</th>\n",
       "      <td>76.94</td>\n",
       "      <td>17.02</td>\n",
       "      <td>6.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr2</th>\n",
       "      <td>83.40</td>\n",
       "      <td>13.97</td>\n",
       "      <td>2.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr21</th>\n",
       "      <td>83.65</td>\n",
       "      <td>12.96</td>\n",
       "      <td>3.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr22</th>\n",
       "      <td>81.01</td>\n",
       "      <td>14.61</td>\n",
       "      <td>4.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr3</th>\n",
       "      <td>83.37</td>\n",
       "      <td>13.85</td>\n",
       "      <td>2.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr4</th>\n",
       "      <td>84.26</td>\n",
       "      <td>12.99</td>\n",
       "      <td>2.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr5</th>\n",
       "      <td>83.37</td>\n",
       "      <td>13.93</td>\n",
       "      <td>2.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr6</th>\n",
       "      <td>84.03</td>\n",
       "      <td>12.70</td>\n",
       "      <td>3.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr7</th>\n",
       "      <td>82.26</td>\n",
       "      <td>14.51</td>\n",
       "      <td>3.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr8</th>\n",
       "      <td>83.97</td>\n",
       "      <td>13.30</td>\n",
       "      <td>2.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr9</th>\n",
       "      <td>83.25</td>\n",
       "      <td>13.70</td>\n",
       "      <td>3.05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "labels012      0      1     2\n",
       "chrom                        \n",
       "chr1       82.74  14.19  3.07\n",
       "chr10      82.65  14.36  2.99\n",
       "chr11      83.85  13.11  3.04\n",
       "chr12      82.79  14.13  3.08\n",
       "chr13      84.76  12.43  2.82\n",
       "chr14      82.91  14.25  2.84\n",
       "chr15      83.63  13.22  3.15\n",
       "chr16      81.79  14.66  3.56\n",
       "chr17      79.85  15.92  4.23\n",
       "chr18      84.90  12.21  2.89\n",
       "chr19      76.94  17.02  6.04\n",
       "chr2       83.40  13.97  2.63\n",
       "chr21      83.65  12.96  3.39\n",
       "chr22      81.01  14.61  4.39\n",
       "chr3       83.37  13.85  2.78\n",
       "chr4       84.26  12.99  2.74\n",
       "chr5       83.37  13.93  2.70\n",
       "chr6       84.03  12.70  3.27\n",
       "chr7       82.26  14.51  3.23\n",
       "chr8       83.97  13.30  2.73\n",
       "chr9       83.25  13.70  3.05"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check distribution of labels 0, 1, 2 in test set\n",
    "contingency_table = pd.crosstab(test['chrom'], test['labels012'])\n",
    "\n",
    "# Assuming contingency_table is your computed contingency table\n",
    "normalized_table2 = (contingency_table.div(contingency_table.sum(axis=1), axis=0) * 100).round(2)\n",
    "normalized_table2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "236bff04-2a47-45f8-a4b7-1cc83ab3bd94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>labels012</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chrom</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>chr1</th>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr10</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr11</th>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr12</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr13</th>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr14</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr15</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr16</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr17</th>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr18</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr19</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>-0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr2</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr21</th>\n",
       "      <td>0.02</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>-0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr22</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr3</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr4</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr5</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr6</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr7</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr8</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chr9</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "labels012     0     1     2\n",
       "chrom                      \n",
       "chr1       0.00 -0.01  0.00\n",
       "chr10      0.00  0.00  0.00\n",
       "chr11     -0.01  0.00  0.01\n",
       "chr12      0.00  0.00  0.00\n",
       "chr13     -0.01  0.00  0.00\n",
       "chr14      0.00  0.00  0.00\n",
       "chr15      0.00  0.00  0.00\n",
       "chr16      0.00  0.00 -0.01\n",
       "chr17     -0.01  0.01  0.00\n",
       "chr18      0.00  0.00  0.00\n",
       "chr19      0.00  0.01 -0.01\n",
       "chr2       0.00  0.00  0.00\n",
       "chr21      0.02 -0.01 -0.01\n",
       "chr22      0.00  0.00 -0.01\n",
       "chr3       0.00  0.00  0.00\n",
       "chr4       0.01  0.00  0.00\n",
       "chr5       0.00  0.00  0.00\n",
       "chr6       0.00  0.00  0.00\n",
       "chr7       0.00  0.00  0.00\n",
       "chr8       0.00  0.00  0.00\n",
       "chr9       0.00  0.00  0.00"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check difference in distribution of labels 0, 1, 2 in train-validation set vs test set\n",
    "normalized_table1 - normalized_table2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e31dc4e9-98f8-44c2-9494-c164be72b625",
   "metadata": {},
   "source": [
    "- distribution of train-validation set matches distribution of test set to 0.02%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c7c5f56b-9140-49bc-9f3d-ed2a82ea2f53",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_30125/3285531863.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  chr20_df['combined'] = chr20_df['chrom'] + '_' + chr20_df['labels012'].astype(str)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(389366, 3504292)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# combine chr20 with rest of test set\n",
    "chr20_df = df[df['chrom'] == 'chr20']\n",
    "chr20_df['combined'] = chr20_df['chrom'] + '_' + chr20_df['labels012'].astype(str)\n",
    "# concatenate chr20_df and test\n",
    "test_df = pd.concat([test, chr20_df], ignore_index=True)\n",
    "len(test_df), len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0793bf25-daed-4266-b620-b2737e3ee056",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>pos_start</th>\n",
       "      <th>pos_end</th>\n",
       "      <th>sequences</th>\n",
       "      <th>labels01</th>\n",
       "      <th>labels012</th>\n",
       "      <th>combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3198699</th>\n",
       "      <td>chr22</td>\n",
       "      <td>17920762</td>\n",
       "      <td>17920963</td>\n",
       "      <td>TGGAAATCTGGGGTCATCTCAGACCCTTCTCTGTTCTCCACTCCTC...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr22_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2552114</th>\n",
       "      <td>chr14</td>\n",
       "      <td>26643125</td>\n",
       "      <td>26643326</td>\n",
       "      <td>AGTTTCTTAATCCTGAGTTCTAATTTGATTGCACTGTGGTCTGAGA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr14_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3189440</th>\n",
       "      <td>chr21</td>\n",
       "      <td>41464158</td>\n",
       "      <td>41464359</td>\n",
       "      <td>AAAAAATGGGAGTTTCTGTGCACAAACTCTCTCCCTGCCTGCTGCC...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr21_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3234028</th>\n",
       "      <td>chr22</td>\n",
       "      <td>49080339</td>\n",
       "      <td>49080540</td>\n",
       "      <td>CATCCCTGGTCGCCTCCCAGGAAGCCTGCCCAGGCCTAGTGCCTGT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr22_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3596339</th>\n",
       "      <td>chr11</td>\n",
       "      <td>78303101</td>\n",
       "      <td>78303302</td>\n",
       "      <td>CTCGGGTGATGAGTGCACCAAAATCTCACAAATCACCACTAAAGAA...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>chr11_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2441643</th>\n",
       "      <td>chr13</td>\n",
       "      <td>37652807</td>\n",
       "      <td>37653008</td>\n",
       "      <td>CTCGGTGAGTCAGAAATTATGTTATCTTATTGCAATCACTAAGCAA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr13_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>473873</th>\n",
       "      <td>chr2</td>\n",
       "      <td>198235696</td>\n",
       "      <td>198235897</td>\n",
       "      <td>AGAGGCAGAGTAGCTATTATACCATCCCCAGCAGGAGATTCATGTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr2_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2813728</th>\n",
       "      <td>chr16</td>\n",
       "      <td>78365185</td>\n",
       "      <td>78365386</td>\n",
       "      <td>GTGCCCAGTTTAGTGCCTGCCAAGAGTAACTGGTGGGTCTGTGTCA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr16_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2036432</th>\n",
       "      <td>chr10</td>\n",
       "      <td>87194820</td>\n",
       "      <td>87195021</td>\n",
       "      <td>GTTAAGAAATTATAAACTTAAGGCCAGGCACAGTGGCTCACGCCTA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr10_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173879</th>\n",
       "      <td>chr1</td>\n",
       "      <td>183634163</td>\n",
       "      <td>183634364</td>\n",
       "      <td>AAAAGTTCTTACACAAAATCTCTGGTTGACAAGCTGTTATTTTCAA...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>chr1_0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         chrom  pos_start    pos_end  \\\n",
       "3198699  chr22   17920762   17920963   \n",
       "2552114  chr14   26643125   26643326   \n",
       "3189440  chr21   41464158   41464359   \n",
       "3234028  chr22   49080339   49080540   \n",
       "3596339  chr11   78303101   78303302   \n",
       "2441643  chr13   37652807   37653008   \n",
       "473873    chr2  198235696  198235897   \n",
       "2813728  chr16   78365185   78365386   \n",
       "2036432  chr10   87194820   87195021   \n",
       "173879    chr1  183634163  183634364   \n",
       "\n",
       "                                                 sequences  labels01  \\\n",
       "3198699  TGGAAATCTGGGGTCATCTCAGACCCTTCTCTGTTCTCCACTCCTC...         0   \n",
       "2552114  AGTTTCTTAATCCTGAGTTCTAATTTGATTGCACTGTGGTCTGAGA...         0   \n",
       "3189440  AAAAAATGGGAGTTTCTGTGCACAAACTCTCTCCCTGCCTGCTGCC...         0   \n",
       "3234028  CATCCCTGGTCGCCTCCCAGGAAGCCTGCCCAGGCCTAGTGCCTGT...         0   \n",
       "3596339  CTCGGGTGATGAGTGCACCAAAATCTCACAAATCACCACTAAAGAA...         1   \n",
       "2441643  CTCGGTGAGTCAGAAATTATGTTATCTTATTGCAATCACTAAGCAA...         0   \n",
       "473873   AGAGGCAGAGTAGCTATTATACCATCCCCAGCAGGAGATTCATGTA...         0   \n",
       "2813728  GTGCCCAGTTTAGTGCCTGCCAAGAGTAACTGGTGGGTCTGTGTCA...         0   \n",
       "2036432  GTTAAGAAATTATAAACTTAAGGCCAGGCACAGTGGCTCACGCCTA...         0   \n",
       "173879   AAAAGTTCTTACACAAAATCTCTGGTTGACAAGCTGTTATTTTCAA...         0   \n",
       "\n",
       "         labels012 combined  \n",
       "3198699          0  chr22_0  \n",
       "2552114          0  chr14_0  \n",
       "3189440          0  chr21_0  \n",
       "3234028          0  chr22_0  \n",
       "3596339          1  chr11_1  \n",
       "2441643          0  chr13_0  \n",
       "473873           0   chr2_0  \n",
       "2813728          0  chr16_0  \n",
       "2036432          0  chr10_0  \n",
       "173879           0   chr1_0  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.to_csv('231117_TP_FP_FN_seqs_labels01_labels012_test.csv', index=False)\n",
    "train.to_csv('231117_TP_FP_FN_seqs_labels01_labels012_train_validation.csv', index=False)\n",
    "train.head(n=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c99e797-862e-412a-8a9b-3eb54fbf6499",
   "metadata": {},
   "source": [
    "## 4. one-hot encode sequences and labels and save Numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "62eaaadc-bed7-4ead-8819-c7eda1d0860d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 57s, sys: 10.7 s, total: 6min 7s\n",
      "Wall time: 6min 7s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((3504292,), (3504292,))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Define a mapping of nucleotides to one-hot vectors\n",
    "nucleotide_mapping = {\n",
    "    'A': [1, 0, 0, 0],\n",
    "    'C': [0, 1, 0, 0],\n",
    "    'G': [0, 0, 1, 0],\n",
    "    'T': [0, 0, 0, 1]\n",
    "}\n",
    "\n",
    "# Function to one-hot encode a DNA sequence\n",
    "def one_hot_encode(sequence):\n",
    "    encoding = np.array([nucleotide_mapping[n] for n in sequence])\n",
    "    return encoding\n",
    "\n",
    "# Apply one-hot encoding to the DNA sequences\n",
    "encoded_sequences = train['sequences'].apply(one_hot_encode)\n",
    "train['sequences'].shape, encoded_sequences.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "45f74ea7-019e-4f6d-b422-4ade16f5a7bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3504292, 201, 4)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_sequences = np.stack(encoded_sequences)\n",
    "encoded_sequences.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e7b068fb-6dd5-44c5-8254-bb672b0db245",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels:\n",
      " [[0 0 0 ... 0 1 1]]\n",
      "One-hot encoded labels:\n",
      " [[1. 1. 1. ... 1. 0. 0.]\n",
      " [0. 0. 0. ... 0. 1. 1.]]\n",
      "CPU times: user 280 ms, sys: 6.08 ms, total: 286 ms\n",
      "Wall time: 283 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# one-hot encode labels\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "labels = train['labels01']\n",
    "\n",
    "one_hot_encoder = OneHotEncoder(categories='auto')\n",
    "labels = np.array(labels).reshape(-1, 1)\n",
    "input_labels = one_hot_encoder.fit_transform(labels).toarray()\n",
    "\n",
    "print('Labels:\\n',labels.T)\n",
    "print('One-hot encoded labels:\\n',input_labels.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "33dbf7d1-db39-4978-8474-fd8aa876c35f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3504292, 2)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "691b18a6-4a82-42ea-a90e-f0741e1e04e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data + labels in npz int8\n",
    "encoded_sequences_int8 = encoded_sequences.astype(np.int8)\n",
    "input_labels_int8 = input_labels.astype(np.int8)\n",
    "np.savez(\"231117_encoded_seqs_labels_int8.npz\", encoded_sequences=encoded_sequences_int8, input_labels=input_labels_int8) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd34d46b-95ab-462c-9028-9743a58debaf",
   "metadata": {},
   "source": [
    "---\n",
    "### NEXT:\n",
    "\n",
    "- test various neural network architectures"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}