BeethovenQuartets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DelimitedFiles\n",
    "using Distances\n",
    "using FileIO\n",
    "using LinearAlgebra\n",
    "using Random\n",
    "using Statistics\n",
    "using DataFrames\n",
    "using DataFramesMeta\n",
    "using Plots\n",
    "using PyCall\n",
    "using CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PyObject <module 'music21' from '/home/alfredo/.local/lib/python3.10/site-packages/music21/__init__.py'>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "m21 = pyimport(\"music21\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "get_stochastic_kseq (generic function with 1 method)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#functions from personal packages.\n",
    "git_path = \"/home/alfredo/Git/Harmony_Evolution/src\"\n",
    "include(joinpath(git_path,\"CEGFunctions.jl\"))\n",
    "include(joinpath(git_path,\"Constr_series.jl\"))\n",
    "include(joinpath(git_path,\"IM_Functions.jl\"))\n",
    "include(joinpath(git_path,\"StochasticFunctions.jl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "xml_path = \"/home/alfredo/MusicPenn/Beethoven_HAnalysis/XMLFiles\" #xml files \n",
    "adata_path = \"/home/alfredo/Git/Harmony_Evolution/Beethoven_Annotations/\" #annotated data\n",
    "out_path = \"/home/alfredo/Git/Harmony_Evolution/Beethoven_Annotations/\"\n",
    "xml_list = readdir(xml_path)\n",
    "ann_files = readdir(adata_path)\n",
    "adata_list = ann_files[findall(x -> occursin(r\".tsv\", x), ann_files)];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "opus_number = map(x-> join(split(x, \"_\")[1:2],\"_\"), xml_list)\n",
    "u_opus = unique(opus_number);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op127_no12_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op127_no12_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op127_no12_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op127_no12_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op130_no13_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op130_no13_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op130_no13_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op130_no13_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op130_no13_mov5.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op130_no13_mov6.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov5.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov6.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op131_no14_mov7.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op132_no15_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op132_no15_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op132_no15_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op132_no15_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op132_no15_mov5.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op135_no16_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op135_no16_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op135_no16_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op135_no16_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no1_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no1_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no1_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no1_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no2_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no2_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no2_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no2_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no3_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no3_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no3_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no3_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no4_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no4_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no4_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no4_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no5_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no5_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no5_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no5_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no6_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no6_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no6_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op18_no6_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no7_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no7_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no7_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no7_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no8_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no8_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no8_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no8_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no9_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no9_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no9_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op59_no9_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op74_no10_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op74_no10_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op74_no10_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op74_no10_mov4.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op95_no11_mov1.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op95_no11_mov2.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op95_no11_mov3.musicxml DONE!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File op95_no11_mov4.musicxml DONE!\n"
     ]
    }
   ],
   "source": [
    "for xml_i in 1:length(xml_list)\n",
    "    local fh_kseq\n",
    "    #loading csv from midi\n",
    "    piece_xml = m21.converter.parse(joinpath(xml_path,xml_list[xml_i]))\n",
    "\n",
    "\n",
    "    #loading hand-annotation data\n",
    "    piece_ann = CSV.read(joinpath(adata_path,adata_list[xml_i]), DataFrame) \n",
    "    df_piece =  get_xml_df(piece_xml)\n",
    "\n",
    "\n",
    "    separated_measures = groupby(df_piece, :Measure)\n",
    "\n",
    "\n",
    "    kseq = Any[]\n",
    "    n_bar = Int[]\n",
    "    unc_seq = Float64[]\n",
    "    for m in separated_measures\n",
    "        out = get_distance_to_keys(get_center_effect(Matrix(m)))\n",
    "        push!(n_bar, m[1,:Measure])\n",
    "        push!(kseq, out[1,1])\n",
    "        push!(unc_seq, round(get_key_ent(out),digits=4))\n",
    "    end\n",
    "    #fundamental key\n",
    "    fun_key = get_rank_freq(kseq)[1,1] #Getting the most repeated key (expected to be the global key)\n",
    "    #fun_key = piece_ann[1,:global_key]\n",
    "    try\n",
    "        fh_kseq = funhar_seq(kseq, fun_key) #mapping the key sequence to a functional harmony sequence, taking as reference the global key\n",
    "    catch\n",
    "        try\n",
    "            new_funkey = key_translate[fun_key]\n",
    "            fh_kseq = funhar_seq(kseq, new_funkey)\n",
    "        catch\n",
    "            fun_key = get_rank_freq(kseq)[1,1]\n",
    "            fh_kseq = funhar_seq(kseq, fun_key)\n",
    "        end\n",
    "    end\n",
    "    nmea = piece_ann.measure\n",
    "    fh_out = []\n",
    "    unc_out = []\n",
    "    kseq_out = []\n",
    "    for n_m in nmea\n",
    "        loc = findfirst(x-> x==n_m, n_bar)\n",
    "        if !isnothing(loc)\n",
    "            push!(fh_out, fh_kseq[loc])\n",
    "            push!(unc_out, unc_seq[loc])\n",
    "            push!(kseq_out, kseq[loc])\n",
    "        else\n",
    "            push!(fh_out, \"N/A\")\n",
    "            push!(unc_out, \"N/A\")\n",
    "            push!(kseq_out, \"N/A\")\n",
    "        end\n",
    "    end\n",
    "\n",
    "    g_key = [get_rank_freq(kseq)[1,1] for i in length(fh_out)]\n",
    "    #construct the output dataframe\n",
    "    df_out = DataFrame(\n",
    "        :Measure => piece_ann[!,:measure],\n",
    "        :Global_Key => piece_ann[!,:global_key],\n",
    "        :Local_Key => piece_ann[!,:local_key],\n",
    "        :Chord => piece_ann[!,:chord],\n",
    "        :Relative_Numeral => piece_ann[!,:numeral],\n",
    "        :CoE_KeyCall => fh_out,\n",
    "        :CoE_Uncert => unc_out,\n",
    "        :CoE_KeySequence => kseq_out,\n",
    "        :CoE_GlobalKey => g_key\n",
    "    )\n",
    "    #exporting...\n",
    "    xml_file = xml_list[xml_i]\n",
    "    name_out = \"CoEKeyCallNEW-$(join([split(xml_file,\".\")[1] \"csv\"],\".\"))\"\n",
    "    CSV.write(joinpath(out_path,name_out), df_out, header=true)\n",
    "    println(\"File $(xml_file) DONE!\")\n",
    "    flush(stdout)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_files = readdir(out_path)\n",
    "compare_list = csv_files[findall(x-> occursin(r\"NEW-\",x),csv_files)];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "unc_wrong = []\n",
    "fwrongs = []\n",
    "mwrongs = []\n",
    "good_guess = []\n",
    "for f in 1:length(compare_list)\n",
    "    bars = 0\n",
    "    g_guess = 0\n",
    "    df_ann = CSV.read(joinpath(out_path,compare_list[f]),DataFrame)\n",
    "    \n",
    "    g_key = get_rank_freq(df_ann[!,:Local_Key])[1,1]\n",
    "    ann_meas = groupby(df_ann[df_ann.Local_Key .==g_key,:],:Measure)\n",
    "    #ann_meas = groupby(df_ann, :Measure)\n",
    "    \n",
    "    tn_m = length(ann_meas)\n",
    "    for i in 1:tn_m\n",
    "        \n",
    "        ann_keys = vcat(ann_meas[i][:,:Local_Key]..., ann_meas[i][:,:Relative_Numeral]..., ann_meas[i][:,:Chord]...)\n",
    "        #ann_keys = vcat(ann_meas[i][:,:Relative_Numeral]...)\n",
    "        coe_key = ann_meas[i][1,:CoE_KeyCall]\n",
    "        if !isempty(findall(x-> x==coe_key, coalesce.(ann_keys,\"N/A\")))\n",
    "            g_guess += 1\n",
    "        else\n",
    "            push!(unc_wrong, ann_meas[i][1,:CoE_Uncert])\n",
    "            push!(fwrongs, f)\n",
    "            push!(mwrongs, i)\n",
    "        end\n",
    "        bars +=1\n",
    "    end\n",
    "    push!(good_guess, g_guess / bars)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the median accuracy for all the movements is: 68.435%\n"
     ]
    }
   ],
   "source": [
    "println(\"the median accuracy for all the movements is: \", round(median(good_guess)*100,digits=3) ,\"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.8.0",
   "language": "julia",
   "name": "julia-1.8"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}