07_03_lstm_compose_predict.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM - Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "import time\n",
    "import os\n",
    "import numpy as np\n",
    "import sys\n",
    "from music21 import instrument, note, stream, chord, duration\n",
    "from models.RNNAttention import create_network, sample_with_temp\n",
    "\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run params\n",
    "section = 'compose'\n",
    "run_id = '0006'\n",
    "music_name = 'cello'\n",
    "run_folder = 'run/{}/'.format(section)\n",
    "run_folder += '_'.join([run_id, music_name])\n",
    "\n",
    "# model params\n",
    "embed_size = 100\n",
    "rnn_units = 256\n",
    "use_attention = True\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load the lookup tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "store_folder = os.path.join(run_folder, 'store')\n",
    "\n",
    "with open(os.path.join(store_folder, 'distincts'), 'rb') as filepath:\n",
    "    distincts = pkl.load(filepath)\n",
    "    note_names, n_notes, duration_names, n_durations = distincts\n",
    "\n",
    "with open(os.path.join(store_folder, 'lookups'), 'rb') as filepath:\n",
    "    lookups = pkl.load(filepath)\n",
    "    note_to_int, int_to_note, duration_to_int, int_to_duration = lookups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## build the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_folder = os.path.join(run_folder, 'weights')\n",
    "weights_file = 'weights.h5'\n",
    "\n",
    "model, att_model = create_network(n_notes, n_durations, embed_size, rnn_units, use_attention)\n",
    "\n",
    "# Load the weights to each node\n",
    "weight_source = os.path.join(weights_folder,weights_file)\n",
    "model.load_weights(weight_source)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## build your own phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prediction params\n",
    "notes_temp=0.5\n",
    "duration_temp = 0.5\n",
    "max_extra_notes = 50\n",
    "max_seq_len = 32\n",
    "seq_len = 32\n",
    "\n",
    "# notes = ['START', 'D3', 'D3', 'E3', 'D3', 'G3', 'F#3','D3', 'D3', 'E3', 'D3', 'G3', 'F#3','D3', 'D3', 'E3', 'D3', 'G3', 'F#3','D3', 'D3', 'E3', 'D3', 'G3', 'F#3']\n",
    "# durations = [0, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2]\n",
    "\n",
    "\n",
    "# notes = ['START', 'F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3','F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3', 'F#3', 'G#3', 'F#3', 'E3']\n",
    "# durations = [0, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2, 0.75, 0.25, 1, 1, 1, 2]\n",
    "\n",
    "\n",
    "notes = ['START']\n",
    "durations = [0]\n",
    "\n",
    "if seq_len is not None:\n",
    "    notes = ['START'] * (seq_len - len(notes)) + notes\n",
    "    durations = [0] * (seq_len - len(durations)) + durations\n",
    "\n",
    "\n",
    "sequence_length = len(notes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate notes from the neural network based on a sequence of notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_output = []\n",
    "notes_input_sequence = []\n",
    "durations_input_sequence = []\n",
    "\n",
    "overall_preds = []\n",
    "\n",
    "for n, d in zip(notes,durations):\n",
    "    note_int = note_to_int[n]\n",
    "    duration_int = duration_to_int[d]\n",
    "    \n",
    "    notes_input_sequence.append(note_int)\n",
    "    durations_input_sequence.append(duration_int)\n",
    "    \n",
    "    prediction_output.append([n, d])\n",
    "    \n",
    "    if n != 'START':\n",
    "        midi_note = note.Note(n)\n",
    "\n",
    "        new_note = np.zeros(128)\n",
    "        new_note[midi_note.pitch.midi] = 1\n",
    "        overall_preds.append(new_note)\n",
    "\n",
    "\n",
    "att_matrix = np.zeros(shape = (max_extra_notes+sequence_length, max_extra_notes))\n",
    "\n",
    "for note_index in range(max_extra_notes):\n",
    "\n",
    "    prediction_input = [\n",
    "        np.array([notes_input_sequence])\n",
    "        , np.array([durations_input_sequence])\n",
    "       ]\n",
    "\n",
    "    notes_prediction, durations_prediction = model.predict(prediction_input, verbose=0)\n",
    "    if use_attention:\n",
    "        att_prediction = att_model.predict(prediction_input, verbose=0)[0]\n",
    "        att_matrix[(note_index-len(att_prediction)+sequence_length):(note_index+sequence_length), note_index] = att_prediction\n",
    "    \n",
    "    new_note = np.zeros(128)\n",
    "    \n",
    "    for idx, n_i in enumerate(notes_prediction[0]):\n",
    "        try:\n",
    "            note_name = int_to_note[idx]\n",
    "            midi_note = note.Note(note_name)\n",
    "            new_note[midi_note.pitch.midi] = n_i\n",
    "            \n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "    overall_preds.append(new_note)\n",
    "            \n",
    "    \n",
    "    i1 = sample_with_temp(notes_prediction[0], notes_temp)\n",
    "    i2 = sample_with_temp(durations_prediction[0], duration_temp)\n",
    "    \n",
    "\n",
    "    note_result = int_to_note[i1]\n",
    "    duration_result = int_to_duration[i2]\n",
    "    \n",
    "    prediction_output.append([note_result, duration_result])\n",
    "\n",
    "    notes_input_sequence.append(i1)\n",
    "    durations_input_sequence.append(i2)\n",
    "    \n",
    "    if len(notes_input_sequence) > max_seq_len:\n",
    "        notes_input_sequence = notes_input_sequence[1:]\n",
    "        durations_input_sequence = durations_input_sequence[1:]\n",
    "        \n",
    "#     print(note_result)\n",
    "#     print(duration_result)\n",
    "        \n",
    "    if note_result == 'START':\n",
    "        break\n",
    "\n",
    "overall_preds = np.transpose(np.array(overall_preds)) \n",
    "print('Generated sequence of {} notes'.format(len(prediction_output)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15,15))\n",
    "ax.set_yticks([int(j) for j in range(35,70)])\n",
    "\n",
    "plt.imshow(overall_preds[35:70,:], origin=\"lower\", cmap='coolwarm', vmin = -0.5, vmax = 0.5, extent=[0, max_extra_notes, 35,70]\n",
    "          \n",
    "          )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## convert the output from the prediction to notes and create a midi file from the notes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_folder = os.path.join(run_folder, 'output')\n",
    "\n",
    "midi_stream = stream.Stream()\n",
    "\n",
    "# create note and chord objects based on the values generated by the model\n",
    "for pattern in prediction_output:\n",
    "    note_pattern, duration_pattern = pattern\n",
    "    # pattern is a chord\n",
    "    if ('.' in note_pattern):\n",
    "        notes_in_chord = note_pattern.split('.')\n",
    "        chord_notes = []\n",
    "        for current_note in notes_in_chord:\n",
    "            new_note = note.Note(current_note)\n",
    "            new_note.duration = duration.Duration(duration_pattern)\n",
    "            new_note.storedInstrument = instrument.Violoncello()\n",
    "            chord_notes.append(new_note)\n",
    "        new_chord = chord.Chord(chord_notes)\n",
    "        midi_stream.append(new_chord)\n",
    "    elif note_pattern == 'rest':\n",
    "    # pattern is a rest\n",
    "        new_note = note.Rest()\n",
    "        new_note.duration = duration.Duration(duration_pattern)\n",
    "        new_note.storedInstrument = instrument.Violoncello()\n",
    "        midi_stream.append(new_note)\n",
    "    elif note_pattern != 'START':\n",
    "    # pattern is a note\n",
    "        new_note = note.Note(note_pattern)\n",
    "        new_note.duration = duration.Duration(duration_pattern)\n",
    "        new_note.storedInstrument = instrument.Violoncello()\n",
    "        midi_stream.append(new_note)\n",
    "\n",
    "\n",
    "\n",
    "midi_stream = midi_stream.chordify()\n",
    "timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "midi_stream.write('midi', fp=os.path.join(output_folder, 'output-' + timestr + '.mid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## attention plot\n",
    "if use_attention:\n",
    "    fig, ax = plt.subplots(figsize=(20,20))\n",
    "\n",
    "    im = ax.imshow(att_matrix[(seq_len-2):,], cmap='coolwarm', interpolation='nearest')\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    # Minor ticks\n",
    "    ax.set_xticks(np.arange(-.5, len(prediction_output)- seq_len, 1), minor=True);\n",
    "    ax.set_yticks(np.arange(-.5, len(prediction_output)- seq_len, 1), minor=True);\n",
    "\n",
    "    # Gridlines based on minor ticks\n",
    "    ax.grid(which='minor', color='black', linestyle='-', linewidth=1)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    # We want to show all ticks...\n",
    "    ax.set_xticks(np.arange(len(prediction_output) - seq_len))\n",
    "    ax.set_yticks(np.arange(len(prediction_output)- seq_len+2))\n",
    "    # ... and label them with the respective list entries\n",
    "    ax.set_xticklabels([n[0] for n in prediction_output[(seq_len):]])\n",
    "    ax.set_yticklabels([n[0] for n in prediction_output[(seq_len - 2):]])\n",
    "\n",
    "    # ax.grid(color='black', linestyle='-', linewidth=1)\n",
    "\n",
    "    ax.xaxis.tick_top()\n",
    "\n",
    "\n",
    "    \n",
    "    plt.setp(ax.get_xticklabels(), rotation=90, ha=\"left\", va = \"center\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gdl_code",
   "language": "python",
   "name": "gdl_code"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}