Autoencoders_folksonomies_OneHotencodding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/trbuser/DATA/blob/master/Autoencoders_folksonomies_OneHotencodding.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dhv7hz3dkgbc",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 522
        },
        "outputId": "7c28499d-a0b9-4396-d421-bbde3c80bfa1"
      },
      "source": [
        "import numpy as np\n",
        "from keras.layers import Input, Dense\n",
        "from keras.models import Model\n",
        "from keras.datasets import mnist\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "from numpy.random import seed\n",
        "from sklearn.preprocessing import minmax_scale\n",
        "\n",
        "\n",
        "########encoding the collection into  OneHotEncoder for dataframe\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "import io\n",
        "data = pd.read_csv(io.BytesIO(uploaded['tags.csv']))\n",
        "\n",
        "#######\"en dataframe\"\n",
        "\n",
        "\n",
        "\n",
        "corpus_df=data[['userId','tag','movieId']] \n",
        "corpus_df=corpus_df.dropna()\n",
        "\n",
        "print(corpus_df)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "#############################################\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "encoder = sklearn.preprocessing.OneHotEncoder()\n",
        "label_encoder = sklearn.preprocessing.LabelEncoder()\n",
        "data_label_encoded = label_encoder.fit_transform(data['category_feature'])\n",
        "data['category_feature'] = data_label_encoded\n",
        "data_feature_one_hot_encoded = encoder.fit_transform(data[['category_feature']].as_matrix())\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "###############\n",
        "from category_encoders import OneHotEncoder\n",
        "\n",
        "encoder = OneHotEncoder(cols=['userId','tag','movieId'], handle_unknown='ignore')\n",
        "ac=encoder.fit_transform(corpus_df)\n",
        "\n",
        "df=ac\n",
        "\n",
        "print(df.shape)\n",
        "print(type(df))\n",
        "\n",
        "\n",
        "train = df.sample(frac=.8, random_state=42)\n",
        "test = df.loc[~df.index.isin(train.index)]\n",
        "\n",
        "\n",
        "########### definition du nombre de dimensions\n",
        "\n",
        "\n",
        "print(\"train.shape[1:]\")\n",
        "print(train.shape[:1])\n",
        "\n",
        "\n",
        "\n",
        "print(\"train\")\n",
        "print(train)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "encoding_dim = 10\n",
        "\n",
        "\n",
        "train=train.values                                                          \n",
        "test=test.values\n",
        "\n",
        "#(32966,1)\n",
        "x_train_simple = train.reshape((len(train), np.prod(train.shape[1:])))\n",
        " \n",
        "x_test_simple = test.reshape((len(test), np.prod(test.shape[1:])))\n",
        "\n",
        "\n",
        "window_length=1454\n",
        "\n",
        "\n",
        "from keras.layers import Input, Dense, Conv1D, MaxPooling1D, UpSampling1D, BatchNormalization, LSTM, RepeatVector\n",
        "from keras.models import Model\n",
        "from keras.models import model_from_json\n",
        "from keras import regularizers\n",
        "\n",
        "\n",
        "\n",
        "input_window = Input(shape=(window_length,))\n",
        "# \"encoded\" is the encoded representation of the input\n",
        "encoded = Dense(encoding_dim, activation='relu')(input_window)\n",
        "# \"decoded\" is the lossy reconstruction of the input\n",
        "decoded = Dense(window_length, activation='sigmoid')(encoded)\n",
        "\n",
        "# this model maps an input to its reconstruction\n",
        "autoencoder = Model(input_window, decoded)\n",
        "\n",
        "# this model maps an input to its encoded representation\n",
        "encoder = Model(input_window, encoded)\n",
        "#mean\n",
        "autoencoder.summary()\n",
        "autoencoder.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
        "history = autoencoder.fit(x_train_simple, x_train_simple,\n",
        "                epochs=20,\n",
        "                batch_size=1024,\n",
        "                shuffle=True,\n",
        "                verbose=2,\n",
        "                validation_data=(x_test_simple, x_test_simple))\n",
        "\n",
        "decoded_stocks = autoencoder.predict(x_test_simple)\n",
        "\n",
        "\n",
        "plt.imshow(x_test_simple, interpolation='none')\n",
        "plt.show()\n",
        "plt.imshow(decoded_stocks, interpolation='none')\n",
        "plt.show()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "plt.plot(x_test_simple,decoded_stocks)\n",
        "plt.show()\n",
        "print(decoded_stocks)\n",
        "print(\"predicted dimensions\")\n",
        "print(decoded_stocks.shape)\n",
        "print(\"types of x_test_simple\")\n",
        "print(x_test_simple.shape)\n",
        "print(type(decoded_stocks))\n",
        "\n",
        "######retourne numpy ndarry \n",
        "  \n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(20)\n",
        "plt.figure()\n",
        "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
        "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.legend()\n",
        "plt.show()    \n",
        "############################### accuracy\n",
        "\n",
        "\n",
        "accuracy = history.history['acc']\n",
        "val_accuracy = history.history['val_acc']\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(len(accuracy))\n",
        "plt.plot(epochs, accuracy, 'g', label='Training accuracy')\n",
        "plt.plot(epochs, val_accuracy, 'r', label='Validation accuracy')\n",
        "plt.title('Training and validation accuracy')\n",
        "plt.legend()\n",
        "plt.figure()\n",
        "plt.plot(epochs, loss, 'g', label='Training loss')\n",
        "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.legend()\n",
        "plt.show()\n",
        "#####################\n",
        "\n",
        "test_samples = 200\n",
        "\n",
        "def plot_examples(stock_input, stock_decoded):\n",
        "    n = 10  \n",
        "    plt.figure(figsize=(20, 4))\n",
        "    for i, idx in enumerate(list(np.arange(0, test_samples, 20))):\n",
        "        # display original\n",
        "        ax = plt.subplot(2, n, i + 1)\n",
        "        if i == 0:\n",
        "            ax.set_ylabel(\"encoded\", fontweight=600)\n",
        "        else:\n",
        "            ax.get_yaxis().set_visible(False)\n",
        "        plt.plot(stock_input[idx])\n",
        "        ax.get_xaxis().set_visible(False)\n",
        "         \n",
        "        \n",
        "\n",
        "        # display reconstruction\n",
        "        ax = plt.subplot(2, n, i + 1 + n)\n",
        "        if i == 0:\n",
        "            ax.set_ylabel(\"found representation Decoded\", fontweight=600)\n",
        "        else:\n",
        "            ax.get_yaxis().set_visible(False)\n",
        "        plt.plot(stock_decoded[idx])\n",
        "        ax.get_xaxis().set_visible(False)\n",
        "        \n",
        "        \n",
        "\n",
        "def plot_history(history):\n",
        "    plt.figure(figsize=(15, 5))\n",
        "    ax = plt.subplot(1, 2, 1)\n",
        "    plt.plot(history.history[\"loss\"])\n",
        "    plt.title(\"Train loss\")\n",
        "    ax = plt.subplot(1, 2, 2)\n",
        "    plt.plot(history.history[\"val_loss\"])\n",
        "    plt.title(\"Test loss\")\n",
        "     \n",
        "\n",
        "\n",
        "plot_history(history)\n",
        "plot_examples(x_test_simple, decoded_stocks) \n",
        "\n",
        "\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-65d03b57-819e-4a24-94c0-e0c295d07276\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-65d03b57-819e-4a24-94c0-e0c295d07276\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving tags.csv to tags (3).csv\n",
            "        userId            tag  movieId\n",
            "0           18    Mark Waters     4141\n",
            "1           65      dark hero      208\n",
            "2           65      dark hero      353\n",
            "3           65  noir thriller      521\n",
            "4           65      dark hero      592\n",
            "...        ...            ...      ...\n",
            "465559  138446        dragged    55999\n",
            "465560  138446  Jason Bateman    55999\n",
            "465561  138446         quirky    55999\n",
            "465562  138446            sad    55999\n",
            "465563  138472  rise to power      923\n",
            "\n",
            "[465548 rows x 3 columns]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-e566b8c963f3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcategory_encoders\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0mencoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcols\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'userId'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'tag'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'movieId'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle_unknown\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m \u001b[0mac\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'sparse'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Dft4rLl0lJB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "import numpy as np\n",
        "from keras.layers import Input, Dense\n",
        "from keras.models import Model\n",
        "from keras.datasets import mnist\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "from numpy.random import seed\n",
        "from sklearn.preprocessing import minmax_scale\n",
        "\n",
        "\n",
        "########encoding the collection into  OneHotEncoder for dataframe\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "import io\n",
        "data = pd.read_csv(io.BytesIO(uploaded['tags.csv']))\n",
        "\n",
        "#######\"en dataframe\"\n",
        "\n",
        "import sklearn\n",
        "\n",
        "encoder = sklearn.preprocessing.OneHotEncoder()\n",
        "label_encoder_1 = sklearn.preprocessing.LabelEncoder()\n",
        "data_label_encoded_tag = label_encoder_1.fit_transform(data['tag'])\n",
        "data['tag'] = data_label_encoded_tag\n",
        "tag_feature_one_hot_encoded = encoder.fit_transform(data[['tag']].as_matrix())\n",
        "\n",
        "\n",
        "\n",
        "encoder_2 = sklearn.preprocessing.OneHotEncoder()\n",
        "label_encoder_2 = sklearn.preprocessing.LabelEncoder()\n",
        "data_label_encoded_movieId = label_encoder_2.fit_transform(data['movieId'])\n",
        "data['movieId'] = data_label_encoded_movieId\n",
        "movieId_feature_one_hot_encoded = encoder_2.fit_transform(data[['movieId']].as_matrix())\n",
        "\n",
        "encoder_3 = sklearn.preprocessing.OneHotEncoder()\n",
        "label_encoder_3 = sklearn.preprocessing.LabelEncoder()\n",
        "data_label_encoded_userId = label_encoder_3.fit_transform(data['userId'])\n",
        "data['userId'] = data_label_encoded_userId\n",
        "userId_feature_one_hot_encoded = encoder_3.fit_transform(data[['userId']].as_matrix())\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AXx17uhf20ja",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        },
        "outputId": "d07d76d1-2a59-4e07-da68-4e389ee53918"
      },
      "source": [
        "pip install --upgrade category_encoders"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting category_encoders\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a0/52/c54191ad3782de633ea3d6ee3bb2837bda0cf3bc97644bb6375cf14150a0/category_encoders-2.1.0-py2.py3-none-any.whl (100kB)\n",
            "\r\u001b[K     |███▎                            | 10kB 23.9MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 20kB 2.2MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 30kB 3.2MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 40kB 2.1MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 51kB 2.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 61kB 3.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 71kB 3.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 81kB 4.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 92kB 4.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 102kB 2.9MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (1.17.4)\n",
            "Requirement already satisfied, skipping upgrade: statsmodels>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.10.2)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn>=0.20.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.21.3)\n",
            "Requirement already satisfied, skipping upgrade: patsy>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.5.1)\n",
            "Requirement already satisfied, skipping upgrade: scipy>=0.19.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (1.3.3)\n",
            "Requirement already satisfied, skipping upgrade: pandas>=0.21.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders) (0.25.3)\n",
            "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20.0->category_encoders) (0.14.1)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from patsy>=0.4.1->category_encoders) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.21.1->category_encoders) (2.6.1)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.21.1->category_encoders) (2018.9)\n",
            "Installing collected packages: category-encoders\n",
            "Successfully installed category-encoders-2.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fRZW8sEb35r6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "from keras.layers import Input, Dense\n",
        "from keras.models import Model\n",
        "from keras.datasets import mnist\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "from numpy.random import seed\n",
        "from sklearn.preprocessing import minmax_scale\n",
        "\n",
        "\n",
        "########encoding the collection into  OneHotEncoder for dataframe\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "import io\n",
        "data = pd.read_csv(io.BytesIO(uploaded['tags.csv']))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "corpus_df=data[['userId','tag','movieId']] \n",
        "corpus_df=corpus_df.dropna()\n",
        "\n",
        "print(corpus_df)\n",
        "\n",
        "corpus_df['tag'] = pd.Categorical(corpus_df['tag'])\n",
        "corpus_df['tag'] = corpus_df.tag.cat.codes\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "df=corpus_df\n",
        "\n",
        "print(df.shape)\n",
        "print(type(df))\n",
        "\n",
        "train = df.sample(frac=.8, random_state=42)\n",
        "test = df.loc[~df.index.isin(train.index)]\n",
        "\n",
        "\n",
        "########### definition du nombre de dimensions\n",
        "\n",
        "\n",
        "print(\"train.shape[1:]\")\n",
        "print(train.shape[:1])\n",
        "\n",
        "\n",
        "\n",
        "print(\"train\")\n",
        "print(train)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "encoding_dim = 1000\n",
        "\n",
        "\n",
        "train=train.values                                                          \n",
        "test=test.values\n",
        "\n",
        "#(32966,1)\n",
        "x_train_simple = train.reshape((len(train), np.prod(train.shape[1:])))\n",
        " \n",
        "x_test_simple = test.reshape((len(test), np.prod(test.shape[1:])))\n",
        "\n",
        "\n",
        "window_length=3\n",
        "\n",
        "\n",
        "from keras.layers import Input, Dense, Conv1D, MaxPooling1D, UpSampling1D, BatchNormalization, LSTM, RepeatVector\n",
        "from keras.models import Model\n",
        "from keras.models import model_from_json\n",
        "from keras import regularizers\n",
        "\n",
        "\n",
        "\n",
        "input_window = Input(shape=(window_length,))\n",
        "# \"encoded\" is the encoded representation of the input\n",
        "encoded = Dense(encoding_dim, activation='relu')(input_window)\n",
        "# \"decoded\" is the lossy reconstruction of the input\n",
        "decoded = Dense(window_length, activation='sigmoid')(encoded)\n",
        "\n",
        "# this model maps an input to its reconstruction\n",
        "autoencoder = Model(input_window, decoded)\n",
        "\n",
        "# this model maps an input to its encoded representation\n",
        "encoder = Model(input_window, encoded)\n",
        "#mean\n",
        "autoencoder.summary()\n",
        "autoencoder.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
        "history = autoencoder.fit(x_train_simple, x_train_simple,\n",
        "                epochs=20,\n",
        "                batch_size=1024,\n",
        "                shuffle=True,\n",
        "                verbose=2,\n",
        "                validation_data=(x_test_simple, x_test_simple))\n",
        "\n",
        "decoded_stocks = autoencoder.predict(x_test_simple)\n",
        "\n",
        "\n",
        "plt.imshow(x_test_simple, interpolation='none')\n",
        "plt.show()\n",
        "plt.imshow(decoded_stocks, interpolation='none')\n",
        "plt.show()\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "plt.plot(x_test_simple,decoded_stocks)\n",
        "plt.show()\n",
        "print(decoded_stocks)\n",
        "print(\"predicted dimensions\")\n",
        "print(decoded_stocks.shape)\n",
        "print(\"types of x_test_simple\")\n",
        "print(x_test_simple.shape)\n",
        "print(type(decoded_stocks))\n",
        "\n",
        "######retourne numpy ndarry \n",
        "  \n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(20)\n",
        "plt.figure()\n",
        "plt.plot(epochs, loss, 'g', label='Training loss')\n",
        "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.legend()\n",
        "plt.show()    \n",
        "############################### accuracy\n",
        "\n",
        "\n",
        "accuracy = history.history['acc']\n",
        "val_accuracy = history.history['val_acc']\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(len(accuracy))\n",
        "plt.plot(epochs, accuracy, 'g', label='Training accuracy')\n",
        "plt.plot(epochs, val_accuracy, 'r', label='Validation accuracy')\n",
        "plt.title('Training and validation accuracy')\n",
        "plt.legend()\n",
        "plt.figure()\n",
        "plt.plot(epochs, loss, 'g', label='Training loss')\n",
        "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.legend()\n",
        "plt.show()\n",
        "#####################\n",
        "\n",
        "test_samples = 2000\n",
        "\n",
        "def plot_examples(stock_input, stock_decoded):\n",
        "    n = 10  \n",
        "    plt.figure(figsize=(20, 4))\n",
        "    for i, idx in enumerate(list(np.arange(0, test_samples, 200))):\n",
        "        # display original\n",
        "        ax = plt.subplot(2, n, i + 1)\n",
        "        if i == 0:\n",
        "            ax.set_ylabel(\"encoded\", fontweight=600)\n",
        "        else:\n",
        "            ax.get_yaxis().set_visible(False)\n",
        "        plt.plot(stock_input[idx])\n",
        "        ax.get_xaxis().set_visible(False)\n",
        "         \n",
        "        \n",
        "\n",
        "        # display reconstruction\n",
        "        ax = plt.subplot(2, n, i + 1 + n)\n",
        "        if i == 0:\n",
        "            ax.set_ylabel(\"found representation Decoded\", fontweight=600)\n",
        "        else:\n",
        "            ax.get_yaxis().set_visible(False)\n",
        "        plt.plot(stock_decoded[idx])\n",
        "        ax.get_xaxis().set_visible(False)\n",
        "        \n",
        "        \n",
        "\n",
        "def plot_history(history):\n",
        "    plt.figure(figsize=(15, 5))\n",
        "    ax = plt.subplot(1, 2, 1)\n",
        "    plt.plot(history.history[\"loss\"])\n",
        "    plt.title(\"Train loss\")\n",
        "    ax = plt.subplot(1, 2, 2)\n",
        "    plt.plot(history.history[\"val_loss\"])\n",
        "    plt.title(\"Test loss\")\n",
        "     \n",
        "\n",
        "\n",
        "plot_history(history)\n",
        "plot_examples(x_test_simple, decoded_stocks) \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        " \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liOkTYDr36O5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}