pca.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_DuIw7bBBuRa",
        "outputId": "019d189f-d50f-4002-a36f-717b1c5599c0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "orchid_test.csv  orchid_train.csv  orchid_val.csv  sample_data\n"
          ]
        }
      ],
      "source": [
        "!ls"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get -y install openjdk-8-jre-headless\n",
        "!pip install pyspark\n",
        "from pyspark import SparkContext\n",
        "sc = SparkContext.getOrCreate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TTNrUBlYChEj",
        "outputId": "2e499541-f8e0-4a18-b1a0-37bde5a5d188"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  libxtst6\n",
            "Suggested packages:\n",
            "  libnss-mdns fonts-dejavu-extra fonts-nanum fonts-ipafont-gothic fonts-ipafont-mincho\n",
            "  fonts-wqy-microhei fonts-wqy-zenhei fonts-indic\n",
            "The following NEW packages will be installed:\n",
            "  libxtst6 openjdk-8-jre-headless\n",
            "0 upgraded, 2 newly installed, 0 to remove and 45 not upgraded.\n",
            "Need to get 30.8 MB of archives.\n",
            "After this operation, 104 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxtst6 amd64 2:1.2.3-1build4 [13.4 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 openjdk-8-jre-headless amd64 8u402-ga-2ubuntu1~22.04 [30.8 MB]\n",
            "Fetched 30.8 MB in 2min 13s (232 kB/s)\n",
            "Selecting previously unselected package libxtst6:amd64.\n",
            "(Reading database ... 121918 files and directories currently installed.)\n",
            "Preparing to unpack .../libxtst6_2%3a1.2.3-1build4_amd64.deb ...\n",
            "Unpacking libxtst6:amd64 (2:1.2.3-1build4) ...\n",
            "Selecting previously unselected package openjdk-8-jre-headless:amd64.\n",
            "Preparing to unpack .../openjdk-8-jre-headless_8u402-ga-2ubuntu1~22.04_amd64.deb ...\n",
            "Unpacking openjdk-8-jre-headless:amd64 (8u402-ga-2ubuntu1~22.04) ...\n",
            "Setting up libxtst6:amd64 (2:1.2.3-1build4) ...\n",
            "Setting up openjdk-8-jre-headless:amd64 (8u402-ga-2ubuntu1~22.04) ...\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/orbd to provide /usr/bin/orbd (orbd) in auto mode\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/servertool to provide /usr/bin/servertool (servertool) in auto mode\n",
            "update-alternatives: using /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/tnameserv to provide /usr/bin/tnameserv (tnameserv) in auto mode\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.5.1.tar.gz (317.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.0/317.0 MB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n",
            "Building wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.5.1-py2.py3-none-any.whl size=317488491 sha256=e7c23480e6dba60a9c08d27cc330528c76380f40f88a2c16fd39269b5f185d1a\n",
            "  Stored in directory: /root/.cache/pip/wheels/80/1d/60/2c256ed38dddce2fdd93be545214a63e02fbd8d74fb0b7f3a6\n",
            "Successfully built pyspark\n",
            "Installing collected packages: pyspark\n",
            "Successfully installed pyspark-3.5.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_raw_data=  sc.textFile(\"orchid_train.csv\")\n",
        "train_str_data= train_raw_data.map(lambda x:x.split(','))\n",
        "train_data = train_str_data.map(lambda x:[float(y)for y in x])\n",
        "print(train_data.take(20)[0][-5:])\n",
        "print(train_data.take(20)[9][-5:])\n",
        "print(train_data.take(20)[18][-5:])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-75HNMBdCNzv",
        "outputId": "e026c97b-72e0-42cc-9bc9-ff4345fa0107"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.1933087557554245, -0.0045260111801326275, 0.1272827833890915, -0.2245592623949051, 0.0]\n",
            "[0.12061325460672379, -0.23026636242866516, -0.023545075207948685, -0.13828127086162567, 1.0]\n",
            "[0.36595019698143005, -0.5046675801277161, -0.5441071391105652, -0.08429872989654541, 2.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "n_cases =train_data.count()\n",
        "label_distinct =train_data.map(lambda x:x[-1]).distinct()\n",
        "n_classes_ =label_distinct.count()\n",
        "print('num of cases= ',n_cases)\n",
        "print('num of classes= ',n_classes_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IFDM69d8DIZU",
        "outputId": "af03444a-be99-4c1e-e808-fb62ebca7187"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "num of cases=  1533\n",
            "num of classes=  219\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_raw_data=  sc.textFile(\"orchid_val.csv\")\n",
        "val_str_data= val_raw_data.map(lambda x:x.split(','))\n",
        "val_data = val_str_data.map(lambda x:[float(y)for y in x])\n",
        "\n",
        "test_raw_data=  sc.textFile(\"orchid_test.csv\")\n",
        "test_str_data= test_raw_data.map(lambda x:x.split(','))\n",
        "test_data = test_str_data.map(lambda x:[float(y)for y in x])"
      ],
      "metadata": {
        "id": "9gvRpbHWEa31"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.regression import LabeledPoint\n",
        "lp_train =train_data.map(lambda x:LabeledPoint(x[-1],x[:-1]) )\n",
        "lp_val = val_data.map(lambda x:LabeledPoint(x[-1],x[:-1]) )\n",
        "lp_test = test_data.map(lambda x:LabeledPoint(x[-1],x[:-1]) )"
      ],
      "metadata": {
        "id": "NmIexWHLCgIN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(5):\n",
        "  data_col =lp_train.map(lambda lp:lp.features[i])\n",
        "  print(\"第\",i+1,\"個特徵\",data_col.stats())\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KTnB0GcMCN2F",
        "outputId": "c75fd3dc-6984-49fe-b59c-5b2b6ccb281c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "第 1 個特徵 (count: 1533, mean: -0.028009288536736458, stdev: 0.45923738706579875, max: 1.4892573356628418, min: -1.3924832344055176)\n",
            "第 2 個特徵 (count: 1533, mean: 0.003849973568314006, stdev: 0.43567297671586475, max: 1.3220020532608032, min: -1.446252465248108)\n",
            "第 3 個特徵 (count: 1533, mean: -0.05128098582584903, stdev: 0.4343921399980737, max: 1.4108004570007324, min: -1.6000101566314697)\n",
            "第 4 個特徵 (count: 1533, mean: -0.11399428229950634, stdev: 0.44183990953013985, max: 1.5220394134521484, min: -1.415898084640503)\n",
            "第 5 個特徵 (count: 1533, mean: 0.01973237649594426, stdev: 0.4270469439187318, max: 1.2699164152145386, min: -1.339835286140442)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "def confusion_matrix(GT_Pred):\n",
        "  GT =GT_Pred.map(lambda x :x[0]).collect()\n",
        "  Pred=GT_Pred.map(lambda x :x[1]).collect()\n",
        "  label_list"
      ],
      "metadata": {
        "id": "lJxZdinmCN4s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.tree import DecisionTree\n",
        "n_classes=n_classes_\n",
        "categorical_features_info={}\n",
        "impurity='gini'\n",
        "maxdepth=6\n",
        "maxbins=3\n",
        "DT_model = DecisionTree.trainClassifier(lp_train,n_classes,categorical_features_info,impurity,maxdepth,maxbins)\n",
        "\n",
        "#train\n",
        "GT=lp_train.map(lambda x:x.label)\n",
        "features=lp_train.map(lambda x:x.features)\n",
        "Pred = DT_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('Train :acc = ',acc)\n",
        "\n",
        "#val\n",
        "GT=lp_val.map(lambda x:x.label)\n",
        "features=lp_val.map(lambda x:x.features)\n",
        "Pred = DT_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('Train :acc = ',acc)\n",
        "\n",
        "#test\n",
        "GT=lp_test.map(lambda x:x.label)\n",
        "features=lp_test.map(lambda x:x.features)\n",
        "Pred = DT_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('test :acc =',acc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ni3KoRayG_VK",
        "outputId": "57d0b1ee-4a4c-436d-a958-c1fe79baf443"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train :acc =  0.2491846053489889\n",
            "Train :acc =  0.182648401826484\n",
            "test :acc = 0.19863013698630136\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.tree import RandomForest\n",
        "n_classes= n_classes_\n",
        "categorical_features_info={}\n",
        "n_trees=100\n",
        "feature_subset_strategy='auto'\n",
        "impurity='gini'\n",
        "maxdepth=8\n",
        "maxbins=5\n",
        "\n",
        "RF_model = RandomForest.trainClassifier(lp_train,n_classes,categorical_features_info,n_trees,feature_subset_strategy,impurity,maxdepth,maxbins)\n",
        "\n",
        "#train\n",
        "GT=lp_train.map(lambda x:x.label)\n",
        "features=lp_train.map(lambda x:x.features)\n",
        "Pred = RF_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('Train :acc = ',acc)\n",
        "\n",
        "#val\n",
        "GT=lp_val.map(lambda x:x.label)\n",
        "features=lp_val.map(lambda x:x.features)\n",
        "Pred = RF_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('Train :acc = ',acc)\n",
        "\n",
        "#test\n",
        "GT=lp_test.map(lambda x:x.label)\n",
        "features=lp_test.map(lambda x:x.features)\n",
        "Pred = RF_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('test :acc =',acc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o5IjBC_0JSh4",
        "outputId": "912258a8-e14b-4ee5-a797-94664c3c3d9d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train :acc =  1.0\n",
            "Train :acc =  0.8904109589041096\n",
            "test :acc = 0.9155251141552512\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.linalg import Vectors\n",
        "from pyspark.mllib.linalg.distributed import RowMatrix\n",
        "#train\n",
        "Mat_train = RowMatrix(lp_train.map(lambda x:Vectors.dense(x.features)))\n",
        "Lab_train = lp_train.map(lambda x:x.label)\n",
        "pc =Mat_train.computePrincipalComponents(512)\n",
        "PCA_features =Mat_train.multiply(pc).rows\n",
        "LP=Lab_train.zip(PCA_features)\n",
        "LP_train_pca = LP.map(lambda x:LabeledPoint(x[0],x[1]))\n",
        "\n",
        "#val\n",
        "Mat_val = RowMatrix(lp_val.map(lambda x:Vectors.dense(x.features)))\n",
        "Lab_val = lp_val.map(lambda x:x.label)\n",
        "#pc =Mat_val.computePrincipalComponents(512)\n",
        "PCA_features =Mat_val.multiply(pc).rows\n",
        "LP=Lab_val.zip(PCA_features)\n",
        "LP_val_pca = LP.map(lambda x:LabeledPoint(x[0],x[1]))\n",
        "\n",
        "\n",
        "#test\n",
        "Mat_test = RowMatrix(lp_test.map(lambda x:Vectors.dense(x.features)))\n",
        "Lab_test = lp_test.map(lambda x:x.label)\n",
        "#pc =Mat_test.computePrincipalComponents(512)\n",
        "PCA_features =Mat_test.multiply(pc).rows\n",
        "LP=Lab_test.zip(PCA_features)\n",
        "LP_test_pca = LP.map(lambda x:LabeledPoint(x[0],x[1]))\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "tWiAshBgMlJR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n_classes= n_classes_\n",
        "categorical_features_info={}\n",
        "n_trees=100\n",
        "feature_subset_strategy='auto'\n",
        "impurity='gini'\n",
        "maxdepth=6\n",
        "maxbins=3\n",
        "\n",
        "RF_model = RandomForest.trainClassifier(LP_train_pca,n_classes,categorical_features_info,n_trees,feature_subset_strategy,impurity,maxdepth,maxbins)\n",
        "\n",
        "#train\n",
        "GT=LP_train_pca.map(lambda x:x.label)\n",
        "features=LP_train_pca.map(lambda x:x.features)\n",
        "Pred = RF_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('Train :acc = ',acc)\n",
        "\n",
        "#val\n",
        "GT=LP_val_pca.map(lambda x:x.label)\n",
        "features=LP_val_pca.map(lambda x:x.features)\n",
        "Pred = RF_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('val :acc = ',acc)\n",
        "\n",
        "#test\n",
        "GT=LP_test_pca.map(lambda x:x.label)\n",
        "features=LP_test_pca.map(lambda x:x.features)\n",
        "Pred = RF_model.predict(features)\n",
        "GT_Pred = GT.zip(Pred)\n",
        "c1 =GT_Pred.filter(lambda x:x[0]==x[1]).count()\n",
        "c2 =GT_Pred.count()\n",
        "acc=c1/c2\n",
        "print('test :acc =',acc)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "crBomRu3QzB2",
        "outputId": "c7dfc853-fd72-44ce-fc57-dde4e716553a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train :acc =  1.0\n",
            "val :acc =  0.9223744292237442\n",
            "test :acc = 0.9018264840182648\n"
          ]
        }
      ]
    }
  ]
}