DIC pa3-part3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from pyspark.sql import SQLContext\n",
    "sc = pyspark.SparkContext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName(\"DIC assignment\").config(\"spark.some.config.option\",\"some-value\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = spark.read.csv('train.csv',inferSchema =True, header= True, escape=\"\\\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = spark.read.csv('test.csv',inferSchema =True, header= True, escape=\"\\\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = df3.dropna(thresh=3,subset=('movie_id','movie_name','plot'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df2.dropna(thresh=4,subset=('movie_id','movie_name','plot','genre'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertBinary(z):\n",
    "    import ast\n",
    "    import re\n",
    "    init_list =[0 for i in range(20)] \n",
    "    string = \"\"\n",
    "    genre = [\"Drama\", \"Comedy\", \"Romance Film\", \"Thriller\", \"Action\", \"World cinema\", \"Crime Fiction\", \"Horror\", \"Black-and-white\", \"Indie\",\"Action/Adventure\", \"Adventure\", \"Family Film\", \"Short Film\", \"Romantic drama\",\"Animation\",\"Musical\",\"Science Fiction\",\"Mystery\",\"Romantic comedy\"]\n",
    "    temp = ast.literal_eval(z)\n",
    "    z = temp   \n",
    "    \n",
    "    for i in range(len(z)): \n",
    "        if z[i] in genre:\n",
    "            index = genre.index(z[i])\n",
    "            init_list[index] = 1  \n",
    "\n",
    "    \n",
    "    for j in init_list:\n",
    "        string += str(j)\n",
    "        string += ' '\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+--------------------+--------------------+\n",
      "|movie_id|          movie_name|                plot|               genre|              binary|\n",
      "+--------+--------------------+--------------------+--------------------+--------------------+\n",
      "|23890098|          Taxi Blues|Shlykov, a hard-w...|['World cinema', ...|1 0 0 0 0 1 0 0 0...|\n",
      "|31186339|    The Hunger Games|The nation of Pan...|['Action/Adventur...|1 0 0 0 1 0 0 0 0...|\n",
      "|20663735|          Narasimham|Poovalli Induchoo...|['Musical', 'Acti...|1 0 0 0 1 0 0 0 0...|\n",
      "| 2231378|  The Lemon Drop Kid|The Lemon Drop Ki...|          ['Comedy']|0 1 0 0 0 0 0 0 0...|\n",
      "|  595909|   A Cry in the Dark|Seventh-day Adven...|['Crime Fiction',...|1 0 0 0 0 1 1 0 0...|\n",
      "| 5272176|            End Game|The president is ...|['Action/Adventur...|1 0 0 1 1 0 0 0 0...|\n",
      "| 1952976|          Dark Water|{{plot}} The film...|['Thriller', 'Dra...|1 0 0 1 0 0 0 1 0...|\n",
      "|24225279|                Sing|The story begins ...|           ['Drama']|1 0 0 0 0 0 0 0 0...|\n",
      "| 2462689|       Meet John Doe|Infuriated at bei...|['Black-and-white...|1 1 1 0 0 0 0 0 1...|\n",
      "|20532852|Destination Meatball|A line of people ...|['Animation', 'Sh...|0 0 0 0 0 0 0 0 0...|\n",
      "|15401493|    Husband for Hire|Lola  attempts to...|          ['Comedy']|0 1 0 0 0 0 0 0 0...|\n",
      "|18188932|         Up and Down|Milan and Goran a...|['Crime Fiction',...|1 1 0 0 0 1 1 0 0...|\n",
      "| 2940516|Ghost In The Noon...|Bumbling pirate c...|          ['Comedy']|0 1 0 0 0 0 0 0 0...|\n",
      "| 1480747|       House Party 2|{{plot}} Followin...|          ['Comedy']|0 1 0 0 0 0 0 0 0...|\n",
      "|24448645|Forest of the Dam...|Despite Lucy's re...|          ['Horror']|0 0 0 0 0 0 0 1 0...|\n",
      "|15072401|Charlie Chan's Se...|Alan Colby, heir ...|['Crime Fiction',...|0 0 0 1 0 0 1 1 0...|\n",
      "| 4018288|     The Biggest Fan|Debbie's favorite...|           ['Drama']|1 0 0 0 0 0 0 0 0...|\n",
      "| 4596602|      Ashes to Ashes|Ashes to Ashes is...|['Crime Fiction',...|0 0 1 1 1 0 1 0 0...|\n",
      "|15224586|        Green Dragon|The film follows ...|  ['Indie', 'Drama']|1 0 0 0 0 0 0 0 0...|\n",
      "|15585766|  The Rats of Tobruk|Three friends are...|           ['Drama']|1 0 0 0 0 0 0 0 0...|\n",
      "+--------+--------------------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import udf\n",
    "from pyspark.sql.types import StringType,IntegerType\n",
    "colsInt = udf(lambda z: convertBinary(z), StringType())\n",
    "spark.udf.register(\"colsInt\", colsInt)\n",
    "df2 = df.withColumn( 'binary',colsInt('genre'))\n",
    "df2.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import RegexTokenizer, StopWordsRemover, CountVectorizer, Word2Vec\n",
    "from pyspark.sql.types import IntegerType\n",
    "import pyspark.sql.functions as f\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "# regular expression tokenizer\n",
    "regexTokenizer = RegexTokenizer(inputCol=\"plot\", outputCol=\"words\", pattern=\"\\\\W\")\n",
    "tokenized = regexTokenizer.transform(df2)\n",
    "\n",
    "# stop words\n",
    "stopwordsRemover = StopWordsRemover(inputCol=\"words\", outputCol=\"filtered\")\n",
    "stop = stopwordsRemover.transform(tokenized)\n",
    "\n",
    "# bag of words count\n",
    "word2vec = Word2Vec(vectorSize = 100,minCount = 5, inputCol = 'filtered', outputCol = 'features')\n",
    "model = word2vec.fit(stop)\n",
    "result = model.transform(stop)\n",
    "\n",
    "split_col = pyspark.sql.functions.split(result['binary'], ' ')\n",
    "result = result.withColumn('Drama', split_col.getItem(0).cast(IntegerType()))\n",
    "result = result.withColumn('Comedy', split_col.getItem(1).cast(IntegerType()))\n",
    "result = result.withColumn('Romance Film', split_col.getItem(2).cast(IntegerType()))\n",
    "result = result.withColumn('Thriller', split_col.getItem(3).cast(IntegerType()))\n",
    "result = result.withColumn('Action', split_col.getItem(4).cast(IntegerType()))\n",
    "result = result.withColumn('World Cinema', split_col.getItem(5).cast(IntegerType()))\n",
    "result = result.withColumn('Crime Fiction', split_col.getItem(6).cast(IntegerType()))\n",
    "result = result.withColumn('Horror', split_col.getItem(7).cast(IntegerType()))\n",
    "result = result.withColumn('Black-and-White', split_col.getItem(8).cast(IntegerType()))\n",
    "result = result.withColumn('Indie', split_col.getItem(9).cast(IntegerType()))\n",
    "result = result.withColumn('Action/Adventure', split_col.getItem(10).cast(IntegerType()))\n",
    "result = result.withColumn('Adventure', split_col.getItem(11).cast(IntegerType()))\n",
    "result = result.withColumn('Family Film', split_col.getItem(12).cast(IntegerType()))\n",
    "result = result.withColumn('Short Film', split_col.getItem(13).cast(IntegerType()))\n",
    "result = result.withColumn('Romantic drama', split_col.getItem(14).cast(IntegerType()))\n",
    "result = result.withColumn('Animation', split_col.getItem(15).cast(IntegerType()))\n",
    "result = result.withColumn('Musical', split_col.getItem(16).cast(IntegerType()))\n",
    "result = result.withColumn('Science Fiction', split_col.getItem(17).cast(IntegerType()))\n",
    "result = result.withColumn('Mystery', split_col.getItem(18).cast(IntegerType()))\n",
    "result = result.withColumn('Romantic Comedy', split_col.getItem(19).cast(IntegerType()))\n",
    "result.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.where(\"movie_id != 20580938\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import RegexTokenizer, StopWordsRemover, CountVectorizer,Word2Vec\n",
    "from pyspark.sql.types import IntegerType\n",
    "import pyspark.sql.functions as f\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "# regular expression tokenizer\n",
    "test_regexTokenizer = RegexTokenizer(inputCol=\"plot\", outputCol=\"words\", pattern=\"\\\\W\")\n",
    "test_tokenized = test_regexTokenizer.transform(test_df)\n",
    "#tokenized.show()\n",
    "# stop words\n",
    "#add_stopwords = [\"http\",\"https\",\"amp\",\"rt\",\"t\",\"c\",\"the\"] \n",
    "test_stopwordsRemover = StopWordsRemover(inputCol=\"words\", outputCol=\"filtered\")\n",
    "test_stop = test_stopwordsRemover.transform(test_tokenized)\n",
    "\n",
    "# bag of words count\n",
    "test_word2vec = Word2Vec(vectorSize = 100, minCount = 5, inputCol = 'filtered', outputCol = 'features')\n",
    "test_model = test_word2vec.fit(test_stop)\n",
    "test_result = test_model.transform(test_stop)\n",
    "test_result.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------+\n",
      "|movie_id|          movie_name|                plot|               words|            filtered|            features|       rawPrediction|         probability|prediction|\n",
      "+--------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------+\n",
      "| 1335380|              Exodus|The film is based...|[the, film, is, b...|[film, based, eve...|[-0.0442120403582...|[1.05978177904513...|[0.74264884078618...|       0.0|\n",
      "|29062594|A la salida nos v...|A group of teenag...|[a, group, of, te...|[group, teenagers...|[-0.0742260838540...|[0.76330854876330...|[0.68207162413300...|       0.0|\n",
      "| 9252321|   Come Back, Africa|This story of a Z...|[this, story, of,...|[story, zulu, fam...|[-0.0573536608455...|[1.11724446591884...|[0.75347723552209...|       0.0|\n",
      "|13455076|       A Merry Mixup|The Stooges play ...|[the, stooges, pl...|[stooges, play, t...|[-0.0523639659372...|[1.18351256831866...|[0.76557878502564...|       0.0|\n",
      "|24165951|        Getting Even|A soldier-of-fort...|[a, soldier, of, ...|[soldier, fortune...|[-0.0865279108184...|[0.87490352414301...|[0.70576499398424...|       0.0|\n",
      "| 1925869|  River of No Return|Set in the Northw...|[set, in, the, no...|[set, northwester...|[-0.0673981012602...|[0.77284267258120...|[0.68413549985331...|       0.0|\n",
      "|10799612|          Amici miei|Like in many othe...|[like, in, many, ...|[like, many, moni...|[-0.0086094259471...|[1.19720848192429...|[0.76802781575216...|       0.0|\n",
      "|28238240|Mickey's Big Game...|Mickey and the Sc...|[mickey, and, the...|[mickey, scorpion...|[-0.0993913817884...|[1.76252203884192...|[0.85352524645354...|       0.0|\n",
      "|17124781|The Good, the Bad...|In the desert wil...|[in, the, desert,...|[desert, wilderne...|[-0.0874793851008...|[1.55599614718823...|[0.82577807524149...|       0.0|\n",
      "|28207941|    The Dancing Fool|Bimbo and Koko ar...|[bimbo, and, koko...|[bimbo, koko, sig...|[0.00422492934324...|[0.83187178195415...|[0.69675056275142...|       0.0|\n",
      "|19174305|              Tahaan|Tahaan  lives wit...|[tahaan, lives, w...|[tahaan, lives, g...|[-0.0534644076439...|[1.04271980066701...|[0.73937445244126...|       0.0|\n",
      "|18392317|     Mysterious Mose|Betty is startled...|[betty, is, start...|[betty, startled,...|[-0.0175673857304...|[1.53341141567727...|[0.82250489686256...|       0.0|\n",
      "|34420857|Kelviyum Naane Pa...|Nirmal ([[Karthik...|[nirmal, karthik,...|[nirmal, karthik,...|[-0.0148776123507...|[0.33500518212693...|[0.58297671405383...|       0.0|\n",
      "| 4039635|   First on the Moon|A group of journa...|[a, group, of, jo...|[group, journalis...|[-0.0595192792400...|[2.19005970980643...|[0.89935331131330...|       0.0|\n",
      "| 8034072|  Journey of a Woman|Vaibhavari Sahay,...|[vaibhavari, saha...|[vaibhavari, saha...|[-0.0130747856987...|[0.48613122809377...|[0.61919462600214...|       0.0|\n",
      "| 4016437|     Sophie's Choice|In 1947, the movi...|[in, 1947, the, m...|[1947, movie, nar...|[-0.0407334308112...|[0.40739625059263...|[0.60046338456610...|       0.0|\n",
      "| 1520023|  Ninja Resurrection|Ninja Resurrectio...|[ninja, resurrect...|[ninja, resurrect...|[-0.0366783723851...|[1.60501486318671...|[0.83271811489368...|       0.0|\n",
      "|24589422|      Maria’s Lovers|In the spring of ...|[in, the, spring,...|[spring, 1946, iv...|[-0.0172060729443...|[0.76706544373479...|[0.68288574868259...|       0.0|\n",
      "|35068740|           Chinnavar|Muthu ([[Prabhu  ...|[muthu, prabhu, a...|[muthu, prabhu, v...|[-0.0019895408084...|[0.00937980365104...|[0.50234493372037...|       0.0|\n",
      "|21132951|              Aparan|Vishwanathan , an...|[vishwanathan, an...|[vishwanathan, in...|[-0.0472234943447...|[0.96711495130166...|[0.72454407305927...|       0.0|\n",
      "+--------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "lr1 = LogisticRegression(labelCol=\"Drama\", featuresCol=\"features\")\n",
    "lrModel1 = lr1.fit(result)\n",
    "output1 = lrModel1.transform(test_result)\n",
    "\n",
    "lr2 = LogisticRegression(labelCol=\"Comedy\", featuresCol=\"features\")\n",
    "lrModel2 = lr2.fit(result)\n",
    "output2 = lrModel2.transform(test_result)\n",
    "\n",
    "lr3 = LogisticRegression(labelCol=\"Romance Film\", featuresCol=\"features\")\n",
    "lrModel3 = lr3.fit(result)\n",
    "output3 = lrModel3.transform(test_result)\n",
    "\n",
    "lr4 = LogisticRegression(labelCol=\"Thriller\", featuresCol=\"features\")\n",
    "lrModel4 = lr4.fit(result)\n",
    "output4 = lrModel4.transform(test_result)\n",
    "\n",
    "lr5 = LogisticRegression(labelCol=\"Action\", featuresCol=\"features\")\n",
    "lrModel5 = lr5.fit(result)\n",
    "output5 = lrModel5.transform(test_result)\n",
    "\n",
    "lr6 = LogisticRegression(labelCol=\"World Cinema\", featuresCol=\"features\")\n",
    "lrModel6 = lr6.fit(result)\n",
    "output6 = lrModel6.transform(test_result)\n",
    "\n",
    "lr7 = LogisticRegression(labelCol=\"Crime Fiction\", featuresCol=\"features\")\n",
    "lrModel7 = lr7.fit(result)\n",
    "output7 = lrModel7.transform(test_result)\n",
    "\n",
    "lr8 = LogisticRegression(labelCol=\"Horror\", featuresCol=\"features\")\n",
    "lrModel8 = lr8.fit(result)\n",
    "output8 = lrModel8.transform(test_result)\n",
    "\n",
    "lr9 = LogisticRegression(labelCol=\"Black-and-White\", featuresCol=\"features\")\n",
    "lrModel9 = lr9.fit(result)\n",
    "output9 = lrModel9.transform(test_result)\n",
    "\n",
    "lr10 = LogisticRegression(labelCol=\"Indie\", featuresCol=\"features\")\n",
    "lrModel10 = lr10.fit(result)\n",
    "output10 = lrModel10.transform(test_result)\n",
    "\n",
    "lr11 = LogisticRegression(labelCol=\"Action/Adventure\", featuresCol=\"features\")\n",
    "lrModel11 = lr11.fit(result)\n",
    "output11 = lrModel11.transform(test_result)\n",
    "\n",
    "lr12 = LogisticRegression(labelCol=\"Adventure\", featuresCol=\"features\")\n",
    "lrModel12 = lr12.fit(result)\n",
    "output12 = lrModel12.transform(test_result)\n",
    "\n",
    "lr13 = LogisticRegression(labelCol=\"Family Film\", featuresCol=\"features\")\n",
    "lrModel13 = lr13.fit(result)\n",
    "output13 = lrModel13.transform(test_result)\n",
    "\n",
    "lr14 = LogisticRegression(labelCol=\"Short Film\", featuresCol=\"features\")\n",
    "lrModel14 = lr14.fit(result)\n",
    "output14 = lrModel14.transform(test_result)\n",
    "\n",
    "lr15 = LogisticRegression(labelCol=\"Romantic drama\", featuresCol=\"features\")\n",
    "lrModel15 = lr15.fit(result)\n",
    "output15 = lrModel15.transform(test_result)\n",
    "\n",
    "lr16 = LogisticRegression(labelCol=\"Animation\", featuresCol=\"features\")\n",
    "lrModel16 = lr16.fit(result)\n",
    "output16 = lrModel16.transform(test_result)\n",
    "\n",
    "lr17 = LogisticRegression(labelCol=\"Musical\", featuresCol=\"features\")\n",
    "lrModel17 = lr17.fit(result)\n",
    "output17 = lrModel17.transform(test_result)\n",
    "\n",
    "lr18 = LogisticRegression(labelCol=\"Science Fiction\", featuresCol=\"features\")\n",
    "lrModel18 = lr18.fit(result)\n",
    "output18 = lrModel18.transform(test_result)\n",
    "\n",
    "lr19 = LogisticRegression(labelCol=\"Mystery\", featuresCol=\"features\")\n",
    "lrModel19 = lr19.fit(result)\n",
    "output19 = lrModel19.transform(test_result)\n",
    "\n",
    "lr20 = LogisticRegression(labelCol=\"Romantic Comedy\", featuresCol=\"features\")\n",
    "lrModel20 = lr20.fit(result)\n",
    "output20 = lrModel20.transform(test_result)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1335380, 29062594, 9252321, 13455076, 24165951, 1925869, 10799612, 28238240, 17124781, 28207941, 19174305, 18392317, 34420857, 4039635, 8034072, 4016437, 1520023, 24589422, 35068740, 21132951]\n"
     ]
    }
   ],
   "source": [
    "movie_list = output1.select('movie_id').collect()\n",
    "movie_array = [int(row.movie_id) for row in movie_list]\n",
    "\n",
    "#print(movie_array[:20])\n",
    "\n",
    "mvv_list1 = output1.select('prediction').collect()\n",
    "mvv_array1 = [int(row.prediction) for row in mvv_list1]\n",
    "#print(mvv_array)\n",
    "\n",
    "mvv_list2 = output2.select('prediction').collect()\n",
    "mvv_array2 = [int(row.prediction) for row in mvv_list2]\n",
    "\n",
    "mvv_list3 = output3.select('prediction').collect()\n",
    "mvv_array3 = [int(row.prediction) for row in mvv_list3]\n",
    "\n",
    "mvv_list4 = output4.select('prediction').collect()\n",
    "mvv_array4 = [int(row.prediction) for row in mvv_list4]\n",
    "\n",
    "mvv_list5 = output5.select('prediction').collect()\n",
    "mvv_array5 = [int(row.prediction) for row in mvv_list5]\n",
    "\n",
    "mvv_list6 = output6.select('prediction').collect()\n",
    "mvv_array6 = [int(row.prediction) for row in mvv_list6]\n",
    "\n",
    "mvv_list7 = output7.select('prediction').collect()\n",
    "mvv_array7 = [int(row.prediction) for row in mvv_list7]\n",
    "\n",
    "mvv_list8 = output8.select('prediction').collect()\n",
    "mvv_array8 = [int(row.prediction) for row in mvv_list8]\n",
    "\n",
    "mvv_list9 = output9.select('prediction').collect()\n",
    "mvv_array9 = [int(row.prediction) for row in mvv_list9]\n",
    "\n",
    "mvv_list10 = output10.select('prediction').collect()\n",
    "mvv_array10 = [int(row.prediction) for row in mvv_list10]\n",
    "\n",
    "mvv_list11 = output11.select('prediction').collect()\n",
    "mvv_array11 = [int(row.prediction) for row in mvv_list11]\n",
    "#print(mvv_array)\n",
    "\n",
    "mvv_list12 = output12.select('prediction').collect()\n",
    "mvv_array12 = [int(row.prediction) for row in mvv_list12]\n",
    "\n",
    "mvv_list13 = output13.select('prediction').collect()\n",
    "mvv_array13 = [int(row.prediction) for row in mvv_list13]\n",
    "\n",
    "mvv_list14 = output14.select('prediction').collect()\n",
    "mvv_array14 = [int(row.prediction) for row in mvv_list14]\n",
    "\n",
    "mvv_list15 = output15.select('prediction').collect()\n",
    "mvv_array15 = [int(row.prediction) for row in mvv_list15]\n",
    "\n",
    "mvv_list16 = output16.select('prediction').collect()\n",
    "mvv_array16 = [int(row.prediction) for row in mvv_list16]\n",
    "\n",
    "mvv_list17 = output17.select('prediction').collect()\n",
    "mvv_array17 = [int(row.prediction) for row in mvv_list17]\n",
    "\n",
    "mvv_list18 = output18.select('prediction').collect()\n",
    "mvv_array18 = [int(row.prediction) for row in mvv_list18]\n",
    "\n",
    "mvv_list19 = output19.select('prediction').collect()\n",
    "mvv_array19 = [int(row.prediction) for row in mvv_list19]\n",
    "\n",
    "mvv_list20 = output20.select('prediction').collect()\n",
    "mvv_array20 = [int(row.prediction) for row in mvv_list20]\n",
    "\n",
    "\n",
    "string_list = []\n",
    "for i in range(len(mvv_array1)):\n",
    "    string = ''\n",
    "    string = str(mvv_array1[i]) + ' ' + str(mvv_array2[i]) + ' ' + str(mvv_array3[i]) + ' ' + str(mvv_array4[i]) + ' ' + str(mvv_array5[i]) + ' ' + str(mvv_array6[i]) + ' ' + str(mvv_array7[i]) + ' ' + str(mvv_array8[i]) + ' ' + str(mvv_array9[i]) + ' ' + str(mvv_array10[i]) + ' ' + str(mvv_array11[i]) + ' ' + str(mvv_array12[i]) + ' ' + str(mvv_array13[i]) + ' ' + str(mvv_array14[i]) + ' ' + str(mvv_array15[i]) + ' ' + str(mvv_array16[i]) + ' ' + str(mvv_array17[i]) + ' ' + str(mvv_array18[i]) + ' ' + str(mvv_array19[i]) + ' ' + str(mvv_array20[i]) \n",
    "    string_list.append(string)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import csv\n",
    "from pyspark.sql import SQLContext\n",
    "\n",
    "sqlContext = SQLContext(sc)\n",
    "temp = sqlContext.createDataFrame(zip(movie_array,string_list),schema=['movie_id','predictions']) \n",
    "temp.toPandas().to_csv('part3.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}