PRISMDECK-TRAINED-MODEL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "30a26155-f48a-4d11-88e8-8c64284d4641",
   "metadata": {},
   "source": [
    "# PrismDeck Trained Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f436e265-dab4-46b1-a7f8-d094933bc77c",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Pyspark Environments/ Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "82c83572-4b47-4ad9-8cc0-e2112c4883a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the PySpark environment variables\n",
    "import os\n",
    "os.environ['SPARK_HOME'] = \"/Users/joey/Downloads/Spark\"\n",
    "os.environ['PYSPARK_DRIVER_PYTHON'] = 'jupyter'\n",
    "os.environ['PYSPARK_DRIVER_PYTHON_OPTS'] = 'lab'\n",
    "os.environ['PYSPARK_PYTHON'] = 'python'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "24e58cba-a0f4-480b-980c-d8c2e8587c82",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.recommendation import ALS\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.sql.functions import mean, col, when\n",
    "# Regression Evaluator  is used to evaluate the performance of the recommendation model. \n",
    "# It assesses how well the model's predicted ratings align with the actual ratings in the test dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eee6a2e5-bad6-47f7-ba81-a377dd5c7d0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "24/01/17 08:41:08 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://192.168.3.149:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.5.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>PrismDeck</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x124566df0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Because of how big the 'ratings.csv' file is, I configured the spark driver memory  to allocate 4gb\n",
    "SparkSession.builder.appName(\"PrismDeck\").config(\"spark.driver.memory\", \"4g\").getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67194915-3128-4038-8d08-d487f956c0b4",
   "metadata": {},
   "source": [
    "## Reading & Loading Data \n",
    "\n",
    "- Creating a SparkSession\n",
    "- Read CSV file into a DataFrame\n",
    "- Load CSV file\n",
    "- Load a sample of the ratings data\n",
    "- Specifying the fraction of the data to sample (10% in this case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f47afc6e-97f2-4d4c-91a4-9964615a4fd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName(\"PrismDeck\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0993c4c0-86f2-47c5-9673-381bb903bc91",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_file_path = \"/Users/joey/Downloads/ml-25m/ratings.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a7a270de-cc54-4437-a550-b2889360f23c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24/01/17 08:41:22 WARN GarbageCollectionMetrics: To enable non-built-in garbage collector(s) List(G1 Concurrent GC), users should configure it(them) to spark.eventLog.gcMetrics.youngGenerationGarbageCollectors or spark.eventLog.gcMetrics.oldGenerationGarbageCollectors\n",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "ratings = spark.read.csv('/Users/joey/Downloads/ml-25m/ratings.csv', header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c869c7f2-3f9f-4aa4-9cfc-e23be5d8ff88",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_sample = ratings.sample(fraction=0.1, seed=123)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "702a884f-dffa-4cf0-80fa-699deae64147",
   "metadata": {},
   "source": [
    "## Displayed Rows in dataset\n",
    "Display the first 10 rows of the 'ratings' DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "65f2507d-a155-4390-ab7f-73b58bcc52b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-------+------+----------+\n",
      "|userId|movieId|rating| timestamp|\n",
      "+------+-------+------+----------+\n",
      "|     1|    296|   5.0|1147880044|\n",
      "|     1|    306|   3.5|1147868817|\n",
      "|     1|    307|   5.0|1147868828|\n",
      "|     1|    665|   5.0|1147878820|\n",
      "|     1|    899|   3.5|1147868510|\n",
      "|     1|   1088|   4.0|1147868495|\n",
      "|     1|   1175|   3.5|1147868826|\n",
      "|     1|   1217|   3.5|1147878326|\n",
      "|     1|   1237|   5.0|1147868839|\n",
      "|     1|   1250|   4.0|1147868414|\n",
      "+------+-------+------+----------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ratings.show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7cedda7-b1be-4560-8795-1a76b7b9b3c6",
   "metadata": {},
   "source": [
    "## ALS Model Setup\n",
    "\n",
    "- ALS (Alternating Least Squares) is a collaborative filtering algorithm used for recommendation.\n",
    "- 'userCol' specifies the column name in the DataFrame representing user IDs.\n",
    "- In collaborative filtering, ALS predicts the ratings of items for users based on their preferences.\n",
    "- 'itemCol' specifies the column name in the DataFrame representing item (movie) IDs.\n",
    "- This is the identifier for the items that users might rate or interact with.\n",
    "- 'ratingCol' specifies the column name in the DataFrame representing the actual ratings given by users.\n",
    "- The algorithm will try to predict ratings in this column.\n",
    "- 'coldStartStrategy' is a strategy for handling new users or items not present in the training dataset.\n",
    "- 'drop' is a strategy that drops any rows in the DataFrame of predictions that contain NaN values.\n",
    "\n",
    "- The configured ALS model will be used for training and making predictions.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5254fb57-1779-44ae-80a6-ffcd1534b3c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "als = ALS(\n",
    "    userCol='userId',\n",
    "    itemCol='movieId',\n",
    "    ratingCol='rating',\n",
    "    coldStartStrategy='drop'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15a0346d-6759-4720-a56e-611f7ffa7227",
   "metadata": {},
   "source": [
    "## Data Splitting and Model Training\n",
    "\n",
    "- Split the data into training and testing sets\n",
    "- 'combined_df' contains my feature-rich data for building the recommendation model.\n",
    "- 'randomsplit' divides the data into two sets: Training data and Testing Data\n",
    "- List inside randomsplit represents the split: 80% = Training data, 20% = Testing data\n",
    "- seed parameter ensures reproducibility. If you use the same seed, you'll get the same split each time you run the code.\n",
    "- This helps with the consistency in model evaluation.\n",
    "- Tuple contains the two split DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0aa4208d-fce1-4b44-b2d2-6a8bf92f2073",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_data, test_data) = ratings.randomSplit([0.8, 0.2], seed=123)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41922a21-f9ba-4d2c-9dd3-b279fd990207",
   "metadata": {},
   "source": [
    "-  Persist the DataFrames\n",
    " - Which means the code following is storing or saving some DataFrames in a way that makes future operations on that data faster.\n",
    "- persistence can happen in the computer's memory (RAM), on disk, or a combination of both."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9194b817-b067-4072-a07f-597e923d379f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[userId: int, movieId: int, rating: double, timestamp: int]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.persist()\n",
    "test_data.persist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bb9527d-2e92-46cb-90cf-ad6dd5eac2a7",
   "metadata": {},
   "source": [
    "- Repartition the DataFrames\n",
    "- partitioning can improve parallelism and performance.\n",
    "- Depending on the size of 'ratings.csv' data and the available memory, repartitioning can help in managing memory more efficiently.\n",
    "- It prevents out-of-memory errors or excessive memory usage\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "25d7aedb-9a76-4276-a2ac-848ff29f7be3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.repartition(10)  # Adjust the number of partitions\n",
    "test_data = test_data.repartition(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37be9751-07cf-43fb-8796-b0dc7d80a5af",
   "metadata": {},
   "source": [
    "- Train the ALS model on the 'Training data'\n",
    "- Training data DataFrame includes the user-item interactions (user ratings)\n",
    "- 'fit' trains the ALS model on the training data\n",
    "- It learns the underlying patterns and relationships in the data to make predictions.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6c3d6369-4465-4f54-9a29-8e944eda2135",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "24/01/17 08:42:33 WARN InstanceBuilder: Failed to load implementation from:dev.ludovic.netlib.blas.JNIBLAS\n",
      "24/01/17 08:42:33 WARN InstanceBuilder: Failed to load implementation from:dev.ludovic.netlib.blas.VectorBLAS\n",
      "24/01/17 08:42:33 WARN InstanceBuilder: Failed to load implementation from:dev.ludovic.netlib.lapack.JNILAPACK\n",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "model = als.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d39c42d5-a4af-4f33-b1b5-eaf6ad0c12fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions on the test data\n",
    "predictions = model.transform(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a856e7b2-bd0f-49ee-95cc-1eca78cedf69",
   "metadata": {},
   "source": [
    "## Model Evaluation/ RMSE (Trained Models Accuracy)\n",
    "\n",
    " Evaluate the model using a regression evaluator\n",
    " - metricName: 'rmse' indicates that the evaluation metric is Root Mean Squared Error (RMSE)\n",
    " - labelCol: 'rating' specifies the column in the DataFrame containing the actual ratings\n",
    " - predictionCol: 'prediction' specifies the column in the DataFrame containing the predicted ratings\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87369b42-fce4-4e08-bffe-3f50e48bdee8",
   "metadata": {},
   "source": [
    "## Model Evaluation/ RMSE (Trained Models Accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "75f83add-eeda-438a-a738-008d2e3aad7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 147:>                                                        (0 + 8) / 8]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Squared Error (RMSE) on test data: 0.8036803070756547\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "evaluator = RegressionEvaluator(\n",
    "    metricName='rmse',\n",
    "    labelCol='rating',\n",
    "    predictionCol='prediction'\n",
    ")\n",
    "rmse = evaluator.evaluate(predictions)\n",
    "print(f\"Root Mean Squared Error (RMSE) on test data: {rmse}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1ead8e2-b320-4aa8-8124-8e4bebbd63b5",
   "metadata": {},
   "source": [
    "## Generating Recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "009c7672-0859-4ac3-986a-f5bad65970cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|userId|recommendations                                                                                                                                                                                                |\n",
      "+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|26    |[{127252, 5.799946}, {203882, 5.4500513}, {203086, 5.3906593}, {144202, 5.3087993}, {196787, 5.1712666}, {194434, 4.973962}, {126941, 4.940329}, {149484, 4.90486}, {166812, 4.8869233}, {151615, 4.8856316}]  |\n",
      "|27    |[{127252, 6.05012}, {175625, 5.850381}, {194334, 5.844524}, {54758, 5.650063}, {179985, 5.6348896}, {203086, 5.6253676}, {182657, 5.5584292}, {177411, 5.5174475}, {151615, 5.500729}, {169648, 5.4808483}]    |\n",
      "|28    |[{151989, 7.2066145}, {194434, 6.9905963}, {203882, 6.7411184}, {201821, 6.568886}, {183947, 6.450201}, {59799, 6.3247685}, {165559, 6.288392}, {127252, 6.207216}, {196787, 6.2031226}, {192089, 6.167577}]   |\n",
      "|31    |[{203882, 3.8306992}, {194334, 3.7271738}, {201821, 3.5896657}, {183947, 3.568052}, {192089, 3.5543435}, {203086, 3.51191}, {194434, 3.4770849}, {166812, 3.458516}, {127252, 3.4416068}, {184299, 3.4280481}] |\n",
      "|34    |[{194434, 5.7649097}, {207888, 5.3904815}, {127252, 5.347031}, {120272, 5.208782}, {145871, 5.175252}, {180323, 5.0794134}, {183947, 5.0709834}, {83006, 5.0370994}, {187951, 5.0225635}, {187949, 5.0225635}] |\n",
      "|44    |[{127252, 6.9799275}, {203882, 6.3564324}, {203086, 6.172238}, {149986, 6.077306}, {196717, 6.0519304}, {194434, 6.0031376}, {145871, 5.9712157}, {187951, 5.933522}, {187949, 5.933522}, {187947, 5.933522}]  |\n",
      "|53    |[{192089, 6.57731}, {194334, 6.5378714}, {203882, 5.9210796}, {139060, 5.884084}, {201821, 5.865512}, {144208, 5.8622165}, {155923, 5.8407845}, {207996, 5.8101068}, {194332, 5.799648}, {183947, 5.7634764}]  |\n",
      "|65    |[{127252, 6.0140624}, {169606, 5.8858304}, {109887, 5.7689085}, {191399, 5.674864}, {130219, 5.5963497}, {1830, 5.547939}, {144202, 5.5190535}, {184347, 5.4557686}, {61913, 5.3925323}, {126737, 5.3861322}]  |\n",
      "|76    |[{151989, 5.68401}, {183947, 5.598275}, {192089, 5.5549197}, {203882, 5.4984183}, {143422, 5.4422455}, {172737, 5.3636055}, {194434, 5.3374705}, {165559, 5.306945}, {93991, 5.2823877}, {203086, 5.274909}]   |\n",
      "|78    |[{192089, 6.475466}, {127252, 6.4716434}, {203882, 6.20645}, {145871, 6.199954}, {151989, 6.1817875}, {180323, 6.1573634}, {59799, 6.06482}, {173153, 6.0201735}, {198841, 6.00076}, {180307, 5.968215}]       |\n",
      "|81    |[{200930, 4.7964487}, {168906, 4.7107925}, {79045, 4.6162815}, {49872, 4.533974}, {93376, 4.522403}, {193257, 4.4600782}, {181209, 4.420847}, {116108, 4.3903418}, {103875, 4.3903418}, {27914, 4.3903418}]    |\n",
      "|85    |[{127252, 6.145437}, {203086, 5.7543254}, {190707, 5.6474376}, {169648, 5.5172424}, {203882, 5.499145}, {151615, 5.4977837}, {166812, 5.455398}, {193257, 5.435601}, {205453, 5.434028}, {192089, 5.433073}]   |\n",
      "|101   |[{203086, 4.9338217}, {127252, 4.8666315}, {192089, 4.8099637}, {194334, 4.7894936}, {203882, 4.7743473}, {183947, 4.6135106}, {172737, 4.567935}, {194434, 4.557661}, {156414, 4.530436}, {166812, 4.516105}] |\n",
      "|103   |[{151989, 6.153132}, {194434, 6.0186744}, {203882, 5.908208}, {183947, 5.84183}, {200930, 5.815773}, {172737, 5.791689}, {203086, 5.646142}, {192089, 5.603412}, {117907, 5.568992}, {150637, 5.56377}]        |\n",
      "|108   |[{127252, 5.1741257}, {194434, 5.1731954}, {203086, 5.0763993}, {203882, 5.0081306}, {157791, 4.938467}, {157789, 4.938467}, {202231, 4.902987}, {187951, 4.776847}, {187949, 4.776847}, {187947, 4.776847}]   |\n",
      "|115   |[{203882, 6.323225}, {203086, 6.02687}, {183947, 5.999662}, {194434, 5.9682264}, {172737, 5.6396675}, {151989, 5.6313972}, {127252, 5.5918503}, {166812, 5.4598265}, {101855, 5.4509706}, {187951, 5.429394}]  |\n",
      "|126   |[{203086, 6.269658}, {203882, 6.1542773}, {127252, 6.114534}, {194434, 5.706501}, {166812, 5.6171966}, {183947, 5.597029}, {205453, 5.54774}, {184299, 5.545183}, {172737, 5.519411}, {151989, 5.5053425}]     |\n",
      "|133   |[{203882, 5.284044}, {183947, 5.2077484}, {172737, 5.109498}, {203086, 5.0981197}, {192089, 5.0486164}, {151989, 4.998507}, {200930, 4.899941}, {143422, 4.8927326}, {194334, 4.83184}, {142891, 4.8309693}]   |\n",
      "|137   |[{203882, 5.5343914}, {203086, 5.521003}, {194334, 5.3950515}, {127252, 5.3758163}, {101862, 5.172437}, {184299, 5.100364}, {166812, 5.0558248}, {172737, 4.9587297}, {101855, 4.9443626}, {192089, 4.9286265}]|\n",
      "|148   |[{203882, 5.538429}, {194434, 5.5319138}, {183947, 5.4301867}, {203086, 5.314853}, {151989, 5.3005414}, {192089, 5.187046}, {172737, 5.1517863}, {165559, 5.1400166}, {157791, 5.1024704}, {157789, 5.1024704}]|\n",
      "+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|movieId|recommendations                                                                                                                                                                                            |\n",
      "+-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|12     |[{87426, 5.4314575}, {149507, 4.750429}, {129669, 4.7246842}, {8583, 4.7114887}, {142811, 4.652433}, {93649, 4.6407547}, {159335, 4.629198}, {85938, 4.599441}, {10861, 4.568414}, {110061, 4.5407877}]    |\n",
      "|26     |[{105130, 4.9823847}, {105801, 4.941435}, {96471, 4.9353614}, {66365, 4.934801}, {4395, 4.926381}, {112919, 4.90449}, {19143, 4.9015417}, {18230, 4.9004655}, {103920, 4.8948765}, {122820, 4.890899}]     |\n",
      "|27     |[{87426, 5.6629896}, {10417, 5.1815124}, {71207, 5.1614075}, {56046, 5.155874}, {52924, 5.135329}, {127635, 5.1323433}, {18230, 5.1316786}, {14103, 5.1231484}, {157063, 5.1139226}, {67467, 5.082174}]    |\n",
      "|28     |[{38121, 5.390257}, {59643, 5.373658}, {103920, 5.3467607}, {151033, 5.3067455}, {137978, 5.3039107}, {105801, 5.278962}, {23121, 5.267126}, {4395, 5.2485924}, {121910, 5.245497}, {161070, 5.232005}]    |\n",
      "|31     |[{87426, 5.507538}, {10417, 5.040228}, {96471, 5.0171237}, {52924, 5.010979}, {18230, 5.001836}, {56046, 4.98077}, {61667, 4.9675035}, {67467, 4.940812}, {129440, 4.9207263}, {71207, 4.9166956}]         |\n",
      "|34     |[{58248, 5.6535344}, {32202, 5.2766004}, {43679, 5.2534246}, {10417, 5.211037}, {118929, 5.1673446}, {13576, 5.1347165}, {141970, 5.1189466}, {62064, 5.1165338}, {140193, 5.0926957}, {93907, 5.079878}]  |\n",
      "|44     |[{87426, 5.6163664}, {149507, 4.953182}, {93649, 4.907864}, {99640, 4.8252788}, {159335, 4.781101}, {10861, 4.686137}, {52858, 4.656885}, {8583, 4.648554}, {61667, 4.626388}, {110061, 4.605067}]         |\n",
      "|53     |[{132629, 5.2853885}, {84907, 5.2117968}, {5752, 5.0984893}, {151033, 5.0881658}, {58894, 5.0338907}, {113621, 5.0335298}, {87216, 5.007686}, {55855, 5.0048923}, {108335, 5.004484}, {103920, 4.9958663}] |\n",
      "|65     |[{87426, 5.528904}, {20479, 4.9314938}, {93649, 4.783749}, {85938, 4.757452}, {159335, 4.693857}, {129669, 4.681377}, {30067, 4.6757164}, {8583, 4.634231}, {55247, 4.6136103}, {160598, 4.61342}]         |\n",
      "|76     |[{87426, 5.393677}, {149507, 5.0399284}, {67467, 4.8094378}, {124934, 4.7180915}, {61667, 4.6832495}, {120911, 4.6430016}, {75309, 4.6383615}, {162109, 4.6358695}, {87392, 4.635015}, {91940, 4.634395}]  |\n",
      "|78     |[{87426, 4.6478148}, {67467, 4.623452}, {19143, 4.620604}, {107363, 4.610509}, {75309, 4.6102324}, {149507, 4.5909452}, {96471, 4.55851}, {105946, 4.555536}, {139326, 4.5502253}, {123176, 4.5389986}]    |\n",
      "|81     |[{87426, 4.926141}, {87816, 4.785672}, {137757, 4.784082}, {142811, 4.73468}, {152146, 4.719393}, {67467, 4.7159743}, {88305, 4.7112412}, {156318, 4.7035394}, {74964, 4.7035007}, {34543, 4.6960487}]     |\n",
      "|85     |[{103920, 4.929072}, {19143, 4.857475}, {158917, 4.8394575}, {123176, 4.8351507}, {123177, 4.8292923}, {160416, 4.813796}, {135960, 4.8128343}, {151033, 4.809008}, {80084, 4.7980967}, {113621, 4.790643}]|\n",
      "|101    |[{58894, 4.9483}, {128620, 4.8931246}, {13338, 4.8615885}, {138914, 4.856338}, {56026, 4.8523827}, {161312, 4.8506694}, {89560, 4.8347077}, {58989, 4.8253636}, {75309, 4.81859}, {136159, 4.813293}]      |\n",
      "|103    |[{87426, 5.457942}, {96471, 4.9759483}, {86668, 4.9277306}, {10417, 4.905048}, {26659, 4.8973613}, {67467, 4.897159}, {61667, 4.8575926}, {149507, 4.847832}, {18230, 4.827996}, {105946, 4.815527}]       |\n",
      "|108    |[{151033, 5.296366}, {87426, 5.2521715}, {24387, 5.173585}, {149507, 5.1211123}, {67467, 5.096871}, {31506, 5.060037}, {18885, 5.0567627}, {15132, 5.0137773}, {33422, 4.9625535}, {7629, 4.9623613}]      |\n",
      "|115    |[{158154, 5.9603148}, {142811, 5.8892035}, {75309, 5.6529193}, {103920, 5.595914}, {148502, 5.591575}, {160416, 5.5604186}, {105801, 5.550692}, {59134, 5.5470448}, {123177, 5.5395207}, {19143, 5.53742}] |\n",
      "|126    |[{87426, 5.0649834}, {26659, 4.4612875}, {114026, 4.385287}, {37835, 4.351963}, {53712, 4.344241}, {52924, 4.3415375}, {96471, 4.333653}, {12810, 4.3323293}, {151238, 4.3089876}, {77817, 4.3041525}]     |\n",
      "|133    |[{87426, 5.387237}, {18885, 5.211004}, {26659, 5.176028}, {86668, 5.1183944}, {33711, 5.0687494}, {148502, 5.0673304}, {115740, 5.063576}, {113441, 5.0556765}, {158154, 5.052484}, {33115, 5.0477147}]    |\n",
      "|137    |[{87426, 5.3380704}, {18885, 5.2390604}, {103920, 5.14626}, {67467, 5.133565}, {107363, 5.0835476}, {113441, 5.080728}, {160416, 5.0751424}, {31506, 5.0529065}, {100653, 5.025002}, {24387, 5.018683}]    |\n",
      "+-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|userId|recommendations                                                                                                                                                                                                |\n",
      "+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|471   |[{127252, 5.9139094}, {203086, 5.5853567}, {192089, 5.3456798}, {203882, 5.283688}, {156414, 5.197998}, {151615, 5.191771}, {198841, 5.1819134}, {196717, 5.1694903}, {194434, 5.16531}, {187951, 5.1023607}]  |\n",
      "|463   |[{127252, 7.1203847}, {193257, 6.704946}, {190707, 6.5830116}, {203086, 6.5639505}, {203882, 6.5409117}, {196787, 6.518819}, {205453, 6.5102763}, {166812, 6.424472}, {200030, 6.3800874}, {187129, 6.37337}]  |\n",
      "|833   |[{127252, 5.117021}, {201821, 5.0028524}, {192089, 4.9809484}, {194434, 4.938762}, {203882, 4.869365}, {187951, 4.844014}, {187949, 4.844014}, {187947, 4.844014}, {157791, 4.834285}, {157789, 4.834285}]     |\n",
      "|496   |[{194434, 6.322712}, {183947, 5.996137}, {201821, 5.939149}, {151989, 5.843431}, {203882, 5.8332357}, {169852, 5.6662474}, {192089, 5.6605372}, {165559, 5.6501117}, {172737, 5.6021204}, {157791, 5.590077}]  |\n",
      "|148   |[{203882, 5.538429}, {194434, 5.5319138}, {183947, 5.4301867}, {203086, 5.314853}, {151989, 5.3005414}, {192089, 5.187046}, {172737, 5.1517863}, {165559, 5.1400166}, {157791, 5.1024704}, {157789, 5.1024704}]|\n",
      "+------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 338:=====================================>                (70 + 8) / 100]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|movieId|recommendations                                                                                                                                                                                             |\n",
      "+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|1580   |[{87426, 5.6314783}, {105130, 5.3395433}, {149507, 5.287355}, {18230, 5.2767253}, {61667, 5.2740593}, {156252, 5.2701845}, {156318, 5.2649546}, {91940, 5.233714}, {105946, 5.212908}, {129440, 5.2123775}] |\n",
      "|44022  |[{87426, 5.555072}, {52924, 5.28932}, {71207, 5.2047014}, {127635, 5.057534}, {54382, 5.047116}, {96471, 5.044008}, {45953, 5.04363}, {56046, 5.03518}, {137831, 5.0351324}, {30897, 5.024629}]             |\n",
      "|3175   |[{87426, 5.2681174}, {149507, 5.1971507}, {105130, 5.1638327}, {31747, 5.1035542}, {18230, 5.0834827}, {105946, 5.0606165}, {126317, 5.029885}, {59460, 5.0255613}, {91940, 5.0195155}, {140193, 5.0143495}]|\n",
      "|175197 |[{87426, 4.767536}, {34485, 4.5620832}, {83125, 4.4510984}, {142811, 4.429291}, {124934, 4.3883395}, {2675, 4.3355517}, {110073, 4.3222814}, {8583, 4.314501}, {75309, 4.3119783}, {15136, 4.3005157}]      |\n",
      "|1088   |[{87426, 5.3093467}, {10417, 5.3084655}, {128747, 5.190329}, {42807, 5.1884675}, {14103, 5.182371}, {65680, 5.1762385}, {22527, 5.122215}, {72284, 5.1214128}, {126090, 5.083607}, {151238, 5.0787015}]     |\n",
      "+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# Generate top 5 movie recommendations for each user\n",
    "user_recommendations = model.recommendForAllUsers(10)\n",
    "# Generate top 5 user recommendations for each movie\n",
    "movie_recommendations = model.recommendForAllItems(10)\n",
    "\n",
    "# Generate top 5 movie recommendations for a specified set of users\n",
    "selected_users = ratings.select(als.getUserCol()).distinct().limit(5)\n",
    "user_subset_recommendations = model.recommendForUserSubset(selected_users, 10)\n",
    "\n",
    "# Generate top 5 user recommendations for a specified set of movies\n",
    "selected_movies = ratings.select(als.getItemCol()).distinct().limit(5)\n",
    "movie_subset_recommendations = model.recommendForItemSubset(selected_movies, 10)\n",
    "\n",
    "# Display the generated recommendations\n",
    "user_recommendations.show(truncate=False)\n",
    "movie_recommendations.show(truncate=False)\n",
    "user_subset_recommendations.show(truncate=False)\n",
    "movie_subset_recommendations.show(truncate=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4563b3c-2487-4ea0-a866-9683459cdf98",
   "metadata": {},
   "source": [
    "## Save Model & Stop Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "721aebdd-c024-4c46-a4b1-85d36691eb4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "model.save('/Users/joey/Downloads/model_training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea4a2558-8e33-4faa-a968-411c4d02e26f",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}