Hacker Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"KMeans Cluster Model\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.clustering import KMeans, LDA, BisectingKMeans, GaussianMixture, PowerIterationClustering\n",
    "from pyspark.ml.feature import VectorAssembler, VectorIndexer\n",
    "from pyspark.ml.feature import StandardScaler\n",
    "from pyspark.ml.evaluation import ClusteringEvaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and verify data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = spark.read.csv('hack_data.csv',header = True, inferSchema = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Session_Connection_Time: double (nullable = true)\n",
      " |-- Bytes Transferred: double (nullable = true)\n",
      " |-- Kali_Trace_Used: integer (nullable = true)\n",
      " |-- Servers_Corrupted: double (nullable = true)\n",
      " |-- Pages_Corrupted: double (nullable = true)\n",
      " |-- Location: string (nullable = true)\n",
      " |-- WPM_Typing_Speed: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Session_Connection_Time=8.0, Bytes Transferred=391.09, Kali_Trace_Used=1, Servers_Corrupted=2.96, Pages_Corrupted=7.0, Location='Slovenia', WPM_Typing_Speed=72.37),\n",
       " Row(Session_Connection_Time=20.0, Bytes Transferred=720.99, Kali_Trace_Used=0, Servers_Corrupted=3.04, Pages_Corrupted=9.0, Location='British Virgin Islands', WPM_Typing_Speed=69.08),\n",
       " Row(Session_Connection_Time=31.0, Bytes Transferred=356.32, Kali_Trace_Used=1, Servers_Corrupted=3.71, Pages_Corrupted=8.0, Location='Tokelau', WPM_Typing_Speed=70.58)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.0\n",
      "391.09\n",
      "1\n",
      "2.96\n",
      "7.0\n",
      "Slovenia\n",
      "72.37\n"
     ]
    }
   ],
   "source": [
    "for item in data.head(1)[0]:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------------+------------------+------------------+-----------------+------------------+-----------+------------------+\n",
      "|summary|Session_Connection_Time| Bytes Transferred|   Kali_Trace_Used|Servers_Corrupted|   Pages_Corrupted|   Location|  WPM_Typing_Speed|\n",
      "+-------+-----------------------+------------------+------------------+-----------------+------------------+-----------+------------------+\n",
      "|  count|                    334|               334|               334|              334|               334|        334|               334|\n",
      "|   mean|     30.008982035928145| 607.2452694610777|0.5119760479041916|5.258502994011977|10.838323353293413|       null|57.342395209580864|\n",
      "| stddev|     14.088200614636158|286.33593163576757|0.5006065264451406| 2.30190693339697|  3.06352633036022|       null| 13.41106336843464|\n",
      "|    min|                    1.0|              10.0|                 0|              1.0|               6.0|Afghanistan|              40.0|\n",
      "|    max|                   60.0|            1330.5|                 1|             10.0|              15.0|   Zimbabwe|              75.0|\n",
      "+-------+-----------------------+------------------+------------------+-----------------+------------------+-----------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Session_Connection_Time',\n",
       " 'Bytes Transferred',\n",
       " 'Kali_Trace_Used',\n",
       " 'Servers_Corrupted',\n",
       " 'Pages_Corrupted',\n",
       " 'Location',\n",
       " 'WPM_Typing_Speed']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(inputCols =['Session_Connection_Time',\n",
    " 'Bytes Transferred',\n",
    " 'Kali_Trace_Used',\n",
    " 'Servers_Corrupted',\n",
    " 'Pages_Corrupted',\n",
    " 'WPM_Typing_Speed'],outputCol='features')\n",
    "final_data = assembler.transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Session_Connection_Time=8.0, Bytes Transferred=391.09, Kali_Trace_Used=1, Servers_Corrupted=2.96, Pages_Corrupted=7.0, Location='Slovenia', WPM_Typing_Speed=72.37, features=DenseVector([8.0, 391.09, 1.0, 2.96, 7.0, 72.37]))]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Session_Connection_Time: double (nullable = true)\n",
      " |-- Bytes Transferred: double (nullable = true)\n",
      " |-- Kali_Trace_Used: integer (nullable = true)\n",
      " |-- Servers_Corrupted: double (nullable = true)\n",
      " |-- Pages_Corrupted: double (nullable = true)\n",
      " |-- Location: string (nullable = true)\n",
      " |-- WPM_Typing_Speed: double (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final_data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Session_Connection_Time=8.0, Bytes Transferred=391.09, Kali_Trace_Used=1, Servers_Corrupted=2.96, Pages_Corrupted=7.0, Location='Slovenia', WPM_Typing_Speed=72.37, features=DenseVector([8.0, 391.09, 1.0, 2.96, 7.0, 72.37]), scaledfeatures=DenseVector([0.5679, 1.3658, 1.9976, 1.2859, 2.2849, 5.3963]))]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler = StandardScaler(inputCol='features',outputCol='scaledfeatures')\n",
    "final_data = scaler.fit(final_data).transform(final_data)\n",
    "final_data.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data,test_data = final_data.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+-----------------+---------------+-----------------+---------------+----------------+----------------+--------------------+--------------------+\n",
      "|Session_Connection_Time|Bytes Transferred|Kali_Trace_Used|Servers_Corrupted|Pages_Corrupted|        Location|WPM_Typing_Speed|            features|      scaledfeatures|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+----------------+----------------+--------------------+--------------------+\n",
      "|                    1.0|           390.69|              1|             2.79|            9.0|Marshall Islands|           71.57|[1.0,390.69,1.0,2...|[0.07098138558313...|\n",
      "|                    2.0|           228.08|              1|             2.48|            8.0|         Bolivia|            70.8|[2.0,228.08,1.0,2...|[0.14196277116626...|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+----------------+----------------+--------------------+--------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_data.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+-----------------+---------------+-----------------+---------------+-----------+----------------+--------------------+--------------------+\n",
      "|Session_Connection_Time|Bytes Transferred|Kali_Trace_Used|Servers_Corrupted|Pages_Corrupted|   Location|WPM_Typing_Speed|            features|      scaledfeatures|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+-----------+----------------+--------------------+--------------------+\n",
      "|                    2.0|            343.1|              1|             2.74|            8.0|    Albania|           70.64|[2.0,343.1,1.0,2....|[0.14196277116626...|\n",
      "|                    6.0|           370.45|              1|             2.72|            8.0|Philippines|           68.33|[6.0,370.45,1.0,2...|[0.42588831349878...|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+-----------+----------------+--------------------+--------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_data.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build and Evaluate Model "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.640031992469459\n"
     ]
    }
   ],
   "source": [
    "classifier = KMeans(k= 2 ,featuresCol='scaledfeatures')\n",
    "model = classifier.fit(train_data)\n",
    "predictions = model.transform(test_data)\n",
    "print(ClusteringEvaluator().evaluate(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+\n",
      "|prediction|count|\n",
      "+----------+-----+\n",
      "|         1|   53|\n",
      "|         0|   52|\n",
      "+----------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.groupBy('prediction').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster Centers: \n",
      "[1.24927239 1.3135289  0.8858819  1.36419975 2.56595463 5.26858977]\n",
      "[2.99367107 2.93449086 1.05135623 3.20744749 4.52408357 3.29363445]\n"
     ]
    }
   ],
   "source": [
    "centers = model.clusterCenters()\n",
    "print(\"Cluster Centers: \")\n",
    "for center in centers:    \n",
    "    print(center)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3041242586860749\n"
     ]
    }
   ],
   "source": [
    "classifier = KMeans(k= 3 ,featuresCol='scaledfeatures')\n",
    "model = classifier.fit(train_data)\n",
    "predictions = model.transform(test_data)\n",
    "print(ClusteringEvaluator().evaluate(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+\n",
      "|prediction|count|\n",
      "+----------+-----+\n",
      "|         1|   25|\n",
      "|         2|   28|\n",
      "|         0|   52|\n",
      "+----------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.groupBy('prediction').count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster Centers: \n",
      "[1.24927239 1.3135289  0.8858819  1.36419975 2.56595463 5.26858977]\n",
      "[2.94835644 2.88876049 0.         3.19300485 4.55176256 3.32260227]\n",
      "[3.03445423 2.9756482  1.99757683 3.22044587 4.49917247 3.26756341]\n"
     ]
    }
   ],
   "source": [
    "centers = model.clusterCenters()\n",
    "print(\"Cluster Centers: \")\n",
    "for center in centers:    \n",
    "    print(center)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### There were 2 hackers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The lower bound on the log likelihood of the entire corpus: -92543.57194332496\n"
     ]
    }
   ],
   "source": [
    "# Build and Train Model\n",
    "classifier = LDA(k=10, maxIter=10)\n",
    "model = classifier.fit(train_data)\n",
    "ll = model.logLikelihood(train_data)\n",
    "print(\"The lower bound on the log likelihood of the entire corpus: \" + str(ll))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The upper bound on perplexity: 0.5683371880795841\n"
     ]
    }
   ],
   "source": [
    "lp = model.logPerplexity(train_data)\n",
    "print(\"The upper bound on perplexity: \" + str(lp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The topics described by their top-weighted terms:\n",
      "+-----+-----------+----------------------------------------------------------------+\n",
      "|topic|termIndices|termWeights                                                     |\n",
      "+-----+-----------+----------------------------------------------------------------+\n",
      "|0    |[1, 5, 0]  |[0.8778912208582277, 0.061600159303839164, 0.038745638419526586]|\n",
      "|1    |[3, 0, 5]  |[0.19323920813914042, 0.16992826297847025, 0.1651360782002795]  |\n",
      "|2    |[5, 3, 4]  |[0.18318189460712053, 0.17863853257783938, 0.17485553307440463] |\n",
      "|3    |[1, 2, 4]  |[0.18181948937914433, 0.17692876753903303, 0.16654630287108652] |\n",
      "|4    |[2, 1, 0]  |[0.1885405071506146, 0.1737984959091707, 0.17144227653864472]   |\n",
      "|5    |[4, 5, 3]  |[0.18304237170145105, 0.17938902163213982, 0.17897332919055967] |\n",
      "|6    |[2, 1, 4]  |[0.18278283845243729, 0.17589499274117384, 0.16974439122250504] |\n",
      "|7    |[4, 3, 2]  |[0.19127979639176304, 0.16993391061108637, 0.16782528885448667] |\n",
      "|8    |[3, 5, 0]  |[0.1872399435577682, 0.1732168231269729, 0.16723520669775804]   |\n",
      "|9    |[5, 0, 4]  |[0.7555500954629804, 0.11840579581269917, 0.05795719412363282]  |\n",
      "+-----+-----------+----------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Describe topics.\n",
    "topics = model.describeTopics(3)\n",
    "print(\"The topics described by their top-weighted terms:\")\n",
    "topics.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+--------------------+\n",
      "|Session_Connection_Time|Bytes Transferred|Kali_Trace_Used|Servers_Corrupted|Pages_Corrupted|    Location|WPM_Typing_Speed|            features|      scaledfeatures|   topicDistribution|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+--------------------+\n",
      "|                    2.0|            343.1|              1|             2.74|            8.0|     Albania|           70.64|[2.0,343.1,1.0,2....|[0.14196277116626...|[0.87936689639428...|\n",
      "|                    6.0|           370.45|              1|             2.72|            8.0| Philippines|           68.33|[6.0,370.45,1.0,2...|[0.42588831349878...|[0.89496041445426...|\n",
      "|                    8.0|            64.83|              0|             3.58|            8.0|Saudi Arabia|           70.21|[8.0,64.83,0.0,3....|[0.56785108466505...|[0.44517314230650...|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+--------------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make predictions\n",
    "predictions = model.transform(test_data)\n",
    "predictions.show(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bisecting k-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build and Train Model\n",
    "classifier = BisectingKMeans().setK(2).setSeed(1)\n",
    "model = classifier.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7792054100391903\n"
     ]
    }
   ],
   "source": [
    "# Make predictions\n",
    "predictions = model.transform(test_data)\n",
    "# Evaluate clustering by computing Silhouette score\n",
    "print(ClusteringEvaluator().evaluate(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster Centers: \n",
      "[ 19.75652174 358.74234783   0.45217391   3.61208696   8.51304348\n",
      "  67.77443478]\n",
      "[4.00000000e+01 8.57770702e+02 5.17543860e-01 6.90728070e+00\n",
      " 1.32017544e+01 4.70793860e+01]\n"
     ]
    }
   ],
   "source": [
    "# Shows the result\n",
    "centers = model.clusterCenters()\n",
    "print(\"Cluster Centers: \")\n",
    "for center in centers:    \n",
    "    print(center)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+----------+\n",
      "|Session_Connection_Time|Bytes Transferred|Kali_Trace_Used|Servers_Corrupted|Pages_Corrupted|    Location|WPM_Typing_Speed|            features|      scaledfeatures|prediction|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+----------+\n",
      "|                    2.0|            343.1|              1|             2.74|            8.0|     Albania|           70.64|[2.0,343.1,1.0,2....|[0.14196277116626...|         0|\n",
      "|                    6.0|           370.45|              1|             2.72|            8.0| Philippines|           68.33|[6.0,370.45,1.0,2...|[0.42588831349878...|         0|\n",
      "|                    8.0|            64.83|              0|             3.58|            8.0|Saudi Arabia|           70.21|[8.0,64.83,0.0,3....|[0.56785108466505...|         0|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+----------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gaussian Mixture Model (GMM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build and Train Model\n",
    "classifier = GaussianMixture().setK(2).setSeed(538009335)\n",
    "model = classifier.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.640031992469459\n"
     ]
    }
   ],
   "source": [
    "# Make predictions\n",
    "predictions = model.transform(test_data)\n",
    "# Evaluate clustering by computing Silhouette score\n",
    "print(ClusteringEvaluator().evaluate(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+--------------------+----------+\n",
      "|Session_Connection_Time|Bytes Transferred|Kali_Trace_Used|Servers_Corrupted|Pages_Corrupted|    Location|WPM_Typing_Speed|            features|      scaledfeatures|         probability|prediction|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+--------------------+----------+\n",
      "|                    2.0|            343.1|              1|             2.74|            8.0|     Albania|           70.64|[2.0,343.1,1.0,2....|[0.14196277116626...|[0.99999999880620...|         0|\n",
      "|                    6.0|           370.45|              1|             2.72|            8.0| Philippines|           68.33|[6.0,370.45,1.0,2...|[0.42588831349878...|[0.99999999827283...|         0|\n",
      "|                    8.0|            64.83|              0|             3.58|            8.0|Saudi Arabia|           70.21|[8.0,64.83,0.0,3....|[0.56785108466505...|[0.99999999874013...|         0|\n",
      "+-----------------------+-----------------+---------------+-----------------+---------------+------------+----------------+--------------------+--------------------+--------------------+----------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.show(3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}