linearly_interpolate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T01:50:28.609259Z",
     "start_time": "2020-01-15T01:50:09.243744Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pyspark.sql.functions as func\n",
    "from pyspark.sql.functions import col\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql.types import *\n",
    "sc = SparkContext.getOrCreate()\n",
    "ss = SparkSession.builder.getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T01:50:30.316001Z",
     "start_time": "2020-01-15T01:50:30.312239Z"
    }
   },
   "outputs": [],
   "source": [
    "def toDoubleSafe(value): \n",
    "    try:\n",
    "        return float(value)\n",
    "    except ValueError:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T01:50:31.226331Z",
     "start_time": "2020-01-15T01:50:31.223172Z"
    }
   },
   "outputs": [],
   "source": [
    "def toIntSafe(value): \n",
    "    try:\n",
    "        return int(value)\n",
    "    except ValueError:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T01:52:22.379300Z",
     "start_time": "2020-01-15T01:52:21.083035Z"
    }
   },
   "outputs": [],
   "source": [
    "argo = sc.textFile(\"argo_data.csv\") \\\n",
    "         .map(lambda x: x.split(\",\")) \\\n",
    "         .map(lambda x: (x[0], \n",
    "                         toDoubleSafe(x[1]), \n",
    "                         toDoubleSafe(x[2]),\n",
    "                         toDoubleSafe(x[3]),\n",
    "                         toDoubleSafe(x[4]),\n",
    "                         toDoubleSafe(x[5]),\n",
    "                         x[6]))\n",
    "argo_schema = StructType([StructField(\"profile_id\", StringType(), True),\n",
    "                          StructField(\"pres\", DoubleType(), True),\n",
    "                          StructField(\"temp\", DoubleType(), True),\n",
    "                          StructField(\"lat\", DoubleType(), True),\n",
    "                          StructField(\"lon\", DoubleType(), True),\n",
    "                          StructField(\"psal\", DoubleType(), True),\n",
    "                          StructField(\"date\", TimestampType(), True),\n",
    "                        ])\n",
    "argo_df = ss.read.csv(\"argo_data.csv\", header=True, schema=argo_schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T01:52:25.165831Z",
     "start_time": "2020-01-15T01:52:23.815209Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----+-----+------+------+------+-------------------+\n",
      "|profile_id|pres| temp|   lat|   lon|  psal|               date|\n",
      "+----------+----+-----+------+------+------+-------------------+\n",
      "|5905377_14|2.91|1.048|-56.73|148.24|33.739|2018-06-17 16:59:45|\n",
      "|5905377_14|4.21|1.049|-56.73|148.24|33.738|2018-06-17 16:59:45|\n",
      "|5905377_14|6.21|1.049|-56.73|148.24|33.738|2018-06-17 16:59:45|\n",
      "+----------+----+-----+------+------+------+-------------------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "argo_df.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T01:52:30.148664Z",
     "start_time": "2020-01-15T01:52:26.076193Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+--------------------+\n",
      "| profile_id|      temp_pres_list|\n",
      "+-----------+--------------------+\n",
      "| 5905293_29|[28.567, 28.566, ...|\n",
      "|4901659_142|[24.296, 24.296, ...|\n",
      "|5904161_236|[-1.671, -1.657, ...|\n",
      "|5902385_124|[8.42, 8.42, 8.42...|\n",
      "| 2902676_53|[30.067, 29.848, ...|\n",
      "|5903601_240|[14.992, 12.59, 1...|\n",
      "|3901205_123|[24.722, 24.729, ...|\n",
      "| 5902476_72|[19.721, 19.72, 1...|\n",
      "|5903808_221|[29.881, 29.9, 29...|\n",
      "|5904597_169|[-0.506, -0.49, -...|\n",
      "|2902656_254|[29.885, 29.894, ...|\n",
      "| 2902717_10|[29.531, 29.54, 2...|\n",
      "|4901409_231|[20.468, 20.353, ...|\n",
      "| 5904767_79|[-0.222, -0.22, -...|\n",
      "| 5905353_25|[10.136, 10.136, ...|\n",
      "|4901565_183|[8.667, 8.667, 8....|\n",
      "|5901691_348|[9.369, 9.371, 9....|\n",
      "|5903840_227|[29.45, 29.448, 2...|\n",
      "|5904715_109|[28.984, 28.998, ...|\n",
      "| 6903232_13|[5.107, 5.108, 5....|\n",
      "+-----------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import *\n",
    "\n",
    "insane_sort = udf(lambda x: [item[0] for item in sorted(x,key=lambda x: x[1])], ArrayType(DoubleType()))\n",
    "\n",
    "array(argo_df['temp'],array(argo_df['pres']))\n",
    "\n",
    "argo_df_listed = argo_df.select('profile_id',array(argo_df['temp'],argo_df['pres']).alias('temp_pres'))\\\n",
    "       .groupBy('profile_id').agg(collect_list('temp_pres').alias('temp_pres_list'))\n",
    "argo_df_listed = argo_df_listed.select('profile_id',insane_sort(argo_df_listed['temp_pres_list']).alias('temp_pres_list'))\n",
    "argo_df_listed.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T02:02:03.568996Z",
     "start_time": "2020-01-15T02:01:59.195778Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(temp_pres_list=[28.567, 28.566, 28.566, 28.567, 28.568, 28.566, 28.565, 28.565, 28.566, 28.567, 28.561, 28.543, 28.496, 28.431, 28.417, 28.413, 28.403, 28.396, 28.396, 28.395, 28.382, 28.355, 28.349, 28.344, 28.331, 28.316, 28.31, 28.288, 28.24, 28.189, 28.154, 28.152, 28.145, 28.118, 28.06, 28.038, 28.038, 28.032, 27.991, 27.943, 27.874, 27.667, 27.097, 26.964, 26.933, 26.914, 26.902, 26.895, 26.886, 26.874, 26.847, 26.817, 26.729, 26.565, 26.333, 25.999, 25.456, 25.029, 24.666, 24.514, 24.27, 23.542, 22.41, 22.128, 21.968, 21.907, 21.557, 21.049, 20.637, 20.002, 19.551, 19.007, 18.695, 17.543, 16.814, 16.152, 15.345, 14.576, 14.383, 14.331, 14.295, 14.237, 13.927, 13.877, 13.85, 13.539, 13.255, 13.189, 13.099, 12.889, 12.753, 12.787, 12.762, 12.729, 12.586, 12.356, 12.26, 12.2, 11.959, 11.894, 11.861, 11.851, 11.84, 11.785, 11.762, 11.757, 11.724, 11.623, 11.547, 11.488, 11.472, 11.463, 11.443, 11.405, 11.339, 11.294, 11.266, 11.246, 11.237, 11.243, 11.242, 11.225, 11.209, 11.203, 11.19, 11.185, 11.181, 11.175, 11.168, 11.151, 11.134, 11.096, 11.049, 11.031, 11.02, 10.993, 10.96, 10.959, 10.957, 10.938, 10.918, 10.892, 10.857, 10.806, 10.79, 10.77, 10.762, 10.757, 10.749, 10.724, 10.689, 10.675, 10.66, 10.647, 10.633, 10.618, 10.561, 10.544, 10.532, 10.515, 10.497, 10.482, 10.459, 10.427, 10.398, 10.378, 10.36, 10.348, 10.341, 10.332, 10.315, 10.279, 10.235, 10.219, 10.174, 10.142, 10.103, 10.074, 10.03, 10.023, 10.014, 10.001, 9.982, 9.967, 9.955, 9.926, 9.869, 9.829, 9.775, 9.675, 9.543, 9.448, 9.351, 9.257, 9.143, 9.106, 9.086, 9.069, 9.021, 8.98, 8.921, 8.891, 8.885, 8.876, 8.863, 8.854, 8.849, 8.841, 8.814, 8.793, 8.763, 8.736, 8.715, 8.68, 8.564, 8.509, 8.501, 8.498, 8.494, 8.481, 8.44, 8.375, 8.278, 8.182, 8.16, 8.15, 8.133, 8.107, 8.086, 8.081, 8.076, 8.061, 8.046, 8.013, 7.967, 7.904, 7.893, 7.882, 7.868, 7.859, 7.856, 7.855, 7.854, 7.85, 7.84, 7.832, 7.827, 7.817, 7.806, 7.794, 7.781, 7.761, 7.743, 7.73, 7.714, 7.704, 7.695, 7.679, 7.658, 7.642, 7.599, 7.586, 7.584, 7.574, 7.571, 7.57, 7.569, 7.567, 7.566, 7.565, 7.564, 7.561, 7.559, 7.557, 7.549, 7.546, 7.541, 7.535, 7.529, 7.519, 7.493, 7.438, 7.375, 7.357, 7.26, 7.088, 7.014, 6.992, 6.974, 6.916, 6.876, 6.872, 6.869, 6.867, 6.862, 6.861, 6.853, 6.848, 6.845, 6.837, 6.832, 6.823, 6.814, 6.806, 6.8, 6.792, 6.787, 6.784, 6.78, 6.778, 6.768, 6.76, 6.755, 6.752, 6.746, 6.736, 6.699, 6.649, 6.632, 6.628, 6.626, 6.625, 6.621, 6.618, 6.612, 6.584, 6.555, 6.549, 6.538, 6.534, 6.527, 6.52, 6.516, 6.509, 6.5, 6.486, 6.457, 6.435, 6.43, 6.422, 6.42, 6.419, 6.417, 6.412, 6.365, 6.346, 6.332, 6.308, 6.29, 6.283, 6.281, 6.279, 6.275, 6.262, 6.246, 6.236, 6.21, 6.169, 6.135, 6.073, 6.055, 6.047, 6.04, 6.031, 6.021, 6.006, 5.97, 5.953, 5.942, 5.93, 5.918, 5.903, 5.873, 5.841, 5.819, 5.79, 5.775, 5.767, 5.766, 5.764, 5.753, 5.745, 5.738, 5.735, 5.724, 5.712, 5.705, 5.693, 5.679, 5.663, 5.637, 5.617, 5.611, 5.595, 5.574, 5.509, 5.475, 5.454, 5.448, 5.443, 5.44, 5.437, 5.434, 5.433, 5.433, 5.432, 5.43, 5.428, 5.422, 5.411, 5.384, 5.362, 5.346, 5.332, 5.328, 5.324, 5.324, 5.318, 5.316, 5.314, 5.306, 5.288, 5.269, 5.252, 5.248, 5.247, 5.247, 5.232, 5.206, 5.197, 5.191, 5.187, 5.175, 5.169, 5.167, 5.166, 5.16, 5.152, 5.143, 5.135, 5.123, 5.114, 5.113, 5.112, 5.111, 5.108, 5.086, 5.049, 5.04, 5.039, 5.037, 5.033, 5.032, 5.032, 5.028, 5.025, 5.023, 5.019, 5.018, 5.015, 5.012, 5.006, 5.001, 5.001, 4.999, 4.992, 4.976, 4.965, 4.955, 4.948, 4.946, 4.944, 4.939, 4.931, 4.925, 4.918, 4.914, 4.906, 4.898, 4.897, 4.896, 4.889, 4.877, 4.873, 4.862, 4.85, 4.841, 4.826, 4.813, 4.811, 4.811, 4.811, 4.811, 4.811, 4.805, 4.793, 4.791, 4.785, 4.779, 4.771, 4.762, 4.755, 4.744, 4.734, 4.728, 4.726, 4.712, 4.688, 4.648, 4.628, 4.624, 4.621, 4.62, 4.62, 4.619, 4.618, 4.613, 4.601, 4.587, 4.584, 4.58, 4.574, 4.571, 4.567, 4.561, 4.554, 4.55, 4.549, 4.549, 4.546, 4.54, 4.531, 4.528, 4.516, 4.503, 4.486, 4.476, 4.462, 4.441, 4.427, 4.408, 4.401, 4.396, 4.387, 4.38, 4.375, 4.373, 4.366, 4.362, 4.359, 4.354, 4.347, 4.338, 4.334, 4.325, 4.319, 4.315, 4.312, 4.308, 4.286, 4.267, 4.249, 4.243, 4.236, 4.229, 4.227, 4.225, 4.225, 4.22, 4.2, 4.189, 4.183, 4.18, 4.178, 4.173, 4.165, 4.159, 4.155, 4.154, 4.152, 4.15, 4.15, 4.148, 4.147, 4.145, 4.143, 4.142, 4.142, 4.125, 4.102, 4.092, 4.089, 4.078, 4.067, 4.065, 4.059, 4.052, 4.039, 4.027, 4.017, 4.002, 3.994, 3.992, 3.987, 3.985, 3.982, 3.981, 3.979, 3.977, 3.974, 3.973, 3.972, 3.972, 3.972, 3.971, 3.971, 3.971, 3.971, 3.967, 3.963, 3.963, 3.962, 3.961, 3.955, 3.947, 3.945, 3.942, 3.936, 3.926, 3.914, 3.905, 3.902, 3.888, 3.871, 3.866, 3.858, 3.853, 3.852, 3.852, 3.851, 3.851, 3.852, 3.852, 3.852, 3.852, 3.839, 3.816, 3.807, 3.794, 3.792, 3.791, 3.788, 3.785, 3.78, 3.774, 3.765, 3.761, 3.757, 3.74, 3.727, 3.715, 3.701, 3.699, 3.696, 3.693, 3.688, 3.681, 3.677, 3.673, 3.672, 3.67, 3.666, 3.663, 3.661, 3.654, 3.65, 3.649, 3.648, 3.647, 3.646, 3.644, 3.642, 3.64, 3.637, 3.626, 3.623, 3.623, 3.617, 3.604, 3.602, 3.597, 3.592, 3.589, 3.583, 3.578, 3.577, 3.575, 3.567, 3.556, 3.537, 3.523, 3.507, 3.502, 3.493, 3.483, 3.48, 3.472, 3.468, 3.464, 3.462, 3.458, 3.443, 3.434, 3.431, 3.428, 3.418, 3.402, 3.4, 3.39, 3.378, 3.369, 3.357, 3.349, 3.331, 3.32, 3.311, 3.299, 3.286, 3.252, 3.234, 3.226, 3.225, 3.223, 3.214, 3.204, 3.2, 3.195, 3.194, 3.193, 3.193, 3.192, 3.19, 3.188, 3.184, 3.182, 3.181, 3.177, 3.173, 3.166, 3.156, 3.144, 3.14, 3.14, 3.14, 3.14, 3.14, 3.14, 3.14, 3.14, 3.141, 3.141, 3.141, 3.14, 3.138, 3.13, 3.119, 3.116, 3.115, 3.102, 3.087, 3.078, 3.076, 3.073, 3.07, 3.068, 3.067, 3.065, 3.064, 3.063, 3.062, 3.057, 3.05, 3.042, 3.038, 3.034, 3.025, 3.008, 3.001, 2.983, 2.965, 2.955, 2.951, 2.95, 2.949, 2.945, 2.937, 2.934, 2.933, 2.933, 2.932, 2.931, 2.931, 2.931, 2.927, 2.914, 2.904, 2.901, 2.9, 2.9, 2.9, 2.899, 2.899, 2.898, 2.898, 2.897, 2.893, 2.893, 2.89, 2.883, 2.878, 2.874, 2.872, 2.868, 2.864, 2.862, 2.861, 2.86, 2.859, 2.857, 2.851, 2.847, 2.846, 2.845, 2.841, 2.838, 2.836, 2.832, 2.821, 2.815, 2.796, 2.787, 2.786, 2.785, 2.785, 2.785, 2.785, 2.783, 2.778, 2.769, 2.763, 2.754, 2.747, 2.741, 2.738, 2.738, 2.738, 2.736, 2.734, 2.734, 2.734, 2.733, 2.724, 2.719, 2.718, 2.717, 2.705, 2.7, 2.698, 2.695, 2.692, 2.688, 2.683, 2.68, 2.679, 2.677, 2.667, 2.655, 2.645, 2.64, 2.635, 2.629, 2.626, 2.625, 2.625, 2.624, 2.623, 2.621, 2.615, 2.611, 2.611, 2.61, 2.609, 2.608, 2.607, 2.606, 2.603, 2.602, 2.601, 2.598, 2.597, 2.595, 2.592, 2.591, 2.59, 2.589, 2.584, 2.576, 2.57, 2.565, 2.558, 2.556, 2.555, 2.553, 2.55, 2.546, 2.542, 2.54, 2.538, 2.537, 2.536, 2.536, 2.536, 2.535, 2.534, 2.533, 2.519, 2.503, 2.5, 2.5, 2.5, 2.499, 2.498, 2.498, 2.498, 2.498, 2.497, 2.497, 2.496, 2.494, 2.49, 2.483, 2.475, 2.47, 2.467, 2.461, 2.459, 2.458, 2.457, 2.456, 2.455, 2.455, 2.453, 2.452, 2.452, 2.451, 2.45, 2.448, 2.447, 2.447, 2.443, 2.438, 2.436, 2.434, 2.432, 2.43, 2.418, 2.412, 2.41, 2.409, 2.409, 2.409, 2.409, 2.408, 2.394, 2.379, 2.37, 2.368, 2.363, 2.359, 2.359, 2.357, 2.357, 2.354, 2.351, 2.349, 2.346, 2.343, 2.341, 2.34, 2.336, 2.335, 2.331, 2.329, 2.327, 2.325, 2.322, 2.322, 2.322, 2.32, 2.32])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row1 = argo_df_listed.select('temp_pres_list').collect()[0]\n",
    "row1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T02:02:56.905891Z",
     "start_time": "2020-01-15T02:02:56.894372Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ParallelCollectionRDD[104] at parallelize at PythonRDD.scala:195"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contents = sc.parallelize([row1])\n",
    "contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T02:25:11.185771Z",
     "start_time": "2020-01-15T02:25:11.172110Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[163] at RDD at PythonRDD.scala:53"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contents_rdd = contents.flatMap(lambda x: x)\n",
    "contents_rdd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T02:15:34.162761Z",
     "start_time": "2020-01-15T02:15:34.033745Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[28.567,\n",
       "  28.566,\n",
       "  28.566,\n",
       "  28.567,\n",
       "  28.568,\n",
       "  28.566,\n",
       "  28.565,\n",
       "  28.565,\n",
       "  28.566,\n",
       "  28.567,\n",
       "  28.561,\n",
       "  28.543,\n",
       "  28.496,\n",
       "  28.431,\n",
       "  28.417,\n",
       "  28.413,\n",
       "  28.403,\n",
       "  28.396,\n",
       "  28.396,\n",
       "  28.395,\n",
       "  28.382,\n",
       "  28.355,\n",
       "  28.349,\n",
       "  28.344,\n",
       "  28.331,\n",
       "  28.316,\n",
       "  28.31,\n",
       "  28.288,\n",
       "  28.24,\n",
       "  28.189,\n",
       "  28.154,\n",
       "  28.152,\n",
       "  28.145,\n",
       "  28.118,\n",
       "  28.06,\n",
       "  28.038,\n",
       "  28.038,\n",
       "  28.032,\n",
       "  27.991,\n",
       "  27.943,\n",
       "  27.874,\n",
       "  27.667,\n",
       "  27.097,\n",
       "  26.964,\n",
       "  26.933,\n",
       "  26.914,\n",
       "  26.902,\n",
       "  26.895,\n",
       "  26.886,\n",
       "  26.874,\n",
       "  26.847,\n",
       "  26.817,\n",
       "  26.729,\n",
       "  26.565,\n",
       "  26.333,\n",
       "  25.999,\n",
       "  25.456,\n",
       "  25.029,\n",
       "  24.666,\n",
       "  24.514,\n",
       "  24.27,\n",
       "  23.542,\n",
       "  22.41,\n",
       "  22.128,\n",
       "  21.968,\n",
       "  21.907,\n",
       "  21.557,\n",
       "  21.049,\n",
       "  20.637,\n",
       "  20.002,\n",
       "  19.551,\n",
       "  19.007,\n",
       "  18.695,\n",
       "  17.543,\n",
       "  16.814,\n",
       "  16.152,\n",
       "  15.345,\n",
       "  14.576,\n",
       "  14.383,\n",
       "  14.331,\n",
       "  14.295,\n",
       "  14.237,\n",
       "  13.927,\n",
       "  13.877,\n",
       "  13.85,\n",
       "  13.539,\n",
       "  13.255,\n",
       "  13.189,\n",
       "  13.099,\n",
       "  12.889,\n",
       "  12.753,\n",
       "  12.787,\n",
       "  12.762,\n",
       "  12.729,\n",
       "  12.586,\n",
       "  12.356,\n",
       "  12.26,\n",
       "  12.2,\n",
       "  11.959,\n",
       "  11.894,\n",
       "  11.861,\n",
       "  11.851,\n",
       "  11.84,\n",
       "  11.785,\n",
       "  11.762,\n",
       "  11.757,\n",
       "  11.724,\n",
       "  11.623,\n",
       "  11.547,\n",
       "  11.488,\n",
       "  11.472,\n",
       "  11.463,\n",
       "  11.443,\n",
       "  11.405,\n",
       "  11.339,\n",
       "  11.294,\n",
       "  11.266,\n",
       "  11.246,\n",
       "  11.237,\n",
       "  11.243,\n",
       "  11.242,\n",
       "  11.225,\n",
       "  11.209,\n",
       "  11.203,\n",
       "  11.19,\n",
       "  11.185,\n",
       "  11.181,\n",
       "  11.175,\n",
       "  11.168,\n",
       "  11.151,\n",
       "  11.134,\n",
       "  11.096,\n",
       "  11.049,\n",
       "  11.031,\n",
       "  11.02,\n",
       "  10.993,\n",
       "  10.96,\n",
       "  10.959,\n",
       "  10.957,\n",
       "  10.938,\n",
       "  10.918,\n",
       "  10.892,\n",
       "  10.857,\n",
       "  10.806,\n",
       "  10.79,\n",
       "  10.77,\n",
       "  10.762,\n",
       "  10.757,\n",
       "  10.749,\n",
       "  10.724,\n",
       "  10.689,\n",
       "  10.675,\n",
       "  10.66,\n",
       "  10.647,\n",
       "  10.633,\n",
       "  10.618,\n",
       "  10.561,\n",
       "  10.544,\n",
       "  10.532,\n",
       "  10.515,\n",
       "  10.497,\n",
       "  10.482,\n",
       "  10.459,\n",
       "  10.427,\n",
       "  10.398,\n",
       "  10.378,\n",
       "  10.36,\n",
       "  10.348,\n",
       "  10.341,\n",
       "  10.332,\n",
       "  10.315,\n",
       "  10.279,\n",
       "  10.235,\n",
       "  10.219,\n",
       "  10.174,\n",
       "  10.142,\n",
       "  10.103,\n",
       "  10.074,\n",
       "  10.03,\n",
       "  10.023,\n",
       "  10.014,\n",
       "  10.001,\n",
       "  9.982,\n",
       "  9.967,\n",
       "  9.955,\n",
       "  9.926,\n",
       "  9.869,\n",
       "  9.829,\n",
       "  9.775,\n",
       "  9.675,\n",
       "  9.543,\n",
       "  9.448,\n",
       "  9.351,\n",
       "  9.257,\n",
       "  9.143,\n",
       "  9.106,\n",
       "  9.086,\n",
       "  9.069,\n",
       "  9.021,\n",
       "  8.98,\n",
       "  8.921,\n",
       "  8.891,\n",
       "  8.885,\n",
       "  8.876,\n",
       "  8.863,\n",
       "  8.854,\n",
       "  8.849,\n",
       "  8.841,\n",
       "  8.814,\n",
       "  8.793,\n",
       "  8.763,\n",
       "  8.736,\n",
       "  8.715,\n",
       "  8.68,\n",
       "  8.564,\n",
       "  8.509,\n",
       "  8.501,\n",
       "  8.498,\n",
       "  8.494,\n",
       "  8.481,\n",
       "  8.44,\n",
       "  8.375,\n",
       "  8.278,\n",
       "  8.182,\n",
       "  8.16,\n",
       "  8.15,\n",
       "  8.133,\n",
       "  8.107,\n",
       "  8.086,\n",
       "  8.081,\n",
       "  8.076,\n",
       "  8.061,\n",
       "  8.046,\n",
       "  8.013,\n",
       "  7.967,\n",
       "  7.904,\n",
       "  7.893,\n",
       "  7.882,\n",
       "  7.868,\n",
       "  7.859,\n",
       "  7.856,\n",
       "  7.855,\n",
       "  7.854,\n",
       "  7.85,\n",
       "  7.84,\n",
       "  7.832,\n",
       "  7.827,\n",
       "  7.817,\n",
       "  7.806,\n",
       "  7.794,\n",
       "  7.781,\n",
       "  7.761,\n",
       "  7.743,\n",
       "  7.73,\n",
       "  7.714,\n",
       "  7.704,\n",
       "  7.695,\n",
       "  7.679,\n",
       "  7.658,\n",
       "  7.642,\n",
       "  7.599,\n",
       "  7.586,\n",
       "  7.584,\n",
       "  7.574,\n",
       "  7.571,\n",
       "  7.57,\n",
       "  7.569,\n",
       "  7.567,\n",
       "  7.566,\n",
       "  7.565,\n",
       "  7.564,\n",
       "  7.561,\n",
       "  7.559,\n",
       "  7.557,\n",
       "  7.549,\n",
       "  7.546,\n",
       "  7.541,\n",
       "  7.535,\n",
       "  7.529,\n",
       "  7.519,\n",
       "  7.493,\n",
       "  7.438,\n",
       "  7.375,\n",
       "  7.357,\n",
       "  7.26,\n",
       "  7.088,\n",
       "  7.014,\n",
       "  6.992,\n",
       "  6.974,\n",
       "  6.916,\n",
       "  6.876,\n",
       "  6.872,\n",
       "  6.869,\n",
       "  6.867,\n",
       "  6.862,\n",
       "  6.861,\n",
       "  6.853,\n",
       "  6.848,\n",
       "  6.845,\n",
       "  6.837,\n",
       "  6.832,\n",
       "  6.823,\n",
       "  6.814,\n",
       "  6.806,\n",
       "  6.8,\n",
       "  6.792,\n",
       "  6.787,\n",
       "  6.784,\n",
       "  6.78,\n",
       "  6.778,\n",
       "  6.768,\n",
       "  6.76,\n",
       "  6.755,\n",
       "  6.752,\n",
       "  6.746,\n",
       "  6.736,\n",
       "  6.699,\n",
       "  6.649,\n",
       "  6.632,\n",
       "  6.628,\n",
       "  6.626,\n",
       "  6.625,\n",
       "  6.621,\n",
       "  6.618,\n",
       "  6.612,\n",
       "  6.584,\n",
       "  6.555,\n",
       "  6.549,\n",
       "  6.538,\n",
       "  6.534,\n",
       "  6.527,\n",
       "  6.52,\n",
       "  6.516,\n",
       "  6.509,\n",
       "  6.5,\n",
       "  6.486,\n",
       "  6.457,\n",
       "  6.435,\n",
       "  6.43,\n",
       "  6.422,\n",
       "  6.42,\n",
       "  6.419,\n",
       "  6.417,\n",
       "  6.412,\n",
       "  6.365,\n",
       "  6.346,\n",
       "  6.332,\n",
       "  6.308,\n",
       "  6.29,\n",
       "  6.283,\n",
       "  6.281,\n",
       "  6.279,\n",
       "  6.275,\n",
       "  6.262,\n",
       "  6.246,\n",
       "  6.236,\n",
       "  6.21,\n",
       "  6.169,\n",
       "  6.135,\n",
       "  6.073,\n",
       "  6.055,\n",
       "  6.047,\n",
       "  6.04,\n",
       "  6.031,\n",
       "  6.021,\n",
       "  6.006,\n",
       "  5.97,\n",
       "  5.953,\n",
       "  5.942,\n",
       "  5.93,\n",
       "  5.918,\n",
       "  5.903,\n",
       "  5.873,\n",
       "  5.841,\n",
       "  5.819,\n",
       "  5.79,\n",
       "  5.775,\n",
       "  5.767,\n",
       "  5.766,\n",
       "  5.764,\n",
       "  5.753,\n",
       "  5.745,\n",
       "  5.738,\n",
       "  5.735,\n",
       "  5.724,\n",
       "  5.712,\n",
       "  5.705,\n",
       "  5.693,\n",
       "  5.679,\n",
       "  5.663,\n",
       "  5.637,\n",
       "  5.617,\n",
       "  5.611,\n",
       "  5.595,\n",
       "  5.574,\n",
       "  5.509,\n",
       "  5.475,\n",
       "  5.454,\n",
       "  5.448,\n",
       "  5.443,\n",
       "  5.44,\n",
       "  5.437,\n",
       "  5.434,\n",
       "  5.433,\n",
       "  5.433,\n",
       "  5.432,\n",
       "  5.43,\n",
       "  5.428,\n",
       "  5.422,\n",
       "  5.411,\n",
       "  5.384,\n",
       "  5.362,\n",
       "  5.346,\n",
       "  5.332,\n",
       "  5.328,\n",
       "  5.324,\n",
       "  5.324,\n",
       "  5.318,\n",
       "  5.316,\n",
       "  5.314,\n",
       "  5.306,\n",
       "  5.288,\n",
       "  5.269,\n",
       "  5.252,\n",
       "  5.248,\n",
       "  5.247,\n",
       "  5.247,\n",
       "  5.232,\n",
       "  5.206,\n",
       "  5.197,\n",
       "  5.191,\n",
       "  5.187,\n",
       "  5.175,\n",
       "  5.169,\n",
       "  5.167,\n",
       "  5.166,\n",
       "  5.16,\n",
       "  5.152,\n",
       "  5.143,\n",
       "  5.135,\n",
       "  5.123,\n",
       "  5.114,\n",
       "  5.113,\n",
       "  5.112,\n",
       "  5.111,\n",
       "  5.108,\n",
       "  5.086,\n",
       "  5.049,\n",
       "  5.04,\n",
       "  5.039,\n",
       "  5.037,\n",
       "  5.033,\n",
       "  5.032,\n",
       "  5.032,\n",
       "  5.028,\n",
       "  5.025,\n",
       "  5.023,\n",
       "  5.019,\n",
       "  5.018,\n",
       "  5.015,\n",
       "  5.012,\n",
       "  5.006,\n",
       "  5.001,\n",
       "  5.001,\n",
       "  4.999,\n",
       "  4.992,\n",
       "  4.976,\n",
       "  4.965,\n",
       "  4.955,\n",
       "  4.948,\n",
       "  4.946,\n",
       "  4.944,\n",
       "  4.939,\n",
       "  4.931,\n",
       "  4.925,\n",
       "  4.918,\n",
       "  4.914,\n",
       "  4.906,\n",
       "  4.898,\n",
       "  4.897,\n",
       "  4.896,\n",
       "  4.889,\n",
       "  4.877,\n",
       "  4.873,\n",
       "  4.862,\n",
       "  4.85,\n",
       "  4.841,\n",
       "  4.826,\n",
       "  4.813,\n",
       "  4.811,\n",
       "  4.811,\n",
       "  4.811,\n",
       "  4.811,\n",
       "  4.811,\n",
       "  4.805,\n",
       "  4.793,\n",
       "  4.791,\n",
       "  4.785,\n",
       "  4.779,\n",
       "  4.771,\n",
       "  4.762,\n",
       "  4.755,\n",
       "  4.744,\n",
       "  4.734,\n",
       "  4.728,\n",
       "  4.726,\n",
       "  4.712,\n",
       "  4.688,\n",
       "  4.648,\n",
       "  4.628,\n",
       "  4.624,\n",
       "  4.621,\n",
       "  4.62,\n",
       "  4.62,\n",
       "  4.619,\n",
       "  4.618,\n",
       "  4.613,\n",
       "  4.601,\n",
       "  4.587,\n",
       "  4.584,\n",
       "  4.58,\n",
       "  4.574,\n",
       "  4.571,\n",
       "  4.567,\n",
       "  4.561,\n",
       "  4.554,\n",
       "  4.55,\n",
       "  4.549,\n",
       "  4.549,\n",
       "  4.546,\n",
       "  4.54,\n",
       "  4.531,\n",
       "  4.528,\n",
       "  4.516,\n",
       "  4.503,\n",
       "  4.486,\n",
       "  4.476,\n",
       "  4.462,\n",
       "  4.441,\n",
       "  4.427,\n",
       "  4.408,\n",
       "  4.401,\n",
       "  4.396,\n",
       "  4.387,\n",
       "  4.38,\n",
       "  4.375,\n",
       "  4.373,\n",
       "  4.366,\n",
       "  4.362,\n",
       "  4.359,\n",
       "  4.354,\n",
       "  4.347,\n",
       "  4.338,\n",
       "  4.334,\n",
       "  4.325,\n",
       "  4.319,\n",
       "  4.315,\n",
       "  4.312,\n",
       "  4.308,\n",
       "  4.286,\n",
       "  4.267,\n",
       "  4.249,\n",
       "  4.243,\n",
       "  4.236,\n",
       "  4.229,\n",
       "  4.227,\n",
       "  4.225,\n",
       "  4.225,\n",
       "  4.22,\n",
       "  4.2,\n",
       "  4.189,\n",
       "  4.183,\n",
       "  4.18,\n",
       "  4.178,\n",
       "  4.173,\n",
       "  4.165,\n",
       "  4.159,\n",
       "  4.155,\n",
       "  4.154,\n",
       "  4.152,\n",
       "  4.15,\n",
       "  4.15,\n",
       "  4.148,\n",
       "  4.147,\n",
       "  4.145,\n",
       "  4.143,\n",
       "  4.142,\n",
       "  4.142,\n",
       "  4.125,\n",
       "  4.102,\n",
       "  4.092,\n",
       "  4.089,\n",
       "  4.078,\n",
       "  4.067,\n",
       "  4.065,\n",
       "  4.059,\n",
       "  4.052,\n",
       "  4.039,\n",
       "  4.027,\n",
       "  4.017,\n",
       "  4.002,\n",
       "  3.994,\n",
       "  3.992,\n",
       "  3.987,\n",
       "  3.985,\n",
       "  3.982,\n",
       "  3.981,\n",
       "  3.979,\n",
       "  3.977,\n",
       "  3.974,\n",
       "  3.973,\n",
       "  3.972,\n",
       "  3.972,\n",
       "  3.972,\n",
       "  3.971,\n",
       "  3.971,\n",
       "  3.971,\n",
       "  3.971,\n",
       "  3.967,\n",
       "  3.963,\n",
       "  3.963,\n",
       "  3.962,\n",
       "  3.961,\n",
       "  3.955,\n",
       "  3.947,\n",
       "  3.945,\n",
       "  3.942,\n",
       "  3.936,\n",
       "  3.926,\n",
       "  3.914,\n",
       "  3.905,\n",
       "  3.902,\n",
       "  3.888,\n",
       "  3.871,\n",
       "  3.866,\n",
       "  3.858,\n",
       "  3.853,\n",
       "  3.852,\n",
       "  3.852,\n",
       "  3.851,\n",
       "  3.851,\n",
       "  3.852,\n",
       "  3.852,\n",
       "  3.852,\n",
       "  3.852,\n",
       "  3.839,\n",
       "  3.816,\n",
       "  3.807,\n",
       "  3.794,\n",
       "  3.792,\n",
       "  3.791,\n",
       "  3.788,\n",
       "  3.785,\n",
       "  3.78,\n",
       "  3.774,\n",
       "  3.765,\n",
       "  3.761,\n",
       "  3.757,\n",
       "  3.74,\n",
       "  3.727,\n",
       "  3.715,\n",
       "  3.701,\n",
       "  3.699,\n",
       "  3.696,\n",
       "  3.693,\n",
       "  3.688,\n",
       "  3.681,\n",
       "  3.677,\n",
       "  3.673,\n",
       "  3.672,\n",
       "  3.67,\n",
       "  3.666,\n",
       "  3.663,\n",
       "  3.661,\n",
       "  3.654,\n",
       "  3.65,\n",
       "  3.649,\n",
       "  3.648,\n",
       "  3.647,\n",
       "  3.646,\n",
       "  3.644,\n",
       "  3.642,\n",
       "  3.64,\n",
       "  3.637,\n",
       "  3.626,\n",
       "  3.623,\n",
       "  3.623,\n",
       "  3.617,\n",
       "  3.604,\n",
       "  3.602,\n",
       "  3.597,\n",
       "  3.592,\n",
       "  3.589,\n",
       "  3.583,\n",
       "  3.578,\n",
       "  3.577,\n",
       "  3.575,\n",
       "  3.567,\n",
       "  3.556,\n",
       "  3.537,\n",
       "  3.523,\n",
       "  3.507,\n",
       "  3.502,\n",
       "  3.493,\n",
       "  3.483,\n",
       "  3.48,\n",
       "  3.472,\n",
       "  3.468,\n",
       "  3.464,\n",
       "  3.462,\n",
       "  3.458,\n",
       "  3.443,\n",
       "  3.434,\n",
       "  3.431,\n",
       "  3.428,\n",
       "  3.418,\n",
       "  3.402,\n",
       "  3.4,\n",
       "  3.39,\n",
       "  3.378,\n",
       "  3.369,\n",
       "  3.357,\n",
       "  3.349,\n",
       "  3.331,\n",
       "  3.32,\n",
       "  3.311,\n",
       "  3.299,\n",
       "  3.286,\n",
       "  3.252,\n",
       "  3.234,\n",
       "  3.226,\n",
       "  3.225,\n",
       "  3.223,\n",
       "  3.214,\n",
       "  3.204,\n",
       "  3.2,\n",
       "  3.195,\n",
       "  3.194,\n",
       "  3.193,\n",
       "  3.193,\n",
       "  3.192,\n",
       "  3.19,\n",
       "  3.188,\n",
       "  3.184,\n",
       "  3.182,\n",
       "  3.181,\n",
       "  3.177,\n",
       "  3.173,\n",
       "  3.166,\n",
       "  3.156,\n",
       "  3.144,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.14,\n",
       "  3.141,\n",
       "  3.141,\n",
       "  3.141,\n",
       "  3.14,\n",
       "  3.138,\n",
       "  3.13,\n",
       "  3.119,\n",
       "  3.116,\n",
       "  3.115,\n",
       "  3.102,\n",
       "  3.087,\n",
       "  3.078,\n",
       "  3.076,\n",
       "  3.073,\n",
       "  3.07,\n",
       "  3.068,\n",
       "  3.067,\n",
       "  3.065,\n",
       "  3.064,\n",
       "  3.063,\n",
       "  3.062,\n",
       "  3.057,\n",
       "  3.05,\n",
       "  3.042,\n",
       "  3.038,\n",
       "  3.034,\n",
       "  3.025,\n",
       "  3.008,\n",
       "  3.001,\n",
       "  2.983,\n",
       "  2.965,\n",
       "  2.955,\n",
       "  2.951,\n",
       "  2.95,\n",
       "  2.949,\n",
       "  2.945,\n",
       "  2.937,\n",
       "  2.934,\n",
       "  2.933,\n",
       "  2.933,\n",
       "  2.932,\n",
       "  2.931,\n",
       "  2.931,\n",
       "  2.931,\n",
       "  2.927,\n",
       "  2.914,\n",
       "  2.904,\n",
       "  2.901,\n",
       "  2.9,\n",
       "  2.9,\n",
       "  2.9,\n",
       "  2.899,\n",
       "  2.899,\n",
       "  2.898,\n",
       "  2.898,\n",
       "  2.897,\n",
       "  2.893,\n",
       "  2.893,\n",
       "  2.89,\n",
       "  2.883,\n",
       "  2.878,\n",
       "  2.874,\n",
       "  2.872,\n",
       "  2.868,\n",
       "  2.864,\n",
       "  2.862,\n",
       "  2.861,\n",
       "  2.86,\n",
       "  2.859,\n",
       "  2.857,\n",
       "  2.851,\n",
       "  2.847,\n",
       "  2.846,\n",
       "  2.845,\n",
       "  2.841,\n",
       "  2.838,\n",
       "  2.836,\n",
       "  2.832,\n",
       "  2.821,\n",
       "  2.815,\n",
       "  2.796,\n",
       "  2.787,\n",
       "  2.786,\n",
       "  2.785,\n",
       "  2.785,\n",
       "  2.785,\n",
       "  2.785,\n",
       "  2.783,\n",
       "  2.778,\n",
       "  2.769,\n",
       "  2.763,\n",
       "  2.754,\n",
       "  2.747,\n",
       "  2.741,\n",
       "  2.738,\n",
       "  2.738,\n",
       "  2.738,\n",
       "  2.736,\n",
       "  2.734,\n",
       "  2.734,\n",
       "  2.734,\n",
       "  2.733,\n",
       "  2.724,\n",
       "  2.719,\n",
       "  2.718,\n",
       "  2.717,\n",
       "  2.705,\n",
       "  2.7,\n",
       "  2.698,\n",
       "  2.695,\n",
       "  2.692,\n",
       "  2.688,\n",
       "  2.683,\n",
       "  2.68,\n",
       "  2.679,\n",
       "  2.677,\n",
       "  2.667,\n",
       "  2.655,\n",
       "  2.645,\n",
       "  2.64,\n",
       "  2.635,\n",
       "  2.629,\n",
       "  2.626,\n",
       "  2.625,\n",
       "  2.625,\n",
       "  2.624,\n",
       "  2.623,\n",
       "  2.621,\n",
       "  2.615,\n",
       "  2.611,\n",
       "  2.611,\n",
       "  2.61,\n",
       "  2.609,\n",
       "  2.608,\n",
       "  2.607,\n",
       "  2.606,\n",
       "  2.603,\n",
       "  2.602,\n",
       "  2.601,\n",
       "  2.598,\n",
       "  2.597,\n",
       "  2.595,\n",
       "  2.592,\n",
       "  2.591,\n",
       "  2.59,\n",
       "  2.589,\n",
       "  2.584,\n",
       "  2.576,\n",
       "  2.57,\n",
       "  2.565,\n",
       "  2.558,\n",
       "  2.556,\n",
       "  2.555,\n",
       "  2.553,\n",
       "  2.55,\n",
       "  2.546,\n",
       "  2.542,\n",
       "  2.54,\n",
       "  2.538,\n",
       "  2.537,\n",
       "  2.536,\n",
       "  2.536,\n",
       "  2.536,\n",
       "  2.535,\n",
       "  2.534,\n",
       "  2.533,\n",
       "  2.519,\n",
       "  2.503,\n",
       "  2.5,\n",
       "  2.5,\n",
       "  2.5,\n",
       "  2.499,\n",
       "  2.498,\n",
       "  2.498,\n",
       "  2.498,\n",
       "  2.498,\n",
       "  2.497,\n",
       "  2.497,\n",
       "  2.496,\n",
       "  2.494,\n",
       "  2.49,\n",
       "  2.483,\n",
       "  2.475,\n",
       "  2.47,\n",
       "  2.467,\n",
       "  2.461,\n",
       "  2.459,\n",
       "  2.458,\n",
       "  2.457,\n",
       "  2.456,\n",
       "  2.455,\n",
       "  2.455,\n",
       "  2.453,\n",
       "  2.452,\n",
       "  2.452,\n",
       "  2.451,\n",
       "  2.45,\n",
       "  2.448,\n",
       "  2.447,\n",
       "  2.447,\n",
       "  2.443,\n",
       "  2.438,\n",
       "  2.436,\n",
       "  2.434,\n",
       "  2.432,\n",
       "  2.43,\n",
       "  2.418,\n",
       "  2.412,\n",
       "  2.41,\n",
       "  2.409,\n",
       "  2.409,\n",
       "  2.409,\n",
       "  2.409,\n",
       "  2.408,\n",
       "  2.394,\n",
       "  2.379,\n",
       "  2.37,\n",
       "  2.368,\n",
       "  2.363,\n",
       "  2.359,\n",
       "  2.359,\n",
       "  2.357,\n",
       "  2.357,\n",
       "  2.354,\n",
       "  2.351,\n",
       "  2.349,\n",
       "  2.346,\n",
       "  2.343,\n",
       "  2.341,\n",
       "  2.34,\n",
       "  2.336,\n",
       "  2.335,\n",
       "  2.331,\n",
       "  2.329,\n",
       "  2.327,\n",
       "  2.325,\n",
       "  2.322,\n",
       "  2.322,\n",
       "  2.322,\n",
       "  2.32,\n",
       "  2.32]]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contents_rdd.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T02:32:41.825327Z",
     "start_time": "2020-01-15T02:32:40.556410Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|            features|index|\n",
      "+--------------------+-----+\n",
      "|[28.567,28.566,28...|    1|\n",
      "+--------------------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "schema = StructType([\n",
    "    StructField(\"num\",ArrayType(DoubleType()),True)])\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.sql import Window\n",
    "\n",
    "#contents_rdd_vec = contents_rdd.map(lambda x:Vectors.dense(x))\n",
    "\n",
    "contents_df = ss.createDataFrame(contents_rdd)\n",
    "assemb = VectorAssembler(outputCol=\"features\", inputCols=contents_df.columns)\n",
    "\n",
    "vec = assemb.transform(contents_df)\n",
    "vec_indexed = vec.select('features',(rank().over(Window.partitionBy().orderBy('features')).alias('index')))\n",
    "vec_indexed.show()\n",
    "\n",
    "\n",
    "# contents_df.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-15T02:25:20.651516Z",
     "start_time": "2020-01-15T02:25:20.642519Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[_1: double, _2: double, _3: double, _4: double, _5: double, _6: double, _7: double, _8: double, _9: double, _10: double, _11: double, _12: double, _13: double, _14: double, _15: double, _16: double, _17: double, _18: double, _19: double, _20: double, _21: double, _22: double, _23: double, _24: double, _25: double, _26: double, _27: double, _28: double, _29: double, _30: double, _31: double, _32: double, _33: double, _34: double, _35: double, _36: double, _37: double, _38: double, _39: double, _40: double, _41: double, _42: double, _43: double, _44: double, _45: double, _46: double, _47: double, _48: double, _49: double, _50: double, _51: double, _52: double, _53: double, _54: double, _55: double, _56: double, _57: double, _58: double, _59: double, _60: double, _61: double, _62: double, _63: double, _64: double, _65: double, _66: double, _67: double, _68: double, _69: double, _70: double, _71: double, _72: double, _73: double, _74: double, _75: double, _76: double, _77: double, _78: double, _79: double, _80: double, _81: double, _82: double, _83: double, _84: double, _85: double, _86: double, _87: double, _88: double, _89: double, _90: double, _91: double, _92: double, _93: double, _94: double, _95: double, _96: double, _97: double, _98: double, _99: double, _100: double, _101: double, _102: double, _103: double, _104: double, _105: double, _106: double, _107: double, _108: double, _109: double, _110: double, _111: double, _112: double, _113: double, _114: double, _115: double, _116: double, _117: double, _118: double, _119: double, _120: double, _121: double, _122: double, _123: double, _124: double, _125: double, _126: double, _127: double, _128: double, _129: double, _130: double, _131: double, _132: double, _133: double, _134: double, _135: double, _136: double, _137: double, _138: double, _139: double, _140: double, _141: double, _142: double, _143: double, _144: double, _145: double, _146: double, _147: double, _148: double, _149: double, _150: double, _151: double, _152: double, _153: double, _154: double, _155: double, _156: double, _157: double, _158: double, _159: double, _160: double, _161: double, _162: double, _163: double, _164: double, _165: double, _166: double, _167: double, _168: double, _169: double, _170: double, _171: double, _172: double, _173: double, _174: double, _175: double, _176: double, _177: double, _178: double, _179: double, _180: double, _181: double, _182: double, _183: double, _184: double, _185: double, _186: double, _187: double, _188: double, _189: double, _190: double, _191: double, _192: double, _193: double, _194: double, _195: double, _196: double, _197: double, _198: double, _199: double, _200: double, _201: double, _202: double, _203: double, _204: double, _205: double, _206: double, _207: double, _208: double, _209: double, _210: double, _211: double, _212: double, _213: double, _214: double, _215: double, _216: double, _217: double, _218: double, _219: double, _220: double, _221: double, _222: double, _223: double, _224: double, _225: double, _226: double, _227: double, _228: double, _229: double, _230: double, _231: double, _232: double, _233: double, _234: double, _235: double, _236: double, _237: double, _238: double, _239: double, _240: double, _241: double, _242: double, _243: double, _244: double, _245: double, _246: double, _247: double, _248: double, _249: double, _250: double, _251: double, _252: double, _253: double, _254: double, _255: double, _256: double, _257: double, _258: double, _259: double, _260: double, _261: double, _262: double, _263: double, _264: double, _265: double, _266: double, _267: double, _268: double, _269: double, _270: double, _271: double, _272: double, _273: double, _274: double, _275: double, _276: double, _277: double, _278: double, _279: double, _280: double, _281: double, _282: double, _283: double, _284: double, _285: double, _286: double, _287: double, _288: double, _289: double, _290: double, _291: double, _292: double, _293: double, _294: double, _295: double, _296: double, _297: double, _298: double, _299: double, _300: double, _301: double, _302: double, _303: double, _304: double, _305: double, _306: double, _307: double, _308: double, _309: double, _310: double, _311: double, _312: double, _313: double, _314: double, _315: double, _316: double, _317: double, _318: double, _319: double, _320: double, _321: double, _322: double, _323: double, _324: double, _325: double, _326: double, _327: double, _328: double, _329: double, _330: double, _331: double, _332: double, _333: double, _334: double, _335: double, _336: double, _337: double, _338: double, _339: double, _340: double, _341: double, _342: double, _343: double, _344: double, _345: double, _346: double, _347: double, _348: double, _349: double, _350: double, _351: double, _352: double, _353: double, _354: double, _355: double, _356: double, _357: double, _358: double, _359: double, _360: double, _361: double, _362: double, _363: double, _364: double, _365: double, _366: double, _367: double, _368: double, _369: double, _370: double, _371: double, _372: double, _373: double, _374: double, _375: double, _376: double, _377: double, _378: double, _379: double, _380: double, _381: double, _382: double, _383: double, _384: double, _385: double, _386: double, _387: double, _388: double, _389: double, _390: double, _391: double, _392: double, _393: double, _394: double, _395: double, _396: double, _397: double, _398: double, _399: double, _400: double, _401: double, _402: double, _403: double, _404: double, _405: double, _406: double, _407: double, _408: double, _409: double, _410: double, _411: double, _412: double, _413: double, _414: double, _415: double, _416: double, _417: double, _418: double, _419: double, _420: double, _421: double, _422: double, _423: double, _424: double, _425: double, _426: double, _427: double, _428: double, _429: double, _430: double, _431: double, _432: double, _433: double, _434: double, _435: double, _436: double, _437: double, _438: double, _439: double, _440: double, _441: double, _442: double, _443: double, _444: double, _445: double, _446: double, _447: double, _448: double, _449: double, _450: double, _451: double, _452: double, _453: double, _454: double, _455: double, _456: double, _457: double, _458: double, _459: double, _460: double, _461: double, _462: double, _463: double, _464: double, _465: double, _466: double, _467: double, _468: double, _469: double, _470: double, _471: double, _472: double, _473: double, _474: double, _475: double, _476: double, _477: double, _478: double, _479: double, _480: double, _481: double, _482: double, _483: double, _484: double, _485: double, _486: double, _487: double, _488: double, _489: double, _490: double, _491: double, _492: double, _493: double, _494: double, _495: double, _496: double, _497: double, _498: double, _499: double, _500: double, _501: double, _502: double, _503: double, _504: double, _505: double, _506: double, _507: double, _508: double, _509: double, _510: double, _511: double, _512: double, _513: double, _514: double, _515: double, _516: double, _517: double, _518: double, _519: double, _520: double, _521: double, _522: double, _523: double, _524: double, _525: double, _526: double, _527: double, _528: double, _529: double, _530: double, _531: double, _532: double, _533: double, _534: double, _535: double, _536: double, _537: double, _538: double, _539: double, _540: double, _541: double, _542: double, _543: double, _544: double, _545: double, _546: double, _547: double, _548: double, _549: double, _550: double, _551: double, _552: double, _553: double, _554: double, _555: double, _556: double, _557: double, _558: double, _559: double, _560: double, _561: double, _562: double, _563: double, _564: double, _565: double, _566: double, _567: double, _568: double, _569: double, _570: double, _571: double, _572: double, _573: double, _574: double, _575: double, _576: double, _577: double, _578: double, _579: double, _580: double, _581: double, _582: double, _583: double, _584: double, _585: double, _586: double, _587: double, _588: double, _589: double, _590: double, _591: double, _592: double, _593: double, _594: double, _595: double, _596: double, _597: double, _598: double, _599: double, _600: double, _601: double, _602: double, _603: double, _604: double, _605: double, _606: double, _607: double, _608: double, _609: double, _610: double, _611: double, _612: double, _613: double, _614: double, _615: double, _616: double, _617: double, _618: double, _619: double, _620: double, _621: double, _622: double, _623: double, _624: double, _625: double, _626: double, _627: double, _628: double, _629: double, _630: double, _631: double, _632: double, _633: double, _634: double, _635: double, _636: double, _637: double, _638: double, _639: double, _640: double, _641: double, _642: double, _643: double, _644: double, _645: double, _646: double, _647: double, _648: double, _649: double, _650: double, _651: double, _652: double, _653: double, _654: double, _655: double, _656: double, _657: double, _658: double, _659: double, _660: double, _661: double, _662: double, _663: double, _664: double, _665: double, _666: double, _667: double, _668: double, _669: double, _670: double, _671: double, _672: double, _673: double, _674: double, _675: double, _676: double, _677: double, _678: double, _679: double, _680: double, _681: double, _682: double, _683: double, _684: double, _685: double, _686: double, _687: double, _688: double, _689: double, _690: double, _691: double, _692: double, _693: double, _694: double, _695: double, _696: double, _697: double, _698: double, _699: double, _700: double, _701: double, _702: double, _703: double, _704: double, _705: double, _706: double, _707: double, _708: double, _709: double, _710: double, _711: double, _712: double, _713: double, _714: double, _715: double, _716: double, _717: double, _718: double, _719: double, _720: double, _721: double, _722: double, _723: double, _724: double, _725: double, _726: double, _727: double, _728: double, _729: double, _730: double, _731: double, _732: double, _733: double, _734: double, _735: double, _736: double, _737: double, _738: double, _739: double, _740: double, _741: double, _742: double, _743: double, _744: double, _745: double, _746: double, _747: double, _748: double, _749: double, _750: double, _751: double, _752: double, _753: double, _754: double, _755: double, _756: double, _757: double, _758: double, _759: double, _760: double, _761: double, _762: double, _763: double, _764: double, _765: double, _766: double, _767: double, _768: double, _769: double, _770: double, _771: double, _772: double, _773: double, _774: double, _775: double, _776: double, _777: double, _778: double, _779: double, _780: double, _781: double, _782: double, _783: double, _784: double, _785: double, _786: double, _787: double, _788: double, _789: double, _790: double, _791: double, _792: double, _793: double, _794: double, _795: double, _796: double, _797: double, _798: double, _799: double, _800: double, _801: double, _802: double, _803: double, _804: double, _805: double, _806: double, _807: double, _808: double, _809: double, _810: double, _811: double, _812: double, _813: double, _814: double, _815: double, _816: double, _817: double, _818: double, _819: double, _820: double, _821: double, _822: double, _823: double, _824: double, _825: double, _826: double, _827: double, _828: double, _829: double, _830: double, _831: double, _832: double, _833: double, _834: double, _835: double, _836: double, _837: double, _838: double, _839: double, _840: double, _841: double, _842: double, _843: double, _844: double, _845: double, _846: double, _847: double, _848: double, _849: double, _850: double, _851: double, _852: double, _853: double, _854: double, _855: double, _856: double, _857: double, _858: double, _859: double, _860: double, _861: double, _862: double, _863: double, _864: double, _865: double, _866: double, _867: double, _868: double, _869: double, _870: double, _871: double, _872: double, _873: double, _874: double, _875: double, _876: double, _877: double, _878: double, _879: double, _880: double, _881: double, _882: double, _883: double, _884: double, _885: double, _886: double, _887: double, _888: double, _889: double, _890: double, _891: double, _892: double, _893: double, _894: double, _895: double, _896: double, _897: double, _898: double, _899: double, _900: double, _901: double, _902: double, _903: double, _904: double, _905: double, _906: double, _907: double, _908: double, _909: double, _910: double, _911: double, _912: double, _913: double, _914: double, _915: double, _916: double, _917: double, _918: double, _919: double, _920: double, _921: double, _922: double, _923: double, _924: double, _925: double, _926: double, _927: double, _928: double, _929: double, _930: double, _931: double, _932: double, _933: double, _934: double, _935: double, _936: double, _937: double, _938: double, _939: double, _940: double, _941: double, _942: double, _943: double, _944: double, _945: double, _946: double, _947: double, _948: double, _949: double, _950: double, _951: double, _952: double, _953: double, _954: double, _955: double, _956: double, _957: double, _958: double, _959: double, _960: double, _961: double, _962: double, _963: double, _964: double, _965: double, _966: double, _967: double, _968: double, _969: double, _970: double, _971: double, _972: double, _973: double, _974: double, _975: double, _976: double, _977: double, _978: double, _979: double, _980: double, _981: double, _982: double, _983: double, _984: double, _985: double, _986: double, _987: double, _988: double, _989: double, _990: double, _991: double, _992: double, _993: double, _994: double, _995: double, _996: double, _997: double, _998: double, _999: double, _1000: double]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contents_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}