7.2_Skateboarding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7.2 Reinforcement Learning - Skateboarding\n",
    "\n",
    "Module - Reinforcement Learning\n",
    "\n",
    "For book, references and training materials, please check this project website [http://activefitness.ai/ai-in-sports-with-python](http://activefitness.ai/ai-in-sports-with-python).\n",
    "\n",
    "Reference: Chapter 10, [Applied Machine Learning for Health and Fitness](https://www.apress.com/us/book/9781484257715)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "Creating environment from the given name, wrapped in a DummyVecEnv.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\tf_util.py:58: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\tf_util.py:67: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\policies.py:115: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\input.py:25: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\policies.py:560: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.flatten instead.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\tensorflow_core\\python\\layers\\core.py:332: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\a2c\\utils.py:156: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\distributions.py:418: The name tf.random_normal is deprecated. Please use tf.random.normal instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\a2c\\a2c.py:143: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\tf_util.py:313: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\tf_util.py:313: The name tf.GraphKeys is deprecated. Please use tf.compat.v1.GraphKeys instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\tensorflow_core\\python\\ops\\clip_ops.py:301: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\a2c\\a2c.py:167: The name tf.train.RMSPropOptimizer is deprecated. Please use tf.compat.v1.train.RMSPropOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\tensorflow_core\\python\\training\\rmsprop.py:119: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\a2c\\a2c.py:177: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\a2c\\a2c.py:179: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\common\\base_class.py:1082: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
      "\n",
      "---------------------------------\n",
      "| explained_variance | -0.00917 |\n",
      "| fps                | 3        |\n",
      "| nupdates           | 1        |\n",
      "| policy_entropy     | 1.42     |\n",
      "| total_timesteps    | 5        |\n",
      "| value_loss         | 0.0137   |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0683  |\n",
      "| fps                | 127      |\n",
      "| nupdates           | 100      |\n",
      "| policy_entropy     | 1.42     |\n",
      "| total_timesteps    | 500      |\n",
      "| value_loss         | 0.0576   |\n",
      "---------------------------------\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\envs\\tf11\\lib\\site-packages\\stable_baselines\\a2c\\utils.py:581: The name tf.Summary is deprecated. Please use tf.compat.v1.Summary instead.\n",
      "\n",
      "---------------------------------\n",
      "| explained_variance | 0.0663   |\n",
      "| fps                | 165      |\n",
      "| nupdates           | 200      |\n",
      "| policy_entropy     | 1.41     |\n",
      "| total_timesteps    | 1000     |\n",
      "| value_loss         | 0.00549  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.159    |\n",
      "| fps                | 177      |\n",
      "| nupdates           | 300      |\n",
      "| policy_entropy     | 1.4      |\n",
      "| total_timesteps    | 1500     |\n",
      "| value_loss         | 0.00859  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0807   |\n",
      "| fps                | 183      |\n",
      "| nupdates           | 400      |\n",
      "| policy_entropy     | 1.39     |\n",
      "| total_timesteps    | 2000     |\n",
      "| value_loss         | 0.0154   |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0296  |\n",
      "| fps                | 186      |\n",
      "| nupdates           | 500      |\n",
      "| policy_entropy     | 1.37     |\n",
      "| total_timesteps    | 2500     |\n",
      "| value_loss         | 0.00509  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.129    |\n",
      "| fps                | 186      |\n",
      "| nupdates           | 600      |\n",
      "| policy_entropy     | 1.36     |\n",
      "| total_timesteps    | 3000     |\n",
      "| value_loss         | 0.267    |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.229    |\n",
      "| fps                | 189      |\n",
      "| nupdates           | 700      |\n",
      "| policy_entropy     | 1.33     |\n",
      "| total_timesteps    | 3500     |\n",
      "| value_loss         | 0.00238  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0779   |\n",
      "| fps                | 193      |\n",
      "| nupdates           | 800      |\n",
      "| policy_entropy     | 1.32     |\n",
      "| total_timesteps    | 4000     |\n",
      "| value_loss         | 1.14     |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -19      |\n",
      "| fps                | 196      |\n",
      "| nupdates           | 900      |\n",
      "| policy_entropy     | 1.29     |\n",
      "| total_timesteps    | 4500     |\n",
      "| value_loss         | 0.0315   |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.237   |\n",
      "| fps                | 197      |\n",
      "| nupdates           | 1000     |\n",
      "| policy_entropy     | 1.28     |\n",
      "| total_timesteps    | 5000     |\n",
      "| value_loss         | 0.00509  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0337  |\n",
      "| fps                | 195      |\n",
      "| nupdates           | 1100     |\n",
      "| policy_entropy     | 1.25     |\n",
      "| total_timesteps    | 5500     |\n",
      "| value_loss         | 0.00178  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0226  |\n",
      "| fps                | 196      |\n",
      "| nupdates           | 1200     |\n",
      "| policy_entropy     | 1.22     |\n",
      "| total_timesteps    | 6000     |\n",
      "| value_loss         | 0.00391  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -3.86    |\n",
      "| fps                | 201      |\n",
      "| nupdates           | 1300     |\n",
      "| policy_entropy     | 1.18     |\n",
      "| total_timesteps    | 6500     |\n",
      "| value_loss         | 0.00866  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0947   |\n",
      "| fps                | 203      |\n",
      "| nupdates           | 1400     |\n",
      "| policy_entropy     | 1.14     |\n",
      "| total_timesteps    | 7000     |\n",
      "| value_loss         | 0.002    |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.94    |\n",
      "| fps                | 206      |\n",
      "| nupdates           | 1500     |\n",
      "| policy_entropy     | 1.12     |\n",
      "| total_timesteps    | 7500     |\n",
      "| value_loss         | 0.00178  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0341   |\n",
      "| fps                | 209      |\n",
      "| nupdates           | 1600     |\n",
      "| policy_entropy     | 1.08     |\n",
      "| total_timesteps    | 8000     |\n",
      "| value_loss         | 0.00586  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -1.72    |\n",
      "| fps                | 211      |\n",
      "| nupdates           | 1700     |\n",
      "| policy_entropy     | 1.04     |\n",
      "| total_timesteps    | 8500     |\n",
      "| value_loss         | 0.000698 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0206  |\n",
      "| fps                | 213      |\n",
      "| nupdates           | 1800     |\n",
      "| policy_entropy     | 1.01     |\n",
      "| total_timesteps    | 9000     |\n",
      "| value_loss         | 0.0066   |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.8     |\n",
      "| fps                | 212      |\n",
      "| nupdates           | 1900     |\n",
      "| policy_entropy     | 0.969    |\n",
      "| total_timesteps    | 9500     |\n",
      "| value_loss         | 0.000911 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.352    |\n",
      "| fps                | 212      |\n",
      "| nupdates           | 2000     |\n",
      "| policy_entropy     | 0.933    |\n",
      "| total_timesteps    | 10000    |\n",
      "| value_loss         | 0.00401  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.506    |\n",
      "| fps                | 208      |\n",
      "| nupdates           | 2100     |\n",
      "| policy_entropy     | 0.903    |\n",
      "| total_timesteps    | 10500    |\n",
      "| value_loss         | 0.000374 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0875   |\n",
      "| fps                | 205      |\n",
      "| nupdates           | 2200     |\n",
      "| policy_entropy     | 0.866    |\n",
      "| total_timesteps    | 11000    |\n",
      "| value_loss         | 0.0201   |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.305    |\n",
      "| fps                | 203      |\n",
      "| nupdates           | 2300     |\n",
      "| policy_entropy     | 0.834    |\n",
      "| total_timesteps    | 11500    |\n",
      "| value_loss         | 0.000819 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0586   |\n",
      "| fps                | 204      |\n",
      "| nupdates           | 2400     |\n",
      "| policy_entropy     | 0.788    |\n",
      "| total_timesteps    | 12000    |\n",
      "| value_loss         | 0.000677 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -1.23    |\n",
      "| fps                | 203      |\n",
      "| nupdates           | 2500     |\n",
      "| policy_entropy     | 0.755    |\n",
      "| total_timesteps    | 12500    |\n",
      "| value_loss         | 0.00568  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0188  |\n",
      "| fps                | 202      |\n",
      "| nupdates           | 2600     |\n",
      "| policy_entropy     | 0.708    |\n",
      "| total_timesteps    | 13000    |\n",
      "| value_loss         | 0.000519 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.317   |\n",
      "| fps                | 200      |\n",
      "| nupdates           | 2700     |\n",
      "| policy_entropy     | 0.674    |\n",
      "| total_timesteps    | 13500    |\n",
      "| value_loss         | 0.000919 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.0324   |\n",
      "| fps                | 196      |\n",
      "| nupdates           | 2800     |\n",
      "| policy_entropy     | 0.642    |\n",
      "| total_timesteps    | 14000    |\n",
      "| value_loss         | 0.000644 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -1.06    |\n",
      "| fps                | 193      |\n",
      "| nupdates           | 2900     |\n",
      "| policy_entropy     | 0.609    |\n",
      "| total_timesteps    | 14500    |\n",
      "| value_loss         | 0.000455 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0756  |\n",
      "| fps                | 191      |\n",
      "| nupdates           | 3000     |\n",
      "| policy_entropy     | 0.567    |\n",
      "| total_timesteps    | 15000    |\n",
      "| value_loss         | 0.000587 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0508  |\n",
      "| fps                | 191      |\n",
      "| nupdates           | 3100     |\n",
      "| policy_entropy     | 0.537    |\n",
      "| total_timesteps    | 15500    |\n",
      "| value_loss         | 0.00156  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.00668  |\n",
      "| fps                | 192      |\n",
      "| nupdates           | 3200     |\n",
      "| policy_entropy     | 0.506    |\n",
      "| total_timesteps    | 16000    |\n",
      "| value_loss         | 0.000143 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.23    |\n",
      "| fps                | 192      |\n",
      "| nupdates           | 3300     |\n",
      "| policy_entropy     | 0.479    |\n",
      "| total_timesteps    | 16500    |\n",
      "| value_loss         | 0.000514 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.241   |\n",
      "| fps                | 189      |\n",
      "| nupdates           | 3400     |\n",
      "| policy_entropy     | 0.456    |\n",
      "| total_timesteps    | 17000    |\n",
      "| value_loss         | 0.000948 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.167   |\n",
      "| fps                | 187      |\n",
      "| nupdates           | 3500     |\n",
      "| policy_entropy     | 0.428    |\n",
      "| total_timesteps    | 17500    |\n",
      "| value_loss         | 0.000182 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.402    |\n",
      "| fps                | 184      |\n",
      "| nupdates           | 3600     |\n",
      "| policy_entropy     | 0.401    |\n",
      "| total_timesteps    | 18000    |\n",
      "| value_loss         | 0.000257 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.463   |\n",
      "| fps                | 181      |\n",
      "| nupdates           | 3700     |\n",
      "| policy_entropy     | 0.371    |\n",
      "| total_timesteps    | 18500    |\n",
      "| value_loss         | 0.000608 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.023   |\n",
      "| fps                | 179      |\n",
      "| nupdates           | 3800     |\n",
      "| policy_entropy     | 0.342    |\n",
      "| total_timesteps    | 19000    |\n",
      "| value_loss         | 8.35e-05 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -5.59    |\n",
      "| fps                | 180      |\n",
      "| nupdates           | 3900     |\n",
      "| policy_entropy     | 0.324    |\n",
      "| total_timesteps    | 19500    |\n",
      "| value_loss         | 0.00152  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.46    |\n",
      "| fps                | 180      |\n",
      "| nupdates           | 4000     |\n",
      "| policy_entropy     | 0.306    |\n",
      "| total_timesteps    | 20000    |\n",
      "| value_loss         | 0.00181  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.0684  |\n",
      "| fps                | 180      |\n",
      "| nupdates           | 4100     |\n",
      "| policy_entropy     | 0.284    |\n",
      "| total_timesteps    | 20500    |\n",
      "| value_loss         | 0.000302 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.761   |\n",
      "| fps                | 181      |\n",
      "| nupdates           | 4200     |\n",
      "| policy_entropy     | 0.259    |\n",
      "| total_timesteps    | 21000    |\n",
      "| value_loss         | 0.000164 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.67    |\n",
      "| fps                | 181      |\n",
      "| nupdates           | 4300     |\n",
      "| policy_entropy     | 0.236    |\n",
      "| total_timesteps    | 21500    |\n",
      "| value_loss         | 0.00218  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.661   |\n",
      "| fps                | 182      |\n",
      "| nupdates           | 4400     |\n",
      "| policy_entropy     | 0.212    |\n",
      "| total_timesteps    | 22000    |\n",
      "| value_loss         | 0.000137 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.341   |\n",
      "| fps                | 183      |\n",
      "| nupdates           | 4500     |\n",
      "| policy_entropy     | 0.201    |\n",
      "| total_timesteps    | 22500    |\n",
      "| value_loss         | 0.000397 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.634   |\n",
      "| fps                | 183      |\n",
      "| nupdates           | 4600     |\n",
      "| policy_entropy     | 0.182    |\n",
      "| total_timesteps    | 23000    |\n",
      "| value_loss         | 0.000511 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | 0.17     |\n",
      "| fps                | 182      |\n",
      "| nupdates           | 4700     |\n",
      "| policy_entropy     | 0.172    |\n",
      "| total_timesteps    | 23500    |\n",
      "| value_loss         | 0.000213 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -7.34    |\n",
      "| fps                | 178      |\n",
      "| nupdates           | 4800     |\n",
      "| policy_entropy     | 0.158    |\n",
      "| total_timesteps    | 24000    |\n",
      "| value_loss         | 0.000177 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -1.57    |\n",
      "| fps                | 175      |\n",
      "| nupdates           | 4900     |\n",
      "| policy_entropy     | 0.143    |\n",
      "| total_timesteps    | 24500    |\n",
      "| value_loss         | 0.00197  |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| explained_variance | -0.234   |\n",
      "| fps                | 171      |\n",
      "| nupdates           | 5000     |\n",
      "| policy_entropy     | 0.128    |\n",
      "| total_timesteps    | 25000    |\n",
      "| value_loss         | 0.000218 |\n",
      "---------------------------------\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "from stable_baselines import A2C\n",
    "\n",
    "model = A2C('MlpPolicy', 'MountainCarContinuous-v0', verbose=1, tensorboard_log=\"logs/a2c_mcc_tensorboard/\")\n",
    "model.learn(total_timesteps=25000)\n",
    "model.save(\"a2c_mcc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6006 (pid 5776), started 0:00:42 ago. (Use '!kill 5776' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-691049453e2d86fe\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-691049453e2d86fe\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          url.port = 6006;\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import datetime\n",
    "# load tensorboard notebook extension\n",
    "%load_ext tensorboard\n",
    "# start monitoring\n",
    "%tensorboard --logdir ./logs/a2c_mcc_tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 2700x2700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "from stable_baselines import A2C\n",
    "from utils.gym import gymplot\n",
    "\n",
    "model = A2C.load(\"a2c_mcc\")\n",
    "\n",
    "env = gym.make('MountainCarContinuous-v0')\n",
    "obs = env.reset()\n",
    "img = gymplot.plot_init(env)\n",
    "while True:\n",
    "    action, _states = model.predict(obs)\n",
    "    obs, rewards, dones, info = env.step(action)\n",
    "    gymplot.plot_next(img, env)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We created a connected humanoid with joints and placed it into a physics environment under forces of gravity, but we didn't teach it the skills to walk, jump, crawl, standup or make any actions that a human learns throughout his life. Let's load a trained model, with some weights, and instead of making random motions let's make it act using predicted movements:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "P3.6 (tf11)",
   "language": "python",
   "name": "tf11"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}