Fix code coverage (HumanCompatibleAI#15)

* Fix codecov in tests

* Bugfix: don't write to .monitor.csv in CWD

* Unbreak non-coverage tests

* Merge imitation/scripts.py and scripts/run_training.py into scripts/train.py

* Move tabular tests to tests directory

* Move scripts into package

* Update docs

* Try with flags

* Fix up codecov config

* TEMPORARY HACK: is CodeCov confused by nested paths?

* Revert "TEMPORARY HACK: is CodeCov confused by nested paths?"

This reverts commit 4392027.

* Try codecov without flags

* TEMPORARY COMMIT: introduce some untested code

* TEMPORARY: Add some untested code

* Remove dummy function

* Fix lint

* README: Add CodeCov info

* README: fix typo

.codecov.yml

@@ -0,0 +1,16 @@
+coverage:
+  status:
+    project:
+      default: false
+      main:
+        target: 80%
+        paths:
+          - "imitation/"
+          - "!imitation/examples/"
+          - "!imitation/scripts/"
+        flags: main
+      auxiliary:
+        target: 0%
+        paths:
+          - "imitation/examples/"
+          - "imitation/scripts/"

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+include=imitation/*
+
+[paths]
+imitation_source =
+	imitation/
+	*/lib/python*/site-packages/imitation

.travis.yml

@@ -9,7 +9,7 @@ install:
   - pip install -r requirements.txt
   - pip install .
 script:
-  - pytest --expensive imitation/
+  - pytest --expensive tests/
 matrix:
   include:
   - name: "Linting"
@@ -24,8 +24,10 @@ matrix:
   - name: "3.7 Coverage Tests"
     python: "3.7"
     script:
-      - py.test --cov-report=xml --cov=imitation --expensive imitation/
+      - py.test --cov-report=xml --cov=${VIRTUAL_ENV}/lib/python3.7/site-packages/imitation --expensive tests/
     after_success:
+      - mv .coverage .coverage.imitation
+      - coverage combine  # rewrite paths
       - codecov
 
   - name: "3.6 Unit Tests"

README.md

@@ -19,14 +19,17 @@ pip install -e .  # install `imitation` in developer mode
 To run:
 ```
 # train demos with normal AIRL
-python scripts/data_collect.py --gin_config configs/cartpole_data_collect.gin
+python -m imitation.scripts.data_collect --gin_config configs/cartpole_data_collect.gin
 # do AIRL magic to get back reward from demos
-python scripts/run_training.py --gin_config configs/cartpole_orig_airl_repro.gin
+python -m imitation.scripts.train --gin_config configs/cartpole_orig_airl_repro.gin
 ```
 
 # Contributing
   * Follow the [Google Python Style Guide](http://google.github.io/styleguide/pyguide.html). Examples of Google-style
 docstrings can be found [here](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html).
   * PEP8 guidelines and line-width=80 are enforced by `ci/code_checks.sh`, which is automatically run by Travis CI.
   * Add units tests covering any new features, or bugs that are being fixed.
-  * Pull requests should pass Travis CI tests before they are merged.
+  * Code coverage is automatically enforced by CodeCov.
+    The exact coverage required by CodeCov depends on the previous
+    code coverage %. Files in `imitation/{examples,scripts}/` have no
+    coverage requirements.

ci/code_checks.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-SOURCE_DIRS="imitation scripts"
+SOURCE_DIRS="imitation"
 
 RET=0
 

imitation/scripts.py → imitation/scripts/train.py

@@ -1,10 +1,9 @@
-"""
-Random experiments. As this file expands, I will probably move individual
-experiments into an scripts/ directory.
+"""Train an IRL algorithm and plot its output.
 
-These scripts are meant to be run in a Jupyter notebook (displays figures)
-but also automatically save timestamped figures to the output/ directory.
-"""
+Can be used as a CLI script, or the `train_and_plot` function can be called
+directly."""
+
+import argparse
 from collections import defaultdict
 import datetime
 import os
@@ -171,3 +170,20 @@ def _savefig_timestamp(prefix="", also_show=True):
     tf.logging.info("plot saved to {}".format(path))
     if also_show:
         plt.show()
+
+
+def main():
+    tf.logging.set_verbosity(tf.logging.INFO)
+
+    train_and_plot()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--gin_config",
+                        default='configs/cartpole_orig_airl_repro.gin')
+    args = parser.parse_args()
+
+    gin.parse_config_file(args.gin_config)
+
+    main()

imitation/util/util.py

@@ -43,7 +43,7 @@ def make_vec_env(env_id, n_envs=8):
     """
     # Use Monitor to support logging the episode reward and length.
     def monitored_env():
-        return Monitor(gym.make(env_id), '', allow_early_resets=True)
+        return Monitor(gym.make(env_id), None, allow_early_resets=True)
     return DummyVecEnv([monitored_env for _ in range(n_envs)])
 
 

notebooks/plot_training.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import imitation.scripts as scripts\n",
+    "from imitation.scripts.train import train_and_plot\n",
     "import tensorflow as tf\n",
     "import gin.tf\n",
     "import os\n",
@@ -28,7 +28,7 @@
     "\n",
     "gin.parse_config_file(\"configs/cartpole_gail.gin\")\n",
     "\n",
-    "scripts.train_and_plot(n_plots_each_per_epoch=1, n_gen_plot_episodes=1)"
+    "train_and_plot(n_plots_each_per_epoch=1, n_gen_plot_episodes=1)"
    ]
   }
  ],