Bug 1057293 - Refactored Python requirements.

- moved requirements files into separate files
- updated docs
- start to use peep

.gitignore

@@ -3,7 +3,7 @@
 *.sw?
 .vagrant
 kuma.box
-settings_local.py
+/settings_local.py
 pip-log.txt
 .coverage
 coverage.xml

.gitmodules

@@ -1,175 +1,3 @@
 [submodule "kumascript"]
 	path = kumascript
 	url = git://github.com/mozilla/kumascript.git
-[submodule "vendor/src/basket-client"]
-	path = vendor/src/basket-client
-	url = git://github.com/mozilla/basket-client.git
-[submodule "vendor/src/bleach"]
-	path = vendor/src/bleach
-	url = git://github.com/mozilla/bleach.git
-[submodule "vendor/src/commonware"]
-	path = vendor/src/commonware
-	url = git://github.com/jsocol/commonware.git
-[submodule "vendor/src/django"]
-	path = vendor/src/django
-	url = git://github.com/django/django.git
-[submodule "vendor/src/django-allauth"]
-	path = vendor/src/django-allauth
-	url = git://github.com/pennersr/django-allauth.git
-[submodule "vendor/src/django-appconf"]
-	path = vendor/src/django-appconf
-	url = git://github.com/jezdez/django-appconf.git
-[submodule "vendor/src/django-constance"]
-	path = vendor/src/django-constance
-	url = git://github.com/jezdez/django-constance.git
-[submodule "vendor/src/django-extensions"]
-	path = vendor/src/django-extensions
-	url = git://github.com/django-extensions/django-extensions.git
-[submodule "vendor/src/django-jinja"]
-	path = vendor/src/django-jinja
-	url = https://github.com/niwinz/django-jinja.git
-[submodule "vendor/src/django-memcached-hashring"]
-	path = vendor/src/django-memcached-hashring
-	url = git://github.com/jezdez/django-memcached-hashring.git
-	ignore = untracked
-[submodule "vendor/src/django-nose"]
-	path = vendor/src/django-nose
-	url = git://github.com/django-nose/django-nose.git
-[submodule "vendor/src/django-rest-framework"]
-	path = vendor/src/django-rest-framework
-	url = git://github.com/tomchristie/django-rest-framework.git
-[submodule "vendor/src/django-smuggler"]
-	path = vendor/src/django-smuggler
-	url = git://github.com/semente/django-smuggler.git
-[submodule "vendor/src/django-soapbox"]
-	path = vendor/src/django-soapbox
-	url = git://github.com/ubernostrum/django-soapbox.git
-[submodule "vendor/src/django-statici18n"]
-	path = vendor/src/django-statici18n
-	url = git://github.com/zyegfryed/django-statici18n.git
-[submodule "vendor/src/django-tidings"]
-	path = vendor/src/django-tidings
-	url = git://github.com/mozilla/django-tidings.git
-	ignore = untracked
-[submodule "vendor/src/django-waffle"]
-	path = vendor/src/django-waffle
-	url = git://github.com/jsocol/django-waffle.git
-[submodule "vendor/src/elasticsearch-py"]
-	path = vendor/src/elasticsearch-py
-	url = git://github.com/elasticsearch/elasticsearch-py.git
-[submodule "vendor/src/hash_ring"]
-	path = vendor/src/hash_ring
-	url = git://github.com/Doist/hash_ring.git
-[submodule "vendor/src/oauthlib"]
-	path = vendor/src/oauthlib
-	url = git://github.com/idan/oauthlib.git
-[submodule "vendor/src/python-magic"]
-	path = vendor/src/python-magic
-	url = git://github.com/ahupp/python-magic.git
-	ignore = dirty
-[submodule "vendor/src/python-memcached"]
-	path = vendor/src/python-memcached
-	url = git://github.com/linsomniac/python-memcached.git
-[submodule "vendor/src/python-openid"]
-	path = vendor/src/python-openid
-	url = git://github.com/openid/python-openid.git
-[submodule "vendor/src/pytidylib"]
-	path = vendor/src/pytidylib
-	url = git://github.com/countergram/pytidylib.git
-[submodule "vendor/src/raven-python"]
-	path = vendor/src/raven-python
-	url = git://github.com/getsentry/raven-python.git
-[submodule "vendor/src/requests"]
-	path = vendor/src/requests
-	url = git://github.com/kennethreitz/requests.git
-[submodule "vendor/src/requests-oauthlib"]
-	path = vendor/src/requests-oauthlib
-	url = git://github.com/requests/requests-oauthlib.git
-[submodule "vendor/src/urllib3"]
-	path = vendor/src/urllib3
-	url = git://github.com/shazow/urllib3.git
-[submodule "vendor/src/urlobject"]
-	path = vendor/src/urlobject
-	url = git://github.com/zacharyvoase/urlobject.git
-[submodule "vendor/src/billiard"]
-	path = vendor/src/billiard
-	url = git://github.com/celery/billiard.git
-[submodule "vendor/src/celery"]
-	path = vendor/src/celery
-	url = git://github.com/celery/celery.git
-[submodule "vendor/src/kombu"]
-	path = vendor/src/kombu
-	url = git://github.com/celery/kombu.git
-[submodule "vendor/src/django-celery"]
-	path = vendor/src/django-celery
-	url = git://github.com/celery/django-celery.git
-[submodule "vendor/src/django-honeypot"]
-	path = vendor/src/django-honeypot
-	url = git://github.com/sunlightlabs/django-honeypot.git
-[submodule "vendor/src/bitly-api-python"]
-	path = vendor/src/bitly-api-python
-	url = https://github.com/bitly/bitly-api-python.git
-[submodule "vendor/src/django-cacheback"]
-	path = vendor/src/django-cacheback
-	url = https://github.com/codeinthehole/django-cacheback.git
-	ignore = untracked
-[submodule "vendor/src/django-ratelimit"]
-	path = vendor/src/django-ratelimit
-	url = git://github.com/jsocol/django-ratelimit.git
-[submodule "vendor/src/django-taggit"]
-	path = vendor/src/django-taggit
-	url = git://github.com/alex/django-taggit.git
-[submodule "vendor/src/django-product-details"]
-	path = vendor/src/django-product-details
-	url = git://github.com/mozilla/django-product-details.git
-[submodule "vendor/src/django-debug-toolbar"]
-	path = vendor/src/django-debug-toolbar
-	url = git://github.com/django-debug-toolbar/django-debug-toolbar.git
-[submodule "vendor/src/html5lib"]
-	path = vendor/src/html5lib
-	url = https://github.com/html5lib/html5lib-python
-[submodule "vendor/src/pyquery"]
-	path = vendor/src/pyquery
-	url = https://github.com/gawel/pyquery
-[submodule "vendor/src/py-amqp"]
-	path = vendor/src/py-amqp
-	url = https://github.com/celery/py-amqp
-[submodule "vendor/src/cssselect"]
-	path = vendor/src/cssselect
-	url = https://github.com/SimonSapin/cssselect
-[submodule "vendor/src/python-dateutil"]
-	path = vendor/src/python-dateutil
-	url = https://github.com/dateutil/dateutil
-[submodule "vendor/src/sqlparse"]
-	path = vendor/src/sqlparse
-	url = https://github.com/andialbrecht/sqlparse
-[submodule "vendor/src/werkzeug"]
-	path = vendor/src/werkzeug
-	url = https://github.com/mitsuhiko/werkzeug
-[submodule "vendor/src/django-devserver"]
-	path = vendor/src/django-devserver
-	url = https://github.com/dcramer/django-devserver
-[submodule "vendor/src/elasticsearch-dsl-py"]
-	path = vendor/src/elasticsearch-dsl-py
-	url = https://github.com/elastic/elasticsearch-dsl-py
-[submodule "vendor/src/django-flat-theme"]
-	path = vendor/src/django-flat-theme
-	url = https://github.com/elky/django-flat-theme.git
-[submodule "vendor/src/django-sundial"]
-	path = vendor/src/django-sundial
-	url = https://github.com/charettes/django-sundial
-[submodule "vendor/src/pipeline"]
-	path = vendor/src/pipeline
-	url = https://github.com/cyberdelia/django-pipeline.git
-[submodule "vendor/src/django-celery-transactions"]
-	path = vendor/src/django-celery-transactions
-	url = git://github.com/fellowshipofone/django-celery-transactions.git
-[submodule "vendor/src/puente"]
-	path = vendor/src/puente
-	url = git://github.com/mozilla/puente.git
-[submodule "vendor/src/requests-mock"]
-	path = vendor/src/requests-mock
-	url = https://github.com/openstack/requests-mock.git
-[submodule "vendor/src/whitenoise"]
-	path = vendor/src/whitenoise
-	url = https://github.com/evansd/whitenoise.git

.travis.yml

@@ -1,6 +1,3 @@
-branches:
-    only:
-        - master
 cache:
     apt: true
     directories:

Dockerfile

@@ -17,8 +17,8 @@ RUN apt-get update && \
 WORKDIR /app
 COPY . /app
 
-RUN pip install --build ./build --cache-dir ./cache --no-deps \
-    -r requirements/compiled.txt && \
+RUN python scripts/peep.py --build ./build --cache-dir ./cache --no-deps \
+    -r requirements/compiled.txt -r requirements/default.txt && \
     rm -r build cache
 
 RUN npm install -g \

docs/celery.rst

@@ -59,9 +59,8 @@ Celery
 Installing
 ----------
 
-Celery (and Django-Celery) is part of our :doc:`vendor library <vendor>`. You
-shouldn't need to do any manual installation.
-
+Celery (and Django-Celery) is part of our dependencies. You shouldn't need to
+do any manual installation.
 
 Configuring and Running
 -----------------------

docs/development.rst

@@ -97,30 +97,75 @@ Tests
 Python dependencies
 -------------------
 
-Pure Packages
-~~~~~~~~~~~~~
+Kuma tracks its Python dependencies with pip_ and peep_.
 
-All of the pure Python dependencies are included in the git repository,
-in the ``vendor`` subdirectory. This allows them to be available on the
-Python path without needing to be installed in the system, allowing multiple
-versions for multiple projects simultaneously.
+The ``requirements`` directory contains the plaintext requirements files
+that are used in the Vagrant VM, during automatic tests with Travis-CI
+and duriing deployment to stage and prod.
 
-Compiled Packages
-~~~~~~~~~~~~~~~~~
+Here's what that folder contains:
 
-There are a small number of compiled packages, including the MySQL Python
-client. You can install these using ``pip`` or via a package manager.
-To use ``pip``, you only need to do the following.
+- ``compiled.txt`` - contains dependencies that require a compiler and may
+  need to be treated differently dependending on environment
+
+- ``default.txt`` - contains the default dependencies that are used in all
+  environments
+
+- ``docs.txt`` - contains dependencies that are required to build the docs
+
+- ``tests.txt`` - a file for tests dependencies, both local and automatic
+
+- ``travis.txt`` - a file used by the ``.travis.yml`` config file when
+  running automatic testing
+
+Adding a requirement
+~~~~~~~~~~~~~~~~~~~~
+
+To add a dependency you have to add it to the appropriate requirement file
+in the ``requirements`` folder. To do that we'll use peep_ to get the hash
+of the distribution file you'd like to install.
 
 First SSH into the Vagrant VM::
 
     vagrant ssh
 
-Then disable the virtualenv that is auto-enabled and install the compiled
-dependencies::
+Add the requirement with the exact version specifier to the requirements
+file most appropriate to the use of the dependency, e.g.
+``requirements/default.txt``::
+
+    django-pipeline==1.6.0
+
+Then download a distribution file from PyPI_ or whatever source you deem
+safe of the dependency you added above, e.g.::
+
+    wget https://pypi.python.org/packages/source/d/django-pipeline/django-pipeline-1.6.0.tar.gz
+
+Check if the file you downloaded contains what you expect and then use peep
+to calculate a hash of the file you downloaded::
+
+    script/peep.py django-pipeline-1.6.0.tar.gz
+
+This will print out a hash in the form of::
+
+    # sha256: paFCZIUSX_kQWjcNx9em6npTILXRgCcjA9QppD-BL-U
+
+Add this string above the line of the requirement string in the requirements
+file, e.g.::
+
+    # sha256: paFCZIUSX_kQWjcNx9em6npTILXRgCcjA9QppD-BL-U
+    django-pipeline==1.6.0
+
+Then verify if the hash stil matches and install the new dependency in the VM::
+
+    script/peep.py install -r requirements/default.txt
+
+Updating a requirement
+~~~~~~~~~~~~~~~~~~~~~~
+
+Follow the same steps as when adding a requirement but replace the old peep
+hash in the requirements file. Don't forget to run afterwards::
 
-    deactivate
-    sudo pip install -r requirements/compiled.txt
+    script/peep.py install -r requirements/default.txt
 
 Front-end dependencies
 ----------------------
@@ -276,3 +321,5 @@ set your ``settings_local.py`` with the following::
 
     CSRF_COOKIE_SECURE = False
 
+.. _pip: https://pip.pypa.io/
+.. _peep: https://pypi.python.org/pypi/peep

docs/index.rst

@@ -13,10 +13,8 @@ Contents:
    tests-performance
    troubleshooting
 
-   vendor
    celery
    email
    elasticsearch
    localization
    ckeditor
-