pw_build: Add tools for creating py distributables

Change-Id: Ifac2ad9f647f416b3aef5812a08187a8313d65ed
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/39664
Commit-Queue: Joe Ethier <[email protected]>
Reviewed-by: Keir Mierle <[email protected]>
Reviewed-by: Wyatt Hepler <[email protected]>

docs/python_build.rst

@@ -285,7 +285,7 @@ the generated wheel with `GN metadata
 <https://gn.googlesource.com/gn/+/master/docs/reference.md#var_metadata>`_.
 Wheels for a Python package and its transitive dependencies can be collected
 from the ``pw_python_package_wheels`` key. See
-:ref:`module-pw_build-python-wheels`.
+:ref:`module-pw_build-python-dist`.
 
 Protocol buffers
 ^^^^^^^^^^^^^^^^

pw_build/py/BUILD.gn

@@ -20,6 +20,7 @@ pw_python_package("py") {
   setup = [ "setup.py" ]
   sources = [
     "pw_build/__init__.py",
+    "pw_build/collect_wheels.py",
     "pw_build/copy_from_cipd.py",
     "pw_build/error.py",
     "pw_build/exec.py",

pw_build/py/pw_build/collect_wheels.py

@@ -0,0 +1,65 @@
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Collect Python wheels from a build into a central directory."""
+
+import argparse
+import logging
+from pathlib import Path
+import shutil
+import sys
+
+_LOG = logging.getLogger(__name__)
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        '--prefix',
+        type=Path,
+        help='Root search path to use in conjunction with --wheels_file')
+    parser.add_argument(
+        '--suffix_file',
+        type=argparse.FileType('r'),
+        help=('File that lists subdirs relative to --prefix, one per line,'
+              'to search for .whl files to copy into --out_dir'))
+    parser.add_argument(
+        '--out_dir',
+        type=Path,
+        help='Path where all the built and collected .whl files should be put')
+
+    return parser.parse_args()
+
+
+def copy_wheels(prefix, suffix_file, out_dir):
+    if not out_dir.exists():
+        out_dir.mkdir()
+
+    for suffix in suffix_file.readlines():
+        path = prefix / suffix.strip()
+        _LOG.debug('Searching for wheels in %s', path)
+        if path == out_dir:
+            continue
+        for wheel in path.glob('**/*.whl'):
+            _LOG.debug('Copying %s to %s', wheel, out_dir)
+            shutil.copy(wheel, out_dir)
+
+
+def main():
+    copy_wheels(**vars(_parse_args()))
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    main()
+    sys.exit(0)

pw_build/python.rst

@@ -4,7 +4,7 @@
 Python GN templates
 -------------------
 The Python build is implemented with GN templates defined in
-``pw_build/python.gni``. That file contains the complete usage documentation.
+``pw_build/python.gni``. See the .gni file for complete usage documentation.
 
 .. seealso:: :ref:`docs-python-build`
 
@@ -34,7 +34,8 @@ same as the directory. For example, these two labels are equivalent:
 The actions in a ``pw_python_package`` (e.g. installing packages and running
 Pylint) are done within a single GN toolchain to avoid duplication in
 multi-toolchain builds. This toolchain can be set with the
-``pw_build_PYTHON_TOOLCHAIN`` GN arg, which defaults to a dummy toolchain.
+``pw_build_PYTHON_TOOLCHAIN`` GN arg, which defaults to
+``$dir_pw_build/python_toolchain:python``.
 
 Arguments
 ---------
@@ -100,30 +101,6 @@ This is an example Python package declaration for a ``pw_my_module`` module.
     pylintrc = "$dir_pigweed/.pylintrc"
   }
 
-
-.. _module-pw_build-python-wheels:
-
-Collecting Python wheels for distribution
------------------------------------------
-The ``.wheel`` subtarget generates a wheel (``.whl``) for the Python package.
-Wheels for a package and its transitive dependencies can be collected by
-traversing the ``pw_python_package_wheels`` `GN metadata
-<https://gn.googlesource.com/gn/+/master/docs/reference.md#var_metadata>`_ key,
-which lists the output directory for each wheel.
-
-The ``pw_mirror_tree`` template can be used to collect wheels in an output
-directory:
-
-.. code-block::
-
-  import("$dir_pw_build/mirror_tree.gni")
-
-  pw_mirror_tree("my_wheels") {
-    path_data_keys = [ "pw_python_package_wheels" ]
-    deps = [ ":python_packages.wheel" ]
-    directory = "$root_out_dir/the_wheels"
-  }
-
 pw_python_script
 ================
 A ``pw_python_script`` represents a set of standalone Python scripts and/or
@@ -160,3 +137,87 @@ pw_python_requirements
 ======================
 Represents a set of local and PyPI requirements, with no associated source
 files. These targets serve the role of a ``requirements.txt`` file.
+
+.. _module-pw_build-python-dist:
+
+---------------------
+Python distributables
+---------------------
+Pigweed also provides some templates to make it easier to bundle Python packages
+for deployment. These templates are found in ``pw_build/python_dist.gni``. See
+the .gni file for complete usage doclumentation.
+
+pw_python_wheels
+================
+Collects Python wheels for one or more ``pw_python_package`` targets, plus any
+additional ``pw_python_package`` targets they depend on, directly or indirectly.
+Note that this does not include Python dependencies that come from outside the
+GN build, like packages from PyPI, for example. Those should still be declared
+in the package's ``setup.py`` file as usual.
+
+Arguments
+---------
+- ``packages`` - List of ``pw_python_package`` targets whose wheels should be
+  included; their dependencies will be pulled in as wheels also.
+
+Wheel collection under the hood
+-------------------------------
+The ``.wheel`` subtarget of every ``pw_python_package`` generates a wheel
+(``.whl``) for the Python package. The ``pw_python_wheels`` template figures
+out which wheels to collect by traversing the ``pw_python_package_wheels``
+`GN metadata
+<https://gn.googlesource.com/gn/+/master/docs/reference.md#var_metadata>`_ key,
+which lists the output directory for each wheel.
+
+The ``pw_mirror_tree`` template is then used to collect wheels in an output
+directory:
+
+.. code-block::
+
+  import("$dir_pw_build/mirror_tree.gni")
+
+  pw_mirror_tree("my_wheels") {
+    path_data_keys = [ "pw_python_package_wheels" ]
+    deps = [ ":python_packages.wheel" ]
+    directory = "$root_out_dir/the_wheels"
+  }
+
+pw_python_zip_with_setup
+========================
+Generates a ``.zip`` archive suitable for deployment outside of the project's
+developer environment. The generated ``.zip`` contains Python wheels
+(``.whl`` files) for one or more ``pw_python_package`` targets, plus wheels for
+any additional ``pw_python_package`` targets in the GN build they depend on,
+directly or indirectly. Dependencies from outside the GN build, such as packages
+from PyPI, must be listed in packages' ``setup.py`` files as usual.
+
+The ``.zip`` also includes simple setup scripts for Linux,
+MacOS, and Windows. The setup scripts automatically create a Python virtual
+environment and install the whole collection of wheels into it using ``pip``.
+
+Optionally, additional files and directories can be included in the archive.
+
+Arguments
+---------
+- ``packages`` - A list of `pw_python_package` targets whose wheels should be
+  included; their dependencies will be pulled in as wheels also.
+- ``inputs`` - An optional list of extra files to include in the generated
+  ``.zip``, formatted the same way as the ``inputs`` argument to ``pw_zip``
+  targets.
+- ``dirs`` - An optional list of directories to include in the generated
+  ``.zip``, formatted the same was as the ``dirs`` argument to ``pw_zip``
+  targets.
+
+Example
+-------
+
+.. code-block::
+
+  import("//build_overrides/pigweed.gni")
+
+  import("$dir_pw_build/python_dist.gni")
+
+  pw_python_zip_with_setup("my_tools") {
+    packages = [ ":some_python_package" ]
+    inputs = [ "$dir_pw_build/python_dist/README.md > /${target_name}/" ]
+  }

pw_build/python_dist.gni

@@ -0,0 +1,126 @@
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("//build_overrides/pigweed.gni")
+
+import("$dir_pw_build/python.gni")
+import("$dir_pw_build/python_action.gni")
+import("$dir_pw_build/zip.gni")
+
+# Builds a directory containing a collection of Python wheels.
+#
+# Given one or more pw_python_package targets, this target will build their
+# .wheel sub-targets along with the .wheel sub-targets of all dependencies,
+# direct and indirect, as understood by GN. The resulting .whl files will be
+# collected into a single directory called 'python_wheels'.
+#
+# Args:
+#   packages: A list of pw_python_package targets whose wheels should be
+#       included; their dependencies will be pulled in as wheels also.
+template("pw_python_wheels") {
+  _outer_name = target_name
+  _wheel_paths_path = "${target_gen_dir}/${target_name}_wheel_paths.txt"
+
+  _deps = []
+  if (defined(invoker.deps)) {
+    _deps = invoker.deps
+  }
+
+  _packages = []
+  foreach(_pkg, invoker.packages) {
+    _pkg_name = get_label_info(_pkg, "label_no_toolchain")
+    _pkg_toolchain = get_label_info(_pkg, "toolchain")
+    _packages += [ "${_pkg_name}.wheel(${_pkg_toolchain})" ]
+  }
+
+  # Build a list of relative paths containing all the wheels we depend on.
+  generated_file("${target_name}._wheel_paths") {
+    data_keys = [ "pw_python_package_wheels" ]
+    rebase = root_build_dir
+    deps = _packages
+    outputs = [ _wheel_paths_path ]
+  }
+
+  pw_python_action(target_name) {
+    deps = _deps + [ ":${_outer_name}._wheel_paths" ]
+    module = "pw_build.collect_wheels"
+
+    args = [
+      "--prefix",
+      rebase_path(root_build_dir),
+      "--suffix",
+      rebase_path(_wheel_paths_path),
+      "--out_dir",
+      rebase_path("${target_out_dir}/python_wheels"),
+    ]
+
+    stamp = true
+  }
+}
+
+# Builds a .zip containing Python wheels and setup scripts.
+#
+# The resulting .zip archive will contain a directory with Python wheels for
+# all pw_python_package targets listed in 'packages', plus wheels for any
+# pw_python_package targets those packages depend on, directly or indirectly,
+# as understood by GN.
+#
+# In addition to Python wheels, the resulting .zip will also contain simple
+# setup scripts for Linux, MacOS, and Windows that take care of creating a
+# Python venv and installing all the included wheels into it, and a README.md
+# file with setup and usage instructions.
+#
+# Args:
+#   packages: A list of pw_python_package targets whose wheels should be
+#       included; their dependencies will be pulled in as wheels also.
+#   inputs: An optional list of extra files to include in the generated .zip,
+#       formatted the same was as the 'inputs' argument to pw_zip targets.
+#   dirs: An optional list of directories to include in the generated .zip,
+#       formatted the same way as the 'dirs' argument to pw_zip targets.
+template("pw_python_zip_with_setup") {
+  _outer_name = target_name
+  _zip_path = "${target_out_dir}/${target_name}.zip"
+  _deps = []
+  if (defined(invoker.deps)) {
+    _deps = invoker.deps
+  }
+  _inputs = []
+  if (defined(invoker.inputs)) {
+    _inputs = invoker.inputs
+  }
+  _dirs = []
+  if (defined(invoker.dirs)) {
+    _dirs = invoker.dirs
+  }
+
+  pw_python_wheels("${target_name}.wheels") {
+    packages = invoker.packages
+    deps = _deps
+  }
+
+  pw_zip("${target_name}") {
+    inputs = _inputs + [
+               "$dir_pw_build/python_dist/setup.bat > /${target_name}/",
+               "$dir_pw_build/python_dist/setup.sh > /${target_name}/",
+             ]
+
+    dirs =
+        _dirs +
+        [ "${target_out_dir}/python_wheels/ > /${target_name}/python_wheels/" ]
+
+    output = _zip_path
+
+    deps = [ ":${_outer_name}.wheels" ]
+  }
+}

pw_build/python_dist/README.md

@@ -0,0 +1,33 @@
+# Python Distributables
+Setup and usage instructions for Pigweed Python distributables.
+
+## Prerequisites
+Python distributables require Python 3.7 or later.
+
+## Setup
+Run the included setup script found inside the unzipped directory.
+
+Linux / MacOS:
+```bash
+setup.sh
+```
+
+Windows:
+```
+setup.bat
+```
+
+The setup script will create a virtual environment called `python-venv`.
+
+### Usage
+Once setup is complete, the Python tools can be invoked as runnable modules:
+
+Linux/MacOS:
+```bash
+python-venv/bin/python -m MODULE_NAME [OPTIONS]
+```
+
+Windows:
+```
+python-venv\Scripts\python -m MODULE_NAME [OPTIONS]
+```