Stop installing setuptools and wheel (#243)

Currently the buildpack performs a system site-packages install of not only pip, but also setuptools and wheel. This has historically been necessary for pip to be able to build source distributions (sdists) for packages that don't ship with compatible wheels. However: - Thanks to PEP 518, packages can now (and many already do) specify an explicit build backend using `[build-system]` in their `pyproject.toml`. The dependencies specified in that config (such as setuptools and wheel) will be installed by pip into an isolated and ephemeral build environment as part of the source distribution build process. Such packages therefore don't need/use globally installed setuptools/wheel versions. - As of pip v22.1, pip will now default to the isolated build environment mode (along with a fallback legacy setuptools build backend), if the setuptools package isn't installed globally. This means that packages that haven't yet migrated to a PEP 518 `pyproject.toml` build backend config can still build even if setuptools isn't installed globally. There are a small number of rarely used packages in the wild that aren't compatible with build isolation mode, however, these typically require more build dependencies than just setuptools, which means they wouldn't have worked with this buildpack anyway. As such, it's no longer necessary for us to install setuptools and wheel globally. This matches the behaviour of the `venv` and `ensurepip` modules in Python 3.12+, where setuptools and wheel installation has also been removed. And it also matches the default behaviour of Poetry too, whose `install --sync` command removes any implicitly installed packages in the current environment (other than pip). See: https://peps.python.org/pep-0518/ https://pip.pypa.io/en/stable/reference/build-system/ https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#build-isolation pypa/pip#10717 python/cpython#101039 pypa/get-pip#218 astral-sh/uv#2252 GUS-W-16437776.
heroku · Aug 7, 2024 · 20c1272 · 20c1272
1 parent 1ee3732
commit 20c1272
Show file tree

Hide file tree

Showing 10 changed files with 52 additions and 162 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Removed
+
+- Stopped explicitly installing setuptools and wheel. They will be automatically installed by pip into an isolated build environment if they are required for building a package. ([#243](https://github.com/heroku/buildpacks-python/pull/243))
+
 ## [0.13.0] - 2024-08-01
 
 ### Changed

diff --git a/requirements/setuptools.txt b/requirements/setuptools.txt
diff --git a/requirements/wheel.txt b/requirements/wheel.txt
diff --git a/src/errors.rs b/src/errors.rs
@@ -129,13 +129,13 @@ fn on_python_layer_error(error: PythonLayerError) {
         PythonLayerError::BootstrapPipCommand(error) => match error {
             StreamedCommandError::Io(io_error) => log_io_error(
                 "Unable to bootstrap pip",
-                "running the command to install pip, setuptools and wheel",
+                "running the command to install pip",
                 &io_error,
             ),
             StreamedCommandError::NonZeroExitStatus(exit_status) => log_error(
                 "Unable to bootstrap pip",
                 formatdoc! {"
-                    The command to install pip, setuptools and wheel did not exit successfully ({exit_status}).
+                    The command to install pip did not exit successfully ({exit_status}).
                     
                     See the log output above for more information.
                     

diff --git a/src/layers/pip_cache.rs b/src/layers/pip_cache.rs
@@ -1,4 +1,4 @@
-use crate::packaging_tool_versions::PackagingToolVersions;
+use crate::packaging_tool_versions::PIP_VERSION;
 use crate::python_version::PythonVersion;
 use crate::{BuildpackError, PythonBuildpack};
 use libcnb::build::BuildContext;
@@ -17,14 +17,13 @@ pub(crate) fn prepare_pip_cache(
     context: &BuildContext<PythonBuildpack>,
     env: &mut Env,
     python_version: &PythonVersion,
-    packaging_tool_versions: &PackagingToolVersions,
 ) -> Result<(), libcnb::Error<BuildpackError>> {
     let new_metadata = PipCacheLayerMetadata {
         arch: context.target.arch.clone(),
         distro_name: context.target.distro_name.clone(),
         distro_version: context.target.distro_version.clone(),
         python_version: python_version.to_string(),
-        packaging_tool_versions: packaging_tool_versions.clone(),
+        pip_version: PIP_VERSION.to_string(),
     };
 
     let layer = context.cached_layer(
@@ -74,15 +73,15 @@ pub(crate) fn prepare_pip_cache(
     Ok(())
 }
 
-// Timestamp based cache invalidation isn't used here since the Python/pip/setuptools/wheel
-// versions will change often enough that it isn't worth the added complexity. Ideally pip
-// would support cleaning up its own cache: https://github.com/pypa/pip/issues/6956
+// Timestamp based cache invalidation isn't used here since the Python and pip versions will
+// change often enough that it isn't worth the added complexity. Ideally pip would support
+// cleaning up its own cache: https://github.com/pypa/pip/issues/6956
 #[derive(Deserialize, PartialEq, Serialize)]
 #[serde(deny_unknown_fields)]
 struct PipCacheLayerMetadata {
     arch: String,
     distro_name: String,
     distro_version: String,
     python_version: String,
-    packaging_tool_versions: PackagingToolVersions,
+    pip_version: String,
 }
diff --git a/src/layers/python.rs b/src/layers/python.rs
@@ -1,4 +1,4 @@
-use crate::packaging_tool_versions::PackagingToolVersions;
+use crate::packaging_tool_versions::PIP_VERSION;
 use crate::python_version::PythonVersion;
 use crate::utils::{self, DownloadUnpackArchiveError, StreamedCommandError};
 use crate::{BuildpackError, PythonBuildpack};
@@ -17,7 +17,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::{fs, io};
 
-/// Creates a layer containing the Python runtime and the packages `pip`, `setuptools` and `wheel`.
+/// Creates a layer containing the Python runtime and pip.
 //
 // We install both Python and the packaging tools into the same layer, since:
 //  - We don't want to mix buildpack/packaging dependencies with the app's own dependencies
@@ -31,25 +31,18 @@ use std::{fs, io};
 //  - This leaves just the system site-packages directory, which exists within the Python
 //    installation directory and Python does not support moving it elsewhere.
 //  - It matches what both local and official Docker image environments do.
-#[allow(clippy::too_many_lines)]
 pub(crate) fn install_python_and_packaging_tools(
     context: &BuildContext<PythonBuildpack>,
     env: &mut Env,
     python_version: &PythonVersion,
-    packaging_tool_versions: &PackagingToolVersions,
 ) -> Result<(), libcnb::Error<BuildpackError>> {
     let new_metadata = PythonLayerMetadata {
         arch: context.target.arch.clone(),
         distro_name: context.target.distro_name.clone(),
         distro_version: context.target.distro_version.clone(),
         python_version: python_version.to_string(),
-        packaging_tool_versions: packaging_tool_versions.clone(),
+        pip_version: PIP_VERSION.to_string(),
     };
-    let PackagingToolVersions {
-        pip_version,
-        setuptools_version,
-        wheel_version,
-    } = packaging_tool_versions;
 
     let layer = context.cached_layer(
         layer_name!("python"),
@@ -71,9 +64,8 @@ pub(crate) fn install_python_and_packaging_tools(
 
     match layer.state {
         LayerState::Restored { .. } => {
-            log_info(format!("Using cached Python {python_version}"));
             log_info(format!(
-                "Using cached pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version}"
+                "Using cached Python {python_version} and pip {PIP_VERSION}"
             ));
         }
         LayerState::Empty { ref cause } => {
@@ -117,9 +109,7 @@ pub(crate) fn install_python_and_packaging_tools(
         return Ok(());
     }
 
-    log_info(format!(
-        "Installing pip {pip_version}, setuptools {setuptools_version} and wheel {wheel_version}"
-    ));
+    log_info(format!("Installing pip {PIP_VERSION}"));
 
     let python_stdlib_dir = layer_path.join(format!(
         "lib/python{}.{}",
@@ -140,9 +130,7 @@ pub(crate) fn install_python_and_packaging_tools(
                 "--no-cache-dir",
                 "--no-input",
                 "--quiet",
-                format!("pip=={pip_version}").as_str(),
-                format!("setuptools=={setuptools_version}").as_str(),
-                format!("wheel=={wheel_version}").as_str(),
+                format!("pip=={PIP_VERSION}").as_str(),
             ])
             .current_dir(&context.app_dir)
             .env_clear()
@@ -170,7 +158,7 @@ struct PythonLayerMetadata {
     distro_name: String,
     distro_version: String,
     python_version: String,
-    packaging_tool_versions: PackagingToolVersions,
+    pip_version: String,
 }
 
 /// Compare cached layer metadata to the new layer metadata to determine if the cache should be
@@ -189,25 +177,15 @@ fn cache_invalidation_reasons(
         distro_name: cached_distro_name,
         distro_version: cached_distro_version,
         python_version: cached_python_version,
-        packaging_tool_versions:
-            PackagingToolVersions {
-                pip_version: cached_pip_version,
-                setuptools_version: cached_setuptools_version,
-                wheel_version: cached_wheel_version,
-            },
+        pip_version: cached_pip_version,
     } = cached_metadata;
 
     let PythonLayerMetadata {
         arch,
         distro_name,
         distro_version,
         python_version,
-        packaging_tool_versions:
-            PackagingToolVersions {
-                pip_version,
-                setuptools_version,
-                wheel_version,
-            },
+        pip_version,
     } = new_metadata;
 
     let mut reasons = Vec::new();
@@ -236,18 +214,6 @@ fn cache_invalidation_reasons(
         ));
     }
 
-    if cached_setuptools_version != setuptools_version {
-        reasons.push(format!(
-            "The setuptools version has changed from {cached_setuptools_version} to {setuptools_version}"
-        ));
-    }
-
-    if cached_wheel_version != wheel_version {
-        reasons.push(format!(
-            "The wheel version has changed from {cached_wheel_version} to {wheel_version}"
-        ));
-    }
-
     reasons
 }
 
@@ -423,11 +389,7 @@ mod tests {
             distro_name: "ubuntu".to_string(),
             distro_version: "22.04".to_string(),
             python_version: "3.11.0".to_string(),
-            packaging_tool_versions: PackagingToolVersions {
-                pip_version: "A.B.C".to_string(),
-                setuptools_version: "D.E.F".to_string(),
-                wheel_version: "G.H.I".to_string(),
-            },
+            pip_version: "A.B.C".to_string(),
         }
     }
 
@@ -462,11 +424,7 @@ mod tests {
             distro_name: "debian".to_string(),
             distro_version: "12".to_string(),
             python_version: "3.11.1".to_string(),
-            packaging_tool_versions: PackagingToolVersions {
-                pip_version: "A.B.C-new".to_string(),
-                setuptools_version: "D.E.F-new".to_string(),
-                wheel_version: "G.H.I-new".to_string(),
-            },
+            pip_version: "A.B.C-new".to_string(),
         };
         assert_eq!(
             cache_invalidation_reasons(&cached_metadata, &new_metadata),
@@ -475,8 +433,6 @@ mod tests {
                 "The OS has changed from ubuntu-22.04 to debian-12",
                 "The Python version has changed from 3.11.0 to 3.11.1",
                 "The pip version has changed from A.B.C to A.B.C-new",
-                "The setuptools version has changed from D.E.F to D.E.F-new",
-                "The wheel version has changed from G.H.I to G.H.I-new"
             ]
         );
     }

diff --git a/src/main.rs b/src/main.rs
@@ -13,7 +13,6 @@ use crate::layers::pip_dependencies::PipDependenciesLayerError;
 use crate::layers::python::{self, PythonLayerError};
 use crate::layers::{pip_cache, pip_dependencies};
 use crate::package_manager::{DeterminePackageManagerError, PackageManager};
-use crate::packaging_tool_versions::PackagingToolVersions;
 use crate::python_version::PythonVersionError;
 use libcnb::build::{BuildContext, BuildResult, BuildResultBuilder};
 use libcnb::detect::{DetectContext, DetectResult, DetectResultBuilder};
@@ -53,7 +52,6 @@ impl Buildpack for PythonBuildpack {
         log_header("Determining Python version");
         let python_version = python_version::determine_python_version(&context.app_dir)
             .map_err(BuildpackError::PythonVersion)?;
-        let packaging_tool_versions = PackagingToolVersions::default();
 
         // We inherit the current process's env vars, since we want `PATH` and `HOME` from the OS
         // to be set (so that later commands can find tools like Git in the base image), along
@@ -62,26 +60,16 @@ impl Buildpack for PythonBuildpack {
         // making sure that buildpack env vars take precedence in layers envs and command usage.
         let mut env = Env::from_current();
 
-        // Create the layer containing the Python runtime, and the packages `pip`, `setuptools` and `wheel`.
-        log_header("Installing Python and packaging tools");
-        python::install_python_and_packaging_tools(
-            &context,
-            &mut env,
-            &python_version,
-            &packaging_tool_versions,
-        )?;
+        // Create the layer containing the Python runtime and pip.
+        log_header("Installing Python and pip");
+        python::install_python_and_packaging_tools(&context, &mut env, &python_version)?;
 
         // Create the layers for the application dependencies and package manager cache.
         // In the future support will be added for package managers other than pip.
         let dependencies_layer_dir = match package_manager {
             PackageManager::Pip => {
                 log_header("Installing dependencies using pip");
-                pip_cache::prepare_pip_cache(
-                    &context,
-                    &mut env,
-                    &python_version,
-                    &packaging_tool_versions,
-                )?;
+                pip_cache::prepare_pip_cache(&context, &mut env, &python_version)?;
                 pip_dependencies::install_dependencies(&context, &mut env)?
             }
         };

diff --git a/src/packaging_tool_versions.rs b/src/packaging_tool_versions.rs
@@ -1,36 +1,10 @@
-use serde::{Deserialize, Serialize};
 use std::str;
 
 // We store these versions in requirements files so that Dependabot can update them.
 // Each file must contain a single package specifier in the format `package==1.2.3`,
 // from which we extract/validate the version substring at compile time.
-const PIP_VERSION: &str = extract_requirement_version(include_str!("../requirements/pip.txt"));
-const SETUPTOOLS_VERSION: &str =
-    extract_requirement_version(include_str!("../requirements/setuptools.txt"));
-const WHEEL_VERSION: &str = extract_requirement_version(include_str!("../requirements/wheel.txt"));
-
-/// The versions of various packaging tools used during the build.
-/// These are always installed, and are independent of the chosen package manager.
-/// Strings are used instead of a semver version, since these packages don't use
-/// semver, and we never introspect the version parts anyway.
-#[allow(clippy::struct_field_names)]
-#[derive(Clone, Deserialize, PartialEq, Serialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct PackagingToolVersions {
-    pub(crate) pip_version: String,
-    pub(crate) setuptools_version: String,
-    pub(crate) wheel_version: String,
-}
-
-impl Default for PackagingToolVersions {
-    fn default() -> Self {
-        Self {
-            pip_version: PIP_VERSION.to_string(),
-            setuptools_version: SETUPTOOLS_VERSION.to_string(),
-            wheel_version: WHEEL_VERSION.to_string(),
-        }
-    }
-}
+pub(crate) const PIP_VERSION: &str =
+    extract_requirement_version(include_str!("../requirements/pip.txt"));
 
 // Extract the version substring from an exact-version package specifier (such as `foo==1.2.3`).
 // This function should only be used to extract the version constants from the buildpack's own