Various build fixes (NVIDIA#450)

This PR:
- Sets the toolchain for the docker image to gcc12, since the OpenMP support is not currently enabled in the llvm build
- Uses [BLAS from netlib](https://www.netlib.org/blas/) instead of [OpenBLAS](https://github.com/xianyi/OpenBLAS/tree/v0.3.21) to supply the blas implementation to avoid sporadic issues with the optimizer
- Fixes a missing image name in the deployment

.github/workflows/ci.yml

@@ -74,8 +74,8 @@ jobs:
     needs: config
     uses: ./.github/workflows/docker_images.yml
     with:
-      devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key.llvm }}
-      devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive.llvm }}
+      devdeps_cache: ${{ fromJson(needs.config.outputs.json).cache_key.gcc12 }}
+      devdeps_archive: ${{ fromJson(needs.config.outputs.json).tar_archive.gcc12 }}
 
   python_wheels:
     name: Create Python wheels

.github/workflows/deployments.yml

@@ -79,7 +79,7 @@ jobs:
       platforms: ${{ inputs.platforms }}
       toolchain: x86_64
       build_args: |
-        manylinux_image=${{ inputs.manylinux_image }}
+        manylinux_image=${{ inputs.manylinux_image || 'quay.io/pypa/manylinux_2_28_x86_64:latest' }}
       registry_cache_from: ${{ (github.event_name == 'push' && 'main') || inputs.cache_base }}
       registry_cache_update: ${{ github.event_name == 'workflow_dispatch' && inputs.update_registry_cache }}
       registry_cache_update_only: ${{ github.event.pull_request.merged == true }}
@@ -119,8 +119,8 @@ jobs:
 
     outputs:
       json: "${{ steps.read_json.outputs.result }}"
-      base_image: ${{ fromJson(steps.read_json.outputs.result).image_hash.llvm }}
-      toolchain: llvm
+      base_image: ${{ fromJson(steps.read_json.outputs.result).image_hash.gcc12 }}
+      toolchain: gcc12
 
     steps:
       - uses: cloudposse/[email protected]

.github/workflows/publishing.yml

@@ -1,9 +1,9 @@
 on:
   workflow_dispatch:
     inputs:
-      release_id:
+      deployment_id:
         type: string
-        description: The release id on GitHub from which to download the assets.
+        description: The workflow id of the deployment that triggered the release creation.
         required: true
   workflow_run:
     workflows:
@@ -30,9 +30,12 @@ jobs:
     steps:
       - name: Download artifacts
         id: artifacts
-        if: github.event_name == 'workflow_run'
         run: |
-          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+          if ${{ inputs.deployment_id }}; then
+            artifacts_url=https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ inputs.deployment_id }}/artifacts
+          else
+            artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+          fi
           artifacts=$(gh api $artifacts_url -q '.artifacts[] | {name: .name, url: .archive_download_url}')
           wheelhouse="$(pwd)/wheels"
           mkdir -p "$wheelhouse"
@@ -68,7 +71,7 @@ jobs:
       - name: Wait for release
         id: assets
         run: |
-          release_id=${{ inputs.release_id || steps.artifacts.outputs.release_id }}
+          release_id=${{ steps.artifacts.outputs.release_id }}
           github_commit=`gh release view $release_id -R ${{ github.repository }} --json targetCommitish --jq .targetCommitish`
 
           echo "release_id=$release_id" >> $GITHUB_OUTPUT

NOTICE

@@ -110,10 +110,10 @@ License at <https://github.com/libcpr/cpr/blob/master/LICENSE>
 
 ----------------------------------------------------------------
 
-Open BLAS - BSD-3-Clause
-<https://www.openblas.net/>
+BLAS - custom license
+<https://netlib.org/blas/>
 
-License at <https://github.com/xianyi/OpenBLAS/blob/develop/LICENSE>
+See also <https://netlib.org/blas/#_licensing>
 
 ----------------------------------------------------------------
 

docker/build/devdeps.Dockerfile

@@ -70,13 +70,22 @@ RUN source /opt/llvm/bootstrap/init_command.sh && \
         bash /scripts/build_llvm.sh -s /llvm-project -c Release -v \
     && rm -rf /llvm-project 
 
+# Todo: 
+# - remove http://apt.llvm.org/jammy/ in the install_toolchain.sh and use
+#   FROM llvmbuild as prereqs
+# - uncomment the source /opt/llvm/bootstrap/init_command.sh below
 FROM ubuntu:22.04 as prereqs
-COPY --from=llvmbuild /opt/llvm/lib/cmake/llvm /opt/llvm/lib/cmake/llvm
+SHELL ["/bin/bash", "-c"]
+COPY --from=llvmbuild /opt/llvm /opt/llvm
 ADD ./scripts/install_prerequisites.sh /scripts/install_prerequisites.sh
-RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates \
+ADD ./scripts/install_toolchain.sh /scripts/install_toolchain.sh
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
     && export LLVM_INSTALL_PREFIX=/opt/llvm \
-    && export OPENBLAS_INSTALL_PREFIX=/usr/local/openblas \
+    && export BLAS_INSTALL_PREFIX=/usr/local/blas \
     && export OPENSSL_INSTALL_PREFIX=/usr/local/openssl \
+    # Making sure that anything that is build from source when installing additional
+    # prerequisites is built using the same toolchain as CUDA Quantum by default.
+    # && source /opt/llvm/bootstrap/init_command.sh \
     && bash /scripts/install_prerequisites.sh \
     && apt-get remove -y ca-certificates \
     && apt-get autoremove -y --purge && apt-get clean && rm -rf /var/lib/apt/lists/*
@@ -114,12 +123,12 @@ ENV CXX="$LLVM_INSTALL_PREFIX/bootstrap/cxx"
 RUN apt-get update && apt-get install -y --no-install-recommends libstdc++-12-dev \
     && apt-get autoremove -y --purge && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# Copy over other prerequisites we build from source.
-COPY --from=prereqs /usr/local/openblas /usr/local/openblas
-COPY --from=prereqs /usr/local/openssl /usr/local/openssl
-ENV OPENBLAS_INSTALL_PREFIX=/usr/local/openblas
+# Copy over additional prerequisites.
+ENV BLAS_INSTALL_PREFIX=/usr/local/blas
 ENV OPENSSL_INSTALL_PREFIX=/usr/local/openssl
 ENV OPENSSL_ROOT_DIR="$OPENSSL_INSTALL_PREFIX"
+COPY --from=prereqs /usr/local/blas "$BLAS_INSTALL_PREFIX"
+COPY --from=prereqs /usr/local/openssl "$OPENSSL_INSTALL_PREFIX"
 
 # Install additional tools for CUDA Quantum documentation generation.
 RUN apt-get update && apt-get install --no-install-recommends -y wget ca-certificates \

docker/build/devdeps.ext.Dockerfile

@@ -116,8 +116,9 @@ ENV UCX_TLS=rc,cuda_copy,cuda_ipc,gdr_copy,sm
 # Install cuQuantum libraries.
 
 ARG CUQUANTUM_INSTALL_PREFIX=/opt/nvidia/cuquantum
-ENV CUQUANTUM_INSTALL_PREFIX="$CUQUANTUM_INSTALL_PREFIX"
+ENV CUQUANTUM_ROOT="$CUQUANTUM_INSTALL_PREFIX"
 ENV LD_LIBRARY_PATH="$CUQUANTUM_INSTALL_PREFIX/lib:$LD_LIBRARY_PATH"
+ENV CPATH="$CUQUANTUM_INSTALL_PREFIX/include:$CPATH"
 
 RUN apt-get update && apt-get install -y --no-install-recommends xz-utils \
     && wget https://developer.download.nvidia.com/compute/cuquantum/redist/cuquantum/linux-x86_64/cuquantum-linux-x86_64-23.06.0.7_cuda11-archive.tar.xz \
@@ -130,8 +131,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends xz-utils \
 # Install cuTensor libraries.
 
 ARG CUTENSOR_INSTALL_PREFIX=/opt/nvidia/cutensor
-ENV CUTENSOR_INSTALL_PREFIX="$CUTENSOR_INSTALL_PREFIX"
+ENV CUTENSOR_ROOT="$CUTENSOR_INSTALL_PREFIX"
 ENV LD_LIBRARY_PATH="$CUTENSOR_INSTALL_PREFIX/lib:$LD_LIBRARY_PATH"
+ENV CPATH="$CUTENSOR_INSTALL_PREFIX/include:$CPATH"
 
 RUN apt-get update && apt-get install -y --no-install-recommends xz-utils \
     && wget https://developer.download.nvidia.com/compute/cutensor/redist/libcutensor/linux-x86_64/libcutensor-linux-x86_64-1.7.0.1-archive.tar.xz \

docker/build/devdeps.manylinux.Dockerfile

@@ -43,14 +43,10 @@ RUN dnf check-update && dnf install -y --nobest --setopt=install_weak_deps=False
     && rm -rf /llvm-project && rm /scripts/build_llvm.sh
 
 # Install additional dependencies required to build the CUDA Quantum wheel.
-RUN dnf check-update && dnf install -y --nobest --setopt=install_weak_deps=False \
-        glibc-static perl-core \
-    && dnf clean all
-
-# Build OpenBLAS from source with OpenMP enabled.
 ADD ./scripts/install_prerequisites.sh /scripts/install_prerequisites.sh
-ENV OPENBLAS_INSTALL_PREFIX=/usr/local/openblas
+ENV BLAS_INSTALL_PREFIX=/usr/local/blas
 ENV OPENSSL_INSTALL_PREFIX=/usr/local/openssl
-RUN dnf check-update && dnf install -y --nobest --setopt=install_weak_deps=False wget \
+RUN dnf check-update && dnf install -y --nobest --setopt=install_weak_deps=False \
+        glibc-static perl-core wget \
     && bash /scripts/install_prerequisites.sh \
     && dnf remove -y wget && dnf clean all

docker/release/cudaq.Dockerfile

@@ -92,6 +92,7 @@ RUN bash "$CUDA_QUANTUM_PATH/bin/migrate_assets.sh" "$CUDA_QUANTUM_PATH/assets"
 ENV PATH "${PATH}:$CUDA_QUANTUM_PATH/bin"
 ENV PYTHONPATH "${PYTHONPATH}:$CUDA_QUANTUM_PATH"
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CUDA_QUANTUM_PATH/lib"
+ENV CPLUS_INCLUDE_PATH="$CPLUS_INCLUDE_PATH:$CUDA_QUANTUM_PATH/include"
 
 # Include additional readmes and samples that are distributed with the image.
 

pyproject.toml

@@ -42,7 +42,7 @@ cmake.args = [
     "-DCUDAQ_DISABLE_CPP_FRONTEND=TRUE",
     "-DCUDAQ_DISABLE_TOOLS=TRUE",
     "-DCUDAQ_BUILD_TESTS=FALSE",
-    "-DBLAS_LIBRARIES=/usr/local/openblas/lib/libopenblas.a",
+    "-DBLAS_LIBRARIES=/usr/local/blas/libblas.a",
     "-DOPENSSL_ROOT_DIR=/usr/local/openssl",
 ]
 

scripts/build_cudaq.sh

@@ -36,6 +36,8 @@
 LLVM_INSTALL_PREFIX=${LLVM_INSTALL_PREFIX:-/opt/llvm}
 CUQUANTUM_INSTALL_PREFIX=${CUQUANTUM_INSTALL_PREFIX:-/opt/nvidia/cuquantum}
 CUDAQ_INSTALL_PREFIX=${CUDAQ_INSTALL_PREFIX:-"$HOME/.cudaq"}
+BLAS_INSTALL_PREFIX=${BLAS_INSTALL_PREFIX:-/usr/local/blas}
+BLAS_LIBRARIES=${BLAS_LIBRARIES:-"$BLAS_INSTALL_PREFIX/libblas.a"}
 
 # Process command line arguments
 (return 0 2>/dev/null) && is_sourced=true || is_sourced=false
@@ -108,8 +110,8 @@ cmake_args="-G Ninja "$repo_root" \
   -DNVQPP_LD_PATH="$NVQPP_LD_PATH" \
   -DCMAKE_BUILD_TYPE=$build_configuration \
   -DCUDAQ_ENABLE_PYTHON=TRUE \
-  -DBLAS_LIBRARIES="${OPENBLAS_INSTALL_PREFIX:+$OPENBLAS_INSTALL_PREFIX/lib/libopenblas.a}" \
   -DCUDAQ_TEST_MOCK_SERVERS=FALSE \
+  -DBLAS_LIBRARIES="${BLAS_LIBRARIES}" \
   -DCMAKE_EXE_LINKER_FLAGS_INIT="$cmake_common_linker_flags_init" \
   -DCMAKE_MODULE_LINKER_FLAGS_INIT="$cmake_common_linker_flags_init" \
   -DCMAKE_SHARED_LINKER_FLAGS_INIT="$cmake_common_linker_flags_init" \

scripts/install_prerequisites.sh

@@ -17,14 +17,14 @@
 #
 # The necessary LLVM components will be installed in the location defined by the
 # LLVM_INSTALL_PREFIX if they do not already exist in that location.
-# If OpenBLAS is not found, it will be built from source and installed the location
-# defined by the OPENBLAS_INSTALL_PREFIX.
+# If BLAS is not found, it will be built from source and installed the location
+# defined by the BLAS_INSTALL_PREFIX.
 # If OpenSSL is not found, it will be built from source and installed the location
 # defined by the OPENSSL_INSTALL_PREFIX.
 
 LLVM_INSTALL_PREFIX=${LLVM_INSTALL_PREFIX:-/opt/llvm}
-OPENBLAS_INSTALL_PREFIX=${OPENBLAS_INSTALL_PREFIX:-/usr/local}
-OPENSSL_INSTALL_PREFIX=${OPENSSL_INSTALL_PREFIX:-/usr/local}
+BLAS_INSTALL_PREFIX=${BLAS_INSTALL_PREFIX:-/usr/local/blas}
+OPENSSL_INSTALL_PREFIX=${OPENSSL_INSTALL_PREFIX:-/usr/lib/ssl}
 
 function temp_install_if_command_unknown {
     if [ ! -x "$(command -v $1)" ]; then
@@ -41,6 +41,7 @@ function remove_temp_installs {
   fi
 }
 
+set -e
 trap remove_temp_installs EXIT
 this_file_dir=`dirname "$(readlink -f "${BASH_SOURCE[0]}")"`
 
@@ -49,7 +50,7 @@ if [ ! -x "$(command -v cmake)" ]; then
     APT_UNINSTALL="$APT_UNINSTALL $2"
 fi
 if [ "$CC" == "" ] && [ "$CXX" == "" ]; then
-  source "$this_file_dir/install_tool.sh" -t gcc12
+  source "$this_file_dir/install_toolchain.sh" -t gcc12
 fi
 
 llvm_dir="$LLVM_INSTALL_PREFIX/lib/cmake/llvm"
@@ -79,33 +80,30 @@ if [ ! -x "$(command -v ar)" ] && [ -x "$(command -v "$LLVM_INSTALL_PREFIX/bin/l
     fi
 fi
 
-if [ ! -f "$OPENBLAS_INSTALL_PREFIX/lib/libopenblas.a" ]; then
+if [ ! -f "$BLAS_INSTALL_PREFIX/libblas.a" ] && [ ! -f "$BLAS_INSTALL_PREFIX/lib/libblas.a" ]; then
   apt-get update
   temp_install_if_command_unknown wget wget
   temp_install_if_command_unknown make make
   temp_install_if_command_unknown gcc gcc
   temp_install_if_command_unknown g++ g++
   temp_install_if_command_unknown gfortran gfortran
 
-  wget https://github.com/xianyi/OpenBLAS/releases/download/v0.3.23/OpenBLAS-0.3.23.tar.gz
-  tar -xf OpenBLAS-0.3.23.tar.gz && cd OpenBLAS-0.3.23
-
-  # TODO: gcc and clang work with different OpenMP libraries (libgomp vs libomp).
-  # To enable OpenMP support here we need to build it with the same compiler toolchain
-  # as we build CUDA Quantum with to ensure we link against the OpenMP library supported
-  # by that compiler. OpenMP support is disabled until we add the tools to build this 
-  # with each toolchains in the dev images.
-  make USE_OPENMP=0 && make install PREFIX="$OPENBLAS_INSTALL_PREFIX"
-  cd .. && rm -rf OpenBLAS-0.3.23*
+  # See also: https://github.com/NVIDIA/cuda-quantum/issues/452
+  wget http://www.netlib.org/blas/blas-3.11.0.tgz
+  tar -xzvf blas-3.11.0.tgz && cd BLAS-3.11.0
+  make && mkdir -p "$BLAS_INSTALL_PREFIX" && mv blas_LINUX.a "$BLAS_INSTALL_PREFIX/libblas.a"
+  cd .. && rm -rf blas-3.11.0.tgz BLAS-3.11.0
+  remove_temp_installs
 fi
 
-if [ ! -d "$OPENSSL_INSTALL_PREFIX" ] || [ -z "$(ls -A "$OPENSSL_INSTALL_PREFIX")" ]; then
-  apt-get update
-  temp_install_if_command_unknown git git
+if [ ! -d "$OPENSSL_INSTALL_PREFIX" ] || [ -z "$(ls -A "$OPENSSL_INSTALL_PREFIX"/openssl*)" ]; then
+  apt-get update && apt-get install -y --no-install-recommends perl
+  temp_install_if_command_unknown wget wget
   temp_install_if_command_unknown make make
 
   wget https://www.openssl.org/source/openssl-3.1.1.tar.gz
   tar -xf openssl-3.1.1.tar.gz && cd openssl-3.1.1
   ./config no-zlib --prefix="$OPENSSL_INSTALL_PREFIX" --openssldir="$OPENSSL_INSTALL_PREFIX"
   make install && cd .. && rm -rf openssl-3.1.1*
+  remove_temp_installs
 fi