Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[testing] Reorganize the test structure. #1243

Merged
merged 5 commits into from
Feb 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/repo_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,10 @@ jobs:
done
}

create_output cxx '*.cpp *.h *.hpp :!:test :!:tpls :!:**/nlopt-src/*'
create_output cxx_headers '*.h *.hpp :!:test :!:tpls :!:**/nlopt-src/*'
create_output cxx '*.cpp *.h *.hpp :!:test :!:targettests :!:tpls :!:**/nlopt-src/*'
create_output cxx_headers '*.h *.hpp :!:test :!:targettests :!:tpls :!:**/nlopt-src/*'
create_output cxx_examples 'docs/sphinx/examples/**/*.cpp'
create_output python '*.py :!:python/tests :!:test :!:tpls :!:docs/sphinx/conf.py'
create_output python '*.py :!:python/tests :!:test :!:targettests :!:tpls :!:docs/sphinx/conf.py'
create_output markdown '*.md :!:tpls'
create_output rst '*.rst :!:tpls'
echo "json=$(echo $json)" >> $GITHUB_OUTPUT
Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/test_in_devenv.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,14 @@ jobs:
run: |
cd $CUDAQ_REPO_ROOT
python3 -m pip install iqm-client
ctest --output-on-failure --test-dir build -E ctest-nvqpp
ctest --output-on-failure --test-dir build -E ctest-nvqpp -E ctest-targettests
ctest_status=$?
/opt/llvm/bin/llvm-lit -v --param nvqpp_site_config=build/test/lit.site.cfg.py build/test
lit_status=$?
if [ ! $ctest_status -eq 0 ] || [ ! $lit_status -eq 0 ]; then
echo "::error file=test_in_devenv.yml::C++ tests failed (ctest status $ctest_status, llvm-lit status $lit_status)."
/opt/llvm/bin/llvm-lit -v --param nvqpp_site_config=build/targettests/lit.site.cfg.py build/targettests
targ_status=$?
if [ ! $ctest_status -eq 0 ] || [ ! $lit_status -eq 0 ] || [ $targ_status -ne 0 ]; then
echo "::error file=test_in_devenv.yml::C++ tests failed (ctest status $ctest_status, llvm-lit status $lit_status, target tests status $targ_status)."
exit 1
fi

Expand Down
5 changes: 4 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,7 @@ if (CUDAQ_ENABLE_PYTHON)
endif()

if(CUDAQ_BUILD_TESTS)
umbrella_lit_testsuite_begin(check-all)
set(INSTALL_GTEST OFF)
add_subdirectory(tpls/googletest-src)
# Bug in GCC 12 leads to spurious warnings (-Wrestrict)
Expand All @@ -544,11 +545,13 @@ if(CUDAQ_BUILD_TESTS)
endif()
include(GoogleTest)
include(CTest)
add_subdirectory(test)
if (NOT CUDAQ_DISABLE_RUNTIME)
add_subdirectory(targettests)
add_subdirectory(unittests)
add_subdirectory(docs)
endif()
add_subdirectory(test)
umbrella_lit_testsuite_end(check-all)
endif()

if (CUDAQ_BUILD_RELOCATABLE_PACKAGE AND NOT CUDAQ_DISABLE_CPP_FRONTEND)
Expand Down
2 changes: 1 addition & 1 deletion docker/build/assets.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ RUN if [ ! -x "$(command -v nvidia-smi)" ] || [ -z "$(nvidia-smi | egrep -o "CUD
fi && cd /cuda-quantum && \
# FIXME: Disabled nlopt doesn't seem to work properly
# tracked in https://github.com/NVIDIA/cuda-quantum/issues/1103
excludes+=" --exclude-regex NloptTester|ctest-nvqpp" && \
excludes+=" --exclude-regex NloptTester|ctest-nvqpp|ctest-targettests" && \
ctest --output-on-failure --test-dir build $excludes

ENV CUDAQ_CPP_STD="c++17"
Expand Down
2 changes: 1 addition & 1 deletion python/tests/backends/test_IQM.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def startUpMockServer():
with os.popen("git rev-parse --show-toplevel") as f:
git_top = f.read().strip()
if os.path.isdir(git_top):
mapping_file = f"{git_top}/test/Supplemental/Apollo Variant.txt"
mapping_file = f"{git_top}/targettests/Supplemental/Apollo Variant.txt"
kwargs["mapping_file"] = mapping_file
cudaq.set_target("iqm", url="http://localhost:{}".format(port), **kwargs)

Expand Down
57 changes: 57 additions & 0 deletions targettests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# ============================================================================ #
# Copyright (c) 2022 - 2023 NVIDIA Corporation & Affiliates. #
# All rights reserved. #
# #
# This source code and the accompanying materials are made available under #
# the terms of the Apache License 2.0 which accompanies this distribution. #
# ============================================================================ #

llvm_canonicalize_cmake_booleans(
CUDA_FOUND
CUDAQ_TEST_REMOTE_SIM
)

configure_lit_site_cfg(
${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
MAIN_CONFIG
${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
PATHS
${PATHS_FOR_PLUGINS}
)

set(CUDAQ_TEST_PARAMS
nvqpp_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py)

get_property(test_cudaq_libraries GLOBAL PROPERTY CUDAQ_RUNTIME_LIBS)

set(CUDAQ_TEST_DEPENDS
cudaq-opt
cudaq-translate
CircuitCheck
FileCheck
cudaq-quake
fixup-linkage
${test_cudaq_libraries}
)

add_custom_target(nvqpp-targettest-depends DEPENDS ${CUDAQ_TEST_DEPENDS})
set_target_properties(nvqpp-targettest-depends PROPERTIES FOLDER "TargetTests")

add_lit_testsuite(check-targets "Running the end-to-end target tests"
${CMAKE_CURRENT_BINARY_DIR}
PARAMS ${CUDAQ_TEST_PARAMS}
DEPENDS ${CUDAQ_TEST_DEPENDS}
)
set_target_properties(check-targets PROPERTIES FOLDER "TargetTests")

add_lit_testsuites(CUDAQ ${CMAKE_CURRENT_SOURCE_DIR}
PARAMS ${CUDAQ_TEST_PARAMS}
DEPENDS ${CUDAQ_TEST_DEPENDS}
FOLDER "Tests/Subdirectories"
)

# Add nvqpp tests to the ctest suite
add_test(NAME ctest-targettests
COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target check-targets
WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion targettests/ionq/bug_qubit.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/callable_kernel_arg.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/cudaq_observe.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/graph_coloring-1.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/graph_coloring.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/if_jit.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/load_value.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/qspan_slices.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/sudoku_2x2-1.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/sudoku_2x2-bit_names.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/sudoku_2x2-reg_name.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/sudoku_2x2.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/swap_gate.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/test-int8_t.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/test-int8_t_free_func.cpp
2 changes: 1 addition & 1 deletion targettests/ionq/variable_size_qreg.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/bug_qubit.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/callable_kernel_arg.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/cudaq_observe.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/if_jit.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/load_value.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/qspan_slices.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/swap_gate.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/test-int8_t.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/test-int8_t_free_func.cpp
2 changes: 1 addition & 1 deletion targettests/iqm/variable_size_qreg.cpp
77 changes: 77 additions & 0 deletions targettests/lit.cfg.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# ============================================================================ #
# Copyright (c) 2022 - 2023 NVIDIA Corporation & Affiliates. #
# All rights reserved. #
# #
# This source code and the accompanying materials are made available under #
# the terms of the Apache License 2.0 which accompanies this distribution. #
# ============================================================================ #

import os
import platform
import re
import subprocess
import sys
import bisect

import lit.util
from lit.llvm import llvm_config
from lit.llvm.subst import ToolSubst
from lit.llvm.subst import FindTool

import lit.formats

# The name of this test suite.
config.name = 'CUDAQ-Target'

# `testFormat`: The test format to use to interpret tests.
config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)

config.suffixes = ['.cpp']

# Exclude a list of directories from the test suite:
# - 'Inputs' contain auxiliary inputs for various tests.
local_excludes = ['ionq', 'iqm', 'oqc', 'quantinuum',
'Inputs', 'CMakeLists.txt', 'README.txt', 'LICENSE.txt']
config.excludes = [exclude for exclude in config.excludes] + local_excludes

config.substitutions.append(('%PATH%', config.environment['PATH']))
config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir))
config.substitutions.append(('%pluginext', config.llvm_plugin_ext))
config.substitutions.append(('%llvmInclude', config.llvm_install + "/include"))
config.substitutions.append(('%cudaq_lib_dir', config.cudaq_lib_dir))
config.substitutions.append(('%cudaq_plugin_ext', config.cudaq_plugin_ext))

llvm_config.use_default_substitutions()

# Ask `llvm-config` about asserts
llvm_config.feature_config([('--assertion-mode', {'ON': 'asserts'})])

# Allow to require specific build targets
config.targets = frozenset(config.targets_to_build.split())
for arch in config.targets_to_build.split():
config.available_features.add(arch.lower() + '-registered-target')

# Allow to filter tests based on environment variables
cpp_stds = ['c++17', 'c++20', 'c++23']
std_up_to = os.environ.get('CUDAQ_CPP_STD', 'c++23').lower()
for std in cpp_stds[:bisect.bisect(cpp_stds, std_up_to)]:
config.available_features.add(std)
std_default = os.environ.get('CUDAQ_CPP_STD')
if std_default is None:
config.substitutions.append(('%cpp_std', ''))
else:
config.substitutions.append(('%cpp_std', '-std=' + std_default.lower()))

# The root path where tests are located.
config.test_source_root = os.path.dirname(__file__)

# The root path where tests should be run.
config.test_exec_root = os.path.join(config.cudaq_obj_root, 'targettests')

# Propagate some variables from the host environment.
llvm_config.with_system_environment(
['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP'])

# Tweak the PATH to include the tools directory.
llvm_config.with_environment('PATH', config.cudaq_tools_dir, append_path=True)
llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
50 changes: 50 additions & 0 deletions targettests/lit.site.cfg.py.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
@LIT_SITE_CFG_IN_HEADER@

# ============================================================================ #
# Copyright (c) 2022 - 2023 NVIDIA Corporation & Affiliates. #
# All rights reserved. #
# #
# This source code and the accompanying materials are made available under #
# the terms of the Apache License 2.0 which accompanies this distribution. #
# ============================================================================ #

import sys

# Helper function to convert cmake variable boolean definitions to True/False.
def cmake_boolvar_to_bool(s):
return s == "1"

config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_BINARY_DIR@")
config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
config.llvm_install = config.llvm_tools_dir + "/.."
config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
config.errc_messages = "@LLVM_LIT_ERRC_MESSAGES@"
config.cudaq_obj_root = "@CUDAQ_BINARY_DIR@"
config.cudaq_src_dir = "@CUDAQ_SOURCE_DIR@"
config.cudaq_tools_dir = lit_config.substitute("@CUDAQ_TOOLS_DIR@")
config.cudaq_intrinsic_modules_dir = "@CUDAQ_INTRINSIC_MODULES_DIR@"
config.cudaq_llvm_tools_dir = "@CMAKE_BINARY_DIR@/bin"
config.cudaq_lib_dir = "@CMAKE_BINARY_DIR@/lib"
config.cudaq_plugin_ext = "@LLVM_PLUGIN_EXT@"
config.python_executable = "@PYTHON_EXECUTABLE@"
config.cc = "@CMAKE_C_COMPILER@"
config.targets_to_build = "@TARGETS_TO_BUILD@"
config.excludes = ["@CUDAQ_EXTRA_CONFIG_EXCLUDES@"]
config.pipefail = False

# Is the nvcc compiler available?
config.cuda_found = "@CUDA_FOUND@"
if cmake_boolvar_to_bool(config.cuda_found):
config.available_features.add('nvcc')

config.test_remote_sim = "@CUDAQ_TEST_REMOTE_SIM@"
if cmake_boolvar_to_bool(config.test_remote_sim):
config.available_features.add('remote-sim')

import lit.llvm
lit.llvm.initialize(lit_config, config)

# Let the main config do the real work.
lit_config.load_config(config, "@CUDAQ_SOURCE_DIR@/targettests/lit.cfg.py")

2 changes: 1 addition & 1 deletion targettests/oqc/bug_qubit.cpp
2 changes: 1 addition & 1 deletion targettests/oqc/callable_kernel_arg.cpp
Loading
Loading