diff --git a/ci/run_ctests.sh b/ci/run_ctests.sh index 68c4648001..ada3b12e57 100755 --- a/ci/run_ctests.sh +++ b/ci/run_ctests.sh @@ -2,10 +2,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Run gtests with per-test-case retry for flaky detection. +# Run gtests through ctest, with per-test-case retry for flaky detection. # # Features: -# - Runs each gtest binary and collects JUnit XML results +# - Selects tests by ctest label (CUOPT_CPP_TEST_GROUP) +# - Collects JUnit XML results # - On failure, parses XML to find failing test cases and retries them individually # - Produces separate XML files per retry so nightly_report.py can classify flaky tests # - Detects segfaults (signal 11) and isolates crashing tests @@ -14,6 +15,9 @@ # GTEST_OUTPUT - gtest XML output prefix (set by test_cpp.sh) # GTEST_MAX_RETRIES - max retries per failing test case (default: 2) # RAPIDS_TESTS_DIR - directory for test results +# CUOPT_CPP_TEST_GROUP - all (default), routing or mathopt +# +# Only --gtest_filter= is accepted as an argument. # CUOPT_CPP_TEST_GROUP - which binaries to run: all (default), routing # (binaries labeled "routing" in cpp/tests) or mathopt # (everything else, including unlabeled binaries) @@ -25,20 +29,27 @@ SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" # shellcheck source=ci/utils/crash_helpers.sh source "${SCRIPT_DIR}/utils/crash_helpers.sh" -# Support customizing the gtests' install location +# Support customizing the tests' install location # First, try the installed location (CI/conda environments) -installed_test_location="${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcuopt/" +installed_test_location="${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcuopt" # Fall back to the build directory (devcontainer environments) -devcontainers_test_location="${SCRIPT_DIR}/../cpp/build/latest/gtests/libcuopt/" +devcontainers_build_dir="${SCRIPT_DIR}/../cpp/build/latest" -if [[ -d "${installed_test_location}" ]]; then +if [[ -f "${installed_test_location}/CTestTestfile.cmake" ]]; then + CTEST_DIR="${installed_test_location}" GTEST_DIR="${installed_test_location}" -elif [[ -d "${devcontainers_test_location}" ]]; then - GTEST_DIR="${devcontainers_test_location}" +elif [[ -f "${devcontainers_build_dir}/CTestTestfile.cmake" ]]; then + CTEST_DIR="${devcontainers_build_dir}" + GTEST_DIR="${devcontainers_build_dir}/gtests/libcuopt" else - echo "Error: Test location not found. Searched:" >&2 + echo "Error: CTestTestfile.cmake not found. Searched:" >&2 echo " - ${installed_test_location}" >&2 - echo " - ${devcontainers_test_location}" >&2 + echo " - ${devcontainers_build_dir}" >&2 + exit 1 +fi + +if ! command -v ctest > /dev/null; then + echo "Error: ctest not found on PATH (install cmake in the test environment)" >&2 exit 1 fi @@ -86,6 +97,39 @@ write_binary_crash_marker() { "Process terminated by ${sig} mid-run. gtest did not emit a JUnit XML because RUN_ALL_TESTS() did not complete; inspect the run log for [FAILED] / stack-trace lines that preceded the crash." } +# ctest only reports "Failed", so recover the exit code from its output. +run_ctest_test() { + local test_name="$1" + local xml_file="$2" + local log + log=$(mktemp) + local -a filter_env=() + [[ -n "${3:-}" ]] && filter_env=(GTEST_FILTER="$3") + rm -f "${xml_file}" + + local ctest_rc=0 + set +o pipefail + env ${filter_env[@]+"${filter_env[@]}"} GTEST_OUTPUT="xml:${xml_file}" ctest --test-dir "${CTEST_DIR}" \ + -R "^${test_name}\$" --no-tests=error --timeout 0 --verbose 2>&1 | tee "${log}" + ctest_rc=${PIPESTATUS[0]} + set -o pipefail + + local rc=0 + if [ "${ctest_rc}" -ne 0 ]; then + if grep -Eiq "Segmentation fault|SegFault" "${log}"; then rc=139 + elif grep -Eiq "Subprocess aborted|Child aborted" "${log}"; then rc=134 + elif grep -Eiq "Subprocess killed" "${log}"; then rc=137 + elif grep -Eq "\(Timeout\)|\*\*\*Timeout" "${log}"; then rc=124 + else + rc=$(sed -n 's/.*Child return code: \([0-9]\+\).*/\1/p' "${log}" | tail -n 1) + rc="${rc:-1}" + [ "${rc}" -eq 0 ] && rc=1 + fi + fi + rm -f "${log}" + return "${rc}" +} + run_gtest_with_retry() { local gt="$1" shift @@ -95,9 +139,9 @@ run_gtest_with_retry() { echo "Running gtest ${test_name}" - # First run — full binary + # First run — full binary, through CTest local rc=0 - "${gt}" --gtest_output="xml:${xml_file}" "$@" || rc=$? + run_ctest_test "${test_name}" "${xml_file}" || rc=$? if [ "${rc}" -eq 0 ]; then return 0 @@ -130,7 +174,7 @@ run_gtest_with_retry() { # plus any that failed. Only retry those, not the ones that passed. echo "INFO: Finding tests that need retry in ${test_name}" local all_tests - all_tests=$("${gt}" --gtest_list_tests "$@" 2>/dev/null \ + all_tests=$("${gt}" --gtest_list_tests 2>/dev/null \ | python3 "${JUNIT_HELPERS}" gtest-list || echo "") # Extract tests that already passed from partial XML @@ -184,7 +228,7 @@ run_gtest_with_retry() { echo " Retry ${attempt}/${GTEST_MAX_RETRIES}: ${tc}" local retry_rc=0 - "${gt}" --gtest_filter="${tc}" --gtest_output="xml:${retry_xml}" "$@" || retry_rc=$? + run_ctest_test "${test_name}" "${retry_xml}" "${tc}" || retry_rc=$? if [ "${retry_rc}" -eq 0 ]; then echo " FLAKY: ${tc} passed on retry ${attempt}" @@ -221,28 +265,46 @@ case "${CUOPT_CPP_TEST_GROUP}" in *) echo "Error: invalid CUOPT_CPP_TEST_GROUP '${CUOPT_CPP_TEST_GROUP}'" >&2; exit 1 ;; esac -# .labels is installed next to each labeled binary (see ConfigureTest). -has_label() { - [[ -f "$1.labels" ]] && grep -qx "$2" "$1.labels" -} +# ctest runs tests from the test directory; some read data relative to CUOPT_HOME. +export CUOPT_HOME="${CUOPT_HOME:-$(realpath "${SCRIPT_DIR}/..")}" -NUM_RUN=0 -for gt in "${GTEST_DIR}"/*_TEST; do - is_routing=false - has_label "${gt}" routing && is_routing=true - if [[ "${CUOPT_CPP_TEST_GROUP}" == "routing" && "${is_routing}" == false ]] || - [[ "${CUOPT_CPP_TEST_GROUP}" == "mathopt" && "${is_routing}" == true ]]; then - continue - fi - NUM_RUN=$((NUM_RUN + 1)) - run_gtest_with_retry "${gt}" "$@" || true +for arg in "$@"; do + case "${arg}" in + --gtest_filter=*) export GTEST_FILTER="${arg#--gtest_filter=}" ;; + *) echo "Error: unsupported argument '${arg}' (only --gtest_filter=... is supported)" >&2; exit 1 ;; + esac done -if [ "${NUM_RUN}" -eq 0 ]; then - echo "Error: no gtest binaries selected for group '${CUOPT_CPP_TEST_GROUP}' in ${GTEST_DIR}" >&2 - OVERALL_RC=1 +case "${CUOPT_CPP_TEST_GROUP}" in + routing) CTEST_SELECT=(-L '^routing$') ;; + mathopt) CTEST_SELECT=(-LE '^routing$') ;; + *) CTEST_SELECT=() ;; +esac + +# Names of the enabled tests matching the given ctest selection. +list_tests() { + ctest --test-dir "${CTEST_DIR}" -N "$@" | sed -n 's/^ *Test *#[0-9]*: //p' | + grep -v -e '^generate_resource_spec$' -e ' (Disabled)$' || true +} + +# Unlabeled tests would silently land in mathopt, so require a group label. +UNLABELED=$(list_tests -LE '^(routing|numopt)$') +if [[ -n "${UNLABELED}" ]]; then + echo "Error: tests without a routing or numopt label (set LABELS in CMake):" >&2 + echo "${UNLABELED}" | sed 's/^/ - /' >&2 + exit 1 +fi + +TEST_LIST=$(list_tests ${CTEST_SELECT[@]+"${CTEST_SELECT[@]}"}) +if [[ -z "${TEST_LIST}" ]]; then + echo "Error: no tests selected for group '${CUOPT_CPP_TEST_GROUP}' in ${CTEST_DIR}" >&2 + exit 1 fi +while read -r test_name; do + run_gtest_with_retry "${GTEST_DIR}/${test_name}" || true +done <<< "${TEST_LIST}" + # Final summary so failures are easy to spot in the raw run log. # nightly_report.py also produces a structured report from the XML files, # but this prints early (before any post-test-script steps) and surfaces diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index e3d4b68095..7ed75b2840 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -71,26 +71,11 @@ endif() set(CUOPT_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -# ################################################################ ------------------------------------------------------------------ -# install_test_labels(TEST_NAME label1 [label2 ...]) -# -# Installs a .labels marker next to the test binary so ci/run_ctests.sh can select -# tests by label. Use it for tests whose LABELS are set outside ConfigureTest. -function(install_test_labels TEST_NAME) - string(REPLACE ";" "\n" _labels_text "${ARGN}") - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}.labels" "${_labels_text}\n") - install( - FILES "${CMAKE_CURRENT_BINARY_DIR}/${TEST_NAME}.labels" - COMPONENT testing - DESTINATION bin/gtests/libcuopt - EXCLUDE_FROM_ALL - ) -endfunction() - # ################################################################ ------------------------------------------------------------------ # ConfigureTest(NAME source1.cu source2.cu [STATIC_LIB] [LABELS label1 label2 ...]) # -# LABELS sets CTest labels for selective local test execution via `ctest -L