From 00b1aef96b1c872799f318ed57bda5facef48528 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman Date: Sat, 26 Apr 2025 10:37:49 +0000 Subject: [PATCH] ctestCheckHook: init Motivation for this hook is simple: there's no single documented way to do trivial things with ctest: 1. Pass additional flags to ctest invocation. 2. Selectively disable tests in a mechanism similar to python's `disabledTests` or rust's composable skips in `checkFlags`. 3. Disable parallel checking. Current state of things has lead to several different solutions: 1. Completely overriding `checkPhase` [1] and invoking ctest manually with the necessary flags. This is most often coupled with `-E` for disabling test or setting parallel level. 2. Wrangling with weird double string/regex escaping and trying to stuff additional parameters and/or exclusion regex via `CMAKE_CTEST_ARGUMENTS`. This approach is especially painful when test names have spaces. This is the reason I originally decided to implement this hook after wrangling with failing darwin tests here [2]. 3. Stuffing additional arguments into `checkFlagsArray` with the `ARGS` makefile parameter [3]. I don't see any reason to keep the status-quo. Doing something along these lines has been suggested [4] for both `ctest` and `meson`. Meson setup-hook has switched from `ninja` to `meson` in [5] with little friction. Doing the same for cmake in a single sweep would prove problematic due to the aforementioned zoo of workarounds and hacks for `ctest`. Doing it via a separate hook would allow us to refactor things piecemeal and without going through staging. The benefit of the hook is immediately clear and it would allow to drive the refactor tractor at a comfortable pace. [1]: pd/pdal/package.nix:117, cc/ccache/package.nix:108, gl/glog/package.nix:79 [2]: https://www.github.com/NixOS/nixpkgs/pull/375955 [3]: op/open62541/package.nix:114 [4]: https://www.github.com/NixOS/nixpkgs/issues/113829 [5]: https://www.github.com/NixOS/nixpkgs/pull/213845 --- doc/hooks/cmake.section.md | 18 ++++++++ doc/redirects.json | 12 +++++ .../ct/ctestCheckHook/ctest-check-hook.sh | 44 +++++++++++++++++++ pkgs/by-name/ct/ctestCheckHook/package.nix | 9 ++++ 4 files changed, 83 insertions(+) create mode 100644 pkgs/by-name/ct/ctestCheckHook/ctest-check-hook.sh create mode 100644 pkgs/by-name/ct/ctestCheckHook/package.nix diff --git a/doc/hooks/cmake.section.md b/doc/hooks/cmake.section.md index 5bcc3c980fc5..4aecc8440c64 100644 --- a/doc/hooks/cmake.section.md +++ b/doc/hooks/cmake.section.md @@ -33,3 +33,21 @@ The default value is `build`. #### `dontUseCmakeConfigure` {#dont-use-cmake-configure} When set to true, don't use the predefined `cmakeConfigurePhase`. + +## Controlling CTest invocation {#cmake-ctest} + +By default tests are run by make in [`checkPhase`](#ssec-check-phase) or by [ninja](#ninja) if `ninja` is +available in `nativeBuildInputs`. Makefile and Ninja generators produce the `test` target, which invokes `ctest` under the hood. +This makes passing additional arguments to `ctest` difficult, so it's possible to invoke it directly in `checkPhase` +by adding `ctestCheckHook` to `nativeCheckInputs`. + +### CTest Variables {#cmake-ctest-variables} + +#### `disabledTests` {#cmake-ctest-disabled-tests} + +Allows to disable running a list of tests. Note that regular expressions are not supported by `disabledTests`, but +it can be combined with `--exclude-regex` option. + +#### `ctestFlags` {#cmake-ctest-flags} + +Additional options passed to `ctest` together with `checkFlags`. diff --git a/doc/redirects.json b/doc/redirects.json index ad13136e23ef..539dd159ce01 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -5,6 +5,18 @@ "chap-release-notes": [ "release-notes.html#chap-release-notes" ], + "cmake-ctest": [ + "index.html#cmake-ctest" + ], + "cmake-ctest-disabled-tests": [ + "index.html#cmake-ctest-disabled-tests" + ], + "cmake-ctest-flags": [ + "index.html#cmake-ctest-flags" + ], + "cmake-ctest-variables": [ + "index.html#cmake-ctest-variables" + ], "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], diff --git a/pkgs/by-name/ct/ctestCheckHook/ctest-check-hook.sh b/pkgs/by-name/ct/ctestCheckHook/ctest-check-hook.sh new file mode 100644 index 000000000000..65d39923f967 --- /dev/null +++ b/pkgs/by-name/ct/ctestCheckHook/ctest-check-hook.sh @@ -0,0 +1,44 @@ +# shellcheck shell=bash disable=SC2154 + +ctestCheckHook() { + echo "Executing ctestCheckHook" + + runHook preCheck + + local buildCores=1 + + if [ "${enableParallelChecking-1}" ]; then + buildCores="$NIX_BUILD_CORES" + fi + + local flagsArray=( + "-j$buildCores" + # This is enabled by the cmakeConfigurePhase by exporting + # CTEST_OUTPUT_ON_FAILURE, but it makes sense it enable it globally here + # as well. + "--output-on-failure" + ) + + local disabledTestsArray=() + concatTo disabledTestsArray disabledTests + + if [ ${#disabledTestsArray[@]} -ne 0 ]; then + local ctestExcludedTestsFile=$NIX_BUILD_TOP/.ctest-excluded-tests + disabledTestsString="$(concatStringsSep "\n" disabledTestsArray)" + echo -e "$disabledTestsString" >"$ctestExcludedTestsFile" + flagsArray+=("--exclude-from-file" "$ctestExcludedTestsFile") + fi + + concatTo flagsArray ctestFlags checkFlags checkFlagsArray + + echoCmd 'ctest flags' "${flagsArray[@]}" + ctest "${flagsArray[@]}" + + echo "Finished ctestCheckHook" + + runHook postCheck +} + +if [ -z "${dontUseCTestCheck-}" ] && [ -z "${checkPhase-}" ]; then + checkPhase=ctestCheckHook +fi diff --git a/pkgs/by-name/ct/ctestCheckHook/package.nix b/pkgs/by-name/ct/ctestCheckHook/package.nix new file mode 100644 index 000000000000..631b2ba8b86a --- /dev/null +++ b/pkgs/by-name/ct/ctestCheckHook/package.nix @@ -0,0 +1,9 @@ +{ + makeSetupHook, + cmake, +}: + +makeSetupHook { + name = "ctestCheckHook"; + propagatedBuildInputs = [ cmake ]; +} ./ctest-check-hook.sh