Part 12: Creating and Running Tests with CTest

Enabling Testing

CTest is CMake's companion tool for running tests. Before you can add any tests, you must explicitly enable testing support in your project. There are two approaches, each with different capabilities.

enable_testing() vs include(CTest)

cmake_minimum_required(VERSION 3.24)
project(MyApp LANGUAGES CXX)

# Option A: Minimal — just enables add_test()
enable_testing()

# Option B: Full CTest support — adds BUILD_TESTING option,
# CDash submission support, and memcheck/coverage targets
include(CTest)

The key difference: include(CTest) calls enable_testing() internally but also creates the BUILD_TESTING cache variable (defaults to ON) and adds dashboard submission support.

The BUILD_TESTING Pattern

Use BUILD_TESTING to let users opt out of building test infrastructure:

cmake_minimum_required(VERSION 3.24)
project(MyLibrary LANGUAGES CXX)

# This creates BUILD_TESTING as a cache variable (default ON)
include(CTest)

add_library(mylib src/mylib.cpp)
target_include_directories(mylib PUBLIC include)

# Only build test targets when testing is enabled
if(BUILD_TESTING)
    add_executable(test_mylib tests/test_mylib.cpp)
    target_link_libraries(test_mylib PRIVATE mylib)

    add_test(NAME mylib_basic COMMAND test_mylib)
endif()

Adding Tests

The add_test() command registers a test with CTest. A test is simply a command that CTest runs and checks the exit code — zero means pass, non-zero means fail.

cmake_minimum_required(VERSION 3.24)
project(Calculator LANGUAGES CXX)

include(CTest)

# Build the test executable
add_executable(test_calc tests/test_calculator.cpp)
target_link_libraries(test_calc PRIVATE calculator_lib)

# Register tests — each is an independent invocation
add_test(NAME calc_addition    COMMAND test_calc --test=add)
add_test(NAME calc_subtraction COMMAND test_calc --test=sub)
add_test(NAME calc_multiply    COMMAND test_calc --test=mul)
add_test(NAME calc_division    COMMAND test_calc --test=div)

Working Directory and Arguments

cmake_minimum_required(VERSION 3.24)
project(FileProcessor LANGUAGES CXX)

include(CTest)

add_executable(test_processor tests/test_processor.cpp)

# Set working directory so the test can find fixture files
add_test(
    NAME processor_csv_parse
    COMMAND test_processor --input sample.csv --format csv
    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests/fixtures
)

# Use generator expressions for the command path
add_test(
    NAME processor_json_parse
    COMMAND $ --input sample.json --format json
    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests/fixtures
)

        flowchart LR
            A[CMakeLists.txt] -->|"enable_testing()\nadd_test()"| B[CTestTestfile.cmake]
            B -->|"ctest"| C[Execute Tests]
            C --> D{Exit Code}
            D -->|"0"| E[PASS]
            D -->|"!= 0"| F[FAIL]
            C -->|"--output-junit"| G[JUnit XML]
            C -->|"-T Test"| H[CDash Submission]
            style A fill:#f8f9fa,stroke:#3B9797
            style B fill:#f8f9fa,stroke:#16476A
            style C fill:#f8f9fa,stroke:#16476A
            style E fill:#d4edda,stroke:#28a745
            style F fill:#f8d7da,stroke:#dc3545
    

Test Properties

Test properties control how CTest executes and evaluates each test. Set them with set_tests_properties():

TIMEOUT

cmake_minimum_required(VERSION 3.24)
project(NetworkTests LANGUAGES CXX)

include(CTest)

add_executable(test_network tests/test_network.cpp)

add_test(NAME network_connect COMMAND test_network --test=connect)
add_test(NAME network_timeout COMMAND test_network --test=slow_response)
add_test(NAME network_stress  COMMAND test_network --test=stress)

# Set timeout per test (seconds)
set_tests_properties(network_connect PROPERTIES TIMEOUT 10)
set_tests_properties(network_timeout PROPERTIES TIMEOUT 30)
set_tests_properties(network_stress  PROPERTIES TIMEOUT 120)

# Set properties on multiple tests at once
set_tests_properties(
    network_connect network_timeout network_stress
    PROPERTIES
        ENVIRONMENT "TEST_SERVER=localhost;TEST_PORT=8080"
)

PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION

Instead of relying solely on exit codes, match test output against patterns:

cmake_minimum_required(VERSION 3.24)
project(OutputTests LANGUAGES CXX)

include(CTest)

add_executable(test_output tests/test_output.cpp)

# Test passes only if stdout contains "All checks passed"
add_test(NAME output_success COMMAND test_output --mode=success)
set_tests_properties(output_success PROPERTIES
    PASS_REGULAR_EXPRESSION "All checks passed"
)

# Test fails if output contains any error marker
add_test(NAME output_no_errors COMMAND test_output --mode=mixed)
set_tests_properties(output_no_errors PROPERTIES
    FAIL_REGULAR_EXPRESSION "ERROR|FATAL|CRITICAL"
)

# Combine: must match pass AND must not match fail
add_test(NAME output_strict COMMAND test_output --mode=verbose)
set_tests_properties(output_strict PROPERTIES
    PASS_REGULAR_EXPRESSION "Result: OK"
    FAIL_REGULAR_EXPRESSION "Warning:|Error:"
)

cmake_minimum_required(VERSION 3.24)
project(ScriptTests LANGUAGES NONE)

include(CTest)

# Test a Python script
add_test(
    NAME python_validation
    COMMAND python3 ${CMAKE_SOURCE_DIR}/scripts/validate.py --strict
)
set_tests_properties(python_validation PROPERTIES
    PASS_REGULAR_EXPRESSION "Validation complete: 0 errors"
    TIMEOUT 60
)

# Test a shell script
add_test(
    NAME setup_check
    COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/check_environment.sh
)
set_tests_properties(setup_check PROPERTIES
    FAIL_REGULAR_EXPRESSION "MISSING|NOT FOUND"
)

Running Tests

The ctest Command

# Run all tests from the build directory
cd build
ctest

# Verbose output — show each test's stdout/stderr
ctest --output-on-failure

# Extra verbose — show command being run and all output
ctest -VV

# Show test output only for failed tests (most common usage)
ctest --output-on-failure --progress

Filtering with -R and -E

# Run only tests matching a regex pattern
ctest -R "network"           # Runs network_connect, network_timeout, etc.
ctest -R "^calc_"            # Runs calc_addition, calc_subtraction, etc.

# Exclude tests matching a pattern
ctest -E "stress|slow"       # Skip stress and slow tests

# Combine include and exclude
ctest -R "network" -E "stress"  # Network tests except stress

# Run a specific test by number (from ctest -N listing)
ctest -I 3,5                 # Run tests #3 through #5

# List all tests without running them
ctest -N

Parallel Execution

CTest can run independent tests in parallel for faster feedback:

# Run up to 8 tests simultaneously
ctest -j8

# Use all available cores
ctest -j$(nproc)

# On Windows (PowerShell)
ctest -j $env:NUMBER_OF_PROCESSORS

Some tests cannot run in parallel (e.g., tests that bind to the same port or modify shared files). Use RESOURCE_LOCK to serialize them:

cmake_minimum_required(VERSION 3.24)
project(DatabaseTests LANGUAGES CXX)

include(CTest)

add_executable(test_db tests/test_database.cpp)

add_test(NAME db_insert  COMMAND test_db --test=insert)
add_test(NAME db_update  COMMAND test_db --test=update)
add_test(NAME db_delete  COMMAND test_db --test=delete)
add_test(NAME db_migrate COMMAND test_db --test=migrate)

# These tests modify the same database — never run them simultaneously
set_tests_properties(db_insert db_update db_delete db_migrate
    PROPERTIES RESOURCE_LOCK "database"
)

# This test uses 4 cores — tell CTest so it schedules correctly
add_test(NAME db_parallel_load COMMAND test_db --test=parallel_load)
set_tests_properties(db_parallel_load PROPERTIES PROCESSORS 4)

Test Fixtures

Fixtures provide setup/teardown semantics — ensuring that prerequisite tests run before dependent tests and cleanup runs afterward:

cmake_minimum_required(VERSION 3.24)
project(IntegrationTests LANGUAGES CXX)

include(CTest)

add_executable(test_integration tests/test_integration.cpp)

# Setup: start the test server
add_test(NAME start_server COMMAND test_integration --action=start_server)
set_tests_properties(start_server PROPERTIES
    FIXTURES_SETUP "ServerFixture"
    TIMEOUT 10
)

# Cleanup: stop the test server
add_test(NAME stop_server COMMAND test_integration --action=stop_server)
set_tests_properties(stop_server PROPERTIES
    FIXTURES_CLEANUP "ServerFixture"
    TIMEOUT 10
)

# Tests that require the server to be running
add_test(NAME api_get    COMMAND test_integration --action=test_get)
add_test(NAME api_post   COMMAND test_integration --action=test_post)
add_test(NAME api_delete COMMAND test_integration --action=test_delete)

set_tests_properties(api_get api_post api_delete PROPERTIES
    FIXTURES_REQUIRED "ServerFixture"
    TIMEOUT 30
)

        flowchart TD
            A[start_server] -->|FIXTURES_SETUP| B[ServerFixture]
            B -->|FIXTURES_REQUIRED| C[api_get]
            B -->|FIXTURES_REQUIRED| D[api_post]
            B -->|FIXTURES_REQUIRED| E[api_delete]
            C --> F[stop_server]
            D --> F
            E --> F
            F -->|FIXTURES_CLEANUP| B
            style A fill:#d4edda,stroke:#28a745
            style F fill:#f8d7da,stroke:#dc3545
            style B fill:#fff3cd,stroke:#ffc107
            style C fill:#f8f9fa,stroke:#3B9797
            style D fill:#f8f9fa,stroke:#3B9797
            style E fill:#f8f9fa,stroke:#3B9797
    

cmake_minimum_required(VERSION 3.24)
project(FullStackTests LANGUAGES CXX)

include(CTest)

add_executable(test_fullstack tests/test_fullstack.cpp)

# Database fixture
add_test(NAME db_setup    COMMAND test_fullstack --setup-db)
add_test(NAME db_teardown COMMAND test_fullstack --teardown-db)
set_tests_properties(db_setup    PROPERTIES FIXTURES_SETUP   "DB")
set_tests_properties(db_teardown PROPERTIES FIXTURES_CLEANUP "DB")

# Server fixture (requires database)
add_test(NAME server_start COMMAND test_fullstack --start-server)
add_test(NAME server_stop  COMMAND test_fullstack --stop-server)
set_tests_properties(server_start PROPERTIES
    FIXTURES_SETUP    "Server"
    FIXTURES_REQUIRED "DB"  # Server needs DB to be ready
)
set_tests_properties(server_stop PROPERTIES FIXTURES_CLEANUP "Server")

# Integration tests require both
add_test(NAME e2e_login COMMAND test_fullstack --test=login)
set_tests_properties(e2e_login PROPERTIES
    FIXTURES_REQUIRED "Server;DB"
)

Labels and Filtering

Labels let you categorize tests and run subsets without knowing individual test names:

cmake_minimum_required(VERSION 3.24)
project(LabeledTests LANGUAGES CXX)

include(CTest)

add_executable(test_suite tests/test_suite.cpp)

add_test(NAME unit_math      COMMAND test_suite --suite=math)
add_test(NAME unit_string    COMMAND test_suite --suite=string)
add_test(NAME integ_api      COMMAND test_suite --suite=api)
add_test(NAME integ_database COMMAND test_suite --suite=database)
add_test(NAME perf_benchmark COMMAND test_suite --suite=benchmark)

# Assign labels for categorization
set_tests_properties(unit_math unit_string PROPERTIES
    LABELS "unit"
)
set_tests_properties(integ_api integ_database PROPERTIES
    LABELS "integration"
)
set_tests_properties(perf_benchmark PROPERTIES
    LABELS "performance;slow"
)

# Run only unit tests
ctest -L unit

# Run integration tests
ctest -L integration

# Exclude slow tests
ctest -LE slow

# Combine label filters
ctest -L "unit|integration" -LE slow

Test Discovery

For large projects with many test files, manually listing each test in CMakeLists.txt is tedious. Batch registration patterns help:

cmake_minimum_required(VERSION 3.24)
project(DiscoveryPatterns LANGUAGES CXX)

include(CTest)

# Pattern: One executable per test file, auto-discovered
file(GLOB TEST_SOURCES tests/test_*.cpp)

foreach(test_source ${TEST_SOURCES})
    # Extract filename without extension: tests/test_math.cpp → test_math
    get_filename_component(test_name ${test_source} NAME_WE)

    add_executable(${test_name} ${test_source})
    target_link_libraries(${test_name} PRIVATE mylib)

    add_test(NAME ${test_name} COMMAND ${test_name})
    set_tests_properties(${test_name} PROPERTIES
        LABELS "unit"
        TIMEOUT 30
    )
endforeach()

Expected Failures

Sometimes you need to mark tests that are expected to fail, or temporarily disable tests:

cmake_minimum_required(VERSION 3.24)
project(FailureTests LANGUAGES CXX)

include(CTest)

add_executable(test_app tests/test_app.cpp)

# WILL_FAIL: test passes when the command returns non-zero
# Useful for testing that invalid input is rejected
add_test(NAME reject_invalid_input COMMAND test_app --input=invalid)
set_tests_properties(reject_invalid_input PROPERTIES WILL_FAIL TRUE)

# DISABLED: test is skipped entirely (shown as "Not Run")
add_test(NAME broken_feature COMMAND test_app --test=broken)
set_tests_properties(broken_feature PROPERTIES DISABLED TRUE)

# SKIP_RETURN_CODE: specific exit code means "skip" not "fail"
# Return code 77 means the test environment isn't available
add_test(NAME optional_gpu_test COMMAND test_app --test=gpu)
set_tests_properties(optional_gpu_test PROPERTIES
    SKIP_RETURN_CODE 77
)

cmake_minimum_required(VERSION 3.24)
project(PlatformTests LANGUAGES CXX)

include(CTest)

add_executable(test_platform tests/test_platform.cpp)

# Only add Linux-specific tests on Linux
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
    add_test(NAME linux_inotify COMMAND test_platform --test=inotify)
    set_tests_properties(linux_inotify PROPERTIES LABELS "linux;filesystem")
endif()

# Windows-specific tests
if(WIN32)
    add_test(NAME win_registry COMMAND test_platform --test=registry)
    set_tests_properties(win_registry PROPERTIES LABELS "windows;registry")
endif()

# Cross-platform tests always available
add_test(NAME xplat_filesystem COMMAND test_platform --test=filesystem)
set_tests_properties(xplat_filesystem PROPERTIES LABELS "crossplatform")

CTest Scripting

CTest can execute scripted workflows using ctest -S script.cmake. This enables automated dashboard submissions to CDash and complex CI workflows:

# ctest_script.cmake — run with: ctest -S ctest_script.cmake
set(CTEST_SOURCE_DIRECTORY "/path/to/source")
set(CTEST_BINARY_DIRECTORY "/path/to/build")

set(CTEST_SITE "ci-server-01")
set(CTEST_BUILD_NAME "Linux-GCC12-Release")

# Configure
ctest_start("Continuous")
ctest_configure()

# Build
ctest_build(NUMBER_ERRORS num_errors NUMBER_WARNINGS num_warnings)
message(STATUS "Build: ${num_errors} errors, ${num_warnings} warnings")

# Test
ctest_test(
    PARALLEL_LEVEL 8
    RETURN_VALUE test_result
    EXCLUDE_LABEL "slow"
)

# Submit results to CDash (if configured)
# ctest_submit()

if(NOT test_result EQUAL 0)
    message(FATAL_ERROR "Tests failed!")
endif()

# Run the script
ctest -S ctest_script.cmake

# Useful CTest scripting commands:
# ctest_start()     — Begin a testing session
# ctest_configure() — Run CMake configure
# ctest_build()     — Build the project
# ctest_test()      — Run tests
# ctest_coverage()  — Collect coverage data
# ctest_memcheck()  — Run memory analysis
# ctest_submit()    — Upload to CDash