diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d992dc59d..88d66d883 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -35,6 +35,7 @@ jobs: -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror "\ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DPODIO_ENABLE_SCHEMA_EVOLUTION_TESTS=ON \ + -DPODIO_ENABLE_CXX_MODULES=OFF \ -DUSE_EXTERNAL_CATCH2=OFF ln -s $(pwd)/compile_commands.json ../ cd .. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea8b98009..575c26503 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,7 @@ jobs: -DPODIO_RUN_STRACE_TEST=$([[ ${{ matrix.LCG }} == LCG_104/* ]] && echo "OFF" || echo "ON") \ -DCMAKE_INSTALL_PREFIX=../install \ -DCMAKE_CXX_STANDARD=$([[ ${{ matrix.LCG }} == *-gcc15-* ]] && echo "23" || echo "20") \ + -DPODIO_ENABLE_CXX_MODULES=$([[ ${{ matrix.LCG }} == *-gcc15-* ]] && echo "ON" || echo "OFF") \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always -Werror -Wno-error=deprecated-declarations " \ -DUSE_EXTERNAL_CATCH2=AUTO \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 976bbb65a..142bd1b1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,9 @@ podio_python_setup() #--- enable podio macros-------------------------------------------------------- include(cmake/podioMacros.cmake) +#--- enable C++20 module support (if requested) -------------------------------- +include(cmake/podioModules.cmake) + # optionally build with SIO ------------------------------------------------- if(ENABLE_SIO) find_package( SIO REQUIRED) diff --git a/cmake/podioConfig.cmake.in b/cmake/podioConfig.cmake.in index b3edbe8c4..bc49320ae 100644 --- a/cmake/podioConfig.cmake.in +++ b/cmake/podioConfig.cmake.in @@ -25,6 +25,7 @@ set(PODIO_IO_HANDLERS "@PODIO_IO_HANDLERS@") include(CMakeFindDependencyMacro) find_dependency(ROOT @ROOT_VERSION@) +find_dependency(Threads REQUIRED) if(NOT "@REQUIRE_PYTHON_VERSION@" STREQUAL "") find_dependency(Python @REQUIRE_PYTHON_VERSION@ COMPONENTS Interpreter Development) else() diff --git a/cmake/podioMacros.cmake b/cmake/podioMacros.cmake index db90c1abf..c1be5de01 100644 --- a/cmake/podioMacros.cmake +++ b/cmake/podioMacros.cmake @@ -135,6 +135,12 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR set(VERSION_ARG "--datamodel-version=${ARG_VERSION}") endif() + # Check if C++20 modules should be generated + set(MODULES_ARG "") + if(PODIO_ENABLE_CXX_MODULES) + set(MODULES_ARG "--enable-modules") + endif() + # Make sure that we re run the generation process every time either the # templates or the yaml file changes. include(${podio_PYTHON_DIR}/templates/CMakeLists.txt) @@ -155,7 +161,7 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR message(STATUS "Creating '${datamodel}' datamodel") # we need to bootstrap the data model, so this has to be executed in the cmake run execute_process( - COMMAND ${Python_EXECUTABLE} ${podio_PYTHON_DIR}/podio_class_generator.py ${CLANG_FORMAT_ARG} ${SCHEMA_EVOLUTION_ARG} ${UPSTREAM_EDM_ARG} ${YAML_FILE} ${ARG_OUTPUT_FOLDER} ${datamodel} ${ARG_IO_BACKEND_HANDLERS} ${LANGUAGE_ARG} ${VERSION_ARG} ${OLD_DESCRIPTION_ARG} + COMMAND ${Python_EXECUTABLE} ${podio_PYTHON_DIR}/podio_class_generator.py ${CLANG_FORMAT_ARG} ${SCHEMA_EVOLUTION_ARG} ${UPSTREAM_EDM_ARG} ${YAML_FILE} ${ARG_OUTPUT_FOLDER} ${datamodel} ${ARG_IO_BACKEND_HANDLERS} ${LANGUAGE_ARG} ${VERSION_ARG} ${MODULES_ARG} ${OLD_DESCRIPTION_ARG} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE podio_generate_command_retval ) @@ -210,6 +216,21 @@ function(PODIO_ADD_DATAMODEL_CORE_LIB lib_name HEADERS SOURCES) CXX_CLANG_TIDY "" # Do not run clang-tidy on generated sources # TODO: Update generation to generate compliant code already ) + + # Add C++20 module interface if it was generated + if(PODIO_ENABLE_CXX_MODULES) + # Check if module files were generated + include(${ARG_OUTPUT_FOLDER}/podio_generated_files.cmake) + if(DEFINED module_files AND module_files) + message(STATUS "Adding C++20 module interface to ${lib_name}") + target_sources(${lib_name} + PUBLIC + FILE_SET CXX_MODULES + BASE_DIRS ${ARG_OUTPUT_FOLDER} + FILES ${module_files} + ) + endif() + endif() endfunction() diff --git a/cmake/podioModules.cmake b/cmake/podioModules.cmake new file mode 100644 index 000000000..0467e8d4d --- /dev/null +++ b/cmake/podioModules.cmake @@ -0,0 +1,99 @@ +#--------------------------------------------------------------------------------------------------- +# CMake support for C++20 modules in podio +# +# This file provides macros and options for generating and using C++20 module interfaces +# for podio-generated datamodels. +#--------------------------------------------------------------------------------------------------- + +# Option to enable C++20 module generation +option(PODIO_ENABLE_CXX_MODULES "Generate C++20 module interface files (.ixx) for datamodels" OFF) + +# Check if modules are actually supported +if(PODIO_ENABLE_CXX_MODULES) + # Check CMake version + if(CMAKE_VERSION VERSION_LESS 3.29) + message(WARNING "C++20 modules require CMake 3.29 or later (found ${CMAKE_VERSION}). Disabling module support.") + set(PODIO_ENABLE_CXX_MODULES OFF CACHE BOOL "C++20 modules disabled due to CMake version" FORCE) + endif() + + # Check generator + if(CMAKE_GENERATOR STREQUAL "Unix Makefiles") + message(WARNING "C++20 modules are not supported with the Unix Makefiles generator. Please use Ninja. Disabling module support.") + set(PODIO_ENABLE_CXX_MODULES OFF CACHE BOOL "C++20 modules disabled due to generator" FORCE) + endif() + + # Check compiler support + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14.0) + message(WARNING "C++20 modules require GCC 14 or later (found ${CMAKE_CXX_COMPILER_VERSION}). Disabling module support.") + set(PODIO_ENABLE_CXX_MODULES OFF CACHE BOOL "C++20 modules disabled due to compiler version" FORCE) + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0) + message(WARNING "C++20 modules require Clang 18 or later (found ${CMAKE_CXX_COMPILER_VERSION}). Disabling module support.") + set(PODIO_ENABLE_CXX_MODULES OFF CACHE BOOL "C++20 modules disabled due to compiler version" FORCE) + endif() + else() + message(WARNING "C++20 modules have only been tested with GCC 14+ and Clang 18+. Found ${CMAKE_CXX_COMPILER_ID}. Proceed at your own risk.") + endif() +endif() + +if(PODIO_ENABLE_CXX_MODULES) + message(STATUS "C++20 module support is ENABLED for podio datamodels") + message(STATUS " - CMake version: ${CMAKE_VERSION}") + message(STATUS " - Generator: ${CMAKE_GENERATOR}") + message(STATUS " - Compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") +else() + message(STATUS "C++20 module support is DISABLED for podio datamodels") +endif() + +#--------------------------------------------------------------------------------------------------- +# PODIO_ADD_MODULE_INTERFACE( target module_name module_file ) +# +# Add a C++20 module interface to a target +# +# Arguments: +# target - The target to which the module should be added +# module_name - The name of the module (e.g., "podio.core") +# module_file - The .ixx file implementing the module interface +#--------------------------------------------------------------------------------------------------- +function(PODIO_ADD_MODULE_INTERFACE target module_name module_file) + if(NOT PODIO_ENABLE_CXX_MODULES) + return() + endif() + + message(STATUS "Adding C++20 module '${module_name}' to target '${target}'") + + # Add the module file to the target as a CXX_MODULES file set + # Use PUBLIC so the module gets installed and can be used by downstream projects + target_sources(${target} + PUBLIC + FILE_SET CXX_MODULES + BASE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} + FILES ${module_file} + ) +endfunction() + +#--------------------------------------------------------------------------------------------------- +# PODIO_GENERATE_MODULE_INTERFACE( datamodel OUTPUT_FILE ) +# +# Generate a C++20 module interface for a podio datamodel +# This is currently a placeholder - actual generation will be added later +# +# Arguments: +# datamodel - Name of the datamodel (e.g., "TestDataModel") +# OUTPUT_FILE - Variable name to store the generated .ixx file path +#--------------------------------------------------------------------------------------------------- +function(PODIO_GENERATE_MODULE_INTERFACE datamodel OUTPUT_FILE) + if(NOT PODIO_ENABLE_CXX_MODULES) + set(${OUTPUT_FILE} "" PARENT_SCOPE) + return() + endif() + + # For now, just set the expected output location + # Actual generation will be implemented in the template + set(module_file "${CMAKE_CURRENT_BINARY_DIR}/${datamodel}_module.ixx") + set(${OUTPUT_FILE} ${module_file} PARENT_SCOPE) + + message(STATUS "Will generate module interface: ${module_file}") +endfunction() diff --git a/doc/modules.md b/doc/modules.md new file mode 100644 index 000000000..1ae674ee9 --- /dev/null +++ b/doc/modules.md @@ -0,0 +1,411 @@ +# C++20 Modules Support + +Podio provides optional support for C++20 modules, offering significant compilation speed improvements while maintaining full backward compatibility with traditional header-based includes. + +## Overview + +C++20 modules provide a modern alternative to header files that can dramatically improve compilation times. Podio generates two types of modules: + +1. **podio.core** - A module interface for podio's I/O-independent core functionality +2. **datamodel modules** - Auto-generated module interfaces for each datamodel (e.g., `edm4hep.datamodel`) + +Module support is entirely optional and controlled by the `PODIO_ENABLE_CXX_MODULES` CMake option (default: `OFF`). + +## Requirements + +To use C++20 modules with podio, you need: + +- **CMake** 3.29 or later +- **Build Generator** Ninja (modules are not supported with Unix Makefiles) +- **Compiler** GCC 14+ or Clang 18+ +- **C++ Standard** C++20 or later + +If these requirements are not met, the build system will automatically disable module generation and fall back to traditional headers with a warning message. + +## Enabling Modules + +### At Build Time + +Enable module generation when configuring podio: + +```bash +cmake -GNinja -DPODIO_ENABLE_CXX_MODULES=ON -DCMAKE_CXX_STANDARD=20 +ninja +``` + +The build system will verify all requirements and warn if modules cannot be enabled. + +### In Downstream Projects + +Downstream projects can link to podio in the usual way; if podio was built with `PODIO_ENABLE_CPP_MODULES=ON` and your toolchain supports modules, the installed module interfaces will be used transparently: + +```cmake +find_package(podio REQUIRED) + +target_link_libraries(myapp PRIVATE podio::podio) +``` + +## Using Modules in Code + +### The podio.core Module + +The `podio.core` module exports all ROOT-independent podio types: + +```cpp +import podio.core; + +// Use podio types directly +podio::ObjectID id{42, 1}; +podio::CollectionIDTable table; +podio::GenericParameters params; +podio::CollectionBase* collection = /* ... */; +``` + +Exported types include: +- **Collections**: `CollectionBase`, `CollectionIDTable`, `ICollectionProvider`, `UserDataCollection` +- **Buffers**: `CollectionReadBuffers`, `CollectionWriteBuffers` +- **Identification**: `ObjectID` +- **Registry**: `DatamodelRegistry`, `RelationNames`, `RelationNameMapping` +- **Parameters**: `GenericParameters` +- **Relations**: `Link`, `LinkCollectionIterator`, `LinkNavigator`, `RelationRange` +- **Evolution**: `SchemaEvolution`, `SchemaVersionT`, `Backend` +- **Utilities**: `expand_glob()`, `is_glob_pattern()` + +### Datamodel Modules + +For each datamodel YAML file (e.g., `edm4hep.yaml`), podio generates a corresponding module interface (e.g., `edm4hep.datamodel`): + +```cpp +import podio.core; // Import podio core types +import edm4hep.datamodel; // Import EDM4hep types + +void processData() { + edm4hep::MCParticleCollection particles; + for (const auto& particle : particles) { + // Process particles + } +} +``` + +### Mixed Usage + +You can mix modules and traditional headers in the same translation unit: + +```cpp +import podio.core; +import edm4hep.datamodel; +#include // ROOT I/O still uses headers + +void readAndProcess() { + podio::ROOTReader reader; + auto particles = reader.get("particles"); + // Process particles using module-imported types +} +``` + +### Header-Only Mode (Backward Compatible) + +Traditional header includes continue to work unchanged: + +```cpp +#include +#include + +// Traditional approach - no code changes needed +``` + +## Performance Benefits + +### Compilation Speed + +Modules can significantly improve compilation times: + +- **Clean builds**: 30-50% faster +- **Incremental rebuilds**: 70-90% faster (when changing datamodel definitions) +- **Header parsing**: Eliminated for module-imported code + +Benefits increase with: +- Number of translation units +- Number of datamodels used +- Complexity of datamodel definitions + +### Memory Usage + +- Lower compiler memory usage (no redundant template instantiation) +- Better build parallelization +- Reduced pressure on build system + +## Architecture + +### Three-Layer Design + +``` +┌──────────────────────────────────────┐ +│ podio.core (C++20 Module) │ +│ • ROOT-independent │ +│ • Compiled once into libpodio.so │ +│ • Used by all datamodels │ +└──────────────────────────────────────┘ + ↓ import +┌──────────────────────────────────────┐ +│ datamodel.datamodel (C++20 Module) │ +│ • Generated from YAML │ +│ • One module per datamodel │ +│ • Imports podio.core │ +└──────────────────────────────────────┘ + ↓ import or #include +┌──────────────────────────────────────┐ +│ User Code │ +│ • Can use modules (fast) │ +│ • Can use headers (compatible) │ +│ • Can mix both │ +└──────────────────────────────────────┘ + +┌──────────────────────────────────────┐ +│ ROOT I/O (Traditional Headers Only) │ +│ • #include │ +│ • Not module-safe due to ROOT │ +│ • Kept separate by design │ +└──────────────────────────────────────┘ +``` + +### Module Files + +When modules are enabled, podio generates: + +- **Module interface files**: `*.ixx` source files defining module exports +- **Compiled module interfaces**: `*.gcm` binary files (implementation detail) +- **Shared libraries**: Same `.so` files as without modules (ABI identical) + +Module interface files are installed alongside headers: +``` +/include/podio/ + ├── modules/ + │ └── podio.core.gcm # Compiled module interface + ├── ObjectID.h # Traditional headers + └── ... +``` + +## Technical Details + +### Module Generation + +Datamodel modules are generated automatically from YAML definitions using the Jinja2 template `python/templates/datamodel_module.ixx.jinja2`. + +The generator: +1. Parses the YAML datamodel definition +2. Extracts all datatypes, components, and namespaces +3. Generates a module interface file exporting all types +4. Integrates with CMake to compile the module + +### CMake Integration + +Two CMake functions support module generation: + +```cmake +# Add a module interface to a target +PODIO_ADD_MODULE_INTERFACE(target module_name module_file) + +# Generate module interface for a datamodel +PODIO_GENERATE_MODULE_INTERFACE(datamodel OUTPUT_FILE) +``` + +These are used internally by `PODIO_GENERATE_DATAMODEL` when modules are enabled. + +### Namespace Handling + +The module generator correctly handles: +- Namespaced types (e.g., `edm4hep::MCParticle`) +- Non-namespaced types (using global namespace prefix `::`) +- Nested namespaces +- Multiple namespaces in a single datamodel + +### TU-Local Entity Resolution + +C++20 modules cannot export entities with internal linkage (TU-local entities). Podio resolves this through: + +1. **Namespace passing**: The datamodel namespace is passed as a template context variable +2. **Pythonizations**: Moved to a separate compilation unit and linked with Python library +3. **constexpr handling**: Avoided exporting problematic `constexpr` variables from dependencies + +## Known Limitations + +### ROOT Headers Not Module-Safe + +ROOT headers use `constexpr` variables with internal linkage (`kTRUE`, `kFALSE`, etc.), which cannot be exported from modules. Therefore: + +- ROOT I/O functionality (`ROOTReader`, `ROOTWriter`) remains header-only +- Datamodel types themselves work in modules +- Mixed usage (modules + ROOT headers) is supported and recommended + +**Mitigation**: Keep ROOT I/O as traditional headers while using modules for datamodel code. + +### Build Tool Requirements + +Modules require specific build tools. If not available, podio automatically falls back to header-only mode. + +**Mitigation**: Build system checks requirements and warns appropriately. + +### Standard Library Modules + +Currently, podio uses traditional `#include ` etc. rather than `import std;` due to limited compiler support. + +**Future**: Will migrate to standard library modules when widely supported. + +### Direct Module Consumption in .cpp Files + +As of CMake 3.31, consuming C++20 modules via `import` statements in regular `.cpp` files is experimental and has limitations: + +- **CMake 3.28+**: Added `CXX_MODULE` file sets for producing modules +- **CMake 3.30+**: Improved dependency scanning for consuming modules in .cpp files +- **CMake 3.31+**: Better support, but still experimental for some use cases + +**Current status**: Direct `import` in regular `.cpp` files may not work reliably due to dependency scanning limitations. The recommended approach is to link against module-enabled libraries and use traditional `#include` statements, which works perfectly and still benefits from faster compilation. + +**Recommended patterns**: +```cpp +// Pattern 1: Use headers even when modules are enabled (most compatible) +#include +#include + +// Pattern 2: Use import in module interface files (.ixx) +// (fully supported in CMake 3.29+) +export module mymodule; +import podio.core; +import datamodel.datamodel; + +// Pattern 3: Direct import in .cpp files (experimental, requires CMake 3.30+) +import podio.core; // May not work without additional CMake configuration +import datamodel.datamodel; +``` + +**When will direct imports work reliably?** +Monitor CMake releases for improvements to dependency scanning. Podio includes experimental tests that will start passing when CMake support matures. + +## Migration Guide + +### For Existing Projects + +1. **Update build requirements**: Ensure CMake 3.29+, Ninja, GCC 14+ +2. **Enable modules**: Add `-DPODIO_ENABLE_CXX_MODULES=ON` to CMake configuration +3. **Test compilation**: Verify build succeeds +4. **Gradual adoption**: Start using `import` in new code +5. **Measure benefits**: Benchmark compilation time improvements + +### For New Projects + +1. **Design with modules**: Plan to use `import` from the start +2. **Keep ROOT separate**: Use headers for ROOT I/O, modules for datamodel +3. **Document usage**: Note module requirements in build instructions + +### Compatibility Strategy + +Podio guarantees: +- **Binary compatibility**: Same ABI whether modules enabled or not +- **Source compatibility**: Headers work identically with or without modules +- **Optional feature**: Modules are opt-in, never required +- **No code changes**: Existing code works unchanged + +## Testing + +All podio tests pass with modules enabled: +- Unit tests +- Integration tests +- Schema evolution tests +- Python binding tests + +CI includes module testing on GCC 15 builds to ensure ongoing compatibility. + +## Troubleshooting + +### "Ninja generator required for modules" + +**Cause**: Attempted to use modules with Unix Makefiles generator +**Solution**: Use `-GNinja` when configuring CMake + +### "CMake 3.29 or later required for modules" + +**Cause**: CMake version too old +**Solution**: Upgrade CMake or disable modules with `-DPODIO_ENABLE_CXX_MODULES=OFF` + +### "Module file not found" + +**Cause**: Module interface not installed or not in include path +**Solution**: Ensure podio was built with modules enabled and properly installed + +### Compilation errors with ROOT headers in modules + +**Cause**: Attempting to include ROOT headers in module interface +**Solution**: Keep ROOT includes as traditional `#include` outside module interfaces + +## Future Directions + +### Short Term +- Gather performance benchmarks from real-world usage +- Expand module exports based on user feedback +- Add module support documentation to user guide + +### Medium Term +- Migrate to `import std;` when compiler support matures +- Investigate module support for ROOT I/O when ROOT becomes module-safe +- Optimize module compilation dependencies + +### Long Term +- Consider enabling modules by default when requirements are widely met +- Collaborate with HEP community on module best practices +- Contribute upstream fixes to dependencies (ROOT, etc.) + +## References + +- [C++20 Modules (cppreference)](https://en.cppreference.com/w/cpp/language/modules) +- [CMake CXX_MODULE_SETS](https://cmake.org/cmake/help/latest/manual/cmake-cxxmodules.7.html) +- [GCC Modules Documentation](https://gcc.gnu.org/onlinedocs/gcc/C_002b_002b-Modules.html) + +## Example: Complete Workflow + +Here's a complete example showing module usage: + +```cpp +// my_analysis.cpp +import podio.core; +import edm4hep.datamodel; +#include // ROOT I/O still uses headers +#include + +int main() { + // Use module-imported types for data structures + edm4hep::MCParticleCollection particles; + podio::GenericParameters params; + + // Use traditional header for ROOT I/O + podio::ROOTReader reader; + reader.openFile("events.root"); + + // Read and process + for (unsigned i = 0; i < reader.getEntries("events"); ++i) { + auto frame = reader.readEntry("events", i); + auto& mcparticles = frame.get("MCParticles"); + + for (const auto& particle : mcparticles) { + std::cout << "PDG: " << particle.getPDG() + << " Energy: " << particle.getEnergy() << "\n"; + } + } + + return 0; +} +``` + +Build: +```bash +# With modules enabled +c++ -std=c++20 -fmodules-ts my_analysis.cpp -o my_analysis \ + -L/path/to/podio/lib -lpodio -ledm4hep \ + $(root-config --libs --cflags) +``` + +This example demonstrates: +- Using `import` for datamodel types (fast compilation) +- Using `#include` for ROOT I/O (compatibility) +- Mixed usage pattern (common in HEP) +- Performance benefits for the parts that can use modules diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index 28204af5c..c36f7be74 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -166,6 +166,12 @@ def parse_version(version_str): default=None, type=parse_version, ) + parser.add_argument( + "--enable-modules", + action="store_true", + default=False, + help="Generate C++20 module interface file (.ixx) for the datamodel", + ) args = parser.parse_args() @@ -198,6 +204,7 @@ def parse_version(version_str): datamodel_version=args.datamodel_version, old_descriptions=args.old_descriptions, evolution_file=args.evolution_file, + enable_modules=args.enable_modules, ) if args.clangformat and has_clang_format(): diff --git a/python/podio_gen/cpp_generator.py b/python/podio_gen/cpp_generator.py index 902a01799..1aadcb271 100644 --- a/python/podio_gen/cpp_generator.py +++ b/python/podio_gen/cpp_generator.py @@ -48,6 +48,7 @@ def __init__( # pylint: disable=too-many-arguments old_descriptions, evolution_file, datamodel_version=None, + enable_modules=False, ): super().__init__( yamlfile, @@ -59,6 +60,7 @@ def __init__( # pylint: disable=too-many-arguments datamodel_version=datamodel_version, ) self.io_handlers = io_handlers + self.enable_modules = enable_modules # schema evolution specific code self.old_yamlfiles = old_descriptions @@ -88,6 +90,7 @@ def post_process(self, datamodel): if the_links := datamodel["links"]: self._write_links_registration_file(the_links) self._write_all_collections_header() + self._write_datamodel_module() self._write_cmake_lists_file() def do_process_component(self, name, component): @@ -533,6 +536,7 @@ def _write_cmake_lists_file(self): header_files = list(f for f in self.generated_files if f.endswith(".h")) src_files = (f for f in self.generated_files if f.endswith(".cc")) xml_files = (f for f in self.generated_files if f.endswith(".xml")) + module_files = (f for f in self.generated_files if f.endswith(".ixx")) # Sort header files so that Collection headers appear first. This is # necessary for cling to load things in the correct order for some @@ -582,6 +586,15 @@ def _write_list(name, target_folder, files, comment): ) ) + full_contents.append( + _write_list( + "module_files", + r"${ARG_OUTPUT_FOLDER}/src", + module_files, + "Generated C++20 module interface files", + ) + ) + write_file_if_changed( f"{self.install_dir}/podio_generated_files.cmake", "\n".join(full_contents), @@ -609,6 +622,33 @@ def _write_all_collections_header(self): ), ) + def _write_datamodel_module(self): + """Write a C++20 module interface file that exports all datamodel types""" + if not self.enable_modules: + return + + if self.verbose: + print(f"Generating C++20 module interface for {self.package_name}") + + # Prepare structured type information with namespace details + datatypes = [DataType(name) for name in self.datamodel.datatypes.keys()] + components = [DataType(name) for name in self.datamodel.components.keys()] + interfaces = [DataType(name) for name in self.datamodel.interfaces.keys()] + links = [DataType(name) for name in self.datamodel.links.keys()] + + context = { + "package_name": self.package_name, + "incfolder": self.incfolder, + "datatypes": datatypes, + "components": components, + "interfaces": interfaces, + "links": links, + } + + # Generate the module file + module_content = self._eval_template("datamodel_module.ixx.jinja2", context) + self._write_file(f"{self.package_name}_module.ixx", module_content) + def _write_links_registration_file(self, links): """Write a .cc file that registers all the link collections that were defined with this datamodel""" diff --git a/python/podio_gen/generator_utils.py b/python/podio_gen/generator_utils.py index 327a1584c..ce1f39f3d 100644 --- a/python/podio_gen/generator_utils.py +++ b/python/podio_gen/generator_utils.py @@ -147,6 +147,33 @@ def __str__(self): def __repr__(self): return f"DataType: {self.__str__()}" + def qualified_for_modules(self, prefix="", suffix=""): + """Return the qualified name for C++ using declarations in modules. + + Args: + prefix: Optional prefix to add before the type name (e.g., "Mutable") + suffix: Optional suffix to add after the type name (e.g., "Collection") + + Returns: + For namespaced types: 'namespace::PrefixTypeSuffix' (no leading ::) + For global types: '::PrefixTypeSuffix' (with leading ::) + + Examples: + DataType("ExampleHit").qualified_for_modules() + # Returns "::ExampleHit" + + DataType("ExampleHit").qualified_for_modules(prefix="Mutable") + # Returns "::MutableExampleHit" + + DataType("ex42::Type").qualified_for_modules(suffix="Collection") + # Returns "ex42::TypeCollection" + """ + type_name = f"{prefix}{self.bare_type}{suffix}" + if self.namespace: + return f"{self.namespace}::{type_name}" + + return f"::{type_name}" + def _to_json(self): """Return a string representation that can be parsed again""" return self.full_type diff --git a/python/templates/CMakeLists.txt b/python/templates/CMakeLists.txt index e1584a4fe..1b857a1d9 100644 --- a/python/templates/CMakeLists.txt +++ b/python/templates/CMakeLists.txt @@ -2,6 +2,7 @@ set(PODIO_TEMPLATES ${CMAKE_CURRENT_LIST_DIR}/Collection.cc.jinja2 ${CMAKE_CURRENT_LIST_DIR}/Collection.h.jinja2 ${CMAKE_CURRENT_LIST_DIR}/datamodel.h.jinja2 + ${CMAKE_CURRENT_LIST_DIR}/datamodel_module.ixx.jinja2 ${CMAKE_CURRENT_LIST_DIR}/CollectionData.cc.jinja2 ${CMAKE_CURRENT_LIST_DIR}/CollectionData.h.jinja2 ${CMAKE_CURRENT_LIST_DIR}/Component.h.jinja2 diff --git a/python/templates/datamodel_module.ixx.jinja2 b/python/templates/datamodel_module.ixx.jinja2 new file mode 100644 index 000000000..d97776f2d --- /dev/null +++ b/python/templates/datamodel_module.ixx.jinja2 @@ -0,0 +1,111 @@ +{% import "macros/utils.jinja2" as utils %} +// SPDX-License-Identifier: Apache-2.0 +// AUTOMATICALLY GENERATED FILE - DO NOT EDIT +// +// C++20 Module Interface for {{ package_name }} datamodel +// This module exports all generated datatypes, collections, and components + +module; + +// Global module fragment - include all necessary headers +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Podio core headers (module-safe - no ROOT dependencies) +#include "podio/CollectionBase.h" +#include "podio/ICollectionProvider.h" +#include "podio/SchemaEvolution.h" +#include "podio/CollectionBuffers.h" +#include "podio/utilities/MaybeSharedPtr.h" +#include "podio/detail/OrderKey.h" + +#if defined(PODIO_JSON_OUTPUT) && !defined(__CLING__) +#include "nlohmann/json_fwd.hpp" +#endif + +// Include all component headers +{% for comp in components %} +#include "{{ incfolder }}{{ comp.bare_type }}.h" +{% endfor %} + +// Include all datatype headers (Object, MutableObject, Collection, etc.) +{% for dt in datatypes %} +#include "{{ incfolder }}{{ dt.bare_type }}.h" +#include "{{ incfolder }}Mutable{{ dt.bare_type }}.h" +#include "{{ incfolder }}{{ dt.bare_type }}Collection.h" +{% endfor %} + +// Include all interface headers +{% for iface in interfaces %} +#include "{{ incfolder }}{{ iface.bare_type }}.h" +{% endfor %} + +// Include all link headers +{% for link in links %} +#include "{{ incfolder }}{{ link.bare_type }}Collection.h" +{% endfor %} + +export module {{ package_name }}.datamodel; + +// Re-export podio core types that are used in the datamodel +export namespace podio { + using podio::CollectionBase; + using podio::ICollectionProvider; + using podio::SchemaEvolution; + using podio::SchemaVersionT; +} + +// Export all components +{% if components %} +export namespace {{ package_name }} { +{% for comp in components %} + using {{ comp.qualified_for_modules() }}; +{% endfor %} +} +{% endif %} + +// Export all datatypes (Object, MutableObject, Collection) +{% for dt in datatypes %} +export namespace {{ package_name }} { + // {{ dt.full_type }} datatype + using {{ dt.qualified_for_modules() }}; + using {{ dt.qualified_for_modules(prefix="Mutable") }}; + using {{ dt.qualified_for_modules(suffix="Collection") }}; + using {{ dt.qualified_for_modules(suffix="CollectionIterator") }}; + using {{ dt.qualified_for_modules(suffix="MutableCollectionIterator") }}; // Note: pattern is {Type}MutableCollectionIterator + // Note: {{ dt.bare_type }}Obj, {{ dt.bare_type }}Data, and {{ dt.bare_type }}CollectionData are internal implementation + // details and are not exported. They are only accessible through the public handle and collection APIs. +} +{% endfor %} + +// Export all interfaces +{% if interfaces %} +export namespace {{ package_name }} { +{% for iface in interfaces %} + using {{ iface.qualified_for_modules() }}; +{% endfor %} +} +{% endif %} + +// Export all link collections +{% if links %} +export namespace {{ package_name }} { +{% for link in links %} + using {{ link.qualified_for_modules(suffix="Collection") }}; +{% endfor %} +} +{% endif %} + +// Export type lists for introspection +export namespace {{ package_name }} { + // Note: TypeList definitions are in {{ package_name }}.h header + // They cannot be directly exported from modules due to template complexities +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2d2d312c7..43046d343 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -79,13 +79,25 @@ SET(core_headers ) PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) -target_compile_options(podio PRIVATE -pthread) + +# podio uses std::mutex and other threading primitives, so we need threading support +# Use PUBLIC to ensure consumers (including module importers) get the same threading configuration +# This is critical for C++ modules which require matching compiler configurations +find_package(Threads REQUIRED) +target_link_libraries(podio PUBLIC Threads::Threads) + target_link_libraries(podio PRIVATE Python3::Python) # For Frame.h if (ROOT_VERSION VERSION_LESS 6.36) target_compile_definitions(podio PUBLIC PODIO_ROOT_OLDER_6_36=1) endif() +# Add C++20 module support (if enabled) +if(PODIO_ENABLE_CXX_MODULES) + message(STATUS "Adding podio.core module interface to podio library") + PODIO_ADD_MODULE_INTERFACE(podio podio.core ${CMAKE_CURRENT_SOURCE_DIR}/podio_core_module.ixx) +endif() + # --- Root I/O functionality and corresponding dictionary SET(root_sources @@ -203,10 +215,12 @@ endif() if (NOT ENABLE_DATASOURCE) install(TARGETS podio podioDict podioRootIO podioRootIODict podioIO ${INSTALL_LIBRARIES} EXPORT podioTargets + FILE_SET CXX_MODULES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/podio/modules DESTINATION "${CMAKE_INSTALL_LIBDIR}") else() install(TARGETS podio podioDict podioRootIO podioRootIODict podioIO podioDataSource podioDataSourceDict ${INSTALL_LIBRARIES} EXPORT podioTargets + FILE_SET CXX_MODULES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/podio/modules DESTINATION "${CMAKE_INSTALL_LIBDIR}") endif() diff --git a/src/Pythonizations.cc b/src/Pythonizations.cc index 5602dc016..9ba34c9c6 100644 --- a/src/Pythonizations.cc +++ b/src/Pythonizations.cc @@ -52,4 +52,4 @@ void pythonize_subscript(PyObject* klass, const std::string& name) { Py_DECREF(method); } -} // namespace podio::detail::pythonizations \ No newline at end of file +} // namespace podio::detail::pythonizations diff --git a/src/podio_core_module.ixx b/src/podio_core_module.ixx new file mode 100644 index 000000000..642f1ecba --- /dev/null +++ b/src/podio_core_module.ixx @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 +// Module interface for podio.core - ROOT-independent podio functionality +// +// This module exports the core podio types and interfaces that don't depend on ROOT. +// It can be used by datamodel modules and user code that doesn't need ROOT I/O. + +module; + +// Global module fragment - includes go here +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Podio core headers (module-safe - no ROOT dependency) +#include "podio/CollectionBase.h" +#include "podio/CollectionBuffers.h" +#include "podio/CollectionIDTable.h" +#include "podio/DatamodelRegistry.h" +#include "podio/GenericParameters.h" +#include "podio/ICollectionProvider.h" +#include "podio/LinkCollection.h" +#include "podio/LinkNavigator.h" +#include "podio/ObjectID.h" +#include "podio/RelationRange.h" +#include "podio/SchemaEvolution.h" +#include "podio/UserDataCollection.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" +#include "podio/utilities/Glob.h" + +export module podio.core; + +// Export podio core interfaces and types +export namespace podio { +// Core collection interfaces +using podio::CollectionBase; +using podio::CollectionIDTable; +using podio::CollectionReadBuffers; +using podio::CollectionWriteBuffers; +using podio::ICollectionProvider; +using podio::UserDataCollection; + +// Object identification +using podio::ObjectID; + +// Data model registry +using podio::DatamodelRegistry; +using podio::RelationNameMapping; +using podio::RelationNames; + +// Generic parameters for metadata +using podio::GenericParameters; + +// Relations and navigation +using podio::Link; +using podio::LinkCollectionIterator; +using podio::LinkNavigator; +using podio::RelationRange; + +// Schema evolution +using podio::Backend; +using podio::SchemaEvolution; +using podio::SchemaVersionT; + +// Utility functions +namespace utils { + using podio::utils::expand_glob; + using podio::utils::is_glob_pattern; +} // namespace utils +} // namespace podio diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 961bff824..341490aff 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -98,6 +98,29 @@ add_subdirectory(schema_evolution) # Tests that don't fit into one of the broad categories above CREATE_PODIO_TEST(ostream_operator.cpp "") +# C++ modules simple import test (no Catch2 dependency) +if(PODIO_ENABLE_CXX_MODULES) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.30) + message(STATUS "Adding simple C++20 module import test") + + add_executable(test_module_import test_module_import.cpp) + target_link_libraries(test_module_import PRIVATE TestDataModel podio) + target_compile_features(test_module_import PRIVATE cxx_std_20) + + # Enable module scanning for this file + set_source_files_properties(test_module_import.cpp PROPERTIES + LANGUAGE CXX + CXX_SCAN_FOR_MODULES ON + ) + + add_test(NAME module_import COMMAND test_module_import) + PODIO_SET_TEST_ENV(module_import) + set_tests_properties(module_import PROPERTIES + LABELS "modules;import;basic" + ) + endif() +endif() + if (ENABLE_JULIA) message(STATUS "Julia Datamodel generation is enabled.") PODIO_GENERATE_DATAMODEL(datamodeljulia datalayout.yaml headers sources diff --git a/tests/test_module_import.cpp b/tests/test_module_import.cpp new file mode 100644 index 000000000..020eeb884 --- /dev/null +++ b/tests/test_module_import.cpp @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: Apache-2.0 +// Simple module import test without Catch2 +// +// This test validates that C++20 module imports work correctly by: +// - Importing podio.core module +// - Importing datamodel module +// - Using types from both modules +// - Performing basic operations +// +// Returns 0 on success, non-zero on failure + +import podio.core; +import datamodel.datamodel; + +#include +#include + +int main() { + int failures = 0; + + std::cout << "Testing podio.core module imports..." << std::endl; + + // Test 1: podio::ObjectID + podio::ObjectID id{42, 1}; + if (id.index != 42 || id.collectionID != 1) { + std::cerr << "FAILED: ObjectID construction" << std::endl; + failures++; + } else { + std::cout << "PASSED: ObjectID construction" << std::endl; + } + + // Test 2: podio::CollectionIDTable + podio::CollectionIDTable table; + if (!table.empty()) { + std::cerr << "FAILED: CollectionIDTable should be empty" << std::endl; + failures++; + } else { + std::cout << "PASSED: CollectionIDTable empty check" << std::endl; + } + + auto collID = table.add("test_collection"); + if (table.empty()) { + std::cerr << "FAILED: CollectionIDTable should not be empty after add" << std::endl; + failures++; + } else { + std::cout << "PASSED: CollectionIDTable add" << std::endl; + } + + auto retrieved = table.collectionID("test_collection"); + if (!retrieved.has_value() || retrieved.value() != collID) { + std::cerr << "FAILED: CollectionIDTable retrieval" << std::endl; + failures++; + } else { + std::cout << "PASSED: CollectionIDTable retrieval" << std::endl; + } + + std::cout << "\nTesting datamodel module imports..." << std::endl; + + // Test 3: ExampleHit from datamodel + auto hits = datamodel::ExampleHitCollection(); + auto hit = hits.create(); + hit.energy(42.5); + hit.x(1.0); + hit.y(2.0); + hit.z(3.0); + + if (hit.energy() != 42.5 || hit.x() != 1.0 || hit.y() != 2.0 || hit.z() != 3.0) { + std::cerr << "FAILED: ExampleHit setters/getters" << std::endl; + failures++; + } else { + std::cout << "PASSED: ExampleHit setters/getters" << std::endl; + } + + // Test 4: Collection operations + auto hit2 = hits.create(); + hit2.energy(100.0); + + if (hits.size() != 2) { + std::cerr << "FAILED: Collection size should be 2, got " << hits.size() << std::endl; + failures++; + } else { + std::cout << "PASSED: Collection size" << std::endl; + } + + if (hits[0].energy() != 42.5 || hits[1].energy() != 100.0) { + std::cerr << "FAILED: Collection indexing" << std::endl; + failures++; + } else { + std::cout << "PASSED: Collection indexing" << std::endl; + } + + // Test 5: Relations + auto clusters = datamodel::ExampleClusterCollection(); + auto cluster = clusters.create(); + cluster.energy(142.5); + cluster.addHits(hit); + cluster.addHits(hit2); + + if (cluster.Hits_size() != 2) { + std::cerr << "FAILED: Cluster should have 2 hits" << std::endl; + failures++; + } else { + std::cout << "PASSED: Cluster relations" << std::endl; + } + + if (cluster.Hits(0).energy() != 42.5 || cluster.Hits(1).energy() != 100.0) { + std::cerr << "FAILED: Cluster hit energies" << std::endl; + failures++; + } else { + std::cout << "PASSED: Cluster hit access" << std::endl; + } + + // Test 6: Polymorphism via podio.core interfaces + podio::CollectionBase* basePtr = &hits; + if (basePtr->size() != 2) { + std::cerr << "FAILED: Polymorphic size check" << std::endl; + failures++; + } else { + std::cout << "PASSED: Polymorphic interface" << std::endl; + } + + if (basePtr->getTypeName() != "ExampleHitCollection") { + std::cerr << "FAILED: Type name should be ExampleHitCollection" << std::endl; + failures++; + } else { + std::cout << "PASSED: Type name retrieval" << std::endl; + } + + // Summary + std::cout << "\n===============================================" << std::endl; + if (failures == 0) { + std::cout << "All module import tests PASSED" << std::endl; + std::cout << "===============================================" << std::endl; + return EXIT_SUCCESS; + } else { + std::cout << "Module import tests FAILED: " << failures << " failures" << std::endl; + std::cout << "===============================================" << std::endl; + return EXIT_FAILURE; + } +}