From b761f0d10dc8511c2d10923692fadc0daab7f797 Mon Sep 17 00:00:00 2001 From: Roger Nelson Date: Mon, 1 Dec 2025 14:05:39 -0500 Subject: [PATCH 01/24] Build all python package dependencies Signed-off-by: Roger Nelson --- cmake/dependencies/python3.cmake | 15 ++++----------- src/build/requirements.txt.in | 3 ++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index 947a98085..dacaf6cbf 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -29,8 +29,7 @@ SET(RV_DEPS_PYTHON_VERSION_SHORT "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}" ) -# This version is used for generating src/build/requirements.txt from requirements.txt.in template All platforms install OpenTimelineIO from git to ensure -# consistent source builds. +# This version is used for generating src/build/requirements.txt from requirements.txt.in template SET(_opentimelineio_version "${RV_DEPS_OTIO_VERSION}" ) @@ -38,10 +37,6 @@ SET(_opentimelineio_version SET(_pyside_version "${RV_DEPS_PYSIDE_VERSION}" ) -# Construct the full git URL for pip to use in requirements.txt Using this avoids @ symbol conflicts in CONFIGURE_FILE -SET(_opentimelineio_pip_url - "git+https://github.com/AcademySoftwareFoundation/OpenTimelineIO@v${_opentimelineio_version}#egg=OpenTimelineIO" -) SET(_python3_download_url "https://github.com/python/cpython/archive/refs/tags/v${_python3_version}.zip" @@ -77,9 +72,6 @@ SET(_build_dir ${RV_DEPS_BASE_DIR}/${_python3_target}/build ) -# Note: OpenTimelineIO is now installed via requirements.txt from git URL for all platforms. This ensures consistent source builds across Windows, Mac, and -# Linux. - FETCHCONTENT_DECLARE( ${_pyside_target} URL ${_pyside_archive_url} @@ -276,12 +268,13 @@ ELSE() SET(_otio_debug_env "") ENDIF() -# Single unified command for all platforms and build types +# Using --no-binary :all: to ensure all packages with native extensions are built from source +# against our custom Python build, preventing ABI compatibility issues. SET(_requirements_install_command ${CMAKE_COMMAND} -E env ${_otio_debug_env} "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" - "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall -r "${_requirements_output_file}" + "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: -r "${_requirements_output_file}" ) IF(RV_TARGET_WINDOWS) diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index d354ac868..2be3bf364 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -1,10 +1,11 @@ # This file contains all the packages that will be packaged with RV. Please add the license next to the package. # NOTE: This is a CMake template file. The actual requirements.txt is generated during CMake configuration. # To update OpenTimelineIO version, edit _opentimelineio_version in cmake/dependencies/python3.cmake +# NOTE: Using --no-binary :all: in pip install ensures all packages are built from source against our custom Python. pip # License: MIT License (MIT) setuptools # License: MIT License -@_opentimelineio_pip_url@ # License: Other/Proprietary License (Modified Apache 2.0 License) +opentimelineio==@_opentimelineio_version@ # License: Other/Proprietary License (Modified Apache 2.0 License) PyOpenGL # License: BSD License (BSD) # MacOS only - PyOpenGL_accelerate is built from source in python3.cmake as a workaround until the fix From 2bc948342482a135b4aa00f4d2ab351d034c0085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 3 Dec 2025 12:42:24 -0500 Subject: [PATCH 02/24] ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 310c4dd7b..a7e2eb736 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,6 +246,41 @@ jobs: exit 1 fi + - name: Install GCC 11 toolchain for Rocky Linux 8 + if: ${{ matrix.rocky-version == '8' }} + run: | + retry_count=0 + max_retries=5 + + until [ $retry_count -ge $max_retries ]; do + echo "Attempt $((retry_count + 1)) of $max_retries" + + if dnf install -y gcc-toolset-11-toolchain; then + echo "GCC 11 toolchain installed successfully" + break + else + retry_count=$((retry_count + 1)) + if [ $retry_count -lt $max_retries ]; then + echo "Attempt $retry_count failed. Retrying in 5 seconds..." + sleep 5 + fi + fi + done + + if [ $retry_count -ge $max_retries ]; then + echo "Failed to install GCC 11 toolchain after $max_retries attempts" + exit 1 + fi + + - name: Enable GCC 11 toolchain for Rocky Linux 8 + if: ${{ matrix.rocky-version == '8' }} + run: | + source /opt/rh/gcc-toolset-11/enable + echo "CC=$(which gcc)" >> $GITHUB_ENV + echo "CXX=$(which g++)" >> $GITHUB_ENV + echo "PATH=$PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV + - name: Check out repository code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7 with: @@ -735,6 +770,11 @@ jobs: run: | python -c "import sys; print(sys.executable)" + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@0b1efabc08b657293548b77fb76cc02d26091c7e + with: + toolchain: stable + - name: Cache CMake for Windows uses: actions/cache@v4 id: windows-cmake-cache From 8b799601fc2d32b14886335f70587b7be29be100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 3 Dec 2025 13:57:21 -0500 Subject: [PATCH 03/24] tentative fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- .github/workflows/ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7e2eb736..a97d1bc18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -276,10 +276,12 @@ jobs: if: ${{ matrix.rocky-version == '8' }} run: | source /opt/rh/gcc-toolset-11/enable - echo "CC=$(which gcc)" >> $GITHUB_ENV - echo "CXX=$(which g++)" >> $GITHUB_ENV - echo "PATH=$PATH" >> $GITHUB_ENV + echo "/opt/rh/gcc-toolset-11/root/usr/bin" >> $GITHUB_PATH + echo "CC=/opt/rh/gcc-toolset-11/root/usr/bin/gcc" >> $GITHUB_ENV + echo "CXX=/opt/rh/gcc-toolset-11/root/usr/bin/g++" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV + gcc --version + g++ --version - name: Check out repository code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # 4.1.7 @@ -797,6 +799,12 @@ jobs: echo "export ACLOCAL_PATH=/c/msys64/usr/share/aclocal" >> ~/.bash_profile shell: msys2 {0} + - name: Add Rust to PATH for msys2 + run: | + ls -al $HOME/.cargo/bin + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + shell: msys2 {0} + - name: Display environment variables run: | echo $PATH From 9c1c7c1527b77235aa9be07a9595bbaa3d9e04b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 3 Dec 2025 14:08:47 -0500 Subject: [PATCH 04/24] try to fix rust for msys2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a97d1bc18..7ff1db275 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -801,8 +801,12 @@ jobs: - name: Add Rust to PATH for msys2 run: | - ls -al $HOME/.cargo/bin - echo "$HOME/.cargo/bin" >> $GITHUB_PATH + echo $CARGO_HOME + + - name: Add Rust to PATH for msys2 + run: | + echo $CARGO_HOME + echo "$CARGO_HOME/bin" >> $GITHUB_PATH shell: msys2 {0} - name: Display environment variables From 7a53e942e3df5df67933f12cff92fb2163ae802f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 3 Dec 2025 15:05:45 -0500 Subject: [PATCH 05/24] try to fix rust for msys2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ff1db275..034abe579 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -777,6 +777,13 @@ jobs: with: toolchain: stable + - name: Verify Rust installation + run: | + rustc --version + cargo --version + where rustc + where cargo + - name: Cache CMake for Windows uses: actions/cache@v4 id: windows-cmake-cache @@ -801,12 +808,15 @@ jobs: - name: Add Rust to PATH for msys2 run: | - echo $CARGO_HOME + echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> ~/.bash_profile + shell: msys2 {0} - - name: Add Rust to PATH for msys2 + - name: Verify Rust in msys2 run: | - echo $CARGO_HOME - echo "$CARGO_HOME/bin" >> $GITHUB_PATH + echo "Checking for Rust in msys2..." + rustc --version || echo "WARNING: Rust not found in msys2 PATH" + cargo --version || echo "WARNING: Cargo not found in msys2 PATH" + echo "PATH: $PATH" shell: msys2 {0} - name: Display environment variables @@ -883,6 +893,15 @@ jobs: export QT_HOME=c:/Qt/${{ matrix.qt-version }}/msvc2019_64 cmake -B _build -G "Visual Studio 17 2022" -A x64 -DRV_DEPS_WIN_PERL_ROOT=c:/Strawberry/perl/bin -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DRV_DEPS_QT_LOCATION=$QT_HOME -DRV_VFX_PLATFORM=${{ matrix.vfx-platform }} shell: msys2 {0} + + - name: Final Rust verification before build + run: | + echo "Verifying Rust is accessible for the build..." + rustc --version + cargo --version + echo "Rust PATH:" + where rustc + shell: msys2 {0} - name: Build OpenRV dependencies if: steps.cmake-dependencies.outputs.cache-hit != 'true' From d5d65a464f319b898724c490c0a86269baf10f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 3 Dec 2025 15:22:18 -0500 Subject: [PATCH 06/24] try to fix rust for msys2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 034abe579..aa31f2873 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -808,7 +808,7 @@ jobs: - name: Add Rust to PATH for msys2 run: | - echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> ~/.bash_profile + echo "export PATH=\"/c/Users/runneradmin/.cargo/bin:\$PATH\"" >> ~/.bash_profile shell: msys2 {0} - name: Verify Rust in msys2 From 85386b61835fa0b1baf2feabaad91e1f79675219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 4 Dec 2025 09:28:15 -0500 Subject: [PATCH 07/24] fix crypography for openssl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index dacaf6cbf..ee5048eaa 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -273,6 +273,7 @@ ENDIF() SET(_requirements_install_command ${CMAKE_COMMAND} -E env ${_otio_debug_env} + "OPENSSL_DIR=${RV_DEPS_OPENSSL_INSTALL_DIR}" "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: -r "${_requirements_output_file}" ) From e7fed17445fb7ca6dfdf86d420bbafd9b725df1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 4 Dec 2025 10:51:44 -0500 Subject: [PATCH 08/24] try to fix crypography build in windows debug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index ee5048eaa..dc00dec9c 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -377,13 +377,16 @@ IF(RV_TARGET_WINDOWS AND CMAKE_BUILD_TYPE MATCHES "^Debug$" ) # OCIO v2.2's pybind11 doesn't find python.lib in Debug since the name is python_d.lib. + # Also, Rust libraries (like cryptography via pyo3) look for python3.lib. ADD_CUSTOM_COMMAND( TARGET ${_python3_target} POST_BUILD COMMENT "Copying Debug Python lib as a unversionned file for Debug" COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_libpath} - COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_in_bin_libpath} DEPENDS ${_python3_target} ${_requirements_output_file} - ${_requirements_input_file} + COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_in_bin_libpath} + COMMAND cmake -E copy_if_different ${_lib_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_lib_dir}/python${PYTHON_VERSION_MAJOR}.lib + COMMAND cmake -E copy_if_different ${_bin_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_bin_dir}/python${PYTHON_VERSION_MAJOR}.lib + DEPENDS ${_python3_target} ${_requirements_output_file} ${_requirements_input_file} ) ENDIF() From 8658b0460fa93c7a9a5acf622697e54c12fb4d4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 4 Dec 2025 10:57:54 -0500 Subject: [PATCH 09/24] fix issue for rocky 8 2023 where the openssl comes from the system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index dc00dec9c..4c1d76ed2 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -273,7 +273,14 @@ ENDIF() SET(_requirements_install_command ${CMAKE_COMMAND} -E env ${_otio_debug_env} - "OPENSSL_DIR=${RV_DEPS_OPENSSL_INSTALL_DIR}" +) + +# Only set OPENSSL_DIR if we built OpenSSL ourselves (not for Rocky Linux 8 CY2023 which uses system OpenSSL) +IF(DEFINED RV_DEPS_OPENSSL_INSTALL_DIR) + LIST(APPEND _requirements_install_command "OPENSSL_DIR=${RV_DEPS_OPENSSL_INSTALL_DIR}") +ENDIF() + +LIST(APPEND _requirements_install_command "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: -r "${_requirements_output_file}" ) From 90afb09a7d872967c4974a5251e59d0917ceabd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 4 Dec 2025 16:01:04 -0500 Subject: [PATCH 10/24] remove debug steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- .github/workflows/ci.yml | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa31f2873..f130676ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -777,13 +777,6 @@ jobs: with: toolchain: stable - - name: Verify Rust installation - run: | - rustc --version - cargo --version - where rustc - where cargo - - name: Cache CMake for Windows uses: actions/cache@v4 id: windows-cmake-cache @@ -811,14 +804,6 @@ jobs: echo "export PATH=\"/c/Users/runneradmin/.cargo/bin:\$PATH\"" >> ~/.bash_profile shell: msys2 {0} - - name: Verify Rust in msys2 - run: | - echo "Checking for Rust in msys2..." - rustc --version || echo "WARNING: Rust not found in msys2 PATH" - cargo --version || echo "WARNING: Cargo not found in msys2 PATH" - echo "PATH: $PATH" - shell: msys2 {0} - - name: Display environment variables run: | echo $PATH @@ -826,6 +811,8 @@ jobs: which python which python3 which cmake + rustc --version || echo "WARNING: Rust not found in msys2 PATH" + cargo --version || echo "WARNING: Cargo not found in msys2 PATH" shell: msys2 {0} - name: Prepare Qt folder @@ -893,15 +880,6 @@ jobs: export QT_HOME=c:/Qt/${{ matrix.qt-version }}/msvc2019_64 cmake -B _build -G "Visual Studio 17 2022" -A x64 -DRV_DEPS_WIN_PERL_ROOT=c:/Strawberry/perl/bin -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DRV_DEPS_QT_LOCATION=$QT_HOME -DRV_VFX_PLATFORM=${{ matrix.vfx-platform }} shell: msys2 {0} - - - name: Final Rust verification before build - run: | - echo "Verifying Rust is accessible for the build..." - rustc --version - cargo --version - echo "Rust PATH:" - where rustc - shell: msys2 {0} - name: Build OpenRV dependencies if: steps.cmake-dependencies.outputs.cache-hit != 'true' From e5b96fe47dd9c305e42a89c1ae8439655dcba718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Tue, 16 Dec 2025 13:58:13 -0500 Subject: [PATCH 11/24] documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- docs/build_system/config_linux_rocky89.md | 45 ++++++++++++++---- docs/build_system/config_macos.md | 9 +++- docs/build_system/config_windows.md | 57 +++++++++++++++++++---- 3 files changed, 90 insertions(+), 21 deletions(-) diff --git a/docs/build_system/config_linux_rocky89.md b/docs/build_system/config_linux_rocky89.md index f0e73d2a0..b9ec4a983 100644 --- a/docs/build_system/config_linux_rocky89.md +++ b/docs/build_system/config_linux_rocky89.md @@ -73,8 +73,9 @@ All other dependencies are shared across variations. 1. [Install tools and build dependencies](rocky_install_tools_and_dependencies) 2. [Install Pyenv / Python](rocky_install_pyenv_and_python) 3. [Install CMake](rocky_install_cmake) -4. [Install Qt](rocky_install_qt) -5. [Build Open RV](rocky_build_openrv) +4. [Install Rust](rocky_install_rust) +5. [Install Qt](rocky_install_qt) +6. [Build Open RV](rocky_build_openrv) 6. [Building with Docker (Optional)](rocky_building_with_docker) @@ -164,8 +165,34 @@ cmake --version # confirm the version of your newly installed version of CMake cmake version 3.31.6 ``` +(rocky_install_rust)= +### 4. Install Rust + +````{warning} +Rust version **1.92 or later** is required to build certain Python dependencies (such as cryptography). +```` + +Install Rust using rustup: + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +``` + +This will install the latest stable version of Rust. After installation, add Rust to your current shell session: + +```bash +source $HOME/.cargo/env +``` + +Verify that Rust is installed correctly: + +```bash +rustc --version +cargo --version +``` + (rocky_install_qt)= -### 4. Install Qt +### 5. Install Qt Download the latest open-source [Qt installer](https://www.qt.io/download-open-source). We do not recommend that you install Qt from other installable sources as it may introduce build issues. @@ -193,19 +220,19 @@ Note 1: If you install Qt at a different installation path, you will need to man Note 2: Qt modules for Logs, Android, iOS and WebAssembly are not required to build Open RV. (rocky_build_openrv)= -### 5. Build Open RV +### 6. Build Open RV Once the platform-specific installation process is complete, building Open RV follows the same process for all platforms. Please refer to the [Common Build Instructions](config_common_build.md) for the complete build process. (rocky_building_with_docker)= -### 6. Building with Docker (Optional) +### 7. Building with Docker (Optional) To build Open RV using Docker, use the provided Dockerfile found in this repository, which should already contain all required dependencies. Please go through the cloning procedure found in the [common build process](config_common_build.md). Once cloned, get back here to build the docker image, run the container, and build Open RV within the docker container. -#### 5.1. Build the image and run +#### 7.1. Build the image and run ````{tabs} ```{code-tab} bash Rocky 9 @@ -223,7 +250,7 @@ docker run -d openrv-rocky8 /bin/bash -c "sleep infinity" -#### 5.2. Create and run the container +#### 7.2. Create and run the container ```bash # Lookup the container id for openrv-rocky{8/9} docker container ls @@ -237,11 +264,11 @@ docker container ls docker container exec -it /bin/bash # replace 'id' with your value ``` -#### 5.3. Build Open RV in the container +#### 7.3. Build Open RV in the container Once you are into the container, you can follow the [common build process](config_common_build.md) to build Open RV. -#### 5.4. Copy the stage folder outside of the container +#### 7.4. Copy the stage folder outside of the container If you are on a host that is the same as, or compatible with, your version of Rocky Linux, you can copy the stage folder outside of the container and execute Open RV. diff --git a/docs/build_system/config_macos.md b/docs/build_system/config_macos.md index 4ce74ba6e..907a7d44a 100644 --- a/docs/build_system/config_macos.md +++ b/docs/build_system/config_macos.md @@ -126,13 +126,18 @@ Most of the build requirements can be installed by running the following brew in ````{tabs} ```{code-tab} bash VFX-CY2024 -brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool python@3.11 yasm clang-format black meson nasm pkg-config glew +brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool python@3.11 yasm clang-format black meson nasm pkg-config glew rust ``` ```{code-tab} bash VFX-CY2023 -brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool python@3.10 yasm clang-format black meson nasm pkg-config glew +brew install ninja readline sqlite3 xz zlib tcl-tk@8 autoconf automake libtool python@3.10 yasm clang-format black meson nasm pkg-config glew rust ``` ```` +````{warning} +Rust version **1.92 or later** is required to build certain Python dependencies (such as cryptography) that contain Rust components. +Homebrew will install the latest stable version of Rust. +```` + Make sure `xcode-select -p` still returns `/Applications/Xcode.app/Contents/Developer`. If that is not the case, run `sudo xcode-select -s /Applications/Xcode.app` (install_qt)= diff --git a/docs/build_system/config_windows.md b/docs/build_system/config_windows.md index 43083e3ad..b20470525 100644 --- a/docs/build_system/config_windows.md +++ b/docs/build_system/config_windows.md @@ -74,10 +74,11 @@ All other dependencies are shared across variations. 4. [Install CMake](install_cmake) 5. [Install Qt](install_windows_qt) 6. [Install Strawberry Perl](install_strawberry_perl) -7. [Install MSYS2](install_msys2) +7. [Install Rust](install_rust) +8. [Install MSYS2](install_msys2) 1. [Install required MSYS2 pacman packages (from an MSYS2-MinGW64 shell)](install_msys2_packages) -8. [Setup environment variables](setup_env) -9. [Build Open RV](build_windows_openrv) +9. [Setup environment variables](setup_env) +10. [Build Open RV](build_windows_openrv) ````{warning} @@ -373,8 +374,35 @@ Take note of the installation path for Strawberry Perl, as it will be required i The default path is `C:\Strawberry`. ```` +(install_rust)= +## 7. Install Rust + +````{warning} +Rust version **1.92 or later** is required to build certain Python dependencies (such as cryptography). +```` + +Install Rust using rustup-init: + +1. Download rustup-init from [https://win.rustup.rs/x86_64](https://win.rustup.rs/x86_64) +2. Run `rustup-init.exe` +3. Follow the prompts to complete the installation (the default options are recommended) +4. After installation, close and reopen your terminal to apply the PATH changes + +Verify that Rust is installed correctly and check the version: + +```bash +rustc --version +cargo --version +``` + +````{note} +The Rust installation will be located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\\.cargo\bin`). Do not forget to add the location to the PATH environment variable in [step 9. Setup environment variables](#setup_env). +```` +# End of Selection +``` + (install_msys2)= -## 7. Install MSYS2 +## 8. Install MSYS2 ````{warning} @@ -389,7 +417,7 @@ Download and install the latest [MSYS2](https://www.msys2.org/). Open RV is **NO MSYS2 is only used for convenience as it comes with a package manager with utility packages required for the Open RV build such as cmake, git, flex, bison, nasm, unzip, zip, etc. (install_msys2_packages)= -### 7.1 Install required MSYS2 pacman packages +### 8.1 Install required MSYS2 pacman packages ````{note} The MSYS2 MingGW64 (mingw64.exe) terminal MUST be used.\ @@ -432,7 +460,7 @@ While installing the MSYS packages, review the list for any missing package. Som Note: To confirm which version/location of any tool used inside the MSYS shell, `where` can be used e.g. `where python`. If there's more than one path return, the top one will be used. (setup_env)= -### 8. Setup environment variables +### 9. Setup environment variables ````{note} This is the step where the path of Strawberry Perl, Python, CMake and Qt will be needed. @@ -447,14 +475,19 @@ These modifications will be added to the `.bash_profile` file located in the Use #### PATH environment variable ````{note} -Update the CMake, Strawberry Perl and Python location to reflect your installation path, using **forward slashes (/)** for a Unix-style path +Update the CMake, Strawberry Perl, Python, and Rust locations to reflect your installation paths, using **forward slashes (/)** for a Unix-style path to prevent issues later on. (e.g., C:\Python310 becomes /c/Python310). + +**For Rust:** Replace `` with your actual Windows username. You can find your username by running `echo %USERNAME%` in a Windows command prompt. +The Rust installation is located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\\.cargo\bin`), +which becomes `/c/Users//.cargo/bin` in MSYS2 format. ```` The following paths **must** be added to the PATH environment variable within MSYS2: - CMake binary directory - Python binary directory +- Rust cargo binary directory - MSYS2's `mingw64/bin` - Strawberry perl directory @@ -462,13 +495,17 @@ The following paths **must** be added to the PATH environment variable within MS ````{tabs} ```{code-tab} bash VFX-CY2024 -echo 'export PATH="/c/Program Files/CMake/bin:/c/Python311:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile +echo 'export PATH="/c/Program Files/CMake/bin:/c/Python311:/c/Users//.cargo/bin:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile ``` ```{code-tab} bash VFX-CY2023 -echo 'export PATH="/c/Program Files/CMake/bin:/c/Python310:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile +echo 'export PATH="/c/Program Files/CMake/bin:/c/Python310:/c/Users//.cargo/bin:/c/msys64/mingw64/bin:$PATH:/c/Strawberry/perl/bin"' >> ~/.bash_profile ``` ```` +````{warning} +Remember to replace `` with your actual Windows username in the commands above. +```` + #### ACLOCAL_PATH ```shell @@ -501,7 +538,7 @@ source ~/.bash_profile ``` (build_windows_openrv)= -## 9. Build Open RV +## 10. Build Open RV Once the platform-specific installation process is complete, building Open RV follows the same process for all platforms. Please refer to the [Common Build Instructions](config_common_build.md) for the complete build process. From c957a0f93ee5d90fa87c7ba10bf2256e86a2bcc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 17 Dec 2025 15:28:43 -0500 Subject: [PATCH 12/24] fix windows doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- docs/build_system/config_windows.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/build_system/config_windows.md b/docs/build_system/config_windows.md index b20470525..0eca08c2a 100644 --- a/docs/build_system/config_windows.md +++ b/docs/build_system/config_windows.md @@ -398,8 +398,6 @@ cargo --version ````{note} The Rust installation will be located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\\.cargo\bin`). Do not forget to add the location to the PATH environment variable in [step 9. Setup environment variables](#setup_env). ```` -# End of Selection -``` (install_msys2)= ## 8. Install MSYS2 From a96eaf4fcb46cb63beb749997298a7dc89601652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Wed, 17 Dec 2025 15:46:14 -0500 Subject: [PATCH 13/24] fix windows doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- docs/build_system/config_windows.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build_system/config_windows.md b/docs/build_system/config_windows.md index 0eca08c2a..185ae2d8d 100644 --- a/docs/build_system/config_windows.md +++ b/docs/build_system/config_windows.md @@ -396,7 +396,7 @@ cargo --version ``` ````{note} -The Rust installation will be located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\\.cargo\bin`). Do not forget to add the location to the PATH environment variable in [step 9. Setup environment variables](#setup_env). +The Rust installation will be located at `%USERPROFILE%\.cargo\bin` (typically `C:\Users\\.cargo\bin`). Do not forget to add the location to the PATH environment variable in [Setup environment variables](#setup_env). ```` (install_msys2)= From 607f46058a152f2a95e8ee090d0bfde6b8057747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 18 Dec 2025 09:11:47 -0500 Subject: [PATCH 14/24] clean up packages installation in requirements.txt only and testing python outside of make_python.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 21 +++- src/build/make_python.py | 128 +++-------------------- src/build/requirements.txt.in | 2 +- src/build/test_python.py | 168 +++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 118 deletions(-) create mode 100644 src/build/test_python.py diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index 4c1d76ed2..de37750a1 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -380,6 +380,23 @@ ADD_CUSTOM_COMMAND( DEPENDS ${_python3_target} ${_requirements_output_file} ${_requirements_input_file} ) +# Test the Python distribution after requirements are installed +SET(${_python3_target}-test-flag + ${_install_dir}/${_python3_target}-test-flag +) + +SET(_test_python_script + "${PROJECT_SOURCE_DIR}/src/build/test_python.py" +) + +ADD_CUSTOM_COMMAND( + COMMENT "Testing Python distribution" + OUTPUT ${${_python3_target}-test-flag} + COMMAND python3 "${_test_python_script}" --python-home "${_install_dir}" --variant "${CMAKE_BUILD_TYPE}" + COMMAND cmake -E touch ${${_python3_target}-test-flag} + DEPENDS ${${_python3_target}-requirements-flag} ${_test_python_script} +) + IF(RV_TARGET_WINDOWS AND CMAKE_BUILD_TYPE MATCHES "^Debug$" ) @@ -454,7 +471,7 @@ IF(RV_TARGET_WINDOWS) ADD_CUSTOM_COMMAND( COMMENT "Installing ${_python3_target}'s include and libs into ${RV_STAGE_LIB_DIR}" OUTPUT ${RV_STAGE_BIN_DIR}/${_python3_lib_name} ${_copy_commands} - DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${_build_flag_depends} + DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} ${_build_flag_depends} ) ADD_CUSTOM_TARGET( @@ -468,7 +485,7 @@ ELSE() COMMAND ${CMAKE_COMMAND} -E copy_directory ${_install_dir}/lib ${RV_STAGE_LIB_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory ${_install_dir}/include ${RV_STAGE_INCLUDE_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory ${_install_dir}/bin ${RV_STAGE_BIN_DIR} - DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${_build_flag_depends} + DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} ${_build_flag_depends} ) ADD_CUSTOM_TARGET( ${_python3_target}-stage-target ALL diff --git a/src/build/make_python.py b/src/build/make_python.py index 3537a0f92..390bc6674 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -138,8 +138,17 @@ def get_python_interpreter_args(python_home: str, variant: str) -> List[str]: ), ) - if not python_interpreters or os.path.exists(python_interpreters[0]) is False: - raise FileNotFoundError() + if not python_interpreters: + raise FileNotFoundError( + f"No Python interpreter found in {python_home}. " + f"Searched for pattern '{python_name_pattern}' in {python_home} (recursively) and {os.path.join(python_home, 'bin')}. " + ) + + if not os.path.exists(python_interpreters[0]): + raise FileNotFoundError( + f"Python interpreter does not exist: {python_interpreters[0]}. " + f"Found interpreters: {python_interpreters}" + ) print(f"Found python interpreters {python_interpreters}") @@ -221,27 +230,6 @@ def patch_python_distribution(python_home: str) -> None: print(f"Ensuring pip with {ensure_pip_args}") subprocess.run(ensure_pip_args).check_returncode() - pip_args = python_interpreter_args + ["-m", "pip"] - - for package in ["pip", "certifi", "six", "wheel", "packaging", "requests", "pydantic"]: - package_install_args = pip_args + [ - "install", - "--upgrade", - "--force-reinstall", - package, - ] - print(f"Installing {package} with {package_install_args}") - subprocess.run(package_install_args).check_returncode() - - wheel_install_args = pip_args + [ - "install", - "--upgrade", - "--force-reinstall", - "wheel", - ] - print(f"Installing wheel with {wheel_install_args}") - subprocess.run(wheel_install_args).check_returncode() - site_packages = glob.glob(os.path.join(python_home, "**", "site-packages"), recursive=True)[0] if os.path.exists(site_packages) is False: @@ -263,96 +251,6 @@ def patch_python_distribution(python_home: str) -> None: sitecustomize_file.write(SITECUSTOMIZE_FILE_CONTENT) -def test_python_distribution(python_home: str) -> None: - """ - Test the Python distribution. - - :param python_home: Package root of an Python package - """ - tmp_dir = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) - os.makedirs(tmp_dir) - - tmp_python_home = os.path.join(tmp_dir, os.path.basename(python_home)) - try: - print(f"Moving {python_home} to {tmp_python_home}") - shutil.move(python_home, tmp_python_home) - - python_interpreter_args = get_python_interpreter_args(tmp_python_home, VARIANT) - - # Note: OpenTimelineIO is installed via requirements.txt for all platforms and build types. - # The git URL in requirements.txt ensures it builds from source with proper linkage. - - wheel_install_arg = python_interpreter_args + [ - "-m", - "pip", - "install", - "cryptography", - ] - - print(f"Validating that we can install a wheel with {wheel_install_arg}") - subprocess.run(wheel_install_arg).check_returncode() - - python_validation_args = python_interpreter_args + [ - "-c", - "\n".join( - [ - # Check for tkinter - "try:", - " import tkinter", - "except:", - " import Tkinter as tkinter", - # Make sure certifi is available - "import certifi", - # Make sure the SSL_CERT_FILE variable is sett - "import os", - "assert certifi.where() == os.environ['SSL_CERT_FILE']", - # Make sure ssl is correctly built and linked - "import ssl", - # Misc - "import sqlite3", - "import ctypes", - "import ssl", - "import _ssl", - "import zlib", - ] - ), - ] - print(f"Validating the python package with {python_validation_args}") - subprocess.run(python_validation_args).check_returncode() - - dummy_ssl_file = os.path.join("Path", "To", "Dummy", "File") - python_validation2_args = python_interpreter_args + [ - "-c", - "\n".join( - [ - "import os", - f"assert os.environ['SSL_CERT_FILE'] == '{dummy_ssl_file}'", - ] - ), - ] - print(f"Validating the python package with {python_validation2_args}") - subprocess.run(python_validation2_args, env={**os.environ, "SSL_CERT_FILE": dummy_ssl_file}).check_returncode() - - python_validation3_args = python_interpreter_args + [ - "-c", - "\n".join( - [ - "import os", - "assert 'SSL_CERT_FILE' not in os.environ", - ] - ), - ] - print(f"Validating the python package with {python_validation3_args}") - subprocess.run( - python_validation3_args, - env={**os.environ, "DO_NOT_SET_SSL_CERT_FILE": "bleh"}, - ).check_returncode() - - finally: - print(f"Moving {tmp_python_home} to {python_home}") - shutil.move(tmp_python_home, python_home) - - def clean() -> None: """ Run the clean step of the build. Removes everything. @@ -608,7 +506,7 @@ def install_python_vfx2023() -> None: os.symlink(os.path.basename(python3_path), python_path) patch_python_distribution(OUTPUT_DIR) - test_python_distribution(OUTPUT_DIR) + # Note: Testing is now done via test_python.py after requirements.txt installation def install_python_vfx2024() -> None: @@ -722,7 +620,7 @@ def install_python_vfx2024() -> None: os.symlink(os.path.basename(python3_path), python_path) patch_python_distribution(OUTPUT_DIR) - test_python_distribution(OUTPUT_DIR) + # Note: Testing is now done via test_python.py after requirements.txt installation if __name__ == "__main__": diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index 2be3bf364..2b14ec97c 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -16,7 +16,7 @@ PyOpenGL # License: BSD License (BSD) # Use PyOpenGL_accelerate only on x86_64 platform. PyOpenGL_accelerate ; (platform_system=='Windows' or platform_system=='Linux') and platform_machine=='x86_64' # License: BSD License (BSD) -# Those are installed by the src/build/make_python.py script, adding them here to list their licenses. +# Additional packages required by RV and for testing the Python distribution certifi # License: Mozilla Public License 2.0 (MPL 2.0) (MPL-2.0) six # License: MIT License (MIT) diff --git a/src/build/test_python.py b/src/build/test_python.py new file mode 100644 index 000000000..de63fa037 --- /dev/null +++ b/src/build/test_python.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ***************************************************************************** +# Copyright 2020 Autodesk, Inc. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# ***************************************************************************** + +""" +Test script for validating a Python distribution built by make_python.py. + +This script validates that the Python distribution is correctly built, +relocatable, and has all necessary dependencies installed. +""" + +import argparse +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import uuid + +# Import the helper function from make_python.py (same directory) +from make_python import get_python_interpreter_args + + +def test_python_distribution(python_home: str, variant: str) -> None: + """ + Test the Python distribution. + + This test validates: + - The distribution is relocatable (works when moved to a temp directory) + - Can install wheels (cryptography package) + - All critical modules are available (ssl, tkinter, certifi, etc.) + - SSL_CERT_FILE environment variable is properly set + - Respects user-provided SSL_CERT_FILE + - Respects DO_NOT_SET_SSL_CERT_FILE flag + + :param python_home: Package root of a Python package + :param variant: Build variant (Debug or Release) + """ + tmp_dir = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) + os.makedirs(tmp_dir) + + tmp_python_home = os.path.join(tmp_dir, os.path.basename(python_home)) + try: + print(f"Moving {python_home} to {tmp_python_home}") + shutil.move(python_home, tmp_python_home) + + python_interpreter_args = get_python_interpreter_args(tmp_python_home, variant) + + # Note: OpenTimelineIO is installed via requirements.txt for all platforms and build types. + # The git URL in requirements.txt ensures it builds from source with proper linkage. + + wheel_install_arg = python_interpreter_args + [ + "-m", + "pip", + "install", + "cryptography", + ] + + print(f"Validating that we can install a wheel with {wheel_install_arg}") + subprocess.run(wheel_install_arg).check_returncode() + + python_validation_args = python_interpreter_args + [ + "-c", + "\n".join( + [ + # Check for tkinter + "try:", + " import tkinter", + "except:", + " import Tkinter as tkinter", + # Make sure certifi is available + "import certifi", + # Make sure the SSL_CERT_FILE variable is set + "import os", + "assert certifi.where() == os.environ['SSL_CERT_FILE']", + # Make sure ssl is correctly built and linked + "import ssl", + # Misc + "import sqlite3", + "import ctypes", + "import ssl", + "import _ssl", + "import zlib", + ] + ), + ] + print(f"Validating the python package with {python_validation_args}") + subprocess.run(python_validation_args).check_returncode() + + dummy_ssl_file = os.path.join("Path", "To", "Dummy", "File") + python_validation2_args = python_interpreter_args + [ + "-c", + "\n".join( + [ + "import os", + f"assert os.environ['SSL_CERT_FILE'] == '{dummy_ssl_file}'", + ] + ), + ] + print(f"Validating the python package with {python_validation2_args}") + subprocess.run(python_validation2_args, env={**os.environ, "SSL_CERT_FILE": dummy_ssl_file}).check_returncode() + + python_validation3_args = python_interpreter_args + [ + "-c", + "\n".join( + [ + "import os", + "assert 'SSL_CERT_FILE' not in os.environ", + ] + ), + ] + print(f"Validating the python package with {python_validation3_args}") + subprocess.run( + python_validation3_args, + env={**os.environ, "DO_NOT_SET_SSL_CERT_FILE": "bleh"}, + ).check_returncode() + + print("All Python distribution tests passed successfully!") + + finally: + print(f"Moving {tmp_python_home} back to {python_home}") + shutil.move(tmp_python_home, python_home) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Test a Python distribution built by make_python.py" + ) + + parser.add_argument( + "--python-home", + dest="python_home", + type=pathlib.Path, + required=True, + help="Path to the Python installation directory to test" + ) + parser.add_argument( + "--variant", + dest="variant", + type=str, + required=True, + choices=["Debug", "Release"], + help="Build variant (Debug or Release)" + ) + + args = parser.parse_args() + + if not os.path.exists(args.python_home): + print(f"Error: Python home directory does not exist: {args.python_home}", file=sys.stderr) + sys.exit(1) + + try: + test_python_distribution(args.python_home, args.variant) + print("\n✓ Python distribution validation completed successfully") + sys.exit(0) + except Exception as e: + print(f"\n✗ Python distribution validation failed: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + sys.exit(1) + From fec9b9501af95247af4ffb05075dcda76bed27ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 18 Dec 2025 11:17:42 -0500 Subject: [PATCH 15/24] move numpy out of pyside script and build it from scratch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/defaults/CY2023.cmake | 6 +++--- cmake/defaults/CY2024.cmake | 2 +- cmake/defaults/CY2025.cmake | 2 +- cmake/defaults/CY2026.cmake | 2 +- cmake/dependencies/python3.cmake | 16 ++++++++++------ src/build/make_pyside6.py | 13 +------------ src/build/requirements.txt.in | 9 ++++++--- 7 files changed, 23 insertions(+), 27 deletions(-) diff --git a/cmake/defaults/CY2023.cmake b/cmake/defaults/CY2023.cmake index 131c7986f..ed9bd85b6 100644 --- a/cmake/defaults/CY2023.cmake +++ b/cmake/defaults/CY2023.cmake @@ -40,9 +40,9 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2023") SET(RV_DEPS_IMATH_LIB_MAJOR "3_1") # NumPy - # NumPY for CY2023 is 1.23.x series but Pyside2 requires < 1.23 - # So we comment this out and make_pyside.py hardcoded to use NumPy < 1.23 - #SET(ENV{RV_DEPS_NUMPY_VERSION} "1.23.5") + # NumPy for CY2023 VFX platform is 1.23.x series, but PySide2 requires < 1.23 + # Using numpy 1.22.4 (last version before 1.23) for PySide2 compatibility + SET(RV_DEPS_NUMPY_VERSION "1.22.4") # OCIO # https://github.com/AcademySoftwareFoundation/OpenColorIO diff --git a/cmake/defaults/CY2024.cmake b/cmake/defaults/CY2024.cmake index 2c8774001..b0abdea43 100644 --- a/cmake/defaults/CY2024.cmake +++ b/cmake/defaults/CY2024.cmake @@ -21,7 +21,7 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2024") # NumPy # https://numpy.org/doc/stable/release.html - SET(ENV{RV_DEPS_NUMPY_VERSION} "1.24.4") + SET(RV_DEPS_NUMPY_VERSION "1.24.4") # OCIO # https://github.com/AcademySoftwareFoundation/OpenColorIO diff --git a/cmake/defaults/CY2025.cmake b/cmake/defaults/CY2025.cmake index ba3ac3467..ed26f4b7b 100644 --- a/cmake/defaults/CY2025.cmake +++ b/cmake/defaults/CY2025.cmake @@ -21,7 +21,7 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2025") # NumPy # https://numpy.org/doc/stable/release.html - SET(ENV{RV_DEPS_NUMPY_VERSION} "1.26.4") + SET(RV_DEPS_NUMPY_VERSION "1.26.4") # OCIO # https://github.com/AcademySoftwareFoundation/OpenColorIO diff --git a/cmake/defaults/CY2026.cmake b/cmake/defaults/CY2026.cmake index 761710d99..220433f94 100644 --- a/cmake/defaults/CY2026.cmake +++ b/cmake/defaults/CY2026.cmake @@ -21,7 +21,7 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2026") # NumPy # https://numpy.org/doc/stable/release.html - SET(ENV{RV_DEPS_NUMPY_VERSION} "2.3.0") + SET(RV_DEPS_NUMPY_VERSION "2.3.0") # OCIO # https://github.com/AcademySoftwareFoundation/OpenColorIO diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index de37750a1..37f2adeb7 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -38,6 +38,10 @@ SET(_pyside_version "${RV_DEPS_PYSIDE_VERSION}" ) +SET(_numpy_version + "${RV_DEPS_NUMPY_VERSION}" +) + SET(_python3_download_url "https://github.com/python/cpython/archive/refs/tags/v${_python3_version}.zip" ) @@ -241,7 +245,7 @@ ELSE() SET(_python3_cmake_library ${_python3_lib}) ENDIF() -# Generate requirements.txt from template with the OpenTimelineIO version substituted +# Generate requirements.txt from template with the OpenTimelineIO and NumPy versions substituted SET(_requirements_input_file "${PROJECT_SOURCE_DIR}/src/build/requirements.txt.in" ) @@ -282,7 +286,7 @@ ENDIF() LIST(APPEND _requirements_install_command "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" - "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: -r "${_requirements_output_file}" + "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: --only-binary pip,setuptools,wheel -r "${_requirements_output_file}" ) IF(RV_TARGET_WINDOWS) @@ -373,7 +377,7 @@ SET(${_python3_target}-requirements-flag ) ADD_CUSTOM_COMMAND( - COMMENT "Installing requirements from ${_requirements_output_file}" + COMMENT "Installing requirements from ${_requirements_output_file} pyside and other dependencies" OUTPUT ${${_python3_target}-requirements-flag} COMMAND ${_requirements_install_command} COMMAND cmake -E touch ${${_python3_target}-requirements-flag} @@ -427,7 +431,7 @@ IF(RV_VFX_PLATFORM STREQUAL CY2023) ${rv_deps_pyside2_SOURCE_DIR}/build_scripts/platforms/windows_desktop.py COMMAND ${_pyside_make_command} --prepare --build COMMAND cmake -E touch ${${_pyside_target}-build-flag} - DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} + DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} USES_TERMINAL ) @@ -438,9 +442,9 @@ ELSEIF(RV_VFX_PLATFORM STRGREATER_EQUAL CY2024) ADD_CUSTOM_COMMAND( COMMENT "Building PySide6 using ${_pyside_make_command_script}" OUTPUT ${${_pyside_target}-build-flag} - COMMAND ${CMAKE_COMMAND} -E env "RV_DEPS_NUMPY_VERSION=$ENV{RV_DEPS_NUMPY_VERSION}" ${_pyside_make_command} --prepare --build + COMMAND ${_pyside_make_command} --prepare --build COMMAND cmake -E touch ${${_pyside_target}-build-flag} - DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} + DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} USES_TERMINAL ) diff --git a/src/build/make_pyside6.py b/src/build/make_pyside6.py index ee3f33d1b..fd95ee536 100755 --- a/src/build/make_pyside6.py +++ b/src/build/make_pyside6.py @@ -159,18 +159,7 @@ def get_fallback_clang_filename_suffix(version): os.environ["LLVM_INSTALL_DIR"] = libclang_install_dir os.environ["CLANG_INSTALL_DIR"] = libclang_install_dir - # PySide6 build requires numpy 1.26.3 - numpy_version = os.environ.get("RV_DEPS_NUMPY_VERSION") - if not numpy_version: - raise ValueError("RV_DEPS_NUMPY_VERSION environment variable is not set.") - install_numpy_args = get_python_interpreter_args(PYTHON_OUTPUT_DIR, VARIANT) + [ - "-m", - "pip", - "install", - f"numpy=={numpy_version}", - ] - print(f"Installing numpy with {install_numpy_args}") - subprocess.run(install_numpy_args).check_returncode() + # Note: numpy is now installed via requirements.txt in python3.cmake before PySide6 builds. cmakelist_path = os.path.join(SOURCE_DIR, "sources", "shiboken6", "ApiExtractor", "CMakeLists.txt") old_cmakelist_path = os.path.join(SOURCE_DIR, "sources", "shiboken6", "ApiExtractor", "CMakeLists.txt.old") diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index 2b14ec97c..3867a32f8 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -1,10 +1,14 @@ # This file contains all the packages that will be packaged with RV. Please add the license next to the package. # NOTE: This is a CMake template file. The actual requirements.txt is generated during CMake configuration. -# To update OpenTimelineIO version, edit _opentimelineio_version in cmake/dependencies/python3.cmake +# To update OpenTimelineIO version, edit RV_DEPS_OTIO_VERSION in cmake/defaults/CYCOMMON.cmake +# To update NumPy version, edit RV_DEPS_NUMPY_VERSION in cmake/defaults/CY*.cmake # NOTE: Using --no-binary :all: in pip install ensures all packages are built from source against our custom Python. +# NOTE: pip, setuptools, and wheel are exceptions - they use pre-built binaries for compatibility. pip # License: MIT License (MIT) setuptools # License: MIT License +wheel # License: MIT License (MIT) +numpy==@_numpy_version@ # License: BSD License (BSD-3-Clause) - Required by PySide6 opentimelineio==@_opentimelineio_version@ # License: Other/Proprietary License (Modified Apache 2.0 License) PyOpenGL # License: BSD License (BSD) @@ -18,9 +22,8 @@ PyOpenGL_accelerate ; (platform_system=='Windows' or platform_system=='Linux') a # Additional packages required by RV and for testing the Python distribution -certifi # License: Mozilla Public License 2.0 (MPL 2.0) (MPL-2.0) +certifi # License: Mozilla Public License 2.0 (MPL 2.0) (MPL-2.0) - required by test_python.py six # License: MIT License (MIT) -wheel # License: MIT License (MIT) packaging # License: Apache Software License, BSD License requests # License: Apache Software License (Apache 2.0) cryptography # License: Apache Software License, BSD License ((Apache-2.0 OR BSD-3-Clause) AND PSF-2.0) From 322f330743840f18ca98da0bbcca9bb6f5baeb2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 18 Dec 2025 12:34:01 -0500 Subject: [PATCH 16/24] small fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- src/build/make_python.py | 1 + src/build/test_python.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/build/make_python.py b/src/build/make_python.py index 390bc6674..082ed4bed 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -52,6 +52,7 @@ all the certificate authorities that are distributed with Firefox. """ import site +import sys try: import os diff --git a/src/build/test_python.py b/src/build/test_python.py index de63fa037..c90a42a93 100644 --- a/src/build/test_python.py +++ b/src/build/test_python.py @@ -158,10 +158,10 @@ def test_python_distribution(python_home: str, variant: str) -> None: try: test_python_distribution(args.python_home, args.variant) - print("\n✓ Python distribution validation completed successfully") + print("\n[OK] Python distribution validation completed successfully") sys.exit(0) except Exception as e: - print(f"\n✗ Python distribution validation failed: {e}", file=sys.stderr) + print(f"\n[FAILED] Python distribution validation failed: {e}", file=sys.stderr) import traceback traceback.print_exc() sys.exit(1) From 2f68b526005d93a0d6fc5775b7327a85a2d84f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 18 Dec 2025 14:32:59 -0500 Subject: [PATCH 17/24] fix issue with sitecustomize.py during build of python dependencies because certifi was moved outside of make_python.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- src/build/make_python.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/build/make_python.py b/src/build/make_python.py index 082ed4bed..da0641006 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -69,10 +69,19 @@ if "SSL_CERT_FILE" not in os.environ and "DO_NOT_SET_SSL_CERT_FILE" not in os.environ: os.environ["SSL_CERT_FILE"] = certifi.where() +except ImportError: + # certifi not installed yet - this is expected during build when pip installs build dependencies + pass except Exception as e: - print("Failed to set certifi.where() as SSL_CERT_FILE.", file=sys.stderr) - print(e, file=sys.stderr) - print("Set DO_NOT_SET_SSL_CERT_FILE to skip this step in RV's Python initialization.", file=sys.stderr) + # Only print errors for unexpected exceptions, and only if verbose mode is enabled + try: + import os as _os + if "PYTHONVERBOSE" in _os.environ: + print("Failed to set certifi.where() as SSL_CERT_FILE.", file=sys.stderr) + print(e, file=sys.stderr) + print("Set DO_NOT_SET_SSL_CERT_FILE to skip this step in RV's Python initialization.", file=sys.stderr) + except: + pass try: import os @@ -106,9 +115,15 @@ site.removeduppaths() except Exception as e: - print("Failed to reorder RV's Python search path", file=sys.stderr) - print(e, file=sys.stderr) - print("Set DO_NOT_REORDER_PYTHON_PATH to skip this step in RV's Python initialization.", file=sys.stderr) + # Only print errors if verbose mode is enabled + try: + import os as _os + if "PYTHONVERBOSE" in _os.environ: + print("Failed to reorder RV's Python search path", file=sys.stderr) + print(e, file=sys.stderr) + print("Set DO_NOT_REORDER_PYTHON_PATH to skip this step in RV's Python initialization.", file=sys.stderr) + except: + pass ''' From 01332eb43e2c2492cb2c63b5c58f2ab71745b4c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Thu, 18 Dec 2025 16:20:30 -0500 Subject: [PATCH 18/24] build known packages with c/c++/rust extension to prevent building pacakges like cpython, meson, ninja, etc from source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 13 ++++++++++--- src/build/requirements.txt.in | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index 37f2adeb7..4479dd682 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -272,8 +272,15 @@ ELSE() SET(_otio_debug_env "") ENDIF() -# Using --no-binary :all: to ensure all packages with native extensions are built from source -# against our custom Python build, preventing ABI compatibility issues. +# Force source builds only for packages with C/C++/Rust extensions that need to link against our custom Python. +# All other packages (including build tools like Cython) can use pre-built wheels, which avoids +# path length issues on Windows and improves build speed. +# Packages built from source: +# - opentimelineio: C++ extensions, uses CMAKE_ARGS for proper linking +# - numpy: C extensions that need our Python ABI +# - PyOpenGL-accelerate: Cython extensions +# - cryptography: C extensions linking to OpenSSL +# - pydantic: Has Rust extensions (pydantic-core) in v2+ SET(_requirements_install_command ${CMAKE_COMMAND} -E env ${_otio_debug_env} @@ -286,7 +293,7 @@ ENDIF() LIST(APPEND _requirements_install_command "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" - "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary :all: --only-binary pip,setuptools,wheel -r "${_requirements_output_file}" + "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary opentimelineio,numpy,PyOpenGL-accelerate,cryptography,pydantic -r "${_requirements_output_file}" ) IF(RV_TARGET_WINDOWS) diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index 3867a32f8..e826620df 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -2,8 +2,9 @@ # NOTE: This is a CMake template file. The actual requirements.txt is generated during CMake configuration. # To update OpenTimelineIO version, edit RV_DEPS_OTIO_VERSION in cmake/defaults/CYCOMMON.cmake # To update NumPy version, edit RV_DEPS_NUMPY_VERSION in cmake/defaults/CY*.cmake -# NOTE: Using --no-binary :all: in pip install ensures all packages are built from source against our custom Python. -# NOTE: pip, setuptools, and wheel are exceptions - they use pre-built binaries for compatibility. +# NOTE: Packages with C/C++/Rust extensions (opentimelineio, numpy, PyOpenGL-accelerate, cryptography, pydantic) +# are built from source to ensure proper linking with our custom Python. Other packages use pre-built wheels. +# See python3.cmake pip # License: MIT License (MIT) setuptools # License: MIT License From 0962044cfbc610c1ec8b9782951d3a036c8a800c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Fri, 19 Dec 2025 08:13:24 -0500 Subject: [PATCH 19/24] cffi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index 4479dd682..f03731134 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -279,8 +279,9 @@ ENDIF() # - opentimelineio: C++ extensions, uses CMAKE_ARGS for proper linking # - numpy: C extensions that need our Python ABI # - PyOpenGL-accelerate: Cython extensions -# - cryptography: C extensions linking to OpenSSL +# - cryptography: Rust extensions with C bindings to OpenSSL # - pydantic: Has Rust extensions (pydantic-core) in v2+ +# - cffi: C extensions (_cffi_backend module), required by cryptography as build dependency SET(_requirements_install_command ${CMAKE_COMMAND} -E env ${_otio_debug_env} @@ -293,7 +294,7 @@ ENDIF() LIST(APPEND _requirements_install_command "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" - "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary opentimelineio,numpy,PyOpenGL-accelerate,cryptography,pydantic -r "${_requirements_output_file}" + "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary opentimelineio,numpy,PyOpenGL-accelerate,cryptography,pydantic,cffi -r "${_requirements_output_file}" ) IF(RV_TARGET_WINDOWS) From 58aa660dfebfdb82c1ccf746235fcdbcff6e5cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Fri, 19 Dec 2025 09:26:38 -0500 Subject: [PATCH 20/24] ruff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- src/build/make_python.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/build/make_python.py b/src/build/make_python.py index da0641006..dbb38b664 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -17,8 +17,6 @@ import sys import subprocess import platform -import tempfile -import uuid from typing import List from datetime import datetime From 8e292592d3a80409899471be8f88aec3aca33a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Fri, 19 Dec 2025 09:32:02 -0500 Subject: [PATCH 21/24] ruff formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- src/build/make_python.py | 5 ++--- src/build/test_python.py | 10 ++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/build/make_python.py b/src/build/make_python.py index dbb38b664..416a57c5b 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -157,11 +157,10 @@ def get_python_interpreter_args(python_home: str, variant: str) -> List[str]: f"No Python interpreter found in {python_home}. " f"Searched for pattern '{python_name_pattern}' in {python_home} (recursively) and {os.path.join(python_home, 'bin')}. " ) - + if not os.path.exists(python_interpreters[0]): raise FileNotFoundError( - f"Python interpreter does not exist: {python_interpreters[0]}. " - f"Found interpreters: {python_interpreters}" + f"Python interpreter does not exist: {python_interpreters[0]}. Found interpreters: {python_interpreters}" ) print(f"Found python interpreters {python_interpreters}") diff --git a/src/build/test_python.py b/src/build/test_python.py index c90a42a93..1f4f36534 100644 --- a/src/build/test_python.py +++ b/src/build/test_python.py @@ -130,16 +130,14 @@ def test_python_distribution(python_home: str, variant: str) -> None: if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Test a Python distribution built by make_python.py" - ) + parser = argparse.ArgumentParser(description="Test a Python distribution built by make_python.py") parser.add_argument( "--python-home", dest="python_home", type=pathlib.Path, required=True, - help="Path to the Python installation directory to test" + help="Path to the Python installation directory to test", ) parser.add_argument( "--variant", @@ -147,7 +145,7 @@ def test_python_distribution(python_home: str, variant: str) -> None: type=str, required=True, choices=["Debug", "Release"], - help="Build variant (Debug or Release)" + help="Build variant (Debug or Release)", ) args = parser.parse_args() @@ -163,6 +161,6 @@ def test_python_distribution(python_home: str, variant: str) -> None: except Exception as e: print(f"\n[FAILED] Python distribution validation failed: {e}", file=sys.stderr) import traceback + traceback.print_exc() sys.exit(1) - From 1e7d934ca267c51ad4ec386f844eb7b655f1130c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Fri, 19 Dec 2025 10:51:50 -0500 Subject: [PATCH 22/24] build from source by default and list the ones that dont need too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 96 +++++++++++++++++++++----------- src/build/requirements.txt.in | 6 +- 2 files changed, 65 insertions(+), 37 deletions(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index f03731134..f1add48e4 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -237,12 +237,15 @@ ELSE() # Not WINDOWS ) ENDIF() -# Set the appropriate library for CMAKE_ARGS based on platform -# Windows needs the import library (.lib), Unix needs the shared library (.so/.dylib) +# Set the appropriate library for CMAKE_ARGS based on platform Windows needs the import library (.lib), Unix needs the shared library (.so/.dylib) IF(RV_TARGET_WINDOWS) - SET(_python3_cmake_library ${_python3_implib}) + SET(_python3_cmake_library + ${_python3_implib} + ) ELSE() - SET(_python3_cmake_library ${_python3_lib}) + SET(_python3_cmake_library + ${_python3_lib} + ) ENDIF() # Generate requirements.txt from template with the OpenTimelineIO and NumPy versions substituted @@ -255,36 +258,42 @@ SET(_requirements_output_file CONFIGURE_FILE(${_requirements_input_file} ${_requirements_output_file} @ONLY) -# OpenTimelineIO needs to be built from source with CMAKE_ARGS to ensure it uses -# the correct custom-built Python libraries. This is required for both old and new -# versions of pybind11, especially pybind11 v2.13.6+ which has stricter detection. -# Note: pybind11's FindPythonLibsNew.cmake uses PYTHON_LIBRARY (all caps), -# PYTHON_INCLUDE_DIR, and PYTHON_EXECUTABLE variables. -# --no-cache-dir: Don't use pip's wheel cache (prevents using wheels built for wrong Python version) +# OpenTimelineIO needs to be built from source with CMAKE_ARGS to ensure it uses the correct custom-built Python libraries. This is required for both old and +# new versions of pybind11, especially pybind11 v2.13.6+ which has stricter detection. Note: pybind11's FindPythonLibsNew.cmake uses PYTHON_LIBRARY (all caps), +# PYTHON_INCLUDE_DIR, and PYTHON_EXECUTABLE variables. --no-cache-dir: Don't use pip's wheel cache (prevents using wheels built for wrong Python version) # --force-reinstall: Reinstall packages even if already installed (ensures fresh build) -# Set OTIO_CXX_DEBUG_BUILD for all Debug builds to ensure OTIO's C++ extensions -# are built with debug symbols and proper optimization levels matching RV's build type. -# On Windows, this also ensures OTIO links against the debug Python library (python311_d.lib). +# Set OTIO_CXX_DEBUG_BUILD for all Debug builds to ensure OTIO's C++ extensions are built with debug symbols and proper optimization levels matching RV's build +# type. On Windows, this also ensures OTIO links against the debug Python library (python311_d.lib). IF(CMAKE_BUILD_TYPE MATCHES "^Debug$") - SET(_otio_debug_env "OTIO_CXX_DEBUG_BUILD=1") + SET(_otio_debug_env + "OTIO_CXX_DEBUG_BUILD=1" + ) ELSE() - SET(_otio_debug_env "") + SET(_otio_debug_env + "" + ) ENDIF() -# Force source builds only for packages with C/C++/Rust extensions that need to link against our custom Python. -# All other packages (including build tools like Cython) can use pre-built wheels, which avoids -# path length issues on Windows and improves build speed. -# Packages built from source: -# - opentimelineio: C++ extensions, uses CMAKE_ARGS for proper linking -# - numpy: C extensions that need our Python ABI -# - PyOpenGL-accelerate: Cython extensions -# - cryptography: Rust extensions with C bindings to OpenSSL -# - pydantic: Has Rust extensions (pydantic-core) in v2+ -# - cffi: C extensions (_cffi_backend module), required by cryptography as build dependency +# List of pure Python packages that are safe to install from pre-built wheels. All other packages (those with C/C++/Rust extensions) will be built from source +# to ensure proper linking against our custom Python build. Packages are built from source unless explicitly listed here +SET(RV_PYTHON_WHEEL_SAFE + "pip" # Package installer (pure Python) + "setuptools" # Build system (pure Python) + "wheel" # Wheel format support (pure Python) + "PyOpenGL" # OpenGL bindings without acceleration (pure Python) + "certifi" # SSL certificate bundle (just data files) + "six" # Python 2/3 compatibility (pure Python) + "packaging" # Version parsing (pure Python) + "requests" # HTTP library (pure Python) + CACHE STRING "Pure Python packages safe to install from wheels" +) + +# Convert list to comma-separated string for pip's --only-binary flag +STRING(REPLACE ";" "," _wheel_safe_packages "${RV_PYTHON_WHEEL_SAFE}") + SET(_requirements_install_command - ${CMAKE_COMMAND} -E env - ${_otio_debug_env} + ${CMAKE_COMMAND} -E env ${_otio_debug_env} ) # Only set OPENSSL_DIR if we built OpenSSL ourselves (not for Rocky Linux 8 CY2023 which uses system OpenSSL) @@ -292,9 +301,28 @@ IF(DEFINED RV_DEPS_OPENSSL_INSTALL_DIR) LIST(APPEND _requirements_install_command "OPENSSL_DIR=${RV_DEPS_OPENSSL_INSTALL_DIR}") ENDIF() -LIST(APPEND _requirements_install_command - "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" - "${_python3_executable}" -s -E -I -m pip install --upgrade --no-cache-dir --force-reinstall --no-binary opentimelineio,numpy,PyOpenGL-accelerate,cryptography,pydantic,cffi -r "${_requirements_output_file}" +# Build all packages from source except those in RV_PYTHON_WHEEL_SAFE. Packages with native extensions (opentimelineio, numpy, PyOpenGL-accelerate, +# cryptography, pydantic, cffi, etc.) will be built from source for proper ABI compatibility. +LIST( + APPEND + _requirements_install_command + "CMAKE_ARGS=-DPYTHON_LIBRARY=${_python3_cmake_library} -DPYTHON_INCLUDE_DIR=${_include_dir} -DPYTHON_EXECUTABLE=${_python3_executable}" + "${_python3_executable}" + -s + -E + -I + -m + pip + install + --upgrade + --no-cache-dir + --force-reinstall + --no-binary + :all: + --only-binary + ${_wheel_safe_packages} + -r + "${_requirements_output_file}" ) IF(RV_TARGET_WINDOWS) @@ -412,8 +440,8 @@ ADD_CUSTOM_COMMAND( IF(RV_TARGET_WINDOWS AND CMAKE_BUILD_TYPE MATCHES "^Debug$" ) - # OCIO v2.2's pybind11 doesn't find python.lib in Debug since the name is python_d.lib. - # Also, Rust libraries (like cryptography via pyo3) look for python3.lib. + # OCIO v2.2's pybind11 doesn't find python.lib in Debug since the name is python_d.lib. Also, Rust libraries (like cryptography via pyo3) look for + # python3.lib. ADD_CUSTOM_COMMAND( TARGET ${_python3_target} POST_BUILD @@ -421,8 +449,8 @@ IF(RV_TARGET_WINDOWS COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_libpath} COMMAND cmake -E copy_if_different ${_python3_implib} ${_python_release_in_bin_libpath} COMMAND cmake -E copy_if_different ${_lib_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_lib_dir}/python${PYTHON_VERSION_MAJOR}.lib - COMMAND cmake -E copy_if_different ${_bin_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_bin_dir}/python${PYTHON_VERSION_MAJOR}.lib - DEPENDS ${_python3_target} ${_requirements_output_file} ${_requirements_input_file} + COMMAND cmake -E copy_if_different ${_bin_dir}/python${PYTHON_VERSION_MAJOR}_d.lib ${_bin_dir}/python${PYTHON_VERSION_MAJOR}.lib DEPENDS ${_python3_target} + ${_requirements_output_file} ${_requirements_input_file} ) ENDIF() diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index e826620df..a24281855 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -2,9 +2,9 @@ # NOTE: This is a CMake template file. The actual requirements.txt is generated during CMake configuration. # To update OpenTimelineIO version, edit RV_DEPS_OTIO_VERSION in cmake/defaults/CYCOMMON.cmake # To update NumPy version, edit RV_DEPS_NUMPY_VERSION in cmake/defaults/CY*.cmake -# NOTE: Packages with C/C++/Rust extensions (opentimelineio, numpy, PyOpenGL-accelerate, cryptography, pydantic) -# are built from source to ensure proper linking with our custom Python. Other packages use pre-built wheels. -# See python3.cmake +# NOTE: By default, all packages are built from source to ensure proper ABI compatibility with our custom Python. +# Pure Python packages safe to install from wheels are listed in RV_PYTHON_WHEEL_SAFE in python3.cmake. +# This approach ensures safety by default: new packages will be built from source unless explicitly marked as wheel-safe. pip # License: MIT License (MIT) setuptools # License: MIT License From bf6475d167e41a207e262987324a8c0d96b5812a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Fri, 19 Dec 2025 12:19:56 -0500 Subject: [PATCH 23/24] add more packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- cmake/dependencies/python3.cmake | 10 +++++++--- src/build/requirements.txt.in | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index f1add48e4..a0a5031d5 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -275,18 +275,22 @@ ELSE() ) ENDIF() -# List of pure Python packages that are safe to install from pre-built wheels. All other packages (those with C/C++/Rust extensions) will be built from source -# to ensure proper linking against our custom Python build. Packages are built from source unless explicitly listed here +# List of packages that are safe to install from pre-built wheels. All other packages (those with C/C++/Rust extensions) will be built from source to ensure +# proper linking against our custom Python build. Packages are built from source unless explicitly listed here. This includes: pure Python packages, build tools +# that don't need ABI compatibility, and packages with data files only. SET(RV_PYTHON_WHEEL_SAFE "pip" # Package installer (pure Python) "setuptools" # Build system (pure Python) "wheel" # Wheel format support (pure Python) + "Cython" # Build tool (compiles Python to C, but the tool itself can use pre-built wheels) + "meson-python" # Build backend (pure Python) + "ninja" # Build tool (native but doesn't link to Python) "PyOpenGL" # OpenGL bindings without acceleration (pure Python) "certifi" # SSL certificate bundle (just data files) "six" # Python 2/3 compatibility (pure Python) "packaging" # Version parsing (pure Python) "requests" # HTTP library (pure Python) - CACHE STRING "Pure Python packages safe to install from wheels" + CACHE STRING "Packages safe to install from wheels (pure Python or build tools)" ) # Convert list to comma-separated string for pip's --only-binary flag diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index a24281855..16c09f0f6 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -3,7 +3,7 @@ # To update OpenTimelineIO version, edit RV_DEPS_OTIO_VERSION in cmake/defaults/CYCOMMON.cmake # To update NumPy version, edit RV_DEPS_NUMPY_VERSION in cmake/defaults/CY*.cmake # NOTE: By default, all packages are built from source to ensure proper ABI compatibility with our custom Python. -# Pure Python packages safe to install from wheels are listed in RV_PYTHON_WHEEL_SAFE in python3.cmake. +# Pure Python packages and build tools safe to install from wheels are listed in RV_PYTHON_WHEEL_SAFE in python3.cmake. # This approach ensures safety by default: new packages will be built from source unless explicitly marked as wheel-safe. pip # License: MIT License (MIT) From 1f2f8114500cc7ab01f9032e4ace90c9bd07384b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9drik=20Fuoco?= Date: Fri, 19 Dec 2025 14:38:55 -0500 Subject: [PATCH 24/24] move sitecustomize outside of make_python, small changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédrik Fuoco --- src/build/make_python.py | 99 ++++---------------------------------- src/build/sitecustomize.py | 80 ++++++++++++++++++++++++++++++ src/build/test_python.py | 2 +- 3 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 src/build/sitecustomize.py diff --git a/src/build/make_python.py b/src/build/make_python.py index 416a57c5b..97938a225 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -19,7 +19,6 @@ import platform from typing import List -from datetime import datetime ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(ROOT_DIR) @@ -33,96 +32,16 @@ LIB_DIR = "" -SITECUSTOMIZE_FILE_CONTENT = f''' -# -# Copyright (c) {datetime.now().year} Autodesk, Inc. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# -""" -Site-level module that ensures OpenSSL will have up to date certificate authorities -on Linux and macOS. It gets imported when the Python interpreter starts up, both -when launching Python as a standalone interpreter or as an embedded one. -The OpenSSL shipped with Desktop requires a list of certificate authorities to be -distributed with the build instead of relying on the OS keychain. In order to keep -an up to date list, we're going to pull it from the certifi module, which incorporates -all the certificate authorities that are distributed with Firefox. -""" -import site -import sys +def get_sitecustomize_content() -> str: + """ + Load and return the sitecustomize.py content. -try: - import os - import certifi - - # Do not set SSL_CERT_FILE to our own if it is already set. Someone could - # have their own certificate authority that they specify with this env var. - # Unfortunately this is not a PATH like environment variable, so we can't - # concatenate multiple paths with ":". - # - # To learn more about SSL_CERT_FILE and how it is being used by OpenSSL when - # verifying certificates, visit - # https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_set_default_verify_paths.html - if "SSL_CERT_FILE" not in os.environ and "DO_NOT_SET_SSL_CERT_FILE" not in os.environ: - os.environ["SSL_CERT_FILE"] = certifi.where() - -except ImportError: - # certifi not installed yet - this is expected during build when pip installs build dependencies - pass -except Exception as e: - # Only print errors for unexpected exceptions, and only if verbose mode is enabled - try: - import os as _os - if "PYTHONVERBOSE" in _os.environ: - print("Failed to set certifi.where() as SSL_CERT_FILE.", file=sys.stderr) - print(e, file=sys.stderr) - print("Set DO_NOT_SET_SSL_CERT_FILE to skip this step in RV's Python initialization.", file=sys.stderr) - except: - pass - -try: - import os - - if "DO_NOT_REORDER_PYTHON_PATH" not in os.environ: - import site - import sys - - prefixes = list(set(site.PREFIXES)) - - # Python libs and site-packages is the first that should be in the PATH - new_path_list = list(set(site.getsitepackages())) - new_path_list.insert(0, os.path.dirname(new_path_list[0])) - - # Then any paths in RV's app package - for path in sys.path: - for prefix in prefixes: - if path.startswith(prefix) is False: - continue - - if os.path.exists(path): - new_path_list.append(path) - - # Then the remaining paths - for path in sys.path: - if os.path.exists(path): - new_path_list.append(path) - - # Save the new sys.path - sys.path = new_path_list - site.removeduppaths() - -except Exception as e: - # Only print errors if verbose mode is enabled - try: - import os as _os - if "PYTHONVERBOSE" in _os.environ: - print("Failed to reorder RV's Python search path", file=sys.stderr) - print(e, file=sys.stderr) - print("Set DO_NOT_REORDER_PYTHON_PATH to skip this step in RV's Python initialization.", file=sys.stderr) - except: - pass -''' + :return: The sitecustomize.py content as a string + """ + template_path = os.path.join(ROOT_DIR, "sitecustomize.py") + with open(template_path, "r") as f: + return f.read() def get_python_interpreter_args(python_home: str, variant: str) -> List[str]: @@ -261,7 +180,7 @@ def patch_python_distribution(python_home: str) -> None: os.remove(site_customize_path) with open(site_customize_path, "w") as sitecustomize_file: - sitecustomize_file.write(SITECUSTOMIZE_FILE_CONTENT) + sitecustomize_file.write(get_sitecustomize_content()) def clean() -> None: diff --git a/src/build/sitecustomize.py b/src/build/sitecustomize.py new file mode 100644 index 000000000..431ebd3a6 --- /dev/null +++ b/src/build/sitecustomize.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2025 Autodesk, Inc. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Site-level module that ensures OpenSSL will have up to date certificate authorities +on Linux and macOS. It gets imported when the Python interpreter starts up, both +when launching Python as a standalone interpreter or as an embedded one. +The OpenSSL shipped with Desktop requires a list of certificate authorities to be +distributed with the build instead of relying on the OS keychain. In order to keep +an up to date list, we're going to pull it from the certifi module, which incorporates +all the certificate authorities that are distributed with Firefox. +""" + +import site +import sys +import os + +try: + import certifi + + # Do not set SSL_CERT_FILE to our own if it is already set. Someone could + # have their own certificate authority that they specify with this env var. + # Unfortunately this is not a PATH like environment variable, so we can't + # concatenate multiple paths with ":". + # + # To learn more about SSL_CERT_FILE and how it is being used by OpenSSL when + # verifying certificates, visit + # https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_set_default_verify_paths.html + if "SSL_CERT_FILE" not in os.environ and "DO_NOT_SET_SSL_CERT_FILE" not in os.environ: + os.environ["SSL_CERT_FILE"] = certifi.where() + +except ImportError: + # certifi not installed yet - this is expected during build when pip installs build dependencies + pass +except Exception as e: + if "PYTHONVERBOSE" in os.environ: + # During regular builds, we silently skip these errors so sitecustomize.py can load even + # if certifi is temporarily unavailable. + print("Failed to set certifi.where() as SSL_CERT_FILE.", file=sys.stderr) + print(e, file=sys.stderr) + print("Set DO_NOT_SET_SSL_CERT_FILE to skip this step in RV's Python initialization.", file=sys.stderr) + +try: + if "DO_NOT_REORDER_PYTHON_PATH" not in os.environ: + import site + import sys + + prefixes = list(set(site.PREFIXES)) + + # Python libs and site-packages is the first that should be in the PATH + new_path_list = list(set(site.getsitepackages())) + new_path_list.insert(0, os.path.dirname(new_path_list[0])) + + # Then any paths in RV's app package + for path in sys.path: + for prefix in prefixes: + if path.startswith(prefix) is False: + continue + + if os.path.exists(path): + new_path_list.append(path) + + # Then the remaining paths + for path in sys.path: + if os.path.exists(path): + new_path_list.append(path) + + # Save the new sys.path + sys.path = new_path_list + site.removeduppaths() + +except Exception as e: + if "PYTHONVERBOSE" in os.environ: + # Display path-reordering failures only when verbose tracing is explicitly requested. + print("Failed to reorder RV's Python search path", file=sys.stderr) + print(e, file=sys.stderr) + print("Set DO_NOT_REORDER_PYTHON_PATH to skip this step in RV's Python initialization.", file=sys.stderr) diff --git a/src/build/test_python.py b/src/build/test_python.py index 1f4f36534..62c3a1ebe 100644 --- a/src/build/test_python.py +++ b/src/build/test_python.py @@ -73,7 +73,7 @@ def test_python_distribution(python_home: str, variant: str) -> None: # Check for tkinter "try:", " import tkinter", - "except:", + "except Exception:", " import Tkinter as tkinter", # Make sure certifi is available "import certifi",