diff --git a/cmake/defaults/CY2023.cmake b/cmake/defaults/CY2023.cmake index 42d7fd799..42ff73c38 100644 --- a/cmake/defaults/CY2023.cmake +++ b/cmake/defaults/CY2023.cmake @@ -61,8 +61,10 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2023") "3_1" ) - # NumPy NumPY for CY2023 is 1.23.x series but Pyside2 requires < 1.23 So we comment this out and make_pyside.py hardcoded to use NumPy < 1.23 - # SET(ENV{RV_DEPS_NUMPY_VERSION} "1.23.5") + # NumPy NumPy for CY2023 VFX platform is 1.23.x series, but PySide2 requires < 1.23 Using numpy 1.22.4 (last version before 1.23) for PySide2 compatibility + SET(RV_DEPS_NUMPY_VERSION + "1.22.4" + ) # OCIO https://github.com/AcademySoftwareFoundation/OpenColorIO SET(RV_DEPS_OCIO_VERSION diff --git a/cmake/defaults/CY2024.cmake b/cmake/defaults/CY2024.cmake index 0470796ef..809bb8536 100644 --- a/cmake/defaults/CY2024.cmake +++ b/cmake/defaults/CY2024.cmake @@ -36,7 +36,7 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2024") ) # NumPy https://numpy.org/doc/stable/release.html - SET(ENV{RV_DEPS_NUMPY_VERSION} + SET(RV_DEPS_NUMPY_VERSION "1.24.4" ) diff --git a/cmake/defaults/CY2025.cmake b/cmake/defaults/CY2025.cmake index 36499ecda..20b7f641b 100644 --- a/cmake/defaults/CY2025.cmake +++ b/cmake/defaults/CY2025.cmake @@ -36,7 +36,7 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2025") ) # NumPy https://numpy.org/doc/stable/release.html - SET(ENV{RV_DEPS_NUMPY_VERSION} + SET(RV_DEPS_NUMPY_VERSION "1.26.4" ) diff --git a/cmake/defaults/CY2026.cmake b/cmake/defaults/CY2026.cmake index 3b3e773fb..d5a5da455 100644 --- a/cmake/defaults/CY2026.cmake +++ b/cmake/defaults/CY2026.cmake @@ -36,7 +36,7 @@ IF(RV_VFX_PLATFORM STREQUAL "CY2026") ) # NumPy https://numpy.org/doc/stable/release.html - SET(ENV{RV_DEPS_NUMPY_VERSION} + SET(RV_DEPS_NUMPY_VERSION "2.3.0" ) diff --git a/cmake/dependencies/python3.cmake b/cmake/dependencies/python3.cmake index fbd2b363a..a0a5031d5 100644 --- a/cmake/dependencies/python3.cmake +++ b/cmake/dependencies/python3.cmake @@ -38,6 +38,10 @@ SET(_pyside_version "${RV_DEPS_PYSIDE_VERSION}" ) +SET(_numpy_version + "${RV_DEPS_NUMPY_VERSION}" +) + SET(_python3_download_url "https://github.com/python/cpython/archive/refs/tags/v${_python3_version}.zip" ) @@ -244,7 +248,7 @@ ELSE() ) ENDIF() -# Generate requirements.txt from template with the OpenTimelineIO version substituted +# Generate requirements.txt from template with the OpenTimelineIO and NumPy versions substituted SET(_requirements_input_file "${PROJECT_SOURCE_DIR}/src/build/requirements.txt.in" ) @@ -271,8 +275,27 @@ ELSE() ) ENDIF() -# Using --no-binary :all: to ensure all packages with native extensions are built from source against our custom Python build, preventing ABI compatibility -# issues. +# List of packages that are safe to install from pre-built wheels. All other packages (those with C/C++/Rust extensions) will be built from source to ensure +# proper linking against our custom Python build. Packages are built from source unless explicitly listed here. This includes: pure Python packages, build tools +# that don't need ABI compatibility, and packages with data files only. +SET(RV_PYTHON_WHEEL_SAFE + "pip" # Package installer (pure Python) + "setuptools" # Build system (pure Python) + "wheel" # Wheel format support (pure Python) + "Cython" # Build tool (compiles Python to C, but the tool itself can use pre-built wheels) + "meson-python" # Build backend (pure Python) + "ninja" # Build tool (native but doesn't link to Python) + "PyOpenGL" # OpenGL bindings without acceleration (pure Python) + "certifi" # SSL certificate bundle (just data files) + "six" # Python 2/3 compatibility (pure Python) + "packaging" # Version parsing (pure Python) + "requests" # HTTP library (pure Python) + CACHE STRING "Packages safe to install from wheels (pure Python or build tools)" +) + +# Convert list to comma-separated string for pip's --only-binary flag +STRING(REPLACE ";" "," _wheel_safe_packages "${RV_PYTHON_WHEEL_SAFE}") + SET(_requirements_install_command ${CMAKE_COMMAND} -E env ${_otio_debug_env} ) @@ -282,6 +305,8 @@ IF(DEFINED RV_DEPS_OPENSSL_INSTALL_DIR) LIST(APPEND _requirements_install_command "OPENSSL_DIR=${RV_DEPS_OPENSSL_INSTALL_DIR}") ENDIF() +# Build all packages from source except those in RV_PYTHON_WHEEL_SAFE. Packages with native extensions (opentimelineio, numpy, PyOpenGL-accelerate, +# cryptography, pydantic, cffi, etc.) will be built from source for proper ABI compatibility. LIST( APPEND _requirements_install_command @@ -298,6 +323,8 @@ LIST( --force-reinstall --no-binary :all: + --only-binary + ${_wheel_safe_packages} -r "${_requirements_output_file}" ) @@ -390,13 +417,30 @@ SET(${_python3_target}-requirements-flag ) ADD_CUSTOM_COMMAND( - COMMENT "Installing requirements from ${_requirements_output_file}" + COMMENT "Installing requirements from ${_requirements_output_file} pyside and other dependencies" OUTPUT ${${_python3_target}-requirements-flag} COMMAND ${_requirements_install_command} COMMAND cmake -E touch ${${_python3_target}-requirements-flag} DEPENDS ${_python3_target} ${_requirements_output_file} ${_requirements_input_file} ) +# Test the Python distribution after requirements are installed +SET(${_python3_target}-test-flag + ${_install_dir}/${_python3_target}-test-flag +) + +SET(_test_python_script + "${PROJECT_SOURCE_DIR}/src/build/test_python.py" +) + +ADD_CUSTOM_COMMAND( + COMMENT "Testing Python distribution" + OUTPUT ${${_python3_target}-test-flag} + COMMAND python3 "${_test_python_script}" --python-home "${_install_dir}" --variant "${CMAKE_BUILD_TYPE}" + COMMAND cmake -E touch ${${_python3_target}-test-flag} + DEPENDS ${${_python3_target}-requirements-flag} ${_test_python_script} +) + IF(RV_TARGET_WINDOWS AND CMAKE_BUILD_TYPE MATCHES "^Debug$" ) @@ -427,7 +471,7 @@ IF(RV_VFX_PLATFORM STREQUAL CY2023) ${rv_deps_pyside2_SOURCE_DIR}/build_scripts/platforms/windows_desktop.py COMMAND ${_pyside_make_command} --prepare --build COMMAND cmake -E touch ${${_pyside_target}-build-flag} - DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} + DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} USES_TERMINAL ) @@ -438,9 +482,9 @@ ELSEIF(RV_VFX_PLATFORM STRGREATER_EQUAL CY2024) ADD_CUSTOM_COMMAND( COMMENT "Building PySide6 using ${_pyside_make_command_script}" OUTPUT ${${_pyside_target}-build-flag} - COMMAND ${CMAKE_COMMAND} -E env "RV_DEPS_NUMPY_VERSION=$ENV{RV_DEPS_NUMPY_VERSION}" ${_pyside_make_command} --prepare --build + COMMAND ${_pyside_make_command} --prepare --build COMMAND cmake -E touch ${${_pyside_target}-build-flag} - DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} + DEPENDS ${_python3_target} ${_pyside_make_command_script} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} USES_TERMINAL ) @@ -471,7 +515,7 @@ IF(RV_TARGET_WINDOWS) ADD_CUSTOM_COMMAND( COMMENT "Installing ${_python3_target}'s include and libs into ${RV_STAGE_LIB_DIR}" OUTPUT ${RV_STAGE_BIN_DIR}/${_python3_lib_name} ${_copy_commands} - DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${_build_flag_depends} + DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} ${_build_flag_depends} ) ADD_CUSTOM_TARGET( @@ -485,7 +529,7 @@ ELSE() COMMAND ${CMAKE_COMMAND} -E copy_directory ${_install_dir}/lib ${RV_STAGE_LIB_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory ${_install_dir}/include ${RV_STAGE_INCLUDE_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory ${_install_dir}/bin ${RV_STAGE_BIN_DIR} - DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${_build_flag_depends} + DEPENDS ${_python3_target} ${${_python3_target}-requirements-flag} ${${_python3_target}-test-flag} ${_build_flag_depends} ) ADD_CUSTOM_TARGET( ${_python3_target}-stage-target ALL diff --git a/src/build/make_pyside6.py b/src/build/make_pyside6.py index ee3f33d1b..fd95ee536 100755 --- a/src/build/make_pyside6.py +++ b/src/build/make_pyside6.py @@ -159,18 +159,7 @@ def get_fallback_clang_filename_suffix(version): os.environ["LLVM_INSTALL_DIR"] = libclang_install_dir os.environ["CLANG_INSTALL_DIR"] = libclang_install_dir - # PySide6 build requires numpy 1.26.3 - numpy_version = os.environ.get("RV_DEPS_NUMPY_VERSION") - if not numpy_version: - raise ValueError("RV_DEPS_NUMPY_VERSION environment variable is not set.") - install_numpy_args = get_python_interpreter_args(PYTHON_OUTPUT_DIR, VARIANT) + [ - "-m", - "pip", - "install", - f"numpy=={numpy_version}", - ] - print(f"Installing numpy with {install_numpy_args}") - subprocess.run(install_numpy_args).check_returncode() + # Note: numpy is now installed via requirements.txt in python3.cmake before PySide6 builds. cmakelist_path = os.path.join(SOURCE_DIR, "sources", "shiboken6", "ApiExtractor", "CMakeLists.txt") old_cmakelist_path = os.path.join(SOURCE_DIR, "sources", "shiboken6", "ApiExtractor", "CMakeLists.txt.old") diff --git a/src/build/make_python.py b/src/build/make_python.py index 3537a0f92..97938a225 100755 --- a/src/build/make_python.py +++ b/src/build/make_python.py @@ -17,11 +17,8 @@ import sys import subprocess import platform -import tempfile -import uuid from typing import List -from datetime import datetime ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(ROOT_DIR) @@ -35,80 +32,16 @@ LIB_DIR = "" -SITECUSTOMIZE_FILE_CONTENT = f''' -# -# Copyright (c) {datetime.now().year} Autodesk, Inc. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 -# -""" -Site-level module that ensures OpenSSL will have up to date certificate authorities -on Linux and macOS. It gets imported when the Python interpreter starts up, both -when launching Python as a standalone interpreter or as an embedded one. -The OpenSSL shipped with Desktop requires a list of certificate authorities to be -distributed with the build instead of relying on the OS keychain. In order to keep -an up to date list, we're going to pull it from the certifi module, which incorporates -all the certificate authorities that are distributed with Firefox. -""" -import site - -try: - import os - import certifi - - # Do not set SSL_CERT_FILE to our own if it is already set. Someone could - # have their own certificate authority that they specify with this env var. - # Unfortunately this is not a PATH like environment variable, so we can't - # concatenate multiple paths with ":". - # - # To learn more about SSL_CERT_FILE and how it is being used by OpenSSL when - # verifying certificates, visit - # https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_set_default_verify_paths.html - if "SSL_CERT_FILE" not in os.environ and "DO_NOT_SET_SSL_CERT_FILE" not in os.environ: - os.environ["SSL_CERT_FILE"] = certifi.where() - -except Exception as e: - print("Failed to set certifi.where() as SSL_CERT_FILE.", file=sys.stderr) - print(e, file=sys.stderr) - print("Set DO_NOT_SET_SSL_CERT_FILE to skip this step in RV's Python initialization.", file=sys.stderr) - -try: - import os - - if "DO_NOT_REORDER_PYTHON_PATH" not in os.environ: - import site - import sys - - prefixes = list(set(site.PREFIXES)) - - # Python libs and site-packages is the first that should be in the PATH - new_path_list = list(set(site.getsitepackages())) - new_path_list.insert(0, os.path.dirname(new_path_list[0])) - - # Then any paths in RV's app package - for path in sys.path: - for prefix in prefixes: - if path.startswith(prefix) is False: - continue - - if os.path.exists(path): - new_path_list.append(path) - - # Then the remaining paths - for path in sys.path: - if os.path.exists(path): - new_path_list.append(path) - - # Save the new sys.path - sys.path = new_path_list - site.removeduppaths() - -except Exception as e: - print("Failed to reorder RV's Python search path", file=sys.stderr) - print(e, file=sys.stderr) - print("Set DO_NOT_REORDER_PYTHON_PATH to skip this step in RV's Python initialization.", file=sys.stderr) -''' +def get_sitecustomize_content() -> str: + """ + Load and return the sitecustomize.py content. + + :return: The sitecustomize.py content as a string + """ + template_path = os.path.join(ROOT_DIR, "sitecustomize.py") + with open(template_path, "r") as f: + return f.read() def get_python_interpreter_args(python_home: str, variant: str) -> List[str]: @@ -138,8 +71,16 @@ def get_python_interpreter_args(python_home: str, variant: str) -> List[str]: ), ) - if not python_interpreters or os.path.exists(python_interpreters[0]) is False: - raise FileNotFoundError() + if not python_interpreters: + raise FileNotFoundError( + f"No Python interpreter found in {python_home}. " + f"Searched for pattern '{python_name_pattern}' in {python_home} (recursively) and {os.path.join(python_home, 'bin')}. " + ) + + if not os.path.exists(python_interpreters[0]): + raise FileNotFoundError( + f"Python interpreter does not exist: {python_interpreters[0]}. Found interpreters: {python_interpreters}" + ) print(f"Found python interpreters {python_interpreters}") @@ -221,27 +162,6 @@ def patch_python_distribution(python_home: str) -> None: print(f"Ensuring pip with {ensure_pip_args}") subprocess.run(ensure_pip_args).check_returncode() - pip_args = python_interpreter_args + ["-m", "pip"] - - for package in ["pip", "certifi", "six", "wheel", "packaging", "requests", "pydantic"]: - package_install_args = pip_args + [ - "install", - "--upgrade", - "--force-reinstall", - package, - ] - print(f"Installing {package} with {package_install_args}") - subprocess.run(package_install_args).check_returncode() - - wheel_install_args = pip_args + [ - "install", - "--upgrade", - "--force-reinstall", - "wheel", - ] - print(f"Installing wheel with {wheel_install_args}") - subprocess.run(wheel_install_args).check_returncode() - site_packages = glob.glob(os.path.join(python_home, "**", "site-packages"), recursive=True)[0] if os.path.exists(site_packages) is False: @@ -260,97 +180,7 @@ def patch_python_distribution(python_home: str) -> None: os.remove(site_customize_path) with open(site_customize_path, "w") as sitecustomize_file: - sitecustomize_file.write(SITECUSTOMIZE_FILE_CONTENT) - - -def test_python_distribution(python_home: str) -> None: - """ - Test the Python distribution. - - :param python_home: Package root of an Python package - """ - tmp_dir = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) - os.makedirs(tmp_dir) - - tmp_python_home = os.path.join(tmp_dir, os.path.basename(python_home)) - try: - print(f"Moving {python_home} to {tmp_python_home}") - shutil.move(python_home, tmp_python_home) - - python_interpreter_args = get_python_interpreter_args(tmp_python_home, VARIANT) - - # Note: OpenTimelineIO is installed via requirements.txt for all platforms and build types. - # The git URL in requirements.txt ensures it builds from source with proper linkage. - - wheel_install_arg = python_interpreter_args + [ - "-m", - "pip", - "install", - "cryptography", - ] - - print(f"Validating that we can install a wheel with {wheel_install_arg}") - subprocess.run(wheel_install_arg).check_returncode() - - python_validation_args = python_interpreter_args + [ - "-c", - "\n".join( - [ - # Check for tkinter - "try:", - " import tkinter", - "except:", - " import Tkinter as tkinter", - # Make sure certifi is available - "import certifi", - # Make sure the SSL_CERT_FILE variable is sett - "import os", - "assert certifi.where() == os.environ['SSL_CERT_FILE']", - # Make sure ssl is correctly built and linked - "import ssl", - # Misc - "import sqlite3", - "import ctypes", - "import ssl", - "import _ssl", - "import zlib", - ] - ), - ] - print(f"Validating the python package with {python_validation_args}") - subprocess.run(python_validation_args).check_returncode() - - dummy_ssl_file = os.path.join("Path", "To", "Dummy", "File") - python_validation2_args = python_interpreter_args + [ - "-c", - "\n".join( - [ - "import os", - f"assert os.environ['SSL_CERT_FILE'] == '{dummy_ssl_file}'", - ] - ), - ] - print(f"Validating the python package with {python_validation2_args}") - subprocess.run(python_validation2_args, env={**os.environ, "SSL_CERT_FILE": dummy_ssl_file}).check_returncode() - - python_validation3_args = python_interpreter_args + [ - "-c", - "\n".join( - [ - "import os", - "assert 'SSL_CERT_FILE' not in os.environ", - ] - ), - ] - print(f"Validating the python package with {python_validation3_args}") - subprocess.run( - python_validation3_args, - env={**os.environ, "DO_NOT_SET_SSL_CERT_FILE": "bleh"}, - ).check_returncode() - - finally: - print(f"Moving {tmp_python_home} to {python_home}") - shutil.move(tmp_python_home, python_home) + sitecustomize_file.write(get_sitecustomize_content()) def clean() -> None: @@ -608,7 +438,7 @@ def install_python_vfx2023() -> None: os.symlink(os.path.basename(python3_path), python_path) patch_python_distribution(OUTPUT_DIR) - test_python_distribution(OUTPUT_DIR) + # Note: Testing is now done via test_python.py after requirements.txt installation def install_python_vfx2024() -> None: @@ -722,7 +552,7 @@ def install_python_vfx2024() -> None: os.symlink(os.path.basename(python3_path), python_path) patch_python_distribution(OUTPUT_DIR) - test_python_distribution(OUTPUT_DIR) + # Note: Testing is now done via test_python.py after requirements.txt installation if __name__ == "__main__": diff --git a/src/build/requirements.txt.in b/src/build/requirements.txt.in index 2be3bf364..16c09f0f6 100644 --- a/src/build/requirements.txt.in +++ b/src/build/requirements.txt.in @@ -1,10 +1,15 @@ # This file contains all the packages that will be packaged with RV. Please add the license next to the package. # NOTE: This is a CMake template file. The actual requirements.txt is generated during CMake configuration. -# To update OpenTimelineIO version, edit _opentimelineio_version in cmake/dependencies/python3.cmake -# NOTE: Using --no-binary :all: in pip install ensures all packages are built from source against our custom Python. +# To update OpenTimelineIO version, edit RV_DEPS_OTIO_VERSION in cmake/defaults/CYCOMMON.cmake +# To update NumPy version, edit RV_DEPS_NUMPY_VERSION in cmake/defaults/CY*.cmake +# NOTE: By default, all packages are built from source to ensure proper ABI compatibility with our custom Python. +# Pure Python packages and build tools safe to install from wheels are listed in RV_PYTHON_WHEEL_SAFE in python3.cmake. +# This approach ensures safety by default: new packages will be built from source unless explicitly marked as wheel-safe. pip # License: MIT License (MIT) setuptools # License: MIT License +wheel # License: MIT License (MIT) +numpy==@_numpy_version@ # License: BSD License (BSD-3-Clause) - Required by PySide6 opentimelineio==@_opentimelineio_version@ # License: Other/Proprietary License (Modified Apache 2.0 License) PyOpenGL # License: BSD License (BSD) @@ -16,11 +21,10 @@ PyOpenGL # License: BSD License (BSD) # Use PyOpenGL_accelerate only on x86_64 platform. PyOpenGL_accelerate ; (platform_system=='Windows' or platform_system=='Linux') and platform_machine=='x86_64' # License: BSD License (BSD) -# Those are installed by the src/build/make_python.py script, adding them here to list their licenses. +# Additional packages required by RV and for testing the Python distribution -certifi # License: Mozilla Public License 2.0 (MPL 2.0) (MPL-2.0) +certifi # License: Mozilla Public License 2.0 (MPL 2.0) (MPL-2.0) - required by test_python.py six # License: MIT License (MIT) -wheel # License: MIT License (MIT) packaging # License: Apache Software License, BSD License requests # License: Apache Software License (Apache 2.0) cryptography # License: Apache Software License, BSD License ((Apache-2.0 OR BSD-3-Clause) AND PSF-2.0) diff --git a/src/build/sitecustomize.py b/src/build/sitecustomize.py new file mode 100644 index 000000000..431ebd3a6 --- /dev/null +++ b/src/build/sitecustomize.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2025 Autodesk, Inc. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Site-level module that ensures OpenSSL will have up to date certificate authorities +on Linux and macOS. It gets imported when the Python interpreter starts up, both +when launching Python as a standalone interpreter or as an embedded one. +The OpenSSL shipped with Desktop requires a list of certificate authorities to be +distributed with the build instead of relying on the OS keychain. In order to keep +an up to date list, we're going to pull it from the certifi module, which incorporates +all the certificate authorities that are distributed with Firefox. +""" + +import site +import sys +import os + +try: + import certifi + + # Do not set SSL_CERT_FILE to our own if it is already set. Someone could + # have their own certificate authority that they specify with this env var. + # Unfortunately this is not a PATH like environment variable, so we can't + # concatenate multiple paths with ":". + # + # To learn more about SSL_CERT_FILE and how it is being used by OpenSSL when + # verifying certificates, visit + # https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_set_default_verify_paths.html + if "SSL_CERT_FILE" not in os.environ and "DO_NOT_SET_SSL_CERT_FILE" not in os.environ: + os.environ["SSL_CERT_FILE"] = certifi.where() + +except ImportError: + # certifi not installed yet - this is expected during build when pip installs build dependencies + pass +except Exception as e: + if "PYTHONVERBOSE" in os.environ: + # During regular builds, we silently skip these errors so sitecustomize.py can load even + # if certifi is temporarily unavailable. + print("Failed to set certifi.where() as SSL_CERT_FILE.", file=sys.stderr) + print(e, file=sys.stderr) + print("Set DO_NOT_SET_SSL_CERT_FILE to skip this step in RV's Python initialization.", file=sys.stderr) + +try: + if "DO_NOT_REORDER_PYTHON_PATH" not in os.environ: + import site + import sys + + prefixes = list(set(site.PREFIXES)) + + # Python libs and site-packages is the first that should be in the PATH + new_path_list = list(set(site.getsitepackages())) + new_path_list.insert(0, os.path.dirname(new_path_list[0])) + + # Then any paths in RV's app package + for path in sys.path: + for prefix in prefixes: + if path.startswith(prefix) is False: + continue + + if os.path.exists(path): + new_path_list.append(path) + + # Then the remaining paths + for path in sys.path: + if os.path.exists(path): + new_path_list.append(path) + + # Save the new sys.path + sys.path = new_path_list + site.removeduppaths() + +except Exception as e: + if "PYTHONVERBOSE" in os.environ: + # Display path-reordering failures only when verbose tracing is explicitly requested. + print("Failed to reorder RV's Python search path", file=sys.stderr) + print(e, file=sys.stderr) + print("Set DO_NOT_REORDER_PYTHON_PATH to skip this step in RV's Python initialization.", file=sys.stderr) diff --git a/src/build/test_python.py b/src/build/test_python.py new file mode 100644 index 000000000..62c3a1ebe --- /dev/null +++ b/src/build/test_python.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# ***************************************************************************** +# Copyright 2020 Autodesk, Inc. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# ***************************************************************************** + +""" +Test script for validating a Python distribution built by make_python.py. + +This script validates that the Python distribution is correctly built, +relocatable, and has all necessary dependencies installed. +""" + +import argparse +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import uuid + +# Import the helper function from make_python.py (same directory) +from make_python import get_python_interpreter_args + + +def test_python_distribution(python_home: str, variant: str) -> None: + """ + Test the Python distribution. + + This test validates: + - The distribution is relocatable (works when moved to a temp directory) + - Can install wheels (cryptography package) + - All critical modules are available (ssl, tkinter, certifi, etc.) + - SSL_CERT_FILE environment variable is properly set + - Respects user-provided SSL_CERT_FILE + - Respects DO_NOT_SET_SSL_CERT_FILE flag + + :param python_home: Package root of a Python package + :param variant: Build variant (Debug or Release) + """ + tmp_dir = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) + os.makedirs(tmp_dir) + + tmp_python_home = os.path.join(tmp_dir, os.path.basename(python_home)) + try: + print(f"Moving {python_home} to {tmp_python_home}") + shutil.move(python_home, tmp_python_home) + + python_interpreter_args = get_python_interpreter_args(tmp_python_home, variant) + + # Note: OpenTimelineIO is installed via requirements.txt for all platforms and build types. + # The git URL in requirements.txt ensures it builds from source with proper linkage. + + wheel_install_arg = python_interpreter_args + [ + "-m", + "pip", + "install", + "cryptography", + ] + + print(f"Validating that we can install a wheel with {wheel_install_arg}") + subprocess.run(wheel_install_arg).check_returncode() + + python_validation_args = python_interpreter_args + [ + "-c", + "\n".join( + [ + # Check for tkinter + "try:", + " import tkinter", + "except Exception:", + " import Tkinter as tkinter", + # Make sure certifi is available + "import certifi", + # Make sure the SSL_CERT_FILE variable is set + "import os", + "assert certifi.where() == os.environ['SSL_CERT_FILE']", + # Make sure ssl is correctly built and linked + "import ssl", + # Misc + "import sqlite3", + "import ctypes", + "import ssl", + "import _ssl", + "import zlib", + ] + ), + ] + print(f"Validating the python package with {python_validation_args}") + subprocess.run(python_validation_args).check_returncode() + + dummy_ssl_file = os.path.join("Path", "To", "Dummy", "File") + python_validation2_args = python_interpreter_args + [ + "-c", + "\n".join( + [ + "import os", + f"assert os.environ['SSL_CERT_FILE'] == '{dummy_ssl_file}'", + ] + ), + ] + print(f"Validating the python package with {python_validation2_args}") + subprocess.run(python_validation2_args, env={**os.environ, "SSL_CERT_FILE": dummy_ssl_file}).check_returncode() + + python_validation3_args = python_interpreter_args + [ + "-c", + "\n".join( + [ + "import os", + "assert 'SSL_CERT_FILE' not in os.environ", + ] + ), + ] + print(f"Validating the python package with {python_validation3_args}") + subprocess.run( + python_validation3_args, + env={**os.environ, "DO_NOT_SET_SSL_CERT_FILE": "bleh"}, + ).check_returncode() + + print("All Python distribution tests passed successfully!") + + finally: + print(f"Moving {tmp_python_home} back to {python_home}") + shutil.move(tmp_python_home, python_home) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Test a Python distribution built by make_python.py") + + parser.add_argument( + "--python-home", + dest="python_home", + type=pathlib.Path, + required=True, + help="Path to the Python installation directory to test", + ) + parser.add_argument( + "--variant", + dest="variant", + type=str, + required=True, + choices=["Debug", "Release"], + help="Build variant (Debug or Release)", + ) + + args = parser.parse_args() + + if not os.path.exists(args.python_home): + print(f"Error: Python home directory does not exist: {args.python_home}", file=sys.stderr) + sys.exit(1) + + try: + test_python_distribution(args.python_home, args.variant) + print("\n[OK] Python distribution validation completed successfully") + sys.exit(0) + except Exception as e: + print(f"\n[FAILED] Python distribution validation failed: {e}", file=sys.stderr) + import traceback + + traceback.print_exc() + sys.exit(1)