diff --git a/v/vllm/vllm_ubi_9.3.sh b/v/vllm/vllm_ubi_9.3.sh index 9bc17df1aa..2a80c1f447 100644 --- a/v/vllm/vllm_ubi_9.3.sh +++ b/v/vllm/vllm_ubi_9.3.sh @@ -1,16 +1,14 @@ #!/bin/bash -e -# # ----------------------------------------------------------------------------- # # Package : vllm # Version : v0.8.4 -# Source repo : https://github.com/vllm-project/vllm.git -# Tested on : UBI 9.3 -# Language : c -# Ci-Check : True +# Source repo : https://github.com/vllm-project/vllm +# Tested on : UBI:9.6 +# Language : Python +# Ci-Check : True # Script License: Apache License 2.0 # Maintainer : Sai Kiran Nukala -# # Disclaimer: This script has been tested in root mode on given # ========== platform using the mentioned version of the package. # It may not work as expected with newer versions of the @@ -19,974 +17,67 @@ # # ---------------------------------------------------------------------------- -set -e - +set -e +# Variables PACKAGE_NAME=vllm PACKAGE_VERSION=${1:-v0.8.4} PACKAGE_URL=https://github.com/vllm-project/vllm.git CURRENT_DIR=$(pwd) PACKAGE_DIR=vllm -yum install -y openblas-devel git make libtool wget gcc-toolset-13 cmake python3.12 python3.12-devel python3.12-pip gcc-toolset-13 gcc-toolset-13-binutils gcc-toolset-13-binutils-devel gcc-toolset-13-gcc-c++ git make cmake binutils -yum install -y git \ - meson ninja-build openblas-devel openssl-devel libjpeg-devel bzip2-devel libffi-devel zlib-devel \ - libtiff-devel freetype-devel make cmake automake autoconf procps-ng \ - python3.12 python3.12-devel python3.12-pip - -export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH -gcc --version - -export GCC_HOME=/opt/rh/gcc-toolset-13/root/usr -export CC=$GCC_HOME/bin/gcc -export CXX=$GCC_HOME/bin/g++ - -OS_NAME=$(cat /etc/os-release | grep ^PRETTY_NAME | cut -d= -f2) -python3.12 -m pip install --upgrade pip build -INSTALL_ROOT="/install-deps" -mkdir -p $INSTALL_ROOT -for package in openblas lame opus libvpx ffmpeg pillow numpy; do - mkdir -p ${INSTALL_ROOT}/${package} - export "${package^^}_PREFIX=${INSTALL_ROOT}/${package}" - echo "Exported ${package^^}_PREFIX=${INSTALL_ROOT}/${package}" -done - -python3.12 -m pip install numpy==2.0.2 cython bottleneck==1.4.2 brotli==1.1.0 pyyaml==6.0.2 aiohttp==3.9.0 argon2_cffi_bindings==21.2.0 -python3.12 -m pip install cassandra_driver==3.29.2 contourpy==1.3.1 cx_oracle==8.3.0 cytoolz==1.0.1 ibm_db==3.2.6 ijson==3.3.0 jenkspy==0.4.1 markupsafe==3.0.2 -python3.12 -m pip install zstd==1.5.6.1 yarl==1.18.3 unicodedata2==15.1.0 pandas==2.2.3 ninja - -echo "-------Installing cmake---------" -wget https://cmake.org/files/v3.28/cmake-3.28.0.tar.gz -tar -zxvf cmake-3.28.0.tar.gz -cd cmake-3.28.0 -./bootstrap -make -make install -cd $CURRENT_DIR - -#installing openblas -cd $CURRENT_DIR -git clone https://github.com/OpenMathLib/OpenBLAS -cd OpenBLAS -git checkout v0.3.29 -git submodule update --init -# Set build options -declare -a build_opts -# Fix ctest not automatically discovering tests -LDFLAGS=$(echo "${LDFLAGS}" | sed "s/-Wl,--gc-sections//g") -export CF="${CFLAGS} -Wno-unused-parameter -Wno-old-style-declaration" -unset CFLAGS -export USE_OPENMP=1 -build_opts+=(USE_OPENMP=${USE_OPENMP}) -# Handle Fortran flags -if [ ! -z "$FFLAGS" ]; then - export FFLAGS="${FFLAGS/-fopenmp/ }" - export FFLAGS="${FFLAGS} -frecursive" - export LAPACK_FFLAGS="${FFLAGS}" -fi -export PLATFORM=$(uname -m) -build_opts+=(BINARY="64") -build_opts+=(DYNAMIC_ARCH=1) -build_opts+=(TARGET="POWER9") -BUILD_BFLOAT16=1 -# Placeholder for future builds that may include ILP64 variants. -build_opts+=(INTERFACE64=0) -build_opts+=(SYMBOLSUFFIX="") -# Build LAPACK -build_opts+=(NO_LAPACK=0) -# Enable threading and set the number of threads -build_opts+=(USE_THREAD=1) -build_opts+=(NUM_THREADS=8) -# Disable CPU/memory affinity handling to avoid problems with NumPy and R -build_opts+=(NO_AFFINITY=1) -# Build OpenBLAS -make ${build_opts[@]} CFLAGS="${CF}" FFLAGS="${FFLAGS}" prefix=${OPENBLAS_PREFIX} -# Install OpenBLAS -CFLAGS="${CF}" FFLAGS="${FFLAGS}" make install PREFIX="${OPENBLAS_PREFIX}" ${build_opts[@]} -export LD_LIBRARY_PATH=${OPENBLAS_PREFIX}/lib:$LD_LIBRARY_PATH -export PKG_CONFIG_PATH=${OPENBLAS_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH -pkg-config --modversion openblas -echo "-----------------------------------------------------Installed openblas-----------------------------------------------------" - -cd $CURRENT_DIR -echo "------------ abseil_cpp cloning-------------------" - -ABSEIL_VERSION=20240116.2 -ABSEIL_URL="https://github.com/abseil/abseil-cpp" -git clone $ABSEIL_URL -b $ABSEIL_VERSION -echo "------------ libprotobuf installing-------------------" -export C_COMPILER=$(which gcc) -export CXX_COMPILER=$(which g++) -#Build libprotobuf -git clone https://github.com/protocolbuffers/protobuf -cd protobuf -git checkout v4.25.3 - -LIBPROTO_DIR=$(pwd) -mkdir -p $LIBPROTO_DIR/local/libprotobuf -LIBPROTO_INSTALL=$LIBPROTO_DIR/local/libprotobuf - -git submodule update --init --recursive -rm -rf ./third_party/googletest | true -rm -rf ./third_party/abseil-cpp | true - -cp -r $CURRENT_DIR/abseil-cpp ./third_party/ - -mkdir build -cd build - -cmake -G "Ninja" \ - ${CMAKE_ARGS} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_C_COMPILER=$C_COMPILER \ - -DCMAKE_CXX_COMPILER=$CXX_COMPILER \ - -DCMAKE_INSTALL_PREFIX=$LIBPROTO_INSTALL \ - -Dprotobuf_BUILD_TESTS=OFF \ - -Dprotobuf_BUILD_LIBUPB=OFF \ - -Dprotobuf_BUILD_SHARED_LIBS=ON \ - -Dprotobuf_ABSL_PROVIDER="module" \ - -Dprotobuf_JSONCPP_PROVIDER="package" \ - -Dprotobuf_USE_EXTERNAL_GTEST=OFF \ - .. - -cmake --build . --verbose -cmake --install . -cd $CURRENT_DIR - -export PATH=$LIBPROTOBUF_PREFIX/bin:$PATH -export PROTOC="$LIBPROTOBUF_PREFIX/bin/protoc" -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 -export LIBRARY_PATH="${LIBPROTOBUF_PREFIX}/lib64:$LD_LIBRARY_PATH" -export LD_LIBRARY_PATH=${LIBPROTOBUF_PREFIX}/lib64:$LD_LIBRARY_PATH -export PKG_CONFIG_PATH=${LIBPROTOBUF_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 - -# echo "----Installing rust------" -curl https://sh.rustup.rs -sSf | sh -s -- -y -source "$HOME/.cargo/env" - -# echo "------------cloning pytorch----------------" -git clone https://github.com/pytorch/pytorch.git -cd pytorch -git checkout v2.6.0 -git submodule sync -git submodule update --init --recursive - -echo "------------applying patch----------------" -wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/p/pytorch/pytorch_v2.6.0.patch -git apply pytorch_v2.6.0.patch - -ARCH=`uname -p` -BUILD_NUM="1" -export OPENBLAS_INCLUDE=/OpenBLAS/local/openblas/include/ -export LD_LIBRARY_PATH="$OpenBLASInstallPATH/lib" -export OpenBLAS_HOME="/usr/include/openblas" -export build_type="cpu" -export cpu_opt_arch="power9" -export cpu_opt_tune="power10" -export CPU_COUNT=$(nproc --all) -export CXXFLAGS="${CXXFLAGS} -D__STDC_FORMAT_MACROS" -export LDFLAGS="$(echo ${LDFLAGS} | sed -e 's/-Wl\,--as-needed//')" -export LDFLAGS="${LDFLAGS} -Wl,-rpath-link,${VIRTUAL_ENV}/lib" -export CXXFLAGS="${CXXFLAGS} -fplt" -export CFLAGS="${CFLAGS} -fplt" -export BLAS=OpenBLAS -export USE_FBGEMM=0 -export USE_SYSTEM_NCCL=1 -export USE_MKLDNN=0 -export USE_NNPACK=0 -export USE_QNNPACK=0 -export USE_XNNPACK=0 -export USE_PYTORCH_QNNPACK=0 -export TH_BINARY_BUILD=1 -export USE_LMDB=1 -export USE_LEVELDB=1 -export USE_NINJA=0 -export USE_MPI=0 -export USE_OPENMP=1 -export USE_TBB=0 -export BUILD_CUSTOM_PROTOBUF=OFF -export BUILD_CAFFE2=1 -export PYTORCH_BUILD_VERSION=2.6.0 -export PYTORCH_BUILD_NUMBER=${BUILD_NUM} -export USE_CUDA=0 -export USE_CUDNN=0 -export USE_TENSORRT=0 -export Protobuf_INCLUDE_DIR=${LIBPROTO_INSTALL}/include -export Protobuf_LIBRARIES=${LIBPROTO_INSTALL}/lib64 -export Protobuf_LIBRARY=${LIBPROTO_INSTALL}/lib64/libprotobuf.so -export Protobuf_LITE_LIBRARY=${LIBPROTO_INSTALL}/lib64/libprotobuf-lite.so -export Protobuf_PROTOC_EXECUTABLE=${LIBPROTO_INSTALL}/bin/protoc -export LD_LIBRARY_PATH=/pytorch/torch/lib64/libprotobuf.so.3.13.0.0:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/pytorch/build/lib/libprotobuf.so.3.13.0.0:$LD_LIBRARY_PATH -export PATH="/protobuf/local/libprotobuf/bin/protoc:${PATH}" -export LD_LIBRARY_PATH="/protobuf/local/libprotobuf/lib64:${LD_LIBRARY_PATH}" -export LD_LIBRARY_PATH="/protobuf/third_party/abseil-cpp/local/abseilcpp/lib:${LD_LIBRARY_PATH}" -export SETUPTOOLS_SCM_PRETEND_VERSION=2.6.0 -sed -i "s/cmake/cmake==3.*/g" requirements.txt - -python3.12 -m pip install -r requirements.txt -MAX_JOBS=$(nproc) python3.12 setup.py install - -cd $CURRENT_DIR - -echo "------------------------clone and build torchaudio-------------------" -git clone https://github.com/pytorch/audio.git -cd audio -git checkout v2.6.0 - -# Apply the patch -echo "------------------------Applying patch-------------------" -wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/t/torchaudio/torchaudio_v2.6.0.patch -git apply torchaudio_v2.6.0.patch -echo "-----------------------Applied patch successfully---------------------------------------" - - -export USE_FFMPEG=OFF -export BUILD_SOX=OFF -export USE_OPENMP=OFF -export BUILD_TORCHAUDIO_PYTHON_EXTENSION=ON -export LIBPROTO_INSTALL=/protobuf/local/libprotobuf -export Protobuf_INCLUDE_DIR=${LIBPROTO_INSTALL}/include -export Protobuf_LIBRARIES=${LIBPROTO_INSTALL}/lib64 -export Protobuf_LIBRARY=${LIBPROTO_INSTALL}/lib64/libprotobuf.so -export Protobuf_LITE_LIBRARY=${LIBPROTO_INSTALL}/lib64/libprotobuf-lite.so -export Protobuf_PROTOC_EXECUTABLE=${LIBPROTO_INSTALL}/bin/protoc -export LD_LIBRARY_PATH=/pytorch/torch/lib64/libprotobuf.so.3.13.0.0:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/pytorch/build/lib/libprotobuf.so.3.13.0.0:$LD_LIBRARY_PATH -export PATH="/protobuf/local/libprotobuf/bin/protoc:${PATH}" -export LD_LIBRARY_PATH="/OpenBLAS:/protobuf/local/libprotobuf/lib64:${LD_LIBRARY_PATH}" -export LD_LIBRARY_PATH="/protobuf/third_party/abseil-cpp/local/abseilcpp/lib:${LD_LIBRARY_PATH}" -export Protobuf_INCLUDE_DIR=/protobuf/local/libprotobuf/include -export Protobuf_LIBRARIES=/protobuf/local/libprotobuf/lib64 -export Protobuf_LIBRARY=/protobuf/local/libprotobuf/lib64/libprotobuf.so -export Protobuf_PROTOC_EXECUTABLE=/protobuf/local/bin/protoc -sed -i 's/2\.6\.0a0/2.6.0/' version.txt -mv .git .git.bak -echo "Installing torchaudio..." -pip3.12 install --no-build-isolation . - - -#installing libvpx -cd $CURRENT_DIR -git clone https://github.com/webmproject/libvpx.git -cd libvpx -git checkout v1.13.1 -export target_platform=$(uname)-$(uname -m) -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH -if [[ ${target_platform} == Linux-* ]]; then - LDFLAGS="$LDFLAGS -pthread" -fi -CPU_DETECT="${CPU_DETECT} --enable-runtime-cpu-detect" - -./configure --prefix=$LIBVPX_PREFIX \ - --as=yasm \ - --enable-shared \ - --disable-static \ - --disable-install-docs \ - --disable-install-srcs \ - --enable-vp8 \ - --enable-postproc \ - --enable-vp9 \ - --enable-vp9-highbitdepth \ - --enable-pic \ - ${CPU_DETECT} \ - --enable-experimental || { cat config.log; exit 1; } - -make -make install PREFIX="${LIBVPX_PREFIX}" -export LD_LIBRARY_PATH=${LIBVPX_PREFIX}/lib:$LD_LIBRARY_PATH -export PKG_CONFIG_PATH=${LIBVPX_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH -pkg-config --modversion vpx -echo "-----------------------------------------------------Installed libvpx------------------------------------------------" - - -#installing lame -cd $CURRENT_DIR -wget https://downloads.sourceforge.net/sourceforge/lame/lame-3.100.tar.gz -tar -xvf lame-3.100.tar.gz -cd lame-3.100 -# remove libtool files -find $LAME_PREFIX -name '*.la' -delete - -./configure --prefix=$LAME_PREFIX \ - --disable-dependency-tracking \ - --disable-debug \ - --enable-shared \ - --enable-static \ - --enable-nasm - -make -make install PREFIX="${LAME_PREFIX}" -export LD_LIBRARY_PATH=/install-deps/lame/lib:$LD_LIBRARY_PATH -export PATH="/install-deps/lame/bin:$PATH" -lame --version -echo "-----------------------------------------------------Installed lame------------------------------------------------" - - -#installing opus -cd $CURRENT_DIR -git clone https://github.com/xiph/opus -cd opus -git checkout v1.3.1 -yum install -y autoconf automake libtool -./autogen.sh -./configure --prefix=$OPUS_PREFIX -make -make install PREFIX="${OPUS_PREFIX}" -export LD_LIBRARY_PATH=${OPUS_PREFIX}/lib:$LD_LIBRARY_PATH -export PKG_CONFIG_PATH=${OPUS_PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH -pkg-config --modversion opus -echo "-----------------------------------------------------Installed opus------------------------------------------------" - - -#installing ffmpeg -cd $CURRENT_DIR -git clone https://github.com/FFmpeg/FFmpeg -cd FFmpeg -git checkout n7.1 -git submodule update --init - -yum install -y gmp-devel freetype-devel - - -USE_NONFREE=no #the options below are set for NO -./configure \ - --prefix="$FFMPEG_PREFIX" \ - --cc=${CC} \ - --disable-doc \ - --enable-gmp \ - --enable-hardcoded-tables \ - --enable-libfreetype \ - --enable-pthreads \ - --enable-postproc \ - --enable-pic \ - --enable-pthreads \ - --enable-shared \ - --enable-static \ - --enable-version3 \ - --enable-zlib \ - --enable-libopus \ - --enable-libmp3lame \ - --enable-libvpx \ - --extra-cflags="-I$LAME_PREFIX/include -I$OPUS_PREFIX/include -I$LIBVPX_PREFIX/include" \ - --extra-ldflags="-L$LAME_PREFIX/lib -L$OPUS_PREFIX/lib -L$LIBVPX_PREFIX/lib" \ - --disable-encoder=h264 \ - --disable-decoder=h264 \ - --disable-decoder=libh264 \ - --disable-decoder=libx264 \ - --disable-decoder=libopenh264 \ - --disable-encoder=libopenh264 \ - --disable-encoder=libx264 \ - --disable-decoder=libx264rgb \ - --disable-encoder=libx264rgb \ - --disable-encoder=hevc \ - --disable-decoder=hevc \ - --disable-encoder=aac \ - --disable-decoder=aac \ - --disable-decoder=aac_fixed \ - --disable-encoder=aac_latm \ - --disable-decoder=aac_latm \ - --disable-encoder=mpeg \ - --disable-encoder=mpeg1video \ - --disable-encoder=mpeg2video \ - --disable-encoder=mpeg4 \ - --disable-encoder=msmpeg4 \ - --disable-encoder=mpeg4_v4l2m2m \ - --disable-encoder=msmpeg4v2 \ - --disable-encoder=msmpeg4v3 \ - --disable-decoder=mpeg \ - --disable-decoder=mpegvideo \ - --disable-decoder=mpeg1video \ - --disable-decoder=mpeg1_v4l2m2m \ - --disable-decoder=mpeg2video \ - --disable-decoder=mpeg2_v4l2m2m \ - --disable-decoder=mpeg4 \ - --disable-decoder=msmpeg4 \ - --disable-decoder=mpeg4_v4l2m2m \ - --disable-decoder=msmpeg4v1 \ - --disable-decoder=msmpeg4v2 \ - --disable-decoder=msmpeg4v3 \ - --disable-encoder=h264_v4l2m2m \ - --disable-decoder=h264_v4l2m2m \ - --disable-encoder=hevc_v4l2m2m \ - --disable-decoder=hevc_v4l2m2m \ - --disable-nonfree --disable-gpl --disable-gnutls --enable-openssl --disable-libopenh264 --disable-libx264 #"${_CONFIG_OPTS[@]}" - -make -make install PREFIX="${FFMPEG_PREFIX}" -export PKG_CONFIG_PATH=${FFMPEG_PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH} -export LD_LIBRARY_PATH=${FFMPEG_PREFIX}/lib:${LD_LIBRARY_PATH} -export PATH="/install-deps/ffmpeg/bin:$PATH" -ffmpeg -version -echo "-----------------------------------------------------Installed ffmpeg------------------------------------------------" - -cd $CURRENT_DIR - -echo "--------------------Installing x264---------------------------------" -git clone https://code.videolan.org/videolan/x264.git -cd x264 -./configure --prefix=/install-deps/x264 --enable-shared --enable-pic --disable-asm -make -make install -export PKG_CONFIG_PATH=/install-deps/x264/lib/pkgconfig:$PKG_CONFIG_PATH -export LD_LIBRARY_PATH=/install-deps/x264/lib:$LD_LIBRARY_PATH -export CFLAGS="-I/install-deps/x264/include $CFLAGS" -export LDFLAGS="-L/install-deps/x264/lib $LDFLAGS" -pkg-config --modversion x264 -cd $CURRENT_DIR - -echo "------------------Building opencv-python-headless-----------------------" -cd $CURRENT_DIR - -git clone https://github.com/opencv/opencv-python -cd opencv-python -git checkout ${1:-86} -git submodule update --init --recursive - -sed -i "s/^[[:space:]]*name=package_name/name=\"${PACKAGE_NAME}\"/" setup.py - -export PROTOBUF_PREFIX=$CURRENT_DIR/protobuf/local/libprotobuf -export OPENBLAS_PREFIX=$CURRENT_DIR/OpenBLAS - -# Adjust these paths so CMake can find headers and libraries -export CMAKE_PREFIX_PATH="$PROTOBUF_PREFIX:$OPENBLAS_PREFIX:$CMAKE_PREFIX_PATH" -export LD_LIBRARY_PATH=$PROTOBUF_PREFIX/lib64:$OPENBLAS_PREFIX:$LD_LIBRARY_PATH -export LIBRARY_PATH=$PROTOBUF_PREFIX/lib64:$OPENBLAS_PREFIX:$LIBRARY_PATH - -export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Release - -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH - -DWITH_EIGEN=1 - -DBUILD_TESTS=0 - -DBUILD_DOCS=0 - -DBUILD_PERF_TESTS=0 - -DBUILD_ZLIB=0 - -DBUILD_TIFF=0 - -DBUILD_PNG=0 - -DBUILD_OPENEXR=1 - -DBUILD_JASPER=0 - -DWITH_ITT=1 - -DBUILD_JPEG=0 - -DBUILD_LIBPROTOBUF_FROM_SOURCES=OFF - -DWITH_V4L=1 - -DWITH_OPENCL=0 - -DWITH_OPENCLAMDFFT=0 - -DWITH_OPENCLAMDBLAS=0 - -DWITH_OPENCL_D3D11_NV=0 - -DWITH_1394=0 - -DWITH_CARBON=0 - -DWITH_OPENNI=0 - -DWITH_FFMPEG=1 - -DFFMPEG_DIR=$FFMPEG_PREFIX - -DWITH_JASPER=0 - -DWITH_VA=0 - -DWITH_VA_INTEL=0 - -DWITH_GSTREAMER=0 - -DWITH_MATLAB=0 - -DWITH_TESSERACT=0 - -DWITH_VTK=0 - -DWITH_GTK=0 - -DWITH_QT=0 - -DWITH_GPHOTO2=0 - -DINSTALL_C_EXAMPLES=0 - -DBUILD_PROTOBUF=OFF - -DPROTOBUF_UPDATE_FILES=ON - -DProtobuf_LIBRARY=$PROTOBUF_PREFIX/lib64/libprotobuf.so - -DProtobuf_INCLUDE_DIR=$PROTOBUF_PREFIX/include - -DWITH_LAPACK=0 - -DHAVE_LAPACK=0 - -DLAPACK_LAPACKE_H=$OPENBLAS_PREFIX/lapack-netlib/LAPACKE/include/lapacke.h - -DLAPACK_CBLAS_H=$OPENBLAS_PREFIX/cblas.h - -DENABLE_SSE=OFF \ - -DENABLE_SSE2=OFF \ - -DENABLE_SSE3=OFF \ - -DENABLE_SSSE3=OFF \ - -DENABLE_SSE41=OFF \ - -DENABLE_SSE42=OFF \ - -DENABLE_AVX=OFF \ - -DENABLE_AVX2=OFF \ - -DENABLE_NEON=OFF \ - -DENABLE_VSX=OFF \ - -DCPU_BASELINE_DISABLE=ON \ - -DCPU_DISPATCH=OFF" - -export C_INCLUDE_PATH=$(python3.12 -c "import numpy; print(numpy.get_include())") -export CPLUS_INCLUDE_PATH=$C_INCLUDE_PATH -ln -sf $CURRENT_DIR/opencv-python/tests/SampleVideo_1280x720_1mb.mp4 SampleVideo_1280x720_1mb.mp4 -pip3.12 install scikit-build setuptools wheel -pip3.12 install -e . - -#installing pillow -cd $CURRENT_DIR -git clone https://github.com/python-pillow/Pillow -cd Pillow -git checkout 11.1.0 -yum install -y libjpeg-turbo-devel -git submodule update --init -python3.12 -m pip install . -echo "-----------------------------------------------------Installed pillow------------------------------------------------" - -echo "-----------------------------------------------------Installing pyav------------------------------------------------" -# clone source repository -cd $CURRENT_DIR -git clone https://github.com/PyAV-Org/PyAV -cd PyAV -git checkout v13.1.0 -git submodule update --init -export CFLAGS="${CFLAGS} -I/install-deps/ffmpeg/include" -export LDFLAGS="${LDFLAGS} -L/install-deps/ffmpeg/lib" -#Build package -python3.12 setup.py build_ext --inplace -echo "-----------------------------------------------------Installed Pyav------------------------------------------------" - -echo "-----------------------------------------------------Installing gensim ------------------------------------------------" -cd $CURRENT_DIR -echo "Installing required Python packages..." -python3.12 -m pip install requests ruamel-yaml 'meson-python<0.13.0,>=0.11.0' setuptools==76.0.0 scipy==1.15.2 Cython==3.0.12 nbformat testfixtures mock nbconvert -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# Clone the repository -git clone https://github.com/RaRe-Technologies/gensim -cd gensim -git checkout 4.3.3 -echo "Building the package using setup.py..." -#Compiled extensions are unavailable. -python3.12 -m pip install scipy -python3.12 setup.py build_ext --inplace -# Build package -python3.12 -m pip install . -echo "-----------------------------------------------------Installed gensim ------------------------------------------------" - - -echo "------HDF5 installing-----------------------" -cd $CURRENT_DIR -git clone https://github.com/HDFGroup/hdf5 -cd hdf5/ -git checkout hdf5-1_12_1 -git submodule update --init -export HDF5_PREFIX=/install-deps/hdf5 -./configure --prefix=$HDF5_PREFIX --enable-cxx --enable-fortran --with-pthread=yes \ - --enable-threadsafe --enable-build-mode=production --enable-unsupported \ - --enable-using-memchecker --enable-clear-file-buffers --with-ssl -make -j$(nproc) -make install -export LD_LIBRARY_PATH=${HDF5_PREFIX}/lib:$LD_LIBRARY_PATH -export HDF5_DIR=${HDF5_PREFIX} -cd $CURRENT_DIR - -echo "----------h5py installing--------------------" -git clone https://github.com/h5py/h5py.git -cd h5py/ -git checkout 3.13.0 -HDF5_DIR=${HDF5_PREFIX} python3.12 -m pip install . -cd $CURRENT_DIR - -echo "--------------ml_dtypes installing---------------" -git clone https://github.com/jax-ml/ml_dtypes.git -cd ml_dtypes -git checkout v0.4.1 -git submodule update --init -export CFLAGS="-I${ML_DIR}/include" -export CXXFLAGS="-I${ML_DIR}/include" -export CC=/opt/rh/gcc-toolset-13/root/bin/gcc -export CXX=/opt/rh/gcc-toolset-13/root/bin/g++ -python3.12 -m pip install . -echo "--------------ml_dtypes installed---------------" - -cd $CURRENT_DIR -echo "------------------------------------------------------------Installing requirements for statsmodels------------------------------------------------------" -yum install -y meson libffi-devel zlib-devel \ - libtiff-devel freetype-devel make cmake automake autoconf procps-ng \ - python3.12 python3.12-devel python3.12-pip -git clone https://github.com/statsmodels/statsmodels.git -cd statsmodels -git checkout v0.14.4 -python3.12 -m pip install . -echo "--------------statsmodels installed---------------" - -cd $CURRENT_DIR -echo "------------------------installing dm_tree-------------------------" -yum install -y gcc-toolset-13 make libtool cmake xz zlib-devel libffi-devel libevent-devel libjpeg-turbo-devel python python-devel -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -export SITE_PACKAGE_PATH="/lib/python3.12/site-packages" -git clone https://github.com/deepmind/tree -cd tree/ -git checkout 0.1.8 -pip3.12 install --upgrade pip -pip3.12 install absl-py attr numpy wrapt attrs -wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/d/dm-tree/update_abseil_version_and_linking_fix.patch -git apply update_abseil_version_and_linking_fix.patch -python3.12 setup.py build_ext --inplace -echo "------------------------installed dm_tree-------------------------" - -cd $CURRENT_DIR -echo "------------------------installing gmpy2-------------------------" -yum install -y make python-devel gmp-devel.ppc64le mpfr-devel.ppc64le libmpc-devel.ppc64le -git clone https://github.com/aleaxit/gmpy.git -cd gmpy -git checkout v2.2.1 -pip3.12 --verbose install --editable .[docs,tests] -pip3.12 install . -echo "------------------------installed gmpy2-------------------------" - -cd $CURRENT_DIR -echo "------------------------installing grpcio-------------------------" -yum install -y python3 python3-devel python3-pip git gcc-toolset-13 gcc-toolset-13-gcc-c++ -git clone https://github.com/grpc/grpc.git -cd grpc/ -git checkout v1.71.0 -git submodule update --init --recursive -pip3.12 install coverage -export GRPC_PYTHON_BUILD_SYSTEM_=true -export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export PATH="/opt/rh/gcc-toolset-13/root/usr/bin:${PATH}" -python3.12 setup.py install -echo "------------------------installed grpcio-------------------------" - -cd $CURRENT_DIR -echo "------------------------installing jpype-------------------------" -yum install -y python-devel java-11-openjdk java-11-openjdk-devel gzip tar make xz cmake yum-utils openblas-devel bzip2-devel bzip2 zip unzip libffi-devel zlib-devel autoconf automake libtool cargo pkgconf-pkg-config.ppc64le info.ppc64le fontconfig.ppc64le fontconfig-devel.ppc64le sqlite-devel -export JAVA_HOME=/usr/lib/jvm/java-11-openjdk -export PATH=$JAVA_HOME/bin:$PATH -git clone https://github.com/jpype-project/jpype.git -cd jpype -git checkout v1.5.2 -wget https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar -O sqlite-jdbc.jar -wget https://repo1.maven.org/maven2/org/hsqldb/hsqldb/2.7.2/hsqldb-2.7.2.jar -O hsqldb.jar -wget https://repo1.maven.org/maven2/com/h2database/h2/2.2.224/h2-2.2.224.jar -O h2.jar -export CLASSPATH=$CLASSPATH:$(pwd)/sqlite-jdbc.jar:$(pwd)/hsqldb.jar:$(pwd)/h2.jar -pip3.12 install -cov numpy tox -pip3.12 install . -echo "------------------------installed jpype-------------------------" - - -cd $CURRENT_DIR -echo "------------------------installing llvmlite-------------------------" -LLVM_PROJECT_GIT_TAG="llvmorg-15.0.7" -WORKING_DIR="$(pwd)" -yum install -y bzip2-devel libffi-devel zlib-devel cmake python3.11 python3.11-devel python3.11-pip -yum install gcc-toolset-13 -y -export GCC_HOME=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -echo "enabling gcc13-toolset" -source /opt/rh/gcc-toolset-13/enable -git clone https://github.com/llvm/llvm-project.git -cd "$WORKING_DIR/llvm-project" -git checkout $LLVM_PROJECT_GIT_TAG -cd "$WORKING_DIR" -git clone https://github.com/numba/llvmlite -cd llvmlite -git checkout v0.44.0rc1 -export LLVM_CONFIG="/llvm-project/build/bin/llvm-config" -cd "$WORKING_DIR/llvm-project" -git apply "$WORKING_DIR/llvmlite/conda-recipes/llvm15-clear-gotoffsetmap.patch" -git apply "$WORKING_DIR/llvmlite/conda-recipes/llvm15-remove-use-of-clonefile.patch" -cp "$WORKING_DIR/llvmlite/conda-recipes/llvmdev/build.sh" . -chmod 777 "$WORKING_DIR/llvm-project/build.sh" && "$WORKING_DIR/llvm-project/build.sh" -LLVM_CONFIG_PATH=$(which llvm-config) -if [ -z "$LLVM_CONFIG_PATH" ]; then - echo "llvm-config not found in PATH, using local path." - export LLVM_CONFIG="$WORKING_DIR/llvm-project/build/bin/llvm-config" -else - echo "llvm-config found at: $LLVM_CONFIG_PATH" - export LLVM_CONFIG="$LLVM_CONFIG_PATH" -fi -echo "Checking for llvm-config.h in: $WORKING_DIR/llvm-project/build/include/llvm/Config" -ls "$WORKING_DIR/llvm-project/build/include/llvm/Config/llvm-config.h" || { echo "llvm-config.h not found. Exiting."; exit 1; } -cd "$WORKING_DIR/llvmlite" -export CXXFLAGS="-I$WORKING_DIR/llvm-project/build/include" -export LLVM_CONFIG="$WORKING_DIR/llvm-project/build/bin/llvm-config" -pip3.12 install . -echo "------------------------installed llvmlite-------------------------" - - -cd $CURRENT_DIR -echo "------------------------installing zfp-------------------------" -yum install -y gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran make python python-devel python-pip cmake -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH -echo "Cloning and installing..." -git clone https://github.com/LLNL/zfp -cd zfp -git checkout 1.0.1 -echo "Checking Python version..." -PYTHON_VERSION=$(python3.12 -c "import sys; print('.'.join(map(str, sys.version_info[:2])))") -IFS='.' read -r MAJOR MINOR <<< "$PYTHON_VERSION" -if [[ "$MAJOR" -gt 3 ]] || { [[ "$MAJOR" -eq 3 ]] && [[ "$MINOR" -ge 12 ]]; }; then - echo "Python version is >= 3.12, installing numpy 2.2.2..." - pip3.12 install numpy==2.2.2 -else - echo "Python version is < 3.12, installing numpy 1.23.5..." - pip3.12 install cython==0.29.36 numpy==1.23.5 -fi - -NUMPY_INCLUDE_DIR=$(python3.12 -c "import numpy; print(numpy.get_include())") - -echo "Creating build directory..." -mkdir -p build -cd build -export CFLAGS="-fPIC -fopenmp" -export LDFLAGS="-fPIC -static-libgcc -static-libstdc++ -fopenmp -Wl,--whole-archive -lgomp -Wl,--no-whole-archive" -echo "Running CMake..." -echo "Running CMake with static linking..." -cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ZFPY=ON -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_INSTALL_PREFIX=$(pwd)/install \ - -DPYTHON_EXECUTABLE=$(which python3.12) \ - -DPYTHON_INCLUDE_DIR=$(python3.12 -c "import sysconfig; print(sysconfig.get_path('include'))") \ - -DPYTHON_LIBRARY=$(python3.12 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ - -DNUMPY_INCLUDE_DIR=$NUMPY_INCLUDE_DIR \ - -DCMAKE_C_FLAGS="-fopenmp" \ - -DCMAKE_CXX_FLAGS="-fopenmp" \ - -DCMAKE_EXE_LINKER_FLAGS="-static-libgcc -static-libstdc++ -fopenmp" - -echo "Building zfp..." -make -j$(nproc) -make install -export CMAKE_PREFIX_PATH=$(pwd)/install -cd .. -sed -i 's/), language_level = "3"]/)]/' setup.py -pip3.12 install . -echo "------------------------installed zfp-------------------------" - - -cd $CURRENT_DIR -echo "------------------------installing numba-------------------------" -yum install -y python3.12 python3.12-devel python3.12-pip gcc-toolset-13 -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH -# export LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib/gcc/ppc64le-redhat-linux/13:$LIBRARY_PATH -export CPATH=/opt/rh/gcc-toolset-13/root/usr/include:$CPATH -yum install -y bzip2-devel libffi-devel zlib-devel cmake -#clone repository -git clone https://github.com/numba/numba -cd numba -git checkout 0.62.0dev0 -export CXXFLAGS=-I/usr/include -PYTHON_VERSION=$(python3.12 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") -sed -i '/#include "dynamic_annotations.h".*\/\*/d' /usr/include/python${PYTHON_VERSION}/internal/pycore_atomic.h -sed -i '1i#include "dynamic_annotations.h" /* _Py_ANNOTATE_MEMORY_ORDER */' /usr/include/python${PYTHON_VERSION}/internal/pycore_atomic.h -python3.12 -m pip install . -echo "------------------------installed numba-------------------------" - -cd $CURRENT_DIR -echo "------------------------installing numexpr-------------------------" -yum install -y python-devel bzip2-devel libffi-devel xz zlib-devel cmake openblas-devel -#clone repository -git clone https://github.com/pydata/numexpr.git -cd numexpr -git checkout v2.8.4 -pip3.12 install -pip3.12 install numpy==1.26.3 -pip3.12 install -e . -python3.12 setup.py install -echo "------------------------installed numexpr-------------------------" - - - - - -cd $CURRENT_DIR -echo "------------------------installing opencv_python_headless-------------------------" -yum install -y gcc-toolset-13 gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran gcc-toolset-13-binutils gcc-toolset-13-binutils-devel python python-pip python-devel git make cmake pkgconfig autoconf automake libtool zlib-devel freetype-devel gmp-devel -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH - -OS_NAME=$(cat /etc/os-release | grep ^PRETTY_NAME | cut -d= -f2) - -INSTALL_ROOT="/install-deps" -mkdir -p $INSTALL_ROOT - -for package in openblas lame opus libvpx ffmpeg libprotobuf protobuf abseilcpp; do - mkdir -p ${INSTALL_ROOT}/${package} - export "${package^^}_PREFIX=${INSTALL_ROOT}/${package}" - echo "Exported ${package^^}_PREFIX=${INSTALL_ROOT}/${package}" -done -python3.12 -m pip install --upgrade pip - -# clone source repository - -cd $CURRENT_DIR -echo "------------------------installing psycopg2-------------------------" -yum install -y sqlite sqlite-devel xz xz-devel postgresql-devel --nobest -# Clone the package repository -git clone https://github.com/psycopg/psycopg2 -cd psycopg2 -git checkout 2.9.10 -python3.12 setup.py install -echo "------------------------installed psycopg2-------------------------" - -cd $CURRENT_DIR -echo "------------------------installing pyodbc-------------------------" -yum install -y unixODBC-devel -git clone https://github.com/mkleehammer/pyodbc pyodbc -cd pyodbc -git checkout 5.2.0 -pip3.12 install -pip3.12 install "chardet<5,>=3.0.2" --force-reinstall -pip3.12 install psutil -python3.12 setup.py install -echo "------------------------installed pyodbc-------------------------" +TORCH_VERSION=v2.8.0 +TORCHVISION_VERSION=v0.23.0 +TORCH_URL=https://github.com/pytorch/pytorch.git +TORCHVISION_URL=https://github.com/pytorch/vision.git +export PYTHON=/usr/bin/python3.12 +# ----------------------------------------------------------------------------- +# System dependencies +# ----------------------------------------------------------------------------- +yum install -y yum-utils + +yum install -y \ + git wget tar gzip xz which procps-ng yum-utils \ + --allowerasing + +yum install -y \ + git make cmake gcc-toolset-13 \ + python3.12 python3.12-devel python3.12-pip m4 \ + unzip zip wget patch \ + openssl-devel sqlite-devel \ + llvm-devel libevent-devel openblas-devel \ + freetype-devel gmp-devel libjpeg-turbo-devel \ + cargo libcurl-devel \ + which procps-ng yum-utils \ + gcc-toolset-13-libatomic-devel \ + --allowerasing -cd $CURRENT_DIR -echo "------------------------installing pyzmq-------------------------" -# Install dependencies -yum install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran \ - cmake bzip2-devel glibc-static libstdc++-static libffi-devel \ - zlib-devel python-devel python3.12-pip pkg-config automake autoconf libtool +# ----------------------------------------------------------------------------- +# Enable GCC toolset (CRITICAL: must stay enabled for entire script) +# ----------------------------------------------------------------------------- +echo "-------------------- Enabling GCC toolset --------------------------" source /opt/rh/gcc-toolset-13/enable -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH - -# Clone the repository -git clone https://github.com/zeromq/pyzmq.git -cd pyzmq -git checkout v26.4.0 -pip3.12 install cython==3.0.12 packaging==24.2 pathspec==0.12.1 scikit-build-core==0.11.1 ninja==1.11.1.4 build -asyncio -timeout -pip3.12 install -e . -echo "------------------------installed pyzmq-------------------------" - - -# cd $CURRENT_DIR -echo "------------------------installing scikit_image-------------------------" -yum install -y gcc gcc-c++ make python python-devel libtool sqlite-devel xz zlib-devel bzip2-devel libffi-devel libevent-devel libjpeg-turbo-devel gcc-gfortran openblas openblas-devel libgomp -OS_NAME=$(cat /etc/os-release | grep ^PRETTY_NAME | cut -d= -f2) -git clone https://github.com/scikit-image/scikit-image -cd scikit-image -git checkout v0.25.2 -python3.12 -m pip install -r requirements.txt -python3.12 -m pip install -r requirements/build.txt -python3.12 -m pip install --upgrade pip -python3.12 -m pip install -e . -echo "------------------------installed scikit_image-------------------------" - -cd $CURRENT_DIR -echo " -------------------------- Installing onetbb -------------------------- " -yum install -y python3 python3-devel python3-pip -yum install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ cmake bzip2-devel glibc-static libstdc++-static libffi-devel zlib-devel pkg-config automake autoconf libtool -yum install gcc-toolset-13 sudo -y -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH -# export GCC_HOME=/opt/rh/gcc-toolset-13/root/usr -# export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH -python3.12 -m pip install --upgrade pip -# Installing Swing from source -echo " ------------------------------ Installing Swig ------------------------------ " -git clone https://github.com/nightlark/swig-pypi.git -cd swig-pypi -pip3.12 install . -echo " ------------------------------ Swig Installed Successfully ------------------------------ " -cd $CURRENT_DIR -# Installing hwloc from source -echo " ------------------------------ Installing hwloc ------------------------------ " -git clone https://github.com/open-mpi/hwloc -cd hwloc -./autogen.sh -./configure -make -make install -echo " ------------------------------ hwloc Installed Successfully ------------------------------ " -cd $CURRENT_DIR -echo "------------Cloning the Repository------------" -git clone https://github.com/uxlfoundation/oneTBB -cd oneTBB -git checkout v2021.8.0 -mkdir build -cd build/ -pwd -cmake -DCMAKE_INSTALL_PREFIX=/tmp/my_installed_onetbb -DTBB_TEST=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC=ON -DCMAKE_CXX_FLAGS="-static-libgcc -static-libstdc++" -DCMAKE_EXE_LINKER_FLAGS="-static" -DTBB_BUILD=ON -DTBB4PY_BUILD=ON .. -echo "------------Building the package------------" -make -j4 python_build -echo "------------Export statements------------" -export TBBROOT=/tmp/my_installed_onetbb/ -export CMAKE_PREFIX_PATH=$TBBROOT -make install -echo " -------------------------- Installed onetbb -------------------------- " - -cd $CURRENT_DIR -echo "----------------------------- Installing matplotlib ----------------------------" -yum install -y python3-pip python3 python python3-devel -yum install -y openblas-devel -yum install -y zlib zlib-devel libjpeg-turbo libjpeg-turbo-devel -echo "Dependencies installed." -# Clone the matplotlib package. -git clone https://github.com/matplotlib/matplotlib.git -cd matplotlib/ -git checkout v3.10.1 -git submodule update --init - -echo "Downloading and preparing qhull..." -# Download qhull -mkdir -p build -wget https://github.com/qhull/qhull/archive/refs/tags/v8.0.2.tar.gz -O qhull-8.0.2.tar.gz -tar -xzf qhull-8.0.2.tar.gz -mv qhull-8.0.2 build/qhull-2020.2 -echo "qhull downloaded and prepared." - -echo "Setting up virtual environment for Python..." -# Setup virtual environment for python -pip3.12 install hypothesis build meson pybind11 meson-python -echo "Python environment set up." - -# Install package dependencies -pip3.12 install 'numpy<2' fontTools setuptools-scm contourpy kiwisolver python-dateutil cycler pyparsing pillow certifi -# pip3.12 install --upgrade setuptools -echo "Dependencies installed." -ln -sf /usr/bin/python3.12 /usr/bin/python3 - -# Build and Install the package (This is dependent on numpy, pillow) -pip3.12 install -e . -echo "----------------------------- Installed matplotlib ----------------------------" - - -cd $CURRENT_DIR -echo "----------------------------- Installing torchvision ----------------------------" -OS_NAME=$(cat /etc/os-release | grep ^PRETTY_NAME | cut -d= -f2) -MAX_JOBS=${MAX_JOBS:-$(nproc)} - -# CURRENT_DIR=$(pwd) - -yum install -y pkgconfig atlas -yum install -y libtool xz zlib-devel bzip2-devel libffi-devel libevent-devel patch gcc-toolset-13 pkg-config gmp-devel freetype-devel -yum install -y gcc-toolset-13-libatomic-devel +export CC=/opt/rh/gcc-toolset-13/root/usr/bin/gcc +export CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++ -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH - -cd $CURRENT_DIR -echo "--------------------scipy installing-------------------------------" -#Building scipy -python3.12 -m pip install beniget==0.4.2.post1 Cython==3.0.11 gast==0.6.0 meson==1.6.0 meson-python==0.17.1 packaging pybind11 pyproject-metadata pythran==0.17.0 setuptools==75.3.0 pooch build hypothesis patchelf -git clone https://github.com/scipy/scipy -cd scipy/ -git checkout v1.15.2 -git submodule update --init -echo "instaling scipy......." -python3.12 -m pip install . -cd $CURRENT_DIR -export C_COMPILER=$(which gcc) -export CXX_COMPILER=$(which g++) -cd $CURRENT_DIR -git clone https://github.com/pytorch/vision.git -cd vision -git checkout v0.21.0 -export LD_LIBRARY_PATH=/home/protobuf/local/libprotobuf/lib64:$LD_LIBRARY_PATH -pip3.12 install -v -e . --no-build-isolation -echo "----------------------------- Installed torchvision ----------------------------" +echo "CC = $CC" +echo "CXX = $CXX" +gcc --version +g++ --version +# ----------------------------------------------------------------------------- +# Python tooling +# ----------------------------------------------------------------------------- +echo "-------------------- Upgrading Python tooling ---------------------" +python3.12 -m pip install --upgrade pip setuptools wheel cython -cd $CURRENT_DIR -echo "--------------------pyarrow installing-------------------------------" -echo "Install dependencies and tools." -yum install -y cmake zlib-devel libjpeg-devel gcc-toolset-13 cmake libevent libtool pkg-config brotli-devel.ppc64le bzip2-devel lz4-devel -# export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -# CURRENT_DIR=$(pwd) -# Installing flex bison c-ares gflags rapidjson xsimd snappy libzstd +# ----------------------------------------------------------------------------- +# Python binary wheels (POWER / devpi) +#----------------------------------------------------------------------------- +# echo "-------------------- Installing Python dependencies ----------------" echo "-----------flex installing------------------" wget https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz tar -xvf flex-2.6.4.tar.gz @@ -998,6 +89,7 @@ make -j$(nproc) echo "Installing flex..." make install cd $CURRENT_DIR +echo "-----------flex installed------------------" echo "-------bison installing----------------------" wget https://ftp.gnu.org/gnu/bison/bison-3.8.2.tar.gz @@ -1011,269 +103,16 @@ echo "Installing bison..." make install cd $CURRENT_DIR -echo "------------ gflags installing-------------------" -git clone https://github.com/gflags/gflags.git -cd gflags -mkdir build && cd build -echo "Running cmake to configure the build..." -cmake .. -echo "Compiling the source code gflags..." -make -j$(nproc) -echo "Installing gflags..." -make install -cd $CURRENT_DIR - -echo "----------Installing c-ares----------------" -#Building c-areas -git clone https://github.com/c-ares/c-ares.git -cd c-ares -git checkout cares-1_19_1 - - -target_platform=$(uname)-$(uname -m) -AR=$(which ar) -PKG_NAME=c-ares - -mkdir -p c_ares_prefix -export C_ARES_PREFIX=$(pwd)/c_ares_prefix - -echo "Building ${PKG_NAME}." - -# Isolate the build. -mkdir build && cd build - -if [[ "$PKG_NAME" == *static ]]; then - CARES_STATIC=ON - CARES_SHARED=OFF -else - CARES_STATIC=OFF - CARES_SHARED=ON -fi - -if [[ "${target_platform}" == Linux-* ]]; then - CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_AR=${AR}" -fi - - -# Generate the build files. -echo "Generating the build files..." -cmake ${CMAKE_ARGS} .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="$C_ARES_PREFIX" \ - -DCARES_STATIC=${CARES_STATIC} \ - -DCARES_SHARED=${CARES_SHARED} \ - -DCARES_INSTALL=ON \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -GNinja - #${SRC_DIR} - -# Build. -echo "Building c-areas..." -ninja || exit 1 - -# Installing -echo "Installing c-areas..." -ninja install || exit 1 - -cd $CURRENT_DIR - -echo "----------c-areas installed-----------------------" - -echo "----------------rapidjson installing------------------" -git clone https://github.com/Tencent/rapidjson.git -cd rapidjson -mkdir build && cd build -echo "Running cmake to configure the build..." -cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -echo "Compiling the source code for rapidjson..." -make -j$(nproc) -echo "Installing rapidjson" -make install -cd $CURRENT_DIR - -echo "--------------xsimd installing-------------------------" -git clone https://github.com/xtensor-stack/xsimd.git -cd xsimd -mkdir build && cd build -echo "Running cmake to configure the build for xsimd.." -cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local -echo "Compiling the source code for xsimd..." -make -j$(nproc) -echo "Installing xsimd..." -make install -cd $CURRENT_DIR - - -echo "-----------------snappy installing----------------" -git clone https://github.com/google/snappy.git -cd snappy -git submodule update --init --recursive - -mkdir -p local/snappy -export SNAPPY_PREFIX=$(pwd)/local/snappy -mkdir build -cd build -echo "Running cmake to configure the build for snappy..." -cmake -DCMAKE_INSTALL_PREFIX=$SNAPPY_PREFIX \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_INSTALL_LIBDIR=lib \ - .. -echo "Compiling the source code for snappy..." -make -j$(nproc) -echo "Installing snappy..." -make install -cd .. -cd $CURRENT_DIR - - -echo "------------libzstd installing-------------------------" -git clone https://github.com/facebook/zstd.git -cd zstd - -echo "Compiling the source code for libzstd..." +echo "-----------------numactl installing-----------------------" +yum install -y git autoconf automake libtool +# Clone the repository +git clone https://github.com/numactl/numactl numactl +cd numactl +git checkout v2.0.19 +./autogen.sh +./configure make -echo "Installing libzstd..." make install -export ZSTD_HOME=/usr/local -export CMAKE_PREFIX_PATH=$ZSTD_HOME -export LD_LIBRARY_PATH=$ZSTD_HOME/lib64:$LD_LIBRARY_PATH -cd $CURRENT_DIR - - - -echo "------------ re2 installing-------------------" - -git clone http://github.com/google/re2 -cd re2 -git checkout 2022-04-01 - -git submodule update --init - -mkdir re2-prefix - -export RE2_PREFIX=$(pwd)/re2-prefix - -export CPU_COUNT=`nproc` - -mkdir build-cmake -pushd build-cmake - -echo "Running cmake to configure the build for re2..." -cmake ${CMAKE_ARGS} -GNinja \ - -DCMAKE_PREFIX_PATH=$RE2_PREFIX \ - -DCMAKE_INSTALL_PREFIX="${RE2_PREFIX}" \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DENABLE_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ - .. -echo "Installing re2..." - ninja -v install - popd -echo "Running make shared-install......" -make -j "${CPU_COUNT}" prefix=${RE2_PREFIX} shared-install -cd $CURRENT_DIR - - - -echo "------------ utf8proc installing-------------------" - -git clone https://github.com/JuliaStrings/utf8proc.git -cd utf8proc -git submodule update --init -git checkout v2.6.1 - -mkdir utf8proc_prefix -export UTF8PROC_PREFIX=$(pwd)/utf8proc_prefix - -# Create build directory -mkdir build -cd build -echo "Running cmake to configure the build for utf8proc..." -# Run cmake to configure the build -cmake -G "Unix Makefiles" \ - -DCMAKE_BUILD_TYPE="Release" \ - -DCMAKE_INSTALL_PREFIX="${UTF8PROC_PREFIX}" \ - -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ - -DBUILD_SHARED_LIBS=1 \ - .. -echo "Build and install utf8proc" -cmake --build . - -echo "Installing utf8proc ..." -cmake --build . --target install -cd $CURRENT_DIR - -echo "------------ orc installing-------------------" - -git clone https://github.com/apache/orc -cd orc -git checkout v2.0.3 -wget https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/master/o/orc/orc.patch -git apply orc.patch -mkdir orc_prefix -export ORC_PREFIX=$(pwd)/orc_prefix -mkdir -p build -cd build -export PROTOBUF_PREFIX=$LIBPROTO_INSTALL -export CMAKE_PREFIX_PATH=$LIBPROTO_INSTALL -export LD_LIBRARY_PATH=$LIBPROTO_INSTALL/lib64 -export GCC=$CC -export GXX=$CXX -export HOST=$(uname)-$(uname -m) -export HOST=$(uname)-$(uname -m) - -CPPFLAGS="${CPPFLAGS} -Wl,-rpath,$VIRTUAL_ENV_PATH/**/lib" - - -declare -a _CMAKE_EXTRA_CONFIG -if [[ "$CONDA_BUILD_CROSS_COMPILATION" == 1 ]]; then - _CMAKE_EXTRA_CONFIG+=(-DHAS_PRE_1970_EXITCODE=0) - _CMAKE_EXTRA_CONFIG+=(-DHAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT=) - _CMAKE_EXTRA_CONFIG+=(-DHAS_POST_2038_EXITCODE=0) - _CMAKE_EXTRA_CONFIG+=(-DHAS_POST_2038_EXITCODE__TRYRUN_OUTPUT=) -fi -if [[ ${HOST} =~ .*darwin.* ]]; then - _CMAKE_EXTRA_CONFIG+=(-DCMAKE_AR=${AR}) - _CMAKE_EXTRA_CONFIG+=(-DCMAKE_RANLIB=${RANLIB}) - _CMAKE_EXTRA_CONFIG+=(-DCMAKE_LINKER=${LD}) -fi -if [[ ${HOST} =~ .*Linux.* ]]; then - CXXFLAGS="${CXXFLAGS//-std=c++17/-std=c++11}" - # I hate you so much CMake. - LIBPTHREAD=$(find ${PREFIX} -name "libpthread.so") - _CMAKE_EXTRA_CONFIG+=(-DPTHREAD_LIBRARY=${LIBPTHREAD}) -fi - -CPPFLAGS="${CPPFLAGS} -Wl,-rpath,$VIRTUAL_ENV_PATH/**/lib" -echo "Running cmake to configure the build for orc..." -source /opt/rh/gcc-toolset-13/enable -cmake ${CMAKE_ARGS} \ - -DCMAKE_PREFIX_PATH=$ORC_PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_JAVA=False \ - -DLZ4_HOME=/usr \ - -DZLIB_HOME=/usr \ - -DZSTD_HOME=/usr/local \ - -DCMAKE_POLICY_DEFAULT_CMP0074=NEW \ - -DProtobuf_ROOT=$PROTOBUF_PREFIX \ - -DPROTOBUF_HOME=$PROTOBUF_PREFIX \ - -DPROTOBUF_EXECUTABLE=$PROTOBUF_PREFIX/bin/protoc \ - -DSNAPPY_HOME=$SNAPPY_PREFIX \ - -DBUILD_LIBHDFSPP=NO \ - -DBUILD_CPP_TESTS=OFF \ - -DCMAKE_INSTALL_PREFIX=$ORC_PREFIX \ - -DCMAKE_C_COMPILER=$(type -p ${CC}) \ - -DCMAKE_CXX_COMPILER=$(type -p ${CXX}) \ - -DCMAKE_C_FLAGS="$CFLAGS" \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS -Wno-unused-parameter" \ - "${_CMAKE_EXTRA_CONFIG[@]}" \ - -GNinja .. - -ninja -echo "Installing orc..." -ninja install cd $CURRENT_DIR @@ -1338,282 +177,185 @@ echo " Building and installing Boost...." # Remove Python headers as we don't build Boost.Python. rm "${BOOST_PREFIX}/include/boost/python.hpp" rm -r "${BOOST_PREFIX}/include/boost/python" -cd $CURRENT_DIR +cd $CURRENT_DIR + + + + +python3.12 -m pip install \ + --trusted-host wheels.developerfirst.ibm.com \ + "abseil-cpp @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/419/275773a4cc480/abseil_cpp-20240116.2-py3-none-any.whl" \ + "av @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/b09/d74303b7aaf7f/av-13.1.0-cp312-cp312-linux_ppc64le.whl" \ + "ffmpeg @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/25d/9b7b985a577b9/ffmpeg-7.1-py3-none-any.whl" \ + "lame @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/923/40f226ef59d45/lame-3.100-py3-none-any.whl" \ + "libprotobuf @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/e53/57a336598c208/libprotobuf-25.4-py3-none-manylinux2014_ppc64le.whl" \ + "libvpx @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/328/cfde06a744e3d/libvpx-1.13.1-py3-none-any.whl" \ + "llvmlite @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/ddd/ebf05d58bd563/llvmlite-0.44.0-cp312-cp312-linux_ppc64le.whl" \ + "numba @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/609/d7a83e5ac7944/numba-0.62.0.dev0-cp312-cp312-linux_ppc64le.whl" \ + "openblas @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/523/31f6718639cba/openblas-0.3.29-py3-none-manylinux2014_ppc64le.whl" \ + "opus @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/bec/40c21ed2fe31b/opus-1.3.1-py3-none-any.whl" \ + "protobuf @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/5e5/a6c4d93fcc5cd/protobuf-4.25.8-cp312-cp312-linux_ppc64le.whl" \ + "scipy @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/4e1/4b512f33efb85/scipy-1.16.0-cp312-cp312-linux_ppc64le.whl" \ + "sentencepiece @ https://wheels.developerfirst.ibm.com/ppc64le/linux/+f/161/51fddd15aec51/sentencepiece-0.2.0-cp312-cp312-linux_ppc64le.whl" \ + certifi charset-normalizer filelock fsspec idna \ + Jinja2 MarkupSafe mpmath networkx requests \ + sympy tqdm typing_extensions urllib3 numpy + + +python3.12 -m pip install cmake pyyaml packaging openpyxl setuptools_scm + +# ----------------------------------------------------------------------------- +# Pillow (devpi) +# ----------------------------------------------------------------------------- +echo "-------------------- Installing Pillow -----------------------------" + +python3.12 -m pip install \ + --trusted-host wheels.developerfirst.ibm.com \ + --extra-index-url https://wheels.developerfirst.ibm.com/ppc64le/linux/+simple/ \ + pillow==11.2.1 + +# ----------------------------------------------------------------------------- +# Resolve OpenBLAS +# ----------------------------------------------------------------------------- +echo "-------------------- Resolving OpenBLAS ----------------------------" + +export OpenBLAS_HOME=/usr/local/lib/python3.12/site-packages/openblas +export OpenBLAS_DIR=${OpenBLAS_HOME} +export BLAS=OpenBLAS +export LD_LIBRARY_PATH=${OpenBLAS_HOME}/lib:${LD_LIBRARY_PATH} +export PKG_CONFIG_PATH="${OpenBLAS_HOME}/lib/pkgconfig:${PKG_CONFIG_PATH}" +export LIBRARY_PATH="${OpenBLAS_HOME}/lib:${LIBRARY_PATH}" + +# ----------------------------------------------------------------------------- +# Build PyArrow 21.0.0 from source +# ----------------------------------------------------------------------------- +echo "-------------------- Building PyArrow 21.0.0 -----------------------" + +export PYARROW_VERSION=21.0.0 +git clone --recursive https://github.com/apache/arrow.git -b apache-arrow-${PYARROW_VERSION} +cd arrow/cpp + +mkdir -p build && cd build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DARROW_PYTHON=ON \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_JEMALLOC=ON \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_PARQUET=ON \ + .. -echo "------------thrift_cpp installing-------------------" -git clone https://github.com/apache/thrift -cd thrift -git checkout 0.21.0 -mkdir thrit-prefix -export THRIFT_PREFIX=$CURRENT_DIR/thrit-prefix - -export BOOST_ROOT=${BOOST_PREFIX} -export ZLIB_ROOT=/usr -export LIBEVENT_ROOT=/usr - -export _ROOT=/usr -export _ROOT_DIR=/usr - -./bootstrap.sh -echo "Configuring thrift-cpp installation..." -./configure --prefix=$THRIFT_PREFIX \ - --with-python=no \ - --with-py3=no \ - --with-ruby=no \ - --with-java=no \ - --with-kotlin=no \ - --with-erlang=no \ - --with-nodejs=no \ - --with-c_glib=no \ - --with-haxe=no \ - --with-rs=no \ - --with-cpp=yes \ - --with-PACKAGE=yes \ - --with-zlib=$ZLIB_ROOT \ - --with-libevent=$LIBEVENT_ROOT \ - --with-boost=$BOOST_ROOT \ - --with-=$_ROOT \ - --enable-tests=no \ - --enable-tutorial=no - -echo "Compiling the source code for thrift-cpp..." make -j$(nproc) -echo "Installing thrift_cpp..." make install -cd $CURRENT_DIR -echo "------------ grpc_cpp installing-------------------" -git clone https://github.com/grpc/grpc -cd grpc -git checkout v1.68.0 -git submodule update --init -mkdir grpc-prefix -export GRPC_PREFIX=$(pwd)/grpc-prefix -AR=`which ar` -RANLIB=`which ranlib` +echo "/usr/local/lib" > /etc/ld.so.conf.d/arrow.conf +echo "/usr/local/lib64" >> /etc/ld.so.conf.d/arrow.conf +ldconfig -PROTOC_BIN=$LIBPROTO_INSTALL/bin/protoc -PROTOBUF_SRC=$LIBPROTO_INSTALL +cd ../../python -export CMAKE_PREFIX_PATH="$C_ARES_PREFIX;$RE2_PREFIX;$LIBPROTO_INSTALL" +pip install -r requirements-build.txt +export PYARROW_PARALLEL=$(nproc) -export LD_LIBRARY_PATH=$LIBPROTO_INSTALL/lib64:${LD_LIBRARY_PATH} +python3.12 setup.py build_ext \ + --build-type=release \ + --bundle-arrow-cpp \ + bdist_wheel \ + --dist-dir ./wheelhouse -target_platform=$(uname)-$(uname -m) +pip install wheelhouse/pyarrow-${PYARROW_VERSION}-*.whl -if [[ "${target_platform}" == osx* ]]; then - export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_STANDARD=14" -else - export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_STANDARD=17" -fi +cd ${CURRENT_DIR} +# ----------------------------------------------------------------------------- +# Build PyTorch 2.8.0 +# ----------------------------------------------------------------------------- +echo "-------------------- Building PyTorch ${TORCH_VERSION} ------------------------" -mkdir -p build-cpp -pushd build-cpp -echo "Running cmake to configure the build for grpc-cpp...." -cmake ${CMAKE_ARGS} .. \ - -GNinja \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=$GRPC_PREFIX \ - -DgRPC_CARES_PROVIDER="package" \ - -DgRPC_GFLAGS_PROVIDER="package" \ - -DgRPC_PROTOBUF_PROVIDER="package" \ - -DProtobuf_ROOT=$PROTOBUF_SRC \ - -DgRPC_SSL_PROVIDER="package" \ - -DgRPC_ZLIB_PROVIDER="package" \ - -DgRPC_ABSL_PROVIDER="package" \ - -DgRPC_RE2_PROVIDER="package" \ - -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \ - -DCMAKE_AR=${AR} \ - -DCMAKE_RANLIB=${RANLIB} \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DProtobuf_PROTOC_EXECUTABLE=$PROTOC_BIN -echo "Installing grpc_cpp..." -ninja install -v -popd +export USE_OPENMP=ON +export USE_MKLDNN=OFF +export USE_MKLDNN_CBLAS=OFF +export USE_CUDA=OFF +export CMAKE_PREFIX_PATH="${OpenBLAS_HOME}:${CMAKE_PREFIX_PATH}" -cd $CURRENT_DIR -echo "-----------------installing pyarrow----------------------" -#cloning pyarrow -git clone https://github.com/apache/arrow -cd arrow -git checkout apache-arrow-19.0.0 -git submodule update --init -mkdir pyarrow_prefix -export PYARROW_PREFIX=$(pwd)/pyarrow_prefix -export ARROW_HOME=$PYARROW_PREFIX -export target_platform=$(uname)-$(uname -m) -export CXX=$(which g++) -export CMAKE_PREFIX_PATH=$C_ARES_PREFIX:$LIBPROTO_INSTALL:$RE2_PREFIX:$GRPC_PREFIX:$ORC_PREFIX:$BOOST_PREFIX:${UTF8PROC_PREFIX}:$THRIFT_PREFIX:$SNAPPY_PREFIX:/usr -export LD_LIBRARY_PATH=$GRPC_PREFIX/lib:$LIBPROTO_INSTALL/lib64 -mkdir cpp/build -pushd cpp/build -EXTRA_CMAKE_ARGS="" - -# Include g++'s system headers -if [ "$(uname)" == "Linux" ]; then - SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') - EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" - sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake -fi +git clone --recursive ${TORCH_URL} +cd pytorch +git checkout ${TORCH_VERSION} +git submodule sync +git submodule update --init --recursive -# Enable CUDA support -if [ "${build_type}" = "cuda" ]; then - if [[ -z "${CUDA_HOME+x}" ]] - then - echo "cuda version=${cudatoolkit} CUDA_HOME=$CUDA_HOME" - CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) - if [[ -n "$CUDA_GDB_EXECUTABLE" ]] - then - CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) - else - echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" - return 1 - fi - fi - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs" -else - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" -fi -# Disable Gandiva -EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF" +python3.12 -m pip install -r requirements.txt +python3.12 setup.py build_ext -j$(nproc) +python3.12 setup.py install -export BOOST_ROOT="${BOOST_PREFIX}" -export Boost_ROOT="${BOOST_PREFIX}" +cd ${CURRENT_DIR} -export CXXFLAGS="-I$${BOOST_PREFIX}/include -I${THRIFT_PREFIX}/include" +# ----------------------------------------------------------------------------- +# Build TorchVision +# ----------------------------------------------------------------------------- +echo "-------------------- Building TorchVision --------------------------" -#SIMD Settings -if [[ "${target_platform}" == "Linux-x86_64" ]]; then - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_SIMD_LEVEL=SSE4_2" -fi -if [[ "${target_platform}" == "Linux-ppc64le" ]]; then - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_ALTIVEC=ON" -fi -if [[ "${target_platform}" != "Linux-s390x" ]]; then - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_USE_LD_GOLD=ON" -fi +git clone ${TORCHVISION_URL} +cd vision +git checkout ${TORCHVISION_VERSION} +USE_FFMPEG=1 USE_JPEG=1 USE_PNG=1 python3.12 setup.py develop +cd ${CURRENT_DIR} -export AR=$(which ar) -export RANLIB=$(which ranlib) -echo "Running cmake to configure the build for pyarrow..." -cmake \ - -DCMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH \ - -DARROW_BOOST_USE_SHARED=ON \ - -DARROW_BUILD_BENCHMARKS=OFF \ - -DARROW_BUILD_STATIC=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_BUILD_UTILITIES=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DARROW_DATASET=ON \ - -DARROW_DEPENDENCY_SOURCE=SYSTEM \ - -DARROW_FLIGHT=ON \ - -DARROW_HDFS=ON \ - -DARROW_JEMALLOC=ON \ - -DARROW_MIMALLOC=ON \ - -DARROW_ORC=ON \ - -DARROW_PACKAGE_PREFIX=$PYARROW_PREFIX \ - -DARROW_PARQUET=ON \ - -DARROW_PYTHON=ON \ - -DARROW_S3=OFF \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_WITH_THRIFT=ON \ - -DCMAKE_BUILD_TYPE=release \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DPYTHON_EXECUTABLE=python \ - -DPython3_EXECUTABLE=python \ - -DProtobuf_PROTOC_EXECUTABLE=${LIBPROTO_INSTALL}/bin/protoc \ - -DORC_INCLUDE_DIR=${ORC_PREFIX}/include \ - -DgRPC_DIR=${GRPC_PREFIX} \ - -DBoost_DIR=${BOOST_PREFIX} \ - -DBoost_INCLUDE_DIR=${BOOST_PREFIX}/include/ \ - -Dutf8proc_LIB=${UTF8PROC_PREFIX}/lib/libutf8proc.so ${UTF8PROC_PREFIX}/lib/libutf8proc.so.2 ${UTF8PROC_PREFIX}/lib/libutf8proc.so.2.4.1 \ - -Dutf8proc_INCLUDE_DIR=${UTF8PROC_PREFIX}/include \ - -DCMAKE_AR=${AR} \ - -DCMAKE_RANLIB=${RANLIB} \ - -GNinja \ - ${EXTRA_CMAKE_ARGS} \ - .. - -echo "Installing pyarrow...." -ninja install -popd +# ----------------------------------------------------------------------------- +# Build vLLM +# ----------------------------------------------------------------------------- +echo "-------------------- Building vLLM ---------------------------------" -cd $CURRENT_DIR -echo "Installing prerequisite for arrow..." -pip3.12 install setuptools-scm - -export PYARROW_BUNDLE_ARROW_CPP=1 -export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH} -export build_type=cpu -cd arrow -export CMAKE_PREFIX_PATH=$ARROW_HOME -# Build dependencies -export PARQUET_HOME=$ARROW_HOME -export SETUPTOOLS_SCM_PRETEND_VERSION=$version -export PYARROW_BUILD_TYPE=release -export PYARROW_BUNDLE_ARROW_CPP_HEADERS=1 -export PYARROW_WITH_DATASET=1 -export PYARROW_WITH_FLIGHT=1 -# Disable Gandiva -export PYARROW_WITH_GANDIVA=0 -export PYARROW_WITH_HDFS=1 -export PYARROW_WITH_ORC=1 -export PYARROW_WITH_PARQUET=1 -export PYARROW_WITH_PLASMA=1 -export PYARROW_WITH_S3=0 -export PYARROW_CMAKE_GENERATOR=Ninja -BUILD_EXT_FLAGS="" - -# Enable CUDA support -if [ "${build_type}" = "cuda" ]; then - export PYARROW_WITH_CUDA=1 -else - export PYARROW_WITH_CUDA=0 -fi -cd python -pip3.12 install . +git clone -b ${PACKAGE_VERSION} ${PACKAGE_URL} +cd ${PACKAGE_DIR} +# Keep your existing filters +sed -i 's/^torch/# torch/' requirements/cpu.txt +sed -i 's/^torchvision/# torchvision/' requirements/cpu.txt +sed -i 's/^torchaudio/# torchaudio/' requirements/cpu.txt +sed -i 's/^outlines_core/# outlines_core/' requirements/common.txt +sed -i 's/^scipy/# scipy/' requirements/common.txt -cd $CURRENT_DIR -yum install -y git autoconf automake libtool -# Clone the repository -git clone https://github.com/numactl/numactl numactl -cd numactl -git checkout v2.0.19 -./autogen.sh -./configure -make -make install +# NEW: scrub deprecated extras if present anywhere +sed -i 's/huggingface-hub\[hf_xet\]/huggingface-hub/g' requirements/*.txt || true +sed -i -e 's/.*torch.*//g' pyproject.toml requirements/*.txt -cd $CURRENT_DIR -git clone $PACKAGE_URL -cd $PACKAGE_DIR -git checkout $PACKAGE_VERSION -yum install -y kmod -python3.12 -m pip install setuptools-rust maturin setuptools_scm -export LD_LIBRARY_PATH=/protobuf/local/libprotobuf/lib64:$LD_LIBRARY_PATH -export CMAKE_PREFIX_PATH=/protobuf/local/libprotobuf -# USE cmake<4 -sed -i 's/"cmake>=3.26",/"cmake>=3.26,<4",/g' pyproject.toml +python3.12 -m pip install \ + --prefer-binary \ + --trusted-host wheels.developerfirst.ibm.com \ + --extra-index-url https://wheels.developerfirst.ibm.com/ppc64le/linux/+simple/ \ + -r requirements/cpu.txt + +python3.12 -m pip install llguidance + +PIP_NO_BUILD_ISOLATION=1 \ +CC=${CC} \ +CXX=${CXX} \ +python3.12 -m pip install xgrammar + +python3.12 -m pip install "huggingface-hub>=0.32.0" "transformers<=4.57.3" +export VLLM_TARGET_DEVICE=cpu +export MAX_JOBS=$(nproc) + +# AVOID eggs: build and install via pip/wheel instead of setup.py install export SETUPTOOLS_SCM_PRETEND_VERSION=0.8.4 +python3.12 setup.py install -if ! (python3.12 -m pip install -v . --no-build-isolation); then - echo "------------------$PACKAGE_NAME:install_fails-------------------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | GitHub | Fail | Install_Fails" - exit 1 -fi +python3.12 setup.py bdist_wheel --dist-dir="${CURRENT_DIR}" + +cd ${CURRENT_DIR} + +echo "===================================================================" +echo " vLLM WHEEL BUILT SUCCESSFULLY (v0.11.1)" +echo "===================================================================" +# 4. Run the "Hello World" example +# Note: This attempts to download a small model (facebook/opt-125m) from HuggingFace. +# If the build machine is offline, this step will fail. +echo "Running basic offline inference example..." -if ! pytest; then +if ! python3.12 ${PACKAGE_DIR}/examples/offline_inference/basic/basic.py; then echo "------------------$PACKAGE_NAME:install_success_but_test_fails---------------------" echo "$PACKAGE_URL $PACKAGE_NAME" echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | GitHub | Fail | Install_success_but_test_Fails"