From 42ec6ed0be3cba907386df9916ee80fb6def9bc8 Mon Sep 17 00:00:00 2001 From: zhouxin Date: Wed, 21 Jan 2026 08:10:46 +0000 Subject: [PATCH 1/4] trigger ci From 25f2f1c3497dfee903d6073be6c20e3e04417579 Mon Sep 17 00:00:00 2001 From: zhouxin Date: Wed, 21 Jan 2026 09:06:25 +0000 Subject: [PATCH 2/4] Test fork pytest --- scripts/unittest_check_gpu.sh | 77 +++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/scripts/unittest_check_gpu.sh b/scripts/unittest_check_gpu.sh index ada1a49e4..708e5ecb4 100644 --- a/scripts/unittest_check_gpu.sh +++ b/scripts/unittest_check_gpu.sh @@ -15,7 +15,7 @@ set +x echo '************************************************************************************************************' -echo "Insalling latest release gpu version torch" +echo "Installing latest release gpu version torch" python -m pip uninstall -y torchaudio python -m pip install torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl python -m pip install torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl @@ -23,10 +23,11 @@ python -m pip install torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64 python -c "import torch; print('torch version information:' ,torch.__version__)" echo '************************************************************************************************************' -echo "Insalling develop gpu version paddle" +echo "Installing develop gpu version paddle" python -m pip uninstall -y paddlepaddle python -m pip uninstall -y paddlepaddle-gpu -python -m pip install --force-reinstall --no-deps -U --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ +# For bypass broken update in paddle, should not merged into master +python -m pip install --force-reinstall --no-deps -U --pre paddlepaddle-gpu==3.4.0.dev20260119 -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ python -m pip install safetensors==0.6.2 python -c "import paddle; print('paddle version information:' , paddle.__version__); commit = paddle.__git_commit__;print('paddle commit information:' , commit)" @@ -41,16 +42,74 @@ echo '************************************************************************** #python -c "import paddleformers; print('paddleformers version information:', paddleformers.__version__)" echo '************************************************************************************************************' -echo "Insalling paconvert requirements" +echo "Installing paconvert requirements" python -m pip install -r requirements.txt echo '************************************************************************************************************' echo "Checking code unit test by pytest ..." -python -m pip install pytest-timeout pytest-xdist pytest-rerunfailures -python -m pytest -n 1 --reruns=3 ./tests; check_error=$? + +# Install pytest and necessary plugins for parallelism and isolation +python -m pip install pytest-timeout pytest-xdist pytest-rerunfailures pytest-forked + +# Detect GPU count using Python to determine the number of parallel workers +# This ensures we use exactly one process per GPU +NUM_GPUS=$(python -c "import torch; print(torch.cuda.device_count())") + +if [ "$NUM_GPUS" -eq "0" ]; then + echo "No GPU detected. Falling back to single process." + PYTEST_WORKERS=1 +else + echo "Detected ${NUM_GPUS} GPUs. Setting pytest workers to ${NUM_GPUS}." + PYTEST_WORKERS=${NUM_GPUS} +fi + +echo "Injecting automatic GPU distribution logic into tests/conftest.py..." + +cat < tests/conftest.py +import pytest +import os + +@pytest.fixture(scope="session", autouse=True) +def distribute_gpus_automatically(worker_id): + """ + Auto-assign GPU based on xdist worker ID. + gw0 -> GPU 0 + gw1 -> GPU 1 + ... + """ + if worker_id == "master": + return + + try: + worker_num = int(worker_id.replace("gw", "")) + except ValueError: + return + + import torch + total_gpus = torch.cuda.device_count() + + if total_gpus > 0: + gpu_idx = worker_num % total_gpus + + os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_idx) + + # print(f"Worker {worker_id} initialized on GPU {gpu_idx}") +EOF + +echo '************************************************************************************************************' +echo "Checking code unit test by pytest with ${PYTEST_WORKERS}-Process ISOLATION ..." + + +# Run pytest with dynamic workers and process isolation +# -n: Number of workers (equal to GPU count) +# --forked: Isolate each test in a separate process to clean up GPU context +# --dist=loadscope: Group tests by module to optimize loading time +python -m pytest -n ${PYTEST_WORKERS} --forked --dist=loadscope --reruns=3 ./tests; check_error=$? + if [ ${check_error} != 0 ];then - echo "Rerun unit test check." - python -m pytest -n 1 --lf ./tests; check_error=$? + echo "Rerun unit test check (Sequential Fallback)." + # Fallback to single process sequential execution if parallel execution fails + python -m pytest -n 1 --forked --lf ./tests; check_error=$? fi echo '************************************************************************************************************' @@ -65,7 +124,7 @@ if [ ${check_error} != 0 ];then echo "Your PR code unit test check failed." echo "Please run the following command." echo "" - echo " python -m pytest tests" + echo " python -m pytest --forked tests" echo "" echo "For more information, please refer to our check guide:" echo "https://github.com/PaddlePaddle/PaConvert#readme." From d723dcecc67651716546953413e2037fe8bf04b3 Mon Sep 17 00:00:00 2001 From: zhouxin Date: Wed, 21 Jan 2026 09:15:37 +0000 Subject: [PATCH 3/4] Test 4 process --- scripts/unittest_check_gpu.sh | 75 ++++------------------------------- 1 file changed, 8 insertions(+), 67 deletions(-) diff --git a/scripts/unittest_check_gpu.sh b/scripts/unittest_check_gpu.sh index 708e5ecb4..c14270d17 100644 --- a/scripts/unittest_check_gpu.sh +++ b/scripts/unittest_check_gpu.sh @@ -15,7 +15,7 @@ set +x echo '************************************************************************************************************' -echo "Installing latest release gpu version torch" +echo "Insalling latest release gpu version torch" python -m pip uninstall -y torchaudio python -m pip install torch-2.7.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl python -m pip install torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64.whl @@ -23,11 +23,10 @@ python -m pip install torchvision-0.22.1+cu118-cp310-cp310-manylinux_2_28_x86_64 python -c "import torch; print('torch version information:' ,torch.__version__)" echo '************************************************************************************************************' -echo "Installing develop gpu version paddle" +echo "Insalling develop gpu version paddle" python -m pip uninstall -y paddlepaddle python -m pip uninstall -y paddlepaddle-gpu -# For bypass broken update in paddle, should not merged into master -python -m pip install --force-reinstall --no-deps -U --pre paddlepaddle-gpu==3.4.0.dev20260119 -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ +python -m pip install --force-reinstall --no-deps -U --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ python -m pip install safetensors==0.6.2 python -c "import paddle; print('paddle version information:' , paddle.__version__); commit = paddle.__git_commit__;print('paddle commit information:' , commit)" @@ -42,74 +41,16 @@ echo '************************************************************************** #python -c "import paddleformers; print('paddleformers version information:', paddleformers.__version__)" echo '************************************************************************************************************' -echo "Installing paconvert requirements" +echo "Insalling paconvert requirements" python -m pip install -r requirements.txt echo '************************************************************************************************************' echo "Checking code unit test by pytest ..." - -# Install pytest and necessary plugins for parallelism and isolation python -m pip install pytest-timeout pytest-xdist pytest-rerunfailures pytest-forked - -# Detect GPU count using Python to determine the number of parallel workers -# This ensures we use exactly one process per GPU -NUM_GPUS=$(python -c "import torch; print(torch.cuda.device_count())") - -if [ "$NUM_GPUS" -eq "0" ]; then - echo "No GPU detected. Falling back to single process." - PYTEST_WORKERS=1 -else - echo "Detected ${NUM_GPUS} GPUs. Setting pytest workers to ${NUM_GPUS}." - PYTEST_WORKERS=${NUM_GPUS} -fi - -echo "Injecting automatic GPU distribution logic into tests/conftest.py..." - -cat < tests/conftest.py -import pytest -import os - -@pytest.fixture(scope="session", autouse=True) -def distribute_gpus_automatically(worker_id): - """ - Auto-assign GPU based on xdist worker ID. - gw0 -> GPU 0 - gw1 -> GPU 1 - ... - """ - if worker_id == "master": - return - - try: - worker_num = int(worker_id.replace("gw", "")) - except ValueError: - return - - import torch - total_gpus = torch.cuda.device_count() - - if total_gpus > 0: - gpu_idx = worker_num % total_gpus - - os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_idx) - - # print(f"Worker {worker_id} initialized on GPU {gpu_idx}") -EOF - -echo '************************************************************************************************************' -echo "Checking code unit test by pytest with ${PYTEST_WORKERS}-Process ISOLATION ..." - - -# Run pytest with dynamic workers and process isolation -# -n: Number of workers (equal to GPU count) -# --forked: Isolate each test in a separate process to clean up GPU context -# --dist=loadscope: Group tests by module to optimize loading time -python -m pytest -n ${PYTEST_WORKERS} --forked --dist=loadscope --reruns=3 ./tests; check_error=$? - +python -m pytest --forked -n 4 --reruns=3 ./tests; check_error=$? if [ ${check_error} != 0 ];then - echo "Rerun unit test check (Sequential Fallback)." - # Fallback to single process sequential execution if parallel execution fails - python -m pytest -n 1 --forked --lf ./tests; check_error=$? + echo "Rerun unit test check." + python -m pytest -n 1 --lf ./tests; check_error=$? fi echo '************************************************************************************************************' @@ -124,7 +65,7 @@ if [ ${check_error} != 0 ];then echo "Your PR code unit test check failed." echo "Please run the following command." echo "" - echo " python -m pytest --forked tests" + echo " python -m pytest tests" echo "" echo "For more information, please refer to our check guide:" echo "https://github.com/PaddlePaddle/PaConvert#readme." From 524a9fa7558ce9950ca1cba7311227b57e695fa7 Mon Sep 17 00:00:00 2001 From: zhouxin Date: Wed, 21 Jan 2026 09:31:34 +0000 Subject: [PATCH 4/4] Test forkl --- scripts/unittest_check_gpu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/unittest_check_gpu.sh b/scripts/unittest_check_gpu.sh index c14270d17..b2d801d64 100644 --- a/scripts/unittest_check_gpu.sh +++ b/scripts/unittest_check_gpu.sh @@ -47,7 +47,7 @@ python -m pip install -r requirements.txt echo '************************************************************************************************************' echo "Checking code unit test by pytest ..." python -m pip install pytest-timeout pytest-xdist pytest-rerunfailures pytest-forked -python -m pytest --forked -n 4 --reruns=3 ./tests; check_error=$? +python -m pytest --forked -n 1 --reruns=3 ./tests; check_error=$? if [ ${check_error} != 0 ];then echo "Rerun unit test check." python -m pytest -n 1 --lf ./tests; check_error=$?