From e395aff54d5147303bb9eaf953b313242c400400 Mon Sep 17 00:00:00 2001 From: Zhuohao-Li Date: Wed, 24 Dec 2025 21:37:39 -0800 Subject: [PATCH 1/6] qwen2.5 --- examples/geo3k_vlm/run_geo3k_vlm.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index 95b1d1f36..4fe5fe72d 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -8,13 +8,17 @@ # Configuration TRAIN_BACKEND=${SLIME_SCRIPT_TRAIN_BACKEND:-"megatron"} -MODEL_NAME=${SLIME_SCRIPT_MODEL_NAME:-"Qwen3-VL-8B-Instruct"} +MODEL_NAME=${SLIME_SCRIPT_MODEL_NAME:-"Qwen2.5-VL-3B-Instruct"} DATASET_NAME=${SLIME_SCRIPT_DATASET_NAME:-"chenhegu/geo3k_imgurl"} NUM_GPUS=${SLIME_SCRIPT_NUM_GPUS:-8} DATASET_LOCAL_NAME=$(basename "$DATASET_NAME") # Validate MODEL_NAME VALID_MODELS=" + Qwen2.5-VL-3B-Instruct + Qwen2.5-VL-7B-Instruct + Qwen2.5-VL-32B-Instruct + Qwen2.5-VL-72B-Instruct Qwen3-VL-2B-Instruct Qwen3-VL-4B-Instruct Qwen3-VL-8B-Instruct From 9216ff0db2df44a157ff3f7f18cc1eff734c8d54 Mon Sep 17 00:00:00 2001 From: Zhuohao-Li Date: Wed, 24 Dec 2025 21:45:53 -0800 Subject: [PATCH 2/6] qwen2.5 --- examples/geo3k_vlm/run_geo3k_vlm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index 4fe5fe72d..79d7fe168 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -191,7 +191,7 @@ else # get MODEL_ARGS from scripts/models for megatron backend SLIME_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)" - MODEL_ARGS_FILE=$(echo "$MODEL_NAME" | sed 's/-Instruct//g; s/-Thinking//g; s/Qwen3-VL-/qwen3-/g; s/-2B/-1.7B/g') + MODEL_ARGS_FILE=$(echo "$MODEL_NAME" | sed 's/-Instruct//g; s/-Thinking//g; s/Qwen2.5-VL-/qwen2.5-/g; s/Qwen3-VL-/qwen3-/g; s/-2B/-1.7B/g') source "${SLIME_DIR}/scripts/models/${MODEL_ARGS_FILE}.sh" fi From d1b5d81e88da67722ba512e06b4dfa09342bfb92 Mon Sep 17 00:00:00 2001 From: Zhuohao-Li Date: Wed, 24 Dec 2025 21:47:04 -0800 Subject: [PATCH 3/6] qwen2.5 --- examples/geo3k_vlm/run_geo3k_vlm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index 79d7fe168..e7bd13ae3 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -85,7 +85,7 @@ fi CKPT_ARGS=( --hf-checkpoint /root/models/${MODEL_NAME} # vl model has rotary base 5000000 - --rotary-base 5000000 + # --rotary-base 5000000 ) ROLLOUT_ARGS=( From 79a71378211300b9291540a1800c1fea34d0485d Mon Sep 17 00:00:00 2001 From: Zhuohao-Li Date: Wed, 24 Dec 2025 21:50:06 -0800 Subject: [PATCH 4/6] qwen2.5 --- examples/geo3k_vlm/run_geo3k_vlm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index e7bd13ae3..e8413e6f0 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -170,7 +170,7 @@ else BACKEND_ARGS=( --train-backend megatron --load /root/models/${MODEL_NAME} - --tensor-model-parallel-size 4 + --tensor-model-parallel-size 2 --sequence-parallel --pipeline-model-parallel-size 1 --context-parallel-size 1 From d9dea160698c6df13db09155baf40ab9d5da3c3c Mon Sep 17 00:00:00 2001 From: Zhuohao-Li Date: Wed, 24 Dec 2025 21:56:04 -0800 Subject: [PATCH 5/6] qwen2.5 --- examples/geo3k_vlm/README.md | 4 ++++ examples/geo3k_vlm/run_geo3k_vlm.sh | 6 +++--- examples/geo3k_vlm/run_geo3k_vlm_sft.sh | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/geo3k_vlm/README.md b/examples/geo3k_vlm/README.md index 800cca542..b6dbfa43c 100644 --- a/examples/geo3k_vlm/README.md +++ b/examples/geo3k_vlm/README.md @@ -2,6 +2,10 @@ Training VLMs with FSDP or Megatron on single-turn reasoning task using GRPO on the [GEO3K dataset](https://huggingface.co/datasets/hiyouga/geometry3k). We used processed version [here](https://huggingface.co/datasets/chenhegu/geo3k_imgurl). +Supported models: +* Qwen2.5-VL +* Qwen3-VL (Dense and Moe) +

FSDP vs Megatron Reward Plot

diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index e8413e6f0..68bfe90b4 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -84,8 +84,8 @@ fi # Common args CKPT_ARGS=( --hf-checkpoint /root/models/${MODEL_NAME} - # vl model has rotary base 5000000 - # --rotary-base 5000000 + # qwen3 vl model has rotary base 5000000, set it when applicable + --rotary-base 5000000 ) ROLLOUT_ARGS=( @@ -170,7 +170,7 @@ else BACKEND_ARGS=( --train-backend megatron --load /root/models/${MODEL_NAME} - --tensor-model-parallel-size 2 + --tensor-model-parallel-size 4 --sequence-parallel --pipeline-model-parallel-size 1 --context-parallel-size 1 diff --git a/examples/geo3k_vlm/run_geo3k_vlm_sft.sh b/examples/geo3k_vlm/run_geo3k_vlm_sft.sh index 94eb02f3e..35bc96c99 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm_sft.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm_sft.sh @@ -6,6 +6,10 @@ DATASET_LOCAL_NAME=$(basename "$DATASET_NAME") # Validate MODEL_NAME VALID_MODELS=" + Qwen2.5-VL-3B-Instruct + Qwen2.5-VL-7B-Instruct + Qwen2.5-VL-32B-Instruct + Qwen2.5-VL-72B-Instruct Qwen3-VL-2B-Instruct Qwen3-VL-4B-Instruct Qwen3-VL-8B-Instruct @@ -151,7 +155,7 @@ else # get MODEL_ARGS from scripts/models for megatron backend SLIME_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)" - MODEL_ARGS_FILE=$(echo "$MODEL_NAME" | sed 's/-Instruct//g; s/-Thinking//g; s/Qwen3-VL-/qwen3-/g; s/-2B/-1.7B/g') + MODEL_ARGS_FILE=$(echo "$MODEL_NAME" | sed 's/-Instruct//g; s/-Thinking//g; s/Qwen2.5-VL-/qwen2.5-/g; s/Qwen3-VL-/qwen3-/g; s/-2B/-1.7B/g') source "${SLIME_DIR}/scripts/models/${MODEL_ARGS_FILE}.sh" fi From 1ce8b826a7768a0d65bbeb427ddc31a227dc954b Mon Sep 17 00:00:00 2001 From: Zhuohao-Li Date: Wed, 24 Dec 2025 21:57:28 -0800 Subject: [PATCH 6/6] qwen2.5 --- examples/geo3k_vlm/run_geo3k_vlm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/geo3k_vlm/run_geo3k_vlm.sh b/examples/geo3k_vlm/run_geo3k_vlm.sh index 68bfe90b4..0185214be 100644 --- a/examples/geo3k_vlm/run_geo3k_vlm.sh +++ b/examples/geo3k_vlm/run_geo3k_vlm.sh @@ -8,7 +8,7 @@ # Configuration TRAIN_BACKEND=${SLIME_SCRIPT_TRAIN_BACKEND:-"megatron"} -MODEL_NAME=${SLIME_SCRIPT_MODEL_NAME:-"Qwen2.5-VL-3B-Instruct"} +MODEL_NAME=${SLIME_SCRIPT_MODEL_NAME:-"Qwen3-VL-8B-Instruct"} DATASET_NAME=${SLIME_SCRIPT_DATASET_NAME:-"chenhegu/geo3k_imgurl"} NUM_GPUS=${SLIME_SCRIPT_NUM_GPUS:-8} DATASET_LOCAL_NAME=$(basename "$DATASET_NAME")