GLM 5.2 双机混部 - shun8686/sglang GitHub Wiki
服务启动参数 - A3
#!/bin/bash
SERVER_PORT=6677
MODEL_PATH="${MODEL_PATH:-/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.2-w8a8}"
NODE_IPS=(
"192.168.0.1"
"192.168.0.2"
)
NNODES=${#NODE_IPS[@]}
MASTER_ADDR="${NODE_IPS[0]}"
MASTER_PORT="5567"
DIST_INIT_ADDR="${MASTER_ADDR}:${MASTER_PORT}"
detect_nic() {
local nic
nic=$(awk '
function is_virtual(name) {
return name ~ /^(lo|docker|tunl|cali|veth|br-|virbr|kube-|flannel|weave|cilium)/
}
NR > 2 {
split($1, a, ":"); ifname = a[1]
if (!is_virtual(ifname)) traffic[ifname] = $2 + $10
}
END {
best = ""; bestv = -1
for (i in traffic) if (traffic[i] > bestv) { best = i; bestv = traffic[i] }
if (best != "") print best
}' /proc/net/dev 2>/dev/null || true)
echo "${nic:-lo}"
}
NIC_NAME="${NIC_NAME:-$(detect_nic)}"
# ========================= 环境变量 (node_envs) =========================
export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True"
export STREAMS_PER_DEVICE="32"
export SGLANG_SET_CPU_AFFINITY="1"
export SGLANG_ENABLE_SPEC_V2="1"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM="1"
export DEEPEP_HCCL_BUFFSIZE="2500"
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ="1"
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS="1024"
export DEEPEP_NORMAL_LONG_SEQ_ROUND="72"
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK="32"
export DEEPEP_HYBRID_DEPLOYMENT="1"
export DEEP_NORMAL_MODE_USE_INT8_QUANT="1"
export HCCL_SOCKET_IFNAME="${NIC_NAME}"
export GLOO_SOCKET_IFNAME="${NIC_NAME}"
# ===== Auto-detect node rank by matching local IPs =============
LOCAL_HOST1=$(hostname -I | awk '{print $1}')
LOCAL_HOST2=$(hostname -I | awk '{print $2}')
NODE_RANK=""
for i in "${!NODE_IPS[@]}"; do
if [ "$LOCAL_HOST1" == "${NODE_IPS[$i]}" ](/shun8686/sglang/wiki/|-"$LOCAL_HOST2"-==-"${NODE_IPS[$i]}"-); then
NODE_RANK="$i"
SERVER_HOST="${NODE_IPS[$i]}"
export HCCL_SOCKET_IFNAME="${HCCL_IFS[$i]}"
break
fi
done
if [ -z "${NODE_RANK}" ](/shun8686/sglang/wiki/--z-"${NODE_RANK}"-); then
echo "ERROR: local IPs [${LOCAL_HOST1} ${LOCAL_HOST2}] not found in NODE_IPS=[${NODE_IPS[*]}]"
exit 1
fi
echo "========================================"
echo "Launching ${NNODES} Nodes"
echo "node-rank : ${NODE_RANK}"
echo "local IPs : ${LOCAL_HOST1} ${LOCAL_HOST2}"
echo "dist-init-addr : ${DIST_INIT_ADDR}"
echo "nnodes : ${NNODES}"
echo "HCCL interface : ${HCCL_SOCKET_IFNAME}"
echo "GLOO interface : ${GLOO_SOCKET_IFNAME}"
echo "========================================"
# ========================= 启动 sglang 服务 =========================
python3 -m sglang.launch_server \
--trust-remote-code \
--model "${MODEL_PATH}" \
--host "${SERVER_HOST}" \
--port "${SERVER_PORT}" \
--attention-backend ascend \
--device npu \
--tp-size 32 \
--nnodes 2 \
--node-rank "${NODE_RANK}"
--dp-size 8 \
--enable-dp-attention \
--chunked-prefill-size 65536 \
--max-prefill-tokens 280000 \
--mem-fraction-static 0.76 \
--context-length 135000 \
--served-model-name glm-5.2-w8a8 \
--cuda-graph-max-bs-decode 4 \
--max-running-requests 32 \
--quantization modelslim \
--moe-a2a-backend deepep \
--deepep-mode auto \
--disable-shared-experts-fusion \
--load-balance-method round_robin \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--enable-metrics \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--dist-init-addr "${DIST_INIT_ADDR}" \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 2 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://127.0.0.1:5000", "config_store_url":"tcp://127.0.0.1:6000", "log_level":"info", "world_size":256, "protocol": "device_sdma", "dram_size": "1GB"}'
服务启动参数 - A3 - 极光
#!/bin/bash
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
source /usr/local/Ascend/cann/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash
export PYTORCH_NPU_ALLOC_CONF="expandable_segments:True"
export STREAMS_PER_DEVICE="32"
export SGLANG_SET_CPU_AFFINITY="1"
export SGLANG_ENABLE_SPEC_V2="1"
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM="1"
export DEEPEP_HCCL_BUFFSIZE="2500"
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ="1"
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS="1024"
export DEEPEP_NORMAL_LONG_SEQ_ROUND="72"
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK="32"
export DEEPEP_HYBRID_DEPLOYMENT="1"
export DEEP_NORMAL_MODE_USE_INT8_QUANT="1"
export SGLANG_HICACHE_HOST_MEM_BACKEND=memfabric
# ========================= 启动 sglang 服务 =========================
python3 -m sglang.launch_server \
--max-total-tokens 65536 \
--chunked-prefill-size 65536 \
--trust-remote-code \
--model "/root/.cache/modelscope/hub/models/Eco-Tech/GLM-5.2-w8a8" \
--attention-backend ascend \
--device npu \
--tp-size 32 \
--dp-size 8 \
--enable-dp-attention \
--max-prefill-tokens 280000 \
--mem-fraction-static 0.88 \
--context-length 135000 \
--served-model-name glm-5.2-w8a8 \
--cuda-graph-max-bs-decode 4 \
--max-running-requests 32 \
--quantization modelslim \
--moe-a2a-backend deepep \
--deepep-mode auto \
--disable-shared-experts-fusion \
--load-balance-method round_robin \
--reasoning-parser glm45 \
--tool-call-parser glm47 \
--enable-metrics \
--speculative-algorithm NEXTN \
--speculative-num-steps 3 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 4 \
--speculative-draft-model-quantization unquant \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 2 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://127.0.0.1:5000", "config_store_url":"tcp://127.0.0.1:6000", "log_level":"info", "world_size":256, "protocol": "device_sdma", "dram_size": "1GB"}'
服务启动参数 - A5 - 原始
#!/bin/bash
# ===== Cleanup =====
unset https_proxy http_proxy HTTPS_PROXY HTTP_PROXY ASCEND_LAUNCH_BLOCKING
pkill -9 python 2>/dev/null || true
pkill -9 sglang 2>/dev/null || true
pkill -9 VLLM 2>/dev/null || true
# ===== Environment =====
#export PYTHONPATH=/home/r00648901/sglang/python:$PYTHONPATH
export DEEPEP_HCCL_BUFFSIZE=2048
export HCCL_CONNECT_TIMEOUT=300
export HCCL_EXEC_TIMEOUT=68
export HCCL_OP_EXPANSION_MODE=AIV
export ACL_DEVICE_SYNC_TIMEOUT=60
# 内存碎片
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
# [FIA]
export ASCEND_USE_FIA=1
# [MLAPO]
export SGLANG_NPU_USE_MLAPO=1
#export SGLANG_NPU_GLM_NEXTN_BF16_KV_CACHE=1
# [DEEPEP]
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=35
# [Prefill Delay]
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
# [MTP]
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export TRANSFORMERS_VERBOSITY=error
# [多机]
export HCCL_HOST_SOCKET_PORT_RANGE=auto
export GLOO_SOCKET_IFNAME=data0.3001
unset HCCL_IF_IP 2>/dev/null || true
unset HCCL_SOCKET_FAMILY 2>/dev/null || true
unset RANK_TABLE_FILllE 2>/dev/null || true
# ===== Model Config =====
MODEL_PATH=/mnt/share/w00936111/GLM-5.2-W8A8C8-mxfp8
SERVED_MODEL_NAME=glm52
SERVER_HOST=141.61.33.18
SERVER_PORT=6677
# ===== Cluster Config ===========================================
# 每台机器: IP + HCCL 网卡名 (一一对应)
# 拓展时只需往三个数组里追加即可，NNODES / TP_SIZE 自动推算
NODE_IPS=(
"141.61.33.18"
"141.61.33.19"
)
HCCL_IFS=(
"enp34s0f1"
"enp34s0f1"
)
NUM_NPUS_PER_NODE=8 # 每机 NPU 数
# ================================================================
MASTER_ADDR="${NODE_IPS[0]}"
MASTER_PORT="5569"
DIST_INIT_ADDR="${MASTER_ADDR}:${MASTER_PORT}"
NNODES=${#NODE_IPS[@]}
TP_SIZE=$(( NNODES * NUM_NPUS_PER_NODE ))
DP_SIZE=4 # DP 并行度
# ===== Auto-detect node rank by matching local IPs =============
LOCAL_HOST1=$(hostname -I | awk '{print $1}')
LOCAL_HOST2=$(hostname -I | awk '{print $2}')
NODE_RANK=""
for i in "${!NODE_IPS[@]}"; do
if [ "$LOCAL_HOST1" == "${NODE_IPS[$i]}" ](/shun8686/sglang/wiki/|-"$LOCAL_HOST2"-==-"${NODE_IPS[$i]}"-); then
NODE_RANK="$i"
export HCCL_SOCKET_IFNAME="${HCCL_IFS[$i]}"
break
fi
done
if [ -z "${NODE_RANK}" ](/shun8686/sglang/wiki/--z-"${NODE_RANK}"-); then
echo "ERROR: local IPs [${LOCAL_HOST1} ${LOCAL_HOST2}] not found in NODE_IPS=[${NODE_IPS[*]}]"
exit 1
fi
echo "========================================"
echo "Launching GLM5.2 ${NNODES} Nodes"
echo "node-rank : ${NODE_RANK}"
echo "local IPs : ${LOCAL_HOST1} ${LOCAL_HOST2}"
echo "dist-init-addr : ${DIST_INIT_ADDR}"
echo "nnodes : ${NNODES}"
echo "tp-size : ${TP_SIZE}"
echo "dp-size : ${DP_SIZE}"
echo "HCCL interface : ${HCCL_SOCKET_IFNAME}"
echo "GLOO interface : ${GLOO_SOCKET_IFNAME}"
echo "========================================"
# ===== Launch =====
python3 -m sglang.launch_server --model-path ${MODEL_PATH} \
--served-model-name "${SERVED_MODEL_NAME}" \
--host "${SERVER_HOST}" \
--port "${SERVER_PORT}" \
--nnodes "${NNODES}" \
--node-rank "${NODE_RANK}" \
--dist-init-addr "${DIST_INIT_ADDR}" \
--tp-size "${TP_SIZE}" \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--watchdog-timeout 9000 \
--max-running-requests 52 \
--mem-fraction-static 0.84 \
--quantization modelslim \
--max-prefill-tokens 2048000 \
--chunked-prefill-size 16384 \
--kv-cache-dtype "fp8_e4m3" \
--speculative-draft-kv-cache-dtype bf16 \
--dp ${DP_SIZE} \
--enable-dp-attention \
--enable-dp-lm-head \
--load-balance-method round_robin \
--moe-a2a-backend deepep \
--deepep-mode auto \
--speculative-algorithm NEXTN \
--speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \
--enable-metrics
服务启动参数 - A5 - 极光
#!/bin/bash
# ===== Cleanup =====
unset https_proxy http_proxy HTTPS_PROXY HTTP_PROXY ASCEND_LAUNCH_BLOCKING
# ===== Environment =====
export DEEPEP_HCCL_BUFFSIZE=2048
export HCCL_CONNECT_TIMEOUT=300
export HCCL_EXEC_TIMEOUT=68
export HCCL_OP_EXPANSION_MODE=AIV
export ACL_DEVICE_SYNC_TIMEOUT=60
# 内存碎片
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
# [FIA]
export ASCEND_USE_FIA=1
# [MLAPO]
export SGLANG_NPU_USE_MLAPO=1
#export SGLANG_NPU_GLM_NEXTN_BF16_KV_CACHE=1
# [DEEPEP]
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=35
# [Prefill Delay]
export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1
export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200
# [MTP]
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export TRANSFORMERS_VERBOSITY=error
# [多机]
export HCCL_HOST_SOCKET_PORT_RANGE=auto
export GLOO_SOCKET_IFNAME=data0.3001
unset HCCL_IF_IP 2>/dev/null || true
unset HCCL_SOCKET_FAMILY 2>/dev/null || true
unset RANK_TABLE_FILllE 2>/dev/null || true
echo "========================================"
echo "HCCL interface : ${HCCL_SOCKET_IFNAME}"
echo "GLOO interface : ${GLOO_SOCKET_IFNAME}"
echo "========================================"
# ===== Launch =====
python3 -m sglang.launch_server \
--model-path /mnt/share/w00936111/GLM-5.2-W8A8C8-mxfp8 \
--served-model-name glm52 \
--tp-size 16 \
--trust-remote-code --device npu --attention-backend ascend --watchdog-timeout 9000 \
--max-running-requests 52 \
--mem-fraction-static 0.84 \
--quantization modelslim \
--max-prefill-tokens 2048000 \
--max-total-tokens 65536 \
--chunked-prefill-size 65536 \
--kv-cache-dtype "fp8_e4m3" \
--speculative-draft-kv-cache-dtype bf16 \
--dp 4 \
--enable-dp-attention \
--enable-dp-lm-head \
--load-balance-method round_robin \
--moe-a2a-backend deepep \
--deepep-mode auto \
--speculative-algorithm NEXTN \
--speculative-num-steps 4 --speculative-eagle-topk 1 --speculative-num-draft-tokens 5 \
--enable-metrics \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 2 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://127.0.0.1:5007", "config_store_url":"tcp://127.0.0.1:6007", "log_level":"info", "world_size":256, "protocol": "device_urma", "dram_size": "1GB"}'