[A5]GLM5.2 ‐ 单机混部 - shun8686/sglang GitHub Wiki
A5 - L2
如下脚本使用B072, 执行pip uninstall custom_ops后能跑通, 141.61.45.119环境
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
# export PYTHONPATH=/mnt/share/d00662834/sglang/python:$PYTHONPATH/
export HCCL_CONNECT_TIMEOUT=300
export HCCL_EXEC_TIMEOUT=300
export HCCL_BUFFSIZE=2048
export HCCL_OP_EXPANSION_MODE=""AIV""
export HCCL_INTRA_PCIE_ENABLE=1
export HCCL_INTRA_ROCE_ENABLE=0
export ACL_DEVICE_SYNC_TIMEOUT=300
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export TASK_QUEUE_ENABLE=1
export STREAMS_PER_DEVICE=32
MODEL_PATH=/mnt/share/GLM-5.2-W4A8C8-A5-0731-copy
export ASCEND_USE_FIA=1
export SGLANG_NPU_USE_MLAPO=0
export SGLANG_ENABLE_SPEC_V2=1
export TRANSFORMERS_VERBOSITY=error
export HCCL_HOST_SOCKET_PORT_RANGE=auto
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
# export DEEPEP_NORMAL_LONG_SEQ_ROUND=9
# export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=8192
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=0
unset HCCL_IF_IP 2>/dev/null || true
unset HCCL_SOCKET_FAMILY 2>/dev/null || true
unset RANK_TABLE_FILE 2>/dev/null || true
mkdir -p logs
python3 -m sglang.launch_server --model-path ${MODEL_PATH} \
--served-model-name glm52 \
--host 127.0.0.1 \
--port 8818 \
--tp-size 8 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--watchdog-timeout 9000 \
--max-running-requests 120 \
--mem-fraction-static 0.8 \
--quantization modelslim \
--max-prefill-tokens 2048000 \
--chunked-prefill-size 16384 \
--enable-metrics \
--cuda-graph-bs-decode 8 \
--load-balance-method round_robin \
--speculative-algorithm EAGLE \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--speculative-draft-kv-cache-dtype bf16 \
--kv-cache-dtype "fp8_e4m3" \
--dp-size 4 \
--enable-dp-attention
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-size 60 \
2>&1 | tee "logs/run_$(date +%Y-%m-%d_%H-%M-%S).log"
A5 - L3 - 手动
B080上卸载custom_ops规避后可以跑通 pip uninstall custom_ops
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
export MF_DEVICE_UB_QOS=1
#export SGLANG_HICACHE_HOST_MEM_BACKEND=memfabric
# export PYTHONPATH=/mnt/share/d00662834/sglang/python:$PYTHONPATH/
export HCCL_CONNECT_TIMEOUT=300
export HCCL_EXEC_TIMEOUT=300
export HCCL_BUFFSIZE=2048
export HCCL_OP_EXPANSION_MODE=""AIV""
export HCCL_INTRA_PCIE_ENABLE=1
export HCCL_INTRA_ROCE_ENABLE=0
# export SGLANG_HICACHE_HYBM_RESERVE_GB=60
# export SGLANG_HICACHE_IO_ASCENDC=1
# export SGLANG_HICACHE_HOST_MEM=hybm
export ACL_DEVICE_SYNC_TIMEOUT=300
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export TASK_QUEUE_ENABLE=1
export STREAMS_PER_DEVICE=32
# MODEL_PATH=/mnt/share/w00936111/GLM-5.2-W4A4C8-mxfp4-A5-0731
# MODEL_PATH=/mnt/share/y00951466/GLM-5.2-W4A4C8-mxfp4-A5-0731-clip
# MODEL_PATH=/home/weights/GLM-5.2-W4A8C8-A5-0731
# MODEL_PATH=/mnt/share/GLM-5.2-W4A8C8-0713
# MODEL_PATH=/mnt/share/w00936111/GLM-5.2-W8A8C8-mxfp8
# MODEL_PATH=/mnt/share/GLM-5.2-W8A8C8-mxfp8
MODEL_PATH=/mnt/share/GLM-5.2-W4A8C8-A5-0731-copy
export ASCEND_USE_FIA=1
export SGLANG_NPU_USE_MLAPO=0
export SGLANG_ENABLE_SPEC_V2=1
export TRANSFORMERS_VERBOSITY=error
export HCCL_HOST_SOCKET_PORT_RANGE=auto
export HCCL_SOCKET_IFNAME=lo
# unset HCCL_SOCKET_IFNAME
export GLOO_SOCKET_IFNAME=lo
# export DEEPEP_NORMAL_LONG_SEQ_ROUND=9
# export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=8192
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=0
unset HCCL_IF_IP 2>/dev/null || true
unset HCCL_SOCKET_FAMILY 2>/dev/null || true
unset RANK_TABLE_FILE 2>/dev/null || true
SERVED_MODEL_NAME=glm52
SERVER_HOST=127.0.0.1
SERVER_PORT=8818
python3 -m sglang.launch_server --model-path ${MODEL_PATH} \
--served-model-name "${SERVED_MODEL_NAME}" \
--host "${SERVER_HOST}" \
--port "${SERVER_PORT}" \
--tp-size 8 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--watchdog-timeout 9000 \
--max-running-requests 120 \
--mem-fraction-static 0.8 \
--quantization modelslim \
--max-prefill-tokens 2048000 \
--max-total-tokens 65536 \
--chunked-prefill-size 65536 \
--enable-metrics \
--cuda-graph-bs-decode 8 \
--load-balance-method round_robin \
--speculative-algorithm EAGLE \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--speculative-draft-kv-cache-dtype bf16 \
--kv-cache-dtype "fp8_e4m3" \
--dp-size 4 \
--enable-dp-attention \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 4 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://127.0.0.1:5007", "config_store_url":"tcp://127.0.0.1:6007", "log_level":"info", "world_size":8, "protocol": "device_urma", "dram_size": "30GB"}' \
2>&1 | tee "logs/run_$(date +%Y-%m-%d_%H-%M-%S).log"
A5 - L3 - 极光
# B080 存在bug启动失败，临时卸载custom_ops规避
pip uninstall custom_ops || true
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
# L2 host mem是否使用MF
#export SGLANG_HICACHE_HOST_MEM_BACKEND=memfabric
export HCCL_CONNECT_TIMEOUT=300
export HCCL_EXEC_TIMEOUT=300
export HCCL_BUFFSIZE=2048
export HCCL_OP_EXPANSION_MODE="AIV"
#export HCCL_INTRA_PCIE_ENABLE=1
#export HCCL_INTRA_ROCE_ENABLE=0
export ACL_DEVICE_SYNC_TIMEOUT=300
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export TASK_QUEUE_ENABLE=1
export STREAMS_PER_DEVICE=32
export ASCEND_USE_FIA=1
export SGLANG_NPU_USE_MLAPO=0
export SGLANG_ENABLE_SPEC_V2=1
export TRANSFORMERS_VERBOSITY=error
export HCCL_HOST_SOCKET_PORT_RANGE=auto
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=0
python3 -m sglang.launch_server \
--model-path /mnt/share/GLM-5.2-W4A8C8-A5-0731-copy \
--tp-size 8 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--watchdog-timeout 9000 \
--max-running-requests 120 \
--mem-fraction-static 0.8 \
--quantization modelslim \
--max-prefill-tokens 2048000 \
--max-total-tokens 65536 \
--chunked-prefill-size 65536 \
--enable-metrics \
--cuda-graph-bs-decode 8 \
--load-balance-method round_robin \
--speculative-algorithm EAGLE \
--speculative-num-steps 4 \
--speculative-eagle-topk 1 \
--speculative-num-draft-tokens 5 \
--speculative-draft-kv-cache-dtype bf16 \
--kv-cache-dtype "fp8_e4m3" \
--dp-size 4 \
--enable-dp-attention \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 4 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://127.0.0.1:5007", "config_store_url":"tcp://127.0.0.1:6007", "log_level":"info", "world_size":8, "protocol": "device_urma", "dram_size": "30GB"}'