[A5] Kimi K3四机混部 - shun8686/sglang GitHub Wiki
A5 - 原始
#!/bin/bash
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=10
sysctl -w kernel.numa_balancing=0
# 下面这行若内核不支持可注释掉，或加判断
sysctl -w kernel.sched_migration_cost_ns=50000 2>/dev/null || true
export SGLANG_SET_CPU_AFFINITY=1
# export TRITON_ALL_BLOCKS_PARALLEL=1
export SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1
#MODEL_PATH=/mnt/share/z00842572/k3-official
MODEL_PATH=/mnt/share/recipes/kimi/kimi_k3_27
DRAFT_MODEL_PATH=/mnt/share/recipes/kimi/RadixArk-Kimi-K3-DSpark
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
export SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1
export SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_RAGGED_VERIFY_MODE=static
export SGLANG_DSPARK_FOLDED_PROPOSAL=0
export SGLANG_DSPARK_FOLDED_SAMPLING=0
export SGLANG_DSPARK_STACKED_CTX_KV=0
export SGLANG_DSPARK_EMBED_IN_GRAPH=0
# use fias v2
export SGLANG_NPU_USE_FIAS_V2_BSND=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=0
#source /home/z00842572/CANN/cann/set_env.sh
#source /home/z00842572/CANN/nnal/atb/set_env.sh
#export ASCEND_LAUNCH_BLOCKING=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export DEEPEP_HCCL_BUFFSIZE=1200
export HCCL_OP_EXPANSION_MODE=AIV
export ASCEND_USE_FIA=1
export SGLANG_W4A8_MXFP4_MOE=1
#export DEEP_USE_MODE="alltoall"
export SGLANG_TORCH_PROFILER_DIR="./sglang_profile/"
export SGLANG_PROFILE_WITH_STACK=False
export SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1
#export PYTHONPATH=/mnt/share/hanwlax/sglang/python:$PYTHONPATH
#export PYTHONPATH=/mnt/share/z00842572/0828/sglang/python:$PYTHONPATH
#export PYTHONPATH=/mnt/share/z00842572/ci_sglang/python:$PYTHONPATH
# export SGLANG_DSPARK_DEBUG_TRACE=${SGLANG_DSPARK_DEBUG_TRACE:-1}
# 定义 Decode 节点 IP 列表（4 机）
D_IP=('141.61.94.103' '141.61.94.111' '141.61.94.107' '141.61.94.139')
# 获取本机所有 IP 地址
LOCAL_IPS=$(hostname -I 2>/dev/null)
# 标记是否匹配
MATCHED=0
# 检查是否属于 Decode 节点
for i in "${!D_IP[@]}"; do
for ip in $LOCAL_IPS; do
if [ "$ip" == "${D_IP[$i]}" ](/shun8686/sglang/wiki/-"$ip"-==-"${D_IP[$i]}"-); then
MATCHED=1
echo "Decode -> ${D_IP[$i]} (node-rank $i)"
# 设置网络接口环境变量
export HCCL_SOCKET_IFNAME=enp35s0f2
export GLOO_SOCKET_IFNAME=data0.173
# 启动服务
sglang serve \
--model-loader-extra-config '{"enable_multithread_load": true}' \
--dist-init-addr ${D_IP[0]}:5000 --nnodes 4 --node-rank $i \
--model-path $MODEL_PATH \
--tokenizer-path $MODEL_PATH \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--tp-size 32 \
--enable-dp-attention \
--dp-size 1 \
--enable-dp-lm-head \
--max-mamba-cache-size 160 \
--mem-fraction-static 0.9 \
--chunked-prefill-size 2048 \
--cuda-graph-bs-decode 8 16 32 \
--max-running-requests 32 \
--host 0.0.0.0 \
--port 30000 \
--max-total-tokens 200000 \
--enable-shared-experts-attn-tp \
--enable-dense-mlp-attn-tp \
--moe-a2a-backend deepep \
--deepep-mode auto \
--speculative-algorithm DSPARK \
--speculative-draft-model-path $DRAFT_MODEL_PATH \
--speculative-draft-model-quantization unquant \
--speculative-dspark-block-size 7 \
--speculative-draft-attention-backend ascend \
--linear-attn-verify-backend triton \
--speculative-eagle-topk 1 \
--watchdog-timeout 9000 \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 4 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://141.61.94.103:5007", "config_store_url":"tcp://141.61.94.103:6007", "log_level":"info", "world_size":32, "protocol": "device_urma", "dram_size": "20GB"}' \
2>&1 | tee "logs/run32p_$(date +%Y-%m-%d_%H-%M-%S).log" "logs/run32p_latest.log"
exit 0 # 正常结束（原为 exit 1，改为 0 表示成功）
fi
done
done
# 如果没有匹配任何 IP，则报错退出
if [ $MATCHED -eq 0 ]; then
echo "Error: Local IP ($LOCAL_IPS) not found in D_IP list."
exit 1
fi
A5 - 极光
GLOO_SOCKET_IFNAME需指定data0.173 当前只支持Kimi K3原始权重, 不支持量化权重
#!/bin/bash
pkill -9 sglang
pkill -9 python
#pip uninstall -y memcache_hybrid || true
#pip uninstall -y memfabric_hybrid || true
#pip install /mnt/share/d00662834/memfabric_hybrid-1.3.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl || true
#pip install /mnt/share/d00662834/memcache_hybrid-1.3.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl || true
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=10
sysctl -w kernel.numa_balancing=0
# 下面这行若内核不支持可注释掉，或加判断
sysctl -w kernel.sched_migration_cost_ns=50000 2>/dev/null || true
export SGLANG_SET_CPU_AFFINITY=1
# export TRITON_ALL_BLOCKS_PARALLEL=1
export SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
unset ASCEND_LAUNCH_BLOCKING
export SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1
export SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_RAGGED_VERIFY_MODE=static
export SGLANG_DSPARK_FOLDED_PROPOSAL=0
export SGLANG_DSPARK_FOLDED_SAMPLING=0
export SGLANG_DSPARK_STACKED_CTX_KV=0
export SGLANG_DSPARK_EMBED_IN_GRAPH=0
# use fias v2
export SGLANG_NPU_USE_FIAS_V2_BSND=1
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=0
#source /home/z00842572/CANN/cann/set_env.sh
#source /home/z00842572/CANN/nnal/atb/set_env.sh
export ASCEND_LAUNCH_BLOCKING=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64
export DEEPEP_HCCL_BUFFSIZE=1200
export HCCL_OP_EXPANSION_MODE=AIV
export ASCEND_USE_FIA=1
export SGLANG_W4A8_MXFP4_MOE=1
#export DEEP_USE_MODE="alltoall"
export SGLANG_TORCH_PROFILER_DIR="./sglang_profile/"
export SGLANG_PROFILE_WITH_STACK=False
export SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1
export GLOO_SOCKET_IFNAME="data0.173"
# 启动服务
python -m sglang.launch_server \
--model-loader-extra-config '{"enable_multithread_load": true}' \
--model-path /mnt/share/recipes/kimi/kimi_k3_27 \
--tokenizer-path /mnt/share/recipes/kimi/kimi_k3_27 \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--dtype bfloat16 \
--mamba-ssm-dtype bfloat16 \
--tp-size 32 \
--enable-dp-attention \
--dp-size 1 \
--enable-dp-lm-head \
--max-mamba-cache-size 160 \
--mem-fraction-static 0.9 \
--chunked-prefill-size 2048 \
--disable-cuda-graph \
--max-running-requests 32 \
--max-total-tokens 200000 \
--enable-shared-experts-attn-tp \
--enable-dense-mlp-attn-tp \
--moe-a2a-backend deepep \
--deepep-mode auto \
--speculative-algorithm DSPARK \
--speculative-draft-model-path /mnt/share/recipes/kimi/RadixArk-Kimi-K3-DSpark \
--speculative-draft-model-quantization unquant \
--speculative-dspark-block-size 7 \
--speculative-draft-attention-backend ascend \
--linear-attn-verify-backend triton \
--speculative-eagle-topk 1 \
--watchdog-timeout 9000 \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 4 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://141.61.94.103:5007", "config_store_url":"tcp://141.61.94.103:6007", "log_level":"info", "world_size":32, "protocol": "device_urma", "dram_size": "20GB"}'