[A3] Kimi K3四机混部 - shun8686/sglang GitHub Wiki
服务启动参数 - A3
如下配置在蓝区4机（209 212 216 217）可以拉起来，B072
#!/bin/bash
set -o pipefail
cd -- "$(dirname -- "${BASH_SOURCE[0]}")" || exit 1
mkdir -p logs || exit 1
export PYTHONUNBUFFERED=1
export SGLANG_DEBUG_NPU_BATCH_TIMING="${SGLANG_DEBUG_NPU_BATCH_TIMING:-1}"
export SGLANG_LOG_FORWARD_ITERS="${SGLANG_LOG_FORWARD_ITERS:-1}"
export SGLANG_LOG_MS="${SGLANG_LOG_MS:-1}"
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=10
sysctl -w kernel.numa_balancing=0
sysctl -w kernel.sched_migration_cost_ns=50000
MODEL_PATH=/home/weights/Kimi-K3-w4a8-int-moe
DRAFT_MODEL_PATH=/home/weights/Kimi-K3-DSpark
CUDA_GRAPH_BS="${CUDA_GRAPH_BS:-2 4 8 12 16}"
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
# Discard inherited code overrides before CANN restores its Python dependencies.
unset PYTHONPATH
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
# profiling
# export SGLANG_NPU_PROFILING=1
export SGLANG_NPU_PROFILING_BS=4
export SGLANG_NPU_PROFILING_PATH="/home/hanwlax/workspace/progress/kimi_k3/profiling"
# A/B
export SGLANG_NPU_USE_FUSED_KDA_EXTEND=1 # 1 means new ops, 0 means old correct ops (for ascendC correctness, local changes), accurracy regressionon A3 (ascencC kda prefill)
# MXFP8 quant (for A5)
export SGLANG_NPU_MOE_SITU_MXFP8_FUSED=0
# fias v2
export SGLANG_NPU_USE_FIAS_V2_BSND=0 # v2 has accpect len regression, need to be fixed
# Experts fine-grained dual stream
export SGLANG_NPU_FINE_GRAINED_MOE_DUAL_STREAM=0
# PYTHONPATH
CODEPATH="/home/hanwlax/test-codes/k3"
SGLANG_PATH="${CODEPATH}/sglang-nz/python"
KERNEL_PATH="${CODEPATH}/sgl-kernel-npu/python/sgl_kernel_npu"
#export PYTHONPATH="${SGLANG_PATH}${KERNEL_PATH:+:${KERNEL_PATH}}${PYTHONPATH:+:${PYTHONPATH}}"
export TRITON_CACHE_DIR="/root/.cache/sglang/triton"
D_IP=('192.168.25.209' '192.168.25.212' '192.168.25.216' '192.168.25.217')
LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'`
LOCAL_HOST2=`hostname -I|awk -F " " '{print$2}'`
echo "${LOCAL_HOST1}"
echo "${LOCAL_HOST2}"
for i in "${!D_IP[@]}";
do
if [ "$LOCAL_HOST1" == "${D_IP[$i]}" ](/shun8686/sglang/wiki/|-"$LOCAL_HOST2"-==-"${D_IP[$i]}"-);
then
echo "Mixed -> ${D_IP[$i]}"
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS=1
export SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1
export SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1
export SGLANG_K3_DENSE_MLP_ATTN_TP=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_RAGGED_VERIFY_MODE=static
export SGLANG_DSPARK_FOLDED_PROPOSAL=0
export SGLANG_DSPARK_FOLDED_SAMPLING=0
export SGLANG_DSPARK_STACKED_CTX_KV=0
export SGLANG_DSPARK_EMBED_IN_GRAPH=0
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export HCCL_SOCKET_IFNAME=enp196s0f0
export GLOO_SOCKET_IFNAME=enp196s0f0
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export HCCL_BUFFSIZE=2000
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_BUFFSIZE=200
export DEEPEP_HCCL_BUFFSIZE=1800
unset ASCEND_CUSTOM_OPP_PATH
unset SGLANG_NPU_FUSED_MOE_MODE
unset ENABLE_PROFILING
unset SGLANG_K3_TRACE_STATE_FILE
unset SGLANG_K3_TRACE_HIDDEN_FILE
unset ASCEND_RT_VISIBLE_DEVICES
unset ASCEND_LAUNCH_BLOCKING
sglang serve \
--model-loader-extra-config '{"enable_multithread_load": true}' \
--dist-init-addr 192.168.25.209:5000 --nnodes 4 --node-rank $i \
--model-path $MODEL_PATH \
--tokenizer-path $MODEL_PATH \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--mamba-ssm-dtype bfloat16 \
--dtype bfloat16 \
--tp-size 64 \
--enable-dp-attention --dp-size 4 --enable-dp-lm-head \
--mem-fraction-static 0.75 \
--max-total-tokens 250000 \
--max-mamba-cache-size 240 \
--chunked-prefill-size 20480 \
--cuda-graph-bs-decode $CUDA_GRAPH_BS \
--reasoning-parser kimi_k3 \
--max-running-requests 64 \
--host 0.0.0.0 \
--port 30000 \
--moe-a2a-backend deepep \
--enable-shared-experts-attn-tp \
--enable-dense-mlp-attn-tp \
--deepep-mode auto \
--speculative-draft-model-quantization unquant \
--speculative-algorithm DSPARK \
--speculative-draft-model-path "$DRAFT_MODEL_PATH" \
--speculative-dspark-block-size 7 \
--speculative-draft-attention-backend ascend \
--speculative-eagle-topk 1 \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 2 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://192.168.25.209:5007", "config_store_url":"tcp://192.168.25.209:6007", "log_level":"info", "world_size":256, "protocol": "device_sdma", "dram_size": "1GB"}' \
--watchdog-timeout 9000 2>&1 | tee "logs/run_32p_mix_$(date +%Y-%m-%d_%H-%M-%S).log"
exit "${PIPESTATUS[0]}"
fi
done
exit 1
服务启动参数 - A3 - 极光
#!/bin/bash
pkill -9 python 2>/dev/null || true
pkill -9 sglang 2>/dev/null || true
pkill -9 VLLM 2>/dev/null || true
export SGLANG_NPU_HICACHE_MAMBA_IO=async
export PYTHONUNBUFFERED=1
export SGLANG_DEBUG_NPU_BATCH_TIMING=1
export SGLANG_LOG_FORWARD_ITERS=1
export SGLANG_LOG_MS=1
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=10
sysctl -w kernel.numa_balancing=0
# sysctl -w kernel.sched_migration_cost_ns=50000
unset https_proxy
unset http_proxy
unset HTTPS_PROXY
unset HTTP_PROXY
# Discard inherited code overrides before CANN restores its Python dependencies.
unset PYTHONPATH
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
# A/B
export SGLANG_NPU_USE_FUSED_KDA_EXTEND=0 # 1 means new ops, 0 means old correct ops (for ascendC correctness, local changes), accurracy regressionon A3 (ascencC kda prefill)
# MXFP8 quant (for A5)
export SGLANG_NPU_MOE_SITU_MXFP8_FUSED=0
# fias v2
export SGLANG_NPU_USE_FIAS_V2_BSND=0 # v2 has accpect len regression, need to be fixed
# Experts fine-grained dual stream
export SGLANG_NPU_FINE_GRAINED_MOE_DUAL_STREAM=0
export TRITON_CACHE_DIR="/root/.cache/sglang/triton"
export SGLANG_SET_CPU_AFFINITY=1
export SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS=1
export SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1
export SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1
export SGLANG_K3_DENSE_MLP_ATTN_TP=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_RAGGED_VERIFY_MODE=static
export SGLANG_DSPARK_FOLDED_PROPOSAL=0
export SGLANG_DSPARK_FOLDED_SAMPLING=0
export SGLANG_DSPARK_STACKED_CTX_KV=0
export SGLANG_DSPARK_EMBED_IN_GRAPH=0
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export HCCL_BUFFSIZE=2000
export DEEPEP_NORMAL_LONG_SEQ_ROUND=64
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_BUFFSIZE=200
export DEEPEP_HCCL_BUFFSIZE=1800
unset ASCEND_CUSTOM_OPP_PATH
unset SGLANG_NPU_FUSED_MOE_MODE
unset ENABLE_PROFILING
unset SGLANG_K3_TRACE_STATE_FILE
unset SGLANG_K3_TRACE_HIDDEN_FILE
unset ASCEND_LAUNCH_BLOCKING
python3 -m sglang.launch_server \
--max-total-tokens 65536 \
--chunked-prefill-size 65536 \
--model-path /home/weights/Kimi-K3-W4A8 \
--tokenizer-path /home/weights/Kimi-K3-W4A8 \
--model-loader-extra-config '{"enable_multithread_load": true}' \
--trust-remote-code \
--attention-backend ascend \
--device npu \
--quantization modelslim \
--mamba-ssm-dtype bfloat16 \
--dtype bfloat16 \
--tp-size 64 \
--enable-dp-attention --dp-size 4 --enable-dp-lm-head \
--enable-shared-experts-attn-tp --enable-dense-mlp-attn-tp \
--mem-fraction-static 0.75 \
--max-mamba-cache-size 240 \
--cuda-graph-bs-decode 2 4 8 12 16 \
--reasoning-parser kimi_k3 \
--max-running-requests 64 \
--moe-a2a-backend deepep \
--enable-shared-experts-attn-tp \
--enable-dense-mlp-attn-tp \
--cuda-graph-backend-decode disabled \
--deepep-mode auto \
--watchdog-timeout 9000 \
--enable-hierarchical-cache \
--hicache-io-backend kernel_ascend \
--enable-cache-report \
--hicache-ratio 2 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://80.5.17.37:5007", "config_store_url":"tcp://80.5.17.37:6007", "log_level":"info", "world_size":256, "protocol": "device_sdma", "dram_size": "1GB"}'