[A3] DeepSeek V4 Flash单机混部 - shun8686/sglang GitHub Wiki
服务启动参数 - A3
26.2 B100在61.47.19.66上测试通过
#!/bin/bash
# ==============================================
# DeepSeek-V4-Flash W8A8 单机 8 卡 PD-mix 启动脚本
# ==============================================
# 清理代理设置
unset http_proxy
unset https_proxy
unset HTTP_PROXY
unset HTTPS_PROXY
unset no_proxy
# 系统性能调优
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl -w vm.swappiness=0
sysctl -w kernel.numa_balancing=0
# 加载昇腾环境
source /usr/local/Ascend/ascend-toolkit/set_env.sh
source /usr/local/Ascend/nnal/atb/set_env.sh
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash
source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash
# ========== 环境变量（来自测试用例 DEEPSEEK_V4_FLASH_W8A8_8P_ENVS）==========
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export STREAMS_PER_DEVICE=32
export INF_NAN_MODE_FORCE_DISABLE=1
export SGLANG_SET_CPU_AFFINITY=1
export HCCL_SOCKET_IFNAME=lo
export GLOO_SOCKET_IFNAME=lo
export HCCL_OP_EXPANSION_MODE=AIV
# deepep 相关
export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128
export DEEPEP_HCCL_BUFFSIZE=1000
export DEEP_NORMAL_MODE_USE_INT8_QUANT=1
export DEEPEP_NORMAL_LONG_SEQ_ROUND=16
export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=2048
export DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ=1
# 跳过 GPU 分支优化
export SGLANG_OPT_FP8_WO_A_GEMM=0
export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=False
export FORCE_DRAFT_MODEL_NON_QUANT=1
export SGLANG_DSV4_FP4_EXPERTS=False
export SGLANG_OPT_FUSE_WQA_WKV=0
export SGLANG_OPT_BF16_FP32_GEMM_ALGO=torch
export SGLANG_OPT_USE_FUSED_HASH_TOPK=False
export SGLANG_OPT_USE_TILELANG_MHC_PRE=False
export SGLANG_OPT_DEEPGEMM_HC_PRENORM=False
export SGLANG_OPT_USE_TILELANG_MHC_POST=False
# MTP相关
export SGLANG_ENABLE_SPEC_V2=1
export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
export SGLANG_DSPARK_FAST_KERNEL=0
export SGLANG_DSPARK_FAST_SAMPLING=0
# ========== 参考脚本中的额外稳定/性能环境变量（可选）==========
#export SGLANG_ENABLE_WAR_BARRIER=1
#export SGLANG_FORCE_COARSE_WAR_BARRIER=1
export SGLANG_ENABLE_WAR_BARRIER=1
export SGLANG_FORCE_COARSE_WAR_BARRIER=1
# ========== 模型路径 ==========
# 请根据实际路径修改，或通过环境变量 MODEL_PATH 传入
MODEL_PATH="${MODEL_PATH:-/mnt/paas/weights/DeepSeek-V4-Flash-0731-w8a8}"
export DEEPEP_HYBRID_DEPLOYMENT=1
# 设置服务 IP 和端口
IP=("${SGLANG_HOST_IP:-0.0.0.0}")
PORT="${SGLANG_PORT:-30000}"
python3 -m sglang.launch_server \
--model-path "${MODEL_PATH}" \
--page-size 128 \
--tp-size 16 \
--trust-remote-code \
--device npu \
--attention-backend dsv4 \
--watchdog-timeout 9000 \
--host "${IP[0]}" \
--port "${PORT}" \
--mem-fraction-static 0.62 \
--max-total-tokens 65536 \
--chunked-prefill-size 65536 \
--max-running-requests 160 \
--dp-size 16 \
--enable-dp-attention \
--moe-a2a-backend deepep \
--deepep-mode auto \
--quantization modelslim \
--enable-dp-lm-head \
--kv-cache-dtype bfloat16 \
--cuda-graph-bs-decode 1 2 4 8 10 \
--speculative-algorithm DSPARK \
--speculative-draft-model-path "${MODEL_PATH}" \
--speculative-draft-model-quantization modelslim \
--speculative-draft-attention-backend ascend \
--speculative-num-draft-tokens 6 \
--enable-hierarchical-cache \
--enable-cache-report \
--hicache-io-backend kernel_ascend \
--hicache-ratio 1.5 \
--hicache-write-policy write_through \
--hicache-storage-backend npu_memcache \
--hicache-storage-backend-extra-config '{"meta_service_url":"tcp://127.0.0.1:5007", "config_store_url":"tcp://127.0.0.1:6007", "log_level":"info", "world_size":16, "protocol": "device_sdma", "dram_size": "10GB"}' \
2>&1 | tee sglang_server__$(date +%Y%m%d_%H%M%S).log
benchserving
MODEL_PATH="${MODEL_PATH:-/home/weights/DeepSeek-V4-Flash-0731-w8a8}"
python3 -m sglang.benchmark.serving \
--host 127.0.0.1 \
--port 30000 \
--backend sglang \
--dataset-name generated-shared-prefix \
--backend sglang \
--gsp-num-groups 3 \
--gsp-prompts-per-group 32 \
--gsp-system-prompt-len 28800 \
--gsp-question-len 3200 \
--gsp-output-len 1024 \
--max-concurrency 32 \
--random-range-ratio 1 \
--seed 100 \
--model $MODEL_PATH \
--tokenizer $MODEL_PATH \
--warmup-requests 16 \
--cache-report
----------------Cache Hit Details-----------------
Total prompt tokens: 3142361
Total cached tokens: 2154496
Device: 1224704
Host: 626688
Storage (NpuMemcacheStore): 303104
Cache hit rate: 68.6%
Device: 56.8%
Host: 29.1%
Storage (NpuMemcacheStore): 14.1%
==================================================
精度
MODEL_PATH=/home/weights/DeepSeek-V4-Flash-0731-w8a8
evalscope eval \
--model ${MODEL_PATH} \
--api-url http://127.0.0.1:30000/v1 \
--api-key EMPTY \
--eval-type openai_api \
--generation-config '{"max_tokens":120000,"top_p":1,"temperature":1,"timeout":6000,"stream":true,"extra_body":{"chat_template_kwargs":{"thinking":true}}}' \
--datasets gpqa_diamond \
--dataset-args '{"gpqa_diamond":{"local_path":"/home/d00662834/datasets/gpqa","subset_list":["gpqa_diamond"]}}' \
--eval-batch-size 32 \
--ignore-errors \
--work-dir ./gpqa/
┌─────────────────────────────┬──────────────┬────────────┬──────────────┬───────┬─────────┐
│ Model │ Dataset │ Metric │ Subset │ Num │ Score │
├─────────────────────────────┼──────────────┼────────────┼──────────────┼───────┼─────────┤
│ DeepSeek-V4-Flash-0731-w8a8 │ GPQA-Diamond │ Accuracy ↑ │ gpqa_diamond │ 198 │ 90.4% │
└─────────────────────────────┴──────────────┴────────────┴──────────────┴───────┴─────────┘
服务启动参数 - A3 - 极光