仅需10万不到，DeepSeek R1 671B大模型本地部署实战指南

git clone https://github.com/kvcache-ai/ktransformers.git
cd ktransformers 
git submodule init 
git submodule update 
git checkout 7a19f3b git rev-parse --short HEAD #应显示 7a19f3b
1.
2.
3.
4.
5.

[submodule "third_party/llama.cpp"]
        path = third_party/llama.cpp
        url = https://github.com/ggerganov/llama.cpp.git
[submodule "third_party/pybind11"]
        path = third_party/pybind11
        url = https://github.com/pybind/pybind11.git
1.
2.
3.
4.
5.
6.

modelscope download unsloth/DeepSeek-R1-GGUF --include "DeepSeek-R1-Q4_K_M/*" --cache_dir /home/user/new/models
1.

curl -LsSf https://astral.sh/uv/install.sh | sh
1.

uv venv ./venv --python 3.11 --python-preference=only-managedsource  venv/bin/activate
1.

$ uv pip install flashinfer-python
1.

$ export TORCH_CUDA_ARCH_LIST="8.6"
uv pip install https://github.com/ubergarm/ktransformers/releases/download/7a19f3b/ktransformers-0.2.2rc1+cu120torch26fancy.amd.ubergarm.7a19f3b.flashinfer-cp311-cp311-linux_x86_64.whl
1.
2.

uv pip install https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.0.5/flash_attn-2.6.3+cu124torch2.6-cp311-cp311-linux_x86_64.whl
1.

支持多GPU配置及通过 `--optimize_config_path` 进行更细粒度的显存卸载设置
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True python3 ktransformers/server/main.py      
--gguf_path /mnt/ai/models/unsloth/DeepSeek-R1-GGUF/DeepSeek-R1-UD-Q2_K_XL/     
--model_path deepseek-ai/DeepSeek-R1 
--model_name unsloth/DeepSeek-R1-UD-Q2_K_XL 
--cpu_infer 16 
--max_new_tokens 8192 
--cache_lens 32768 
--total_context 32768
--cache_q4 true
--temperature 0.6 
--top_p 0.95 
--optimize_config_path ktransformers/optimize/optimize_rules/DeepSeek-R1-Chat.yaml 
--force_think 
--use_cuda_graph 
--host 127.0.0.1 
--port 8080
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.

# 安装额外编译依赖项，包括CUDA工具链等，例如：
# sudo apt-get install build-essential cmake ...
source venv/bin/activate
uv pip install -r requirements-local_chat.txt
uv pip install setuptools wheel packaging
# 建议跳过可选网站应用，使用`open-webui`或`litellm`等替代方案
cd ktransformers/website/
npm install @vue/cli
npm run build
cd ../..
# 如果拥有充足的CPU核心和内存资源，可显著提升构建速度
# $ export MAX_JOBS=8
# $ export CMAKE_BUILD_PARALLEL_LEVEL=8
# 安装flash_attn
uv pip install flash_attn --no-build-isolation
# 可选实验性使用flashinfer替代triton
# 除非您是已经成功上手的进阶用户，否则暂不建议使用
# 使用以下命令安装：
# $ uv pip install flashinfer-python
# 仅适用于以下情况：
# 配备Intel双路CPU且内存>1TB可容纳两份完整模型内存副本(每路CPU一份副本)
# AMD EPYC NPS0双路平台可能无需此配置？
# $ export USE_NUMA=1
# 安装ktransformers
KTRANSFORMERS_FORCE_BUILD=TRUE uv pip install . --no-build-isolation
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.

KTRANSFORMERS_FORCE_BUILD=TRUE uv build
1.

uv pip install ./dist/ktransformers-0.2.2rc1+cu120torch26fancy-cp311-cp311-linux_x86_64.whl
1.

ktransformers --model_path /home/user/new/ktran0.2.2/ktransformers/models/deepseek-ai/DeepSeek-R1  --gguf_path /home/user/new/models/unsloth/DeepSeek-R1-GGUF/DeepSeek-R1-Q4_K_M   --port 8080
1.

/tmp/cc8uoJt1.s:23667: Error: no such instruction: `vpdpbusd %ymm3,%ymm15,%ymm1'
的报错，
File "<string>", line 327, in build_extension     
File "/usr/local/python3/lib/python3.11/subprocess.py", 
line 571, in run       
raise CalledProcessError(retcode, process.args,   subprocess.CalledProcessError: 
Command '['cmake', '--build', '.', '--verbose', '--parallel=128']' 
returned non-zero exit status 1.   [end of output]
1.
2.
3.
4.
5.
6.
7.
8.

-DLLAMA_NATIVE=OFF -DLLAMA_AVX=ON -DLLAMA_AVX2=ON -DLLAMA_AVX512=OFF -DLLAMA_AVXVNNI=OFF
1.