• cpu i5-10600k

• 主板 Z490 arous pro ax

• 内存 16+8+16+8 共48G

• 显卡 RTX2080ti 22G x2 nvlink （300瓦bios水冷）

推理框架：自编译llama.cpp docker镜像(官方镜像不支持nccl通信)
dockerfile

# syntax=docker/dockerfile:1
# CUDA 13.0.3 + NCCL build for RTX 2080 Ti (SM 75) / llama.cpp tensor parallelism

ARG CUDA_VERSION=13.0.3
ARG UBUNTU_VERSION=24.04
ARG LLAMA_CPP_REF=b10362

FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build

ARG LLAMA_CPP_REF
ARG CUDA_ARCH=75
ARG BUILD_JOBS=4

ENV DEBIAN_FRONTEND=noninteractive \
    CC=gcc-13 \
    CXX=g++-13 \
    CUDAHOSTCXX=g++-13

# The CUDA Ubuntu repository in the NVIDIA CUDA base image supplies libnccl-dev.
RUN apt-get update && apt-get install -y --no-install-recommends \
        ca-certificates git cmake make \
        gcc-13 g++-13 \
        libnccl-dev \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /src
RUN git clone --depth 1 --branch "${LLAMA_CPP_REF}" https://github.com/ggml-org/llama.cpp.git .

RUN cmake -S . -B build \
        -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH} \
        -DGGML_NATIVE=OFF \
        -DGGML_CUDA=ON \
        -DGGML_CUDA_NCCL=ON \
        -DGGML_BACKEND_DL=ON \
        -DLLAMA_BUILD_TESTS=OFF \
        -DLLAMA_BUILD_EXAMPLES=ON \
    # CMake's install manifest includes the enabled example binaries.  Build all
    # enabled targets before installing, rather than just llama-server.
    && cmake --build build -j${BUILD_JOBS} \
    && cmake --install build --prefix /opt/llama

# The user has the CUDA devel image locally, so use it as the runtime base too.
# Change this to nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
# once that runtime image is available locally.
FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS server

ENV DEBIAN_FRONTEND=noninteractive \
    LLAMA_ARG_HOST=0.0.0.0 \
    LD_LIBRARY_PATH=/opt/llama/lib:${LD_LIBRARY_PATH}

RUN apt-get update && apt-get install -y --no-install-recommends \
        ca-certificates curl libgomp1 libnccl2 \
    && rm -rf /var/lib/apt/lists/*

COPY --from=build /opt/llama /opt/llama
RUN ldconfig \
    && ldd /opt/llama/bin/llama-server | grep -E 'nccl|not found' || true

WORKDIR /opt/llama
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
    CMD curl -fsS http://localhost:8080/health || exit 1

ENTRYPOINT ["/opt/llama/bin/llama-server"]

 docker build --no-cache --progress=plain   -f Dockerfile.llama-cpp-cuda130-nccl   -t llama.cpp:cuda13.0.3-nccl-sm75   --build-arg LLAMA_CPP_REF=master   --build-arg CUDA_ARCH=75   --build-arg BUILD_JOBS=4 .

-m /models/Qwen3.8-27B.Q6_K.gguf --alias Qwen3.8-27B -mm /models/Qwen3.8-27B.mmproj-q8_0.gguf --host 0.0.0.0 --port 8080  --fit off  -c 262144 -np 2 -ngl -1 --threads 6 --threads-batch 12 --split-mode tensor --flash-attn on --spec-type draft-mtp --spec-draft-n-max 3 --no-context-shift --jinja --cache-type-k q8_0 --cache-type-v q8_0 -cram 12288 -ctxcp 32 -kvu --temp 0.6 --top-p 0.5 --top-k 15 --chat-template-kwargs "{\"enable_thinking\":true,\"preserve_thinking\":false,\"reasoning_effort\":\"medium\"}" --api-key xxxx --no-ui  

2080 Ti 22GB ×2 性能测试

并发数：1｜超时：30s

提示词长度 预填充耗时 预填充速度 输出长度 输出耗时 输出速度
2,048 tokens 2,266.30 ms 903.68 tokens/s 512 tokens 8,504.73 ms 60.20 tokens/s
4,096 tokens 4,151.52 ms 986.63 tokens/s 512 tokens 7,855.27 ms 65.18 tokens/s
6,144 tokens 6,126.74 ms 1,002.82 tokens/s 512 tokens 8,005.43 ms 63.96 tokens/s
8,192 tokens 8,136.60 ms 1,006.81 tokens/s 512 tokens 8,021.08 ms 63.83 tokens/s
10,240 tokens 10,208.45 ms 1,003.09 tokens/s 512 tokens 6,271.73 ms 81.64 tokens/s

所有测试均为 1/1 并发成功。

总体性能
**预填充吞吐范围：**903.68～1,006.81 tokens/s
**输出吞吐范围：**60.20～81.64 tokens/s
**平均预填充吞吐：**980.61 tokens/s
**平均输出吞吐：**66.96 tokens/s

2080 Ti 22GB ×2 性能测试

并发数：2｜超时：30s

提示词长度 预填充耗时 总预填充速度 输出长度 输出耗时 总输出速度
2,048 tokens 3,852.35 ms 1,063.25 tokens/s 512 tokens 12,374.84 ms 82.75 tokens/s
4,096 tokens 7,779.40 ms 1,053.04 tokens/s 512 tokens 13,557.93 ms 75.53 tokens/s
6,144 tokens 12,875.19 ms 954.39 tokens/s 512 tokens 12,327.89 ms 83.06 tokens/s
8,192 tokens 17,385.96 ms 942.37 tokens/s 512 tokens 13,142.39 ms 77.92 tokens/s
10,240 tokens 14,149.75 ms 1,447.38 tokens/s 512 tokens 16,787.24 ms 61.00 tokens/s

所有测试均为 2/2 并发成功。

总体性能
**总预填充吞吐范围：**942.37～1,447.38 tokens/s
**总输出吞吐范围：**61.00～83.06 tokens/s
**平均总预填充吞吐：**1,092.09 tokens/s
**平均总输出吞吐：**76.05 tokens/s