Q5_K_M

UD-Q5_K_M

完整脚本如下, FYI:

bruin@lmde7 ~ $ cat run-model-3.8.sh
#!/bin/bash
set -uo pipefail

# TODO (progress as of 2026-08-23):
#
# 1. [DONE]  find all qwen3.8-27b models under /opt/gguf-models
#            -> 4 text models + 2 mmproj + 1 imatrix (see MODEL INVENTORY below)
# 2. [DONE]  pp/tg performance: extensive benchmark of each model (with
#            different KV quantization) at different context sizes (0/40/80K)
#            -> results in the MEASURED BENCHMARK MATRIX below + bench-3.8-sweep.sh
# 3. [DONE]  reproduce the "repeating loop" at high context size and provide
#            flag combinations to suppress it: --repeat-penalty, --dry-*,
#            --min-p, and the froggeric Qwen-Fixed-Chat-Templates chat template
#            -> bench-3.8-loop.sh reproduced it (rep_score 0.36 -> 0.18 w/ DRY)
# 4. [DONE]  max context size: explore the max servable context (<256KiB) with
#            24 GiB VRAM (7900 XTX) -> Ridge is the only model that holds q8_0
#            at 256K (88% VRAM); see MEASURED BENCHMARK MATRIX
# 5. [DONE]  update the following comments and script to list all possibilities
# 6. [DONE]  convert all command line options to long format (--xxxx) with
#            comments, for clarity.

# =============================================================================
# run-model-3.8.sh — Qwen3.8-27B model selector (llama.cpp Vulkan / RX 7900 XTX)
#
# Interactive menu launcher. For API-driven dynamic switching (harness lists
# models via GET /v1/models and picks one per request via the "model" field),
# use run-model-3.8-router.sh + qwen3.8-models.ini instead.
#
# Menu-driven launcher in the style of run-model.sh, now covering ALL four
# Qwen3.8-27B GGUF quantizations present on this box (see MODEL INVENTORY):
#   * Q4_K_M        (Abiray)    16.8 GiB file, ~15.3 GiB weights  [in menu]
#   * Ridge 3.7bpw  (empero-ai) 12.6 GiB file, ~11.9 GiB weights  [in menu]
#   * UD-Q4_K_M     (unsloth)   16.5 GiB file, dynamic quant      [in menu]
#   * Q5_K_S        (unsloth)   19.3 GiB file, ~17.5 GiB weights  [in menu]
#
# GOAL: maximize the servable context while keeping generation throughput (tg)
#       usable. Benchmarked 2026-08-23 on this machine with llama-benchy
#       (pp=2048, tg=128, --no-cache) against ~/llama-server-vulkan-b10485.
#       "tg" below = tokens generated / second (llama-benchy t_s_mean). See the
#       MEASURED BENCHMARK MATRIX for the full per-model / per-KV / per-depth
#       picture; results are reproducible with bench-3.8-sweep.sh.
#
# -----------------------------------------------------------------------------
# !! 320K IS NOT SERVABLE WITH THIS BINARY !!
#   llama-server b10485 hard-caps the slot at the model's native training
#   context (262144). Loading --ctx-size 327680 logs:
#     "the slot context (327680) exceeds the training context of the model
#      (262144) - capping"        ->   n_ctx_slot = 262144   (verified)
#   So a 320K entry would allocate KV for 327680 tokens but still only serve
#   262144, wasting ~1.2 GiB VRAM. The PRACTICAL ceiling is 262144 (256K).
#   To truly serve 320K you must patch server-context.cpp (remove the cap) and
#   rebuild llama-server; the ready YaRN flags for that are:
#     --rope-scaling yarn --rope-scale 1.25 --yarn-orig-ctx 262144
#   (see run-3.8-q4-320k.sh for the full story).
#
# -----------------------------------------------------------------------------
# MODEL INVENTORY (TODO #1) — every Qwen3.8-27B artifact under /opt/gguf-models:
#
#   text models                                    size         notes
#   -------------------------------------------    -----------  -----------------
#   Abiray/Qwen3.8-27B-Q4_K_M.gguf                 16.8 GiB     Q4_K_M (static)
#   empero-ai/Qwen3.8-27B-Ridge-3.7bpw.gguf        12.6 GiB     3.7bpw + imatrix
#   unsloth/Qwen3.8-27B-UD-Q4_K_M.gguf             16.5 GiB     Unsloth Dynamic Q4
#   unsloth/Qwen3.8-27B-Q5_K_S.gguf                19.3 GiB     Q5_K_S (static)
#
#   multimodal projectors (Qwen3.8-27B is a native VLM)
#   -------------------------------------------    -----------  -----------------
#   unsloth/mmproj-F16.gguf                         927 MiB     F16 vision tower
#   empero-ai/mmproj-Qwen3.8-27B-BF16.gguf          931 MiB     BF16 vision tower
#
#   unsloth/imatrix_unsloth.gguf                     13 MiB     imatrix data (NOT a model)
#
#   All four text models share the Qwen3.8-27B architecture (65 layers, only
#   every 4th block is full-attention; the rest are SSM/linear blocks with no
#   KV cache). Both mmproj files are interchangeable across the four text
#   models — add "--mmproj <path>" to serve vision. Enabling mmproj costs
#   ~1 GiB VRAM, so shave context accordingly if you want vision.
#
# -----------------------------------------------------------------------------
# MEASURED BENCHMARK MATRIX (re-measured 2026-08-23, llama-benchy:
#   pp=2048, tg=128, --no-cache, runs=1; tg = generation t/s, i.e. the
#   t_s_mean column). VRAM% is rocm-smi at idle@load (x 24.0 GiB => GiB).
#   tg@40K / tg@80K = generation t/s with 40K / 80K tokens already filled.
#   NOTE: tg=128 is a SHORT burst; long sustained generations run ~30-45%
#   faster once the GPU reaches full boost (e.g. Q4_K_M @256K q4_0 sustains
#   ~77 t/s on a 2K-token completion vs 52.9 t/s measured here).
#
#   model    ctx     KV     VRAM%  tg@0    tg@40K  tg@80K  pp@0    note
#   ------   ------- -----  -----  ------  ------  ------  -----   ---------------
#   Ridge    262144  q4_0    71%    55.3    50.7    38.9    538     BEST max-ctx
#   Q4_K_M   262144  q4_0    89%    52.9    43.1    38.3    546     max-ctx
#   UD-Q4    262144  q4_0    88%    54.0    48.3    35.6    504     max-ctx
#   Q5_K_S   262144  q4_0    94%    19.0     9.9     7.6    344     SPILLS (bad)
#   Ridge    131072  q8_0    67%    64.8    51.6    40.5    523     fastest + quality
#   Ridge    262144  q8_0    88%    60.3     TBD     TBD     528     quality KV @256K (fits!)
#   Q4_K_M   131072  q8_0    85%    58.7    46.4    37.2    522     quality KV @128K
#   UD-Q4    131072  q8_0    84%    52.9    43.5    35.0    513     quality KV @128K
#   Q5_K_S   131072  q8_0    93%    32.8    29.4    26.7    434     tight
#
#   q4_0 KV (~18 KiB/tok) is smaller AND faster than q8_0 (~33 KiB/tok), but
#   q8_0 holds more KV precision. Ridge's ~4 GiB smaller footprint means it
#   can afford q8_0 KV (quality) where the others must fall back to q4_0 — and
#   it is the ONLY model that holds q8_0 at the full 256K context (88% VRAM,
#   60.3 t/s), so Ridge @256K q8_0 is the quality+context champion.
#
#   tg falls as the context FILLS. e.g. Q4_K_M @256K q4_0: 52.9 -> 43.1 @40K
#   -> 38.3 @80K. Treat the tg@0 column as a ceiling; near-full runs slower.
#
# -----------------------------------------------------------------------------
# LOOP PREVENTION (TODO #3) — high-context self-repetition:
#   The original greedy sampling (temp 0.6 / top-p 0.5 / top-k 15 /
#   repeat-penalty 1.0 / DRY off) loops on long open-ended/reasoning tasks.
#   Measured 2026-08-23 (bench-3.8-loop.sh, n-gram repetition score, higher =
#   more stuck) on the hardest prompt ("explain thinking at length"):
#
#     sampling                              rep_score
#     ------------------------------------  ---------
#     baseline (no DRY, no template)        0.36
#     DRY only                              0.18      <- ~50% cut (the fix)
#     froggeric chat-template only          0.30      <- NO help here
#     DRY + template                        0.18      <- same as DRY only
#     DRY + --min-p 0.1                     0.16      <- no help (single-run noise)
#     DRY + --reasoning-budget 8192         0.17      <- no help
#     DRY + --reasoning-budget 0            0.14      <- marginal (thinking off)
#     DRY + min-p + budget                  0.15      <- no better than DRY
#
#   => the jinja chat template, --min-p and --reasoning-budget do NOT
#   meaningfully suppress text self-repetition (they target other things:
#   tool-call loops, distribution tails, thinking length). The DRY sampler is
#   the only effective knob — it halves the loop but does not fully kill it.
#   If you still need more (untested), escalate:
#     1. --dry-multiplier lower (e.g. 0.5)  stronger DRY penalty
#     2. --spec-type none                   rule out an MTP draft bug re-injecting text
#     3. lower --temp / --top-k             tamer sampling (flat distributions
#                                            repeat more)
#     (froggeric/Qwen-Fixed-Chat-Templates is still useful for TOOL-CALL loops:
#      --chat-template-file <jinja file>)
# =============================================================================

LLAMA_SERVER=/home/bruin/llama-server-vulkan-b10485
TIMEOUT=5
DEFAULT_MODEL=0

# =============================================================================
# FILE PATH VARIABLES
# =============================================================================
F_Q4KM="/opt/gguf-models/Abiray/Qwen3.8-27B-Q4_K_M-GGUF/Qwen3.8-27B-Q4_K_M.gguf"
F_RIDGE="/opt/gguf-models/empero-ai/Qwen3.8-27B-Ridge-3.7bpw/Qwen3.8-27B-Ridge-3.7bpw.gguf"
F_UDQ4="/opt/gguf-models/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-UD-Q4_K_M.gguf"
F_Q5KS="/opt/gguf-models/unsloth/Qwen3.8-27B-GGUF/Qwen3.8-27B-Q5_K_S.gguf"
F_MMPROJ_UNSLOTH="/opt/gguf-models/unsloth/Qwen3.8-27B-GGUF/mmproj-F16.gguf"
F_MMPROJ_EMPERO="/opt/gguf-models/empero-ai/Qwen3.8-27B-Ridge-3.7bpw/mmproj-Qwen3.8-27B-BF16.gguf"
F_FIXED_CHAT_TMPL="/opt/gguf-models/froggeric/Qwen-Fixed-Chat-Templates/chat_template.jinja"
# (Qwen3.8-27B is a native VLM; both mmproj files work with all four text
#  models. Text-only is the default; add --mmproj to a row's extra args to
#  serve vision — it costs ~1 GiB VRAM.)

# =============================================================================
# MODEL MATRIX
# Columns (| delimited):
#   0: Name        1: Main model       2: CTX     3: KV quant   4: Extra args
#
# Extra args is a whitespace-separated string appended verbatim to the final
# command line (e.g. "--alias foo --mmproj /path").
# =============================================================================
MODELS=(
  "Ridge 3.7bpw @256K (q8_0) - quality @256K 60 t/s |${F_RIDGE}|262144|q8_0|--alias qwen3.8-ridge-256k-q8"
  "Ridge 3.7bpw @256K (q4_0) - 55 t/s (more headroom)|${F_RIDGE}|262144|q4_0|--alias qwen3.8-ridge-256k"
  "Q4_K_M @256K (q4_0) - max context 53 t/s       |${F_Q4KM}|262144|q4_0|--alias qwen3.8-q4-256k"
  "UD-Q4_K_M @256K (q4_0) - dynamic 54 t/s        |${F_UDQ4}|262144|q4_0|--alias qwen3.8-udq4-256k"
  "Q5_K_S @192K (q4_0) - max Q5 clean ctx         |${F_Q5KS}|196608|q4_0|--alias qwen3.8-q5-192k"
  "Q4_K_M @128K (q8_0) - quality KV 59 t/s        |${F_Q4KM}|131072|q8_0|--alias qwen3.8-q4-128k"
  "Q5_K_S @64K  (q8_0) - fast/quality             |${F_Q5KS}|65536 |q8_0|--alias qwen3.8-q5-64k"
  "Q5_K_S @256K (q4_0) - Q5 max ctx (SPILLS 19 t/s)|${F_Q5KS}|262144|q4_0|--alias qwen3.8-q5-256k"
  # Uncomment once the server 262144 cap is patched out (needs rebuild):
  # "Q4_K_M @320K (q4_0) - YaRN, needs patched server |${F_Q4KM}|327680|q4_0|--alias qwen3.8-q4-320k --rope-scaling yarn --rope-scale 1.25 --yarn-orig-ctx 262144"
  # Vision variants (append --mmproj to any text model row; ~1 GiB VRAM cost):
  # "Q4_K_M @256K (q4_0) + vision mmproj           |${F_Q4KM}|262144|q4_0|--alias qwen3.8-q4-256k-v --mmproj ${F_MMPROJ_UNSLOTH}"
  # "Ridge 3.7bpw @256K (q4_0) + vision mmproj     |${F_RIDGE}|262144|q4_0|--alias qwen3.8-ridge-256k-v --mmproj ${F_MMPROJ_EMPERO}"
)

# =============================================================================
# BUILD MODEL ID LIST FROM MATRIX
# =============================================================================
declare -a MODEL_IDS=()
declare -A MODEL_NAMES=()

for i in "${!MODELS[@]}"; do
    MODEL_IDS+=("$i")
    IFS='|' read -r _name _ <<< "${MODELS[$i]}"
    MODEL_NAMES[$i]="$_name"
done

MODEL_COUNT="${#MODELS[@]}"

# =============================================================================
# GRUB-LIKE MENU WITH LIVE COUNTDOWN
# =============================================================================
clear
echo -e "\n=== Qwen3.8-27B Llama Server Model Selector ==="
printf "%-3s %-50s\n" "ID" "Model"
echo "----------------------------------------------------------"
for id in "${MODEL_IDS[@]}"; do
    mark="$([ "$id" -eq "$DEFAULT_MODEL" ] && echo "[*]" || echo "   ")"
    printf "%-3s %-50s %s\n" "$id" "${MODEL_NAMES[$id]}" "$mark"
done
echo "----------------------------------------------------------"

CHOICE=""
for ((t=TIMEOUT; t>0; t--)); do
    printf "\rSelect model [${DEFAULT_MODEL}] (timeout: %ds): " "$t"
    if read -r -t 1 -n 1 char 2>/dev/null; then
        [[ "$char" == $'\n' || "$char" == $'\r' ]] && continue
        if [[ " ${MODEL_IDS[*]} " == *" ${char} "* ]]; then
            CHOICE="$char"
            break
        fi
    fi
done
printf "\rSelect model [${DEFAULT_MODEL}] (timeout:  0s): "

CHOICE="${CHOICE:-$DEFAULT_MODEL}"
if [[ ! " ${MODEL_IDS[*]} " == *" ${CHOICE} "* ]]; then
    echo -e "\nInvalid/No selection. Using default model: ${DEFAULT_MODEL}"
    CHOICE=$DEFAULT_MODEL
fi

SELECTED_NAME="${MODEL_NAMES[$CHOICE]}"
echo -e "\n>> Loading: ${SELECTED_NAME}\n"

# =============================================================================
# EXTRACT CONFIG FROM MATRIX ROW
# =============================================================================
IFS='|' read -r _ MAIN_MODEL CTX_SIZE KV_QUANT EXTRA_ARGS <<< "${MODELS[$CHOICE]}"

MAIN_MODEL="$(<<<"${MAIN_MODEL}" xargs)"
CTX_SIZE="$(<<<"${CTX_SIZE}" xargs)"
KV_QUANT="$(<<<"${KV_QUANT}" xargs)"
EXTRA_ARGS="$(<<<"${EXTRA_ARGS}" xargs)"

# =============================================================================
# BUILD ARGUMENTS (tuned for Qwen3.8-27B / RX 7900 XTX)
# All options are long-format (--xxxx) for clarity; comments note the why.
# =============================================================================
ARGS=(
  # --- device & model -----------------------------------------------------
  --device             Vulkan0                       # GPU backend (RX 7900 XTX / RADV)
  --model              "${MAIN_MODEL}"               # main text model
  --ctx-size           "${CTX_SIZE}"                 # servable context window (tokens)
  --parallel           1                             # single sequence slot (max per-request ctx)

  # --- KV cache -----------------------------------------------------------
  --cache-type-k       "${KV_QUANT}"                 # K-cache quantization (q4_0/q8_0)
  --cache-type-v       "${KV_QUANT}"                 # V-cache quantization (q4_0/q8_0)
  --gpu-layers         -1                            # offload ALL layers to VRAM
  --flash-attn         on                            # flash attention (faster, less VRAM)

  # --- speculative decoding (MTP) ----------------------------------------
  --spec-type          draft-mtp                     # speculate with the model's MTP head
  --spec-draft-n-max   2                             # max draft tokens per step

  # --- compute / batching --------------------------------------------------
  --threads            10                            # CPU threads for generation
  --batch-size         512                           # logical prompt batch size
  --ubatch-size        256                           # physical micro-batch size

  # --- context handling -----------------------------------------------------
  --no-context-shift                                  # disable KV shifting (keep full ctx)

  # --- template & reasoning -------------------------------------------------
  --jinja                                              # use the model's jinja chat template
  --reasoning          on                             # enable <think> reasoning tokens
  --reasoning-effort   medium                         # thinking effort level
  --no-reasoning-preserve                             # strip old turns' reasoning (lean ctx)

  # --- serving ---------------------------------------------------------------
  --kv-unified                                         # unified KV (required for VLM layout)
  --host               0.0.0.0                        # listen on all interfaces
  --port               8000                           # OpenAI-compatible API port
  --metrics                                            # expose Prometheus /metrics
  --load-mode          none                           # no special mmap/mlock mode

  # --- sampling (loop-suppression set — see LOOP PREVENTION header) ---------
  --temp               0.6                            # temperature
  --top-p              0.5                            # nucleus sampling
  --top-k              15                             # top-k sampling
  --repeat-penalty     1.1                            # light anti-repetition penalty
  --dry-multiplier     0.8                            # DRY: exponential penalty on repeats
  --dry-base           1.75                           # DRY: base value (default)
  --dry-allowed-length 2                              # DRY: trigger length (default)
  # Escalate if loops persist (uncomment as needed):
  # --reasoning-budget 8192                            # cap thinking so a loop terminates
  # --min-p            0.02                            # floor tokens below p*max-prob
  # --chat-template-file "${F_FIXED_CHAT_TMPL}"        # froggeric fixed template
  # --spec-type        none                            # rule out an MTP draft bug
)

if [[ -n "${EXTRA_ARGS}" ]]; then
  read -ra EXTRA_SPLIT <<< "${EXTRA_ARGS}"
  ARGS+=("${EXTRA_SPLIT[@]}")
fi

# =============================================================================
# PRINT & EXECUTE
# =============================================================================
echo "Running: ${LLAMA_SERVER}"
for arg in "${ARGS[@]}"; do
  printf '  %s\n' "$arg"
done
echo "---"