# ninfer-serve.sh
#!/bin/bash
# NInfer host control: ninfer-serve.sh start | stop | status
# Binary reports usage.prompt_tokens_details.cached_tokens (OpenAI chat-completions),
# so DSH shows real KV prefix cache hit % instead of 0%.
# Default = text-only at 262K context (full int8 KV pool, ~10.45 GiB).
# Vision is optional: VISION=1 ninfer-serve.sh start switches to 192K context + --vision
# with media buffers trimmed from defaults (1G+2G) to 256M+512M so the fixed Vision
# buffers fit the ~10.4 GiB free after weights on the 5090 (at 262K they do not).
# C=2: median agent prompt scales to ~86K at 192K; two frontiers (~172K) fit the pool,
# so requests reuse their frontier (97-99% cached).
set -u
PORT=18080
LOG=/home/user/ninfer_serve.log
APP=/home/user/ninfer-build/apps/ninfer-serve
ART=/home/user/ninfer_artifacts/qwen3_8_27b_nvfp4.ninfer

if [ "${VISION:-0}" = "1" ]; then
  CTX=196608
  VISION_FLAGS="--vision --media-cache-mib 256 --media-live-mib 512"
else
  CTX=262144
  VISION_FLAGS=""
fi

is_running() { pgrep -f "ninfer-serve.*qwen3_8_27b" >/dev/null 2>&1; }

case "${1:-start}" in
  start)
    if is_running; then
      echo "already running (pid $(pgrep -of 'ninfer-serve.*qwen3_8_27b'))"
      exit 0
    fi
    export LD_LIBRARY_PATH=/home/user/ninfer_deps/prefix/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}
    cd /home/eason || exit 1
    setsid nohup "$APP" "$ART" \
      --host 0.0.0.0 --port $PORT --device 0 \
      --max-context $CTX --prefill-chunk 1024 --kv-dtype int8 \
      $VISION_FLAGS \
      --max-concurrency 2 --pending-timeout-ms 600000 \
      --spec mtp --draft-tokens 3 --lm-head-draft >>"$LOG" 2>&1 &
    for i in $(seq 1 30); do
      sleep 5
      if curl -s http://127.0.0.1:$PORT/health | grep -q ok; then
        echo "NInfer up: pid $(pgrep -of 'ninfer-serve.*qwen3_8_27b'), ctx=$CTX vision=${VISION:-0}, log $LOG"
        exit 0
      fi
    done
    echo "server did not become healthy in 150s; last log lines:"; tail -5 "$LOG"; exit 1
    ;;
  stop)
    if ! is_running; then echo "not running"; exit 0; fi
    pkill -f "ninfer-serve.*qwen3_8_27b"
    for i in $(seq 1 10); do sleep 2; is_running || break; done
    if is_running; then echo "still running (try: fuser -k $PORT/tcp)"; exit 1; fi
    echo "stopped"
    ;;
  status)
    if is_running; then
      echo "running pid $(pgrep -of 'ninfer-serve.*qwen3_8_27b')"; curl -s http://127.0.0.1:$PORT/health; echo
    else
      echo "not running"
    fi
    ;;
  *) echo "usage: $0 start|stop|status (VISION=1 for vision mode)"; exit 2;;
esac

記得把{ip}換掉

:: start.bat
@echo off
setlocal
title NInfer Start

rem --- self-elevate (netsh portproxy + firewall need admin) ---
net session >nul 2>&1
if %errorlevel% neq 0 (
  echo Requesting administrator rights...
  powershell -NoProfile -Command "Start-Process '%~f0' -Verb RunAs"
  exit /b
)

echo [1/4] Starting NInfer server in WSL Ubuntu (port 18080, MTP3 on)...
wsl -d Ubuntu -- bash /mnt/c/Users/user/Desktop/ninfer-serve.sh start
if errorlevel 1 goto :fail

for /f "usebackq" %%i in (`wsl -d Ubuntu -- hostname -I`) do set WSL_IP=%%i
echo [2/4] Repairing LAN portproxy: 0.0.0.0:18080 -> %WSL_IP%:18080 ...
netsh interface portproxy delete v4tov4 listenaddress=0.0.0.0 listenport=18080 >nul 2>&1
netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=18080 connectaddress=%WSL_IP% connectport=18080

echo [3/4] Ensuring firewall rule ...
powershell -NoProfile -Command "if (-not (Get-NetFirewallRule -DisplayName 'NInfer serve 18080' -ErrorAction SilentlyContinue)) { New-NetFirewallRule -DisplayName 'NInfer serve 18080' -Direction Inbound -Protocol TCP -LocalPort 18080 -Action Allow -Profile Private,Public | Out-Null; Write-Host 'firewall rule added' } else { Write-Host 'firewall rule exists' }"

echo [4/4] Verifying via LAN IP ...
powershell -NoProfile -Command "try { $r = Invoke-RestMethod -Uri http://{ip}:18080/health -TimeoutSec 8; Write-Host ('LAN OK: ' + ($r | ConvertTo-Json -Compress)) } catch { Write-Host ('LAN FAIL: ' + $_.Exception.Message) }"

echo.
echo Done. Server URL for other devices: http://{ip}:18080
echo Model qwen3.8-27b (nvfp4), MTP3 on, no vision, KV int8.
pause
exit /b 0

:fail
echo Start failed - check WSL log: wsl -d Ubuntu -- tail -20 /home/eason/ninfer_serve.log
pause

因為一張抵兩張，這下子又多一張5090可以利用了

1. prefill 稅不是 5090 獨有。論壇 TID:1010（Strix Halo）實測過同一現象：頻寬強/算力弱的平台，長 prompt prefill 才是瓶頸，decode 快救不回來。你的 98.6% 把「agent 負載 = prefill 負載」從個案變成規律——以後評估本地 agent 引擎，先看 prefill tok/s 和 KV 容量規劃，decode t/s 排第二。

2. 「C=2 最優」是 NInfer 的引擎特性，不是普適結論。你 log 裡 prefilling=1（prefill 串行）；vLLM 的 chunked prefill 會把 prefill/decode 交錯排程，同樣負載下最優 C 會不一樣。但「C × 中位 prompt ≤ KV pool」這條公式是普適的——它正是論壇 TID:988（vLLM prefix-cache hitrate 0.0%）的根因：pool 裝不下 frontier → 全逐出 → full_reset。你表裡 50K 以下 97-99% hit vs 160K 以上 0% 的斷崖，和 vLLM 的 LRU 逐出行為同構。

3. MTP 接受率 3.8 低於 3.6 與論壇數據一致：TID:1131 實測 Qwen3.8 AWQ 上 MTP/NEXTN 接受率崩潰；TID:1164 的 7900XTX Q4_K_M（n-max 5）實測 0.71-0.97。你 NVFP4 官方 90.8% / 實測 54-88% 證實一件事：接受率跟量化格式和 draft 設置強相關，不是引擎退化。第 9 節沒測的 groupwise-int profile 建議補——AWQ/int4 在 3.8 上接受率大概率往下掉，NVFP4 對 draft head 友好得多。

「縮 context 不能救 cache」這句完全同意，補個機制：agent 框架的 prompt 長度和 window 是正比的（工具結果、歷史全往裡塞），縮 window 只是把「塞滿」的閾值提前，命中率不變。真正能救的是 checkpoint + 穩定前綴——你第 6 節在做的事，方向完全正確。

一句話：這篇把「本地 agent 的瓶頸是 prefill 和 KV 規劃」釘死了，和論壇這個月的 KV 稅系列觀察閉環。歡迎多來幾篇這種測法寫清楚的。