分享下我的风狗策略，新手看的，看总有人坏卡
-
大佬勿喷，旨在防止卡过热，具体温度电压以及风扇控制代码需要以各自设备为准
#!/bin/bash
============================================================
GPU五态温度状态机（10秒巡检，仅状态翻转瞬间出手）
四卡同步：3x CMP 170HX + 1x V100(VM)
【静音态】 全部卡 < 40°C -> 110W + 锁800 + 风扇 0x20/0x25
【工作态】 上穿 45°C -> 130W + 解锁频率(rgc) + 风扇 0x2F/0x3F
【强启态】 任一卡 >= 60°C -> 130W + 风扇 0x3F/0x4F
【强冷态】 任一卡 >= 64°C -> 130W + 风扇拉满 0x64/0x64 （大声报警）
【回落】 强启/强冷后 <=50°C -> 风扇回 0x2F/0x3F，保持 130W
温度采样失败/非法 = 直接进强冷态（宁可错杀）
滞回带 40~45；写入带回读重试；日志 /var/log/gpu_fan_trigger.log
2026-10-27 修复：V100 SSH 提权、锁频校验、静默失败、温度校验、风扇RPM反馈
============================================================
LOG=/var/log/gpu_fan_trigger.log
VM_USER=fan
VM_HOST=192.168.122.29
SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=3)log_msg() { echo "$(date '+%F %T') $*" >> "$LOG"; }
root 无 VM 免密密钥；统一以 fan 用户身份复用现有密钥，BatchMode 禁止交互
vm_ssh() { sudo -n -u fan -H ssh "${SSH_OPTS[@]}" "${VM_USER}@${VM_HOST}" "$@"; }
lock_800() { # 以命令退出码校验，当前SM仅记信息，不再用文本硬匹配
local i rc ok_host=1 ok_vm=1 sm
for i in 1 2 3; do
nvidia-smi -lgc 800 >/dev/null 2>&1
rc=$?
if [ "$rc" -eq 0 ]; then ok_host=0; break; fi
sleep 1
done
[ "$ok_host" -eq 0 ] || log_msg "[警告] 主机锁频800失败(重试3次) rc=$rc"for i in 1 2 3; do vm_ssh "sudo -n nvidia-smi -lgc 800" >/dev/null 2>&1 rc=$? if [ "$rc" -eq 0 ]; then ok_vm=0; break; fi sleep 1 done [ "$ok_vm" -eq 0 ] || log_msg "[警告] V100锁频800失败(重试3次) rc=$rc" if [ "$ok_host" -eq 0 ]; then sm=$(nvidia-smi --query-gpu=clocks.sm --format=csv,noheader,nounits 2>/dev/null | paste -sd/ -) log_msg "[信息] 主机锁频800已下发 当前SM=${sm:-?}MHz" fi [ "$ok_host" -eq 0 ] && [ "$ok_vm" -eq 0 ] && return 0 return 1}
set_fans() {
local z0=${1#0x} z1=${2#0x} i R0 R1
z0=${z0,,}
z1=${z1,,}
for i in 1 2 3; do
ipmitool raw 0x30 0x70 0x66 0x01 0x00 "0x$z0" >/dev/null 2>&1
ipmitool raw 0x30 0x70 0x66 0x01 0x01 "0x$z1" >/dev/null 2>&1
sleep 1
R0=$(ipmitool raw 0x30 0x70 0x66 0x00 0x00 2>/dev/null | tr -dc '0-9a-fA-F')
R1=$(ipmitool raw 0x30 0x70 0x66 0x00 0x01 2>/dev/null | tr -dc '0-9a-fA-F')
[ "${R0,,}" = "$z0" ] && [ "${R1,,}" = "$z1" ] && return 0
done
return 1
}FAN_FAIL_LOGGED=0
apply_fans() { # 失败返回非0，且同一次故障只报一次，下轮继续重试
local z0=$1 z1=$2 label=$3
if set_fans "$z0" "$z1"; then
FAN_FAIL_LOGGED=0
return 0
fi
if [ "$FAN_FAIL_LOGGED" -eq 0 ]; then
log_msg "[报警] 风扇写入失败 zone0=$z0 zone1=$z1 ($label)，下轮重试"
FAN_FAIL_LOGGED=1
fi
return 1
}check_fan_rpm() { # 高档位抽检真实转速，防止占空比正确但风扇堵转/掉线
local label=$1 total alive min
read -r total alive min < <(
ipmitool sensor 2>/dev/null | awk -F'|' '
$1 ~ /FAN/ {
v=$2+0; total++
if (v > 0) { alive++; if (min == 0 || v < min) min=v }
}
END { print total+0, alive+0, min+0 }'
)
total=${total:-0}; alive=${alive:-0}; min=${min:-0}
if [ "$alive" -lt "$total" ] || [ "$min" -lt 800 ]; then
log_msg "[报警] 风扇转速异常 ($label) total=${total:-0} alive=${alive:-0} min=${min:-0}RPM"
return 1
fi
return 0
}set_host_power() {
local pl=$1 rc
nvidia-smi -pl "$pl" >/dev/null 2>&1
rc=$?
[ "$rc" -eq 0 ] || log_msg "[警告] 主机功率设置失败 pl=${pl}W rc=$rc"
return $rc
}sync_v100_power() {
local pl=$1 rc
vm_ssh "sudo -n nvidia-smi -pl $pl" >/dev/null 2>&1
rc=$?
[ "$rc" -eq 0 ] || log_msg "[警告] V100功率设置失败 pl=${pl}W rc=$rc"
return $rc
}sync_v100_clocks() {
local action=$1 cmd rc
if [ "$action" = "lock" ]; then
cmd="sudo -n nvidia-smi -lgc 800"
else
cmd="sudo -n nvidia-smi -rgc"
fi
vm_ssh "$cmd" >/dev/null 2>&1
rc=$?
[ "$rc" -eq 0 ] || log_msg "[警告] V100频率${action}失败 rc=$rc"
return $rc
}===== 铁律执法：显存 =====
1) 单进程占用 >= 65331MiB(63.8G) -> 杀
2) 单进程出现在 >= 2 张卡上(跨卡加模) -> 杀，无例外
vram_law() {
declare -A SEEN
while IFS=',' read -r u p m; do
u="${u//[[:space:]]/}"
p=$(echo "$p" | tr -dc '0-9')
m=$(echo "$m" | tr -dc '0-9')
[ -n "$p" ] && [ -n "$m" ] || continue
if [ "$m" -ge 65331 ]; then
echo "$(date '+%F %T') [铁律] pid=$p 单卡${m}MiB>=63.8G -> 击毙" >> $LOG
kill -9 "$p" 2>/dev/null
continue
fi
if [ -n "${SEEN[$p]:-}" ] && [ "${SEEN[$p]}" != "$u" ]; then
echo "$(date '+%F %T') [铁律] pid=$p 跨卡加模(${SEEN[$p]}+${u}) -> 击毙" >> $LOG
kill -9 "$p" 2>/dev/null
continue
fi
SEEN[$p]=$u
done < <(nvidia-smi --query-compute-apps=gpu_uuid,pid,used_memory --format=csv,noheader,nounits 2>/dev/null)
}启动自愈：按当前外挂占空比推断状态，并校正两区一致
CUR0=$(ipmitool raw 0x30 0x70 0x66 0x00 0x00 2>/dev/null | tr -dc '0-9a-fA-F')
CUR1=$(ipmitool raw 0x30 0x70 0x66 0x00 0x01 2>/dev/null | tr -dc '0-9a-fA-F')
CUR0=${CUR0,,}
CUR1=${CUR1,,}
case "$CUR1" in
4f) STATE=boost; EXPECT0=3f; EXPECT1=4f ;;
64) STATE=panic; EXPECT0=64; EXPECT1=64 ;;
2f|3f) STATE=work; EXPECT0=2f; EXPECT1=3f ;;
20|25) STATE=idle; EXPECT0=20; EXPECT1=25 ;;
*) STATE=work; EXPECT0=2f; EXPECT1=3f ;;
esac
if [ "$CUR0" != "$EXPECT0" ] || [ "$CUR1" != "$EXPECT1" ]; then
if apply_fans "0x$EXPECT0" "0x$EXPECT1" "启动校正"; then
log_msg "[启动] 风扇区校正 ${CUR0:-?}/${CUR1:-?} -> $EXPECT0/$EXPECT1"
fi
fi
case "$STATE" in
idle)
set_host_power 110
sync_v100_power 110
lock_800 ;;
work)
set_host_power 130
sync_v100_power 130
nvidia-smi -rgc >/dev/null 2>&1
sync_v100_clocks unlock ;;
*)
set_host_power 130
sync_v100_power 130 ;;
esac
log_msg "[启动] 初始状态=$STATE 风扇=${CUR0:-?}/${CUR1:-?}"VMT_BAD_LOGGED=0
while true; do
RAW=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null | sort -n | tail -1)# 采样失败或非数字 -> 强冷态（大声报警） if ! [[ "$RAW" =~ ^[0-9]+$ ]] || [ "$RAW" -lt 0 ] || [ "$RAW" -gt 150 ]; then if [ "$STATE" != "panic" ]; then sync_v100_power 130 if apply_fans 0x64 0x64 "采样失败"; then set_host_power 130 log_msg "[报警] 主机温度采样失败/非法(raw=${RAW:-空}) -> 进强冷态0x64/0x64" check_fan_rpm "采样失败" STATE=panic fi fi sleep 10; continue fi T=$RAW # V100(VM直通)温度互通：HTTP状态+纯数字+范围三重校验；VM关机(000)静默跳过 VMT_RESP=$(curl -sm 2 -w '\n%{http_code}' http://192.168.122.29:9100/metrics 2>/dev/null) VMT_CODE=${VMT_RESP##*$'\n'} VMT_BODY=${VMT_RESP%$'\n'*} if [ "$VMT_CODE" = "200" ]; then VMT=$(printf '%s' "$VMT_BODY" | tr -d '[:space:]') if [[ "$VMT" =~ ^[0-9]{1,3}$ ]] && [ "$VMT" -ge 0 ] && [ "$VMT" -le 120 ]; then [ "$VMT" -gt "$T" ] && T=$VMT VMT_BAD_LOGGED=0 else if [ "$VMT_BAD_LOGGED" -eq 0 ]; then log_msg "[警告] V100温度格式非法 body=${VMT_BODY:0:40}，本次忽略" VMT_BAD_LOGGED=1 fi fi elif [ "$VMT_CODE" != "000" ] && [ "$VMT_BAD_LOGGED" -eq 0 ]; then log_msg "[警告] V100温度HTTP异常 code=$VMT_CODE，本次忽略" VMT_BAD_LOGGED=1 fi if [ "$T" -ge 64 ]; then if [ "$STATE" != "panic" ]; then if apply_fans 0x64 0x64 "强冷"; then set_host_power 130 sync_v100_power 130 log_msg "[强冷] ${T}C>=64C -> 130W+风扇拉满0x64" check_fan_rpm "强冷" STATE=panic fi fi elif [ "$T" -ge 60 ] && [ "$STATE" != "boost" ]; then if apply_fans 0x3F 0x4F "强启"; then set_host_power 130 sync_v100_power 130 log_msg "[强启] ${T}C>=60C -> 130W+风扇3F/4F" check_fan_rpm "强启" STATE=boost fi elif [ "$T" -le 50 ] && { [ "$STATE" = "boost" ] || [ "$STATE" = "panic" ]; }; then if apply_fans 0x2F 0x3F "回落"; then set_host_power 130 sync_v100_power 130 log_msg "[回落] ${T}C<=50C -> 墙130W,风扇2F/3F" STATE=rest fi elif [ "$T" -ge 45 ] && [ "$STATE" = "idle" ]; then if apply_fans 0x2F 0x3F "工作"; then set_host_power 130 sync_v100_power 130 nvidia-smi -rgc >/dev/null 2>&1 sync_v100_clocks unlock log_msg "[工作] 上穿45C -> 130W+解锁频率+风扇2F/3F" STATE=work fi elif [ "$T" -lt 40 ] && [ "$STATE" != "boost" ] && [ "$STATE" != "panic" ] && [ "$STATE" != "idle" ]; then set_host_power 110 sync_v100_power 110 lock_800 if apply_fans 0x20 0x25 "静音"; then log_msg "[静音] ${T}C<40C -> 110W+锁800+风扇20/25" STATE=idle fi fi vram_law sleep 10done
-
-
-
-
-
-
-
-
-
-
-
-
-
-
