#!/bin/bash

============================================================

GPU五态温度状态机（10秒巡检，仅状态翻转瞬间出手）

四卡同步：3x CMP 170HX + 1x V100(VM)

【静音态】 全部卡 < 40°C -> 110W + 锁800 + 风扇 0x20/0x25

【工作态】 上穿 45°C -> 130W + 解锁频率(rgc) + 风扇 0x2F/0x3F

【强启态】 任一卡 >= 60°C -> 130W + 风扇 0x3F/0x4F

【强冷态】 任一卡 >= 64°C -> 130W + 风扇拉满 0x64/0x64 （大声报警）

【回落】 强启/强冷后 <=50°C -> 风扇回 0x2F/0x3F，保持 130W

温度采样失败/非法 = 直接进强冷态（宁可错杀）

滞回带 40~45；写入带回读重试；日志 /var/log/gpu_fan_trigger.log

2026-10-27 修复：V100 SSH 提权、锁频校验、静默失败、温度校验、风扇RPM反馈

============================================================

LOG=/var/log/gpu_fan_trigger.log
VM_USER=fan
VM_HOST=192.168.122.29
SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=3)

log_msg() { echo "$(date '+%F %T') $*" >> "$LOG"; }

root 无 VM 免密密钥；统一以 fan 用户身份复用现有密钥，BatchMode 禁止交互

vm_ssh() { sudo -n -u fan -H ssh "${SSH_OPTS[@]}" "${VM_USER}@${VM_HOST}" "$@"; }

lock_800() { # 以命令退出码校验，当前SM仅记信息，不再用文本硬匹配
local i rc ok_host=1 ok_vm=1 sm
for i in 1 2 3; do
nvidia-smi -lgc 800 >/dev/null 2>&1
rc=$?
if [ "$rc" -eq 0 ]; then ok_host=0; break; fi
sleep 1
done
[ "$ok_host" -eq 0 ] || log_msg "[警告] 主机锁频800失败(重试3次) rc=$rc"

for i in 1 2 3; do
    vm_ssh "sudo -n nvidia-smi -lgc 800" >/dev/null 2>&1
    rc=$?
    if [ "$rc" -eq 0 ]; then ok_vm=0; break; fi
    sleep 1
done
[ "$ok_vm" -eq 0 ] || log_msg "[警告] V100锁频800失败(重试3次) rc=$rc"

if [ "$ok_host" -eq 0 ]; then
    sm=$(nvidia-smi --query-gpu=clocks.sm --format=csv,noheader,nounits 2>/dev/null | paste -sd/ -)
    log_msg "[信息] 主机锁频800已下发 当前SM=${sm:-?}MHz"
fi
[ "$ok_host" -eq 0 ] && [ "$ok_vm" -eq 0 ] && return 0
return 1

}

set_fans() {
local z0=${1#0x} z1=${2#0x} i R0 R1
z0=${z0,,}
z1=${z1,,}
for i in 1 2 3; do
ipmitool raw 0x30 0x70 0x66 0x01 0x00 "0x$z0" >/dev/null 2>&1
ipmitool raw 0x30 0x70 0x66 0x01 0x01 "0x$z1" >/dev/null 2>&1
sleep 1
R0=$(ipmitool raw 0x30 0x70 0x66 0x00 0x00 2>/dev/null | tr -dc '0-9a-fA-F')
R1=$(ipmitool raw 0x30 0x70 0x66 0x00 0x01 2>/dev/null | tr -dc '0-9a-fA-F')
[ "${R0,,}" = "$z0" ] && [ "${R1,,}" = "$z1" ] && return 0
done
return 1
}

FAN_FAIL_LOGGED=0
apply_fans() { # 失败返回非0，且同一次故障只报一次，下轮继续重试
local z0=$1 z1=$2 label=$3
if set_fans "$z0" "$z1"; then
FAN_FAIL_LOGGED=0
return 0
fi
if [ "$FAN_FAIL_LOGGED" -eq 0 ]; then
log_msg "[报警] 风扇写入失败 zone0=$z0 zone1=$z1 ($label)，下轮重试"
FAN_FAIL_LOGGED=1
fi
return 1
}

check_fan_rpm() { # 高档位抽检真实转速，防止占空比正确但风扇堵转/掉线
local label=$1 total alive min
read -r total alive min < <(
ipmitool sensor 2>/dev/null | awk -F'|' '
$1 ~ /FAN/ {
v=$2+0; total++
if (v > 0) { alive++; if (min == 0 || v < min) min=v }
}
END { print total+0, alive+0, min+0 }'
)
total=${total:-0}; alive=${alive:-0}; min=${min:-0}
if [ "$alive" -lt "$total" ] || [ "$min" -lt 800 ]; then
log_msg "[报警] 风扇转速异常 ($label) total=${total:-0} alive=${alive:-0} min=${min:-0}RPM"
return 1
fi
return 0
}

set_host_power() {
local pl=$1 rc
nvidia-smi -pl "$pl" >/dev/null 2>&1
rc=$?
[ "$rc" -eq 0 ] || log_msg "[警告] 主机功率设置失败 pl=${pl}W rc=$rc"
return $rc
}

sync_v100_power() {
local pl=$1 rc
vm_ssh "sudo -n nvidia-smi -pl $pl" >/dev/null 2>&1
rc=$?
[ "$rc" -eq 0 ] || log_msg "[警告] V100功率设置失败 pl=${pl}W rc=$rc"
return $rc
}

sync_v100_clocks() {
local action=$1 cmd rc
if [ "$action" = "lock" ]; then
cmd="sudo -n nvidia-smi -lgc 800"
else
cmd="sudo -n nvidia-smi -rgc"
fi
vm_ssh "$cmd" >/dev/null 2>&1
rc=$?
[ "$rc" -eq 0 ] || log_msg "[警告] V100频率${action}失败 rc=$rc"
return $rc
}

===== 铁律执法：显存 =====

1) 单进程占用 >= 65331MiB(63.8G) -> 杀

2) 单进程出现在 >= 2 张卡上(跨卡加模) -> 杀，无例外

vram_law() {
declare -A SEEN
while IFS=',' read -r u p m; do
u="${u//[[:space:]]/}"
p=$(echo "$p" | tr -dc '0-9')
m=$(echo "$m" | tr -dc '0-9')
[ -n "$p" ] && [ -n "$m" ] || continue
if [ "$m" -ge 65331 ]; then
echo "$(date '+%F %T') [铁律] pid=$p 单卡${m}MiB>=63.8G -> 击毙" >> $LOG
kill -9 "$p" 2>/dev/null
continue
fi
if [ -n "${SEEN[$p]:-}" ] && [ "${SEEN[$p]}" != "$u" ]; then
echo "$(date '+%F %T') [铁律] pid=$p 跨卡加模(${SEEN[$p]}+${u}) -> 击毙" >> $LOG
kill -9 "$p" 2>/dev/null
continue
fi
SEEN[$p]=$u
done < <(nvidia-smi --query-compute-apps=gpu_uuid,pid,used_memory --format=csv,noheader,nounits 2>/dev/null)
}

启动自愈：按当前外挂占空比推断状态，并校正两区一致

CUR0=$(ipmitool raw 0x30 0x70 0x66 0x00 0x00 2>/dev/null | tr -dc '0-9a-fA-F')
CUR1=$(ipmitool raw 0x30 0x70 0x66 0x00 0x01 2>/dev/null | tr -dc '0-9a-fA-F')
CUR0=${CUR0,,}
CUR1=${CUR1,,}
case "$CUR1" in
4f) STATE=boost; EXPECT0=3f; EXPECT1=4f ;;
64) STATE=panic; EXPECT0=64; EXPECT1=64 ;;
2f|3f) STATE=work; EXPECT0=2f; EXPECT1=3f ;;
20|25) STATE=idle; EXPECT0=20; EXPECT1=25 ;;
*) STATE=work; EXPECT0=2f; EXPECT1=3f ;;
esac
if [ "$CUR0" != "$EXPECT0" ] || [ "$CUR1" != "$EXPECT1" ]; then
if apply_fans "0x$EXPECT0" "0x$EXPECT1" "启动校正"; then
log_msg "[启动] 风扇区校正 ${CUR0:-?}/${CUR1:-?} -> $EXPECT0/$EXPECT1"
fi
fi
case "$STATE" in
idle)
set_host_power 110
sync_v100_power 110
lock_800 ;;
work)
set_host_power 130
sync_v100_power 130
nvidia-smi -rgc >/dev/null 2>&1
sync_v100_clocks unlock ;;
*)
set_host_power 130
sync_v100_power 130 ;;
esac
log_msg "[启动] 初始状态=$STATE 风扇=${CUR0:-?}/${CUR1:-?}"

VMT_BAD_LOGGED=0
while true; do
RAW=$(nvidia-smi --query-gpu=temperature.gpu --format=csv,noheader,nounits 2>/dev/null | sort -n | tail -1)

# 采样失败或非数字 -> 强冷态（大声报警）
if ! [[ "$RAW" =~ ^[0-9]+$ ]] || [ "$RAW" -lt 0 ] || [ "$RAW" -gt 150 ]; then
    if [ "$STATE" != "panic" ]; then
        sync_v100_power 130
        if apply_fans 0x64 0x64 "采样失败"; then
            set_host_power 130
            log_msg "[报警] 主机温度采样失败/非法(raw=${RAW:-空}) -> 进强冷态0x64/0x64"
            check_fan_rpm "采样失败"
            STATE=panic
        fi
    fi
    sleep 10; continue
fi
T=$RAW

# V100(VM直通)温度互通：HTTP状态+纯数字+范围三重校验；VM关机(000)静默跳过
VMT_RESP=$(curl -sm 2 -w '\n%{http_code}' http://192.168.122.29:9100/metrics 2>/dev/null)
VMT_CODE=${VMT_RESP##*$'\n'}
VMT_BODY=${VMT_RESP%$'\n'*}
if [ "$VMT_CODE" = "200" ]; then
    VMT=$(printf '%s' "$VMT_BODY" | tr -d '[:space:]')
    if [[ "$VMT" =~ ^[0-9]{1,3}$ ]] && [ "$VMT" -ge 0 ] && [ "$VMT" -le 120 ]; then
        [ "$VMT" -gt "$T" ] && T=$VMT
        VMT_BAD_LOGGED=0
    else
        if [ "$VMT_BAD_LOGGED" -eq 0 ]; then
            log_msg "[警告] V100温度格式非法 body=${VMT_BODY:0:40}，本次忽略"
            VMT_BAD_LOGGED=1
        fi
    fi
elif [ "$VMT_CODE" != "000" ] && [ "$VMT_BAD_LOGGED" -eq 0 ]; then
    log_msg "[警告] V100温度HTTP异常 code=$VMT_CODE，本次忽略"
    VMT_BAD_LOGGED=1
fi

if [ "$T" -ge 64 ]; then
    if [ "$STATE" != "panic" ]; then
        if apply_fans 0x64 0x64 "强冷"; then
            set_host_power 130
            sync_v100_power 130
            log_msg "[强冷] ${T}C>=64C -> 130W+风扇拉满0x64"
            check_fan_rpm "强冷"
            STATE=panic
        fi
    fi

elif [ "$T" -ge 60 ] && [ "$STATE" != "boost" ]; then
    if apply_fans 0x3F 0x4F "强启"; then
        set_host_power 130
        sync_v100_power 130
        log_msg "[强启] ${T}C>=60C -> 130W+风扇3F/4F"
        check_fan_rpm "强启"
        STATE=boost
    fi

elif [ "$T" -le 50 ] && { [ "$STATE" = "boost" ] || [ "$STATE" = "panic" ]; }; then
    if apply_fans 0x2F 0x3F "回落"; then
        set_host_power 130
        sync_v100_power 130
        log_msg "[回落] ${T}C<=50C -> 墙130W,风扇2F/3F"
        STATE=rest
    fi

elif [ "$T" -ge 45 ] && [ "$STATE" = "idle" ]; then
    if apply_fans 0x2F 0x3F "工作"; then
        set_host_power 130
        sync_v100_power 130
        nvidia-smi -rgc >/dev/null 2>&1
        sync_v100_clocks unlock
        log_msg "[工作] 上穿45C -> 130W+解锁频率+风扇2F/3F"
        STATE=work
    fi

elif [ "$T" -lt 40 ] && [ "$STATE" != "boost" ] && [ "$STATE" != "panic" ] && [ "$STATE" != "idle" ]; then
    set_host_power 110
    sync_v100_power 110
    lock_800
    if apply_fans 0x20 0x25 "静音"; then
        log_msg "[静音] ${T}C<40C -> 110W+锁800+风扇20/25"
        STATE=idle
    fi
fi
vram_law
sleep 10

done