#!/bin/sh
# fillrate-task: kernel-log-read
# Strictly read-only kernel and GPU log collection, safe while rentals run.
# Built to chase intermittent Vast "bad nvidia_smi" / "GPUs not found?" polls.
#
# Prints, to stdout only (the runner log captures it):
#   1. dmesg -T filtered for Xid, NVRM, nvidia, pcie, AER, MCE, hung
#   2. journalctl -k --since "03:00 today" with the same filter
#   3. nvidia-smi -q for ECC, retired pages, row remapping, and throttle reasons
#   4. timestamps of kaalia.log lines for failed nvidia_smi polls
# Total output is capped at OUT_CAP bytes (default 400000) and each section at
# LINE_CAP lines (default 2000, newest kept). Every command runs under timeout.
#
# Writes no files, restarts nothing, installs nothing, and never touches
# containers, docker, vastai, or GPU settings. The FILLRATE_* overrides are for
# tests; a real run uses the defaults.

set -u

KAALIA_LOG=${FILLRATE_KAALIA_LOG:-/var/lib/vastai_kaalia/kaalia.log}
SINCE=${FILLRATE_KLOG_SINCE:-03:00 today}
OUT_CAP=${FILLRATE_KLOG_OUT_CAP:-400000}
LINE_CAP=${FILLRATE_KLOG_LINE_CAP:-2000}
T=${FILLRATE_KLOG_TIMEOUT:-60}
EXPIRES=${FILLRATE_EXPIRES:-0}
FILTER='Xid|NVRM|nvidia|pcie|AER|MCE|hung'
BAD='GPUs not found|bad nvidia_smi|nvidia_smi[^ ]*\(\) .*(error|fail)|on_read error'

main() {
  echo "=== kernel-log-read machine=${FILLRATE_MACHINE_ID:-unknown} host=$(hostname) at $(date '+%Y-%m-%d %H:%M:%S %Z') since=\"$SINCE\" ==="
  echo "uptime: $(uptime)"
  echo "boot: $(uptime -s 2>/dev/null)"

  echo "=== 1. dmesg -T | grep -i -E '$FILTER' (last $LINE_CAP) ==="
  if command -v dmesg > /dev/null 2>&1; then
    timeout "$T" dmesg -T 2>&1 | grep -i -E "$FILTER" | tail -n "$LINE_CAP"
    echo "(matches: $(timeout "$T" dmesg -T 2>/dev/null | grep -c -i -E "$FILTER"))"
  else
    echo "dmesg not found"
  fi

  echo "=== 2. journalctl -k --since \"$SINCE\" | grep -i -E '$FILTER' (last $LINE_CAP) ==="
  if command -v journalctl > /dev/null 2>&1; then
    timeout "$T" journalctl -k --since "$SINCE" --no-pager -o short-iso 2>&1 | grep -i -E "$FILTER" | tail -n "$LINE_CAP"
    echo "(matches: $(timeout "$T" journalctl -k --since "$SINCE" --no-pager 2>/dev/null | grep -c -i -E "$FILTER"))"
  else
    echo "journalctl not found"
  fi

  echo "=== 3. nvidia-smi -q -d ECC,PAGE_RETIREMENT,ROW_REMAPPER,PERFORMANCE ==="
  if command -v nvidia-smi > /dev/null 2>&1; then
    timeout "$T" nvidia-smi -q -d ECC,PAGE_RETIREMENT,ROW_REMAPPER,PERFORMANCE 2>&1 | tail -n "$LINE_CAP"
  else
    echo "nvidia-smi not found"
  fi

  echo "=== 4. kaalia failed nvidia_smi poll timestamps ($KAALIA_LOG) ==="
  if [ -r "$KAALIA_LOG" ]; then
    timeout "$T" grep -E "$BAD" "$KAALIA_LOG" | sed -n 's/^\[\([^]]*\)\].*\] P[0-9]* *\(.*\)$/\1  \2/p' | cut -c1-120 | tail -n "$LINE_CAP"
    echo "(count: $(grep -c -E "$BAD" "$KAALIA_LOG"))"
  else
    echo "kaalia.log not readable"
  fi
  echo "=== kernel-log-read done ==="
}

if [ "$(date +%s)" -gt "$EXPIRES" ]; then
  echo "refusing: ticket expired"
  echo "summary: kernel-log-read result=refused reason=expired"
  exit 1
fi

main 2>&1 | head -c "$OUT_CAP"
echo
echo "summary: kernel-log-read result=ok out_cap=$OUT_CAP"
exit 0