#!/bin/sh
# dpaa2-linkfix -- detect and recover the dpaa2 "ni" internal link-state desync,
# and capture everything needed to find the actual defect.
# Workaround for FreeBSD bug 279352 ("dpaa2 (dpni0 device): interface goes
# catatonic periodically").  No upstream fix exists; PR open since 2024-05-27.
#
#   dpaa2-linkfix event <ifname>    called by devd on a link-state change
#   dpaa2-linkfix poll  <ifname>    called by cron as a safety net
#   dpaa2-linkfix dump  <ifname>    capture state only, change nothing
#
# THE BUG.  The ni's internal link state goes DOWN without the driver re-issuing
# DPAA2_CMD_MAC_SET_LINK_STATE (it is guarded by
# `if (link_state != sc->link_state)` in dpaa2_ni_miibus_statchg()).  ifconfig
# still reports LOWER_UP and the switch still reports the port up, but no packets
# pass.  Only a real media transition resyncs it.
#
# WHY devd IS THE RIGHT HOOK.  devd cannot see the fault itself -- no interrupt
# means no if_link_state_change() means no devd event.  But the fault is *created
# during a real link transition*, which devd does see (bz@ captured "2 link states
# coalesced" immediately before a wedge, PR 279352 comment 16).  So we reconcile
# the driver's internal link state against reality every time the link moves.
# The cron poll is the safety net for a desync with no event at all (PR c17).
#
# THE LOOP HAZARD.  Our own media flap emits LINK_DOWN + LINK_UP, which re-invokes
# this script.  Guarded three ways: an atomic mkdir lock, a self-inflicted quiet
# period, and an hourly rate limit.
#
# DEBUG.log is the point of this script as much as the recovery is.  We dump full
# state BEFORE and AFTER the media flap, so the recovery itself is an experiment:
# the before/after diff shows exactly which state the flap resynced.

MODE=${1:-poll}
IF=${2:-dpni0}
UNIT=$(echo "$IF" | sed 's/^dpni//')

SW=172.16.1.2                   # switch, used only to verify recovery
OOB=dpni1                       # second, unaddressed link to the same switch
SETTLE=4                        # let the MC settle after a real transition
QUIET=45                        # ignore events for this long after our own flap
MAX_PER_HOUR=4
LOCK=/var/run/dpaa2-linkfix.$IF.lock
QUIETF=/var/run/dpaa2-linkfix.$IF.quiet
HIST=/var/run/dpaa2-linkfix.$IF.hist

WORK=/var/tmp/ai/switch
DEBUG=$WORK/DEBUG.log
CURL=/usr/local/bin/curl
NTFY=https://ntfy.sh/skunkwerks

PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
export PATH

log() { logger -t dpaa2-linkfix "$IF/$MODE: $*"; }

# Push a notification.  Backgrounded and time-limited so a slow or dead uplink
# can never delay the recovery.  During a wedge the uplinks (dpni8/dpni9) stay
# up, so this does get out.
notify() {
    [ -x "$CURL" ] || return 0
    ( "$CURL" -s -m 10 \
        -H "Title: r02 $IF dpaa2 bug 279352" \
        -d "r02 $IF: $*" "$NTFY" >/dev/null 2>&1 & ) >/dev/null 2>&1
    return 0
}

mc_state() {
    sysctl -n "dev.dpaa2_ni.$UNIT.link" 2>/dev/null |
        awk -F': ' '/Link State/{print $2}' | awk '{print $1}'
}

# DO NOT reintroduce an ifconfig call here.  2026-08-20: `ifconfig` is not a
# read-only command on this board.  Any invocation - even `ifconfig dpni0` -
# sends a netlink RTM_GETLINK with NLM_F_DUMP (ifconfig_netlink.c:149), and the
# kernel then runs get_operstate_ether() -> SIOCGIFMEDIA -> mii_pollstat() ->
# MDIO on EVERY interface (netlink/route/iface.c:126).  On r02 that is an
# 8-PHY sweep of a shared memac MDIO bus, which is what generates the phantom
# link transitions that wedge the DPMAC in the first place.  This poll used to
# do 288 such sweeps a day, and in event mode it was a positive feedback loop:
# flap -> devd -> ifconfig -> more flaps.
#
# LOWER_UP cannot be read without causing the sweep: it is not a native FreeBSD
# flag, it is synthesised at iface.c:315 from the operstate that the MDIO poll
# just produced.  So the test is dropped entirely and we gate on mc_state()
# alone - which is the authoritative signal anyway, being the MC firmware's own
# view, i.e. the thing that actually gates the datapath.  False positives on a
# genuine cable-out are bounded by the MAX_PER_HOUR rate limit below.
#
# See LOG.md §3c and CURRENT.md §8.

# ---------------------------------------------------------------------------
# Full state capture.  $1 = label (e.g. "BEFORE-HEAL").
# Everything here is read-only except the out-of-band ARP probe, which injects
# raw frames on an unaddressed interface and changes no configuration.
# ---------------------------------------------------------------------------
dump() {
    LABEL=$1
    mkdir -p "$WORK"
    {
        echo
        echo "================================================================"
        echo "== $LABEL  $IF  $(date '+%Y-%m-%d %H:%M:%S')  mode=$MODE"
        echo "== uptime:$(uptime)"
        echo "================================================================"

        echo "---- 1. THE SIGNATURE: MC internal link state vs ifconfig ----"
        echo "# Read ONLY the UP/DOWN word. '(IGNORE)'/'(VALID)' is a driver"
        echo "# display bug (an always-zero reserved bit) and is always IGNORE."
        sysctl "dev.dpaa2_ni.$UNIT.link" 2>&1
        echo "---- sampled 3x, 1s apart, to see if it is flapping ----"
        for i in 1 2 3; do echo "  [$i] $(mc_state)"; sleep 1; done

        echo "---- 2. ifconfig -m $IF ----"
        ifconfig -m "$IF" 2>&1

        echo "---- 3. INTERRUPTS: did the DPNI link-change IRQ actually fire? ----"
        echo "# This is the crux of the upstream claim 'we do not see an"
        echo "# interrupt for a link-state change'.  Compare across dumps."
        vmstat -i 2>&1 | grep -i "dpaa2\|dpni\|total" || echo "(none)"

        echo "---- 4. all dpaa2_ni.$UNIT sysctls (incl. full stats) ----"
        sysctl "dev.dpaa2_ni.$UNIT" 2>&1

        echo "---- 5. dpaa2_mac objects ----"
        sysctl dev.dpaa2_mac 2>&1 | grep -v "^dev.dpaa2_mac.[0-9]*.%" || true

        echo "---- 6. link state of EVERY dpni (is only $IF affected?) ----"
        for u in 0 1 2 3 4 5 6 7 8 9; do
            S=$(sysctl -n "dev.dpaa2_ni.$u.link" 2>/dev/null |
                awk -F': ' '/Link State/{print $2}')
            R=$(sysctl -n "dev.dpaa2_ni.$u.link" 2>/dev/null |
                awk -F': ' '/Link Rate/{print $2}')
            # NOTE: this ifconfig triggers a full 8-PHY MDIO sweep (see the
            # comment on mc_state()).  Tolerated only because dump() runs when
            # we are already wedged and the evidence is worth more than the
            # extra dice roll.  Do not call dump() speculatively.
            F=$(ifconfig "dpni$u" 2>/dev/null |
                sed -n '1s/.*flags=[0-9a-fx]*<\([^>]*\)>.*/\1/p')
            echo "  dpni$u internal='$S' rate=$R flags=$F"
        done

        echo "---- 7. PHY / miibus ----"
        sysctl dev.vscphy 2>&1 | grep -vE "%(pnpinfo|location|parent|driver)" | head -40

        echo "---- 8. reachability over $IF ----"
        ping -c 3 -t 3 "$SW" 2>&1

        echo "---- 9. reachability over $OOB (OUT OF BAND, raw ARP, no config) ----"
        echo "# $OOB is link-active to the same switch but unaddressed, so it"
        echo "# bypasses the IP stack entirely.  If the switch answers here but"
        echo "# not on $IF, the fault is confined to $IF's DPMAC/datapath."
        if [ -x /usr/sbin/tcpdump ] || [ -x /usr/bin/tcpdump ]; then
            tcpdump -i "$OOB" -n -c 4 -s 128 arp > "$WORK/.oob.$$" 2>&1 &
            T=$!
            sleep 1
            python3 "$WORK/arpsend.py" "$OOB" 172.16.254.253 "$SW" 6 2>&1
            sleep 3
            kill $T 2>/dev/null; wait $T 2>/dev/null
            cat "$WORK/.oob.$$" 2>/dev/null
            rm -f "$WORK/.oob.$$"
        fi

        echo "---- 10. the servers ----"
        for a in 2a02:1b8:10:205::6 2a02:1b8:10:205::7; do
            echo "  ping6 $a:"; ping6 -c 2 -i 1 "$a" 2>&1 | tail -3
        done

        echo "---- 11. neighbour / arp / route ----"
        ndp -an 2>&1 | head -20
        arp -an 2>&1 | head -20
        netstat -rn -f inet6 2>&1 | grep -i "205::" || true

        echo "---- 12. interface error counters ----"
        netstat -idW 2>&1 | head -20
        echo "---- mbufs ----"
        netstat -m 2>&1 | head -8

        echo "---- 13. recent kernel messages (link state changed / coalesced) ----"
        dmesg 2>&1 | grep -iE "dpni|dpaa2|dpmac|link" | tail -30

        echo "---- 14. is r02 still up to the world? (expected: yes) ----"
        ping6 -c 2 -i 1 2606:4700:4700::1111 2>&1 | tail -3
        birdc show protocols 2>/dev/null | head -15

        echo "== END $LABEL =="
    } >> "$DEBUG" 2>&1
}

# --- mode: dump only -------------------------------------------------------
if [ "$MODE" = dump ]; then
    dump "MANUAL-DUMP"
    echo "appended to $DEBUG"
    exit 0
fi

# --- guard 1: only one instance at a time (mkdir is atomic) ----------------
if ! mkdir "$LOCK" 2>/dev/null; then
    exit 0
fi
trap 'rmdir "$LOCK" 2>/dev/null' EXIT INT TERM

# --- guard 2: ignore the events our own flap generates ---------------------
if [ -f "$QUIETF" ]; then
    AGE=$(( $(date +%s) - $(cat "$QUIETF" 2>/dev/null || echo 0) ))
    if [ "$AGE" -lt "$QUIET" ]; then
        exit 0
    fi
fi

[ "$MODE" = event ] && sleep "$SETTLE"

S=$(mc_state)

# Breadcrumb, event mode only (~5/day given dpni0's flap rate).  Without this a
# healthy flap leaves no trace at all and we could not tell a working hook from a
# silently broken one.  It also gives us a local, timestamped flap log.
if [ "$MODE" = event ]; then
    log "link transition seen: internal=$S rate=$(sysctl -n dev.dpaa2_ni.$UNIT.link 2>/dev/null | awk -F': ' '/Link Rate/{print $2}')"
fi

# The signature: the MC firmware's internal view says DOWN.  Keying on the
# sysctl rather than on reachability means an unrelated outage can never trick
# us into flapping a healthy interface.  The former second term (ifconfig
# LOWER_UP = yes) was removed on 2026-08-20 - see the comment on mc_state()
# above; reading it was itself triggering the fault.
if [ "$S" != DOWN ]; then
    exit 0
fi

# --- guard 3: hourly rate limit -------------------------------------------
NOW=$(date +%s)
if [ -f "$HIST" ]; then
    N=$(awk -v now="$NOW" 'now - $1 < 3600' "$HIST" | wc -l | tr -d ' ')
    if [ "$N" -ge "$MAX_PER_HOUR" ]; then
        log "SUPPRESSED, already healed $N times this hour; link may be genuinely dead"
        notify "SUPPRESSED - $N heals this hour, link may be genuinely dead"
        exit 0
    fi
    awk -v now="$NOW" 'now - $1 < 7200' "$HIST" > "$HIST.n" && mv "$HIST.n" "$HIST"
fi
echo "$NOW" >> "$HIST"

log "bug 279352 signature: MC internal link state DOWN on $IF - healing"
notify "WEDGED (MC internal link DOWN). Capturing + healing."

# --- evidence BEFORE we change anything -----------------------------------
dump "BEFORE-HEAL"

# --- the fix: the sequence the maintainers validated ----------------------
# 'none' or '10baseT/UTP', then back to auto.  NOT 100baseTX (duplex mismatch
# risk for no benefit).
date +%s > "$QUIETF"
ifconfig "$IF" media none
sleep 3
ifconfig "$IF" media autoselect
sleep 8
date +%s > "$QUIETF"

S=$(mc_state)
USED="media none"
if [ "$S" != UP ]; then
    log "still $S after 'media none', trying 10baseT/UTP (PR 279352 c16)"
    ifconfig "$IF" media 10baseT/UTP
    sleep 3
    ifconfig "$IF" media autoselect
    sleep 8
    date +%s > "$QUIETF"
    S=$(mc_state)
    USED="media none, then 10baseT/UTP"
fi

# --- evidence AFTER: the diff against BEFORE-HEAL is the experiment -------
dump "AFTER-HEAL (applied: $USED)"

if ping -c 2 -t 3 "$SW" >/dev/null 2>&1; then
    log "HEALED via '$USED': internal link=$S, switch reachable again"
    notify "HEALED via '$USED'. internal link=$S. See DEBUG.log"
else
    log "HEAL FAILED via '$USED': internal link=$S, switch still unreachable."
    log "mode 2/3 per LOG.md 3b - a reboot may be needed; media flap not enough"
    notify "HEAL FAILED via '$USED'. internal link=$S. Still down - may need reboot."
fi