poll and repair a specific dpaa2 interface

dpaa2-linkfix.sh · 12 KiB · Bash Raw

#!/bin/sh
# dpaa2-linkfix -- detect and recover the dpaa2 "ni" internal link-state desync,
# and capture everything needed to find the actual defect.
# Workaround for FreeBSD bug 279352 ("dpaa2 (dpni0 device): interface goes
# catatonic periodically").  No upstream fix exists; PR open since 2024-05-27.
#
#   dpaa2-linkfix event <ifname>    called by devd on a link-state change
#   dpaa2-linkfix poll  <ifname>    called by cron as a safety net
#   dpaa2-linkfix dump  <ifname>    capture state only, change nothing
#
# THE BUG.  The ni's internal link state goes DOWN without the driver re-issuing
# DPAA2_CMD_MAC_SET_LINK_STATE (it is guarded by
# `if (link_state != sc->link_state)` in dpaa2_ni_miibus_statchg()).  ifconfig
# still reports LOWER_UP and the switch still reports the port up, but no packets
# pass.  Only a real media transition resyncs it.
#
# WHY devd IS THE RIGHT HOOK.  devd cannot see the fault itself -- no interrupt
# means no if_link_state_change() means no devd event.  But the fault is *created
# during a real link transition*, which devd does see (bz@ captured "2 link states
# coalesced" immediately before a wedge, PR 279352 comment 16).  So we reconcile
# the driver's internal link state against reality every time the link moves.
# The cron poll is the safety net for a desync with no event at all (PR c17).
#
# THE LOOP HAZARD.  Our own media flap emits LINK_DOWN + LINK_UP, which re-invokes
# this script.  Guarded three ways: an atomic mkdir lock, a self-inflicted quiet
# period, and an hourly rate limit.
#
# DEBUG.log is the point of this script as much as the recovery is.  We dump full
# state BEFORE and AFTER the media flap, so the recovery itself is an experiment:
# the before/after diff shows exactly which state the flap resynced.

MODE=${1:-poll}
IF=${2:-dpni0}
UNIT=$(echo "$IF" | sed 's/^dpni//')

SW=172.16.1.2                   # switch, used only to verify recovery
OOB=dpni1                       # second, unaddressed link to the same switch
SETTLE=4                        # let the MC settle after a real transition
QUIET=45                        # ignore events for this long after our own flap
MAX_PER_HOUR=4
LOCK=/var/run/dpaa2-linkfix.$IF.lock
QUIETF=/var/run/dpaa2-linkfix.$IF.quiet
HIST=/var/run/dpaa2-linkfix.$IF.hist

WORK=/var/tmp/ai/switch
DEBUG=$WORK/DEBUG.log
CURL=/usr/local/bin/curl
NTFY=https://ntfy.sh/skunkwerks

PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
export PATH

log() { logger -t dpaa2-linkfix "$IF/$MODE: $*"; }

# Push a notification.  Backgrounded and time-limited so a slow or dead uplink
# can never delay the recovery.  During a wedge the uplinks (dpni8/dpni9) stay
# up, so this does get out.
notify() {
    [ -x "$CURL" ] || return 0
    ( "$CURL" -s -m 10 \
        -H "Title: r02 $IF dpaa2 bug 279352" \
        -d "r02 $IF: $*" "$NTFY" >/dev/null 2>&1 & ) >/dev/null 2>&1
    return 0
}

mc_state() {
    sysctl -n "dev.dpaa2_ni.$UNIT.link" 2>/dev/null |
        awk -F': ' '/Link State/{print $2}' | awk '{print $1}'
}

# DO NOT reintroduce an ifconfig call here.  2026-08-20: `ifconfig` is not a
# read-only command on this board.  Any invocation - even `ifconfig dpni0` -
# sends a netlink RTM_GETLINK with NLM_F_DUMP (ifconfig_netlink.c:149), and the
# kernel then runs get_operstate_ether() -> SIOCGIFMEDIA -> mii_pollstat() ->
# MDIO on EVERY interface (netlink/route/iface.c:126).  On r02 that is an
# 8-PHY sweep of a shared memac MDIO bus, which is what generates the phantom
# link transitions that wedge the DPMAC in the first place.  This poll used to
# do 288 such sweeps a day, and in event mode it was a positive feedback loop:
# flap -> devd -> ifconfig -> more flaps.
#
# LOWER_UP cannot be read without causing the sweep: it is not a native FreeBSD
# flag, it is synthesised at iface.c:315 from the operstate that the MDIO poll
# just produced.  So the test is dropped entirely and we gate on mc_state()
# alone - which is the authoritative signal anyway, being the MC firmware's own
# view, i.e. the thing that actually gates the datapath.  False positives on a
# genuine cable-out are bounded by the MAX_PER_HOUR rate limit below.
#
# See LOG.md §3c and CURRENT.md §8.

# ---------------------------------------------------------------------------
# Full state capture.  $1 = label (e.g. "BEFORE-HEAL").
# Everything here is read-only except the out-of-band ARP probe, which injects
# raw frames on an unaddressed interface and changes no configuration.
# ---------------------------------------------------------------------------
dump() {
    LABEL=$1
    mkdir -p "$WORK"
    {
        echo
        echo "================================================================"
        echo "== $LABEL  $IF  $(date '+%Y-%m-%d %H:%M:%S')  mode=$MODE"
        echo "== uptime:$(uptime)"
        echo "================================================================"

echo "---- 1. THE SIGNATURE: MC internal link state vs ifconfig ----"
        echo "# Read ONLY the UP/DOWN word. '(IGNORE)'/'(VALID)' is a driver"
        echo "# display bug (an always-zero reserved bit) and is always IGNORE."
        sysctl "dev.dpaa2_ni.$UNIT.link" 2>&1
        echo "---- sampled 3x, 1s apart, to see if it is flapping ----"
        for i in 1 2 3; do echo "  [$i] $(mc_state)"; sleep 1; done

echo "---- 2. ifconfig -m $IF ----"
        ifconfig -m "$IF" 2>&1

echo "---- 3. INTERRUPTS: did the DPNI link-change IRQ actually fire? ----"
        echo "# This is the crux of the upstream claim 'we do not see an"
        echo "# interrupt for a link-state change'.  Compare across dumps."
        vmstat -i 2>&1 | grep -i "dpaa2\|dpni\|total" || echo "(none)"

echo "---- 4. all dpaa2_ni.$UNIT sysctls (incl. full stats) ----"
        sysctl "dev.dpaa2_ni.$UNIT" 2>&1

echo "---- 5. dpaa2_mac objects ----"
        sysctl dev.dpaa2_mac 2>&1 | grep -v "^dev.dpaa2_mac.[0-9]*.%" || true

echo "---- 6. link state of EVERY dpni (is only $IF affected?) ----"
        for u in 0 1 2 3 4 5 6 7 8 9; do
            S=$(sysctl -n "dev.dpaa2_ni.$u.link" 2>/dev/null |
                awk -F': ' '/Link State/{print $2}')
            R=$(sysctl -n "dev.dpaa2_ni.$u.link" 2>/dev/null |
                awk -F': ' '/Link Rate/{print $2}')
            # NOTE: this ifconfig triggers a full 8-PHY MDIO sweep (see the
            # comment on mc_state()).  Tolerated only because dump() runs when
            # we are already wedged and the evidence is worth more than the
            # extra dice roll.  Do not call dump() speculatively.
            F=$(ifconfig "dpni$u" 2>/dev/null |
                sed -n '1s/.*flags=[0-9a-fx]*<$[^>]*$>.*/\1/p')
            echo "  dpni$u internal='$S' rate=$R flags=$F"
        done

echo "---- 8. reachability over $IF ----"
        ping -c 3 -t 3 "$SW" 2>&1

echo "---- 9. reachability over $OOB (OUT OF BAND, raw ARP, no config) ----"
        echo "# $OOB is link-active to the same switch but unaddressed, so it"
        echo "# bypasses the IP stack entirely.  If the switch answers here but"
        echo "# not on $IF, the fault is confined to $IF's DPMAC/datapath."
        if [ -x /usr/sbin/tcpdump ] || [ -x /usr/bin/tcpdump ]; then
            tcpdump -i "$OOB" -n -c 4 -s 128 arp > "$WORK/.oob.$$" 2>&1 &
            T=$!
            sleep 1
            python3 "$WORK/arpsend.py" "$OOB" 172.16.254.253 "$SW" 6 2>&1
            sleep 3
            kill $T 2>/dev/null; wait $T 2>/dev/null
            cat "$WORK/.oob.$$" 2>/dev/null
            rm -f "$WORK/.oob.$$"
        fi

echo "---- 10. the servers ----"
        for a in 2a02:1b8:10:205::6 2a02:1b8:10:205::7; do
            echo "  ping6 $a:"; ping6 -c 2 -i 1 "$a" 2>&1 | tail -3
        done

echo "---- 11. neighbour / arp / route ----"
        ndp -an 2>&1 | head -20
        arp -an 2>&1 | head -20
        netstat -rn -f inet6 2>&1 | grep -i "205::" || true

echo "---- 12. interface error counters ----"
        netstat -idW 2>&1 | head -20
        echo "---- mbufs ----"
        netstat -m 2>&1 | head -8

echo "---- 14. is r02 still up to the world? (expected: yes) ----"
        ping6 -c 2 -i 1 2606:4700:4700::1111 2>&1 | tail -3
        birdc show protocols 2>/dev/null | head -15

echo "== END $LABEL =="
    } >> "$DEBUG" 2>&1
}

# --- mode: dump only -------------------------------------------------------
if [ "$MODE" = dump ]; then
    dump "MANUAL-DUMP"
    echo "appended to $DEBUG"
    exit 0
fi

# --- guard 1: only one instance at a time (mkdir is atomic) ----------------
if ! mkdir "$LOCK" 2>/dev/null; then
    exit 0
fi
trap 'rmdir "$LOCK" 2>/dev/null' EXIT INT TERM

# --- guard 2: ignore the events our own flap generates ---------------------
if [ -f "$QUIETF" ]; then
    AGE=$(( $(date +%s) - $(cat "$QUIETF" 2>/dev/null || echo 0) ))
    if [ "$AGE" -lt "$QUIET" ]; then
        exit 0
    fi
fi

[ "$MODE" = event ] && sleep "$SETTLE"

S=$(mc_state)

# Breadcrumb, event mode only (~5/day given dpni0's flap rate).  Without this a
# healthy flap leaves no trace at all and we could not tell a working hook from a
# silently broken one.  It also gives us a local, timestamped flap log.
if [ "$MODE" = event ]; then
    log "link transition seen: internal=$S rate=$(sysctl -n dev.dpaa2_ni.$UNIT.link 2>/dev/null | awk -F': ' '/Link Rate/{print $2}')"
fi

# The signature: the MC firmware's internal view says DOWN.  Keying on the
# sysctl rather than on reachability means an unrelated outage can never trick
# us into flapping a healthy interface.  The former second term (ifconfig
# LOWER_UP = yes) was removed on 2026-08-20 - see the comment on mc_state()
# above; reading it was itself triggering the fault.
if [ "$S" != DOWN ]; then
    exit 0
fi

# --- guard 3: hourly rate limit -------------------------------------------
NOW=$(date +%s)
if [ -f "$HIST" ]; then
    N=$(awk -v now="$NOW" 'now - $1 < 3600' "$HIST" | wc -l | tr -d ' ')
    if [ "$N" -ge "$MAX_PER_HOUR" ]; then
        log "SUPPRESSED, already healed $N times this hour; link may be genuinely dead"
        notify "SUPPRESSED - $N heals this hour, link may be genuinely dead"
        exit 0
    fi
    awk -v now="$NOW" 'now - $1 < 7200' "$HIST" > "$HIST.n" && mv "$HIST.n" "$HIST"
fi
echo "$NOW" >> "$HIST"

log "bug 279352 signature: MC internal link state DOWN on $IF - healing"
notify "WEDGED (MC internal link DOWN). Capturing + healing."

# --- evidence BEFORE we change anything -----------------------------------
dump "BEFORE-HEAL"

# --- the fix: the sequence the maintainers validated ----------------------
# 'none' or '10baseT/UTP', then back to auto.  NOT 100baseTX (duplex mismatch
# risk for no benefit).
date +%s > "$QUIETF"
ifconfig "$IF" media none
sleep 3
ifconfig "$IF" media autoselect
sleep 8
date +%s > "$QUIETF"

S=$(mc_state)
USED="media none"
if [ "$S" != UP ]; then
    log "still $S after 'media none', trying 10baseT/UTP (PR 279352 c16)"
    ifconfig "$IF" media 10baseT/UTP
    sleep 3
    ifconfig "$IF" media autoselect
    sleep 8
    date +%s > "$QUIETF"
    S=$(mc_state)
    USED="media none, then 10baseT/UTP"
fi

# --- evidence AFTER: the diff against BEFORE-HEAL is the experiment -------
dump "AFTER-HEAL (applied: $USED)"

if ping -c 2 -t 3 "$SW" >/dev/null 2>&1; then
    log "HEALED via '$USED': internal link=$S, switch reachable again"
    notify "HEALED via '$USED'. internal link=$S. See DEBUG.log"
else
    log "HEAL FAILED via '$USED': internal link=$S, switch still unreachable."
    log "mode 2/3 per LOG.md 3b - a reboot may be needed; media flap not enough"
    notify "HEAL FAILED via '$USED'. internal link=$S. Still down - may need reboot."
fi

1	#!/bin/sh
2	# dpaa2-linkfix -- detect and recover the dpaa2 "ni" internal link-state desync,
3	# and capture everything needed to find the actual defect.
4	# Workaround for FreeBSD bug 279352 ("dpaa2 (dpni0 device): interface goes
5	# catatonic periodically"). No upstream fix exists; PR open since 2024-05-27.
6	#
7	# dpaa2-linkfix event <ifname> called by devd on a link-state change
8	# dpaa2-linkfix poll <ifname> called by cron as a safety net
9	# dpaa2-linkfix dump <ifname> capture state only, change nothing
10	#
11	# THE BUG. The ni's internal link state goes DOWN without the driver re-issuing
12	# DPAA2_CMD_MAC_SET_LINK_STATE (it is guarded by
13	# `if (link_state != sc->link_state)` in dpaa2_ni_miibus_statchg()). ifconfig
14	# still reports LOWER_UP and the switch still reports the port up, but no packets
15	# pass. Only a real media transition resyncs it.
16	#
17	# WHY devd IS THE RIGHT HOOK. devd cannot see the fault itself -- no interrupt
18	# means no if_link_state_change() means no devd event. But the fault is *created
19	# during a real link transition*, which devd does see (bz@ captured "2 link states
20	# coalesced" immediately before a wedge, PR 279352 comment 16). So we reconcile
21	# the driver's internal link state against reality every time the link moves.
22	# The cron poll is the safety net for a desync with no event at all (PR c17).
23	#
24	# THE LOOP HAZARD. Our own media flap emits LINK_DOWN + LINK_UP, which re-invokes
25	# this script. Guarded three ways: an atomic mkdir lock, a self-inflicted quiet
26	# period, and an hourly rate limit.
27	#
28	# DEBUG.log is the point of this script as much as the recovery is. We dump full
29	# state BEFORE and AFTER the media flap, so the recovery itself is an experiment:
30	# the before/after diff shows exactly which state the flap resynced.
31
32	MODE=${1:-poll}
33	IF=${2:-dpni0}
34	UNIT=$(echo "$IF" \| sed 's/^dpni//')
35
36	SW=172.16.1.2 # switch, used only to verify recovery
37	OOB=dpni1 # second, unaddressed link to the same switch
38	SETTLE=4 # let the MC settle after a real transition
39	QUIET=45 # ignore events for this long after our own flap
40	MAX_PER_HOUR=4
41	LOCK=/var/run/dpaa2-linkfix.$IF.lock
42	QUIETF=/var/run/dpaa2-linkfix.$IF.quiet
43	HIST=/var/run/dpaa2-linkfix.$IF.hist
44
45	WORK=/var/tmp/ai/switch
46	DEBUG=$WORK/DEBUG.log
47	CURL=/usr/local/bin/curl
48	NTFY=https://ntfy.sh/skunkwerks
49
50	PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
51	export PATH
52
53	log() { logger -t dpaa2-linkfix "$IF/$MODE: $*"; }
54
55	# Push a notification. Backgrounded and time-limited so a slow or dead uplink
56	# can never delay the recovery. During a wedge the uplinks (dpni8/dpni9) stay
57	# up, so this does get out.
58	notify() {
59	[ -x "$CURL" ] \|\| return 0
60	( "$CURL" -s -m 10 \
61	-H "Title: r02 $IF dpaa2 bug 279352" \
62	-d "r02 $IF: $*" "$NTFY" >/dev/null 2>&1 & ) >/dev/null 2>&1
63	return 0
64	}
65
66	mc_state() {
67	sysctl -n "dev.dpaa2_ni.$UNIT.link" 2>/dev/null \|
68	awk -F': ' '/Link State/{print $2}' \| awk '{print $1}'
69	}
70
71	# DO NOT reintroduce an ifconfig call here. 2026-08-20: `ifconfig` is not a
72	# read-only command on this board. Any invocation - even `ifconfig dpni0` -
73	# sends a netlink RTM_GETLINK with NLM_F_DUMP (ifconfig_netlink.c:149), and the
74	# kernel then runs get_operstate_ether() -> SIOCGIFMEDIA -> mii_pollstat() ->
75	# MDIO on EVERY interface (netlink/route/iface.c:126). On r02 that is an
76	# 8-PHY sweep of a shared memac MDIO bus, which is what generates the phantom
77	# link transitions that wedge the DPMAC in the first place. This poll used to
78	# do 288 such sweeps a day, and in event mode it was a positive feedback loop:
79	# flap -> devd -> ifconfig -> more flaps.
80	#
81	# LOWER_UP cannot be read without causing the sweep: it is not a native FreeBSD
82	# flag, it is synthesised at iface.c:315 from the operstate that the MDIO poll
83	# just produced. So the test is dropped entirely and we gate on mc_state()
84	# alone - which is the authoritative signal anyway, being the MC firmware's own
85	# view, i.e. the thing that actually gates the datapath. False positives on a
86	# genuine cable-out are bounded by the MAX_PER_HOUR rate limit below.
87	#
88	# See LOG.md §3c and CURRENT.md §8.
89
90	# ---------------------------------------------------------------------------
91	# Full state capture. $1 = label (e.g. "BEFORE-HEAL").
92	# Everything here is read-only except the out-of-band ARP probe, which injects
93	# raw frames on an unaddressed interface and changes no configuration.
94	# ---------------------------------------------------------------------------
95	dump() {
96	LABEL=$1
97	mkdir -p "$WORK"
98	{
99	echo
100	echo "================================================================"
101	echo "== $LABEL $IF $(date '+%Y-%m-%d %H:%M:%S') mode=$MODE"
102	echo "== uptime:$(uptime)"
103	echo "================================================================"
104
105	echo "---- 1. THE SIGNATURE: MC internal link state vs ifconfig ----"
106	echo "# Read ONLY the UP/DOWN word. '(IGNORE)'/'(VALID)' is a driver"
107	echo "# display bug (an always-zero reserved bit) and is always IGNORE."
108	sysctl "dev.dpaa2_ni.$UNIT.link" 2>&1
109	echo "---- sampled 3x, 1s apart, to see if it is flapping ----"
110	for i in 1 2 3; do echo " [$i] $(mc_state)"; sleep 1; done
111
112	echo "---- 2. ifconfig -m $IF ----"
113	ifconfig -m "$IF" 2>&1
114
115	echo "---- 3. INTERRUPTS: did the DPNI link-change IRQ actually fire? ----"
116	echo "# This is the crux of the upstream claim 'we do not see an"
117	echo "# interrupt for a link-state change'. Compare across dumps."
118	vmstat -i 2>&1 \| grep -i "dpaa2\\|dpni\\|total" \|\| echo "(none)"
119
120	echo "---- 4. all dpaa2_ni.$UNIT sysctls (incl. full stats) ----"
121	sysctl "dev.dpaa2_ni.$UNIT" 2>&1
122
123	echo "---- 5. dpaa2_mac objects ----"
124	sysctl dev.dpaa2_mac 2>&1 \| grep -v "^dev.dpaa2_mac.[0-9]*.%" \|\| true
125
126	echo "---- 6. link state of EVERY dpni (is only $IF affected?) ----"
127	for u in 0 1 2 3 4 5 6 7 8 9; do
128	S=$(sysctl -n "dev.dpaa2_ni.$u.link" 2>/dev/null \|
129	awk -F': ' '/Link State/{print $2}')
130	R=$(sysctl -n "dev.dpaa2_ni.$u.link" 2>/dev/null \|
131	awk -F': ' '/Link Rate/{print $2}')
132	# NOTE: this ifconfig triggers a full 8-PHY MDIO sweep (see the
133	# comment on mc_state()). Tolerated only because dump() runs when
134	# we are already wedged and the evidence is worth more than the
135	# extra dice roll. Do not call dump() speculatively.
136	F=$(ifconfig "dpni$u" 2>/dev/null \|
137	sed -n '1s/.flags=[0-9a-fx]<$[^>]$>./\1/p')
138	echo " dpni$u internal='$S' rate=$R flags=$F"
139	done
140
141	echo "---- 7. PHY / miibus ----"
142	sysctl dev.vscphy 2>&1 \| grep -vE "%(pnpinfo\|location\|parent\|driver)" \| head -40
143
144	echo "---- 8. reachability over $IF ----"
145	ping -c 3 -t 3 "$SW" 2>&1
146
147	echo "---- 9. reachability over $OOB (OUT OF BAND, raw ARP, no config) ----"
148	echo "# $OOB is link-active to the same switch but unaddressed, so it"
149	echo "# bypasses the IP stack entirely. If the switch answers here but"
150	echo "# not on $IF, the fault is confined to $IF's DPMAC/datapath."
151	if [ -x /usr/sbin/tcpdump ] \|\| [ -x /usr/bin/tcpdump ]; then
152	tcpdump -i "$OOB" -n -c 4 -s 128 arp > "$WORK/.oob.$$" 2>&1 &
153	T=$!
154	sleep 1
155	python3 "$WORK/arpsend.py" "$OOB" 172.16.254.253 "$SW" 6 2>&1
156	sleep 3
157	kill $T 2>/dev/null; wait $T 2>/dev/null
158	cat "$WORK/.oob.$$" 2>/dev/null
159	rm -f "$WORK/.oob.$$"
160	fi
161
162	echo "---- 10. the servers ----"
163	for a in 2a02:1b8:10:205::6 2a02:1b8:10:205::7; do
164	echo " ping6 $a:"; ping6 -c 2 -i 1 "$a" 2>&1 \| tail -3
165	done
166
167	echo "---- 11. neighbour / arp / route ----"
168	ndp -an 2>&1 \| head -20
169	arp -an 2>&1 \| head -20
170	netstat -rn -f inet6 2>&1 \| grep -i "205::" \|\| true
171
172	echo "---- 12. interface error counters ----"
173	netstat -idW 2>&1 \| head -20
174	echo "---- mbufs ----"
175	netstat -m 2>&1 \| head -8
176
177	echo "---- 13. recent kernel messages (link state changed / coalesced) ----"
178	dmesg 2>&1 \| grep -iE "dpni\|dpaa2\|dpmac\|link" \| tail -30
179
180	echo "---- 14. is r02 still up to the world? (expected: yes) ----"
181	ping6 -c 2 -i 1 2606:4700:4700::1111 2>&1 \| tail -3
182	birdc show protocols 2>/dev/null \| head -15
183
184	echo "== END $LABEL =="
185	} >> "$DEBUG" 2>&1
186	}
187
188	# --- mode: dump only -------------------------------------------------------
189	if [ "$MODE" = dump ]; then
190	dump "MANUAL-DUMP"
191	echo "appended to $DEBUG"
192	exit 0
193	fi
194
195	# --- guard 1: only one instance at a time (mkdir is atomic) ----------------
196	if ! mkdir "$LOCK" 2>/dev/null; then
197	exit 0
198	fi
199	trap 'rmdir "$LOCK" 2>/dev/null' EXIT INT TERM
200
201	# --- guard 2: ignore the events our own flap generates ---------------------
202	if [ -f "$QUIETF" ]; then
203	AGE=$(( $(date +%s) - $(cat "$QUIETF" 2>/dev/null \|\| echo 0) ))
204	if [ "$AGE" -lt "$QUIET" ]; then
205	exit 0
206	fi
207	fi
208
209	[ "$MODE" = event ] && sleep "$SETTLE"
210
211	S=$(mc_state)
212
213	# Breadcrumb, event mode only (~5/day given dpni0's flap rate). Without this a
214	# healthy flap leaves no trace at all and we could not tell a working hook from a
215	# silently broken one. It also gives us a local, timestamped flap log.
216	if [ "$MODE" = event ]; then
217	log "link transition seen: internal=$S rate=$(sysctl -n dev.dpaa2_ni.$UNIT.link 2>/dev/null \| awk -F': ' '/Link Rate/{print $2}')"
218	fi
219
220	# The signature: the MC firmware's internal view says DOWN. Keying on the
221	# sysctl rather than on reachability means an unrelated outage can never trick
222	# us into flapping a healthy interface. The former second term (ifconfig
223	# LOWER_UP = yes) was removed on 2026-08-20 - see the comment on mc_state()
224	# above; reading it was itself triggering the fault.
225	if [ "$S" != DOWN ]; then
226	exit 0
227	fi
228
229	# --- guard 3: hourly rate limit -------------------------------------------
230	NOW=$(date +%s)
231	if [ -f "$HIST" ]; then
232	N=$(awk -v now="$NOW" 'now - $1 < 3600' "$HIST" \| wc -l \| tr -d ' ')
233	if [ "$N" -ge "$MAX_PER_HOUR" ]; then
234	log "SUPPRESSED, already healed $N times this hour; link may be genuinely dead"
235	notify "SUPPRESSED - $N heals this hour, link may be genuinely dead"
236	exit 0
237	fi
238	awk -v now="$NOW" 'now - $1 < 7200' "$HIST" > "$HIST.n" && mv "$HIST.n" "$HIST"
239	fi
240	echo "$NOW" >> "$HIST"
241
242	log "bug 279352 signature: MC internal link state DOWN on $IF - healing"
243	notify "WEDGED (MC internal link DOWN). Capturing + healing."
244
245	# --- evidence BEFORE we change anything -----------------------------------
246	dump "BEFORE-HEAL"
247
248	# --- the fix: the sequence the maintainers validated ----------------------
249	# 'none' or '10baseT/UTP', then back to auto. NOT 100baseTX (duplex mismatch
250	# risk for no benefit).
251	date +%s > "$QUIETF"
252	ifconfig "$IF" media none
253	sleep 3
254	ifconfig "$IF" media autoselect
255	sleep 8
256	date +%s > "$QUIETF"
257
258	S=$(mc_state)
259	USED="media none"
260	if [ "$S" != UP ]; then
261	log "still $S after 'media none', trying 10baseT/UTP (PR 279352 c16)"
262	ifconfig "$IF" media 10baseT/UTP
263	sleep 3
264	ifconfig "$IF" media autoselect
265	sleep 8
266	date +%s > "$QUIETF"
267	S=$(mc_state)
268	USED="media none, then 10baseT/UTP"
269	fi
270
271	# --- evidence AFTER: the diff against BEFORE-HEAL is the experiment -------
272	dump "AFTER-HEAL (applied: $USED)"
273
274	if ping -c 2 -t 3 "$SW" >/dev/null 2>&1; then
275	log "HEALED via '$USED': internal link=$S, switch reachable again"
276	notify "HEALED via '$USED'. internal link=$S. See DEBUG.log"
277	else
278	log "HEAL FAILED via '$USED': internal link=$S, switch still unreachable."
279	log "mode 2/3 per LOG.md 3b - a reboot may be needed; media flap not enough"
280	notify "HEAL FAILED via '$USED'. internal link=$S. Still down - may need reboot."
281	fi
282