#!/bin/sh #wan-watchdog.sh v.20260903.01 # various updates and optimizations using Claude Code. # Settings: BOOTWAIT=5 # Under 45 seconds, automatically sets TESTMODE=1: Script will run once in Testing Mode and then quit (not loop). # You can also invoke testing mode by running './wan-watchdog.sh test'.That will over-ride the timer (0 seconds and always in test mode). # The reason you might want to still use the timer-method, a shorter bootwait (under 45 seconds, like 40 seconds), is to leave the script as-is, # allowing the router to reboot and rc.local to call the script, without looping the script, for testing purposes. # Where do you want the connection error messages written to? log_file="/root/wan_watchdoglog.txt" # Besides writing the log start date/time, only failures or changes to the connection are written to file. # ALERT!: Running in TESTMODE will divert log to /root/wwatch.log. # Re-test connection interval: How many seconds to wait after a completed process of checking connectivity, to do it again? LOOPWAIT=90 WAITFOR=3 # Seconds to wait inbetween certain steps (adds momentary pauses between steps, useful for watching the script when debugging in test mode) MODEMTYPE="GENERICWAN" # Determines the wan or cell modem reset procedure used within the modem_reset function. Use GENERICWAN, QUECTELQMI, SIERRAQMI or SIERRAMBIM. At this time, different reset procedures are used, depending on specific real-world issues with the modem type and the mode it's operating in (QMI or MBIM). They may be merged in future versions. It's found, at present testing, that the two different brands of modems exhibit slightly different lock-up characteristics, and different procedures based on actual experiences, are needed to recover their function. # GENERIC / GENERICWAN was recently added for people with fiber/ethernet or other non-cellular WAN link. NVRELOAD=0 # Testing a 3rd-level modem restoration technique, currently only used for SIERRAMBIM: If the Sierra modem fails in the 'up:false' condition, this can happen when the cell tower is fine, but instead be caused by corruption in the modem's settings. Enabling this option will do an extended reset. # 20260220: Update!: Further testing and research, revealed that generally, DO NOT use the above method (leave it at '0'). What you probably should be doing if you run into the described condition, is delete all the modem images and preferences (Sierra AT!ENTERCND="A710",AT!IMAGE=0 command) and reflash only your carrier firmware, with qmi-firmware-update flash tool (not fwdld) # CAUTION: Before you set this to '1', you should have entered management mode, when your modem was working, and done an AT!NVBACKUP=3, to save your modem's working settings in the NVRAM backup location 3. # UCI Logical Name of the bare wan connection (no VPN): CONFIGURED_WWANNAME="qmi" # On cellular modem routers, usually 'mbim' or 'qmi' - whatever you named the interface in Device -> Interfaces. # ^^ As 'ifstatus' and 'ifup' recognizes it. This is different than the 'ifconfig' kernel name of the device, which will be derived later. MODEMUSBDEVNODE="/dev/cdc-wdm0" # Used in cellular modem routers, for the (wan) modem_reset function. This setting is normally /dev/cdc-wdm0 on either QMI or MBIM-configured single-modem systems. # If you are NOT using a VPN: It may be optimal for you to replace these with your provider's DNS servers, even if you are not using them for DNS. Used for checking if both ipv6 and ipv4 are up and working. # If you are using a VPN: Your provider's DNS servers will likely not respond to pings. DNS6SITES="2001:4860:4860::8844 2001:4860:4860::8888" DNS4SITES="8.4.4.8 8.8.8.8" #DNS6SITES="2001:4888:32:ff00:3c1:d:: 2001:4888:3b:ff00:3c2:d::" # Replace with YOUR provider's IPV6 DNS servers. (These sample ones are Verizons's) #DNS4SITES="198.224.151.135 198.224.150.135" # Replace with YOUR provider's IPV4 DNS servers. # DARKMOON replaces traditional SQM (luci-app-sqm/sqm-scripts) and cake-autorate, due to significantly lower CPU utilization, and a nice Luci interface. You can either use the old version of this script for sqm with or without cake-autorate, # or use this version and newer, with darkmoon (cake-autorate-reborn), that takes over the job of sqm & cake-autorate. DARKMOON=0 # Enable Native C version of Cake-Autorate (will NOT work with SQM). Works with a straight WWAN connection or VPN. # Use Software Offloading: # SQM and Cake and other techniques like 'ip route' where-ever packets have to be processed by the kernel, do not work correctly with Hardware Offloading. # VPN (1 - Yes / 0 - No): # Define whether you want to use VPN's: 1 for yes 0 for no. You may, for whatever reason, not want to use VPN's on your router at some time, e.g. during testing. # ATTENTION: Reminder when you set-up all your VPN profiles, to configure them to NOT 'Start on boot'. Let this script start them instead. VPN=1 # Put here the 'friendly' UCI names of your VPN Interfaces (Luci->Network->Interfaces. Name on the top, not on the bottom.): # ATTENTION: None of your VPN profiles (interfaces) should be set to 'Start at boot' - UNCHECK that. And manually test each one to make sure they work. VPN1="protonvpn1" VPN2="protonvpn2" VPN3="protonvpn3" VPN4="protonvpn4" # Changelog: Newest comments at the top # # v20260903.01: QUECTELQMI Level 2b — three changes from the 2026-09-02 event (GPIO+ifup # ... recovered the modem; watchdog declared reboot because it sampled ifstatus/fping while # ... qmi.sh was still in its UCI delay). (1) After ifup, poll ifstatus until "up":true or # ... delay+65s, then fping; do not fail on one snapshot. (2) If the AT dump shows registered # ... (CEREG 1 or 5, COPS has an operator, or QENG not SEARCH), poll ifstatus/fping again # ... instead of treating that as reboot-worthy. (3) Honor kill before the post-modem_reset # ... fping and before safe_reboot, so a stop during recovery does not log "reboot required". # v20260831.03: QUECTELQMI — after all recovery levels are exhausted, also log # ... uqmi --get-serving-system, AT+CEREG?, and AT+COPS? next to the existing AT+CPIN?/AT+QENG dump. # v20260831.02: QUECTELQMI Level 2b — if uqmi still returns "Failed to connect to service" after # ... the GPIO wait, rebind qmi_wwan (same unbind/bind as Level 0) before ifup. GPIO can leave the # ... host QMI USB path wedged while the modem itself is already answering AT. Then ifup as before # ... (single 65s wait). # v20260831.01: QUECTELQMI Level 2b — removed the extra 65s retry and extra 20s settle that were # ... added in v20260830.01. After ifup qmi, wait 65s once, then ifstatus + fping. If it has not # ... come up in 65s, more waiting does not help; continue to AT diagnostics / reboot decision. # v20260830.01: QUECTELQMI Level 2b — after GPIO reset, ifup qmi still needs a full ~60s for the # ... QMI data session, same as a normal ifup. The previous 45s wait (and the comment that the modem # ... was already tower-registered so the bearer would come up in seconds) was too short: 2026-08-30 # ... log showed ifstatus=false / AT+QENG NOCONN at that 45s mark, the script declared reboot, and # ... connectivity came back immediately after the watchdog was killed. Fix: wait 65s after ifup # ... (same as Level 1 / Level 2a). # v20260804.02: Found by watching a test-mode run: after an fping failure with no VPN active (VPN=0 and # ... no VPN interface manually running), the script still ran a second check_fping before modem_reset — # ... pointless, since tearing down a VPN that was never up changes nothing, and that check_fping call # ... (with its own internal timeouts/retries) just delays getting to modem_reset. Fix: the post-teardown # ... check_fping is now only run if a VPN was actually active and torn down (vpn_is_active=1) — since only # ... in that case could the teardown plausibly have fixed the failure (bad VPN tunnel, good underlying # ... WAN). If no VPN was active, the script now skips straight to modem_reset. # v20260804.01: Fixed a real bug found by running './wan-watchdog.sh test': the router actually rebooted # ... during a test run, even though test mode is documented to run once, log to wwatch.log, and exit # ... without ever calling `reboot`. Root cause: modem_reset() had THREE places that called safe_reboot() # ... directly (two in the SIERRAQMI-branch hard-down conditions, one in QUECTELQMI's "all recovery levels # ... exhausted" block after Level 2b) — completely bypassing TESTMODE. The only TESTMODE-aware reboot gate # ... in the whole script lived at the very bottom ('if [ $TESTMODE = 0 ] && [ $NEEDREBOOT = 1 ]'), which # ... only runs based on NEEDREBOOT/CONNECTIONSTATUS as set by the *caller's* post-modem_reset check_fping — # ... it never protected modem_reset()'s own internal reboot calls, since those never went through # ... NEEDREBOOT at all. Fix (per design correction from the user, restoring the original architecture): # ... removed all three direct safe_reboot() calls from inside modem_reset(). When recovery is exhausted, # ... modem_reset() now just logs and returns, exactly like every other failure path in the function. The # ... caller's existing check_fping-after-modem_reset logic (around line 1234) naturally still fails, which # ... sets CONNECTIONSTATUS="0" and NEEDREBOOT="1" the normal way — and only the single TESTMODE-gated block # ... at the bottom of the script ever calls safe_reboot(). safe_reboot() itself is unchanged: it should # ... never need its own TESTMODE awareness now that it has exactly one caller, gated exactly once. # v20260801.01: QUECTELQMI — two changes based on log evidence review of every past "Failed to connect to # ... service" event (not just the 2026-08-01 one that prompted Level 0): # ... (1) Reinstated an AT probe (ATI on ttyUSB2) as the gate in front of Level 0's qmi_wwan rebind. The # ... 2026-07-28.03 note below argued no gate was needed since the rebind is cheap (~7s) and falls through # ... regardless. But log review found two confirmed cases (2026-07-10 20:17, 2026-07-13 22:26) where the # ... *initial* uqmi query already read "Failed to connect to service" and the fix that actually worked was # ... GPIO lte_rst — a real modem-side reset, not a host-side driver problem. So the signature alone does not # ... reliably mean "host-side wedge, modem is fine": it can equally mean a genuinely down modem. AT-responsiveness # ... is the one signal that has actually distinguished the two cases in practice (confirmed 2026-08-01 when AT # ... kept responding throughout a QMI outage that only a router reboot fixed). So: uqmi fails → probe AT first. # ... AT responds → modem is alive, QMI failure is host-side → try the qmi_wwan rebind. AT does not respond → # ... modem itself is unresponsive → skip the rebind (it would just waste time) and go straight to GPIO (Level 2b). # ... (2) Level 2b's post-GPIO diagnostic uqmi query wait increased 60s→75s. Same log review: in both the Jul 10 # ... and Jul 13 successful GPIO recoveries, that diagnostic query *still* read "Failed to connect to service" at # ... the 60s mark even though the modem had, in fact, recovered (ifup+fping succeeded moments later) — the query # ... fired before the modem finished re-enumerating, producing a misleading line in the log. This query has never # ... gated the recovery decision (ifup+fping is what actually decides success), only its logged accuracy. # v20260728.03: QUECTELQMI — added Level 0: qmi_wwan driver rebind, tried before any modem-side reset # ... (Level 2b GPIO / Level 2c PWRKEY). Prompted by a 2026-08-01 event where uqmi reported "Failed to # ... connect to service" throughout, yet AT/ATI over ttyUSB2 (driver 'option') responded normally the # ... whole time — the QMI interface (driver 'qmi_wwan', same physical USB device, separate interface/ # ... driver instance) was wedged host-side, not the modem. Neither GPIO reset nor PWRKEY power-cycle # ... fixed it that day; only a full router reboot did, which points at the router's own USB/qmi_wwan # ... driver state, not the modem. The uqmi "Failed to connect to service" signature alone is now the # ... trigger: unbind/rebind just the qmi_wwan USB interface (forces the kernel to tear down and # ... recreate ${MODEMUSBDEVNODE} without power-cycling the modem or touching the AT ports), then # ... re-check with the same uqmi status query. Success clears modem_failed and falls through into the # ... existing Level 1 ifdown/ifup path for reconnection+fping verification. The complementary case — # ... uqmi responds normally (e.g. "online") but data still doesn't pass — is unchanged: that already # ... flows through Level 1 into Level 2b, since a responsive QMI stack means the fault isn't a host-side # ... wedge. (Superseded 2026-08-01.01 above: an AT probe now gates whether the rebind is even attempted.) # v20260801.02: QUECTELQMI — removed Level 2c (PWRKEY forced power-cycle) from the automatic escalation # ... chain. It was added 2026-07-28.01 on the theory that lte_rst's brief ~200ms rail cycle might not be # ... a deep enough reset for some faults. It has been exercised exactly once since (2026-08-01) and failed # ... — only a full router reboot recovered the modem that day, not PWRKEY. Combined with the Level 0 # ... finding above (that day's fault was the router's own qmi_wwan/USB host state, not the modem chip), # ... PWRKEY's failure is consistent with it never having been the actual fix: it power-cycles the modem, # ... which was never the broken part. There is no case on record where PWRKEY succeeded after lte_rst # ... failed. Escalation is now: AT/uqmi probe → qmi_wwan rebind (Level 0) → lte_rst (Level 2b) → router # ... reboot. The lte_pwrkey GPIO wiring/pin itself is untouched — only the automatic Level 2c step is gone. # v20260728.02: Fixed 'kill ' not stopping the script promptly. Every wait in the recovery # ... escalation levels (1, 2a, 2b, 2c) was a plain blocking `sleep N`, not the interruptible _sleep() # ... wrapper — only the idle LOOPWAIT used it. A plain foreground sleep defers signal/trap handling # ... until it exits, so a kill sent during recovery could sit for minutes (worst case, Level # ... 2c's 35+60+45s chain) before taking effect. All sleeps now go through _sleep(), so a plain # ... `kill ` (SIGTERM) stops the script immediately, same as before adding the PID-lockfile check. # v20260728.01: QUECTELQMI — added Level 2c: PWRKEY forced power-cycle, tried after Level 2b (lte_rst) # ... fails. lte_rst only restarts the processor/firmware (VDD_EXT/VDD_RF cycle ~200ms, VBAT never # ... drops — confirmed against Quectel EG18 hardware design Fig.17). lte_pwrkey is the module's real # ... power switch: holding it low >=800ms while running forces power-down (rails off within ~30s per # ... Fig.14); holding it low >=500ms while off forces power-on (~17s to Active, Fig.13). Prompted by # ... a 2026-07-27 event where Level 2b's brief rail cycle didn't recover the modem — theory is a # ... fault latched somewhere (USB PHY, a regulator) needed longer than ~200ms to clear. Two fixed 1s # ... pulses (power-off, wait 35s, power-on) are unambiguous here since the modem is known to have # ... been running before the crash, and it can't re-reach Running state (and thus become eligible for # ... the 800ms shutdown threshold again) within a 1s hold — full boot takes ~17s. # v20260728.01: QUECTELQMI — all `uqmi` calls now pass `-t 5000` (uqmi's own uloop-integrated response # ... timeout, ms) instead of relying solely on the external `timeout 15` wrapper. uqmi has no default # ... timeout of its own; without -t it can only be bounded by an outer process kill. -t is tied to the # ... same event loop as the pending QMI request, prints an explicit "Request timed out" on expiry (so # ... a genuine timeout is now distinguishable in the log from an immediate QMI-level error response — # ... both previously surfaced as the same "Failed to connect to service" text), and needs no outer # ... `timeout` wrapper: qmi_device_open() uses O_NONBLOCK and all uqmi I/O is non-blocking uloop/epoll, # ... so there's no code path left that -t doesn't already cover. # v20260713.01: QUECTELQMI — three changes, all based on log evidence across 9 recovery events. # ... (1) Level 2b post-GPIO reboot wait 20s→60s: EG18-NA takes ~60s to fully reboot and reattach to tower; # ... 20s was too short, firing uqmi diagnostic and ifup before the modem finished booting. Post-ifup wait # ... reduced 65s→20s: modem already tower-registered during 60s reboot, bearer negotiation completes in seconds. # ... (2) Fast-path: if initial uqmi status is "Failed to connect to service" on modem_reset() entry, Level 1 # ... is skipped and modem_failed=1 set immediately. Log shows Level 1 always fails in this state; skipping # ... saves ~90s of futile ifdown/ifup/wait. # ... (3) Level 2a (uqmi software reset) disabled by default via try_uqmi_reset=0: in 9 events it was accepted # ... once and still failed to recover the modem; in all other cases the QMI stack was unresponsive. Level 2b # ... (GPIO) is the effective fix. Code preserved; set try_uqmi_reset=1 for other Quectel modems. # v20260708.02: QUECTELQMI modem_reset() — after ifdown/ifup reports up=true, now verifies with fping before returning. # ... Quectel EG18-NA (and similar Quectel 4G modems) exhibit a known data-plane light-crash: the QMI control plane # ... reports "online" and ifstatus reports "up": true, but actual packet forwarding is broken. Sierra modems do not # ... exhibit this — if Sierra reports up=true, the data path is reliably working. For Quectel, up=true is not # ... sufficient confirmation. If fping fails after ifdown/ifup despite up=true, the script now escalates internally # ... to the full modem reset (uqmi reset → GPIO lte_rst fallback) instead of returning to the caller and triggering # ... a router reboot. This avoids the unnecessary full router reboot that happened on 2026-07-08. # v20260708.01: No functional changes. Added this note: To manually stop the watchdog, use 'kill '. The script exits cleanly. # ... (As of v20260728.02, this is immediate everywhere, including mid-recovery — see that entry.) # v20260704.01: Since the last revision, there has been further polishing by Claude Code. QUECTELQMI has been split from SIERRAQMI: The Sierra portion was very reliable, # ... as Sierra modems are reliable: #... meaning, if the modem data link failed, the communication from the router to the modem was still good. So 'uqmi' commands to RESET (reboot) the modem, work. # ... Both Sierra and Quectel use firmware from Qualcomm. But Quectel has hardware stability issues, at least with the couple of common models I have directly tested (4GLTE) in real-world operations. # ... Consequently, where SIERRAQMI and QUECTELQMI in the modem_reset() function, pointed to the same routines earlier, QUECTELQMI now has it's own section. # .... Most noteably, in the more aggressive recovery option (when a simple ifdown/ifup does not work), the router toggles the LTE_RST pin on the modem chip. # .... Again, on Qualcomm-based firmware, the terminology for a modem reboot is 'reset', but it only reboots the modem, without changing settings on it. # v20260527.01: There were possibly some typos somewhere. Anyway, I have Claude to help with some things, and essentially I noticed that somehow this script # ... was not correctly detecting the WWANDEVICE from the CONFIGURED_WWANNAME. There were some changes made and some testing and it's working. # ... The idea is you ONLY have to set your CONFIGURED_WWANNAME at the top, and the script will derive the WWANDEVICE names as needed (wwan0, eth1, wg0, wg1, etc), as used by Darkmoon # v20260501.01: Changed support over to darkmoon, a C-language-based native service for variable QOS: https://github.com/kamikaonashi/openwrt-package-darkmoon # ... The source as of this writing is missing a Makefile for luci-app-darkmoon. # ... Information on it is in the forum thread: https://forum.openwrt.org/t/cake-autorate-reborn-a-lightweight-c-rewrite-with-luci-ui-integration-v1-0-1/246965/36 # ... It is VASTLY more efficient at CPU usage, than cake-autorate, although the inspiration and what this watchdog script was originally written to use. # v20260404.01: Fixed some missing log-recording when the second-level SIERRAQMI/QUECTELQMI routine fails to bring the modem back up: # ... Was simply missing some tee -a logfile commands # ... ATINOUT swapped with SOCAT, as SOCAT is included in the main openwrt sources. # ... Added more extensive logging after second-level failure: SIM status, AT!GSTATUS? and the equivalent Quectel command # v20260310.01: [Old notes concerning SQM/CAKE removed. Trimmed changelog notes older than 20260219.01] # v20260223.01: Added some fall-through conditions. # v20260222.01: Updated check_fping was missing a ' ' (typo), causing a fall-through condition instead of corrective action. # ... Various modem_reset techniques had inconsistent delay timers after 'ifup' command. Takes around 65 seconds (maximum) in testing for QMI, but only 35-40 seconds # ... for MBIM. Having too short of a delay can cause a false-negative condition. # ... Updated the internal documentation (these notes, remarks in the script so you know what's happening) # v20260219.01: Some improvements in the layout and wording output and sent to the log, when there is an fping failure condition. # ... Added: fall-through condition for check_fping. # ... Improved: fall-through logging of variables in the Cake/SQM portion # ... Added: a GENERIC/GENERICWAN 'modem_reset' profile. This script will log if there is an issue, but the script will not do actually anything unless you add some instructions under that profile. # ... This is for people who have fiber or some other upstream, and want to use this script for it's wireguard/sqm/cake functions. # ... While it's obviously needed, this script evolved from a modem watchdog script, and turned into that + VPN then + SQM & Cake-Autorate. # ... Added: './wan-watchdog.sh test' will run in TESTMODE, regardless of the BOOTWAIT timer. Handy for trouble-shooting. # Setup: # Call this script from rc.local with '/root/wan-watchdog.sh &'. Add a sleep function prior to that, if your modem needs ~60 seconds to get online. # Your rc.local calls this script at startup: # # _sleep 60 # Gives the router a little more time to complete boot-up processes. # /root/wan-watchdog.sh & # Starts this script and allows it to run in the background. # BOOTWAIT TESTMODE determination: # When BOOTWAIT is less than 45 seconds, this script runs in Testing Mode: # Ordinary mode: designed for start-on-boot and loop, checking the connection, etc. # Testing mode: Will cause this script to run 1 time then exit, and if there is e.g. a total wwan failure, it will echo to the screen 'reboot' but not actually reboot the router. # The wait period gives your cell modem time to connect. Should be at least 45 seconds under ordinary circumstances. # Bootwait usually 45-360 seconds. Initial waiting period before testing. # Controlling it via BOOTWAIT: in case you reboot your router manually with the script in testing mode: the script does not automatically keep running, reboot or go into a boot-loop. # Prereq: verify required tools and configuration before entering the main loop. # Each check logs the failure reason and exits immediately so the problem is obvious. # 1. fping — required for all connectivity checks regardless of modem type. if ! command -v fping > /dev/null 2>&1; then echo "PREREQ FAIL: 'fping' is not installed. Install with: opkg update && opkg install fping" | tee -a "$log_file" exit 1 fi # 2. socat — required for AT command communication with cellular modems. # Not needed for GENERIC/GENERICWAN (fiber/ethernet uplinks have no modem AT interface). if [ "$MODEMTYPE" != "GENERIC" ] && [ "$MODEMTYPE" != "GENERICWAN" ]; then if ! command -v socat > /dev/null 2>&1; then echo "PREREQ FAIL: 'socat' is not installed. Required for AT commands when MODEMTYPE=${MODEMTYPE}. Install with: opkg update && opkg install socat" | tee -a "$log_file" exit 1 fi fi # 3. SQM — must not be running. SQM (sqm-scripts/luci-app-sqm) conflicts with Darkmoon on the # same WAN interface; they both manage qdisc and will interfere with each other. # If you previously used SQM, disable it before using this script with Darkmoon. if [ -f /etc/init.d/sqm ] && service sqm status 2>/dev/null | grep -q "running"; then echo "PREREQ FAIL: SQM (sqm-scripts) is running. SQM and Darkmoon cannot both manage the WAN interface. Stop and disable SQM: service sqm stop && /etc/init.d/sqm disable" | tee -a "$log_file" exit 1 fi # 4. Darkmoon — only checked if DARKMOON=1. Must be installed (not necessarily running yet). if [ "$DARKMOON" = "1" ] && ! [ -f /etc/init.d/darkmoon ]; then echo "PREREQ FAIL: DARKMOON is set to 1 but darkmoon is not installed (/etc/init.d/darkmoon not found). Install the darkmoon package or set DARKMOON=0 above." | tee -a "$log_file" exit 1 fi # Make sure there isn't another wan-watchdog.sh process running: LOCKFILE=/var/run/wan-watchdog.pid if [ -f "$LOCKFILE" ] && [ -d "/proc/$(cat "$LOCKFILE")" ]; then exit 1 fi echo $$ > "$LOCKFILE" STOP_REQUESTED=0 trap 'rm -f "$LOCKFILE"' EXIT trap 'STOP_REQUESTED=1' INT TERM # _sleep: backgrounds the sleep and waits on it, so a TERM/INT trap fires and interrupts it immediately # instead of waiting for the shell to return to a foreground command boundary. Used for every wait in # this script, including inside the recovery escalation levels, so `kill ` stops it right away — # not just during idle polling. (v20260728.02: previously only the idle LOOPWAIT used this; every # recovery-level `sleep N` was plain and blocking, which is why `kill` could take minutes to take effect # during a Level 2b/2c recovery attempt.) _sleep() { sleep "$1" & wait $!; } stop_if_requested() { if [ "$STOP_REQUESTED" = "1" ]; then echo "Stop requested — exiting cleanly." | tee -a "$log_file" exit 0 fi } # Test BOOTWAIT value to determine test run or normal run mode: if [ $BOOTWAIT -ge 45 ] && [ -z "$1" ] ; then # if greater or equal to 45 and there is nothing appended to running "./wan-watchdog.sh" echo echo "Watchdog script running in normal mode. (This message not logged)" TESTMODE=0 echo "Waiting BOOTWAIT value (${BOOTWAIT}) seconds before continuing..." sleep $BOOTWAIT # This startup delay must be first, to ensure the time and date recorded in the log is correct. else # fall-through condition: # Handle non-null, non-'test' argument if [ "$1" != "test" ] && [ -n "$1" ] ; then # if some gibberish was appended: "./wan-watchdog.sh TEST", or "... gibberish", exit. echo echo "Something was appended to run wan-watchdog.sh. It was: $1. But I don't have a condition for that: Exiting immediately..." exit # exit script immediately fi # Run in test mode: either 'test' arg given OR BOOTWAIT < 45 echo echo "Watchdog script running in test-mode." TESTMODE=1 log_file="/root/wwatch.log" echo | tee -a "$log_file" echo "Writing errors to alternative logfile: ${log_file}" echo "Wan-watchdog script started in TESTING mode on $(date)." | tee -a "$log_file" echo "Run once and exit. No loop, and no reboot if needed (e.g. unrecoverable wan connection failure)." echo echo "For ordinary startup: Increase the BOOTWAIT value or don't use 'test' switch.". fi # Check for existence of a logfile. Create one if there isn't one: if ! [ -f ${log_file} ] ; then echo # new blank line echo "Logfile ${log_file} does not exist... creating." echo "Logfile created." >> $log_file fi # include functions needed for retrieving current gateway, ipv4 wan and ipv6 wan IP addresses: . /lib/functions/network.sh # Below is the custom FPING function: (not from /lib/functions/network.sh, but made specifically for this script instead) check_fping() { # Run the fping command and check if it succeeds # Interval in ms, count number is number of failures before function returns FAILURE. You can adjust these numbers as-needed. # This pings your provider's DNS servers or those sites of your choosing, # If 1 response each from the IPV4 and IPV6 is successful, then it registers SUCCESS. # If either all IPv4 or all IPv6 completely fail, or both fail, it responds with FAILURE. local resultipv6=0 local resultipv4=0 echo "Pinging your selected ipv4 and ipv6 sites:" # edit the lines to use your provider's ipv4 and ipv6 dns servers. Global sites like dns.opendns.com may not respond. # IPV6 hosts: fping --alive --interval=400 --timeout=8000 --period=3000 --count=3 --reachable=1 --addr -6 ${DNS6SITES} || resultipv6=$? # IPV4 hosts: fping --alive --interval=400 --timeout=8000 --period=3000 --count=3 --reachable=1 --addr -4 ${DNS4SITES} || resultipv4=$? # if both ipv4 & 6 are successful, return 0 / success, and continue: if [ $resultipv6 -eq 0 ] && [ $resultipv4 -eq 0 ]; then echo "Communications working - ordinary fping success. (not logged)" return 0 # Return success # if either ipv4 or ipv6 are unsuccessful, reply with an error, create a newline in the log and post an error with the date: elif [ $resultipv6 -eq 1 ] || [ $resultipv4 -eq 1 ] ; then echo | tee -a "$log_file" echo "On $(date): ALERT!: fping:" | tee -a "$log_file" # if it was ipv6 that failed, report & log that specifically: if [ $resultipv6 -eq 1 ] ; then echo "fping: no response: FAILURE on IPv6 test!" | tee -a "$log_file" # only log on failure fi # if it was ipv4 that failed, report & log that specifically: if [ $resultipv4 -eq 1 ]; then echo "fping: no response: FAILURE on IPv4 test!" | tee -a "$log_file" # only log on failure fi return 1 # Return failure else # check_fping 'fall through' else condition. No previous conditions met: log everything and quit the script. echo "On $(date): ALERT!:" | tee -a "$log_file" echo "Fall-through condition reached, unexpected error: failure fping" | tee -a "$log_file" echo "FIXME! Some other error or condition unknown in the check_fping function section of the script" | tee -a "$log_file" echo "Current values for:" | tee -a "$log_file" echo "resultipv6: ${resultipv6}" | tee -a "$log_file" echo "resultipv4: ${resultipv4}" | tee -a "$log_file" echo "NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "MODEMTYPE: ${MODEMTYPE}" | tee -a "$log_file" echo "CONFIGURED_WWANNAME: ${CONFIGURED_WWANNAME}" | tee -a "$log_file" echo "CONFIGURED_WWANDEVICE: ${CONFIGURED_WWANDEVICE}" | tee -a "$log_file" echo "CONNECTIONSTATUS: ${CONNECTIONSTATUS}" | tee -a "$log_file" echo "WWAN_CHANGED: ${WWAN_CHANGED}" | tee -a "$log_file" echo "WWAN_CHANGED is dynamically set to whether the active wan interface was detected by this script as changed." echo "DARKMOON preference is ${DARKMOON}" | tee -a "$log_file" echo "Doing a network_flush_cache and re-reading NETx_IF_NAME..." echo "Flushing network cache and re-testing NETx_IF_NAME results.." network_flush_cache network_find_wan NET_IF_NAME network_find_wan6 NET6_IF_NAME echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "Exiting script..." | tee -a "$log_file" exit # Immediately exit script fi } safe_reboot() { if [ -f /root/rebooted.txt ]; then reboot_time=$(date -r /root/rebooted.txt +%s) now=$(date +%s) elapsed=$(( now - reboot_time )) if [ "$elapsed" -lt 900 ]; then echo "On $(date): REBOOT SUPPRESSED — rebooted.txt exists and was written only ${elapsed}s ago (within 15-minute window)." | tee -a "$log_file" echo "Not rebooting again to avoid a reboot loop (e.g. tower down). Exiting script." | tee -a "$log_file" exit fi fi echo "$(date)" > /root/rebooted.txt echo "On $(date): Writing reboot timestamp to /root/rebooted.txt." | tee -a "$log_file" service darkmoon stop echo "Rebooting router in 5 seconds..." | tee -a "$log_file" _sleep 5 reboot } at_cmd() { local cmd="$1" local wait="${2:-3}" { printf "%s\r\n" "$cmd"; _sleep "$wait"; } | timeout "$((wait + 3))" socat - /dev/ttyUSB2,b115200,raw,echo=0,icanon=0 } modem_reset() { echo | tee -a "$log_file" echo "Something wrong, fping failing. Attempting to do recovery routines for WAN/Modem Type: ${MODEMTYPE}" | tee -a "$log_file" # GENERIC WAN routine v20260219.01 (fiber/ethernet/coax cable etc, not cellular modems) if [ "$MODEMTYPE" = "GENERIC" ] || [ "$MODEMTYPE" = "GENERICWAN" ] ; then echo "Running selected GENERIC WAN recovery profile." | tee -a "$log_file" echo "We will attempt a simple 'ifup', but you should modify the script to do what you want when the WAN connection fails." | tee -a "$log_file" echo "Note! An 'online' response does not actually mean the WAN is working." | tee -a "$log_file" echo "But, if modem_reset function is running, there is no actual connectivity." | tee -a "$log_file" sleep $WAITFOR # first only probe and then log and memorize the WAN status: if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then # If up status = true, but there's no connectivity, log it first: # If 'up' status is 'true', then # The WAN is crashed-out 'soft-crash' internally, but there is no throughput: echo "Logging the previous WAN link uptime duration, in seconds:" | tee -a "$log_file" sleep $WAITFOR ifstatus ${CONFIGURED_WWANNAME} | grep "uptime" | tee -a "$log_file" # Display what the previous uptime is (when uptime > 0, it implies up:true). # Only if uptime is greater than 60 seconds (it should normally be!), then calculate and display cell link uptime in days, hours and minutes: if [ "$(ifstatus "${CONFIGURED_WWANNAME}" | grep -o '"uptime": [0-9]*' | cut -d' ' -f2)" -gt 60 ] ; then # if uptime is greater than 60 seconds (should be).. # Log & Display cell link uptime as Days, Hours and Minutes: ifstatus ${CONFIGURED_WWANNAME} | grep -o '"uptime": [0-9]*' | cut -d' ' -f2 | awk '{s=$1; d=int(s/86400); h=int(s%86400/3600); m=int(s%3600/60); print " Up for: "d" days, "h" hours, "m" minutes."}' | tee -a "$log_file" fi # set a local function variable so we can remember that the previous status was online+ifstatus was true. local previouswanstatus='true' elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false' ; then # if uptime is false, something else is wrong.. ifstatus "${CONFIGURED_WWANNAME}" | grep '"up": false' | tee -a "$log_file" # log it if the WAN is in offline/false mode. # set a local function variable so we can remember that the previous status was offline+ifstatus was false. local previouswanstatus='false' fi echo "Here's where you insert your custom routines, what you want the router to try to do to recover your WAN connection." | tee -a "$log_file" echo "For now, we will send a generic 'ifup' to the designated CONFIGURED_WWANNAME interface." | tee -a "$log_file" echo "Sending: ifup ${CONFIGURED_WWANNAME}" | tee -a "$log_file" # as reset is not the same as ifup ifup ${CONFIGURED_WWANNAME} echo "Waiting another 50 seconds for the actual IP data interface to come up..." | tee -a "$log_file" _sleep 50 # re-test what the status of the CONFIGURED_WWANNAME is, after repair steps: if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then # If up status is true, we assume everything is fixed. Display and log the result, and return out of the function ifstatus ${CONFIGURED_WWANNAME} | grep "up" | tee -a "$log_file" # We need to show the 'up' status, and uptime. Should be about 10 seconds. # If the WAN was successfully reset/recovered, it will typically show up=true, and an uptime of about 10 seconds: # > we are not going to calculate the Days Hours Minutes again, because it would only have been a few seconds of uptime. return # Exits the modem restart function. Let fping do a follow-up test to the connection. If it fails, the router will reboot! # 'up:false' - Else, if the up status is was and is still 'false', then this could a more abnormal condition. elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false' && [ "$previouswanstatus" = "false" ]; then # The WAN is in an abnormal condition. 'ifup' etc did not work. Log and exit function. ifstatus ${CONFIGURED_WWANNAME} | grep "up" | tee -a "$log_file" # log and display the up status. echo "*!ALERT!* exiting, unresolvable condition for the ${MODEMTYPE}." | tee -a "$log_file" echo "GENERIC WAN abnormally stuck in offline mode! Investigate it manually. Exiting script directly, to avoid unnecessary reboot which probably will not fix the issue.." | tee -a "$log_file" exit # Exit the script # 'up:false' - Else, if the up status is 'false', and it was previous up:true, this is an abnormal condition. Report and exit script. elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false' && [ "$previouswanstatus" = "true" ]; then # The WAN is in an abnormal condition. 'ifup' etc did not work. Log and exit function. ifstatus ${CONFIGURED_WWANNAME} | grep "up" | tee -a "$log_file" # log and display the up status. echo "*!ALERT!* exiting, unresolvable condition for the ${MODEMTYPE}." | tee -a "$log_file" echo "GENERIC WAN is in offline(up:false) mode, but was previously reporting up:true! Investigate it manually. Exiting script directly, to avoid unnecessary reboot which probably will not fix the issue.." | tee -a "$log_file" exit # Exit the script else # v20260219.01 Fall through for exception in GENERICWAN fix conditional procedure echo | tee -a "$log_file" echo "On $(date): ALERT!:" | tee -a "$log_file" echo "Fall-through condition reached in GENERICWAN portion of modem_reset function, unexpected error: failure" | tee -a "$log_file" echo "aborting entire script...." | tee -a "$log_file" echo "Current values for:" | tee -a "$log_file" echo "previouswanstatus: ${previouswanstatus}" | tee -a "$log_file" echo "MODEMTYPE: ${MODEMTYPE}" | tee -a "$log_file" echo "NVRELOAD: ${NVRELOAD}" | tee -a "$log_file" echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "CONFIGURED_WWANNAME: ${CONFIGURED_WWANNAME}" | tee -a "$log_file" echo "CONFIGURED_WWANDEVICE: ${CONFIGURED_WWANDEVICE}" | tee -a "$log_file" echo "CONNECTIONSTATUS: ${CONNECTIONSTATUS}" | tee -a "$log_file" echo "WWAN_CHANGED: ${WWAN_CHANGED}" | tee -a "$log_file" echo "WWAN_CHANGED is also used to set whether the active wan interface was detected by this script as changed." echo "DARKMOON: is ${DARKMOON}" | tee -a "$log_file" echo "Doing a network_flush_cache and re-reading NETx_IF_NAME..." network_flush_cache network_find_wan NET_IF_NAME network_find_wan6 NET6_IF_NAME echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "Exiting script..." | tee -a "$log_file" exit # Immediately exit script fi # SIERRAQMI routine v20260404.01 # 20260404: fixed some missing instructions to save the output to log file. # 20260222: restructured the logic in detecting up/down status. It uses the same method but the logical path it follows is clearer: # It reports what the 'ifstatus' responds with, regardless if it's up/true or down/false. It also now saves that status in a local variable for comparison, later. # Then it takes action and and re-checks the status, and compares it with the previous result. elif [ "$MODEMTYPE" = "SIERRAQMI" ] ; then echo "Querying online status via uqmi:" | tee -a "$log_file" echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode..." | tee -a "$log_file" uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode | tee -a "$log_file" # Log what uqmi cli *thinks* the modem status is (for reference). echo "Note! An 'online' response does not actually mean the modem is working. Typically, they crash-out in the 'online' state" | tee -a "$log_file" echo "But if this function is running, there is no actual connectivity." | tee -a "$log_file" sleep $WAITFOR # check reported uptime, if it is up, and log if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then # If up status = true, let's log that and uptime, first: # If 'up' status is 'true', then # The modem is crashed-out 'soft-crash' internally, but there is no throughput: echo "'ifstatus ${CONFIGURED_WWANNAME}' is responding that connection is 'up', although since this routine (modem_reset) is running, then the cell modem is probably 'light'-crashed internally." | tee -a "$log_file" echo "Logging the previous cell link uptime duration, in seconds:" | tee -a "$log_file" sleep $WAITFOR ifstatus ${CONFIGURED_WWANNAME} | grep "uptime" | tee -a "$log_file" # Display what the previous uptime is (when uptime > 0, it implies up:true). # Only if uptime is greater than 60 seconds (it should normally be!), then calculate and display cell link uptime in days, hours and minutes: if [ "$(ifstatus "${CONFIGURED_WWANNAME}" | grep -o '"uptime": [0-9]*' | cut -d' ' -f2)" -gt 60 ] ; then # if uptime is greater than 60 seconds (should be).. # Log & Display cell link uptime as Days, Hours and Minutes: ifstatus ${CONFIGURED_WWANNAME} | grep -o '"uptime": [0-9]*' | cut -d' ' -f2 | awk '{s=$1; d=int(s/86400); h=int(s%86400/3600); m=int(s%3600/60); print " Up for: "d" days, "h" hours, "m" minutes."}' | tee -a "$log_file" fi # set a local function variable so we can remember that the previous status was online+ifstatus was true. local previouswanstatus='true' # If false/down, log it, and make a note of the current status, for later comparison: elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false'; then echo "'ifstatus ${CONFIGURED_WWANNAME}' is intially responding that connection is 'down'." | tee -a "$log_file" # set a local function variable so we can remember that the previous status was offline+ifstatus was false. local previouswanstatus='false' fi # now take a simple ifdown/ifup action, usually successful: sleep $WAITFOR echo "Attempting basic ifdown then ifup on ${CONFIGURED_WWANNAME}, to re-establish connectivity:" | tee -a "$log_file" ifdown ${CONFIGURED_WWANNAME} _sleep 5 ifup ${CONFIGURED_WWANNAME} echo "Wait 65 seconds to give the modem time to reconnect..." | tee -a "$log_file" _sleep 65 # give the modem time to reconnect (takes at least 25 seconds for a typical QMI Quectel config. YMMV) # if it responds as up, we consider it a success if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true' ; then # Log the up status and uptime, presuming a successful reconnect, pass back to fping to check. ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" return # Seems successful! Exit the modem restart function for a follow-up check_fping # if it is reporting subsequently, that the up status is (still) false/down, log it and do a more invasive approach: # invasive process: elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false' ; then ifstatus "${CONFIGURED_WWANNAME}" | grep '"up": false' | tee -a "$log_file" # log it if the modem is in offline/false mode. # try a more invasive approach to getting the wan interface up echo "Previous simple ifdown/ifup unsuccessful." | tee -a "$log_file" echo "Full RESET of modem:" | tee -a "$log_file" echo "Sending ifdown ${CONFIGURED_WWANNAME} to stop QMI before modem reset..." | tee -a "$log_file" ifdown ${CONFIGURED_WWANNAME} _sleep 4 echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --set-device-operating-mode reset" | tee -a "$log_file" uqmi -t 5000 -d ${MODEMUSBDEVNODE} --set-device-operating-mode reset | tee -a "$log_file" echo "Wait 65 seconds to give the modem time to reset and re-enumerate on USB..." | tee -a "$log_file" _sleep 65 # Extended from 25s — modem re-enumerates on USB after reset, takes longer than a simple reconnect echo "Getting operating mode status..." | tee -a "$log_file" echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode" | tee -a "$log_file" uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode | tee -a "$log_file" echo "Sending: ifup ${CONFIGURED_WWANNAME}" | tee -a "$log_file" # as reset is not the same as ifup ifup ${CONFIGURED_WWANNAME} echo "Waiting 65 seconds for the actual IP data interface to come up..." | tee -a "$log_file" _sleep 65 # followup testing, after more invasive process, checking modem self-reporting online/offline status # Result: (1) we tried a simple 'ifup' to fix a no-throughput condition on the wan, but (2) the follow-up up status returned down/false. # so (3) we tried a more advanced reset + ifup, and (4) now it's (reported to be) working. # success! if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true' ; then echo "SUCCESS! Modem reporting:" | tee -a "$log_file" ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" # Log the up status and uptime, presumes a successful reconnect (if not, the followup fping will reboot the router when detecting the failure). return # Success! Exit the modem restart function for a follow-up check_fping fi # same as (1,2,3), but (4) the follow-up status still returns false/down both times. Hard down condition. # sadness if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false' && [ "$previouswanstatus" = "false" ]; then # if uptime is still false, something is wrong.. echo "FAILURE! Modem reporting:" | tee -a "$log_file" ifstatus "${CONFIGURED_WWANNAME}" | grep '"up": false' | tee -a "$log_file" # log the status is in offline/false mode. echo "Some sort of severe error condition:" | tee -a "$log_file" echo " • WAN Connection ${MODEMTYPE} had a no-throughput condition." | tee -a "$log_file" echo " • Modem initially reported as down/false/offline." | tee -a "$log_file" echo " • The basic reconnect attempted, with no up/true response, then advanced reconnect/reset, also with no up/true reported." | tee -a "$log_file" echo " • Modem still reporting as down/false/offline." | tee -a "$log_file" echo "modem_reset could not recover the connection. Leaving reboot decision to the caller's post-modem_reset fping check (NEEDREBOOT/CONNECTIONSTATUS, TESTMODE-gated)." | tee -a "$log_file" # unusual condition, can happen sometimes due to modem corruption, sim slot issues # ... (4) the follow-up status returned down/false, yet it was originally reported true (with no throughput) elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false' && [ "$previouswanstatus" = "true" ]; then # if uptime has reverted to false, something is very wrong.. ifstatus "${CONFIGURED_WWANNAME}" | grep '"up": false' | tee -a "$log_file" # log it. # this is an odd condition, usually related to modem corruption of some sort. It was previously self-reporting 'up' with no throughput, # and after subsequent attempts to bring it truly online, it is now reporting down/offline/false. echo "Some sort of severe error condition:" | tee -a "$log_file" echo " • WAN Connection ${MODEMTYPE} had a no-throughput condition." | tee -a "$log_file" echo " • Modem previously reported online/true/up, as it does, despite no throughput condition." | tee -a "$log_file" echo " • Neither the basic nor advanced reconnect/reset was successful." | tee -a "$log_file" echo " • And in the end, the modem reverted/reports as down/false/offline." | tee -a "$log_file" echo " • Sometimes, it is a SIM problem:" | tee -a "$log_file" echo " • Recording AT+CPIN? status:" | tee -a "$log_file" at_cmd "AT+CPIN?" | tee -a "$log_file" echo " • If output above shows SIM_STATUS_ILLEGAL, you may simply have to re-insert the SIM, and if your device does not detect hotswapped SIMs, manually send a AT+CFUN=1,1 to reboot the modem only, then manually check again with AT+CPIN?. It should say READY." | tee -a "$log_file" echo " • Recording AT!GSTATUS? (only works for SIERRA):" | tee -a "$log_file" at_cmd "AT!GSTATUS?" 5 | tee -a "$log_file" echo "modem_reset could not recover the connection. Leaving reboot decision to the caller's post-modem_reset fping check (NEEDREBOOT/CONNECTIONSTATUS, TESTMODE-gated)." | tee -a "$log_file" fi else # v20260219.01 Fall through for SIERRAQMI: echo | tee -a "$log_file" echo "On $(date): ALERT!:" | tee -a "$log_file" echo "Fall-through condition reached in QMI portion of modem_reset function, unexpected error: failure" | tee -a "$log_file" echo "Current values for:" | tee -a "$log_file" echo "previouswanstatus: ${previouswanstatus}" | tee -a "$log_file" echo "MODEMTYPE: ${MODEMTYPE}" | tee -a "$log_file" echo "NVRELOAD: ${NVRELOAD}" | tee -a "$log_file" echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "CONFIGURED_WWANNAME: ${CONFIGURED_WWANNAME}" | tee -a "$log_file" echo "CONNECTIONSTATUS: ${CONNECTIONSTATUS}" | tee -a "$log_file" echo "WWAN_CHANGED: ${WWAN_CHANGED}" | tee -a "$log_file" echo "WWAN_CHANGED is also used to set whether the active wan interface was detected by this script as changed." echo "DARKMOON: is ${DARKMOON}" | tee -a "$log_file" echo "Doing a network_flush_cache and re-reading NETx_IF_NAME (in case there is a variance).." network_flush_cache network_find_wan NET_IF_NAME network_find_wan6 NET6_IF_NAME echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "Exiting script..." | tee -a "$log_file" exit # Immediately exit script fi # QUECTELQMI routine v20260801.01 # Linear recovery flow using modem_failed state variable. Unlike SIERRAQMI, Quectel modems # commonly report ifstatus=up while the data plane is broken, so fping is used to verify # connectivity after each level — not just ifstatus. Sierra modems do not exhibit this. # Level 0: (only if uqmi reports "Failed to connect to service") AT probe on ttyUSB2 → # if AT responds, qmi_wwan driver unbind/rebind → re-check uqmi status. # If AT does not respond, skip straight to Level 2b. # Level 1: ifdown/ifup → ifstatus + fping (skipped if QMI stack unresponsive on entry) # Level 2a: uqmi software reset → ifstatus + fping (disabled by default via try_uqmi_reset=0; # has not produced a recovery on EG18-NA — set try_uqmi_reset=1 for other Quectel modems) # Level 2b: GPIO lte_rst hardware reset → ifdown → ifup → ifstatus + fping # modem_failed=1 on any failure; cleared only on confirmed fping success (or Level 0 restoring QMI # responsiveness). If all levels are exhausted, modem_reset just returns — it never calls safe_reboot # directly (fixed 2026-08-04, see changelog). The caller's own post-modem_reset check_fping sets # NEEDREBOOT/CONNECTIONSTATUS, and only the TESTMODE-gated block at the end of the script decides # whether to actually reboot. No PWRKEY power-cycle step — removed 2026-08-01.02, see changelog: # never once fixed anything a full reboot didn't, and the modem chip was never the actual fault in # the case it was added for. elif [ "$MODEMTYPE" = "QUECTELQMI" ] ; then local modem_failed=0 local uqmi_reset_result local initial_uqmi_status local qmi_usb_iface local rebind_uqmi_status local at_response # try_uqmi_reset: set to 1 to enable Level 2a (uqmi software reset) before GPIO. # Disabled by default — on EG18-NA, the reset command has never produced a recovery: # when the QMI stack is responsive it is accepted but the modem does not come back; # when unresponsive it fails immediately. GPIO (Level 2b) is the reliable fix. # Other Quectel modems may respond differently — set to 1 to find out. local try_uqmi_reset=0 # --- Initial diagnostics --- echo "Querying online status via uqmi:" | tee -a "$log_file" echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode..." | tee -a "$log_file" initial_uqmi_status=$(uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode 2>&1) echo "$initial_uqmi_status" | tee -a "$log_file" echo "Note! An 'online' response does not actually mean the modem is working. Typically, they crash-out in the 'online' state" | tee -a "$log_file" echo "But if this function is running, there is no actual connectivity." | tee -a "$log_file" sleep $WAITFOR if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then echo "'ifstatus ${CONFIGURED_WWANNAME}' is responding that connection is 'up', although since this routine (modem_reset) is running, then the cell modem is probably 'light'-crashed internally." | tee -a "$log_file" echo "Logging the previous cell link uptime duration, in seconds:" | tee -a "$log_file" sleep $WAITFOR ifstatus ${CONFIGURED_WWANNAME} | grep "uptime" | tee -a "$log_file" if [ "$(ifstatus "${CONFIGURED_WWANNAME}" | grep -o '"uptime": [0-9]*' | cut -d' ' -f2)" -gt 60 ] ; then ifstatus ${CONFIGURED_WWANNAME} | grep -o '"uptime": [0-9]*' | cut -d' ' -f2 | awk '{s=$1; d=int(s/86400); h=int(s%86400/3600); m=int(s%3600/60); print " Up for: "d" days, "h" hours, "m" minutes."}' | tee -a "$log_file" fi elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false'; then echo "'ifstatus ${CONFIGURED_WWANNAME}' is initially responding that connection is 'down'." | tee -a "$log_file" fi # Fast-path: if QMI stack is completely unresponsive on entry, skip Level 1 directly to Level 2b. # Level 1 (ifdown/ifup) and Level 2a (uqmi reset) both always fail when uqmi returns "Failed to # connect to service" — confirmed across all such events in the log. Level 1 adds ~90s of futile # waiting in this state. Level 2a runs briefly regardless (its own ifdown handles interface teardown # before the GPIO pulse). Other Quectel modems where uqmi is responsive still go through Level 1→2a. if echo "$initial_uqmi_status" | grep -q "Failed to connect to service"; then echo "QMI stack unresponsive on entry — skipping Level 1 (ifdown/ifup), proceeding to Level 0/2a/2b." | tee -a "$log_file" modem_failed=1 fi # --- Level 0: AT probe, then qmi_wwan driver rebind (host-side fix, tried before any modem reset) --- # "Failed to connect to service" means the QMI control-plane handshake got no response — that alone # does not tell us whether the modem itself crashed or just the QMI side of it. The AT interface # (ttyUSB2, kernel driver 'option') and the QMI interface (${MODEMUSBDEVNODE}, kernel driver # 'qmi_wwan') are separate USB interfaces on the same physical device, each owned by an independent # driver instance, so probing AT tells us which case we're in: # - AT responds: the modem is alive; the QMI side is wedged host-side (a stuck URB, a lost # callback in qmi_wwan) — confirmed 2026-08-01, where AT/ATI kept responding normally over # ttyUSB2 the whole time uqmi reported "Failed to connect to service", and only a full router # reboot (resets the router's own USB host stack) cleared it, not a modem-side GPIO/PWRKEY cycle. # In this case, unbind/rebind just the qmi_wwan interface — forces the kernel to tear down and # recreate ${MODEMUSBDEVNODE} without power-cycling the modem or touching the AT ports. # - AT does not respond: the modem itself is unresponsive, not just QMI — confirmed by log review # of 2026-07-10 and 2026-07-13, where the *initial* uqmi failure was fixed by an actual GPIO # lte_rst reset, not anything host-side. Rebinding qmi_wwan wouldn't have helped there, so skip # straight to GPIO (Level 2b) rather than spend ~7s on a rebind that can't fix a dead modem. if [ "$modem_failed" = "1" ]; then echo "Level 0: uqmi reported QMI-stack failure — probing AT interface (ttyUSB2) to tell a host-side qmi_wwan wedge apart from a genuinely unresponsive modem..." | tee -a "$log_file" at_response=$(at_cmd "ATI" 3) echo "$at_response" | tee -a "$log_file" if echo "$at_response" | grep -q "OK"; then echo "AT interface responsive — modem is alive; treating this as a host-side qmi_wwan wedge. Rebinding qmi_wwan driver for ${MODEMUSBDEVNODE} before attempting any modem-side reset..." | tee -a "$log_file" qmi_usb_iface=$(basename "$(readlink -f "/sys/class/usbmisc/$(basename "${MODEMUSBDEVNODE}")/device")") if [ -n "$qmi_usb_iface" ] && [ -e "/sys/bus/usb/drivers/qmi_wwan/${qmi_usb_iface}" ]; then echo "Unbinding ${qmi_usb_iface} from qmi_wwan..." | tee -a "$log_file" echo "$qmi_usb_iface" > /sys/bus/usb/drivers/qmi_wwan/unbind _sleep 2 echo "Rebinding ${qmi_usb_iface} to qmi_wwan..." | tee -a "$log_file" echo "$qmi_usb_iface" > /sys/bus/usb/drivers/qmi_wwan/bind echo "Waiting 5 seconds for ${MODEMUSBDEVNODE} to reappear..." | tee -a "$log_file" _sleep 5 echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode (post-rebind status)" | tee -a "$log_file" rebind_uqmi_status=$(uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode 2>&1) echo "$rebind_uqmi_status" | tee -a "$log_file" if ! echo "$rebind_uqmi_status" | grep -q "Failed to connect to service"; then echo "Level 0 recovery successful — QMI stack responsive again after qmi_wwan rebind. Proceeding to reconnect (Level 1)." | tee -a "$log_file" modem_failed=0 else echo "FAIL: Level 0 — qmi_wwan rebind did not restore QMI responsiveness. Proceeding to modem-side reset." | tee -a "$log_file" fi else echo "FAIL: Level 0 — could not resolve qmi_wwan USB interface path for ${MODEMUSBDEVNODE}. Proceeding to modem-side reset." | tee -a "$log_file" fi else echo "AT interface not responding either — modem itself appears unresponsive, not just a host-side driver wedge. Skipping qmi_wwan rebind, proceeding directly to modem-side reset (GPIO)." | tee -a "$log_file" fi fi # --- Level 1: basic ifdown/ifup --- if [ "$modem_failed" = "0" ]; then sleep $WAITFOR echo "Attempting basic ifdown then ifup on ${CONFIGURED_WWANNAME}, to re-establish connectivity:" | tee -a "$log_file" ifdown ${CONFIGURED_WWANNAME} _sleep 5 ifup ${CONFIGURED_WWANNAME} echo "Waiting 65 seconds to give the modem time to reconnect..." | tee -a "$log_file" _sleep 65 if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" echo "Level 1: ifstatus=up after ifdown/ifup. Verifying with fping (Quectel data-plane check)..." | tee -a "$log_file" echo "Waiting 20 seconds for connection to settle before fping test..." | tee -a "$log_file" _sleep 20 if check_fping; then echo "Level 1 recovery successful — connectivity confirmed after ifdown/ifup." | tee -a "$log_file" return else echo "FAIL: Level 1 — ifstatus=up but fping still failing. Quectel data-plane light-crash confirmed." | tee -a "$log_file" modem_failed=1 fi else ifstatus "${CONFIGURED_WWANNAME}" | grep '"up"' | tee -a "$log_file" echo "FAIL: Level 1 — ifstatus=false after ifdown/ifup." | tee -a "$log_file" modem_failed=1 fi fi # --- Level 2a: uqmi software reset --- if [ "$modem_failed" = "1" ] && [ "$try_uqmi_reset" = "1" ]; then echo "Level 2a: attempting uqmi software reset..." | tee -a "$log_file" ifdown ${CONFIGURED_WWANNAME} _sleep 4 echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --set-device-operating-mode reset" | tee -a "$log_file" uqmi_reset_result=$(uqmi -t 5000 -d ${MODEMUSBDEVNODE} --set-device-operating-mode reset 2>&1) echo "$uqmi_reset_result" | tee -a "$log_file" if echo "$uqmi_reset_result" | grep -q "Failed to connect to service"; then echo "FAIL: Level 2a — uqmi reset returned 'Failed to connect to service'. QMI stack unresponsive; modem did not receive reset command. Skipping to GPIO reset." | tee -a "$log_file" # modem_failed stays 1; no point waiting for a reset that was never sent else echo "Level 2a: uqmi reset accepted. Waiting 65 seconds for modem to re-enumerate on USB..." | tee -a "$log_file" _sleep 65 echo "Sending: ifup ${CONFIGURED_WWANNAME}" | tee -a "$log_file" ifup ${CONFIGURED_WWANNAME} echo "Waiting 65 seconds for the IP data interface to come up..." | tee -a "$log_file" _sleep 65 if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" echo "Level 2a: ifstatus=up after uqmi reset. Verifying with fping..." | tee -a "$log_file" if check_fping; then echo "Level 2a recovery successful — connectivity confirmed after uqmi reset." | tee -a "$log_file" modem_failed=0 return else echo "FAIL: Level 2a — ifstatus=up but fping still failing after uqmi reset. Proceeding to GPIO reset." | tee -a "$log_file" # modem_failed stays 1 fi else ifstatus "${CONFIGURED_WWANNAME}" | grep '"up"' | tee -a "$log_file" echo "FAIL: Level 2a — ifstatus=false after uqmi reset. Proceeding to GPIO reset." | tee -a "$log_file" # modem_failed stays 1 fi fi fi # --- Level 2b: GPIO lte_rst hardware reset --- # GPIO lte_rst is a modem hardware cold-start (equivalent to power-cycling the modem chip). # ifdown before the pulse ensures clean interface state; ifup after tells netifd to reconnect # once the modem has re-enumerated. The 75s GPIO wait is only for USB re-enumerate / modem # boot. ifup qmi still needs the usual ~65s for the QMI data session — do not shorten it. if [ "$modem_failed" = "1" ]; then echo "Level 2b: GPIO lte_rst hardware reset..." | tee -a "$log_file" ifdown ${CONFIGURED_WWANNAME} _sleep 4 echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode (pre-GPIO status)" | tee -a "$log_file" uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode | tee -a "$log_file" echo "Pulsing GPIO lte_rst low for 1 second to force modem hardware reboot..." | tee -a "$log_file" echo 0 > /sys/class/gpio/lte_rst/value _sleep 1 echo 1 > /sys/class/gpio/lte_rst/value echo "GPIO lte_rst pulsed. Waiting 75 seconds for modem to reboot and re-enumerate on USB..." | tee -a "$log_file" _sleep 75 stop_if_requested echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode (post-GPIO status)" | tee -a "$log_file" post_gpio_uqmi=$(uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode 2>&1) echo "$post_gpio_uqmi" | tee -a "$log_file" if echo "$post_gpio_uqmi" | grep -q "Failed to connect to service"; then echo "Level 2b: uqmi unresponsive after GPIO — rebinding qmi_wwan before ifup..." | tee -a "$log_file" qmi_usb_iface=$(basename "$(readlink -f "/sys/class/usbmisc/$(basename "${MODEMUSBDEVNODE}")/device")") if [ -n "$qmi_usb_iface" ] && [ -e "/sys/bus/usb/drivers/qmi_wwan/${qmi_usb_iface}" ]; then echo "Unbinding ${qmi_usb_iface} from qmi_wwan..." | tee -a "$log_file" echo "$qmi_usb_iface" > /sys/bus/usb/drivers/qmi_wwan/unbind _sleep 2 echo "Rebinding ${qmi_usb_iface} to qmi_wwan..." | tee -a "$log_file" echo "$qmi_usb_iface" > /sys/bus/usb/drivers/qmi_wwan/bind echo "Waiting 5 seconds for ${MODEMUSBDEVNODE} to reappear..." | tee -a "$log_file" _sleep 5 echo "Sending: uqmi -d ${MODEMUSBDEVNODE} --get-device-operating-mode (post-rebind status)" | tee -a "$log_file" uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-device-operating-mode 2>&1 | tee -a "$log_file" else echo "FAIL: Level 2b — could not resolve qmi_wwan USB interface path for ${MODEMUSBDEVNODE}. Continuing to ifup." | tee -a "$log_file" fi fi echo "Sending: ifup ${CONFIGURED_WWANNAME}" | tee -a "$log_file" ifup ${CONFIGURED_WWANNAME} qmi_delay=$(uci -q get "network.${CONFIGURED_WWANNAME}.delay") [ -z "$qmi_delay" ] && qmi_delay=0 ifup_ceiling=$((qmi_delay + 65)) echo "Polling ifstatus for up to ${ifup_ceiling}s after ifup (qmi delay ${qmi_delay}s + 65s)..." | tee -a "$log_file" waited=0 ifstatus_up=0 while [ "$waited" -lt "$ifup_ceiling" ]; do stop_if_requested if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then ifstatus_up=1 break fi _sleep 5 waited=$((waited + 5)) done if [ "$ifstatus_up" = "1" ]; then ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" echo "Level 2b: ifstatus=up after GPIO reset (${waited}s). Verifying with fping..." | tee -a "$log_file" if check_fping; then echo "Level 2b recovery successful — connectivity confirmed after GPIO reset." | tee -a "$log_file" modem_failed=0 return else echo "FAIL: Level 2b — ifstatus=up but fping still failing after GPIO reset." | tee -a "$log_file" # modem_failed stays 1 fi else ifstatus "${CONFIGURED_WWANNAME}" | grep '"up"' | tee -a "$log_file" echo "FAIL: Level 2b — ifstatus still false after ${ifup_ceiling}s poll." | tee -a "$log_file" # modem_failed stays 1 fi fi # --- All recovery levels exhausted --- # No Level 2c (PWRKEY power-cycle) here — removed 2026-08-01.02. It was tried exactly once # (2026-08-01) after Level 2b failed, and also failed; only a full router reboot recovered the # modem that day. Since that day's actual fault was a host-side qmi_wwan/USB wedge (now caught # earlier by Level 0), not the modem chip itself, PWRKEY power-cycling the modem was never going # to be the fix. There is no case on record where PWRKEY succeeded after lte_rst failed. if [ "$modem_failed" = "1" ]; then echo "QUECTELQMI: all recovery levels exhausted (Level 0 AT probe/qmi_wwan rebind, Level 1 ifdown/ifup, Level 2a uqmi reset, Level 2b GPIO lte_rst)." | tee -a "$log_file" echo " • AT+CPIN? (SIM status):" | tee -a "$log_file" at_cmd "AT+CPIN?" | tee -a "$log_file" echo " • If AT+CPIN? shows SIM_STATUS_ILLEGAL, try re-inserting the SIM or sending AT+CFUN=1,1 to reboot the modem only, then re-check with AT+CPIN?. It should return READY." | tee -a "$log_file" echo " • AT+QENG=\"servingcell\" (cell status):" | tee -a "$log_file" qeng_out=$(at_cmd 'AT+QENG="servingcell"' 5) echo "$qeng_out" | tee -a "$log_file" echo " • uqmi --get-serving-system (QMI registration state):" | tee -a "$log_file" uqmi -t 5000 -d ${MODEMUSBDEVNODE} --get-serving-system 2>&1 | tee -a "$log_file" echo " • AT+CEREG? (EPS registration):" | tee -a "$log_file" cereg_out=$(at_cmd "AT+CEREG?") echo "$cereg_out" | tee -a "$log_file" echo " • AT+COPS? (operator selection):" | tee -a "$log_file" cops_out=$(at_cmd "AT+COPS?") echo "$cops_out" | tee -a "$log_file" stop_if_requested at_registered=0 echo "$cereg_out" | grep -qE '\+CEREG: [0-9]+,[15]' && at_registered=1 echo "$cops_out" | grep -qE '\+COPS:.*,".+"' && at_registered=1 if echo "$qeng_out" | grep -q 'servingcell' && ! echo "$qeng_out" | grep -qi 'SEARCH'; then at_registered=1 fi if [ "$at_registered" = "1" ]; then echo "AT shows registered (CEREG/COPS/QENG) — not treating this as reboot-worthy yet; polling ifstatus/fping (netifd may still be catching up)..." | tee -a "$log_file" waited=0 while [ "$waited" -lt 65 ]; do stop_if_requested if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" echo "ifstatus=up during AT-registered poll (${waited}s). Verifying with fping..." | tee -a "$log_file" if check_fping; then echo "Level 2b recovery successful — connectivity confirmed after AT-registered poll." | tee -a "$log_file" modem_failed=0 return fi break fi _sleep 5 waited=$((waited + 5)) done if [ "$modem_failed" = "1" ]; then echo "AT was registered but ifstatus/fping still failing after extra 65s poll." | tee -a "$log_file" fi fi if [ "$modem_failed" = "1" ]; then echo "modem_reset could not recover the connection. Leaving reboot decision to the caller's post-modem_reset fping check (NEEDREBOOT/CONNECTIONSTATUS, TESTMODE-gated)." | tee -a "$log_file" fi fi # SIERRAMBIM routine v20260201.01 elif [ "$MODEMTYPE" = "SIERRAMBIM" ] ; then # check and log the status of the modem: true if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then # If up status = true, go through the modem reset / reboot procedure for the 'soft-crash' condition: # If 'up' status is 'true', then # The modem is crashed-out 'soft-crash' internally, but there is no throughput: # up:true - modem responds to a status query as 'up', but typically in this state where fping has failed, there is no throughput. # ... this is the typical crashed condition for quectel and sierra modems. echo "'ifstatus ${CONFIGURED_WWANNAME}' is responding that connection is 'up', although since this routine (modem_reset) is running, then the cell modem is probably 'light'-crashed internally." echo "Logging the previous cell link uptime duration, in seconds:" | tee -a "$log_file" ifstatus ${CONFIGURED_WWANNAME} | grep "uptime" | tee -a "$log_file" # Display what the previous uptime is (when uptime > 0, it implies up:true). # Only if uptime is greater than 60 seconds (it should normally be at this point!), then calculate and display cell link uptime in days, hours and minutes: if [ "$(ifstatus "${CONFIGURED_WWANNAME}" | grep -o '"uptime": [0-9]*' | cut -d' ' -f2)" -gt 60 ] ; then # if uptime is greater than 60 seconds (should be).. # Display cell link uptime as Days, Hours and Minutes: ifstatus ${CONFIGURED_WWANNAME} | grep -o '"uptime": [0-9]*' | cut -d' ' -f2 | awk '{s=$1; d=int(s/86400); h=int(s%86400/3600); m=int(s%3600/60); print " Up for: "d" days, "h" hours, "m" minutes."}' | tee -a "$log_file" fi echo "Note! An 'up: true', response does not actually mean the modem interface is fully working." echo "Typically, they crash-out internally in the 'online' / 'true' state with no throughput." echo "but critically, there is no actual connectivity." sleep $WAITFOR echo "${MODEMTYPE} restart procedure:" | tee -a "$log_file" echo "Attempting to send AT!RESET (reboot of the cell modem only):" | tee -a "$log_file" local resultatreset resultatreset=$(at_cmd "AT!RESET") echo "$resultatreset" | tee -a "$log_file" echo "Wait 35 seconds to give the modem time to boot and reconnect." | tee -a "$log_file" _sleep 35 # Mandatory pause minimum for the cell modem to reconnect to tower. echo "Re-querying status with 'AT!GSTATUS?'" | tee -a "$log_file" local resultgstatus # modem self-status to tower, provider, regardless of the 'ifup/ifstatus' openwrt interface status resultgstatus=$(at_cmd "AT!GSTATUS?" 5) echo "$resultgstatus" | tee -a "$log_file" echo "'ifup ${CONFIGURED_WWANNAME}' & pause afterwards, waiting for interface to come back up" ifup ${CONFIGURED_WWANNAME} | tee -a "$log_file" echo "Mandatory: Waiting 45 seconds for the openwrt IP interface data to come up..." | tee -a "$log_file" _sleep 45 # wait for interface to come back up/reload. ifstatus ${CONFIGURED_WWANNAME} | grep "up" | tee -a "$log_file" # We need to show the 'up' status, and uptime. Should be about 10 seconds. # If the modem was successfully reset, it will show up=true, and an uptime of about 10 seconds: # we had success since the status of up is true. # we are not going to calculate the Days Hours Minutes again, because it would only have been a few seconds of uptime. return # Exits the modem restart function. # 'up:false' - Else, if the up status is initially 'false', then this could a more abnormal condition, especially if the modem was working just-prior. # Could be the tower, or a corruption issue in the modem. elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false'; then # Display results and log: ifstatus ${CONFIGURED_WWANNAME} | grep "up" | tee -a "$log_file" # log that it's 'false' (says 'cell phone link is down') echo "** 'ifstatus ${CONFIGURED_WWANNAME}' is reporting the cell modem link is down (false)." | tee -a "$log_file" echo " If the tower is up (please verify), this condition usually means the unusual instance when the modem had a corruption of it's internal registers." | tee -a "$log_file" echo " NVRELOAD set to ${NVRELOAD}." sleep $WAITFOR # do a basic ifup to see if the interface is simply 'down'. Usually this step is not necessary, but in edge cases, maybe it was manually brought down prior to running this script echo " Attempting to 'ifup' the modem interface - maybe it's simply down?" | tee -a "$log_file" echo " 'ifup ${CONFIGURED_WWANNAME}' & pause afterwards, while waiting for interface to come back up" | tee -a "$log_file" ifup ${CONFIGURED_WWANNAME} | tee -a "$log_file" echo " Mandatory: Waiting 45 seconds for the openwrt IP interface data to come up..." | tee -a "$log_file" _sleep 45 # wait for interface to come back up/reload. # testing result, when started with a down+false condition: if ifstatus "${CONFIGURED_WWANNAME}" | grep '"up": true'; then # We need to check the 'up' status. Uptime should be about 10 seconds. ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" # Log the up status and uptime, presumes a successful reconnect (if not, the followup fping will reboot the router when detecting the failure). echo "'ifup ${CONFIGURED_WWANNAME}' success. Modem responds as up. Exiting modem_reset function, to further test with fping.." # If the modem was successfully reset, it will show up=true, and an uptime of about 10 seconds: # we had success since the status of up is true. # we are not going to calculate the Days Hours Minutes again, because it would only have been a few seconds of uptime. return # Exits the modem restart function. elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false'; then # extended modem reset functions: echo " ** Even after " if [ $NVRELOAD -eq 1 ] ; then echo " NVRELOAD selected. A more invasive reset of the modem. Maybe the nvram settings got screwed-up (this happens, occasionally)" | tee -a "$log_file" echo " Enter management functions on sierra wireless:" | tee -a "$log_file" _sleep 1 && echo " Putting modem in a mode to accept higher-level management commands" local responsefromatcmd responsefromatcmd=$(at_cmd 'AT!ENTERCND="A710"') echo " Response: ${responsefromatcmd}" | tee -a "$log_file" echo " Tell the modem to reload nvram settings, saved manually previously with !NVBACKUP=3" | tee -a "$log_file" responsefromatcmd=$(at_cmd "AT!RMARESET=3") echo " Response: ${responsefromatcmd}" | tee -a "$log_file" _sleep 7 && echo "modem is reloading NVRAM variables..." echo " Tell the modem to reboot. This is required after the above reload command." | tee -a "$log_file" responsefromatcmd=$(at_cmd "AT!RESET") echo " Response: ${responsefromatcmd}" | tee -a "$log_file" _sleep 20 && echo "modem is rebooting..." | tee -a "$log_file" ifup ${CONFIGURED_WWANNAME} | tee -a "$log_file" # restarting the interface echo " Waiting 45 seconds to give the modem time to reconnect." | tee -a "$log_file" _sleep 45 # Mandatory pause minimum for the cell modem to reconnect to tower. echo " Re-querying status with 'ifstatus ${CONFIGURED_WWANNAME}'" | tee -a "$log_file" echo " This time, the modem, if successfully reset, will show up is true, and an uptime of about 10 seconds:" | tee -a "$log_file" if ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": true'; then ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" # We only need to see the 'up' status, and uptime. return # Exits the modem restart function. elif ifstatus "${CONFIGURED_WWANNAME}" | grep -q '"up": false'; then # something is REALLY REALLY wrong. The modem should be up by now. Display results and log: ifstatus "${CONFIGURED_WWANNAME}" | grep "up" | tee -a "$log_file" echo "** Exiting the entire script from within the modem_reset function. Nothing got the modem working again. Maybe the tower is down?" | tee -a "$log_file" exit # Exit the entire script. Using EXIT at this point, is based on local critical decision-making: Is the tower really up? Is the bill paid? Or does the cell modem has an unknown issue. fi # exit extended reset testing elif [ $NVRELOAD -eq 0 ] ; then echo "Modem abnormally stuck in offline mode! NVRELOAD is disabled in the script. Investigate it manually. Exiting script directly, to avoid unnecessary reboot which probably will not fix the issue.." | tee -a "$log_file" echo "Maybe the tower is down? But you chose not to have the script try to fix the modem corruption issue. You have to decide at this point what you want to do." | tee -a "$log_file" echo "Exiting the wan-watchdog script..." | tee -a "$log_file" exit # Exit the entire script. fi fi fi else # v20260219.01 Fall through, usually due to incorrect MODEMTYPE declaration echo | tee -a "$log_file" echo "On $(date): ALERT!:" | tee -a "$log_file" echo "Fall-through condition reached:" | tee -a "$log_file" echo "FIXME! Some other error or condition unanticipated in the main condition checking in the modem_reset function section of the script" | tee -a "$log_file" echo "Usually it's MODEMTYPE is not set correctly in script" | tee -a "$log_file" echo "Current values for:" | tee -a "$log_file" echo "previouswanstatus: ${previouswanstatus}" | tee -a "$log_file" echo "MODEMTYPE: ${MODEMTYPE}" | tee -a "$log_file" echo "NVRELOAD: ${NVRELOAD}" | tee -a "$log_file" echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "CONFIGURED_WWANNAME: ${CONFIGURED_WWANNAME}" | tee -a "$log_file" echo "CONNECTIONSTATUS: ${CONNECTIONSTATUS}" | tee -a "$log_file" echo "WWAN_CHANGED: ${WWAN_CHANGED}" | tee -a "$log_file" echo "WWAN_CHANGED is also used to set whether the active wan interface was detected by this script as changed." echo "DARKMOON: is ${DARKMOON}" | tee -a "$log_file" echo "Doing a network_flush_cache and re-reading NETx_IF_NAME (in case of a variance)..." network_flush_cache network_find_wan NET_IF_NAME network_find_wan6 NET6_IF_NAME echo "Current NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "Current NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "Exiting script..." | tee -a "$log_file" exit # Immediately exit script fi } ########################### start main script ####################### echo # new blank line echo "Screen message: Logging to: ${log_file}" if [ "$TESTMODE" = 0 ]; then service sysntpd restart # restart the time-server on the router, only after BOOTWAIT. _sleep 5 # give 5 seconds for sysntpd to retrieve and reset the clock to current time. fi echo | tee -a "$log_file" # Blank line to delineate new logging sequence (on restart, startup of router) echo "Wan-Watchdog script started / Router booted on $(date)" | tee -a "$log_file" echo # new blank line echo "Entering wan & vpn testing:" network_flush_cache network_find_wan NET_IF_NAME network_find_wan6 NET6_IF_NAME NEEDREBOOT="0" # Initialize WWAN_CHANGED="0" # Initialize. WWAN_CHANGED is also used to set whether the active wan interface was detected by this script as changed. CONNECTIONSTATUS="1" # Initialize # Connection Status: 0 Not connected-connection attempt failure, exit and reboot. # Connection Status: 1 Unknown State, try to take corrective action # Connection Status: 2 WWAN interface connected, fping success # Connection Status: 3 WWAN & VPN connected, fping success ACTIVE_WWANNAME="none" # Initialize the -active and tested- VPN profile 'protonvpn1' or 'qmi, to 'none'. ACTIVE_WWANDEVICE="none" # ATTENTION: Look at the following output during testing, to determine if you correctly set CONFIGURED_WWANDEVICE & CONFIGURED_WWANNAME, assuming you are connected in a basic way to your default internet connection when you test-run this echo "Active NET_IF_NAME reports as ${NET_IF_NAME}" # if NIN is null, then the WAN is not connected (usually during testing where CONFIGURED_WWANNAME has been manually stopped). echo "Active NET6_IF_NAME reports as ${NET6_IF_NAME}" # same as above echo "Configured WWANNNAME is set to ${CONFIGURED_WWANNAME}" echo "MODEMTYPE (for modem_reset / wan reset) is set to ${MODEMTYPE}" echo "MODEMUSBDEVNODE is set to ${MODEMUSBDEVNODE}" # Ordinarily either NIN or N6IN value, must match CONFIGURED_WWANNAME, if the router is connected to the internet when you test-run this script, and you have no VPN profiles active. # If neither NET or NET6 IF match CONFIGURED_WWANNAME, stop this script and correct the value for $CONFIGURED_WWANNAME stored at the top. # You could set a name of 'mbim' when creating the device, and openwrt will make 'mbim_4' and 'mbim_6' virtual adapters. # Alternatively, you make 'qmi' and openwrt uses 'qmi' for ipv6, and a virtual adapter of 'qmi_4' for ipv4. # This script attempts to determine all possibilities, and writes an error to the log with info if nothing matches, and exits. echo "Sleeping for ${WAITFOR} seconds..." sleep $WAITFOR while [ $CONNECTIONSTATUS -ge 1 ]; do # While CONNECTION STATUS is 1 or greater, do the following loop: # if1 if check_fping ; then # do a traffic throughput test, does it actually transfer data? echo echo "Successful traffic throughput test (check_fping function) response." echo network_flush_cache network_find_wan NET_IF_NAME network_find_wan NET6_IF_NAME echo "Current values for:" echo "NET_IF_NAME: ${NET_IF_NAME}" echo "NET6_IF_NAME: ${NET6_IF_NAME}" # from settings at the top: echo "CONFIGURED_WWANNAME interface: ${CONFIGURED_WWANNAME}" # if2 if [ "$VPN" = "1" ] ; then # If VPN selector is turned ON, try/test VPN connections 1 through 4. echo "VPN Option selector turned-on '1'. Getting current interface connection values..." echo "Checking if any VPN connections are currently up..." sleep $WAITFOR # case1 is_vpn_active="0" case "$NET_IF_NAME" in "$VPN1"|"$VPN2"|"$VPN3"|"$VPN4") echo "NET_IF_NAME (active wan) is one of the configured VPN's: ${NET_IF_NAME}. It is reported as up, and traffic throughput test is successful!" echo "Basic WWAN Connection is up, plus 1 out of 4 possible VPN profiles/interfaces are connected." echo "Setting value of ACTIVE_WWANNAME to ${NET_IF_NAME}.." ACTIVE_WWANNAME=$NET_IF_NAME is_vpn_active="1" ;; esac if [ "$is_vpn_active" -eq 1 ]; then # detect if VPN has changed from previous run of script: echo "VPN detected as ${ACTIVE_WWANNAME} and is active:" echo "1 (out of 4 possible) VPN profiles/interfaces are connected." sleep $WAITFOR echo "Comparing current VPN to previous wwanname saved to /tmp/currentwwanname.txt, if any previous run of this script on this boot of the router..." # Sets PREVIOUS_WWANNAME to value read from previous run of script, otherwise sets to 'none'. if [ -f /tmp/currentwwanname.txt ]; then PREVIOUS_WWANNAME=$(cat /tmp/currentwwanname.txt) else PREVIOUS_WWANNAME="none" fi CONNECTIONSTATUS=3 echo "Comparing ACTIVE_WWANNAME to PREVIOUS_WWANNAME:" echo "PREVIOUS_WWANNAME is: ${PREVIOUS_WWANNAME}.." if [ "$ACTIVE_WWANNAME" = "$PREVIOUS_WWANNAME" ]; then # If ACTIVE_WWANNAME matches PREVIOUS_WWANNAME in this loop or run, then do nothing and continue echo "Current VPN is the same as the previously-connected VPN profile, continuing..." WWAN_CHANGED=0 sleep $WAITFOR else # if the values do not match, then the VPN has changed from previous script run, write the current vpn to 'currentwwanname.txt' echo "VPN(WAN) has changed. Previous VPN(WAN) was ${PREVIOUS_WWANNAME}. Current VPN is ${ACTIVE_WWANNAME}" | tee -a "$log_file" echo "Writing changes to currentvpn.txt..." WWAN_CHANGED=1 echo "$ACTIVE_WWANNAME" > /tmp/currentwwanname.txt # write the current vpn value to a file, e.g. to be used for recall and comparison later. sleep $WAITFOR fi # and if vpn = 1 but the ACTIVE_WWANNAME / NET_IF_NAME is not one of the configured VPN profiles named on the header of this script: # if the WWAN interface is up, & VPN selector is turned-on, and no VPN is up yet. # Here we use a rather complex comparison, since between the qmi and mbim protocols and various carriers, your simple 'qmi' can show up in various ways: # qmi/qmi_4, qmi_4/qmi_6, qmi etc. The point is the configured WWANNAME matches the ACTIVEWWAN, but the exact match needs to be 'fuzzy'. elif [ "$is_vpn_active" -eq 0 ] && { [ "$NET_IF_NAME" = "$CONFIGURED_WWANNAME" ] || \ [ "$NET6_IF_NAME" = "$CONFIGURED_WWANNAME" ] || \ [ "$NET_IF_NAME" = "${CONFIGURED_WWANNAME}_4" ] || \ [ "$NET6_IF_NAME" = "${CONFIGURED_WWANNAME}_6" ]; }; then echo echo "The active internet-facing interface (${NET_IF_NAME} or $NET_IF_NAME_6} or ${NET_IF_NAME_4}) does not match one of the available VPN profile names, but does match the configured WWANNAME, and the internet connection is working." echo "VPN selector is ON but VPN not up yet. Trying to get a VPN connection up..." | tee -a "$log_file" sleep $WAITFOR echo "Trying $VPN1..." | tee -a "$log_file" echo "ifup ${VPN1}" | tee -a "$log_file" ifup $VPN1 | tee -a "$log_file" _sleep 7 # wait for it to connect network_flush_cache network_find_wan NET_IF_NAME # if openwrt is saying the VPN is up, and it actually is up (throughput), then: if [ "$NET_IF_NAME" = "$VPN1" ] && check_fping ; then ACTIVE_WWANNAME="$VPN1" CONNECTIONSTATUS=3 WWAN_CHANGED=1 echo "${VPN1} is up, and fping success!" | tee -a "$log_file" else # connection attempt had failed one way or another.. echo "Unable to connect to ${VPN1}... trying ${VPN2} profile..." | tee -a "$log_file" echo "ifdown ${VPN1}" | tee -a "$log_file" ifdown $VPN1 | tee -a "$log_file" sleep $WAITFOR echo "Trying to connect to ${VPN2}.." echo "ifup ${VPN2}" | tee -a "$log_file" ifup $VPN2 | tee -a "$log_file" _sleep 7 network_flush_cache network_find_wan NET_IF_NAME if [ "$NET_IF_NAME" = "$VPN2" ] && check_fping ; then ACTIVE_WWANNAME=$VPN2 CONNECTIONSTATUS=3 WWAN_CHANGED=1 echo "${VPN2} is up and fping success!" | tee -a "$log_file" else echo "Unable to connect to ${VPN2}, trying ${VPN3} profile..." | tee -a "$log_file" echo "ifdown ${VPN2}" | tee -a "$log_file" ifdown $VPN2 | tee -a "$log_file" sleep $WAITFOR echo "ifup ${VPN3}" | tee -a "$log_file" ifup $VPN3 | tee -a "$log_file" _sleep 7 network_flush_cache network_find_wan NET_IF_NAME if [ "$NET_IF_NAME" = "$VPN3" ] && check_fping ; then ACTIVE_WWANNAME=$VPN3 CONNECTIONSTATUS=3 WWAN_CHANGED=1 echo "${VPN3} is up, and fping success!" | tee -a "$log_file" else echo "Unable to connect to ${VPN3}, trying ${VPN4} profile..." | tee -a "$log_file" echo "ifdown ${VPN3}" | tee -a "$log_file" ifdown $VPN3 | tee -a "$log_file" sleep $WAITFOR echo "ifup ${VPN4}" | tee -a "$log_file" ifup $VPN4 | tee -a "$log_file" _sleep 7 network_flush_cache network_find_wan NET_IF_NAME if [ "$NET_IF_NAME" = "$VPN4" ] && check_fping ; then ACTIVE_WWANNAME=$VPN4 CONNECTIONSTATUS=3 WWAN_CHANGED=1 echo "${VPN4} is up and fping success!" | tee -a "$log_file" else echo "Unable to start VPN4, or connect to any previous VPN1-3 profiles on $(date)" | tee -a "$log_file" fi # Finished VPN4 connect/all VPNx attempts fi # Finished VPN3 connect attempt fi # Finished VPN2 connect attempt fi # Finished VPN1 connect attempt #if3 else # something is wrong with the basic configuration of the script / interfaces mismatch echo "** Something went wrong! **" | tee -a "$log_file" echo "Most likely this scripts VPN/WWAN setup configuration variables are not matched to your router's actual interface names." | tee -a "$log_file" echo "Exiting..." | tee -a "$log_file" exit #if3 # finished trying to establish a VPN connection when VPN is configured ON but there is no VPN connection # VPN = 1 finished. VPN connection is connected, and establishing required VPN connection if not active already. fi #if2 # Else if VPN Selector is set to 0 / no VPN (only plain wan (wwan0), evaluate whether any VPN's are active elif [ "$VPN" -eq 0 ] ; then echo "VPN=${VPN}. VPN Selector is turned-off." echo "Checking to see if the current NET_IF_NAME or NET6_IF_NAME matches CONFIGURED_WWANNAME (or a _4 _6 variation):" echo "Current values:" echo "NET_IF_NAME: ${NET_IF_NAME}" echo "NET6_IF_NAME: ${NET6_IF_NAME}" echo "CONFIGURED_WWANNAME interface: ${CONFIGURED_WWANNAME}" if [ "${CONFIGURED_WWANNAME}" = "$NET_IF_NAME" ] || \ [ "${CONFIGURED_WWANNAME}" = "$NET6_IF_NAME" ] || \ [ "${CONFIGURED_WWANNAME}_4" = "$NET_IF_NAME" ] || \ [ "${CONFIGURED_WWANNAME}_6" = "$NET6_IF_NAME" ]; then echo "The active network wan interface name matches the configured wwan name, and VPN is selected OFF. Do nothing here/continue." else # the VPN is running, shut it down! echo "Neither the NET_IF_NAME (${NET_IF_NAME}) or NET6_IF_NAME (${NET6_IF_NAME}) match the CONFIGURED_WWANNAME (${CONFIGURED_WWANNAME})" | tee -a "$log_file" echo "Probably a VPN connection was manually started. Shutting down VPN1-4 interfaces (ifdown), e.g. in case a VPN was started manually prior to this script run. VPN selector is OFF." | tee -a "$log_file" # VPN may have been manually started prior to current script invocation, or may be running from a prior invocation with the options changed afterwards. ifdown $VPN1 ifdown $VPN2 ifdown $VPN3 ifdown $VPN4 WWAN_CHANGED=1 fi ACTIVE_WWANNAME=$NET_IF_NAME # note: oftentimes we need to take action on what the ACTIVE_WWANNAME is, not what it's necessarily configured as, due to discrepencies mentioned earlier in how Openwrt handles the names when it's got virtual interfaces. echo "Active WAN is ${ACTIVE_WWANNAME}, VPN turned off in script, and no VPN active: Setting 'CONNECTIONSTATUS' = 2" CONNECTIONSTATUS=2 sleep $WAITFOR #if1 fi # finished for VPN 1/0 Selector check, establishing VPN connection, determining if active VPN has changed from prior VPN interface. else # fping failure - No connectivity: on initial or subsequent fping test, whether through VPN or without. These responses should always be logged. # This is directly after an fping testing failure, so no need for an additional blank line echo # Usually means we have to do a modem-cycle (reset, etc) echo "On $(date): ALERT: fping failure - No WAN or VPN connectivity." | tee -a "$log_file" # Log when no connectivity. if [ $VPN -eq 1 ] ; then # If VPN is selected, and basic connectivity is failing, echo "VPN Selector is ON. WWAN/VPN is failing at initial fping test." | tee -a "$log_file" echo "This would ordinarily be the case, e.g. if you forcibly stopped your WWAN prior to running this script, to test." | tee -a "$log_file" elif [ $VPN -eq 0 ] ; then echo "VPN Selector is turned-off in this script and there is no WAN connectivity." | tee -a "$log_file" fi echo "Script will now attempt to (re)connect cell link, according to the designated ${MODEMTYPE} procedure." | tee -a "$log_file" CONNECTIONSTATUS="1" # reset to unknown ACTIVE_WWANNAME="none" # Only tear down VPN interfaces if any are actually active vpn_is_active=0 for _vpn in "$VPN1" "$VPN2" "$VPN3" "$VPN4"; do [ -n "$_vpn" ] || continue if ifstatus "$_vpn" | grep -q '"up": true'; then vpn_is_active=1 break fi done if [ "$VPN" -eq 1 ] && [ "$vpn_is_active" -eq 1 ]; then echo "VPN=1 and active VPN interface(s) detected. Shutting down VPN interfaces to test naked WAN connection..." | tee -a "$log_file" ifdown $VPN1; ifdown $VPN2; ifdown $VPN3; ifdown $VPN4 sleep $WAITFOR elif [ "$vpn_is_active" -eq 1 ]; then echo "VPN=0 but active VPN interface(s) detected (manually started). Shutting down to restore bare WAN..." | tee -a "$log_file" ifdown $VPN1; ifdown $VPN2; ifdown $VPN3; ifdown $VPN4 sleep $WAITFOR else echo "No active VPN interfaces detected. Skipping VPN teardown." fi # Only worth re-testing fping here if a VPN was actually torn down above — that's the only thing # that just changed. If no VPN was active, nothing changed, so re-running check_fping would just # re-confirm the same failure and waste the check_fping interval/timeout before modem_reset ever # gets a chance to run. Short-circuits: when vpn_is_active=0, check_fping is not called at all. if [ "$vpn_is_active" -eq 1 ] && check_fping ; then echo "check_fping SUCCESS follow-up after teardown of any VPN's: restart procedure worked, no need for reboot." | tee -a "$log_file" echo | tee -a "$log_file" sleep $WAITFOR ACTIVE_WWANNAME=$NET_IF_NAME CONNECTIONSTATUS="2" NEEDREBOOT="0" else if [ "$vpn_is_active" -eq 1 ]; then echo "check_fping still failing after VPN teardown — underlying WAN connection is the problem, not just the VPN. Proceeding to modem_reset." | tee -a "$log_file" else echo "No VPN was active, so there was nothing to tear down that could have fixed this — skipping the redundant check_fping retest and going directly to modem_reset." | tee -a "$log_file" fi modem_reset # Modem reset function as defined above. See function for details. _sleep 3 stop_if_requested if check_fping ; then echo "check_fping SUCCESS follow-up after teardown of any VPN's and subsequent modem_reset: restart procedure worked, no need for reboot." | tee -a "$log_file" echo | tee -a "$log_file" sleep $WAITFOR CONNECTIONSTATUS="2" NEEDREBOOT="0" ACTIVE_WWANNAME=$NET_IF_NAME else echo "fping follow-up after modem_reset FAILURE: Reboot required." | tee -a "$log_file" CONNECTIONSTATUS="0" NEEDREBOOT="1" fi fi # end of follow-up probe to see if WWAN interface restart worked... fi # End of initial connectivity throughput test (check_fping) ######## Connection Shaping: darkmoon (was previously a combined sqm & cake-autorate) portion of script ########## # Now let's start or restart, as needed, darkmoon # Should check darkmoon status and correct if necessary, here # and what it is currently using for it's 'ul_if' value: echo echo echo "DARKMOON portion of wan-watchdog.sh:" echo echo "Detecting interface/DARKMOON status & preferences:" echo "DARKMOON preference ON/OFF is ${DARKMOON}" echo echo "If 'running', darkmoon is running." echo "If 'inactive', darkmoon is not running." echo "If 'not found', darkmoon is not installed." echo # Capture status once to avoid multiple service calls DARKMOON_STATUS=$(service darkmoon status) echo "Status: Darkmoon status: $DARKMOON_STATUS" echo "CONNECTIONSTATUS is ${CONNECTIONSTATUS}" echo "Pulling active darkmoon wan interface setting (kernel device name):" DARKMOON_INTERFACE=$(uci -q get darkmoon.primary.ul_if) # this only returns the upload interface, which is exactly the same format as the kernel interface name, echo "DARKMOON_INTERFACE is ${DARKMOON_INTERFACE}" echo "ACTIVE_WWANNAME is ${ACTIVE_WWANNAME}" network_get_device ACTIVE_WWANDEVICE ${ACTIVE_WWANNAME} # pull the ACTIVE_WWANDEVICE from the ACTIVE_WWANNAME echo "ACTIVE_WWANDEVICE is ${ACTIVE_WWANDEVICE}" echo "CONFIGURED_WWANNAME UCI name is ${CONFIGURED_WWANNAME}" network_get_device CONFIGURED_WWANDEVICE "${CONFIGURED_WWANNAME}" # pull the CONFIGURED_WWANDEVICE from the CONFIGURED_WWANNAME echo "CONFIGURED_WWANDEVICE kernel name is ${CONFIGURED_WWANDEVICE}" echo "WWAN_CHANGED is ${WWAN_CHANGED}." # making it easy to the darkmoon setting to the active interface directly. The darkmoon dl_if is always a derivative of the active interface: i.e. ifb-$ACTIVE_INTERFACE sleep $WAITFOR # if the VPN is selected (1) / enabled (on), and running (working) (therefore connectionstatus=3), WWAN/VPN has not changed from prior, and darkmoon is already running, or # if the VPN is selected OFF (0), and this is a plain wwan (therefore connectionstatus=2), and the WWAN/VPN has not changed from prior, and darkmoon is running, # This is most typical state when the script is running in a loop the second, third etc times. [23] means 2 or 3. if [[ "$CONNECTIONSTATUS" == [23] ]] && \ [[ "$DARKMOON_STATUS" == "running" ]] && \ [ "$DARKMOON" = "1" ] && \ [ "$WWAN_CHANGED" = "0" ] && \ [ "$DARKMOON_INTERFACE" = "$ACTIVE_WWANDEVICE" ] ; then echo "Connection Status is ${CONNECTIONSTATUS}: VPN ${VPN}. DARKMOON is set ON and running, WWAN/VPN has NOT been flagged as changed, and the DARKMOON interface matches the active interface: Do nothing." # else if the WWAN/VPN is active, has not changed, but darkmoon is not yet running, typically due to set to 'disabled' on startup, then: simply start darkmoon: elif [[ "$CONNECTIONSTATUS" == [23] ]] && \ [[ "$DARKMOON_STATUS" == "not running" || "$DARKMOON_STATUS" == "inactive" ]] && \ [ "$DARKMOON" = "1" ] && \ [ "$WWAN_CHANGED" = "0" ] && \ [ "$DARKMOON_INTERFACE" = "$ACTIVE_WWANDEVICE" ]; then echo "CONNECTIONSTATUS is ${CONNECTIONSTATUS}: VPN is ${VPN}, WWAN/VPN interface has not changed, DarkMoon is NOT running, yet DarkMoon selector is ON. Starting Darkmoon..." sleep $WAITFOR echo "service darkmoon start" service darkmoon start sleep $WAITFOR # else if connectionstatus is 2 or 3, but the ACTIVE_WWANNAME internet path, either WWAN or VPN, has changed OR the active interface does not match darkmoon's, and darkmoon is set ON, (re)start it with the new interface: elif [[ "$CONNECTIONSTATUS" == [23] ]] && \ [ "$DARKMOON" = "1" ] && \ { [ "$WWAN_CHANGED" = "1" ] || [[ "$DARKMOON_INTERFACE" != "$ACTIVE_WWANDEVICE" ]]; } ; then echo "Connection Status is ${CONNECTIONSTATUS}: VPN is set to ${VPN}. DARKMOON is set to ON. But WAN/VPN has changed: restart DARKMOON with new WAN/VPN interface" | tee -a "$log_file" echo "Stopping darkmoon (if running)..." echo "service darkmoon stop" service darkmoon stop sleep $WAITFOR echo "Pulling new value for ACTIVE_WWANNAME: ${NET_IF_NAME}" ACTIVE_WWANNAME=$NET_IF_NAME echo "Writing current ACTIVE_WWANNAME to /tmp/currentwwanname.txt." echo "DarkMoon will pick up currrent kernel device through the detected ACTIVE_WWANDEVICE as derived from the detected ACTIVE_WWANNAME." # ... write the current WWAN name to a file, to be used by this script if restarted to compare interfaces. echo "${ACTIVE_WWANNAME}" > /tmp/currentwwanname.txt network_get_device ACTIVE_WWANDEVICE "${ACTIVE_WWANNAME}" # sqm, darkmoon, route etc use the kernel name, not UCI 'friendly' name echo "ACTIVE_WWANDEVICE is ${ACTIVE_WWANDEVICE}" # This will print 'wwan0' or 'eth1' or 'wireguard-protonvpn1', whatever the kernel device name is. echo echo "Updating the Darkmoon interfaces to the new ACTIVE_WWANDEVICE ${ACTIVE_WWANDEVICE}:" echo "uci set darkmoon.primary.ul_if=${ACTIVE_WWANDEVICE}" uci set darkmoon.primary.ul_if="$ACTIVE_WWANDEVICE" echo "uci set darkmoon.primary.dl_if=ifb-${ACTIVE_WWANDEVICE}" uci set darkmoon.primary.dl_if="ifb-$ACTIVE_WWANDEVICE" echo "uci commit darkmoon" uci commit darkmoon sleep $WAITFOR echo "service darkmoon start" service darkmoon start WWAN_CHANGED="0" # wwan_changed is no longer '1', because we've taken care of setting the active to match the preferences sleep $WAITFOR # typical when script has been stopped, DARKMOON selector has been turned off, and script is re-run # : if darkmoon is running, but darkmoon is set to OFF, then shut down cake-autorate: elif [[ "$CONNECTIONSTATUS" == [23] ]] && [ "$(service darkmoon status)" = "running" ] && [ "$DARKMOON" = "0" ]; then echo "Connection Status is ${CONNECTIONSTATUS}, DarkMoon is running, but is selected OFF. Stopping Darkmoon:" echo "service darkmoon stop" service darkmoon stop # This can be the case during manually running the script, or if you decided not to use cake-autorate for now, but forgot to disable it from startup # or it had been started previously but you don't want it running rn. sleep $WAITFOR # else if Connection is 2 or 3 - good, w or wo VPN, DARKMOON is selected OFF and is not running, do nothing elif [[ "$CONNECTIONSTATUS" == [23] ]] && \ [[ "$(service darkmoon status)" == "not running" || "$(service darkmoon status)" == "inactive" ]] && \ [ "$DARKMOON" = "0" ] ; then echo "CONNECTIONSTATUS is ${CONNECTIONSTATUS}. DARKMOON is selected OFF and Darkmoon is not running. Do nothing." sleep $WAITFOR # normal conditions when Darkmoon is not (yet) installed at all on the router What are you waiting for?: # if Connection status is 2 or 3 - good w or wo VPN, and Darkmoon is selected OFF not installed, continue.. elif ([[ "$CONNECTIONSTATUS" == [23] ]]) && \ [ ! -f "/etc/init.d/darkmoon" ] && \ [ "$DARKMOON" = "0" ]; then echo "DARKMOON: not enabled in script & not installed on router." echo "Connectionstatus is 2/3 (good)" echo "Do nothing - continue running" # else if Connection is 2 / good w or wo VPN, Darkmoon is selected ON yet not installed, log it & exit script. elif ([[ "$CONNECTIONSTATUS" = [23] ]]) && \ [ ! -f "/etc/init.d/darkmoon" ] && \ [ "$DARKMOON" = "1" ]; then echo "Darkmoon is not installed, but enabled in script." | tee -a "$log_file" echo "Connectionstatus is 2 or 3 (good)" | tee -a "$log_file" echo "Quitting... FIXME: Install Darkmoon or disable it in this script." | tee -a "$log_file" exit # Immediately exit script # else if we need a reboot because there was an fping failure, & connection status will be '0'. We didn't use to catch this, but # now we do (20260219.xx updates added a 'fall-through' condition, below). So we have to account for it. # fping previously failed (no throughput) after a modem_reset, reboot required. elif [ "$CONNECTIONSTATUS" = "0" ] && [ "$NEEDREBOOT" = "1" ] ; then echo "Skipping Darkmoon portion of script, fping failed after modem_reset: reboot required" | tee -a "$log_file" # fall-through condition: else # 'fall through' else condition. No previous conditions met: log everything and quit the script. echo "On $(date): ALERT!: DARKMOON section fall-through" | tee -a "$log_file" echo "FIXME! Some other error or condition unanticipated, in the Darkmoon section of the script" | tee -a "$log_file" echo "Current values for:" | tee -a "$log_file" echo "NET_IF_NAME: ${NET_IF_NAME}" | tee -a "$log_file" echo "NET6_IF_NAME: ${NET6_IF_NAME}" | tee -a "$log_file" echo "MODEMTYPE: ${MODEMTYPE}" | tee -a "$log_file" echo "ACTIVE_WWANNAME: ${ACTIVE_WWANNAME}" | tee -a "$log_file" echo "ACTIVE_WWANDEVICE: ${ACTIVE_WWANDEVICE}" | tee -a "$log_file" echo "CONFIGURED_WWANNAME interface: ${CONFIGURED_WWANNAME}" | tee -a "$log_file" echo "CONFIGURED_WWANDEVICE device: ${CONFIGURED_WWANDEVICE}" | tee -a "$log_file" echo "CONNECTIONSTATUS: ${CONNECTIONSTATUS}" | tee -a "$log_file" echo "WWAN_CHANGED: ${WWAN_CHANGED}" | tee -a "$log_file" echo "DARKMOON: ${DARKMOON}" | tee -a "$log_file" echo "NEEDREBOOT: ${NEEDREBOOT}" | tee -a "$log_file" echo "Exiting script..." | tee -a "$log_file" exit # Immediately exit script fi # Finished starting darkmoon for current connection, if selected, restarting services or terminating as necessary. echo # Pause for LOOPWAIT then re-test connection, or if in testing-mode, go directly to exit: if [ $TESTMODE = 0 ] ; then # now check to see if you should loop or break if [ "$STOP_REQUESTED" = "1" ]; then echo "Stop requested — exiting cleanly after completing current cycle." | tee -a "$log_file" exit 0 fi echo "Pausing ${LOOPWAIT} seconds and testing again..." _sleep $LOOPWAIT if [ "$STOP_REQUESTED" = "1" ]; then echo "Stop requested — exiting cleanly." | tee -a "$log_file" exit 0 fi elif [ $TESTMODE = 1 ] ; then echo "Would loop at this point, but it is in Testing mode. Exiting.." | tee -a "$log_file" echo "Test run ended: $(date)" | tee -a "$log_file" exit fi echo done # When CONNECTION STATUS no longer 1 or greater, or break the while-do loop if [ $TESTMODE = 0 ] && [ $NEEDREBOOT = 1 ]; then stop_if_requested echo "Preparing to reboot router (fping failed after modem_reset):" | tee -a "$log_file" safe_reboot elif [ $TESTMODE = 1 ] && [ $NEEDREBOOT = 1 ]; then echo "Failed WWAN connectivity check. Reboot required, but script in testing mode... no auto-reboot. Exit." | tee -a "$log_file" fi