/*===================================================================== * hellmon.c - THE ROUTER FROM HELL - Drop/Error Watcher (C edition) v2 * * Single-binary replacement for hellwatch.sh, purpose-built for * Magikarp (Gentoo/OpenRC, Intel X710 / i40e, WireGuard + BIRD OSPF). * * v2 additions: * - PAGES: [1] Overview [2] Interfaces [3] Burst catcher * switch live with 1/2/3, n/p, or arrow keys; q quits * - Interfaces page: link state/speed/MTU, carrier_changes (alerts * on flap!), live rx/tx throughput + pps, the full netdev error * family (missed/fifo/overrun/carrier/collisions/multicast), * qdisc drops, i40e priv flags, and all moving ethtool counters * - Integrated BURST CATCHER (ported from burst-catch.sh): arms the * kfree_skb stacktrace trigger the moment tx_dropped ticks on the * watched iface, snapshots the ftrace buffer + forks a tcpdump * for the capture window, then summarizes TX-path frames. * Non-blocking state machine: the UI keeps updating mid-capture. * Evidence saved to /tmp/burst-YYYYmmdd-HHMMSS/. * * What it watches (unchanged from v1): * - WireGuard devs/peers, handshake age, endpoint roams * - UDP underlay, IP/ICMP stack, softnet, conntrack * - NIC counters via ETHTOOL_GSTATS ioctl; i40e priv flags * - BIRD OSPF neighbors incl. MISSING detection from bird.conf * * Counters are delta-tracked: red = incremented this interval, * yellow = incremented since program start, dim = quiet. * * Build: gcc -O2 -Wall -Wextra -o hellmon hellmon.c * Usage: sudo ./hellmon [options] (see --help) *=====================================================================*/ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*--------------------------------------------------------------------- * Defaults (Magikarp). All overridable on the command line. *-------------------------------------------------------------------*/ static const char *DEF_WAN_IF = "enp16s0f0np0"; static const char *DEF_LAN_IF = "enp16s0f1np1"; static const char *DEF_MGMT_IF = "enp42s0"; static const char *DEF_BIRDCONF = "/etc/bird.conf"; static const char *DEF_WGDIR = "/etc/wireguard"; static const char *DEF_BIRDC = "birdc"; #define MAX_IFACES 32 #define MAX_WGDEVS 8 #define MAX_PEERS 64 #define MAX_OSPF_EXP 32 #define MAX_ETH_STATS 1024 #define IFNAME_LEN IFNAMSIZ #define STALE_HANDSHAKE 150 /* keepalive 25s -> 150s = ~6 missed */ /*--------------------------------------------------------------------- * Pages *-------------------------------------------------------------------*/ enum { PG_OVERVIEW = 0, PG_IFACES, PG_BURST, PG_HELLGATE, PG_COUNT }; static const char *PG_NAMES[PG_COUNT] = { "Overview", "Interfaces", "BurstCatch", "Hellgate" }; static int g_page = PG_OVERVIEW; /*--------------------------------------------------------------------- * Options *-------------------------------------------------------------------*/ static int g_interval = 2; static int g_rediscover = 60; static long g_count = -1; /* -1 = forever; 1 = --once */ static int g_color = -1; /* -1 auto, 0 off, 1 on */ static int g_clear = 1; /* clear screen between ticks */ static const char *g_wan = NULL; static const char *g_lan = NULL; static const char *g_birdconf = NULL; static const char *g_wgdir = NULL; static const char *g_birdc = NULL; /* burst catcher */ static int g_burst_arm = 1; /* -D disables */ static const char *g_burst_if = NULL; /* default: LAN */ static int g_burst_secs = 5; /* -s capture window */ static int g_burst_pcap = 1; /* fork tcpdump during window */ /* colors (set at runtime) */ static const char *RED="", *YEL="", *GRN="", *DIM="", *RST="", *BOLD=""; /*--------------------------------------------------------------------- * Growable output buffer (render once per tick -> no flicker) *-------------------------------------------------------------------*/ typedef struct { char *s; size_t len, cap; } sbuf_t; static void sb_init(sbuf_t *b){ b->cap=65536; b->len=0; b->s=malloc(b->cap); b->s[0]=0; } static void sb_reset(sbuf_t *b){ b->len=0; b->s[0]=0; } static void sb_vappend(sbuf_t *b, const char *fmt, va_list ap){ va_list ap2; va_copy(ap2, ap); int need = vsnprintf(NULL, 0, fmt, ap2); va_end(ap2); if (need < 0) return; while (b->len + (size_t)need + 1 > b->cap) { b->cap *= 2; b->s = realloc(b->s, b->cap); } vsnprintf(b->s + b->len, b->cap - b->len, fmt, ap); b->len += (size_t)need; } static void sb_addf(sbuf_t *b, const char *fmt, ...){ va_list ap; va_start(ap, fmt); sb_vappend(b, fmt, ap); va_end(ap); } static sbuf_t PAGES[PG_COUNT]; /* rendered every tick, all of them */ static sbuf_t ALERTS; /* alert headline (shared) */ static int n_alerts = 0; static void alert(const char *fmt, ...){ char tmp[512]; va_list ap; va_start(ap, fmt); vsnprintf(tmp, sizeof tmp, fmt, ap); va_end(ap); if (n_alerts) sb_addf(&ALERTS, " | "); sb_addf(&ALERTS, "%s", tmp); n_alerts++; } /*--------------------------------------------------------------------- * Counter registry: key -> base/prev/cur. Linear array; a few hundred * entries scanned every tick is nothing. *-------------------------------------------------------------------*/ typedef struct { char key[128]; unsigned long long base, prev, cur; int init; } ctr_t; static ctr_t *g_ctrs = NULL; static int g_nctrs = 0, g_ctr_cap = 0; static ctr_t *ctr_get(const char *key){ for (int i = 0; i < g_nctrs; i++) if (!strcmp(g_ctrs[i].key, key)) return &g_ctrs[i]; if (g_nctrs == g_ctr_cap) { g_ctr_cap = g_ctr_cap ? g_ctr_cap * 2 : 256; g_ctrs = realloc(g_ctrs, (size_t)g_ctr_cap * sizeof(ctr_t)); } ctr_t *c = &g_ctrs[g_nctrs++]; memset(c, 0, sizeof *c); snprintf(c->key, sizeof c->key, "%s", key); return c; } /* Record value; append "val (+d/Ns, +b total)" colorized; optionally * raise alert on per-interval growth. Returns per-interval delta. */ static long long ctr_fmtx(sbuf_t *b, const char *key, unsigned long long val, int do_alert){ ctr_t *c = ctr_get(key); if (!c->init) { c->base = c->prev = val; c->init = 1; } long long d = (long long)val - (long long)c->prev; long long t = (long long)val - (long long)c->base; c->cur = val; if (d > 0) { sb_addf(b, "%s%llu (+%lld/%ds, +%lld total)%s", RED, val, d, g_interval, t, RST); if (do_alert) alert("%s +%lld", key, d); } else if (t > 0) { sb_addf(b, "%llu %s(+%lld since start)%s", val, YEL, t, RST); } else { sb_addf(b, "%s%llu%s", DIM, val, RST); } return d; } #define ctr_fmt(b, key, val) ctr_fmtx(b, key, val, 1) /* rate tracking (no display; returns units/sec since last tick) */ static double ctr_rate(const char *key, unsigned long long val, double dt){ ctr_t *c = ctr_get(key); if (!c->init) { c->base = c->prev = val; c->init = 1; } double r = dt > 0.05 ? ((double)val - (double)c->prev) / dt : 0.0; c->cur = val; return r < 0 ? 0.0 : r; } static void ctr_roll(void){ for (int i = 0; i < g_nctrs; i++) g_ctrs[i].prev = g_ctrs[i].cur; } /*--------------------------------------------------------------------- * Small file / string helpers *-------------------------------------------------------------------*/ static int read_file_str(const char *path, char *buf, size_t n){ int fd = open(path, O_RDONLY); if (fd < 0) return -1; ssize_t r = read(fd, buf, n - 1); close(fd); if (r < 0) return -1; buf[r] = 0; return 0; } static unsigned long long read_file_ull(const char *path){ char buf[64]; if (read_file_str(path, buf, sizeof buf) < 0) return 0; return strtoull(buf, NULL, 10); } static unsigned long long sysstat(const char *ifname, const char *stat){ char p[256]; snprintf(p, sizeof p, "/sys/class/net/%s/statistics/%s", ifname, stat); return read_file_ull(p); } static int if_exists(const char *ifname){ char p[256]; snprintf(p, sizeof p, "/sys/class/net/%s", ifname); return access(p, F_OK) == 0; } static void rstrip(char *s){ size_t n = strlen(s); while (n && (s[n-1]=='\n' || s[n-1]=='\r' || s[n-1]==' ' || s[n-1]=='\t')) s[--n] = 0; } static char *lskip(char *s){ while (*s==' '||*s=='\t') s++; return s; } /* human-ish byte formatter (iec) */ static void fmt_iec(unsigned long long v, char *out, size_t n){ static const char *u[] = {"","K","M","G","T","P"}; double d = (double)v; int i = 0; while (d >= 1024.0 && i < 5) { d /= 1024.0; i++; } if (i == 0) snprintf(out, n, "%llu", v); else snprintf(out, n, "%.1f%s", d, u[i]); } static void fmt_iec_d(double v, char *out, size_t n){ static const char *u[] = {"","K","M","G","T","P"}; int i = 0; while (v >= 1024.0 && i < 5) { v /= 1024.0; i++; } if (i == 0) snprintf(out, n, "%.0f", v); else snprintf(out, n, "%.1f%s", v, u[i]); } static double now_mono(void){ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; } /*--------------------------------------------------------------------- * /proc/net/snmp + /proc/net/netstat + /proc/net/snmp6 -> key/val map *-------------------------------------------------------------------*/ typedef struct { char key[80]; unsigned long long val; } kv_t; static kv_t *g_nst = NULL; static int g_nst_n = 0, g_nst_cap = 0; static void nst_put(const char *key, unsigned long long v){ if (g_nst_n == g_nst_cap) { g_nst_cap = g_nst_cap ? g_nst_cap * 2 : 512; g_nst = realloc(g_nst, (size_t)g_nst_cap * sizeof(kv_t)); } snprintf(g_nst[g_nst_n].key, sizeof g_nst[0].key, "%s", key); g_nst[g_nst_n].val = v; g_nst_n++; } static unsigned long long nst_get(const char *key){ for (int i = 0; i < g_nst_n; i++) if (!strcmp(g_nst[i].key, key)) return g_nst[i].val; return 0; } static void nst_load_paired(const char *path){ FILE *f = fopen(path, "r"); if (!f) return; char h[4096], v[4096]; while (fgets(h, sizeof h, f) && fgets(v, sizeof v, f)) { char *hc = strchr(h, ':'), *vc = strchr(v, ':'); if (!hc || !vc) continue; *hc = 0; *vc = 0; if (strcmp(h, v)) continue; /* prefix mismatch: bail row */ char prefix[32]; snprintf(prefix, sizeof prefix, "%.31s", h); char *hp = hc + 1, *vp = vc + 1, *hs, *vs, *hsave, *vsave; hs = strtok_r(hp, " \t\n", &hsave); vs = strtok_r(vp, " \t\n", &vsave); while (hs && vs) { char key[80]; snprintf(key, sizeof key, "%s%s", prefix, hs); nst_put(key, strtoull(vs, NULL, 10)); hs = strtok_r(NULL, " \t\n", &hsave); vs = strtok_r(NULL, " \t\n", &vsave); } } fclose(f); } static void nst_load_snmp6(const char *path){ FILE *f = fopen(path, "r"); if (!f) return; char line[256], key[80]; unsigned long long v; while (fgets(line, sizeof line, f)) if (sscanf(line, "%79s %llu", key, &v) == 2) nst_put(key, v); fclose(f); } static void nst_refresh(void){ g_nst_n = 0; nst_load_paired("/proc/net/snmp"); nst_load_paired("/proc/net/netstat"); nst_load_snmp6("/proc/net/snmp6"); } /*--------------------------------------------------------------------- * softnet: sum col2 (drops) and col3 (time_squeeze) across CPUs (hex) *-------------------------------------------------------------------*/ static void softnet(unsigned long long *drops, unsigned long long *squeeze){ *drops = 0; *squeeze = 0; FILE *f = fopen("/proc/net/softnet_stat", "r"); if (!f) return; char line[1024]; while (fgets(line, sizeof line, f)) { unsigned long long a, b, c; if (sscanf(line, "%llx %llx %llx", &a, &b, &c) == 3) { *drops += b; *squeeze += c; } } fclose(f); } /*--------------------------------------------------------------------- * ethtool via ioctl: stat names cached per NIC; values read each tick. *-------------------------------------------------------------------*/ typedef struct { char ifname[IFNAME_LEN]; int n_stats; char (*names)[ETH_GSTRING_LEN]; /* cached at discovery */ int n_watch; int watch_idx[MAX_ETH_STATS]; /* indices of interesting stats */ int n_pflags; char (*pflag_names)[ETH_GSTRING_LEN]; uint32_t pflags; int pflags_ok; char driver[32]; } ethdev_t; static int g_ethsock = -1; static int eth_ioctl(const char *ifname, void *data){ struct ifreq ifr; memset(&ifr, 0, sizeof ifr); snprintf(ifr.ifr_name, IFNAMSIZ, "%.15s", ifname); ifr.ifr_data = data; return ioctl(g_ethsock, SIOCETHTOOL, &ifr); } static int eth_sset_count(const char *ifname, int set){ struct { struct ethtool_sset_info hdr; uint32_t buf[1]; } sset; memset(&sset, 0, sizeof sset); sset.hdr.cmd = ETHTOOL_GSSET_INFO; sset.hdr.sset_mask = 1ULL << set; if (eth_ioctl(ifname, &sset) < 0) return -1; if (!sset.hdr.sset_mask) return 0; return (int)sset.buf[0]; } static int eth_gstrings(const char *ifname, int set, int count, char (**out)[ETH_GSTRING_LEN]){ struct ethtool_gstrings *gs = calloc(1, sizeof *gs + (size_t)count * ETH_GSTRING_LEN); gs->cmd = ETHTOOL_GSTRINGS; gs->string_set = set; gs->len = count; if (eth_ioctl(ifname, gs) < 0) { free(gs); return -1; } char (*names)[ETH_GSTRING_LEN] = malloc((size_t)count * ETH_GSTRING_LEN); memcpy(names, gs->data, (size_t)count * ETH_GSTRING_LEN); free(gs); *out = names; return 0; } static const char *ETH_WATCH_SUBSTR[] = { "drop", "discard", "error", "err", "missed", "fifo", "crc", "fault", "jabber", "oversize", "undersize", "fragment", "unknown_protocol", "out_of_buffer", "no_dma", "congst", "alloc_fail", "busy", "timeout", "illegal", "csum_bad", "rx_length", "tx_restart", "linearize", "force_wb", "xoff", "xon", /* v2: pause-frame counters */ NULL }; static int eth_stat_interesting(const char *name){ for (int i = 0; ETH_WATCH_SUBSTR[i]; i++) if (strcasestr(name, ETH_WATCH_SUBSTR[i])) return 1; return 0; } static void ethdev_discover(ethdev_t *e, const char *ifname){ memset(e, 0, sizeof *e); snprintf(e->ifname, sizeof e->ifname, "%s", ifname); struct ethtool_drvinfo drv; memset(&drv, 0, sizeof drv); drv.cmd = ETHTOOL_GDRVINFO; if (eth_ioctl(ifname, &drv) == 0) snprintf(e->driver, sizeof e->driver, "%s", drv.driver); int n = eth_sset_count(ifname, ETH_SS_STATS); if (n > 0 && n <= MAX_ETH_STATS && eth_gstrings(ifname, ETH_SS_STATS, n, &e->names) == 0) { e->n_stats = n; e->n_watch = 0; for (int i = 0; i < n; i++) if (eth_stat_interesting(e->names[i])) e->watch_idx[e->n_watch++] = i; } int np = eth_sset_count(ifname, ETH_SS_PRIV_FLAGS); if (np > 0 && np <= 32 && eth_gstrings(ifname, ETH_SS_PRIV_FLAGS, np, &e->pflag_names) == 0) { struct ethtool_value ev; memset(&ev, 0, sizeof ev); ev.cmd = ETHTOOL_GPFLAGS; if (eth_ioctl(ifname, &ev) == 0) { e->n_pflags = np; e->pflags = ev.data; e->pflags_ok = 1; } } } static void ethdev_free(ethdev_t *e){ free(e->names); e->names = NULL; free(e->pflag_names); e->pflag_names = NULL; e->n_stats = e->n_watch = e->n_pflags = 0; } static void ethdev_render(sbuf_t *b, ethdev_t *e){ if (!e->n_stats) return; struct ethtool_stats *st = calloc(1, sizeof *st + (size_t)e->n_stats * sizeof(uint64_t)); st->cmd = ETHTOOL_GSTATS; st->n_stats = e->n_stats; if (eth_ioctl(e->ifname, st) < 0) { free(st); return; } for (int w = 0; w < e->n_watch; w++) { int i = e->watch_idx[w]; unsigned long long v = st->data[i]; char key[160]; snprintf(key, sizeof key, "%.16s/%.100s", e->ifname, e->names[i]); ctr_t *c = ctr_get(key); if (v == 0 && (!c->init || (c->base == 0 && c->cur == 0))) { if (!c->init) { c->base = c->prev = c->cur = 0; c->init = 1; } continue; } sb_addf(b, " %-34s: ", e->names[i]); ctr_fmt(b, key, v); if (!strcmp(e->names[i], "rx_unknown_protocol")) sb_addf(b, " %s(i40e: usually LLDP/unknown ethertype)%s", DIM, RST); if (strcasestr(e->names[i], "xoff_rx")) sb_addf(b, " %s(pause frames FROM switch)%s", DIM, RST); sb_addf(b, "\n"); } free(st); } static void ethdev_render_pflags(sbuf_t *b, ethdev_t *e){ if (!e->pflags_ok || !e->n_pflags) return; sb_addf(b, " %spriv-flags:%s ", DIM, RST); for (int i = 0; i < e->n_pflags; i++) { int on = !!(e->pflags & (1u << i)); const char *nm = e->pflag_names[i]; if (strcasestr(nm, "disable-fw-lldp") && !on) sb_addf(b, "%s%s:off(!)%s ", YEL, nm, RST); else sb_addf(b, "%s%s:%s%s ", DIM, nm, on ? "on" : "off", RST); } sb_addf(b, "\n"); } /*--------------------------------------------------------------------- * qdisc drops: `tc -s qdisc show dev X` (one popen per iface per tick) *-------------------------------------------------------------------*/ static unsigned long long qdisc_drops(const char *ifname){ char cmd[256]; snprintf(cmd, sizeof cmd, "tc -s qdisc show dev %s 2>/dev/null", ifname); FILE *p = popen(cmd, "r"); if (!p) return 0; unsigned long long total = 0; char line[512]; while (fgets(line, sizeof line, p)) { char *s = strstr(line, "dropped "); if (s) total += strtoull(s + 8, NULL, 10); } pclose(p); return total; } /* first qdisc kind on the dev (e.g. "mq", "fq", "fq_codel") */ static void qdisc_kind(const char *ifname, char *out, size_t n){ out[0] = 0; char cmd[256]; snprintf(cmd, sizeof cmd, "tc qdisc show dev %s 2>/dev/null | head -2", ifname); FILE *p = popen(cmd, "r"); if (!p) return; char line[256], kinds[2][32] = {{0},{0}}; int k = 0; while (k < 2 && fgets(line, sizeof line, p)) { char kind[32]; if (sscanf(line, "qdisc %31s", kind) == 1) snprintf(kinds[k++], 32, "%s", kind); } pclose(p); if (k == 2) snprintf(out, n, "%s/%s", kinds[0], kinds[1]); else if (k == 1) snprintf(out, n, "%s", kinds[0]); } /*--------------------------------------------------------------------- * WireGuard *-------------------------------------------------------------------*/ typedef struct { char pubkey[64]; char name[48]; } peer_name_t; typedef struct { char pubkey[64]; char endpoint[80]; } peer_ep_t; static char g_wgdevs[MAX_WGDEVS][IFNAME_LEN]; static int g_n_wgdevs = 0; static peer_name_t g_peer_names[MAX_PEERS]; static int g_n_peer_names = 0; static peer_ep_t g_peer_eps[MAX_PEERS]; static int g_n_peer_eps = 0; /* v2.1: per-device peer health, refreshed by wg_render each tick, so * link-state display can say "up (3/3 peers fresh)" instead of the * kernel's eternal operstate=unknown for carrier-less virtual devs */ typedef struct { char dev[IFNAME_LEN]; int total, fresh; /* fresh = handshake within STALE_HANDSHAKE */ long best_age; /* youngest handshake age, -1 if none */ } wg_health_t; static wg_health_t g_wg_health[MAX_WGDEVS]; static int g_n_wg_health = 0; static wg_health_t *wg_health_get(const char *dev){ for (int i = 0; i < g_n_wg_health; i++) if (!strcmp(g_wg_health[i].dev, dev)) return &g_wg_health[i]; if (g_n_wg_health >= MAX_WGDEVS) return NULL; wg_health_t *h = &g_wg_health[g_n_wg_health++]; memset(h, 0, sizeof *h); snprintf(h->dev, sizeof h->dev, "%.15s", dev); h->best_age = -1; return h; } static const char *peer_name(const char *pub){ for (int i = 0; i < g_n_peer_names; i++) if (!strcmp(g_peer_names[i].pubkey, pub)) return g_peer_names[i].name; return NULL; } static const char *peer_prev_ep(const char *pub){ for (int i = 0; i < g_n_peer_eps; i++) if (!strcmp(g_peer_eps[i].pubkey, pub)) return g_peer_eps[i].endpoint; return NULL; } static void peer_set_ep(const char *pub, const char *ep){ for (int i = 0; i < g_n_peer_eps; i++) if (!strcmp(g_peer_eps[i].pubkey, pub)) { snprintf(g_peer_eps[i].endpoint, sizeof g_peer_eps[0].endpoint, "%s", ep); return; } if (g_n_peer_eps < MAX_PEERS) { snprintf(g_peer_eps[g_n_peer_eps].pubkey, sizeof g_peer_eps[0].pubkey, "%s", pub); snprintf(g_peer_eps[g_n_peer_eps].endpoint, sizeof g_peer_eps[0].endpoint, "%s", ep); g_n_peer_eps++; } } static void load_peer_names(const char *conf){ FILE *f = fopen(conf, "r"); if (!f) return; char line[512], pending[128] = ""; int in_peer = 0; while (fgets(line, sizeof line, f)) { rstrip(line); char *s = lskip(line); if (*s == '[') { in_peer = (strncasecmp(s, "[peer]", 6) == 0); pending[0] = 0; continue; } if (!in_peer) continue; if (*s == '#') { char *c = lskip(s + 1); if (strchr(c, '=')) continue; /* disabled directive */ if (*c) snprintf(pending, sizeof pending, "%s", c); continue; } if (!strncasecmp(s, "publickey", 9)) { char *eq = strchr(s, '='); if (!eq) continue; char *k = lskip(eq + 1); rstrip(k); char key[64]; int ki = 0; for (char *q = k; *q && ki < 63; q++) if (!isspace((unsigned char)*q)) key[ki++] = *q; key[ki] = 0; if (pending[0] && g_n_peer_names < MAX_PEERS) { peer_name_t *pn = &g_peer_names[g_n_peer_names++]; snprintf(pn->pubkey, sizeof pn->pubkey, "%s", key); snprintf(pn->name, sizeof pn->name, "%s", pending); char *sp = strpbrk(pn->name, " \t"); if (sp) *sp = 0; } pending[0] = 0; } } fclose(f); } static void wg_discover(void){ g_n_wgdevs = 0; g_n_peer_names = 0; DIR *d = opendir("/sys/class/net"); if (!d) return; struct dirent *de; while ((de = readdir(d)) && g_n_wgdevs < MAX_WGDEVS) { if (de->d_name[0] == '.') continue; char p[512], buf[512]; snprintf(p, sizeof p, "/sys/class/net/%s/uevent", de->d_name); if (read_file_str(p, buf, sizeof buf) < 0) continue; if (!strstr(buf, "DEVTYPE=wireguard")) continue; snprintf(g_wgdevs[g_n_wgdevs++], IFNAME_LEN, "%.15s", de->d_name); char conf[512]; snprintf(conf, sizeof conf, "%s/%s.conf", g_wgdir, de->d_name); load_peer_names(conf); } closedir(d); } static void wg_render(sbuf_t *b, const char *dev, time_t now){ sb_addf(b, "%s=== WireGuard (%s) link ===%s\n", BOLD, dev, RST); char key[160]; snprintf(key, sizeof key, "%s/tx_errors", dev); sb_addf(b, " TX errors (no peer/AllowedIPs match): "); ctr_fmt(b, key, sysstat(dev, "tx_errors")); sb_addf(b, "\n"); snprintf(key, sizeof key, "%s/tx_dropped", dev); sb_addf(b, " TX drops : "); ctr_fmt(b, key, sysstat(dev, "tx_dropped")); sb_addf(b, "\n"); snprintf(key, sizeof key, "%s/rx_errors", dev); sb_addf(b, " RX errors (auth/decrypt fail) : "); ctr_fmt(b, key, sysstat(dev, "rx_errors")); sb_addf(b, "\n"); snprintf(key, sizeof key, "%s/rx_dropped", dev); sb_addf(b, " RX drops : "); ctr_fmt(b, key, sysstat(dev, "rx_dropped")); sb_addf(b, "\n"); sb_addf(b, "\n%s=== WireGuard (%s) peers ===%s\n", BOLD, dev, RST); wg_health_t *hl = wg_health_get(dev); if (hl) { hl->total = 0; hl->fresh = 0; hl->best_age = -1; } char cmd[256]; snprintf(cmd, sizeof cmd, "wg show %s dump 2>/dev/null", dev); FILE *p = popen(cmd, "r"); if (!p) { sb_addf(b, " %s(wg unavailable)%s\n", YEL, RST); return; } char line[1024]; int row = 0; while (fgets(line, sizeof line, p)) { rstrip(line); row++; if (row == 1) continue; /* interface line */ char *f[8] = {0}; int nf = 0; char *save = NULL; for (char *t = strtok_r(line, "\t", &save); t && nf < 8; t = strtok_r(NULL, "\t", &save)) f[nf++] = t; if (nf < 7) continue; const char *pub = f[0], *ep = f[2]; time_t hs = (time_t)strtoll(f[4], NULL, 10); unsigned long long rx = strtoull(f[5], NULL, 10); unsigned long long tx = strtoull(f[6], NULL, 10); const char *nm = peer_name(pub); char shortname[16]; if (!nm) { snprintf(shortname, sizeof shortname, "%.12s", pub); nm = shortname; } sb_addf(b, " %-12s hs:", nm); if (hl) hl->total++; if (hs == 0) { sb_addf(b, "%sNEVER%s", RED, RST); alert("peer:%s no handshake ever", nm); } else { long age = (long)(now - hs); if (hl && (hl->best_age < 0 || age < hl->best_age)) hl->best_age = age; if (age > STALE_HANDSHAKE) { sb_addf(b, "%s%lds STALE%s", RED, age, RST); alert("peer:%s handshake %lds", nm, age); } else { if (hl) hl->fresh++; sb_addf(b, "%s%lds%s", GRN, age, RST); } } const char *prev = peer_prev_ep(pub); if (prev && strcmp(prev, ep)) { sb_addf(b, " ep:%s%s (ROAMED from %s)%s", YEL, ep, prev, RST); alert("peer:%s endpoint roamed", nm); } else sb_addf(b, " ep:%s", ep); peer_set_ep(pub, ep); char rxs[32], txs[32]; fmt_iec(rx, rxs, sizeof rxs); fmt_iec(tx, txs, sizeof txs); sb_addf(b, " rx:%s tx:%s\n", rxs, txs); } pclose(p); } /*--------------------------------------------------------------------- * BIRD OSPF *-------------------------------------------------------------------*/ typedef struct { char ip[48]; char name[64]; } ospf_exp_t; static ospf_exp_t g_ospf_exp[MAX_OSPF_EXP]; static int g_n_ospf_exp = 0; static int looks_ipv4(const char *s, char *out, size_t n){ for (const char *p = s; *p; p++) { unsigned a, b, c, d; int len; if (!isdigit((unsigned char)*p)) continue; if (sscanf(p, "%u.%u.%u.%u%n", &a, &b, &c, &d, &len) == 4 && a < 256 && b < 256 && c < 256 && d < 256) { snprintf(out, n, "%u.%u.%u.%u", a, b, c, d); return 1; } while (isdigit((unsigned char)p[1])) p++; } return 0; } static void load_ospf_expected(const char *conf){ g_n_ospf_exp = 0; FILE *f = fopen(conf, "r"); if (!f) return; char line[512]; int in_neigh = 0; while (fgets(line, sizeof line, f)) { rstrip(line); char *s = lskip(line); if (*s == '#') continue; if (!in_neigh) { if (strstr(s, "neighbors") && strchr(s, '{')) in_neigh = 1; continue; } if (strchr(s, '}')) { in_neigh = 0; continue; } char name[64] = ""; char *h = strchr(s, '#'); if (h) { char *c = lskip(h + 1); snprintf(name, sizeof name, "%s", c); rstrip(name); *h = 0; } char ip[48]; if (looks_ipv4(s, ip, sizeof ip) && g_n_ospf_exp < MAX_OSPF_EXP) { ospf_exp_t *e = &g_ospf_exp[g_n_ospf_exp++]; snprintf(e->ip, sizeof e->ip, "%s", ip); snprintf(e->name, sizeof e->name, "%s", name[0] ? name : ip); } } fclose(f); } static void ospf_render(sbuf_t *b){ char probe[256]; snprintf(probe, sizeof probe, "command -v %s >/dev/null 2>&1", g_birdc); if (system(probe) != 0) return; sb_addf(b, "\n%s=== OSPF neighbors (bird) ===%s\n", BOLD, RST); char cmd[256]; snprintf(cmd, sizeof cmd, "%s show ospf neighbors 2>&1", g_birdc); FILE *p = popen(cmd, "r"); if (!p) return; char seen[MAX_OSPF_EXP][48]; int n_seen = 0, has_table = 0; char first[256] = ""; char line[512]; while (fgets(line, sizeof line, p)) { rstrip(line); if (!first[0] && line[0]) snprintf(first, sizeof first, "%.255s", line); if (strcasestr(line, "router id")) { has_table = 1; continue; } char *s = lskip(line); unsigned a,b2,c,d; int len; if (sscanf(s, "%u.%u.%u.%u%n", &a, &b2, &c, &d, &len) != 4 || a>255||b2>255||c>255||d>255) continue; char last_ip[48] = ""; const char *q = s; while (*q) { char ip[48]; if (looks_ipv4(q, ip, sizeof ip)) { snprintf(last_ip, sizeof last_ip, "%s", ip); q = strstr(q, ip) + strlen(ip); } else break; } if (last_ip[0] && n_seen < MAX_OSPF_EXP) snprintf(seen[n_seen++], 48, "%s", last_ip); if (strcasestr(s, "full")) sb_addf(b, " %s%s%s\n", GRN, s, RST); else { sb_addf(b, " %s%s%s\n", RED, s, RST); alert("OSPF: %s", s); } } pclose(p); if (has_table) { for (int i = 0; i < g_n_ospf_exp; i++) { int found = 0; for (int j = 0; j < n_seen; j++) if (!strcmp(g_ospf_exp[i].ip, seen[j])) { found = 1; break; } if (!found) { sb_addf(b, " %s%s (%s) MISSING - configured in bird.conf, " "no adjacency%s\n", RED, g_ospf_exp[i].ip, g_ospf_exp[i].name, RST); alert("OSPF: %s MISSING", g_ospf_exp[i].name); } } } else { sb_addf(b, " %sbirdc returned no neighbor table -- skipping " "MISSING check%s\n", YEL, RST); sb_addf(b, " %sbirdc said: %.120s%s\n", DIM, first, RST); alert("birdc unreadable"); } } /*--------------------------------------------------------------------- * Interface list assembly *-------------------------------------------------------------------*/ static char g_ifaces[MAX_IFACES][IFNAME_LEN]; static int g_n_ifaces = 0; static void add_iface(const char *name){ if (g_n_ifaces >= MAX_IFACES) return; for (int i = 0; i < g_n_ifaces; i++) if (!strcmp(g_ifaces[i], name)) return; snprintf(g_ifaces[g_n_ifaces++], IFNAME_LEN, "%s", name); } static void iface_discover(void){ g_n_ifaces = 0; if (if_exists(g_wan)) add_iface(g_wan); if (if_exists(g_lan)) add_iface(g_lan); DIR *d = opendir("/sys/class/net"); if (d) { struct dirent *de; size_t wl = strlen(g_wan), ll = strlen(g_lan); while ((de = readdir(d))) { if (de->d_name[0] == '.') continue; if ((!strncmp(de->d_name, g_lan, ll) && de->d_name[ll] == '.') || (!strncmp(de->d_name, g_wan, wl) && de->d_name[wl] == '.')) add_iface(de->d_name); } closedir(d); } for (int i = 0; i < g_n_wgdevs; i++) add_iface(g_wgdevs[i]); if (if_exists("he6")) add_iface("he6"); if (if_exists(DEF_MGMT_IF)) add_iface(DEF_MGMT_IF); } /*--------------------------------------------------------------------- * v2: per-tick interface snapshot (read counters ONCE, render on both * the overview and the detail page without double-counting) *-------------------------------------------------------------------*/ typedef struct { unsigned long long rx_bytes, tx_bytes, rx_packets, tx_packets; unsigned long long rx_errors, tx_errors, rx_dropped, tx_dropped; unsigned long long rx_missed, rx_fifo, rx_over, rx_crc, rx_frame; unsigned long long tx_fifo, tx_carrier, collisions, multicast; unsigned long long carrier_changes, qdrops; unsigned long flags; /* IFF_* from /sys */ int is_wg; char operstate[16]; char qkind[64]; long speed_mbit; /* -1 unknown */ long mtu; double rx_bps, tx_bps, rx_pps, tx_pps; /* rates since last tick */ } ifsnap_t; static ifsnap_t g_snap[MAX_IFACES]; static void ifsnap_take(int idx, const char *ifn, double dt){ ifsnap_t *s = &g_snap[idx]; memset(s->operstate, 0, sizeof s->operstate); s->rx_bytes = sysstat(ifn, "rx_bytes"); s->tx_bytes = sysstat(ifn, "tx_bytes"); s->rx_packets = sysstat(ifn, "rx_packets"); s->tx_packets = sysstat(ifn, "tx_packets"); s->rx_errors = sysstat(ifn, "rx_errors"); s->tx_errors = sysstat(ifn, "tx_errors"); s->rx_dropped = sysstat(ifn, "rx_dropped"); s->tx_dropped = sysstat(ifn, "tx_dropped"); s->rx_missed = sysstat(ifn, "rx_missed_errors"); s->rx_fifo = sysstat(ifn, "rx_fifo_errors"); s->rx_over = sysstat(ifn, "rx_over_errors"); s->rx_crc = sysstat(ifn, "rx_crc_errors"); s->rx_frame = sysstat(ifn, "rx_frame_errors"); s->tx_fifo = sysstat(ifn, "tx_fifo_errors"); s->tx_carrier = sysstat(ifn, "tx_carrier_errors"); s->collisions = sysstat(ifn, "collisions"); s->multicast = sysstat(ifn, "multicast"); s->qdrops = qdisc_drops(ifn); qdisc_kind(ifn, s->qkind, sizeof s->qkind); char p[256], buf[64]; snprintf(p, sizeof p, "/sys/class/net/%s/flags", ifn); s->flags = read_file_str(p, buf, sizeof buf) == 0 ? strtoul(buf, NULL, 0) : 0; s->is_wg = 0; for (int i = 0; i < g_n_wgdevs; i++) if (!strcmp(g_wgdevs[i], ifn)) { s->is_wg = 1; break; } snprintf(p, sizeof p, "/sys/class/net/%s/carrier_changes", ifn); s->carrier_changes = read_file_ull(p); snprintf(p, sizeof p, "/sys/class/net/%s/operstate", ifn); if (read_file_str(p, buf, sizeof buf) == 0) { rstrip(buf); snprintf(s->operstate, sizeof s->operstate, "%.15s", buf); } snprintf(p, sizeof p, "/sys/class/net/%s/speed", ifn); s->speed_mbit = read_file_str(p, buf, sizeof buf) == 0 ? strtol(buf, NULL, 10) : -1; snprintf(p, sizeof p, "/sys/class/net/%s/mtu", ifn); s->mtu = (long)read_file_ull(p); char key[192]; snprintf(key, sizeof key, "%s/r.rxb", ifn); s->rx_bps = ctr_rate(key, s->rx_bytes, dt); snprintf(key, sizeof key, "%s/r.txb", ifn); s->tx_bps = ctr_rate(key, s->tx_bytes, dt); snprintf(key, sizeof key, "%s/r.rxp", ifn); s->rx_pps = ctr_rate(key, s->rx_packets, dt); snprintf(key, sizeof key, "%s/r.txp", ifn); s->tx_pps = ctr_rate(key, s->tx_packets, dt); } /* Resolve a display state + color for an interface. Physical NICs use * operstate as-is. Carrier-less virtual devices (wireguard, sit/he6, * lo) report operstate=unknown forever, so fall back to IFF_UP / * IFF_LOWER_UP -- and for WG devices, judge by peer handshake health. */ #ifndef IFF_LOWER_UP_HM #define IFF_LOWER_UP_HM 0x10000 #endif static void link_state(const ifsnap_t *s, const char *ifname, char *out, size_t n, const char **col){ if (s->is_wg) { const wg_health_t *h = NULL; for (int i = 0; i < g_n_wg_health; i++) if (!strcmp(g_wg_health[i].dev, ifname)) { h = &g_wg_health[i]; break; } if (!(s->flags & 0x1)) { /* IFF_UP */ snprintf(out, n, "down"); *col = RED; return; } if (h && h->total > 0) { if (h->fresh > 0) { snprintf(out, n, "up %d/%d peers", h->fresh, h->total); *col = GRN; } else { snprintf(out, n, "up 0/%d peers STALE", h->total); *col = RED; } } else { snprintf(out, n, "up (no peer data)"); *col = YEL; } return; } if (!strcmp(s->operstate, "unknown")) { /* other virtual */ if ((s->flags & 0x1) && ((s->flags & 0x40) || (s->flags & IFF_LOWER_UP_HM))) { snprintf(out, n, "up*"); /* * = no carrier */ *col = GRN; } else if (s->flags & 0x1) { snprintf(out, n, "up? (no-carrier)"); *col = YEL; } else { snprintf(out, n, "down"); *col = RED; } return; } snprintf(out, n, "%s", s->operstate[0] ? s->operstate : "?"); *col = strcmp(s->operstate, "up") ? RED : GRN; } /*--------------------------------------------------------------------- * v2: BURST CATCHER * Non-blocking state machine driven from the main loop. * idle: poll tx_dropped on the watched iface each tick * capturing: stacktrace trigger armed, tcpdump forked; when the * window elapses, harvest trace -> /tmp/burst-/ *-------------------------------------------------------------------*/ #define BURST_HIST 8 #define TRACE_READ_MAX (4u * 1024u * 1024u) typedef struct { char when[24]; char dir[96]; long long trigger_delta; /* drops seen at detection */ long long window_drops; /* drops during the capture window */ int stacks; /* "" count */ char summary[192]; /* top TX-path frames */ } burst_rec_t; static burst_rec_t g_bursts[BURST_HIST]; static int g_n_bursts = 0; /* total caught (ring) */ static int g_burst_state = 0; /* 0 idle, 1 capturing */ static double g_burst_t0 = 0; static unsigned long long g_burst_prev = 0; static int g_burst_prev_ok = 0; static unsigned long long g_burst_startval = 0; static long long g_burst_trigdelta = 0; static pid_t g_tcpdump_pid = -1; static char g_tracedir[64] = ""; /* "" = ftrace unavailable */ static char g_burst_dir[96] = ""; static char g_burst_why[128] = ""; /* reason disabled, if any */ static const char *TXPATH_FRAMES[] = { "__dev_queue_xmit", "sch_direct_xmit", "dev_hard_start_xmit", "nf_flow_queue_xmit", "vlan_dev_hard_start_xmit", "ip6_finish_output", "ip_finish_output", "ip6_output", "ip_output", "ndisc_send_skb", "arp_solicit", "i40e_", "fq_", "qdisc_drop", NULL }; static int write_str(const char *path, const char *s){ int fd = open(path, O_WRONLY); if (fd < 0) return -1; ssize_t r = write(fd, s, strlen(s)); close(fd); return r < 0 ? -1 : 0; } static void burst_find_tracedir(void){ static const char *cand[] = { "/sys/kernel/tracing", "/sys/kernel/debug/tracing", NULL }; g_tracedir[0] = 0; for (int i = 0; cand[i]; i++) { char p[128]; snprintf(p, sizeof p, "%s/events/skb/kfree_skb/trigger", cand[i]); if (access(p, W_OK) == 0) { snprintf(g_tracedir, sizeof g_tracedir, "%s", cand[i]); return; } } } static void burst_trigger(int on){ if (!g_tracedir[0]) return; char p[128]; snprintf(p, sizeof p, "%s/events/skb/kfree_skb/trigger", g_tracedir); if (on) { write_str(p, "!stacktrace"); /* clear stale, best effort */ write_str(p, "stacktrace"); snprintf(p, sizeof p, "%s/trace", g_tracedir); write_str(p, " "); /* clear ring buffer */ } else { write_str(p, "!stacktrace"); } } static void burst_begin(long long delta, time_t now){ struct tm tmv; localtime_r(&now, &tmv); char ts[24]; strftime(ts, sizeof ts, "%Y%m%d-%H%M%S", &tmv); snprintf(g_burst_dir, sizeof g_burst_dir, "/tmp/burst-%s", ts); mkdir(g_burst_dir, 0755); burst_trigger(1); g_tcpdump_pid = -1; if (g_burst_pcap) { pid_t pid = fork(); if (pid == 0) { char pcap[160], cnt[16]; snprintf(pcap, sizeof pcap, "%s/burst.pcap", g_burst_dir); snprintf(cnt, sizeof cnt, "400"); int nul = open("/dev/null", O_RDWR); if (nul >= 0) { dup2(nul, 1); dup2(nul, 2); close(nul); } execlp("tcpdump", "tcpdump", "-i", g_burst_if, "-n", "-c", cnt, "-w", pcap, (char *)NULL); _exit(127); } g_tcpdump_pid = pid; } g_burst_state = 1; g_burst_t0 = now_mono(); g_burst_trigdelta = delta; alert("BURST +%lld on %s (capturing)", delta, g_burst_if); } /* scan saved trace: count stacks + interesting TX-path frames */ static void burst_summarize(const char *tracefile, burst_rec_t *r){ r->stacks = 0; r->summary[0] = 0; FILE *f = fopen(tracefile, "r"); if (!f) return; int counts[32] = {0}; char line[512]; while (fgets(line, sizeof line, f)) { if (strstr(line, "")) { r->stacks++; continue; } char *arrow = strstr(line, "=> "); if (!arrow) continue; char *fn = arrow + 3; for (int i = 0; TXPATH_FRAMES[i]; i++) if (!strncmp(fn, TXPATH_FRAMES[i], strlen(TXPATH_FRAMES[i]))) { counts[i]++; break; } } fclose(f); /* top 3 into summary */ size_t off = 0; for (int pick = 0; pick < 3; pick++) { int best = -1; for (int i = 0; TXPATH_FRAMES[i]; i++) if (counts[i] > 0 && (best < 0 || counts[i] > counts[best])) best = i; if (best < 0) break; off += (size_t)snprintf(r->summary + off, sizeof r->summary - off, "%s%d %s", pick ? ", " : "", counts[best], TXPATH_FRAMES[best]); counts[best] = 0; if (off >= sizeof r->summary - 1) break; } if (!r->summary[0]) snprintf(r->summary, sizeof r->summary, "no tx-path frames (hw counter / rx-side only)"); } static void burst_finish(void){ /* harvest ftrace ring into the burst dir */ char src[128], dst[160]; snprintf(src, sizeof src, "%s/trace", g_tracedir); snprintf(dst, sizeof dst, "%s/stacks.txt", g_burst_dir); int in = open(src, O_RDONLY); int out = open(dst, O_WRONLY | O_CREAT | O_TRUNC, 0644); if (in >= 0 && out >= 0) { char buf[65536]; ssize_t r; size_t total = 0; while ((r = read(in, buf, sizeof buf)) > 0 && total < TRACE_READ_MAX) { if (write(out, buf, (size_t)r) < 0) break; total += (size_t)r; } } if (in >= 0) close(in); if (out >= 0) close(out); burst_trigger(0); if (g_tcpdump_pid > 0) { kill(g_tcpdump_pid, SIGTERM); int st; waitpid(g_tcpdump_pid, &st, 0); g_tcpdump_pid = -1; } unsigned long long endval = sysstat(g_burst_if, "tx_dropped"); burst_rec_t *r = &g_bursts[g_n_bursts % BURST_HIST]; memset(r, 0, sizeof *r); time_t now = time(NULL); struct tm tmv; localtime_r(&now, &tmv); strftime(r->when, sizeof r->when, "%H:%M:%S", &tmv); snprintf(r->dir, sizeof r->dir, "%s", g_burst_dir); r->trigger_delta = g_burst_trigdelta; r->window_drops = (long long)endval - (long long)g_burst_startval; burst_summarize(dst, r); g_n_bursts++; char cpath[160], cbuf[160]; snprintf(cpath, sizeof cpath, "%s/counters.txt", g_burst_dir); snprintf(cbuf, sizeof cbuf, "iface=%s trigger_delta=%lld start=%llu end=%llu " "window_drops=%lld\n", g_burst_if, g_burst_trigdelta, g_burst_startval, endval, r->window_drops); FILE *cf = fopen(cpath, "w"); if (cf) { fputs(cbuf, cf); fclose(cf); } g_burst_state = 0; g_burst_prev = endval; /* don't re-trigger on our own window */ g_burst_prev_ok = 1; } /* called once per tick while idle; detects increments */ static void burst_poll(void){ if (!g_burst_arm || !g_tracedir[0] || g_burst_state != 0) return; unsigned long long v = sysstat(g_burst_if, "tx_dropped"); if (!g_burst_prev_ok) { g_burst_prev = v; g_burst_prev_ok = 1; return; } long long d = (long long)v - (long long)g_burst_prev; g_burst_prev = v; if (d > 0) { g_burst_startval = v; burst_begin(d, time(NULL)); } } /* called frequently (sub-interval); returns 1 if a capture finished */ static int burst_maybe_finish(void){ if (g_burst_state == 1 && now_mono() - g_burst_t0 >= g_burst_secs) { burst_finish(); return 1; } return 0; } static void burst_render(sbuf_t *b){ sb_addf(b, "%s=== Burst catcher ===%s\n", BOLD, RST); if (!g_tracedir[0]) { sb_addf(b, " %sDISABLED: %s%s\n", YEL, g_burst_why[0] ? g_burst_why : "ftrace kfree_skb trigger not writable " "(need root / tracefs)", RST); return; } sb_addf(b, " watching : %s (tx_dropped, poll %ds)\n", g_burst_if, g_interval); sb_addf(b, " tracefs : %s\n", g_tracedir); sb_addf(b, " window : %ds pcap: %s armed: %s%s%s " "(key 'b' toggles)\n", g_burst_secs, g_burst_pcap ? "yes (400 pkts)" : "no", g_burst_arm ? GRN : YEL, g_burst_arm ? "YES" : "no", RST); if (g_burst_state == 1) sb_addf(b, " state : %sCAPTURING%s (%.1fs remain) -> %s\n", RED, RST, (double)g_burst_secs - (now_mono() - g_burst_t0), g_burst_dir); else sb_addf(b, " state : %sidle%s\n", DIM, RST); sb_addf(b, "\n%s--- caught bursts (newest first, %d total) ---%s\n", BOLD, g_n_bursts, RST); if (!g_n_bursts) { sb_addf(b, " %s(none yet -- waiting for %s/tx_dropped to move)%s\n", DIM, g_burst_if, RST); return; } int shown = g_n_bursts < BURST_HIST ? g_n_bursts : BURST_HIST; for (int k = 0; k < shown; k++) { burst_rec_t *r = &g_bursts[(g_n_bursts - 1 - k) % BURST_HIST]; sb_addf(b, " %s%s%s +%lld trig, %lld in window, %d stacks\n", BOLD, r->when, RST, r->trigger_delta, r->window_drops, r->stacks); sb_addf(b, " tx-path: %s%s%s\n", YEL, r->summary, RST); sb_addf(b, " %s%s/{stacks.txt,burst.pcap,counters.txt}%s\n", DIM, r->dir, RST); } } /*--------------------------------------------------------------------- * Section renders (v1 sections, now routed to pages) *-------------------------------------------------------------------*/ static void render_udp(sbuf_t *b){ sb_addf(b, "%s=== UDP underlay ===%s\n", BOLD, RST); sb_addf(b, " UdpRcvbufErrors (WG socket overrun?): "); ctr_fmt(b, "UdpRcvbufErrors", nst_get("UdpRcvbufErrors")); sb_addf(b, "\n"); sb_addf(b, " UdpSndbufErrors : "); ctr_fmt(b, "UdpSndbufErrors", nst_get("UdpSndbufErrors")); sb_addf(b, "\n"); sb_addf(b, " UdpInErrors : "); ctr_fmt(b, "UdpInErrors", nst_get("UdpInErrors")); sb_addf(b, "\n"); } static void render_ip(sbuf_t *b){ sb_addf(b, "\n%s=== IP stack ===%s\n", BOLD, RST); unsigned long long noroutes = nst_get("IpExtInNoRoutes"); if (!noroutes) noroutes = nst_get("InNoRoutes"); sb_addf(b, " InNoRoutes : "); ctr_fmt(b, "IpInNoRoutes", noroutes); sb_addf(b, "\n"); sb_addf(b, " InDiscards : "); ctr_fmt(b, "IpInDiscards", nst_get("IpInDiscards")); sb_addf(b, "\n"); sb_addf(b, " OutDiscards : "); ctr_fmt(b, "IpOutDiscards", nst_get("IpOutDiscards")); sb_addf(b, "\n"); sb_addf(b, " OutNoRoutes : "); ctr_fmt(b, "IpOutNoRoutes", nst_get("IpOutNoRoutes")); sb_addf(b, "\n"); sb_addf(b, " FragFails (DF set): "); ctr_fmt(b, "IpFragFails", nst_get("IpFragFails")); sb_addf(b, "\n"); sb_addf(b, " ReasmFails : "); ctr_fmt(b, "IpReasmFails", nst_get("IpReasmFails")); sb_addf(b, "\n"); sb_addf(b, " ICMP DestUnreach out (incl frag-needed): "); ctr_fmt(b, "IcmpOutDestUnreachs", nst_get("IcmpOutDestUnreachs")); sb_addf(b, "\n"); sb_addf(b, " Ip6 InDiscards : "); ctr_fmt(b, "Ip6InDiscards", nst_get("Ip6InDiscards")); sb_addf(b, "\n"); sb_addf(b, " Ip6 OutNoRoutes : "); ctr_fmt(b, "Ip6OutNoRoutes", nst_get("Ip6OutNoRoutes")); sb_addf(b, "\n"); sb_addf(b, " Icmp6 PktTooBig out: "); ctr_fmt(b, "Icmp6OutPktToobigs", nst_get("Icmp6OutPktToobigs")); sb_addf(b, "\n"); /* v2: rp_filter kills are invisible everywhere else; surface them */ sb_addf(b, " IPReversePathFilter: "); ctr_fmt(b, "TcpExtIPReversePathFilter", nst_get("TcpExtIPReversePathFilter")); sb_addf(b, " %s(strict rp_filter eating asymmetric OSPF paths?)%s\n", DIM, RST); } static void render_softnet(sbuf_t *b){ unsigned long long dr, sq; softnet(&dr, &sq); sb_addf(b, "\n%s=== Softnet (kernel ingress) ===%s\n", BOLD, RST); sb_addf(b, " Drops : "); ctr_fmt(b, "softnet_drops", dr); sb_addf(b, "\n"); sb_addf(b, " Time squeeze : "); ctr_fmt(b, "softnet_squeeze", sq); sb_addf(b, " %s(budget exhausted; sustained growth at 10G = raise " "netdev_budget)%s\n", DIM, RST); } static void render_conntrack(sbuf_t *b){ unsigned long long cnt = read_file_ull("/proc/sys/net/netfilter/nf_conntrack_count"); unsigned long long max = read_file_ull("/proc/sys/net/netfilter/nf_conntrack_max"); if (!max) max = 1; unsigned pct = (unsigned)(cnt * 100 / max); const char *col = DIM; if (pct > 70) col = YEL; if (pct > 90) { col = RED; alert("conntrack %u%%", pct); } sb_addf(b, "\n%s=== Conntrack ===%s\n %s%llu / %llu (%u%%)%s\n", BOLD, RST, col, cnt, max, pct, RST); } /* Overview page: one compact line per interface (no alerts here; the * detail page owns alerting for these keys) */ static void render_iface_brief(sbuf_t *b, int idx, const char *ifname){ ifsnap_t *s = &g_snap[idx]; char rxr[32], txr[32], key[192], st[40]; const char *sc; link_state(s, ifname, st, sizeof st, &sc); fmt_iec_d(s->rx_bps, rxr, sizeof rxr); fmt_iec_d(s->tx_bps, txr, sizeof txr); sb_addf(b, " %-18s %s%-18s%s rx:%sB/s tx:%sB/s ", ifname, sc, st, RST, rxr, txr); snprintf(key, sizeof key, "%s/l.rx_dropped", ifname); sb_addf(b, "rxD:"); ctr_fmtx(b, key, s->rx_dropped, 0); snprintf(key, sizeof key, "%s/l.tx_dropped", ifname); sb_addf(b, " txD:"); ctr_fmtx(b, key, s->tx_dropped, 0); snprintf(key, sizeof key, "%s/l.qdrops", ifname); sb_addf(b, " qD:"); ctr_fmtx(b, key, s->qdrops, 0); sb_addf(b, "\n"); } /* Interfaces page: the works */ static void render_iface_full(sbuf_t *b, int idx, const char *ifname, ethdev_t *eths, int n_eths){ ifsnap_t *s = &g_snap[idx]; char key[192], rxr[32], txr[32], st[40]; const char *sc; link_state(s, ifname, st, sizeof st, &sc); sb_addf(b, "\n%s=== %s ===%s\n", BOLD, ifname, RST); /* link line */ sb_addf(b, " link : %s%s%s", sc, st, RST); if (s->speed_mbit > 0) sb_addf(b, " %ldMb/s", s->speed_mbit); sb_addf(b, " mtu:%ld", s->mtu); if (s->qkind[0]) sb_addf(b, " qdisc:%s%s%s", strstr(s->qkind, "fq_codel") ? GRN : (strstr(s->qkind, "/fq") || !strcmp(s->qkind,"fq")) ? YEL : DIM, s->qkind, RST); sb_addf(b, " carrier_changes:"); snprintf(key, sizeof key, "%s/carrier_changes", ifname); if (ctr_fmt(b, key, s->carrier_changes) > 0) sb_addf(b, " %s<< LINK FLAP%s", RED, RST); sb_addf(b, "\n"); /* rates */ fmt_iec_d(s->rx_bps, rxr, sizeof rxr); fmt_iec_d(s->tx_bps, txr, sizeof txr); sb_addf(b, " rate : rx %sB/s (%.0f pps) tx %sB/s (%.0f pps) " "mcast total:%llu\n", rxr, s->rx_pps, txr, s->tx_pps, s->multicast); /* netdev error family */ snprintf(key, sizeof key, "%s/l.rx_errors", ifname); sb_addf(b, " rx : err:"); ctr_fmt(b, key, s->rx_errors); snprintf(key, sizeof key, "%s/l.rx_dropped", ifname); sb_addf(b, " drop:"); ctr_fmt(b, key, s->rx_dropped); snprintf(key, sizeof key, "%s/l.rx_missed", ifname); sb_addf(b, " missed:"); ctr_fmt(b, key, s->rx_missed); snprintf(key, sizeof key, "%s/l.rx_fifo", ifname); sb_addf(b, " fifo:"); ctr_fmt(b, key, s->rx_fifo); snprintf(key, sizeof key, "%s/l.rx_over", ifname); sb_addf(b, " overrun:"); ctr_fmt(b, key, s->rx_over); snprintf(key, sizeof key, "%s/l.rx_crc", ifname); sb_addf(b, " crc:"); ctr_fmt(b, key, s->rx_crc); snprintf(key, sizeof key, "%s/l.rx_frame", ifname); sb_addf(b, " frame:"); ctr_fmt(b, key, s->rx_frame); sb_addf(b, "\n"); snprintf(key, sizeof key, "%s/l.tx_errors", ifname); sb_addf(b, " tx : err:"); ctr_fmt(b, key, s->tx_errors); snprintf(key, sizeof key, "%s/l.tx_dropped", ifname); sb_addf(b, " drop:"); ctr_fmt(b, key, s->tx_dropped); snprintf(key, sizeof key, "%s/l.tx_fifo", ifname); sb_addf(b, " fifo:"); ctr_fmt(b, key, s->tx_fifo); snprintf(key, sizeof key, "%s/l.tx_carrier", ifname); sb_addf(b, " carrier:"); ctr_fmt(b, key, s->tx_carrier); snprintf(key, sizeof key, "%s/l.collisions", ifname); sb_addf(b, " colls:"); ctr_fmt(b, key, s->collisions); sb_addf(b, "\n"); snprintf(key, sizeof key, "%s/l.qdrops", ifname); sb_addf(b, " qdisc drops: "); ctr_fmt(b, key, s->qdrops); if (s->qkind[0] && (strstr(s->qkind, "/fq") || !strcmp(s->qkind, "fq")) && !strstr(s->qkind, "fq_codel")) sb_addf(b, " %s(fq flow_limit on a router: bursty forwarded flows " "get clipped -- use fq_codel)%s", YEL, RST); sb_addf(b, "\n"); for (int i = 0; i < n_eths; i++) { if (strcmp(eths[i].ifname, ifname)) continue; ethdev_render_pflags(b, ðs[i]); ethdev_render(b, ðs[i]); } } /*--------------------------------------------------------------------- * Terminal handling + key input *-------------------------------------------------------------------*/ static struct termios g_tio_saved; static int g_tio_set = 0; static void term_raw(void){ if (!isatty(0)) return; if (tcgetattr(0, &g_tio_saved) < 0) return; struct termios t = g_tio_saved; t.c_lflag &= ~(ICANON | ECHO); t.c_cc[VMIN] = 0; t.c_cc[VTIME] = 0; if (tcsetattr(0, TCSANOW, &t) == 0) g_tio_set = 1; } static void restore_term(void){ if (g_tio_set) tcsetattr(0, TCSANOW, &g_tio_saved); if (g_color) fputs("\x1b[?25h", stdout); /* show cursor */ fflush(stdout); } static void full_cleanup(void){ if (g_burst_state == 1) { burst_trigger(0); if (g_tcpdump_pid > 0) kill(g_tcpdump_pid, SIGTERM); } restore_term(); } static void on_sigint(int sig){ (void)sig; full_cleanup(); _exit(0); } /*--------------------------------------------------------------------- * Page 4: hellgate (XDP gate) - see hellgate_page.c *-------------------------------------------------------------------*/ #include "hellgate_page.c" /*--------------------------------------------------------------------- * Painting *-------------------------------------------------------------------*/ static time_t g_last_ts = 0; static int g_neths_shown = 0; static ethdev_t *g_eths_ptr = NULL; static void paint(void){ if (g_clear) fputs("\x1b[H\x1b[2J\x1b[3J", stdout); char ts[64]; struct tm tmv; localtime_r(&g_last_ts, &tmv); strftime(ts, sizeof ts, "%a %b %e %H:%M:%S %Y", &tmv); printf("%s%s interval=%ds (red = incremented this interval)%s\n", BOLD, ts, g_interval, RST); /* tab bar */ printf(" "); for (int i = 0; i < PG_COUNT; i++) { if (i == g_page) printf("%s[%d:%s]%s ", BOLD, i + 1, PG_NAMES[i], RST); else printf("%s[%d:%s]%s ", DIM, i + 1, PG_NAMES[i], RST); } printf("%s keys: 1-4/n/p page b=arm burst q=quit%s", DIM, RST); if (g_burst_state == 1) printf(" %s[CAPTURING]%s", RED, RST); printf("\n"); if (n_alerts) printf("%s%s!! ACTIVE: %s%s\n", RED, BOLD, ALERTS.s, RST); else printf("%sAll quiet.%s\n", GRN, RST); printf("\n%s", PAGES[g_page].s); fflush(stdout); } /*--------------------------------------------------------------------- * usage / main *-------------------------------------------------------------------*/ static void usage(const char *argv0){ printf( "hellmon v2 - THE ROUTER FROM HELL drop/error monitor (Magikarp edition)\n\n" "Usage: %s [options]\n\n" " -i SEC sample interval (default 2)\n" " -n COUNT exit after COUNT samples (default: run forever)\n" " -1, --once single snapshot of ALL pages, no clearing (cron friendly)\n" " -w IF WAN interface (default %s)\n" " -l IF LAN interface (default %s)\n" " -b FILE bird config (default %s)\n" " -W DIR wireguard confs (default %s)\n" " -B CMD birdc command (default %s)\n" " -r SEC topology rediscover period (default 60)\n" " -C force color on -M force color off (mono)\n" " -k keep scrollback (don't clear screen between samples)\n" "burst catcher:\n" " -I IF iface to watch for tx_dropped bursts (default: LAN)\n" " -s SEC capture window seconds (default 5)\n" " -P no pcap (skip forking tcpdump during captures)\n" " -D disable the burst catcher entirely\n" "keys (live): 1/2/3 or n/p switch page, b toggles burst arming, q quits\n", argv0, DEF_WAN_IF, DEF_LAN_IF, DEF_BIRDCONF, DEF_WGDIR, DEF_BIRDC); } int main(int argc, char **argv){ g_wan = DEF_WAN_IF; g_lan = DEF_LAN_IF; g_birdconf = DEF_BIRDCONF; g_wgdir = DEF_WGDIR; g_birdc = DEF_BIRDC; for (int i = 1; i < argc; i++) { const char *a = argv[i]; if (!strcmp(a, "-h") || !strcmp(a, "--help")) { usage(argv[0]); return 0; } else if (!strcmp(a, "-1") || !strcmp(a, "--once")) { g_count = 1; g_clear = 0; } else if (!strcmp(a, "-C")) g_color = 1; else if (!strcmp(a, "-M")) g_color = 0; else if (!strcmp(a, "-k")) g_clear = 0; else if (!strcmp(a, "-P")) g_burst_pcap = 0; else if (!strcmp(a, "-D")) g_burst_arm = 0; else if (!strcmp(a, "-i") && i+1 < argc) g_interval = atoi(argv[++i]); else if (!strcmp(a, "-n") && i+1 < argc) g_count = atol(argv[++i]); else if (!strcmp(a, "-r") && i+1 < argc) g_rediscover = atoi(argv[++i]); else if (!strcmp(a, "-w") && i+1 < argc) g_wan = argv[++i]; else if (!strcmp(a, "-l") && i+1 < argc) g_lan = argv[++i]; else if (!strcmp(a, "-b") && i+1 < argc) g_birdconf = argv[++i]; else if (!strcmp(a, "-W") && i+1 < argc) g_wgdir = argv[++i]; else if (!strcmp(a, "-B") && i+1 < argc) g_birdc = argv[++i]; else if (!strcmp(a, "-I") && i+1 < argc) g_burst_if = argv[++i]; else if (!strcmp(a, "-s") && i+1 < argc) g_burst_secs = atoi(argv[++i]); else { fprintf(stderr, "unknown option: %s\n", a); return 2; } } if (g_interval < 1) g_interval = 1; if (g_burst_secs < 1) g_burst_secs = 1; if (!g_burst_if) g_burst_if = g_lan; if (g_color == -1) g_color = isatty(1); if (g_color) { RED="\x1b[31;1m"; YEL="\x1b[33;1m"; GRN="\x1b[32m"; DIM="\x1b[2m"; RST="\x1b[0m"; BOLD="\x1b[1m"; } if (geteuid() != 0) fprintf(stderr, "%swarning: not root; some counters (conntrack, " "wg peers, burst catcher) will be empty%s\n", YEL, RST); g_ethsock = socket(AF_INET, SOCK_DGRAM, 0); if (g_ethsock < 0) { perror("socket"); return 1; } burst_find_tracedir(); if (!g_tracedir[0]) snprintf(g_burst_why, sizeof g_burst_why, "kfree_skb trigger not writable (need root + " "CONFIG_FTRACE / mounted tracefs)"); signal(SIGINT, on_sigint); signal(SIGTERM, on_sigint); atexit(full_cleanup); if (g_color && g_clear) fputs("\x1b[?25l", stdout); /* hide cursor */ int interactive = g_clear && isatty(0) && g_count != 1; if (interactive) term_raw(); for (int i = 0; i < PG_COUNT; i++) sb_init(&PAGES[i]); sb_init(&ALERTS); ethdev_t eths[2]; int n_eths = 0; wg_discover(); iface_discover(); load_ospf_expected(g_birdconf); if (if_exists(g_wan)) ethdev_discover(ðs[n_eths++], g_wan); if (if_exists(g_lan) && strcmp(g_wan, g_lan)) ethdev_discover(ðs[n_eths++], g_lan); g_eths_ptr = eths; time_t last_disc = time(NULL); double prev_tick = now_mono(); int first_tick = 1; long iter = 0; int quit = 0; while (!quit && (g_count < 0 || iter < g_count)) { time_t now = time(NULL); g_last_ts = now; if (g_hg_force_redisc) { last_disc = 0; g_hg_force_redisc = 0; } if (now - last_disc >= g_rediscover) { wg_discover(); iface_discover(); load_ospf_expected(g_birdconf); for (int i = 0; i < n_eths; i++) ethdev_free(ðs[i]); n_eths = 0; if (if_exists(g_wan)) ethdev_discover(ðs[n_eths++], g_wan); if (if_exists(g_lan) && strcmp(g_wan, g_lan)) ethdev_discover(ðs[n_eths++], g_lan); last_disc = now; } g_neths_shown = n_eths; double tick_now = now_mono(); double dt = first_tick ? (double)g_interval : tick_now - prev_tick; prev_tick = tick_now; first_tick = 0; for (int i = 0; i < PG_COUNT; i++) sb_reset(&PAGES[i]); sb_reset(&ALERTS); n_alerts = 0; nst_refresh(); for (int i = 0; i < g_n_ifaces; i++) ifsnap_take(i, g_ifaces[i], dt); burst_poll(); /* ---- page 1: overview ---- */ sbuf_t *b = &PAGES[PG_OVERVIEW]; for (int i = 0; i < g_n_wgdevs; i++) { wg_render(b, g_wgdevs[i], now); sb_addf(b, "\n"); } render_udp(b); render_ip(b); render_softnet(b); render_conntrack(b); sb_addf(b, "\n%s=== Interfaces (page 2 for detail) ===%s\n", BOLD, RST); for (int i = 0; i < g_n_ifaces; i++) render_iface_brief(b, i, g_ifaces[i]); ospf_render(b); /* ---- page 2: interfaces ---- */ b = &PAGES[PG_IFACES]; sb_addf(b, "%s=== Interface detail ===%s nics[", BOLD, RST); for (int i = 0; i < n_eths; i++) sb_addf(b, "%s%s(%s)", i ? " " : "", eths[i].ifname, eths[i].driver); sb_addf(b, "]\n"); for (int i = 0; i < g_n_ifaces; i++) render_iface_full(b, i, g_ifaces[i], eths, n_eths); /* ---- page 3: burst catcher ---- */ burst_render(&PAGES[PG_BURST]); /* ---- page 4: hellgate ---- */ hg_render(&PAGES[PG_HELLGATE], dt); /* ---- paint ---- */ if (g_count == 1) { /* --once: dump all pages sequentially */ char ts[64]; struct tm tmv; localtime_r(&now, &tmv); strftime(ts, sizeof ts, "%a %b %e %H:%M:%S %Y", &tmv); printf("%s%s%s\n", BOLD, ts, RST); if (n_alerts) printf("%s%s!! ACTIVE: %s%s\n", RED, BOLD, ALERTS.s, RST); else printf("%sAll quiet.%s\n", GRN, RST); for (int i = 0; i < PG_COUNT; i++) { printf("\n%s########## PAGE %d: %s ##########%s\n\n", BOLD, i + 1, PG_NAMES[i], RST); fputs(PAGES[i].s, stdout); } fflush(stdout); } else { paint(); } ctr_roll(); iter++; if (g_count >= 0 && iter >= g_count) break; /* ---- wait for next tick, servicing keys + burst window ---- */ double deadline = now_mono() + g_interval; while (!quit) { double rem = deadline - now_mono(); if (rem <= 0) break; if (rem > 0.25) rem = 0.25; /* slice: keys + burst stay live */ struct timeval tv; tv.tv_sec = (time_t)rem; tv.tv_usec = (suseconds_t)((rem - (double)tv.tv_sec) * 1e6); fd_set rfds; FD_ZERO(&rfds); if (interactive) FD_SET(0, &rfds); int rc = select(interactive ? 1 : 0, &rfds, NULL, NULL, &tv); if (burst_maybe_finish()) paint(); if (rc > 0 && FD_ISSET(0, &rfds)) { unsigned char kb[8]; ssize_t kn = read(0, kb, sizeof kb); for (ssize_t k = 0; k < kn; k++) { unsigned char c = kb[k]; if (c == 27 && k + 2 < kn && kb[k+1] == '[') { if (kb[k+2] == 'C') c = 'n'; /* right */ else if (kb[k+2] == 'D') c = 'p'; /* left */ else if (kb[k+2] == 'A') c = 'k'; /* up */ else if (kb[k+2] == 'B') c = 'j'; /* down */ k += 2; } if (g_page == PG_HELLGATE && hg_handle_key(c)) { sb_reset(&PAGES[PG_HELLGATE]); hg_render(&PAGES[PG_HELLGATE], 0.0); paint(); continue; } switch (c) { case '1': case '2': case '3': case '4': g_page = c - '1'; paint(); break; case 'n': g_page = (g_page + 1) % PG_COUNT; paint(); break; case 'p': g_page = (g_page + PG_COUNT - 1) % PG_COUNT; paint(); break; case 'b': g_burst_arm = !g_burst_arm; paint(); break; case 'q': quit = 1; break; default: break; } } } } } full_cleanup(); return 0; }