From 13359f9ed0007c5e90b00a40491f2fc3607e052d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20=E2=80=9EKAMI=E2=80=9D=20Szalai?= Date: Fri, 23 Oct 2026 09:20:28 +0200 Subject: [PATCH] Surface recovered Invalid channels on the dashboard, wire --include-invalid into the deep check Follow-up to the previous commit on this branch: that one added --invalid/--include-invalid to check_channels.py but deliberately left them unwired, because generate_dashboard.py's "most recent row per (list, channel)" view assumed every row was a working channel - turning --include-invalid on in a workflow would have bloated the "needs attention" table with hundreds of already-known-dead Invalid rows, while hiding the one thing that's actually interesting: an Invalid channel that's alive again wouldn't show up there at all, since that table only lists rows that are *not* alive. - generate_dashboard.py: every existing count (cards, trend, by-list, "needs attention") is now built from `>` (working) rows only, via a `marker_of()` helper that treats history written before this field existed as `>` too, since only working channels were ever checked back then - so this is a no-op for every dashboard that hasn't started using --include-invalid yet. - New "Flagged Invalid, answering again" section lists `x` rows whose latest check came back alive/flaky - the dashboard's equivalent of check_channels.py's own `RECOVERED?` line - with a note that this is a shortlist for a human, not something to auto-restore. - check_channels_deep.yml now runs with --include-invalid, and its step-summary blurb mentions the new section. Tested: wrote a small synthetic history.jsonl mixing a marker-less legacy row, `>` rows, and `x` rows (one alive, one dead) and confirmed the channel count/cards only reflect the `>` rows, the alive `x` row shows up under "Flagged Invalid, answering again", the dead `x` row shows up nowhere (expected - it's just confirming the status quo), and an all-`>` history renders the new section's "none right now" empty state. Also ran it against this repo's real 77MB .github/checker-history/history.jsonl (all pre-dates this change, so no `x` rows in it) - renders in ~1.5s, unchanged output. pylint: 10.00/10 on both files. --- .github/workflows/check_channels_deep.yml | 4 +- generate_dashboard.py | 79 +++++++++++++++++++++-- 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_channels_deep.yml b/.github/workflows/check_channels_deep.yml index 75204d9..85db648 100644 --- a/.github/workflows/check_channels_deep.yml +++ b/.github/workflows/check_channels_deep.yml @@ -28,7 +28,7 @@ jobs: - name: Run the deep check (HTTP probes + ffprobe confirmation on dead channels) run: | - python3 check_channels.py --attempts 3 --timeout 12 --pause 5 --workers 40 --confirm-dead --json run.jsonl | tee check_summary.txt + python3 check_channels.py --attempts 3 --timeout 12 --pause 5 --workers 40 --confirm-dead --include-invalid --json run.jsonl | tee check_summary.txt - name: Append to history and prune entries older than 90 days run: | @@ -73,7 +73,7 @@ jobs: { echo "## Deep channel check - $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "" - echo "Channels that looked \`dead\` over HTTP were given a second opinion by \`ffprobe\`. A channel still reported \`dead\` here survived both checks and is safe to act on; one reported \`disputed\` looked dead over HTTP but ffprobe found a real stream behind it - needs a human look, not an automatic edit." + echo "Channels that looked \`dead\` over HTTP were given a second opinion by \`ffprobe\`. A channel still reported \`dead\` here survived both checks and is safe to act on; one reported \`disputed\` looked dead over HTTP but ffprobe found a real stream behind it - needs a human look, not an automatic edit. This run also re-checks every channel already flagged Invalid (\`--include-invalid\`) - one reported \`RECOVERED?\` answered like a live stream again and shows up on the dashboard's own 'Flagged Invalid, answering again' section, also pending a human look before anyone restores it." echo "" echo '```' cat check_summary.txt diff --git a/generate_dashboard.py b/generate_dashboard.py index 49c518d..1841bcd 100644 --- a/generate_dashboard.py +++ b/generate_dashboard.py @@ -12,10 +12,20 @@ The page shows: - a trend of the alive share across every run kept in the history - one row per list (country) with its current breakdown - every channel that is not currently `alive`, searchable and filterable by state + - any channel flagged Invalid in the lists that answered like a live stream again, + if the history has rows for those (see check_channels.py's --invalid/--include-invalid) "Currently" means: for each (list, channel), the most recent row in the history - older rows for the same channel are trend data, not part of today's snapshot. +A row's `marker` says whether it came from a working (`>`) or an Invalid (`x`) list +entry; older history written before that field existed has no `marker` and is always +treated as `>`, since only working channels were ever checked back then. Every count +above - the state breakdown, the trend, the by-list table, "needs attention" - is +built from `>` rows only, so turning on --include-invalid in a workflow never changes +what those numbers mean; the `x` rows only ever feed the separate "flagged Invalid" +section below them. + Usage: ./generate_dashboard.py # reads the default history path ./generate_dashboard.py --history path/to.jsonl --out path/to.html @@ -32,6 +42,15 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_HISTORY = os.path.join(BASE_DIR, ".github", "checker-history", "history.jsonl") DEFAULT_OUT = os.path.join(BASE_DIR, "docs", "index.html") +# history written before check_channels.py recorded `marker` has none - it only ever +# checked working channels back then, so the absence of the field means `>` too +WORKING = ">" + + +def marker_of(row): + """Return `row`'s marker, treating pre-`marker` history as a working (`>`) row.""" + return row.get("marker", WORKING) + # rendering order and the CSS custom property carrying each state's color STATE_ORDER = ["alive", "flaky", "blocked", "unreachable", "disputed", "dead"] STATE_VAR = {s: f"var(--s-{s})" for s in STATE_ORDER} @@ -236,13 +255,43 @@ def build_problem_rows(current): return "".join(rows) -def render(history_path, generated_at): +def build_recovered_rows(recovered): + """Return one `` per Invalid-flagged channel that answered like a live stream. + + Unlike `build_problem_rows`, every row here is worth printing - there is no + `alive` case to skip, since being `alive`/`flaky` again is exactly what makes a + row belong in this table in the first place. + """ + ordered = sorted(recovered, key=lambda r: (r["list"], r["channel"])) + rows = [] + for row in ordered: + rows.append( + f"" + f'{escape(row["list"])}' + f'{escape(row["channel"])}' + f'' + f'{escape(row["state"])}' + f'{escape(row["url"])}' + f'{escape(row["checked_at"][:10])}' + "" + ) + return "".join(rows) + + +def render(history_path, generated_at): # pylint: disable=too-many-locals """Build the full dashboard page for the history found at `history_path`.""" rows = read_history(history_path) if not rows: return EMPTY_PAGE.format(path=escape(history_path)) - current = latest_snapshot(rows) + # every top-of-page count, the trend, and "needs attention" are built from working + # (`>`) rows only, so turning --include-invalid on in a workflow can only ever add + # the separate "flagged Invalid" section below - it never changes what these numbers + # mean, and never hides a dead working channel behind hundreds of expected-dead + # Invalid ones. + working_rows = [row for row in rows if marker_of(row) == WORKING] + + current = latest_snapshot(working_rows) overall_counts = defaultdict(int) for row in current: overall_counts[row["state"]] += 1 @@ -252,18 +301,29 @@ def render(history_path, generated_at): for row in current: by_list[row["list"]].append(row) - trend = run_trend(rows) + trend = run_trend(working_rows) last_checked = rows[-1]["checked_at"] alive_pct_overall = ( overall_counts.get("alive", 0) / total_channels * 100 if total_channels else 0 ) + # the opposite case from "needs attention": a channel that is *supposed* to be + # dead (it is sitting on an `[x]` row) but just answered like a live stream - + # recorded only when a workflow actually ran check_channels.py with --invalid or + # --include-invalid, so this is empty rather than wrong on a history that never did + invalid_current = latest_snapshot([row for row in rows if marker_of(row) != WORKING]) + recovered = [row for row in invalid_current if row["state"] in ("alive", "flaky")] + cards = build_cards(overall_counts) list_rows = build_list_rows(by_list) problem_rows = build_problem_rows(current) + recovered_rows = build_recovered_rows(recovered) state_options = "".join( f'' for s in STATE_ORDER if s != "alive" ) + recovered_subline = ( + f" · {len(recovered)} flagged Invalid now answering again" if recovered else "" + ) # pylint: disable=line-too-long return f""" @@ -418,7 +478,7 @@ footer a {{ color: var(--accent); }}

Channel Signal — {alive_pct_overall:.0f}% on air

-

{total_channels} channels · {len(by_list)} lists · last swept {escape(last_checked)} · page built {escape(generated_at)}

+

{total_channels} channels · {len(by_list)} lists{recovered_subline} · last swept {escape(last_checked)} · page built {escape(generated_at)}

{cards}
@@ -457,6 +517,17 @@ footer a {{ color: var(--accent); }} +
+

Flagged Invalid, answering again

+

Channels sitting on an [x] row that answered like a live stream when a workflow last ran check_channels.py with --invalid or --include-invalid. Not proof the source is back for good - a single good answer can be a flaky CDN having a moment, not a restored channel - so this is a shortlist to look at by hand, not something to auto-restore.

+
+ + + {"".join(recovered_rows) or ''} +
ListChannelStateURLLast checked
none right now
+
+
+