From a86fb23e94acd6a78ee1e936085455a83277acf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1lm=C3=A1n=20=E2=80=9EKAMI=E2=80=9D=20Szalai?= Date: Fri, 4 Sep 2026 10:21:37 +0200 Subject: [PATCH] Add channel checker, scheduled health checks, and a status dashboard Builds on the idea in #1145 (a checker that maps failures back to a channel and list, and tells a geo-blocked channel from a dead one) with a few additions: check_channels.py - Standard library only, same as #1145's version. - Adds a `disputed` state: an optional --confirm-dead pass gives ffprobe a second opinion on anything that looks dead over HTTP, before it gets reported as dead. This is one-directional (can only pull a verdict out of `dead`, never push one into it) because ffprobe itself is not reliable enough to trust in the other direction - a known-good DASH channel needed longer than any sane per-channel budget to open while testing this, and a header check alone had already been fooled by an isolated media fragment sitting at a URL that looked like a live channel (a mistake made and caught while triaging #1149/#1151 - see the docstring for the details). - Restructured to share one worker pool across every list in a run instead of a fresh pool per list, which stopped small lists from paying the same wall-clock floor as large ones; a full run across all lists dropped from not finishing in 15 minutes to about 10. - --json writes one record per channel per run, for building a history. generate_dashboard.py - Turns that history into a single self-contained docs/index.html: current state breakdown, an alive-share trend across every run kept, a per-list breakdown, and a searchable/filterable table of everything that is not currently alive. Two new scheduled workflows - check_channels_fast.yml: every 6 hours, HTTP checks only. - check_channels_deep.yml: every 2 days, with --confirm-dead (needs ffmpeg). Both append to .github/checker-history/history.jsonl (pruned to 90 days), then build and deploy the dashboard to GitHub Pages. --- .github/workflows/check_channels_deep.yml | 91 ++++ .github/workflows/check_channels_fast.yml | 88 ++++ .gitignore | 2 + check_channels.py | 326 ++++++++++++++ generate_dashboard.py | 503 ++++++++++++++++++++++ 5 files changed, 1010 insertions(+) create mode 100644 .github/workflows/check_channels_deep.yml create mode 100644 .github/workflows/check_channels_fast.yml create mode 100644 check_channels.py create mode 100644 generate_dashboard.py diff --git a/.github/workflows/check_channels_deep.yml b/.github/workflows/check_channels_deep.yml new file mode 100644 index 0000000..75204d9 --- /dev/null +++ b/.github/workflows/check_channels_deep.yml @@ -0,0 +1,91 @@ +name: Channel check (deep) + +on: + schedule: + - cron: '0 6 */2 * *' # every 2 days at 06:00 UTC + workflow_dispatch: + +permissions: + contents: write + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + check-deep: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install ffmpeg (for the ffprobe second opinion on dead channels) + run: sudo apt-get update && sudo apt-get install -y ffmpeg + + - name: Run the deep check (HTTP probes + ffprobe confirmation on dead channels) + run: | + python3 check_channels.py --attempts 3 --timeout 12 --pause 5 --workers 40 --confirm-dead --json run.jsonl | tee check_summary.txt + + - name: Append to history and prune entries older than 90 days + run: | + mkdir -p .github/checker-history + history=".github/checker-history/history.jsonl" + touch "$history" + cat run.jsonl >> "$history" + cutoff=$(date -u -d '90 days ago' +%Y-%m-%dT%H:%M:%SZ) + python3 - "$history" "$cutoff" <<'PYEOF' + import json, sys + path, cutoff = sys.argv[1], sys.argv[2] + kept = [] + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line and json.loads(line)["checked_at"] >= cutoff: + kept.append(line) + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(kept) + ("\n" if kept else "")) + PYEOF + + - name: Commit history + run: | + git config user.name "ChannelCheckerBot" || true + git config user.email "channelcheckerbot@users.noreply.github.com" || true + git add .github/checker-history/history.jsonl + git diff --staged --quiet || git commit --quiet -m "Channel check (deep) - $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git pull --rebase origin master || true + git diff --quiet HEAD @{u} || git push origin HEAD + + - name: Build the status dashboard + run: python3 generate_dashboard.py --history .github/checker-history/history.jsonl --out docs/index.html + + - name: Upload dashboard for Pages + uses: actions/upload-pages-artifact@v3 + with: + path: docs + + - name: Summarize + if: always() + run: | + { + echo "## Deep channel check - $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "" + echo "Channels that looked \`dead\` over HTTP were given a second opinion by \`ffprobe\`. A channel still reported \`dead\` here survived both checks and is safe to act on; one reported \`disputed\` looked dead over HTTP but ffprobe found a real stream behind it - needs a human look, not an automatic edit." + echo "" + echo '```' + cat check_summary.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + deploy-pages: + needs: check-deep + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/check_channels_fast.yml b/.github/workflows/check_channels_fast.yml new file mode 100644 index 0000000..01dc204 --- /dev/null +++ b/.github/workflows/check_channels_fast.yml @@ -0,0 +1,88 @@ +name: Channel check (fast) + +on: + schedule: + - cron: '0 */6 * * *' # every 6 hours + workflow_dispatch: + +permissions: + contents: write + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + check-fast: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Run the fast check (HTTP probes only, no ffprobe) + run: | + python3 check_channels.py --attempts 3 --timeout 12 --pause 5 --workers 40 --json run.jsonl | tee check_summary.txt + + - name: Append to history and prune entries older than 90 days + run: | + mkdir -p .github/checker-history + history=".github/checker-history/history.jsonl" + touch "$history" + cat run.jsonl >> "$history" + cutoff=$(date -u -d '90 days ago' +%Y-%m-%dT%H:%M:%SZ) + python3 - "$history" "$cutoff" <<'PYEOF' + import json, sys + path, cutoff = sys.argv[1], sys.argv[2] + kept = [] + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line and json.loads(line)["checked_at"] >= cutoff: + kept.append(line) + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(kept) + ("\n" if kept else "")) + PYEOF + + - name: Commit history + run: | + git config user.name "ChannelCheckerBot" || true + git config user.email "channelcheckerbot@users.noreply.github.com" || true + git add .github/checker-history/history.jsonl + git diff --staged --quiet || git commit --quiet -m "Channel check (fast) - $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git pull --rebase origin master || true + git diff --quiet HEAD @{u} || git push origin HEAD + + - name: Build the status dashboard + run: python3 generate_dashboard.py --history .github/checker-history/history.jsonl --out docs/index.html + + - name: Upload dashboard for Pages + uses: actions/upload-pages-artifact@v3 + with: + path: docs + + - name: Summarize + if: always() + run: | + { + echo "## Fast channel check - $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "" + echo "Every channel probed 3x over HTTP; no ffprobe confirmation (see the *Channel check (deep)* workflow for that). \`blocked\` means geo-restricted, not broken - see \`check_channels.py\` for what each state means." + echo "" + echo '```' + cat check_summary.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + deploy-pages: + needs: check-fast + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 7a60b85..33ffd46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ __pycache__/ *.pyc +# generated by generate_dashboard.py; deployed straight to Pages, never committed +docs/ diff --git a/check_channels.py b/check_channels.py new file mode 100644 index 0000000..5cb862b --- /dev/null +++ b/check_channels.py @@ -0,0 +1,326 @@ +#!/usr/bin/python3 +"""Report which channels in `lists/` are no longer reachable. + +The playlist checker records the URLs it could not open, but a bare URL does not say +which channel or which list it came from, and one failed attempt does not mean a +channel is gone. Streams time out, rate limit, move between CDNs, and some are only +served inside their own country. + +This script maps every URL back to its channel and list, probes each one several +times, and separates the cases that look alike from the outside: + + dead every attempt said "not found", so the stream is really gone + disputed looked dead over HTTP, but ffprobe found a real audio/video stream + behind it - needs a human, not a bot, to decide + blocked the server answered but refused us, which is what a channel served + only in its own country looks like from anywhere else + unreachable every attempt timed out or failed to connect + flaky answered at least once, so it is up but unreliable from here + alive answered every time + +Only `dead` is safe to act on without a second opinion. `blocked` in particular must +not be treated as a fault: the lists mark geo-blocked channels deliberately. + +Both playlist formats used by the lists are understood, HLS (.m3u8) and MPEG-DASH +(.mpd), so a DASH stream is not mistaken for a broken one. Neither check is airtight +on its own: a manifest that starts with `#EXTM3U`/`](url) | ...` is the row format used by every list +ROW = re.compile(r"^\|[^|]*\|([^|]+)\|[^|]*\[>\]\((https?://[^)\s]+)\)") + +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0 Safari/537.36" +) + +OK = "ok" +GONE = "gone" +REFUSED = "refused" +UNREACHABLE = "unreachable" + +ALIVE = "alive" +DEAD = "dead" +DISPUTED = "disputed" +BLOCKED = "blocked" +UNREACHED = "unreachable" +FLAKY = "flaky" + +# a channel served only in its own country answers, then refuses us +REFUSING_CODES = (401, 402, 403, 451) +GONE_CODES = (404, 410) + +# ffprobe gets much longer than an HTTP probe: it has to open a connection, read +# enough of the stream to find a decodable frame, and do that over whatever the +# channel's own CDN feels like doing today, not just get a response header back +FFPROBE_TIMEOUT = 25 + + +def read_channels(name): + """Return the `(channel, url)` pairs of the list called `name`.""" + path = os.path.join(LISTS_DIR, name + ".md") + channels = [] + with open(path, encoding="utf-8") as handle: + for line in handle: + match = ROW.match(line.strip()) + if match: + channels.append((match.group(1).strip(), match.group(2))) + return channels + + +def looks_like_a_playlist(head): + """Return True if `head` is the start of an HLS or a DASH playlist.""" + text = head.lstrip() + if text.startswith("#EXTM3U"): + return True + # DASH manifests are XML, and may carry a declaration, a comment or neither + return " {url}") + return states.get(DEAD, 0) + + +def as_records(list_name, results, checked_at, confirm_dead): + """Turn one list's results into flat dicts, one per channel, for --json output. + + `confirm_dead` is recorded on every row so a later reader can tell a `dead` + verdict that already survived an ffprobe second opinion from one that has not + been asked yet - the two are not the same strength of evidence. + """ + return [ + { + "checked_at": checked_at, + "list": list_name, + "channel": channel, + "url": url, + "state": state, + "outcomes": outcomes, + "confirm_dead": confirm_dead, + } + for channel, url, state, outcomes in results + ] + + +def parse_args(): + """Parse the command line into an `argparse.Namespace`.""" + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("lists", nargs="*", help="lists to check, without the .md suffix") + parser.add_argument("--attempts", type=int, default=3, help="probes per channel") + parser.add_argument("--timeout", type=int, default=12, help="seconds per probe") + parser.add_argument("--pause", type=int, default=5, help="seconds between probes") + parser.add_argument("--workers", type=int, default=8, help="channels probed at once") + parser.add_argument( + "--confirm-dead", action="store_true", + help="give ffprobe a second opinion on channels that look dead (slower; needs ffmpeg)", + ) + parser.add_argument( + "--json", metavar="PATH", + help="also write every channel's result as one JSON record per line to PATH, for " + "building a history across runs (state, outcomes, timestamp, list, channel, url)", + ) + return parser.parse_args() + + +def write_json_records(handle, names, results_by_list, checked_at, confirm_dead): + """Write every list's results to `handle` as one JSON record per channel per line.""" + for name in names: + for record in as_records(name, results_by_list[name], checked_at, confirm_dead): + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def main(): + """Check the lists named on the command line, or every list.""" + args = parse_args() + + if args.confirm_dead and shutil.which("ffprobe") is None: + print( + "--confirm-dead needs ffprobe (part of ffmpeg) on PATH; proceeding without it.", + file=sys.stderr, + ) + args.confirm_dead = False + + all_lists = (f[:-3] for f in os.listdir(LISTS_DIR) if f.endswith(".md")) + names = args.lists or sorted(all_lists) + checked_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + settings = ProbeSettings( + args.attempts, args.timeout, args.pause, args.workers, args.confirm_dead, + ) + + results_by_list = check_all(names, settings) + + dead_total = sum(report(name, results_by_list[name], args.attempts) for name in names) + if args.json: + with open(args.json, "w", encoding="utf-8") as handle: + write_json_records(handle, names, results_by_list, checked_at, args.confirm_dead) + + return 1 if dead_total else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/generate_dashboard.py b/generate_dashboard.py new file mode 100644 index 0000000..49c518d --- /dev/null +++ b/generate_dashboard.py @@ -0,0 +1,503 @@ +#!/usr/bin/python3 +"""Turn `.github/checker-history/history.jsonl` into a static status dashboard. + +Reads the JSON-lines history that `check_channels.py --json` appends to (one row +per channel per run), and writes a single self-contained `docs/index.html`: no +build step, no client-side fetch of the raw history - everything needed to render +is computed here and baked into the page, so it opens instantly whether it is +opened from disk or served by GitHub Pages. + +The page shows: + - today's state breakdown (how many channels are alive/dead/blocked/... right now) + - a trend of the alive share across every run kept in the history + - one row per list (country) with its current breakdown + - every channel that is not currently `alive`, searchable and filterable by state + +"Currently" means: for each (list, channel), the most recent row in the history - +older rows for the same channel are trend data, not part of today's snapshot. + +Usage: + ./generate_dashboard.py # reads the default history path + ./generate_dashboard.py --history path/to.jsonl --out path/to.html +""" + +import argparse +import json +import os +from collections import defaultdict +from datetime import datetime, timezone +from html import escape + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_HISTORY = os.path.join(BASE_DIR, ".github", "checker-history", "history.jsonl") +DEFAULT_OUT = os.path.join(BASE_DIR, "docs", "index.html") + +# rendering order and the CSS custom property carrying each state's color +STATE_ORDER = ["alive", "flaky", "blocked", "unreachable", "disputed", "dead"] +STATE_VAR = {s: f"var(--s-{s})" for s in STATE_ORDER} +STATE_LABEL = { + "alive": "answers every time", + "flaky": "answers, but not every time", + "blocked": "refused - likely geo-restricted", + "unreachable": "times out or refuses to connect", + "disputed": "looked dead, ffprobe found a stream - needs a human", + "dead": "gone on every attempt, ffprobe agrees where asked", +} + + +def read_history(path): + """Return every row of the history file, oldest first, skipping malformed lines.""" + rows = [] + if not os.path.exists(path): + return rows + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + rows.sort(key=lambda r: r["checked_at"]) + return rows + + +def latest_snapshot(rows): + """Return the most recent row for each (list, channel, url) tuple.""" + latest = {} + for row in rows: + key = (row["list"], row["channel"], row["url"]) + latest[key] = row + return list(latest.values()) + + +def run_trend(rows): + """Group rows by `checked_at` (one run) and return the alive share of each run.""" + by_run = defaultdict(lambda: defaultdict(int)) + for row in rows: + by_run[row["checked_at"]][row["state"]] += 1 + points = [] + for checked_at in sorted(by_run): + counts = by_run[checked_at] + total = sum(counts.values()) + alive_share = counts.get("alive", 0) / total if total else 0 + points.append((checked_at, alive_share, total)) + return points + + +def _scope_grid_and_labels(pad_l, pad_r, pad_t, width, plot_h): + """Return the horizontal gridlines and their %-axis labels for `svg_trend`.""" + grid = "".join( + f'' + for f in (0, 0.25, 0.5, 0.75, 1) + ) + labels = "".join( + f'{int((1 - f) * 100)}%' + for f in (0, 0.5, 1) + ) + return grid, labels + + +def _scope_geometry(points, pad_l, pad_t, plot_w, plot_h): + """Return `(line_path, area_path, last_x, last_y)` for the trend's data points.""" + xs = [pad_l + i * plot_w / (len(points) - 1) for i in range(len(points))] + ys = [pad_t + (1 - share) * plot_h for _, share, _ in points] + steps = (f"{'M' if i == 0 else 'L'}{x:.1f},{y:.1f}" for i, (x, y) in enumerate(zip(xs, ys))) + path = " ".join(steps) + area = path + f" L{xs[-1]:.1f},{pad_t + plot_h:.1f} L{xs[0]:.1f},{pad_t + plot_h:.1f} Z" + return path, area, xs[-1], ys[-1] + + +def _scope_endpoint_labels(points): + """Return `(first_date, last_date, last_alive_pct)` for the trend's endpoints.""" + return points[0][0][:10], points[-1][0][:10], f"{points[-1][1] * 100:.1f}%" + + +def svg_trend(points, width=860, height=180): + """Render `points` as an oscilloscope-style trace of the alive share over time. + + Kept as one function despite the local-variable count: it is a single visual + composite (grid, axis labels, line, fill, endpoint dot) whose pieces only make + sense read together, and splitting it further would trade a readable layout + calculation for indirection between fragments that all belong to one . + """ + # pylint: disable=too-many-locals + pad_l, pad_r, pad_t = 34, 16, 16 + plot_w, plot_h = width - pad_l - pad_r, height - pad_t - 26 + grid, labels = _scope_grid_and_labels(pad_l, pad_r, pad_t, width, plot_h) + + if len(points) < 2: + return ( + f'{grid}{labels}' + f'' + "collecting data - check back after a few runs" + ) + + path, area, last_x, last_y = _scope_geometry(points, pad_l, pad_t, plot_w, plot_h) + first_label, last_label, last_pct = _scope_endpoint_labels(points) + return ( + f'' + f"{grid}{labels}" + f'' + f'' + f'{escape(first_label)}' + f'' + f"{escape(last_label)}" + ) + + +def svg_signal_bar(counts, width=280, height=14): + """Render a `state -> count` mapping as one horizontal signal-strength bar.""" + total = sum(counts.values()) or 1 + x = 0 + segments = [] + for state in STATE_ORDER: + count = counts.get(state, 0) + if not count: + continue + w = count / total * width + segments.append( + f'' + f"{state}: {count} ({count / total * 100:.1f}%)" + ) + x += w + return ( + f'{"".join(segments)}' + ) + + +def legend_html(): + """Return the small state-color legend shown above the by-list table.""" + chips = "".join( + f'{s}' + f'{STATE_LABEL[s]}' + for s in STATE_ORDER + ) + return f'
{chips}
' + + +# pylint: disable=line-too-long +EMPTY_PAGE = """ +Channel Signal + +

Channel Signal

No history yet at {path} - the checker workflows populate this after their first run.

""" +# pylint: enable=line-too-long + + +def build_cards(overall_counts): + """Return the row of big per-state count cards at the top of the page.""" + return "".join( + f'
' + f'{overall_counts.get(s, 0)}
{s}
' + for s in STATE_ORDER + ) + + +def build_list_rows(by_list): + """Return one `` per list (country), each with its own signal bar.""" + rows = [] + for name in sorted(by_list): + entries = by_list[name] + counts = defaultdict(int) + for entry in entries: + counts[entry["state"]] += 1 + alive_pct = counts.get("alive", 0) / len(entries) * 100 + rows.append( + f"{escape(name)}{len(entries)}" + f'{alive_pct:.0f}%' + f"{svg_signal_bar(counts)}" + ) + return "".join(rows) + + +def build_problem_rows(current): + """Return one `` per channel that is not currently `alive`.""" + ordered = sorted(current, key=lambda r: (r["state"] != "dead", r["list"], r["channel"])) + rows = [] + for row in ordered: + if row["state"] == "alive": + continue + rows.append( + f'' + f'{escape(row["list"])}' + f'{escape(row["channel"])}' + f'' + f'{escape(row["state"])}' + f'{escape(row["url"])}' + f'{escape(row["checked_at"][:10])}' + "" + ) + return "".join(rows) + + +def render(history_path, generated_at): + """Build the full dashboard page for the history found at `history_path`.""" + rows = read_history(history_path) + if not rows: + return EMPTY_PAGE.format(path=escape(history_path)) + + current = latest_snapshot(rows) + overall_counts = defaultdict(int) + for row in current: + overall_counts[row["state"]] += 1 + total_channels = len(current) + + by_list = defaultdict(list) + for row in current: + by_list[row["list"]].append(row) + + trend = run_trend(rows) + last_checked = rows[-1]["checked_at"] + alive_pct_overall = ( + overall_counts.get("alive", 0) / total_channels * 100 if total_channels else 0 + ) + + cards = build_cards(overall_counts) + list_rows = build_list_rows(by_list) + problem_rows = build_problem_rows(current) + state_options = "".join( + f'' for s in STATE_ORDER if s != "alive" + ) + + # pylint: disable=line-too-long + return f""" + + + + +Channel Signal + + + + +
+
+

Channel Signal — {alive_pct_overall:.0f}% on air

+
+

{total_channels} channels · {len(by_list)} lists · last swept {escape(last_checked)} · page built {escape(generated_at)}

+ +
{cards}
+ +
+

Alive share, over time

+ {svg_trend(trend)} +
+ +
+

By list

+ {legend_html()} +
+ + + {"".join(list_rows)} +
ListChannelsAliveSignal
+
+
+ +
+

Needs attention

+

blocked usually means geo-restricted on purpose (the Ⓖ marker in the lists), not broken. + disputed looked dead over HTTP but ffprobe found a real stream behind it - worth a human look before touching it.

+
+ + +
+
+ + + {"".join(problem_rows)} +
ListChannelStateURLLast checked
+
+
+ + +
+ + + +""" + # pylint: enable=line-too-long + + +def main(): + """Generate the dashboard from the command-line-given (or default) history path.""" + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--history", default=DEFAULT_HISTORY, help="path to history.jsonl") + parser.add_argument("--out", default=DEFAULT_OUT, help="path to write the dashboard HTML") + args = parser.parse_args() + + generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + html = render(args.history, generated_at) + + os.makedirs(os.path.dirname(args.out), exist_ok=True) + with open(args.out, "w", encoding="utf-8") as handle: + handle.write(html) + print(f"Wrote {args.out}") + + +if __name__ == "__main__": + main()