"""Process runtime diagnosis (ported from sh2/src/process_runtime_status.py).""" from __future__ import annotations import re from typing import Any _FFMPEG_PROGRESS_RE = re.compile( r"frame=\s*(?P\d+).*?" r"time=(?P\d+):(?P\d+):(?P\d+(?:\.\d+)?).*?" r"speed=\s*(?P\d+(?:\.\d+)?)x", re.IGNORECASE, ) def normalize_log_text(text: str) -> str: if not text: return "" return text.replace("\r\n", "\n").replace("\r", "\n") def process_family(process_name: str) -> str: low = (process_name or "").strip().lower() if low.startswith("camera") or low.startswith("douyin_youtube") or low.startswith("pull_stream"): return "youtube" if low.startswith("youtube"): return "youtube" if low.startswith("tiktok"): return "tiktok" if low.startswith("obs"): return "obs" return "generic" def _has_any(text: str, patterns: tuple[str, ...]) -> bool: return any(p in text for p in patterns) def _looks_like_youtube_key_rejected(tail: str) -> bool: youtube_output = _has_any( tail, ( "rtmps://a.rtmps.youtube.com/live2/", "rtmp://a.rtmp.youtube.com/live2/", "youtube.com/live2/", ), ) if not youtube_output: return False if not _has_any(tail, ("error opening output", "error opening output file", "error opening output files")): return False return _has_any( tail, ( "io error: end of file", "input/output error", "error in the pull function", "tls @", "401", "403", "unauthorized", "forbidden", ), ) def _looks_like_ffmpeg_progress(tail: str) -> bool: return "frame=" in tail and "time=" in tail and _has_any(tail, ("bitrate=", "speed=")) def _looks_like_ffmpeg_starting(tail: str) -> bool: return _has_any( tail, ( "starting ffmpeg", "ffmpeg command", "opening input", "opening output", "rtmps://a.rtmps.youtube.com/live2/", "rtmp://a.rtmp.youtube.com/live2/", "youtube.com/live2/", ), ) def _ffmpeg_progress_samples(tail: str) -> list[dict[str, float | int]]: samples: list[dict[str, float | int]] = [] for match in _FFMPEG_PROGRESS_RE.finditer(tail or ""): try: frame = int(match.group("frame")) media_time = ( int(match.group("h")) * 3600 + int(match.group("m")) * 60 + float(match.group("s")) ) speed = float(match.group("speed")) except (TypeError, ValueError): continue samples.append({"frame": frame, "time": media_time, "speed": speed}) return samples[-8:] def _ffmpeg_progress_stalled(samples: list[dict[str, float | int]]) -> bool: if len(samples) < 3: return False tail = samples[-3:] frames = {int(row["frame"]) for row in tail} times = {round(float(row["time"]), 2) for row in tail} return len(frames) == 1 and len(times) == 1 def _ffmpeg_speed_low(samples: list[dict[str, float | int]], threshold: float = 0.98) -> bool: if len(samples) < 4: return False tail = samples[-4:] if not all(float(row["speed"]) < threshold for row in tail): return False return int(tail[-1]["frame"]) > int(tail[0]["frame"]) or float(tail[-1]["time"]) > float(tail[0]["time"]) _DISCONNECT_PATTERNS = ( "client disconnected", "connection reset by peer", "broken pipe", "connection timed out", "i/o error", "error submitting a packet to muxer", "failed to update header", ) def _looks_like_stream_disconnected(tail: str) -> bool: return _has_any(tail, _DISCONNECT_PATTERNS) def summarize_process_activity( process_name: str, process_status: str, recent_log: str, recent_error: str, *, log_age_seconds: float | None = None, ffmpeg_pids: list[int] | None = None, ) -> dict[str, Any]: pids = sorted({int(pid) for pid in (ffmpeg_pids or []) if int(pid) > 0}) status = (process_status or "").strip().lower() family = process_family(process_name) tail = normalize_log_text(f"{recent_log}\n{recent_error}")[-8000:].lower() progress_samples = _ffmpeg_progress_samples(tail) latest_progress = progress_samples[-1] if progress_samples else {} progress_stalled = _ffmpeg_progress_stalled(progress_samples) speed_low = _ffmpeg_speed_low(progress_samples) business_status = "inactive" business_note = "进程未运行" log_recent = log_age_seconds is None or log_age_seconds <= 20 has_recent_ffmpeg_progress = _looks_like_ffmpeg_progress(tail) and log_recent has_ffmpeg_worker = bool(pids) if family == "youtube" and not has_recent_ffmpeg_progress and _looks_like_youtube_key_rejected(tail): business_status = "youtube_key_rejected" business_note = "YouTube 拒绝了 RTMPS 推流,串流密钥可能无效、过期或与直播活动不匹配" elif status in {"online", "running"}: if _has_any( tail, ("请在 ", "配置 key", "stream_key", "key=你的stream_key", "串流密钥", "至少需要一个有效"), ): business_status = "misconfigured" business_note = "缺少或无法使用 YouTube 串流密钥" elif family == "youtube" and _looks_like_stream_disconnected(tail) and not has_recent_ffmpeg_progress: business_status = "stream_disconnected" business_note = "推流连接已断开,正在等待自动重连或需检查网络" elif family == "youtube" and has_recent_ffmpeg_progress and has_ffmpeg_worker and progress_stalled: business_status = "source_stalled" business_note = "FFmpeg 在运行,但帧/时间进度已停滞" elif family == "youtube" and has_recent_ffmpeg_progress and has_ffmpeg_worker and speed_low: business_status = "relay_slow" business_note = "FFmpeg 在运行,但近期推流速度低于实时" elif has_recent_ffmpeg_progress or (has_ffmpeg_worker and _looks_like_ffmpeg_progress(tail)): business_status = "streaming" business_note = "FFmpeg 正在推流" elif family == "youtube" and has_ffmpeg_worker and _looks_like_ffmpeg_starting(tail) and log_recent: business_status = "relay_starting" business_note = "FFmpeg 已启动,尚未观察到视频进度" elif log_age_seconds is not None and log_age_seconds > 180: business_status = "stale" business_note = f"进程在线但日志 {int(log_age_seconds)} 秒未更新" elif family == "youtube" and _has_any( tail, ("等待直播", "循环等待", "检测直播间中", "没有正在监测的直播", "无录制任务", "网址内容获取失败"), ): business_status = "waiting_source" business_note = "正在等待源站开播" elif family == "youtube" and _has_any( tail, ("开始推流到 youtube", "推流到 youtube", "正在直播中", "[relay_meta]"), ): business_status = "source_live" business_note = "源站已开播,但暂未检测到 FFmpeg 推流" elif "traceback" in tail or " error " in tail or "[error]" in tail or "错误信息" in tail: business_status = "error" business_note = "近期日志中出现异常" else: business_status = "monitoring" business_note = "进程在线,正在监测源站" elif status in {"stopped", "stopping", "not_found"}: business_status = "stopped" business_note = "进程未运行" elif status == "invalid": business_note = "无效的进程名" return { "business_status": business_status, "business_note": business_note, "ffmpeg_pids": pids, "ffmpeg_active": bool(pids), "log_age_seconds": log_age_seconds, "ffmpeg_frame": latest_progress.get("frame"), "ffmpeg_time_seconds": latest_progress.get("time"), "ffmpeg_speed": latest_progress.get("speed"), "ffmpeg_progress_stalled": progress_stalled, "ffmpeg_speed_low": speed_low, "is_pushing": business_status in {"streaming", "relay_slow"} or (business_status == "source_stalled" and has_recent_ffmpeg_progress and has_ffmpeg_worker), }