's'

web2_config_governance.py

@@ -115,3 +115,43 @@ def write_config_text(config_dir: Path, rel_name: str, content: str, *, reason:
     target.parent.mkdir(parents=True, exist_ok=True)
     target.write_text(content, encoding="utf-8")
     return target
+
+
+def _normalize_url_line_for_dedupe(line: str) -> str:
+    from web2_relay_pro import extract_douyin_url_from_config_line
+
+    url = extract_douyin_url_from_config_line(line)
+    if url:
+        return url.rstrip("/").lower()
+    t = (line or "").strip()
+    if not t or t.startswith("#"):
+        return ""
+    return t.rstrip("/").lower()
+
+
+def dedupe_url_config_content(content: str) -> tuple[str, int]:
+    """去除 URL_config 中重复的抖音直播间行（参照 d2ypp2 config_governance dedupe_urls）。"""
+    seen: set[str] = set()
+    removed = 0
+    out: list[str] = []
+    for raw in content.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            out.append(raw.rstrip())
+            continue
+        key = _normalize_url_line_for_dedupe(line)
+        if not key:
+            out.append(raw.rstrip())
+            continue
+        if "douyin" not in key:
+            out.append(raw.rstrip())
+            continue
+        if key in seen:
+            removed += 1
+            continue
+        seen.add(key)
+        out.append(raw.rstrip())
+    text = "\n".join(out).rstrip()
+    if text:
+        text += "\n"
+    return text, removed