eric/gitlab-instance-0a899031_sh
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

's'

Browse Source
This commit is contained in:
eric
2026-05-16 19:24:30 -05:00
parent 19beec12a5
commit 75a0ca4e31
260 changed files with 51345 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
d2ypp2/tools/bootstrap_remote_repo_paramiko.py Normal file
View File

@@ -0,0 +1,71 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""新 git clone 后：修正属主、创建 .venv、安装 requirements（需与 deploy_live_paramiko 相同环境变量）。"""
from __future__ import annotations
import os
import sys
try:
import paramiko
except ImportError:
print("请先安装: pip install paramiko", file=sys.stderr)
sys.exit(1)
HOST = os.environ.get("LIVE_DEPLOY_HOST", "192.168.21.105")
USER = os.environ.get("LIVE_DEPLOY_USER", "live")
PASS = os.environ.get("LIVE_DEPLOY_PASS", "12345678")
REMOTE_BASE = os.environ.get("LIVE_DEPLOY_PATH", "/home/live/douyinyoutube").rstrip("/")
def main() -> int:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(HOST, username=USER, password=PASS, timeout=30, banner_timeout=40, auth_timeout=30)
try:
def run(title: str, cmd: str, timeout: int = 600) -> int:
print(f"\n=== {title} ===")
wrapped = (
"export LC_ALL=C.UTF-8 LANG=C.UTF-8; "
f"cd {REMOTE_BASE} 2>/dev/null || exit 9; "
+ cmd
)
_, stdout, stderr = c.exec_command(wrapped, timeout=timeout, get_pty=False)
out = (stdout.read() or b"").decode("utf-8", "replace")
err = (stderr.read() or b"").decode("utf-8", "replace")
code = stdout.channel.recv_exit_status()
if out.strip():
print(out.rstrip())
if err.strip():
print(err.rstrip(), file=sys.stderr)
print(f"exit={code}")
return code
code = run(
"chown (root 克隆后必做)",
f"echo '{PASS}' | sudo -S chown -R {USER}:{USER} {REMOTE_BASE} && echo chown_ok",
timeout=60,
)
if code != 0:
return 1
run("chmod +x 常用脚本", "chmod +x start.sh web.sh dev.sh 2>/dev/null; chmod +x scripts/launch.py 2>/dev/null; echo chmod_ok", timeout=20)
venv_cmd = (
f"if [ -x .venv/bin/pip ]; then echo venv_exists; "
f"else python3 -m venv .venv && echo venv_created; fi && "
f".venv/bin/pip install -q --upgrade pip && "
f".venv/bin/pip install -r requirements.txt"
)
code = run("venv + pip install -r requirements.txt", venv_cmd, timeout=900)
if code != 0:
return 2
print("\n完成。接着在本机执行: python tools/deploy_live_paramiko.py")
return 0
finally:
c.close()
if __name__ == "__main__":
raise SystemExit(main())

435
d2ypp2/tools/deploy_live_paramiko.py Normal file
View File

@@ -0,0 +1,435 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""通过 Paramiko（SFTP + exec）部署到 ARM 板，无交互 SSH。日志写入 .deploy_logs/。"""
from __future__ import annotations
import io
import os
import posixpath
import sys
import time
from datetime import datetime
from pathlib import Path
try:
import paramiko
except ImportError:
print("请先安装: pip install paramiko", file=sys.stderr)
sys.exit(1)
HOST = os.environ.get("LIVE_DEPLOY_HOST", "192.168.21.105")
USER = os.environ.get("LIVE_DEPLOY_USER", "live")
PASS = os.environ.get("LIVE_DEPLOY_PASS", "12345678")
REMOTE_BASE = os.environ.get("LIVE_DEPLOY_PATH", "/home/live/douyinyoutube").rstrip("/")
REPO_ROOT = Path(__file__).resolve().parent.parent
def _discover_repo_files() -> list[str]:
"""与 youtube.py 运行相关的源码，避免只同步白名单时缺模块。"""
extra: list[str] = []
backup_dir = REPO_ROOT / "backup"
if backup_dir.is_dir():
for p in sorted(backup_dir.glob("*.py")):
extra.append(str(p.relative_to(REPO_ROOT)).replace("\\", "/"))
return extra
FILES = [
"src/utils.py",
"src/logger.py",
"src/spider.py",
"src/stream.py",
"src/proxy.py",
"src/http_clients/async_http.py",
"src/http_clients/sync_http.py",
"src/android_control.py",
"src/scrcpy_stream.py",
"src/async_thread_loop.py",
"src/youtube_ini_sync.py",
"src/control_plane.py",
"src/web_process_backend.py",
"src/hub_routes.py",
"src/hub_kv_store.py",
"src/initializer.py",
"src/live_config/__init__.py",
"src/live_config/loader.py",
"web.py",
"youtube.py",
"tiktok.py",
"main.py",
"ffmpeg_install.py",
"scripts/launch.py",
"scripts/env_check.py",
"scripts/hardware_probe.py",
"config/hardware.env.example",
"config/system-stack.env.example",
"ecosystem.config.cjs",
"docs/install-profiles.md",
"douyin_youtube_ffplay.py",
"scripts/linux/install_hub.sh",
"scripts/linux/upgrade_ffmpeg_newer.sh",
"scripts/linux/reinstall_live_edge.sh",
"scripts/linux/service_ctl.sh",
"scripts/linux/lib/common.sh",
"services/srs/docker-compose.yml",
"services/srs/data/index.html",
"services/neko/docker-compose.yml",
"services/neko/chromium.conf",
"services/neko/google-chrome.conf",
"services/neko/browser-launch.sh",
"services/neko/policies/policies.json",
"services/neko/.gitignore",
"live-platform/config/defaults/services/neko.json",
"live-platform/tools/render_caddyfile.py",
"live-platform/tools/render_stack_env.py",
"web-console/package.json",
"web-console/package-lock.json",
"web-console/app/globals.css",
"web-console/app/layout.tsx",
"web-console/app/page.tsx",
"web-console/middleware.ts",
"web-console/components/OverviewCharts.tsx",
"web-console/components/live-product/LiveControlApp.tsx",
"web-console/components/live-product/LiveAndroidWorkstation.tsx",
"web-console/components/live-product/LiveAndroidStreamConsole.tsx",
"web-console/components/live-product/LiveYoutubeUnmannedView.tsx",
"web-console/components/live-product/LiveProcessLiveLogCard.tsx",
"web-console/components/live-product/LiveTiktokHdmiView.tsx",
"web-console/lib/panelUrls.ts",
"web-console/lib/androidScrcpyWs.ts",
"web-console/lib/scrcpyH264Decoder.ts",
"web-console/lib/youtubeProChannels.ts",
"web-console/lib/youtubeConfigUtils.ts",
"web-console/components/console/LiveConsoleWorkspace.tsx",
]
FILES = sorted(set(FILES + _discover_repo_files()))
LOG_DIR = REPO_ROOT / ".deploy_logs"
TAIL_LINES = 90
def _safe_print(text: str) -> None:
try:
print(text)
except UnicodeEncodeError:
enc = getattr(sys.stdout, "encoding", None) or "utf-8"
try:
sys.stdout.buffer.write(text.encode(enc, errors="replace") + b"\n")
except Exception:
sys.stdout.buffer.write(text.encode("ascii", errors="backslashreplace") + b"\n")
def _decode(data: bytes) -> str:
if not data:
return ""
try:
return data.decode("utf-8", errors="replace")
except Exception:
return data.decode("ascii", errors="backslashreplace")
def _tail(text: str, n: int = TAIL_LINES) -> str:
lines = text.splitlines()
if len(lines) <= n:
return text
return "\n".join(lines[-n:])
def _log_append(path: Path, text: str) -> None:
with path.open("a", encoding="utf-8", errors="replace") as fh:
fh.write(text)
if not text.endswith("\n"):
fh.write("\n")
def mkdir_p(sftp: paramiko.SFTPClient, remote_dir: str) -> None:
remote_dir = remote_dir.replace("\\", "/").rstrip("/")
if not remote_dir or remote_dir == "/":
return
parts = [p for p in remote_dir.split("/") if p]
cur = ""
for p in parts:
cur = f"{cur}/{p}" if cur else f"/{p}"
try:
sftp.stat(cur)
except OSError:
try:
sftp.mkdir(cur)
except OSError:
pass
def _shell_bytes_lf(data: bytes) -> bytes:
"""Windows 工作区常见 CRLF；上传到 Linux 后 bash 会把 pipefail\\r 当成非法选项。"""
return data.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
def sftp_put(sftp: paramiko.SFTPClient, local: Path, remote: str) -> None:
remote = remote.replace("\\", "/")
parent = posixpath.dirname(remote)
mkdir_p(sftp, parent)
if local.suffix.lower() == ".sh":
sftp.putfo(io.BytesIO(_shell_bytes_lf(local.read_bytes())), remote)
else:
sftp.put(str(local), remote)
def connect() -> tuple[paramiko.SSHClient, paramiko.SFTPClient]:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
HOST,
username=USER,
password=PASS,
timeout=30,
banner_timeout=40,
auth_timeout=30,
)
return client, client.open_sftp()
def run_raw(ssh: paramiko.SSHClient, cmd: str, timeout: int) -> tuple[int, str, str]:
# 非交互；统一 UTF-8，减少远端 locale 乱码
wrapped = (
"export LC_ALL=C.UTF-8 LANG=C.UTF-8; "
"cd ~ 2>/dev/null || true; "
+ cmd
)
_stdin, stdout, stderr = ssh.exec_command(wrapped, timeout=timeout, get_pty=False)
out_b = stdout.read()
err_b = stderr.read()
code = stdout.channel.recv_exit_status()
return code, _decode(out_b), _decode(err_b)
def run_logged(
ssh: paramiko.SSHClient,
log_path: Path,
title: str,
cmd: str,
timeout: int = 120,
) -> tuple[int, str]:
block = f"\n========== {title} ==========\n$ {cmd}\n"
_log_append(log_path, block)
code, out, err = run_raw(ssh, cmd, timeout=timeout)
full = f"exit={code}\n--- stdout ---\n{out}\n--- stderr ---\n{err}\n"
_log_append(log_path, full)
_safe_print(f"\n--- {title} (最后 {TAIL_LINES} 行) ---\n{_tail(full)}")
return code, full
def main() -> int:
LOG_DIR.mkdir(parents=True, exist_ok=True)
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
log_path = LOG_DIR / f"deploy_{stamp}.log"
log_path.write_text(f"deploy {stamp} -> {USER}@{HOST}:{REMOTE_BASE}\n", encoding="utf-8")
if hasattr(sys.stdout, "reconfigure"):
try:
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
_safe_print(f"完整日志: {log_path}")
client, sftp = None, None
try:
client, sftp = connect()
except Exception as e:
_safe_print(f"连接失败: {e}")
_log_append(log_path, f"CONNECT FAIL: {e!r}\n")
return 1
try:
run_logged(client, log_path, "pwd", f"cd {REMOTE_BASE} && pwd", timeout=30)
run_logged(client, log_path, "id", "id", timeout=15)
run_logged(client, log_path, "hostname", "hostname", timeout=15)
run_logged(
client,
log_path,
"repo_exists",
f"test -d {REMOTE_BASE} && echo ok || echo MISSING",
timeout=15,
)
for rel in FILES:
lp = REPO_ROOT / rel
if not lp.is_file():
_safe_print(f"本地缺少文件: {lp}")
return 2
rp = f"{REMOTE_BASE}/{rel.replace(chr(92), '/')}"
_log_append(log_path, f"SFTP PUT {lp} -> {rp}\n")
sftp_put(sftp, lp, rp)
_safe_print(f"已上传: {rel}")
run_logged(
client,
log_path,
"clear_python_caches",
f"rm -rf {REMOTE_BASE}/src/__pycache__ {REMOTE_BASE}/backup/__pycache__ 2>/dev/null; echo cache_cleared",
timeout=20,
)
run_logged(
client,
log_path,
"verify_utils_py_snippet",
f"sed -n '1,28p' {REMOTE_BASE}/src/utils.py",
timeout=15,
)
py_compile = (
f"cd {REMOTE_BASE} && python3 -m py_compile web.py youtube.py tiktok.py ffmpeg_install.py "
"backup/__init__.py backup/msg_push.py backup/robust_relay.py "
"src/utils.py src/logger.py src/spider.py src/stream.py src/proxy.py "
"src/http_clients/async_http.py src/http_clients/sync_http.py "
"src/control_plane.py src/web_process_backend.py src/hub_routes.py src/hub_kv_store.py "
"src/async_thread_loop.py src/youtube_ini_sync.py "
"src/live_config/__init__.py src/live_config/loader.py "
"scripts/hardware_probe.py douyin_youtube_ffplay.py main.py"
)
code_py, _ = run_logged(client, log_path, "python_py_compile", py_compile, timeout=90)
if code_py != 0:
_safe_print("远程 py_compile 失败，见日志")
return 3
_log_append(
log_path,
"\n[提示] web_console_build 在远端执行 npm + next build，ARM 上常需 5～20 分钟；"
"未完成前日志不会出现 exit=，属正常现象。\n",
)
_safe_print("远端正在构建 web-console（请耐心等待，ARM 可能较久）…")
build_cmd = (
f"cd {REMOTE_BASE}/web-console && "
"(npm ci 2>/dev/null || npm install) && "
"(command -v pnpm >/dev/null 2>&1 && pnpm run build || npm run build)"
)
code_b, _ = run_logged(client, log_path, "web_console_build", build_cmd, timeout=2400)
if code_b != 0:
_safe_print("[警告] web-console build 非 0（可能未装 Node/pnpm），完整输出见日志")
sudo_snap = (
f"echo '{PASS}' | sudo -S bash -c "
f"'systemctl list-units --type=service --state=running 2>/dev/null | head -50 || true'"
)
run_logged(client, log_path, "sudo_systemctl_running_head", sudo_snap, timeout=45)
restart_cmd = (
f"echo '{PASS}' | sudo -S bash -c '"
"systemctl try-restart live-console.service 2>/dev/null || true; "
"systemctl try-restart live-edge.service 2>/dev/null || true; "
"systemctl try-restart douyinyoutube.service 2>/dev/null || true; "
"systemctl try-restart douyin-live.service 2>/dev/null || true; "
"systemctl try-restart live-web.service 2>/dev/null || true; "
"echo try_restart_done'"
)
run_logged(client, log_path, "sudo_try_restart_units", restart_cmd, timeout=60)
run_logged(
client,
log_path,
"sudo_restart_live_console",
f"echo '{PASS}' | sudo -S systemctl restart live-console.service",
timeout=120,
)
_safe_print("等待控制面监听（最多约 45s）…")
time.sleep(4)
run_logged(
client,
log_path,
"curl_health_after_restart",
"for i in $(seq 1 22); do "
"if curl -sf -m 4 http://127.0.0.1:8001/health >/dev/null 2>&1; then echo \"health_ok_8001\"; exit 0; fi; "
"if curl -sf -m 4 http://127.0.0.1:8101/health >/dev/null 2>&1; then echo \"health_ok_8101\"; exit 0; fi; "
"sleep 2; "
"done; echo curl_health_failed; exit 0",
timeout=120,
)
# 若健康检查失败，再尝试 reset-failed + start（部分板子 restart 后 unit 未 active）
recover_console = (
f"echo '{PASS}' | sudo -S bash -c '"
"systemctl reset-failed live-console.service 2>/dev/null; "
"systemctl start live-console.service 2>/dev/null; "
"sleep 5; "
"systemctl is-active live-console.service || true; "
"journalctl -u live-console.service -n 25 --no-pager 2>/dev/null || true'"
)
run_logged(client, log_path, "recover_live_console_if_needed", recover_console, timeout=90)
run_logged(
client,
log_path,
"curl_health_after_recover",
"for i in $(seq 1 12); do "
"if curl -sf -m 4 http://127.0.0.1:8001/health >/dev/null 2>&1; then echo health_ok_recover; exit 0; fi; "
"sleep 2; done; echo still_failed; exit 0",
timeout=60,
)
run_logged(
client,
log_path,
"curl_port80_root",
"curl -sS -m 12 -o /dev/null -w 'http_code=%{http_code}\\n' http://127.0.0.1/ 2>&1 || echo curl_80_fail",
timeout=20,
)
run_logged(
client,
log_path,
"chmod_reinstall_edge",
f"chmod +x {REMOTE_BASE}/scripts/linux/reinstall_live_edge.sh 2>/dev/null || true",
timeout=15,
)
run_logged(
client,
log_path,
"reinstall_live_edge",
f"echo '{PASS}' | sudo -S bash {REMOTE_BASE}/scripts/linux/reinstall_live_edge.sh",
timeout=180,
)
# Caddy 刚 enable --now 时 bind :80 可能晚于立即 curl（曾出现 http_code=000）
time.sleep(12)
run_logged(
client,
log_path,
"curl_port80_after_edge",
"for i in 1 2 3 4 5 6 7 8; do "
"code=$(curl -sS -m 10 -o /dev/null -w '%{http_code}' http://127.0.0.1/ 2>/dev/null || echo 000); "
"echo \"try$i http_code=$code\"; "
"[ \"$code\" = 200 ] && exit 0; "
"sleep 4; "
"done; exit 1",
timeout=120,
)
run_logged(
client,
log_path,
"pm2_or_process",
f"command -v pm2 >/dev/null 2>&1 && pm2 list || "
f"(pgrep -af 'launch.py|uvicorn|web.py' | head -15 || true)",
timeout=30,
)
run_logged(
client,
log_path,
"curl_server_info",
"curl -sS -m 10 'http://127.0.0.1:8001/server_info' 2>&1 | head -c 4000 || true",
timeout=20,
)
run_logged(
client,
log_path,
"docker_srs_neko_ps",
"docker ps -a --format '{{.Names}} {{.Status}}' 2>/dev/null | grep -E 'live-srs|live-neko' || echo '(no live-srs / live-neko in docker ps)'",
timeout=25,
)
return 0
finally:
if sftp:
sftp.close()
if client:
client.close()
if __name__ == "__main__":
raise SystemExit(main())

62
d2ypp2/tools/probe_board_stack.py Normal file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""SSH 到板子：打印 ENABLE_NEKO 与 Docker 中与 srs/neko 相关的容器。
环境变量与 deploy_live_paramiko.py 一致：
LIVE_DEPLOY_HOST（默认 192.168.21.105）
LIVE_DEPLOY_USER（默认 live）
LIVE_DEPLOY_PASS（默认 12345678）
"""
from __future__ import annotations
import os
import sys
try:
import paramiko
except ImportError:
print("请先执行: pip install paramiko", file=sys.stderr)
sys.exit(1)
HOST = os.environ.get("LIVE_DEPLOY_HOST", "192.168.21.105")
USER = os.environ.get("LIVE_DEPLOY_USER", "live")
PASS = os.environ.get("LIVE_DEPLOY_PASS", "12345678")
REMOTE_SH = r"""
echo "--- ENABLE_NEKO in known env files ---"
for f in /opt/live/config/system-stack.env /opt/live/generated/system-stack.env /home/live/douyinyoutube/config/system-stack.env; do
if [ -f "$f" ]; then
echo "=== $f ==="
grep -E '^ENABLE_NEKO=' "$f" 2>/dev/null || echo '(no ENABLE_NEKO line)'
fi
done
echo "--- docker: srs / neko (running or all) ---"
docker ps -a --format '{{.Names}} {{.Status}}' 2>/dev/null | grep -iE 'neko|srs' || echo '(no container name matching neko|srs)'
echo "--- docker ps -a (first 25) ---"
docker ps -a --format '{{.Names}} {{.Status}}' 2>/dev/null | head -25
"""
def main() -> int:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
c.connect(HOST, username=USER, password=PASS, timeout=20)
except Exception as e:
print(f"SSH_CONNECT_FAILED {HOST} {type(e).__name__}: {e}", file=sys.stderr)
return 2
_, stdout, stderr = c.exec_command(
"export LC_ALL=C.UTF-8 LANG=C.UTF-8; " + REMOTE_SH,
timeout=90,
)
out = stdout.read().decode("utf-8", errors="replace")
err = stderr.read().decode("utf-8", errors="replace")
code = stdout.channel.recv_exit_status()
c.close()
print(out, end="")
if err.strip():
print(err, file=sys.stderr, end="")
return code
if __name__ == "__main__":
raise SystemExit(main())

42
d2ypp2/tools/ssh_neko_start.py Normal file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""SSH：在板子上执行 service_ctl.sh neko start（拉镜像可能较久）。"""
from __future__ import annotations
import os
import sys
import paramiko
HOST = os.environ.get("LIVE_DEPLOY_HOST", "192.168.21.105")
USER = os.environ.get("LIVE_DEPLOY_USER", "live")
PASS = os.environ.get("LIVE_DEPLOY_PASS", "12345678")
REMOTE_BASE = os.environ.get("LIVE_DEPLOY_PATH", "/home/live/douyinyoutube").rstrip("/")
CMD = (
f"export LC_ALL=C.UTF-8 LANG=C.UTF-8; "
f"cd {REMOTE_BASE} && bash scripts/linux/service_ctl.sh neko start"
)
def main() -> int:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
c.connect(HOST, username=USER, password=PASS, timeout=25)
except Exception as e:
print(f"SSH failed: {e}", file=sys.stderr)
return 2
_, stdout, stderr = c.exec_command(CMD, timeout=600)
out = stdout.read().decode("utf-8", errors="replace")
err = stderr.read().decode("utf-8", errors="replace")
code = stdout.channel.recv_exit_status()
c.close()
print(out, end="")
if err.strip():
print("--- stderr ---", file=sys.stderr)
print(err, file=sys.stderr, end="")
return code
if __name__ == "__main__":
raise SystemExit(main())

57
d2ypp2/tools/verify_live_console.py Normal file
View File

@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""重启 live-console 并验收 HTTP（Paramiko）。"""
from __future__ import annotations
import sys
import time
from pathlib import Path
import paramiko
HOST = "192.168.21.105"
USER = "live"
PASS = "12345678"
LOG = Path(__file__).resolve().parent.parent / ".deploy_logs" / "verify_live_console.log"
def main() -> int:
LOG.parent.mkdir(parents=True, exist_ok=True)
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(HOST, username=USER, password=PASS, timeout=30)
def run(cmd: str, timeout: int = 90) -> tuple[int, str, str]:
_, stdout, stderr = c.exec_command(
"export LC_ALL=C.UTF-8 LANG=C.UTF-8; " + cmd,
timeout=timeout,
)
out = stdout.read().decode("utf-8", errors="replace")
err = stderr.read().decode("utf-8", errors="replace")
code = stdout.channel.recv_exit_status()
block = f"\n$ {cmd}\nexit={code}\n--- out ---\n{out}\n--- err ---\n{err}\n"
with LOG.open("a", encoding="utf-8", errors="replace") as fh:
fh.write(block)
tail = "\n".join(block.splitlines()[-60:])
print(tail)
return code, out, err
try:
run(
f"echo '{PASS}' | sudo -S systemctl restart live-console.service",
timeout=120,
)
time.sleep(5)
run("systemctl is-active live-console.service", timeout=20)
run("ss -tlnp 2>/dev/null | grep 8001 || true", timeout=20)
run(
"curl -sS -m 15 http://127.0.0.1:8001/health && echo "
"&& curl -sS -m 15 http://127.0.0.1:8001/server_info | head -c 1200",
timeout=40,
)
return 0
finally:
c.close()
if __name__ == "__main__":
raise SystemExit(main())