's'

2026-06-08 03:06:12 -05:00
parent 60dfe538b0
commit ebfec37e47
34 changed files with 1753 additions and 31 deletions
--- a/scripts/ssh-build-frontend-prod.py
+++ b/scripts/ssh-build-frontend-prod.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Build Next.js on production and restart cnomadcna.service."""
+from __future__ import annotations
+
+import sys
+
+import paramiko
+
+HOST = "82.157.112.245"
+USER = "ubuntu"
+PASSWORD = "Xiao4669805"
+REPO = "/home/ubuntu/gitlab-instance-0a899031_cnomadcna"
+BUILD_CMD = (
+    f"cd {REPO} && "
+    "APP_ENV=production NEXT_PUBLIC_APP_ENV=production NODE_ENV=production "
+    "pnpm build && systemctl restart cnomadcna.service && "
+    "systemctl is-active cnomadcna.service"
+)
+
+
+def main() -> int:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        _, o, e = c.exec_command(
+            f"echo '{PASSWORD}' | sudo -S bash -lc \"{BUILD_CMD}\"",
+            timeout=900,
+            get_pty=True,
+        )
+        out = o.read().decode("utf-8", errors="replace")
+        err = e.read().decode("utf-8", errors="replace")
+        code = o.channel.recv_exit_status()
+        if out:
+            sys.stdout.buffer.write(out.encode("utf-8", errors="replace"))
+        if err:
+            sys.stdout.buffer.write(err.encode("utf-8", errors="replace"))
+        return code
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/scripts/ssh-check-docker-mirror.py
+++ b/scripts/ssh-check-docker-mirror.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+CMDS = [
+    "echo '{pw}' | sudo -S cat /etc/docker/daemon.json 2>/dev/null || echo 'no daemon.json'",
+    "echo '{pw}' | sudo -S docker info 2>/dev/null | grep -A5 'Registry Mirrors' || true",
+    "echo '{pw}' | sudo -S docker images --format '{{.Repository}}:{{.Tag}}' | sort",
+    "echo '{pw}' | sudo -S bash -lc 'cd /opt/nomadro-services/integrations && docker compose ps -a 2>/dev/null'",
+    "echo '{pw}' | sudo -S nginx -t 2>&1",
+    "curl -sI https://nomadro.cn/ | head -3",
+]
+
+
+def main() -> None:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        for cmd in CMDS:
+            cmd = cmd.format(pw=PASSWORD)
+            print(f"\n=== {cmd.replace(PASSWORD, '***')} ===")
+            _, o, e = c.exec_command(cmd, timeout=90, get_pty=True)
+            sys.stdout.buffer.write(o.read())
+            sys.stdout.buffer.write(e.read())
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-check-ntfy.py
+++ b/scripts/ssh-check-ntfy.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+CMDS = [
+    "docker ps -a | grep -i ntfy || true",
+    "curl -sI http://127.0.0.1:9180/ | head -8",
+    "curl -sI https://ntfy.nomadro.cn/ | head -10",
+    "curl -skI https://ntfy.nomadro.cn/ | head -10",
+    "sudo cat /www/server/panel/vhost/nginx/ntfy.nomadro.cn.conf 2>/dev/null | head -60",
+    "sudo ls /etc/letsencrypt/live/ | grep ntfy || true",
+    "sudo certbot certificates 2>/dev/null | grep -A3 ntfy || true",
+]
+
+
+def main() -> None:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        for cmd in CMDS:
+            full = f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\""
+            print(f"\n=== {cmd} ===")
+            _, o, e = c.exec_command(full, timeout=60, get_pty=True)
+            sys.stdout.buffer.write(o.read())
+            sys.stdout.buffer.write(e.read())
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-fix-deploy.py
+++ b/scripts/ssh-fix-deploy.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+CMDS = [
+    "echo '{pw}' | sudo -S ls /www/server/panel/vhost/nginx/ | grep nomadro",
+    "echo '{pw}' | sudo -S head -40 /www/server/panel/vhost/nginx/lounge.nomadro.cn.conf 2>/dev/null || echo '{pw}' | sudo -S head -40 /www/server/panel/vhost/nginx/mirotalk.nomadro.cn.conf",
+    "echo '{pw}' | sudo -S ls /etc/letsencrypt/live/ 2>/dev/null | head -20",
+    "echo '{pw}' | sudo -S docker images | head -20",
+    "echo '{pw}' | sudo -S cat /opt/nomadro-services/integrations/docker-compose.yml | head -30",
+]
+
+
+def main() -> None:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        for cmd in CMDS:
+            cmd = cmd.format(pw=PASSWORD)
+            print(f"\n=== {cmd.replace(PASSWORD, '***')} ===")
+            _, o, e = c.exec_command(cmd, timeout=60, get_pty=True)
+            out = o.read().decode("utf-8", errors="replace")
+            err = e.read().decode("utf-8", errors="replace")
+            sys.stdout.buffer.write(out.encode("utf-8", errors="replace"))
+            sys.stdout.buffer.write(err.encode("utf-8", errors="replace"))
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-fix-lounge.py
+++ b/scripts/ssh-fix-lounge.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Diagnose and fix The Lounge + Ergo IRC on production VPS."""
+from __future__ import annotations
+
+import sys
+
+import paramiko
+
+import os
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "")
+
+COMMANDS = [
+    "hostname",
+    "systemctl is-active thelounge.service ergo.service 2>&1 || true",
+    "systemctl status thelounge.service --no-pager -l 2>&1 | tail -25",
+    "systemctl status ergo.service --no-pager -l 2>&1 | tail -25",
+    "ss -ltnp 2>/dev/null | grep -E ':9000|:6667' || netstat -ltnp 2>/dev/null | grep -E ':9000|:6667' || true",
+    "curl -sI http://127.0.0.1:9000/ 2>&1 | head -8",
+    "curl -sI https://lounge.nomadro.cn/ 2>&1 | head -10",
+    "journalctl -u thelounge.service -n 30 --no-pager 2>&1",
+    "journalctl -u ergo.service -n 20 --no-pager 2>&1",
+]
+
+
+def run_remote(commands: list[str]) -> None:
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(HOST, username=USER, password=PASSWORD, timeout=25)
+    try:
+        for cmd in commands:
+            print(f"\n=== {cmd} ===")
+            _, stdout, stderr = client.exec_command(cmd, timeout=45)
+            out = stdout.read().decode("utf-8", errors="replace")
+            err = stderr.read().decode("utf-8", errors="replace")
+            if out:
+                print(out.rstrip())
+            if err:
+                print("ERR:", err.rstrip())
+    finally:
+        client.close()
+
+
+if __name__ == "__main__":
+    run_remote(COMMANDS if len(sys.argv) == 1 else sys.argv[1:])
--- a/scripts/ssh-git-pull-prod.py
+++ b/scripts/ssh-git-pull-prod.py
@@ -40,9 +40,11 @@ def main() -> int:
    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    c.connect(HOST, username=USER, password=SSH_PASSWORD, timeout=30)
    try:
+        run(c, f"cp {REPO}/.env.local /tmp/nomadcna.env.local.bak 2>/dev/null || true")
        run(c, f"cd {REPO} && git fetch {auth_url} {BRANCH}")
        run(c, f"cd {REPO} && git checkout {BRANCH}")
-        run(c, f"cd {REPO} && git pull {auth_url} {BRANCH}")
+        run(c, f"cd {REPO} && git reset --hard FETCH_HEAD")
+        run(c, f"cp /tmp/nomadcna.env.local.bak {REPO}/.env.local 2>/dev/null || true")
        run(c, "systemctl restart cnomadcna-api.service")
        run(c, "systemctl is-active cnomadcna-api.service")
        run(c, f"cd {REPO} && git log -1 --oneline")
--- a/scripts/ssh-ntfy-auth.py
+++ b/scripts/ssh-ntfy-auth.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+CMDS = [
+    "docker inspect nomadro-ntfy --format '{{json .Mounts}}'",
+    "sudo ls -la /opt/nomadro-services/ 2>/dev/null",
+    "sudo cat /opt/nomadro-services/docker-compose.yml 2>/dev/null | head -80",
+    "sudo ls -la /var/lib/ntfy/ 2>/dev/null",
+    "sudo cat /var/lib/ntfy/server.yml 2>/dev/null | head -40",
+    "docker exec nomadro-ntfy ntfy token list 2>/dev/null || true",
+]
+
+
+def main() -> None:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        for cmd in CMDS:
+            print(f"\n=== {cmd} ===")
+            _, o, e = c.exec_command(f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\"", timeout=60, get_pty=True)
+            sys.stdout.buffer.write(o.read())
+            sys.stdout.buffer.write(e.read())
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-patch-join-page.py
+++ b/scripts/ssh-patch-join-page.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""Patch join page TS fix on production."""
+import base64
+import sys
+
+import paramiko
+
+HOST = "82.157.112.245"
+USER = "ubuntu"
+PASSWORD = "Xiao4669805"
+LOCAL = r"c:\Users\admin\Desktop\gitlab-instance-0a899031_cnomadcna\app\[locale]\join\page.tsx"
+REMOTE = "/home/ubuntu/gitlab-instance-0a899031_cnomadcna/app/[locale]/join/page.tsx"
+TMP = "/tmp/join-page.tsx"
+
+content = open(LOCAL, encoding="utf-8").read()
+b64 = base64.b64encode(content.encode("utf-8")).decode()
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+cmd = (
+    f"python3 -c \"import base64; open('{TMP}','wb').write(base64.b64decode('{b64}'))\" "
+    f"&& cp {TMP} {REMOTE} && chown ubuntu:ubuntu {REMOTE} && echo uploaded"
+)
+_, o, e = c.exec_command(f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\"", timeout=120)
+out = o.read().decode()
+err = e.read().decode()
+code = o.channel.recv_exit_status()
+sys.stdout.write(out)
+sys.stdout.write(err)
+c.close()
+sys.exit(code)
--- a/scripts/ssh-probe-server.py
+++ b/scripts/ssh-probe-server.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+CMDS = [
+    "hostname; free -h | head -2; df -h / | tail -1",
+    "docker --version 2>&1; docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' 2>&1",
+    "ss -ltnp 2>/dev/null | grep -E ':5432|:3306|:9000|:9300|:8090|:4002|:8000|:9180|:9100|:3000|:8080|:1935|:1337|:8025|:9001' || true",
+    "ls /www/server/panel/vhost/nginx/*.conf 2>/dev/null | xargs -I{} basename {}",
+    "ls /etc/systemd/system/*.service 2>/dev/null | xargs -I{} basename {} | grep -E 'nomad|lounge|mirotalk|ergo|cnomad' || true",
+    "which nginx certbot 2>&1",
+    "curl -sI https://nomadro.cn/ | head -3",
+]
+
+
+def main() -> None:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=25)
+    try:
+        for cmd in CMDS:
+            print(f"\n=== {cmd} ===")
+            _, o, e = c.exec_command(cmd, timeout=45, get_pty=True)
+            out = o.read().decode("utf-8", errors="replace").strip()
+            err = e.read().decode("utf-8", errors="replace").strip()
+            if out:
+                print(out)
+            if err:
+                print("ERR:", err)
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-restore-lounge.py
+++ b/scripts/ssh-restore-lounge.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Restore corrupted The Lounge config.js on production VPS."""
+from __future__ import annotations
+
+import paramiko
+
+import os
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "")
+
+INSPECT = [
+    "cat /var/lib/thelounge/config.js 2>&1 | head -20",
+    "ls -la /var/lib/thelounge/ 2>&1",
+    "ls -la /var/lib/thelounge/*.bak /var/lib/thelounge/config.js.* 2>&1 || true",
+    "cat /etc/systemd/system/thelounge.service 2>&1",
+    "sudo ls -la /var/lib/thelounge/ 2>&1",
+]
+
+LOUNGE_CONFIG = r"""'use strict';
+
+module.exports = {
+  host: '127.0.0.1',
+  port: 9000,
+  public: true,
+  theme: 'morning',
+  prefetch: false,
+  disableLog: false,
+  fileUpload: false,
+  transports: ['websocket', 'polling'],
+  defaults: {
+    name: 'NomadCNA',
+    username: 'nomad',
+    realname: 'NomadCNA Community',
+    join: '#nomadcna,#online-events',
+  },
+  lockNetwork: true,
+  messageStorage: [],
+  useHexIp: false,
+  maxHistory: 10000,
+  displayNetwork: true,
+  networks: [
+    {
+      name: 'NomadCNA',
+      host: '127.0.0.1',
+      port: 6667,
+      tls: false,
+      rejectUnauthorized: false,
+      nick: 'nomad',
+      username: 'nomad',
+      realname: 'NomadCNA Community',
+      join: '#nomadcna,#online-events',
+      commands: [],
+    },
+  ],
+};
+"""
+
+
+def run(commands: list[str]) -> None:
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(HOST, username=USER, password=PASSWORD, timeout=25)
+    try:
+        for cmd in commands:
+            print(f"\n=== {cmd} ===")
+            _, stdout, stderr = client.exec_command(cmd, timeout=60, get_pty=True)
+            out = stdout.read().decode("utf-8", errors="replace")
+            err = stderr.read().decode("utf-8", errors="replace")
+            if out:
+                print(out.rstrip())
+            if err:
+                print("ERR:", err.rstrip())
+    finally:
+        client.close()
+
+
+def restore() -> None:
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    client.connect(HOST, username=USER, password=PASSWORD, timeout=25)
+    try:
+        sftp = client.open_sftp()
+        remote_tmp = "/tmp/thelounge-config.js"
+        with sftp.file(remote_tmp, "w") as f:
+            f.write(LOUNGE_CONFIG)
+        sftp.close()
+
+        cmds = [
+            f"echo '{PASSWORD}' | sudo -S cp /var/lib/thelounge/config.js /var/lib/thelounge/config.js.broken-$(date +%Y%m%d%H%M%S) 2>/dev/null || true",
+            f"echo '{PASSWORD}' | sudo -S cp {remote_tmp} /var/lib/thelounge/config.js",
+            f"echo '{PASSWORD}' | sudo -S chown thelounge:thelounge /var/lib/thelounge/config.js",
+            f"echo '{PASSWORD}' | sudo -S chmod 640 /var/lib/thelounge/config.js",
+            f"echo '{PASSWORD}' | sudo -S systemctl restart thelounge.service",
+            "sleep 3",
+            "systemctl is-active thelounge.service",
+            "curl -sI http://127.0.0.1:9000/ | head -8",
+            "curl -sI https://lounge.nomadro.cn/ | head -8",
+            "journalctl -u thelounge.service -n 15 --no-pager",
+        ]
+        for cmd in cmds:
+            print(f"\n=== {cmd.replace(PASSWORD, '***')} ===")
+            _, stdout, stderr = client.exec_command(cmd, timeout=60, get_pty=True)
+            out = stdout.read().decode("utf-8", errors="replace")
+            err = stderr.read().decode("utf-8", errors="replace")
+            if out:
+                print(out.rstrip())
+            if err:
+                print("ERR:", err.rstrip())
+    finally:
+        client.close()
+
+
+VERIFY = [
+    "systemctl is-active thelounge.service ergo.service",
+    "curl -s 'https://lounge.nomadro.cn/socket.io/?EIO=4&transport=polling&t=health' | cut -c1-120",
+    "{ printf 'NICK health\\r\\nUSER health 0 * :NomadCNA Health\\r\\nJOIN #nomadcna\\r\\nQUIT :healthcheck\\r\\n'; sleep 1; } | timeout 5s nc 127.0.0.1 6667 2>&1 | head -5",
+    "curl -sI 'https://lounge.nomadro.cn/' | head -3",
+]
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) > 1 and sys.argv[1] == "restore":
+        restore()
+    elif len(sys.argv) > 1 and sys.argv[1] == "verify":
+        run(VERIFY)
+    else:
+        run(INSPECT)
--- a/scripts/ssh-setup-ntfy-acl.py
+++ b/scripts/ssh-setup-ntfy-acl.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Configure ntfy ACL: public read on nomadcna-* topics, backend publish token."""
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+
+def run(c: paramiko.SSHClient, cmd: str, timeout: int = 120) -> str:
+    print(f"\n$ {cmd.replace(PASSWORD, '***')}")
+    _, o, e = c.exec_command(f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\"", timeout=timeout, get_pty=True)
+    out = o.read().decode("utf-8", errors="replace")
+    err = e.read().decode("utf-8", errors="replace")
+    sys.stdout.buffer.write(out.encode("utf-8", errors="replace"))
+    sys.stdout.buffer.write(err.encode("utf-8", errors="replace"))
+    return out + err
+
+
+def main() -> None:
+    api_pass = secrets.token_urlsafe(16)
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        run(c, "docker exec nomadro-ntfy ntfy user add --role=admin nomadcna-api --pass='{}' 2>/dev/null || true".format(api_pass))
+        run(c, "docker exec nomadro-ntfy ntfy access nomadcna-* read everyone")
+        run(c, "docker exec nomadro-ntfy ntfy access nomadcna-* write nomadcna-api")
+        token_out = run(c, "docker exec nomadro-ntfy ntfy token add --user=nomadcna-api nomadcna-backend 2>&1")
+        token_match = re.search(r"tk_[a-zA-Z0-9_-]+", token_out)
+        token = token_match.group(0) if token_match else ""
+
+        if token:
+            run(c, f"grep -q NTFY_AUTH_TOKEN /opt/nomadro-services/.env || echo 'NTFY_AUTH_TOKEN={token}' >> /opt/nomadro-services/.env")
+            run(c, f"grep -q NTFY_AUTH_TOKEN /home/ubuntu/gitlab-instance-0a899031_cnomadcna/.env.local 2>/dev/null || echo 'NTFY_AUTH_TOKEN={token}' >> /home/ubuntu/gitlab-instance-0a899031_cnomadcna/.env.local")
+
+        test = run(
+            c,
+            f"curl -s -X POST https://ntfy.nomadro.cn/nomadcna-healthcheck "
+            f"-H 'Authorization: Bearer {token}' -H 'Title: NomadCNA' -d 'push ok'",
+        )
+        print("\n=== Token (save to FastAPI env) ===")
+        print(token or "(check token add output above)")
+        print("=== Test response ===")
+        print(test.strip()[:200])
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-setup-ntfy-acl2.py
+++ b/scripts/ssh-setup-ntfy-acl2.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+
+def run(c: paramiko.SSHClient, cmd: str) -> str:
+    print(f"\n$ {cmd.replace(PASSWORD, '***')}")
+    _, o, e = c.exec_command(f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\"", timeout=120, get_pty=True)
+    out = o.read().decode("utf-8", errors="replace")
+    err = e.read().decode("utf-8", errors="replace")
+    combined = out + err
+    sys.stdout.buffer.write(combined.encode("utf-8", errors="replace"))
+    return combined
+
+
+def main() -> None:
+    api_pass = secrets.token_urlsafe(12)
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        run(c, "docker exec nomadro-ntfy ntfy user list 2>&1")
+        run(c, "docker exec nomadro-ntfy ntfy access --help 2>&1 | head -25")
+        run(c, f"docker exec nomadro-ntfy ntfy user add --role=admin nomadcna-api --pass={api_pass} 2>&1")
+        run(c, "docker exec nomadro-ntfy ntfy access nomadcna-* everyone read 2>&1")
+        run(c, "docker exec nomadro-ntfy ntfy access nomadcna-* nomadcna-api write-only 2>&1")
+        token_out = run(c, "docker exec nomadro-ntfy ntfy token add -l nomadcna-backend nomadcna-api 2>&1")
+        token_match = re.search(r"tk_[a-zA-Z0-9_-]+", token_out)
+        token = token_match.group(0) if token_match else ""
+        if not token:
+            run(c, "docker exec nomadro-ntfy ntfy token list nomadcna-api 2>&1")
+        if token:
+            repo_env = "/home/ubuntu/gitlab-instance-0a899031_cnomadcna/.env.local"
+            run(c, f"grep -q '^NTFY_AUTH_TOKEN=' {repo_env} 2>/dev/null && sed -i 's/^NTFY_AUTH_TOKEN=.*/NTFY_AUTH_TOKEN={token}/' {repo_env} || echo 'NTFY_AUTH_TOKEN={token}' >> {repo_env}")
+            run(
+                c,
+                f"curl -s -X POST https://ntfy.nomadro.cn/nomadcna-healthcheck "
+                f"-H 'Authorization: Bearer {token}' -H 'Title: NomadCNA' -d 'push ok'",
+            )
+            run(c, "systemctl restart cnomadcna-api.service 2>/dev/null || true")
+        print(f"\nTOKEN={token}")
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-setup-ntfy-acl3.py
+++ b/scripts/ssh-setup-ntfy-acl3.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import sys
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+
+
+def run(c: paramiko.SSHClient, cmd: str) -> str:
+    print(f"\n$ {cmd.replace(PASSWORD, '***')}")
+    _, o, e = c.exec_command(f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\"", timeout=120, get_pty=True)
+    combined = o.read().decode("utf-8", errors="replace") + e.read().decode("utf-8", errors="replace")
+    sys.stdout.buffer.write(combined.encode("utf-8", errors="replace"))
+    return combined
+
+
+def main() -> None:
+    api_pass = secrets.token_urlsafe(16)
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    try:
+        run(c, "docker exec nomadro-ntfy ntfy user add --help 2>&1 | head -30")
+        run(c, f"docker exec nomadro-ntfy ntfy user add nomadcna-api --role=writer --passwd={api_pass} 2>&1")
+        run(c, "docker exec nomadro-ntfy ntfy access '*' 'nomadcna-*' read 2>&1")
+        run(c, "docker exec nomadro-ntfy ntfy access nomadcna-api 'nomadcna-*' write-only 2>&1")
+        run(c, "docker exec nomadro-ntfy ntfy user list 2>&1")
+        token_out = run(c, "docker exec nomadro-ntfy ntfy token add -l nomadcna-backend nomadcna-api 2>&1")
+        token = re.search(r"tk_[a-zA-Z0-9_-]+", token_out)
+        token_val = token.group(0) if token else ""
+        if not token_val:
+            token_out2 = run(c, "docker exec nomadro-ntfy ntfy token add -l nomadcna-backend nomadro 2>&1")
+            token2 = re.search(r"tk_[a-zA-Z0-9_-]+", token_out2)
+            token_val = token2.group(0) if token2 else ""
+        if token_val:
+            repo = "/home/ubuntu/gitlab-instance-0a899031_cnomadcna/.env.local"
+            run(
+                c,
+                f"touch {repo} && grep -q '^NTFY_AUTH_TOKEN=' {repo} && "
+                f"sed -i 's/^NTFY_AUTH_TOKEN=.*/NTFY_AUTH_TOKEN={token_val}/' {repo} || "
+                f"echo 'NTFY_AUTH_TOKEN={token_val}' >> {repo}",
+            )
+            run(
+                c,
+                f"curl -s -X POST https://ntfy.nomadro.cn/nomadcna-healthcheck "
+                f"-H 'Authorization: Bearer {token_val}' -H 'Title: NomadCNA' -d 'push ok'",
+            )
+            run(c, "systemctl restart cnomadcna-api.service")
+            run(c, "systemctl is-active cnomadcna-api.service")
+        print(f"\nNTFY_AUTH_TOKEN={token_val}")
+    finally:
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-sync-ntfy-backend.py
+++ b/scripts/ssh-sync-ntfy-backend.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+import paramiko
+
+HOST = os.getenv("NOMAD_SSH_HOST", "82.157.112.245")
+USER = os.getenv("NOMAD_SSH_USER", "ubuntu")
+PASSWORD = os.getenv("NOMAD_SSH_PASSWORD", "Xiao4669805")
+REPO = "/home/ubuntu/gitlab-instance-0a899031_cnomadcna"
+ROOT = Path(__file__).resolve().parent.parent
+
+FILES = [
+    "backend/ntfy_push.py",
+    "backend/main.py",
+    "backend/settings.py",
+]
+
+
+def main() -> None:
+    c = paramiko.SSHClient()
+    c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+    sftp = c.open_sftp()
+    try:
+        for rel in FILES:
+            local = ROOT / rel
+            tmp = f"/tmp/{Path(rel).name}"
+            dest = f"{REPO}/{rel.replace(chr(92), '/')}"
+            sftp.put(str(local), tmp)
+            _, o, _ = c.exec_command(
+                f"echo '{PASSWORD}' | sudo -S cp {tmp} {dest} && sudo chown ubuntu:ubuntu {dest}",
+                timeout=60,
+                get_pty=True,
+            )
+            print(f"uploaded {rel}: {o.read().decode().strip()}")
+        _, o, _ = c.exec_command(
+            f"echo '{PASSWORD}' | sudo -S systemctl restart cnomadcna-api.service && systemctl is-active cnomadcna-api.service",
+            timeout=60,
+            get_pty=True,
+        )
+        sys.stdout.buffer.write(o.read())
+    finally:
+        sftp.close()
+        c.close()
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/ssh-verify-prod.py
+++ b/scripts/ssh-verify-prod.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""Quick production health checks after deploy."""
+from __future__ import annotations
+
+import paramiko
+
+HOST = "82.157.112.245"
+USER = "ubuntu"
+PASSWORD = "Xiao4669805"
+REPO = "/home/ubuntu/gitlab-instance-0a899031_cnomadcna"
+
+cmds = [
+    "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8000/docs",
+    f"test -f {REPO}/backend/ntfy_push.py && echo ntfy_push:ok",
+    "grep -E '^(NTFY_|LISTMONK_)' /home/ubuntu/gitlab-instance-0a899031_cnomadcna/.env.local | sed 's/=.*/=***/'",
+    "systemctl is-active cnomadcna-api.service",
+    "systemctl is-active cnomadcna.service",
+    "systemctl cat cnomadcna.service | head -20",
+]
+
+c = paramiko.SSHClient()
+c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+c.connect(HOST, username=USER, password=PASSWORD, timeout=30)
+for cmd in cmds:
+    _, o, _ = c.exec_command(f"echo '{PASSWORD}' | sudo -S bash -lc \"{cmd}\"", timeout=60)
+    out = o.read().decode().strip()
+    print(f"$ {cmd}\n  -> {out}\n")
+c.close()