Files
gitlab-instance-0a899031_no…/deploy/observability/remote_deploy.py

545 lines
20 KiB
Python

#!/usr/bin/env python3
"""
Deploy Umami + ntfy, configure Caddy, create track sites, inject trackers,
install notify/healthcheck helpers.
Secrets stay on the server under /etc/nomadro/ (never committed).
"""
from __future__ import annotations
import json
import os
import secrets
import string
import time
import urllib.error
import urllib.request
from pathlib import Path
import paramiko
HOST = "107.173.30.245"
USER = "root"
PASS = os.environ.get("NOMADRO_SSH_PASS", "Xiao4669805")
REPO = Path(__file__).resolve().parents[2]
UMAMI_DIR = "/opt/umami"
NTFY_DIR = "/opt/ntfy"
CONF_DIR = "/etc/nomadro"
SITES = [
("nomadro.com", "Nomadro (blog + app)"),
("portal.nomadro.com", "Nomadro portal"),
("dylive.nomadro.com", "DouyinWEB / dylive"),
("nomadro.live", "TikTok Promo / nomadro.live"),
("wx.nomadro.com", "HackRobot / wx"),
("x.nomadro.com", "XAuto / x"),
]
# layout paths on server for Next.js tracker injection
NEXT_LAYOUTS = [
("/var/www/nomadro.com/app/layout.tsx", "nomadro.com", "systemctl restart nomadro", "pnpm"),
("/opt/douyinWEB/frontend/src/app/layout.tsx", "dylive.nomadro.com", "systemctl restart douyinweb-web", "npm"),
("/opt/tiktokWEB/frontend/src/app/layout.tsx", "nomadro.live", "systemctl restart tiktok-promo-frontend", "npm"),
("/opt/wauto/web/app/layout.tsx", "wx.nomadro.com", "systemctl restart hackrobot-web", "npm"),
]
def gen_password(n: int = 24) -> str:
alphabet = string.ascii_letters + string.digits
return "".join(secrets.choice(alphabet) for _ in range(n))
def ssh_connect() -> paramiko.SSHClient:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(HOST, username=USER, password=PASS, timeout=30, allow_agent=False, look_for_keys=False)
return c
def run(c: paramiko.SSHClient, cmd: str, timeout: int = 300) -> str:
print("$", cmd[:160])
_i, o, e = c.exec_command(cmd, timeout=timeout)
out = (o.read() + e.read()).decode("utf-8", errors="replace")
if out.strip():
print(out[-2500:] if len(out) > 2500 else out)
return out
def put_dir(sftp: paramiko.SFTPClient, local: Path, remote: str) -> None:
try:
sftp.stat(remote)
except OSError:
# mkdir -p via parents
parts = remote.strip("/").split("/")
cur = ""
for p in parts:
cur += "/" + p
try:
sftp.stat(cur)
except OSError:
sftp.mkdir(cur)
for path in local.rglob("*"):
if path.is_dir():
continue
if path.name.startswith(".env") and path.name != ".env.example":
continue
rel = path.relative_to(local).as_posix()
rpath = f"{remote}/{rel}"
rdir = "/".join(rpath.split("/")[:-1])
try:
sftp.stat(rdir)
except OSError:
parts = rdir.strip("/").split("/")
cur = ""
for p in parts:
cur += "/" + p
try:
sftp.stat(cur)
except OSError:
sftp.mkdir(cur)
sftp.put(str(path), rpath)
print("put", rpath)
def http_json(url: str, method: str = "GET", data: dict | None = None, token: str | None = None, timeout: int = 30):
body = None if data is None else json.dumps(data).encode()
headers = {"Content-Type": "application/json", "Accept": "application/json"}
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, data=body, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
raw = r.read().decode()
return r.status, json.loads(raw) if raw else {}
except urllib.error.HTTPError as e:
raw = e.read().decode(errors="replace")
try:
parsed = json.loads(raw) if raw else {}
except json.JSONDecodeError:
parsed = {"raw": raw}
return e.code, parsed
def wait_umami(c: paramiko.SSHClient, tries: int = 60) -> None:
for i in range(tries):
out = run(c, "curl -fsS -o /dev/null -w '%{http_code}' http://127.0.0.1:3010/api/heartbeat || echo fail")
if "200" in out:
print("umami ready")
return
time.sleep(3)
raise RuntimeError("Umami did not become healthy")
def wait_ntfy(c: paramiko.SSHClient, tries: int = 40) -> None:
for i in range(tries):
out = run(c, "curl -fsS http://127.0.0.1:2586/v1/health || true")
if "healthy" in out:
print("ntfy ready")
return
time.sleep(2)
raise RuntimeError("ntfy did not become healthy")
def ensure_umami_sites(admin_user: str, admin_pass: str) -> dict[str, str]:
"""Create websites; return domain -> websiteId. Prefer public HTTPS once Caddy is up."""
base = "https://umami.nomadro.com"
# fallback local via SSH tunnel not needed if caddy works; try public first
code, login = http_json(
f"{base}/api/auth/login",
"POST",
{"username": admin_user, "password": admin_pass},
)
if code != 200 or "token" not in login:
# try default then change
code, login = http_json(
f"{base}/api/auth/login",
"POST",
{"username": "admin", "password": "umami"},
)
if code != 200:
raise RuntimeError(f"Umami login failed: {code} {login}")
token = login["token"]
uid = login["user"]["id"]
http_json(f"{base}/api/users/{uid}", "POST", {"password": admin_pass}, token=token)
# re-login
code, login = http_json(
f"{base}/api/auth/login",
"POST",
{"username": "admin", "password": admin_pass},
)
if code != 200:
raise RuntimeError(f"Umami re-login failed: {code} {login}")
token = login["token"]
code, listing = http_json(f"{base}/api/websites", token=token)
existing = {}
if code == 200:
items = listing if isinstance(listing, list) else listing.get("data") or listing.get("websites") or []
for w in items:
if isinstance(w, dict) and w.get("domain"):
existing[w["domain"]] = w["id"]
result = {}
for domain, name in SITES:
if domain in existing:
result[domain] = existing[domain]
print("umami site exists", domain, existing[domain])
continue
code, created = http_json(
f"{base}/api/websites",
"POST",
{"name": name, "domain": domain},
token=token,
)
if code not in (200, 201) or "id" not in created:
raise RuntimeError(f"create website {domain} failed: {code} {created}")
result[domain] = created["id"]
print("umami site created", domain, created["id"])
return result
UMAMI_MARKER = "umami.nomadro.com/script.js"
def inject_layout_snippet(content: str, website_id: str) -> str:
snippet_line = (
f' <script defer src="https://umami.nomadro.com/script.js" '
f'data-website-id="{website_id}"></script>'
)
if UMAMI_MARKER in content:
# replace existing website id
import re
return re.sub(
r'<script defer src="https://umami\.nomadro\.com/script\.js" data-website-id="[^"]*"></script>',
snippet_line.strip(),
content,
)
# Prefer inside <head> if present
if "<head>" in content:
return content.replace("<head>", f"<head>\n{snippet_line}", 1)
# App router often has no head — inject before </body> or after <html...>
if "<body" in content:
return content.replace("<body", f"{snippet_line}\n <body", 1)
if "</html>" in content:
return content.replace("</html>", f"{snippet_line}\n </html>", 1)
return content + "\n" + snippet_line + "\n"
def main() -> None:
umami_admin_pass = gen_password(20)
umami_db_pass = gen_password(24)
umami_app_secret = secrets.token_hex(32)
ntfy_admin_pass = gen_password(20)
c = ssh_connect()
sftp = c.open_sftp()
run(c, f"mkdir -p {UMAMI_DIR} {NTFY_DIR} {CONF_DIR} /usr/local/bin /var/lib/nomadro-health")
# Upload stacks
put_dir(sftp, REPO / "deploy" / "umami", UMAMI_DIR)
put_dir(sftp, REPO / "deploy" / "ntfy", NTFY_DIR)
put_dir(sftp, REPO / "deploy" / "observability", f"{CONF_DIR}/observability-src")
for name in ("umami.nomadro.com.caddy", "ntfy.nomadro.com.caddy"):
sftp.put(str(REPO / "deploy" / "caddy-sites" / name), f"/etc/caddy/sites/{name}")
print("caddy site", name)
# Write umami .env
umami_env = f"POSTGRES_PASSWORD={umami_db_pass}\nAPP_SECRET={umami_app_secret}\n"
with sftp.file(f"{UMAMI_DIR}/.env", "w") as f:
f.write(umami_env)
# Save admin credentials (may update password after first login)
with sftp.file(f"{CONF_DIR}/umami.env", "w") as f:
f.write(f"UMAMI_URL=https://umami.nomadro.com\nUMAMI_USER=admin\nUMAMI_PASSWORD={umami_admin_pass}\n")
# Start stacks
run(c, f"cd {UMAMI_DIR} && docker compose pull && docker compose up -d", timeout=600)
run(c, f"cd {NTFY_DIR} && docker compose pull && docker compose up -d", timeout=600)
# Reload Caddy
run(c, "caddy validate --config /etc/caddy/Caddyfile && systemctl reload caddy")
wait_umami(c)
wait_ntfy(c)
# ntfy users + token (non-interactive)
# Ensure auth db dir writable
run(
c,
f"""cd {NTFY_DIR}
# create admin if missing
if ! docker compose exec -T ntfy ntfy user list 2>/dev/null | grep -q '^admin'; then
docker compose exec -T ntfy ntfy user add admin --role=admin --password '{ntfy_admin_pass}' || \\
printf '%s\\n%s\\n' '{ntfy_admin_pass}' '{ntfy_admin_pass}' | docker compose exec -T ntfy ntfy user add admin --role=admin
fi
docker compose exec -T ntfy ntfy access admin server-alerts read-write || true
docker compose exec -T ntfy ntfy access admin deploy read-write || true
# token
TOKEN=$(docker compose exec -T ntfy ntfy token list admin 2>/dev/null | awk 'NR==2{{print $1}}')
if [ -z "$TOKEN" ] || [ "$TOKEN" = "TOKEN" ]; then
TOKEN=$(docker compose exec -T ntfy ntfy token add admin 2>/dev/null | tail -n1 | tr -d '\\r')
fi
echo "TOKEN=$TOKEN"
cat > {CONF_DIR}/ntfy.env <<EOF
NTFY_BASE_URL=https://ntfy.nomadro.com
NTFY_USER=admin
NTFY_PASSWORD={ntfy_admin_pass}
NTFY_TOKEN=$TOKEN
NTFY_TOPIC=server-alerts
EOF
chmod 600 {CONF_DIR}/ntfy.env {CONF_DIR}/umami.env
""",
timeout=120,
)
# Install notify + healthcheck
run(
c,
f"""
cp {CONF_DIR}/observability-src/notify-ntfy.sh /usr/local/bin/notify-ntfy
cp {CONF_DIR}/observability-src/healthcheck.sh /usr/local/bin/nomadro-healthcheck
chmod +x /usr/local/bin/notify-ntfy /usr/local/bin/nomadro-healthcheck
cat >/etc/cron.d/nomadro-health <<'EOF'
*/5 * * * * root /usr/local/bin/nomadro-healthcheck >/dev/null 2>&1
EOF
chmod 644 /etc/cron.d/nomadro-health
""",
)
# Wait for public HTTPS
for _ in range(30):
out = run(c, "curl -fsS -o /dev/null -w '%{http_code}' https://umami.nomadro.com/api/heartbeat || echo fail")
if "200" in out:
break
time.sleep(3)
# Create Umami websites + change default password
# Do this from the server to avoid local TLS/DNS quirks
sites_json = json.dumps({d: n for d, n in SITES})
remote_bootstrap = f"""
python3 - <<'PY'
import json, urllib.request, urllib.error, time, os
admin_pass = {umami_admin_pass!r}
base = "https://umami.nomadro.com"
sites = {sites_json}
def req(url, method="GET", data=None, token=None):
body = None if data is None else json.dumps(data).encode()
headers = {{"Content-Type":"application/json","Accept":"application/json"}}
if token:
headers["Authorization"] = f"Bearer {{token}}"
r = urllib.request.Request(url, data=body, headers=headers, method=method)
try:
with urllib.request.urlopen(r, timeout=30) as resp:
raw = resp.read().decode()
return resp.status, json.loads(raw) if raw else {{}}
except urllib.error.HTTPError as e:
raw = e.read().decode(errors="replace")
try:
return e.code, json.loads(raw) if raw else {{}}
except Exception:
return e.code, {{"raw": raw}}
# login: try custom password then default
code, login = req(base + "/api/auth/login", "POST", {{"username":"admin","password":admin_pass}})
if code != 200:
code, login = req(base + "/api/auth/login", "POST", {{"username":"admin","password":"umami"}})
if code != 200:
raise SystemExit(f"login failed {{code}} {{login}}")
token = login["token"]
uid = login["user"]["id"]
req(base + f"/api/users/{{uid}}", "POST", {{"password": admin_pass}}, token=token)
code, login = req(base + "/api/auth/login", "POST", {{"username":"admin","password":admin_pass}})
if code != 200:
raise SystemExit(f"re-login failed {{code}} {{login}}")
token = login["token"]
code, listing = req(base + "/api/websites", token=token)
existing = {{}}
items = listing if isinstance(listing, list) else (listing.get("data") or listing.get("websites") or [])
for w in items:
if isinstance(w, dict) and w.get("domain"):
existing[w["domain"]] = w["id"]
result = {{}}
for domain, name in sites.items():
if domain in existing:
result[domain] = existing[domain]
continue
code, created = req(base + "/api/websites", "POST", {{"name": name, "domain": domain}}, token=token)
if code not in (200, 201) or "id" not in created:
raise SystemExit(f"create {{domain}} failed {{code}} {{created}}")
result[domain] = created["id"]
with open("{CONF_DIR}/umami-sites.env", "w") as f:
for d, wid in result.items():
key = d.replace(".", "_").replace("-", "_").upper()
f.write(f"UMAMI_ID_{{key}}={{wid}}\\n")
f.write("UMAMI_SCRIPT=https://umami.nomadro.com/script.js\\n")
print(json.dumps(result, indent=2))
PY
"""
out = run(c, remote_bootstrap, timeout=120)
# parse last json object from output
site_ids = {}
try:
start = out.rfind("{")
end = out.rfind("}")
if start >= 0 and end > start:
site_ids = json.loads(out[start : end + 1])
except json.JSONDecodeError:
pass
if not site_ids:
# read file from server
out2 = run(c, f"cat {CONF_DIR}/umami-sites.env")
for line in out2.splitlines():
if line.startswith("UMAMI_ID_"):
# reverse map roughly
pass
# Read site ids properly
out_env = run(c, f"cat {CONF_DIR}/umami-sites.env")
env_map = {}
for line in out_env.splitlines():
if "=" in line and line.startswith("UMAMI_ID_"):
k, v = line.split("=", 1)
env_map[k] = v.strip()
def id_for(domain: str) -> str:
key = "UMAMI_ID_" + domain.replace(".", "_").replace("-", "_").upper()
if key in env_map:
return env_map[key]
if domain in site_ids:
return site_ids[domain]
raise KeyError(domain)
# Inject WordPress mu-plugin tracker (from repo template + real website id)
wid_nomadro = id_for("nomadro.com")
mu_text = (REPO / "deploy" / "wordpress" / "mu-plugins" / "nomadro-adsense.php").read_text(
encoding="utf-8"
).replace("REPLACE_UMAMI_WEBSITE_ID", wid_nomadro)
with sftp.file("/tmp/nomadro-adsense.php", "w") as f:
f.write(mu_text)
run(
c,
"""
docker compose -f /var/www/nomadro-wp/docker-compose.yml cp /tmp/nomadro-adsense.php wordpress:/var/www/html/wp-content/mu-plugins/nomadro-adsense.php
docker compose -f /var/www/nomadro-wp/docker-compose.yml exec -T wordpress chown www-data:www-data /var/www/html/wp-content/mu-plugins/nomadro-adsense.php
mkdir -p /var/www/nomadro-wp/mu-plugins
cp /tmp/nomadro-adsense.php /var/www/nomadro-wp/mu-plugins/nomadro-adsense.php
""",
)
run(
c,
f"""
touch /var/www/nomadro.com/.env.local
grep -q '^NEXT_PUBLIC_UMAMI_WEBSITE_ID=' /var/www/nomadro.com/.env.local \\
&& sed -i 's/^NEXT_PUBLIC_UMAMI_WEBSITE_ID=.*/NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid_nomadro}/' /var/www/nomadro.com/.env.local \\
|| echo 'NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid_nomadro}' >> /var/www/nomadro.com/.env.local
""",
)
# Inject into Next layouts + rebuild
for layout, domain, restart_cmd, pkg in NEXT_LAYOUTS:
try:
wid = id_for(domain if domain != "nomadro.com" else "nomadro.com")
# portal uses same as nomadro for main app layout
if "nomadro.com/app/layout" in layout:
wid = id_for("nomadro.com")
except KeyError as e:
print("skip missing id", e)
continue
run(
c,
f"""
python3 - <<'PY'
from pathlib import Path
import re
path = Path({layout!r})
wid = {wid!r}
text = path.read_text(encoding='utf-8')
line = f' <script defer src="https://umami.nomadro.com/script.js" data-website-id="{{wid}}"></script>'
if "umami.nomadro.com/script.js" in text:
text = re.sub(
r'<script defer src="https://umami\\.nomadro\\.com/script\\.js" data-website-id="[^"]*"></script>',
line.strip(),
text,
)
else:
if "<head>" in text:
text = text.replace("<head>", "<head>\\n" + line, 1)
elif "<body" in text:
# App Router: put script as sibling before body content via head in html
if "<html" in text and "</html>" in text and "<head" not in text:
text = re.sub(r"(<html[^>]*>)", r"\\1\\n <head>\\n" + line + "\\n </head>", text, count=1)
else:
text = text.replace("<body", line + "\\n <body", 1)
else:
text += "\\n" + line + "\\n"
path.write_text(text, encoding='utf-8')
print("patched", path)
PY
""",
)
# Sync nomadro app Umami component + layout before rebuild
try:
sftp.put(
str(REPO / "components" / "umami-analytics.tsx"),
"/var/www/nomadro.com/components/umami-analytics.tsx",
)
sftp.put(str(REPO / "app" / "layout.tsx"), "/var/www/nomadro.com/app/layout.tsx")
except Exception as ex:
print("nomadro file sync warn:", ex)
# Rebuild sites that need it (sequential; can be slow)
rebuilds = [
("cd /var/www/nomadro.com && pnpm build && systemctl restart nomadro", 900),
("cd /opt/douyinWEB/frontend && npm run build && systemctl restart douyinweb-web", 900),
(
"cd /opt/tiktokWEB/frontend && npm run build && "
"cp -a .next/static .next/standalone/.next/static 2>/dev/null || true; "
"systemctl restart tiktok-promo-frontend",
900,
),
(
"cd /opt/wauto/web && npm run build && "
"cp -a public .next/standalone/ 2>/dev/null || true; "
"cp -a .next/static .next/standalone/.next/static 2>/dev/null || true; "
"systemctl restart hackrobot-web",
900,
),
]
for cmd, t in rebuilds:
try:
run(c, cmd, timeout=t)
except Exception as ex:
print("rebuild warn:", cmd, ex)
# Also sync local WP mu-plugin in repo and local layout
# Test notify
run(c, "/usr/local/bin/notify-ntfy 'Observability online' 'Umami + ntfy deployed on $(hostname)' high 'tada,rocket' || true")
# Verify
run(
c,
"""
echo '=== umami heartbeat ==='
curl -fsS https://umami.nomadro.com/api/heartbeat; echo
echo '=== ntfy health ==='
curl -fsS https://ntfy.nomadro.com/v1/health; echo
echo '=== credentials ==='
ls -la /etc/nomadro/
echo '=== containers ==='
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | grep -E 'umami|ntfy|NAMES'
""",
)
sftp.close()
c.close()
print("\nDONE. Logins in /etc/nomadro/umami.env and /etc/nomadro/ntfy.env on the server.")
if __name__ == "__main__":
main()