eric/gitlab-instance-0a899031_nomadro
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Umami/ntfy 可观测性栈与 WordPress AdSense 部署

Browse Source
This commit is contained in:
eric
2026-08-15 23:49:19 -05:00
parent ba0b1058b0
commit 08f484780d
61 changed files with 2723 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
deploy/observability/README.md Normal file
View File

@@ -0,0 +1,44 @@
# Nomadro observability (Umami analytics + ntfy notifications)
## DNS
- `umami.nomadro.com` → server
- `ntfy.nomadro.com` → server
## URLs
- Analytics: https://umami.nomadro.com
- Notifications: https://ntfy.nomadro.com
## Credentials (server only)
```bash
cat /etc/nomadro/umami.env # admin login
cat /etc/nomadro/ntfy.env # publish token + topic
cat /etc/nomadro/umami-sites.env
```
## Tracked sites
| Domain | Surface |
| --- | --- |
| nomadro.com | WordPress + Next (zh) |
| portal.nomadro.com | registered in Umami |
| dylive.nomadro.com | DouyinWEB |
| nomadro.live | TikTok Promo |
| wx.nomadro.com | HackRobot |
| x.nomadro.com | XAuto |
## Notifications
```bash
notify-ntfy "Title" "message" high "rotating_light" server-alerts
```
Health checks run every 5 minutes via `/etc/cron.d/nomadro-health` and push to topic `server-alerts`.
## Redeploy
```bash
python deploy/observability/finish_deploy.py
```

29
deploy/observability/_git_inventory.py Normal file
View File

@@ -0,0 +1,29 @@
#!/usr/bin/env python3
"""Inventory git repos on server with dirty status."""
import paramiko
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect("107.173.30.245", username="root", password="Xiao4669805", timeout=20, allow_agent=False, look_for_keys=False)
cmd = r"""
set +e
for d in /var/www/nomadro.com /opt/douyinWEB /opt/tiktokWEB /opt/wauto /opt/xauto /opt/umami /opt/ntfy /var/www/nomadro-wp /var/www/nomadro-router; do
echo "======== $d"
if [ -d "$d/.git" ]; then
cd "$d"
git remote -v 2>/dev/null | head -3
git status -sb 2>/dev/null
git rev-parse --abbrev-ref HEAD 2>/dev/null
echo "dirty files:" $(git status --porcelain 2>/dev/null | wc -l)
else
echo "NO_GIT"
ls -la "$d" 2>/dev/null | head -8
fi
done
# also find other git repos under /opt /var/www
find /opt /var/www -maxdepth 3 -type d -name .git 2>/dev/null | sed 's|/.git||'
"""
_i, o, e = c.exec_command(cmd, timeout=60)
print((o.read() + e.read()).decode())
c.close()

337
deploy/observability/finish_deploy.py Normal file
View File

@@ -0,0 +1,337 @@
#!/usr/bin/env python3
"""Finish ntfy auth, Umami sites, tracker injection, rebuilds."""
from __future__ import annotations
import json
import os
import secrets
import string
import time
from pathlib import Path
import paramiko
HOST = "107.173.30.245"
PASS = os.environ.get("NOMADRO_SSH_PASS", "Xiao4669805")
REPO = Path(__file__).resolve().parents[2]
CONF = "/etc/nomadro"
NTFY_DIR = "/opt/ntfy"
SITES = {
"nomadro.com": "Nomadro (blog + app)",
"portal.nomadro.com": "Nomadro portal",
"dylive.nomadro.com": "DouyinWEB / dylive",
"nomadro.live": "TikTok Promo / nomadro.live",
"wx.nomadro.com": "HackRobot / wx",
"x.nomadro.com": "XAuto / x",
}
NEXT_LAYOUTS = [
("/var/www/nomadro.com/app/layout.tsx", "nomadro.com"),
("/opt/douyinWEB/frontend/src/app/layout.tsx", "dylive.nomadro.com"),
("/opt/tiktokWEB/frontend/src/app/layout.tsx", "nomadro.live"),
("/opt/wauto/web/app/layout.tsx", "wx.nomadro.com"),
]
def gen(n=20):
return "".join(secrets.choice(string.ascii_letters + string.digits) for _ in range(n))
def run(c, cmd, timeout=300):
print("$", cmd[:140])
_i, o, e = c.exec_command(cmd, timeout=timeout)
# channel timeout
chan = _i.channel if hasattr(_i, "channel") else o.channel
chan.settimeout(timeout)
out = (o.read() + e.read()).decode("utf-8", errors="replace")
if out.strip():
print(out[-3000:] if len(out) > 3000 else out)
return out
def main():
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(HOST, username="root", password=PASS, timeout=30, allow_agent=False, look_for_keys=False)
sftp = c.open_sftp()
# Load or create umami admin password
try:
with sftp.file(f"{CONF}/umami.env", "r") as f:
umami_env = f.read().decode()
except OSError:
umami_env = ""
umami_pass = None
for line in umami_env.splitlines():
if line.startswith("UMAMI_PASSWORD="):
umami_pass = line.split("=", 1)[1].strip()
if not umami_pass:
umami_pass = gen()
with sftp.file(f"{CONF}/umami.env", "w") as f:
f.write(
f"UMAMI_URL=https://umami.nomadro.com\nUMAMI_USER=admin\nUMAMI_PASSWORD={umami_pass}\n"
)
ntfy_pass = gen()
# ntfy user + token — use expect-like via script file to avoid hang
run(
c,
f"""
set -e
cd {NTFY_DIR}
# Remove incomplete user if any
docker compose exec -T ntfy ntfy user del admin 2>/dev/null || true
# Non-interactive add: ntfy reads password from NTFY_PASSWORD env in recent versions
docker compose exec -T -e NTFY_PASSWORD='{ntfy_pass}' ntfy ntfy user add --role=admin admin 2>/tmp/ntfy_user_err || true
if ! docker compose exec -T ntfy ntfy user list 2>/dev/null | grep -q admin; then
# fallback: hash + config — use `ntfy user hash` then add with password hash if available
HASH=$(docker compose exec -T ntfy ntfy hash '{ntfy_pass}' 2>/dev/null | tr -d '\\r' || true)
echo "hash=$HASH"
# try password flag variants
docker compose exec -T ntfy ntfy user add admin --role=admin --password '{ntfy_pass}' 2>/tmp/ntfy_err2 || true
fi
docker compose exec -T ntfy ntfy user list || true
docker compose exec -T ntfy ntfy access admin server-alerts rw || docker compose exec -T ntfy ntfy access admin server-alerts read-write || true
docker compose exec -T ntfy ntfy access admin deploy rw || docker compose exec -T ntfy ntfy access admin deploy read-write || true
# Create token
TOKEN_OUT=$(docker compose exec -T ntfy ntfy token add admin 2>&1 | tr -d '\\r')
echo "TOKEN_OUT=$TOKEN_OUT"
TOKEN=$(echo "$TOKEN_OUT" | grep -Eo 'tk_[A-Za-z0-9]+' | tail -1)
if [ -z "$TOKEN" ]; then
TOKEN=$(echo "$TOKEN_OUT" | awk 'NF{{print $NF}}' | tail -1)
fi
echo "TOKEN=$TOKEN"
cat > {CONF}/ntfy.env <<EOF
NTFY_BASE_URL=https://ntfy.nomadro.com
NTFY_USER=admin
NTFY_PASSWORD={ntfy_pass}
NTFY_TOKEN=$TOKEN
NTFY_TOPIC=server-alerts
EOF
chmod 600 {CONF}/ntfy.env {CONF}/umami.env
cat {CONF}/ntfy.env
""",
timeout=90,
)
# Install helpers
run(
c,
f"""
cp {CONF}/observability-src/notify-ntfy.sh /usr/local/bin/notify-ntfy
cp {CONF}/observability-src/healthcheck.sh /usr/local/bin/nomadro-healthcheck
chmod +x /usr/local/bin/notify-ntfy /usr/local/bin/nomadro-healthcheck
cat >/etc/cron.d/nomadro-health <<'EOF'
*/5 * * * * root /usr/local/bin/nomadro-healthcheck >/dev/null 2>&1
EOF
""",
)
# Wait public umami
for _ in range(40):
out = run(c, "curl -fsS -o /dev/null -w '%{http_code}' https://umami.nomadro.com/api/heartbeat || echo x")
if "200" in out:
break
time.sleep(3)
sites_json = json.dumps(SITES)
out = run(
c,
f"""
python3 - <<'PY'
import json, urllib.request, urllib.error
admin_pass = {umami_pass!r}
base = "https://umami.nomadro.com"
sites = json.loads({sites_json!r})
def req(url, method="GET", data=None, token=None):
body = None if data is None else json.dumps(data).encode()
headers = {{"Content-Type":"application/json","Accept":"application/json"}}
if token:
headers["Authorization"] = f"Bearer {{token}}"
r = urllib.request.Request(url, data=body, headers=headers, method=method)
try:
with urllib.request.urlopen(r, timeout=45) as resp:
raw = resp.read().decode()
return resp.status, json.loads(raw) if raw else {{}}
except urllib.error.HTTPError as e:
raw = e.read().decode(errors="replace")
try:
return e.code, json.loads(raw) if raw else {{}}
except Exception:
return e.code, {{"raw": raw}}
code, login = req(base+"/api/auth/login","POST",{{"username":"admin","password":admin_pass}})
if code != 200:
code, login = req(base+"/api/auth/login","POST",{{"username":"admin","password":"umami"}})
if code != 200:
raise SystemExit(f"login failed {{code}} {{login}}")
token = login["token"]; uid = login["user"]["id"]
code2, upd = req(base+f"/api/users/{{uid}}","POST",{{"password":admin_pass}}, token=token)
print("password update", code2, upd)
code, login = req(base+"/api/auth/login","POST",{{"username":"admin","password":admin_pass}})
if code != 200:
raise SystemExit(f"re-login failed {{code}} {{login}}")
token = login["token"]
code, listing = req(base+"/api/websites", token=token)
items = listing if isinstance(listing, list) else (listing.get("data") or listing.get("websites") or [])
existing = {{w["domain"]: w["id"] for w in items if isinstance(w, dict) and w.get("domain")}}
result = {{}}
for domain, name in sites.items():
if domain in existing:
result[domain] = existing[domain]
continue
code, created = req(base+"/api/websites","POST",{{"name":name,"domain":domain}}, token=token)
print("create", domain, code, created)
if code not in (200,201) or "id" not in created:
raise SystemExit(f"create failed {{domain}}")
result[domain] = created["id"]
with open("{CONF}/umami-sites.env","w") as f:
for d, wid in result.items():
key = d.replace(".","_").replace("-","_").upper()
f.write(f"UMAMI_ID_{{key}}={{wid}}\\n")
f.write("UMAMI_SCRIPT=https://umami.nomadro.com/script.js\\n")
print("SITE_IDS="+json.dumps(result))
PY
""",
timeout=120,
)
env_map = {}
for line in run(c, f"cat {CONF}/umami-sites.env").splitlines():
if line.startswith("UMAMI_ID_") and "=" in line:
k, v = line.split("=", 1)
env_map[k] = v.strip()
def id_for(domain: str) -> str:
key = "UMAMI_ID_" + domain.replace(".", "_").replace("-", "_").upper()
return env_map[key]
wid_nomadro = id_for("nomadro.com")
mu = (REPO / "deploy/wordpress/mu-plugins/nomadro-adsense.php").read_text(encoding="utf-8")
# Only replace the const assignment so comparison/placeholder logic stays intact
mu = mu.replace(
"const NOMADRO_UMAMI_WEBSITE_ID = 'REPLACE_UMAMI_WEBSITE_ID';",
f"const NOMADRO_UMAMI_WEBSITE_ID = '{wid_nomadro}';",
1,
)
with sftp.file("/tmp/nomadro-adsense.php", "w") as f:
f.write(mu)
run(
c,
"""
docker compose -f /var/www/nomadro-wp/docker-compose.yml cp /tmp/nomadro-adsense.php wordpress:/var/www/html/wp-content/mu-plugins/nomadro-adsense.php
docker compose -f /var/www/nomadro-wp/docker-compose.yml exec -T wordpress chown www-data:www-data /var/www/html/wp-content/mu-plugins/nomadro-adsense.php
mkdir -p /var/www/nomadro-wp/mu-plugins && cp /tmp/nomadro-adsense.php /var/www/nomadro-wp/mu-plugins/nomadro-adsense.php
""",
)
run(
c,
f"""
touch /var/www/nomadro.com/.env.local
grep -q '^NEXT_PUBLIC_UMAMI_WEBSITE_ID=' /var/www/nomadro.com/.env.local \\
&& sed -i 's/^NEXT_PUBLIC_UMAMI_WEBSITE_ID=.*/NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid_nomadro}/' /var/www/nomadro.com/.env.local \\
|| echo 'NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid_nomadro}' >> /var/www/nomadro.com/.env.local
""",
)
sftp.put(str(REPO / "components/umami-analytics.tsx"), "/var/www/nomadro.com/components/umami-analytics.tsx")
sftp.put(str(REPO / "app/layout.tsx"), "/var/www/nomadro.com/app/layout.tsx")
for layout, domain in NEXT_LAYOUTS:
wid = id_for(domain)
run(
c,
f"""
python3 - <<'PY'
from pathlib import Path
import re
path = Path({layout!r})
wid = {wid!r}
text = path.read_text(encoding='utf-8')
line = f'<script defer src="https://umami.nomadro.com/script.js" data-website-id="{{wid}}"></script>'
if "umami.nomadro.com/script.js" in text:
text = re.sub(r'<script defer src="https://umami\\.nomadro\\.com/script\\.js" data-website-id="[^"]*"></script>', line, text)
else:
if "<html" in text and "<head" not in text:
text = re.sub(r'(<html[^>]*>)', r'\\1\\n <head>\\n '+line+r'\\n </head>', text, count=1)
elif "<head>" in text:
text = text.replace("<head>", "<head>\\n "+line, 1)
elif "<body" in text:
text = text.replace("<body", " "+line+"\\n <body", 1)
else:
text += "\\n"+line+"\\n"
path.write_text(text, encoding='utf-8')
print('patched', path)
PY
""",
)
# Also write env files for other next apps
for path, domain in [
("/opt/douyinWEB/frontend/.env.local", "dylive.nomadro.com"),
("/opt/tiktokWEB/frontend/.env.local", "nomadro.live"),
("/opt/wauto/web/.env.local", "wx.nomadro.com"),
]:
wid = id_for(domain)
run(
c,
f"""
touch {path}
grep -q '^NEXT_PUBLIC_UMAMI_WEBSITE_ID=' {path} \\
&& sed -i 's/^NEXT_PUBLIC_UMAMI_WEBSITE_ID=.*/NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid}/' {path} \\
|| echo 'NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid}' >> {path}
""",
)
rebuilds = [
("cd /var/www/nomadro.com && pnpm build && systemctl restart nomadro", 900),
("cd /opt/douyinWEB/frontend && npm run build && systemctl restart douyinweb-web", 900),
(
"cd /opt/tiktokWEB/frontend && npm run build && "
"mkdir -p .next/standalone/.next && cp -a .next/static .next/standalone/.next/static 2>/dev/null || true; "
"systemctl restart tiktok-promo-frontend",
900,
),
(
"cd /opt/wauto/web && npm run build && "
"mkdir -p .next/standalone/.next && cp -a public .next/standalone/ 2>/dev/null || true; "
"cp -a .next/static .next/standalone/.next/static 2>/dev/null || true; "
"systemctl restart hackrobot-web",
900,
),
]
for cmd, t in rebuilds:
try:
run(c, cmd, timeout=t)
except Exception as ex:
print("rebuild warn", ex)
# Test notify (may fail if token empty)
run(c, "/usr/local/bin/notify-ntfy 'Observability online' 'Umami + ntfy ready' high 'tada' || true")
run(
c,
"""
echo '=== verify ==='
curl -fsS https://umami.nomadro.com/api/heartbeat; echo
curl -fsS https://ntfy.nomadro.com/v1/health; echo
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | grep -E 'umami|ntfy|NAMES'
echo '--- umami sites ---'; cat /etc/nomadro/umami-sites.env
echo '--- sample homepage umami ---'
curl -fsS -A 'Mozilla/5.0' https://nomadro.com/ | grep -o 'umami.nomadro.com/script.js' | head -3
curl -fsS -A 'Googlebot' https://nomadro.com/ | grep -o 'data-website-id="[^"]*"' | head -3
""",
)
sftp.close()
c.close()
print("DONE")
if __name__ == "__main__":
main()

39
deploy/observability/healthcheck.sh Normal file
View File

@@ -0,0 +1,39 @@
#!/bin/bash
# Healthcheck key public services; notify ntfy on failure transitions.
set -euo pipefail
STATE_DIR=/var/lib/nomadro-health
NOTIFY=/usr/local/bin/notify-ntfy
mkdir -p "$STATE_DIR"
check() {
local name="$1"
local cmd="$2"
local ok=0
if eval "$cmd" >/dev/null 2>&1; then
ok=1
fi
local state_file="$STATE_DIR/${name}.ok"
local prev=1
[[ -f "$state_file" ]] && prev=$(cat "$state_file")
echo "$ok" >"$state_file"
if [[ "$ok" -eq 0 && "$prev" -eq 1 ]]; then
"$NOTIFY" "Service down: $name" "Host $(hostname) — $name failed health check at $(date -u +%Y-%m-%dT%H:%M:%SZ)" high "rotating_light,server" || true
elif [[ "$ok" -eq 1 && "$prev" -eq 0 ]]; then
"$NOTIFY" "Service recovered: $name" "Host $(hostname) — $name is healthy again" default "white_check_mark,server" || true
fi
}
check caddy "systemctl is-active --quiet caddy"
check docker "systemctl is-active --quiet docker"
check nomadro "systemctl is-active --quiet nomadro"
check nomadro-router "systemctl is-active --quiet nomadro-router"
check wordpress "curl -fsS -o /dev/null http://127.0.0.1:8088/"
check umami "curl -fsS -o /dev/null http://127.0.0.1:3010/api/heartbeat"
check ntfy "curl -fsS http://127.0.0.1:2586/v1/health | grep -q healthy"
check dylive "curl -fsS -o /dev/null -H 'Host: dylive.nomadro.com' http://127.0.0.1:4015/"
check nomadro-live "curl -fsS -o /dev/null http://127.0.0.1:3015/"
check wx "curl -fsS -o /dev/null http://127.0.0.1:3000/"
check public-nomadro "curl -fsS -o /dev/null https://nomadro.com/"
check public-umami "curl -fsS -o /dev/null https://umami.nomadro.com/api/heartbeat"
check public-ntfy "curl -fsS https://ntfy.nomadro.com/v1/health | grep -q healthy"

29
deploy/observability/notify-ntfy.sh Normal file
View File

@@ -0,0 +1,29 @@
#!/bin/bash
# Publish a notification to local ntfy (https://ntfy.nomadro.com).
# Usage: notify-ntfy.sh "title" "message" [priority] [tags] [topic]
set -euo pipefail
CONF="${NOMADRO_NTFY_CONF:-/etc/nomadro/ntfy.env}"
# shellcheck disable=SC1090
[[ -f "$CONF" ]] && source "$CONF"
TITLE="${1:-Alert}"
MESSAGE="${2:-}"
PRIORITY="${3:-default}"
TAGS="${4:-warning}"
TOPIC="${5:-${NTFY_TOPIC:-server-alerts}}"
BASE_URL="${NTFY_BASE_URL:-https://ntfy.nomadro.com}"
TOKEN="${NTFY_TOKEN:-}"
if [[ -z "$TOKEN" ]]; then
echo "notify-ntfy: NTFY_TOKEN missing in $CONF" >&2
exit 1
fi
curl -fsS \
-H "Authorization: Bearer ${TOKEN}" \
-H "Title: ${TITLE}" \
-H "Priority: ${PRIORITY}" \
-H "Tags: ${TAGS}" \
-d "$MESSAGE" \
"${BASE_URL%/}/${TOPIC}" >/dev/null

544
deploy/observability/remote_deploy.py Normal file
View File

@@ -0,0 +1,544 @@
#!/usr/bin/env python3
"""
Deploy Umami + ntfy, configure Caddy, create track sites, inject trackers,
install notify/healthcheck helpers.
Secrets stay on the server under /etc/nomadro/ (never committed).
"""
from __future__ import annotations
import json
import os
import secrets
import string
import time
import urllib.error
import urllib.request
from pathlib import Path
import paramiko
HOST = "107.173.30.245"
USER = "root"
PASS = os.environ.get("NOMADRO_SSH_PASS", "Xiao4669805")
REPO = Path(__file__).resolve().parents[2]
UMAMI_DIR = "/opt/umami"
NTFY_DIR = "/opt/ntfy"
CONF_DIR = "/etc/nomadro"
SITES = [
("nomadro.com", "Nomadro (blog + app)"),
("portal.nomadro.com", "Nomadro portal"),
("dylive.nomadro.com", "DouyinWEB / dylive"),
("nomadro.live", "TikTok Promo / nomadro.live"),
("wx.nomadro.com", "HackRobot / wx"),
("x.nomadro.com", "XAuto / x"),
]
# layout paths on server for Next.js tracker injection
NEXT_LAYOUTS = [
("/var/www/nomadro.com/app/layout.tsx", "nomadro.com", "systemctl restart nomadro", "pnpm"),
("/opt/douyinWEB/frontend/src/app/layout.tsx", "dylive.nomadro.com", "systemctl restart douyinweb-web", "npm"),
("/opt/tiktokWEB/frontend/src/app/layout.tsx", "nomadro.live", "systemctl restart tiktok-promo-frontend", "npm"),
("/opt/wauto/web/app/layout.tsx", "wx.nomadro.com", "systemctl restart hackrobot-web", "npm"),
]
def gen_password(n: int = 24) -> str:
alphabet = string.ascii_letters + string.digits
return "".join(secrets.choice(alphabet) for _ in range(n))
def ssh_connect() -> paramiko.SSHClient:
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(HOST, username=USER, password=PASS, timeout=30, allow_agent=False, look_for_keys=False)
return c
def run(c: paramiko.SSHClient, cmd: str, timeout: int = 300) -> str:
print("$", cmd[:160])
_i, o, e = c.exec_command(cmd, timeout=timeout)
out = (o.read() + e.read()).decode("utf-8", errors="replace")
if out.strip():
print(out[-2500:] if len(out) > 2500 else out)
return out
def put_dir(sftp: paramiko.SFTPClient, local: Path, remote: str) -> None:
try:
sftp.stat(remote)
except OSError:
# mkdir -p via parents
parts = remote.strip("/").split("/")
cur = ""
for p in parts:
cur += "/" + p
try:
sftp.stat(cur)
except OSError:
sftp.mkdir(cur)
for path in local.rglob("*"):
if path.is_dir():
continue
if path.name.startswith(".env") and path.name != ".env.example":
continue
rel = path.relative_to(local).as_posix()
rpath = f"{remote}/{rel}"
rdir = "/".join(rpath.split("/")[:-1])
try:
sftp.stat(rdir)
except OSError:
parts = rdir.strip("/").split("/")
cur = ""
for p in parts:
cur += "/" + p
try:
sftp.stat(cur)
except OSError:
sftp.mkdir(cur)
sftp.put(str(path), rpath)
print("put", rpath)
def http_json(url: str, method: str = "GET", data: dict | None = None, token: str | None = None, timeout: int = 30):
body = None if data is None else json.dumps(data).encode()
headers = {"Content-Type": "application/json", "Accept": "application/json"}
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, data=body, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
raw = r.read().decode()
return r.status, json.loads(raw) if raw else {}
except urllib.error.HTTPError as e:
raw = e.read().decode(errors="replace")
try:
parsed = json.loads(raw) if raw else {}
except json.JSONDecodeError:
parsed = {"raw": raw}
return e.code, parsed
def wait_umami(c: paramiko.SSHClient, tries: int = 60) -> None:
for i in range(tries):
out = run(c, "curl -fsS -o /dev/null -w '%{http_code}' http://127.0.0.1:3010/api/heartbeat || echo fail")
if "200" in out:
print("umami ready")
return
time.sleep(3)
raise RuntimeError("Umami did not become healthy")
def wait_ntfy(c: paramiko.SSHClient, tries: int = 40) -> None:
for i in range(tries):
out = run(c, "curl -fsS http://127.0.0.1:2586/v1/health || true")
if "healthy" in out:
print("ntfy ready")
return
time.sleep(2)
raise RuntimeError("ntfy did not become healthy")
def ensure_umami_sites(admin_user: str, admin_pass: str) -> dict[str, str]:
"""Create websites; return domain -> websiteId. Prefer public HTTPS once Caddy is up."""
base = "https://umami.nomadro.com"
# fallback local via SSH tunnel not needed if caddy works; try public first
code, login = http_json(
f"{base}/api/auth/login",
"POST",
{"username": admin_user, "password": admin_pass},
)
if code != 200 or "token" not in login:
# try default then change
code, login = http_json(
f"{base}/api/auth/login",
"POST",
{"username": "admin", "password": "umami"},
)
if code != 200:
raise RuntimeError(f"Umami login failed: {code} {login}")
token = login["token"]
uid = login["user"]["id"]
http_json(f"{base}/api/users/{uid}", "POST", {"password": admin_pass}, token=token)
# re-login
code, login = http_json(
f"{base}/api/auth/login",
"POST",
{"username": "admin", "password": admin_pass},
)
if code != 200:
raise RuntimeError(f"Umami re-login failed: {code} {login}")
token = login["token"]
code, listing = http_json(f"{base}/api/websites", token=token)
existing = {}
if code == 200:
items = listing if isinstance(listing, list) else listing.get("data") or listing.get("websites") or []
for w in items:
if isinstance(w, dict) and w.get("domain"):
existing[w["domain"]] = w["id"]
result = {}
for domain, name in SITES:
if domain in existing:
result[domain] = existing[domain]
print("umami site exists", domain, existing[domain])
continue
code, created = http_json(
f"{base}/api/websites",
"POST",
{"name": name, "domain": domain},
token=token,
)
if code not in (200, 201) or "id" not in created:
raise RuntimeError(f"create website {domain} failed: {code} {created}")
result[domain] = created["id"]
print("umami site created", domain, created["id"])
return result
UMAMI_MARKER = "umami.nomadro.com/script.js"
def inject_layout_snippet(content: str, website_id: str) -> str:
snippet_line = (
f' <script defer src="https://umami.nomadro.com/script.js" '
f'data-website-id="{website_id}"></script>'
)
if UMAMI_MARKER in content:
# replace existing website id
import re
return re.sub(
r'<script defer src="https://umami\.nomadro\.com/script\.js" data-website-id="[^"]*"></script>',
snippet_line.strip(),
content,
)
# Prefer inside <head> if present
if "<head>" in content:
return content.replace("<head>", f"<head>\n{snippet_line}", 1)
# App router often has no head — inject before </body> or after <html...>
if "<body" in content:
return content.replace("<body", f"{snippet_line}\n <body", 1)
if "</html>" in content:
return content.replace("</html>", f"{snippet_line}\n </html>", 1)
return content + "\n" + snippet_line + "\n"
def main() -> None:
umami_admin_pass = gen_password(20)
umami_db_pass = gen_password(24)
umami_app_secret = secrets.token_hex(32)
ntfy_admin_pass = gen_password(20)
c = ssh_connect()
sftp = c.open_sftp()
run(c, f"mkdir -p {UMAMI_DIR} {NTFY_DIR} {CONF_DIR} /usr/local/bin /var/lib/nomadro-health")
# Upload stacks
put_dir(sftp, REPO / "deploy" / "umami", UMAMI_DIR)
put_dir(sftp, REPO / "deploy" / "ntfy", NTFY_DIR)
put_dir(sftp, REPO / "deploy" / "observability", f"{CONF_DIR}/observability-src")
for name in ("umami.nomadro.com.caddy", "ntfy.nomadro.com.caddy"):
sftp.put(str(REPO / "deploy" / "caddy-sites" / name), f"/etc/caddy/sites/{name}")
print("caddy site", name)
# Write umami .env
umami_env = f"POSTGRES_PASSWORD={umami_db_pass}\nAPP_SECRET={umami_app_secret}\n"
with sftp.file(f"{UMAMI_DIR}/.env", "w") as f:
f.write(umami_env)
# Save admin credentials (may update password after first login)
with sftp.file(f"{CONF_DIR}/umami.env", "w") as f:
f.write(f"UMAMI_URL=https://umami.nomadro.com\nUMAMI_USER=admin\nUMAMI_PASSWORD={umami_admin_pass}\n")
# Start stacks
run(c, f"cd {UMAMI_DIR} && docker compose pull && docker compose up -d", timeout=600)
run(c, f"cd {NTFY_DIR} && docker compose pull && docker compose up -d", timeout=600)
# Reload Caddy
run(c, "caddy validate --config /etc/caddy/Caddyfile && systemctl reload caddy")
wait_umami(c)
wait_ntfy(c)
# ntfy users + token (non-interactive)
# Ensure auth db dir writable
run(
c,
f"""cd {NTFY_DIR}
# create admin if missing
if ! docker compose exec -T ntfy ntfy user list 2>/dev/null | grep -q '^admin'; then
docker compose exec -T ntfy ntfy user add admin --role=admin --password '{ntfy_admin_pass}' || \\
printf '%s\\n%s\\n' '{ntfy_admin_pass}' '{ntfy_admin_pass}' | docker compose exec -T ntfy ntfy user add admin --role=admin
fi
docker compose exec -T ntfy ntfy access admin server-alerts read-write || true
docker compose exec -T ntfy ntfy access admin deploy read-write || true
# token
TOKEN=$(docker compose exec -T ntfy ntfy token list admin 2>/dev/null | awk 'NR==2{{print $1}}')
if [ -z "$TOKEN" ] || [ "$TOKEN" = "TOKEN" ]; then
TOKEN=$(docker compose exec -T ntfy ntfy token add admin 2>/dev/null | tail -n1 | tr -d '\\r')
fi
echo "TOKEN=$TOKEN"
cat > {CONF_DIR}/ntfy.env <<EOF
NTFY_BASE_URL=https://ntfy.nomadro.com
NTFY_USER=admin
NTFY_PASSWORD={ntfy_admin_pass}
NTFY_TOKEN=$TOKEN
NTFY_TOPIC=server-alerts
EOF
chmod 600 {CONF_DIR}/ntfy.env {CONF_DIR}/umami.env
""",
timeout=120,
)
# Install notify + healthcheck
run(
c,
f"""
cp {CONF_DIR}/observability-src/notify-ntfy.sh /usr/local/bin/notify-ntfy
cp {CONF_DIR}/observability-src/healthcheck.sh /usr/local/bin/nomadro-healthcheck
chmod +x /usr/local/bin/notify-ntfy /usr/local/bin/nomadro-healthcheck
cat >/etc/cron.d/nomadro-health <<'EOF'
*/5 * * * * root /usr/local/bin/nomadro-healthcheck >/dev/null 2>&1
EOF
chmod 644 /etc/cron.d/nomadro-health
""",
)
# Wait for public HTTPS
for _ in range(30):
out = run(c, "curl -fsS -o /dev/null -w '%{http_code}' https://umami.nomadro.com/api/heartbeat || echo fail")
if "200" in out:
break
time.sleep(3)
# Create Umami websites + change default password
# Do this from the server to avoid local TLS/DNS quirks
sites_json = json.dumps({d: n for d, n in SITES})
remote_bootstrap = f"""
python3 - <<'PY'
import json, urllib.request, urllib.error, time, os
admin_pass = {umami_admin_pass!r}
base = "https://umami.nomadro.com"
sites = {sites_json}
def req(url, method="GET", data=None, token=None):
body = None if data is None else json.dumps(data).encode()
headers = {{"Content-Type":"application/json","Accept":"application/json"}}
if token:
headers["Authorization"] = f"Bearer {{token}}"
r = urllib.request.Request(url, data=body, headers=headers, method=method)
try:
with urllib.request.urlopen(r, timeout=30) as resp:
raw = resp.read().decode()
return resp.status, json.loads(raw) if raw else {{}}
except urllib.error.HTTPError as e:
raw = e.read().decode(errors="replace")
try:
return e.code, json.loads(raw) if raw else {{}}
except Exception:
return e.code, {{"raw": raw}}
# login: try custom password then default
code, login = req(base + "/api/auth/login", "POST", {{"username":"admin","password":admin_pass}})
if code != 200:
code, login = req(base + "/api/auth/login", "POST", {{"username":"admin","password":"umami"}})
if code != 200:
raise SystemExit(f"login failed {{code}} {{login}}")
token = login["token"]
uid = login["user"]["id"]
req(base + f"/api/users/{{uid}}", "POST", {{"password": admin_pass}}, token=token)
code, login = req(base + "/api/auth/login", "POST", {{"username":"admin","password":admin_pass}})
if code != 200:
raise SystemExit(f"re-login failed {{code}} {{login}}")
token = login["token"]
code, listing = req(base + "/api/websites", token=token)
existing = {{}}
items = listing if isinstance(listing, list) else (listing.get("data") or listing.get("websites") or [])
for w in items:
if isinstance(w, dict) and w.get("domain"):
existing[w["domain"]] = w["id"]
result = {{}}
for domain, name in sites.items():
if domain in existing:
result[domain] = existing[domain]
continue
code, created = req(base + "/api/websites", "POST", {{"name": name, "domain": domain}}, token=token)
if code not in (200, 201) or "id" not in created:
raise SystemExit(f"create {{domain}} failed {{code}} {{created}}")
result[domain] = created["id"]
with open("{CONF_DIR}/umami-sites.env", "w") as f:
for d, wid in result.items():
key = d.replace(".", "_").replace("-", "_").upper()
f.write(f"UMAMI_ID_{{key}}={{wid}}\\n")
f.write("UMAMI_SCRIPT=https://umami.nomadro.com/script.js\\n")
print(json.dumps(result, indent=2))
PY
"""
out = run(c, remote_bootstrap, timeout=120)
# parse last json object from output
site_ids = {}
try:
start = out.rfind("{")
end = out.rfind("}")
if start >= 0 and end > start:
site_ids = json.loads(out[start : end + 1])
except json.JSONDecodeError:
pass
if not site_ids:
# read file from server
out2 = run(c, f"cat {CONF_DIR}/umami-sites.env")
for line in out2.splitlines():
if line.startswith("UMAMI_ID_"):
# reverse map roughly
pass
# Read site ids properly
out_env = run(c, f"cat {CONF_DIR}/umami-sites.env")
env_map = {}
for line in out_env.splitlines():
if "=" in line and line.startswith("UMAMI_ID_"):
k, v = line.split("=", 1)
env_map[k] = v.strip()
def id_for(domain: str) -> str:
key = "UMAMI_ID_" + domain.replace(".", "_").replace("-", "_").upper()
if key in env_map:
return env_map[key]
if domain in site_ids:
return site_ids[domain]
raise KeyError(domain)
# Inject WordPress mu-plugin tracker (from repo template + real website id)
wid_nomadro = id_for("nomadro.com")
mu_text = (REPO / "deploy" / "wordpress" / "mu-plugins" / "nomadro-adsense.php").read_text(
encoding="utf-8"
).replace("REPLACE_UMAMI_WEBSITE_ID", wid_nomadro)
with sftp.file("/tmp/nomadro-adsense.php", "w") as f:
f.write(mu_text)
run(
c,
"""
docker compose -f /var/www/nomadro-wp/docker-compose.yml cp /tmp/nomadro-adsense.php wordpress:/var/www/html/wp-content/mu-plugins/nomadro-adsense.php
docker compose -f /var/www/nomadro-wp/docker-compose.yml exec -T wordpress chown www-data:www-data /var/www/html/wp-content/mu-plugins/nomadro-adsense.php
mkdir -p /var/www/nomadro-wp/mu-plugins
cp /tmp/nomadro-adsense.php /var/www/nomadro-wp/mu-plugins/nomadro-adsense.php
""",
)
run(
c,
f"""
touch /var/www/nomadro.com/.env.local
grep -q '^NEXT_PUBLIC_UMAMI_WEBSITE_ID=' /var/www/nomadro.com/.env.local \\
&& sed -i 's/^NEXT_PUBLIC_UMAMI_WEBSITE_ID=.*/NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid_nomadro}/' /var/www/nomadro.com/.env.local \\
|| echo 'NEXT_PUBLIC_UMAMI_WEBSITE_ID={wid_nomadro}' >> /var/www/nomadro.com/.env.local
""",
)
# Inject into Next layouts + rebuild
for layout, domain, restart_cmd, pkg in NEXT_LAYOUTS:
try:
wid = id_for(domain if domain != "nomadro.com" else "nomadro.com")
# portal uses same as nomadro for main app layout
if "nomadro.com/app/layout" in layout:
wid = id_for("nomadro.com")
except KeyError as e:
print("skip missing id", e)
continue
run(
c,
f"""
python3 - <<'PY'
from pathlib import Path
import re
path = Path({layout!r})
wid = {wid!r}
text = path.read_text(encoding='utf-8')
line = f' <script defer src="https://umami.nomadro.com/script.js" data-website-id="{{wid}}"></script>'
if "umami.nomadro.com/script.js" in text:
text = re.sub(
r'<script defer src="https://umami\\.nomadro\\.com/script\\.js" data-website-id="[^"]*"></script>',
line.strip(),
text,
)
else:
if "<head>" in text:
text = text.replace("<head>", "<head>\\n" + line, 1)
elif "<body" in text:
# App Router: put script as sibling before body content via head in html
if "<html" in text and "</html>" in text and "<head" not in text:
text = re.sub(r"(<html[^>]*>)", r"\\1\\n <head>\\n" + line + "\\n </head>", text, count=1)
else:
text = text.replace("<body", line + "\\n <body", 1)
else:
text += "\\n" + line + "\\n"
path.write_text(text, encoding='utf-8')
print("patched", path)
PY
""",
)
# Sync nomadro app Umami component + layout before rebuild
try:
sftp.put(
str(REPO / "components" / "umami-analytics.tsx"),
"/var/www/nomadro.com/components/umami-analytics.tsx",
)
sftp.put(str(REPO / "app" / "layout.tsx"), "/var/www/nomadro.com/app/layout.tsx")
except Exception as ex:
print("nomadro file sync warn:", ex)
# Rebuild sites that need it (sequential; can be slow)
rebuilds = [
("cd /var/www/nomadro.com && pnpm build && systemctl restart nomadro", 900),
("cd /opt/douyinWEB/frontend && npm run build && systemctl restart douyinweb-web", 900),
(
"cd /opt/tiktokWEB/frontend && npm run build && "
"cp -a .next/static .next/standalone/.next/static 2>/dev/null || true; "
"systemctl restart tiktok-promo-frontend",
900,
),
(
"cd /opt/wauto/web && npm run build && "
"cp -a public .next/standalone/ 2>/dev/null || true; "
"cp -a .next/static .next/standalone/.next/static 2>/dev/null || true; "
"systemctl restart hackrobot-web",
900,
),
]
for cmd, t in rebuilds:
try:
run(c, cmd, timeout=t)
except Exception as ex:
print("rebuild warn:", cmd, ex)
# Also sync local WP mu-plugin in repo and local layout
# Test notify
run(c, "/usr/local/bin/notify-ntfy 'Observability online' 'Umami + ntfy deployed on $(hostname)' high 'tada,rocket' || true")
# Verify
run(
c,
"""
echo '=== umami heartbeat ==='
curl -fsS https://umami.nomadro.com/api/heartbeat; echo
echo '=== ntfy health ==='
curl -fsS https://ntfy.nomadro.com/v1/health; echo
echo '=== credentials ==='
ls -la /etc/nomadro/
echo '=== containers ==='
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | grep -E 'umami|ntfy|NAMES'
""",
)
sftp.close()
c.close()
print("\nDONE. Logins in /etc/nomadro/umami.env and /etc/nomadro/ntfy.env on the server.")
if __name__ == "__main__":
main()

197
deploy/observability/sync_and_deploy_ops.py Normal file
View File

@@ -0,0 +1,197 @@
#!/usr/bin/env python3
"""Commit & push dirty repos (local + server) with eric credentials, deploy douyinWEB ops."""
from __future__ import annotations
import os
import subprocess
import time
from pathlib import Path
import paramiko
HOST = "107.173.30.245"
PASS = "Xiao4669805"
GIT_USER = "eric"
GIT_PASS = "Xiao4669805"
AUTH = f"{GIT_USER}:{GIT_PASS}@"
DOUYIN = Path(r"C:\Users\admin\Desktop\douyinWEB")
NOMADRO = Path(r"C:\project\gitlab-instance-0a899031_nomadro")
def local(cmd: list[str] | str, cwd: Path) -> str:
print(f"\n[{cwd.name}] $", cmd if isinstance(cmd, str) else " ".join(cmd))
r = subprocess.run(
cmd,
cwd=str(cwd),
shell=isinstance(cmd, str),
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
)
out = (r.stdout or "") + (r.stderr or "")
print(out[-3000:] if len(out) > 3000 else out)
if r.returncode != 0:
raise RuntimeError(f"cmd failed in {cwd}: {cmd}\n{out}")
return out
def ensure_auth_remote(cwd: Path) -> None:
url = local(["git", "remote", "get-url", "origin"], cwd).strip()
if "gitea.dsx2020.com" in url and AUTH not in url:
if url.startswith("https://"):
new = url.replace("https://", f"https://{AUTH}", 1)
# strip existing userinfo
if "@gitea" in url:
new = "https://" + AUTH + url.split("@", 1)[1]
local(["git", "remote", "set-url", "origin", new], cwd)
def commit_push_local(cwd: Path, paths: list[str], message: str) -> None:
ensure_auth_remote(cwd)
local(["git", "add", *paths], cwd)
st = local(["git", "status", "--porcelain"], cwd)
if not st.strip():
print(f"[{cwd.name}] nothing to commit")
# still try push in case ahead
local(["git", "push", "-u", "origin", "HEAD"], cwd)
return
local(["git", "commit", "-m", message], cwd)
local(["git", "push", "-u", "origin", "HEAD"], cwd)
def main() -> None:
# 1) Local douyinWEB — ops page
commit_push_local(
DOUYIN,
[
"frontend/src/app/ops/page.tsx",
"frontend/src/components/Shell.tsx",
"frontend/src/app/layout.tsx",
"backend/app/ops_panel.py",
"backend/app/notify.py",
"backend/app/main.py",
],
"feat(ops): Umami/ntfy/DNS 运维控制台页面与 API",
)
# 2) Local nomadro — observability stack
commit_push_local(
NOMADRO,
[
"app/layout.tsx",
"components/umami-analytics.tsx",
"deploy/Caddyfile",
"deploy/README.md",
"deploy/caddy-sites",
"deploy/umami",
"deploy/ntfy",
"deploy/observability",
"deploy/router",
"deploy/wordpress",
"deploy/deploy.sh",
"deploy/setup-server.sh",
],
"feat: Umami/ntfy 可观测性栈与 WordPress AdSense 部署",
)
# 3) Server repos: commit layout umami injections + sync + deploy douyin
c = paramiko.SSHClient()
c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
c.connect(HOST, username="root", password=PASS, timeout=30, allow_agent=False, look_for_keys=False)
def run(cmd: str, timeout: int = 600) -> str:
print("\n$ ", cmd[:160])
_i, o, e = c.exec_command(cmd, timeout=timeout)
out = (o.read() + e.read()).decode("utf-8", errors="replace")
print(out[-3500:] if len(out) > 3500 else out)
return out
# Ensure remotes have credentials
for repo in ("/opt/douyinWEB", "/opt/tiktokWEB", "/opt/wauto", "/var/www/nomadro.com"):
run(
f"""
cd {repo} || exit 0
url=$(git remote get-url origin 2>/dev/null || true)
[ -z "$url" ] && exit 0
case "$url" in
*{AUTH}*) ;;
https://*@gitea.dsx2020.com/*)
rest=$(echo "$url" | sed 's|https://[^@]*@|https://|')
git remote set-url origin "https://{AUTH}${{rest#https://}}"
;;
https://gitea.dsx2020.com/*)
git remote set-url origin "https://{AUTH}${{url#https://}}"
;;
esac
git remote -v | head -2
"""
)
# Pull douyin first then apply — better: push already done, pull on server
run(
"""
cd /opt/douyinWEB
git fetch origin
git stash push -u -m 'pre-ops-deploy' -- frontend/src/app/layout.tsx 2>/dev/null || true
git checkout test1
git pull --ff-only origin test1
""",
timeout=120,
)
# Commit other server-only dirty files (tiktok/wauto/nomadro layouts) if still dirty
run(
r"""
commit_if_dirty() {
local dir="$1"; local msg="$2"; shift 2
cd "$dir" || return 0
git add "$@" 2>/dev/null || true
if [ -n "$(git status --porcelain -- "$@" 2>/dev/null)" ]; then
git -c user.email=ops@nomadro.com -c user.name=ops commit -m "$msg" || true
git push origin HEAD || true
else
echo "$dir: clean for selected paths"
fi
}
commit_if_dirty /opt/tiktokWEB 'chore: inject Umami analytics tracker' frontend/src/app/layout.tsx
commit_if_dirty /opt/wauto 'chore: inject Umami analytics tracker' web/app/layout.tsx
commit_if_dirty /var/www/nomadro.com 'chore: Umami analytics component' app/layout.tsx components/umami-analytics.tsx
"""
)
# Deploy douyinWEB frontend + restart API
run(
"""
cd /opt/douyinWEB/frontend
# ensure umami env
grep -q '^NEXT_PUBLIC_UMAMI_WEBSITE_ID=' .env.local 2>/dev/null || echo 'NEXT_PUBLIC_UMAMI_WEBSITE_ID=f18d3484-be2e-4344-8882-b80c5b53f9cf' >> .env.local
npm run build
systemctl restart douyinweb-web
systemctl restart douyinweb-api
sleep 2
systemctl is-active douyinweb-web douyinweb-api
curl -fsS -o /dev/null -w 'ops_page:%{http_code}\\n' https://dylive.nomadro.com/ops || true
curl -fsS -o /dev/null -w 'ops_api:%{http_code}\\n' -H 'Cookie: ' https://dylive.nomadro.com/api/ops || true
""",
timeout=600,
)
# Sync nomadro.com from git if needed
run(
"""
cd /var/www/nomadro.com
git fetch origin
git pull --ff-only origin test || git pull --ff-only || true
# copy observability deploy assets already in repo; ensure router files synced if present
""",
timeout=120,
)
c.close()
print("\nALL DONE")
if __name__ == "__main__":
main()