vpnbot-xray-patches/scripts/pilot_and_promote.py
loop-uh 38c41f4e7c
All checks were successful
Follow official Xray dev releases / candidate (push) Successful in 19s
Test and build VPnBot Xray patches / test-and-build (push) Successful in 2m58s
Выпуск Xray: канарейку называет хаб, промоутер не держит своего узла
14.11.2026 узел tinl3 удалили из парка, а /etc/vpnbot-xray-release-promoter.env всё ещё называл его: каждый запуск падал на пропавшем known_hosts, ни одна сборка не могла быть доказана. Решение владельца 15.11.2026: канарейка — метка xray-canary в реестре хаба (vpnbot_node_manager 10879fb), промоутер спрашивает хаб на каждом запуске.

- hub_canary(): manage_node.py --db inventory xray-canary --apply; узел, адрес, пользователь, ключ и known_hosts берутся из ответа хаба. Отказ хаба (нет годного VLESS-узла) — PipelineError, следующий запуск спрашивает снова.
- canary_with_v3(): CPU канарейки читается по SSH (vpnbot_cpu_profile.py --field cpu_profile); узел не v3 промоутер возвращает хабу меткой no-xray-canary и спрашивает следующий, не больше MAX_CANARY_HANDOVERS раз.
- Поля канарейки и jump-маршрута сняты из env (пример env говорит, что узел называет хаб).

Тесты: HubCanaryTests — узел из хаба, env не называет узла, отказ хаба останавливает запуск, узел с базовым CPU возвращается хабу.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 12:02:29 -05:00

623 lines
23 KiB
Python
Executable file

#!/usr/bin/env python3
"""Install the newest candidate on one canary and promote it after proof."""
from __future__ import annotations
import dataclasses
import fcntl
import io
import json
import os
import re
import shlex
import stat
import subprocess
import sys
import tempfile
import time
import zipfile
from pathlib import Path
from typing import Any
import release_pipeline
@dataclasses.dataclass(frozen=True)
class Settings:
releases_url: str
token_file: Path
canary_node: str
canary_host: str
canary_port: int
canary_user: str
identity_file: Path
known_hosts_file: Path
updater_path: str
xray_path: str
config_dir: str
service_name: str
state_dir: Path
canary_script: Path
mptcp_check_script: Path
timeout_seconds: int
def env(name: str, default: str = "") -> str:
return str(os.environ.get(name, default)).strip()
#: The hub's registry CLI and store: the canary is the node the hub marks
#: (``vpnbot_node_manager/xray_canary.py``, owner decision 15.11.2026). Until
#: then the node was written into this unit's env by hand, and the deleted
#: tinl3 failed every run from 14.11 on its missing pin.
HUB_PYTHON = "/opt/vpnbot-node-manager/venv/bin/python"
HUB_MANAGE = "/opt/vpnbot-node-manager/scripts/manage_node.py"
HUB_INVENTORY = "/etc/vpnbot-node-manager/inventory.sqlite3"
def hub_canary() -> dict[str, Any]:
"""The canary's direct endpoint as the hub names it now; it moves the mark itself."""
command = [
env("VPNBOT_XRAY_HUB_PYTHON", HUB_PYTHON),
env("VPNBOT_XRAY_HUB_MANAGE", HUB_MANAGE),
"--db",
env("VPNBOT_XRAY_HUB_INVENTORY", HUB_INVENTORY),
"xray-canary",
"--apply",
]
completed = subprocess.run(command, capture_output=True, text=True, timeout=120, check=False)
if completed.returncode != 0:
raise release_pipeline.PipelineError(
f"the hub named no Xray canary (exit {completed.returncode}): {completed.stderr.strip()[-300:]}"
)
lines = completed.stdout.strip().splitlines()
try:
canary = json.loads(lines[-1])["canary"] if lines else None
except (ValueError, KeyError, TypeError):
canary = None
if not isinstance(canary, dict):
raise release_pipeline.PipelineError("the hub's canary answer is not readable")
return canary
#: The node's own CPU verdict (node-baseline ``shared/cpu_profile.py``): the
#: candidate pilot must prove the GOAMD64 v3 build, so a baseline CPU cannot be
#: the canary.
NODE_CPU_PROFILE = "/usr/local/lib/vpnbot-node/vpnbot_cpu_profile.py"
#: The hub's label that keeps a node out of the canary choice.
NO_XRAY_CANARY_LABEL = "no-xray-canary"
#: How many nodes one run may hand back to the hub before it gives up.
MAX_CANARY_HANDOVERS = 8
def hub_exclude(node_id: str) -> None:
"""Tell the hub this node cannot be the canary; it moves the mark on the next question."""
completed = subprocess.run(
[
env("VPNBOT_XRAY_HUB_PYTHON", HUB_PYTHON),
env("VPNBOT_XRAY_HUB_MANAGE", HUB_MANAGE),
"--db",
env("VPNBOT_XRAY_HUB_INVENTORY", HUB_INVENTORY),
"label",
"add",
"--node-id",
node_id,
"--label",
NO_XRAY_CANARY_LABEL,
"--apply",
],
capture_output=True,
text=True,
timeout=120,
check=False,
)
if completed.returncode != 0:
raise release_pipeline.PipelineError(
f"the hub refused to exclude {node_id} from the canary: {completed.stderr.strip()[-300:]}"
)
def canary_with_v3(settings: Settings) -> Settings:
"""The hub's canary, handed back to the hub while its CPU cannot run the v3 build."""
for _ in range(MAX_CANARY_HANDOVERS):
profile = require_remote_success(
remote_command(settings, ["python3", NODE_CPU_PROFILE, "--field", "cpu_profile"], timeout=30),
"reading canary CPU profile",
)
if profile == "v3":
return settings
print(f"Canary {settings.canary_node} has CPU profile {profile or '<none>'}: handed back to the hub")
hub_exclude(settings.canary_node)
settings = load_settings()
raise release_pipeline.PipelineError("the hub named no canary with a GOAMD64 v3 CPU")
def load_settings() -> Settings:
script_dir = Path(__file__).resolve().parent
canary = hub_canary()
settings = Settings(
releases_url=env(
"VPNBOT_XRAY_FORGEJO_RELEASES_URL",
"https://iianajhlkh.zapret.moe/api/v1/repos/zapretkvn/vpnbot-xray-patches/releases",
),
token_file=Path(env("VPNBOT_XRAY_FORGEJO_TOKEN_FILE")),
canary_node=str(canary.get("node_id") or ""),
canary_host=str(canary.get("host") or ""),
canary_port=int(canary.get("port") or 0),
canary_user=str(canary.get("user") or ""),
identity_file=Path(str(canary.get("identity_file") or "")),
known_hosts_file=Path(str(canary.get("known_hosts_file") or "")),
updater_path=env("VPNBOT_XRAY_CANARY_UPDATER", "/usr/local/bin/vpnbot-xray-core-updater"),
xray_path=env("VPNBOT_XRAY_CANARY_BINARY", "/opt/vpnbot/xray-core/bin/xray"),
config_dir=env("VPNBOT_XRAY_CANARY_CONFIG_DIR", "/opt/vpnbot/xray-core/config"),
service_name=env("VPNBOT_XRAY_CANARY_SERVICE", "vpnbot-xray.service"),
state_dir=Path(env("VPNBOT_XRAY_PROMOTER_STATE_DIR", "/var/lib/vpnbot-xray-release-promoter")),
canary_script=Path(
env("VPNBOT_XRAY_CANARY_SCRIPT", str(script_dir / "canary_active_revoke.py"))
),
mptcp_check_script=Path(
env("VPNBOT_XRAY_MPTCP_CHECK_SCRIPT", str(script_dir / "check_no_mptcp_listener.py"))
),
timeout_seconds=max(60, int(env("VPNBOT_XRAY_PROMOTER_TIMEOUT_SECONDS", "900"))),
)
if not settings.releases_url.startswith("https://"):
raise release_pipeline.PipelineError("Forgejo releases URL must use HTTPS")
for label, value in {
"canary node": settings.canary_node,
"canary host": settings.canary_host,
"canary user": settings.canary_user,
}.items():
if not value or not re.fullmatch(r"[A-Za-z0-9_.:-]+", value):
raise release_pipeline.PipelineError(f"invalid {label}")
if not 1 <= settings.canary_port <= 65535:
raise release_pipeline.PipelineError("invalid canary SSH port")
for label, path in {
"token file": settings.token_file,
"SSH identity": settings.identity_file,
"known_hosts": settings.known_hosts_file,
"canary script": settings.canary_script,
"MPTCP check script": settings.mptcp_check_script,
}.items():
if not path.is_file() or path.is_symlink():
raise release_pipeline.PipelineError(f"{label} is missing or unsafe: {path}")
token_mode = stat.S_IMODE(settings.token_file.stat().st_mode)
if token_mode & 0o077:
raise release_pipeline.PipelineError("Forgejo token file must not be group/world-readable")
for value in (
settings.updater_path,
settings.xray_path,
settings.config_dir,
settings.service_name,
):
if not re.fullmatch(r"[A-Za-z0-9_./@+-]+", value):
raise release_pipeline.PipelineError(f"unsafe remote setting: {value}")
return settings
def utc_now() -> str:
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def load_token(path: Path) -> str:
token = path.read_text(encoding="utf-8").strip()
if not token or any(character.isspace() for character in token):
raise release_pipeline.PipelineError("Forgejo token file is empty or malformed")
return token
def ssh_base(settings: Settings) -> list[str]:
return [
"ssh",
"-i",
str(settings.identity_file),
"-p",
str(settings.canary_port),
"-o",
"BatchMode=yes",
"-o",
"IdentitiesOnly=yes",
"-o",
"StrictHostKeyChecking=yes",
"-o",
f"UserKnownHostsFile={settings.known_hosts_file}",
"-o",
"ConnectTimeout=15",
f"{settings.canary_user}@{settings.canary_host}",
]
def remote_command(
settings: Settings,
args: list[str],
*,
stdin: bytes | None = None,
timeout: int | None = None,
) -> subprocess.CompletedProcess[bytes]:
remote = shlex.join(args)
return subprocess.run(
[*ssh_base(settings), remote],
input=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=timeout or settings.timeout_seconds,
)
def require_remote_success(result: subprocess.CompletedProcess[bytes], action: str) -> str:
stdout = result.stdout.decode("utf-8", errors="replace").strip()
stderr = result.stderr.decode("utf-8", errors="replace").strip()
if result.returncode != 0:
raise release_pipeline.PipelineError(
f"{action} failed with exit={result.returncode}: {stderr or stdout or '<no output>'}"
)
return stdout
def remote_version(settings: Settings) -> tuple[str, str]:
output = require_remote_success(
remote_command(settings, [settings.xray_path, "version"], timeout=30),
"reading canary Xray version",
)
first_line = output.splitlines()[0] if output else ""
match = re.search(r"\bXray\s+([0-9]+(?:\.[0-9]+){2}(?:-[A-Za-z0-9_.-]+)?)", first_line)
if not match:
raise release_pipeline.PipelineError(f"cannot parse canary Xray version: {first_line}")
return output, f"v{match.group(1)}"
def updater(settings: Settings, tag: str, channel: str) -> str:
if not (
release_pipeline.CANDIDATE_RE.fullmatch(tag)
or release_pipeline.PROVEN_RE.fullmatch(tag)
):
raise release_pipeline.PipelineError(f"unsafe updater target tag: {tag}")
cpu_profile = "v3" if channel == "candidate" else "auto"
result = remote_command(
settings,
[
"/usr/bin/env",
f"XRAY_CORE_RELEASE_CHANNEL={channel}",
f"XRAY_CORE_VERSION={tag}",
f"XRAY_CORE_CPU_PROFILE={cpu_profile}",
settings.updater_path,
"--force",
],
)
output = require_remote_success(result, f"installing {tag} on canary")
if channel == "candidate" and "cpu_profile=v3" not in output:
raise release_pipeline.PipelineError(
"candidate updater did not prove the required GOAMD64 v3 artifact"
)
return output
def write_state(path: Path, payload: dict[str, Any]) -> None:
release_pipeline.write_atomic(path, release_pipeline.canonical_json_bytes(payload), mode=0o600)
def state_path(settings: Settings, candidate_tag: str) -> Path:
if not release_pipeline.CANDIDATE_RE.fullmatch(candidate_tag):
raise release_pipeline.PipelineError("invalid candidate tag for state path")
return settings.state_dir / f"{candidate_tag}.json"
def select_candidate(
settings: Settings, token: str
) -> tuple[dict[str, Any], dict[str, Any], bytes] | None:
releases = release_pipeline.list_forgejo_releases(settings.releases_url, token=token)
known_tags = {str(item.get("tag_name") or "") for item in releases if not item.get("draft")}
for candidate in releases:
tag = str(candidate.get("tag_name") or "")
if (
candidate.get("draft")
or not candidate.get("prerelease")
or not release_pipeline.CANDIDATE_RE.fullmatch(tag)
):
continue
manifest = release_pipeline.read_release_manifest(candidate, token=token)
if manifest is None:
raise release_pipeline.PipelineError(f"candidate {tag} has no release manifest")
if manifest["release"]["candidate_tag"] != tag:
raise release_pipeline.PipelineError(f"candidate {tag} manifest tag mismatch")
proven_tag = str(manifest["release"]["proven_tag"])
if proven_tag in known_tags:
continue
manifest_asset = release_pipeline.release_assets(candidate)[release_pipeline.MANIFEST_NAME]
manifest_bytes = release_pipeline.request_bytes(
release_pipeline.asset_url(manifest_asset), token=token
)
return candidate, manifest, manifest_bytes
return None
def ensure_previous_is_proven(settings: Settings, token: str, tag: str) -> None:
release = release_pipeline.release_by_tag(settings.releases_url, tag, token=token)
if release is None or release.get("draft") or release.get("prerelease"):
raise release_pipeline.PipelineError(
f"canary current version {tag} is not an available proven rollback target"
)
def resolve_previous_proven_tag(
settings: Settings,
token: str,
version_statement: str,
) -> str:
if release_pipeline.CAPABILITY not in version_statement:
raise release_pipeline.PipelineError(
"canary current Xray lacks the required active-revoke capability"
)
exact = re.search(r"v[0-9]+(?:\.[0-9]+){2}-vpnbot\.[1-9][0-9]*", version_statement)
if exact:
tag = exact.group(0)
ensure_previous_is_proven(settings, token, tag)
return tag
numeric = re.search(r"\bXray\s+([0-9]+(?:\.[0-9]+){2})\b", version_statement)
if not numeric:
raise release_pipeline.PipelineError(
"cannot map the canary current Xray version to a proven rollback release"
)
prefix = f"v{numeric.group(1)}-vpnbot."
releases = release_pipeline.list_forgejo_releases(settings.releases_url, token=token)
for release in releases:
tag = str(release.get("tag_name") or "")
if (
not release.get("draft")
and not release.get("prerelease")
and release_pipeline.PROVEN_RE.fullmatch(tag)
and tag.startswith(prefix)
):
return tag
raise release_pipeline.PipelineError(
f"no proven rollback release matches installed Xray {numeric.group(1)}"
)
#: The archive whose binary the MPTCP check starts (the compatible amd64 build).
MPTCP_CHECK_ASSET = "Xray-linux-64.zip"
def assert_no_mptcp_listener(settings: Settings, payloads: dict[str, bytes]) -> None:
"""Refuse a candidate whose Xray listens with MPTCP, before any canary install.
The check (``check_no_mptcp_listener.py``) runs the binary in a network
namespace of its own, where MPTCP is on by kernel default whatever this
host set, and fails closed when it cannot judge.
"""
archive = payloads.get(MPTCP_CHECK_ASSET)
if archive is None:
raise release_pipeline.PipelineError(f"candidate has no {MPTCP_CHECK_ASSET} for the MPTCP check")
with tempfile.TemporaryDirectory(prefix="vpnbot-xray-mptcp-") as directory:
binary = Path(directory) / "xray"
with zipfile.ZipFile(io.BytesIO(archive)) as bundle:
binary.write_bytes(bundle.read("xray"))
binary.chmod(0o755)
result = subprocess.run(
[sys.executable, "-I", str(settings.mptcp_check_script), str(binary)],
capture_output=True,
text=True,
timeout=180,
check=False,
)
if result.returncode != 0:
detail = (result.stderr or result.stdout or "").strip().splitlines()[-1:] or ["no output"]
raise release_pipeline.PipelineError(f"candidate MPTCP check failed: {detail[0][:300]}")
def run_canary(settings: Settings, expected_proven_tag: str) -> dict[str, Any]:
script = settings.canary_script.read_bytes()
result = remote_command(
settings,
[
"/usr/bin/python3",
"-",
"--xray",
settings.xray_path,
"--config-dir",
settings.config_dir,
"--service",
settings.service_name,
"--expected-tag",
expected_proven_tag,
"--capability",
release_pipeline.CAPABILITY,
"--live-audit-capability",
release_pipeline.LIVE_USER_AUDIT_CAPABILITY,
"--cutoff-seconds",
"10",
],
stdin=script,
)
stdout = result.stdout.decode("utf-8", errors="replace").strip()
stderr = result.stderr.decode("utf-8", errors="replace").strip()
last_line = stdout.splitlines()[-1] if stdout else ""
try:
payload = json.loads(last_line)
except json.JSONDecodeError as exc:
raise release_pipeline.PipelineError(
f"canary returned non-JSON output: {stderr or stdout or '<no output>'}"
) from exc
if result.returncode != 0 or payload.get("result") != "passed":
raise release_pipeline.PipelineError(
f"active-revoke canary failed: {payload.get('error') or stderr or stdout}"
)
checks = payload.get("checks")
if not isinstance(checks, dict) or not checks or not all(value is True for value in checks.values()):
raise release_pipeline.PipelineError("canary returned incomplete checks")
return payload
def promote(
settings: Settings,
token: str,
candidate_tag: str,
proof_path: Path,
) -> None:
environment = os.environ.copy()
environment["FORGEJO_TOKEN"] = token
result = subprocess.run(
[
sys.executable,
str(Path(release_pipeline.__file__).resolve()),
"promote",
"--forgejo-releases-url",
settings.releases_url,
"--candidate-tag",
candidate_tag,
"--proof",
str(proof_path),
],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=settings.timeout_seconds,
env=environment,
)
if result.returncode != 0:
raise release_pipeline.PipelineError(
result.stderr.strip() or result.stdout.strip() or "candidate promotion failed"
)
print(result.stdout.strip())
def process_candidate(
settings: Settings,
token: str,
candidate: dict[str, Any],
manifest: dict[str, Any],
manifest_bytes: bytes,
) -> None:
candidate_tag = str(manifest["release"]["candidate_tag"])
proven_tag = str(manifest["release"]["proven_tag"])
state_file = state_path(settings, candidate_tag)
proof_file = settings.state_dir / f"{candidate_tag}.proof.json"
state: dict[str, Any] = {}
if state_file.exists():
loaded = release_pipeline.load_json(state_file)
if not isinstance(loaded, dict):
raise release_pipeline.PipelineError(f"invalid promoter state: {state_file}")
state = loaded
manifest_hash = release_pipeline.sha256_bytes(manifest_bytes)
if state and state.get("manifest_sha256") != manifest_hash:
raise release_pipeline.PipelineError("candidate manifest changed after the pilot started")
if state.get("phase") == "pilot_passed":
if not proof_file.is_file():
raise release_pipeline.PipelineError("pilot state exists without its proof file")
promote(settings, token, candidate_tag, proof_file)
state["phase"] = "promoted"
state["promoted_at"] = utc_now()
write_state(state_file, state)
return
_manifest, _manifest_bytes, payloads = release_pipeline.validated_candidate_payloads(candidate, token=token)
# Before the canary sees it: the candidate must not listen with MPTCP
# (CVE-2026-98432; Xray-core v26.10.31 turned MPTCP off by default).
assert_no_mptcp_listener(settings, payloads)
current_statement, _ = remote_version(settings)
previous_tag = str(
state.get("previous_proven_tag")
or resolve_previous_proven_tag(settings, token, current_statement)
)
ensure_previous_is_proven(settings, token, previous_tag)
state = {
"schema_version": 1,
"phase": "installing",
"candidate_tag": candidate_tag,
"proven_tag": proven_tag,
"manifest_sha256": manifest_hash,
"canary_node": settings.canary_node,
"previous_proven_tag": previous_tag,
"previous_version_statement": state.get("previous_version_statement") or current_statement,
"started_at": state.get("started_at") or utc_now(),
}
write_state(state_file, state)
try:
updater_output = updater(settings, candidate_tag, "candidate")
state["phase"] = "installed"
state["updater_output"] = updater_output[-2000:]
state["installed_at"] = utc_now()
write_state(state_file, state)
canary = run_canary(settings, proven_tag)
except Exception as pilot_error:
rollback_error = ""
try:
updater(settings, previous_tag, "stable")
except Exception as exc:
rollback_error = f"; rollback also failed: {exc}"
state["phase"] = "failed_rolled_back" if not rollback_error else "failed_rollback_failed"
state["failed_at"] = utc_now()
state["error"] = f"{pilot_error}{rollback_error}"
write_state(state_file, state)
raise release_pipeline.PipelineError(state["error"]) from pilot_error
installed_statement, _ = remote_version(settings)
checks = dict(canary["checks"])
checks["candidate_manifest_valid"] = True
checks["candidate_assets_valid"] = True
checks["candidate_installed_by_atomic_updater"] = True
checks["candidate_opens_no_mptcp_listener"] = True
proof = {
"schema_version": 1,
"result": "passed",
"candidate_tag": candidate_tag,
"proven_tag": proven_tag,
"manifest_sha256": manifest_hash,
"canary_node": settings.canary_node,
"canary_host": settings.canary_host,
"started_at": state["started_at"],
"passed_at": utc_now(),
"previous_version_statement": state["previous_version_statement"],
"installed_version_statement": installed_statement,
"checks": checks,
"active_revoke": canary.get("active_revoke"),
}
write_state(proof_file, proof)
state["phase"] = "pilot_passed"
state["passed_at"] = proof["passed_at"]
state["proof_file"] = str(proof_file)
write_state(state_file, state)
promote(settings, token, candidate_tag, proof_file)
state["phase"] = "promoted"
state["promoted_at"] = utc_now()
write_state(state_file, state)
def main() -> int:
try:
settings = load_settings()
settings.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
settings.state_dir.chmod(0o700)
lock_path = settings.state_dir / "promoter.lock"
with lock_path.open("w", encoding="utf-8") as lock:
fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
token = load_token(settings.token_file)
try:
selected = select_candidate(settings, token)
except release_pipeline.SourceUnavailableError as exc:
# Nothing was mutated yet: the next timer run reads again, and
# the release observer alone escalates a persistent outage.
print(f"Promoter run deferred: source_unavailable={exc.code}: {exc}")
return 0
if selected is None:
print("No unproven VPnBot Xray candidate is waiting")
return 0
candidate, manifest, manifest_bytes = selected
settings = canary_with_v3(settings)
process_candidate(settings, token, candidate, manifest, manifest_bytes)
return 0
except BlockingIOError:
print("Another VPnBot Xray promoter run is active", file=sys.stderr)
return 75
except Exception as exc:
print(f"vpnbot-xray-release-promoter: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())