#!/usr/bin/env python3 """Deploy a built static website to Bunny Storage or an SSH endpoint.""" from __future__ import annotations import argparse from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed import hashlib import json import mimetypes import os import subprocess import sys import tarfile import time import tomllib import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass from pathlib import Path from typing import Any, TypeVar MANIFEST_NAME = ".static-site-manifest.json" MANIFEST_VERSION = 1 BUNNY_RETRY_DELAYS = (1, 2, 4, 8, 16) BUNNY_TRANSIENT_STATUS_CODES = {401, 408, 429, 500, 502, 503, 504} DEFAULT_BUNNY_TRANSFER_WORKERS = 16 T = TypeVar("T") class DeployError(RuntimeError): """A static-site deployment could not be completed safely.""" def load_edge_rules(path: Path) -> list[dict[str, Any]]: """Read the [[edge_rules]] tables of a TOML file. Only the structure is checked here; the Gandalf gateway validates the rules when a deployment starts, before anything is uploaded. """ try: data = tomllib.loads(path.read_text(encoding="utf-8")) except OSError as exc: raise DeployError(f"cannot read edge rules {path}: {exc}") from exc except tomllib.TOMLDecodeError as exc: raise DeployError(f"edge rules {path} are not valid TOML: {exc}") from exc unknown = set(data) - {"edge_rules"} if unknown: raise DeployError( f"edge rules {path} has unknown keys: {', '.join(sorted(unknown))}" ) rules = data.get("edge_rules", []) if not isinstance(rules, list) or not all(isinstance(rule, dict) for rule in rules): raise DeployError(f"edge rules {path}: edge_rules must be [[edge_rules]] tables") try: # Dates and times have no JSON form and no meaning in a rule. json.dumps(rules) except TypeError as exc: raise DeployError(f"edge rules {path} contain an unsupported value: {exc}") from exc return rules def positive_int(value: str) -> int: try: parsed = int(value) except ValueError as exc: raise argparse.ArgumentTypeError("must be an integer") from exc if parsed < 1: raise argparse.ArgumentTypeError("must be at least 1") return parsed @dataclass(frozen=True) class LocalFile: path: str source: Path size: int sha256: str def collect_files(root: Path) -> dict[str, LocalFile]: root = root.resolve() if not root.is_dir(): raise DeployError(f"build directory does not exist: {root}") files: dict[str, LocalFile] = {} for source in sorted(root.rglob("*")): if source.is_symlink(): raise DeployError(f"symbolic links are not supported: {source.relative_to(root)}") if not source.is_file(): continue relative = source.relative_to(root).as_posix() if relative == MANIFEST_NAME: raise DeployError(f"{MANIFEST_NAME} is reserved by the deployer") digest = hashlib.sha256(source.read_bytes()).hexdigest() files[relative] = LocalFile(relative, source, source.stat().st_size, digest) if "index.html" not in files: raise DeployError("build directory must contain index.html") return files def manifest_for(files: dict[str, LocalFile]) -> dict[str, Any]: return { "version": MANIFEST_VERSION, "files": { path: {"sha256": item.sha256, "size": item.size} for path, item in sorted(files.items()) }, } def manifest_bytes(manifest: dict[str, Any]) -> bytes: return (json.dumps(manifest, sort_keys=True, separators=(",", ":")) + "\n").encode() def parse_manifest(raw: bytes | None) -> dict[str, dict[str, Any]]: if raw is None: return {} try: value = json.loads(raw) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise DeployError(f"remote {MANIFEST_NAME} is not valid JSON") from exc if not isinstance(value, dict) or value.get("version") != MANIFEST_VERSION: raise DeployError(f"remote {MANIFEST_NAME} has an unsupported version") files = value.get("files") if not isinstance(files, dict) or not all(isinstance(key, str) for key in files): raise DeployError(f"remote {MANIFEST_NAME} has an invalid files map") for path, metadata in files.items(): parts = Path(path).parts if ( not path or path.startswith("/") or ".." in parts or path == MANIFEST_NAME or not isinstance(metadata, dict) ): raise DeployError(f"remote {MANIFEST_NAME} contains an unsafe path") return files class BunnyStorage: def __init__(self, zone: str, password: str, endpoint: str) -> None: if not zone or not password: raise DeployError("BUNNY_STORAGE_ZONE and BUNNY_STORAGE_PASSWORD are required") endpoint = endpoint.removeprefix("https://").rstrip("/") if not endpoint or "/" in endpoint: raise DeployError("Bunny storage endpoint must be a hostname") self.zone = zone self.password = password self.base_url = f"https://{endpoint}/{urllib.parse.quote(zone, safe='')}" def _request( self, method: str, path: str, *, body: bytes | None = None, headers: dict[str, str] | None = None, missing_ok: bool = False, ) -> bytes | None: url_path = urllib.parse.quote(path, safe="/") request_headers = {"AccessKey": self.password, **(headers or {})} for attempt in range(len(BUNNY_RETRY_DELAYS) + 1): request = urllib.request.Request( f"{self.base_url}/{url_path}", data=body, headers=request_headers, method=method, ) try: with urllib.request.urlopen(request, timeout=60) as response: return response.read() except urllib.error.HTTPError as exc: if missing_ok and exc.code == 404: return None detail = exc.read().decode("utf-8", errors="replace") if ( exc.code in BUNNY_TRANSIENT_STATUS_CODES and attempt < len(BUNNY_RETRY_DELAYS) ): time.sleep(BUNNY_RETRY_DELAYS[attempt]) continue raise DeployError( f"Bunny Storage {method} {path} failed: HTTP {exc.code}: {detail}" ) from exc except urllib.error.URLError as exc: if attempt < len(BUNNY_RETRY_DELAYS): time.sleep(BUNNY_RETRY_DELAYS[attempt]) continue raise DeployError( f"Bunny Storage {method} {path} failed: {exc.reason}" ) from exc raise AssertionError("unreachable Bunny Storage retry state") def get_manifest(self) -> bytes | None: return self._request("GET", MANIFEST_NAME, missing_ok=True) def download(self, path: str) -> bytes | None: return self._request("GET", path, missing_ok=True) def upload(self, item: LocalFile) -> None: content_type = mimetypes.guess_type(item.path)[0] or "application/octet-stream" self._request( "PUT", item.path, body=item.source.read_bytes(), headers={ "Checksum": item.sha256.upper(), "Content-Type": content_type, }, ) def upload_manifest(self, manifest: dict[str, Any]) -> None: body = manifest_bytes(manifest) self._request( "PUT", MANIFEST_NAME, body=body, headers={ "Checksum": hashlib.sha256(body).hexdigest().upper(), "Content-Type": "application/json", }, ) def delete(self, path: str) -> None: self._request("DELETE", path, missing_ok=True) def bunny_plan( local: dict[str, LocalFile], remote: dict[str, dict[str, Any]] ) -> tuple[list[str], list[str]]: upload = [ path for path, item in local.items() if not isinstance(remote.get(path), dict) or remote[path].get("sha256") != item.sha256 or remote[path].get("size") != item.size ] delete = sorted(set(remote) - set(local)) # Publish assets before documents that may start referring to them. upload.sort(key=lambda path: (path.lower().endswith(('.html', '.htm')), path)) return upload, delete def run_concurrently( items: list[T], action: Callable[[T], None], workers: int ) -> None: if workers < 1: raise DeployError("Bunny transfer workers must be at least 1") if not items: return with ThreadPoolExecutor(max_workers=min(workers, len(items))) as executor: futures = [executor.submit(action, item) for item in items] for future in as_completed(futures): future.result() def apply_bunny_plan( storage: BunnyStorage, files: dict[str, LocalFile], upload: list[str], delete: list[str], workers: int = DEFAULT_BUNNY_TRANSFER_WORKERS, ) -> None: assets = [ path for path in upload if not path.lower().endswith((".html", ".htm")) ] documents = [ path for path in upload if path.lower().endswith((".html", ".htm")) ] run_concurrently(assets, lambda path: storage.upload(files[path]), workers) run_concurrently(documents, lambda path: storage.upload(files[path]), workers) run_concurrently(delete, storage.delete, workers) storage.upload_manifest(manifest_for(files)) def purge_bunny_pull_zone(pull_zone_id: str, api_key: str) -> None: if not api_key: raise DeployError("BUNNY_API_KEY is required when BUNNY_PULL_ZONE_ID is set") request = urllib.request.Request( f"https://api.bunny.net/pullzone/{urllib.parse.quote(pull_zone_id, safe='')}/purgeCache", headers={"AccessKey": api_key}, method="POST", ) try: with urllib.request.urlopen(request, timeout=60): return except urllib.error.HTTPError as exc: detail = exc.read().decode("utf-8", errors="replace") raise DeployError(f"Bunny cache purge failed: HTTP {exc.code}: {detail}") from exc except urllib.error.URLError as exc: raise DeployError(f"Bunny cache purge failed: {exc.reason}") from exc def purge_bunny_gateway(endpoint: str, zone: str, password: str) -> None: parsed = urllib.parse.urlsplit(endpoint) if ( parsed.scheme != "https" or not parsed.hostname or parsed.username is not None or parsed.password is not None or parsed.port is not None or parsed.path not in ("", "/") or parsed.query or parsed.fragment ): raise DeployError("BUNNY_PURGE_ENDPOINT must be an HTTPS origin without a path") url = ( f"https://{parsed.hostname}/v1/static-sites/" f"{urllib.parse.quote(zone, safe='')}/purge" ) request = urllib.request.Request( url, data=b"", headers={"Authorization": f"Bearer {password}", "Accept": "application/json"}, method="POST", ) try: with urllib.request.urlopen(request, timeout=60): return except urllib.error.HTTPError as exc: exc.read() raise DeployError(f"Bunny purge gateway failed: HTTP {exc.code}") from exc except urllib.error.URLError as exc: raise DeployError(f"Bunny purge gateway failed: {exc.reason}") from exc def gateway_origin(endpoint: str) -> str: parsed = urllib.parse.urlsplit(endpoint) if ( parsed.scheme != "https" or not parsed.hostname or parsed.username is not None or parsed.password is not None or parsed.port is not None or parsed.path not in ("", "/") or parsed.query or parsed.fragment ): raise DeployError("BUNNY_PURGE_ENDPOINT must be an HTTPS origin without a path") return f"https://{parsed.hostname}" def deployment_gateway_request( endpoint: str, zone: str, password: str, method: str, path: str, payload: dict[str, Any], ) -> dict[str, Any]: url = ( f"{gateway_origin(endpoint)}/v1/static-sites/" f"{urllib.parse.quote(zone, safe='')}/{path.lstrip('/')}" ) request = urllib.request.Request( url, data=json.dumps(payload).encode(), headers={ "Authorization": f"Bearer {password}", "Accept": "application/json", "Content-Type": "application/json", }, method=method, ) try: with urllib.request.urlopen(request, timeout=60) as response: raw = response.read() return json.loads(raw) if raw else {} except urllib.error.HTTPError as exc: detail = exc.read() try: message = str(json.loads(detail).get("error") or "") except (UnicodeDecodeError, json.JSONDecodeError, AttributeError): message = "" suffix = f": {message}" if message else "" raise DeployError( f"Bunny deployment gateway {method} failed: HTTP {exc.code}{suffix}" ) from exc except urllib.error.URLError as exc: raise DeployError(f"Bunny deployment gateway failed: {exc.reason}") from exc def request_identity_token(audience: str) -> str: """The running workflow's OIDC identity token, from Forgejo or GitHub.""" url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "") token = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "") if not url or not token: raise DeployError( "no BUNNY_STORAGE_PASSWORD and no workflow identity: let the job " "request an identity token (Forgejo: enable-openid-connect: true; " "GitHub: permissions: id-token: write) or set the password" ) separator = "&" if "?" in url else "?" request = urllib.request.Request( f"{url}{separator}audience={urllib.parse.quote(audience, safe='')}", headers={"Authorization": f"bearer {token}", "Accept": "application/json"}, ) try: with urllib.request.urlopen(request, timeout=30) as response: value = json.loads(response.read()).get("value") except urllib.error.HTTPError as exc: exc.read() raise DeployError( f"the workflow identity token could not be requested: HTTP {exc.code}" ) from exc except (urllib.error.URLError, TimeoutError, ValueError, AttributeError) as exc: raise DeployError( f"the workflow identity token could not be requested: {exc}" ) from exc if not isinstance(value, str) or not value: raise DeployError("the CI provider returned no identity token") return value def start_bunny_deployment( endpoint: str, zone: str, password: str, idempotency_key: str, revision: str | None, edge_rules: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = {"idempotency_key": idempotency_key, "revision": revision} if edge_rules is not None: # Sent here as well so the gateway rejects broken rules before the upload. payload["edge_rules"] = edge_rules result = deployment_gateway_request( endpoint, zone, password, "POST", "deployments", payload, ) deployment = result.get("deployment") if not isinstance(deployment, dict) or not deployment.get("deployment_id"): raise DeployError("Bunny deployment gateway returned an invalid lease") # A deployment by repository identity receives the Storage Zone password. return { **deployment, "storage_password": result.get("storage_password"), "storage_endpoint": result.get("storage_endpoint"), } def finalize_bunny_deployment( endpoint: str, zone: str, password: str, deployment_id: str, idempotency_key: str, revision: str | None, manifest_sha256: str, edge_rules: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: payload: dict[str, Any] = { "idempotency_key": idempotency_key, "revision": revision, "manifest_sha256": manifest_sha256, } if edge_rules is not None: payload["edge_rules"] = edge_rules result = deployment_gateway_request( endpoint, zone, password, "POST", f"deployments/{urllib.parse.quote(deployment_id, safe='')}/finalize", payload, ) deployment = result.get("deployment") if not isinstance(deployment, dict): raise DeployError("Bunny deployment gateway returned an invalid finalization") return {**deployment, "edge_rules_changed": bool(result.get("edge_rules_changed"))} def abort_bunny_deployment( endpoint: str, zone: str, password: str, deployment_id: str, idempotency_key: str, ) -> None: deployment_gateway_request( endpoint, zone, password, "DELETE", f"deployments/{urllib.parse.quote(deployment_id, safe='')}", {"idempotency_key": idempotency_key}, ) def deploy_bunny(args: argparse.Namespace) -> dict[str, Any]: if args.purge_endpoint and args.pull_zone_id: raise DeployError( "BUNNY_PURGE_ENDPOINT and BUNNY_PULL_ZONE_ID are mutually exclusive" ) # Without the password, the workflow's identity token authenticates, and # the gateway hands out the password for this deployment only. identity = not args.password if identity and not args.purge_endpoint: raise DeployError( "BUNNY_STORAGE_PASSWORD is required without the Gandalf gateway " "(BUNNY_PURGE_ENDPOINT)" ) edge_rules_path = getattr(args, "edge_rules", None) if edge_rules_path and not args.purge_endpoint: raise DeployError("edge rules can only be applied through BUNNY_PURGE_ENDPOINT") # Through the gateway, no file means no rules: it removes rules from earlier deploys. edge_rules = load_edge_rules(edge_rules_path) if edge_rules_path else [] files = collect_files(args.directory) manifest = manifest_for(files) manifest_sha256 = hashlib.sha256(manifest_bytes(manifest)).hexdigest() revision = args.revision.strip() or None rules_sha256 = hashlib.sha256( json.dumps(edge_rules, sort_keys=True).encode() ).hexdigest() idempotency_key = args.idempotency_key.strip() or hashlib.sha256( f"{args.zone}\0{revision or ''}\0{manifest_sha256}\0{rules_sha256}".encode() ).hexdigest() def report(upload: list[str] | None, delete: list[str] | None) -> dict[str, Any]: return { "backend": "bunny", "changed": None if upload is None else bool(upload or delete), "dry_run": args.dry_run, "upload": upload, "delete": delete, "purged": False, "deployment": None, "edge_rules": [rule.get("name") for rule in edge_rules], } def plan(storage: BunnyStorage) -> tuple[list[str], list[str]]: return bunny_plan(files, parse_manifest(storage.get_manifest())) if identity: if args.dry_run: # Comparing with the deployed files needs the password. return {**report(None, None), "files": len(files)} storage = None result = report(None, None) else: storage = BunnyStorage(args.zone, args.password, args.endpoint) upload, delete = plan(storage) result = report(upload, delete) if args.dry_run: return result if args.purge_endpoint: audience = gateway_origin(args.purge_endpoint) def credential() -> str: # Identity tokens are short-lived; each request gets a fresh one. return args.password or request_identity_token(audience) lease = start_bunny_deployment( args.purge_endpoint, args.zone, credential(), idempotency_key, revision, edge_rules, ) try: if identity: password = str(lease.get("storage_password") or "") if not password: raise DeployError( "the gateway handed out no storage credential for this deployment" ) # Keeps the password out of the CI log. print(f"::add-mask::{password}", flush=True) storage = BunnyStorage( args.zone, password, str(lease.get("storage_endpoint") or args.endpoint), ) upload, delete = plan(storage) result = report(upload, delete) assert storage is not None if result["changed"]: apply_bunny_plan(storage, files, upload, delete, args.workers) deployment = finalize_bunny_deployment( args.purge_endpoint, args.zone, credential(), str(lease["deployment_id"]), idempotency_key, revision, manifest_sha256, edge_rules, ) except Exception: try: abort_bunny_deployment( args.purge_endpoint, args.zone, credential(), str(lease["deployment_id"]), idempotency_key, ) except DeployError: pass raise result["deployment"] = deployment result["purged"] = bool(deployment.get("purged")) elif args.pull_zone_id: assert storage is not None if result["changed"]: apply_bunny_plan(storage, files, upload, delete, args.workers) purge_bunny_pull_zone(args.pull_zone_id, args.api_key) result["purged"] = True elif result["changed"]: assert storage is not None apply_bunny_plan(storage, files, upload, delete, args.workers) return result def deploy_ssh(args: argparse.Namespace) -> dict[str, Any]: files = collect_files(args.directory) if args.dry_run: return { "backend": "ssh", "changed": True, "dry_run": True, "files": len(files), "target": args.target, "site": args.site, } command = ["ssh", "-p", str(args.port)] if args.identity: command.extend(["-i", str(args.identity)]) command.extend([args.target, args.site]) process = subprocess.Popen(command, stdin=subprocess.PIPE) assert process.stdin is not None try: with tarfile.open(fileobj=process.stdin, mode="w|gz") as archive: for path, item in sorted(files.items()): archive.add(item.source, arcname=path, recursive=False) except BrokenPipeError: pass return_code = process.wait() if return_code: raise DeployError(f"SSH deployment failed with exit code {return_code}") return { "backend": "ssh", "changed": True, "dry_run": False, "files": len(files), "target": args.target, "site": args.site, } def parser() -> argparse.ArgumentParser: result = argparse.ArgumentParser( description="Deploy a completed static-site build to Bunny or an SSH endpoint." ) subparsers = result.add_subparsers(dest="backend", required=True) bunny = subparsers.add_parser("bunny", help="Sync a build to Bunny Storage.") bunny.add_argument("directory", type=Path) bunny.add_argument("--zone", default=os.environ.get("BUNNY_STORAGE_ZONE", "")) bunny.add_argument( "--endpoint", default=os.environ.get("BUNNY_STORAGE_ENDPOINT", "storage.bunnycdn.com"), help="Regional storage hostname (default: BUNNY_STORAGE_ENDPOINT or Frankfurt).", ) bunny.add_argument( "--pull-zone-id", default=os.environ.get("BUNNY_PULL_ZONE_ID", "") ) bunny.add_argument( "--purge-endpoint", default=os.environ.get("BUNNY_PURGE_ENDPOINT", "") ) bunny.add_argument( "--revision", default=( os.environ.get("STATIC_SITE_REVISION") or os.environ.get("GITHUB_SHA") or os.environ.get("FORGEJO_SHA") or os.environ.get("CI_COMMIT_SHA", "") ), ) bunny.add_argument( "--edge-rules", type=Path, default=os.environ.get("STATIC_SITE_EDGE_RULES") or None, help="TOML file with [[edge_rules]] for the site's Pull Zones (needs the gateway).", ) bunny.add_argument( "--idempotency-key", default=os.environ.get("STATIC_SITE_IDEMPOTENCY_KEY", ""), ) bunny.add_argument( "--workers", type=positive_int, default=os.environ.get( "BUNNY_TRANSFER_WORKERS", str(DEFAULT_BUNNY_TRANSFER_WORKERS) ), help="Maximum concurrent Bunny Storage transfers.", ) bunny.add_argument("--dry-run", action="store_true") bunny.set_defaults( handler=deploy_bunny, password=os.environ.get("BUNNY_STORAGE_PASSWORD", ""), api_key=os.environ.get("BUNNY_API_KEY", ""), ) ssh = subparsers.add_parser("ssh", help="Atomically deploy through static-deploy SSH.") ssh.add_argument("directory", type=Path) ssh.add_argument("--target", required=True, help="SSH target, e.g. static-deploy@citadel.") ssh.add_argument("--site", required=True, help="Configured caddy_static_sites name.") ssh.add_argument("--port", type=int, default=22) ssh.add_argument("--identity", type=Path) ssh.add_argument("--dry-run", action="store_true") ssh.set_defaults(handler=deploy_ssh) return result def main() -> int: args = parser().parse_args() try: report = args.handler(args) except DeployError as exc: print(f"static-site: {exc}", file=sys.stderr) return 1 print(json.dumps(report, indent=2, sort_keys=True)) return 0 if __name__ == "__main__": raise SystemExit(main())