perf(static-sites): parallelize Bunny transfers

static_site.py

@@ -4,6 +4,8 @@
 from __future__ import annotations
 
 import argparse
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
 import hashlib
 import json
 import mimetypes
@@ -17,19 +19,32 @@ import urllib.parse
 import urllib.request
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, TypeVar
 
 
 MANIFEST_NAME = ".static-site-manifest.json"
 MANIFEST_VERSION = 1
 BUNNY_RETRY_DELAYS = (1, 2, 4, 8, 16)
 BUNNY_TRANSIENT_STATUS_CODES = {401, 408, 429, 500, 502, 503, 504}
+DEFAULT_BUNNY_TRANSFER_WORKERS = 16
+
+T = TypeVar("T")
 
 
 class DeployError(RuntimeError):
     """A static-site deployment could not be completed safely."""
 
 
+def positive_int(value: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("must be an integer") from exc
+    if parsed < 1:
+        raise argparse.ArgumentTypeError("must be at least 1")
+    return parsed
+
+
 @dataclass(frozen=True)
 class LocalFile:
     path: str
@@ -140,6 +155,9 @@ class BunnyStorage:
                     f"Bunny Storage {method} {path} failed: HTTP {exc.code}: {detail}"
                 ) from exc
             except urllib.error.URLError as exc:
+                if attempt < len(BUNNY_RETRY_DELAYS):
+                    time.sleep(BUNNY_RETRY_DELAYS[attempt])
+                    continue
                 raise DeployError(
                     f"Bunny Storage {method} {path} failed: {exc.reason}"
                 ) from exc
@@ -192,6 +210,39 @@ def bunny_plan(
     return upload, delete
 
 
+def run_concurrently(
+    items: list[T], action: Callable[[T], None], workers: int
+) -> None:
+    if workers < 1:
+        raise DeployError("Bunny transfer workers must be at least 1")
+    if not items:
+        return
+    with ThreadPoolExecutor(max_workers=min(workers, len(items))) as executor:
+        futures = [executor.submit(action, item) for item in items]
+        for future in as_completed(futures):
+            future.result()
+
+
+def apply_bunny_plan(
+    storage: BunnyStorage,
+    files: dict[str, LocalFile],
+    upload: list[str],
+    delete: list[str],
+    workers: int = DEFAULT_BUNNY_TRANSFER_WORKERS,
+) -> None:
+    assets = [
+        path for path in upload if not path.lower().endswith((".html", ".htm"))
+    ]
+    documents = [
+        path for path in upload if path.lower().endswith((".html", ".htm"))
+    ]
+
+    run_concurrently(assets, lambda path: storage.upload(files[path]), workers)
+    run_concurrently(documents, lambda path: storage.upload(files[path]), workers)
+    run_concurrently(delete, storage.delete, workers)
+    storage.upload_manifest(manifest_for(files))
+
+
 def purge_bunny_pull_zone(pull_zone_id: str, api_key: str) -> None:
     if not api_key:
         raise DeployError("BUNNY_API_KEY is required when BUNNY_PULL_ZONE_ID is set")
@@ -264,11 +315,7 @@ def deploy_bunny(args: argparse.Namespace) -> dict[str, Any]:
         return result
 
     if result["changed"]:
-        for path in upload:
-            storage.upload(files[path])
-        for path in delete:
-            storage.delete(path)
-        storage.upload_manifest(manifest_for(files))
+        apply_bunny_plan(storage, files, upload, delete, args.workers)
     if args.purge_endpoint:
         purge_bunny_gateway(args.purge_endpoint, args.zone, args.password)
         result["purged"] = True
@@ -335,6 +382,14 @@ def parser() -> argparse.ArgumentParser:
     bunny.add_argument(
         "--purge-endpoint", default=os.environ.get("BUNNY_PURGE_ENDPOINT", "")
     )
+    bunny.add_argument(
+        "--workers",
+        type=positive_int,
+        default=os.environ.get(
+            "BUNNY_TRANSFER_WORKERS", str(DEFAULT_BUNNY_TRANSFER_WORKERS)
+        ),
+        help="Maximum concurrent Bunny Storage transfers.",
+    )
     bunny.add_argument("--dry-run", action="store_true")
     bunny.set_defaults(
         handler=deploy_bunny,