fix: wait out a passing outage while following a deployment

README.md

@@ -10,9 +10,11 @@ deploys a pushed container image digest through the Gandalf gateway.
 3. It records the deployment and notifies the node the app is placed on.
 4. The node applies it: backup gate, pull, restart, health check, and rollback on
    failure.
-5. This action follows the deployment and prints the node's progress.
+5. This action follows the deployment and prints the node's progress. While it waits, it
+   rides out a forge or gateway that can't be reached or answers 429/5xx (with one warning
+   per outage), since the deployment goes on without it.
 6. It fails the job when the deployment is rolled back, fails, or gets no result within
-   `timeout`.
+   `timeout`. A refused request (4xx) fails it at once.
 
 A newer deployment of the same app that supersedes this one ends the job with a notice.
 

app_deploy.py

@@ -29,6 +29,15 @@ class DeployError(Exception):
     pass
 
 
+class TransientError(DeployError):
+    """A request that may work when tried again: no connection, a timeout, or HTTP 429/5xx."""
+
+
+def _http_error(prefix: str, code: int, message: str = "") -> DeployError:
+    kind = TransientError if code == 429 or code >= 500 else DeployError
+    return kind(f"{prefix}: HTTP {code}{': ' + message if message else ''}")
+
+
 def request_identity_token(audience: str, opener=urllib.request.urlopen) -> str:
     """The running workflow's OIDC identity token, from Forgejo or GitHub."""
     url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "")
@@ -48,8 +57,10 @@ def request_identity_token(audience: str, opener=urllib.request.urlopen) -> str:
             value = json.loads(response.read()).get("value")
     except urllib.error.HTTPError as exc:
         exc.read()
-        raise DeployError(f"the workflow identity token could not be requested: HTTP {exc.code}") from exc
-    except (urllib.error.URLError, TimeoutError, ValueError, AttributeError) as exc:
+        raise _http_error("the workflow identity token could not be requested", exc.code) from exc
+    except (urllib.error.URLError, TimeoutError, OSError) as exc:
+        raise TransientError(f"the workflow identity token could not be requested: {exc}") from exc
+    except (ValueError, AttributeError) as exc:
         raise DeployError(f"the workflow identity token could not be requested: {exc}") from exc
     if not isinstance(value, str) or not value:
         raise DeployError("the CI provider returned no identity token")
@@ -82,9 +93,11 @@ def gateway_request(
             message = str(json.loads(detail).get("error") or "")
         except (UnicodeDecodeError, json.JSONDecodeError, AttributeError):
             message = ""
-        raise DeployError(f"Gandalf {request.method} {path} failed: HTTP {exc.code}{': ' + message if message else ''}") from exc
+        raise _http_error(f"Gandalf {request.method} {path} failed", exc.code, message) from exc
     except urllib.error.URLError as exc:
-        raise DeployError(f"Gandalf could not be reached: {exc.reason}") from exc
+        raise TransientError(f"Gandalf could not be reached: {exc.reason}") from exc
+    except (TimeoutError, OSError) as exc:
+        raise TransientError(f"Gandalf could not be reached: {exc}") from exc
 
 
 def deploy(
@@ -122,6 +135,7 @@ def deploy(
     deadline = clock() + timeout
     seen = None
     current = started
+    unreachable = False
     while True:
         status, detail = current["status"], current.get("detail")
         if (status, detail) != seen:
@@ -133,10 +147,19 @@ def deploy(
             log(f"::error::no result from {started['node']} after {int(timeout)} s (status {status})")
             return 1
         sleep(POLL_SECONDS)
-        # A fresh identity token for each request: they expire within minutes.
+        # A fresh identity token for each request: they expire within minutes. The
+        # deployment goes on without us, so a passing outage (the forge or the gateway
+        # restarting, say) only delays the answer until the deadline.
+        try:
             current = call(
                 gateway, f"/v1/apps/{service}/deployments/{deployment_id}", identity(audience)
             )["deployment"]
+        except TransientError as exc:
+            if not unreachable:
+                log(f"::warning::{exc}; still waiting")
+            unreachable = True
+            continue
+        unreachable = False
     if status == "live":
         return 0
     if status == "superseded":

tests/test_app_deploy_action.py

@@ -15,7 +15,7 @@ IMAGE = f"forge.example/eric/notes@sha256:{'a' * 64}"
 
 
 class FakeGateway:
-    """Answers the start and then the given statuses, one per poll."""
+    """Answers the start and then the given statuses, one per poll (an exception is raised)."""
 
     def __init__(self, statuses, notified=True):
         self.statuses = list(statuses)
@@ -25,6 +25,8 @@ class FakeGateway:
     def __call__(self, gateway, path, token, body=None):
         self.requests.append((path, token, body))
         status = self.statuses.pop(0) if len(self.statuses) > 1 else self.statuses[0]
+        if isinstance(status, Exception):
+            raise status
         detail = {"applying": "pulling", "rolled_back": "not healthy after 120 s"}.get(status)
         return {
             "deployment": {
@@ -95,6 +97,46 @@ def test_waiting_ends_with_the_timeout():
     assert lines[-1].startswith("::error::no result from citadel after 10 s")
 
 
+def test_a_passing_outage_while_waiting_is_waited_out():
+    down = app_deploy.TransientError("Gandalf could not be reached: [Errno 101] Network is unreachable")
+    code, _gateway, lines = run(["pending", down, down, app_deploy.TransientError("HTTP 502"), "live"])
+    assert code == 0
+    # One warning per outage, then the progress goes on.
+    assert [line for line in lines if line.startswith("::warning::")] == [
+        "::warning::Gandalf could not be reached: [Errno 101] Network is unreachable; still waiting"
+    ]
+    assert lines[-1] == "live"
+
+
+def test_an_outage_that_outlasts_the_deadline_fails():
+    down = app_deploy.TransientError("Gandalf could not be reached: timed out")
+    code, _gateway, lines = run(["pending", down], timeout=10)
+    assert code == 1
+    assert lines[-1].startswith("::error::no result from citadel after 10 s")
+
+
+def test_a_refusal_while_waiting_still_fails():
+    with pytest.raises(app_deploy.DeployError, match="HTTP 403") as caught:
+        run(["pending", app_deploy._http_error("Gandalf GET /v1/apps/notes/deployments/d-1 failed", 403)])
+    assert not isinstance(caught.value, app_deploy.TransientError)
+
+
+@pytest.mark.parametrize("code, transient", [(400, False), (401, False), (403, False), (429, True), (502, True), (503, True)])
+def test_http_errors_are_transient_only_when_retrying_can_help(code, transient):
+    assert isinstance(app_deploy._http_error("x", code), app_deploy.TransientError) is transient
+
+
+def test_an_unreachable_token_endpoint_is_transient(monkeypatch):
+    monkeypatch.setenv("ACTIONS_ID_TOKEN_REQUEST_URL", "https://forge.example/token")
+    monkeypatch.setenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "t")
+
+    def unreachable(_request, timeout):
+        raise app_deploy.urllib.error.URLError(OSError(101, "Network is unreachable"))
+
+    with pytest.raises(app_deploy.TransientError, match="Network is unreachable"):
+        app_deploy.request_identity_token("https://g", unreachable)
+
+
 def test_an_unnotified_node_is_mentioned():
     _code, _gateway, lines = run(["live"], notified=False)
     assert "next poll" in lines[0]