diff --git a/README.md b/README.md index c8c2c18..8c812dc 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,11 @@ deploys a pushed container image digest through the Gandalf gateway. 3. It records the deployment and notifies the node the app is placed on. 4. The node applies it: backup gate, pull, restart, health check, and rollback on failure. -5. This action follows the deployment and prints the node's progress. +5. This action follows the deployment and prints the node's progress. While it waits, it + rides out a forge or gateway that can't be reached or answers 429/5xx (with one warning + per outage), since the deployment goes on without it. 6. It fails the job when the deployment is rolled back, fails, or gets no result within - `timeout`. + `timeout`. A refused request (4xx) fails it at once. A newer deployment of the same app that supersedes this one ends the job with a notice. diff --git a/app_deploy.py b/app_deploy.py index da58b86..a0b4009 100644 --- a/app_deploy.py +++ b/app_deploy.py @@ -29,6 +29,15 @@ class DeployError(Exception): pass +class TransientError(DeployError): + """A request that may work when tried again: no connection, a timeout, or HTTP 429/5xx.""" + + +def _http_error(prefix: str, code: int, message: str = "") -> DeployError: + kind = TransientError if code == 429 or code >= 500 else DeployError + return kind(f"{prefix}: HTTP {code}{': ' + message if message else ''}") + + def request_identity_token(audience: str, opener=urllib.request.urlopen) -> str: """The running workflow's OIDC identity token, from Forgejo or GitHub.""" url = os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL", "") @@ -48,8 +57,10 @@ def request_identity_token(audience: str, opener=urllib.request.urlopen) -> str: value = json.loads(response.read()).get("value") except urllib.error.HTTPError as exc: exc.read() - raise DeployError(f"the workflow identity token could not be requested: HTTP {exc.code}") from exc - except (urllib.error.URLError, TimeoutError, ValueError, AttributeError) as exc: + raise _http_error("the workflow identity token could not be requested", exc.code) from exc + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise TransientError(f"the workflow identity token could not be requested: {exc}") from exc + except (ValueError, AttributeError) as exc: raise DeployError(f"the workflow identity token could not be requested: {exc}") from exc if not isinstance(value, str) or not value: raise DeployError("the CI provider returned no identity token") @@ -82,9 +93,11 @@ def gateway_request( message = str(json.loads(detail).get("error") or "") except (UnicodeDecodeError, json.JSONDecodeError, AttributeError): message = "" - raise DeployError(f"Gandalf {request.method} {path} failed: HTTP {exc.code}{': ' + message if message else ''}") from exc + raise _http_error(f"Gandalf {request.method} {path} failed", exc.code, message) from exc except urllib.error.URLError as exc: - raise DeployError(f"Gandalf could not be reached: {exc.reason}") from exc + raise TransientError(f"Gandalf could not be reached: {exc.reason}") from exc + except (TimeoutError, OSError) as exc: + raise TransientError(f"Gandalf could not be reached: {exc}") from exc def deploy( @@ -122,6 +135,7 @@ def deploy( deadline = clock() + timeout seen = None current = started + unreachable = False while True: status, detail = current["status"], current.get("detail") if (status, detail) != seen: @@ -133,10 +147,19 @@ def deploy( log(f"::error::no result from {started['node']} after {int(timeout)} s (status {status})") return 1 sleep(POLL_SECONDS) - # A fresh identity token for each request: they expire within minutes. - current = call( - gateway, f"/v1/apps/{service}/deployments/{deployment_id}", identity(audience) - )["deployment"] + # A fresh identity token for each request: they expire within minutes. The + # deployment goes on without us, so a passing outage (the forge or the gateway + # restarting, say) only delays the answer until the deadline. + try: + current = call( + gateway, f"/v1/apps/{service}/deployments/{deployment_id}", identity(audience) + )["deployment"] + except TransientError as exc: + if not unreachable: + log(f"::warning::{exc}; still waiting") + unreachable = True + continue + unreachable = False if status == "live": return 0 if status == "superseded": diff --git a/tests/test_app_deploy_action.py b/tests/test_app_deploy_action.py index 1a32788..217350c 100644 --- a/tests/test_app_deploy_action.py +++ b/tests/test_app_deploy_action.py @@ -15,7 +15,7 @@ IMAGE = f"forge.example/eric/notes@sha256:{'a' * 64}" class FakeGateway: - """Answers the start and then the given statuses, one per poll.""" + """Answers the start and then the given statuses, one per poll (an exception is raised).""" def __init__(self, statuses, notified=True): self.statuses = list(statuses) @@ -25,6 +25,8 @@ class FakeGateway: def __call__(self, gateway, path, token, body=None): self.requests.append((path, token, body)) status = self.statuses.pop(0) if len(self.statuses) > 1 else self.statuses[0] + if isinstance(status, Exception): + raise status detail = {"applying": "pulling", "rolled_back": "not healthy after 120 s"}.get(status) return { "deployment": { @@ -95,6 +97,46 @@ def test_waiting_ends_with_the_timeout(): assert lines[-1].startswith("::error::no result from citadel after 10 s") +def test_a_passing_outage_while_waiting_is_waited_out(): + down = app_deploy.TransientError("Gandalf could not be reached: [Errno 101] Network is unreachable") + code, _gateway, lines = run(["pending", down, down, app_deploy.TransientError("HTTP 502"), "live"]) + assert code == 0 + # One warning per outage, then the progress goes on. + assert [line for line in lines if line.startswith("::warning::")] == [ + "::warning::Gandalf could not be reached: [Errno 101] Network is unreachable; still waiting" + ] + assert lines[-1] == "live" + + +def test_an_outage_that_outlasts_the_deadline_fails(): + down = app_deploy.TransientError("Gandalf could not be reached: timed out") + code, _gateway, lines = run(["pending", down], timeout=10) + assert code == 1 + assert lines[-1].startswith("::error::no result from citadel after 10 s") + + +def test_a_refusal_while_waiting_still_fails(): + with pytest.raises(app_deploy.DeployError, match="HTTP 403") as caught: + run(["pending", app_deploy._http_error("Gandalf GET /v1/apps/notes/deployments/d-1 failed", 403)]) + assert not isinstance(caught.value, app_deploy.TransientError) + + +@pytest.mark.parametrize("code, transient", [(400, False), (401, False), (403, False), (429, True), (502, True), (503, True)]) +def test_http_errors_are_transient_only_when_retrying_can_help(code, transient): + assert isinstance(app_deploy._http_error("x", code), app_deploy.TransientError) is transient + + +def test_an_unreachable_token_endpoint_is_transient(monkeypatch): + monkeypatch.setenv("ACTIONS_ID_TOKEN_REQUEST_URL", "https://forge.example/token") + monkeypatch.setenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "t") + + def unreachable(_request, timeout): + raise app_deploy.urllib.error.URLError(OSError(101, "Network is unreachable")) + + with pytest.raises(app_deploy.TransientError, match="Network is unreachable"): + app_deploy.request_identity_token("https://g", unreachable) + + def test_an_unnotified_node_is_mentioned(): _code, _gateway, lines = run(["live"], notified=False) assert "next poll" in lines[0]