feat: add live sync retry backoff

crates/geth-node/src/lib.rs

@@ -11415,7 +11415,11 @@ fn ssh_revocation_from_stored(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::sync::{record_live_sync_success, run_live_sync_once};
+    use crate::sync::{
+        SyncRunMode, record_live_sync_failure, record_live_sync_success, run_live_sync_once,
+        should_live_sync_stream,
+    };
+    use std::collections::BTreeMap;
     use tokio::io::AsyncReadExt;
 
     fn skip_iroh_integration_tests() -> bool {
@@ -12022,6 +12026,64 @@ mod tests {
         assert_eq!(peers[0].streams[0].next_action, "none");
     }
 
+    #[test]
+    fn live_sync_failures_back_off_background_streams() {
+        let store = Store::open_memory().expect("open");
+        store
+            .upsert_peer_card(&StoredPeerCard {
+                peer_id: "node:left".to_owned(),
+                card_json: "{}".to_owned(),
+                updated_at_ms: 1,
+            })
+            .expect("insert peer");
+        record_live_sync_failure(
+            &store,
+            "node:left",
+            "kv:prefs",
+            &NodeError::IrohPeer("peer unavailable".to_owned()),
+        )
+        .expect("record failure");
+
+        let response = sync_status_local(&store).expect("sync status");
+        let ControlResponse::SyncStatus { peers, .. } = response else {
+            panic!("unexpected sync status response");
+        };
+        assert_eq!(peers.len(), 1);
+        let stream = &peers[0].streams[0];
+        assert_eq!(stream.stream, "kv:prefs");
+        assert_eq!(stream.state, "failed");
+        assert_eq!(stream.consecutive_failures, 1);
+        assert!(stream.retry_after_ms.is_some());
+        assert!(
+            stream
+                .retry_in_ms
+                .is_some_and(|retry_in_ms| retry_in_ms > 0)
+        );
+        assert!(stream.next_action.contains("geth sync now node:left"));
+
+        let watermarks = BTreeMap::from([("kv:prefs".to_owned(), 42)]);
+        assert!(
+            !should_live_sync_stream(
+                &store,
+                "node:left",
+                "kv:prefs",
+                Some(&watermarks),
+                SyncRunMode::Background,
+            )
+            .expect("background decision")
+        );
+        assert!(
+            should_live_sync_stream(
+                &store,
+                "node:left",
+                "kv:prefs",
+                Some(&watermarks),
+                SyncRunMode::Manual,
+            )
+            .expect("manual decision")
+        );
+    }
+
     #[test]
     fn operator_errors_include_recovery_commands() {
         let peer_error =